mikolov-rnnlm-lib.h

Go to the documentation of this file.

// lm/mikolov-rnnlm-lib.h

// Copyright       2015  Guoguo Chen  Hainan Xu
//            2010-2012  Tomas Mikolov

// See ../../COPYING for clarification regarding multiple authors
//
// This file is based on version 0.3e of the RNNLM language modeling
// toolkit by Tomas Mikolov.  Changes made by authors other than
// Tomas Mikolov are licensed under the Apache License, the short form
// os which is below.  The original code by Tomas Mikolov is licensed
// under the BSD 3-clause license, whose text is further below.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.
//
//
// Original BSD 3-clause license text:
// Copyright (c) 2010-2012 Tomas Mikolov
//
// All rights reserved. Redistribution and use in source and binary forms, with
// or without modification, are permitted provided that the following conditions
// are met: 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following
// disclaimer. 2. Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the
// distribution. 3. Neither name of copyright holders nor the names of its
// contributors may be used to endorse or promote products derived from this
// software without specific prior written permission. THIS SOFTWARE IS PROVIDED
// BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
// EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
// OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
// EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#ifndef KALDI_LM_MIKOLOV_RNNLM_LIB_H_
#define KALDI_LM_MIKOLOV_RNNLM_LIB_H_

#include <string>
#include <vector>
#include "util/stl-utils.h"

namespace rnnlm {

#define MAX_STRING 100
#define MAX_FILENAME_STRING 300

typedef double real;      //  doubles for NN weights
typedef double direct_t;  //  doubles for ME weights;

struct neuron {
  real ac;    // actual value stored in neuron
  real er;    // error value in neuron, used by learning algorithm
};

struct synapse {
  real weight;  // weight of synapse
};

struct vocab_word {
  int cn;
  char word[MAX_STRING];

  real prob;
  int class_index;
};

const unsigned int PRIMES[] = {108641969, 116049371, 125925907, 133333309,
  145678979, 175308587, 197530793, 234567803, 251851741, 264197411,
  330864029, 399999781,
  407407183, 459258997, 479012069, 545678687, 560493491, 607407037, 629629243,
  656789717, 716048933, 718518067, 725925469, 733332871, 753085943, 755555077,
  782715551, 790122953, 812345159, 814814293, 893826581, 923456189, 940740127,
  953085797, 985184539, 990122807};
const unsigned int PRIMES_SIZE  =  sizeof(PRIMES) / sizeof(PRIMES[0]);

const int MAX_NGRAM_ORDER = 20;

enum FileTypeEnum {TEXT, BINARY, COMPRESSED};  // COMPRESSED not yet implemented

class CRnnLM {
 protected:
  char train_file[MAX_FILENAME_STRING];
  char valid_file[MAX_FILENAME_STRING];
  char test_file[MAX_FILENAME_STRING];
  char rnnlm_file[MAX_FILENAME_STRING];
  char lmprob_file[MAX_FILENAME_STRING];

  int rand_seed;
  int version;
  int filetype;

  int use_lmprob;
  real gradient_cutoff;

  real dynamic;

  real alpha;
  real starting_alpha;
  int alpha_divide;
  double logp, llogp;
  float min_improvement;
  int iter;
  int vocab_max_size;
  int vocab_size;
  int train_words;
  int train_cur_pos;
  int counter;

  int anti_k;

  real beta;

  int class_size;
  int **class_words;
  int *class_cn;
  int *class_max_cn;
  int old_classes;

  struct vocab_word *vocab;
  void sortVocab();
  int *vocab_hash;
  int vocab_hash_size;

  int layer0_size;
  int layer1_size;
  int layerc_size;
  int layer2_size;

  long long direct_size;
  int direct_order;
  int history[MAX_NGRAM_ORDER];

  int bptt;
  int bptt_block;
  int *bptt_history;
  neuron *bptt_hidden;
  struct synapse *bptt_syn0;

  int gen;

  int independent;

  struct neuron *neu0;    // neurons in input layer
  struct neuron *neu1;    // neurons in hidden layer
  struct neuron *neuc;    // neurons in hidden layer
  struct neuron *neu2;    // neurons in output layer

  struct synapse *syn0;   // weights between input and hidden layer
  struct synapse *syn1;   // weights between hidden and output layer
                          // (or hidden and compression if compression>0)
  struct synapse *sync;   // weights between hidden and compression layer
  direct_t *syn_d;        // direct parameters between input and output layer
                          // (similar to Maximum Entropy model parameters)

  // backup used in training:
  struct neuron *neu0b;
  struct neuron *neu1b;
  struct neuron *neucb;
  struct neuron *neu2b;

  struct synapse *syn0b;
  struct synapse *syn1b;
  struct synapse *syncb;
  direct_t *syn_db;

  // backup used in n-bset rescoring:
  struct neuron *neu1b2;

  unordered_map<std::string, float> unk_penalty;
  std::string unk_sym;

 public:

  int alpha_set, train_file_set;

  CRnnLM();

  ~CRnnLM();

  real random(real min, real max);

  void setRnnLMFile(const std::string &str);
  int getHiddenLayerSize() const { return layer1_size; }
  void setRandSeed(int newSeed);

  int getWordHash(const char *word);
  void readWord(char *word, FILE *fin);
  int searchVocab(const char *word);

  void saveWeights();      // saves current weights and unit activations
  void initNet();
  void goToDelimiter(int delim, FILE *fi);
  void restoreNet();
  void netReset();         // will erase just hidden layer state + bptt history
                           // + maxent history (called at end of sentences in
                           // the independent mode)

  void computeNet(int last_word, int word);
  void copyHiddenLayerToInput();

  void matrixXvector(struct neuron *dest, struct neuron *srcvec,
                     struct synapse *srcmatrix, int matrix_width,
                     int from, int to, int from2, int to2, int type);

  void restoreContextFromVector(const std::vector<float> &context_in);
  void saveContextToVector(std::vector<float> *context_out);

  float computeConditionalLogprob(
      std::string current_word,
      const std::vector<std::string> &history_words,
      const std::vector<float> &context_in,
      std::vector<float> *context_out);

  void setUnkSym(const std::string &unk);
  void setUnkPenalty(const std::string &filename);
  float getUnkPenalty(const std::string &word);
  bool isUnk(const std::string &word);
};

}  // namespace rnnlm

#endif  // KALDI_LM_MIKOLOV_RNNLM_LIB_H_