arpa-file-parser.h

Go to the documentation of this file.

// lm/arpa-file-parser.h

// Copyright 2014  Guoguo Chen
// Copyright 2016  Smart Action Company LLC (kkm)

// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.

#ifndef KALDI_LM_ARPA_FILE_PARSER_H_
#define KALDI_LM_ARPA_FILE_PARSER_H_

#include <fst/fst-decl.h>

#include <string>
#include <vector>

#include "base/kaldi-types.h"
#include "itf/options-itf.h"

namespace kaldi {

struct ArpaParseOptions {
  enum OovHandling {
    kRaiseError,     
    kAddToSymbols,   
    kReplaceWithUnk,  
    kSkipNGram       
  };

  ArpaParseOptions():
      bos_symbol(-1), eos_symbol(-1), unk_symbol(-1),
      oov_handling(kRaiseError), max_warnings(30) { }

  void Register(OptionsItf *opts) {
    // Registering only the max_warnings count, since other options are
    // treated differently by client programs: some want integer symbols,
    // while other are passed words in their command line.
    opts->Register("max-arpa-warnings", &max_warnings,
                   "Maximum warnings to report on ARPA parsing, "
                   "0 to disable, -1 to show all");
  }

  int32 bos_symbol;  
  int32 eos_symbol;  
  int32 unk_symbol;  
  OovHandling oov_handling;  
  int32 max_warnings;  
};

struct NGram {
  NGram() : logprob(0.0), backoff(0.0) { }
  std::vector<int32> words;  
  float logprob;             
  float backoff;             
};

class ArpaFileParser {
 public:
  ArpaFileParser(const ArpaParseOptions& options, fst::SymbolTable* symbols);
  virtual ~ArpaFileParser();

  void Read(std::istream &is);

  const ArpaParseOptions& Options() const { return options_; }

 protected:
  virtual void ReadStarted() { }

  virtual void HeaderAvailable() { }

  virtual void ConsumeNGram(const NGram&) = 0;

  virtual void ReadComplete() { }

  const fst::SymbolTable* Symbols() const { return symbols_; }

  int32 LineNumber() const { return line_number_; }

  std::string LineReference() const;

  bool ShouldWarn();

  const std::vector<int32>& NgramCounts() const { return ngram_counts_; }

 private:
  ArpaParseOptions options_;
  fst::SymbolTable* symbols_;  // the pointer is not owned here.
  int32 line_number_;
  uint32 warning_count_;
  std::string current_line_;
  std::vector<int32> ngram_counts_;
};

}  // namespace kaldi

#endif  // KALDI_LM_ARPA_FILE_PARSER_H_