#include <const-arpa-lm.h>

Collaboration diagram for ConstArpaLm:

Public Member Functions
	ConstArpaLm ()

	ConstArpaLm (const int32 bos_symbol, const int32 eos_symbol, const int32 unk_symbol, const int32 ngram_order, const int32 num_words, const int32 overflow_buffer_size, const int64 lm_states_size, int32 unigram_states, int32 overflow_buffer, int32 *lm_states)

	~ConstArpaLm ()

void	Read (std::istream &is, bool binary)

void	Write (std::ostream &os, bool binary) const

void	WriteArpa (std::ostream &os) const

float	GetNgramLogprob (const int32 word, const std::vector< int32 > &hist) const

bool	HistoryStateExists (const std::vector< int32 > &hist) const

int32	BosSymbol () const

int32	EosSymbol () const

int32	UnkSymbol () const

int32	NgramOrder () const

Private Member Functions
void	ReadInternal (std::istream &is, bool binary)

void	ReadInternalOldFormat (std::istream &is, bool binary)

float	GetNgramLogprobRecurse (const int32 word, const std::vector< int32 > &hist) const

int32 *	GetLmState (const std::vector< int32 > &seq) const

bool	GetChildInfo (const int32 word, int32 parent, int32 child_info) const

void	DecodeChildInfo (const int32 child_info, int32 parent, int32 child_lm_state, float logprob) const

void	WriteArpaRecurse (int32 lm_state, const std::vector< int32 > &seq, std::vector< ArpaLine > output) const

Private Attributes
bool	memory_assigned_

bool	initialized_

int32	bos_symbol_

int32	eos_symbol_

int32	unk_symbol_

int32	ngram_order_

int32	num_words_

int32	overflow_buffer_size_

int64	lm_states_size_

int32 *	lm_states_end_

int32 **	unigram_states_

int32 **	overflow_buffer_

int32 *	lm_states_

Detailed Description

Definition at line 211 of file const-arpa-lm.h.

Constructor & Destructor Documentation

◆ ConstArpaLm() [1/2]

ConstArpaLm ( )

inline

Definition at line 216 of file const-arpa-lm.h.

                 {
     lm_states_ = NULL;
     unigram_states_ = NULL;
     overflow_buffer_ = NULL;
     memory_assigned_ = false;
     initialized_ = false;
   }

◆ ConstArpaLm() [2/2]

ConstArpaLm	(	const int32	bos_symbol,
		const int32	eos_symbol,
		const int32	unk_symbol,
		const int32	ngram_order,
		const int32	num_words,
		const int32	overflow_buffer_size,
		const int64	lm_states_size,
		int32 **	unigram_states,
		int32 **	overflow_buffer,
		int32 *	lm_states
	)

inline

Definition at line 226 of file const-arpa-lm.h.

References KALDI_ASSERT.

                                                          :
       bos_symbol_(bos_symbol), eos_symbol_(eos_symbol),
       unk_symbol_(unk_symbol), ngram_order_(ngram_order),
       num_words_(num_words), overflow_buffer_size_(overflow_buffer_size),
       lm_states_size_(lm_states_size), unigram_states_(unigram_states),
       overflow_buffer_(overflow_buffer), lm_states_(lm_states) {
     KALDI_ASSERT(unigram_states_ != NULL);
     KALDI_ASSERT(overflow_buffer_ != NULL);
     KALDI_ASSERT(lm_states_ != NULL);
     KALDI_ASSERT(ngram_order_ > 0);
     KALDI_ASSERT(bos_symbol_ < num_words_ && bos_symbol_ > 0);
     KALDI_ASSERT(eos_symbol_ < num_words_ && eos_symbol_ > 0);
     KALDI_ASSERT(unk_symbol_ < num_words_ &&
                  (unk_symbol_ > 0 || unk_symbol_ == -1));
     lm_states_end_ = lm_states_ + lm_states_size_ - 1;
     memory_assigned_ = false;
     initialized_ = true;
   }

◆ ~ConstArpaLm()

~ConstArpaLm ( )

inline

Definition at line 249 of file const-arpa-lm.h.

                  {
     if (memory_assigned_) {
       delete[] lm_states_;
       delete[] unigram_states_;
       delete[] overflow_buffer_;
     }
   }

Member Function Documentation

◆ BosSymbol()

int32 BosSymbol ( ) const

inline

Definition at line 276 of file const-arpa-lm.h.

Referenced by ConstArpaLmDeterministicFst::ConstArpaLmDeterministicFst().

276 { return bos_symbol_; }

kaldi::ConstArpaLm::bos_symbol_

int32 bos_symbol_

Definition: const-arpa-lm.h:332

◆ DecodeChildInfo()

void DecodeChildInfo	(	const int32	child_info,
		int32 *	parent,
		int32 **	child_lm_state,
		float *	logprob
	)		const

private

Definition at line 882 of file const-arpa-lm.cc.

References Int32AndFloat::f, and KALDI_ASSERT.

                                                         {
   KALDI_ASSERT(initialized_);
 
   KALDI_ASSERT(logprob != NULL);
   if (child_info % 2 == 0) {
     // Child is a leaf, only returns the log probability.
     *child_lm_state = NULL;
     Int32AndFloat logprob_i(child_info);
     *logprob = logprob_i.f;
   } else {
     int32 child_offset = child_info / 2;
     if (child_offset > 0) {
       *child_lm_state = parent + child_offset;
       Int32AndFloat logprob_i(**child_lm_state);
       *logprob = logprob_i.f;
     } else {
       KALDI_ASSERT(-child_offset < overflow_buffer_size_);
       *child_lm_state = overflow_buffer_[-child_offset];
       Int32AndFloat logprob_i(**child_lm_state);
       *logprob = logprob_i.f;
     }
     KALDI_ASSERT(*child_lm_state >= lm_states_);
     KALDI_ASSERT(*child_lm_state <= lm_states_end_);
   }
 }

◆ EosSymbol()

int32 EosSymbol ( ) const

inline

Definition at line 277 of file const-arpa-lm.h.

Referenced by ConstArpaLmDeterministicFst::Final().

277 { return eos_symbol_; }

kaldi::ConstArpaLm::eos_symbol_

int32 eos_symbol_

Definition: const-arpa-lm.h:335

◆ GetChildInfo()

bool GetChildInfo	(	const int32	word,
		int32 *	parent,
		int32 *	child_info
	)		const

private

Definition at line 849 of file const-arpa-lm.cc.

References KALDI_ASSERT.

                                                                        {
   KALDI_ASSERT(initialized_);
 
   KALDI_ASSERT(parent != NULL);
   KALDI_ASSERT(parent >= lm_states_);
   KALDI_ASSERT(child_info != NULL);
 
   KALDI_ASSERT(parent + 2 <= lm_states_end_);
   int32 num_children = *(parent + 2);
   KALDI_ASSERT(parent + 2 + 2 * num_children <= lm_states_end_);
 
   if (num_children == 0) return false;
 
   // A binary search into the children memory block.
   int32 start_index = 1;
   int32 end_index = num_children;
   while (start_index <= end_index) {
     int32 mid_index = round((start_index + end_index) / 2);
     int32 mid_word = *(parent + 1 + 2 * mid_index);
     if (mid_word == word) {
       *child_info = *(parent + 2 + 2 * mid_index);
       return true;
     } else if (mid_word < word) {
       start_index = mid_index + 1;
     } else {
       end_index = mid_index - 1;
     }
   }
 
   return false;
 }

◆ GetLmState()

int32 * GetLmState ( const std::vector< int32 > & seq ) const

private

Definition at line 821 of file const-arpa-lm.cc.

References rnnlm::i, KALDI_ASSERT, and ArpaLine::logprob.

                                                                 {
   KALDI_ASSERT(initialized_);
 
   // No LmState exists for empty word sequence.
   if (seq.size() == 0) return NULL;
 
   // If <unk> is defined, then the word sequence should have already been mapped
   // to <unk> is necessary; this is for the case where <unk> is not defined.
   if (seq[0] >= num_words_ || unigram_states_[seq[0]] == NULL) return NULL;
   int32* parent = unigram_states_[seq[0]];
 
   int32 child_info;
   int32* child_lm_state = NULL;
   float logprob;
   for (int32 i = 1; i < seq.size(); ++i) {
     if (!GetChildInfo(seq[i], parent, &child_info)) {
       return NULL;
     }
     DecodeChildInfo(child_info, parent, &child_lm_state, &logprob);
     if (child_lm_state == NULL) {
       return NULL;
     } else {
       parent = child_lm_state;
     }
   }
   return parent;
 }

◆ GetNgramLogprob()

float GetNgramLogprob	(	const int32	word,
		const std::vector< int32 > &	hist
	)		const

Definition at line 748 of file const-arpa-lm.cc.

References rnnlm::i, and KALDI_ASSERT.

Referenced by ConstArpaLmDeterministicFst::Final(), and ConstArpaLmDeterministicFst::GetArc().

                                                                        {
   KALDI_ASSERT(initialized_);
 
   // If the history size plus one is larger than <ngram_order_>, remove the old
   // words.
   std::vector<int32> mapped_hist(hist);
   while (mapped_hist.size() >= ngram_order_) {
     mapped_hist.erase(mapped_hist.begin(), mapped_hist.begin() + 1);
   }
   KALDI_ASSERT(mapped_hist.size() + 1 <= ngram_order_);
 
   // TODO(guoguo): check with Dan if this is reasonable.
   // Maps possible out-of-vocabulary words to <unk>. If a word does not have a
   // corresponding LmState, we treat it as <unk>. We map it to <unk> if <unk> is
   // specified.
   int32 mapped_word = word;
   if (unk_symbol_ != -1) {
     KALDI_ASSERT(mapped_word >= 0);
     if (mapped_word >= num_words_ || unigram_states_[mapped_word] == NULL) {
       mapped_word = unk_symbol_;
     }
     for (int32 i = 0; i < mapped_hist.size(); ++i) {
       KALDI_ASSERT(mapped_hist[i] >= 0);
       if (mapped_hist[i] >= num_words_ ||
           unigram_states_[mapped_hist[i]] == NULL) {
         mapped_hist[i] = unk_symbol_;
       }
     }
   }
 
   // Loops up n-gram probability.
   return GetNgramLogprobRecurse(mapped_word, mapped_hist);
 }

◆ GetNgramLogprobRecurse()

float GetNgramLogprobRecurse	(	const int32	word,
		const std::vector< int32 > &	hist
	)		const

private

Definition at line 783 of file const-arpa-lm.cc.

References ArpaLine::backoff_logprob, Int32AndFloat::f, KALDI_ASSERT, and ArpaLine::logprob.

                                                           {
   KALDI_ASSERT(initialized_);
   KALDI_ASSERT(hist.size() + 1 <= ngram_order_);
 
   // Unigram case.
   if (hist.size() == 0) {
     if (word >= num_words_ || unigram_states_[word] == NULL) {
       // If <unk> is defined, then the word sequence should have already been
       // mapped to <unk> is necessary; this is for the case where <unk> is not
       // defined.
       return std::numeric_limits<float>::min();
     } else {
       Int32AndFloat logprob_i(*unigram_states_[word]);
       return logprob_i.f;
     }
   }
 
   // High n-gram orders.
   float logprob = 0.0;
   float backoff_logprob = 0.0;
   int32* state;
   if ((state = GetLmState(hist)) != NULL) {
     int32 child_info;
     int32* child_lm_state = NULL;
     if (GetChildInfo(word, state, &child_info)) {
       DecodeChildInfo(child_info, state, &child_lm_state, &logprob);
       return logprob;
     } else {
       Int32AndFloat backoff_logprob_i(*(state + 1));
       backoff_logprob = backoff_logprob_i.f;
     }
   }
   std::vector<int32> new_hist(hist);
   new_hist.erase(new_hist.begin(), new_hist.begin() + 1);
   return backoff_logprob + GetNgramLogprobRecurse(word, new_hist);
 }

◆ HistoryStateExists()

bool HistoryStateExists ( const std::vector< int32 > & hist ) const

Definition at line 721 of file const-arpa-lm.cc.

References KALDI_ASSERT.

Referenced by ConstArpaLmDeterministicFst::GetArc().

                                                                        {
   // We do not create LmState for empty word sequence, but technically it is the
   // history state of all unigrams.
   if (hist.size() == 0) {
     return true;
   }
 
   // Tries to locate the LmState of the given word sequence.
   int32* lm_state = GetLmState(hist);
   if (lm_state == NULL) {
     // <lm_state> does not exist means <hist> has no child.
     return false;
   } else {
     // Note that we always create LmState for unigrams, so even if <lm_state> is
     // not NULL, we still have to check if it has child.
     KALDI_ASSERT(lm_state >= lm_states_);
     KALDI_ASSERT(lm_state + 2 <= lm_states_end_);
     // <lm_state + 2> points to <num_children>.
     if (*(lm_state + 2) > 0) {
       return true;
     } else {
       return false;
     }
   }
   return true;
 }

◆ NgramOrder()

int32 NgramOrder ( ) const

inline

Definition at line 279 of file const-arpa-lm.h.

References logprob.

Referenced by ConstArpaLmDeterministicFst::GetArc().

279 { return ngram_order_; }

kaldi::ConstArpaLm::ngram_order_

int32 ngram_order_

Definition: const-arpa-lm.h:342

◆ Read()

void Read	(	std::istream &	is,
		bool	binary
	)

Definition at line 573 of file const-arpa-lm.cc.

References KALDI_ASSERT, and KALDI_ERR.

                                                   {
   KALDI_ASSERT(!initialized_);
   if (!binary) {
     KALDI_ERR << "text-mode reading is not implemented for ConstArpaLm.";
   }
 
   int first_char = is.peek();
   if (first_char == 4) {  // Old on-disk format starts with length of int32.
     ReadInternalOldFormat(is, binary);
   } else {                // New on-disk format starts with token <ConstArpaLm>.
     ReadInternal(is, binary);
   }
 }

◆ ReadInternal()

void ReadInternal	(	std::istream &	is,
		bool	binary
	)

private

Definition at line 587 of file const-arpa-lm.cc.

References kaldi::ExpectToken(), rnnlm::i, KALDI_ASSERT, KALDI_ERR, and kaldi::ReadBasicType().

                                                           {
   KALDI_ASSERT(!initialized_);
   if (!binary) {
     KALDI_ERR << "text-mode reading is not implemented for ConstArpaLm.";
   }
 
   ExpectToken(is, binary, "<ConstArpaLm>");
 
   // Misc info.
   ExpectToken(is, binary, "<LmInfo>");
   ReadBasicType(is, binary, &bos_symbol_);
   ReadBasicType(is, binary, &eos_symbol_);
   ReadBasicType(is, binary, &unk_symbol_);
   ReadBasicType(is, binary, &ngram_order_);
   ExpectToken(is, binary, "</LmInfo>");
 
   // LmStates section.
   ExpectToken(is, binary, "<LmStates>");
   ReadBasicType(is, binary, &lm_states_size_);
   lm_states_ = new int32[lm_states_size_];
   is.read(reinterpret_cast<char *>(lm_states_),
           sizeof(int32) * lm_states_size_);
   if (!is.good()) {
     KALDI_ERR << "ConstArpaLm <LmStates> section reading failed.";
   }
   ExpectToken(is, binary, "</LmStates>");
 
   // Unigram section. We write memory offset to disk instead of the absolute
   // pointers.
   ExpectToken(is, binary, "<LmUnigram>");
   ReadBasicType(is, binary, &num_words_);
   unigram_states_ = new int32*[num_words_];
   int64* tmp_unigram_address = new int64[num_words_];
   is.read(reinterpret_cast<char *>(tmp_unigram_address),
           sizeof(int64) * num_words_);
   if (!is.good()) {
     KALDI_ERR << "ConstArpaLm <LmUnigram> section reading failed.";
   }
   for (int32 i = 0; i < num_words_; ++i) {
     // Check out how we compute the relative address in ConstArpaLm::Write().
     unigram_states_[i] = (tmp_unigram_address[i] == 0) ? NULL
         : lm_states_ + tmp_unigram_address[i] - 1;
   }
   delete[] tmp_unigram_address;
   tmp_unigram_address = NULL;
   ExpectToken(is, binary, "</LmUnigram>");
 
   // Overflow section. We write memory offset to disk instead of the absolute
   // pointers.
   ExpectToken(is, binary, "<LmOverflow>");
   ReadBasicType(is, binary, &overflow_buffer_size_);
   overflow_buffer_ = new int32*[overflow_buffer_size_];
   int64* tmp_overflow_address = new int64[overflow_buffer_size_];
   is.read(reinterpret_cast<char *>(tmp_overflow_address),
           sizeof(int64) * overflow_buffer_size_);
   if (!is.good()) {
     KALDI_ERR << "ConstArpaLm <LmOverflow> section reading failed.";
   }
   for (int32 i = 0; i < overflow_buffer_size_; ++i) {
     // Check out how we compute the relative address in ConstArpaLm::Write().
     overflow_buffer_[i] = (tmp_overflow_address[i] == 0) ? NULL
         : lm_states_ + tmp_overflow_address[i] - 1;
   }
   delete[] tmp_overflow_address;
   tmp_overflow_address = NULL;
   ExpectToken(is, binary, "</LmOverflow>");
   ExpectToken(is, binary, "</ConstArpaLm>");
 
   KALDI_ASSERT(ngram_order_ > 0);
   KALDI_ASSERT(bos_symbol_ < num_words_ && bos_symbol_ > 0);
   KALDI_ASSERT(eos_symbol_ < num_words_ && eos_symbol_ > 0);
   KALDI_ASSERT(unk_symbol_ < num_words_ &&
                (unk_symbol_ > 0 || unk_symbol_ == -1));
   lm_states_end_ = lm_states_ + lm_states_size_ - 1;
   memory_assigned_ = true;
   initialized_ = true;
 }

◆ ReadInternalOldFormat()

void ReadInternalOldFormat	(	std::istream &	is,
		bool	binary
	)

private

Definition at line 665 of file const-arpa-lm.cc.

References rnnlm::i, KALDI_ASSERT, KALDI_ERR, and kaldi::ReadBasicType().

                                                                    {
   KALDI_ASSERT(!initialized_);
   if (!binary) {
     KALDI_ERR << "text-mode reading is not implemented for ConstArpaLm.";
   }
 
   // Misc info.
   ReadBasicType(is, binary, &bos_symbol_);
   ReadBasicType(is, binary, &eos_symbol_);
   ReadBasicType(is, binary, &unk_symbol_);
   ReadBasicType(is, binary, &ngram_order_);
 
   // LmStates section.
   // In the deprecated version, <lm_states_size_> used to be type of int32,
   // which was a bug. We therefore use int32 for read for back-compatibility.
   int32 lm_states_size_int32;
   ReadBasicType(is, binary, &lm_states_size_int32);
   lm_states_size_ = static_cast<int64>(lm_states_size_int32);
   lm_states_ = new int32[lm_states_size_];
   for (int64 i = 0; i < lm_states_size_; ++i) {
     ReadBasicType(is, binary, &lm_states_[i]);
   }
 
   // Unigram section. We write memory offset to disk instead of the absolute
   // pointers.
   ReadBasicType(is, binary, &num_words_);
   unigram_states_ = new int32*[num_words_];
   for (int32 i = 0; i < num_words_; ++i) {
     int64 tmp_address;
     ReadBasicType(is, binary, &tmp_address);
     // Check out how we compute the relative address in ConstArpaLm::Write().
     unigram_states_[i] =
         (tmp_address == 0) ? NULL : lm_states_ + tmp_address - 1;
   }
 
   // Overflow section. We write memory offset to disk instead of the absolute
   // pointers.
   ReadBasicType(is, binary, &overflow_buffer_size_);
   overflow_buffer_ = new int32*[overflow_buffer_size_];
   for (int32 i = 0; i < overflow_buffer_size_; ++i) {
     int64 tmp_address;
     ReadBasicType(is, binary, &tmp_address);
     // Check out how we compute the relative address in ConstArpaLm::Write().
     overflow_buffer_[i] =
         (tmp_address == 0) ? NULL : lm_states_ + tmp_address - 1;
   }
   KALDI_ASSERT(ngram_order_ > 0);
   KALDI_ASSERT(bos_symbol_ < num_words_ && bos_symbol_ > 0);
   KALDI_ASSERT(eos_symbol_ < num_words_ && eos_symbol_ > 0);
   KALDI_ASSERT(unk_symbol_ < num_words_ &&
                (unk_symbol_ > 0 || unk_symbol_ == -1));
   lm_states_end_ = lm_states_ + lm_states_size_ - 1;
   memory_assigned_ = true;
   initialized_ = true;
 }

◆ UnkSymbol()

int32 UnkSymbol ( ) const

inline

Definition at line 278 of file const-arpa-lm.h.

278 { return unk_symbol_; }

kaldi::ConstArpaLm::unk_symbol_

int32 unk_symbol_

Definition: const-arpa-lm.h:339

◆ Write()

void Write	(	std::ostream &	os,
		bool	binary
	)		const

Definition at line 497 of file const-arpa-lm.cc.

References rnnlm::i, KALDI_ASSERT, KALDI_ERR, kaldi::WriteBasicType(), and kaldi::WriteToken().

Referenced by ConstArpaLmBuilder::Write().

                                                          {
   KALDI_ASSERT(initialized_);
   if (!binary) {
     KALDI_ERR << "text-mode writing is not implemented for ConstArpaLm.";
   }
 
   WriteToken(os, binary, "<ConstArpaLm>");
 
   // Misc info.
   WriteToken(os, binary, "<LmInfo>");
   WriteBasicType(os, binary, bos_symbol_);
   WriteBasicType(os, binary, eos_symbol_);
   WriteBasicType(os, binary, unk_symbol_);
   WriteBasicType(os, binary, ngram_order_);
   WriteToken(os, binary, "</LmInfo>");
 
   // LmStates section.
   WriteToken(os, binary, "<LmStates>");
   WriteBasicType(os, binary, lm_states_size_);
   os.write(reinterpret_cast<char *>(lm_states_),
            sizeof(int32) * lm_states_size_);
   if (!os.good()) {
     KALDI_ERR << "ConstArpaLm <LmStates> section writing failed.";
   }
   WriteToken(os, binary, "</LmStates>");
 
   // Unigram section. We write memory offset to disk instead of the absolute
   // pointers.
   WriteToken(os, binary, "<LmUnigram>");
   WriteBasicType(os, binary, num_words_);
   int64* tmp_unigram_address = new int64[num_words_];
   for (int32 i = 0; i < num_words_; ++i) {
     // The relative address here is a little bit tricky:
     // 1. If the original address is NULL, then we set the relative address to
     //    zero.
     // 2. If the original address is not NULL, we set it to the following:
     //      unigram_states_[i] - lm_states_ + 1
     //    we plus 1 to ensure that the above value is positive.
     tmp_unigram_address[i] = (unigram_states_[i] == NULL) ? 0 :
         unigram_states_[i] - lm_states_ + 1;
   }
   os.write(reinterpret_cast<char *>(tmp_unigram_address),
            sizeof(int64) * num_words_);
   if (!os.good()) {
     KALDI_ERR << "ConstArpaLm <LmUnigram> section writing failed.";
   }
   delete[] tmp_unigram_address;   // Releases the memory.
   tmp_unigram_address = NULL;
   WriteToken(os, binary, "</LmUnigram>");
 
   // Overflow section. We write memory offset to disk instead of the absolute
   // pointers.
   WriteToken(os, binary, "<LmOverflow>");
   WriteBasicType(os, binary, overflow_buffer_size_);
   int64* tmp_overflow_address = new int64[overflow_buffer_size_];
   for (int32 i = 0; i < overflow_buffer_size_; ++i) {
     // The relative address here is a little bit tricky:
     // 1. If the original address is NULL, then we set the relative address to
     //    zero.
     // 2. If the original address is not NULL, we set it to the following:
     //      overflow_buffer_[i] - lm_states_ + 1
     //    we plus 1 to ensure that the above value is positive.
     tmp_overflow_address[i] = (overflow_buffer_[i] == NULL) ? 0 :
         overflow_buffer_[i] - lm_states_ + 1;
   }
   os.write(reinterpret_cast<char *>(tmp_overflow_address),
            sizeof(int64) * overflow_buffer_size_);
   if (!os.good()) {
     KALDI_ERR << "ConstArpaLm <LmOverflow> section writing failed.";
   }
   delete[] tmp_overflow_address;
   tmp_overflow_address = NULL;
   WriteToken(os, binary, "</LmOverflow>");
   WriteToken(os, binary, "</ConstArpaLm>");
 }

◆ WriteArpa()

void WriteArpa ( std::ostream & os ) const

Definition at line 952 of file const-arpa-lm.cc.

References ArpaLine::backoff_logprob, rnnlm::i, rnnlm::j, KALDI_ASSERT, and ArpaLine::words.

                                                 {
   KALDI_ASSERT(initialized_);
 
   std::vector<ArpaLine> tmp_output;
   for (int32 i = 0; i < num_words_; ++i) {
     if (unigram_states_[i] != NULL) {
       std::vector<int32> seq(1, i);
       WriteArpaRecurse(unigram_states_[i], seq, &tmp_output);
     }
   }
 
   // Sorts ArpaLines and collects head information.
   std::sort(tmp_output.begin(), tmp_output.end());
   std::vector<int32> ngram_count(1, 0);
   for (int32 i = 0; i < tmp_output.size(); ++i) {
     if (tmp_output[i].words.size() >= ngram_count.size()) {
       ngram_count.resize(tmp_output[i].words.size() + 1);
       ngram_count[tmp_output[i].words.size()] = 1;
     } else {
       ngram_count[tmp_output[i].words.size()] += 1;
     }
   }
 
   // Writes the header.
   os << std::endl;
   os << "\\data\\" << std::endl;
   for (int32 i = 1; i < ngram_count.size(); ++i) {
     os << "ngram " << i << "=" << ngram_count[i] << std::endl;
   }
 
   // Writes n-grams.
   int32 current_order = 0;
   for (int32 i = 0; i < tmp_output.size(); ++i) {
     // Beginning of a n-gram section.
     if (tmp_output[i].words.size() != current_order) {
       current_order = tmp_output[i].words.size();
       os << std::endl;
       os << "\\" << current_order << "-grams:" << std::endl;
     }
 
     // Writes logprob.
     os << tmp_output[i].logprob << '\t';
 
     // Writes word sequence.
     for (int32 j = 0; j < tmp_output[i].words.size(); ++j) {
       os << tmp_output[i].words[j];
       if (j != tmp_output[i].words.size() - 1) {
         os << " ";
       }
     }
 
     // Writes backoff_logprob if it is not zero.
     if (tmp_output[i].backoff_logprob != 0.0) {
       os << '\t' << tmp_output[i].backoff_logprob;
     }
     os << std::endl;
   }
 
   os << std::endl << "\\end\\" << std::endl;
 }

◆ WriteArpaRecurse()

void WriteArpaRecurse	(	int32 *	lm_state,
		const std::vector< int32 > &	seq,
		std::vector< ArpaLine > *	output
	)		const

private

Definition at line 911 of file const-arpa-lm.cc.

References ArpaLine::backoff_logprob, Int32AndFloat::f, rnnlm::i, KALDI_ASSERT, ArpaLine::logprob, and ArpaLine::words.

                                                                       {
   if (lm_state == NULL) return;
 
   KALDI_ASSERT(lm_state >= lm_states_);
   KALDI_ASSERT(lm_state + 2 <= lm_states_end_);
 
   // Inserts the current LmState to <output>.
   ArpaLine arpa_line;
   arpa_line.words = seq;
   Int32AndFloat logprob_i(*lm_state);
   arpa_line.logprob = logprob_i.f;
   Int32AndFloat backoff_logprob_i(*(lm_state + 1));
   arpa_line.backoff_logprob = backoff_logprob_i.f;
   output->push_back(arpa_line);
 
   // Scans for possible children, and recursively adds child to <output>.
   int32 num_children = *(lm_state + 2);
   KALDI_ASSERT(lm_state + 2 + 2 * num_children <= lm_states_end_);
   for (int32 i = 0; i < num_children; ++i) {
     std::vector<int32> new_seq(seq);
     new_seq.push_back(*(lm_state + 3 + 2 * i));
     int32 child_info = *(lm_state + 4 + 2 * i);
     float logprob;
     int32* child_lm_state = NULL;
     DecodeChildInfo(child_info, lm_state, &child_lm_state, &logprob);
 
     if (child_lm_state == NULL) {
       // Leaf case.
       ArpaLine child_arpa_line;
       child_arpa_line.words = new_seq;
       child_arpa_line.logprob = logprob;
       child_arpa_line.backoff_logprob = 0.0;
       output->push_back(child_arpa_line);
     } else {
       WriteArpaRecurse(child_lm_state, new_seq, output);
     }
   }
 }

Member Data Documentation

◆ bos_symbol_

int32 bos_symbol_

private

Definition at line 332 of file const-arpa-lm.h.

◆ eos_symbol_

int32 eos_symbol_

private

Definition at line 335 of file const-arpa-lm.h.

◆ initialized_

bool initialized_

private

Definition at line 329 of file const-arpa-lm.h.

◆ lm_states_

int32* lm_states_

private

Definition at line 384 of file const-arpa-lm.h.

◆ lm_states_end_

int32* lm_states_end_

private

Definition at line 357 of file const-arpa-lm.h.

◆ lm_states_size_

int64 lm_states_size_

private

Definition at line 353 of file const-arpa-lm.h.

◆ memory_assigned_

bool memory_assigned_

private

Definition at line 326 of file const-arpa-lm.h.

◆ ngram_order_

int32 ngram_order_

private

Definition at line 342 of file const-arpa-lm.h.

◆ num_words_

int32 num_words_

private

Definition at line 346 of file const-arpa-lm.h.

◆ overflow_buffer_

int32** overflow_buffer_

private

Definition at line 368 of file const-arpa-lm.h.

◆ overflow_buffer_size_

int32 overflow_buffer_size_

private

Definition at line 350 of file const-arpa-lm.h.

◆ unigram_states_

int32** unigram_states_

private

Definition at line 362 of file const-arpa-lm.h.

◆ unk_symbol_

int32 unk_symbol_

private

Definition at line 339 of file const-arpa-lm.h.

The documentation for this class was generated from the following files:

lm/const-arpa-lm.h
lm/const-arpa-lm.cc

Public Member Functions

Private Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

◆ ConstArpaLm() [1/2]

◆ ConstArpaLm() [2/2]

◆ ~ConstArpaLm()

Member Function Documentation

◆ BosSymbol()

◆ DecodeChildInfo()

◆ EosSymbol()

◆ GetChildInfo()

◆ GetLmState()

◆ GetNgramLogprob()

◆ GetNgramLogprobRecurse()

◆ HistoryStateExists()

◆ NgramOrder()

◆ Read()

◆ ReadInternal()

◆ ReadInternalOldFormat()

◆ UnkSymbol()

◆ Write()

◆ WriteArpa()

◆ WriteArpaRecurse()

Member Data Documentation

◆ bos_symbol_

◆ eos_symbol_

◆ initialized_

◆ lm_states_

◆ lm_states_end_

◆ lm_states_size_

◆ memory_assigned_

◆ ngram_order_

◆ num_words_

◆ overflow_buffer_

◆ overflow_buffer_size_

◆ unigram_states_

◆ unk_symbol_