This class is responsible for arranging examples in groups that have the same strucure (i.e. More...

#include <nnet-chain-example.h>

Collaboration diagram for ChainExampleMerger:

[legend]

Public Member Functions
	ChainExampleMerger (const ExampleMergingConfig &config, NnetChainExampleWriter *writer)

void	AcceptExample (NnetChainExample *a)

void	Finish ()

int32	ExitStatus ()

	~ChainExampleMerger ()

Private Types
typedef unordered_map< NnetChainExample , std::vector< NnetChainExample >, NnetChainExampleStructureHasher, NnetChainExampleStructureCompare >	MapType

Private Member Functions
void	WriteMinibatch (std::vector< NnetChainExample > *egs)

Private Attributes
bool	finished_

int32	num_egs_written_

const ExampleMergingConfig &	config_

NnetChainExampleWriter *	writer_

ExampleMergingStats	stats_

MapType	eg_to_egs_

Detailed Description

This class is responsible for arranging examples in groups that have the same strucure (i.e.

the same input and output indexes), and outputting them in suitable minibatches as defined by ExampleMergingConfig.

Definition at line 234 of file nnet-chain-example.h.

Member Typedef Documentation

◆ MapType

typedef unordered_map<NnetChainExample*, std::vector<NnetChainExample*>, NnetChainExampleStructureHasher, NnetChainExampleStructureCompare> MapType

private

Definition at line 272 of file nnet-chain-example.h.

Constructor & Destructor Documentation

◆ ChainExampleMerger()

ChainExampleMerger	(	const ExampleMergingConfig &	config,
		NnetChainExampleWriter *	writer
	)

Definition at line 453 of file nnet-chain-example.cc.

                                                                       :
     finished_(false), num_egs_written_(0),
     config_(config), writer_(writer) { }

◆ ~ChainExampleMerger()

~ChainExampleMerger ( )

inline

Definition at line 254 of file nnet-chain-example.h.

254 { Finish(); };

kaldi::nnet3::ChainExampleMerger::Finish

void Finish()

Definition: nnet-chain-example.cc:505

Member Function Documentation

◆ AcceptExample()

void AcceptExample ( NnetChainExample * a )

Definition at line 459 of file nnet-chain-example.cc.

References ChainExampleMerger::config_, ChainExampleMerger::eg_to_egs_, ChainExampleMerger::finished_, kaldi::nnet3::GetNnetChainExampleSize(), rnnlm::i, KALDI_ASSERT, ExampleMergingConfig::MinibatchSize(), and ChainExampleMerger::WriteMinibatch().

                                                            {
   KALDI_ASSERT(!finished_);
   // If an eg with the same structure as 'eg' is already a key in the
   // map, it won't be replaced, but if it's new it will be made
   // the key.  Also we remove the key before making the vector empty.
   // This way we ensure that the eg in the key is always the first
   // element of the vector.
   std::vector<NnetChainExample*> &vec = eg_to_egs_[eg];
   vec.push_back(eg);
   int32 eg_size = GetNnetChainExampleSize(*eg),
       num_available = vec.size();
   bool input_ended = false;
   int32 minibatch_size = config_.MinibatchSize(eg_size, num_available,
                                                input_ended);
   if (minibatch_size != 0) {  // we need to write out a merged eg.
     KALDI_ASSERT(minibatch_size == num_available);
 
     std::vector<NnetChainExample*> vec_copy(vec);
     eg_to_egs_.erase(eg);
 
     // MergeChainExamples() expects a vector of NnetChainExample, not of pointers,
     // so use swap to create that without doing any real work.
     std::vector<NnetChainExample> egs_to_merge(minibatch_size);
     for (int32 i = 0; i < minibatch_size; i++) {
       egs_to_merge[i].Swap(vec_copy[i]);
       delete vec_copy[i];  // we owned those pointers.
     }
     WriteMinibatch(&egs_to_merge);
   }
 }

◆ ExitStatus()

int32 ExitStatus ( )

inline

Definition at line 252 of file nnet-chain-example.h.

252 { Finish(); return (num_egs_written_ > 0 ? 0 : 1); }

kaldi::nnet3::ChainExampleMerger::Finish

void Finish()

Definition: nnet-chain-example.cc:505

kaldi::nnet3::ChainExampleMerger::num_egs_written_

int32 num_egs_written_

Definition: nnet-chain-example.h:263

◆ Finish()

void Finish ( )

Definition at line 505 of file nnet-chain-example.cc.

References ChainExampleMerger::config_, ExampleMergingStats::DiscardedExamples(), ChainExampleMerger::eg_to_egs_, ChainExampleMerger::finished_, kaldi::nnet3::GetNnetChainExampleSize(), rnnlm::i, KALDI_ASSERT, ExampleMergingConfig::MinibatchSize(), ExampleMergingStats::PrintStats(), ChainExampleMerger::stats_, and ChainExampleMerger::WriteMinibatch().

                                 {
   if (finished_) return;  // already finished.
   finished_ = true;
 
   // we'll convert the map eg_to_egs_ to a vector of vectors to avoid
   // iterator invalidation problems.
   std::vector<std::vector<NnetChainExample*> > all_egs;
   all_egs.reserve(eg_to_egs_.size());
 
   MapType::iterator iter = eg_to_egs_.begin(), end = eg_to_egs_.end();
   for (; iter != end; ++iter)
     all_egs.push_back(iter->second);
   eg_to_egs_.clear();
 
   for (size_t i = 0; i < all_egs.size(); i++) {
     int32 minibatch_size;
     std::vector<NnetChainExample*> &vec = all_egs[i];
     KALDI_ASSERT(!vec.empty());
     int32 eg_size = GetNnetChainExampleSize(*(vec[0]));
     bool input_ended = true;
     while (!vec.empty() &&
            (minibatch_size = config_.MinibatchSize(eg_size, vec.size(),
                                                    input_ended)) != 0) {
       // MergeChainExamples() expects a vector of
       // NnetChainExample, not of pointers, so use swap to create that
       // without doing any real work.
       std::vector<NnetChainExample> egs_to_merge(minibatch_size);
       for (int32 i = 0; i < minibatch_size; i++) {
         egs_to_merge[i].Swap(vec[i]);
         delete vec[i];  // we owned those pointers.
       }
       vec.erase(vec.begin(), vec.begin() + minibatch_size);
       WriteMinibatch(&egs_to_merge);
     }
     if (!vec.empty()) {
       int32 eg_size = GetNnetChainExampleSize(*(vec[0]));
       NnetChainExampleStructureHasher eg_hasher;
       size_t structure_hash = eg_hasher(*(vec[0]));
       int32 num_discarded = vec.size();
       stats_.DiscardedExamples(eg_size, structure_hash, num_discarded);
       for (int32 i = 0; i < num_discarded; i++)
         delete vec[i];
       vec.clear();
     }
   }
   stats_.PrintStats();
 }

◆ WriteMinibatch()

void WriteMinibatch ( std::vector< NnetChainExample > * egs )

private

Definition at line 490 of file nnet-chain-example.cc.

References ExampleMergingConfig::compress, ChainExampleMerger::config_, kaldi::nnet3::GetNnetChainExampleSize(), KALDI_ASSERT, kaldi::nnet3::MergeChainExamples(), ChainExampleMerger::num_egs_written_, ChainExampleMerger::stats_, TableWriter< Holder >::Write(), ChainExampleMerger::writer_, and ExampleMergingStats::WroteExample().

Referenced by ChainExampleMerger::AcceptExample(), and ChainExampleMerger::Finish().

                                       {
   KALDI_ASSERT(!egs->empty());
   int32 eg_size = GetNnetChainExampleSize((*egs)[0]);
   NnetChainExampleStructureHasher eg_hasher;
   size_t structure_hash = eg_hasher((*egs)[0]);
   int32 minibatch_size = egs->size();
   stats_.WroteExample(eg_size, structure_hash, minibatch_size);
   NnetChainExample merged_eg;
   MergeChainExamples(config_.compress, egs, &merged_eg);
   std::ostringstream key;
   key << "merged-" << (num_egs_written_++) << "-" << minibatch_size;
   writer_->Write(key.str(), merged_eg);
 }