This class is responsible for arranging examples in groups that have the same strucure (i.e. More...

#include <nnet-example-utils.h>

Collaboration diagram for ExampleMerger:

[legend]

Public Member Functions
	ExampleMerger (const ExampleMergingConfig &config, NnetExampleWriter *writer)

void	AcceptExample (NnetExample *a)

void	Finish ()

int32	ExitStatus ()

	~ExampleMerger ()

Private Types
typedef unordered_map< NnetExample , std::vector< NnetExample >, NnetExampleStructureHasher, NnetExampleStructureCompare >	MapType

Private Member Functions
void	WriteMinibatch (const std::vector< NnetExample > &egs)

Private Attributes
bool	finished_

int32	num_egs_written_

const ExampleMergingConfig &	config_

NnetExampleWriter *	writer_

ExampleMergingStats	stats_

MapType	eg_to_egs_

Detailed Description

This class is responsible for arranging examples in groups that have the same strucure (i.e.

the same input and output indexes), and outputting them in suitable minibatches as defined by ExampleMergingConfig.

Definition at line 480 of file nnet-example-utils.h.

Member Typedef Documentation

◆ MapType

typedef unordered_map<NnetExample*, std::vector<NnetExample*>, NnetExampleStructureHasher, NnetExampleStructureCompare> MapType

private

Definition at line 515 of file nnet-example-utils.h.

Constructor & Destructor Documentation

◆ ExampleMerger()

ExampleMerger	(	const ExampleMergingConfig &	config,
		NnetExampleWriter *	writer
	)

Definition at line 1188 of file nnet-example-utils.cc.

                                                        :
     finished_(false), num_egs_written_(0),
     config_(config), writer_(writer) { }

◆ ~ExampleMerger()

~ExampleMerger ( )

inline

Definition at line 500 of file nnet-example-utils.h.

500 { Finish(); };

kaldi::nnet3::ExampleMerger::Finish

void Finish()

Definition: nnet-example-utils.cc:1239

Member Function Documentation

◆ AcceptExample()

void AcceptExample ( NnetExample * a )

Definition at line 1194 of file nnet-example-utils.cc.

References ExampleMerger::config_, ExampleMerger::eg_to_egs_, ExampleMerger::finished_, kaldi::nnet3::GetNnetExampleSize(), rnnlm::i, KALDI_ASSERT, ExampleMergingConfig::MinibatchSize(), and ExampleMerger::WriteMinibatch().

Referenced by main().

                                                  {
   KALDI_ASSERT(!finished_);
   // If an eg with the same structure as 'eg' is already a key in the
   // map, it won't be replaced, but if it's new it will be made
   // the key.  Also we remove the key before making the vector empty.
   // This way we ensure that the eg in the key is always the first
   // element of the vector.
   std::vector<NnetExample*> &vec = eg_to_egs_[eg];
   vec.push_back(eg);
   int32 eg_size = GetNnetExampleSize(*eg),
       num_available = vec.size();
   bool input_ended = false;
   int32 minibatch_size = config_.MinibatchSize(eg_size, num_available,
                                                input_ended);
   if (minibatch_size != 0) {  // we need to write out a merged eg.
     KALDI_ASSERT(minibatch_size == num_available);
 
     std::vector<NnetExample*> vec_copy(vec);
     eg_to_egs_.erase(eg);
 
     // MergeExamples() expects a vector of NnetExample, not of pointers,
     // so use swap to create that without doing any real work.
     std::vector<NnetExample> egs_to_merge(minibatch_size);
     for (int32 i = 0; i < minibatch_size; i++) {
       egs_to_merge[i].Swap(vec_copy[i]);
       delete vec_copy[i];  // we owned those pointers.
     }
     WriteMinibatch(egs_to_merge);
   }
 }

◆ ExitStatus()

int32 ExitStatus ( )

inline

Definition at line 498 of file nnet-example-utils.h.

Referenced by main().

498 { Finish(); return (num_egs_written_ > 0 ? 0 : 1); }

kaldi::nnet3::ExampleMerger::Finish

void Finish()

Definition: nnet-example-utils.cc:1239

kaldi::nnet3::ExampleMerger::num_egs_written_

int32 num_egs_written_

Definition: nnet-example-utils.h:507

◆ Finish()

void Finish ( )

Definition at line 1239 of file nnet-example-utils.cc.

References ExampleMerger::config_, ExampleMergingStats::DiscardedExamples(), ExampleMerger::eg_to_egs_, ExampleMerger::finished_, kaldi::nnet3::GetNnetExampleSize(), rnnlm::i, KALDI_ASSERT, ExampleMergingConfig::MinibatchSize(), ExampleMergingStats::PrintStats(), ExampleMerger::stats_, and ExampleMerger::WriteMinibatch().

Referenced by main().

                            {
   if (finished_) return;  // already finished.
   finished_ = true;
 
   // we'll convert the map eg_to_egs_ to a vector of vectors to avoid
   // iterator invalidation problems.
   std::vector<std::vector<NnetExample*> > all_egs;
   all_egs.reserve(eg_to_egs_.size());
 
   MapType::iterator iter = eg_to_egs_.begin(), end = eg_to_egs_.end();
   for (; iter != end; ++iter)
     all_egs.push_back(iter->second);
   eg_to_egs_.clear();
 
   for (size_t i = 0; i < all_egs.size(); i++) {
     int32 minibatch_size;
     std::vector<NnetExample*> &vec = all_egs[i];
     KALDI_ASSERT(!vec.empty());
     int32 eg_size = GetNnetExampleSize(*(vec[0]));
     bool input_ended = true;
     while (!vec.empty() &&
            (minibatch_size = config_.MinibatchSize(eg_size, vec.size(),
                                                    input_ended)) != 0) {
       // MergeExamples() expects a vector of NnetExample, not of pointers,
       // so use swap to create that without doing any real work.
       std::vector<NnetExample> egs_to_merge(minibatch_size);
       for (int32 i = 0; i < minibatch_size; i++) {
         egs_to_merge[i].Swap(vec[i]);
         delete vec[i];  // we owned those pointers.
       }
       vec.erase(vec.begin(), vec.begin() + minibatch_size);
       WriteMinibatch(egs_to_merge);
     }
     if (!vec.empty()) {
       int32 eg_size = GetNnetExampleSize(*(vec[0]));
       NnetExampleStructureHasher eg_hasher;
       size_t structure_hash = eg_hasher(*(vec[0]));
       int32 num_discarded = vec.size();
       stats_.DiscardedExamples(eg_size, structure_hash, num_discarded);
       for (int32 i = 0; i < num_discarded; i++)
         delete vec[i];
       vec.clear();
     }
   }
   stats_.PrintStats();
 }

◆ WriteMinibatch()

void WriteMinibatch ( const std::vector< NnetExample > & egs )

private

Definition at line 1225 of file nnet-example-utils.cc.

References ExampleMergingConfig::compress, ExampleMerger::config_, kaldi::nnet3::GetNnetExampleSize(), KALDI_ASSERT, kaldi::nnet3::MergeExamples(), ExampleMerger::num_egs_written_, ExampleMerger::stats_, TableWriter< Holder >::Write(), ExampleMerger::writer_, and ExampleMergingStats::WroteExample().

Referenced by ExampleMerger::AcceptExample(), and ExampleMerger::Finish().

                                                                     {
   KALDI_ASSERT(!egs.empty());
   int32 eg_size = GetNnetExampleSize(egs[0]);
   NnetExampleStructureHasher eg_hasher;
   size_t structure_hash = eg_hasher(egs[0]);
   int32 minibatch_size = egs.size();
   stats_.WroteExample(eg_size, structure_hash, minibatch_size);
   NnetExample merged_eg;
   MergeExamples(egs, config_.compress, &merged_eg);
   std::ostringstream key;
   key << "merged-" << (num_egs_written_++) << "-" << minibatch_size;
   writer_->Write(key.str(), merged_eg);
 }