nnet3-subset-egs.cc File Reference
Include dependency graph for nnet3-subset-egs.cc:

Go to the source code of this file.

Functions

int main (int argc, char *argv[])
 

Function Documentation

◆ main()

int main ( int  argc,
char *  argv[] 
)

Definition at line 25 of file nnet3-subset-egs.cc.

References SequentialTableReader< Holder >::Done(), ParseOptions::GetArg(), rnnlm::i, KALDI_LOG, SequentialTableReader< Holder >::Key(), rnnlm::n, SequentialTableReader< Holder >::Next(), ParseOptions::NumArgs(), ParseOptions::PrintUsage(), kaldi::RandInt(), ParseOptions::Read(), ParseOptions::Register(), SequentialTableReader< Holder >::Value(), and kaldi::WithProb().

25  {
26  try {
27  using namespace kaldi;
28  using namespace kaldi::nnet3;
29  typedef kaldi::int32 int32;
30  typedef kaldi::int64 int64;
31 
32  const char *usage =
33  "Creates a random subset of the input examples, of a specified size.\n"
34  "Uses no more memory than the size of the subset.\n"
35  "\n"
36  "Usage: nnet3-subset-egs [options] <egs-rspecifier> [<egs-wspecifier2> ...]\n"
37  "\n"
38  "e.g.\n"
39  "nnet3-copy-egs [args] ark:egs.1.ark ark:- | nnet-subset-egs --n=1000 ark:- ark:subset.egs\n";
40 
41  int32 srand_seed = 0;
42  int32 n = 1000;
43  bool randomize_order = true;
44  ParseOptions po(usage);
45  po.Register("srand", &srand_seed, "Seed for random number generator ");
46  po.Register("n", &n, "Number of examples to output");
47  po.Register("randomize-order", &randomize_order, "If true, randomize the order "
48  "of the output");
49 
50  po.Read(argc, argv);
51 
52  srand(srand_seed);
53 
54  if (po.NumArgs() != 2) {
55  po.PrintUsage();
56  exit(1);
57  }
58 
59  std::string examples_rspecifier = po.GetArg(1),
60  examples_wspecifier = po.GetArg(2);
61 
62  std::vector<std::pair<std::string, NnetExample> > egs;
63  egs.reserve(n);
64 
65  SequentialNnetExampleReader example_reader(examples_rspecifier);
66 
67  int64 num_read = 0;
68  for (; !example_reader.Done(); example_reader.Next()) {
69  num_read++;
70  if (num_read <= n) {
71  egs.resize(egs.size() + 1);
72  egs.back().first = example_reader.Key();
73  egs.back().second = example_reader.Value();
74  } else {
75  BaseFloat keep_prob = n / static_cast<BaseFloat>(num_read);
76  if (WithProb(keep_prob)) { // With probability "keep_prob"
77  int32 index = RandInt(0, n-1);
78  egs[index].first = example_reader.Key();
79  egs[index].second = example_reader.Value();
80  }
81  }
82  }
83  if (randomize_order)
84  std::random_shuffle(egs.begin(), egs.end());
85 
86  NnetExampleWriter writer(examples_wspecifier);
87  for (size_t i = 0; i < egs.size(); i++) {
88  writer.Write(egs[i].first, egs[i].second);
89  }
90 
91  KALDI_LOG << "Selected a subset of " << egs.size() << " out of " << num_read
92  << " neural-network training examples ";
93 
94  return (num_read != 0 ? 0 : 1);
95  } catch(const std::exception &e) {
96  std::cerr << e.what() << '\n';
97  return -1;
98  }
99 }
This code computes Goodness of Pronunciation (GOP) and extracts phone-level pronunciation feature for...
Definition: chain.dox:20
bool WithProb(BaseFloat prob, struct RandomState *state)
Definition: kaldi-math.cc:72
A templated class for writing objects to an archive or script file; see The Table concept...
Definition: kaldi-table.h:368
kaldi::int32 int32
float BaseFloat
Definition: kaldi-types.h:29
The class ParseOptions is for parsing command-line options; see Parsing command-line options for more...
Definition: parse-options.h:36
struct rnnlm::@11::@12 n
A templated class for reading objects sequentially from an archive or script file; see The Table conc...
Definition: kaldi-table.h:287
#define KALDI_LOG
Definition: kaldi-error.h:153
int32 RandInt(int32 min_val, int32 max_val, struct RandomState *state)
Definition: kaldi-math.cc:95