nnet-subset-egs.cc File Reference
Include dependency graph for nnet-subset-egs.cc:

Go to the source code of this file.

Functions

int main (int argc, char *argv[])
 

Function Documentation

◆ main()

int main ( int  argc,
char *  argv[] 
)

Definition at line 26 of file nnet-subset-egs.cc.

References SequentialTableReader< Holder >::Done(), ParseOptions::GetArg(), rnnlm::i, KALDI_LOG, SequentialTableReader< Holder >::Key(), rnnlm::n, SequentialTableReader< Holder >::Next(), ParseOptions::NumArgs(), ParseOptions::PrintUsage(), kaldi::RandInt(), ParseOptions::Read(), ParseOptions::Register(), SequentialTableReader< Holder >::Value(), and kaldi::WithProb().

26  {
27  try {
28  using namespace kaldi;
29  using namespace kaldi::nnet2;
30  typedef kaldi::int32 int32;
31  typedef kaldi::int64 int64;
32 
33  const char *usage =
34  "Creates a random subset of the input examples, of a specified size.\n"
35  "Uses no more memory than the size of the subset.\n"
36  "\n"
37  "Usage: nnet-subset-egs [options] <egs-rspecifier> [<egs-wspecifier2> ...]\n"
38  "\n"
39  "e.g.\n"
40  "nnet-subset-egs [args] ark:- | nnet-subset-egs --n=1000 ark:- ark:subset.egs\n";
41 
42  int32 srand_seed = 0;
43  int32 n = 1000;
44  bool randomize_order = true;
45  ParseOptions po(usage);
46  po.Register("srand", &srand_seed, "Seed for random number generator ");
47  po.Register("n", &n, "Number of examples to output");
48  po.Register("randomize-order", &randomize_order, "If true, randomize the order "
49  "of the output");
50 
51  po.Read(argc, argv);
52 
53  srand(srand_seed);
54 
55  if (po.NumArgs() != 2) {
56  po.PrintUsage();
57  exit(1);
58  }
59 
60  std::string examples_rspecifier = po.GetArg(1),
61  examples_wspecifier = po.GetArg(2);
62 
63  std::vector<std::pair<std::string, NnetExample> > egs;
64  egs.reserve(n);
65 
66  SequentialNnetExampleReader example_reader(examples_rspecifier);
67 
68  int64 num_read = 0;
69  for (; !example_reader.Done(); example_reader.Next()) {
70  num_read++;
71  if (num_read <= n) {
72  egs.resize(egs.size() + 1);
73  egs.back().first = example_reader.Key();
74  egs.back().second = example_reader.Value();
75  } else {
76  BaseFloat keep_prob = n / static_cast<BaseFloat>(num_read);
77  if (WithProb(keep_prob)) { // With probability "keep_prob"
78  int32 index = RandInt(0, n-1);
79  egs[index].first = example_reader.Key();
80  egs[index].second = example_reader.Value();
81  }
82  }
83  }
84  if (randomize_order)
85  std::random_shuffle(egs.begin(), egs.end());
86 
87  NnetExampleWriter writer(examples_wspecifier);
88  for (size_t i = 0; i < egs.size(); i++) {
89  writer.Write(egs[i].first, egs[i].second);
90  }
91 
92  KALDI_LOG << "Selected a subset of " << egs.size() << " out of " << num_read
93  << " neural-network training examples ";
94 
95  return (num_read != 0 ? 0 : 1);
96  } catch(const std::exception &e) {
97  std::cerr << e.what() << '\n';
98  return -1;
99  }
100 }
This code computes Goodness of Pronunciation (GOP) and extracts phone-level pronunciation feature for...
Definition: chain.dox:20
bool WithProb(BaseFloat prob, struct RandomState *state)
Definition: kaldi-math.cc:72
A templated class for writing objects to an archive or script file; see The Table concept...
Definition: kaldi-table.h:368
kaldi::int32 int32
float BaseFloat
Definition: kaldi-types.h:29
The class ParseOptions is for parsing command-line options; see Parsing command-line options for more...
Definition: parse-options.h:36
struct rnnlm::@11::@12 n
A templated class for reading objects sequentially from an archive or script file; see The Table conc...
Definition: kaldi-table.h:287
#define KALDI_LOG
Definition: kaldi-error.h:153
int32 RandInt(int32 min_val, int32 max_val, struct RandomState *state)
Definition: kaldi-math.cc:95