nnet-subset-egs.cc
Go to the documentation of this file.
1 // nnet2bin/nnet-subset-egs.cc
2 
3 // Copyright 2012 Johns Hopkins University (author: Daniel Povey)
4 // Copyright 2014 Vimal Manohar
5 
6 // See ../../COPYING for clarification regarding multiple authors
7 //
8 // Licensed under the Apache License, Version 2.0 (the "License");
9 // you may not use this file except in compliance with the License.
10 // You may obtain a copy of the License at
11 //
12 // http://www.apache.org/licenses/LICENSE-2.0
13 //
14 // THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 // KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
16 // WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
17 // MERCHANTABLITY OR NON-INFRINGEMENT.
18 // See the Apache 2 License for the specific language governing permissions and
19 // limitations under the License.
20 
21 #include "base/kaldi-common.h"
22 #include "util/common-utils.h"
23 #include "hmm/transition-model.h"
25 
26 int main(int argc, char *argv[]) {
27  try {
28  using namespace kaldi;
29  using namespace kaldi::nnet2;
30  typedef kaldi::int32 int32;
31  typedef kaldi::int64 int64;
32 
33  const char *usage =
34  "Creates a random subset of the input examples, of a specified size.\n"
35  "Uses no more memory than the size of the subset.\n"
36  "\n"
37  "Usage: nnet-subset-egs [options] <egs-rspecifier> [<egs-wspecifier2> ...]\n"
38  "\n"
39  "e.g.\n"
40  "nnet-subset-egs [args] ark:- | nnet-subset-egs --n=1000 ark:- ark:subset.egs\n";
41 
42  int32 srand_seed = 0;
43  int32 n = 1000;
44  bool randomize_order = true;
45  ParseOptions po(usage);
46  po.Register("srand", &srand_seed, "Seed for random number generator ");
47  po.Register("n", &n, "Number of examples to output");
48  po.Register("randomize-order", &randomize_order, "If true, randomize the order "
49  "of the output");
50 
51  po.Read(argc, argv);
52 
53  srand(srand_seed);
54 
55  if (po.NumArgs() != 2) {
56  po.PrintUsage();
57  exit(1);
58  }
59 
60  std::string examples_rspecifier = po.GetArg(1),
61  examples_wspecifier = po.GetArg(2);
62 
63  std::vector<std::pair<std::string, NnetExample> > egs;
64  egs.reserve(n);
65 
66  SequentialNnetExampleReader example_reader(examples_rspecifier);
67 
68  int64 num_read = 0;
69  for (; !example_reader.Done(); example_reader.Next()) {
70  num_read++;
71  if (num_read <= n) {
72  egs.resize(egs.size() + 1);
73  egs.back().first = example_reader.Key();
74  egs.back().second = example_reader.Value();
75  } else {
76  BaseFloat keep_prob = n / static_cast<BaseFloat>(num_read);
77  if (WithProb(keep_prob)) { // With probability "keep_prob"
78  int32 index = RandInt(0, n-1);
79  egs[index].first = example_reader.Key();
80  egs[index].second = example_reader.Value();
81  }
82  }
83  }
84  if (randomize_order)
85  std::random_shuffle(egs.begin(), egs.end());
86 
87  NnetExampleWriter writer(examples_wspecifier);
88  for (size_t i = 0; i < egs.size(); i++) {
89  writer.Write(egs[i].first, egs[i].second);
90  }
91 
92  KALDI_LOG << "Selected a subset of " << egs.size() << " out of " << num_read
93  << " neural-network training examples ";
94 
95  return (num_read != 0 ? 0 : 1);
96  } catch(const std::exception &e) {
97  std::cerr << e.what() << '\n';
98  return -1;
99  }
100 }
101 
102 
This code computes Goodness of Pronunciation (GOP) and extracts phone-level pronunciation feature for...
Definition: chain.dox:20
void PrintUsage(bool print_command_line=false)
Prints the usage documentation [provided in the constructor].
bool WithProb(BaseFloat prob, struct RandomState *state)
Definition: kaldi-math.cc:72
int main(int argc, char *argv[])
A templated class for writing objects to an archive or script file; see The Table concept...
Definition: kaldi-table.h:368
kaldi::int32 int32
void Register(const std::string &name, bool *ptr, const std::string &doc)
float BaseFloat
Definition: kaldi-types.h:29
The class ParseOptions is for parsing command-line options; see Parsing command-line options for more...
Definition: parse-options.h:36
struct rnnlm::@11::@12 n
A templated class for reading objects sequentially from an archive or script file; see The Table conc...
Definition: kaldi-table.h:287
int Read(int argc, const char *const *argv)
Parses the command line options and fills the ParseOptions-registered variables.
std::string GetArg(int param) const
Returns one of the positional parameters; 1-based indexing for argc/argv compatibility.
int NumArgs() const
Number of positional parameters (c.f. argc-1).
#define KALDI_LOG
Definition: kaldi-error.h:153
Note on how to parse this filename: it contains functions relatied to neural-net training examples...
int32 RandInt(int32 min_val, int32 max_val, struct RandomState *state)
Definition: kaldi-math.cc:95