nnet3-discriminative-shuffle-egs.cc
Go to the documentation of this file.
1 // nnet3bin/nnet3-discriminative-shuffle-egs.cc
2 
3 // Copyright 2012-2015 Johns Hopkins University (author: Daniel Povey)
4 // 2014-2015 Vimal Manohar
5 
6 // See ../../COPYING for clarification regarding multiple authors
7 //
8 // Licensed under the Apache License, Version 2.0 (the "License");
9 // you may not use this file except in compliance with the License.
10 // You may obtain a copy of the License at
11 //
12 // http://www.apache.org/licenses/LICENSE-2.0
13 //
14 // THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 // KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
16 // WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
17 // MERCHANTABLITY OR NON-INFRINGEMENT.
18 // See the Apache 2 License for the specific language governing permissions and
19 // limitations under the License.
20 
21 #include "base/kaldi-common.h"
22 #include "util/common-utils.h"
23 #include "hmm/transition-model.h"
25 
26 int main(int argc, char *argv[]) {
27  try {
28  using namespace kaldi;
29  using namespace kaldi::nnet3;
30  typedef kaldi::int32 int32;
31  typedef kaldi::int64 int64;
32 
33  const char *usage =
34  "Copy nnet3 discriminative training examples from the input to output,\n"
35  "while randomly shuffling the order. This program will keep all of the examples\n"
36  "in memory at once, unless you use the --buffer-size option\n"
37  "\n"
38  "Usage: nnet3-discriminative-shuffle-egs [options] <egs-rspecifier> <egs-wspecifier>\n"
39  "\n"
40  "nnet3-discriminative-shuffle-egs --srand=1 ark:train.egs ark:shuffled.egs\n";
41 
42  int32 srand_seed = 0;
43  int32 buffer_size = 0;
44  ParseOptions po(usage);
45  po.Register("srand", &srand_seed, "Seed for random number generator ");
46  po.Register("buffer-size", &buffer_size, "If >0, size of a buffer we use "
47  "to do limited-memory partial randomization. Otherwise, do "
48  "full randomization.");
49 
50  po.Read(argc, argv);
51 
52  srand(srand_seed);
53 
54  if (po.NumArgs() != 2) {
55  po.PrintUsage();
56  exit(1);
57  }
58 
59  std::string examples_rspecifier = po.GetArg(1),
60  examples_wspecifier = po.GetArg(2);
61 
62  int64 num_done = 0;
63 
64  std::vector<std::pair<std::string, NnetDiscriminativeExample*> > egs;
65 
66  SequentialNnetDiscriminativeExampleReader example_reader(examples_rspecifier);
67  NnetDiscriminativeExampleWriter example_writer(examples_wspecifier);
68  if (buffer_size == 0) { // Do full randomization
69  // Putting in an extra level of indirection here to avoid excessive
70  // computation and memory demands when we have to resize the vector.
71 
72  for (; !example_reader.Done(); example_reader.Next())
73  egs.push_back(std::pair<std::string, NnetDiscriminativeExample*>(
74  example_reader.Key(),
75  new NnetDiscriminativeExample(example_reader.Value())));
76 
77  std::random_shuffle(egs.begin(), egs.end());
78  } else {
79  KALDI_ASSERT(buffer_size > 0);
80  egs.resize(buffer_size,
81  std::pair<std::string, NnetDiscriminativeExample*>("", NULL));
82  for (; !example_reader.Done(); example_reader.Next()) {
83  int32 index = RandInt(0, buffer_size - 1);
84  if (egs[index].second == NULL) {
85  egs[index] = std::pair<std::string, NnetDiscriminativeExample*>(
86  example_reader.Key(),
87  new NnetDiscriminativeExample(example_reader.Value()));
88  } else {
89  example_writer.Write(egs[index].first, *(egs[index].second));
90  egs[index].first = example_reader.Key();
91  *(egs[index].second) = example_reader.Value();
92  num_done++;
93  }
94  }
95  }
96  for (size_t i = 0; i < egs.size(); i++) {
97  if (egs[i].second != NULL) {
98  example_writer.Write(egs[i].first, *(egs[i].second));
99  delete egs[i].second;
100  num_done++;
101  }
102  }
103 
104  KALDI_LOG << "Shuffled order of " << num_done
105  << " neural-network training examples "
106  << (buffer_size ? "using a buffer (partial randomization)" : "");
107 
108  return (num_done == 0 ? 1 : 0);
109  } catch(const std::exception &e) {
110  std::cerr << e.what() << '\n';
111  return -1;
112  }
113 }
114 
115 
116 
This code computes Goodness of Pronunciation (GOP) and extracts phone-level pronunciation feature for...
Definition: chain.dox:20
void PrintUsage(bool print_command_line=false)
Prints the usage documentation [provided in the constructor].
A templated class for writing objects to an archive or script file; see The Table concept...
Definition: kaldi-table.h:368
kaldi::int32 int32
void Write(const std::string &key, const T &value) const
void Register(const std::string &name, bool *ptr, const std::string &doc)
The class ParseOptions is for parsing command-line options; see Parsing command-line options for more...
Definition: parse-options.h:36
A templated class for reading objects sequentially from an archive or script file; see The Table conc...
Definition: kaldi-table.h:287
int Read(int argc, const char *const *argv)
Parses the command line options and fills the ParseOptions-registered variables.
std::string GetArg(int param) const
Returns one of the positional parameters; 1-based indexing for argc/argv compatibility.
int NumArgs() const
Number of positional parameters (c.f. argc-1).
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:185
int main(int argc, char *argv[])
#define KALDI_LOG
Definition: kaldi-error.h:153
NnetDiscriminativeExample is like NnetExample, but specialized for sequence training.
int32 RandInt(int32 min_val, int32 max_val, struct RandomState *state)
Definition: kaldi-math.cc:95