lattice-determinize.cc
Go to the documentation of this file.
1 // latbin/lattice-determinize.cc
2 
3 // Copyright 2009-2012 Microsoft Corporation
4 // 2012-2013 Johns Hopkins University (Author: Daniel Povey)
5 
6 // See ../../COPYING for clarification regarding multiple authors
7 //
8 // Licensed under the Apache License, Version 2.0 (the "License");
9 // you may not use this file except in compliance with the License.
10 // You may obtain a copy of the License at
11 //
12 // http://www.apache.org/licenses/LICENSE-2.0
13 //
14 // THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 // KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
16 // WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
17 // MERCHANTABLITY OR NON-INFRINGEMENT.
18 // See the Apache 2 License for the specific language governing permissions and
19 // limitations under the License.
20 
21 
22 #include "base/kaldi-common.h"
23 #include "util/common-utils.h"
24 #include "fstext/fstext-lib.h"
25 #include "lat/kaldi-lattice.h"
26 #include "lat/lattice-functions.h"
27 #include "lat/push-lattice.h"
28 #include "lat/minimize-lattice.h"
29 
30 namespace kaldi {
31 
32 bool DeterminizeLatticeWrapper(const Lattice &lat,
33  const std::string &key,
34  bool prune,
35  BaseFloat beam,
36  BaseFloat beam_ratio,
37  int32 max_mem,
38  int32 max_loop,
39  BaseFloat delta,
40  int32 num_loops,
41  CompactLattice *clat) {
43  lat_opts.max_mem = max_mem;
44  lat_opts.max_loop = max_loop;
45  lat_opts.delta = delta;
46  BaseFloat cur_beam = beam;
47  for (int32 i = 0; i < num_loops;) { // we increment i below.
48 
49  if (lat.Start() == fst::kNoStateId) {
50  KALDI_WARN << "Detected empty lattice, skipping " << key;
51  return false;
52  }
53 
54  // The work gets done in the next line.
55  if (DeterminizeLattice(lat, clat, lat_opts, NULL)) {
56  if (prune) PruneLattice(cur_beam, clat);
57  return true;
58  } else { // failed to determinize..
59  KALDI_WARN << "Failed to determinize lattice (presumably max-states "
60  << "reached), reducing lattice-beam to "
61  << (cur_beam*beam_ratio) << " and re-trying.";
62  for (; i < num_loops; i++) {
63  cur_beam *= beam_ratio;
64  Lattice pruned_lat(lat);
65  PruneLattice(cur_beam, &pruned_lat);
66  if (NumArcs(lat) == NumArcs(pruned_lat)) {
67  cur_beam *= beam_ratio;
68  KALDI_WARN << "Pruning did not have an effect on the original "
69  << "lattice size; reducing beam to "
70  << cur_beam << " and re-trying.";
71  } else if (DeterminizeLattice(pruned_lat, clat, lat_opts, NULL)) {
72  if (prune) PruneLattice(cur_beam, clat);
73  return true;
74  } else {
75  KALDI_WARN << "Determinization failed again; reducing beam again to "
76  << (cur_beam*beam_ratio) << " and re-trying.";
77  }
78  }
79  }
80  }
81  KALDI_WARN << "Decreased pruning beam --num-loops=" << num_loops
82  << " times and was not able to determinize: failed for "
83  << key;
84  return false;
85 }
86 
87 }
88 
89 int main(int argc, char *argv[]) {
90  try {
91  using namespace kaldi;
92  typedef kaldi::int32 int32;
93  typedef kaldi::int64 int64;
94  using fst::SymbolTable;
95  using fst::VectorFst;
96  using fst::StdArc;
97 
98  const char *usage =
99  "This program is deprecated, please used lattice-determinize-pruned.\n"
100  "lattice-determinize lattices (and apply a pruning beam)\n"
101  " (see http://kaldi-asr.org/doc/lattices.html for more explanation)\n"
102  " note: this program is tyically only useful if you generated state-level\n"
103  " lattices, e.g. called gmm-latgen-simple with --determinize=false\n"
104  "\n"
105  "Usage: lattice-determinize [options] lattice-rspecifier lattice-wspecifier\n"
106  " e.g.: lattice-determinize --acoustic-scale=0.1 --beam=15.0 ark:1.lats ark:det.lats\n";
107 
108  ParseOptions po(usage);
109  BaseFloat acoustic_scale = 1.0;
110  BaseFloat beam = 10.0;
111  BaseFloat beam_ratio = 0.9;
112  int32 num_loops = 20;
113  int32 max_mem = 50000000; // 50 MB
114  int32 max_loop = 500000;
115  BaseFloat delta = fst::kDelta;
116  bool prune = false;
117  bool minimize = false;
118 
119  po.Register("acoustic-scale", &acoustic_scale,
120  "Scaling factor for acoustic likelihoods");
121  po.Register("beam", &beam,
122  "Pruning beam [applied after acoustic scaling]-- also used "
123  "to handle determinization failures, set --prune=false to "
124  "disable routine pruning");
125  po.Register("delta", &delta, "Tolerance used in determinization");
126  po.Register("prune", &prune, "If true, prune determinized lattices "
127  "with the --beam option.");
128  po.Register("max-mem", &max_mem, "Maximum approximate memory usage in "
129  "determinization (real usage might be many times this)");
130  po.Register("max-loop", &max_loop, "Option to detect a certain "
131  "type of failure in lattice determinization (not critical)");
132  po.Register("beam-ratio", &beam_ratio, "Ratio by which to "
133  "decrease beam if we reach the max-arcs.");
134  po.Register("num-loops", &num_loops, "Number of times to "
135  "decrease beam by beam-ratio if determinization fails.");
136  po.Register("minimize", &minimize,
137  "If true, push and minimize after determinization");
138 
139  po.Read(argc, argv);
140 
141  if (po.NumArgs() != 2) {
142  po.PrintUsage();
143  exit(1);
144  }
145 
146  std::string lats_rspecifier = po.GetArg(1),
147  lats_wspecifier = po.GetArg(2);
148 
149 
150  // Read as regular lattice-- this is the form we need it in for efficient
151  // pruning.
152  SequentialLatticeReader lattice_reader(lats_rspecifier);
153 
154  // Write as compact lattice.
155  CompactLatticeWriter compact_lattice_writer(lats_wspecifier);
156 
157  int32 n_done = 0, n_error = 0;
158 
159  // depth stats (for diagnostics).
160  double sum_depth_in = 0.0,
161  sum_depth_out = 0.0, sum_t = 0.0;
162 
163  if (acoustic_scale == 0.0)
164  KALDI_ERR << "Do not use a zero acoustic scale (cannot be inverted)";
165  LatticeWeight beam_weight(beam, static_cast<BaseFloat>(0.0));
166 
167  for (; !lattice_reader.Done(); lattice_reader.Next()) {
168  std::string key = lattice_reader.Key();
169  Lattice lat = lattice_reader.Value();
170  Invert(&lat); // make it so word labels are on the input.
171 
172  lattice_reader.FreeCurrent();
173  fst::ScaleLattice(fst::AcousticLatticeScale(acoustic_scale), &lat);
174 
175  CompactLattice clat;
176  if (DeterminizeLatticeWrapper(lat, key, prune,
177  beam, beam_ratio, max_mem, max_loop,
178  delta, num_loops, &clat)) {
179  if (minimize) {
182  MinimizeCompactLattice(&clat);
183  }
184 
185  int32 t;
187  double depth = CompactLatticeDepth(clat, &t);
188  sum_depth_in += lat.NumStates();
189  sum_depth_out += depth * t;
190  sum_t += t;
191 
192  fst::ScaleLattice(fst::AcousticLatticeScale(1.0/acoustic_scale), &clat);
193  compact_lattice_writer.Write(key, clat);
194  n_done++;
195  } else {
196  n_error++; // will have already printed warning.
197  }
198  }
199 
200  if (sum_t != 0.0) {
201  KALDI_LOG << "Average input-lattice depth (measured at at state level) is "
202  << (sum_depth_in / sum_t) << ", output depth is "
203  << (sum_depth_out / sum_t) << ", over " << sum_t << " frames "
204  << " (average num-frames = " << (sum_t / n_done) << ").";
205  }
206  KALDI_LOG << "Done " << n_done << " lattices, errors on " << n_error;
207  return (n_done != 0 ? 0 : 1);
208  } catch(const std::exception &e) {
209  std::cerr << e.what();
210  return -1;
211  }
212 }
This code computes Goodness of Pronunciation (GOP) and extracts phone-level pronunciation feature for...
Definition: chain.dox:20
int main(int argc, char *argv[])
bool DeterminizeLatticeWrapper(const Lattice &lat, const std::string &key, bool prune, BaseFloat beam, BaseFloat beam_ratio, int32 max_mem, int32 max_loop, BaseFloat delta, int32 num_loops, CompactLattice *clat)
void PrintUsage(bool print_command_line=false)
Prints the usage documentation [provided in the constructor].
fst::StdArc StdArc
bool PushCompactLatticeStrings(MutableFst< ArcTpl< CompactLatticeWeightTpl< Weight, IntType > > > *clat)
This function pushes the transition-ids as far towards the start as they will go. ...
A templated class for writing objects to an archive or script file; see The Table concept...
Definition: kaldi-table.h:368
kaldi::int32 int32
void Write(const std::string &key, const T &value) const
void Register(const std::string &name, bool *ptr, const std::string &doc)
bool PushCompactLatticeWeights(MutableFst< ArcTpl< CompactLatticeWeightTpl< Weight, IntType > > > *clat)
This function pushes the weights in the CompactLattice so that all states except possibly the start s...
std::vector< std::vector< double > > AcousticLatticeScale(double acwt)
float BaseFloat
Definition: kaldi-types.h:29
The class ParseOptions is for parsing command-line options; see Parsing command-line options for more...
Definition: parse-options.h:36
bool MinimizeCompactLattice(MutableFst< ArcTpl< CompactLatticeWeightTpl< Weight, IntType > > > *clat, float delta)
This function minimizes the compact lattice.
BaseFloat CompactLatticeDepth(const CompactLattice &clat, int32 *num_frames)
Returns the depth of the lattice, defined as the average number of arcs crossing any given frame...
void ScaleLattice(const std::vector< std::vector< ScaleFloat > > &scale, MutableFst< ArcTpl< Weight > > *fst)
Scales the pairs of weights in LatticeWeight or CompactLatticeWeight by viewing the pair (a...
A templated class for reading objects sequentially from an archive or script file; see The Table conc...
Definition: kaldi-table.h:287
fst::VectorFst< LatticeArc > Lattice
Definition: kaldi-lattice.h:44
int Read(int argc, const char *const *argv)
Parses the command line options and fills the ParseOptions-registered variables.
#define KALDI_ERR
Definition: kaldi-error.h:147
std::string GetArg(int param) const
Returns one of the positional parameters; 1-based indexing for argc/argv compatibility.
#define KALDI_WARN
Definition: kaldi-error.h:150
fst::VectorFst< CompactLatticeArc > CompactLattice
Definition: kaldi-lattice.h:46
int NumArgs() const
Number of positional parameters (c.f. argc-1).
bool PruneLattice(BaseFloat beam, LatType *lat)
bool DeterminizeLattice(const Fst< ArcTpl< Weight > > &ifst, MutableFst< ArcTpl< Weight > > *ofst, DeterminizeLatticeOptions opts, bool *debug_ptr)
This function implements the normal version of DeterminizeLattice, in which the output strings are re...
void TopSortCompactLatticeIfNeeded(CompactLattice *clat)
Topologically sort the compact lattice if not already topologically sorted.
Arc::StateId NumArcs(const ExpandedFst< Arc > &fst)
Returns the total number of arcs in an FST.
#define KALDI_LOG
Definition: kaldi-error.h:153