lattice-determinize-pruned.cc
Go to the documentation of this file.
1 // latbin/lattice-determinize-pruned.cc
2 
3 // Copyright 2013 Daniel Povey (Johns Hopkins University)
4 
5 // See ../../COPYING for clarification regarding multiple authors
6 //
7 // Licensed under the Apache License, Version 2.0 (the "License");
8 // you may not use this file except in compliance with the License.
9 // You may obtain a copy of the License at
10 //
11 // http://www.apache.org/licenses/LICENSE-2.0
12 //
13 // THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 // KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
15 // WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
16 // MERCHANTABLITY OR NON-INFRINGEMENT.
17 // See the Apache 2 License for the specific language governing permissions and
18 // limitations under the License.
19 #include "base/kaldi-common.h"
20 #include "util/common-utils.h"
21 #include "lat/kaldi-lattice.h"
23 #include "lat/lattice-functions.h"
24 #include "lat/push-lattice.h"
25 #include "lat/minimize-lattice.h"
26 
27 int main(int argc, char *argv[]) {
28  try {
29  using namespace kaldi;
30  typedef kaldi::int32 int32;
31 
32  const char *usage =
33  "Determinize lattices, keeping only the best path (sequence of acoustic states)\n"
34  "for each input-symbol sequence. This version does pruning as part of the\n"
35  "determinization algorithm, which is more efficient and prevents blowup.\n"
36  "See http://kaldi-asr.org/doc/lattices.html for more information on lattices.\n"
37  "\n"
38  "Usage: lattice-determinize-pruned [options] lattice-rspecifier lattice-wspecifier\n"
39  " e.g.: lattice-determinize-pruned --acoustic-scale=0.1 --beam=6.0 ark:in.lats ark:det.lats\n";
40 
41  ParseOptions po(usage);
42  bool write_compact = true;
43  BaseFloat acoustic_scale = 1.0;
44  BaseFloat beam = 10.0;
45  bool minimize = false;
46  fst::DeterminizeLatticePrunedOptions opts; // Options used in DeterminizeLatticePruned--
47  // this options class does not have its own Register function as it's viewed as
48  // being more part of "fst world", so we register its elements independently.
49  opts.max_mem = 50000000;
50  opts.max_loop = 0; // was 500000;
51 
52  po.Register("write-compact", &write_compact,
53  "If true, write in normal (compact) form. "
54  "--write-compact=false allows you to retain frame-level "
55  "acoustic score information, but this requires the input "
56  "to be in non-compact form e.g. undeterminized lattice "
57  "straight from decoding.");
58  po.Register("acoustic-scale", &acoustic_scale,
59  "Scaling factor for acoustic likelihoods");
60  po.Register("beam", &beam, "Pruning beam [applied after acoustic scaling].");
61  po.Register("minimize", &minimize,
62  "If true, push and minimize after determinization");
63  opts.Register(&po);
64  po.Read(argc, argv);
65 
66  if (po.NumArgs() != 2) {
67  po.PrintUsage();
68  exit(1);
69  }
70 
71  std::string lats_rspecifier = po.GetArg(1),
72  lats_wspecifier = po.GetArg(2);
73 
74 
75  // Read as regular lattice-- this is the form the determinization code
76  // accepts.
77  SequentialLatticeReader lat_reader(lats_rspecifier);
78 
79  CompactLatticeWriter compact_lat_writer;
80  LatticeWriter lat_writer;
81 
82  if (write_compact)
83  compact_lat_writer.Open(lats_wspecifier);
84  else
85  lat_writer.Open(lats_wspecifier);
86 
87  int32 n_done = 0, n_warn = 0;
88 
89  // depth stats (for diagnostics).
90  double sum_depth_in = 0.0,
91  sum_depth_out = 0.0, sum_t = 0.0;
92 
93  if (acoustic_scale == 0.0)
94  KALDI_ERR << "Do not use a zero acoustic scale (cannot be inverted)";
95 
96  for (; !lat_reader.Done(); lat_reader.Next()) {
97  std::string key = lat_reader.Key();
98  Lattice lat = lat_reader.Value();
99 
100  KALDI_VLOG(2) << "Processing lattice " << key;
101 
102  // Compute a map from each (t, tid) to (sum_of_acoustic_scores, count)
103  unordered_map<std::pair<int32,int32>, std::pair<BaseFloat, int32>,
104  PairHasher<int32> > acoustic_scores;
105  if (!write_compact)
106  ComputeAcousticScoresMap(lat, &acoustic_scores);
107 
108  Invert(&lat); // so word labels are on the input side.
109  lat_reader.FreeCurrent();
110  fst::ScaleLattice(fst::AcousticLatticeScale(acoustic_scale), &lat);
111  if (!TopSort(&lat)) {
112  KALDI_WARN << "Could not topologically sort lattice: this probably means it"
113  " has bad properties e.g. epsilon cycles. Your LM or lexicon might "
114  "be broken, e.g. LM with epsilon cycles or lexicon with empty words.";
115  }
116  fst::ArcSort(&lat, fst::ILabelCompare<LatticeArc>());
117  CompactLattice det_clat;
118  if (!DeterminizeLatticePruned(lat, beam, &det_clat, opts)) {
119  KALDI_WARN << "For key " << key << ", determinization did not succeed"
120  "(partial output will be pruned tighter than the specified beam.)";
121  n_warn++;
122  }
123  fst::Connect(&det_clat);
124  if (det_clat.NumStates() == 0) {
125  KALDI_WARN << "For key " << key << ", determinized and trimmed lattice "
126  "was empty.";
127  n_warn++;
128  }
129  if (minimize) {
130  PushCompactLatticeStrings(&det_clat);
131  PushCompactLatticeWeights(&det_clat);
132  MinimizeCompactLattice(&det_clat);
133  }
134 
135  int32 t;
137  double depth = CompactLatticeDepth(det_clat, &t);
138  sum_depth_in += lat.NumStates();
139  sum_depth_out += depth * t;
140  sum_t += t;
141 
142  if (write_compact) {
143  fst::ScaleLattice(fst::AcousticLatticeScale(1.0/acoustic_scale), &det_clat);
144  compact_lat_writer.Write(key, det_clat);
145  } else {
146  Lattice out_lat;
147  fst::ConvertLattice(det_clat, &out_lat);
148 
149  // Replace each arc (t, tid) with the averaged acoustic score from
150  // the computed map
151  ReplaceAcousticScoresFromMap(acoustic_scores, &out_lat);
152  lat_writer.Write(key, out_lat);
153  }
154 
155  n_done++;
156  }
157 
158  if (sum_t != 0.0) {
159  KALDI_LOG << "Average input-lattice depth (measured at at state level) is "
160  << (sum_depth_in / sum_t) << ", output depth is "
161  << (sum_depth_out / sum_t) << ", over " << sum_t << " frames "
162  << " (average num-frames = " << (sum_t / n_done) << ").";
163  }
164  KALDI_LOG << "Done " << n_done << " lattices, determinization finished "
165  << "earlier than specified by the beam (or output was empty) on "
166  << n_warn << " of these.";
167  return (n_done != 0 ? 0 : 1);
168  } catch(const std::exception &e) {
169  std::cerr << e.what();
170  return -1;
171  }
172 }
This code computes Goodness of Pronunciation (GOP) and extracts phone-level pronunciation feature for...
Definition: chain.dox:20
bool Open(const std::string &wspecifier)
bool DeterminizeLatticePruned(const ExpandedFst< ArcTpl< Weight > > &ifst, double beam, MutableFst< ArcTpl< CompactLatticeWeightTpl< Weight, IntType > > > *ofst, DeterminizeLatticePrunedOptions opts)
void PrintUsage(bool print_command_line=false)
Prints the usage documentation [provided in the constructor].
void ReplaceAcousticScoresFromMap(const unordered_map< std::pair< int32, int32 >, std::pair< BaseFloat, int32 >, PairHasher< int32 > > &acoustic_scores, Lattice *lat)
This function restores acoustic scores computed using the function ComputeAcousticScoresMap into the ...
bool PushCompactLatticeStrings(MutableFst< ArcTpl< CompactLatticeWeightTpl< Weight, IntType > > > *clat)
This function pushes the transition-ids as far towards the start as they will go. ...
A templated class for writing objects to an archive or script file; see The Table concept...
Definition: kaldi-table.h:368
kaldi::int32 int32
void Write(const std::string &key, const T &value) const
void Register(const std::string &name, bool *ptr, const std::string &doc)
int main(int argc, char *argv[])
bool PushCompactLatticeWeights(MutableFst< ArcTpl< CompactLatticeWeightTpl< Weight, IntType > > > *clat)
This function pushes the weights in the CompactLattice so that all states except possibly the start s...
std::vector< std::vector< double > > AcousticLatticeScale(double acwt)
float BaseFloat
Definition: kaldi-types.h:29
The class ParseOptions is for parsing command-line options; see Parsing command-line options for more...
Definition: parse-options.h:36
bool MinimizeCompactLattice(MutableFst< ArcTpl< CompactLatticeWeightTpl< Weight, IntType > > > *clat, float delta)
This function minimizes the compact lattice.
BaseFloat CompactLatticeDepth(const CompactLattice &clat, int32 *num_frames)
Returns the depth of the lattice, defined as the average number of arcs crossing any given frame...
void ScaleLattice(const std::vector< std::vector< ScaleFloat > > &scale, MutableFst< ArcTpl< Weight > > *fst)
Scales the pairs of weights in LatticeWeight or CompactLatticeWeight by viewing the pair (a...
void ConvertLattice(const ExpandedFst< ArcTpl< Weight > > &ifst, MutableFst< ArcTpl< CompactLatticeWeightTpl< Weight, Int > > > *ofst, bool invert)
Convert lattice from a normal FST to a CompactLattice FST.
A templated class for reading objects sequentially from an archive or script file; see The Table conc...
Definition: kaldi-table.h:287
fst::VectorFst< LatticeArc > Lattice
Definition: kaldi-lattice.h:44
int Read(int argc, const char *const *argv)
Parses the command line options and fills the ParseOptions-registered variables.
#define KALDI_ERR
Definition: kaldi-error.h:147
#define KALDI_WARN
Definition: kaldi-error.h:150
std::string GetArg(int param) const
Returns one of the positional parameters; 1-based indexing for argc/argv compatibility.
fst::VectorFst< CompactLatticeArc > CompactLattice
Definition: kaldi-lattice.h:46
int NumArgs() const
Number of positional parameters (c.f. argc-1).
void ComputeAcousticScoresMap(const Lattice &lat, unordered_map< std::pair< int32, int32 >, std::pair< BaseFloat, int32 >, PairHasher< int32 > > *acoustic_scores)
This function computes the mapping from the pair (frame-index, transition-id) to the pair (sum-of-aco...
#define KALDI_VLOG(v)
Definition: kaldi-error.h:156
void TopSortCompactLatticeIfNeeded(CompactLattice *clat)
Topologically sort the compact lattice if not already topologically sorted.
#define KALDI_LOG
Definition: kaldi-error.h:153
A hashing function-object for pairs of ints.
Definition: stl-utils.h:235