All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
compute-wer-bootci.cc File Reference
Include dependency graph for compute-wer-bootci.cc:

Go to the source code of this file.

Namespaces

 kaldi
 Relabels neural network egs with the read pdf-id alignments.
 

Functions

void GetEditsSingleHyp (const std::string &hyp_rspecifier, const std::string &ref_rspecifier, const std::string &mode, std::vector< std::pair< int32, int32 > > &edit_word_per_hyp)
 
void GetEditsDualHyp (const std::string &hyp_rspecifier, const std::string &hyp_rspecifier2, const std::string &ref_rspecifier, const std::string &mode, std::vector< std::pair< int32, int32 > > &edit_word_per_hyp, std::vector< std::pair< int32, int32 > > &edit_word_per_hyp2)
 
void GetBootstrapWERInterval (const std::vector< std::pair< int32, int32 > > &edit_word_per_hyp, int32 replications, BaseFloat *mean, BaseFloat *interval)
 
void GetBootstrapWERTwoSystemComparison (const std::vector< std::pair< int32, int32 > > &edit_word_per_hyp, const std::vector< std::pair< int32, int32 > > &edit_word_per_hyp2, int32 replications, BaseFloat *p_improv)
 
int main (int argc, char *argv[])
 

Function Documentation

int main ( int  argc,
char *  argv[] 
)

Definition at line 159 of file compute-wer-bootci.cc.

References ParseOptions::GetArg(), kaldi::GetBootstrapWERInterval(), kaldi::GetBootstrapWERTwoSystemComparison(), kaldi::GetEditsDualHyp(), kaldi::GetEditsSingleHyp(), KALDI_ERR, ParseOptions::NumArgs(), ParseOptions::PrintUsage(), ParseOptions::Read(), and ParseOptions::Register().

159  {
160  using namespace kaldi;
161  typedef kaldi::int32 int32;
162 
163  try {
164  const char *usage =
165  "Compute a bootstrapping of WER to extract the 95\% confidence interval.\n"
166  "Take a reference and a transcription file, in integer or text format,\n"
167  "and outputs overall WER statistics to standard output along with its\n"
168  "confidence interval using the bootstrap methos of Bisani and Ney.\n"
169  "If a second transcription file corresponding to the same reference is\n"
170  "provided, a bootstrap comparison of the two transcription is performed\n"
171  "to estimate the probability of improvement.\n"
172  "\n"
173  "Usage: compute-wer-bootci [options] <ref-rspecifier> <hyp-rspecifier> [<hyp2-rspecifier>]\n"
174  "E.g.: compute-wer-bootci --mode=present ark:data/train/text ark:hyp_text\n"
175  "or compute-wer-bootci ark:data/train/text ark:hyp_text ark:hyp_text2\n"
176  "See also: compute-wer\n";
177 
178  ParseOptions po(usage);
179 
180  std::string mode = "strict";
181  po.Register("mode", &mode,
182  "Scoring mode: \"present\"|\"all\"|\"strict\":\n"
183  " \"present\" means score those we have transcriptions for\n"
184  " \"all\" means treat absent transcriptions as empty\n"
185  " \"strict\" means die if all in ref not also in hyp");
186 
187  int32 replications = 10000;
188  po.Register("replications", &replications,
189  "Number of replications to compute the intervals");
190 
191  po.Read(argc, argv);
192 
193  if (po.NumArgs() < 2 || po.NumArgs() > 3) {
194  po.PrintUsage();
195  exit(1);
196  }
197 
198  std::string ref_rspecifier = po.GetArg(1);
199  std::string hyp_rspecifier = po.GetArg(2);
200  std::string hyp2_rspecifier = (po.NumArgs() == 3?po.GetArg(3):"");
201 
202  if (mode != "strict" && mode != "present" && mode != "all") {
203  KALDI_ERR <<
204  "--mode option invalid: expected \"present\"|\"all\"|\"strict\", got "
205  << mode;
206  }
207 
208  //Get editions per each utterance
209  std::vector<std::pair<int32, int32> > edit_word_per_hyp, edit_word_per_hyp2;
210  if(hyp2_rspecifier.empty())
211  GetEditsSingleHyp(hyp_rspecifier, ref_rspecifier, mode, edit_word_per_hyp);
212  else
213  GetEditsDualHyp(hyp_rspecifier, hyp2_rspecifier, ref_rspecifier, mode,
214  edit_word_per_hyp, edit_word_per_hyp2);
215 
216  //Extract WER for a number of replications of the same size
217  //as the hypothesis extracted
218  BaseFloat mean_wer = 0.0, interval = 0.0,
219  mean_wer2 = 0.0, interval2 = 0.0,
220  p_improv = 0.0;
221 
222  GetBootstrapWERInterval(edit_word_per_hyp, replications,
223  &mean_wer, &interval);
224 
225  if(!hyp2_rspecifier.empty()) {
226  GetBootstrapWERInterval(edit_word_per_hyp2, replications,
227  &mean_wer2, &interval2);
228 
229  GetBootstrapWERTwoSystemComparison(edit_word_per_hyp, edit_word_per_hyp2,
230  replications, &p_improv);
231  }
232 
233  // Print the output,
234  std::cout.precision(2);
235  std::cerr.precision(2);
236  std::cout << "Set1: %WER " << std::fixed << 100*mean_wer <<
237  " 95\% Conf Interval [ " << 100*mean_wer-100*interval <<
238  ", " << 100*mean_wer+100*interval << " ]" << '\n';
239 
240  if(!hyp2_rspecifier.empty()) {
241  std::cout << "Set2: %WER " << std::fixed << 100*mean_wer2 <<
242  " 95\% Conf Interval [ " << 100*mean_wer2-100*interval2 <<
243  ", " << 100*mean_wer2+100*interval2 << " ]" << '\n';
244 
245  std::cout << "Probability of Set2 improving Set1: " << std::fixed <<
246  100*p_improv << '\n';
247  }
248 
249  return 0;
250  } catch(const std::exception &e) {
251  std::cerr << e.what();
252  return -1;
253  }
254 }
Relabels neural network egs with the read pdf-id alignments.
Definition: chain.dox:20
void GetEditsSingleHyp(const std::string &hyp_rspecifier, const std::string &ref_rspecifier, const std::string &mode, std::vector< std::pair< int32, int32 > > &edit_word_per_hyp)
void GetEditsDualHyp(const std::string &hyp_rspecifier, const std::string &hyp_rspecifier2, const std::string &ref_rspecifier, const std::string &mode, std::vector< std::pair< int32, int32 > > &edit_word_per_hyp, std::vector< std::pair< int32, int32 > > &edit_word_per_hyp2)
float BaseFloat
Definition: kaldi-types.h:29
The class ParseOptions is for parsing command-line options; see Parsing command-line options for more...
Definition: parse-options.h:36
#define KALDI_ERR
Definition: kaldi-error.h:127
void GetBootstrapWERInterval(const std::vector< std::pair< int32, int32 > > &edit_word_per_hyp, int32 replications, BaseFloat *mean, BaseFloat *interval)
void GetBootstrapWERTwoSystemComparison(const std::vector< std::pair< int32, int32 > > &edit_word_per_hyp, const std::vector< std::pair< int32, int32 > > &edit_word_per_hyp2, int32 replications, BaseFloat *p_improv)