compute-wer-bootci.cc File Reference

#include "base/kaldi-common.h"
#include "util/common-utils.h"
#include "util/parse-options.h"
#include "tree/context-dep.h"
#include "util/edit-distance.h"
#include "base/kaldi-math.h"

Include dependency graph for compute-wer-bootci.cc:

Go to the source code of this file.

Namespaces
	kaldi
	This code computes Goodness of Pronunciation (GOP) and extracts phone-level pronunciation feature for mispronunciations detection tasks, the reference:

Functions
void	GetEditsSingleHyp (const std::string &hyp_rspecifier, const std::string &ref_rspecifier, const std::string &mode, std::vector< std::pair< int32, int32 > > &edit_word_per_hyp)
void	GetEditsDualHyp (const std::string &hyp_rspecifier, const std::string &hyp_rspecifier2, const std::string &ref_rspecifier, const std::string &mode, std::vector< std::pair< int32, int32 > > &edit_word_per_hyp, std::vector< std::pair< int32, int32 > > &edit_word_per_hyp2)
void	GetBootstrapWERInterval (const std::vector< std::pair< int32, int32 > > &edit_word_per_hyp, int32 replications, BaseFloat *mean, BaseFloat *interval)
void	GetBootstrapWERTwoSystemComparison (const std::vector< std::pair< int32, int32 > > &edit_word_per_hyp, const std::vector< std::pair< int32, int32 > > &edit_word_per_hyp2, int32 replications, BaseFloat *p_improv)
int	main (int argc, char *argv[])

Function Documentation

main()

int main	(	int	argc,
		char *	argv[]
	)

Definition at line 159 of file compute-wer-bootci.cc.

References ParseOptions::GetArg(), kaldi::GetBootstrapWERInterval(), kaldi::GetBootstrapWERTwoSystemComparison(), kaldi::GetEditsDualHyp(), kaldi::GetEditsSingleHyp(), KALDI_ERR, ParseOptions::NumArgs(), ParseOptions::PrintUsage(), ParseOptions::Read(), and ParseOptions::Register().

                                 {
  using namespace kaldi;
  typedef kaldi::int32 int32;

  try {
    const char *usage =
      "Compute a bootstrapping of WER to extract the 95% confidence interval.\n"
      "Take a reference and a transcription file, in integer or text format,\n"
      "and outputs overall WER statistics to standard output along with its\n"
      "confidence interval using the bootstrap method of Bisani and Ney.\n"
      "If a second transcription file corresponding to the same reference is\n"
      "provided, a bootstrap comparison of the two transcription is performed\n"
      "to estimate the probability of improvement.\n"
      "\n"
      "Usage: compute-wer-bootci [options] <ref-rspecifier> <hyp-rspecifier> [<hyp2-rspecifier>]\n"
      "E.g.: compute-wer-bootci --mode=present ark:data/train/text ark:hyp_text\n"
      "or compute-wer-bootci ark:data/train/text ark:hyp_text ark:hyp_text2\n"
      "See also: compute-wer\n";

    ParseOptions po(usage);

    std::string mode = "strict";
    po.Register("mode", &mode,
                "Scoring mode: \"present\"|\"all\"|\"strict\":\n"
                "  \"present\" means score those we have transcriptions for\n"
                "  \"all\" means treat absent transcriptions as empty\n"
                "  \"strict\" means die if all in ref not also in hyp");

    int32 replications = 10000;
    po.Register("replications", &replications,
            "Number of replications to compute the intervals");

    po.Read(argc, argv);

    if (po.NumArgs() < 2 || po.NumArgs() > 3) {
      po.PrintUsage();
      exit(1);
    }

    std::string ref_rspecifier = po.GetArg(1);
    std::string hyp_rspecifier = po.GetArg(2);
    std::string hyp2_rspecifier = (po.NumArgs() == 3?po.GetArg(3):"");

    if (mode != "strict" && mode != "present" && mode != "all") {
      KALDI_ERR <<
          "--mode option invalid: expected \"present\"|\"all\"|\"strict\", got "
          << mode;
    }

    //Get editions per each utterance
    std::vector<std::pair<int32, int32> > edit_word_per_hyp, edit_word_per_hyp2;
    if(hyp2_rspecifier.empty())
      GetEditsSingleHyp(hyp_rspecifier, ref_rspecifier, mode, edit_word_per_hyp);
    else
      GetEditsDualHyp(hyp_rspecifier, hyp2_rspecifier, ref_rspecifier, mode,
              edit_word_per_hyp, edit_word_per_hyp2);

    //Extract WER for a number of replications of the same size
    //as the hypothesis extracted
    BaseFloat mean_wer = 0.0, interval = 0.0,
              mean_wer2 = 0.0, interval2 = 0.0,
              p_improv = 0.0;

    GetBootstrapWERInterval(edit_word_per_hyp, replications,
            &mean_wer, &interval);

    if(!hyp2_rspecifier.empty()) {
      GetBootstrapWERInterval(edit_word_per_hyp2, replications,
              &mean_wer2, &interval2);

      GetBootstrapWERTwoSystemComparison(edit_word_per_hyp, edit_word_per_hyp2,
             replications, &p_improv);
    }

    // Print the output,
    std::cout.precision(2);
    std::cerr.precision(2);
    std::cout << "Set1: %WER " << std::fixed << 100*mean_wer <<
              " 95% Conf Interval [ " << 100*mean_wer-100*interval <<
              ", " << 100*mean_wer+100*interval << " ]" << '\n';

    if(!hyp2_rspecifier.empty()) {
        std::cout << "Set2: %WER " << std::fixed << 100*mean_wer2 <<
            " 95% Conf Interval [ " << 100*mean_wer2-100*interval2 <<
            ", " << 100*mean_wer2+100*interval2 << " ]" << '\n';

        std::cout << "Probability of Set2 improving Set1: " << std::fixed <<
            100*p_improv << '\n';
    }

    return 0;
  } catch(const std::exception &e) {
    std::cerr << e.what();
    return -1;
  }
}