online-gmm-decoding.h

Go to the documentation of this file.

// online2/online-gmm-decoding.h

// Copyright 2014  Johns Hopkins University (author: Daniel Povey)

// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.


#ifndef KALDI_ONLINE2_ONLINE_GMM_DECODING_H_
#define KALDI_ONLINE2_ONLINE_GMM_DECODING_H_

#include <string>
#include <vector>
#include <deque>

#include "matrix/matrix-lib.h"
#include "util/common-utils.h"
#include "base/kaldi-error.h"
#include "transform/basis-fmllr-diag-gmm.h"
#include "transform/fmllr-diag-gmm.h"
#include "online2/online-feature-pipeline.h"
#include "online2/online-gmm-decodable.h"
#include "online2/online-endpoint.h"
#include "decoder/lattice-faster-online-decoder.h"
#include "hmm/transition-model.h"
#include "gmm/am-diag-gmm.h"
#include "hmm/posterior.h"


namespace kaldi {



struct OnlineGmmDecodingAdaptationPolicyConfig {
  BaseFloat adaptation_first_utt_delay;
  BaseFloat adaptation_first_utt_ratio;
  BaseFloat adaptation_delay;
  BaseFloat adaptation_ratio;
  OnlineGmmDecodingAdaptationPolicyConfig():
      adaptation_first_utt_delay(2.0),
      adaptation_first_utt_ratio(1.5),
      adaptation_delay(5.0),
      adaptation_ratio(2.0) { }

  void Register(OptionsItf *opts) {
    opts->Register("adaptation-first-utt-delay", &adaptation_first_utt_delay,
                   "Delay before first basis-fMLLR adaptation for first utterance "
                   "of each speaker");
    opts->Register("adaptation-first-utt-ratio", &adaptation_first_utt_ratio,
                   "Ratio that controls frequency of fMLLR adaptation for first "
                   "utterance of each speaker");
    opts->Register("adaptation-delay", &adaptation_delay,
                   "Delay before first basis-fMLLR adaptation for not-first "
                   "utterances of each speaker");
    opts->Register("adaptation-ratio", &adaptation_ratio,
                   "Ratio that controls frequency of fMLLR adaptation for "
                   "not-first utterances of each speaker");
  }
  
  void Check() const;
  
  bool DoAdapt(BaseFloat chunk_begin_secs,
               BaseFloat chunk_end_secs,
               bool is_first_utterance) const;
      
};


struct OnlineGmmDecodingConfig {
  BaseFloat fmllr_lattice_beam;
  
  BasisFmllrOptions basis_opts; // options for basis-fMLLR adaptation.

  LatticeFasterDecoderConfig faster_decoder_opts;
  
  OnlineGmmDecodingAdaptationPolicyConfig adaptation_policy_opts;
  
  // rxfilename for model trained with online-CMN features
  // (only needed if different from model_rxfilename)
  std::string online_alimdl_rxfilename;
  // rxfilename for model used for estimating fMLLR transforms
  std::string model_rxfilename;
  // rxfilename for possible discriminatively trained model
  // (only needed if different from model_rxfilename)
  std::string rescore_model_rxfilename;
  // rxfilename for the BasisFmllrEstimate object containing the basis
  // used for basis-fMLLR.
  std::string fmllr_basis_rxfilename;

  BaseFloat acoustic_scale;

  std::string silence_phones;
  BaseFloat silence_weight;
  

  OnlineGmmDecodingConfig():  fmllr_lattice_beam(3.0), acoustic_scale(0.1),
                              silence_weight(0.1) { }
  
  void Register(OptionsItf *opts) {
    { // register basis_opts with prefix, there are getting to be too many
      // options.
      ParseOptions basis_po("basis", opts);
      basis_opts.Register(&basis_po);
    }
    adaptation_policy_opts.Register(opts);
    faster_decoder_opts.Register(opts);
    opts->Register("acoustic-scale", &acoustic_scale,
                   "Scaling factor for acoustic likelihoods");
    opts->Register("silence-phones", &silence_phones,
                   "Colon-separated list of integer ids of silence phones, e.g. "
                   "1:2:3 (affects adaptation).");
    opts->Register("silence-weight", &silence_weight,
                   "Weight applied to silence frames for fMLLR estimation (if "
                   "--silence-phones option is supplied)");
    opts->Register("fmllr-lattice-beam", &fmllr_lattice_beam, "Beam used in "
                   "pruning lattices for fMLLR estimation");
    opts->Register("online-alignment-model", &online_alimdl_rxfilename,
                   "(Extended) filename for model trained with online CMN "
                   "features, e.g. from apply-cmvn-online.");
    opts->Register("model", &model_rxfilename, "(Extended) filename for model, "
                   "typically the one used for fMLLR computation.  Required option.");
    opts->Register("rescore-model", &rescore_model_rxfilename, "(Extended) filename "
                   "for model to rescore lattices with, e.g. discriminatively trained"
                   "model, if it differs from that supplied to --model option.  Must"
                   "have the same tree.");
    opts->Register("fmllr-basis", &fmllr_basis_rxfilename, "(Extended) filename "
                   "of fMLLR basis object, as output by gmm-basis-fmllr-training");
  }
};


class OnlineGmmDecodingModels {
 public:
  OnlineGmmDecodingModels(const OnlineGmmDecodingConfig &config);

  const TransitionModel &GetTransitionModel() const;

  const AmDiagGmm &GetOnlineAlignmentModel() const;

  const AmDiagGmm &GetModel() const;

  const AmDiagGmm &GetFinalModel() const;

  const BasisFmllrEstimate &GetFmllrBasis() const;
  
 private:
  // The transition-model is only needed for its integer ids, and these need to
  // be identical for all 3 models, so we only store one (it doesn't matter
  // which one).
  TransitionModel tmodel_; 
  // The model trained with online-CMVN features
  // (if supplied, otherwise use model_)
  AmDiagGmm online_alignment_model_;
  // The ML-trained model used to get transforms (required)
  AmDiagGmm model_;
  // The discriminatively trained model
  // (if supplied, otherwise use model_)
  AmDiagGmm rescore_model_;
  // The following object contains the basis elements for
  // "Basis fMLLR".
  BasisFmllrEstimate fmllr_basis_;
};


struct OnlineGmmAdaptationState {
  OnlineCmvnState cmvn_state;
  FmllrDiagGmmAccs spk_stats;
  Matrix<BaseFloat> transform;

  // Writing and reading of the state of the object
  void Write(std::ostream &out_stream, bool binary) const;
  void Read(std::istream &in_stream, bool binary);


};

class SingleUtteranceGmmDecoder {
 public:
  SingleUtteranceGmmDecoder(const OnlineGmmDecodingConfig &config,
                            const OnlineGmmDecodingModels &models,
                            const OnlineFeaturePipeline &feature_prototype,
                            const fst::Fst<fst::StdArc> &fst,
                            const OnlineGmmAdaptationState &adaptation_state);
  
  OnlineFeaturePipeline &FeaturePipeline() { return *feature_pipeline_; }

  void AdvanceDecoding();

  void FinalizeDecoding();

  bool HaveTransform() const;
  
  void EstimateFmllr(bool end_of_utterance);
  
  void GetAdaptationState(OnlineGmmAdaptationState *adaptation_state) const;

  void GetLattice(bool rescore_if_needed,
                  bool end_of_utterance,
                  CompactLattice *clat) const;

  void GetBestPath(bool end_of_utterance,
                   Lattice *best_path) const;

  BaseFloat FinalRelativeCost() { return decoder_.FinalRelativeCost(); }


  bool EndpointDetected(const OnlineEndpointConfig &config);

  ~SingleUtteranceGmmDecoder();
 private:
  bool GetGaussianPosteriors(bool end_of_utterance, GaussPost *gpost);

  bool RescoringIsNeeded() const;

  OnlineGmmDecodingConfig config_;
  std::vector<int32> silence_phones_; // sorted, unique list of silence phones,
                                      // derived from config_
  const OnlineGmmDecodingModels &models_;
  OnlineFeaturePipeline *feature_pipeline_;  // owned here.
  const OnlineGmmAdaptationState &orig_adaptation_state_;
  // adaptation_state_ generally reflects the "current" state of the
  // adaptation. Note: adaptation_state_.cmvn_state is just copied from
  // orig_adaptation_state, the function GetAdaptationState() gets the CMVN
  // state.
  OnlineGmmAdaptationState adaptation_state_;
  LatticeFasterOnlineDecoder decoder_;
};

  

}  // namespace kaldi



#endif  // KALDI_ONLINE2_ONLINE_GMM_DECODING_H_