online-feature-pipeline.h

Go to the documentation of this file.

// online2/online-feature-pipeline.h

// Copyright 2013-2014   Johns Hopkins University (author: Daniel Povey)

// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.


#ifndef KALDI_ONLINE2_ONLINE_FEATURE_PIPELINE_H_
#define KALDI_ONLINE2_ONLINE_FEATURE_PIPELINE_H_

#include <string>
#include <vector>
#include <deque>

#include "matrix/matrix-lib.h"
#include "util/common-utils.h"
#include "base/kaldi-error.h"
#include "feat/online-feature.h"
#include "feat/pitch-functions.h"

namespace kaldi {


struct OnlineFeaturePipelineCommandLineConfig {
  std::string feature_type;
  std::string mfcc_config;
  std::string plp_config;
  std::string fbank_config;
  bool add_pitch;
  std::string pitch_config;
  std::string pitch_process_config;
  std::string cmvn_config;
  std::string global_cmvn_stats_rxfilename;
  bool add_deltas;
  std::string delta_config;
  bool splice_feats;
  std::string splice_config;
  std::string lda_rxfilename;

  OnlineFeaturePipelineCommandLineConfig() :
    feature_type("mfcc"), add_pitch(false), add_deltas(false),
    splice_feats(false) { }

  void Register(OptionsItf *opts) {
    opts->Register("feature-type", &feature_type,
                   "Base feature type [mfcc, plp, fbank]");
    opts->Register("mfcc-config", &mfcc_config, "Configuration file for "
                   "MFCC features (e.g. conf/mfcc.conf)");
    opts->Register("plp-config", &plp_config, "Configuration file for "
                   "PLP features (e.g. conf/plp.conf)");
    opts->Register("fbank-config", &fbank_config, "Configuration file for "
                   "filterbank features (e.g. conf/fbank.conf)");
    opts->Register("add-pitch", &add_pitch, "Append pitch features to raw "
                   "MFCC/PLP features.");
    opts->Register("pitch-config", &pitch_config, "Configuration file for "
                   "pitch features (e.g. conf/pitch.conf)");
    opts->Register("pitch-process-config", &pitch_process_config,
                   "Configuration file for post-processing pitch features "
                   "(e.g. conf/pitch_process.conf)");
    opts->Register("cmvn-config", &cmvn_config, "Configuration class "
                   "file for online CMVN features (e.g. conf/online_cmvn.conf)");
    opts->Register("global-cmvn-stats", &global_cmvn_stats_rxfilename,
                   "(Extended) filename for global CMVN stats, e.g. obtained "
                   "from 'matrix-sum scp:data/train/cmvn.scp -'");
    opts->Register("add-deltas", &add_deltas,
                   "Append delta features.");
    opts->Register("delta-config", &delta_config, "Configuration file for "
                   "delta feature computation (if not supplied, will not apply "
                   "delta features; supply empty config to use defaults.)");
    opts->Register("splice-feats", &splice_feats, "Splice features with left and "
                   "right context.");
    opts->Register("splice-config", &splice_config, "Configuration file "
                   "for frame splicing, if done (e.g. prior to LDA)");
    opts->Register("lda-matrix", &lda_rxfilename, "Filename of LDA matrix (if "
                   "using LDA), e.g. exp/foo/final.mat");
  }
};



struct OnlineFeaturePipelineConfig {
  OnlineFeaturePipelineConfig():
      feature_type("mfcc"), add_pitch(false), add_deltas(true),
      splice_feats(false) { }

  OnlineFeaturePipelineConfig(
      const OnlineFeaturePipelineCommandLineConfig &cmdline_config);

  BaseFloat FrameShiftInSeconds() const;

  std::string feature_type;  // "mfcc" or "plp" or "fbank"

  MfccOptions mfcc_opts;  // options for MFCC computation,
                          // if feature_type == "mfcc"
  PlpOptions plp_opts;  // Options for PLP computation, if feature_type == "plp"
  FbankOptions fbank_opts;  // Options for filterbank computation, if
                            // feature_type == "fbank"

  bool add_pitch;
  PitchExtractionOptions pitch_opts;  // Options for pitch extraction, if done.
  ProcessPitchOptions pitch_process_opts;  // Options for pitch
                                                   // processing

  OnlineCmvnOptions cmvn_opts;  // Options for online CMN/CMVN computation.

  bool add_deltas;
  DeltaFeaturesOptions delta_opts;  // Options for delta computation, if done.

  bool splice_feats;
  OnlineSpliceOptions splice_opts;  // Options for frame splicing, if done.

  std::string lda_rxfilename;  // Filename for reading LDA or LDA+MLLT matrix,
                               // if used.
  std::string global_cmvn_stats_rxfilename;  // Filename used for reading global
                                             // CMVN stats
};



class OnlineFeaturePipeline: public OnlineFeatureInterface {
 public:
  explicit OnlineFeaturePipeline(const OnlineFeaturePipelineConfig &cfg);

  virtual int32 Dim() const;
  virtual bool IsLastFrame(int32 frame) const;
  virtual int32 NumFramesReady() const;
  virtual void GetFrame(int32 frame, VectorBase<BaseFloat> *feat);

  // This is supplied for debug purposes.
  void GetAsMatrix(Matrix<BaseFloat> *feats);

  void FreezeCmvn();  // stop it from moving further (do this when you start
                      // using fMLLR). This will crash if NumFramesReady() == 0.

  void SetCmvnState(const OnlineCmvnState &cmvn_state);
  void GetCmvnState(OnlineCmvnState *cmvn_state);

  void AcceptWaveform(BaseFloat sampling_rate,
                      const VectorBase<BaseFloat> &waveform);

  BaseFloat FrameShiftInSeconds() const {
    return config_.FrameShiftInSeconds();
  }

  // InputFinished() tells the class you won't be providing any
  // more waveform.  This will help flush out the last few frames
  // of delta or LDA features, and finalize the pitch features
  // (making them more accurate).
  void InputFinished();

  // This object is used to set the fMLLR transform.  Call it with
  // the empty matrix if you want to stop it using any transform.
  void SetTransform(const MatrixBase<BaseFloat> &transform);


  // Returns true if an fMLLR transform has been set using
  // SetTransform().
  bool HaveFmllrTransform() { return fmllr_ != NULL; }

  OnlineFeaturePipeline *New() const;

  virtual ~OnlineFeaturePipeline();

 private:
  OnlineFeaturePipeline(const OnlineFeaturePipelineConfig &cfg,
                        const Matrix<BaseFloat> &lda_mat,
                        const Matrix<BaseFloat> &global_cmvn_stats);

  void Init();

  OnlineFeaturePipelineConfig config_;
  Matrix<BaseFloat> lda_mat_;  // LDA matrix, if supplied.
  Matrix<BaseFloat> global_cmvn_stats_;  // Global CMVN stats.

  OnlineBaseFeature *base_feature_;        // MFCC/PLP/Fbank
  OnlinePitchFeature *pitch_;              // Raw pitch
  OnlineProcessPitch *pitch_feature_;  // Processed pitch
  OnlineFeatureInterface *feature_;        // CMVN (+ processed pitch)

  OnlineCmvn *cmvn_;
  OnlineFeatureInterface *splice_or_delta_;  // This may be NULL if we're not
                                             // doing splicing or deltas.

  OnlineFeatureInterface *lda_;  // If non-NULL, the LDA or LDA+MLLT transform.

  OnlineFeatureInterface* UnadaptedFeature() const;

  OnlineFeatureInterface *fmllr_;  // non-NULL if we currently have an fMLLR
                                   // transform.

  OnlineFeatureInterface* AdaptedFeature() const;
};




}  // namespace kaldi



#endif  // KALDI_ONLINE2_ONLINE_FEATURE_PIPELINE_H_