online-nnet2-feature-pipeline.h

Go to the documentation of this file.

// online2/online-nnet2-feature-pipeline.h

// Copyright 2013-2014   Johns Hopkins University (author: Daniel Povey)

// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.


#ifndef KALDI_ONLINE2_ONLINE_NNET2_FEATURE_PIPELINE_H_
#define KALDI_ONLINE2_ONLINE_NNET2_FEATURE_PIPELINE_H_

#include <string>
#include <vector>
#include <deque>

#include "matrix/matrix-lib.h"
#include "util/common-utils.h"
#include "base/kaldi-error.h"
#include "feat/online-feature.h"
#include "feat/pitch-functions.h"
#include "online2/online-ivector-feature.h"

namespace kaldi {



struct OnlineNnet2FeaturePipelineConfig {
  std::string feature_type;  // "plp" or "mfcc" or "fbank"
  std::string mfcc_config;
  std::string plp_config;
  std::string fbank_config;
  std::string cmvn_config;
  std::string global_cmvn_stats_rxfilename;

  // Note: if we do add pitch, it will not be added to the features we give to
  // the iVector extractor but only to the features we give to the neural
  // network, after the base features but before the iVector.  We don't think
  // the iVector will be particularly helpful in normalizing the pitch features.
  bool add_pitch;

  // the following contains the type of options that you could give to
  // compute-and-process-kaldi-pitch-feats.
  std::string online_pitch_config;

  // The configuration variables in ivector_extraction_config relate to the
  // iVector extractor and options related to it, see type
  // OnlineIvectorExtractionConfig.
  std::string ivector_extraction_config;

  // Config that relates to how we weight silence for (ivector) adaptation
  // this is registered directly to the command line as you might want to
  // play with it in test time.
  OnlineSilenceWeightingConfig silence_weighting_config;

  OnlineNnet2FeaturePipelineConfig():
      feature_type("mfcc"), add_pitch(false) { }


  void Register(OptionsItf *opts) {
    opts->Register("feature-type", &feature_type,
                   "Base feature type [mfcc, plp, fbank]");
    opts->Register("mfcc-config", &mfcc_config, "Configuration file for "
                   "MFCC features (e.g. conf/mfcc.conf)");
    opts->Register("plp-config", &plp_config, "Configuration file for "
                   "PLP features (e.g. conf/plp.conf)");
    opts->Register("fbank-config", &fbank_config, "Configuration file for "
                   "filterbank features (e.g. conf/fbank.conf)");
    opts->Register("cmvn-config", &cmvn_config, "Configuration file for "
                   "online cmvn features (e.g. conf/online_cmvn.conf). "
                   "Controls features on nnet3 input (not ivector features). "
                   "If not set, the OnlineCmvn is disabled.");
    opts->Register("global-cmvn-stats", &global_cmvn_stats_rxfilename,
                   "filename with global stats for OnlineCmvn for features "
                   "on nnet3 input (not ivector features)");
    opts->Register("add-pitch", &add_pitch, "Append pitch features to raw "
                   "MFCC/PLP/filterbank features [but not for iVector extraction]");
    opts->Register("online-pitch-config", &online_pitch_config, "Configuration "
                   "file for online pitch features, if --add-pitch=true (e.g. "
                   "conf/online_pitch.conf)");
    opts->Register("ivector-extraction-config", &ivector_extraction_config,
                   "Configuration file for online iVector extraction, "
                   "see class OnlineIvectorExtractionConfig in the code");
    silence_weighting_config.RegisterWithPrefix("ivector-silence-weighting", opts);
  }
};


struct OnlineNnet2FeaturePipelineInfo {
  OnlineNnet2FeaturePipelineInfo():
      feature_type("mfcc"), add_pitch(false), use_cmvn(false) { }

  OnlineNnet2FeaturePipelineInfo(
      const OnlineNnet2FeaturePipelineConfig &config);

  BaseFloat FrameShiftInSeconds() const;

  std::string feature_type; 

  MfccOptions mfcc_opts;  
  PlpOptions plp_opts;    
  FbankOptions fbank_opts;  

  bool add_pitch;
  PitchExtractionOptions pitch_opts;  
  ProcessPitchOptions pitch_process_opts;  

  bool use_cmvn;
  OnlineCmvnOptions cmvn_opts; 
  std::string global_cmvn_stats_rxfilename;  

  bool use_ivectors;
  OnlineIvectorExtractionInfo ivector_extractor_info;

  OnlineSilenceWeightingConfig silence_weighting_config;

  int32 IvectorDim() { return ivector_extractor_info.extractor.IvectorDim(); }
 private:
  KALDI_DISALLOW_COPY_AND_ASSIGN(OnlineNnet2FeaturePipelineInfo);
};



class OnlineNnet2FeaturePipeline: public OnlineFeatureInterface {
 public:
  explicit OnlineNnet2FeaturePipeline(
      const OnlineNnet2FeaturePipelineInfo &info);


  virtual int32 Dim() const;

  virtual bool IsLastFrame(int32 frame) const;
  virtual int32 NumFramesReady() const;
  virtual void GetFrame(int32 frame, VectorBase<BaseFloat> *feat);

  void UpdateFrameWeights(
      const std::vector<std::pair<int32, BaseFloat> > &delta_weights);

  void SetAdaptationState(
      const OnlineIvectorExtractorAdaptationState &adaptation_state);

  void GetAdaptationState(
      OnlineIvectorExtractorAdaptationState *adaptation_state) const;

  void SetCmvnState(const OnlineCmvnState &cmvn_state);
  void GetCmvnState(OnlineCmvnState *cmvn_state);

  void AcceptWaveform(BaseFloat sampling_rate,
                      const VectorBase<BaseFloat> &waveform);

  BaseFloat FrameShiftInSeconds() const { return info_.FrameShiftInSeconds(); }

  void InputFinished();

  OnlineIvectorFeature *IvectorFeature() {
    return ivector_feature_;
  }

  const OnlineIvectorFeature *IvectorFeature() const {
    return ivector_feature_;
  }

  OnlineFeatureInterface *InputFeature() {
    return nnet3_feature_;
  }

  virtual ~OnlineNnet2FeaturePipeline();

 private:
  const OnlineNnet2FeaturePipelineInfo &info_;

  OnlineBaseFeature *base_feature_;    

  OnlinePitchFeature *pitch_;          
  OnlineProcessPitch *pitch_feature_;  

  OnlineCmvn *cmvn_feature_;
  Matrix<BaseFloat> lda_mat_;          
  Matrix<double> global_cmvn_stats_;   

  OnlineFeatureInterface *feature_plus_optional_pitch_;

  OnlineFeatureInterface *feature_plus_optional_cmvn_;

  OnlineIvectorFeature *ivector_feature_;  

  OnlineFeatureInterface *nnet3_feature_;

  OnlineFeatureInterface *final_feature_;

  int32 dim_;
};


}  // namespace kaldi



#endif  // KALDI_ONLINE2_ONLINE_NNET2_FEATURE_PIPELINE_H_