online-feature.h

Go to the documentation of this file.

// feat/online-feature.h

// Copyright 2013   Johns Hopkins University (author: Daniel Povey)
//           2014   Yanqing Sun, Junjie Wang,
//                  Daniel Povey, Korbinian Riedhammer

// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.


#ifndef KALDI_FEAT_ONLINE_FEATURE_H_
#define KALDI_FEAT_ONLINE_FEATURE_H_

#include <string>
#include <vector>
#include <deque>

#include "matrix/matrix-lib.h"
#include "util/common-utils.h"
#include "base/kaldi-error.h"
#include "feat/feature-functions.h"
#include "feat/feature-mfcc.h"
#include "feat/feature-plp.h"
#include "feat/feature-fbank.h"
#include "itf/online-feature-itf.h"

namespace kaldi {


class RecyclingVector {
public:
  RecyclingVector(int items_to_hold = -1);

  Vector<BaseFloat> *At(int index) const;

  void PushBack(Vector<BaseFloat> *item);

  int Size() const;

  ~RecyclingVector();

private:
  std::deque<Vector<BaseFloat>*> items_;
  int items_to_hold_;
  int first_available_index_;
};


template<class C>
class OnlineGenericBaseFeature: public OnlineBaseFeature {
 public:
  //
  // First, functions that are present in the interface:
  //
  virtual int32 Dim() const { return computer_.Dim(); }

  // Note: IsLastFrame() will only ever return true if you have called
  // InputFinished() (and this frame is the last frame).
  virtual bool IsLastFrame(int32 frame) const {
    return input_finished_ && frame == NumFramesReady() - 1;
  }
  virtual BaseFloat FrameShiftInSeconds() const {
    return computer_.GetFrameOptions().frame_shift_ms / 1000.0f;
  }

  virtual int32 NumFramesReady() const { return features_.Size(); }

  virtual void GetFrame(int32 frame, VectorBase<BaseFloat> *feat);

  // Next, functions that are not in the interface.


  // Constructor from options class
  explicit OnlineGenericBaseFeature(const typename C::Options &opts);

  // This would be called from the application, when you get
  // more wave data.  Note: the sampling_rate is only provided so
  // the code can assert that it matches the sampling rate
  // expected in the options.
  virtual void AcceptWaveform(BaseFloat sampling_rate,
                              const VectorBase<BaseFloat> &waveform);


  // InputFinished() tells the class you won't be providing any
  // more waveform.  This will help flush out the last frame or two
  // of features, in the case where snip-edges == false; it also
  // affects the return value of IsLastFrame().
  virtual void InputFinished();

 private:
  // This function computes any additional feature frames that it is possible to
  // compute from 'waveform_remainder_', which at this point may contain more
  // than just a remainder-sized quantity (because AcceptWaveform() appends to
  // waveform_remainder_ before calling this function).  It adds these feature
  // frames to features_, and shifts off any now-unneeded samples of input from
  // waveform_remainder_ while incrementing waveform_offset_ by the same amount.
  void ComputeFeatures();

  void MaybeCreateResampler(BaseFloat sampling_rate);

  C computer_;  // class that does the MFCC or PLP or filterbank computation

  // resampler in cases when the input sampling frequency is not equal to
  // the expected sampling rate
  std::unique_ptr<LinearResample> resampler_;

  FeatureWindowFunction window_function_;

  // features_ is the Mfcc or Plp or Fbank features that we have already computed.

  RecyclingVector features_;

  // True if the user has called "InputFinished()"
  bool input_finished_;

  // The sampling frequency, extracted from the config.  Should
  // be identical to the waveform supplied.
  BaseFloat sampling_frequency_;

  // waveform_offset_ is the number of samples of waveform that we have
  // already discarded, i.e. that were prior to 'waveform_remainder_'.
  int64 waveform_offset_;

  // waveform_remainder_ is a short piece of waveform that we may need to keep
  // after extracting all the whole frames we can (whatever length of feature
  // will be required for the next phase of computation).
  Vector<BaseFloat> waveform_remainder_;
};

typedef OnlineGenericBaseFeature<MfccComputer> OnlineMfcc;
typedef OnlineGenericBaseFeature<PlpComputer> OnlinePlp;
typedef OnlineGenericBaseFeature<FbankComputer> OnlineFbank;


class OnlineMatrixFeature: public OnlineFeatureInterface {
 public:
  explicit OnlineMatrixFeature(const MatrixBase<BaseFloat> &mat): mat_(mat) { }

  virtual int32 Dim() const { return mat_.NumCols(); }

  virtual BaseFloat FrameShiftInSeconds() const {
    return 0.01f;
  }

  virtual int32 NumFramesReady() const { return mat_.NumRows(); }

  virtual void GetFrame(int32 frame, VectorBase<BaseFloat> *feat) {
    feat->CopyFromVec(mat_.Row(frame));
  }

  virtual bool IsLastFrame(int32 frame) const {
    return (frame + 1 == mat_.NumRows());
  }


 private:
  const MatrixBase<BaseFloat> &mat_;
};


// Note the similarity with SlidingWindowCmnOptions, but there
// are also differences.  One which doesn't appear in the config
// itself, because it's a difference between the setups, is that
// in OnlineCmn, we carry over data from the previous utterance,
// or, if no previous utterance is available, from global stats,
// or, if previous utterances are available but the total amount
// of data is less than prev_frames, we pad with up to "global_frames"
// frames from the global stats.
struct OnlineCmvnOptions {
  int32 cmn_window;
  int32 speaker_frames;  // must be <= cmn_window
  int32 global_frames;  // must be <= speaker_frames.
  bool normalize_mean;  // Must be true if normalize_variance==true.
  bool normalize_variance;

  int32 modulus;  // not configurable from command line, relates to how the
                  // class computes the cmvn internally.  smaller->more
                  // time-efficient but less memory-efficient.  Must be >= 1.
  int32 ring_buffer_size;  // not configurable from command line; size of ring
                           // buffer used for caching CMVN stats.  Must be >=
                           // modulus.
  std::string skip_dims; // Colon-separated list of dimensions to skip normalization
                         // of, e.g. 13:14:15.

  OnlineCmvnOptions():
      cmn_window(600),
      speaker_frames(600),
      global_frames(200),
      normalize_mean(true),
      normalize_variance(false),
      modulus(20),
      ring_buffer_size(20),
      skip_dims("") { }

  void Check() const {
    KALDI_ASSERT(speaker_frames <= cmn_window && global_frames <= speaker_frames
                 && modulus > 0);
  }

  void Register(ParseOptions *po) {
    po->Register("cmn-window", &cmn_window, "Number of frames of sliding "
                 "context for cepstral mean normalization.");
    po->Register("global-frames", &global_frames, "Number of frames of "
                 "global-average cepstral mean normalization stats to use for "
                 "first utterance of a speaker");
    po->Register("speaker-frames", &speaker_frames, "Number of frames of "
                 "previous utterance(s) from this speaker to use in cepstral "
                 "mean normalization");
    // we name the config string "norm-vars" for compatibility with
    // ../featbin/apply-cmvn.cc
    po->Register("norm-vars", &normalize_variance, "If true, do "
                 "cepstral variance normalization in addition to cepstral mean "
                 "normalization ");
    po->Register("norm-means", &normalize_mean, "If true, do mean normalization "
                 "(note: you cannot normalize the variance but not the mean)");
    po->Register("skip-dims", &skip_dims, "Dimensions to skip normalization of "
                 "(colon-separated list of integers)");}
};



struct OnlineCmvnState {
  // The following is the total CMVN stats for this speaker (up till now), in
  // the same format.
  Matrix<double> speaker_cmvn_stats;

  // The following is the global CMVN stats, in the usual
  // format, of dimension 2 x (dim+1), as [  sum-stats          count
  //                                       sum-squared-stats   0    ]
  Matrix<double> global_cmvn_stats;

  // If nonempty, contains CMVN stats representing the "frozen" state
  // of CMVN that reflects how we were normalizing the data when the
  // user called the Freeze() function in class OnlineCmvn.
  Matrix<double> frozen_state;

  OnlineCmvnState() { }

  explicit OnlineCmvnState(const Matrix<double> &global_stats):
      global_cmvn_stats(global_stats) { }

  // Copy constructor
  OnlineCmvnState(const OnlineCmvnState &other);

  void Write(std::ostream &os, bool binary) const;
  void Read(std::istream &is, bool binary);

  // Use the default assignment operator.
};

class OnlineCmvn: public OnlineFeatureInterface {
 public:

  //
  // First, functions that are present in the interface:
  //
  virtual int32 Dim() const { return src_->Dim(); }

  virtual bool IsLastFrame(int32 frame) const {
    return src_->IsLastFrame(frame);
  }
  virtual BaseFloat FrameShiftInSeconds() const {
    return src_->FrameShiftInSeconds();
  }

  // The online cmvn does not introduce any additional latency.
  virtual int32 NumFramesReady() const { return src_->NumFramesReady(); }

  virtual void GetFrame(int32 frame, VectorBase<BaseFloat> *feat);

  //
  // Next, functions that are not in the interface.
  //

  OnlineCmvn(const OnlineCmvnOptions &opts,
             const OnlineCmvnState &cmvn_state,
             OnlineFeatureInterface *src);

  OnlineCmvn(const OnlineCmvnOptions &opts,
             OnlineFeatureInterface *src);

  // Outputs any state information from this utterance to "cmvn_state".
  // The value of "cmvn_state" before the call does not matter: the output
  // depends on the value of OnlineCmvnState the class was initialized
  // with, the input feature values up to cur_frame, and the effects
  // of the user possibly having called Freeze().
  // If cur_frame is -1, it will just output the unmodified original
  // state that was supplied to this object.
  void GetState(int32 cur_frame,
                OnlineCmvnState *cmvn_state);

  // This function can be used to modify the state of the CMVN computation
  // from outside, but must only be called before you have processed any data
  // (otherwise it will crash).  This "state" is really just the information
  // that is propagated between utterances, not the state of the computation
  // inside an utterance.
  void SetState(const OnlineCmvnState &cmvn_state);

  // From this point it will freeze the CMN to what it would have been if
  // measured at frame "cur_frame", and it will stop it from changing
  // further. This also applies retroactively for this utterance, so if you
  // call GetFrame() on previous frames, it will use the CMVN stats
  // from cur_frame; and it applies in the future too if you then
  // call OutputState() and use this state to initialize the next
  // utterance's CMVN object.
  void Freeze(int32 cur_frame);

  virtual ~OnlineCmvn();
 private:

  static void SmoothOnlineCmvnStats(const MatrixBase<double> &speaker_stats,
                                    const MatrixBase<double> &global_stats,
                                    const OnlineCmvnOptions &opts,
                                    MatrixBase<double> *stats);

  void GetMostRecentCachedFrame(int32 frame,
                                int32 *cached_frame,
                                MatrixBase<double> *stats);

  void CacheFrame(int32 frame, const MatrixBase<double> &stats);

  inline void InitRingBufferIfNeeded();

  void ComputeStatsForFrame(int32 frame,
                            MatrixBase<double> *stats);


  OnlineCmvnOptions opts_;
  std::vector<int32> skip_dims_; // Skip CMVN for these dimensions.  Derived from opts_.
  OnlineCmvnState orig_state_;   // reflects the state before we saw this
                                 // utterance.
  Matrix<double> frozen_state_;  // If the user called Freeze(), this variable
                                 // will reflect the CMVN state that we froze
                                 // at.

  // The variable below reflects the raw (count, x, x^2) statistics of the
  // input, computed every opts_.modulus frames.  raw_stats_[n / opts_.modulus]
  // contains the (count, x, x^2) statistics for the frames from
  // std::max(0, n - opts_.cmn_window) through n.
  std::vector<Matrix<double>*> cached_stats_modulo_;
  // the variable below is a ring-buffer of cached stats.  the int32 is the
  // frame index.
  std::vector<std::pair<int32, Matrix<double> > > cached_stats_ring_;

  // Some temporary variables used inside functions of this class, which
  // put here to avoid reallocation.
  Matrix<double> temp_stats_;
  Vector<BaseFloat> temp_feats_;
  Vector<double> temp_feats_dbl_;

  OnlineFeatureInterface *src_;  // Not owned here
};


struct OnlineSpliceOptions {
  int32 left_context;
  int32 right_context;
  OnlineSpliceOptions(): left_context(4), right_context(4) { }
  void Register(ParseOptions *po) {
    po->Register("left-context", &left_context, "Left-context for frame "
                 "splicing prior to LDA");
    po->Register("right-context", &right_context, "Right-context for frame "
                 "splicing prior to LDA");
  }
};

class OnlineSpliceFrames: public OnlineFeatureInterface {
 public:
  //
  // First, functions that are present in the interface:
  //
  virtual int32 Dim() const {
    return src_->Dim() * (1 + left_context_ + right_context_);
  }

  virtual bool IsLastFrame(int32 frame) const {
    return src_->IsLastFrame(frame);
  }
  virtual BaseFloat FrameShiftInSeconds() const {
    return src_->FrameShiftInSeconds();
  }

  virtual int32 NumFramesReady() const;

  virtual void GetFrame(int32 frame, VectorBase<BaseFloat> *feat);

  //
  // Next, functions that are not in the interface.
  //
  OnlineSpliceFrames(const OnlineSpliceOptions &opts,
                     OnlineFeatureInterface *src):
      left_context_(opts.left_context), right_context_(opts.right_context),
      src_(src) { }

 private:
  int32 left_context_;
  int32 right_context_;
  OnlineFeatureInterface *src_;  // Not owned here
};

class OnlineTransform: public OnlineFeatureInterface {
 public:
  //
  // First, functions that are present in the interface:
  //
  virtual int32 Dim() const { return offset_.Dim(); }

  virtual bool IsLastFrame(int32 frame) const {
    return src_->IsLastFrame(frame);
  }
  virtual BaseFloat FrameShiftInSeconds() const {
    return src_->FrameShiftInSeconds();
  }

  virtual int32 NumFramesReady() const { return src_->NumFramesReady(); }

  virtual void GetFrame(int32 frame, VectorBase<BaseFloat> *feat);

  virtual void GetFrames(const std::vector<int32> &frames,
                         MatrixBase<BaseFloat> *feats);

  //
  // Next, functions that are not in the interface.
  //

  OnlineTransform(const MatrixBase<BaseFloat> &transform,
                  OnlineFeatureInterface *src);


 private:
  OnlineFeatureInterface *src_;  // Not owned here
  Matrix<BaseFloat> linear_term_;
  Vector<BaseFloat> offset_;
};

class OnlineDeltaFeature: public OnlineFeatureInterface {
 public:
  //
  // First, functions that are present in the interface:
  //
  virtual int32 Dim() const;

  virtual bool IsLastFrame(int32 frame) const {
    return src_->IsLastFrame(frame);
  }
  virtual BaseFloat FrameShiftInSeconds() const {
    return src_->FrameShiftInSeconds();
  }

  virtual int32 NumFramesReady() const;

  virtual void GetFrame(int32 frame, VectorBase<BaseFloat> *feat);

  //
  // Next, functions that are not in the interface.
  //
  OnlineDeltaFeature(const DeltaFeaturesOptions &opts,
                     OnlineFeatureInterface *src);

 private:
  OnlineFeatureInterface *src_;  // Not owned here
  DeltaFeaturesOptions opts_;
  DeltaFeatures delta_features_;  // This class contains just a few
                                  // coefficients.
};


class OnlineCacheFeature: public OnlineFeatureInterface {
 public:
  virtual int32 Dim() const { return src_->Dim(); }

  virtual bool IsLastFrame(int32 frame) const {
    return src_->IsLastFrame(frame);
  }
  virtual BaseFloat FrameShiftInSeconds() const {
    return src_->FrameShiftInSeconds();
  }

  virtual int32 NumFramesReady() const { return src_->NumFramesReady(); }

  virtual void GetFrame(int32 frame, VectorBase<BaseFloat> *feat);

  virtual void GetFrames(const std::vector<int32> &frames,
                         MatrixBase<BaseFloat> *feats);

  virtual ~OnlineCacheFeature() { ClearCache(); }

  // Things that are not in the shared interface:

  void ClearCache();  // this should be called if you change the underlying
                      // features in some way.

  explicit OnlineCacheFeature(OnlineFeatureInterface *src): src_(src) { }
 private:

  OnlineFeatureInterface *src_;  // Not owned here
  std::vector<Vector<BaseFloat>* > cache_;
};




class OnlineAppendFeature: public OnlineFeatureInterface {
 public:
  virtual int32 Dim() const { return src1_->Dim() + src2_->Dim(); }

  virtual bool IsLastFrame(int32 frame) const {
    return (src1_->IsLastFrame(frame) || src2_->IsLastFrame(frame));
  }
  // Hopefully sources have the same rate
  virtual BaseFloat FrameShiftInSeconds() const {
    return src1_->FrameShiftInSeconds();
  }

  virtual int32 NumFramesReady() const {
    return std::min(src1_->NumFramesReady(), src2_->NumFramesReady());
  }

  virtual void GetFrame(int32 frame, VectorBase<BaseFloat> *feat);

  virtual ~OnlineAppendFeature() {  }

  OnlineAppendFeature(OnlineFeatureInterface *src1,
      OnlineFeatureInterface *src2): src1_(src1), src2_(src2) { }
 private:

  OnlineFeatureInterface *src1_;
  OnlineFeatureInterface *src2_;
};

}  // namespace kaldi

#endif  // KALDI_FEAT_ONLINE_FEATURE_H_