fmllr-raw.h

Go to the documentation of this file.

// transform/fmllr-raw.h

// Copyright 2013  Johns Hopkins University (author: Daniel Povey)

// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.


#ifndef KALDI_TRANSFORM_FMLLR_RAW_H_
#define KALDI_TRANSFORM_FMLLR_RAW_H_

#include <vector>

#include "base/kaldi-common.h"
#include "gmm/am-diag-gmm.h"
#include "gmm/mle-full-gmm.h"
#include "transform/transform-common.h"
#include "util/kaldi-table.h"
#include "util/kaldi-holder.h"

namespace kaldi {


/*
  This header contains classes and functions related to computing Constrained
  MLLR (equivalently, fMLLR) on the raw MFCCs or similar, when they have been
  spliced and projected with something like LDA+MLLT, but where our model is
  built on top of the spliced and projected features.  We back-project the
  model estimation back to the original MFCCs so our transform optimizes the
  data likelihood given our model in the projected space.  We have to include
  the rejected dimensions in this likelihood, too.  The objective function
  includes N times the log-determinant of the square part of the transform,
  where N is the number of times we spliced consecutive features (e.g. N = 9,
  if we spliced +- 4 frames of context).

  For concreteness (but without losing generality), assuming we spliced
  13-dimensional MFCCs across 9 frames to get 117-dimensional features.
   
  Each of the 117-dim features is a linear function of the 13(13+1) transform
  parameters.  We have a particular vectorization of these parameters, from
  which (with the transform) we work out the full quadratic auxiliary function
  w.r.t. the parameters.

  This gives us a generic quadratic scalar function of the 13(13+1) parameters.
  How to get this quadratic w.r.t. one row?  Always keep updated the current
  derivative w.r.t. one row.  The quadratic w.r.t. that row can be read off.
  The log-determinant is easy to work out from the cofactor.

  So the full stats will be a (13(13+1)) by (13(13+1)) SpMatrix, plus
  a bias term.

  The update will iterate row by row, and work out the quadratic function
  of the row.
*/


struct FmllrRawOptions {
  BaseFloat min_count;
  int32 num_iters;
  FmllrRawOptions(): min_count(100.0), num_iters(20) { }
  void Register(OptionsItf *opts) {
    opts->Register("fmllr-min-count", &min_count,
                   "Minimum count required to update fMLLR");
    opts->Register("fmllr-num-iters", &num_iters,
                   "Number of iterations in fMLLR update phase.");
  }
};

class FmllrRawAccs {
 public:
  FmllrRawAccs() { }

  int32 RawDim() const { return raw_dim_; }
  int32 FullDim() const { return full_transform_.NumRows(); }
  int32 SpliceWidth() const { return FullDim() / RawDim(); }
  int32 ModelDim() const { return model_dim_; }
  
  // Initializer takes the raw dimension of the features (e.g. 13 for typicaly
  // MFCC features, and the full transform (e.g. an LDA+MLLT transform).  This
  // full transform is the transform extended with the "rejected rows" that
  // we would normally discard; we need them for this type of estimation.
  FmllrRawAccs(int32 raw_dim,
               int32 model_dim,
               const Matrix<BaseFloat> &full_transform);

  
  BaseFloat AccumulateForGmm(const DiagGmm &gmm,
                             const VectorBase<BaseFloat> &data,
                             BaseFloat weight);
  
  void AccumulateFromPosteriors(const DiagGmm &gmm,
                                const VectorBase<BaseFloat> &data,
                                const VectorBase<BaseFloat> &posteriors);

  void Update(const FmllrRawOptions &opts,
              MatrixBase<BaseFloat> *raw_fmllr_mat,
              BaseFloat *objf_impr,
              BaseFloat *count);

  void SetZero();
 private:
  struct SingleFrameStats {
    Vector<BaseFloat> s; // [FullDim() + 1]-dimensional spliced data, plus 1.0
    Vector<BaseFloat> transformed_data; // [FullDim()] Data times full transform, with offset.
    double count;
    Vector<double> a; // linear term in per-frame auxf; dim is model-dim.
    Vector<double> b; // quadratic term in per-frame auxf; dim is model-dim.
  };
  
  void CommitSingleFrameStats();

  void InitSingleFrameStats(const VectorBase<BaseFloat> &data);
  
  bool DataHasChanged(const VectorBase<BaseFloat> &data) const; // compares it to the
  // data in single_frame_stats_, returns true if it's different.

  
  double GetAuxf(const Vector<double> &simple_linear_stats,
                 const SpMatrix<double> &simple_quadratic_stats,
                 const Matrix<double> &fmllr_mat) const;

  void ConvertToSimpleStats(
      Vector<double> *simple_linear_stats,
      SpMatrix<double> *simple_quadratic_stats) const;

  void ComputeM(
      std::vector<Matrix<double> > *M) const;
  
  void ConvertToPerRowStats(
      const Vector<double> &simple_linear_stats,
      const SpMatrix<double> &simple_quadratic_stats_sp,
      Matrix<double> *linear_stats,
      std::vector<SpMatrix<double> > *diag_stats,
      std::vector<std::vector<Matrix<double> > > *off_diag_stats) const;
  
  int32 raw_dim_; // Raw MFCC dimension.
  int32 model_dim_; // Model dimension

  Matrix<BaseFloat> full_transform_; // Does not include any offset term
  // (last column).
  Vector<BaseFloat> transform_offset_; // The offset term (or zero).
  

  SingleFrameStats single_frame_stats_;
  
  double count_; // The data-count.  Note: in accounting for the determinant, we will
                 // have to multiply this by the number of times the data is spliced
                 // together on each frame.

  SpMatrix<double> temp_; // [full_dim + 1][full_dim + 1], outer product of s.
  Matrix<double> Q_; // linear stats, indexed [model_dim + 1][full_dim + 1]
  Matrix<double> S_; // quadratic stats, indexed
                     // [model_dim + 1][((full_dim+1)*(full_dim+2))/2]
  
  KALDI_DISALLOW_COPY_AND_ASSIGN(FmllrRawAccs);
};



} // namespace kaldi

#endif  // KALDI_TRANSFORM_FMLLR_RAW_H_