FmllrSgmm2Accs Class Reference

Class for computing the accumulators needed for the maximum-likelihood estimate of FMLLR transforms for a subspace GMM acoustic model. More...

#include <fmllr-sgmm2.h>

Collaboration diagram for FmllrSgmm2Accs:

Public Member Functions
	FmllrSgmm2Accs ()
	~FmllrSgmm2Accs ()
void	Init (int32 dim, int32 num_gaussians)
void	SetZero ()
void	Write (std::ostream &out_stream, bool binary) const
void	Read (std::istream &in_stream, bool binary, bool add)
BaseFloat	Accumulate (const AmSgmm2 &sgmm, const VectorBase< BaseFloat > &data, const Sgmm2PerFrameDerivedVars &frame_vars, int32 state_index, BaseFloat weight, Sgmm2PerSpkDerivedVars *spk)
	Accumulation routine that computes the Gaussian posteriors and calls the AccumulateFromPosteriors function with the computed posteriors. More...
void	AccumulateFromPosteriors (const AmSgmm2 &sgmm, const Sgmm2PerSpkDerivedVars &spk, const VectorBase< BaseFloat > &data, const std::vector< int32 > &gauss_select, const Matrix< BaseFloat > &posteriors, int32 state_index)
void	AccumulateForFmllrSubspace (const AmSgmm2 &sgmm, const Sgmm2FmllrGlobalParams &fmllr_globals, SpMatrix< double > *grad_scatter)
BaseFloat	FmllrObjGradient (const AmSgmm2 &sgmm, const Matrix< BaseFloat > &xform, Matrix< BaseFloat > *grad_out, Matrix< BaseFloat > *G_out) const
bool	Update (const AmSgmm2 &model, const Sgmm2FmllrGlobalParams &fmllr_globals, const Sgmm2FmllrConfig &opts, Matrix< BaseFloat > *out_xform, BaseFloat *frame_count, BaseFloat *auxf_improv) const
	Computes the FMLLR transform from the accumulated stats, using the pre-transforms in fmllr_globals. More...
int32	Dim () const
	Accessors. More...
const AffineXformStats &	stats () const

Private Member Functions
	KALDI_DISALLOW_COPY_AND_ASSIGN (FmllrSgmm2Accs)

Private Attributes
AffineXformStats	stats_
	Accumulated stats. More...
int32	dim_
	Dimension of feature vectors. More...

Detailed Description

Class for computing the accumulators needed for the maximum-likelihood estimate of FMLLR transforms for a subspace GMM acoustic model.

Definition at line 122 of file fmllr-sgmm2.h.

Constructor & Destructor Documentation

FmllrSgmm2Accs()

FmllrSgmm2Accs ( )

inline

Definition at line 124 of file fmllr-sgmm2.h.

: dim_(-1) {}

~FmllrSgmm2Accs()

~FmllrSgmm2Accs ( )

inline

Definition at line 125 of file fmllr-sgmm2.h.

{}

Member Function Documentation

Accumulate()

BaseFloat Accumulate	(	const AmSgmm2 &	sgmm,
		const VectorBase< BaseFloat > &	data,
		const Sgmm2PerFrameDerivedVars &	frame_vars,
		int32	state_index,
		BaseFloat	weight,
		Sgmm2PerSpkDerivedVars *	spk
	)

Accumulation routine that computes the Gaussian posteriors and calls the AccumulateFromPosteriors function with the computed posteriors.

The 'data' argument is not FMLLR-transformed and is needed in addition to the the 'frame_vars' since the latter only contains a copy of the transformed feature vector.

Definition at line 156 of file fmllr-sgmm2.cc.

References AmSgmm2::ComponentPosteriors(), Sgmm2PerFrameDerivedVars::gselect, and MatrixBase< Real >::Scale().

Referenced by TestSgmm2FmllrAccsIO(), and TestSgmm2FmllrSubspace().

                                                                  {
  // Calulate Gaussian posteriors and collect statistics
  Matrix<BaseFloat> posteriors;
  BaseFloat log_like = model.ComponentPosteriors(frame_vars, pdf_index,
                                                 spk, &posteriors);
  posteriors.Scale(weight);
  AccumulateFromPosteriors(model, *spk, data, frame_vars.gselect, posteriors,
                           pdf_index);
  return log_like;
}

AccumulateForFmllrSubspace()

void AccumulateForFmllrSubspace	(	const AmSgmm2 &	sgmm,
		const Sgmm2FmllrGlobalParams &	fmllr_globals,
		SpMatrix< double > *	grad_scatter
	)

Definition at line 205 of file fmllr-sgmm2.cc.

References SpMatrix< Real >::AddVec2(), kaldi::ApplyHessianXformToGradient(), kaldi::ApplyPreXformToGradient(), VectorBase< Real >::CopyRowsFromMat(), AmSgmm2::FeatureDim(), KALDI_LOG, KALDI_WARN, kaldi::kSetZero, kaldi::kUndefined, VectorBase< Real >::Scale(), and MatrixBase< Real >::SetUnit().

Referenced by TestSgmm2FmllrSubspace().

                                                                           {
  if (stats_.beta_ <= 0.0) {
    KALDI_WARN << "Not committing any stats since no stats accumulated.";
    return;
  }
  int32 dim = sgmm.FeatureDim();
  Matrix<BaseFloat> xform(dim, dim + 1, kUndefined);
  xform.SetUnit();
  Matrix<BaseFloat> grad(dim, dim + 1, kSetZero);
  this->FmllrObjGradient(sgmm, xform, &grad, NULL);
  Matrix<BaseFloat> pre_xformed_grad(dim, dim + 1, kSetZero);
  ApplyPreXformToGradient(globals, grad, &pre_xformed_grad);
  Matrix<BaseFloat> hess_xformed_grad(dim, dim + 1, kSetZero);
  ApplyHessianXformToGradient(globals, pre_xformed_grad, &hess_xformed_grad);
  Vector<double> grad_vec(dim * (dim + 1));
  grad_vec.CopyRowsFromMat(hess_xformed_grad);
  grad_vec.Scale(1 / std::sqrt(stats_.beta_));
  grad_scatter->AddVec2(1.0, grad_vec);
  KALDI_LOG << "Frame counts for when committing fMLLR subspace stats are "
            << stats_.beta_;
}

AccumulateFromPosteriors()

void AccumulateFromPosteriors	(	const AmSgmm2 &	sgmm,
		const Sgmm2PerSpkDerivedVars &	spk,
		const VectorBase< BaseFloat > &	data,
		const std::vector< int32 > &	gauss_select,
		const Matrix< BaseFloat > &	posteriors,
		int32	state_index
	)

Definition at line 171 of file fmllr-sgmm2.cc.

References SpMatrix< Real >::AddVec2(), AmSgmm2::GetVarScaledSubstateSpeakerMean(), rnnlm::i, kaldi::kSetZero, AmSgmm2::NumSubstatesForGroup(), AmSgmm2::Pdf2Group(), and VectorBase< Real >::Range().

Referenced by kaldi::AccumulateForUtterance().

              {
  Vector<double> var_scaled_mean(dim_), extended_data(dim_+1);
  extended_data.Range(0, dim_).CopyFromVec(data);
  extended_data(dim_) = 1.0;
  SpMatrix<double> scatter(dim_+1, kSetZero);
  scatter.AddVec2(1.0, extended_data);
  int32 j1 = model.Pdf2Group(j2);
  for (int32 ki = 0, ki_max = gselect.size(); ki < ki_max; ki++) {
    int32 i = gselect[ki];

    for (int32 m = 0; m < model.NumSubstatesForGroup(j1); m++) {
      // posterior gamma_{jkmi}(t)                             eq.(39)
      BaseFloat gammat_jmi = posteriors(ki, m);

      // Accumulate statistics for non-zero gaussian posterior
      if (gammat_jmi > 0.0) {
        stats_.beta_ += gammat_jmi;
        model.GetVarScaledSubstateSpeakerMean(j1, m, i, spk,
                                              &var_scaled_mean);
        // Eq. (52): K += \gamma_{jmi} \Sigma_{i}^{-1} \mu_{jmi}^{(s)} x^{+T}
        stats_.K_.AddVecVec(gammat_jmi, var_scaled_mean, extended_data);
        // Eq. (53): G_{i} += \gamma_{jmi} x^{+} x^{+T}
        stats_.G_[i].AddSp(gammat_jmi, scatter);
      }  // non-zero posteriors
    }  // loop over substates
  }  // loop over selected Gaussians
}

Dim()

int32 Dim ( ) const

inline

Accessors.

Definition at line 171 of file fmllr-sgmm2.h.

{ return dim_; }

FmllrObjGradient()

BaseFloat FmllrObjGradient	(	const AmSgmm2 &	sgmm,
		const Matrix< BaseFloat > &	xform,
		Matrix< BaseFloat > *	grad_out,
		Matrix< BaseFloat > *	G_out
	)		const

Definition at line 229 of file fmllr-sgmm2.cc.

References MatrixBase< Real >::AddMat(), MatrixBase< Real >::AddSpMat(), MatrixBase< Real >::CopyFromMat(), AmSgmm2::FeatureDim(), AmSgmm2::GetInvCovars(), rnnlm::i, KALDI_ASSERT, kaldi::kNoTrans, kaldi::kSetZero, kaldi::kTrans, MatrixBase< Real >::LogDet(), AmSgmm2::NumGauss(), MatrixBase< Real >::Range(), MatrixBase< Real >::Scale(), and kaldi::TraceMatMat().

                                                                          {
  int32 dim = sgmm.FeatureDim(),
      num_gauss = sgmm.NumGauss();
  KALDI_ASSERT(stats_.G_.size() == static_cast<size_t>(num_gauss));
  Matrix<double> xform_d(xform);
  SubMatrix<double> A(xform_d, 0, dim, 0, dim);
  Matrix<double> xform_g(dim, dim + 1), total_g(dim, dim + 1);
  SpMatrix<double> inv_covar(dim);
  double obj = stats_.beta_ * A.LogDet() +
      TraceMatMat(xform_d, stats_.K_, kTrans);
  for (int32 i = 0; i < num_gauss; i++) {
    sgmm.GetInvCovars(i, &inv_covar);
    xform_g.AddMatSp(1.0, xform_d, kNoTrans, stats_.G_[i], 0.0);
    total_g.AddSpMat(1.0, inv_covar, xform_g, kNoTrans, 1.0);
  }
  obj -= 0.5 * TraceMatMat(xform_d, total_g, kTrans);
  if (G_out != NULL) G_out->CopyFromMat(total_g);

  // Compute the gradient: P = \beta [(A^{-1})^{T} , 0] + K - S
  if (grad_out != NULL) {
    Matrix<double> grad_d(dim, dim + 1, kSetZero);
    grad_d.Range(0, dim, 0, dim).CopyFromMat(A);
    grad_d.Range(0, dim, 0, dim).InvertDouble();
    grad_d.Range(0, dim, 0, dim).Transpose();
    grad_d.Scale(stats_.beta_);
    grad_d.AddMat(-1.0, total_g, kNoTrans);
    grad_d.AddMat(1.0, stats_.K_, kNoTrans);
    grad_out->CopyFromMat(grad_d);
  }

  return obj;
}

Init()

void Init	(	int32	dim,
		int32	num_gaussians
	)

Definition at line 146 of file fmllr-sgmm2.cc.

Referenced by main(), TestSgmm2FmllrAccsIO(), and TestSgmm2FmllrSubspace().

                                                        {
  if (dim == 0) {  // empty stats
    dim_ = 0;  // non-zero dimension is meaningless in empty stats
    stats_.Init(0, 0);  // clear the stats
  } else {
    dim_ = dim;
    stats_.Init(dim, num_gaussians);
  }
}

KALDI_DISALLOW_COPY_AND_ASSIGN()

KALDI_DISALLOW_COPY_AND_ASSIGN ( FmllrSgmm2Accs  )

private

Read()

void Read	(	std::istream &	in_stream,
		bool	binary,
		bool	add
	)

Definition at line 275 of file fmllr-sgmm2.cc.

References kaldi::ExpectToken(), KALDI_ASSERT, and kaldi::ReadBasicType().

Referenced by TestSgmm2FmllrAccsIO().

                                                               {
  ExpectToken(in, binary, "<FMLLRACCS>");
  ExpectToken(in, binary, "<DIMENSION>");
  ReadBasicType(in, binary, &dim_);
  KALDI_ASSERT(dim_ > 0);
  ExpectToken(in, binary, "<STATS>");
  stats_.Read(in, binary, add);
  ExpectToken(in, binary, "</FMLLRACCS>");
}

SetZero()

void SetZero ( )

inline

Definition at line 128 of file fmllr-sgmm2.h.

Referenced by main().

{ stats_.SetZero(); }

stats()

const AffineXformStats& stats ( ) const

inline

Definition at line 172 of file fmllr-sgmm2.h.

{ return stats_; }

Update()

bool Update	(	const AmSgmm2 &	model,
		const Sgmm2FmllrGlobalParams &	fmllr_globals,
		const Sgmm2FmllrConfig &	opts,
		Matrix< BaseFloat > *	out_xform,
		BaseFloat *	frame_count,
		BaseFloat *	auxf_improv
	)		const

Computes the FMLLR transform from the accumulated stats, using the pre-transforms in fmllr_globals.

Expects the transform matrix out_xform to be initialized to the correct size. Returns true if the transform was updated (i.e. had enough counts).

Definition at line 356 of file fmllr-sgmm2.cc.

References MatrixBase< Real >::AddMat(), kaldi::ApplyHessianXformToGradient(), kaldi::ApplyInvHessianXformToChange(), kaldi::ApplyInvPreXformToChange(), kaldi::ApplyPreXformToGradient(), kaldi::AssertEqual(), Sgmm2FmllrConfig::bases_occ_scale, kaldi::CalcFmllrStepSize(), MatrixBase< Real >::CopyFromMat(), Sgmm2FmllrGlobalParams::fmllr_bases_, Sgmm2FmllrConfig::fmllr_iters, Sgmm2FmllrConfig::fmllr_min_count, Sgmm2FmllrConfig::fmllr_min_count_basis, Sgmm2FmllrConfig::fmllr_min_count_full, Sgmm2FmllrGlobalParams::HasBasis(), Sgmm2FmllrGlobalParams::IsEmpty(), KALDI_ASSERT, KALDI_ERR, KALDI_LOG, KALDI_VLOG, KALDI_WARN, kaldi::kNoTrans, kaldi::kSetZero, kaldi::kTrans, kaldi::kUndefined, MatrixBase< Real >::LogDet(), Sgmm2FmllrConfig::num_fmllr_bases, MatrixBase< Real >::NumCols(), MatrixBase< Real >::NumRows(), MatrixBase< Real >::Scale(), MatrixBase< Real >::SetZero(), and kaldi::TraceMatMat().

Referenced by main(), TestSgmm2FmllrAccsIO(), and TestSgmm2FmllrSubspace().

                                                                              {
  BaseFloat auxf_improv = 0.0, logdet = 0.0;
  KALDI_ASSERT(out_xform->NumRows() == dim_ && out_xform->NumCols() == dim_+1);
  BaseFloat mincount = (globals.HasBasis() ?
      std::min(opts.fmllr_min_count_basis, opts.fmllr_min_count_full) :
      opts.fmllr_min_count);
  bool using_subspace = (globals.HasBasis() ?
      (stats_.beta_ < opts.fmllr_min_count_full) : false);

  if (globals.IsEmpty())
    KALDI_ERR << "Must set up pre-transforms before estimating FMLLR.";

  KALDI_VLOG(1) << "Mincount = " << mincount << "; Basis: "
                << std::string(globals.HasBasis()? "yes; " : "no; ")
                << "Using subspace: " << std::string(using_subspace? "yes; "
                    : "no; ");

  int32 num_bases = 0;
  if (using_subspace) {
    KALDI_ASSERT(globals.fmllr_bases_.size() != 0);
    int32 max_bases = std::min(static_cast<int32>(globals.fmllr_bases_.size()),
                               opts.num_fmllr_bases);
    num_bases = (opts.bases_occ_scale <= 0.0)? max_bases :
        std::min(max_bases, static_cast<int32>(std::floor(opts.bases_occ_scale
                                                          * stats_.beta_)));
    KALDI_VLOG(1) << "Have " << stats_.beta_ << " frames for speaker: Using "
                  << num_bases << " fMLLR bases.";
  }

  // initialization just to get rid of compile errors.
  BaseFloat auxf_old = 0, auxf_new = 0;
  if (frame_count != NULL) *frame_count = stats_.beta_;

  // If occupancy is greater than the min count, update the transform
  if (stats_.beta_ >= mincount) {
    for (int32 iter = 0; iter < opts.fmllr_iters; iter++) {
      Matrix<BaseFloat> grad(dim_, dim_ + 1, kSetZero);
      Matrix<BaseFloat> G(dim_, dim_ + 1, kSetZero);
      auxf_new = this->FmllrObjGradient(sgmm, *out_xform, &grad, &G);

      // For diagnostic purposes
      KALDI_VLOG(3) << "Iter " << iter << ": Auxiliary function = "
          << (auxf_new / stats_.beta_) << " per frame over " << stats_.beta_
          << " frames";

      if (iter > 0) {
        // For diagnostic purposes
        KALDI_VLOG(2) << "Iter " << iter << ": Auxiliary function improvement: "
            << ((auxf_new - auxf_old) / stats_.beta_) << " per frame over "
            << (stats_.beta_) << " frames";
        auxf_improv += auxf_new - auxf_old;
      }

      Matrix<BaseFloat> pre_xformed_grad(dim_, dim_ + 1, kSetZero);
      ApplyPreXformToGradient(globals, grad, &pre_xformed_grad);
//      std::cout << "Pre-X Grad = " << pre_xformed_grad << std::endl;

      // Transform P_sk with the Hessian
      Matrix<BaseFloat> hess_xformed_grad(dim_, dim_ + 1, kSetZero);
      ApplyHessianXformToGradient(globals, pre_xformed_grad,
                                  &hess_xformed_grad);
//      std::cout << "Hess-X Grad = " << hess_xformed_grad << std::endl;

      // Update the actual FMLLR transform matrices
      Matrix<BaseFloat> hess_xformed_delta(dim_, dim_ + 1, kUndefined);
      if (using_subspace) {
        // Note that in this case we can simply store the speaker-specific
        // coefficients for each of the basis matrices. The current
        // implementation stores the computed transform to simplify the code!
        hess_xformed_delta.SetZero();
        for (int32 b = 0; b < num_bases; b++) {  // Eq (B.20)
          hess_xformed_delta.AddMat(TraceMatMat(globals.fmllr_bases_[b],
                                                hess_xformed_grad, kTrans),
                                    globals.fmllr_bases_[b], kNoTrans);
        }
        hess_xformed_delta.Scale(1 / stats_.beta_);
      } else {
        hess_xformed_delta.CopyFromMat(hess_xformed_grad);
        hess_xformed_delta.Scale(1 / stats_.beta_);  // Eq. (B.19)
      }

//      std::cout << "Hess-X Delta = " << hess_xformed_delta << std::endl;

      // Transform Delta with the Hessian
      Matrix<BaseFloat> pre_xformed_delta(dim_, dim_ + 1, kSetZero);
      ApplyInvHessianXformToChange(globals, hess_xformed_delta,
                                   &pre_xformed_delta);

      // Apply inverse pre-transform to Delta
      Matrix<BaseFloat> delta(dim_, dim_ + 1, kSetZero);
      ApplyInvPreXformToChange(globals, pre_xformed_delta, &delta);

#ifdef KALDI_PARANOID
      // Check whether co-ordinate transformation is correct.
      {
        BaseFloat tr1 = TraceMatMat(delta, grad, kTrans);
        BaseFloat tr2 = TraceMatMat(pre_xformed_delta, pre_xformed_grad,
                                    kTrans);
        BaseFloat tr3 = TraceMatMat(hess_xformed_delta, hess_xformed_grad,
                                    kTrans);
        AssertEqual(tr1, tr2, 1e-5);
        AssertEqual(tr2, tr3, 1e-5);
      }
#endif

      // Calculate the optimal step size
      SubMatrix<BaseFloat> A(*out_xform, 0, dim_, 0, dim_);
      BaseFloat step_size = CalcFmllrStepSize(stats_, sgmm, delta, A, G,
                                              opts.fmllr_iters);

      // Update: W <-- W + k \Delta   Eq. (B.34)
      out_xform->AddMat(step_size, delta, kNoTrans);
      auxf_old = auxf_new;

      // Check the objective function change for the last iteration
      if (iter == opts.fmllr_iters - 1) {
        auxf_new = this->FmllrObjGradient(sgmm, *out_xform, NULL, NULL);
        logdet = A.LogDet();
        // SubMatrix A points to the memory location of out_xform, and so will
        // contain the updated value

        KALDI_VLOG(2) << "Iter " << iter << ": Auxiliary function improvement: "
            << ((auxf_new - auxf_old) / stats_.beta_) << " per frame over "
            << (stats_.beta_) << " frames";
        auxf_improv += auxf_new - auxf_old;
      }
    }
    if (auxf_out != NULL) *auxf_out = auxf_improv;
    auxf_improv /= (stats_.beta_ + 1.0e-10);

    KALDI_LOG << "Auxiliary function improvement for FMLLR = " << auxf_improv
        << " per frame over " << stats_.beta_ << " frames. Log-determinant = "
        << logdet;
    return true;
  } else {
    KALDI_ASSERT(stats_.beta_ < mincount);
//    std::cerr.precision(10);
//    std::cerr.setf(std::ios::fixed,std::ios::floatfield);
    KALDI_WARN << "Not updating FMLLR because count is " << stats_.beta_
        << " < " << (mincount);
    if (auxf_out != NULL) *auxf_out = 0.0;
    return false;
  }  // Do not use the transform if it does not have enough counts
  KALDI_ASSERT(false);  // Should never be reached.
}

Write()

void Write	(	std::ostream &	out_stream,
		bool	binary
	)		const

Definition at line 266 of file fmllr-sgmm2.cc.

References kaldi::WriteBasicType(), and kaldi::WriteToken().

Referenced by TestSgmm2FmllrAccsIO().

                                                             {
  WriteToken(out, binary, "<FMLLRACCS>");
  WriteToken(out, binary, "<DIMENSION>");
  WriteBasicType(out, binary, dim_);
  WriteToken(out, binary, "<STATS>");
  stats_.Write(out, binary);
  WriteToken(out, binary, "</FMLLRACCS>");
}

Member Data Documentation

dim_

int32 dim_

private

Dimension of feature vectors.

Definition at line 176 of file fmllr-sgmm2.h.

stats_

AffineXformStats stats_

private

Accumulated stats.

Definition at line 175 of file fmllr-sgmm2.h.

---

The documentation for this class was generated from the following files:

• sgmm2/fmllr-sgmm2.h
• sgmm2/fmllr-sgmm2.cc