natural-gradient-online.h

Go to the documentation of this file.

// nnet3/natural-gradient-online.h

// Copyright 2013-2015   Johns Hopkins University (author: Daniel Povey)

// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.

#ifndef KALDI_NNET3_NATURAL_GRADIENT_ONLINE_H_
#define KALDI_NNET3_NATURAL_GRADIENT_ONLINE_H_

#include <iostream>
#include "base/kaldi-common.h"
#include "matrix/matrix-lib.h"
#include "cudamatrix/cu-matrix-lib.h"

namespace kaldi {
namespace nnet3 {


class OnlineNaturalGradient {
 public:
  OnlineNaturalGradient();

  void SetRank(int32 rank);
  void SetUpdatePeriod(int32 update_period);
  // num_samples_history is a time-constant (in samples) that determines eta.
  void SetNumSamplesHistory(BaseFloat num_samples_history);
  // num_minibatches_history is a time-constant measured in minibatches that
  // provides an alternative way to set eta (the constant that determines how
  // fast we update the Fisher matrix).  If set to a value >0, it overrides any
  // value of 'num_samples_history' that is present.
  void SetNumMinibatchesHistory(BaseFloat num_minibatches_history);

  void SetAlpha(BaseFloat alpha);
  void TurnOnDebug() { self_debug_ = true; }
  BaseFloat GetNumSamplesHistory() const { return num_samples_history_; }
  BaseFloat GetNumMinibatchesHistory() const { return num_minibatches_history_; }
  BaseFloat GetAlpha() const { return alpha_; }
  int32 GetRank() const { return rank_; }
  int32 GetUpdatePeriod() const { return update_period_; }

  // see comment where 'frozen_' is declared.
  inline void Freeze(bool frozen) { frozen_ = frozen; }

  void PreconditionDirections(CuMatrixBase<BaseFloat> *X,
                              BaseFloat *scale);



  // Copy constructor.
  explicit OnlineNaturalGradient(const OnlineNaturalGradient &other);
  // Assignent operator
  OnlineNaturalGradient &operator = (const OnlineNaturalGradient &other);

  // Shallow swap
  void Swap(OnlineNaturalGradient *other);
 private:


  // This is an internal function called from PreconditionDirections().
  // Note: WJKL_t (dimension 2*R by D + R) is [ W_t L_t; J_t K_t ].
  void PreconditionDirectionsInternal(const BaseFloat rho_t,
                                      const BaseFloat tr_X_Xt,
                                      bool updating,
                                      const Vector<BaseFloat> &d_t,
                                      CuMatrixBase<BaseFloat> *WJKL_t,
                                      CuMatrixBase<BaseFloat> *X_t);


  // Works out from t_ and various class variables whether we will update
  // the parameters on this iteration (returns true if so).
  bool Updating() const;

  void ComputeEt(const VectorBase<BaseFloat> &d_t,
                 BaseFloat beta_t,
                 VectorBase<BaseFloat> *e_t,
                 VectorBase<BaseFloat> *sqrt_e_t,
                 VectorBase<BaseFloat> *inv_sqrt_e_t) const;

  void ComputeZt(int32 N,
                 BaseFloat rho_t,
                 const VectorBase<BaseFloat> &d_t,
                 const VectorBase<BaseFloat> &inv_sqrt_e_t,
                 const MatrixBase<BaseFloat> &K_t,
                 const MatrixBase<BaseFloat> &L_t,
                 SpMatrix<double> *Z_t) const;
  // Computes W_{t+1}.  Overwrites J_t.
  void ComputeWt1(int32 N,
                  const VectorBase<BaseFloat> &d_t,
                  const VectorBase<BaseFloat> &d_t1,
                  BaseFloat rho_t,
                  BaseFloat rho_t1,
                  const MatrixBase<BaseFloat> &U_t,
                  const VectorBase<BaseFloat> &sqrt_c_t,
                  const VectorBase<BaseFloat> &inv_sqrt_e_t,
                  const CuMatrixBase<BaseFloat> &W_t,
                  CuMatrixBase<BaseFloat> *J_t,
                  CuMatrixBase<BaseFloat> *W_t1) const;

  // This function is called if C_t has high condition number; it makes sure
  // that R_{t+1} is orthogonal.  See the section in the extended comment above
  // on "keeping R_t orthogonal".
  void ReorthogonalizeRt1(const VectorBase<BaseFloat> &d_t1,
                          BaseFloat rho_t1,
                          CuMatrixBase<BaseFloat> *W_t1,
                          CuMatrixBase<BaseFloat> *temp_W,
                          CuMatrixBase<BaseFloat> *temp_O);

  void Init(const CuMatrixBase<BaseFloat> &R0);

  // Initialize to some small 'default' values, called from Init().  Init() then
  // does a few iterations of update with the first batch's data to give more
  // reasonable values.
  void InitDefault(int32 D);

  // initializes R, which is assumed to have at least as many columns as rows,
  // to a specially designed matrix with orthonormal rows, that has no zero rows
  // or columns.
  static void InitOrthonormalSpecial(CuMatrixBase<BaseFloat> *R);

  // Returns the value eta (with 0 < eta < 1) which reflects how fast we update
  // the estimate of the Fisher matrix (larger == faster).  This is a function
  // rather than a constant because we set this indirectly, via
  // num_samples_history_ or num_minibatches_history_.  The argument N is the
  // number of vectors we're preconditioning, which is the number of rows in the
  // argument R to PreconditionDirections(); you can think of it as the number
  // of vectors we're preconditioning (and in the common case it's some multiple
  // of the minibatch size)
  BaseFloat Eta(int32 N) const;

  // called if self_debug_ = true, makes sure the members satisfy certain
  // properties.
  void SelfTest() const;

  // Configuration values:

  // The rank of the correction to the unit matrix (e.g. 20).
  int32 rank_;

  // After a few initial iterations of updating whenever we can, we start only
  // updating the Fisher-matrix parameters every "update_period_" minibatches;
  // this saves time.
  int32 update_period_;


  // num_samples_history_ determines the value of eta, which in turn affects how
  // fast we update our estimate of the covariance matrix.  We've done it this
  // way in order to make it easy to have a single configuration value that
  // doesn't have to be changed when we change the minibatch size.
  // Note: if num_minibatches_history_ is >0.0, it overrides this.
  BaseFloat num_samples_history_;


  // num_minibatches_history_ is simpler alternative to num_samples_history_ for
  // determining the value of eta, which in turn affects how fast we update our
  // estimate of the covariance matrix.  eta will be set to 1.0 /
  // num_minibatches_history_.  We require that num_minibatches_history_ > 0.0;
  // it will normally be something like 10.0, if set.  It makes sense to set
  // 'num_minibatches_history_' when the rows of the matrix we are
  // preconditioning can't be interpreted as independent samples, so the number
  // of rows is not relevant to determining how fast to update the covariance
  // matrix.
  BaseFloat num_minibatches_history_;


  // alpha controls how much we smooth the Fisher matrix with the unit matrix.
  // e.g. alpha = 4.0.
  BaseFloat alpha_;

  // epsilon is an absolute floor on the unit-matrix scaling factor rho_t in our
  // Fisher estimate, which we set to 1.0e-10.  We don't actually make this
  // configurable from the command line.  It's needed to avoid crashes on
  // all-zero inputs.
  BaseFloat epsilon_;

  // delta is a relative floor on the unit-matrix scaling factor rho_t in our
  // Fisher estimate, which we set to 1.0e-05: this is relative to the largest
  // value of D_t.  It's needed to control roundoff error.  We apply the same
  // floor to the eigenvalues in D_t.
  BaseFloat delta_;

  // this is set to true if the user has called the function Freeze(true), until
  // they call  Freeze(false).  It's used to disable the natural gradient
  // update (and stop incrementing t_).  However, if the object is uninitialized
  // (t_ == 0) it doesn't prevent it from being initialized.  This is used
  // in adversarial training to ensure that the Fisher matrix is updated only
  // the *second* time we see the same data (to avoid biasing the update).
  bool frozen_;

  // t is a counter that measures how many times the user has previously called
  // PreconditionDirections(); it's 0 if that has never been called.
  int32 t_;

  // If true, activates certain checks.
  bool self_debug_;

  CuMatrix<BaseFloat> W_t_;
  BaseFloat rho_t_;
  Vector<BaseFloat> d_t_;
};

} // namespace nnet3
} // namespace kaldi


#endif