AffineComponentPreconditioned Class Reference

#include <nnet-component.h>

Inheritance diagram for AffineComponentPreconditioned:

Collaboration diagram for AffineComponentPreconditioned:

Public Member Functions
virtual std::string	Type () const
virtual void	Read (std::istream &is, bool binary)
virtual void	Write (std::ostream &os, bool binary) const
	Write component to stream. More...
void	Init (BaseFloat learning_rate, int32 input_dim, int32 output_dim, BaseFloat param_stddev, BaseFloat bias_stddev, BaseFloat alpha, BaseFloat max_change)
void	Init (BaseFloat learning_rate, BaseFloat alpha, BaseFloat max_change, std::string matrix_filename)
virtual void	InitFromString (std::string args)
	Initialize, typically from a line of a config file. More...
virtual std::string	Info () const
virtual Component *	Copy () const
	Copy component (deep copy). More...
	AffineComponentPreconditioned ()
void	SetMaxChange (BaseFloat max_change)
Public Member Functions inherited from AffineComponent
	AffineComponent (const AffineComponent &other)
	AffineComponent (const CuMatrixBase< BaseFloat > &linear_params, const CuVectorBase< BaseFloat > &bias_params, BaseFloat learning_rate)
virtual int32	InputDim () const
	Get size of input vectors. More...
virtual int32	OutputDim () const
	Get size of output vectors. More...
void	Init (BaseFloat learning_rate, int32 input_dim, int32 output_dim, BaseFloat param_stddev, BaseFloat bias_stddev)
void	Init (BaseFloat learning_rate, std::string matrix_filename)
virtual void	Resize (int32 input_dim, int32 output_dim)
Component *	CollapseWithNext (const AffineComponent &next) const
Component *	CollapseWithNext (const FixedAffineComponent &next) const
Component *	CollapseWithNext (const FixedScaleComponent &next) const
Component *	CollapseWithPrevious (const FixedAffineComponent &prev) const
	AffineComponent ()
virtual bool	BackpropNeedsInput () const
virtual bool	BackpropNeedsOutput () const
virtual void	Propagate (const ChunkInfo &in_info, const ChunkInfo &out_info, const CuMatrixBase< BaseFloat > &in, CuMatrixBase< BaseFloat > *out) const
	Perform forward pass propagation Input->Output. More...
virtual void	Scale (BaseFloat scale)
	This new virtual function scales the parameters by this amount. More...
virtual void	Add (BaseFloat alpha, const UpdatableComponent &other)
	This new virtual function adds the parameters of another updatable component, times some constant, to the current parameters. More...
virtual void	Backprop (const ChunkInfo &in_info, const ChunkInfo &out_info, const CuMatrixBase< BaseFloat > &in_value, const CuMatrixBase< BaseFloat > &out_value, const CuMatrixBase< BaseFloat > &out_deriv, Component *to_update, CuMatrix< BaseFloat > *in_deriv) const
	Perform backward pass propagation of the derivative, and also either update the model (if to_update == this) or update another model or compute the model derivative (otherwise). More...
virtual void	SetZero (bool treat_as_gradient)
	Set parameters to zero, and if treat_as_gradient is true, we'll be treating this as a gradient so set the learning rate to 1 and make any other changes necessary (there's a variable we have to set for the MixtureProbComponent). More...
virtual BaseFloat	DotProduct (const UpdatableComponent &other) const
	Here, "other" is a component of the same specific type. More...
virtual void	PerturbParams (BaseFloat stddev)
	We introduce a new virtual function that only applies to class UpdatableComponent. More...
virtual void	SetParams (const VectorBase< BaseFloat > &bias, const MatrixBase< BaseFloat > &linear)
const CuVector< BaseFloat > &	BiasParams ()
const CuMatrix< BaseFloat > &	LinearParams ()
virtual int32	GetParameterDim () const
	The following new virtual function returns the total dimension of the parameters in this class. More...
virtual void	Vectorize (VectorBase< BaseFloat > *params) const
	Turns the parameters into vector form. More...
virtual void	UnVectorize (const VectorBase< BaseFloat > &params)
	Converts the parameters from vector form. More...
virtual void	LimitRank (int32 dimension, AffineComponent **a, AffineComponent **b) const
	This function is for getting a low-rank approximations of this AffineComponent by two AffineComponents. More...
void	Widen (int32 new_dimension, BaseFloat param_stddev, BaseFloat bias_stddev, std::vector< NonlinearComponent *> c2, AffineComponent *c3)
	This function is implemented in widen-nnet.cc. More...
Public Member Functions inherited from UpdatableComponent
	UpdatableComponent (const UpdatableComponent &other)
void	Init (BaseFloat learning_rate)
	UpdatableComponent (BaseFloat learning_rate)
	UpdatableComponent ()
virtual	~UpdatableComponent ()
void	SetLearningRate (BaseFloat lrate)
	Sets the learning rate of gradient descent. More...
BaseFloat	LearningRate () const
	Gets the learning rate of gradient descent. More...
Public Member Functions inherited from Component
	Component ()
virtual int32	Index () const
	Returns the index in the sequence of layers in the neural net; intended only to be used in debugging information. More...
virtual void	SetIndex (int32 index)
virtual std::vector< int32 >	Context () const
	Return a vector describing the temporal context this component requires for each frame of output, as a sorted list. More...
void	Propagate (const ChunkInfo &in_info, const ChunkInfo &out_info, const CuMatrixBase< BaseFloat > &in, CuMatrix< BaseFloat > *out) const
	A non-virtual propagate function that first resizes output if necessary. More...
virtual	~Component ()

Protected Member Functions
	KALDI_DISALLOW_COPY_AND_ASSIGN (AffineComponentPreconditioned)
BaseFloat	GetScalingFactor (const CuMatrix< BaseFloat > &in_value_precon, const CuMatrix< BaseFloat > &out_deriv_precon)
	The following function is only called if max_change_ > 0. More...
virtual void	Update (const CuMatrixBase< BaseFloat > &in_value, const CuMatrixBase< BaseFloat > &out_deriv)
Protected Member Functions inherited from AffineComponent
virtual void	UpdateSimple (const CuMatrixBase< BaseFloat > &in_value, const CuMatrixBase< BaseFloat > &out_deriv)
const AffineComponent &	operator= (const AffineComponent &other)

Protected Attributes
BaseFloat	alpha_
BaseFloat	max_change_
Protected Attributes inherited from AffineComponent
CuMatrix< BaseFloat >	linear_params_
CuVector< BaseFloat >	bias_params_
bool	is_gradient_
Protected Attributes inherited from UpdatableComponent
BaseFloat	learning_rate_
	learning rate (0.0..0.01) More...

Additional Inherited Members
Static Public Member Functions inherited from Component
static Component *	ReadNew (std::istream &is, bool binary)
	Read component from stream. More...
static Component *	NewFromString (const std::string &initializer_line)
	Initialize the Component from one line that will contain first the type, e.g. More...
static Component *	NewComponentOfType (const std::string &type)
	Return a new Component of the given type e.g. More...

Detailed Description

Definition at line 948 of file nnet-component.h.

Constructor & Destructor Documentation

AffineComponentPreconditioned()

AffineComponentPreconditioned ( )

inline

Definition at line 964 of file nnet-component.h.

: alpha_(1.0), max_change_(0.0) { }

Member Function Documentation

Copy()

Component * Copy ( ) const

virtual

Copy component (deep copy).

Reimplemented from AffineComponent.

Definition at line 1500 of file nnet-component.cc.

References AffineComponentPreconditioned::alpha_, AffineComponent::bias_params_, AffineComponent::is_gradient_, UpdatableComponent::learning_rate_, AffineComponent::linear_params_, and AffineComponentPreconditioned::max_change_.

                                                     {
  AffineComponentPreconditioned *ans = new AffineComponentPreconditioned();
  ans->learning_rate_ = learning_rate_;
  ans->linear_params_ = linear_params_;
  ans->bias_params_ = bias_params_;
  ans->alpha_ = alpha_;
  ans->max_change_ = max_change_;
  ans->is_gradient_ = is_gradient_;
  return ans;
}

GetScalingFactor()

BaseFloat GetScalingFactor ( const CuMatrix< BaseFloat > &  in_value_precon, const CuMatrix< BaseFloat > &  out_deriv_precon  )

protected

The following function is only called if max_change_ > 0.

It returns the greatest value alpha <= 1.0 such that (alpha times the sum over the row-index of the two matrices of the product the l2 norms of the two rows times learning_rate_) is <= max_change.

Definition at line 1512 of file nnet-component.cc.

References CuVectorBase< Real >::AddDiagMat2(), Component::Index(), KALDI_ASSERT, KALDI_LOG, kaldi::kNoTrans, UpdatableComponent::learning_rate_, CuMatrixBase< Real >::NumRows(), and kaldi::VecVec().

                                                 {
  static int scaling_factor_printed = 0;

  KALDI_ASSERT(in_value_precon.NumRows() == out_deriv_precon.NumRows());
  CuVector<BaseFloat> in_norm(in_value_precon.NumRows()),
      out_deriv_norm(in_value_precon.NumRows());
  in_norm.AddDiagMat2(1.0, in_value_precon, kNoTrans, 0.0);
  out_deriv_norm.AddDiagMat2(1.0, out_deriv_precon, kNoTrans, 0.0);
  // Get the actual l2 norms, not the squared l2 norm.
  in_norm.ApplyPow(0.5);
  out_deriv_norm.ApplyPow(0.5);
  BaseFloat sum = learning_rate_ * VecVec(in_norm, out_deriv_norm);
  // sum is the product of norms that we are trying to limit
  // to max_value_.
  KALDI_ASSERT(sum == sum && sum - sum == 0.0 &&
               "NaN in backprop");
  KALDI_ASSERT(sum >= 0.0);
  if (sum <= max_change_) return 1.0;
  else {
    BaseFloat ans = max_change_ / sum;
    if (scaling_factor_printed < 10) {
      KALDI_LOG << "Limiting step size to " << max_change_
                << " using scaling factor " << ans << ", for component index "
                << Index();
      scaling_factor_printed++;
    }
    return ans;
  }
}

Info()

std::string Info ( ) const

virtual

Reimplemented from AffineComponent.

Definition at line 1481 of file nnet-component.cc.

References AffineComponent::bias_params_, AffineComponent::InputDim(), kaldi::kTrans, UpdatableComponent::LearningRate(), AffineComponent::linear_params_, AffineComponent::OutputDim(), kaldi::TraceMatMat(), AffineComponent::Type(), and kaldi::VecVec().

                                                    {
  std::stringstream stream;
  BaseFloat linear_params_size = static_cast<BaseFloat>(linear_params_.NumRows())
      * static_cast<BaseFloat>(linear_params_.NumCols());
  BaseFloat linear_stddev =
      std::sqrt(TraceMatMat(linear_params_, linear_params_, kTrans) /
                linear_params_size),
      bias_stddev = std::sqrt(VecVec(bias_params_, bias_params_) /
                              bias_params_.Dim());
  stream << Type() << ", input-dim=" << InputDim()
         << ", output-dim=" << OutputDim()
         << ", linear-params-stddev=" << linear_stddev
         << ", bias-params-stddev=" << bias_stddev
         << ", learning-rate=" << LearningRate()
         << ", alpha=" << alpha_
         << ", max-change=" << max_change_;
  return stream.str();
}

Init() [1/2]

void Init	(	BaseFloat	learning_rate,
		int32	input_dim,
		int32	output_dim,
		BaseFloat	param_stddev,
		BaseFloat	bias_stddev,
		BaseFloat	alpha,
		BaseFloat	max_change
	)

Definition at line 1442 of file nnet-component.cc.

References AffineComponent::bias_params_, UpdatableComponent::Init(), KALDI_ASSERT, and AffineComponent::linear_params_.

Referenced by kaldi::nnet2::UnitTestAffineComponentPreconditioned().

                                           {
  UpdatableComponent::Init(learning_rate);
  KALDI_ASSERT(input_dim > 0 && output_dim > 0);
  linear_params_.Resize(output_dim, input_dim);
  bias_params_.Resize(output_dim);
  KALDI_ASSERT(output_dim > 0 && input_dim > 0 && param_stddev >= 0.0);
  linear_params_.SetRandn(); // sets to random normally distributed noise.
  linear_params_.Scale(param_stddev);
  bias_params_.SetRandn();
  bias_params_.Scale(bias_stddev);
  alpha_ = alpha;
  KALDI_ASSERT(alpha_ > 0.0);
  max_change_ = max_change; // Note: any value of max_change_is valid, but
  // only values > 0.0 will actually activate the code.
}

Init() [2/2]

void Init	(	BaseFloat	learning_rate,
		BaseFloat	alpha,
		BaseFloat	max_change,
		std::string	matrix_filename
	)

Definition at line 1426 of file nnet-component.cc.

References AffineComponent::bias_params_, UpdatableComponent::Init(), KALDI_ASSERT, AffineComponent::linear_params_, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), CuMatrixBase< Real >::Range(), and kaldi::ReadKaldiObject().

                                                                    {
  UpdatableComponent::Init(learning_rate);
  alpha_ = alpha;
  max_change_ = max_change;
  CuMatrix<BaseFloat> mat;
  ReadKaldiObject(matrix_filename, &mat); // will abort on failure.
  KALDI_ASSERT(mat.NumCols() >= 2);
  int32 input_dim = mat.NumCols() - 1, output_dim = mat.NumRows();
  linear_params_.Resize(output_dim, input_dim);
  bias_params_.Resize(output_dim);
  linear_params_.CopyFromMat(mat.Range(0, output_dim, 0, input_dim));
  bias_params_.CopyColFromMat(mat, input_dim);
}

InitFromString()

void InitFromString ( std::string  args )

virtual

Initialize, typically from a line of a config file.

The "args" will contain any parameters that need to be passed to the Component, e.g. dimensions.

Reimplemented from AffineComponent.

Definition at line 1390 of file nnet-component.cc.

References AffineComponent::Init(), AffineComponent::InputDim(), KALDI_ASSERT, KALDI_ERR, UpdatableComponent::learning_rate_, AffineComponent::OutputDim(), and kaldi::nnet2::ParseFromString().

Referenced by kaldi::nnet2::UnitTestAffineComponentPreconditioned().

                                                                 {
  std::string orig_args(args);
  std::string matrix_filename;
  BaseFloat learning_rate = learning_rate_;
  BaseFloat alpha = 0.1, max_change = 0.0;
  int32 input_dim = -1, output_dim = -1;
  ParseFromString("learning-rate", &args, &learning_rate); // optional.
  ParseFromString("alpha", &args, &alpha);
  ParseFromString("max-change", &args, &max_change);

  if (ParseFromString("matrix", &args, &matrix_filename)) {
    Init(learning_rate, alpha, max_change, matrix_filename);
    if (ParseFromString("input-dim", &args, &input_dim))
      KALDI_ASSERT(input_dim == InputDim() &&
                   "input-dim mismatch vs. matrix.");
    if (ParseFromString("output-dim", &args, &output_dim))
      KALDI_ASSERT(output_dim == OutputDim() &&
                   "output-dim mismatch vs. matrix.");
  } else {
    bool ok = true;
    ok = ok && ParseFromString("input-dim", &args, &input_dim);
    ok = ok && ParseFromString("output-dim", &args, &output_dim);
    BaseFloat param_stddev = 1.0 / std::sqrt(input_dim),
        bias_stddev = 1.0;
    ParseFromString("param-stddev", &args, &param_stddev);
    ParseFromString("bias-stddev", &args, &bias_stddev);
    if (!ok)
      KALDI_ERR << "Bad initializer " << orig_args;
    Init(learning_rate, input_dim, output_dim, param_stddev,
         bias_stddev, alpha, max_change);
  }
  if (!args.empty())
    KALDI_ERR << "Could not process these elements in initializer: "
              << args;
}

KALDI_DISALLOW_COPY_AND_ASSIGN()

KALDI_DISALLOW_COPY_AND_ASSIGN ( AffineComponentPreconditioned  )

protected

Read()

void Read ( std::istream &  is, bool  binary  )

virtual

Reimplemented from AffineComponent.

Definition at line 1360 of file nnet-component.cc.

References AffineComponent::bias_params_, kaldi::nnet2::ExpectOneOrTwoTokens(), kaldi::ExpectToken(), KALDI_ASSERT, UpdatableComponent::learning_rate_, AffineComponent::linear_params_, kaldi::ReadBasicType(), kaldi::ReadToken(), and AffineComponent::Type().

                                                                    {
  std::ostringstream ostr_beg, ostr_end;
  ostr_beg << "<" << Type() << ">"; // e.g. "<AffineComponentPreconditioned>"
  ostr_end << "</" << Type() << ">"; // e.g. "</AffineComponentPreconditioned>"
  // might not see the "<AffineComponentPreconditioned>" part because
  // of how ReadNew() works.
  ExpectOneOrTwoTokens(is, binary, ostr_beg.str(), "<LearningRate>");
  ReadBasicType(is, binary, &learning_rate_);
  ExpectToken(is, binary, "<LinearParams>");
  linear_params_.Read(is, binary);
  ExpectToken(is, binary, "<BiasParams>");
  bias_params_.Read(is, binary);
  ExpectToken(is, binary, "<Alpha>");
  ReadBasicType(is, binary, &alpha_);
  // todo: remove back-compat code.  Will just be:
  // ExpectToken(is, binary, "<MaxChange>");
  // ReadBasicType(is, binary, &max_change_);
  // ExpectToken(is, binary, ostr_end);
  // [end of function]
  std::string tok;
  ReadToken(is, binary, &tok);
  if (tok == "<MaxChange>") {
    ReadBasicType(is, binary, &max_change_);
    ExpectToken(is, binary, ostr_end.str());
  } else {
    max_change_ = 0.0;
    KALDI_ASSERT(tok == ostr_end.str());
  }
}

SetMaxChange()

void SetMaxChange ( BaseFloat  max_change )

inline

Definition at line 965 of file nnet-component.h.

References KALDI_DISALLOW_COPY_AND_ASSIGN.

Referenced by kaldi::nnet2::SetMaxChange().

{ max_change_ = max_change; }

Type()

virtual std::string Type ( ) const

inlinevirtual

Reimplemented from AffineComponent.

Definition at line 950 of file nnet-component.h.

References kaldi::cu::Copy().

{ return "AffineComponentPreconditioned"; }

Update()

void Update ( const CuMatrixBase< BaseFloat > &  in_value, const CuMatrixBase< BaseFloat > &  out_deriv  )

protectedvirtual

Reimplemented from AffineComponent.

Definition at line 1544 of file nnet-component.cc.

References AffineComponent::bias_params_, CuVectorBase< Real >::CopyColFromMat(), kaldi::kNoTrans, kaldi::kTrans, kaldi::kUndefined, UpdatableComponent::learning_rate_, AffineComponent::linear_params_, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), kaldi::nnet2::PreconditionDirectionsAlphaRescaled(), CuMatrixBase< Real >::Range(), and CuMatrix< Real >::Resize().

                                              {
  CuMatrix<BaseFloat> in_value_temp;

  in_value_temp.Resize(in_value.NumRows(),
                       in_value.NumCols() + 1, kUndefined);
  in_value_temp.Range(0, in_value.NumRows(),
                      0, in_value.NumCols()).CopyFromMat(in_value);

  // Add the 1.0 at the end of each row "in_value_temp"
  in_value_temp.Range(0, in_value.NumRows(),
                      in_value.NumCols(), 1).Set(1.0);

  CuMatrix<BaseFloat> in_value_precon(in_value_temp.NumRows(),
                                      in_value_temp.NumCols(), kUndefined),
      out_deriv_precon(out_deriv.NumRows(),
                       out_deriv.NumCols(), kUndefined);
  // each row of in_value_precon will be that same row of
  // in_value, but multiplied by the inverse of a Fisher
  // matrix that has been estimated from all the other rows,
  // smoothed by some appropriate amount times the identity
  // matrix (this amount is proportional to \alpha).
  PreconditionDirectionsAlphaRescaled(in_value_temp, alpha_, &in_value_precon);
  PreconditionDirectionsAlphaRescaled(out_deriv, alpha_, &out_deriv_precon);

  BaseFloat minibatch_scale = 1.0;

  if (max_change_ > 0.0)
    minibatch_scale = GetScalingFactor(in_value_precon, out_deriv_precon);


  CuSubMatrix<BaseFloat> in_value_precon_part(in_value_precon,
                                            0, in_value_precon.NumRows(),
                                            0, in_value_precon.NumCols() - 1);
  // this "precon_ones" is what happens to the vector of 1's representing
  // offsets, after multiplication by the preconditioner.
  CuVector<BaseFloat> precon_ones(in_value_precon.NumRows());

  precon_ones.CopyColFromMat(in_value_precon, in_value_precon.NumCols() - 1);

  BaseFloat local_lrate = minibatch_scale * learning_rate_;
  bias_params_.AddMatVec(local_lrate, out_deriv_precon, kTrans,
                         precon_ones, 1.0);
  linear_params_.AddMatMat(local_lrate, out_deriv_precon, kTrans,
                           in_value_precon_part, kNoTrans, 1.0);
}

Write()

void Write ( std::ostream &  os, bool  binary  ) const

virtual

Write component to stream.

Reimplemented from AffineComponent.

Definition at line 1463 of file nnet-component.cc.

References AffineComponent::bias_params_, UpdatableComponent::learning_rate_, AffineComponent::linear_params_, AffineComponent::Type(), kaldi::WriteBasicType(), and kaldi::WriteToken().

                                                                           {
  std::ostringstream ostr_beg, ostr_end;
  ostr_beg << "<" << Type() << ">"; // e.g. "<AffineComponent>"
  ostr_end << "</" << Type() << ">"; // e.g. "</AffineComponent>"
  WriteToken(os, binary, ostr_beg.str());
  WriteToken(os, binary, "<LearningRate>");
  WriteBasicType(os, binary, learning_rate_);
  WriteToken(os, binary, "<LinearParams>");
  linear_params_.Write(os, binary);
  WriteToken(os, binary, "<BiasParams>");
  bias_params_.Write(os, binary);
  WriteToken(os, binary, "<Alpha>");
  WriteBasicType(os, binary, alpha_);
  WriteToken(os, binary, "<MaxChange>");
  WriteBasicType(os, binary, max_change_);
  WriteToken(os, binary, ostr_end.str());
}

Member Data Documentation

alpha_

BaseFloat alpha_

protected

Definition at line 968 of file nnet-component.h.

Referenced by AffineComponentPreconditioned::Copy().

max_change_

BaseFloat max_change_

protected

Definition at line 969 of file nnet-component.h.

Referenced by AffineComponentPreconditioned::Copy().

---

The documentation for this class was generated from the following files:

• nnet2/nnet-component.h
• nnet2/nnet-component.cc