NaturalGradientAffineComponent Class Reference

#include <nnet-simple-component.h>

Inheritance diagram for NaturalGradientAffineComponent:

Collaboration diagram for NaturalGradientAffineComponent:

Public Member Functions
virtual std::string	Type () const
	Returns a string such as "SigmoidComponent", describing the type of the object. More...
virtual void	Read (std::istream &is, bool binary)
	Read function (used after we know the type of the Component); accepts input that is missing the token that describes the component type, in case it has already been consumed. More...
virtual void	Write (std::ostream &os, bool binary) const
	Write component to stream. More...
	NaturalGradientAffineComponent ()
void	InitFromConfig (ConfigLine *cfl)
	Initialize, from a ConfigLine object. More...
virtual std::string	Info () const
	Returns some text-form information about this component, for diagnostics. More...
virtual Component *	Copy () const
	Copies component (deep copy). More...
virtual void	Scale (BaseFloat scale)
	This virtual function when called on – an UpdatableComponent scales the parameters by "scale" when called by an UpdatableComponent. More...
virtual void	Add (BaseFloat alpha, const Component &other)
	This virtual function when called by – an UpdatableComponent adds the parameters of another updatable component, times some constant, to the current parameters. More...
virtual void	FreezeNaturalGradient (bool freeze)
	freezes/unfreezes NaturalGradient updates, if applicable (to be overriden by components that use Natural Gradient). More...
virtual void	ConsolidateMemory ()
	This virtual function relates to memory management, and avoiding fragmentation. More...
	NaturalGradientAffineComponent (const NaturalGradientAffineComponent &other)
	NaturalGradientAffineComponent (const CuMatrixBase< BaseFloat > &linear_params, const CuVectorBase< BaseFloat > &bias_params)
Public Member Functions inherited from AffineComponent
virtual int32	InputDim () const
	Returns input-dimension of this component. More...
virtual int32	OutputDim () const
	Returns output-dimension of this component. More...
BaseFloat	OrthonormalConstraint () const
	AffineComponent ()
virtual int32	Properties () const
	Return bitmask of the component's properties. More...
virtual void *	Propagate (const ComponentPrecomputedIndexes *indexes, const CuMatrixBase< BaseFloat > &in, CuMatrixBase< BaseFloat > *out) const
	Propagate function. More...
virtual void	Backprop (const std::string &debug_info, const ComponentPrecomputedIndexes *indexes, const CuMatrixBase< BaseFloat > &in_value, const CuMatrixBase< BaseFloat > &, const CuMatrixBase< BaseFloat > &out_deriv, void *memo, Component *to_update, CuMatrixBase< BaseFloat > *in_deriv) const
	Backprop function; depending on which of the arguments 'to_update' and 'in_deriv' are non-NULL, this can compute input-data derivatives and/or perform model update. More...
virtual void	PerturbParams (BaseFloat stddev)
	This function is to be used in testing. More...
virtual BaseFloat	DotProduct (const UpdatableComponent &other) const
	Computes dot-product between parameters of two instances of a Component. More...
virtual int32	NumParameters () const
	The following new virtual function returns the total dimension of the parameters in this class. More...
virtual void	Vectorize (VectorBase< BaseFloat > *params) const
	Turns the parameters into vector form. More...
virtual void	UnVectorize (const VectorBase< BaseFloat > &params)
	Converts the parameters from vector form. More...
virtual void	SetParams (const CuVectorBase< BaseFloat > &bias, const CuMatrixBase< BaseFloat > &linear)
const CuVector< BaseFloat > &	BiasParams () const
CuVector< BaseFloat > &	BiasParams ()
const CuMatrix< BaseFloat > &	LinearParams () const
CuMatrix< BaseFloat > &	LinearParams ()
	AffineComponent (const AffineComponent &other)
	AffineComponent (const CuMatrixBase< BaseFloat > &linear_params, const CuVectorBase< BaseFloat > &bias_params, BaseFloat learning_rate)
virtual void	Resize (int32 input_dim, int32 output_dim)
void	Init (int32 input_dim, int32 output_dim, BaseFloat param_stddev, BaseFloat bias_stddev)
Public Member Functions inherited from UpdatableComponent
	UpdatableComponent (const UpdatableComponent &other)
	UpdatableComponent ()
virtual	~UpdatableComponent ()
virtual void	SetUnderlyingLearningRate (BaseFloat lrate)
	Sets the learning rate of gradient descent- gets multiplied by learning_rate_factor_. More...
virtual void	SetActualLearningRate (BaseFloat lrate)
	Sets the learning rate directly, bypassing learning_rate_factor_. More...
virtual void	SetAsGradient ()
	Sets is_gradient_ to true and sets learning_rate_ to 1, ignoring learning_rate_factor_. More...
virtual BaseFloat	LearningRateFactor ()
virtual void	SetLearningRateFactor (BaseFloat lrate_factor)
void	SetUpdatableConfigs (const UpdatableComponent &other)
BaseFloat	LearningRate () const
	Gets the learning rate to be used in gradient descent. More...
BaseFloat	MaxChange () const
	Returns the per-component max-change value, which is interpreted as the maximum change (in l2 norm) in parameters that is allowed per minibatch for this component. More...
void	SetMaxChange (BaseFloat max_change)
BaseFloat	L2Regularization () const
	Returns the l2 regularization constant, which may be set in any updatable component (usually from the config file). More...
void	SetL2Regularization (BaseFloat a)
Public Member Functions inherited from Component
virtual void	StoreStats (const CuMatrixBase< BaseFloat > &in_value, const CuMatrixBase< BaseFloat > &out_value, void *memo)
	This function may store stats on average activation values, and for some component types, the average value of the derivative of the nonlinearity. More...
virtual void	ZeroStats ()
	Components that provide an implementation of StoreStats should also provide an implementation of ZeroStats(), to set those stats to zero. More...
virtual void	GetInputIndexes (const MiscComputationInfo &misc_info, const Index &output_index, std::vector< Index > *desired_indexes) const
	This function only does something interesting for non-simple Components. More...
virtual bool	IsComputable (const MiscComputationInfo &misc_info, const Index &output_index, const IndexSet &input_index_set, std::vector< Index > *used_inputs) const
	This function only does something interesting for non-simple Components, and it exists to make it possible to manage optionally-required inputs. More...
virtual void	ReorderIndexes (std::vector< Index > *input_indexes, std::vector< Index > *output_indexes) const
	This function only does something interesting for non-simple Components. More...
virtual ComponentPrecomputedIndexes *	PrecomputeIndexes (const MiscComputationInfo &misc_info, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, bool need_backprop) const
	This function must return NULL for simple Components. More...
virtual void	DeleteMemo (void *memo) const
	This virtual function only needs to be overwritten by Components that return a non-NULL memo from their Propagate() function. More...
	Component ()
virtual	~Component ()

Private Member Functions
NaturalGradientAffineComponent &	operator= (const NaturalGradientAffineComponent &)
virtual void	Update (const std::string &debug_info, const CuMatrixBase< BaseFloat > &in_value, const CuMatrixBase< BaseFloat > &out_deriv)

Private Attributes
OnlineNaturalGradient	preconditioner_in_
OnlineNaturalGradient	preconditioner_out_

Additional Inherited Members
Static Public Member Functions inherited from Component
static Component *	ReadNew (std::istream &is, bool binary)
	Read component from stream (works out its type). Dies on error. More...
static Component *	NewComponentOfType (const std::string &type)
	Returns a new Component of the given type e.g. More...
Protected Member Functions inherited from AffineComponent
void	Init (std::string matrix_filename)
virtual void	UpdateSimple (const CuMatrixBase< BaseFloat > &in_value, const CuMatrixBase< BaseFloat > &out_deriv)
const AffineComponent &	operator= (const AffineComponent &other)
Protected Member Functions inherited from UpdatableComponent
void	InitLearningRatesFromConfig (ConfigLine *cfl)
std::string	ReadUpdatableCommon (std::istream &is, bool binary)
void	WriteUpdatableCommon (std::ostream &is, bool binary) const
Protected Attributes inherited from AffineComponent
CuMatrix< BaseFloat >	linear_params_
CuVector< BaseFloat >	bias_params_
BaseFloat	orthonormal_constraint_
Protected Attributes inherited from UpdatableComponent
BaseFloat	learning_rate_
	learning rate (typically 0.0..0.01) More...
BaseFloat	learning_rate_factor_
	learning rate factor (normally 1.0, but can be set to another < value so that when < you call SetLearningRate(), that value will be scaled by this factor. More...
BaseFloat	l2_regularize_
	L2 regularization constant. More...
bool	is_gradient_
	True if this component is to be treated as a gradient rather than as parameters. More...
BaseFloat	max_change_
	configuration value for imposing max-change More...

Detailed Description

Definition at line 825 of file nnet-simple-component.h.

Constructor & Destructor Documentation

NaturalGradientAffineComponent() [1/3]

NaturalGradientAffineComponent ( )

inline

Definition at line 831 of file nnet-simple-component.h.

References Component::Add(), Component::ConsolidateMemory(), PnormComponent::Copy(), kaldi::nnet3::FreezeNaturalGradient(), Component::Info(), PnormComponent::InitFromConfig(), and Component::Scale().

Referenced by NaturalGradientAffineComponent::Copy().

{ }

NaturalGradientAffineComponent() [2/3]

NaturalGradientAffineComponent ( const NaturalGradientAffineComponent &  other )

explicit

Definition at line 2987 of file nnet-simple-component.cc.

                                                :
    AffineComponent(other),
    preconditioner_in_(other.preconditioner_in_),
    preconditioner_out_(other.preconditioner_out_) { }

NaturalGradientAffineComponent() [3/3]

NaturalGradientAffineComponent	(	const CuMatrixBase< BaseFloat > &	linear_params,
		const CuVectorBase< BaseFloat > &	bias_params
	)

Definition at line 2853 of file nnet-simple-component.cc.

References CuVectorBase< Real >::Dim(), KALDI_ASSERT, CuMatrixBase< Real >::NumRows(), NaturalGradientAffineComponent::preconditioner_in_, NaturalGradientAffineComponent::preconditioner_out_, OnlineNaturalGradient::SetRank(), and OnlineNaturalGradient::SetUpdatePeriod().

                                               :
    AffineComponent(linear_params, bias_params, 0.001) {
  KALDI_ASSERT(bias_params.Dim() == linear_params.NumRows() &&
               bias_params.Dim() != 0);

  // set some default natural gradient configs.
  preconditioner_in_.SetRank(20);
  preconditioner_out_.SetRank(80);
  preconditioner_in_.SetUpdatePeriod(4);
  preconditioner_out_.SetUpdatePeriod(4);
}

Member Function Documentation

Add()

void Add ( BaseFloat  alpha, const Component &  other  )

virtual

This virtual function when called by – an UpdatableComponent adds the parameters of another updatable component, times some constant, to the current parameters.

– a NonlinearComponent (or another component that stores stats, like BatchNormComponent)– it relates to adding stats. Otherwise it will normally do nothing.

Reimplemented from AffineComponent.

Definition at line 3049 of file nnet-simple-component.cc.

References AffineComponent::bias_params_, KALDI_ASSERT, and AffineComponent::linear_params_.

                                                                                   {
  const NaturalGradientAffineComponent *other =
      dynamic_cast<const NaturalGradientAffineComponent*>(&other_in);
  KALDI_ASSERT(other != NULL);
  linear_params_.AddMat(alpha, other->linear_params_);
  bias_params_.AddVec(alpha, other->bias_params_);
}

ConsolidateMemory()

void ConsolidateMemory ( )

virtual

This virtual function relates to memory management, and avoiding fragmentation.

It is called only once per model, after we do the first minibatch of training. The default implementation does nothing, but it can be overridden by child classes, where it may re-initialize certain quantities that may possibly have been allocated during the forward pass (e.g. certain statistics; OnlineNaturalGradient objects). We use our own CPU-based allocator (see cu-allocator.h) and since it can't do paging since we're not in control of the GPU page table, fragmentation can be a problem. The allocator always tries to put things in 'low-address memory' (i.e. at smaller memory addresses) near the beginning of the block it allocated, to avoid fragmentation; but if permanent things (belonging to the model) are allocated in the forward pass, they can permanently stay in high memory. This function helps to prevent that, by re-allocating those things into low-address memory (It's important that it's called after all the temporary buffers for the forward-backward have been freed, so that there is low-address memory available)).

Reimplemented from Component.

Definition at line 3062 of file nnet-simple-component.cc.

References NaturalGradientAffineComponent::preconditioner_in_, NaturalGradientAffineComponent::preconditioner_out_, and OnlineNaturalGradient::Swap().

                                                       {
  OnlineNaturalGradient temp_in(preconditioner_in_);
  preconditioner_in_.Swap(&temp_in);
  OnlineNaturalGradient temp_out(preconditioner_out_);
  preconditioner_out_.Swap(&temp_out);
}

Copy()

Component * Copy ( ) const

virtual

Copies component (deep copy).

Reimplemented from AffineComponent.

Definition at line 2983 of file nnet-simple-component.cc.

References NaturalGradientAffineComponent::NaturalGradientAffineComponent().

                                                      {
  return new NaturalGradientAffineComponent(*this);
}

FreezeNaturalGradient()

void FreezeNaturalGradient ( bool  freeze )

virtual

freezes/unfreezes NaturalGradient updates, if applicable (to be overriden by components that use Natural Gradient).

Reimplemented from UpdatableComponent.

Definition at line 3057 of file nnet-simple-component.cc.

References OnlineNaturalGradient::Freeze(), NaturalGradientAffineComponent::preconditioner_in_, and NaturalGradientAffineComponent::preconditioner_out_.

                                                                      {
  preconditioner_in_.Freeze(freeze);
  preconditioner_out_.Freeze(freeze);
}

Info()

std::string Info ( ) const

virtual

Returns some text-form information about this component, for diagnostics.

Starts with the type of the component. E.g. "SigmoidComponent dim=900", although most components will have much more info.

Reimplemented from AffineComponent.

Definition at line 2972 of file nnet-simple-component.cc.

References OnlineNaturalGradient::GetAlpha(), OnlineNaturalGradient::GetNumSamplesHistory(), OnlineNaturalGradient::GetRank(), OnlineNaturalGradient::GetUpdatePeriod(), AffineComponent::Info(), NaturalGradientAffineComponent::preconditioner_in_, and NaturalGradientAffineComponent::preconditioner_out_.

                                                     {
  std::ostringstream stream;
  stream << AffineComponent::Info();
  stream << ", rank-in=" << preconditioner_in_.GetRank()
         << ", rank-out=" << preconditioner_out_.GetRank()
         << ", num-samples-history=" << preconditioner_in_.GetNumSamplesHistory()
         << ", update-period=" << preconditioner_in_.GetUpdatePeriod()
         << ", alpha=" << preconditioner_in_.GetAlpha();
  return stream.str();
}

InitFromConfig()

void InitFromConfig ( ConfigLine *  cfl )

virtual

Initialize, from a ConfigLine object.

Parameters

[in]

cfl

A ConfigLine containing any parameters that are needed for initialization. For example: "dim=100 param-stddev=0.1"

Reimplemented from AffineComponent.

Definition at line 2867 of file nnet-simple-component.cc.

References AffineComponent::bias_params_, ConfigLine::GetValue(), ConfigLine::HasUnusedValues(), UpdatableComponent::InitLearningRatesFromConfig(), AffineComponent::InputDim(), UpdatableComponent::is_gradient_, KALDI_ASSERT, KALDI_ERR, AffineComponent::linear_params_, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), AffineComponent::orthonormal_constraint_, AffineComponent::OutputDim(), NaturalGradientAffineComponent::preconditioner_in_, NaturalGradientAffineComponent::preconditioner_out_, CuMatrixBase< Real >::Range(), kaldi::ReadKaldiObject(), OnlineNaturalGradient::SetAlpha(), OnlineNaturalGradient::SetNumSamplesHistory(), OnlineNaturalGradient::SetRank(), OnlineNaturalGradient::SetUpdatePeriod(), ConfigLine::UnusedValues(), and ConfigLine::WholeLine().

                                                                   {
  bool ok = true;
  std::string matrix_filename;

  is_gradient_ = false;  // not configurable; there's no reason you'd want this

  InitLearningRatesFromConfig(cfl);

  if (cfl->GetValue("matrix", &matrix_filename)) {
    CuMatrix<BaseFloat> mat;
    ReadKaldiObject(matrix_filename, &mat); // will abort on failure.
    KALDI_ASSERT(mat.NumCols() >= 2);
    int32 input_dim = mat.NumCols() - 1, output_dim = mat.NumRows();
    linear_params_.Resize(output_dim, input_dim);
    bias_params_.Resize(output_dim);
    linear_params_.CopyFromMat(mat.Range(0, output_dim, 0, input_dim));
    bias_params_.CopyColFromMat(mat, input_dim);
    if (cfl->GetValue("input-dim", &input_dim))
      KALDI_ASSERT(input_dim == InputDim() &&
                   "input-dim mismatch vs. matrix.");
    if (cfl->GetValue("output-dim", &output_dim))
      KALDI_ASSERT(output_dim == OutputDim() &&
                   "output-dim mismatch vs. matrix.");
  } else {
    int32 input_dim = -1, output_dim = -1;

    ok = ok && cfl->GetValue("input-dim", &input_dim);
    ok = ok && cfl->GetValue("output-dim", &output_dim);
    if (!ok)
      KALDI_ERR << "Bad initializer " << cfl->WholeLine();
    BaseFloat param_stddev = 1.0 / std::sqrt(input_dim),
        bias_stddev = 1.0, bias_mean = 0.0;
    cfl->GetValue("param-stddev", &param_stddev);
    cfl->GetValue("bias-stddev", &bias_stddev);
    cfl->GetValue("bias-mean", &bias_mean);
    linear_params_.Resize(output_dim, input_dim);
    bias_params_.Resize(output_dim);
    KALDI_ASSERT(output_dim > 0 && input_dim > 0 && param_stddev >= 0.0 &&
                 bias_stddev >= 0.0);
    linear_params_.SetRandn(); // sets to random normally distributed noise.
    linear_params_.Scale(param_stddev);
    bias_params_.SetRandn();
    bias_params_.Scale(bias_stddev);
    bias_params_.Add(bias_mean);
  }

  orthonormal_constraint_ = 0.0;
  cfl->GetValue("orthonormal-constraint", &orthonormal_constraint_);

  // Set natural-gradient configs.
  BaseFloat num_samples_history = 2000.0,
      alpha = 4.0;
  int32 rank_in = -1, rank_out = -1,
      update_period = 4;
  cfl->GetValue("num-samples-history", &num_samples_history);
  cfl->GetValue("alpha", &alpha);
  cfl->GetValue("rank-in", &rank_in);
  cfl->GetValue("rank-out", &rank_out);
  cfl->GetValue("update-period", &update_period);

  if (rank_in < 0)
    rank_in = std::min<int32>(20, (InputDim() + 1) / 2);
  if (rank_out < 0)
    rank_out = std::min<int32>(80, (OutputDim() + 1) / 2);

  preconditioner_in_.SetNumSamplesHistory(num_samples_history);
  preconditioner_out_.SetNumSamplesHistory(num_samples_history);
  preconditioner_in_.SetAlpha(alpha);
  preconditioner_out_.SetAlpha(alpha);
  preconditioner_in_.SetRank(rank_in);
  preconditioner_out_.SetRank(rank_out);
  preconditioner_in_.SetUpdatePeriod(update_period);
  preconditioner_out_.SetUpdatePeriod(update_period);

  if (cfl->HasUnusedValues())
    KALDI_ERR << "Could not process these elements in initializer: "
              << cfl->UnusedValues();
  if (!ok)
    KALDI_ERR << "Bad initializer " << cfl->WholeLine();
}

operator=()

NaturalGradientAffineComponent& operator= ( const NaturalGradientAffineComponent &  )

private

Read()

void Read ( std::istream &  is, bool  binary  )

virtual

Read function (used after we know the type of the Component); accepts input that is missing the token that describes the component type, in case it has already been consumed.

Reimplemented from AffineComponent.

Definition at line 2786 of file nnet-simple-component.cc.

References kaldi::nnet3::ExpectToken(), UpdatableComponent::is_gradient_, KALDI_ERR, kaldi::PeekToken(), kaldi::ReadBasicType(), kaldi::ReadToken(), and UpdatableComponent::ReadUpdatableCommon().

                                                                     {
  ReadUpdatableCommon(is, binary);  // Read the opening tag and learning rate
  ExpectToken(is, binary, "<LinearParams>");
  linear_params_.Read(is, binary);
  ExpectToken(is, binary, "<BiasParams>");
  bias_params_.Read(is, binary);

  BaseFloat num_samples_history, alpha;
  int32 rank_in, rank_out, update_period;

  ExpectToken(is, binary, "<RankIn>");
  ReadBasicType(is, binary, &rank_in);
  ExpectToken(is, binary, "<RankOut>");
  ReadBasicType(is, binary, &rank_out);
  if (PeekToken(is, binary) == 'O') {
    ExpectToken(is, binary, "<OrthonormalConstraint>");
    ReadBasicType(is, binary, &orthonormal_constraint_);
  } else {
    orthonormal_constraint_ = 0.0;
  }
  ExpectToken(is, binary, "<UpdatePeriod>");
  ReadBasicType(is, binary, &update_period);
  ExpectToken(is, binary, "<NumSamplesHistory>");
  ReadBasicType(is, binary, &num_samples_history);
  ExpectToken(is, binary, "<Alpha>");
  ReadBasicType(is, binary, &alpha);

  preconditioner_in_.SetNumSamplesHistory(num_samples_history);
  preconditioner_out_.SetNumSamplesHistory(num_samples_history);
  preconditioner_in_.SetAlpha(alpha);
  preconditioner_out_.SetAlpha(alpha);
  preconditioner_in_.SetRank(rank_in);
  preconditioner_out_.SetRank(rank_out);
  preconditioner_in_.SetUpdatePeriod(update_period);
  preconditioner_out_.SetUpdatePeriod(update_period);

  if (PeekToken(is, binary) == 'M') {
    // MaxChangePerSample, long ago removed; back compatibility.
    ExpectToken(is, binary, "<MaxChangePerSample>");
    BaseFloat temp;
    ReadBasicType(is, binary, &temp);
  }
  if (PeekToken(is, binary) == 'I') {
    // for back compatibility; we don't write this here any
    // more as it's written and read in Write/ReadUpdatableCommon
    ExpectToken(is, binary, "<IsGradient>");
    ReadBasicType(is, binary, &is_gradient_);
  }
  if (PeekToken(is, binary) == 'U') {
    ExpectToken(is, binary, "<UpdateCount>");
    // back-compatibility branch (these configs were added and then removed).
    double temp;
    ReadBasicType(is, binary, &temp);
    ExpectToken(is, binary, "<ActiveScalingCount>");
    ReadBasicType(is, binary, &temp);
    ExpectToken(is, binary, "<MaxChangeScaleStats>");
    ReadBasicType(is, binary, &temp);
  }
  std::string token;
  ReadToken(is, binary, &token);
  // the following has to handle a couple variants of
  if (token.find("NaturalGradientAffineComponent>") == std::string::npos)
    KALDI_ERR << "Expected <NaturalGradientAffineComponent> or "
              << "</NaturalGradientAffineComponent>, got " << token;
}

Scale()

void Scale ( BaseFloat  scale )

virtual

This virtual function when called on – an UpdatableComponent scales the parameters by "scale" when called by an UpdatableComponent.

– a Nonlinear component (or another component that stores stats, like BatchNormComponent)– it relates to scaling activation stats, not parameters. Otherwise it will normally do nothing.

Reimplemented from AffineComponent.

Definition at line 3039 of file nnet-simple-component.cc.

References AffineComponent::bias_params_, and AffineComponent::linear_params_.

                                                          {
  if (scale == 0.0) {
    linear_params_.SetZero();
    bias_params_.SetZero();
  } else {
    linear_params_.Scale(scale);
    bias_params_.Scale(scale);
  }
}

Type()

virtual std::string Type ( ) const

inlinevirtual

Returns a string such as "SigmoidComponent", describing the type of the object.

Reimplemented from AffineComponent.

Definition at line 827 of file nnet-simple-component.h.

References PnormComponent::Read(), and PnormComponent::Write().

{ return "NaturalGradientAffineComponent"; }

Update()

void Update ( const std::string &  debug_info, const CuMatrixBase< BaseFloat > &  in_value, const CuMatrixBase< BaseFloat > &  out_deriv  )

privatevirtual

Reimplemented from AffineComponent.

Definition at line 2993 of file nnet-simple-component.cc.

References AffineComponent::bias_params_, CuVectorBase< Real >::CopyColFromMat(), kaldi::kNoTrans, kaldi::kTrans, kaldi::kUndefined, UpdatableComponent::learning_rate_, AffineComponent::linear_params_, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), OnlineNaturalGradient::PreconditionDirections(), NaturalGradientAffineComponent::preconditioner_in_, NaturalGradientAffineComponent::preconditioner_out_, CuMatrixBase< Real >::Range(), and CuMatrix< Real >::Resize().

                                              {
  CuMatrix<BaseFloat> in_value_temp;

  in_value_temp.Resize(in_value.NumRows(),
                       in_value.NumCols() + 1, kUndefined);
  in_value_temp.Range(0, in_value.NumRows(),
                      0, in_value.NumCols()).CopyFromMat(in_value);

  // Add the 1.0 at the end of each row "in_value_temp"
  in_value_temp.Range(0, in_value.NumRows(),
                      in_value.NumCols(), 1).Set(1.0);

  CuMatrix<BaseFloat> out_deriv_temp(out_deriv);

  // These "scale" values get will get multiplied into the learning rate (faster
  // than having the matrices scaled inside the preconditioning code).
  BaseFloat in_scale, out_scale;

  preconditioner_in_.PreconditionDirections(&in_value_temp, &in_scale);
  preconditioner_out_.PreconditionDirections(&out_deriv_temp, &out_scale);

  // "scale" is a scaling factor coming from the PreconditionDirections calls
  // (it's faster to have them output a scaling factor than to have them scale
  // their outputs).
  BaseFloat scale = in_scale * out_scale;

  CuSubMatrix<BaseFloat> in_value_precon_part(in_value_temp,
                                              0, in_value_temp.NumRows(),
                                              0, in_value_temp.NumCols() - 1);
  // this "precon_ones" is what happens to the vector of 1's representing
  // offsets, after multiplication by the preconditioner.
  CuVector<BaseFloat> precon_ones(in_value_temp.NumRows());

  precon_ones.CopyColFromMat(in_value_temp, in_value_temp.NumCols() - 1);

  BaseFloat local_lrate = scale * learning_rate_;

  bias_params_.AddMatVec(local_lrate, out_deriv_temp, kTrans,
                         precon_ones, 1.0);
  linear_params_.AddMatMat(local_lrate, out_deriv_temp, kTrans,
                           in_value_precon_part, kNoTrans, 1.0);
}

Write()

void Write ( std::ostream &  os, bool  binary  ) const

virtual

Write component to stream.

Reimplemented from AffineComponent.

Definition at line 2948 of file nnet-simple-component.cc.

References AffineComponent::bias_params_, OnlineNaturalGradient::GetAlpha(), OnlineNaturalGradient::GetNumSamplesHistory(), OnlineNaturalGradient::GetRank(), OnlineNaturalGradient::GetUpdatePeriod(), AffineComponent::linear_params_, AffineComponent::orthonormal_constraint_, NaturalGradientAffineComponent::preconditioner_in_, NaturalGradientAffineComponent::preconditioner_out_, kaldi::WriteBasicType(), kaldi::WriteToken(), and UpdatableComponent::WriteUpdatableCommon().

                                                              {
  WriteUpdatableCommon(os, binary);  // Write the opening tag and learning rate
  WriteToken(os, binary, "<LinearParams>");
  linear_params_.Write(os, binary);
  WriteToken(os, binary, "<BiasParams>");
  bias_params_.Write(os, binary);
  WriteToken(os, binary, "<RankIn>");
  WriteBasicType(os, binary, preconditioner_in_.GetRank());
  WriteToken(os, binary, "<RankOut>");
  WriteBasicType(os, binary, preconditioner_out_.GetRank());
  if (orthonormal_constraint_ != 0.0) {
    WriteToken(os, binary, "<OrthonormalConstraint>");
    WriteBasicType(os, binary, orthonormal_constraint_);
  }
  WriteToken(os, binary, "<UpdatePeriod>");
  WriteBasicType(os, binary, preconditioner_in_.GetUpdatePeriod());
  WriteToken(os, binary, "<NumSamplesHistory>");
  WriteBasicType(os, binary, preconditioner_in_.GetNumSamplesHistory());
  WriteToken(os, binary, "<Alpha>");
  WriteBasicType(os, binary, preconditioner_in_.GetAlpha());
  WriteToken(os, binary, "</NaturalGradientAffineComponent>");
}

Member Data Documentation

preconditioner_in_

OnlineNaturalGradient preconditioner_in_

private

Definition at line 852 of file nnet-simple-component.h.

Referenced by NaturalGradientAffineComponent::ConsolidateMemory(), NaturalGradientAffineComponent::FreezeNaturalGradient(), NaturalGradientAffineComponent::Info(), LinearComponent::Info(), NaturalGradientAffineComponent::InitFromConfig(), LinearComponent::InitFromConfig(), NaturalGradientAffineComponent::NaturalGradientAffineComponent(), LinearComponent::Read(), NaturalGradientAffineComponent::Update(), NaturalGradientAffineComponent::Write(), and LinearComponent::Write().

preconditioner_out_

OnlineNaturalGradient preconditioner_out_

private

Definition at line 854 of file nnet-simple-component.h.

Referenced by NaturalGradientAffineComponent::ConsolidateMemory(), NaturalGradientAffineComponent::FreezeNaturalGradient(), NaturalGradientAffineComponent::Info(), LinearComponent::Info(), NaturalGradientAffineComponent::InitFromConfig(), LinearComponent::InitFromConfig(), NaturalGradientAffineComponent::NaturalGradientAffineComponent(), LinearComponent::Read(), NaturalGradientAffineComponent::Update(), NaturalGradientAffineComponent::Write(), and LinearComponent::Write().

---

The documentation for this class was generated from the following files:

• nnet3/nnet-simple-component.h
• nnet3/nnet-simple-component.cc