Keywords: natural gradient descent, NG-SGD, naturalgradient. More...

#include <nnet-component.h>

Inheritance diagram for AffineComponentPreconditionedOnline:

Collaboration diagram for AffineComponentPreconditionedOnline:

[legend]

Public Member Functions
virtual std::string	Type () const

virtual void	Read (std::istream &is, bool binary)

virtual void	Write (std::ostream &os, bool binary) const
	Write component to stream. More...

void	Init (BaseFloat learning_rate, int32 input_dim, int32 output_dim, BaseFloat param_stddev, BaseFloat bias_stddev, int32 rank_in, int32 rank_out, int32 update_period, BaseFloat num_samples_history, BaseFloat alpha, BaseFloat max_change_per_sample)

void	Init (BaseFloat learning_rate, int32 rank_in, int32 rank_out, int32 update_period, BaseFloat num_samples_history, BaseFloat alpha, BaseFloat max_change_per_sample, std::string matrix_filename)

virtual void	Resize (int32 input_dim, int32 output_dim)

	AffineComponentPreconditionedOnline (const AffineComponent &orig, int32 rank_in, int32 rank_out, int32 update_period, BaseFloat eta, BaseFloat alpha)

virtual void	InitFromString (std::string args)
	Initialize, typically from a line of a config file. More...

virtual std::string	Info () const

virtual Component *	Copy () const
	Copy component (deep copy). More...

	AffineComponentPreconditionedOnline ()

Public Member Functions inherited from AffineComponent
	AffineComponent (const AffineComponent &other)

	AffineComponent (const CuMatrixBase< BaseFloat > &linear_params, const CuVectorBase< BaseFloat > &bias_params, BaseFloat learning_rate)

virtual int32	InputDim () const
	Get size of input vectors. More...

virtual int32	OutputDim () const
	Get size of output vectors. More...

void	Init (BaseFloat learning_rate, int32 input_dim, int32 output_dim, BaseFloat param_stddev, BaseFloat bias_stddev)

void	Init (BaseFloat learning_rate, std::string matrix_filename)

Component *	CollapseWithNext (const AffineComponent &next) const

Component *	CollapseWithNext (const FixedAffineComponent &next) const

Component *	CollapseWithNext (const FixedScaleComponent &next) const

Component *	CollapseWithPrevious (const FixedAffineComponent &prev) const

	AffineComponent ()

virtual bool	BackpropNeedsInput () const

virtual bool	BackpropNeedsOutput () const

virtual void	Propagate (const ChunkInfo &in_info, const ChunkInfo &out_info, const CuMatrixBase< BaseFloat > &in, CuMatrixBase< BaseFloat > *out) const
	Perform forward pass propagation Input->Output. More...

virtual void	Scale (BaseFloat scale)
	This new virtual function scales the parameters by this amount. More...

virtual void	Add (BaseFloat alpha, const UpdatableComponent &other)
	This new virtual function adds the parameters of another updatable component, times some constant, to the current parameters. More...

virtual void	Backprop (const ChunkInfo &in_info, const ChunkInfo &out_info, const CuMatrixBase< BaseFloat > &in_value, const CuMatrixBase< BaseFloat > &out_value, const CuMatrixBase< BaseFloat > &out_deriv, Component to_update, CuMatrix< BaseFloat > in_deriv) const
	Perform backward pass propagation of the derivative, and also either update the model (if to_update == this) or update another model or compute the model derivative (otherwise). More...

virtual void	SetZero (bool treat_as_gradient)
	Set parameters to zero, and if treat_as_gradient is true, we'll be treating this as a gradient so set the learning rate to 1 and make any other changes necessary (there's a variable we have to set for the MixtureProbComponent). More...

virtual BaseFloat	DotProduct (const UpdatableComponent &other) const
	Here, "other" is a component of the same specific type. More...

virtual void	PerturbParams (BaseFloat stddev)
	We introduce a new virtual function that only applies to class UpdatableComponent. More...

virtual void	SetParams (const VectorBase< BaseFloat > &bias, const MatrixBase< BaseFloat > &linear)

const CuVector< BaseFloat > &	BiasParams ()

const CuMatrix< BaseFloat > &	LinearParams ()

virtual int32	GetParameterDim () const
	The following new virtual function returns the total dimension of the parameters in this class. More...

virtual void	Vectorize (VectorBase< BaseFloat > *params) const
	Turns the parameters into vector form. More...

virtual void	UnVectorize (const VectorBase< BaseFloat > &params)
	Converts the parameters from vector form. More...

virtual void	LimitRank (int32 dimension, AffineComponent a, AffineComponent b) const
	This function is for getting a low-rank approximations of this AffineComponent by two AffineComponents. More...

void	Widen (int32 new_dimension, BaseFloat param_stddev, BaseFloat bias_stddev, std::vector< NonlinearComponent > c2, AffineComponent c3)
	This function is implemented in widen-nnet.cc. More...

Public Member Functions inherited from UpdatableComponent
	UpdatableComponent (const UpdatableComponent &other)

void	Init (BaseFloat learning_rate)

	UpdatableComponent (BaseFloat learning_rate)

	UpdatableComponent ()

virtual	~UpdatableComponent ()

void	SetLearningRate (BaseFloat lrate)
	Sets the learning rate of gradient descent. More...

BaseFloat	LearningRate () const
	Gets the learning rate of gradient descent. More...

Public Member Functions inherited from Component
	Component ()

virtual int32	Index () const
	Returns the index in the sequence of layers in the neural net; intended only to be used in debugging information. More...

virtual void	SetIndex (int32 index)

virtual std::vector< int32 >	Context () const
	Return a vector describing the temporal context this component requires for each frame of output, as a sorted list. More...

void	Propagate (const ChunkInfo &in_info, const ChunkInfo &out_info, const CuMatrixBase< BaseFloat > &in, CuMatrix< BaseFloat > *out) const
	A non-virtual propagate function that first resizes output if necessary. More...

virtual	~Component ()

Private Member Functions
	KALDI_DISALLOW_COPY_AND_ASSIGN (AffineComponentPreconditionedOnline)

BaseFloat	GetScalingFactor (const CuVectorBase< BaseFloat > &in_products, BaseFloat gamma_prod, CuVectorBase< BaseFloat > *out_products)
	The following function is only called if max_change_per_sample_ > 0, it returns a scaling factor alpha <= 1.0 (1.0 in the normal case) that enforces the "max-change" constraint. More...

void	SetPreconditionerConfigs ()

virtual void	Update (const CuMatrixBase< BaseFloat > &in_value, const CuMatrixBase< BaseFloat > &out_deriv)

Private Attributes
int32	rank_in_

int32	rank_out_

int32	update_period_

BaseFloat	num_samples_history_

BaseFloat	alpha_

OnlinePreconditioner	preconditioner_in_

OnlinePreconditioner	preconditioner_out_

BaseFloat	max_change_per_sample_

Additional Inherited Members
Static Public Member Functions inherited from Component
static Component *	ReadNew (std::istream &is, bool binary)
	Read component from stream. More...

static Component *	NewFromString (const std::string &initializer_line)
	Initialize the Component from one line that will contain first the type, e.g. More...

static Component *	NewComponentOfType (const std::string &type)
	Return a new Component of the given type e.g. More...

Protected Member Functions inherited from AffineComponent
virtual void	UpdateSimple (const CuMatrixBase< BaseFloat > &in_value, const CuMatrixBase< BaseFloat > &out_deriv)

const AffineComponent &	operator= (const AffineComponent &other)

Protected Attributes inherited from AffineComponent
CuMatrix< BaseFloat >	linear_params_

CuVector< BaseFloat >	bias_params_

bool	is_gradient_

Protected Attributes inherited from UpdatableComponent
BaseFloat	learning_rate_
	learning rate (0.0..0.01) More...

Detailed Description

Keywords: natural gradient descent, NG-SGD, naturalgradient.

For the top-level of the natural gradient code look here, and also in nnet-precondition-online.h. AffineComponentPreconditionedOnline is, like AffineComponentPreconditioned, a version of AffineComponent that has a non-(multiple of unit) learning-rate matrix. See nnet-precondition-online.h for a description of the technique.

Definition at line 997 of file nnet-component.h.

Constructor & Destructor Documentation

◆ AffineComponentPreconditionedOnline() [1/2]

AffineComponentPreconditionedOnline	(	const AffineComponent &	orig,
		int32	rank_in,
		int32	rank_out,
		int32	update_period,
		BaseFloat	eta,
		BaseFloat	alpha
	)

Definition at line 1728 of file nnet-component.cc.

References AffineComponentPreconditionedOnline::alpha_, AffineComponent::bias_params_, AffineComponent::is_gradient_, UpdatableComponent::learning_rate_, AffineComponent::linear_params_, AffineComponentPreconditionedOnline::num_samples_history_, AffineComponentPreconditionedOnline::rank_in_, AffineComponentPreconditionedOnline::rank_out_, AffineComponentPreconditionedOnline::SetPreconditionerConfigs(), and AffineComponentPreconditionedOnline::update_period_.

                                                    :
     max_change_per_sample_(0.1) {
   this->linear_params_ = orig.linear_params_;
   this->bias_params_ = orig.bias_params_;
   this->learning_rate_ = orig.learning_rate_;
   this->is_gradient_ = orig.is_gradient_;
   this->rank_in_ = rank_in;
   this->rank_out_ = rank_out;
   this->update_period_ = update_period;
   this->num_samples_history_ = num_samples_history;
   this->alpha_ = alpha;
   SetPreconditionerConfigs();
 }

◆ AffineComponentPreconditionedOnline() [2/2]

AffineComponentPreconditionedOnline ( )

inline

Definition at line 1030 of file nnet-component.h.

References KALDI_DISALLOW_COPY_AND_ASSIGN.

Referenced by AffineComponentPreconditionedOnline::Copy().

1030 : max_change_per_sample_(0.0) { }

kaldi::nnet2::AffineComponentPreconditionedOnline::max_change_per_sample_

BaseFloat max_change_per_sample_

Definition: nnet-component.h:1048

Member Function Documentation

◆ Copy()

Component * Copy ( ) const

virtual

Copy component (deep copy).

Reimplemented from AffineComponent.

Definition at line 1821 of file nnet-component.cc.

                                                            {
   AffineComponentPreconditionedOnline *ans = new AffineComponentPreconditionedOnline();
   ans->learning_rate_ = learning_rate_;
   ans->rank_in_ = rank_in_;
   ans->rank_out_ = rank_out_;
   ans->update_period_ = update_period_;
   ans->num_samples_history_ = num_samples_history_;
   ans->alpha_ = alpha_;
   ans->linear_params_ = linear_params_;
   ans->bias_params_ = bias_params_;
   ans->preconditioner_in_ = preconditioner_in_;
   ans->preconditioner_out_ = preconditioner_out_;
   ans->max_change_per_sample_ = max_change_per_sample_;
   ans->is_gradient_ = is_gradient_;
   ans->SetPreconditionerConfigs();
   return ans;
 }

◆ GetScalingFactor()

BaseFloat GetScalingFactor	(	const CuVectorBase< BaseFloat > &	in_products,
		BaseFloat	gamma_prod,
		CuVectorBase< BaseFloat > *	out_products
	)

private

The following function is only called if max_change_per_sample_ > 0, it returns a scaling factor alpha <= 1.0 (1.0 in the normal case) that enforces the "max-change" constraint.

"in_products" is the inner product with itself of each row of the matrix of preconditioned input features; "out_products" is the same for the output derivatives. gamma_prod is a product of two scalars that are output by the preconditioning code (for the input and output), which we will need to multiply into the learning rate. out_products is a pointer because we modify it in-place.

Definition at line 1841 of file nnet-component.cc.

References CuVectorBase< Real >::ApplyPow(), CuVectorBase< Real >::Dim(), Component::Index(), KALDI_ASSERT, KALDI_LOG, UpdatableComponent::learning_rate_, AffineComponentPreconditionedOnline::max_change_per_sample_, CuVectorBase< Real >::MulElements(), and CuVectorBase< Real >::Sum().

Referenced by AffineComponentPreconditionedOnline::Update().

                                            {
   static int scaling_factor_printed = 0;
   int32 minibatch_size = in_products.Dim();
 
   out_products->MulElements(in_products);
   out_products->ApplyPow(0.5);
   BaseFloat prod_sum = out_products->Sum();
   BaseFloat tot_change_norm = learning_rate_scale * learning_rate_ * prod_sum,
       max_change_norm = max_change_per_sample_ * minibatch_size;
   // tot_change_norm is the product of norms that we are trying to limit
   // to max_value_.
   KALDI_ASSERT(tot_change_norm - tot_change_norm == 0.0 && "NaN in backprop");
   KALDI_ASSERT(tot_change_norm >= 0.0);
   if (tot_change_norm <= max_change_norm) return 1.0;
   else {
     BaseFloat factor = max_change_norm / tot_change_norm;
     if (scaling_factor_printed < 10) {
       KALDI_LOG << "Limiting step size using scaling factor "
                 << factor << ", for component index " << Index();
       scaling_factor_printed++;
     }
     return factor;
   }
 }

◆ Info()

std::string Info ( ) const

virtual

Reimplemented from AffineComponent.

Definition at line 1798 of file nnet-component.cc.

References AffineComponentPreconditionedOnline::alpha_, AffineComponent::bias_params_, AffineComponent::InputDim(), kaldi::kTrans, UpdatableComponent::LearningRate(), AffineComponent::linear_params_, AffineComponentPreconditionedOnline::max_change_per_sample_, AffineComponentPreconditionedOnline::num_samples_history_, AffineComponent::OutputDim(), AffineComponentPreconditionedOnline::rank_in_, AffineComponentPreconditionedOnline::rank_out_, kaldi::TraceMatMat(), AffineComponentPreconditionedOnline::Type(), AffineComponentPreconditionedOnline::update_period_, and kaldi::VecVec().

                                                           {
   std::stringstream stream;
   BaseFloat linear_params_size = static_cast<BaseFloat>(linear_params_.NumRows())
       * static_cast<BaseFloat>(linear_params_.NumCols());
   BaseFloat linear_stddev =
       std::sqrt(TraceMatMat(linear_params_, linear_params_, kTrans) /
                 linear_params_size),
       bias_stddev = std::sqrt(VecVec(bias_params_, bias_params_) /
                               bias_params_.Dim());
   stream << Type() << ", input-dim=" << InputDim()
          << ", output-dim=" << OutputDim()
          << ", linear-params-stddev=" << linear_stddev
          << ", bias-params-stddev=" << bias_stddev
          << ", learning-rate=" << LearningRate()
          << ", rank-in=" << rank_in_
          << ", rank-out=" << rank_out_
          << ", num_samples_history=" << num_samples_history_
          << ", update_period=" << update_period_
          << ", alpha=" << alpha_
          << ", max-change-per-sample=" << max_change_per_sample_;
   return stream.str();
 }

◆ Init() [1/2]

void Init	(	BaseFloat	learning_rate,
		int32	input_dim,
		int32	output_dim,
		BaseFloat	param_stddev,
		BaseFloat	bias_stddev,
		int32	rank_in,
		int32	rank_out,
		int32	update_period,
		BaseFloat	num_samples_history,
		BaseFloat	alpha,
		BaseFloat	max_change_per_sample
	)

Definition at line 1745 of file nnet-component.cc.

References AffineComponentPreconditionedOnline::alpha_, AffineComponent::bias_params_, UpdatableComponent::Init(), KALDI_ASSERT, AffineComponent::linear_params_, AffineComponentPreconditionedOnline::max_change_per_sample_, AffineComponentPreconditionedOnline::num_samples_history_, AffineComponentPreconditionedOnline::rank_in_, AffineComponentPreconditionedOnline::rank_out_, AffineComponentPreconditionedOnline::SetPreconditionerConfigs(), and AffineComponentPreconditionedOnline::update_period_.

                                      {
   UpdatableComponent::Init(learning_rate);
   linear_params_.Resize(output_dim, input_dim);
   bias_params_.Resize(output_dim);
   KALDI_ASSERT(output_dim > 0 && input_dim > 0 && param_stddev >= 0.0 &&
                bias_stddev >= 0.0);
   linear_params_.SetRandn(); // sets to random normally distributed noise.
   linear_params_.Scale(param_stddev);
   bias_params_.SetRandn();
   bias_params_.Scale(bias_stddev);
   rank_in_ = rank_in;
   rank_out_ = rank_out;
   update_period_ = update_period;
   num_samples_history_ = num_samples_history;
   alpha_ = alpha;
   SetPreconditionerConfigs();
   KALDI_ASSERT(max_change_per_sample >= 0.0);
   max_change_per_sample_ = max_change_per_sample;
 }

◆ Init() [2/2]

void Init	(	BaseFloat	learning_rate,
		int32	rank_in,
		int32	rank_out,
		int32	update_period,
		BaseFloat	num_samples_history,
		BaseFloat	alpha,
		BaseFloat	max_change_per_sample,
		std::string	matrix_filename
	)

Definition at line 1704 of file nnet-component.cc.

References AffineComponent::bias_params_, UpdatableComponent::Init(), KALDI_ASSERT, AffineComponent::linear_params_, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), CuMatrixBase< Real >::Range(), and kaldi::ReadKaldiObject().

                                {
   UpdatableComponent::Init(learning_rate);
   rank_in_ = rank_in;
   rank_out_ = rank_out;
   update_period_ = update_period;
   num_samples_history_ = num_samples_history;
   alpha_ = alpha;
   SetPreconditionerConfigs();
   KALDI_ASSERT(max_change_per_sample >= 0.0);
   max_change_per_sample_ = max_change_per_sample;
   CuMatrix<BaseFloat> mat;
   ReadKaldiObject(matrix_filename, &mat); // will abort on failure.
   KALDI_ASSERT(mat.NumCols() >= 2);
   int32 input_dim = mat.NumCols() - 1, output_dim = mat.NumRows();
   linear_params_.Resize(output_dim, input_dim);
   bias_params_.Resize(output_dim);
   linear_params_.CopyFromMat(mat.Range(0, output_dim, 0, input_dim));
   bias_params_.CopyColFromMat(mat, input_dim);
 }

◆ InitFromString()

void InitFromString ( std::string args )

virtual

Initialize, typically from a line of a config file.

The "args" will contain any parameters that need to be passed to the Component, e.g. dimensions.

Reimplemented from AffineComponent.

Definition at line 1648 of file nnet-component.cc.

References AffineComponent::Init(), AffineComponent::InputDim(), KALDI_ASSERT, KALDI_ERR, UpdatableComponent::learning_rate_, AffineComponent::OutputDim(), and kaldi::nnet2::ParseFromString().

Referenced by kaldi::nnet2::UnitTestAffineComponentPreconditionedOnline().

                                                                        {
   std::string orig_args(args);
   bool ok = true;
   std::string matrix_filename;
   BaseFloat learning_rate = learning_rate_;
   BaseFloat num_samples_history = 2000.0, alpha = 4.0,
       max_change_per_sample = 0.1;
   int32 input_dim = -1, output_dim = -1, rank_in = 30, rank_out = 80,
       update_period = 1;
   ParseFromString("learning-rate", &args, &learning_rate); // optional.
   ParseFromString("num-samples-history", &args, &num_samples_history);
   ParseFromString("alpha", &args, &alpha);
   ParseFromString("max-change-per-sample", &args, &max_change_per_sample);
   ParseFromString("rank-in", &args, &rank_in);
   ParseFromString("rank-out", &args, &rank_out);
   ParseFromString("update-period", &args, &update_period);
 
   if (ParseFromString("matrix", &args, &matrix_filename)) {
     Init(learning_rate, rank_in, rank_out, update_period,
          num_samples_history, alpha, max_change_per_sample,
          matrix_filename);
     if (ParseFromString("input-dim", &args, &input_dim))
       KALDI_ASSERT(input_dim == InputDim() &&
                    "input-dim mismatch vs. matrix.");
     if (ParseFromString("output-dim", &args, &output_dim))
       KALDI_ASSERT(output_dim == OutputDim() &&
                    "output-dim mismatch vs. matrix.");
   } else {
     ok = ok && ParseFromString("input-dim", &args, &input_dim);
     ok = ok && ParseFromString("output-dim", &args, &output_dim);
     BaseFloat param_stddev = 1.0 / std::sqrt(input_dim),
         bias_stddev = 1.0;
     ParseFromString("param-stddev", &args, &param_stddev);
     ParseFromString("bias-stddev", &args, &bias_stddev);
     Init(learning_rate, input_dim, output_dim, param_stddev,
          bias_stddev, rank_in, rank_out, update_period,
          num_samples_history, alpha, max_change_per_sample);
   }
   if (!args.empty())
     KALDI_ERR << "Could not process these elements in initializer: "
               << args;
   if (!ok)
     KALDI_ERR << "Bad initializer " << orig_args;
 }

◆ KALDI_DISALLOW_COPY_AND_ASSIGN()

KALDI_DISALLOW_COPY_AND_ASSIGN ( AffineComponentPreconditionedOnline )

private

◆ Read()

void Read	(	std::istream &	is,
		bool	binary
	)

virtual

Reimplemented from AffineComponent.

Definition at line 1608 of file nnet-component.cc.

References AffineComponent::bias_params_, kaldi::nnet2::ExpectOneOrTwoTokens(), kaldi::ExpectToken(), KALDI_ASSERT, UpdatableComponent::learning_rate_, AffineComponent::linear_params_, kaldi::ReadBasicType(), kaldi::ReadToken(), and AffineComponent::Type().

                                                                           {
   std::ostringstream ostr_beg, ostr_end;
   ostr_beg << "<" << Type() << ">";
   ostr_end << "</" << Type() << ">";
   // might not see the "<AffineComponentPreconditionedOnline>" part because
   // of how ReadNew() works.
   ExpectOneOrTwoTokens(is, binary, ostr_beg.str(), "<LearningRate>");
   ReadBasicType(is, binary, &learning_rate_);
   ExpectToken(is, binary, "<LinearParams>");
   linear_params_.Read(is, binary);
   ExpectToken(is, binary, "<BiasParams>");
   bias_params_.Read(is, binary);
   std::string tok;
   ReadToken(is, binary, &tok);
   if (tok == "<Rank>") {  // back-compatibility (temporary)
     ReadBasicType(is, binary, &rank_in_);
     rank_out_ = rank_in_;
   } else {
     KALDI_ASSERT(tok == "<RankIn>");
     ReadBasicType(is, binary, &rank_in_);
     ExpectToken(is, binary, "<RankOut>");
     ReadBasicType(is, binary, &rank_out_);
   }
   ReadToken(is, binary, &tok);
   if (tok == "<UpdatePeriod>") {
     ReadBasicType(is, binary, &update_period_);
     ExpectToken(is, binary, "<NumSamplesHistory>");
   } else {
     update_period_ = 1;
     KALDI_ASSERT(tok == "<NumSamplesHistory>");
   }
   ReadBasicType(is, binary, &num_samples_history_);
   ExpectToken(is, binary, "<Alpha>");
   ReadBasicType(is, binary, &alpha_);
   ExpectToken(is, binary, "<MaxChangePerSample>");
   ReadBasicType(is, binary, &max_change_per_sample_);
   ExpectToken(is, binary, ostr_end.str());
   SetPreconditionerConfigs();
 }

◆ Resize()

void Resize	(	int32	input_dim,
		int32	output_dim
	)

virtual

Reimplemented from AffineComponent.

Definition at line 1594 of file nnet-component.cc.

References AffineComponent::bias_params_, KALDI_ASSERT, and AffineComponent::linear_params_.

                                        {
   KALDI_ASSERT(input_dim > 1 && output_dim > 1);
   if (rank_in_ >= input_dim) rank_in_ = input_dim - 1;
   if (rank_out_ >= output_dim) rank_out_ = output_dim - 1;
   bias_params_.Resize(output_dim);
   linear_params_.Resize(output_dim, input_dim);
   OnlinePreconditioner temp;
   preconditioner_in_ = temp;
   preconditioner_out_ = temp;
   SetPreconditionerConfigs();
 }

◆ SetPreconditionerConfigs()

void SetPreconditionerConfigs ( )

private

Definition at line 1693 of file nnet-component.cc.

Referenced by AffineComponentPreconditionedOnline::AffineComponentPreconditionedOnline(), AffineComponentPreconditionedOnline::Copy(), and AffineComponentPreconditionedOnline::Init().

                                                                    {
   preconditioner_in_.SetRank(rank_in_);
   preconditioner_in_.SetNumSamplesHistory(num_samples_history_);
   preconditioner_in_.SetAlpha(alpha_);
   preconditioner_in_.SetUpdatePeriod(update_period_);
   preconditioner_out_.SetRank(rank_out_);
   preconditioner_out_.SetNumSamplesHistory(num_samples_history_);
   preconditioner_out_.SetAlpha(alpha_);
   preconditioner_out_.SetUpdatePeriod(update_period_);
 }

◆ Type()

virtual std::string Type ( ) const

inlinevirtual

Reimplemented from AffineComponent.

Definition at line 999 of file nnet-component.h.

References kaldi::cu::Copy().

Referenced by AffineComponentPreconditionedOnline::Info(), SpliceComponent::InitFromString(), SpliceMaxComponent::InitFromString(), SumGroupComponent::InitFromString(), PermuteComponent::InitFromString(), DctComponent::InitFromString(), FixedLinearComponent::InitFromString(), FixedAffineComponent::InitFromString(), FixedScaleComponent::InitFromString(), FixedBiasComponent::InitFromString(), and AffineComponentPreconditionedOnline::Write().

                                  {
     return "AffineComponentPreconditionedOnline";
   }

◆ Update()

void Update	(	const CuMatrixBase< BaseFloat > &	in_value,
		const CuMatrixBase< BaseFloat > &	out_deriv
	)

privatevirtual

Reimplemented from AffineComponent.

Definition at line 1869 of file nnet-component.cc.

References AffineComponent::bias_params_, CuVectorBase< Real >::CopyColFromMat(), AffineComponentPreconditionedOnline::GetScalingFactor(), kaldi::kNoTrans, kaldi::kTrans, kaldi::kUndefined, UpdatableComponent::learning_rate_, AffineComponent::linear_params_, AffineComponentPreconditionedOnline::max_change_per_sample_, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), OnlinePreconditioner::PreconditionDirections(), AffineComponentPreconditionedOnline::preconditioner_in_, AffineComponentPreconditionedOnline::preconditioner_out_, CuMatrixBase< Real >::Range(), and CuMatrix< Real >::Resize().

                                               {
   CuMatrix<BaseFloat> in_value_temp;
 
   in_value_temp.Resize(in_value.NumRows(),
                        in_value.NumCols() + 1, kUndefined);
   in_value_temp.Range(0, in_value.NumRows(),
                       0, in_value.NumCols()).CopyFromMat(in_value);
 
   // Add the 1.0 at the end of each row "in_value_temp"
   in_value_temp.Range(0, in_value.NumRows(),
                       in_value.NumCols(), 1).Set(1.0);
 
   CuMatrix<BaseFloat> out_deriv_temp(out_deriv);
 
   CuMatrix<BaseFloat> row_products(2,
                                    in_value.NumRows());
   CuSubVector<BaseFloat> in_row_products(row_products, 0),
       out_row_products(row_products, 1);
 
   // These "scale" values get will get multiplied into the learning rate (faster
   // than having the matrices scaled inside the preconditioning code).
   BaseFloat in_scale, out_scale;
 
   preconditioner_in_.PreconditionDirections(&in_value_temp, &in_row_products,
                                             &in_scale);
   preconditioner_out_.PreconditionDirections(&out_deriv_temp, &out_row_products,
                                              &out_scale);
 
   // "scale" is a scaling factor coming from the PreconditionDirections calls
   // (it's faster to have them output a scaling factor than to have them scale
   // their outputs).
   BaseFloat scale = in_scale * out_scale;
   BaseFloat minibatch_scale = 1.0;
 
   if (max_change_per_sample_ > 0.0)
     minibatch_scale = GetScalingFactor(in_row_products, scale,
                                        &out_row_products);
 
   CuSubMatrix<BaseFloat> in_value_precon_part(in_value_temp,
                                               0, in_value_temp.NumRows(),
                                               0, in_value_temp.NumCols() - 1);
   // this "precon_ones" is what happens to the vector of 1's representing
   // offsets, after multiplication by the preconditioner.
   CuVector<BaseFloat> precon_ones(in_value_temp.NumRows());
 
   precon_ones.CopyColFromMat(in_value_temp, in_value_temp.NumCols() - 1);
 
   BaseFloat local_lrate = scale * minibatch_scale * learning_rate_;
   bias_params_.AddMatVec(local_lrate, out_deriv_temp, kTrans,
                          precon_ones, 1.0);
   linear_params_.AddMatMat(local_lrate, out_deriv_temp, kTrans,
                            in_value_precon_part, kNoTrans, 1.0);
 }

◆ Write()

void Write	(	std::ostream &	os,
		bool	binary
	)		const

virtual

Write component to stream.

Reimplemented from AffineComponent.

Definition at line 1772 of file nnet-component.cc.

References AffineComponentPreconditionedOnline::alpha_, AffineComponent::bias_params_, UpdatableComponent::learning_rate_, AffineComponent::linear_params_, AffineComponentPreconditionedOnline::max_change_per_sample_, AffineComponentPreconditionedOnline::num_samples_history_, AffineComponentPreconditionedOnline::rank_in_, AffineComponentPreconditionedOnline::rank_out_, AffineComponentPreconditionedOnline::Type(), AffineComponentPreconditionedOnline::update_period_, kaldi::WriteBasicType(), and kaldi::WriteToken().

                                                                                  {
   std::ostringstream ostr_beg, ostr_end;
   ostr_beg << "<" << Type() << ">"; // e.g. "<AffineComponent>"
   ostr_end << "</" << Type() << ">"; // e.g. "</AffineComponent>"
   WriteToken(os, binary, ostr_beg.str());
   WriteToken(os, binary, "<LearningRate>");
   WriteBasicType(os, binary, learning_rate_);
   WriteToken(os, binary, "<LinearParams>");
   linear_params_.Write(os, binary);
   WriteToken(os, binary, "<BiasParams>");
   bias_params_.Write(os, binary);
   WriteToken(os, binary, "<RankIn>");
   WriteBasicType(os, binary, rank_in_);
   WriteToken(os, binary, "<RankOut>");
   WriteBasicType(os, binary, rank_out_);
   WriteToken(os, binary, "<UpdatePeriod>");
   WriteBasicType(os, binary, update_period_);
   WriteToken(os, binary, "<NumSamplesHistory>");
   WriteBasicType(os, binary, num_samples_history_);
   WriteToken(os, binary, "<Alpha>");
   WriteBasicType(os, binary, alpha_);
   WriteToken(os, binary, "<MaxChangePerSample>");
   WriteBasicType(os, binary, max_change_per_sample_);
   WriteToken(os, binary, ostr_end.str());
 }