AffineComponent Class Reference

#include <nnet-component.h>

Inheritance diagram for AffineComponent:

Collaboration diagram for AffineComponent:

Public Member Functions
	AffineComponent (const AffineComponent &other)
	AffineComponent (const CuMatrixBase< BaseFloat > &linear_params, const CuVectorBase< BaseFloat > &bias_params, BaseFloat learning_rate)
virtual int32	InputDim () const
	Get size of input vectors. More...
virtual int32	OutputDim () const
	Get size of output vectors. More...
void	Init (BaseFloat learning_rate, int32 input_dim, int32 output_dim, BaseFloat param_stddev, BaseFloat bias_stddev)
void	Init (BaseFloat learning_rate, std::string matrix_filename)
virtual void	Resize (int32 input_dim, int32 output_dim)
Component *	CollapseWithNext (const AffineComponent &next) const
Component *	CollapseWithNext (const FixedAffineComponent &next) const
Component *	CollapseWithNext (const FixedScaleComponent &next) const
Component *	CollapseWithPrevious (const FixedAffineComponent &prev) const
virtual std::string	Info () const
virtual void	InitFromString (std::string args)
	Initialize, typically from a line of a config file. More...
	AffineComponent ()
virtual std::string	Type () const
virtual bool	BackpropNeedsInput () const
virtual bool	BackpropNeedsOutput () const
virtual void	Propagate (const ChunkInfo &in_info, const ChunkInfo &out_info, const CuMatrixBase< BaseFloat > &in, CuMatrixBase< BaseFloat > *out) const
	Perform forward pass propagation Input->Output. More...
virtual void	Scale (BaseFloat scale)
	This new virtual function scales the parameters by this amount. More...
virtual void	Add (BaseFloat alpha, const UpdatableComponent &other)
	This new virtual function adds the parameters of another updatable component, times some constant, to the current parameters. More...
virtual void	Backprop (const ChunkInfo &in_info, const ChunkInfo &out_info, const CuMatrixBase< BaseFloat > &in_value, const CuMatrixBase< BaseFloat > &out_value, const CuMatrixBase< BaseFloat > &out_deriv, Component *to_update, CuMatrix< BaseFloat > *in_deriv) const
	Perform backward pass propagation of the derivative, and also either update the model (if to_update == this) or update another model or compute the model derivative (otherwise). More...
virtual void	SetZero (bool treat_as_gradient)
	Set parameters to zero, and if treat_as_gradient is true, we'll be treating this as a gradient so set the learning rate to 1 and make any other changes necessary (there's a variable we have to set for the MixtureProbComponent). More...
virtual void	Read (std::istream &is, bool binary)
virtual void	Write (std::ostream &os, bool binary) const
	Write component to stream. More...
virtual BaseFloat	DotProduct (const UpdatableComponent &other) const
	Here, "other" is a component of the same specific type. More...
virtual Component *	Copy () const
	Copy component (deep copy). More...
virtual void	PerturbParams (BaseFloat stddev)
	We introduce a new virtual function that only applies to class UpdatableComponent. More...
virtual void	SetParams (const VectorBase< BaseFloat > &bias, const MatrixBase< BaseFloat > &linear)
const CuVector< BaseFloat > &	BiasParams ()
const CuMatrix< BaseFloat > &	LinearParams ()
virtual int32	GetParameterDim () const
	The following new virtual function returns the total dimension of the parameters in this class. More...
virtual void	Vectorize (VectorBase< BaseFloat > *params) const
	Turns the parameters into vector form. More...
virtual void	UnVectorize (const VectorBase< BaseFloat > &params)
	Converts the parameters from vector form. More...
virtual void	LimitRank (int32 dimension, AffineComponent **a, AffineComponent **b) const
	This function is for getting a low-rank approximations of this AffineComponent by two AffineComponents. More...
void	Widen (int32 new_dimension, BaseFloat param_stddev, BaseFloat bias_stddev, std::vector< NonlinearComponent *> c2, AffineComponent *c3)
	This function is implemented in widen-nnet.cc. More...
Public Member Functions inherited from UpdatableComponent
	UpdatableComponent (const UpdatableComponent &other)
void	Init (BaseFloat learning_rate)
	UpdatableComponent (BaseFloat learning_rate)
	UpdatableComponent ()
virtual	~UpdatableComponent ()
void	SetLearningRate (BaseFloat lrate)
	Sets the learning rate of gradient descent. More...
BaseFloat	LearningRate () const
	Gets the learning rate of gradient descent. More...
Public Member Functions inherited from Component
	Component ()
virtual int32	Index () const
	Returns the index in the sequence of layers in the neural net; intended only to be used in debugging information. More...
virtual void	SetIndex (int32 index)
virtual std::vector< int32 >	Context () const
	Return a vector describing the temporal context this component requires for each frame of output, as a sorted list. More...
void	Propagate (const ChunkInfo &in_info, const ChunkInfo &out_info, const CuMatrixBase< BaseFloat > &in, CuMatrix< BaseFloat > *out) const
	A non-virtual propagate function that first resizes output if necessary. More...
virtual	~Component ()

Protected Member Functions
virtual void	Update (const CuMatrixBase< BaseFloat > &in_value, const CuMatrixBase< BaseFloat > &out_deriv)
virtual void	UpdateSimple (const CuMatrixBase< BaseFloat > &in_value, const CuMatrixBase< BaseFloat > &out_deriv)
const AffineComponent &	operator= (const AffineComponent &other)

Protected Attributes
CuMatrix< BaseFloat >	linear_params_
CuVector< BaseFloat >	bias_params_
bool	is_gradient_
Protected Attributes inherited from UpdatableComponent
BaseFloat	learning_rate_
	learning rate (0.0..0.01) More...

Friends
class	SoftmaxComponent
class	AffineComponentPreconditionedOnline

Additional Inherited Members
Static Public Member Functions inherited from Component
static Component *	ReadNew (std::istream &is, bool binary)
	Read component from stream. More...
static Component *	NewFromString (const std::string &initializer_line)
	Initialize the Component from one line that will contain first the type, e.g. More...
static Component *	NewComponentOfType (const std::string &type)
	Return a new Component of the given type e.g. More...

Detailed Description

Definition at line 843 of file nnet-component.h.

Constructor & Destructor Documentation

AffineComponent() [1/3]

AffineComponent

(

const AffineComponent &

other

)

Definition at line 1017 of file nnet-component.cc.

                                                                :
    UpdatableComponent(component),
    linear_params_(component.linear_params_),
    bias_params_(component.bias_params_),
    is_gradient_(component.is_gradient_) { }

AffineComponent() [2/3]

AffineComponent	(	const CuMatrixBase< BaseFloat > &	linear_params,
		const CuVectorBase< BaseFloat > &	bias_params,
		BaseFloat	learning_rate
	)

Definition at line 1023 of file nnet-component.cc.

References CuVectorBase< Real >::Dim(), AffineComponent::is_gradient_, KALDI_ASSERT, and CuMatrixBase< Real >::NumRows().

                                                         :
    UpdatableComponent(learning_rate),
    linear_params_(linear_params),
    bias_params_(bias_params) {
  KALDI_ASSERT(linear_params.NumRows() == bias_params.Dim()&&
               bias_params.Dim() != 0);
  is_gradient_ = false;
}

AffineComponent() [3/3]

AffineComponent ( )

inline

Definition at line 876 of file nnet-component.h.

Referenced by AffineComponent::Copy().

: is_gradient_(false) { } // use Init to really initialize.

Member Function Documentation

Add()

void Add ( BaseFloat  alpha, const UpdatableComponent &  other  )

virtual

This new virtual function adds the parameters of another updatable component, times some constant, to the current parameters.

Implements UpdatableComponent.

Definition at line 1009 of file nnet-component.cc.

References AffineComponent::bias_params_, KALDI_ASSERT, and AffineComponent::linear_params_.

                                                                             {
  const AffineComponent *other =
      dynamic_cast<const AffineComponent*>(&other_in);
  KALDI_ASSERT(other != NULL);
  linear_params_.AddMat(alpha, other->linear_params_);
  bias_params_.AddVec(alpha, other->bias_params_);
}

Backprop()

void Backprop ( const ChunkInfo &  in_info, const ChunkInfo &  out_info, const CuMatrixBase< BaseFloat > &  in_value, const CuMatrixBase< BaseFloat > &  out_value, const CuMatrixBase< BaseFloat > &  out_deriv, Component *  to_update, CuMatrix< BaseFloat > *  in_deriv  ) const

virtual

Perform backward pass propagation of the derivative, and also either update the model (if to_update == this) or update another model or compute the model derivative (otherwise).

Note: in_value and out_value are the values of the input and output of the component, and these may be dummy variables if respectively BackpropNeedsInput() or BackpropNeedsOutput() return false for that component (not all components need these).

num_chunks lets us treat the input matrix as contiguous-in-time chunks of equal size; it only matters if splicing is involved.

Implements Component.

Definition at line 1176 of file nnet-component.cc.

References CuMatrixBase< Real >::AddMatMat(), AffineComponent::InputDim(), AffineComponent::is_gradient_, kaldi::kNoTrans, AffineComponent::linear_params_, CuMatrixBase< Real >::NumRows(), CuMatrix< Real >::Resize(), AffineComponent::Update(), and AffineComponent::UpdateSimple().

                                                                    {
  AffineComponent *to_update = dynamic_cast<AffineComponent*>(to_update_in);
  in_deriv->Resize(out_deriv.NumRows(), InputDim());
  // Propagate the derivative back to the input.
  in_deriv->AddMatMat(1.0, out_deriv, kNoTrans, linear_params_, kNoTrans,
                      0.0);

  if (to_update != NULL) {
    // Next update the model (must do this 2nd so the derivatives we propagate
    // are accurate, in case this == to_update_in.)
    if (to_update->is_gradient_)
      to_update->UpdateSimple(in_value, out_deriv);
    else  // the call below is to a virtual function that may be re-implemented
      to_update->Update(in_value, out_deriv);  // by child classes.
  }
}

BackpropNeedsInput()

virtual bool BackpropNeedsInput ( ) const

inlinevirtual

Reimplemented from Component.

Definition at line 878 of file nnet-component.h.

{ return true; }

BackpropNeedsOutput()

virtual bool BackpropNeedsOutput ( ) const

inlinevirtual

Reimplemented from Component.

Definition at line 879 of file nnet-component.h.

References kaldi::cu::Copy(), kaldi::nnet3::DotProduct(), kaldi::nnet3::PerturbParams(), and Component::Propagate().

{ return false; }

BiasParams()

const CuVector<BaseFloat>& BiasParams ( )

inline

Definition at line 903 of file nnet-component.h.

Referenced by kaldi::nnet2::FixNnet(), and LimitRankClass::operator()().

{ return bias_params_; }

CollapseWithNext() [1/3]

Component * CollapseWithNext

(

const AffineComponent &

next

)

const

Definition at line 1295 of file nnet-component.cc.

References AffineComponent::bias_params_, AffineComponent::Copy(), AffineComponent::InputDim(), KALDI_ASSERT, kaldi::kNoTrans, AffineComponent::linear_params_, and AffineComponent::OutputDim().

Referenced by Nnet::Collapse(), and Nnet::ResizeOutputLayer().

                                                 {
  AffineComponent *ans = dynamic_cast<AffineComponent*>(this->Copy());
  KALDI_ASSERT(ans != NULL);
  // Note: it's possible that "ans" is really of a derived type such
  // as AffineComponentPreconditioned, but this will still work.
  // the "copy" call will copy things like learning rates, "alpha" value
  // for preconditioned component, etc.
  ans->linear_params_.Resize(next_component.OutputDim(), InputDim());
  ans->bias_params_ = next_component.bias_params_;

  ans->linear_params_.AddMatMat(1.0, next_component.linear_params_, kNoTrans,
                                this->linear_params_, kNoTrans, 0.0);
  ans->bias_params_.AddMatVec(1.0, next_component.linear_params_, kNoTrans,
                              this->bias_params_, 1.0);
  return ans;
}

CollapseWithNext() [2/3]

Component * CollapseWithNext

(

const FixedAffineComponent &

next

)

const

Definition at line 1313 of file nnet-component.cc.

References FixedAffineComponent::bias_params_, FixedAffineComponent::Copy(), AffineComponent::InputDim(), KALDI_ASSERT, kaldi::kNoTrans, FixedAffineComponent::linear_params_, and FixedAffineComponent::OutputDim().

                                                      {
  // If at least one was non-updatable, make the whole non-updatable.
  FixedAffineComponent *ans =
      dynamic_cast<FixedAffineComponent*>(next_component.Copy());
  KALDI_ASSERT(ans != NULL);
  ans->linear_params_.Resize(next_component.OutputDim(), InputDim());
  ans->bias_params_ = next_component.bias_params_;

  ans->linear_params_.AddMatMat(1.0, next_component.linear_params_, kNoTrans,
                                this->linear_params_, kNoTrans, 0.0);
  ans->bias_params_.AddMatVec(1.0, next_component.linear_params_, kNoTrans,
                              this->bias_params_, 1.0);
  return ans;
}

CollapseWithNext() [3/3]

Component * CollapseWithNext

(

const FixedScaleComponent &

next

)

const

Definition at line 1329 of file nnet-component.cc.

References AffineComponent::Copy(), FixedScaleComponent::InputDim(), KALDI_ASSERT, AffineComponent::OutputDim(), and FixedScaleComponent::scales_.

                                                     {
  KALDI_ASSERT(this->OutputDim() == next_component.InputDim());
  AffineComponent *ans =
      dynamic_cast<AffineComponent*>(this->Copy());
  KALDI_ASSERT(ans != NULL);
  ans->linear_params_.MulRowsVec(next_component.scales_);
  ans->bias_params_.MulElements(next_component.scales_);

  return ans;
}

CollapseWithPrevious()

Component * CollapseWithPrevious

(

const FixedAffineComponent &

prev

)

const

Definition at line 1343 of file nnet-component.cc.

References AffineComponent::bias_params_, FixedAffineComponent::bias_params_, FixedAffineComponent::Copy(), FixedAffineComponent::InputDim(), KALDI_ASSERT, kaldi::kNoTrans, AffineComponent::linear_params_, FixedAffineComponent::linear_params_, and AffineComponent::OutputDim().

                                                      {
  // If at least one was non-updatable, make the whole non-updatable.
  FixedAffineComponent *ans =
      dynamic_cast<FixedAffineComponent*>(prev_component.Copy());
  KALDI_ASSERT(ans != NULL);

  ans->linear_params_.Resize(this->OutputDim(), prev_component.InputDim());
  ans->bias_params_ = this->bias_params_;

  ans->linear_params_.AddMatMat(1.0, this->linear_params_, kNoTrans,
                                prev_component.linear_params_, kNoTrans, 0.0);
  ans->bias_params_.AddMatVec(1.0, this->linear_params_, kNoTrans,
                              prev_component.bias_params_, 1.0);
  return ans;
}

Copy()

Component * Copy ( ) const

virtual

Copy component (deep copy).

Implements Component.

Reimplemented in AffineComponentPreconditionedOnline, and AffineComponentPreconditioned.

Definition at line 1080 of file nnet-component.cc.

References AffineComponent::AffineComponent(), AffineComponent::bias_params_, AffineComponent::is_gradient_, UpdatableComponent::learning_rate_, and AffineComponent::linear_params_.

Referenced by AffineComponent::CollapseWithNext(), and AffineComponent::LimitRank().

                                       {
  AffineComponent *ans = new AffineComponent();
  ans->learning_rate_ = learning_rate_;
  ans->linear_params_ = linear_params_;
  ans->bias_params_ = bias_params_;
  ans->is_gradient_ = is_gradient_;
  return ans;
}

DotProduct()

BaseFloat DotProduct ( const UpdatableComponent &  other ) const

virtual

Here, "other" is a component of the same specific type.

This function computes the dot product in parameters, and is computed while automatically adjusting learning rates; typically, one of the two will actually contain the gradient.

Implements UpdatableComponent.

Definition at line 1089 of file nnet-component.cc.

References AffineComponent::bias_params_, kaldi::kTrans, AffineComponent::linear_params_, kaldi::TraceMatMat(), and kaldi::VecVec().

                                                                              {
  const AffineComponent *other =
      dynamic_cast<const AffineComponent*>(&other_in);
  return TraceMatMat(linear_params_, other->linear_params_, kTrans)
      + VecVec(bias_params_, other->bias_params_);
}

GetParameterDim()

int32 GetParameterDim ( ) const

virtual

The following new virtual function returns the total dimension of the parameters in this class.

E.g. used for L-BFGS update

Reimplemented from UpdatableComponent.

Definition at line 1247 of file nnet-component.cc.

References AffineComponent::InputDim(), and AffineComponent::OutputDim().

                                             {
  return (InputDim() + 1) * OutputDim();
}

Info()

std::string Info ( ) const

virtual

Reimplemented from UpdatableComponent.

Reimplemented in AffineComponentPreconditionedOnline, and AffineComponentPreconditioned.

Definition at line 1063 of file nnet-component.cc.

References AffineComponent::bias_params_, AffineComponent::InputDim(), kaldi::kTrans, UpdatableComponent::LearningRate(), AffineComponent::linear_params_, AffineComponent::OutputDim(), kaldi::TraceMatMat(), AffineComponent::Type(), and kaldi::VecVec().

                                      {
  std::stringstream stream;
  BaseFloat linear_params_size = static_cast<BaseFloat>(linear_params_.NumRows())
      * static_cast<BaseFloat>(linear_params_.NumCols());
  BaseFloat linear_stddev =
      std::sqrt(TraceMatMat(linear_params_, linear_params_, kTrans) /
                linear_params_size),
      bias_stddev = std::sqrt(VecVec(bias_params_, bias_params_) /
                              bias_params_.Dim());
  stream << Type() << ", input-dim=" << InputDim()
         << ", output-dim=" << OutputDim()
         << ", linear-params-stddev=" << linear_stddev
         << ", bias-params-stddev=" << bias_stddev
         << ", learning-rate=" << LearningRate();
  return stream.str();
}

Init() [1/2]

void Init	(	BaseFloat	learning_rate,
		int32	input_dim,
		int32	output_dim,
		BaseFloat	param_stddev,
		BaseFloat	bias_stddev
	)

Definition at line 1096 of file nnet-component.cc.

References AffineComponent::bias_params_, UpdatableComponent::Init(), KALDI_ASSERT, and AffineComponent::linear_params_.

Referenced by kaldi::nnet2::GenRandomNnet(), AffineComponent::InitFromString(), AffineComponentPreconditioned::InitFromString(), AffineComponentPreconditionedOnline::InitFromString(), and kaldi::nnet2::UnitTestAffineComponent().

                                                                          {
  UpdatableComponent::Init(learning_rate);
  linear_params_.Resize(output_dim, input_dim);
  bias_params_.Resize(output_dim);
  KALDI_ASSERT(output_dim > 0 && input_dim > 0 && param_stddev >= 0.0);
  linear_params_.SetRandn(); // sets to random normally distributed noise.
  linear_params_.Scale(param_stddev);
  bias_params_.SetRandn();
  bias_params_.Scale(bias_stddev);
}

Init() [2/2]

void Init	(	BaseFloat	learning_rate,
		std::string	matrix_filename
	)

Definition at line 1109 of file nnet-component.cc.

References AffineComponent::bias_params_, UpdatableComponent::Init(), KALDI_ASSERT, AffineComponent::linear_params_, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), CuMatrixBase< Real >::Range(), and kaldi::ReadKaldiObject().

                                                      {
  UpdatableComponent::Init(learning_rate);
  CuMatrix<BaseFloat> mat;
  ReadKaldiObject(matrix_filename, &mat); // will abort on failure.
  KALDI_ASSERT(mat.NumCols() >= 2);
  int32 input_dim = mat.NumCols() - 1, output_dim = mat.NumRows();
  linear_params_.Resize(output_dim, input_dim);
  bias_params_.Resize(output_dim);
  linear_params_.CopyFromMat(mat.Range(0, output_dim, 0, input_dim));
  bias_params_.CopyColFromMat(mat, input_dim);
}

InitFromString()

void InitFromString ( std::string  args )

virtual

Initialize, typically from a line of a config file.

The "args" will contain any parameters that need to be passed to the Component, e.g. dimensions.

Implements Component.

Reimplemented in AffineComponentPreconditionedOnline, and AffineComponentPreconditioned.

Definition at line 1122 of file nnet-component.cc.

References AffineComponent::Init(), AffineComponent::InputDim(), KALDI_ASSERT, KALDI_ERR, UpdatableComponent::learning_rate_, AffineComponent::OutputDim(), and kaldi::nnet2::ParseFromString().

Referenced by kaldi::nnet2::UnitTestAffineComponent().

                                                   {
  std::string orig_args(args);
  bool ok = true;
  BaseFloat learning_rate = learning_rate_;
  std::string matrix_filename;
  int32 input_dim = -1, output_dim = -1;
  ParseFromString("learning-rate", &args, &learning_rate); // optional.
  if (ParseFromString("matrix", &args, &matrix_filename)) {
    Init(learning_rate, matrix_filename);
    if (ParseFromString("input-dim", &args, &input_dim))
      KALDI_ASSERT(input_dim == InputDim() &&
                   "input-dim mismatch vs. matrix.");
    if (ParseFromString("output-dim", &args, &output_dim))
      KALDI_ASSERT(output_dim == OutputDim() &&
                   "output-dim mismatch vs. matrix.");
  } else {
    ok = ok && ParseFromString("input-dim", &args, &input_dim);
    ok = ok && ParseFromString("output-dim", &args, &output_dim);
    BaseFloat param_stddev = 1.0 / std::sqrt(input_dim),
        bias_stddev = 1.0;
    ParseFromString("param-stddev", &args, &param_stddev);
    ParseFromString("bias-stddev", &args, &bias_stddev);
    Init(learning_rate, input_dim, output_dim,
         param_stddev, bias_stddev);
  }
  if (!args.empty())
    KALDI_ERR << "Could not process these elements in initializer: "
              << args;
  if (!ok)
    KALDI_ERR << "Bad initializer " << orig_args;
}

InputDim()

virtual int32 InputDim ( ) const

inlinevirtual

Get size of input vectors.

Implements Component.

Definition at line 852 of file nnet-component.h.

Referenced by AffineComponent::Backprop(), SpliceComponent::Backprop(), SpliceMaxComponent::Backprop(), BlockAffineComponent::Backprop(), SumGroupComponent::Backprop(), AffineComponent::CollapseWithNext(), AffineComponent::GetParameterDim(), BlockAffineComponent::GetParameterDim(), AffineComponent::Info(), AffineComponentPreconditioned::Info(), AffineComponentPreconditionedOnline::Info(), AffineComponent::InitFromString(), AffineComponentPreconditioned::InitFromString(), AffineComponentPreconditionedOnline::InitFromString(), AffineComponent::LimitRank(), SoftmaxComponent::MixUp(), DctComponent::Propagate(), DropoutComponent::Propagate(), AdditiveNoiseComponent::Propagate(), Nnet::ResizeOutputLayer(), AffineComponent::UnVectorize(), AffineComponent::Vectorize(), AffineComponent::Widen(), and kaldi::nnet2::WidenNnet().

{ return linear_params_.NumCols(); }

LimitRank()

void LimitRank ( int32  dimension, AffineComponent **  a, AffineComponent **  b  ) const

virtual

This function is for getting a low-rank approximations of this AffineComponent by two AffineComponents.

Definition at line 1261 of file nnet-component.cc.

References AffineComponent::bias_params_, AffineComponent::Copy(), MatrixBase< Real >::DestructiveSvd(), AffineComponent::InputDim(), KALDI_ASSERT, KALDI_LOG, kaldi::kCopyData, kaldi::kSetZero, AffineComponent::linear_params_, MatrixBase< Real >::MulRowsVec(), MatrixBase< Real >::NumCols(), MatrixBase< Real >::NumRows(), Vector< Real >::Resize(), Matrix< Real >::Resize(), kaldi::SortSvd(), and VectorBase< Real >::Sum().

Referenced by Nnet::LimitRankOfLastLayer().

                                                                                {
  KALDI_ASSERT(d <= InputDim());

  // We'll limit the rank of just the linear part, keeping the bias vector full.
  Matrix<BaseFloat> M (linear_params_);
  int32 rows = M.NumRows(), cols = M.NumCols(), rc_min = std::min(rows, cols);
  Vector<BaseFloat> s(rc_min);
  Matrix<BaseFloat> U(rows, rc_min), Vt(rc_min, cols);
  // Do the destructive svd M = U diag(s) V^T.  It actually outputs the transpose of V.
  M.DestructiveSvd(&s, &U, &Vt);
  SortSvd(&s, &U, &Vt); // Sort the singular values from largest to smallest.
  BaseFloat old_svd_sum = s.Sum();
  U.Resize(rows, d, kCopyData);
  s.Resize(d, kCopyData);
  Vt.Resize(d, cols, kCopyData);
  BaseFloat new_svd_sum = s.Sum();
  KALDI_LOG << "Reduced rank from "
            << rc_min <<  " to " << d << ", SVD sum reduced from "
            << old_svd_sum << " to " << new_svd_sum;

  // U.MulColsVec(s); // U <-- U diag(s)
  Vt.MulRowsVec(s); // Vt <-- diag(s) Vt.

  *a = dynamic_cast<AffineComponent*>(this->Copy());
  *b = dynamic_cast<AffineComponent*>(this->Copy());

  (*a)->bias_params_.Resize(d, kSetZero);
  (*a)->linear_params_ = Vt;

  (*b)->bias_params_ = this->bias_params_;
  (*b)->linear_params_ = U;
}

LinearParams()

const CuMatrix<BaseFloat>& LinearParams ( )

inline

Definition at line 904 of file nnet-component.h.

Referenced by kaldi::nnet2::FixNnet(), and LimitRankClass::operator()().

{ return linear_params_; }

operator=()

const AffineComponent& operator= ( const AffineComponent &  other )

protected

OutputDim()

virtual int32 OutputDim ( ) const

inlinevirtual

Get size of output vectors.

Implements Component.

Definition at line 853 of file nnet-component.h.

Referenced by SpliceComponent::Backprop(), PermuteComponent::Backprop(), DctComponent::Backprop(), AffineComponent::CollapseWithNext(), AffineComponent::CollapseWithPrevious(), AffineComponent::GetParameterDim(), BlockAffineComponent::GetParameterDim(), AffineComponent::Info(), AffineComponentPreconditioned::Info(), AffineComponentPreconditionedOnline::Info(), AffineComponent::InitFromString(), AffineComponentPreconditioned::InitFromString(), AffineComponentPreconditionedOnline::InitFromString(), AffineComponent::UnVectorize(), AffineComponent::Vectorize(), AffineComponent::Widen(), and kaldi::nnet2::WidenNnet().

{ return linear_params_.NumRows(); }

PerturbParams()

void PerturbParams ( BaseFloat  stddev )

virtual

We introduce a new virtual function that only applies to class UpdatableComponent.

This is used in testing.

Implements UpdatableComponent.

Definition at line 1053 of file nnet-component.cc.

References AffineComponent::bias_params_, AffineComponent::linear_params_, CuVectorBase< Real >::SetRandn(), and CuMatrixBase< Real >::SetRandn().

                                                    {
  CuMatrix<BaseFloat> temp_linear_params(linear_params_);
  temp_linear_params.SetRandn();
  linear_params_.AddMat(stddev, temp_linear_params);

  CuVector<BaseFloat> temp_bias_params(bias_params_);
  temp_bias_params.SetRandn();
  bias_params_.AddVec(stddev, temp_bias_params);
}

Propagate()

void Propagate ( const ChunkInfo &  in_info, const ChunkInfo &  out_info, const CuMatrixBase< BaseFloat > &  in, CuMatrixBase< BaseFloat > *  out  ) const

virtual

Perform forward pass propagation Input->Output.

Each row is one frame or training example. Interpreted as "num_chunks" equally sized chunks of frames; this only matters for layers that do things like context splicing. Typically this variable will either be 1 (when we're processing a single contiguous chunk of data) or will be the same as in.NumFrames(), but other values are possible if some layers do splicing.

Implements Component.

Definition at line 1155 of file nnet-component.cc.

References CuMatrixBase< Real >::AddMatMat(), AffineComponent::bias_params_, ChunkInfo::CheckSize(), CuMatrixBase< Real >::CopyRowsFromVec(), KALDI_ASSERT, kaldi::kNoTrans, kaldi::kTrans, AffineComponent::linear_params_, and ChunkInfo::NumChunks().

                                                                     {
  in_info.CheckSize(in);
  out_info.CheckSize(*out);
  KALDI_ASSERT(in_info.NumChunks() == out_info.NumChunks());

  // No need for asserts as they'll happen within the matrix operations.
  out->CopyRowsFromVec(bias_params_); // copies bias_params_ to each row
  // of *out.
  out->AddMatMat(1.0, in, kNoTrans, linear_params_, kTrans, 1.0);
}

Read()

void Read ( std::istream &  is, bool  binary  )

virtual

Implements Component.

Reimplemented in AffineComponentPreconditionedOnline, and AffineComponentPreconditioned.

Definition at line 1199 of file nnet-component.cc.

References AffineComponent::bias_params_, kaldi::nnet2::ExpectOneOrTwoTokens(), kaldi::ExpectToken(), AffineComponent::is_gradient_, KALDI_ASSERT, UpdatableComponent::learning_rate_, AffineComponent::linear_params_, CuVector< Real >::Read(), kaldi::ReadBasicType(), kaldi::ReadToken(), and AffineComponent::Type().

                                                      {
  std::ostringstream ostr_beg, ostr_end;
  ostr_beg << "<" << Type() << ">"; // e.g. "<AffineComponent>"
  ostr_end << "</" << Type() << ">"; // e.g. "</AffineComponent>"
  // might not see the "<AffineComponent>" part because
  // of how ReadNew() works.
  ExpectOneOrTwoTokens(is, binary, ostr_beg.str(), "<LearningRate>");
  ReadBasicType(is, binary, &learning_rate_);
  ExpectToken(is, binary, "<LinearParams>");
  linear_params_.Read(is, binary);
  ExpectToken(is, binary, "<BiasParams>");
  bias_params_.Read(is, binary);
  std::string tok;
  // back-compatibility code.  TODO: re-do this later.
  ReadToken(is, binary, &tok);
  if (tok == "<AvgInput>") { // discard the following.
    CuVector<BaseFloat> avg_input;
    avg_input.Read(is, binary);
    BaseFloat avg_input_count;
    ExpectToken(is, binary, "<AvgInputCount>");
    ReadBasicType(is, binary, &avg_input_count);
    ReadToken(is, binary, &tok);
  }
  if (tok == "<IsGradient>") {
    ReadBasicType(is, binary, &is_gradient_);
    ExpectToken(is, binary, ostr_end.str());
  } else {
    is_gradient_ = false;
    KALDI_ASSERT(tok == ostr_end.str());
  }
}

Resize()

void Resize ( int32  input_dim, int32  output_dim  )

virtual

Reimplemented in AffineComponentPreconditionedOnline.

Definition at line 1003 of file nnet-component.cc.

References KALDI_ASSERT.

Referenced by Nnet::ResizeOutputLayer().

                                                              {
  KALDI_ASSERT(input_dim > 0 && output_dim > 0);
  bias_params_.Resize(output_dim);
  linear_params_.Resize(output_dim, input_dim);
}

Scale()

void Scale ( BaseFloat  scale )

virtual

This new virtual function scales the parameters by this amount.

Implements UpdatableComponent.

Definition at line 997 of file nnet-component.cc.

                                           {
  linear_params_.Scale(scale);
  bias_params_.Scale(scale);
}

SetParams()

void SetParams ( const VectorBase< BaseFloat > &  bias, const MatrixBase< BaseFloat > &  linear  )

virtual

Definition at line 1046 of file nnet-component.cc.

References AffineComponent::bias_params_, KALDI_ASSERT, and AffineComponent::linear_params_.

Referenced by kaldi::nnet2::FixNnet(), SoftmaxComponent::MixUp(), and LimitRankClass::operator()().

                                                                     {
  bias_params_ = bias;
  linear_params_ = linear;
  KALDI_ASSERT(bias_params_.Dim() == linear_params_.NumRows());
}

SetZero()

void SetZero ( bool  treat_as_gradient )

virtual

Set parameters to zero, and if treat_as_gradient is true, we'll be treating this as a gradient so set the learning rate to 1 and make any other changes necessary (there's a variable we have to set for the MixtureProbComponent).

Implements UpdatableComponent.

Definition at line 1036 of file nnet-component.cc.

References AffineComponent::bias_params_, AffineComponent::is_gradient_, AffineComponent::linear_params_, and UpdatableComponent::SetLearningRate().

                                                    {
  if (treat_as_gradient) {
    SetLearningRate(1.0);
  }
  linear_params_.SetZero();
  bias_params_.SetZero();
  if (treat_as_gradient)
    is_gradient_ = true;
}

Type()

virtual std::string Type ( ) const

inlinevirtual

Implements Component.

Reimplemented in AffineComponentPreconditionedOnline, and AffineComponentPreconditioned.

Definition at line 877 of file nnet-component.h.

Referenced by AffineComponent::Info(), AffineComponentPreconditioned::Info(), AffineComponent::Read(), AffineComponentPreconditioned::Read(), AffineComponentPreconditionedOnline::Read(), AffineComponent::Write(), and AffineComponentPreconditioned::Write().

{ return "AffineComponent"; }

UnVectorize()

void UnVectorize ( const VectorBase< BaseFloat > &  params )

virtual

Converts the parameters from vector form.

Reimplemented from UpdatableComponent.

Definition at line 1255 of file nnet-component.cc.

References AffineComponent::bias_params_, AffineComponent::InputDim(), AffineComponent::linear_params_, AffineComponent::OutputDim(), and VectorBase< Real >::Range().

                                                                     {
  linear_params_.CopyRowsFromVec(params.Range(0, InputDim() * OutputDim()));
  bias_params_.CopyFromVec(params.Range(InputDim() * OutputDim(),
                                        OutputDim()));
}

Update()

virtual void Update ( const CuMatrixBase< BaseFloat > &  in_value, const CuMatrixBase< BaseFloat > &  out_deriv  )

inlineprotectedvirtual

Reimplemented in AffineComponentPreconditionedOnline, and AffineComponentPreconditioned.

Definition at line 925 of file nnet-component.h.

Referenced by AffineComponent::Backprop().

                                                {
    UpdateSimple(in_value, out_deriv);
  }

UpdateSimple()

void UpdateSimple ( const CuMatrixBase< BaseFloat > &  in_value, const CuMatrixBase< BaseFloat > &  out_deriv  )

protectedvirtual

Definition at line 1169 of file nnet-component.cc.

References AffineComponent::bias_params_, kaldi::kNoTrans, kaldi::kTrans, UpdatableComponent::learning_rate_, and AffineComponent::linear_params_.

Referenced by AffineComponent::Backprop(), and BlockAffineComponentPreconditioned::Update().

                                                                             {
  bias_params_.AddRowSumMat(learning_rate_, out_deriv, 1.0);
  linear_params_.AddMatMat(learning_rate_, out_deriv, kTrans,
                           in_value, kNoTrans, 1.0);
}

Vectorize()

void Vectorize ( VectorBase< BaseFloat > *  params ) const

virtual

Turns the parameters into vector form.

We put the vector form on the CPU, because in the kinds of situations where we do this, we'll tend to use too much memory for the GPU.

Reimplemented from UpdatableComponent.

Definition at line 1250 of file nnet-component.cc.

References AffineComponent::bias_params_, AffineComponent::InputDim(), AffineComponent::linear_params_, AffineComponent::OutputDim(), and VectorBase< Real >::Range().

                                                                   {
  params->Range(0, InputDim() * OutputDim()).CopyRowsFromMat(linear_params_);
  params->Range(InputDim() * OutputDim(),
                OutputDim()).CopyFromVec(bias_params_);
}

Widen()

void Widen	(	int32	new_dimension,
		BaseFloat	param_stddev,
		BaseFloat	bias_stddev,
		std::vector< NonlinearComponent *>	c2,
		AffineComponent *	c3
	)

This function is implemented in widen-nnet.cc.

Definition at line 28 of file widen-nnet.cc.

References AffineComponent::bias_params_, rnnlm::i, AffineComponent::InputDim(), KALDI_ASSERT, KALDI_WARN, kaldi::kCopyData, AffineComponent::linear_params_, and AffineComponent::OutputDim().

Referenced by kaldi::nnet2::WidenNnet().

                                                 {
  int32 old_dim = this->OutputDim(), extra_dim = new_dim - old_dim;
  KALDI_ASSERT(!c2.empty());
  if (new_dim <= old_dim) {
    KALDI_WARN << "Not widening component because new dim "
               << new_dim << " <= old dim " << old_dim;
    return;
  }
  
  this->bias_params_.Resize(new_dim,
                            kCopyData);
  this->bias_params_.Range(old_dim, extra_dim).SetRandn();
  this->bias_params_.Range(old_dim, extra_dim).Scale(bias_stddev);

  this->linear_params_.Resize(new_dim, InputDim(), kCopyData);
  this->linear_params_.Range(old_dim, extra_dim,
                             0, InputDim()).SetRandn();
  this->linear_params_.Range(old_dim, extra_dim,
                             0, InputDim()).Scale(param_stddev);

  for (size_t i = 0; i < c2.size(); i++) // Change dimension of nonlinear
    c2[i]->SetDim(new_dim);              // components
    
  // Change dimension of next affine component [extend with zeros,
  // so the existing outputs do not change in value]
  c3->linear_params_.Resize(c3->OutputDim(), new_dim, kCopyData);
}

Write()

void Write ( std::ostream &  os, bool  binary  ) const

virtual

Write component to stream.

Implements Component.

Reimplemented in AffineComponentPreconditionedOnline, and AffineComponentPreconditioned.

Definition at line 1231 of file nnet-component.cc.

References AffineComponent::bias_params_, AffineComponent::is_gradient_, UpdatableComponent::learning_rate_, AffineComponent::linear_params_, AffineComponent::Type(), kaldi::WriteBasicType(), and kaldi::WriteToken().

                                                             {
  std::ostringstream ostr_beg, ostr_end;
  ostr_beg << "<" << Type() << ">"; // e.g. "<AffineComponent>"
  ostr_end << "</" << Type() << ">"; // e.g. "</AffineComponent>"
  WriteToken(os, binary, ostr_beg.str());
  WriteToken(os, binary, "<LearningRate>");
  WriteBasicType(os, binary, learning_rate_);
  WriteToken(os, binary, "<LinearParams>");
  linear_params_.Write(os, binary);
  WriteToken(os, binary, "<BiasParams>");
  bias_params_.Write(os, binary);
  WriteToken(os, binary, "<IsGradient>");
  WriteBasicType(os, binary, is_gradient_);
  WriteToken(os, binary, ostr_end.str());
}

Friends And Related Function Documentation

AffineComponentPreconditionedOnline

friend class AffineComponentPreconditionedOnline

friend

Definition at line 923 of file nnet-component.h.

SoftmaxComponent

friend class SoftmaxComponent

friend

Definition at line 844 of file nnet-component.h.

Member Data Documentation

bias_params_

CuVector<BaseFloat> bias_params_

protected

Definition at line 938 of file nnet-component.h.

Referenced by AffineComponent::Add(), BlockAffineComponent::Add(), AffineComponentPreconditionedOnline::AffineComponentPreconditionedOnline(), AffineComponent::CollapseWithNext(), AffineComponent::CollapseWithPrevious(), AffineComponent::Copy(), AffineComponentPreconditioned::Copy(), AffineComponentPreconditionedOnline::Copy(), BlockAffineComponent::Copy(), BlockAffineComponentPreconditioned::Copy(), FixedAffineComponent::Copy(), AffineComponent::DotProduct(), BlockAffineComponent::DotProduct(), AffineComponent::Info(), AffineComponentPreconditioned::Info(), AffineComponentPreconditionedOnline::Info(), FixedAffineComponent::Info(), AffineComponent::Init(), AffineComponentPreconditioned::Init(), AffineComponentPreconditionedOnline::Init(), BlockAffineComponent::Init(), FixedAffineComponent::Init(), AffineComponent::LimitRank(), SoftmaxComponent::MixUp(), AffineComponent::PerturbParams(), BlockAffineComponent::PerturbParams(), AffineComponent::Propagate(), BlockAffineComponent::Propagate(), FixedAffineComponent::Propagate(), AffineComponent::Read(), AffineComponentPreconditioned::Read(), AffineComponentPreconditionedOnline::Read(), BlockAffineComponent::Read(), BlockAffineComponentPreconditioned::Read(), FixedAffineComponent::Read(), AffineComponentPreconditionedOnline::Resize(), BlockAffineComponent::Scale(), AffineComponent::SetParams(), AffineComponent::SetZero(), BlockAffineComponent::SetZero(), AffineComponent::UnVectorize(), BlockAffineComponent::UnVectorize(), AffineComponentPreconditioned::Update(), AffineComponentPreconditionedOnline::Update(), BlockAffineComponentPreconditioned::Update(), AffineComponent::UpdateSimple(), BlockAffineComponent::UpdateSimple(), AffineComponent::Vectorize(), BlockAffineComponent::Vectorize(), AffineComponent::Widen(), AffineComponent::Write(), AffineComponentPreconditioned::Write(), AffineComponentPreconditionedOnline::Write(), BlockAffineComponent::Write(), BlockAffineComponentPreconditioned::Write(), and FixedAffineComponent::Write().

is_gradient_

bool is_gradient_

protected

Definition at line 940 of file nnet-component.h.

Referenced by AffineComponent::AffineComponent(), AffineComponentPreconditionedOnline::AffineComponentPreconditionedOnline(), AffineComponent::Backprop(), AffineComponent::Copy(), AffineComponentPreconditioned::Copy(), AffineComponentPreconditionedOnline::Copy(), BlockAffineComponentPreconditioned::Copy(), BlockAffineComponentPreconditioned::Init(), AffineComponent::Read(), BlockAffineComponentPreconditioned::Read(), AffineComponent::SetZero(), BlockAffineComponentPreconditioned::SetZero(), BlockAffineComponentPreconditioned::Update(), AffineComponent::Write(), and BlockAffineComponentPreconditioned::Write().

linear_params_

CuMatrix<BaseFloat> linear_params_

protected

Definition at line 937 of file nnet-component.h.

Referenced by AffineComponent::Add(), BlockAffineComponent::Add(), AffineComponentPreconditionedOnline::AffineComponentPreconditionedOnline(), AffineComponent::Backprop(), BlockAffineComponent::Backprop(), FixedAffineComponent::Backprop(), AffineComponent::CollapseWithNext(), AffineComponent::CollapseWithPrevious(), AffineComponent::Copy(), AffineComponentPreconditioned::Copy(), AffineComponentPreconditionedOnline::Copy(), BlockAffineComponent::Copy(), BlockAffineComponentPreconditioned::Copy(), FixedAffineComponent::Copy(), AffineComponent::DotProduct(), BlockAffineComponent::DotProduct(), AffineComponent::Info(), AffineComponentPreconditioned::Info(), AffineComponentPreconditionedOnline::Info(), FixedAffineComponent::Info(), AffineComponent::Init(), AffineComponentPreconditioned::Init(), AffineComponentPreconditionedOnline::Init(), BlockAffineComponent::Init(), FixedAffineComponent::Init(), AffineComponent::LimitRank(), SoftmaxComponent::MixUp(), AffineComponent::PerturbParams(), BlockAffineComponent::PerturbParams(), AffineComponent::Propagate(), BlockAffineComponent::Propagate(), FixedAffineComponent::Propagate(), AffineComponent::Read(), AffineComponentPreconditioned::Read(), AffineComponentPreconditionedOnline::Read(), BlockAffineComponent::Read(), BlockAffineComponentPreconditioned::Read(), FixedAffineComponent::Read(), AffineComponentPreconditionedOnline::Resize(), BlockAffineComponent::Scale(), AffineComponent::SetParams(), AffineComponent::SetZero(), BlockAffineComponent::SetZero(), AffineComponent::UnVectorize(), BlockAffineComponent::UnVectorize(), AffineComponentPreconditioned::Update(), AffineComponentPreconditionedOnline::Update(), BlockAffineComponentPreconditioned::Update(), AffineComponent::UpdateSimple(), BlockAffineComponent::UpdateSimple(), AffineComponent::Vectorize(), BlockAffineComponent::Vectorize(), AffineComponent::Widen(), AffineComponent::Write(), AffineComponentPreconditioned::Write(), AffineComponentPreconditionedOnline::Write(), BlockAffineComponent::Write(), BlockAffineComponentPreconditioned::Write(), and FixedAffineComponent::Write().

---

The documentation for this class was generated from the following files:

• nnet2/nnet-component.h
• nnet2/nnet-component.cc
• nnet2/widen-nnet.cc