This class is for single-threaded training of neural nets using the 'chain' model. More...

#include <nnet-chain-training.h>

Collaboration diagram for NnetChainTrainer:

[legend]

Public Member Functions
	NnetChainTrainer (const NnetChainTrainingOptions &config, const fst::StdVectorFst &den_fst, Nnet *nnet)

void	Train (const NnetChainExample &eg)

bool	PrintTotalStats () const

	~NnetChainTrainer ()

Private Member Functions
void	TrainInternal (const NnetChainExample &eg, const NnetComputation &computation)

void	TrainInternalBackstitch (const NnetChainExample &eg, const NnetComputation &computation, bool is_backstitch_step1)

void	ProcessOutputs (bool is_backstitch_step2, const NnetChainExample &eg, NnetComputer *computer)

Private Attributes
const NnetChainTrainingOptions	opts_

chain::DenominatorGraph	den_graph_

Nnet *	nnet_

Nnet *	delta_nnet_

CachingOptimizingCompiler	compiler_

int32	num_minibatches_processed_

MaxChangeStats	max_change_stats_

unordered_map< std::string, ObjectiveFunctionInfo, StringHasher >	objf_info_

int32	srand_seed_

Detailed Description

This class is for single-threaded training of neural nets using the 'chain' model.

Definition at line 55 of file nnet-chain-training.h.

Constructor & Destructor Documentation

◆ NnetChainTrainer()

NnetChainTrainer	(	const NnetChainTrainingOptions &	config,
		const fst::StdVectorFst &	den_fst,
		Nnet *	nnet
	)

Definition at line 27 of file nnet-chain-training.cc.

References NnetTrainerOptions::backstitch_training_interval, NnetChainTrainer::compiler_, Nnet::Copy(), NnetChainTrainer::delta_nnet_, KALDI_ASSERT, KALDI_LOG, KALDI_WARN, NnetTrainerOptions::max_param_change, NnetTrainerOptions::momentum, NnetChainTrainer::nnet_, NnetChainTrainingOptions::nnet_config, NnetTrainerOptions::read_cache, CachingOptimizingCompiler::ReadCache(), kaldi::nnet3::ScaleNnet(), Input::Stream(), NnetTrainerOptions::zero_component_stats, and kaldi::nnet3::ZeroComponentStats().

                                               :
     opts_(opts),
     den_graph_(den_fst, nnet->OutputDim("output")),
     nnet_(nnet),
     compiler_(*nnet, opts_.nnet_config.optimize_config,
               opts_.nnet_config.compiler_config),
     num_minibatches_processed_(0),
     max_change_stats_(*nnet),
     srand_seed_(RandInt(0, 100000)) {
   if (opts.nnet_config.zero_component_stats)
     ZeroComponentStats(nnet);
   KALDI_ASSERT(opts.nnet_config.momentum >= 0.0 &&
                opts.nnet_config.max_param_change >= 0.0 &&
                opts.nnet_config.backstitch_training_interval > 0);
   delta_nnet_ = nnet_->Copy();
   ScaleNnet(0.0, delta_nnet_);
 
   if (opts.nnet_config.read_cache != "") {
     bool binary;
     try {
       Input ki(opts.nnet_config.read_cache, &binary);
       compiler_.ReadCache(ki.Stream(), binary);
       KALDI_LOG << "Read computation cache from " << opts.nnet_config.read_cache;
     } catch (...) {
       KALDI_WARN << "Could not open cached computation. "
                     "Probably this is the first training iteration.";
     }
   }
 }

◆ ~NnetChainTrainer()

~NnetChainTrainer ( )

Definition at line 287 of file nnet-chain-training.cc.

References NnetTrainerOptions::binary_write_cache, NnetChainTrainer::compiler_, NnetChainTrainer::delta_nnet_, KALDI_LOG, NnetChainTrainingOptions::nnet_config, NnetChainTrainer::opts_, Output::Stream(), NnetTrainerOptions::write_cache, and CachingOptimizingCompiler::WriteCache().

                                     {
   if (opts_.nnet_config.write_cache != "") {
     Output ko(opts_.nnet_config.write_cache, opts_.nnet_config.binary_write_cache);
     compiler_.WriteCache(ko.Stream(), opts_.nnet_config.binary_write_cache);
     KALDI_LOG << "Wrote computation cache to " << opts_.nnet_config.write_cache;
   }
   delete delta_nnet_;
 }

Member Function Documentation

◆ PrintTotalStats()

bool PrintTotalStats ( ) const

Definition at line 273 of file nnet-chain-training.cc.

References NnetChainTrainer::max_change_stats_, NnetChainTrainer::nnet_, NnetChainTrainer::objf_info_, MaxChangeStats::Print(), and ObjectiveFunctionInfo::PrintTotalStats().

                                              {
   unordered_map<std::string, ObjectiveFunctionInfo, StringHasher>::const_iterator
       iter = objf_info_.begin(),
       end = objf_info_.end();
   bool ans = false;
   for (; iter != end; ++iter) {
     const std::string &name = iter->first;
     const ObjectiveFunctionInfo &info = iter->second;
     ans = info.PrintTotalStats(name) || ans;
   }
   max_change_stats_.Print(*nnet_);
   return ans;
 }

◆ ProcessOutputs()

void ProcessOutputs	(	bool	is_backstitch_step2,
		const NnetChainExample &	eg,
		NnetComputer *	computer
	)

private

Definition at line 204 of file nnet-chain-training.cc.

References NnetComputer::AcceptInput(), NnetChainTrainingOptions::apply_deriv_weights, NnetChainTrainingOptions::chain_config, NnetChainTrainer::den_graph_, NnetChainSupervision::deriv_weights, Nnet::GetNodeIndex(), NnetComputer::GetOutput(), Nnet::IsOutputNode(), KALDI_ERR, kaldi::kTrans, kaldi::kUndefined, CuMatrixBase< Real >::MulRowsVec(), NnetChainSupervision::name, NnetChainTrainer::nnet_, NnetChainTrainingOptions::nnet_config, NnetChainTrainer::num_minibatches_processed_, NVTX_RANGE, NnetChainTrainer::objf_info_, NnetChainTrainer::opts_, NnetChainExample::outputs, NnetTrainerOptions::print_interval, CuMatrixBase< Real >::Scale(), NnetChainSupervision::supervision, and kaldi::TraceMatMat().

Referenced by NnetChainTrainer::TrainInternal(), and NnetChainTrainer::TrainInternalBackstitch().

                                                               {
   NVTX_RANGE(__func__);
   // normally the eg will have just one output named 'output', but
   // we don't assume this.
   // In backstitch training, the output-name with the "_backstitch" suffix is
   // the one computed after the first, backward step of backstitch.
   const std::string suffix = (is_backstitch_step2 ? "_backstitch" : "");
   std::vector<NnetChainSupervision>::const_iterator iter = eg.outputs.begin(),
       end = eg.outputs.end();
   for (; iter != end; ++iter) {
     const NnetChainSupervision &sup = *iter;
     int32 node_index = nnet_->GetNodeIndex(sup.name);
     if (node_index < 0 ||
         !nnet_->IsOutputNode(node_index))
       KALDI_ERR << "Network has no output named " << sup.name;
 
     const CuMatrixBase<BaseFloat> &nnet_output = computer->GetOutput(sup.name);
     CuMatrix<BaseFloat> nnet_output_deriv(nnet_output.NumRows(),
                                           nnet_output.NumCols(),
                                           kUndefined);
 
     bool use_xent = (opts_.chain_config.xent_regularize != 0.0);
     std::string xent_name = sup.name + "-xent";  // typically "output-xent".
     CuMatrix<BaseFloat> xent_deriv;
 
     BaseFloat tot_objf, tot_l2_term, tot_weight;
 
     ComputeChainObjfAndDeriv(opts_.chain_config, den_graph_,
                              sup.supervision, nnet_output,
                              &tot_objf, &tot_l2_term, &tot_weight,
                              &nnet_output_deriv,
                              (use_xent ? &xent_deriv : NULL));
 
     if (use_xent) {
       // this block computes the cross-entropy objective.
       const CuMatrixBase<BaseFloat> &xent_output = computer->GetOutput(
           xent_name);
       // at this point, xent_deriv is posteriors derived from the numerator
       // computation.  note, xent_objf has a factor of '.supervision.weight'
       BaseFloat xent_objf = TraceMatMat(xent_output, xent_deriv, kTrans);
       objf_info_[xent_name + suffix].UpdateStats(xent_name + suffix,
                                         opts_.nnet_config.print_interval,
                                         num_minibatches_processed_,
                                         tot_weight, xent_objf);
     }
 
     if (opts_.apply_deriv_weights && sup.deriv_weights.Dim() != 0) {
       CuVector<BaseFloat> cu_deriv_weights(sup.deriv_weights);
       nnet_output_deriv.MulRowsVec(cu_deriv_weights);
       if (use_xent)
         xent_deriv.MulRowsVec(cu_deriv_weights);
     }
 
     computer->AcceptInput(sup.name, &nnet_output_deriv);
 
     objf_info_[sup.name + suffix].UpdateStats(sup.name + suffix,
                                      opts_.nnet_config.print_interval,
                                      num_minibatches_processed_,
                                      tot_weight, tot_objf, tot_l2_term);
 
     if (use_xent) {
       xent_deriv.Scale(opts_.chain_config.xent_regularize);
       computer->AcceptInput(xent_name, &xent_deriv);
     }
   }
 }

◆ Train()

void Train ( const NnetChainExample & eg )

Definition at line 60 of file nnet-chain-training.cc.

References NnetTrainerOptions::backstitch_training_interval, NnetTrainerOptions::backstitch_training_scale, NnetChainTrainingOptions::chain_config, CachingOptimizingCompiler::Compile(), NnetChainTrainer::compiler_, kaldi::nnet3::ConsolidateMemory(), NnetChainTrainer::delta_nnet_, kaldi::nnet3::FreezeNaturalGradient(), kaldi::nnet3::GetChainComputationRequest(), KALDI_ASSERT, NnetTrainerOptions::momentum, NnetChainTrainer::nnet_, NnetChainTrainingOptions::nnet_config, NnetChainTrainer::num_minibatches_processed_, NVTX_RANGE, NnetChainTrainer::opts_, kaldi::nnet3::ResetGenerators(), NnetChainTrainer::srand_seed_, NnetTrainerOptions::store_component_stats, NnetChainTrainer::TrainInternal(), and NnetChainTrainer::TrainInternalBackstitch().

                                                              {
   NVTX_RANGE(__func__);
   bool need_model_derivative = true;
   const NnetTrainerOptions &nnet_config = opts_.nnet_config;
   bool use_xent_regularization = (opts_.chain_config.xent_regularize != 0.0);
   ComputationRequest request;
   GetChainComputationRequest(*nnet_, chain_eg, need_model_derivative,
                              nnet_config.store_component_stats,
                              use_xent_regularization, need_model_derivative,
                              &request);
   std::shared_ptr<const NnetComputation> computation = compiler_.Compile(request);
 
   if (nnet_config.backstitch_training_scale > 0.0 && num_minibatches_processed_
       % nnet_config.backstitch_training_interval ==
       srand_seed_ % nnet_config.backstitch_training_interval) {
     // backstitch training is incompatible with momentum > 0
     KALDI_ASSERT(nnet_config.momentum == 0.0);
     FreezeNaturalGradient(true, delta_nnet_);
     bool is_backstitch_step1 = true;
     srand(srand_seed_ + num_minibatches_processed_);
     ResetGenerators(nnet_);
     TrainInternalBackstitch(chain_eg, *computation, is_backstitch_step1);
     FreezeNaturalGradient(false, delta_nnet_); // un-freeze natural gradient
     is_backstitch_step1 = false;
     srand(srand_seed_ + num_minibatches_processed_);
     ResetGenerators(nnet_);
     TrainInternalBackstitch(chain_eg, *computation, is_backstitch_step1);
   } else { // conventional training
     TrainInternal(chain_eg, *computation);
   }
   if (num_minibatches_processed_ == 0) {
     ConsolidateMemory(nnet_);
     ConsolidateMemory(delta_nnet_);
   }
   num_minibatches_processed_++;
 }

◆ TrainInternal()

void TrainInternal	(	const NnetChainExample &	eg,
		const NnetComputation &	computation
	)

private

Definition at line 97 of file nnet-chain-training.cc.

References NnetComputer::AcceptInputs(), kaldi::nnet3::ApplyL2Regularization(), NnetTrainerOptions::batchnorm_stats_scale, NnetTrainerOptions::compute_config, kaldi::nnet3::ConstrainOrthonormal(), NnetChainTrainer::delta_nnet_, kaldi::nnet3::GetNumNvalues(), NnetChainExample::inputs, NnetTrainerOptions::l2_regularize_factor, NnetChainTrainer::max_change_stats_, NnetTrainerOptions::max_param_change, NnetTrainerOptions::momentum, NnetChainTrainer::nnet_, NnetChainTrainingOptions::nnet_config, NVTX_RANGE, NnetChainTrainer::opts_, NnetChainTrainer::ProcessOutputs(), NnetComputer::Run(), kaldi::nnet3::ScaleBatchnormStats(), kaldi::nnet3::ScaleNnet(), and kaldi::nnet3::UpdateNnetWithMaxChange().

Referenced by NnetChainTrainer::Train().

                                                                          {
   NVTX_RANGE(__func__);
   const NnetTrainerOptions &nnet_config = opts_.nnet_config;
   // note: because we give the 1st arg (nnet_) as a pointer to the
   // constructor of 'computer', it will use that copy of the nnet to
   // store stats.
   NnetComputer computer(nnet_config.compute_config, computation,
                         nnet_, delta_nnet_);
 
   // give the inputs to the computer object.
   computer.AcceptInputs(*nnet_, eg.inputs);
   computer.Run();
 
   this->ProcessOutputs(false, eg, &computer);
   computer.Run();
 
   // If relevant, add in the part of the gradient that comes from
   // parameter-level L2 regularization.
   ApplyL2Regularization(*nnet_,
                         GetNumNvalues(eg.inputs, false) *
                         nnet_config.l2_regularize_factor,
                         delta_nnet_);
 
   // Updates the parameters of nnet
   bool success = UpdateNnetWithMaxChange(
       *delta_nnet_,
       nnet_config.max_param_change,
       1.0, 1.0 - nnet_config.momentum, nnet_,
       &max_change_stats_);
 
   // Scale down the batchnorm stats (keeps them fresh... this affects what
   // happens when we use the model with batchnorm test-mode set).
   ScaleBatchnormStats(nnet_config.batchnorm_stats_scale, nnet_);
 
   // The following will only do something if we have a LinearComponent
   // or AffineComponent with orthonormal-constraint set to a nonzero value.
   ConstrainOrthonormal(nnet_);
 
   // Scale delta_nnet
   if (success)
     ScaleNnet(nnet_config.momentum, delta_nnet_);
   else
     ScaleNnet(0.0, delta_nnet_);
 }

◆ TrainInternalBackstitch()

void TrainInternalBackstitch	(	const NnetChainExample &	eg,
		const NnetComputation &	computation,
		bool	is_backstitch_step1
	)

private

Definition at line 143 of file nnet-chain-training.cc.

References NnetComputer::AcceptInputs(), kaldi::nnet3::ApplyL2Regularization(), NnetTrainerOptions::backstitch_training_scale, NnetTrainerOptions::batchnorm_stats_scale, NnetTrainerOptions::compute_config, kaldi::nnet3::ConstrainOrthonormal(), NnetChainTrainer::delta_nnet_, kaldi::nnet3::GetNumNvalues(), NnetChainExample::inputs, NnetTrainerOptions::l2_regularize_factor, NnetChainTrainer::max_change_stats_, NnetTrainerOptions::max_param_change, NnetChainTrainer::nnet_, NnetChainTrainingOptions::nnet_config, NnetChainTrainer::opts_, NnetChainTrainer::ProcessOutputs(), NnetComputer::Run(), kaldi::nnet3::ScaleBatchnormStats(), kaldi::nnet3::ScaleNnet(), and kaldi::nnet3::UpdateNnetWithMaxChange().

Referenced by NnetChainTrainer::Train().

                                                                          {
   const NnetTrainerOptions &nnet_config = opts_.nnet_config;
   // note: because we give the 1st arg (nnet_) as a pointer to the
   // constructor of 'computer', it will use that copy of the nnet to
   // store stats.
   NnetComputer computer(nnet_config.compute_config, computation,
                         nnet_, delta_nnet_);
   // give the inputs to the computer object.
   computer.AcceptInputs(*nnet_, eg.inputs);
   computer.Run();
 
   bool is_backstitch_step2 = !is_backstitch_step1;
   this->ProcessOutputs(is_backstitch_step2, eg, &computer);
   computer.Run();
 
   BaseFloat max_change_scale, scale_adding;
   if (is_backstitch_step1) {
     // max-change is scaled by backstitch_training_scale;
     // delta_nnet is scaled by -backstitch_training_scale when added to nnet;
     max_change_scale = nnet_config.backstitch_training_scale;
     scale_adding = -nnet_config.backstitch_training_scale;
   } else {
     // max-change is scaled by 1 + backstitch_training_scale;
     // delta_nnet is scaled by 1 + backstitch_training_scale when added to nnet;
     max_change_scale = 1.0 + nnet_config.backstitch_training_scale;
     scale_adding = 1.0 + nnet_config.backstitch_training_scale;
     // If relevant, add in the part of the gradient that comes from L2
     // regularization.  It may not be optimally inefficient to do it on both
     // passes of the backstitch, like we do here, but it probably minimizes
     // any harmful interactions with the max-change.
     ApplyL2Regularization(*nnet_,
         1.0 / scale_adding * GetNumNvalues(eg.inputs, false) *
         nnet_config.l2_regularize_factor, delta_nnet_);
   }
 
   // Updates the parameters of nnet
   UpdateNnetWithMaxChange(
       *delta_nnet_, nnet_config.max_param_change,
       max_change_scale, scale_adding, nnet_,
       &max_change_stats_);
 
   if (is_backstitch_step1) {
     // The following will only do something if we have a LinearComponent or
     // AffineComponent with orthonormal-constraint set to a nonzero value. We
     // choose to do this only on the 1st backstitch step, for efficiency.
     ConstrainOrthonormal(nnet_);
   }
 
   if (!is_backstitch_step1) {
     // Scale down the batchnorm stats (keeps them fresh... this affects what
     // happens when we use the model with batchnorm test-mode set).  Do this
     // after backstitch step 2 so that the stats are scaled down before we start
     // the next minibatch.
     ScaleBatchnormStats(nnet_config.batchnorm_stats_scale, nnet_);
   }
 
   ScaleNnet(0.0, delta_nnet_);
 }