This class is for single-threaded discriminative training of neural nets. More...

#include <nnet-discriminative-training.h>

Collaboration diagram for NnetDiscriminativeTrainer:

[legend]

Public Member Functions
	NnetDiscriminativeTrainer (const NnetDiscriminativeOptions &config, const TransitionModel &tmodel, const VectorBase< BaseFloat > &priors, Nnet *nnet)

void	Train (const NnetDiscriminativeExample &eg)

bool	PrintTotalStats () const

	~NnetDiscriminativeTrainer ()

Private Member Functions
void	ProcessOutputs (const NnetDiscriminativeExample &eg, NnetComputer *computer)

Private Attributes
const NnetDiscriminativeOptions	opts_

const TransitionModel &	tmodel_

CuVector< BaseFloat >	log_priors_

Nnet *	nnet_

Nnet *	delta_nnet_

CachingOptimizingCompiler	compiler_

int32	num_minibatches_processed_

unordered_map< std::string, DiscriminativeObjectiveFunctionInfo, StringHasher >	objf_info_

Detailed Description

This class is for single-threaded discriminative training of neural nets.

Definition at line 87 of file nnet-discriminative-training.h.

Constructor & Destructor Documentation

◆ NnetDiscriminativeTrainer()

NnetDiscriminativeTrainer	(	const NnetDiscriminativeOptions &	config,
		const TransitionModel &	tmodel,
		const VectorBase< BaseFloat > &	priors,
		Nnet *	nnet
	)

Definition at line 27 of file nnet-discriminative-training.cc.

References NnetDiscriminativeTrainer::compiler_, Nnet::Copy(), NnetDiscriminativeTrainer::delta_nnet_, KALDI_ASSERT, KALDI_LOG, KALDI_WARN, NnetDiscriminativeTrainer::log_priors_, NnetTrainerOptions::max_param_change, NnetTrainerOptions::momentum, NnetDiscriminativeTrainer::nnet_, NnetDiscriminativeOptions::nnet_config, Input::Open(), NnetTrainerOptions::read_cache, CachingOptimizingCompiler::ReadCache(), kaldi::nnet3::ScaleNnet(), Input::Stream(), NnetTrainerOptions::zero_component_stats, and kaldi::nnet3::ZeroComponentStats().

                                               :
     opts_(opts), tmodel_(tmodel), log_priors_(priors),
     nnet_(nnet),
     compiler_(*nnet, opts_.nnet_config.optimize_config),
     num_minibatches_processed_(0) {
   if (opts.nnet_config.zero_component_stats)
     ZeroComponentStats(nnet);
   if (opts.nnet_config.momentum == 0.0 &&
       opts.nnet_config.max_param_change == 0.0) {
     delta_nnet_= NULL;
   } else {
     KALDI_ASSERT(opts.nnet_config.momentum >= 0.0 &&
                  opts.nnet_config.max_param_change >= 0.0);
     delta_nnet_ = nnet_->Copy();
     ScaleNnet(0.0, delta_nnet_);
   }
   if (opts.nnet_config.read_cache != "") {
     bool binary;
     Input ki;
     if (ki.Open(opts.nnet_config.read_cache, &binary)) {
       compiler_.ReadCache(ki.Stream(), binary);
       KALDI_LOG << "Read computation cache from "
                 << opts.nnet_config.read_cache;
     } else {
       KALDI_WARN << "Could not open cached computation. "
                     "Probably this is the first training iteration.";
     }
   }
   log_priors_.ApplyLog();
 }

◆ ~NnetDiscriminativeTrainer()

~NnetDiscriminativeTrainer ( )

Definition at line 264 of file nnet-discriminative-training.cc.

References NnetTrainerOptions::binary_write_cache, NnetDiscriminativeTrainer::compiler_, NnetDiscriminativeTrainer::delta_nnet_, NnetDiscriminativeOptions::nnet_config, NnetDiscriminativeTrainer::opts_, Output::Stream(), NnetTrainerOptions::write_cache, and CachingOptimizingCompiler::WriteCache().

                                                       {
   delete delta_nnet_;
 
   if (opts_.nnet_config.write_cache != "") {
     Output ko(opts_.nnet_config.write_cache, opts_.nnet_config.binary_write_cache);
     compiler_.WriteCache(ko.Stream(), opts_.nnet_config.binary_write_cache);
   }
 }

Member Function Documentation

◆ PrintTotalStats()

bool PrintTotalStats ( ) const

Definition at line 191 of file nnet-discriminative-training.cc.

References DiscriminativeOptions::criterion, NnetDiscriminativeOptions::discriminative_config, NnetDiscriminativeTrainer::objf_info_, NnetDiscriminativeTrainer::opts_, and DiscriminativeObjectiveFunctionInfo::PrintTotalStats().

Referenced by main().

                                                       {
   unordered_map<std::string, DiscriminativeObjectiveFunctionInfo,
     StringHasher>::const_iterator
       iter = objf_info_.begin(),
       end = objf_info_.end();
   bool ans = false;
   for (; iter != end; ++iter) {
     const std::string &name = iter->first;
     const DiscriminativeObjectiveFunctionInfo &info = iter->second;
     bool ret = info.PrintTotalStats(name, opts_.discriminative_config.criterion);
     ans = ans || ret;
   }
 
   return ans;
 }

◆ ProcessOutputs()

void ProcessOutputs	(	const NnetDiscriminativeExample &	eg,
		NnetComputer *	computer
	)

private

Definition at line 109 of file nnet-discriminative-training.cc.

References NnetComputer::AcceptInput(), NnetDiscriminativeOptions::apply_deriv_weights, kaldi::discriminative::ComputeDiscriminativeObjfAndDeriv(), DiscriminativeOptions::criterion, NnetDiscriminativeSupervision::deriv_weights, NnetDiscriminativeOptions::discriminative_config, Nnet::GetNodeIndex(), NnetComputer::GetOutput(), Nnet::IsOutputNode(), KALDI_ERR, kaldi::kTrans, kaldi::kUndefined, NnetDiscriminativeTrainer::log_priors_, CuMatrixBase< Real >::MulRowsVec(), NnetDiscriminativeSupervision::name, NnetDiscriminativeTrainer::nnet_, NnetDiscriminativeOptions::nnet_config, NnetDiscriminativeTrainer::num_minibatches_processed_, NnetDiscriminativeTrainer::objf_info_, NnetDiscriminativeTrainer::opts_, NnetDiscriminativeExample::outputs, NnetTrainerOptions::print_interval, CuMatrix< Real >::Resize(), CuMatrixBase< Real >::Scale(), NnetDiscriminativeSupervision::supervision, NnetDiscriminativeTrainer::tmodel_, DiscriminativeObjectiveInfo::tot_objf, DiscriminativeObjectiveInfo::tot_t_weighted, kaldi::TraceMatMat(), and DiscriminativeOptions::xent_regularize.

Referenced by NnetDiscriminativeTrainer::Train().

                                                                        {
   // normally the eg will have just one output named 'output', but
   // we don't assume this.
   std::vector<NnetDiscriminativeSupervision>::const_iterator iter = eg.outputs.begin(),
       end = eg.outputs.end();
   for (; iter != end; ++iter) {
     const NnetDiscriminativeSupervision &sup = *iter;
     int32 node_index = nnet_->GetNodeIndex(sup.name);
     if (node_index < 0 ||
         !nnet_->IsOutputNode(node_index))
       KALDI_ERR << "Network has no output named " << sup.name;
 
     const CuMatrixBase<BaseFloat> &nnet_output = computer->GetOutput(sup.name);
 
     CuMatrix<BaseFloat> nnet_output_deriv(nnet_output.NumRows(),
                                           nnet_output.NumCols(),
                                           kUndefined);
 
     bool use_xent = (opts_.discriminative_config.xent_regularize != 0.0);
     std::string xent_name = sup.name + "-xent";  // typically "output-xent".
     CuMatrix<BaseFloat> xent_deriv;
     if (use_xent)
       xent_deriv.Resize(nnet_output.NumRows(), nnet_output.NumCols(),
                                kUndefined);
 
     discriminative::DiscriminativeObjectiveInfo stats(opts_.discriminative_config);
 
     if (objf_info_.count(sup.name) == 0) {
       objf_info_[sup.name].stats.Configure(opts_.discriminative_config);
       objf_info_[sup.name].stats.Reset();
     }
 
     ComputeDiscriminativeObjfAndDeriv(opts_.discriminative_config,
                                       tmodel_, log_priors_,
                                       sup.supervision, nnet_output,
                                       &stats,
                                       &nnet_output_deriv,
                                       (use_xent ? &xent_deriv : NULL));
 
     if (use_xent) {
       // this block computes the cross-entropy objective.
       const CuMatrixBase<BaseFloat> &xent_output = computer->GetOutput(xent_name);
       // at this point, xent_deriv is posteriors derived from the numerator
       // computation.  note, xent_objf has a factor of '.supervision.weight'
       BaseFloat xent_objf = TraceMatMat(xent_output, xent_deriv, kTrans);
       if (xent_objf != xent_objf) {
         BaseFloat default_objf = -10;
         xent_objf = default_objf;
       }
 
       discriminative::DiscriminativeObjectiveInfo xent_stats;
       xent_stats.tot_t_weighted = stats.tot_t_weighted;
       xent_stats.tot_objf = xent_objf;
 
       objf_info_[xent_name].UpdateStats(xent_name, "xent",
                                         opts_.nnet_config.print_interval,
                                         num_minibatches_processed_, xent_stats);
     }
 
     if (opts_.apply_deriv_weights && sup.deriv_weights.Dim() != 0) {
       CuVector<BaseFloat> cu_deriv_weights(sup.deriv_weights);
       nnet_output_deriv.MulRowsVec(cu_deriv_weights);
       if (use_xent)
         xent_deriv.MulRowsVec(cu_deriv_weights);
     }
 
     computer->AcceptInput(sup.name, &nnet_output_deriv);
 
     objf_info_[sup.name].UpdateStats(sup.name, opts_.discriminative_config.criterion,
                                      opts_.nnet_config.print_interval,
                                      num_minibatches_processed_++,
                                      stats);
 
     if (use_xent) {
       xent_deriv.Scale(opts_.discriminative_config.xent_regularize);
       computer->AcceptInput(xent_name, &xent_deriv);
     }
   }
 }

◆ Train()

void Train ( const NnetDiscriminativeExample & eg )

Definition at line 63 of file nnet-discriminative-training.cc.

References NnetComputer::AcceptInputs(), kaldi::nnet3::AddNnet(), CachingOptimizingCompiler::Compile(), NnetDiscriminativeTrainer::compiler_, NnetTrainerOptions::compute_config, NnetDiscriminativeTrainer::delta_nnet_, NnetDiscriminativeOptions::discriminative_config, kaldi::nnet3::DotProduct(), kaldi::nnet3::GetDiscriminativeComputationRequest(), NnetDiscriminativeExample::inputs, KALDI_LOG, KALDI_WARN, NnetTrainerOptions::max_param_change, NnetTrainerOptions::momentum, NnetDiscriminativeTrainer::nnet_, NnetDiscriminativeOptions::nnet_config, NnetDiscriminativeTrainer::opts_, NnetDiscriminativeTrainer::ProcessOutputs(), NnetComputer::Run(), kaldi::nnet3::ScaleNnet(), NnetTrainerOptions::store_component_stats, and DiscriminativeOptions::xent_regularize.

Referenced by main().

                                                                          {
   bool need_model_derivative = true;
   const NnetTrainerOptions &nnet_config = opts_.nnet_config;
   bool use_xent_regularization = (opts_.discriminative_config.xent_regularize != 0.0);
   ComputationRequest request;
   GetDiscriminativeComputationRequest(*nnet_, eg, need_model_derivative,
                                       nnet_config.store_component_stats,
                                       use_xent_regularization,
                                       need_model_derivative,
                                       &request);
   std::shared_ptr<const NnetComputation> computation = compiler_.Compile(request);
 
   NnetComputer computer(nnet_config.compute_config, *computation,
                         *nnet_,
                         (delta_nnet_ == NULL ? nnet_ : delta_nnet_));
   // give the inputs to the computer object.
   computer.AcceptInputs(*nnet_, eg.inputs);
   computer.Run();
 
   this->ProcessOutputs(eg, &computer);
   computer.Run();
 
   if (delta_nnet_ != NULL) {
     BaseFloat scale = (1.0 - nnet_config.momentum);
     if (nnet_config.max_param_change != 0.0) {
       BaseFloat param_delta =
           std::sqrt(DotProduct(*delta_nnet_, *delta_nnet_)) * scale;
       if (param_delta > nnet_config.max_param_change) {
         if (param_delta - param_delta != 0.0) {
           KALDI_WARN << "Infinite parameter change, will not apply.";
           ScaleNnet(0.0, delta_nnet_);
         } else {
           scale *= nnet_config.max_param_change / param_delta;
           KALDI_LOG << "Parameter change too big: " << param_delta << " > "
                     << "--max-param-change=" << nnet_config.max_param_change
                     << ", scaling by "
                     << nnet_config.max_param_change / param_delta;
         }
       }
     }
     AddNnet(*delta_nnet_, scale, nnet_);
     ScaleNnet(nnet_config.momentum, delta_nnet_);
   }
 }