NnetRescaler Class Reference

Collaboration diagram for NnetRescaler:

Public Member Functions
	NnetRescaler (const NnetRescaleConfig &config, const std::vector< NnetExample > &examples, Nnet *nnet)
void	Rescale ()

Private Member Functions
void	FormatInput (const std::vector< NnetExample > &data, CuMatrix< BaseFloat > *input)
	takes the input and formats as a single matrix, in forward_data_[0]. More...
void	RescaleComponent (int32 c, int32 num_chunks, CuMatrixBase< BaseFloat > *cur_data_in, CuMatrix< BaseFloat > *next_data)
void	ComputeRelevantIndexes ()
BaseFloat	GetTargetAvgDeriv (int32 c)

Private Attributes
const NnetRescaleConfig &	config_
const std::vector< NnetExample > &	examples_
Nnet *	nnet_
std::vector< ChunkInfo >	chunk_info_out_
std::set< int32 >	relevant_indexes_

Detailed Description

Definition at line 26 of file rescale-nnet.cc.

Constructor & Destructor Documentation

NnetRescaler()

NnetRescaler ( const NnetRescaleConfig &  config, const std::vector< NnetExample > &  examples, Nnet *  nnet  )

inline

Definition at line 28 of file rescale-nnet.cc.

References NnetRescaler::ComputeRelevantIndexes(), NnetRescaler::FormatInput(), NnetRescaler::GetTargetAvgDeriv(), NnetRescaler::Rescale(), and NnetRescaler::RescaleComponent().

                          :
      config_(config), examples_(examples), nnet_(nnet) {}

Member Function Documentation

ComputeRelevantIndexes()

void ComputeRelevantIndexes ( )

private

Definition at line 89 of file rescale-nnet.cc.

References Nnet::GetComponent(), NnetRescaler::nnet_, Nnet::NumComponents(), and NnetRescaler::relevant_indexes_.

Referenced by NnetRescaler::NnetRescaler(), and NnetRescaler::Rescale().

                                          {
  for (int32 c = 0; c + 1 < nnet_->NumComponents(); c++)
    if (dynamic_cast<AffineComponent*>(&nnet_->GetComponent(c)) != NULL &&
        (dynamic_cast<NonlinearComponent*>(&nnet_->GetComponent(c+1)) != NULL &&
         dynamic_cast<SoftmaxComponent*>(&nnet_->GetComponent(c+1)) == NULL))
      relevant_indexes_.insert(c);
}

FormatInput()

void FormatInput ( const std::vector< NnetExample > &  data, CuMatrix< BaseFloat > *  input  )

private

takes the input and formats as a single matrix, in forward_data_[0].

Definition at line 56 of file rescale-nnet.cc.

References NnetRescaler::chunk_info_out_, Nnet::ComputeChunkInfo(), CuMatrixBase< Real >::CopyFromMat(), CuMatrixBase< Real >::CopyRowsFromVec(), Nnet::InputDim(), KALDI_ASSERT, Nnet::LeftContext(), NnetRescaler::nnet_, CuMatrix< Real >::Resize(), and Nnet::RightContext().

Referenced by NnetRescaler::NnetRescaler(), and NnetRescaler::Rescale().

                                                           {
  KALDI_ASSERT(data.size() > 0);
  int32 num_splice = nnet_->LeftContext() + 1 + nnet_->RightContext();
  KALDI_ASSERT(data[0].input_frames.NumRows() == num_splice);

  int32 feat_dim = data[0].input_frames.NumCols(),
         spk_dim = data[0].spk_info.Dim(),
         tot_dim = feat_dim + spk_dim; // we append these at the neural net
                                       // input... note, spk_dim might be 0.
  KALDI_ASSERT(tot_dim == nnet_->InputDim());
  int32 num_chunks = data.size();

  input->Resize(num_splice * num_chunks,
                tot_dim);
  for (int32 chunk = 0; chunk < num_chunks; chunk++) {
    CuSubMatrix<BaseFloat> dest(*input,
                                chunk * num_splice, num_splice,
                                0, feat_dim);
    Matrix<BaseFloat> src(data[chunk].input_frames);
    dest.CopyFromMat(src);
    if (spk_dim != 0) {
      CuSubMatrix<BaseFloat> spk_dest(*input,
                                      chunk * num_splice, num_splice,
                                      feat_dim, spk_dim);
      spk_dest.CopyRowsFromVec(data[chunk].spk_info);
    }
  }
  // TODO : filter out the unnecessary rows from the input
  nnet_->ComputeChunkInfo(num_splice, num_chunks, &chunk_info_out_);

}

GetTargetAvgDeriv()

BaseFloat GetTargetAvgDeriv ( int32  c )

private

Definition at line 98 of file rescale-nnet.cc.

References NnetRescaler::config_, Nnet::GetComponent(), KALDI_ASSERT, KALDI_ERR, NnetRescaler::nnet_, NnetRescaler::relevant_indexes_, NnetRescaleConfig::target_avg_deriv, NnetRescaleConfig::target_first_layer_avg_deriv, and NnetRescaleConfig::target_last_layer_avg_deriv.

Referenced by NnetRescaler::NnetRescaler(), and NnetRescaler::RescaleComponent().

                                                 {
  KALDI_ASSERT(relevant_indexes_.count(c) == 1);
  BaseFloat factor;
  if (dynamic_cast<SigmoidComponent*>(&(nnet_->GetComponent(c + 1))) != NULL)
    factor = 0.25;
  else if (dynamic_cast<TanhComponent*>(&(nnet_->GetComponent(c + 1))) != NULL)
    factor = 1.0;
  else
    KALDI_ERR << "This type of nonlinear component is not handled: index  " << c;
  
  int32 last_c = *std::max_element(relevant_indexes_.begin(), relevant_indexes_.end()),
      first_c = *std::min_element(relevant_indexes_.begin(), relevant_indexes_.end());
  if (c == first_c)
    return factor * config_.target_first_layer_avg_deriv;
  else if (c == last_c)
    return factor * config_.target_last_layer_avg_deriv;
  else
    return factor * config_.target_avg_deriv;
}

Rescale()

void Rescale

(

)

Definition at line 200 of file rescale-nnet.cc.

References NnetRescaler::chunk_info_out_, NnetRescaler::ComputeRelevantIndexes(), NnetRescaler::examples_, NnetRescaler::FormatInput(), Nnet::GetComponent(), NnetRescaler::nnet_, Nnet::NumComponents(), Component::Propagate(), NnetRescaler::relevant_indexes_, NnetRescaler::RescaleComponent(), and CuMatrix< Real >::Swap().

Referenced by NnetRescaler::NnetRescaler(), and kaldi::nnet2::RescaleNnet().

                           {
  ComputeRelevantIndexes(); // set up relevant_indexes_.
  CuMatrix<BaseFloat> cur_data, next_data;
  FormatInput(examples_, &cur_data);
  int32 num_chunks = examples_.size();
  for (int32 c = 0; c < nnet_->NumComponents(); c++) {
    Component &component = nnet_->GetComponent(c);
    if (relevant_indexes_.count(c - 1) == 1) {
      // the following function call also appropriately sets "next_data"
      // after doing the rescaling
      RescaleComponent(c - 1, num_chunks, &cur_data, &next_data);
    } else {
      component.Propagate(chunk_info_out_[c], chunk_info_out_[c+1], cur_data, &next_data);
    }
    cur_data.Swap(&next_data);
  }
}

RescaleComponent()

void RescaleComponent ( int32  c, int32  num_chunks, CuMatrixBase< BaseFloat > *  cur_data_in, CuMatrix< BaseFloat > *  next_data  )

private

Definition at line 121 of file rescale-nnet.cc.

References Component::Backprop(), NnetRescaler::chunk_info_out_, NnetRescaler::config_, NnetRescaleConfig::delta, Nnet::GetComponent(), NnetRescaler::GetTargetAvgDeriv(), KALDI_ASSERT, KALDI_ERR, KALDI_LOG, KALDI_VLOG, NnetRescaleConfig::max_change, NnetRescaleConfig::min_change, NnetRescaler::nnet_, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), Component::Propagate(), UpdatableComponent::Scale(), and CuMatrixBase< Real >::Sum().

Referenced by NnetRescaler::NnetRescaler(), and NnetRescaler::Rescale().

                                    {
  int32 rows = cur_data_in->NumRows(), cols = cur_data_in->NumCols();
  // Only handle sigmoid or tanh here.
  if (dynamic_cast<SigmoidComponent*>(&(nnet_->GetComponent(c + 1))) == NULL &&
      dynamic_cast<TanhComponent*>(&(nnet_->GetComponent(c + 1))) == NULL)
    KALDI_ERR << "This type of nonlinear component is not handled: index  " << c;
  KALDI_ASSERT(chunk_info_out_[0].NumChunks() == num_chunks); //TODO verify how this component can be used
                                                             // rewrite the
                                                             // chunk_info_out_
                                                             // computation
  // the nonlinear component:
  NonlinearComponent &nc =
      *(dynamic_cast<NonlinearComponent*>(&(nnet_->GetComponent(c + 1))));
  ChunkInfo in_info, out_info;
  in_info = chunk_info_out_[c+1];
  out_info = chunk_info_out_[c+2];

  BaseFloat orig_avg_deriv, target_avg_deriv = GetTargetAvgDeriv(c);
  BaseFloat cur_scaling = 1.0; // current rescaling factor (on input).
  int32 num_iters = 10;
  
  CuMatrix<BaseFloat> cur_data(*cur_data_in),
      ones(rows, cols), in_deriv(rows, cols);
      
  ones.Set(1.0);
  nc.Propagate(in_info, out_info, cur_data, next_data);
  nc.Backprop(in_info, out_info, cur_data, *next_data, ones, NULL, &in_deriv);
  BaseFloat cur_avg_deriv;
  cur_avg_deriv = in_deriv.Sum() / (rows * cols);
  orig_avg_deriv = cur_avg_deriv;
  for (int32 iter = 0; iter < num_iters; iter++) {
    // We already have "cur_avg_deriv"; perturb the scale and compute
    // the next avg_deriv, so we can see how it changes with the scale.
    cur_data.CopyFromMat(*cur_data_in);
    cur_data.Scale(cur_scaling + config_.delta);
    nc.Propagate(in_info, out_info, cur_data, next_data);
    nc.Backprop(in_info, out_info, cur_data, *next_data, ones, NULL, &in_deriv);
    BaseFloat next_avg_deriv = in_deriv.Sum() / (rows * cols);
    KALDI_ASSERT(next_avg_deriv < cur_avg_deriv);
    // "gradient" is how avg_deriv changes as we change the scale.
    // should be negative.
    BaseFloat gradient = (next_avg_deriv - cur_avg_deriv) / config_.delta;
    KALDI_ASSERT(gradient < 0.0);
    BaseFloat proposed_change = (target_avg_deriv - cur_avg_deriv) / gradient;
    KALDI_VLOG(2) << "cur_avg_deriv = " << cur_avg_deriv << ", target_avg_deriv = "
                  << target_avg_deriv << ", gradient = " << gradient
                  << ", proposed_change " << proposed_change; 
    // Limit size of proposed change in "cur_scaling", to ensure stability.
    if (fabs(proposed_change / cur_scaling) > config_.max_change)
      proposed_change = cur_scaling * config_.max_change *
          (proposed_change > 0.0 ? 1.0 : -1.0);
    cur_scaling += proposed_change;
    
    cur_data.CopyFromMat(*cur_data_in);
    cur_data.Scale(cur_scaling);
    nc.Propagate(in_info, out_info, cur_data, next_data);
    nc.Backprop(in_info, out_info, cur_data, *next_data, ones, NULL, &in_deriv);
    cur_avg_deriv = in_deriv.Sum() / (rows * cols);
    if (fabs(proposed_change) < config_.min_change) break; // Terminate the
    // optimization
  }
  UpdatableComponent *uc = dynamic_cast<UpdatableComponent*>(
      &nnet_->GetComponent(c));
  KALDI_ASSERT(uc != NULL);
  uc->Scale(cur_scaling); // scale the parameters of the previous
  // AffineComponent.
  
  KALDI_LOG << "For component " << c << ", scaling parameters by "
            << cur_scaling << "; average "
            << "derivative changed from " << orig_avg_deriv << " to "
            << cur_avg_deriv << "; target was " << target_avg_deriv;
}

Member Data Documentation

chunk_info_out_

std::vector<ChunkInfo> chunk_info_out_

private

Definition at line 50 of file rescale-nnet.cc.

Referenced by NnetRescaler::FormatInput(), NnetRescaler::Rescale(), and NnetRescaler::RescaleComponent().

config_

const NnetRescaleConfig& config_

private

Definition at line 47 of file rescale-nnet.cc.

Referenced by NnetRescaler::GetTargetAvgDeriv(), and NnetRescaler::RescaleComponent().

examples_

const std::vector<NnetExample>& examples_

private

Definition at line 48 of file rescale-nnet.cc.

Referenced by NnetRescaler::Rescale().

nnet_

Nnet* nnet_

private

Definition at line 49 of file rescale-nnet.cc.

Referenced by NnetRescaler::ComputeRelevantIndexes(), NnetRescaler::FormatInput(), NnetRescaler::GetTargetAvgDeriv(), NnetRescaler::Rescale(), and NnetRescaler::RescaleComponent().

relevant_indexes_

std::set<int32> relevant_indexes_

private

Definition at line 51 of file rescale-nnet.cc.

Referenced by NnetRescaler::ComputeRelevantIndexes(), NnetRescaler::GetTargetAvgDeriv(), and NnetRescaler::Rescale().

---

The documentation for this class was generated from the following file:

• nnet2/rescale-nnet.cc