nnet-utils.h

Go to the documentation of this file.

// nnet3/nnet-utils.h

// Copyright   2015  Johns Hopkins University (author: Daniel Povey)
//             2016  Daniel Galvez
// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.

#ifndef KALDI_NNET3_NNET_UTILS_H_
#define KALDI_NNET3_NNET_UTILS_H_

#include "base/kaldi-common.h"
#include "util/kaldi-io.h"
#include "matrix/matrix-lib.h"
#include "nnet3/nnet-common.h"
#include "nnet3/nnet-component-itf.h"
#include "nnet3/nnet-descriptor.h"
#include "nnet3/nnet-computation.h"
#include "nnet3/nnet-example.h"

namespace kaldi {
namespace nnet3 {



void EvaluateComputationRequest(
    const Nnet &nnet,
    const ComputationRequest &request,
    std::vector<std::vector<bool> > *is_computable);


int32 NumOutputNodes(const Nnet &nnet);

int32 NumInputNodes(const Nnet &nnet);

void PerturbParams(BaseFloat stddev,
                   Nnet *nnet);


BaseFloat DotProduct(const Nnet &nnet1,
                     const Nnet &nnet2);

void ComponentDotProducts(const Nnet &nnet1,
                          const Nnet &nnet2,
                          VectorBase<BaseFloat> *dot_prod);

std::string PrintVectorPerUpdatableComponent(const Nnet &nnet,
                                             const VectorBase<BaseFloat> &vec);

bool IsSimpleNnet(const Nnet &nnet);

void ZeroComponentStats(Nnet *nnet);


void ComputeSimpleNnetContext(const Nnet &nnet,
                              int32 *left_context,
                              int32 *right_context);


void SetLearningRate(BaseFloat learning_rate,
                     Nnet *nnet);

void ScaleNnet(BaseFloat scale, Nnet *nnet);

void SetNnetAsGradient(Nnet *nnet);


void SetRequireDirectInput(bool b, Nnet *nnet);


void AddNnet(const Nnet &src, BaseFloat alpha, Nnet *dest);

void AddNnetComponents(const Nnet &src, const Vector<BaseFloat> &alphas,
                       BaseFloat scale, Nnet *dest);

bool NnetIsRecurrent(const Nnet &nnet);

int32 NumParameters(const Nnet &src);

void VectorizeNnet(const Nnet &src,
                   VectorBase<BaseFloat> *params);


void UnVectorizeNnet(const VectorBase<BaseFloat> &params,
                     Nnet *dest);

int32 NumUpdatableComponents(const Nnet &dest);

void FreezeNaturalGradient(bool freeze, Nnet *nnet);

void ConvertRepeatedToBlockAffine(Nnet *nnet);

std::string NnetInfo(const Nnet &nnet);

void SetDropoutProportion(BaseFloat dropout_proportion, Nnet *nnet);


bool HasBatchnorm(const Nnet &nnet);

void SetBatchnormTestMode(bool test_mode, Nnet *nnet);


void RecomputeStats(const std::vector<NnetExample> &egs, Nnet *nnet);



void SetDropoutTestMode(bool test_mode, Nnet *nnet);

void ResetGenerators(Nnet *nnet);

void FindOrphanComponents(const Nnet &nnet, std::vector<int32> *components);

void FindOrphanNodes(const Nnet &nnet, std::vector<int32> *nodes);



struct CollapseModelConfig {
  bool collapse_dropout;  // dropout then affine/conv.
  bool collapse_batchnorm;  // batchnorm then affine.
  bool collapse_affine;  // affine or fixed-affine then affine.
  bool collapse_scale;  // affine then fixed-scale.
  CollapseModelConfig(): collapse_dropout(false),
                         collapse_batchnorm(false),
                         collapse_affine(true),
                         collapse_scale(true) { }
};

void CollapseModel(const CollapseModelConfig &config,
                   Nnet *nnet);

void ReadEditConfig(std::istream &config_file, Nnet *nnet);

bool UpdateNnetWithMaxChange(const Nnet &delta_nnet,
                             BaseFloat max_param_change,
                             BaseFloat max_change_scale,
                             BaseFloat scale, Nnet *nnet,
                             std::vector<int32> *
                             num_max_change_per_component_applied,
                             int32 *num_max_change_global_applied);

struct MaxChangeStats;

// This overloaded version of UpdateNnetWithMaxChange() is a convenience
// wrapper for when you have a MaxChangeStats object to keep track
// of how many times the max-change was applied.  See documentation above.
bool UpdateNnetWithMaxChange(const Nnet &delta_nnet,
                             BaseFloat max_param_change,
                             BaseFloat max_change_scale,
                             BaseFloat scale, Nnet *nnet,
                             MaxChangeStats *stats);


void ApplyL2Regularization(const Nnet &nnet,
                           BaseFloat l2_regularize_scale,
                           Nnet *delta_nnet);


void ScaleBatchnormStats(BaseFloat batchnorm_stats_scale,
                         Nnet *nnet);


void ConstrainOrthonormal(Nnet *nnet);


void ConsolidateMemory(Nnet *nnet);

int32 GetNumNvalues(const std::vector<NnetIo> &io_vec,
                    bool exhaustive);


struct MaxChangeStats {
  int32 num_max_change_global_applied;
  int32 num_minibatches_processed;
  std::vector<int32> num_max_change_per_component_applied;

  MaxChangeStats(const Nnet &nnet):
      num_max_change_global_applied(0),
      num_minibatches_processed(0),
      num_max_change_per_component_applied(NumUpdatableComponents(nnet), 0) { }

  // Prints the max-change stats.  Usually will be called at the end
  // of the program.  The nnet is only needed for structural information,
  // to work out the component names.
  void Print(const Nnet &nnet) const;
};



} // namespace nnet3
} // namespace kaldi

#endif