nnet-component-itf.h

Go to the documentation of this file.

// nnet3/nnet-component-itf.h

// Copyright      2015  Johns Hopkins University (author: Daniel Povey)
//                2015  Guoguo Chen
//                2015  Xiaohui Zhang

// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.

#ifndef KALDI_NNET3_NNET_COMPONENT_ITF_H_
#define KALDI_NNET3_NNET_COMPONENT_ITF_H_

#include <iostream>
#include "nnet3/nnet-common.h"
#include "nnet3/nnet-parse.h"
#include "base/kaldi-error.h"

namespace kaldi {
namespace nnet3 {

// enum used to store various binary component properties.
// We give it a name ComponentProperties, but don't use this
// type for the bitmasks: instead use int32 for this type, e.g.
// int32 properties = kSimpleComponent|kBackpropNeedsOutput.
enum ComponentProperties {
  kSimpleComponent = 0x001,  // true if number of rows of input equals number of rows
                             // of output and this component doesn't care about the indexes
                             // (i.e. it maps each row of input to each row of output without
                             // regard to the index values).  Will normally be true.
  kUpdatableComponent = 0x002,  // true if the component has parameters that can
                                // be updated.  Components that return this flag
                                // must be dynamic_castable to type
                                // UpdatableComponent (but components of type
                                // UpdatableComponent do not have to return this
                                // flag, e.g.  if this instance is not really
                                // updatable).
  kPropagateInPlace = 0x004,  // true if we can do the propagate operation in-place
                              // (input and output matrices are the same).
                              // Note: if doing backprop, you'd also need to check
                              // that the kBackpropNeedsInput property is not true.
  kPropagateAdds = 0x008,  // true if the Propagate function adds to, rather
                           // than setting, its output, for non-in-place
                           // propagation.  The Component chooses whether to add
                           // or set, and the calling code has to accommodate
                           // it.
  kReordersIndexes = 0x010,  // true if the ReorderIndexes function might reorder
                             // the indexes (otherwise we can skip calling it).
                             // Must not be set for simple components.
  kBackpropAdds = 0x020,   // true if the Backprop function adds to, rather than
                           // setting, the "in_deriv" output for non-in-place
                           // backprop.  The Component chooses whether to add or
                           // set, and the calling code has to accommodate it.
  kBackpropNeedsInput = 0x040,  // true if backprop operation needs access to
                                // forward-pass input.
  kBackpropNeedsOutput = 0x080,  // true if backprop operation needs access to
                                 // forward-pass output (e.g. true for Sigmoid).
  kBackpropInPlace = 0x100,   // true if we can do the backprop operation in-place
                             // (input and output matrices may be the same).
  kStoresStats = 0x200,      // true if the StoreStats operation stores
                             // statistics e.g. on average node activations and
                             // derivatives of the nonlinearity, (as it does for
                             // Tanh, Sigmoid, ReLU and Softmax).
  kInputContiguous = 0x400,  // true if the component requires its input data (and
                              // input derivatives) to have Stride()== NumCols().
  kOutputContiguous = 0x800,  // true if the component requires its input data (and
                               // output derivatives) to have Stride()== NumCols().
  kUsesMemo = 0x1000,  // true if the component returns a void* pointer from its
                       // Propagate() function that needs to be passed into the
                       // corresponding Backprop function.
  kRandomComponent = 0x2000   // true if the component has some kind of
                              // randomness, like DropoutComponent (these should
                              // inherit from class RandomComponent.
};


// This is a base class for a helper-class of class Component, which is used to
// store any pre-computed indexes it needs for its forward and backward
// computations.  For components which are not "Simple" components (i.e. the
// kSimpleComponent property is false), and which may therefore "care" about
// which index the input and output matrix's rows represent (i.e. about
// which "struct Index" each row corresponds to), their CreateIndexes() function
// will be called prior to Propagate() and Backprop(), to create an object which
// must be a child class of class ComponentPrecomputedIndexes, where they
// can store any indexes that they need.
class ComponentPrecomputedIndexes {
 public:
  virtual ComponentPrecomputedIndexes *Copy() const = 0;
  virtual void Write(std::ostream &os, bool binary) const = 0;
  virtual void Read(std::istream &os, bool binary) = 0;
  virtual std::string Type() const = 0;
  static ComponentPrecomputedIndexes* ReadNew(std::istream &is, bool binary);
  // cpi stands for component_precomputed_indexes
  static ComponentPrecomputedIndexes* NewComponentPrecomputedIndexesOfType(
                                           const std::string &cpi_type);
  virtual ~ComponentPrecomputedIndexes() { }
};


class IndexSet;  // Forward declaration; declared in nnet-computation-graph.h.

class Component {
 public:
  virtual void* Propagate(const ComponentPrecomputedIndexes *indexes,
                          const CuMatrixBase<BaseFloat> &in,
                          CuMatrixBase<BaseFloat> *out) const = 0;

  virtual void Backprop(const std::string &debug_info,
                        const ComponentPrecomputedIndexes *indexes,
                        const CuMatrixBase<BaseFloat> &in_value,
                        const CuMatrixBase<BaseFloat> &out_value,
                        const CuMatrixBase<BaseFloat> &out_deriv,
                        void *memo,
                        Component *to_update, // may be NULL; may be identical
                                              // to "this" or different.
                        CuMatrixBase<BaseFloat> *in_deriv) const = 0;

  virtual void StoreStats(const CuMatrixBase<BaseFloat> &in_value,
                          const CuMatrixBase<BaseFloat> &out_value,
                          void *memo) { }

  virtual void ZeroStats() { }



  virtual void GetInputIndexes(const MiscComputationInfo &misc_info,
                               const Index &output_index,
                               std::vector<Index> *desired_indexes) const;

  virtual bool IsComputable(const MiscComputationInfo &misc_info,
                            const Index &output_index,
                            const IndexSet &input_index_set,
                            std::vector<Index> *used_inputs) const;

  virtual void ReorderIndexes(std::vector<Index> *input_indexes,
                              std::vector<Index> *output_indexes) const {}



  virtual ComponentPrecomputedIndexes* PrecomputeIndexes(
      const MiscComputationInfo &misc_info,
      const std::vector<Index> &input_indexes,
      const std::vector<Index> &output_indexes,
      bool need_backprop) const { return NULL;  }


  virtual std::string Type() const = 0;

  virtual void InitFromConfig(ConfigLine *cfl) = 0;

  virtual int32 InputDim() const = 0;

  virtual int32 OutputDim() const = 0;

  virtual int32 Properties() const = 0;

  static Component* ReadNew(std::istream &is, bool binary);

  virtual Component* Copy() const = 0;

  static Component *NewComponentOfType(const std::string &type);

  virtual void Read(std::istream &is, bool binary) = 0;

  virtual void Write(std::ostream &os, bool binary) const = 0;

  virtual std::string Info() const;

  virtual void Scale(BaseFloat scale) {};

  virtual void Add(BaseFloat alpha, const Component &other) {};

  virtual void DeleteMemo(void *memo) const { KALDI_ASSERT(memo == NULL); }

  virtual void ConsolidateMemory() { }

  Component() { }

  virtual ~Component() { }

 private:
  KALDI_DISALLOW_COPY_AND_ASSIGN(Component);
};


class RandomComponent: public Component {
 public:
  // This function is required in testing code and in other places we need
  // consistency in the random number generation (e.g. when optimizing
  // validation-set performance), but check where else we call srand().  You'll
  // need to call srand prior to making this call.
  void ResetGenerator() { random_generator_.SeedGpu(); }

  // Call this with 'true' to set 'test mode' where the behavior is different
  // from normal mode.
  void SetTestMode(bool test_mode) { test_mode_ = test_mode; }

  RandomComponent(): test_mode_(false) { }

  RandomComponent(const RandomComponent &other):
      test_mode_(other.test_mode_) {}
 protected:
  CuRand<BaseFloat> random_generator_;

  // This is true if we want a different behavior for inference from  that for
  // training.
  bool test_mode_;
};

class UpdatableComponent: public Component {
 public:
  UpdatableComponent(const UpdatableComponent &other);

  // If these defaults are changed, the defaults in
  // InitLearningRatesFromConfig() should be changed too.
  UpdatableComponent(): learning_rate_(0.001), learning_rate_factor_(1.0),
                        l2_regularize_(0.0), is_gradient_(false),
                        max_change_(0.0) { }

  virtual ~UpdatableComponent() { }

  virtual BaseFloat DotProduct(const UpdatableComponent &other) const = 0;

  virtual void PerturbParams(BaseFloat stddev) = 0;

  virtual void SetUnderlyingLearningRate(BaseFloat lrate) {
    learning_rate_ = lrate * learning_rate_factor_;
  }

  virtual void SetActualLearningRate(BaseFloat lrate) { learning_rate_ = lrate; }

  virtual void SetAsGradient() { learning_rate_ = 1.0; is_gradient_ = true; }

  virtual BaseFloat LearningRateFactor() { return learning_rate_factor_; }

  // Sets the learning rate factors to lrate_factor.
  virtual void SetLearningRateFactor(BaseFloat lrate_factor) {
    learning_rate_factor_ = lrate_factor;
  }

  // Copies the learning-rate, learning-rate-factor, l2-regularize, is-gradient
  // and max-change values from 'other'.
  void SetUpdatableConfigs(const UpdatableComponent &other);

  virtual void FreezeNaturalGradient(bool freeze) { }

  BaseFloat LearningRate() const { return learning_rate_; }

  BaseFloat MaxChange() const { return max_change_; }

  void SetMaxChange(BaseFloat max_change) { max_change_ = max_change; }

  BaseFloat L2Regularization() const { return l2_regularize_; }

  void SetL2Regularization(BaseFloat a) { l2_regularize_ = a; }

  virtual std::string Info() const;

  virtual int32 NumParameters() const { KALDI_ASSERT(0); return 0; }

  virtual void Vectorize(VectorBase<BaseFloat> *params) const { KALDI_ASSERT(0); }
  virtual void UnVectorize(const VectorBase<BaseFloat> &params) {
    KALDI_ASSERT(0);
  }

 protected:
  // to be called from child classes, extracts any learning rate information
  // from the config line and sets them appropriately.
  void InitLearningRatesFromConfig(ConfigLine *cfl);

  // To be used in child-class Read() functions, this function reads the opening
  // tag <ThisComponentType> and the learning-rate factor and the learning-rate.
  //
  // Its return value may not always be needed to be inspected by calling code;
  // if there was a token that it read but could not process it returns it, else
  // it returns "".
  std::string ReadUpdatableCommon(std::istream &is, bool binary);

  // To be used in child-class Write() functions, writes the opening
  // <ThisComponentType> tag and the learning-rate factor (if not 1.0) and the
  // learning rate;
  void WriteUpdatableCommon(std::ostream &is, bool binary) const;

  BaseFloat learning_rate_; 
  BaseFloat learning_rate_factor_; 
  BaseFloat l2_regularize_;  
  bool is_gradient_;  
  BaseFloat max_change_; 

 private:
  const UpdatableComponent &operator = (const UpdatableComponent &other); // Disallow.
};


/*   NonlinearComponent is a base-class for things like sigmoid, softmax and
     ReLU: nonlinearities that don't change the dimension.  This base-class
     takes care of storing statistics on the average activations and derivatives
     encountered during training, and model initialization and I/O.

     Supported parameters on the config line:

       dim           Dimension of the input and output of the component.
                     (Caution: for NormalizeComponent, there is a member
                     "add-log-stddev" which if true,  will increase the output
                     dim by one, so it will be "dim" plus one.

       self-repair-scale=0.0   A scale for the self-repair mechanism (which nudges
                     the activation values towards the 'good' regions when a particular
                     dimension of the activations seem to be oversaturated or otherwise
                     unbalanced.  This is typically set from the script level to values
                     like 1.0e-04 to 1.0e-05.

       self-repair-lower-threshold=-1000  A lower threshold for the self-repair mechanism;
                     it will be interpreted in a component-specific way, typically a lower
                     limit on the average derivative or activation below which the
                     self-repair mechanism is activated.  -1000 is a special value which
                     will cause a component-specific default to be used.

       self-repair-upper-threshold=-1000  An upper threshold for the self-repair mechanism;
                     it will be interpreted in a component-specific way, typically an upper
                     limit on the average derivative or activation above which the
                     self-repair mechanism is activated.  -1000 is a special value which
                     will cause a component-specific default to be used.

       block-dim     Defaults to dim, but may be any divisor of dim.  It affects the
                     self-repair, which will be done while treating the input/output as
                     repeating blocks of size 'block-dim' (e.g. blocks of filters).  It allows
                     us to do self-repair on the filter level in CNNs.
                     Currently this only makes a difference for RectifiedLinearComponent.
*/
class NonlinearComponent: public Component {
 public:

  NonlinearComponent();
  explicit NonlinearComponent(const NonlinearComponent &other);

  virtual int32 InputDim() const { return dim_; }
  virtual int32 OutputDim() const { return dim_; }

  // We implement InitFromConfig at this level and this version is sufficient
  // for most of the child classes.  Note: it's overridden by class
  // NormalizeComponent.
  virtual void InitFromConfig(ConfigLine *cfl);

  virtual void Read(std::istream &is, bool binary);

  virtual void ZeroStats();

  virtual std::string Info() const;

  virtual void Write(std::ostream &os, bool binary) const;

  virtual void Scale(BaseFloat scale);
  virtual void Add(BaseFloat alpha, const Component &other);

  virtual void ConsolidateMemory();

  // The following functions are unique to NonlinearComponent.
  // They mostly relate to diagnostics.
  const CuVector<double> &ValueSum() const { return value_sum_; }
  const CuVector<double> &DerivSum() const { return deriv_sum_; }

  double Count() const { return count_; }

 protected:
  enum { kUnsetThreshold = -1000 };

  friend class SigmoidComponent;
  friend class TanhComponent;
  friend class SoftmaxComponent;
  friend class LogSoftmaxComponent;
  friend class RectifiedLinearComponent;

  // This function updates the stats "value_sum_", "deriv_sum_", and
  // count_. (If deriv == NULL, it won't update "deriv_sum_").
  // It will be called from the Backprop function of child classes.
  void StoreStatsInternal(const CuMatrixBase<BaseFloat> &out_value,
                          const CuMatrixBase<BaseFloat> *deriv = NULL);

  // This function may be called from child class members during backprop.  It
  // stores the 'oderiv_sumsq_' stats.
  void StoreBackpropStats(const CuMatrixBase<BaseFloat> &out_deriv);


  const NonlinearComponent &operator = (const NonlinearComponent &other); // Disallow.

  // dim_ is the input dimension (and almost always the output dimension) of the
  // component.
  int32 dim_;
  // block_dim_ will normally be the same as dim_, but it may be any nonzero
  // divisor of dim_; if so, each vector is treated as a number of blocks
  // appended together, and this affects the stats accumulation and self-repair.
  // Currently this is only supported for RectifiedLinearComponent.
  int32 block_dim_;
  CuVector<double> value_sum_; // stats at the output.
  CuVector<double> deriv_sum_; // stats of the derivative of the nonlinearity
                               // (only applicable to element-by-element
                               // nonlinearities, not Softmax.
  // Count corresponding to the stats in 'value_sum_' and 'deriv_sum_'
  double count_;

  CuVector<double> oderiv_sumsq_;  // Sum-square of the derivative of the
                                   // objective function, that we're propagating
                                   // back.  Accumulated during the backprop;
                                   // used for diagnostics.
  // Count corresponding to the stats in 'oderiv_sumsq_'.
  double oderiv_count_;

  // some stats for self-repairing nonlinearities.
  double num_dims_self_repaired_;
  double num_dims_processed_;

  // some configuration values relating to self-repairing nonlinearities.
  BaseFloat self_repair_lower_threshold_;
  BaseFloat self_repair_upper_threshold_;
  BaseFloat self_repair_scale_;
};

} // namespace nnet3
} // namespace kaldi


#endif