nnet-descriptor.h

Go to the documentation of this file.

// nnet3/nnet-descriptor.h

// Copyright   2012-2015  Johns Hopkins University (author: Daniel Povey)

// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.

#ifndef KALDI_NNET3_NNET_DESCRIPTOR_H_
#define KALDI_NNET3_NNET_DESCRIPTOR_H_

#include "base/kaldi-common.h"
#include "util/kaldi-io.h"
#include "matrix/matrix-lib.h"
#include "nnet3/nnet-common.h"
#include "nnet3/nnet-component-itf.h"

#include <iostream>
#include <sstream>
#include <vector>
#include <map>


namespace kaldi {
namespace nnet3 {

class ForwardingDescriptor {
 public:
  // Given an Index that's requested at the output of this descriptor, maps it
  // to a (node_index, Index) pair that says where we are to get the data from.
  //
  virtual Cindex MapToInput(const Index &output) const = 0;

  // Return the feature dimension.
  virtual int32 Dim(const Nnet &nnet) const = 0;

  virtual ForwardingDescriptor *Copy() const = 0;

  virtual int32 Modulus() const { return 1; }

  // Write to string that will be one line of a config-file-like format.  The
  // opposite of Parse.
  virtual void WriteConfig(std::ostream &os,
                           const std::vector<std::string> &node_names) const = 0;

  // that this descriptor may access.
  virtual void GetNodeDependencies(std::vector<int32> *node_indexes) const = 0;

  virtual BaseFloat GetScaleForNode(int32 node_index) const = 0;

  virtual ~ForwardingDescriptor() { }
  ForwardingDescriptor() { }
 private:
  KALDI_DISALLOW_COPY_AND_ASSIGN(ForwardingDescriptor);
};

class SimpleForwardingDescriptor: public ForwardingDescriptor {
 public:
  virtual Cindex MapToInput(const Index &index) const;
  virtual int32 Dim(const Nnet &nnet) const;
  virtual ForwardingDescriptor *Copy() const;
  virtual void GetNodeDependencies(std::vector<int32> *node_indexes) const;
  virtual BaseFloat GetScaleForNode(int32 node_index) const;

  // Write to string that will be one line of a config-file-like format.  The
  // opposite of Parse.
  // written form is just the node-name of src_node_.
  virtual void WriteConfig(std::ostream &os,
                           const std::vector<std::string> &node_names) const;

  SimpleForwardingDescriptor(int32 src_node,
                             BaseFloat scale = 1.0):
      src_node_(src_node), scale_(scale) {
    KALDI_ASSERT(src_node >= 0);
  }
  virtual ~SimpleForwardingDescriptor() { }
 private:
  int32 src_node_;  // index of the source NetworkNode.
  BaseFloat scale_;  // Scale of the node in the expression; this will be 1.0
                     // unless you used a Scale(...) expression in your
                     // Descriptor.
};

class OffsetForwardingDescriptor: public ForwardingDescriptor {
 public:
  virtual Cindex MapToInput(const Index &ind) const;
  virtual int32 Dim(const Nnet &nnet) const { return src_->Dim(nnet); }
  virtual ForwardingDescriptor *Copy() const;

  // written form is: Offset(<src-written-form>, t-offset [, x-offset])
  virtual void WriteConfig(std::ostream &os,
                           const std::vector<std::string> &node_names) const;

  virtual int32 Modulus() const { return src_->Modulus(); }

  virtual void GetNodeDependencies(std::vector<int32> *node_indexes) const;
  virtual BaseFloat GetScaleForNode(int32 node_index) const;

  // takes ownership of src.
  OffsetForwardingDescriptor(ForwardingDescriptor *src,
                             Index offset): src_(src), offset_(offset) { }

  virtual ~OffsetForwardingDescriptor() { delete src_; }


  // this function is not in the shared interface. it's used
  // in class ModelCollapser.
  const ForwardingDescriptor &Src() const { return *src_; }
 private:
  ForwardingDescriptor *src_;  // Owned here.
  Index offset_;  // The index-offset to be added to the index.
};

class SwitchingForwardingDescriptor: public ForwardingDescriptor {
 public:
  virtual Cindex MapToInput(const Index &ind) const;
  virtual int32 Dim(const Nnet &nnet) const { return src_[0]->Dim(nnet); }
  virtual ForwardingDescriptor *Copy() const;
  // Written form is "Switch(<written-form-of-src1>, <written-form-of-src2>, ... )"
  virtual void WriteConfig(std::ostream &os,
                          const std::vector<std::string> &node_names) const;

  virtual int32 Modulus() const;

  // that this descriptor may access.
  virtual void GetNodeDependencies(std::vector<int32> *node_indexes) const;
  virtual BaseFloat GetScaleForNode(int32 node_index) const;

  // takes ownership of items in src.
  SwitchingForwardingDescriptor(std::vector<ForwardingDescriptor*> &src):
      src_(src) { }
  virtual ~SwitchingForwardingDescriptor() { DeletePointers(&src_); }
 private:
  // Pointers are owned here.
  std::vector<ForwardingDescriptor*> src_;
};



class RoundingForwardingDescriptor: public ForwardingDescriptor {
 public:
  virtual Cindex MapToInput(const Index &ind) const;
  virtual int32 Dim(const Nnet &nnet) const { return src_->Dim(nnet); }
  virtual ForwardingDescriptor *Copy() const;
  // Written form is "Round(<written-form-of-src>, <t_modulus>)"
  virtual void WriteConfig(std::ostream &os,
                          const std::vector<std::string> &node_names) const;

  virtual int32 Modulus() const { return t_modulus_; }

  // that this descriptor may access.
  virtual void GetNodeDependencies(std::vector<int32> *node_indexes) const;
  virtual BaseFloat GetScaleForNode(int32 node_index) const;

  // takes ownership of src.
  RoundingForwardingDescriptor(ForwardingDescriptor *src,
                               int32 t_modulus):
      src_(src), t_modulus_(t_modulus) { }

  virtual ~RoundingForwardingDescriptor() { delete src_; }
 private:
  ForwardingDescriptor *src_;
  int32 t_modulus_;
};

class ReplaceIndexForwardingDescriptor: public ForwardingDescriptor {
 public:
  enum VariableName { kN = 0, kT = 1, kX = 2};

  virtual Cindex MapToInput(const Index &ind) const;
  virtual int32 Dim(const Nnet &nnet) const { return src_->Dim(nnet); }
  virtual ForwardingDescriptor *Copy() const;
  // Written form is "ReplaceIndex(<written-form-of-src>, <variable-name>, <value>)"
  // where <variable-name> is either "t" or "x".
  virtual void WriteConfig(std::ostream &os,
                          const std::vector<std::string> &node_names) const;

  // that this descriptor may access.
  virtual void GetNodeDependencies(std::vector<int32> *node_indexes) const;
  virtual BaseFloat GetScaleForNode(int32 node_index) const;

  // takes ownership of src.
  ReplaceIndexForwardingDescriptor(ForwardingDescriptor *src,
                                   VariableName variable_name,
                                   int32 value):
      src_(src), variable_name_(variable_name), value_(value) { }

  virtual ~ReplaceIndexForwardingDescriptor() { delete src_; }
 private:
  ForwardingDescriptor *src_;
  VariableName variable_name_;
  int32 value_;
};


class CindexSet;

class SumDescriptor {
 public:
  virtual void GetDependencies(const Index &ind,
                               std::vector<Cindex> *dependencies) const = 0;

  virtual bool IsComputable(const Index &ind,
                            const CindexSet &cindex_set,
                            std::vector<Cindex> *used_inputs) const = 0;

  virtual int32 Dim(const Nnet &nnet) const = 0;

  virtual SumDescriptor *Copy() const = 0;

  virtual ~SumDescriptor() { }

  virtual void GetNodeDependencies(std::vector<int32> *node_indexes) const = 0;

  virtual BaseFloat GetScaleForNode(int32 node_index) const = 0;

  // see Modulus function of ForwardingDescriptor for explanation.
  virtual int32 Modulus() const = 0;

  virtual void WriteConfig(std::ostream &os,
                           const std::vector<std::string> &node_names) const = 0;
};

class OptionalSumDescriptor: public SumDescriptor {
 public:
  virtual void GetDependencies(const Index &ind,
                               std::vector<Cindex> *dependencies) const;
  virtual bool IsComputable(const Index &ind,
                            const CindexSet &cindex_set,
                            std::vector<Cindex> *used_inputs) const {
      return src_->IsComputable(ind, cindex_set, used_inputs) || true;
  }

  virtual int32 Dim(const Nnet &nnet) const;

  // This function appends to "node_indexes" a list (not necessarily sorted or
  // unique) of all the node indexes that this descriptor may forward data from.
  virtual void GetNodeDependencies(std::vector<int32> *node_indexes) const;
  virtual BaseFloat GetScaleForNode(int32 node_index) const;
  virtual int32 Modulus() const { return src_->Modulus(); }
  virtual void WriteConfig(std::ostream &os,
                           const std::vector<std::string> &node_names) const;
  virtual SumDescriptor *Copy() const;

  OptionalSumDescriptor(SumDescriptor *src): src_(src) { }
  virtual ~OptionalSumDescriptor() { delete src_; }
 private:
  SumDescriptor *src_;
};

class SimpleSumDescriptor: public SumDescriptor {
 public:
  virtual void GetDependencies(const Index &ind,
                               std::vector<Cindex> *dependencies) const;
  virtual bool IsComputable(const Index &ind,
                            const CindexSet &cindex_set,
                            std::vector<Cindex> *used_inputs) const;
  virtual int32 Dim(const Nnet &nnet) const;

  virtual BaseFloat GetScaleForNode(int32 node_index) const;

  // This function appends to "node_indexes" a list (not necessarily sorted or
  // unique) of all the node indexes that this descriptor may forward data from.
  virtual void GetNodeDependencies(std::vector<int32> *node_indexes) const;
  virtual int32 Modulus() const { return src_->Modulus(); }
  virtual void WriteConfig(std::ostream &os,
                           const std::vector<std::string> &node_names) const;
  virtual SumDescriptor *Copy() const;

  SimpleSumDescriptor(ForwardingDescriptor *src): src_(src) { }
  virtual ~SimpleSumDescriptor() { delete src_; }

  // this function is not in the shared interface. it's used
  // in class ModelCollapser.
  const ForwardingDescriptor &Src() const { return *src_; }
 private:
  ForwardingDescriptor *src_;
};


class ConstantSumDescriptor: public SumDescriptor {
 public:
  virtual void GetDependencies(const Index &ind,
                               std::vector<Cindex> *dependencies) const { }
  virtual bool IsComputable(const Index &ind,
                            const CindexSet &cindex_set,
                            std::vector<Cindex> *used_inputs) const {
    return true;
  }
  virtual int32 Dim(const Nnet &nnet) const { return dim_; }
  virtual BaseFloat GetScaleForNode(int32 node_index) const;

  virtual void GetNodeDependencies(std::vector<int32> *node_indexes) const { }
  virtual int32 Modulus() const { return 1; }
  virtual void WriteConfig(std::ostream &os,
                           const std::vector<std::string> &node_names) const;
  virtual SumDescriptor *Copy() const;

  ConstantSumDescriptor(BaseFloat value, int32 dim);
  virtual ~ConstantSumDescriptor() {}
 private:
  BaseFloat value_;
  int32 dim_;
};

class BinarySumDescriptor: public SumDescriptor {
 public:
  enum Operation {
    kSumOperation,  // A + B
    kFailoverOperation, // A if defined, else B.
  };
  virtual void GetDependencies(const Index &ind,
                               std::vector<Cindex> *dependencies) const;
  virtual bool IsComputable(const Index &ind,
                            const CindexSet &cindex_set,
                            std::vector<Cindex> *used_inputs) const;
  virtual int32 Dim(const Nnet &nnet) const;
  virtual BaseFloat GetScaleForNode(int32 node_index) const;

  // This function appends to "node_indexes" a list (not necessarily sorted or
  // unique) of all the node indexes that this descriptor may forward data from.
  virtual void GetNodeDependencies(std::vector<int32> *node_indexes) const;
  virtual int32 Modulus() const;
  virtual void WriteConfig(std::ostream &os,
                           const std::vector<std::string> &node_names) const;
  virtual SumDescriptor *Copy() const;
  BinarySumDescriptor(Operation op, SumDescriptor *src1, SumDescriptor *src2):
      op_(op), src1_(src1), src2_(src2) {}
  virtual ~BinarySumDescriptor() { delete src1_; delete src2_; }
 private:
  Operation op_;
  SumDescriptor *src1_;
  SumDescriptor *src2_;
};


// A Descriptor concatenates over its parts, so its feature-dimension will
// be the sum of the feature-dimensions of its parts.  In a valid Descriptor,
// "parts" will be nonempty.  Each part may be (in general) a summation, but
// usually a summation with just one term.
class Descriptor {
 public:
  int32 Dim(const Nnet &nnet) const;

  // The Parse method is used for reading a config-file-style represenation.
  // Internally this uses class GeneralDescriptor to read and normalize the
  // input.  Assumes the input has already been tokenized into an array of
  // strings by DescriptorTokenize(); it moves the begin-pointer "next_token" to
  // account for each token that it consumes.  Prints warning and returns false on
  // error (including if there was junk after the last token).  The input tokens
  // should be terminated with a token that says "end of input".
  bool Parse(const std::vector<std::string> &node_names,
             const std::string **next_token);

  // Write in config-file format.
  // if parts_.size() == 1, written form is just "<written-form-of-part0>"
  // otherwise, written form is "Append(<written-form-of-part0>, <written-form-of-part1>,  ... )".
  void WriteConfig(std::ostream &os,
                   const std::vector<std::string> &node_names) const;

  void GetDependencies(const Index &index,
                       std::vector<Cindex> *used_inputs) const;

  bool IsComputable(const Index &ind,
                    const CindexSet &cindex_set,
                    std::vector<Cindex> *used_inputs) const;

  // This function outputs to "node_indexes" a list (not necessarily sorted or
  // unique) of all the node indexes that this descriptor may forward data from.
  void GetNodeDependencies(std::vector<int32> *node_indexes) const;

  // see Modulus function of ForwardingDescriptor for explanation.
  int32 Modulus() const;

  int32 NumParts() const { return parts_.size(); }
  const SumDescriptor &Part(int32 n) const;

  Descriptor() { }
  Descriptor(const Descriptor &other) { *this = other; }
  Descriptor &operator = (const Descriptor &other);
  Descriptor(const std::vector<SumDescriptor*> &parts):
      parts_(parts) { }
  ~Descriptor() { Destroy(); }
 private:
  void Destroy();
  // the elements of parts_ are owned here.
  std::vector<SumDescriptor*> parts_;
};


struct GeneralDescriptor {
  enum DescriptorType { kAppend, kSum, kFailover, kIfDefined, kOffset, kSwitch,
                        kRound, kReplaceIndex, kScale, kConst, kNodeName };

  // The Parse method is used for reading a config-file-style represenation.
  // Assumes the input has already been tokenized into an array of strings, and
  // it moves the begin-pointer "next_token" to account for token that it
  // consumes.  Calls KALDI_ERR on error.  The list of tokens should be
  // terminated with a string saying "end of input".  Does not check that all
  // the input has been consumed-- the caller should do that [check that
  // **next_token == "end of input" after calling.]
  static GeneralDescriptor *Parse(const std::vector<std::string> &node_names,
                                  const std::string **next_token);

  explicit GeneralDescriptor(DescriptorType t, int32 value1 = -1,
                             int32 value2 = -1, BaseFloat alpha = 0.0):
      descriptor_type_(t), value1_(value1), value2_(value2),
      alpha_(alpha) { }


  ~GeneralDescriptor() { DeletePointers(&descriptors_); }

  GeneralDescriptor *GetNormalizedDescriptor() const;

  Descriptor *ConvertToDescriptor();

  // prints in text form-- this is really only used for debug.
  void Print(const std::vector<std::string> &node_names,
             std::ostream &os);

 private:
  KALDI_DISALLOW_COPY_AND_ASSIGN(GeneralDescriptor);

  DescriptorType descriptor_type_;

  // value1_ is only relevant if:
  //    (a) descriptor_type_ == kReplaceIndex (value1_ is 1 for t, 2 for x)
  //    (b) descriptor_type_ == kNodeName (value1_ is the index of the node)
  //    (c) descriptor_type_ == kOffset (value1_ is the t offset).
  //    (d) descriptor_type_ == kConst (value1_ is the dimension and alpha_
  //                                   is the value).
  int32 value1_;
  // value2_ is only relevant if
  //    (a) descriptor_type == kReplaceIndex (value2_ is the value
  //                                          we replace the index with).
  //    (b) descriptor_type_ == kOffset (value2_ is the x offset)
  int32 value2_;

  // alpha is only relevant if
  //  (a) descriptor_type == kScale, and this will be the scaling factor.
  //  (b) descriptor_type == kConst; this is the value, and value1_ is set to the
  //          dimension.
  BaseFloat alpha_;

  // For any descriptor types that take args of type kDescriptor, a list of those
  // args.  Pointers owned here.
  std::vector<GeneralDescriptor*> descriptors_;

  //  parses an Append() or Sum() or Switch() expression after the "Append(" or
  //  "Sum(" or "Switch(" has been read.
  void ParseAppendOrSumOrSwitch(const std::vector<std::string> &node_names,
                                const std::string **next_token);
  // parse an IfDefined() expression after the IfDefined( has already been
  // read.
  void ParseIfDefined(const std::vector<std::string> &node_names,
                      const std::string **next_token);
  // ... and so on.
  void ParseOffset(const std::vector<std::string> &node_names,
                   const std::string **next_token);
  void ParseSwitch(const std::vector<std::string> &node_names,
                   const std::string **next_token);
  void ParseFailover(const std::vector<std::string> &node_names,
                     const std::string **next_token);
  void ParseRound(const std::vector<std::string> &node_names,
                  const std::string **next_token);
  void ParseScale(const std::vector<std::string> &node_names,
                  const std::string **next_token);
  void ParseConst(const std::vector<std::string> &node_names,
                  const std::string **next_token);
  void ParseReplaceIndex(const std::vector<std::string> &node_names,
                         const std::string **next_token);



  // Used inside NormalizeAppend().  Return the number of terms there
  // would be in a single consolidated Append() expressions, and asserts that in
  // whichever branch of any other expressions we take, the number of terms is
  // the same.
  int32 NumAppendTerms() const;
  // Used inside NormalizeAppend().  Gets one of the appended terms from this
  // descriptor, with 0 <= term < NumAppendTerms().  Answer is newly allocated.
  GeneralDescriptor *GetAppendTerm(int32 term) const;


  // Normalizes w.r.t. Append expressions by moving Append() to the outside.
  // Called only at the top level.
  GeneralDescriptor *NormalizeAppend() const;

  // This call does all other types of normalization except for normalizing
  // Append() expressions (which is assumed to have been done already).  Returns
  // true if anything was changed.
  static bool Normalize(GeneralDescriptor *ptr);

  SumDescriptor *ConvertToSumDescriptor() const;
  ForwardingDescriptor *ConvertToForwardingDescriptor() const;

};




} // namespace nnet3
} // namespace kaldi

#endif