differentiable-transform.h

Go to the documentation of this file.

// transform/differentiable-transform.h

// Copyright      2018  Johns Hopkins University (author: Daniel Povey)

// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.


#ifndef KALDI_TRANSFORM_DIFFERENTIABLE_TRANSFORM_H_
#define KALDI_TRANSFORM_DIFFERENTIABLE_TRANSFORM_H_

#include <vector>

#include "base/kaldi-common.h"
#include "util/kaldi-table.h"
#include "util/kaldi-holder.h"

namespace kaldi {


namespace differentiable_transform {

class MinibatchInfoItf {
 public:

  virtual ~MinibatchInfoItf() { }
};


class SpeakerStatsItf {

  virtual ~SpeakerStatsItf() { }
};



class DifferentiableTransform {
 public:

  virtual int32 Dim() const = 0;


  int32 NumClasses() const { return num_classes_; }


  virtual void SetNumClasses(int32 num_classes) { num_classes_ = num_classes; }

  virtual MinibatchInfoItf* TrainingForward(
      const CuMatrixBase<BaseFloat> &input,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors,
      CuMatrixBase<BaseFloat> *output) const = 0;


  virtual void TrainingBackward(
      const CuMatrixBase<BaseFloat> &input,
      const CuMatrixBase<BaseFloat> &output_deriv,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors,
      const MinibatchInfoItf &minibatch_info,
      CuMatrixBase<BaseFloat> *input_deriv) const = 0;


  virtual int32 NumFinalIterations() = 0;

  virtual void Accumulate(
      int32 final_iter,
      const CuMatrixBase<BaseFloat> &input,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors) = 0;

  // To be called after repeated alls to Accumulate(), does any estimation that
  // is required in training time (normally per-speaker means and possibly
  // variances.
  virtual void Estimate(int32 final_iter) = 0;

  // Returns an object representing sufficient statistics for estimating a
  // speaker-dependent transform.  This object will initially have zero
  // counts in its statistics.  It will represent the stats for a single
  // speaker.
  virtual SpeakerStatsItf *GetEmptySpeakerStats() = 0;


  // Accumulate statistics for a segment of test data, storing them in the
  // object 'speaker_stats'.  There is no assumption that the soft-counts in
  // 'posteriors' are positive; this allows you to change your mind about the
  // traceback, in test-time, by subtracting the stats that you no longer want
  // to use.
  virtual void TestingAccumulate(
      const MatrixBase<BaseFloat> &input,
      const Posterior &posteriors,
      SpeakerStatsItf *speaker_stats) const = 0;

  // Applies the transformation implied by the statistics in 'speaker_stats' to
  // 'input', storing in the result in 'output'.  It will do any estimation
  // procedure that is required first, if applicable.
  virtual void TestingForward(
      const MatrixBase<BaseFloat> &input,
      const SpeakerStatsItf &speaker_stats,
      MatrixBase<BaseFloat> *output) const = 0;


  // Read transform from stream (works out its type).  Dies on error.
  static DifferentiableTransform* ReadNew(std::istream &is, bool binary);

  // Copies transform (deep copy).
  virtual DifferentiableTransform* Copy() const = 0;

  // Returns a new transform of the given type e.g. "MeanNormalize",
  // or NULL if no such component type exists.
  static DifferentiableTransform *NewTransformOfType(const std::string &type);

  // Write transform to stream
  virtual void Write(std::ostream &os, bool binary) const = 0;

  // Reads transform from stream (normally you would previously have created
  // the transform object of the correct type using ReadNew().
  virtual void Read(std::istream &is, bool binary) = 0;

 protected:
  int32 num_classes_;


};


class NoOpTransform: public DifferentiableTransform {
 public:

  int32 Dim() const override { return dim_; }
  int32 NumClasses() const override { return num_classes_; }
  MinibatchInfoItf* TrainingForward(
      const CuMatrixBase<BaseFloat> &input,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors,
      CuMatrixBase<BaseFloat> *output) const override {
    output->CopyFromMat(input);
    return NULL;
  }
  virtual void TrainingBackward(
      const CuMatrixBase<BaseFloat> &input,
      const CuMatrixBase<BaseFloat> &output_deriv,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors,
      const MinibatchInfoItf &minibatch_info,
      CuMatrixBase<BaseFloat> *input_deriv) const override {
    input_deriv->AddMat(1.0, output_deriv);
  }

  virtual int32 NumFinalIterations() { return 0; }

  void Accumulate(
      int32 final_iter,
      const CuMatrixBase<BaseFloat> &input,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors) override { }



  SpeakerStatsItf *GetEmptySpeakerStats() override { return NULL; }

  void TestingAccumulate(
      const MatrixBase<BaseFloat> &input,
      const Posterior &posteriors,
      SpeakerStatsItf *speaker_stats) const override { }
  void TestingForward(
      const MatrixBase<BaseFloat> &input,
      const SpeakerStatsItf &speaker_stats,
      MatrixBase<BaseFloat> *output) override {
    output->CopyFromMat(input);
  }

  void Estimate(int32 final_iter) override { }

  NoOpTransform(const NoOpTransform &other):
      dim_(other.dim_), num_classes_(other.num_classes_) { }

  DifferentiableTransform* Copy() const override {
    return new NoOpTransform(*this);
  }

  void Write(std::ostream &os, bool binary) const override;

  void Read(std::istream &is, bool binary) override;

 private:
  int32 dim_;
  int32 num_classes_;
};


class SequenceTransform: public DifferentiableTransform {
 public:

  int32 Dim() const override;
  int32 SetNumClasses() const override;

  MinibatchInfoItf* TrainingForward(
      const CuMatrixBase<BaseFloat> &input,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors,
      CuMatrixBase<BaseFloat> *output) const override;
  virtual void TrainingBackward(
      const CuMatrixBase<BaseFloat> &input,
      const CuMatrixBase<BaseFloat> &output_deriv,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors,
      const MinibatchInfoItf &minibatch_info,
      CuMatrixBase<BaseFloat> *input_deriv) const override;

  virtual int32 NumFinalIterations();

  void Accumulate(
      int32 final_iter,
      const CuMatrixBase<BaseFloat> &input,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors) override;

  SpeakerStatsItf *GetEmptySpeakerStats() override;

  void TestingAccumulate(
      const MatrixBase<BaseFloat> &input,
      const Posterior &posteriors,
      SpeakerStatsItf *speaker_stats) const override;

  virtual void TestingForward(
      const MatrixBase<BaseFloat> &input,
      const SpeakerStatsItf &speaker_stats,
      MatrixBase<BaseFloat> *output) override;

  void Estimate(int32 final_iter) override;

  SequenceTransform(const SequenceTransform &other);

  DifferentiableTransform* Copy() const override {
    return new SequenceTransform(*this);
  }

  void Write(std::ostream &os, bool binary) const override;

  void Read(std::istream &is, bool binary) override;

 private:
  std::vector<DifferentiableTransform*> transforms_;
};


class AppendTransform: public DifferentiableTransform {
 public:

  int32 Dim() const override;
  int32 SetNumClasses() const override;

  MinibatchInfoItf* TrainingForward(
      const CuMatrixBase<BaseFloat> &input,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors,
      CuMatrixBase<BaseFloat> *output) const override;
  virtual void TrainingBackward(
      const CuMatrixBase<BaseFloat> &input,
      const CuMatrixBase<BaseFloat> &output_deriv,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors,
      const MinibatchInfoItf &minibatch_info,
      CuMatrixBase<BaseFloat> *input_deriv) const override;

  virtual int32 NumFinalIterations();

  void Accumulate(
      int32 final_iter,
      const CuMatrixBase<BaseFloat> &input,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors) override;

  virtual void TestingForward(
      const MatrixBase<BaseFloat> &input,
      const SpeakerStatsItf &speaker_stats,
      MatrixBase<BaseFloat> *output) override;

  void Estimate(int32 final_iter) override;

  AppendTransform(const AppendTransform &other);

  DifferentiableTransform* Copy() const override {
    return new AppendTransform(*this);
  }

  void Write(std::ostream &os, bool binary) const override;

  void Read(std::istream &is, bool binary) override;

 private:
  std::vector<DifferentiableTransform*> transforms_;
};



class AppendTransform: public DifferentiableTransform {
  int32 Dim() const override;
  int32 NumClasses() const override;
  MinibatchInfoItf* TrainingForward(
      const CuMatrixBase<BaseFloat> &input,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors,
      CuMatrixBase<BaseFloat> *output) const override {
    output->CopyFromMat(input);
    return NULL;
  }
  virtual void TrainingBackward(
      const CuMatrixBase<BaseFloat> &input,
      const CuMatrixBase<BaseFloat> &output_deriv,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors,
      const MinibatchInfoItf &minibatch_info,
      CuMatrixBase<BaseFloat> *input_deriv) const override;

  void Accumulate(
      const CuMatrixBase<BaseFloat> &input,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors) override;

  void Estimate() override { }

  AppendTransform(const AppendTransform &other);

  DifferentiableTransform* Copy() const override;

  void Write(std::ostream &os, bool binary) const override;

  void Read(std::istream &is, bool binary) override;
 private:
  std::vector<DifferentiableTransform*> transforms_;
};


class SimpleMeanTransform: public DifferentiableTransform {
 public:
  int32 Dim() const override;
  int32 NumClasses() const override;
  MinibatchInfoItf* TrainingForward(
      const CuMatrixBase<BaseFloat> &input,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors,
      CuMatrixBase<BaseFloat> *output) const override {
    output->CopyFromMat(input);
    return NULL;
  }
  virtual void TrainingBackward(
      const CuMatrixBase<BaseFloat> &input,
      const CuMatrixBase<BaseFloat> &output_deriv,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors,
      const MinibatchInfoItf &minibatch_info,
      CuMatrixBase<BaseFloat> *input_deriv) const override;

  void Accumulate(
      const CuMatrixBase<BaseFloat> &input,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors) override;

  virtual void TestingForward(
      const MatrixBase<BaseFloat> &input,
      const SpeakerStatsItf &speaker_stats,
      MatrixBase<BaseFloat> *output) override;


  void Estimate() override { }

  AppendTransform(const AppendTransform &other);

  DifferentiableTransform* Copy() const override;

  void Write(std::ostream &os, bool binary) const override;

  void Read(std::istream &is, bool binary) override;
 private:

  // OK: how to compute stats
  class MinibatchInfo: public MinibatchInfoItf {

    // Stores the total weights, per frame, that correspond to the Posteriors
    // supplied to TrainingForward().
    CuVector<BaseFloat> frame_weights;

    // The total of frame_weights.
    BaseFloat total_weight;
  };

  // dim_ is the feature dimension
  int32 dim_;

  // The class-dependent means.  Dimension is num_classes_ by dim_.
  // Note: these will not be set up during training, they will only
  // be set up after calling Accumulate() and Estimate(), which happens
  // in test time.
  CuMatrix<BaseFloat> means_;

  // mean_stats_ and count_ are used in Accumulate() to accumulate
  // statistics to adapt the mean.
  CuMatrix<double> mean_stats_;
  double count_;

};


class FmllrTransform: public DifferentiableTransform {
 public:
  int32 Dim() const override;
  int32 NumClasses() const override;
  MinibatchInfoItf* TrainingForward(
      const CuMatrixBase<BaseFloat> &input,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors,
      CuMatrixBase<BaseFloat> *output) const override;
  virtual void TrainingBackward(
      const CuMatrixBase<BaseFloat> &input,
      const CuMatrixBase<BaseFloat> &output_deriv,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors,
      const MinibatchInfoItf &minibatch_info,
      CuMatrixBase<BaseFloat> *input_deriv) const override;
  void Accumulate(
      int32 final_iter,
      const CuMatrixBase<BaseFloat> &input,
      int32 num_chunks,
      int32 num_spk,
      const Posterior &posteriors) override;

  SpeakerStatsItf *GetEmptySpeakerStats() override;

  void TestingAccumulate(
      const MatrixBase<BaseFloat> &input,
      const Posterior &posteriors,
      SpeakerStatsItf *speaker_stats) const override;

  virtual void TestingForward(
      const MatrixBase<BaseFloat> &input,
      const SpeakerStatsItf &speaker_stats,
      MatrixBase<BaseFloat> *output) override;

  void Estimate(int32 final_iter) override { }

  FmllrTransform(const FmllrTransform &other);

  DifferentiableTransform* Copy() const override;

  void Write(std::ostream &os, bool binary) const override;

  void Read(std::istream &is, bool binary) override;
 private:

  // OK: how to compute stats
  class MinibatchInfo: public MinibatchInfoItf {

    // Stores the total weights, per frame, that correspond to the Posteriors
    // supplied to TrainingForward().  frame_weights.Dim() equals
    // input.NumRows().
    CuVector<BaseFloat> frame_weights;

    // The total of frame_weights per speaker.
    CuVector<BaseFloat> frame_weights;

    BaseFloat total_weight;
  };

  class SpeakerStats: public SpeakerStatsItf {

  };

  // dim_ is the feature dimension
  int32 dim_;

  // The class-dependent means.  Dimension is num_classes_ by dim_.
  // Note: these will not be set up during training, they will only
  // be set up after calling Accumulate() and Estimate(), which happens
  // in test time.
  CuMatrix<BaseFloat> means_;

  // mean_stats_ and count_ are used in Accumulate() to accumulate
  // statistics to adapt the mean.
  CuMatrix<double> mean_stats_;
  double count_;

};


} // namespace differentiable_transform
} // namespace kaldi

#endif  // KALDI_TRANSFORM_DIFFERENTIABLE_TRANSFORM_H_