This struct represents the structure of a convolution computation. More...

#include <convolution.h>

Collaboration diagram for ConvolutionComputation:

Classes
struct	ConvolutionStep

Public Member Functions
void	Write (std::ostream &os, bool binary) const

void	Read (std::istream &is, bool binary)

void	ComputeDerived ()

void	Check () const

Public Attributes
int32	num_filters_in

int32	num_filters_out

int32	height_in

int32	height_out

int32	num_t_in

int32	num_t_out

int32	num_images

int32	temp_rows

int32	temp_cols

std::vector< ConvolutionStep >	steps

Detailed Description

This struct represents the structure of a convolution computation.

This is used inside the PrecomputedIndexes object for the TimeHeightConvolutionComponent (it depends on the inputs and outputs as well as the layer).

CAUTION*: this is after certain transformations of the problem, so the height_in may not always be the "real" height of the input image (it may be a multiple thereof), and the num_t_in may not always be the "real" number of distinct time-steps on the input of the computation (it may be a divisor thereof). ConvolutionComputation contains the info needed to actually perform the computation.

Definition at line 252 of file convolution.h.

Member Function Documentation

◆ Check()

void Check ( ) const

Definition at line 349 of file convolution.cc.

References ConvolutionComputation::ConvolutionStep::backward_columns, ConvolutionComputation::ConvolutionStep::columns, ConvolutionComputation::ConvolutionStep::columns_are_contiguous, CuArrayBase< T >::CopyToVec(), CuArrayBase< T >::Dim(), ConvolutionComputation::ConvolutionStep::first_column, ConvolutionModel::height_in, ConvolutionComputation::ConvolutionStep::height_map, ConvolutionModel::height_out, rnnlm::i, ConvolutionComputation::ConvolutionStep::input_time_shift, KALDI_ASSERT, ConvolutionModel::num_filters_in, ConvolutionModel::num_filters_out, and ConvolutionComputation::ConvolutionStep::params_start_col.

Referenced by kaldi::nnet3::time_height_convolution::UnPadModelHeight().

                                          {
   KALDI_ASSERT(num_filters_in > 0 && num_filters_out > 0 &&
                height_in > 0 && height_out > 0);
   KALDI_ASSERT(num_t_in >= num_t_out &&
                num_t_out > 0 && num_images > 0);
   KALDI_ASSERT((temp_rows == 0 && temp_cols == 0) ||
                (temp_rows <= num_t_out * num_images &&
                 temp_cols > 0));
   KALDI_ASSERT(temp_rows % num_images == 0);
   bool temp_mat_required = false;
   int32 num_steps = steps.size();
   int32 num_extra_input_times = num_t_in - num_t_out,
       input_cols = num_filters_in * height_in,
       smallest_time_shift = 1000,
       largest_time_shift = 0;
   // check 'steps'
   for (int32 s = 0; s < num_steps; s++) {
     const ConvolutionStep &step = steps[s];
     KALDI_ASSERT(step.input_time_shift >= 0 &&
                  step.input_time_shift <= num_extra_input_times);
     if (step.input_time_shift < smallest_time_shift)
       smallest_time_shift = step.input_time_shift;
     if (step.input_time_shift > largest_time_shift)
       largest_time_shift = step.input_time_shift;
     KALDI_ASSERT(step.params_start_col >= 0 &&
                  step.params_start_col % num_filters_in == 0);
     if (s != 0) {
       KALDI_ASSERT(step.input_time_shift != steps[s-1].input_time_shift);
     }
     std::vector<int32> columns;
     step.columns.CopyToVec(&columns);
     KALDI_ASSERT(step.first_column == columns[0]);
     KALDI_ASSERT(step.columns.Dim() == step.height_map.size() * num_filters_in);
     bool all_negative = true;
     int32 temp_height = step.height_map.size();
     bool contiguous = true;
     for (int32 i = 0; i < temp_height; i++) {
       int32 h = step.height_map[i];
       KALDI_ASSERT(h >= -1 && h < height_in);
       if (i > 0 && step.height_map[i-1] != h-1)
         contiguous = false;
       if (h == -1) {
         contiguous = false;
         for (int32 f = 0; f < num_filters_in; f++) {
           KALDI_ASSERT(columns[i * num_filters_in + f] == -1);
         }
       } else {
         all_negative = false;
         for (int32 f = 0; f < num_filters_in; f++) {
           KALDI_ASSERT(columns[i * num_filters_in + f] ==
                        h * num_filters_in + f);
         }
       }
     }
     KALDI_ASSERT(contiguous == step.columns_are_contiguous);
     if (!contiguous || columns.size() != input_cols) {
       // we would need the temporary matrix.  Make sure the
       // temporary matrix is big enough.
       temp_mat_required = true;
       KALDI_ASSERT(columns.size() <= temp_cols);
     }
     KALDI_ASSERT(!all_negative);
 
     std::vector<int32> columns_reconstructed(columns.size(), -1);
     // reconstruct 'columns' from backward_columns as a way to
     // check that backward_columns is correct.
     // they are reverse-direction maps, but we may need
     // step.backward_columns.size() > 1 because of elements
     // in the input that are duplicated in the temp matrix.
     for (size_t k = 0; k < step.backward_columns.size(); k++) {
       std::vector<int32> backward_columns;
       step.backward_columns[k].CopyToVec(&backward_columns);
       KALDI_ASSERT(int32(backward_columns.size()) ==
                    num_filters_in * height_in);
       for (int32 l = 0; l < num_filters_in * height_in; l++) {
         int32 c = backward_columns[l];
         KALDI_ASSERT(c < int32(columns.size()));
         if (c != -1) {
           KALDI_ASSERT(columns_reconstructed[c] == -1);
           columns_reconstructed[c] = l;
         }
       }
     }
     KALDI_ASSERT(columns_reconstructed == columns);
   }
   // check that all rows of the input were used.
   KALDI_ASSERT(smallest_time_shift == 0 &&
                largest_time_shift == num_extra_input_times);
 
   // check that the temp matrix is only allocated if it is required.
   KALDI_ASSERT((temp_cols != 0) == temp_mat_required);
 }

◆ ComputeDerived()

void ComputeDerived ( )

Definition at line 1271 of file convolution.cc.

References ConvolutionComputation::ConvolutionStep::backward_columns, ConvolutionComputation::ConvolutionStep::columns, ConvolutionComputation::ConvolutionStep::columns_are_contiguous, CuArray< T >::CopyFromVec(), ConvolutionComputation::ConvolutionStep::first_column, ConvolutionModel::height_in, ConvolutionComputation::ConvolutionStep::height_map, rnnlm::i, KALDI_ASSERT, ConvolutionModel::num_filters_in, kaldi::nnet3::time_height_convolution::ReverseColumnMapping(), and kaldi::nnet3::time_height_convolution::VectorIsContiguous().

Referenced by kaldi::nnet3::time_height_convolution::UnPadModelHeight().

                                             {
   KALDI_ASSERT(!steps.empty());
 
   int32 input_dim = height_in * num_filters_in;
 
   int32 largest_required_temp_cols = 0;
   for (std::vector<ConvolutionStep>::iterator iter = steps.begin();
        iter != steps.end(); ++iter) {
     ConvolutionStep &step = *iter;
     std::vector<int32> columns;
     int32 temp_height = step.height_map.size();
     columns.resize(temp_height * num_filters_in);
     for (int32 h = 0; h < temp_height; h++) {
       KALDI_ASSERT(step.height_map[h] >= -1 && step.height_map[h] < height_in);
       if (step.height_map[h] != -1) {
         for (int32 f = 0; f < num_filters_in; f++)
           columns[h * num_filters_in + f] = step.height_map[h] * num_filters_in + f;
       } else {
         for (int32 f = 0; f < num_filters_in; f++)
           columns[h * num_filters_in + f] = -1;
       }
     }
     step.columns.CopyFromVec(columns);
     std::vector<std::vector<int32> > backward_columns;
     ReverseColumnMapping(columns, input_dim, &backward_columns);
     step.backward_columns.resize(backward_columns.size());
     for (size_t i = 0; i < backward_columns.size(); i++)
       step.backward_columns[i].CopyFromVec(backward_columns[i]);
 
     // we could replace height_map with columns in the line below and get the
     // same answer, but it would be a little slower.
     step.columns_are_contiguous =
         (step.height_map[0] != -1 && VectorIsContiguous(step.height_map));
     step.first_column = columns[0];
 
 
     bool need_temp_matrix =
         !(step.columns_are_contiguous && step.height_map[0] == 0 &&
           step.height_map.size() == height_in);
     if (need_temp_matrix) {
       largest_required_temp_cols = std::max<int32>(
           largest_required_temp_cols, static_cast<int32>(columns.size()));
     }
   }
   KALDI_ASSERT(temp_cols == largest_required_temp_cols);
 }

◆ Read()

void Read	(	std::istream &	is,
		bool	binary
	)

Definition at line 315 of file convolution.cc.

References ConvolutionModel::Check(), ConvolutionModel::ComputeDerived(), kaldi::ExpectOneOrTwoTokens(), kaldi::nnet3::ExpectToken(), ConvolutionModel::height_in, ConvolutionComputation::ConvolutionStep::height_map, ConvolutionModel::height_out, ConvolutionComputation::ConvolutionStep::input_time_shift, ConvolutionModel::num_filters_in, ConvolutionModel::num_filters_out, ConvolutionComputation::ConvolutionStep::params_start_col, kaldi::ReadBasicType(), and kaldi::ReadIntegerVector().

Referenced by kaldi::nnet3::time_height_convolution::TestComputationIo().

                                                              {
   ExpectOneOrTwoTokens(is, binary, "<ConvComputation>", "<NumFiltersInOut>");
   ReadBasicType(is, binary, &num_filters_in);
   ReadBasicType(is, binary, &num_filters_out);
   ExpectToken(is, binary, "<HeightInOut>");
   ReadBasicType(is, binary, &height_in);
   ReadBasicType(is, binary, &height_out);
   ExpectToken(is, binary, "<NumTInOut>");
   ReadBasicType(is, binary, &num_t_in);
   ReadBasicType(is, binary, &num_t_out);
   ExpectToken(is, binary, "<NumImages>");
   ReadBasicType(is, binary, &num_images);
   ExpectToken(is, binary, "<TempRowsCols>");
   ReadBasicType(is, binary, &temp_rows);
   ReadBasicType(is, binary, &temp_cols);
   int32 num_steps;
   ExpectToken(is, binary, "<NumSteps>");
   ReadBasicType(is, binary, &num_steps);
   steps.resize(num_steps);
   for (int32 s = 0; s < num_steps; s++) {
     ConvolutionStep &step = steps[s];
     ExpectToken(is, binary, "<TimeShift>");
     ReadBasicType(is, binary, &step.input_time_shift);
     ExpectToken(is, binary, "<ParamsStartCol>");
     ReadBasicType(is, binary, &step.params_start_col);
     ExpectToken(is, binary, "<HeightMap>");
     ReadIntegerVector(is, binary, &step.height_map);
   }
   ExpectToken(is, binary, "</ConvComputation>");
   ComputeDerived();
   Check();
 }

◆ Write()

void Write	(	std::ostream &	os,
		bool	binary
	)		const

Definition at line 283 of file convolution.cc.

References ConvolutionModel::height_in, ConvolutionComputation::ConvolutionStep::height_map, ConvolutionModel::height_out, ConvolutionComputation::ConvolutionStep::input_time_shift, ConvolutionModel::num_filters_in, ConvolutionModel::num_filters_out, ConvolutionComputation::ConvolutionStep::params_start_col, kaldi::WriteBasicType(), kaldi::WriteIntegerVector(), and kaldi::WriteToken().

Referenced by kaldi::nnet3::time_height_convolution::TestComputationIo().

                                                                     {
   WriteToken(os, binary, "<ConvComputation>");
   WriteToken(os, binary, "<NumFiltersInOut>");
   WriteBasicType(os, binary, num_filters_in);
   WriteBasicType(os, binary, num_filters_out);
   WriteToken(os, binary, "<HeightInOut>");
   WriteBasicType(os, binary, height_in);
   WriteBasicType(os, binary, height_out);
   WriteToken(os, binary, "<NumTInOut>");
   WriteBasicType(os, binary, num_t_in);
   WriteBasicType(os, binary, num_t_out);
   WriteToken(os, binary, "<NumImages>");
   WriteBasicType(os, binary, num_images);
   WriteToken(os, binary, "<TempRowsCols>");
   WriteBasicType(os, binary, temp_rows);
   WriteBasicType(os, binary, temp_cols);
   int32 num_steps = steps.size();
   WriteToken(os, binary, "<NumSteps>");
   WriteBasicType(os, binary, num_steps);
   for (int32 s = 0; s < num_steps; s++) {
     const ConvolutionStep &step = steps[s];
     WriteToken(os, binary, "<TimeShift>");
     WriteBasicType(os, binary, step.input_time_shift);
     WriteToken(os, binary, "<ParamsStartCol>");
     WriteBasicType(os, binary, step.params_start_col);
     WriteToken(os, binary, "<HeightMap>");
     WriteIntegerVector(os, binary, step.height_map);
   }
   WriteToken(os, binary, "</ConvComputation>");
 }