Classes
struct	ConvolutionComputation
	This struct represents the structure of a convolution computation. More...

struct	ConvolutionComputationIo

struct	ConvolutionComputationOptions
	This struct contains options for compiling the convolutional computation. More...

struct	ConvolutionModel
	This comment explains the basic framework used for everything related to time-height convolution. More...

Functions
static void	GetRandomConvolutionModel (ConvolutionModel *model)

static void	GetRandomConvolutionIndexes (const ConvolutionModel &model, std::vector< Index > input_indexes, std::vector< Index > output_indexes)

void	UnitTestTimeHeightConvolutionIo ()

void	TestComputationIo (const ConvolutionComputation &computation)

void	ZeroBlankRows (const std::vector< Index > &indexes, CuMatrix< BaseFloat > *matrix)

void	ConvolveForwardSimple (const ConvolutionModel &model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const CuMatrixBase< BaseFloat > &input_cu, const CuMatrixBase< BaseFloat > &params_cu, CuMatrixBase< BaseFloat > *output_cu)

void	TestRunningComputation (const ConvolutionModel &conv_model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputation &computation)

void	TestDataBackprop (const ConvolutionModel &conv_model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputation &computation)

void	TestParamsBackprop (const ConvolutionModel &conv_model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputation &computation)

void	UnitTestTimeHeightConvolutionCompile ()

void	UnitTestTimeHeightConvolution ()

static void	ReverseColumnMapping (const std::vector< int32 > &columns, int32 input_dim, std::vector< std::vector< int32 > > *backward_columns)
	This function, used in ConvolutionComputation::ComputeDerived(), reverses a mapping that may not be unique. More...

static bool	VectorIsContiguous (const std::vector< int32 > &vec)

static void	ConvolveForwardInternal (const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &params, CuMatrixBase< BaseFloat > temp_mat, CuMatrixBase< BaseFloat > output)

void	ConvolveForward (const ConvolutionComputation &conv_comp, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &params, CuMatrixBase< BaseFloat > *output)
	This does the forward computation of convolution. More...

static void	ConvolveBackwardDataInternal (const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &params, const CuMatrixBase< BaseFloat > &output_deriv, CuMatrixBase< BaseFloat > temp_mat, CuMatrixBase< BaseFloat > input_deriv)

void	ConvolveBackwardData (const ConvolutionComputation &conv_comp, const CuMatrixBase< BaseFloat > &params, const CuMatrixBase< BaseFloat > &output_deriv, CuMatrixBase< BaseFloat > *input_deriv)
	This does the part of the backward derivative computation of convolution, that propagates derivatives back to the input data. More...

static void	ConvolveBackwardParamsInternal (const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &output_deriv, BaseFloat alpha, CuMatrixBase< BaseFloat > temp_mat, CuMatrixBase< BaseFloat > params_deriv)

void	ConvolveBackwardParams (const ConvolutionComputation &conv_comp, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &output_deriv, BaseFloat alpha, CuMatrixBase< BaseFloat > *params_deriv)
	This does the part of the backward derivative computation of convolution, that computes derivatives w.r.t. More...

void	PadModelHeight (const ConvolutionModel &model, ConvolutionModel *model_padded)
	This function takes a model that might require zero padding in the height dimension and outputs a model accepting a possibly-larger input dimension which does not require zero padding. More...

static void	ComputeTempMatrixSize (const ConvolutionComputationOptions &opts, ConvolutionComputation *computation)
	This function sets 'temp_rows' and 'temp_cols' in 'computation'. More...

void	UnPadModelHeight (const ConvolutionComputationOptions &opts, const ConvolutionModel &model, const ConvolutionModel &model_padded, ConvolutionComputation *computation)
	This function modifies, if necessary, a computation that has been built for the model 'model_padded', so that it can work for the original model 'model'. More...

void	PadComputationInputTime (const ConvolutionModel &model, ConvolutionComputationIo *io)
	This function extends the set of input indexes that the computation has, to account for any required zero-padding in the time dimension. More...

static int32	RoundDownToMultipleOf (int32 i, int32 n)

static void	ShiftAllTimeOffsets (int32 shift, ConvolutionModel *model)

static int32	PrepareIoForAppending (ConvolutionComputationIo io, ConvolutionComputationIo io_appended)

void	AppendInputFrames (const ConvolutionModel &model, ConvolutionComputationIo io, ConvolutionModel model_appended, ConvolutionComputationIo *io_appended)
	This function takes an input model and I/O specification, and it modifies both of them if necessary to ensure that the output 'io_appended' object has the same input and output time strides (i.e. More...

static bool	TimeValueInInput (const ConvolutionComputationIo &io, int32 t)

void	CheckModelAndIo (const ConvolutionModel &model, const ConvolutionComputationIo &io, bool allow_extra_input=false)
	Check that this model and this I/O request are compatible in terms of required context, etc, and crash if not. More...

void	CompileConvolutionComputation (const ConvolutionModel &model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputationOptions &opts, ConvolutionComputation computation, std::vector< Index > input_indexes_modified, std::vector< Index > *output_indexes_modified)
	This function does the compilation for a convolution computation; it's a wrapper for the functions below, which should not have to be called by the end user. More...

static int32	FindGcdOfDifferences (std::vector< int32 > &vec)

static void	RegularizeTList (std::vector< int32 > &t_values, int32 start, int32 step, int32 *num_values)

static void	CreateIndexes (const std::vector< std::pair< int32, int32 > > &n_x_pairs, int32 t_start, int32 t_step, int32 num_t_values, int32 reorder_t, std::vector< Index > *indexes)
	Creates a vector of indexes with a regular structure, according to these specifications. More...

static void	SetSomeIndexesBlank (const std::vector< Index > &ref_indexes, std::vector< Index > *indexes)
	This function modifies 'indexes' by, for any Indexes which was not present in 'ref_indexes', setting the 't' value to kNoTime. More...

void	GetComputationIo (const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, ConvolutionComputationIo *io)
	This function takes lists of input and output indexes to a computation (e.g. More...

void	GetIndexesForComputation (const ConvolutionComputationIo &io, const std::vector< Index > &orig_input_indexes, const std::vector< Index > &orig_output_indexes, std::vector< Index > input_indexes, std::vector< Index > output_indexes)
	This function computes the reordered and possibly padded indexes corresponding to the computation in 'io'. More...

void	MakeComputation (const ConvolutionModel &model, ConvolutionComputationIo &io, const ConvolutionComputationOptions &opts, ConvolutionComputation *computation)

Function Documentation

◆ AppendInputFrames()

void AppendInputFrames	(	const ConvolutionModel &	model,
		ConvolutionComputationIo *	io,
		ConvolutionModel *	model_appended,
		ConvolutionComputationIo *	io_appended
	)

This function takes an input model and I/O specification, and it modifies both of them if necessary to ensure that the output 'io_appended' object has the same input and output time strides (i.e.

t_stride_in == t_stride_out). This is done by appending the input frames across several time values and viewing them as single frames of larger dimension.

The reason why 'io' is non-const is that it may be necessary to pad the number of input frames to ensure that the number of input frames is divisible by a multiple of t_stride_out / t_stride_in (if we pad the input frames, we pad to the right).

The model in 'model_appended' may have larger height_in, and different values of 'offsets' and derived variables thereof, versus the model in 'model'.

This is stage 3 of compilation.

Definition at line 1203 of file convolution.cc.

References ConvolutionModel::all_time_offsets, ConvolutionModel::Check(), ConvolutionModel::ComputeDerived(), ConvolutionModel::height_in, ConvolutionModel::Offset::height_offset, ConvolutionModel::height_out, ConvolutionModel::height_subsample_out, rnnlm::i, KALDI_ASSERT, ConvolutionModel::num_filters_in, ConvolutionModel::num_filters_out, ConvolutionModel::offsets, PrepareIoForAppending(), ConvolutionModel::required_time_offsets, RoundDownToMultipleOf(), ShiftAllTimeOffsets(), ConvolutionComputationIo::start_t_in, ConvolutionComputationIo::start_t_out, ConvolutionComputationIo::t_step_in, ConvolutionComputationIo::t_step_out, and ConvolutionModel::Offset::time_offset.

Referenced by CompileConvolutionComputation().

                                                               {
   int32 ratio = PrepareIoForAppending(io, io_appended);
 
   if (ratio == 1) {
     // we are not doing any appending of frames.
     *model_appended = model;
     return;
   }
 
   // we also need the time-step of the output (which is also now the
   // time-step of the appended input).
   // We know that the time step is not zero, because in that case we would
   // have ratio == 1 and would have returned above.
   int32 time_step_out = io_appended->t_step_out;
   KALDI_ASSERT(time_step_out == io_appended->t_step_in && time_step_out != 0);
   int32 orig_time_step_in = io->t_step_in;
   KALDI_ASSERT(orig_time_step_in * ratio == time_step_out);
 
   // make sure the difference between first input and output frames is what we
   // expect, else something could go wrong here.
   int32 first_time_offset = *(model.all_time_offsets.begin());
   KALDI_ASSERT(io->start_t_in - io->start_t_out == first_time_offset);
 
   ConvolutionModel model_temp(model);
   // shift so that the first time offset is zero.  this makes
   // the model conversion easier.
   ShiftAllTimeOffsets(-first_time_offset, &model_temp);
 
   model_appended->num_filters_in = model.num_filters_in;
   model_appended->num_filters_out = model.num_filters_out;
   model_appended->height_in = ratio * model.height_in;
   model_appended->height_out = model.height_out;
   model_appended->height_subsample_out = model.height_subsample_out;
   int32 num_offsets = model_temp.offsets.size(),
       old_height = model.height_in;
   model_appended->offsets.resize(num_offsets);
   model_appended->all_time_offsets.clear();
   for (int32 i = 0; i < num_offsets; i++) {
     const ConvolutionModel::Offset &old_offset = model_temp.offsets[i];
     ConvolutionModel::Offset &new_offset = model_appended->offsets[i];
     // The following two lines are important!!  They are the core of how
     // we handle subsampling in this framework.
     new_offset.time_offset = RoundDownToMultipleOf(old_offset.time_offset,
                                                    time_step_out);
     KALDI_ASSERT((old_offset.time_offset - new_offset.time_offset) %
                  orig_time_step_in == 0);
     int32 row_offset = (old_offset.time_offset - new_offset.time_offset) /
         orig_time_step_in;
     new_offset.height_offset = old_offset.height_offset +
         row_offset * old_height;
     model_appended->all_time_offsets.insert(new_offset.time_offset);
   }
 
   // Because the 'appended' model will always be used after zero-padding on the
   // time axis, we can just pretend that all desired time-offsets are required.
   // It's a kind of free error-checking.
   model_appended->required_time_offsets = model_appended->all_time_offsets;
 
   // Undo the time-shifting that we did before.
   ShiftAllTimeOffsets(first_time_offset, model_appended);
 
   model_appended->ComputeDerived();
   KALDI_ASSERT(model_appended->Check(false, false));
 }

◆ CheckModelAndIo()

void CheckModelAndIo	(	const ConvolutionModel &	model,
		const ConvolutionComputationIo &	io,
		bool	allow_extra_input = `false`
	)

Check that this model and this I/O request are compatible in terms of required context, etc, and crash if not.

if allow_extra_input == false, this will crash if the input 'io' object has time values that would never be used because they are before/after the first/last desired time values.

Definition at line 1329 of file convolution.cc.

References ConvolutionModel::all_time_offsets, KALDI_ASSERT, KALDI_ERR, rnnlm::n, ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::num_t_out, kaldi::RandInt(), ConvolutionModel::required_time_offsets, ConvolutionComputationIo::start_t_in, ConvolutionComputationIo::start_t_out, ConvolutionComputationIo::t_step_in, ConvolutionComputationIo::t_step_out, and TimeValueInInput().

Referenced by CompileConvolutionComputation().

                                              {
   KALDI_ASSERT(io.num_t_in > 0 && io.num_t_out > 0 &&
                !model.required_time_offsets.empty() &&
                !model.all_time_offsets.empty());
   if (!allow_extra_input) {
     KALDI_ASSERT(io.start_t_in >= io.start_t_out +
                  *model.all_time_offsets.begin());
     int32 last_t_in = io.start_t_in + io.t_step_in * (io.num_t_in - 1),
         last_t_out = io.start_t_out + io.t_step_out * (io.num_t_out - 1);
     KALDI_ASSERT(last_t_in <= last_t_out +
                  *model.all_time_offsets.rbegin());
   }
 
   std::set<int32> input_times_to_check;
   for (int32 n = 0; n < std::min(5, io.num_t_out); n++) {
     int32 t_out = io.start_t_out +
         RandInt(0, io.num_t_out - 1) * io.t_step_out;
     for (std::set<int32>::const_iterator iter =
              model.required_time_offsets.begin();
          iter != model.required_time_offsets.end();
          ++iter) {
       int32 offset = *iter;
       input_times_to_check.insert(t_out + offset);
     }
   }
   for (std::set<int32>::const_iterator iter = input_times_to_check.begin();
        iter != input_times_to_check.end(); ++iter) {
     int32 t = *iter;
     if (!TimeValueInInput(io, t)) {
       KALDI_ERR << "Error checking model and IO: time " << t
                 << " is required but not in the input.";
     }
   }
 }

◆ CompileConvolutionComputation()

void CompileConvolutionComputation	(	const ConvolutionModel &	model,
		const std::vector< Index > &	input_indexes,
		const std::vector< Index > &	output_indexes,
		const ConvolutionComputationOptions &	opts,
		ConvolutionComputation *	computation,
		std::vector< Index > *	input_indexes_modified,
		std::vector< Index > *	output_indexes_modified
	)

This function does the compilation for a convolution computation; it's a wrapper for the functions below, which should not have to be called by the end user.

Parameters

[in]	model	The convolution model that this computation is for.
[in]	input_indexes	The list of Indexes available at the input of the computation.
[in]	output_indexes	The list of Indexes requested to be computed at the output of the computation. It is an error if all dependencies are not satisfied (specifically: for each Index (n,t,x) in 'output_indexes', the Index (n,t+time_offset,x) must be present in 'input_indexes' for each time_offset in model.required_time_offsets.
[out]	computation	If non-NULL, the compiled computation will be written to this location.

Definition at line 1367 of file convolution.cc.

References AppendInputFrames(), CheckModelAndIo(), GetComputationIo(), GetIndexesForComputation(), MakeComputation(), PadComputationInputTime(), PadModelHeight(), and UnPadModelHeight().

Referenced by TimeHeightConvolutionComponent::PrecomputeIndexes(), TimeHeightConvolutionComponent::ReorderIndexes(), and UnitTestTimeHeightConvolutionCompile().

                                                {
 
   // stage zero [preparing the input and output in a regular grid.]
   ConvolutionComputationIo io;
   GetComputationIo(input_indexes, output_indexes, &io);
 
   CheckModelAndIo(model, io, false);
 
   // stage 1.
   PadComputationInputTime(model, &io);
 
   CheckModelAndIo(model, io, false);
 
   // stage 2.
   ConvolutionModel model_padded;
   PadModelHeight(model, &model_padded);
 
   CheckModelAndIo(model_padded, io, false);
 
   // stage 3.
   ConvolutionModel model_appended;
   ConvolutionComputationIo io_appended;
   // make a 'fake' model and io for possibly-appended input frames.  'io' is
   // non-const because we may need to pad with a few extra frames.
   AppendInputFrames(model_padded, &io,
                     &model_appended, &io_appended);
 
   CheckModelAndIo(model_appended, io_appended, true);
 
   // stage 4.
   MakeComputation(model_appended, io_appended, opts, computation);
 
   // 'reverse' of stage 2.  [stage 3 kind of does its own
   // 'reverse' by modifying its input IO object.]
   // The computation is still specified for the appended input,
   // but the execution code can figure that out itself.
   UnPadModelHeight(opts, model, model_padded, computation);
 
   GetIndexesForComputation(io, input_indexes, output_indexes,
                            input_indexes_modified, output_indexes_modified);
 }

◆ ComputeTempMatrixSize()

static void kaldi::nnet3::time_height_convolution::ComputeTempMatrixSize	(	const ConvolutionComputationOptions &	opts,
		ConvolutionComputation *	computation
	)

static

This function sets 'temp_rows' and 'temp_cols' in 'computation'.

Definition at line 956 of file convolution.cc.

References ConvolutionComputation::height_in, ConvolutionComputation::ConvolutionStep::height_map, rnnlm::i, KALDI_WARN, ConvolutionComputationOptions::max_memory_mb, ConvolutionComputation::num_filters_in, ConvolutionComputation::num_images, ConvolutionComputation::num_t_out, ConvolutionComputation::steps, ConvolutionComputation::temp_cols, ConvolutionComputation::temp_rows, and VectorIsContiguous().

Referenced by MakeComputation(), and UnPadModelHeight().

                                                                        {
   int32 temp_rows = 0, temp_cols = 0;
   for (size_t i = 0; i < computation->steps.size(); i++) {
     const ConvolutionComputation::ConvolutionStep &step = computation->steps[i];
     int32 height_map_size = step.height_map.size(),
         this_num_cols = height_map_size * computation->num_filters_in;
     bool columns_are_contiguous =
         (step.height_map[0] != -1 && VectorIsContiguous(step.height_map));
     bool need_temp_matrix = true;
     if (columns_are_contiguous && step.height_map[0] == 0 &&
         this_num_cols == computation->num_filters_in * computation->height_in) {
       // the only situation in which we wouldn't need the temporary matrix
       // for this step, is where the columns are all of the input matrix.
       need_temp_matrix = false;
     }
     if (need_temp_matrix && this_num_cols > temp_cols)
       temp_cols = this_num_cols;
   }
   if (temp_cols > 0) {
     // work out how many rows the temporary matrix should have, taking
     // into account the specified memory limit.
     temp_rows = computation->num_t_out * computation->num_images;
     BaseFloat num_megabytes = (4 * (temp_rows / 1000.0) * (temp_cols / 1000.0)),
         megabyte_limit = opts.max_memory_mb;
     // C++ rounds down; here, we want to round up so we add one.
     int32 ratio = 1.0 + num_megabytes / megabyte_limit;
 
     // divide the number of time steps into 'ratio' pieces that are as equal as
     // possible; round up when dividing, to make sure that new_temp_rows * ratio
     // >= temp_rows so that we don't have a small leftover piece.
     int32 new_num_t_out = (computation->num_t_out + ratio - 1) / ratio;
     temp_rows = new_num_t_out * computation->num_images;
     BaseFloat new_num_megabytes = (4 * (temp_rows / 1000.0) * (temp_cols / 1000.0));
     // make sure we're within the memory limit.
     if (new_num_megabytes > 1.01 * megabyte_limit) {
       KALDI_WARN << "Memory consumed in convolution is more than requested "
                  << "(maybe very long time sequence?)";
     }
   }
   computation->temp_rows = temp_rows;
   computation->temp_cols = temp_cols;
 
 }

◆ ConvolveBackwardData()

void ConvolveBackwardData	(	const ConvolutionComputation &	conv_comp,
		const CuMatrixBase< BaseFloat > &	params,
		const CuMatrixBase< BaseFloat > &	output_deriv,
		CuMatrixBase< BaseFloat > *	input_deriv
	)

This does the part of the backward derivative computation of convolution, that propagates derivatives back to the input data.

See also ConvolveBackwardParams(), which is for the parameter derivative.

Parameters

[in]	conv_comp	A struct that describes the convolution computation (should be the same as in the corresponding forward pass).
[in]	params	The parameters used in the forward convolution. This should be of dimension num_filters_out by (X * num_filters_in), where X is the total number of pixels in the patches, which equals model.offsets.size() in the model for which the computation was compiled. E.g. for a regular 3x3 kernel, X would be 9.
[in]	output_deriv	The derivative of the objective function w.r.t. the output of the convolution. Should be of dimension conv_comp.num_t_out * conv_comp.num_images by conv_comp.height_out num_filters_out. It must satisfy output_deriv.NumCols() == output_deriv.Stride().
[out]	input_deriv	If non-NULL, the backpropagated derivative of the objective function w.r.t. the input will be added to this matrix. Should be the same dimension as the input to the original ConvolveForward() call.

Definition at line 682 of file convolution.cc.

Referenced by TimeHeightConvolutionComponent::Backprop(), and TestDataBackprop().

                                           {
   KALDI_ASSERT(input_deriv->NumCols() == input_deriv->Stride() &&
                output_deriv.NumCols() == output_deriv.Stride());
   KALDI_ASSERT(params.NumRows() == cc.num_filters_out);
   KALDI_ASSERT(output_deriv.NumRows() == cc.num_t_out * cc.num_images &&
                output_deriv.NumCols() == cc.height_out * cc.num_filters_out);
   // the input might need to be reshaped but we can check its total size.
   KALDI_ASSERT(input_deriv->NumRows() * input_deriv->NumCols() ==
                cc.num_images * cc.num_t_in * cc.height_in * cc.num_filters_in);
 
   int32 input_rows = input_deriv->NumRows(),
       required_input_rows = cc.num_images * cc.num_t_in;
 
   // this if-statement handles reshaping the input and recursing if there
   // is subsampling.
   if (input_rows != required_input_rows) {
     if (input_rows % required_input_rows != 0)
       KALDI_ERR << "Input matrix has wrong size.";  // error in calling code.
     // nr is a multiple of required_nr.  Reshape the matrix.
     // we already checked that its Stride() == NumCols();
     int32 num_cols = input_deriv->NumCols(),
         multiple = input_rows / required_input_rows,
         new_num_cols = num_cols * multiple,
         new_stride = new_num_cols;
     CuSubMatrix<BaseFloat> input_deriv_reshaped(
         input_deriv->Data(), required_input_rows,
         new_num_cols, new_stride);
     ConvolveBackwardData(cc, params, output_deriv, &input_deriv_reshaped);
     return;
   }
 
   CuMatrix<BaseFloat> temp_mat(cc.temp_rows, cc.temp_cols,
                                kSetZero, kStrideEqualNumCols);
 
   // this if-statement handles breaking up the arguments
   // and the computation into row-ranges if the temporary
   // matrix would have been excessively large, and we've decided
   // to give it fewer rows than the output (this saves
   // memory).  normally we won't take this if-statement
   // so ignore it if you're trying to understand the framework.
   if (cc.temp_rows != 0 && cc.temp_rows != input_rows) {
     KALDI_ASSERT(cc.temp_rows % cc.num_images == 0);
     int32 num_time_steps_per_chunk = cc.temp_rows / cc.num_images;
     int32 num_extra_in = cc.num_t_in - cc.num_t_out;
 
     for (int32 t_start = 0; t_start < cc.num_t_out;
          t_start += num_time_steps_per_chunk) {
       int32 num_t_left = cc.num_t_out - t_start,
           this_num_t_out = std::min<int32>(num_t_left,
                                            num_time_steps_per_chunk),
           this_num_t_in = this_num_t_out + num_extra_in;
       CuSubMatrix<BaseFloat> input_deriv_part(
           *input_deriv, t_start * cc.num_images,
           this_num_t_in * cc.num_images,
           0, input_deriv->NumCols());
       CuSubMatrix<BaseFloat> output_deriv_part(
           output_deriv, t_start * cc.num_images,
           this_num_t_out * cc.num_images,
           0, output_deriv.NumCols());
       CuSubMatrix<BaseFloat> temp_part(
           temp_mat, 0, this_num_t_out * cc.num_images,
           0, temp_mat.NumCols());
       ConvolveBackwardDataInternal(cc, params, output_deriv_part,
                                    &temp_part, &input_deriv_part);
     }
     return;
   }
   ConvolveBackwardDataInternal(cc, params, output_deriv,
                                &temp_mat, input_deriv);
 }

◆ ConvolveBackwardDataInternal()

static void kaldi::nnet3::time_height_convolution::ConvolveBackwardDataInternal	(	const ConvolutionComputation &	cc,
		const CuMatrixBase< BaseFloat > &	params,
		const CuMatrixBase< BaseFloat > &	output_deriv,
		CuMatrixBase< BaseFloat > *	temp_mat,
		CuMatrixBase< BaseFloat > *	input_deriv
	)

static

Definition at line 603 of file convolution.cc.

Referenced by ConvolveBackwardData().

                                           {
   KALDI_ASSERT(temp_mat->Stride() == temp_mat->NumCols());
 
   // num_t_out supersedes cc.num_t_out (they'll only be different in
   // cases where we are doing the computation in pieces to save memory).
   int32 input_rows = input_deriv->NumRows(),
       output_rows = output_deriv.NumRows();
 
   KALDI_ASSERT(output_rows <= input_rows &&
                input_rows % cc.num_images == 0 &&
                output_rows % cc.num_images == 0);
 
   int32 num_steps = cc.steps.size();
   for (int32 s = 0; s < num_steps; s++) {
     const ConvolutionComputation::ConvolutionStep &step = cc.steps[s];
     int32 input_row_start = step.input_time_shift * cc.num_images;
     CuSubMatrix<BaseFloat> input_deriv_part(*input_deriv,
                                             input_row_start, output_rows,
                                             0, input_deriv->NumCols());
     int32 temp_num_cols = step.columns.Dim(),
         param_cols = temp_num_cols / cc.height_out;
     CuSubMatrix<BaseFloat> params_part(params,
                                        0, params.NumRows(),
                                        step.params_start_col,
                                        param_cols);
     CuSubMatrix<BaseFloat> output_deriv_reshaped(
         output_deriv.Data(), output_rows * cc.height_out,
         cc.num_filters_out, cc.num_filters_out);
 
     if (!step.columns_are_contiguous ||
         temp_num_cols != input_deriv->NumCols()) {
       // In most cases we will take this branch, where we have to propagate the
       // input-derivative via a temporary matrix.  (however, different steps may
       // require different num-cols of the temporary matrix, so we create
       // sub-parts of 'temp_mat'.
 
       // We create the sub-matrix 'temp_mat_part' in a lower-level way, using
       // pointers, because we need to ensure that its num-cols and the stride
       // are the same (this is necessary so that we can do reshaping in
       // ConvolutionReshapedMultiply()).
       CuSubMatrix<BaseFloat> temp_mat_part(temp_mat->Data(),
                                            temp_mat->NumRows(),
                                            temp_num_cols, temp_num_cols),
           temp_mat_part_reshaped(
               temp_mat_part.Data(), temp_mat_part.NumRows() * cc.height_out,
               temp_num_cols / cc.height_out, temp_num_cols / cc.height_out);
 
       temp_mat_part_reshaped.AddMatMat(1.0, output_deriv_reshaped, kNoTrans,
                                        params_part, kNoTrans, 0.0);
 
       if (!step.columns_are_contiguous) {
         for (size_t i = 0; i < step.backward_columns.size(); i++) {
           input_deriv_part.AddCols(temp_mat_part, step.backward_columns[i]);
         }
       } else {
         // we're just taking a sub-matrix of the input matrix, but we still need
         // to make a copy because we need the stride == num-cols (so that the
         // reshaping will work).
         int32 num_cols = step.columns.Dim();
         input_deriv_part.ColRange(step.first_column,
                                   num_cols).AddMat(1.0, temp_mat_part);
       }
     } else {
       CuSubMatrix<BaseFloat> input_deriv_reshaped(
           input_deriv_part.Data(), input_deriv_part.NumRows() * cc.height_out,
           input_deriv_part.NumCols() / cc.height_out,
           input_deriv_part.NumCols() / cc.height_out);
       input_deriv_reshaped.AddMatMat(1.0, output_deriv_reshaped, kNoTrans,
                                      params_part, kNoTrans, 1.0);
     }
   }
 }

◆ ConvolveBackwardParams()

void ConvolveBackwardParams	(	const ConvolutionComputation &	conv_comp,
		const CuMatrixBase< BaseFloat > &	input,
		const CuMatrixBase< BaseFloat > &	output_deriv,
		BaseFloat	alpha,
		CuMatrixBase< BaseFloat > *	params_deriv
	)

This does the part of the backward derivative computation of convolution, that computes derivatives w.r.t.

the parameters. See also ConvolveBackwardData(), which computes derivatives w.r.t. the input data.

Parameters

[in]	conv_comp	A struct that describes the computation that was performed in the forward pass.
[in]	input	The input to the original forward convolution. This should be of dimension (or should be reshapable to the dimension) conv_comp.num_t_in * conv_comp.num_images by conv_comp.height_in * num_filters_in. [highest-stride indexes come first in these multiplications]. It must satisfy input.NumCols() == input.Stride().
[in]	output_deriv	The derivative of the objective function w.r.t. the output of the convolution. Should be of dimension conv_comp.num_t_out * conv_comp.num_images by conv_comp.height_out num_filters_out. It must satisfy output_deriv.NumCols() == output_deriv.Stride().
[in]	alpha	This scalar is multiplied into the derivative when we add to params_deriv, i.e. params_deriv += alpha derivative.
[out]	params_deriv	The derivative of the objective function w.r.t the parameters (the 'params' given to the ConvolveForward function) is added to this location. This matrix should be of dimension conv_comp.NumRows() by conv_comp.NumCols().

Definition at line 840 of file convolution.cc.

Referenced by TestParamsBackprop(), TimeHeightConvolutionComponent::UpdateNaturalGradient(), and TimeHeightConvolutionComponent::UpdateSimple().

                                            {
   KALDI_ASSERT(input.NumCols() == input.Stride() &&
               output_deriv.NumCols() == output_deriv.Stride());
   KALDI_ASSERT(params_deriv->NumRows() == cc.num_filters_out);
   KALDI_ASSERT(output_deriv.NumRows() == cc.num_t_out * cc.num_images &&
                output_deriv.NumCols() == cc.height_out * cc.num_filters_out);
   // the input might need to be reshaped but we can check its total size.
   KALDI_ASSERT(input.NumRows() * input.NumCols() == cc.num_images *
                cc.num_t_in * cc.height_in * cc.num_filters_in);
 
   int32 input_rows = input.NumRows(),
       required_input_rows = cc.num_images * cc.num_t_in;
 
   // this if-statement handles reshaping the input and recursing if there
   // is subsampling.
   if (input_rows != required_input_rows) {
     if (input_rows % required_input_rows != 0)
       KALDI_ERR << "Input matrix has wrong size.";  // error in calling code.
     // nr is a multiple of required_nr.  Reshape the matrix.
     // we already checked that its Stride() == NumCols();
     int32 num_cols = input.NumCols(),
         multiple = input_rows / required_input_rows,
         new_num_cols = num_cols * multiple,
         new_stride = new_num_cols;
     CuSubMatrix<BaseFloat> input_reshaped(
         input.Data(), required_input_rows, new_num_cols, new_stride);
     ConvolveBackwardParams(cc, input_reshaped, output_deriv, alpha,
                            params_deriv);
     return;
   }
 
   CuMatrix<BaseFloat> temp_mat(cc.temp_rows, cc.temp_cols,
                                kUndefined, kStrideEqualNumCols);
 
   // this if-statement handles breaking up the arguments
   // and the computation into row-ranges if the temporary
   // matrix would have been excessively large, and we've decided
   // to give it fewer rows than the output (this saves
   // memory).  normally we won't take this if-statement
   // so ignore it if you're trying to understand the framework.
   if (cc.temp_rows != 0 && cc.temp_rows != input_rows) {
     KALDI_ASSERT(cc.temp_rows % cc.num_images == 0);
     int32 num_time_steps_per_chunk = cc.temp_rows / cc.num_images;
     int32 num_extra_in = cc.num_t_in - cc.num_t_out;
 
     for (int32 t_start = 0; t_start < cc.num_t_out;
          t_start += num_time_steps_per_chunk) {
       int32 num_t_left = cc.num_t_out - t_start,
           this_num_t_out = std::min<int32>(num_t_left,
                                            num_time_steps_per_chunk),
           this_num_t_in = this_num_t_out + num_extra_in;
       CuSubMatrix<BaseFloat> input_part(
           input, t_start * cc.num_images,
           this_num_t_in * cc.num_images,
           0, input.NumCols());
       CuSubMatrix<BaseFloat> output_deriv_part(
           output_deriv, t_start * cc.num_images,
           this_num_t_out * cc.num_images,
           0, output_deriv.NumCols());
       CuSubMatrix<BaseFloat> temp_part(temp_mat,
                                        0, this_num_t_out * cc.num_images,
                                        0, temp_mat.NumCols());
       ConvolveBackwardParamsInternal(cc, input_part, output_deriv_part,
                                      alpha, &temp_part, params_deriv);
     }
     return;
   }
   ConvolveBackwardParamsInternal(cc, input, output_deriv,
                                  alpha, &temp_mat, params_deriv);
 }

◆ ConvolveBackwardParamsInternal()

static void kaldi::nnet3::time_height_convolution::ConvolveBackwardParamsInternal	(	const ConvolutionComputation &	cc,
		const CuMatrixBase< BaseFloat > &	input,
		const CuMatrixBase< BaseFloat > &	output_deriv,
		BaseFloat	alpha,
		CuMatrixBase< BaseFloat > *	temp_mat,
		CuMatrixBase< BaseFloat > *	params_deriv
	)

static

Definition at line 763 of file convolution.cc.

Referenced by ConvolveBackwardParams().

                                            {
   KALDI_ASSERT(temp_mat->Stride() == temp_mat->NumCols());
 
   // num_t_out supersedes cc.num_t_out (they'll only be different in
   // cases where we are doing the computation in pieces to save memory).
   int32 input_rows = input.NumRows(),
       output_rows = output_deriv.NumRows();
 
   KALDI_ASSERT(output_rows <= input_rows &&
                input_rows % cc.num_images == 0 &&
                output_rows % cc.num_images == 0);
 
   int32 num_steps = cc.steps.size();
   for (int32 s = 0; s < num_steps; s++) {
     const ConvolutionComputation::ConvolutionStep &step = cc.steps[s];
     int32 input_row_start = step.input_time_shift * cc.num_images;
     // note: 'input_part' will normally be almost all of 'input', perhaps
     // minus one or two time steps at the start or end.
     CuSubMatrix<BaseFloat> input_part(input,
                                       input_row_start, output_rows,
                                       0, input.NumCols());
     int32 temp_num_cols = step.columns.Dim(),
         param_cols = temp_num_cols / cc.height_out;
     CuSubMatrix<BaseFloat> params_deriv_part(*params_deriv,
                                        0, params_deriv->NumRows(),
                                        step.params_start_col,
                                        param_cols);
     CuSubMatrix<BaseFloat> output_deriv_reshaped(
         output_deriv.Data(), output_rows * cc.height_out,
         cc.num_filters_out, cc.num_filters_out);
     if (!step.columns_are_contiguous ||
         temp_num_cols != input.NumCols()) {
       // In most cases we will take this branch, where we have to copy the input
       // to a temporary matrix.  (however, different steps may require different
       // num-cols of the temporary matrix, so we create sub-parts of 'temp_mat'.
 
       // We create the sub-matrix 'temp_mat_part' in a lower-level way, using
       // pointers, because we need to ensure that its num-cols and the stride
       // are the same (this is necessary so that we can do reshaping in
       // ConvolutionReshapedMultiply()).
       CuSubMatrix<BaseFloat> temp_mat_part(temp_mat->Data(),
                                            temp_mat->NumRows(),
                                            temp_num_cols, temp_num_cols);
       if (!step.columns_are_contiguous) {
         // we're doing a column mapping.
         temp_mat_part.CopyCols(input_part, step.columns);
       } else {
         // we're just taking a sub-matrix of the input matrix, but we still need
         // to make a copy because we need the stride == num-cols (so that the
         // reshaping will work).
         temp_mat_part.CopyFromMat(input_part.ColRange(step.first_column,
                                                       step.columns.Dim()));
       }
       CuSubMatrix<BaseFloat> temp_mat_part_reshaped(
           temp_mat_part.Data(), temp_mat_part.NumRows() * cc.height_out,
           temp_num_cols / cc.height_out, temp_num_cols / cc.height_out);
 
       params_deriv_part.AddMatMat(alpha, output_deriv_reshaped, kTrans,
                                   temp_mat_part_reshaped, kNoTrans, 1.0);
     } else {
       CuSubMatrix<BaseFloat> input_reshaped(
           input_part.Data(), input_part.NumRows() * cc.height_out,
           input_part.NumCols() / cc.height_out,
           input_part.NumCols() / cc.height_out);
 
       params_deriv_part.AddMatMat(alpha, output_deriv_reshaped, kTrans,
                                   input_reshaped, kNoTrans, 1.0);
     }
   }
 }

◆ ConvolveForward()

void ConvolveForward	(	const ConvolutionComputation &	conv_comp,
		const CuMatrixBase< BaseFloat > &	input,
		const CuMatrixBase< BaseFloat > &	params,
		CuMatrixBase< BaseFloat > *	output
	)

This does the forward computation of convolution.

(note: this is convolution without a bias term; you have to handle that separately).

Parameters

[in]	conv_comp	A struct that describes the computation to be performed.
[in]	input	The input to the convolution. This should be of dimension (or should be reshapable to the dimension) conv_comp.num_t_in * conv_comp.num_images by conv_comp.height_in * num_filters_in. [highest-stride indexes come first in these multiplications]. It must satisfy input.NumCols() == input.Stride().
[in]	params	The parameters of the convolution. This should be of dimension conv_comp.ParamRows() by conv_comp.ParamCols().
[out]	output	The output of the convolution (this function adds to* the output). Should be of dimension conv_comp.num_t_out * conv_comp.num_images by conv_comp.height_out * num_filters_out. It must satisfy output.NumCols() == output.Stride().

Definition at line 524 of file convolution.cc.

Referenced by TimeHeightConvolutionComponent::Propagate(), TestDataBackprop(), TestParamsBackprop(), and TestRunningComputation().

                                      {
   KALDI_ASSERT(input.NumCols() == input.Stride() &&
                output->NumCols() == output->Stride());
   KALDI_ASSERT(params.NumRows() == cc.num_filters_out);
   KALDI_ASSERT(output->NumRows() == cc.num_t_out * cc.num_images &&
                output->NumCols() == cc.height_out * cc.num_filters_out);
   // the input might need to be reshaped but we can check its total size.
   KALDI_ASSERT(input.NumRows() * input.NumCols() == cc.num_images *
                cc.num_t_in * cc.height_in * cc.num_filters_in);
 
   int32 input_rows = input.NumRows(),
       required_input_rows = cc.num_images * cc.num_t_in;
 
   // this if-statement handles reshaping the input and recursing if there
   // is subsampling.
   if (input_rows != required_input_rows) {
     if (input_rows % required_input_rows != 0)
       KALDI_ERR << "Input matrix has wrong size.";  // error in calling code.
     // nr is a multiple of required_nr.  Reshape the matrix.
     // we already checked that its Stride() == NumCols();
     int32 num_cols = input.NumCols(),
         multiple = input_rows / required_input_rows,
         new_num_cols = num_cols * multiple,
         new_stride = new_num_cols;
     CuSubMatrix<BaseFloat> input_reshaped(
         input.Data(), required_input_rows, new_num_cols, new_stride);
     ConvolveForward(cc, input_reshaped, params, output);
     return;
   }
 
   CuMatrix<BaseFloat> temp_mat(cc.temp_rows, cc.temp_cols,
                                kUndefined, kStrideEqualNumCols);
 
   // this if-statement handles breaking up the arguments
   // and the computation into row-ranges if the temporary
   // matrix would have been excessively large, and we've decided
   // to give it fewer rows than the output (this saves
   // memory).  normally we won't take this if-statement
   // so ignore it if you're trying to understand the framework.
   if (cc.temp_rows != 0 && cc.temp_rows != input_rows) {
     KALDI_ASSERT(cc.temp_rows % cc.num_images == 0);
     int32 num_time_steps_per_chunk = cc.temp_rows / cc.num_images;
     int32 num_extra_in = cc.num_t_in - cc.num_t_out;
 
     for (int32 t_start = 0; t_start < cc.num_t_out;
          t_start += num_time_steps_per_chunk) {
       int32 num_t_left = cc.num_t_out - t_start,
           this_num_t_out = std::min<int32>(num_t_left,
                                            num_time_steps_per_chunk),
           this_num_t_in = this_num_t_out + num_extra_in;
       CuSubMatrix<BaseFloat> input_part(input, t_start * cc.num_images,
                                         this_num_t_in * cc.num_images,
                                         0, input.NumCols());
       CuSubMatrix<BaseFloat> output_part(*output, t_start * cc.num_images,
                                          this_num_t_out * cc.num_images,
                                          0, output->NumCols());
       CuSubMatrix<BaseFloat> temp_part(temp_mat, 0,
                                        this_num_t_out * cc.num_images,
                                        0, temp_mat.NumCols());
       ConvolveForwardInternal(cc, input_part, params,
                               &temp_part, &output_part);
     }
     return;
   }
   ConvolveForwardInternal(cc, input, params, &temp_mat, output);
 }

◆ ConvolveForwardInternal()

static void kaldi::nnet3::time_height_convolution::ConvolveForwardInternal	(	const ConvolutionComputation &	cc,
		const CuMatrixBase< BaseFloat > &	input,
		const CuMatrixBase< BaseFloat > &	params,
		CuMatrixBase< BaseFloat > *	temp_mat,
		CuMatrixBase< BaseFloat > *	output
	)

static

Definition at line 448 of file convolution.cc.

Referenced by ConvolveForward().

                                      {
   KALDI_ASSERT(temp_mat->Stride() == temp_mat->NumCols());
 
   // num_t_out supersedes cc.num_t_out (they'll only be different in
   // cases where we are doing the computation in pieces to save memory).
   int32 input_rows = input.NumRows(),
       output_rows = output->NumRows();
 
   KALDI_ASSERT(output_rows <= input_rows &&
                input_rows % cc.num_images == 0 &&
                output_rows % cc.num_images == 0);
 
   int32 num_steps = cc.steps.size();
   for (int32 s = 0; s < num_steps; s++) {
     const ConvolutionComputation::ConvolutionStep &step = cc.steps[s];
     int32 input_row_start = step.input_time_shift * cc.num_images;
     // note: 'input_part' will normally be almost all of 'input', perhaps
     // minus one or two time steps at the start or end.
     CuSubMatrix<BaseFloat> input_part(input,
                                       input_row_start, output_rows,
                                       0, input.NumCols());
     int32 temp_num_cols = step.columns.Dim(),
         param_cols = temp_num_cols / cc.height_out;
     CuSubMatrix<BaseFloat> params_part(params,
                                        0, params.NumRows(),
                                        step.params_start_col,
                                        param_cols);
     CuSubMatrix<BaseFloat> output_reshaped(
         output->Data(), output_rows * cc.height_out,
         cc.num_filters_out, cc.num_filters_out);
     if (!step.columns_are_contiguous ||
         temp_num_cols != input.NumCols()) {
       // In most cases we will take this branch, where we have to copy the input
       // to a temporary matrix.  (however, different steps may require different
       // num-cols of the temporary matrix, so we create sub-parts of 'temp_mat'.
 
       // We create the sub-matrix 'temp_mat_part' in a lower-level way, using
       // pointers, because we need to ensure that its num-cols and the stride
       // are the same (this is necessary so that we can do reshaping in
       // ConvolutionReshapedMultiply()).
       CuSubMatrix<BaseFloat> temp_mat_part(temp_mat->Data(),
                                            temp_mat->NumRows(),
                                            temp_num_cols, temp_num_cols);
       if (!step.columns_are_contiguous) {
         // we're doing a column mapping.
         temp_mat_part.CopyCols(input_part, step.columns);
       } else {
         // we're just taking a sub-matrix of the input matrix, but we still need
         // to make a copy because we need the stride == num-cols (so that the
         // reshaping will work).
         temp_mat_part.CopyFromMat(input_part.ColRange(step.first_column,
                                                       step.columns.Dim()));
       }
       CuSubMatrix<BaseFloat> temp_mat_part_reshaped(
           temp_mat_part.Data(), temp_mat_part.NumRows() * cc.height_out,
           temp_num_cols / cc.height_out, temp_num_cols / cc.height_out);
 
       output_reshaped.AddMatMat(1.0, temp_mat_part_reshaped, kNoTrans,
                                 params_part, kTrans, 1.0);
     } else {
       CuSubMatrix<BaseFloat> input_reshaped(
           input_part.Data(), input_part.NumRows() * cc.height_out,
           input_part.NumCols() / cc.height_out,
           input_part.NumCols() / cc.height_out);
 
       output_reshaped.AddMatMat(1.0, input_reshaped, kNoTrans,
                                 params_part, kTrans, 1.0);
     }
   }
 }

◆ ConvolveForwardSimple()

void kaldi::nnet3::time_height_convolution::ConvolveForwardSimple	(	const ConvolutionModel &	model,
		const std::vector< Index > &	input_indexes,
		const std::vector< Index > &	output_indexes,
		const CuMatrixBase< BaseFloat > &	input_cu,
		const CuMatrixBase< BaseFloat > &	params_cu,
		CuMatrixBase< BaseFloat > *	output_cu
	)

Definition at line 218 of file convolution-test.cc.

References CuMatrixBase< Real >::CopyFromMat(), ConvolutionModel::height_in, ConvolutionModel::height_out, ConvolutionModel::height_subsample_out, kaldi::nnet3::kNoTime, kaldi::kNoTrans, ConvolutionModel::num_filters_in, ConvolutionModel::num_filters_out, MatrixBase< Real >::NumRows(), ConvolutionModel::offsets, and Index::t.

Referenced by TestRunningComputation().

                                         {
   // these loops will be very slow on GPU, so do it all on CPU.
   Matrix<BaseFloat> input(input_cu), params(params_cu),
       output(*output_cu);
   std::unordered_map<Index, int32, IndexHasher> index_to_row;
   int32 input_rows = input.NumRows(),
       output_rows = output.NumRows();
   for (int32 r_in = 0; r_in < input_rows; r_in++) {
     if (input_indexes[r_in].t != kNoTime) {
       index_to_row[input_indexes[r_in]] = r_in;
     }
   }
   int32 num_offsets = model.offsets.size(),
       num_filters_in = model.num_filters_in,
       num_filters_out = model.num_filters_out,
       height_in = model.height_in,
       height_out = model.height_out,
       height_subsample_out = model.height_subsample_out;
   for (int32 r_out = 0; r_out < output_rows; r_out++) {
     Index index_out = output_indexes[r_out];
     if (index_out.t == kNoTime)
       continue;
     SubVector<BaseFloat> output_row(output, r_out);
     for (int32 o = 0; o < num_offsets; o++) {
       int32 time_offset = model.offsets[o].time_offset,
           height_offset = model.offsets[o].height_offset;
       Index index_in(index_out);
       index_in.t += time_offset;
       std::unordered_map<Index, int32, IndexHasher>::const_iterator iter =
           index_to_row.find(index_in);
       if (iter != index_to_row.end()) {
         SubMatrix<BaseFloat> params_part(params, 0, params.NumRows(),
                                          o * num_filters_in, num_filters_in);
         int32 r_in = iter->second;
         SubVector<BaseFloat> input_row(input, r_in);
         for (int32 h_out_subsampled = 0;
              h_out_subsampled < height_out;
              h_out_subsampled++) {
           int32 h_out = h_out_subsampled * height_subsample_out,
               h_in = h_out + height_offset;
           if (h_in < 0 || h_in >= height_in)
             continue;
           SubVector<BaseFloat> output_part(output_row,
                                            h_out_subsampled * num_filters_out,
                                            num_filters_out),
               input_part(input_row, h_in * num_filters_in, num_filters_in);
           output_part.AddMatVec(1.0, params_part, kNoTrans, input_part, 1.0);
         }
       }
     }
   }
   output_cu->CopyFromMat(output);
 }

◆ CreateIndexes()

static void kaldi::nnet3::time_height_convolution::CreateIndexes	(	const std::vector< std::pair< int32, int32 > > &	n_x_pairs,
		int32	t_start,
		int32	t_step,
		int32	num_t_values,
		int32	reorder_t,
		std::vector< Index > *	indexes
	)

static

Creates a vector of indexes with a regular structure, according to these specifications.

'n_x_pairs' is the list of (n,x) pairs to include; they will appear in this order.

't_start', 't_step' and 'num_t_values' define the set of 't' values to include (note: t_step >= 0; they will appear in the natural order).

If reorder_t == 1 (the normal case), then the order is simple: 't' has the higher stride, then (n, x). So we'll output first all (n, x) pairs for t_start, then all pairs for t_start + t_step, and so on.

If instead reorder_t > 1, then the order is a little different [note: we expect that num_t_values % reorder_t == 0). Consider, for example, reorder_t == 2. In that case the first block has the first two t values, the second block has the next two t values, and so on. And within each block, the 't' values have the smallest stride (of 1).

Definition at line 1470 of file convolution.cc.

References KALDI_ASSERT, and Index::n.

Referenced by GetIndexesForComputation(), and BlockFactorizedTdnnComponent::NumOutputBlocks().

                                                                       {
   KALDI_ASSERT(reorder_t >= 1 && num_t_values % reorder_t == 0 && t_step >= 0);
   if (t_step == 0) {
     KALDI_ASSERT(num_t_values == 1);
     t_step = 1;
   }
   int32 num_n_x_pairs = n_x_pairs.size();
   indexes->clear();
   indexes->reserve(num_n_x_pairs * num_t_values);
   int32 outer_t_step = t_step * reorder_t,
       t_end = t_start + (num_t_values * t_step);
   Index index;
   for (int32 t_block = t_start; t_block < t_end; t_block += outer_t_step) {
     for (int32 nx = 0; nx < num_n_x_pairs; nx++) {
       index.n = n_x_pairs[nx].first;
       index.x = n_x_pairs[nx].second;
       for (int32 t = t_block; t < t_block + outer_t_step; t += t_step) {
         index.t = t;
         indexes->push_back(index);
       }
     }
   }
   // we can remove the next assert after a while.
   KALDI_ASSERT(indexes->size() == num_n_x_pairs * num_t_values);
 }

◆ FindGcdOfDifferences()

static int32 kaldi::nnet3::time_height_convolution::FindGcdOfDifferences ( std::vector< int32 > & vec )

static

Definition at line 1420 of file convolution.cc.

References kaldi::Gcd(), and rnnlm::i.

Referenced by RegularizeTList().

                                                          {
   size_t size = vec.size();
   int32 ans = 0;
   for (size_t i = 0; i + 1 < size; i++) {
     int32 diff = vec[i+1] - vec[i];
     // diff should not be zero.
     ans = Gcd(ans, diff);
   }
   return ans;
 }

◆ GetComputationIo()

void GetComputationIo	(	const std::vector< Index > &	input_indexes,
		const std::vector< Index > &	output_indexes,
		ConvolutionComputationIo *	io
	)

This function takes lists of input and output indexes to a computation (e.g.

as supplied to ReorderIndexes()), and figures out a regular structure for them (i.e. the smallest grid that will completely cover all the t,n pairs). This function ignores any 't' values that are kNoTime.

Definition at line 1519 of file convolution.cc.

References kaldi::nnet3::GetNxList(), kaldi::nnet3::GetTList(), kaldi::GetVerboseLevel(), KALDI_ASSERT, ConvolutionComputationIo::num_images, ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::num_t_out, RegularizeTList(), ConvolutionComputationIo::reorder_t_in, ConvolutionComputationIo::start_t_in, ConvolutionComputationIo::start_t_out, ConvolutionComputationIo::t_step_in, and ConvolutionComputationIo::t_step_out.

Referenced by CompileConvolutionComputation(), RestrictedAttentionComponent::GetComputationStructure(), TdnnComponent::PrecomputeIndexes(), and TdnnComponent::ReorderIndexes().

                                   {
   std::vector<std::pair<int32, int32> > n_x_pairs;
   GetNxList(input_indexes, &n_x_pairs);
   KALDI_ASSERT(!n_x_pairs.empty());
   io->num_images = n_x_pairs.size();
   if (GetVerboseLevel() >= 3) {  // a debugging step.
     std::vector<std::pair<int32, int32> > n_x_pairs_2;
     GetNxList(output_indexes, &n_x_pairs_2);
     KALDI_ASSERT(n_x_pairs_2 == n_x_pairs);
   }
   std::vector<int32> t_values;
   GetTList(input_indexes, &t_values);
   RegularizeTList(t_values, &(io->start_t_in),
                   &(io->t_step_in), &(io->num_t_in));
   GetTList(output_indexes, &t_values);
   RegularizeTList(t_values, &(io->start_t_out),
                   &(io->t_step_out), &(io->num_t_out));
   io->reorder_t_in = 1;
 }

◆ GetIndexesForComputation()

void GetIndexesForComputation	(	const ConvolutionComputationIo &	io,
		const std::vector< Index > &	orig_input_indexes,
		const std::vector< Index > &	orig_output_indexes,
		std::vector< Index > *	input_indexes,
		std::vector< Index > *	output_indexes
	)

This function computes the reordered and possibly padded indexes corresponding to the computation in 'io'.

Note: the computation may have undergone various manipulations (padding, etc.) after being obtained by the function GetComputationIo(). The original input and output indexes are needed because they dictate the set of (n, x) pairs; and because they determine when to use 'real' indexes and when to use 'blank' padding values (i.e. when to replace the t values in the indexes by kNoTime).

Definition at line 1543 of file convolution.cc.

References CreateIndexes(), kaldi::nnet3::GetNxList(), KALDI_ASSERT, ConvolutionComputationIo::num_images, ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::num_t_out, ConvolutionComputationIo::reorder_t_in, SetSomeIndexesBlank(), ConvolutionComputationIo::start_t_in, ConvolutionComputationIo::start_t_out, ConvolutionComputationIo::t_step_in, and ConvolutionComputationIo::t_step_out.

Referenced by CompileConvolutionComputation(), TdnnComponent::PrecomputeIndexes(), and TdnnComponent::ReorderIndexes().

                                       {
   std::unordered_set<Index, IndexHasher> input_set, output_set;
   for (std::vector<Index>::const_iterator iter = orig_input_indexes.begin();
        iter != orig_input_indexes.end(); ++iter)
     input_set.insert(*iter);
   for (std::vector<Index>::const_iterator iter = orig_output_indexes.begin();
        iter != orig_output_indexes.end(); ++iter)
     output_set.insert(*iter);
   std::vector<std::pair<int32, int32> > n_x_pairs;
   GetNxList(orig_input_indexes, &n_x_pairs);
   KALDI_ASSERT(n_x_pairs.size() == io.num_images);
   CreateIndexes(n_x_pairs, io.start_t_in, io.t_step_in, io.num_t_in,
                 io.reorder_t_in, input_indexes);
   SetSomeIndexesBlank(orig_input_indexes, input_indexes);
   CreateIndexes(n_x_pairs, io.start_t_out, io.t_step_out, io.num_t_out,
                 1, output_indexes);
   SetSomeIndexesBlank(orig_output_indexes, output_indexes);
 }

◆ GetRandomConvolutionIndexes()

static void kaldi::nnet3::time_height_convolution::GetRandomConvolutionIndexes	(	const ConvolutionModel &	model,
		std::vector< Index > *	input_indexes,
		std::vector< Index > *	output_indexes
	)

static

Definition at line 78 of file convolution-test.cc.

References ConvolutionModel::all_time_offsets, ConvolutionModel::Check(), rnnlm::i, kaldi::IsSortedAndUniq(), rnnlm::j, KALDI_ASSERT, Index::n, rnnlm::n, kaldi::RandInt(), ConvolutionModel::required_time_offsets, kaldi::SortAndUniq(), Index::t, and Index::x.

Referenced by UnitTestTimeHeightConvolutionCompile().

                                                                           {
   KALDI_ASSERT(model.Check());
 
   std::vector<std::pair<int32, int32> > n_x_pairs;
   int32 num_n_x_pairs = RandInt(1, 3);
   for (int32 i = 0; i < num_n_x_pairs; i++) {
     int32 n = RandInt(0, 3), x = RandInt(0, 1);
     n_x_pairs.push_back(std::pair<int32, int32>(n, x));
   }
   SortAndUniq(&n_x_pairs);
   num_n_x_pairs = n_x_pairs.size();
 
 
   // 'output_t_values' is the set of *possible* output
   // t values; we'll later sub-sample from these.
   std::vector<int32> output_t_values;
 
   {
     int32 out_t_start = RandInt(-5, 5), out_t_step = RandInt(1, 3),
         num_t_out = RandInt(1, 4);
     for (int32 i = 0; i < num_t_out; i++)
       output_t_values.push_back(out_t_start + i * out_t_step);
   }
 
   input_indexes->clear();
   output_indexes->clear();
   for (size_t i = 0; i < n_x_pairs.size(); i++) {
     std::vector<int32> chosen_output_t_values;
     while (chosen_output_t_values.empty()) {
       for (size_t j = 0; j < output_t_values.size(); j++)
         if (RandInt(0, 1) != 0)
           chosen_output_t_values.push_back(output_t_values[j]);
     }
     KALDI_ASSERT(IsSortedAndUniq(chosen_output_t_values));
 
     std::set<int32> required_input_t_values,
         usable_input_t_values;
     for (size_t j = 0; j < chosen_output_t_values.size(); j++) {
       std::set<int32>::const_iterator iter;
       int32 t_out = chosen_output_t_values[j];
       for (iter = model.required_time_offsets.begin();
            iter != model.required_time_offsets.end(); iter++) {
         int32 offset = *iter;
         required_input_t_values.insert(t_out + offset);
       }
       for (iter = model.all_time_offsets.begin();
            iter != model.all_time_offsets.end(); iter++) {
         int32 offset = *iter;
         usable_input_t_values.insert(t_out + offset);
       }
     }
 
     // add to output_indexes
     for (size_t j = 0; j < chosen_output_t_values.size(); j++) {
       int32 t_out = chosen_output_t_values[j];
       Index index;
       index.n = n_x_pairs[i].first;
       index.x = n_x_pairs[i].second;
       index.t = t_out;
       output_indexes->push_back(index);
     }
 
     std::vector<int32> chosen_input_t_values(required_input_t_values.begin(),
                                              required_input_t_values.end());
     for (std::set<int32>::const_iterator iter = usable_input_t_values.begin();
          iter != usable_input_t_values.end(); ++iter) {
       int32 t = *iter;
       if (RandInt(0, 1) == 0)
         chosen_input_t_values.push_back(t);
     }
     SortAndUniq(&chosen_input_t_values);
 
     // add to input_indexes
     for (size_t j = 0; j < chosen_input_t_values.size(); j++) {
       int32 t_in = chosen_input_t_values[j];
       Index index;
       index.n = n_x_pairs[i].first;
       index.x = n_x_pairs[i].second;
       index.t = t_in;
       input_indexes->push_back(index);
     }
   }
 }

◆ GetRandomConvolutionModel()

static void kaldi::nnet3::time_height_convolution::GetRandomConvolutionModel ( ConvolutionModel * model )

static

Definition at line 28 of file convolution-test.cc.

References ConvolutionModel::Check(), ConvolutionModel::ComputeDerived(), ConvolutionModel::height_in, ConvolutionModel::Offset::height_offset, ConvolutionModel::height_out, ConvolutionModel::height_subsample_out, rnnlm::i, ConvolutionModel::Info(), KALDI_WARN, ConvolutionModel::num_filters_in, ConvolutionModel::num_filters_out, ConvolutionModel::offsets, kaldi::RandInt(), ConvolutionModel::required_time_offsets, kaldi::SortAndUniq(), and ConvolutionModel::Offset::time_offset.

Referenced by UnitTestTimeHeightConvolutionCompile(), and UnitTestTimeHeightConvolutionIo().

                                                                {
 start:
   {
     model->num_filters_in = RandInt(1, 10);
     model->num_filters_out = RandInt(1, 10);
     model->height_in = RandInt(1, 10);
     int32 min_height_offset = RandInt(-2, 0),
         max_height_offset = RandInt(0, 2),
         min_time_offset = RandInt(-2, 0),
         max_time_offset = RandInt(0, 2);
 
     model->height_out = RandInt(1, model->height_in);
     model->height_subsample_out = 1;
     if (RandInt(0, 1) == 0) {
       if (model->height_out % 2 == 0) {
         model->height_out /= 2;
         model->height_subsample_out = 2;
       } else if (model->height_out % 3 == 0) {
         model->height_out /= 3;
         model->height_subsample_out = 3;
       }
     }
     std::vector<int32> all_time_offsets;
     int32 max_offsets = RandInt(1, 10);
     model->offsets.clear();
     model->required_time_offsets.clear();
     for (int32 i = 0; i < max_offsets; i++) {
       ConvolutionModel::Offset o;
       o.time_offset = RandInt(min_time_offset, max_time_offset);
       o.height_offset = RandInt(min_height_offset, max_height_offset);
       all_time_offsets.push_back(o.time_offset);
       model->offsets.push_back(o);
     }
     SortAndUniq(&(model->offsets));
     SortAndUniq(&all_time_offsets);
     std::random_shuffle(all_time_offsets.begin(), all_time_offsets.end());
     int32 num_required_offsets = RandInt(1, all_time_offsets.size());
     for (int32 i = 0; i < num_required_offsets; i++)
       model->required_time_offsets.insert(all_time_offsets[i]);
     model->ComputeDerived();
   }
   if (!model->Check()) {
     KALDI_WARN << "Regenerating model because it didn't pass the check: "
                << model->Info();
     goto start;
   }
 }

◆ MakeComputation()

void MakeComputation	(	const ConvolutionModel &	model,
		ConvolutionComputationIo &	io,
		const ConvolutionComputationOptions &	opts,
		ConvolutionComputation *	computation
	)

Definition at line 1568 of file convolution.cc.

Referenced by CompileConvolutionComputation().

                                                           {
   KALDI_ASSERT(io.t_step_in == io.t_step_out);
   computation->num_filters_in = model.num_filters_in;
   computation->num_filters_out = model.num_filters_out;
   computation->height_in = model.height_in;
   computation->height_out = model.height_out;
   computation->num_t_in = io.num_t_in;
   computation->num_t_out = io.num_t_out;
   computation->num_images = io.num_images;
   KALDI_ASSERT(io.reorder_t_in == 1);
   // first work out the steps of the computation, then
   // work out the dim of the temp matrix
 
   KALDI_ASSERT(IsSortedAndUniq(model.offsets));
   // Each distinct value of 'time_offset' in model.offsets
   // becomes one step of the computation.
 
   // if io.t_step_in was zero, use 1 (so divisions and the like will work as
   // expected).
   int32 t_step = std::max<int32>(1, io.t_step_in),
       num_t_extra = io.num_t_in - io.num_t_out;
 
   computation->steps.clear();
 
   int32 num_offsets = model.offsets.size(),
       cur_start_offset = 0, cur_end_offset = 0;
   for(; cur_start_offset < num_offsets; cur_start_offset = cur_end_offset) {
     cur_end_offset = cur_start_offset;
     while (cur_end_offset < num_offsets &&
            model.offsets[cur_end_offset].time_offset ==
            model.offsets[cur_start_offset].time_offset)
       cur_end_offset++;
     // we are processing the range of indexes into 'offsets'
     // from cur_start_offset to cur_end_offset - 1.
     int32 this_num_offsets = cur_end_offset - cur_start_offset;
     int32 time_offset = model.offsets[cur_start_offset].time_offset;
 
     ConvolutionComputation::ConvolutionStep step;
     // modified_time_offset will be used in working out the 'input_time_shift'
     // that determines which submatrix of the input matrix we'll use.
     // It equals the time-offset corrected for any time-difference between
     // the start of the output and of the input.
     int32 modified_time_offset = time_offset + io.start_t_out - io.start_t_in;
     KALDI_ASSERT(modified_time_offset >= 0 &&
                  modified_time_offset % t_step == 0);
     step.input_time_shift = modified_time_offset / t_step;
     KALDI_ASSERT(step.input_time_shift <= num_t_extra);
     step.params_start_col = model.num_filters_in * cur_start_offset;
     step.height_map.clear();
     step.height_map.reserve(model.height_out * this_num_offsets);
     for (int32 h_out = 0;
          h_out < model.height_out * model.height_subsample_out;
          h_out += model.height_subsample_out) {
       for (int32 o = cur_start_offset; o < cur_end_offset; o++) {
         int32 this_height_offset = model.offsets[o].height_offset,
             h_in = h_out + this_height_offset;
         // by the time we call MakeComputation, the user should already have
         // called PadModelHeight, so there should be no need for zero padding on
         // the height axis, hence the following check.  [we'll later modify the
         // resulting computation in UnPadModelHeight, and that's where
         // zero-padding gets taken account of.]
         KALDI_ASSERT(h_in >= 0 && h_in < model.height_in);
         step.height_map.push_back(h_in);
       }
     }
     computation->steps.push_back(step);
   }
   ComputeTempMatrixSize(opts, computation);
 }

◆ PadComputationInputTime()

void PadComputationInputTime	(	const ConvolutionModel &	model,
		ConvolutionComputationIo *	io
	)

This function extends the set of input indexes that the computation has, to account for any required zero-padding in the time dimension.

It reads model.all_time_offsets and model.time_offsets_modulus; and it may modify members start_t_in t_stride_in and num_t_in of *io.

This is stage 1 of compilation.

Definition at line 1051 of file convolution.cc.

References ConvolutionModel::all_time_offsets, kaldi::Gcd(), KALDI_ASSERT, ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::num_t_out, ConvolutionComputationIo::start_t_in, ConvolutionComputationIo::start_t_out, ConvolutionComputationIo::t_step_in, ConvolutionComputationIo::t_step_out, and ConvolutionModel::time_offsets_modulus.

Referenced by CompileConvolutionComputation().

                                                            {
   if (model.time_offsets_modulus == 0) {
     // this can only happen if model->all_time_offsets.size() == 1,
     // and no padding could be required here. W return to avoid
     // special cases below in Gcd().
     return;
   }
   int32 min_time_offset = *model.all_time_offsets.begin(),
       max_time_offset = *model.all_time_offsets.rbegin();
 
   // it makes everything much simpler if we just enforce that the stride of the
   // input divides model.time_offsets_modulus and also the output stride.
   // (enforcing this may make the input stride smaller).  This may in certain
   // very odd cases cause us to require more inputs [actually 'blanks'] than
   // we really need, but it avoids a lot of careful thought.
   int32 old_t_step_in = io->t_step_in;
   io->t_step_in = Gcd(io->t_step_in, model.time_offsets_modulus);
   if (io->t_step_out != 0)
     io->t_step_in = Gcd(io->t_step_in, io->t_step_out);
 
   // to ensure that we cover all the original input points, now that
   // we changed the stride we may need to increase num_t_in.
   io->num_t_in = 1 + (old_t_step_in * (io->num_t_in - 1)) / io->t_step_in;
 
   // by 'desired' we mean usable as an input, not necessarily
   // required in the sense of 'required_time_offsets'.
   int32 first_desired_input_t = io->start_t_out + min_time_offset;
   if (first_desired_input_t < io->start_t_in) {
     KALDI_ASSERT((io->start_t_in - first_desired_input_t) %
                  io->t_step_in == 0);
     io->num_t_in += (io->start_t_in - first_desired_input_t) / io->t_step_in;
     io->start_t_in = first_desired_input_t;
   }
 
   int32 last_desired_input_t =
       io->start_t_out + (io->num_t_out - 1) * io->t_step_out + max_time_offset,
       last_input_t = io->start_t_in + (io->num_t_in - 1) * io->t_step_in;
   // if the following assert fails, it means we had provided more input than was
   // needed, which is not expected.  This could cause problems later, in
   // AppendInputFrames().
   KALDI_ASSERT(last_desired_input_t >= last_input_t);
   if (last_desired_input_t > last_input_t) {
     KALDI_ASSERT((last_desired_input_t - last_input_t) %
                  io->t_step_in == 0);
     io->num_t_in += (last_desired_input_t - last_input_t) / io->t_step_in;
   }
 }

◆ PadModelHeight()

void PadModelHeight	(	const ConvolutionModel &	model,
		ConvolutionModel *	model_padded
	)

This function takes a model that might require zero padding in the height dimension and outputs a model accepting a possibly-larger input dimension which does not require zero padding.

*model_padded may differ from 'model' in its height_in and its 'offsets' variable (the height-offsets need to be shifted if we pad at the bottom). We then work out the computation in terms of the model that doesn't need padding (which is easier), and later convert it back to work in the space where there is no padding.

This is stage 2 of compilation.

Definition at line 918 of file convolution.cc.

References ConvolutionModel::Check(), ConvolutionModel::height_in, ConvolutionModel::height_out, ConvolutionModel::height_subsample_out, rnnlm::i, KALDI_ASSERT, and ConvolutionModel::offsets.

Referenced by CompileConvolutionComputation().

                                                     {
   *model_padded = model;
   KALDI_ASSERT(!model.offsets.empty());
   int32 min_height_offset = model.offsets[0].height_offset,
       max_height_offset = model.offsets[0].height_offset,
       num_offsets = model.offsets.size();
   for (int32 i = 1; i < num_offsets; i++) {
     min_height_offset = std::min<int32>(min_height_offset,
                                         model.offsets[i].height_offset);
     max_height_offset = std::max<int32>(max_height_offset,
                                         model.offsets[i].height_offset);
   }
   int32 max_output_height = model.height_subsample_out * (model.height_out - 1),
       max_required_input = max_height_offset + max_output_height,
       min_required_input = min_height_offset + 0;
   int32 bottom_padding = -min_required_input,
       top_padding = max_required_input - (model.height_in - 1);
   if (bottom_padding < 0)
     bottom_padding = 0;
   if (top_padding < 0)
     top_padding = 0;
   model_padded->height_in += bottom_padding + top_padding;
   for (int32 i = 0; i < num_offsets; i++)
     model_padded->offsets[i].height_offset += bottom_padding;
 
   // The reason why we say 'allow_height_padding = false' below is obvious--
   // we've 'manually' padded by changing the model, so this modified model
   // should not require height padding.  The reason we set 'check_heights_used'
   // is a little more non-obvious.  The very lowest and hightest heights
   // should always be used, but there may, in unusual models, be other heights
   // that are not used.  We found this in random testing.
   KALDI_ASSERT(model_padded->Check(false, false));
 }

◆ PrepareIoForAppending()

static int32 kaldi::nnet3::time_height_convolution::PrepareIoForAppending	(	ConvolutionComputationIo *	io,
		ConvolutionComputationIo *	io_appended
	)

static

Definition at line 1158 of file convolution.cc.

References KALDI_ASSERT, ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::num_t_out, ConvolutionComputationIo::reorder_t_in, ConvolutionComputationIo::t_step_in, and ConvolutionComputationIo::t_step_out.

Referenced by AppendInputFrames().

                                                                           {
   // first make sure that the output has nonzero stride (it would only have zero
   // stride if there was only one output time index, which is unusual).  if
   // there's only one output time index we can set the stride to whatever we
   // want without affecting the list of output indexes.
   int32 ratio;
   if (io->t_step_out == 0) {
     KALDI_ASSERT(io->num_t_out == 1);
     io->t_step_out = io->t_step_in;
   }
   if (io->t_step_out == io->t_step_in) {
     // there is nothing to do; the output and input strides are the same.
     *io_appended = *io;
     ratio = 1;
     return ratio;
   }
   // Now, we ensured in PadComputationInputTime that if the output stride is
   // nonzero, then the input stride must divide the output stride; and if the
   // output stride was zero then we would have set it to the input stride just
   // above; and if both were zero we would have returned above.  So we can just
   // assert that the input stride divides the output stride.
   KALDI_ASSERT(io->t_step_out % io->t_step_in == 0);
   ratio = io->t_step_out / io->t_step_in;
   // ratio says how many input indexes we have for each output index,
   // ignoring end effects.  It is the number of input indexes we will
   // append together and 'pretend'
 
   // record this ratio in the 'input' I/O object, which we are also
   // modifying to record the extra required padding.
   io->reorder_t_in = ratio;
   if (io->num_t_in % ratio != 0) {
     // Round up the number of input frames to the nearest multiple (via
     // zero-padding) so we get an whole number of appended input frames.
     io->num_t_in += ratio - (io->num_t_in % ratio);
   }
 
   // OK, from this point we create the output io object.
   *io_appended = *io;
   io_appended->reorder_t_in = 1;
   io_appended->t_step_in = io->t_step_out;
   io_appended->num_t_in /= ratio;
   return ratio;
 }

◆ RegularizeTList()

static void kaldi::nnet3::time_height_convolution::RegularizeTList	(	std::vector< int32 > &	t_values,
		int32 *	start,
		int32 *	step,
		int32 *	num_values
	)

static

Definition at line 1431 of file convolution.cc.

References FindGcdOfDifferences(), kaldi::IsSortedAndUniq(), and KALDI_ASSERT.

Referenced by GetComputationIo().

                                                {
   KALDI_ASSERT(!t_values.empty() && IsSortedAndUniq(t_values));
   *start = t_values[0];
   *step = FindGcdOfDifferences(t_values);
   if (*step == 0) {
     KALDI_ASSERT(t_values.size() == 1);
     *num_values = 1;
   } else {
     int32 last_value = t_values.back();
     *num_values = 1 + (last_value - *start) / *step;
     KALDI_ASSERT((last_value - *start) % *step == 0);
   }
 }

◆ ReverseColumnMapping()

static void kaldi::nnet3::time_height_convolution::ReverseColumnMapping	(	const std::vector< int32 > &	columns,
		int32	input_dim,
		std::vector< std::vector< int32 > > *	backward_columns
	)

static

This function, used in ConvolutionComputation::ComputeDerived(), reverses a mapping that may not be unique.

'columns' is a column mapping where each member is either -1 (meaning, copy a zero), or a number between 0 and input_dim - 1.

Its output, 'backward_columns', is the reverse mapping, but it's a vector of vectors instead of just a vector because the mapping may have been many-to-one. Each element of 'backward_columns' will be of dimension input_dim. For each columns[i] = j such that j != -1, for some k we will have (*backward_columns)[k][j] = i.

Definition at line 44 of file convolution.cc.

References rnnlm::i, rnnlm::j, and KALDI_ASSERT.

Referenced by ConvolutionComputation::ComputeDerived().

                                                     {
   int32 columns_dim = columns.size();
   std::vector<std::vector<int32> > temp(input_dim);
   for (int32 i = 0; i < columns_dim; i++) {
     int32 j = columns[i];
     KALDI_ASSERT(j >= -1 && j < input_dim);
     if (j != -1)
       temp[j].push_back(i);
   }
   // 'max_overlap' is the largest number of times that some j >= 0 appears in
   // 'columns'.
   int32 max_overlap = 0;
   for (int32 j = 0; j < input_dim; j++)
     max_overlap = std::max(max_overlap,
                            static_cast<int32>(temp[j].size()));
   backward_columns->resize(max_overlap);
   for (int32 k = 0; k < max_overlap; k++) {
     (*backward_columns)[k].clear();
     (*backward_columns)[k].resize(input_dim, -1);
   }
   for (int32 j = 0; j < input_dim; j++) {
     for (int32 k = 0; k < static_cast<int32>(temp[j].size()); k++) {
       int32 i = temp[j][k];
       (*backward_columns)[k][j] = i;
     }
   }
 }

◆ RoundDownToMultipleOf()

static int32 kaldi::nnet3::time_height_convolution::RoundDownToMultipleOf	(	int32	i,
		int32	n
	)

static

Definition at line 1103 of file convolution.cc.

References kaldi::DivideRoundingDown().

Referenced by AppendInputFrames().

                                                      {
   return n * DivideRoundingDown(i, n);
 }

◆ SetSomeIndexesBlank()

static void kaldi::nnet3::time_height_convolution::SetSomeIndexesBlank	(	const std::vector< Index > &	ref_indexes,
		std::vector< Index > *	indexes
	)

static

This function modifies 'indexes' by, for any Indexes which was not present in 'ref_indexes', setting the 't' value to kNoTime.

This will cause the nnet3 framework to ignore such Indexes for certain purposes, it supresses certain error conditions that would otherwise happen from inserting unnecessary indexes into the input and output.

Definition at line 1505 of file convolution.cc.

References kaldi::nnet3::kNoTime.

Referenced by GetIndexesForComputation().

                                                            {
   std::unordered_set<Index, IndexHasher> ref_set;
   for (std::vector<Index>::const_iterator iter = ref_indexes.begin();
        iter != ref_indexes.end(); ++iter)
     ref_set.insert(*iter);
 
   for (std::vector<Index>::iterator iter = indexes->begin();
        iter != indexes->end(); ++iter) {
     if (ref_set.count(*iter) == 0)
       iter->t = kNoTime;
   }
 }

◆ ShiftAllTimeOffsets()

static void kaldi::nnet3::time_height_convolution::ShiftAllTimeOffsets	(	int32	shift,
		ConvolutionModel *	model
	)

static

Definition at line 1110 of file convolution.cc.

References ConvolutionModel::all_time_offsets, ConvolutionModel::offsets, and ConvolutionModel::required_time_offsets.

Referenced by AppendInputFrames().

                                                          {
   { // shift 'offsets'.
     std::vector<ConvolutionModel::Offset>::iterator
         iter = model->offsets.begin(),
         end = model->offsets.end();
     for (; iter != end; ++iter)
       iter->time_offset += shift;
   }
   std::set<int32> temp;
   std::set<int32>::const_iterator iter;
   for (iter = model->required_time_offsets.begin();
        iter != model->required_time_offsets.end(); ++iter)
     temp.insert(*iter + shift);
   model->required_time_offsets.swap(temp);
   temp.clear();
   for (iter = model->all_time_offsets.begin();
        iter != model->all_time_offsets.end(); ++iter)
     temp.insert(*iter + shift);
   model->all_time_offsets.swap(temp);
 }

◆ TestComputationIo()

void kaldi::nnet3::time_height_convolution::TestComputationIo ( const ConvolutionComputation & computation )

Definition at line 182 of file convolution-test.cc.

References KALDI_ASSERT, kaldi::RandInt(), ConvolutionComputation::Read(), and ConvolutionComputation::Write().

Referenced by UnitTestTimeHeightConvolutionCompile().

                                                                   {
   std::ostringstream os1, os2;
   bool binary = (RandInt(0, 1) == 0);
   computation.Write(os1, binary);
   std::istringstream is(os1.str());
   ConvolutionComputation computation2;
   computation2.Read(is, binary);
   computation2.Write(os2, binary);
   KALDI_ASSERT(os1.str() == os2.str());
   computation2.Check();
 }

◆ TestDataBackprop()

void kaldi::nnet3::time_height_convolution::TestDataBackprop	(	const ConvolutionModel &	conv_model,
		const std::vector< Index > &	input_indexes,
		const std::vector< Index > &	output_indexes,
		const ConvolutionComputation &	computation
	)

Definition at line 308 of file convolution-test.cc.

References kaldi::ApproxEqual(), ConvolveBackwardData(), ConvolveForward(), ConvolutionModel::InputDim(), KALDI_ERR, KALDI_LOG, kaldi::kSetZero, kaldi::kStrideEqualNumCols, kaldi::kTrans, ConvolutionModel::OutputDim(), ConvolutionModel::ParamCols(), ConvolutionModel::ParamRows(), CuMatrixBase< Real >::SetRandn(), kaldi::TraceMatMat(), and ZeroBlankRows().

Referenced by UnitTestTimeHeightConvolutionCompile().

                                                                  {
   CuMatrix<BaseFloat>
       input_deriv(input_indexes.size(), conv_model.InputDim(),
                   kSetZero, kStrideEqualNumCols),
       input(input_indexes.size(), conv_model.InputDim(),
             kSetZero, kStrideEqualNumCols),
       output(output_indexes.size(), conv_model.OutputDim(),
              kSetZero, kStrideEqualNumCols),
       output_deriv(output_indexes.size(), conv_model.OutputDim(),
                    kSetZero, kStrideEqualNumCols),
       params(conv_model.ParamRows(), conv_model.ParamCols());
 
   input.SetRandn();
   params.SetRandn();
   output_deriv.SetRandn();
 
   ZeroBlankRows(output_indexes, &output_deriv);
   ConvolveBackwardData(computation, params, output_deriv, &input_deriv);
   ZeroBlankRows(input_indexes, &input_deriv);
   ZeroBlankRows(input_indexes, &input);
 
   // define the objf as TraceMatMat(output_deriv, output, kTrans).
   // we can work it out from the backpropagated data-derivative.
   BaseFloat expected_objf = TraceMatMat(input_deriv, input, kTrans);
 
   ConvolveForward(computation, input, params, &output);
   ZeroBlankRows(output_indexes, &output);
 
   BaseFloat observed_objf = TraceMatMat(output, output_deriv, kTrans);
 
   KALDI_LOG << "Expected objf = " << expected_objf
             << ", observed objf = " << observed_objf;
   if (!ApproxEqual(expected_objf, observed_objf, 0.1) &&
       fabs(expected_objf) < 1.0) {
     KALDI_ERR << "Difference in objf too large.";
   }
 }

◆ TestParamsBackprop()

void kaldi::nnet3::time_height_convolution::TestParamsBackprop	(	const ConvolutionModel &	conv_model,
		const std::vector< Index > &	input_indexes,
		const std::vector< Index > &	output_indexes,
		const ConvolutionComputation &	computation
	)

Definition at line 350 of file convolution-test.cc.

References kaldi::ApproxEqual(), ConvolveBackwardParams(), ConvolveForward(), ConvolutionModel::InputDim(), KALDI_ERR, KALDI_LOG, kaldi::kSetZero, kaldi::kStrideEqualNumCols, kaldi::kTrans, ConvolutionModel::OutputDim(), ConvolutionModel::ParamCols(), ConvolutionModel::ParamRows(), kaldi::RandInt(), CuMatrixBase< Real >::SetRandn(), kaldi::TraceMatMat(), and ZeroBlankRows().

Referenced by UnitTestTimeHeightConvolutionCompile().

                                                                    {
   CuMatrix<BaseFloat>
       input(input_indexes.size(), conv_model.InputDim(),
             kSetZero, kStrideEqualNumCols),
       output(output_indexes.size(), conv_model.OutputDim(),
              kSetZero, kStrideEqualNumCols),
       output_deriv(output_indexes.size(), conv_model.OutputDim(),
                    kSetZero, kStrideEqualNumCols),
       params(conv_model.ParamRows(), conv_model.ParamCols()),
       params_deriv(conv_model.ParamRows(), conv_model.ParamCols());
 
   input.SetRandn();
   params.SetRandn();
   output_deriv.SetRandn();
 
   BaseFloat alpha = 0.5 * RandInt(1, 3);
 
   ZeroBlankRows(output_indexes, &output_deriv);
   ZeroBlankRows(input_indexes, &input);
 
   ConvolveBackwardParams(computation, input, output_deriv, alpha,
                          &params_deriv);
 
   BaseFloat expected_objf = TraceMatMat(params_deriv, params, kTrans) / alpha;
 
   ConvolveForward(computation, input, params, &output);
 
   ZeroBlankRows(output_indexes, &output);
 
   BaseFloat observed_objf = TraceMatMat(output, output_deriv, kTrans);
 
   KALDI_LOG << "Expected objf = " << expected_objf
             << ", observed objf = " << observed_objf;
   if (!ApproxEqual(expected_objf, observed_objf, 0.1) &&
       fabs(expected_objf) < 1.0) {
     KALDI_ERR << "Difference in objf too large.";
   }
 }

◆ TestRunningComputation()

void kaldi::nnet3::time_height_convolution::TestRunningComputation	(	const ConvolutionModel &	conv_model,
		const std::vector< Index > &	input_indexes,
		const std::vector< Index > &	output_indexes,
		const ConvolutionComputation &	computation
	)

Definition at line 280 of file convolution-test.cc.

References ConvolveForward(), ConvolveForwardSimple(), ConvolutionModel::Info(), ConvolutionModel::InputDim(), KALDI_ERR, KALDI_LOG, kaldi::kSetZero, kaldi::kStrideEqualNumCols, ConvolutionModel::OutputDim(), ConvolutionModel::ParamCols(), ConvolutionModel::ParamRows(), CuMatrixBase< Real >::SetRandn(), and ZeroBlankRows().

Referenced by UnitTestTimeHeightConvolutionCompile().

                                                                        {
   CuMatrix<BaseFloat> input(input_indexes.size(), conv_model.InputDim(),
                             kSetZero, kStrideEqualNumCols),
       output(output_indexes.size(), conv_model.OutputDim(),
              kSetZero, kStrideEqualNumCols),
       output2(output),
       params(conv_model.ParamRows(), conv_model.ParamCols());
   input.SetRandn();
   params.SetRandn();
   ZeroBlankRows(input_indexes, &input);
   ConvolveForward(computation, input, params, &output);
   ZeroBlankRows(output_indexes, &output);
 
   ConvolveForwardSimple(conv_model, input_indexes, output_indexes,
                         input, params, &output2);
   KALDI_LOG << "Tested convolution for model: "
             << conv_model.Info();
   if (!output.ApproxEqual(output2, 0.001)) {
     KALDI_LOG << "Output is: " << output;
     KALDI_LOG << "Output2 is: " << output2;
     KALDI_ERR << "Convolution test failure.";
   }
 }

◆ TimeValueInInput()

static bool kaldi::nnet3::time_height_convolution::TimeValueInInput	(	const ConvolutionComputationIo &	io,
		int32	t
	)

static

Definition at line 1321 of file convolution.cc.

References ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::start_t_in, and ConvolutionComputationIo::t_step_in.

Referenced by CheckModelAndIo().

                                       {
   int32 t_step_in = std::max<int32>(1, io.t_step_in);
   return (t >= io.start_t_in &&
           t < io.start_t_in + (t_step_in * io.num_t_in) &&
           (t - io.start_t_in) % t_step_in == 0);
 }

◆ UnitTestTimeHeightConvolution()

void kaldi::nnet3::time_height_convolution::UnitTestTimeHeightConvolution ( )

Definition at line 437 of file convolution-test.cc.

References UnitTestTimeHeightConvolutionCompile(), and UnitTestTimeHeightConvolutionIo().

Referenced by main().

                                      {
   UnitTestTimeHeightConvolutionIo();
   UnitTestTimeHeightConvolutionCompile();
 }

◆ UnitTestTimeHeightConvolutionCompile()

void kaldi::nnet3::time_height_convolution::UnitTestTimeHeightConvolutionCompile ( )

Definition at line 394 of file convolution-test.cc.

References CompileConvolutionComputation(), GetRandomConvolutionIndexes(), GetRandomConvolutionModel(), rnnlm::i, KALDI_LOG, TestComputationIo(), TestDataBackprop(), TestParamsBackprop(), TestRunningComputation(), and kaldi::nnet3::WriteIndexVector().

Referenced by UnitTestTimeHeightConvolution().

                                             {
   for (int32 i = 0; i < 10; i++) {
     KALDI_LOG << "iter = " << i;
     // Create a ConvolutionModel
     ConvolutionModel conv_model;
     GetRandomConvolutionModel(&conv_model);
     std::vector<Index> input_indexes, output_indexes;
     GetRandomConvolutionIndexes(conv_model, &input_indexes, &output_indexes);
 
     ConvolutionComputationOptions opts;
     ConvolutionComputation computation;
     std::vector<Index> input_indexes_modified, output_indexes_modified;
     CompileConvolutionComputation(conv_model, input_indexes, output_indexes,
                                   opts, &computation,
                                   &input_indexes_modified,
                                   &output_indexes_modified);
     TestComputationIo(computation);
     TestRunningComputation(conv_model,
                            input_indexes_modified,
                            output_indexes_modified,
                            computation);
     TestDataBackprop(conv_model,
                      input_indexes_modified,
                      output_indexes_modified,
                      computation);
     TestParamsBackprop(conv_model,
                        input_indexes_modified,
                        output_indexes_modified,
                        computation);
     std::ostringstream os;
     os << "\nInput-indexes: ";
     WriteIndexVector(os, false, input_indexes);
     os << "\nInput-indexes-modified: ";
     WriteIndexVector(os, false, input_indexes_modified);
     os << "\nOutput-indexes: ";
     WriteIndexVector(os, false, output_indexes);
     os << "\nOutput-indexes-modified: ";
     WriteIndexVector(os, false, output_indexes_modified);
     KALDI_LOG << os.str();
   }
 }

◆ UnitTestTimeHeightConvolutionIo()

void kaldi::nnet3::time_height_convolution::UnitTestTimeHeightConvolutionIo ( )

Definition at line 165 of file convolution-test.cc.

References GetRandomConvolutionModel(), rnnlm::i, KALDI_ASSERT, KALDI_LOG, kaldi::RandInt(), ConvolutionModel::Read(), and ConvolutionModel::Write().

Referenced by UnitTestTimeHeightConvolution().

                                        {
   for (int32 i = 0; i < 10; i++) {
     KALDI_LOG << "iter = " << i;
     // Create a ConvolutionModel and test its I/O.
     ConvolutionModel conv_model;
     GetRandomConvolutionModel(&conv_model);
     std::ostringstream os1, os2;
     bool binary = (RandInt(0, 1) == 0);
     conv_model.Write(os1, binary);
     std::istringstream is(os1.str());
     ConvolutionModel conv_model2;
     conv_model2.Read(is, binary);
     conv_model2.Write(os2, binary);
     KALDI_ASSERT(os1.str() == os2.str() && conv_model2.Check());
   }
 }

◆ UnPadModelHeight()

void UnPadModelHeight	(	const ConvolutionComputationOptions &	opts,
		const ConvolutionModel &	model,
		const ConvolutionModel &	model_padded,
		ConvolutionComputation *	computation
	)

This function modifies, if necessary, a computation that has been built for the model 'model_padded', so that it can work for the original model 'model'.

This may involve modifying the members 'height_in', 'temp_cols', and the column-related members of the elements of the 'steps' array. View it as the reverse step for 'PadModelHeight'.

This function has to be aware that the computation will have been compiled after 'AppendInputFrames()' was called [this makes a difference in setups with subsampling], so the computation may have been built for input frames that were appended over several of the frames that 'model_padded' would require.

This is the reverse step for stage 2 of compilation (it's a transformation of the computation).

Definition at line 1001 of file convolution.cc.

References ConvolutionComputation::Check(), ConvolutionComputation::ComputeDerived(), ComputeTempMatrixSize(), ConvolutionModel::height_in, ConvolutionComputation::height_in, ConvolutionComputation::ConvolutionStep::height_map, ConvolutionModel::height_out, ConvolutionComputation::height_out, rnnlm::i, KALDI_ASSERT, ConvolutionModel::offsets, and ConvolutionComputation::steps.

Referenced by CompileConvolutionComputation().

                                                            {
   // First work out how much padding was done in PadModelHeight().
   int32 bottom_padding = (model_padded.offsets[0].height_offset -
                           model.offsets[0].height_offset),
       total_padding = model_padded.height_in - model.height_in,
       top_padding = total_padding - bottom_padding;
 
   int32 old_computation_height_in = computation->height_in;
   // The computation may have been built for the input appended over
   // several frames. Check that it is for an input height that's a multiple of
   // the model input height.
   KALDI_ASSERT(old_computation_height_in % model_padded.height_in == 0 &&
                computation->height_out == model.height_out);
 
   // 'ratio' is the same ratio from AppendInputFrames(), it's the number
   // of input frames in 'model' and 'model_padded' that get appended
   // to form a single frame in the computation.
   int32 num_steps = computation->steps.size(),
       unpadded_input_height = model.height_in,
       padded_input_height = model_padded.height_in,
       ratio = old_computation_height_in / padded_input_height;
 
   computation->height_in = ratio * unpadded_input_height;
   for (int32 s = 0; s < num_steps; s++) {
     ConvolutionComputation::ConvolutionStep &step = computation->steps[s];
     int32 height_map_size = step.height_map.size();
     for (int32 i = 0; i < height_map_size; i++) {
       int32 c = step.height_map[i];
       KALDI_ASSERT(c >= 0);  // there should be no -1's in the padded computation.
       // below, h is the actual height in terms of the padded computation, and m
       // is an index that goes from zero to (num-appended-frames - 1).
       int32 h = c % padded_input_height,
           m = c / padded_input_height;
       KALDI_ASSERT(m < ratio);
       if (h < bottom_padding || h >= padded_input_height - top_padding) {
         step.height_map[i] = -1;
       } else {
         step.height_map[i] = (h - bottom_padding) + m * unpadded_input_height;
       }
     }
   }
   ComputeTempMatrixSize(opts, computation);
   computation->ComputeDerived();
   computation->Check();
 }

◆ VectorIsContiguous()

static bool kaldi::nnet3::time_height_convolution::VectorIsContiguous ( const std::vector< int32 > & vec )

static

Definition at line 77 of file convolution.cc.

References rnnlm::i, and KALDI_ASSERT.

Referenced by ConvolutionComputation::ComputeDerived(), and ComputeTempMatrixSize().

                                                             {
   KALDI_ASSERT(!vec.empty());
   int32 s = vec.size();
   for (int32 i = 0; i + 1 < s; i++)
     if (vec[i+1] != vec[i] + 1)
       return false;
   return true;
 }

◆ ZeroBlankRows()

void kaldi::nnet3::time_height_convolution::ZeroBlankRows	(	const std::vector< Index > &	indexes,
		CuMatrix< BaseFloat > *	matrix
	)

Definition at line 198 of file convolution-test.cc.

References VectorBase< Real >::Data(), KALDI_ASSERT, kaldi::nnet3::kNoTime, kaldi::kUndefined, CuMatrixBase< Real >::MulRowsVec(), CuMatrixBase< Real >::NumRows(), VectorBase< Real >::Set(), and CuVector< Real >::Swap().

Referenced by TestDataBackprop(), TestParamsBackprop(), and TestRunningComputation().

                                                 {
   KALDI_ASSERT(static_cast<int32>(indexes.size()) == matrix->NumRows());
   int32 num_rows = matrix->NumRows();
   if (num_rows == 0) return;
   Vector<BaseFloat> mask(num_rows, kUndefined);
   mask.Set(1.0);
   const Index *indexes_ptr = &(indexes[0]);
   BaseFloat *mask_ptr = mask.Data();
   for (int32 r = 0; r < num_rows; r++) {
     if (indexes_ptr[r].t == kNoTime)
       mask_ptr[r] = 0.0;
   }
   CuVector<BaseFloat> cu_mask;
   cu_mask.Swap(&mask);
   matrix->MulRowsVec(cu_mask);
 }

Classes

Functions

Function Documentation

◆ AppendInputFrames()

◆ CheckModelAndIo()

◆ CompileConvolutionComputation()

◆ ComputeTempMatrixSize()

◆ ConvolveBackwardData()

◆ ConvolveBackwardDataInternal()

◆ ConvolveBackwardParams()

◆ ConvolveBackwardParamsInternal()

◆ ConvolveForward()

◆ ConvolveForwardInternal()

◆ ConvolveForwardSimple()

◆ CreateIndexes()

◆ FindGcdOfDifferences()

◆ GetComputationIo()

◆ GetIndexesForComputation()

◆ GetRandomConvolutionIndexes()

◆ GetRandomConvolutionModel()

◆ MakeComputation()

◆ PadComputationInputTime()

◆ PadModelHeight()

◆ PrepareIoForAppending()

◆ RegularizeTList()

◆ ReverseColumnMapping()

◆ RoundDownToMultipleOf()

◆ SetSomeIndexesBlank()

◆ ShiftAllTimeOffsets()

◆ TestComputationIo()

◆ TestDataBackprop()

◆ TestParamsBackprop()

◆ TestRunningComputation()

◆ TimeValueInInput()

◆ UnitTestTimeHeightConvolution()

◆ UnitTestTimeHeightConvolutionCompile()

◆ UnitTestTimeHeightConvolutionIo()

◆ UnPadModelHeight()

◆ VectorIsContiguous()

◆ ZeroBlankRows()