All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
kaldi::nnet3::time_height_convolution Namespace Reference

Classes

struct  ConvolutionComputation
 This struct represents the structure of a convolution computation. More...
 
struct  ConvolutionComputationIo
 
struct  ConvolutionComputationOptions
 This struct contains options for compiling the convolutional computation. More...
 
struct  ConvolutionModel
 This comment explains the basic framework used for everything related to time-height convolution. More...
 

Functions

static void GetRandomConvolutionModel (ConvolutionModel *model)
 
static void GetRandomConvolutionIndexes (const ConvolutionModel &model, std::vector< Index > *input_indexes, std::vector< Index > *output_indexes)
 
void UnitTestTimeHeightConvolutionIo ()
 
void TestComputationIo (const ConvolutionComputation &computation)
 
void ZeroBlankRows (const std::vector< Index > &indexes, CuMatrix< BaseFloat > *matrix)
 
void ConvolveForwardSimple (const ConvolutionModel &model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const CuMatrixBase< BaseFloat > &input_cu, const CuMatrixBase< BaseFloat > &params_cu, CuMatrixBase< BaseFloat > *output_cu)
 
void TestRunningComputation (const ConvolutionModel &conv_model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputation &computation)
 
void TestDataBackprop (const ConvolutionModel &conv_model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputation &computation)
 
void TestParamsBackprop (const ConvolutionModel &conv_model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputation &computation)
 
void UnitTestTimeHeightConvolutionCompile ()
 
void UnitTestTimeHeightConvolution ()
 
static void ReverseColumnMapping (const std::vector< int32 > &columns, int32 input_dim, std::vector< std::vector< int32 > > *backward_columns)
 This function, used in ConvolutionComputation::ComputeDerived(), reverses a mapping that may not be unique. More...
 
static bool VectorIsContiguous (const std::vector< int32 > &vec)
 
static void ConvolveForwardInternal (const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &params, CuMatrixBase< BaseFloat > *temp_mat, CuMatrixBase< BaseFloat > *output)
 
void ConvolveForward (const ConvolutionComputation &conv_comp, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &params, CuMatrixBase< BaseFloat > *output)
 This does the forward computation of convolution. More...
 
static void ConvolveBackwardDataInternal (const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &params, const CuMatrixBase< BaseFloat > &output_deriv, CuMatrixBase< BaseFloat > *temp_mat, CuMatrixBase< BaseFloat > *input_deriv)
 
void ConvolveBackwardData (const ConvolutionComputation &conv_comp, const CuMatrixBase< BaseFloat > &params, const CuMatrixBase< BaseFloat > &output_deriv, CuMatrixBase< BaseFloat > *input_deriv)
 This does the part of the backward derivative computation of convolution, that propagates derivatives back to the input data. More...
 
static void ConvolveBackwardParamsInternal (const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &output_deriv, BaseFloat alpha, CuMatrixBase< BaseFloat > *temp_mat, CuMatrixBase< BaseFloat > *params_deriv)
 
void ConvolveBackwardParams (const ConvolutionComputation &conv_comp, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &output_deriv, BaseFloat alpha, CuMatrixBase< BaseFloat > *params_deriv)
 This does the part of the backward derivative computation of convolution, that computes derivatives w.r.t. More...
 
void PadModelHeight (const ConvolutionModel &model, ConvolutionModel *model_padded)
 This function takes a model that might require zero padding in the height dimension and outputs a model accepting a possibly-larger input dimension which does not require zero padding. More...
 
static void ComputeTempMatrixSize (const ConvolutionComputationOptions &opts, ConvolutionComputation *computation)
 This function sets 'temp_rows' and 'temp_cols' in 'computation'. More...
 
void UnPadModelHeight (const ConvolutionComputationOptions &opts, const ConvolutionModel &model, const ConvolutionModel &model_padded, ConvolutionComputation *computation)
 This function modifies, if necessary, a computation that has been built for the model 'model_padded', so that it can work for the original model 'model'. More...
 
void PadComputationInputTime (const ConvolutionModel &model, ConvolutionComputationIo *io)
 This function extends the set of input indexes that the computation has, to account for any required zero-padding in the time dimension. More...
 
static int32 RoundDownToMultipleOf (int32 i, int32 n)
 
static void ShiftAllTimeOffsets (int32 shift, ConvolutionModel *model)
 
static int32 PrepareIoForAppending (ConvolutionComputationIo *io, ConvolutionComputationIo *io_appended)
 
void AppendInputFrames (const ConvolutionModel &model, ConvolutionComputationIo *io, ConvolutionModel *model_appended, ConvolutionComputationIo *io_appended)
 This function takes an input model and I/O specification, and it modifies both of them if necessary to ensure that the output 'io_appended' object has the same input and output time strides (i.e. More...
 
static bool TimeValueInInput (const ConvolutionComputationIo &io, int32 t)
 
void CheckModelAndIo (const ConvolutionModel &model, const ConvolutionComputationIo &io, bool allow_extra_input=false)
 Check that this model and this I/O request are compatible in terms of required context, etc, and crash if not. More...
 
void CompileConvolutionComputation (const ConvolutionModel &model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputationOptions &opts, ConvolutionComputation *computation, std::vector< Index > *input_indexes_modified, std::vector< Index > *output_indexes_modified)
 This function does the compilation for a convolution computation; it's a wrapper for the functions below, which should not have to be called by the end user. More...
 
static void GetNxList (const std::vector< Index > &indexes, std::vector< std::pair< int32, int32 > > *pairs)
 
static void GetTList (const std::vector< Index > &indexes, std::vector< int32 > *t_values)
 
static int32 FindGcdOfDifferences (std::vector< int32 > &vec)
 
static void RegularizeTList (std::vector< int32 > &t_values, int32 *start, int32 *step, int32 *num_values)
 
static void CreateIndexes (const std::vector< std::pair< int32, int32 > > &n_x_pairs, int32 t_start, int32 t_step, int32 num_t_values, int32 reorder_t, std::vector< Index > *indexes)
 Creates a vector of indexes with a regular structure, according to these specifications. More...
 
static void SetSomeIndexesBlank (const std::vector< Index > &ref_indexes, std::vector< Index > *indexes)
 This function modifies 'indexes' by, for any Indexes which was not present in 'ref_indexes', setting the 't' value to kNoTime. More...
 
void GetComputationIo (const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, ConvolutionComputationIo *io)
 This function takes lists of input and output indexes to a computation (e.g. More...
 
void GetIndexesForComputation (const ConvolutionComputationIo &io, const std::vector< Index > &orig_input_indexes, const std::vector< Index > &orig_output_indexes, std::vector< Index > *input_indexes, std::vector< Index > *output_indexes)
 This function computes the reordered and possibly padded indexes corresponding to the computation in 'io'. More...
 
void MakeComputation (const ConvolutionModel &model, ConvolutionComputationIo &io, const ConvolutionComputationOptions &opts, ConvolutionComputation *computation)
 

Function Documentation

void AppendInputFrames ( const ConvolutionModel &  model,
ConvolutionComputationIo *  io,
ConvolutionModel *  model_appended,
ConvolutionComputationIo *  io_appended 
)

This function takes an input model and I/O specification, and it modifies both of them if necessary to ensure that the output 'io_appended' object has the same input and output time strides (i.e.

t_stride_in == t_stride_out). This is done by appending the input frames across several time values and viewing them as single frames of larger dimension.

The reason why 'io' is non-const is that it may be necessary to pad the number of input frames to ensure that the number of input frames is divisible by a multiple of t_stride_out / t_stride_in (if we pad the input frames, we pad to the right).

The model in 'model_appended' may have larger height_in, and different values of 'offsets' and derived variables thereof, versus the model in 'model'.

This is stage 3 of compilation.

Definition at line 1202 of file convolution.cc.

References ConvolutionModel::all_time_offsets, ConvolutionModel::Check(), ConvolutionModel::ComputeDerived(), ConvolutionModel::height_in, ConvolutionModel::Offset::height_offset, ConvolutionModel::height_out, ConvolutionModel::height_subsample_out, rnnlm::i, KALDI_ASSERT, ConvolutionModel::num_filters_in, ConvolutionModel::num_filters_out, ConvolutionModel::offsets, PrepareIoForAppending(), ConvolutionModel::required_time_offsets, RoundDownToMultipleOf(), ShiftAllTimeOffsets(), ConvolutionComputationIo::start_t_in, ConvolutionComputationIo::start_t_out, ConvolutionComputationIo::t_step_in, ConvolutionComputationIo::t_step_out, and ConvolutionModel::Offset::time_offset.

Referenced by CompileConvolutionComputation().

1205  {
1206  int32 ratio = PrepareIoForAppending(io, io_appended);
1207 
1208  if (ratio == 1) {
1209  // we are not doing any appending of frames.
1210  *model_appended = model;
1211  return;
1212  }
1213 
1214  // we also need the time-step of the output (which is also now the
1215  // time-step of the appended input).
1216  // We know that the time step is not zero, because in that case we would
1217  // have ratio == 1 and would have returned above.
1218  int32 time_step_out = io_appended->t_step_out;
1219  KALDI_ASSERT(time_step_out == io_appended->t_step_in && time_step_out != 0);
1220  int32 orig_time_step_in = io->t_step_in;
1221  KALDI_ASSERT(orig_time_step_in * ratio == time_step_out);
1222 
1223  // make sure the difference between first input and output frames is what we
1224  // expect, else something could go wrong here.
1225  int32 first_time_offset = *(model.all_time_offsets.begin());
1226  KALDI_ASSERT(io->start_t_in - io->start_t_out == first_time_offset);
1227 
1228  ConvolutionModel model_temp(model);
1229  // shift so that the first time offset is zero. this makes
1230  // the model conversion easier.
1231  ShiftAllTimeOffsets(-first_time_offset, &model_temp);
1232 
1233  model_appended->num_filters_in = model.num_filters_in;
1234  model_appended->num_filters_out = model.num_filters_out;
1235  model_appended->height_in = ratio * model.height_in;
1236  model_appended->height_out = model.height_out;
1237  model_appended->height_subsample_out = model.height_subsample_out;
1238  int32 num_offsets = model_temp.offsets.size(),
1239  old_height = model.height_in;
1240  model_appended->offsets.resize(num_offsets);
1241  model_appended->all_time_offsets.clear();
1242  for (int32 i = 0; i < num_offsets; i++) {
1243  const ConvolutionModel::Offset &old_offset = model_temp.offsets[i];
1244  ConvolutionModel::Offset &new_offset = model_appended->offsets[i];
1245  // The following two lines are important!! They are the core of how
1246  // we handle subsampling in this framework.
1247  new_offset.time_offset = RoundDownToMultipleOf(old_offset.time_offset,
1248  time_step_out);
1249  KALDI_ASSERT((old_offset.time_offset - new_offset.time_offset) %
1250  orig_time_step_in == 0);
1251  int32 row_offset = (old_offset.time_offset - new_offset.time_offset) /
1252  orig_time_step_in;
1253  new_offset.height_offset = old_offset.height_offset +
1254  row_offset * old_height;
1255  model_appended->all_time_offsets.insert(new_offset.time_offset);
1256  }
1257 
1258  // Because the 'appended' model will always be used after zero-padding on the
1259  // time axis, we can just pretend that all desired time-offsets are required.
1260  // It's a kind of free error-checking.
1261  model_appended->required_time_offsets = model_appended->all_time_offsets;
1262 
1263  // Undo the time-shifting that we did before.
1264  ShiftAllTimeOffsets(first_time_offset, model_appended);
1265 
1266  model_appended->ComputeDerived();
1267  KALDI_ASSERT(model_appended->Check(false, false));
1268 }
static int32 RoundDownToMultipleOf(int32 i, int32 n)
static int32 PrepareIoForAppending(ConvolutionComputationIo *io, ConvolutionComputationIo *io_appended)
static void ShiftAllTimeOffsets(int32 shift, ConvolutionModel *model)
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
void CheckModelAndIo ( const ConvolutionModel &  model,
const ConvolutionComputationIo &  io,
bool  allow_extra_input = false 
)

Check that this model and this I/O request are compatible in terms of required context, etc, and crash if not.

if allow_extra_input == false, this will crash if the input 'io' object has time values that would never be used because they are before/after the first/last desired time values.

Definition at line 1328 of file convolution.cc.

References ConvolutionModel::all_time_offsets, KALDI_ASSERT, KALDI_ERR, rnnlm::n, ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::num_t_out, kaldi::RandInt(), ConvolutionModel::required_time_offsets, ConvolutionComputationIo::start_t_in, ConvolutionComputationIo::start_t_out, ConvolutionComputationIo::t_step_in, ConvolutionComputationIo::t_step_out, and TimeValueInInput().

Referenced by CompileConvolutionComputation().

1330  {
1331  KALDI_ASSERT(io.num_t_in > 0 && io.num_t_out > 0 &&
1332  !model.required_time_offsets.empty() &&
1333  !model.all_time_offsets.empty());
1334  if (!allow_extra_input) {
1335  KALDI_ASSERT(io.start_t_in >= io.start_t_out +
1336  *model.all_time_offsets.begin());
1337  int32 last_t_in = io.start_t_in + io.t_step_in * (io.num_t_in - 1),
1338  last_t_out = io.start_t_out + io.t_step_out * (io.num_t_out - 1);
1339  KALDI_ASSERT(last_t_in <= last_t_out +
1340  *model.all_time_offsets.rbegin());
1341  }
1342 
1343  std::set<int32> input_times_to_check;
1344  for (int32 n = 0; n < std::min(5, io.num_t_out); n++) {
1345  int32 t_out = io.start_t_out +
1346  RandInt(0, io.num_t_out - 1) * io.t_step_out;
1347  for (std::set<int32>::const_iterator iter =
1348  model.required_time_offsets.begin();
1349  iter != model.required_time_offsets.end();
1350  ++iter) {
1351  int32 offset = *iter;
1352  input_times_to_check.insert(t_out + offset);
1353  }
1354  }
1355  for (std::set<int32>::const_iterator iter = input_times_to_check.begin();
1356  iter != input_times_to_check.end(); ++iter) {
1357  int32 t = *iter;
1358  if (!TimeValueInInput(io, t)) {
1359  KALDI_ERR << "Error checking model and IO: time " << t
1360  << " is required but not in the input.";
1361  }
1362  }
1363 }
struct rnnlm::@11::@12 n
#define KALDI_ERR
Definition: kaldi-error.h:127
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static bool TimeValueInInput(const ConvolutionComputationIo &io, int32 t)
int32 RandInt(int32 min_val, int32 max_val, struct RandomState *state)
Definition: kaldi-math.cc:94
void CompileConvolutionComputation ( const ConvolutionModel &  model,
const std::vector< Index > &  input_indexes,
const std::vector< Index > &  output_indexes,
const ConvolutionComputationOptions &  opts,
ConvolutionComputation *  computation,
std::vector< Index > *  input_indexes_modified,
std::vector< Index > *  output_indexes_modified 
)

This function does the compilation for a convolution computation; it's a wrapper for the functions below, which should not have to be called by the end user.

Parameters
[in]modelThe convolution model that this computation is for.
[in]input_indexesThe list of Indexes available at the input of the computation.
[in]output_indexesThe list of Indexes requested to be computed at the output of the computation. It is an error if all dependencies are not satisfied (specifically: for each Index (n,t,x) in 'output_indexes', the Index (n,t+time_offset,x) must be present in 'input_indexes' for each time_offset in model.required_time_offsets.
[out]computationIf non-NULL, the compiled computation will be written to this location.

Definition at line 1366 of file convolution.cc.

References AppendInputFrames(), CheckModelAndIo(), GetComputationIo(), GetIndexesForComputation(), MakeComputation(), PadComputationInputTime(), PadModelHeight(), and UnPadModelHeight().

Referenced by TimeHeightConvolutionComponent::PrecomputeIndexes(), TimeHeightConvolutionComponent::ReorderIndexes(), and UnitTestTimeHeightConvolutionCompile().

1373  {
1374 
1375  // stage zero [preparing the input and output in a regular grid.]
1376  ConvolutionComputationIo io;
1377  GetComputationIo(input_indexes, output_indexes, &io);
1378 
1379  CheckModelAndIo(model, io, false);
1380 
1381  // stage 1.
1382  PadComputationInputTime(model, &io);
1383 
1384  CheckModelAndIo(model, io, false);
1385 
1386  // stage 2.
1387  ConvolutionModel model_padded;
1388  PadModelHeight(model, &model_padded);
1389 
1390  CheckModelAndIo(model_padded, io, false);
1391 
1392  // stage 3.
1393  ConvolutionModel model_appended;
1394  ConvolutionComputationIo io_appended;
1395  // make a 'fake' model and io for possibly-appended input frames. 'io' is
1396  // non-const because we may need to pad with a few extra frames.
1397  AppendInputFrames(model_padded, &io,
1398  &model_appended, &io_appended);
1399 
1400  CheckModelAndIo(model_appended, io_appended, true);
1401 
1402  // stage 4.
1403  MakeComputation(model_appended, io_appended, opts, computation);
1404 
1405  // 'reverse' of stage 2. [stage 3 kind of does its own
1406  // 'reverse' by modifying its input IO object.]
1407  // The computation is still specified for the appended input,
1408  // but the execution code can figure that out itself.
1409  UnPadModelHeight(opts, model, model_padded, computation);
1410 
1411  GetIndexesForComputation(io, input_indexes, output_indexes,
1412  input_indexes_modified, output_indexes_modified);
1413 }
void GetIndexesForComputation(const ConvolutionComputationIo &io, const std::vector< Index > &orig_input_indexes, const std::vector< Index > &orig_output_indexes, std::vector< Index > *input_indexes, std::vector< Index > *output_indexes)
This function computes the reordered and possibly padded indexes corresponding to the computation in ...
void UnPadModelHeight(const ConvolutionComputationOptions &opts, const ConvolutionModel &model, const ConvolutionModel &model_padded, ConvolutionComputation *computation)
This function modifies, if necessary, a computation that has been built for the model 'model_padded'...
void MakeComputation(const ConvolutionModel &model, ConvolutionComputationIo &io, const ConvolutionComputationOptions &opts, ConvolutionComputation *computation)
void AppendInputFrames(const ConvolutionModel &model, ConvolutionComputationIo *io, ConvolutionModel *model_appended, ConvolutionComputationIo *io_appended)
This function takes an input model and I/O specification, and it modifies both of them if necessary t...
void PadModelHeight(const ConvolutionModel &model, ConvolutionModel *model_padded)
This function takes a model that might require zero padding in the height dimension and outputs a mod...
Definition: convolution.cc:917
void GetComputationIo(const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, ConvolutionComputationIo *io)
This function takes lists of input and output indexes to a computation (e.g.
void CheckModelAndIo(const ConvolutionModel &model, const ConvolutionComputationIo &io, bool allow_extra_input)
Check that this model and this I/O request are compatible in terms of required context, etc, and crash if not.
void PadComputationInputTime(const ConvolutionModel &model, ConvolutionComputationIo *io)
This function extends the set of input indexes that the computation has, to account for any required ...
static void kaldi::nnet3::time_height_convolution::ComputeTempMatrixSize ( const ConvolutionComputationOptions &  opts,
ConvolutionComputation *  computation 
)
static

This function sets 'temp_rows' and 'temp_cols' in 'computation'.

Definition at line 955 of file convolution.cc.

References ConvolutionComputation::height_in, ConvolutionComputation::ConvolutionStep::height_map, rnnlm::i, KALDI_WARN, ConvolutionComputationOptions::max_memory_mb, ConvolutionComputation::num_filters_in, ConvolutionComputation::num_images, ConvolutionComputation::num_t_out, ConvolutionComputation::steps, ConvolutionComputation::temp_cols, ConvolutionComputation::temp_rows, and VectorIsContiguous().

Referenced by MakeComputation(), and UnPadModelHeight().

956  {
957  int32 temp_rows = 0, temp_cols = 0;
958  for (size_t i = 0; i < computation->steps.size(); i++) {
959  const ConvolutionComputation::ConvolutionStep &step = computation->steps[i];
960  int32 height_map_size = step.height_map.size(),
961  this_num_cols = height_map_size * computation->num_filters_in;
962  bool columns_are_contiguous =
963  (step.height_map[0] != -1 && VectorIsContiguous(step.height_map));
964  bool need_temp_matrix = true;
965  if (columns_are_contiguous && step.height_map[0] == 0 &&
966  this_num_cols == computation->num_filters_in * computation->height_in) {
967  // the only situation in which we wouldn't need the temporary matrix
968  // for this step, is where the columns are all of the input matrix.
969  need_temp_matrix = false;
970  }
971  if (need_temp_matrix && this_num_cols > temp_cols)
972  temp_cols = this_num_cols;
973  }
974  if (temp_cols > 0) {
975  // work out how many rows the temporary matrix should have, taking
976  // into account the specified memory limit.
977  temp_rows = computation->num_t_out * computation->num_images;
978  BaseFloat num_megabytes = (4 * temp_rows * temp_cols) / 1000000.0,
979  megabyte_limit = opts.max_memory_mb;
980  // C++ rounds down; here, we want to round up so we add one.
981  int32 ratio = 1.0 + num_megabytes / megabyte_limit;
982 
983  // divide the number of time steps into 'ratio' pieces that are as equal as
984  // possible; round up when dividing, to make sure that new_temp_rows * ratio
985  // >= temp_rows so that we don't have a small leftover piece.
986  int32 new_num_t_out = (computation->num_t_out + ratio - 1) / ratio;
987  temp_rows = new_num_t_out * computation->num_images;
988  BaseFloat new_num_megabytes = (4 * temp_rows * temp_cols) / 1000000.0;
989  // make sure we're within the memory limit.
990  if (new_num_megabytes > megabyte_limit) {
991  KALDI_WARN << "Memory consumed in convolution is more than requested "
992  << "(maybe very long time sequence?)";
993  }
994  }
995  computation->temp_rows = temp_rows;
996  computation->temp_cols = temp_cols;
997 
998 }
static bool VectorIsContiguous(const std::vector< int32 > &vec)
Definition: convolution.cc:76
float BaseFloat
Definition: kaldi-types.h:29
#define KALDI_WARN
Definition: kaldi-error.h:130
void ConvolveBackwardData ( const ConvolutionComputation &  conv_comp,
const CuMatrixBase< BaseFloat > &  params,
const CuMatrixBase< BaseFloat > &  output_deriv,
CuMatrixBase< BaseFloat > *  input_deriv 
)

This does the part of the backward derivative computation of convolution, that propagates derivatives back to the input data.

See also ConvolveBackwardParams(), which is for the parameter derivative.

Parameters
[in]conv_compA struct that describes the convolution computation (should be the same as in the corresponding forward pass).
[in]paramsThe parameters used in the forward convolution. This should be of dimension num_filters_out by (X * num_filters_in), where X is the total number of pixels in the patches, which equals model.offsets.size() in the model for which the computation was compiled. E.g. for a regular 3x3 kernel, X would be 9.
[in]output_derivThe derivative of the objective function w.r.t. the output of the convolution. Should be of dimension conv_comp.num_t_out * conv_comp.num_images by conv_comp.height_out num_filters_out. It must satisfy output_deriv.NumCols() == output_deriv.Stride().
[out]input_derivIf non-NULL, the backpropagated derivative of the objective function w.r.t. the input will be *added to* this matrix. Should be the same dimension as the input to the original ConvolveForward() call.

Definition at line 681 of file convolution.cc.

References ConvolveBackwardDataInternal(), CuMatrixBase< Real >::Data(), ConvolutionComputation::height_in, ConvolutionComputation::height_out, KALDI_ASSERT, KALDI_ERR, kaldi::kSetZero, kaldi::kStrideEqualNumCols, ConvolutionComputation::num_filters_in, ConvolutionComputation::num_filters_out, ConvolutionComputation::num_images, ConvolutionComputation::num_t_in, ConvolutionComputation::num_t_out, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), CuMatrixBase< Real >::Stride(), ConvolutionComputation::temp_cols, and ConvolutionComputation::temp_rows.

Referenced by TimeHeightConvolutionComponent::Backprop(), and TestDataBackprop().

685  {
686  KALDI_ASSERT(input_deriv->NumCols() == input_deriv->Stride() &&
687  output_deriv.NumCols() == output_deriv.Stride());
688  KALDI_ASSERT(params.NumRows() == cc.num_filters_out);
689  KALDI_ASSERT(output_deriv.NumRows() == cc.num_t_out * cc.num_images &&
690  output_deriv.NumCols() == cc.height_out * cc.num_filters_out);
691  // the input might need to be reshaped but we can check its total size.
692  KALDI_ASSERT(input_deriv->NumRows() * input_deriv->NumCols() ==
693  cc.num_images * cc.num_t_in * cc.height_in * cc.num_filters_in);
694 
695  int32 input_rows = input_deriv->NumRows(),
696  required_input_rows = cc.num_images * cc.num_t_in;
697 
698  // this if-statement handles reshaping the input and recursing if there
699  // is subsampling.
700  if (input_rows != required_input_rows) {
701  if (input_rows % required_input_rows != 0)
702  KALDI_ERR << "Input matrix has wrong size."; // error in calling code.
703  // nr is a multiple of required_nr. Reshape the matrix.
704  // we already checked that its Stride() == NumCols();
705  int32 num_cols = input_deriv->NumCols(),
706  multiple = input_rows / required_input_rows,
707  new_num_cols = num_cols * multiple,
708  new_stride = new_num_cols;
709  CuSubMatrix<BaseFloat> input_deriv_reshaped(
710  input_deriv->Data(), required_input_rows,
711  new_num_cols, new_stride);
712  ConvolveBackwardData(cc, params, output_deriv, &input_deriv_reshaped);
713  return;
714  }
715 
716  CuMatrix<BaseFloat> temp_mat(cc.temp_rows, cc.temp_cols,
718 
719  // this if-statement handles breaking up the arguments
720  // and the computation into row-ranges if the temporary
721  // matrix would have been excessively large, and we've decided
722  // to give it fewer rows than the output (this saves
723  // memory). normally we won't take this if-statement
724  // so ignore it if you're trying to understand the framework.
725  if (cc.temp_rows != 0 && cc.temp_rows != input_rows) {
726  KALDI_ASSERT(cc.temp_rows % cc.num_images == 0);
727  int32 num_time_steps_per_chunk = cc.temp_rows / cc.num_images;
728  int32 num_extra_in = cc.num_t_in - cc.num_t_out;
729 
730  for (int32 t_start = 0; t_start < cc.num_t_out;
731  t_start += num_time_steps_per_chunk) {
732  int32 num_t_left = cc.num_t_out - t_start,
733  this_num_t_out = std::min<int32>(num_t_left,
734  num_time_steps_per_chunk),
735  this_num_t_in = this_num_t_out + num_extra_in;
736  CuSubMatrix<BaseFloat> input_deriv_part(
737  *input_deriv, t_start * cc.num_images,
738  this_num_t_in * cc.num_images,
739  0, input_deriv->NumCols());
740  CuSubMatrix<BaseFloat> output_deriv_part(
741  output_deriv, t_start * cc.num_images,
742  this_num_t_out * cc.num_images,
743  0, output_deriv.NumCols());
744  CuSubMatrix<BaseFloat> temp_part(
745  temp_mat, 0, this_num_t_out * cc.num_images,
746  0, temp_mat.NumCols());
747  ConvolveBackwardDataInternal(cc, params, output_deriv_part,
748  &temp_part, &input_deriv_part);
749  }
750  return;
751  }
752  ConvolveBackwardDataInternal(cc, params, output_deriv,
753  &temp_mat, input_deriv);
754 }
static void ConvolveBackwardDataInternal(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &params, const CuMatrixBase< BaseFloat > &output_deriv, CuMatrixBase< BaseFloat > *temp_mat, CuMatrixBase< BaseFloat > *input_deriv)
Definition: convolution.cc:602
MatrixIndexT NumCols() const
Definition: cu-matrix.h:196
void ConvolveBackwardData(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &params, const CuMatrixBase< BaseFloat > &output_deriv, CuMatrixBase< BaseFloat > *input_deriv)
This does the part of the backward derivative computation of convolution, that propagates derivatives...
Definition: convolution.cc:681
MatrixIndexT NumRows() const
Dimensions.
Definition: cu-matrix.h:195
#define KALDI_ERR
Definition: kaldi-error.h:127
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
MatrixIndexT Stride() const
Definition: cu-matrix.h:197
const Real * Data() const
Return data pointer (const).
Definition: cu-matrix.h:625
static void kaldi::nnet3::time_height_convolution::ConvolveBackwardDataInternal ( const ConvolutionComputation &  cc,
const CuMatrixBase< BaseFloat > &  params,
const CuMatrixBase< BaseFloat > &  output_deriv,
CuMatrixBase< BaseFloat > *  temp_mat,
CuMatrixBase< BaseFloat > *  input_deriv 
)
static

Definition at line 602 of file convolution.cc.

References CuMatrixBase< Real >::AddMatMat(), ConvolutionComputation::ConvolutionStep::backward_columns, ConvolutionComputation::ConvolutionStep::columns, ConvolutionComputation::ConvolutionStep::columns_are_contiguous, CuMatrixBase< Real >::Data(), CuArray< T >::Dim(), ConvolutionComputation::ConvolutionStep::first_column, ConvolutionComputation::height_out, rnnlm::i, ConvolutionComputation::ConvolutionStep::input_time_shift, KALDI_ASSERT, kaldi::kNoTrans, ConvolutionComputation::num_filters_out, ConvolutionComputation::num_images, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), ConvolutionComputation::ConvolutionStep::params_start_col, ConvolutionComputation::steps, and CuMatrixBase< Real >::Stride().

Referenced by ConvolveBackwardData().

607  {
608  KALDI_ASSERT(temp_mat->Stride() == temp_mat->NumCols());
609 
610  // num_t_out supersedes cc.num_t_out (they'll only be different in
611  // cases where we are doing the computation in pieces to save memory).
612  int32 input_rows = input_deriv->NumRows(),
613  output_rows = output_deriv.NumRows();
614 
615  KALDI_ASSERT(output_rows <= input_rows &&
616  input_rows % cc.num_images == 0 &&
617  output_rows % cc.num_images == 0);
618 
619  int32 num_steps = cc.steps.size();
620  for (int32 s = 0; s < num_steps; s++) {
621  const ConvolutionComputation::ConvolutionStep &step = cc.steps[s];
622  int32 input_row_start = step.input_time_shift * cc.num_images;
623  CuSubMatrix<BaseFloat> input_deriv_part(*input_deriv,
624  input_row_start, output_rows,
625  0, input_deriv->NumCols());
626  int32 temp_num_cols = step.columns.Dim(),
627  param_cols = temp_num_cols / cc.height_out;
628  CuSubMatrix<BaseFloat> params_part(params,
629  0, params.NumRows(),
630  step.params_start_col,
631  param_cols);
632  CuSubMatrix<BaseFloat> output_deriv_reshaped(
633  output_deriv.Data(), output_rows * cc.height_out,
634  cc.num_filters_out, cc.num_filters_out);
635 
636  if (!step.columns_are_contiguous ||
637  temp_num_cols != input_deriv->NumCols()) {
638  // In most cases we will take this branch, where we have to propagate the
639  // input-derivative via a temporary matrix. (however, different steps may
640  // require different num-cols of the temporary matrix, so we create
641  // sub-parts of 'temp_mat'.
642 
643  // We create the sub-matrix 'temp_mat_part' in a lower-level way, using
644  // pointers, because we need to ensure that its num-cols and the stride
645  // are the same (this is necessary so that we can do reshaping in
646  // ConvolutionReshapedMultiply()).
647  CuSubMatrix<BaseFloat> temp_mat_part(temp_mat->Data(),
648  temp_mat->NumRows(),
649  temp_num_cols, temp_num_cols),
650  temp_mat_part_reshaped(
651  temp_mat_part.Data(), temp_mat_part.NumRows() * cc.height_out,
652  temp_num_cols / cc.height_out, temp_num_cols / cc.height_out);
653 
654  temp_mat_part_reshaped.AddMatMat(1.0, output_deriv_reshaped, kNoTrans,
655  params_part, kNoTrans, 0.0);
656 
657  if (!step.columns_are_contiguous) {
658  for (size_t i = 0; i < step.backward_columns.size(); i++) {
659  input_deriv_part.AddCols(temp_mat_part, step.backward_columns[i]);
660  }
661  } else {
662  // we're just taking a sub-matrix of the input matrix, but we still need
663  // to make a copy because we need the stride == num-cols (so that the
664  // reshaping will work).
665  int32 num_cols = step.columns.Dim();
666  input_deriv_part.ColRange(step.first_column,
667  num_cols).AddMat(1.0, temp_mat_part);
668  }
669  } else {
670  CuSubMatrix<BaseFloat> input_deriv_reshaped(
671  input_deriv_part.Data(), input_deriv_part.NumRows() * cc.height_out,
672  input_deriv_part.NumCols() / cc.height_out,
673  input_deriv_part.NumCols() / cc.height_out);
674  input_deriv_reshaped.AddMatMat(1.0, output_deriv_reshaped, kNoTrans,
675  params_part, kNoTrans, 1.0);
676  }
677  }
678 }
MatrixIndexT NumCols() const
Definition: cu-matrix.h:196
MatrixIndexT NumRows() const
Dimensions.
Definition: cu-matrix.h:195
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
MatrixIndexT Stride() const
Definition: cu-matrix.h:197
const Real * Data() const
Return data pointer (const).
Definition: cu-matrix.h:625
void ConvolveBackwardParams ( const ConvolutionComputation &  conv_comp,
const CuMatrixBase< BaseFloat > &  input,
const CuMatrixBase< BaseFloat > &  output_deriv,
BaseFloat  alpha,
CuMatrixBase< BaseFloat > *  params_deriv 
)

This does the part of the backward derivative computation of convolution, that computes derivatives w.r.t.

the parameters. See also ConvolveBackwardData(), which computes derivatives w.r.t. the input data.

Parameters
[in]conv_compA struct that describes the computation that was performed in the forward pass.
[in]inputThe input to the original forward convolution. This should be of dimension (or should be reshapable to the dimension) conv_comp.num_t_in * conv_comp.num_images by conv_comp.height_in * num_filters_in. [highest-stride indexes come first in these multiplications]. It must satisfy input.NumCols() == input.Stride().
[in]output_derivThe derivative of the objective function w.r.t. the output of the convolution. Should be of dimension conv_comp.num_t_out * conv_comp.num_images by conv_comp.height_out num_filters_out. It must satisfy output_deriv.NumCols() == output_deriv.Stride().
[in]alphaThis scalar is multiplied into the derivative when we add to params_deriv, i.e. *params_deriv += alpha * derivative.
[out]params_derivThe derivative of the objective function w.r.t the parameters (the 'params' given to the ConvolveForward function) is *added* to this location. This matrix should be of dimension conv_comp.NumRows() by conv_comp.NumCols().

Definition at line 839 of file convolution.cc.

References ConvolveBackwardParamsInternal(), CuMatrixBase< Real >::Data(), ConvolutionComputation::height_in, ConvolutionComputation::height_out, KALDI_ASSERT, KALDI_ERR, kaldi::kStrideEqualNumCols, kaldi::kUndefined, ConvolutionComputation::num_filters_in, ConvolutionComputation::num_filters_out, ConvolutionComputation::num_images, ConvolutionComputation::num_t_in, ConvolutionComputation::num_t_out, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), CuMatrixBase< Real >::Stride(), ConvolutionComputation::temp_cols, and ConvolutionComputation::temp_rows.

Referenced by TestParamsBackprop(), TimeHeightConvolutionComponent::UpdateNaturalGradient(), and TimeHeightConvolutionComponent::UpdateSimple().

844  {
845  KALDI_ASSERT(input.NumCols() == input.Stride() &&
846  output_deriv.NumCols() == output_deriv.Stride());
847  KALDI_ASSERT(params_deriv->NumRows() == cc.num_filters_out);
848  KALDI_ASSERT(output_deriv.NumRows() == cc.num_t_out * cc.num_images &&
849  output_deriv.NumCols() == cc.height_out * cc.num_filters_out);
850  // the input might need to be reshaped but we can check its total size.
851  KALDI_ASSERT(input.NumRows() * input.NumCols() == cc.num_images *
852  cc.num_t_in * cc.height_in * cc.num_filters_in);
853 
854  int32 input_rows = input.NumRows(),
855  required_input_rows = cc.num_images * cc.num_t_in;
856 
857  // this if-statement handles reshaping the input and recursing if there
858  // is subsampling.
859  if (input_rows != required_input_rows) {
860  if (input_rows % required_input_rows != 0)
861  KALDI_ERR << "Input matrix has wrong size."; // error in calling code.
862  // nr is a multiple of required_nr. Reshape the matrix.
863  // we already checked that its Stride() == NumCols();
864  int32 num_cols = input.NumCols(),
865  multiple = input_rows / required_input_rows,
866  new_num_cols = num_cols * multiple,
867  new_stride = new_num_cols;
868  CuSubMatrix<BaseFloat> input_reshaped(
869  input.Data(), required_input_rows, new_num_cols, new_stride);
870  ConvolveBackwardParams(cc, input_reshaped, output_deriv, alpha,
871  params_deriv);
872  return;
873  }
874 
875  CuMatrix<BaseFloat> temp_mat(cc.temp_rows, cc.temp_cols,
877 
878  // this if-statement handles breaking up the arguments
879  // and the computation into row-ranges if the temporary
880  // matrix would have been excessively large, and we've decided
881  // to give it fewer rows than the output (this saves
882  // memory). normally we won't take this if-statement
883  // so ignore it if you're trying to understand the framework.
884  if (cc.temp_rows != 0 && cc.temp_rows != input_rows) {
885  KALDI_ASSERT(cc.temp_rows % cc.num_images == 0);
886  int32 num_time_steps_per_chunk = cc.temp_rows / cc.num_images;
887  int32 num_extra_in = cc.num_t_in - cc.num_t_out;
888 
889  for (int32 t_start = 0; t_start < cc.num_t_out;
890  t_start += num_time_steps_per_chunk) {
891  int32 num_t_left = cc.num_t_out - t_start,
892  this_num_t_out = std::min<int32>(num_t_left,
893  num_time_steps_per_chunk),
894  this_num_t_in = this_num_t_out + num_extra_in;
895  CuSubMatrix<BaseFloat> input_part(
896  input, t_start * cc.num_images,
897  this_num_t_in * cc.num_images,
898  0, input.NumCols());
899  CuSubMatrix<BaseFloat> output_deriv_part(
900  output_deriv, t_start * cc.num_images,
901  this_num_t_out * cc.num_images,
902  0, output_deriv.NumCols());
903  CuSubMatrix<BaseFloat> temp_part(temp_mat,
904  0, this_num_t_out * cc.num_images,
905  0, temp_mat.NumCols());
906  ConvolveBackwardParamsInternal(cc, input_part, output_deriv_part,
907  alpha, &temp_part, params_deriv);
908  }
909  return;
910  }
911  ConvolveBackwardParamsInternal(cc, input, output_deriv,
912  alpha, &temp_mat, params_deriv);
913 }
void ConvolveBackwardParams(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &output_deriv, BaseFloat alpha, CuMatrixBase< BaseFloat > *params_deriv)
This does the part of the backward derivative computation of convolution, that computes derivatives w...
Definition: convolution.cc:839
static void ConvolveBackwardParamsInternal(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &output_deriv, BaseFloat alpha, CuMatrixBase< BaseFloat > *temp_mat, CuMatrixBase< BaseFloat > *params_deriv)
Definition: convolution.cc:762
MatrixIndexT NumCols() const
Definition: cu-matrix.h:196
MatrixIndexT NumRows() const
Dimensions.
Definition: cu-matrix.h:195
#define KALDI_ERR
Definition: kaldi-error.h:127
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
MatrixIndexT Stride() const
Definition: cu-matrix.h:197
const Real * Data() const
Return data pointer (const).
Definition: cu-matrix.h:625
static void kaldi::nnet3::time_height_convolution::ConvolveBackwardParamsInternal ( const ConvolutionComputation &  cc,
const CuMatrixBase< BaseFloat > &  input,
const CuMatrixBase< BaseFloat > &  output_deriv,
BaseFloat  alpha,
CuMatrixBase< BaseFloat > *  temp_mat,
CuMatrixBase< BaseFloat > *  params_deriv 
)
static

Definition at line 762 of file convolution.cc.

References CuMatrixBase< Real >::AddMatMat(), ConvolutionComputation::ConvolutionStep::columns, ConvolutionComputation::ConvolutionStep::columns_are_contiguous, CuMatrixBase< Real >::CopyCols(), CuMatrixBase< Real >::Data(), CuArray< T >::Dim(), ConvolutionComputation::ConvolutionStep::first_column, ConvolutionComputation::height_out, ConvolutionComputation::ConvolutionStep::input_time_shift, KALDI_ASSERT, kaldi::kNoTrans, kaldi::kTrans, ConvolutionComputation::num_filters_out, ConvolutionComputation::num_images, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), ConvolutionComputation::ConvolutionStep::params_start_col, ConvolutionComputation::steps, and CuMatrixBase< Real >::Stride().

Referenced by ConvolveBackwardParams().

768  {
769  KALDI_ASSERT(temp_mat->Stride() == temp_mat->NumCols());
770 
771  // num_t_out supersedes cc.num_t_out (they'll only be different in
772  // cases where we are doing the computation in pieces to save memory).
773  int32 input_rows = input.NumRows(),
774  output_rows = output_deriv.NumRows();
775 
776  KALDI_ASSERT(output_rows <= input_rows &&
777  input_rows % cc.num_images == 0 &&
778  output_rows % cc.num_images == 0);
779 
780  int32 num_steps = cc.steps.size();
781  for (int32 s = 0; s < num_steps; s++) {
782  const ConvolutionComputation::ConvolutionStep &step = cc.steps[s];
783  int32 input_row_start = step.input_time_shift * cc.num_images;
784  // note: 'input_part' will normally be almost all of 'input', perhaps
785  // minus one or two time steps at the start or end.
786  CuSubMatrix<BaseFloat> input_part(input,
787  input_row_start, output_rows,
788  0, input.NumCols());
789  int32 temp_num_cols = step.columns.Dim(),
790  param_cols = temp_num_cols / cc.height_out;
791  CuSubMatrix<BaseFloat> params_deriv_part(*params_deriv,
792  0, params_deriv->NumRows(),
793  step.params_start_col,
794  param_cols);
795  CuSubMatrix<BaseFloat> output_deriv_reshaped(
796  output_deriv.Data(), output_rows * cc.height_out,
797  cc.num_filters_out, cc.num_filters_out);
798  if (!step.columns_are_contiguous ||
799  temp_num_cols != input.NumCols()) {
800  // In most cases we will take this branch, where we have to copy the input
801  // to a temporary matrix. (however, different steps may require different
802  // num-cols of the temporary matrix, so we create sub-parts of 'temp_mat'.
803 
804  // We create the sub-matrix 'temp_mat_part' in a lower-level way, using
805  // pointers, because we need to ensure that its num-cols and the stride
806  // are the same (this is necessary so that we can do reshaping in
807  // ConvolutionReshapedMultiply()).
808  CuSubMatrix<BaseFloat> temp_mat_part(temp_mat->Data(),
809  temp_mat->NumRows(),
810  temp_num_cols, temp_num_cols);
811  if (!step.columns_are_contiguous) {
812  // we're doing a column mapping.
813  temp_mat_part.CopyCols(input_part, step.columns);
814  } else {
815  // we're just taking a sub-matrix of the input matrix, but we still need
816  // to make a copy because we need the stride == num-cols (so that the
817  // reshaping will work).
818  temp_mat_part.CopyFromMat(input_part.ColRange(step.first_column,
819  step.columns.Dim()));
820  }
821  CuSubMatrix<BaseFloat> temp_mat_part_reshaped(
822  temp_mat_part.Data(), temp_mat_part.NumRows() * cc.height_out,
823  temp_num_cols / cc.height_out, temp_num_cols / cc.height_out);
824 
825  params_deriv_part.AddMatMat(alpha, output_deriv_reshaped, kTrans,
826  temp_mat_part_reshaped, kNoTrans, 1.0);
827  } else {
828  CuSubMatrix<BaseFloat> input_reshaped(
829  input_part.Data(), input_part.NumRows() * cc.height_out,
830  input_part.NumCols() / cc.height_out,
831  input_part.NumCols() / cc.height_out);
832 
833  params_deriv_part.AddMatMat(alpha, output_deriv_reshaped, kTrans,
834  input_reshaped, kNoTrans, 1.0);
835  }
836  }
837 }
MatrixIndexT NumCols() const
Definition: cu-matrix.h:196
MatrixIndexT NumRows() const
Dimensions.
Definition: cu-matrix.h:195
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
MatrixIndexT Stride() const
Definition: cu-matrix.h:197
const Real * Data() const
Return data pointer (const).
Definition: cu-matrix.h:625
void ConvolveForward ( const ConvolutionComputation &  conv_comp,
const CuMatrixBase< BaseFloat > &  input,
const CuMatrixBase< BaseFloat > &  params,
CuMatrixBase< BaseFloat > *  output 
)

This does the forward computation of convolution.

(note: this is convolution without a bias term; you have to handle that separately).

Parameters
[in]conv_compA struct that describes the computation to be performed.
[in]inputThe input to the convolution. This should be of dimension (or should be reshapable to the dimension) conv_comp.num_t_in * conv_comp.num_images by conv_comp.height_in * num_filters_in. [highest-stride indexes come first in these multiplications]. It must satisfy input.NumCols() == input.Stride().
[in]paramsThe parameters of the convolution. This should be of dimension conv_comp.ParamRows() by conv_comp.ParamCols().
[out]outputThe output of the convolution (this function adds to* the output). Should be of dimension conv_comp.num_t_out * conv_comp.num_images by conv_comp.height_out * num_filters_out. It must satisfy output.NumCols() == output.Stride().

Definition at line 523 of file convolution.cc.

References ConvolveForwardInternal(), CuMatrixBase< Real >::Data(), ConvolutionComputation::height_in, ConvolutionComputation::height_out, KALDI_ASSERT, KALDI_ERR, kaldi::kStrideEqualNumCols, kaldi::kUndefined, ConvolutionComputation::num_filters_in, ConvolutionComputation::num_filters_out, ConvolutionComputation::num_images, ConvolutionComputation::num_t_in, ConvolutionComputation::num_t_out, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), CuMatrixBase< Real >::Stride(), ConvolutionComputation::temp_cols, and ConvolutionComputation::temp_rows.

Referenced by TimeHeightConvolutionComponent::Propagate(), TestDataBackprop(), TestParamsBackprop(), and TestRunningComputation().

527  {
528  KALDI_ASSERT(input.NumCols() == input.Stride() &&
529  output->NumCols() == output->Stride());
530  KALDI_ASSERT(params.NumRows() == cc.num_filters_out);
531  KALDI_ASSERT(output->NumRows() == cc.num_t_out * cc.num_images &&
532  output->NumCols() == cc.height_out * cc.num_filters_out);
533  // the input might need to be reshaped but we can check its total size.
534  KALDI_ASSERT(input.NumRows() * input.NumCols() == cc.num_images *
535  cc.num_t_in * cc.height_in * cc.num_filters_in);
536 
537  int32 input_rows = input.NumRows(),
538  required_input_rows = cc.num_images * cc.num_t_in;
539 
540  // this if-statement handles reshaping the input and recursing if there
541  // is subsampling.
542  if (input_rows != required_input_rows) {
543  if (input_rows % required_input_rows != 0)
544  KALDI_ERR << "Input matrix has wrong size."; // error in calling code.
545  // nr is a multiple of required_nr. Reshape the matrix.
546  // we already checked that its Stride() == NumCols();
547  int32 num_cols = input.NumCols(),
548  multiple = input_rows / required_input_rows,
549  new_num_cols = num_cols * multiple,
550  new_stride = new_num_cols;
551  CuSubMatrix<BaseFloat> input_reshaped(
552  input.Data(), required_input_rows, new_num_cols, new_stride);
553  ConvolveForward(cc, input_reshaped, params, output);
554  return;
555  }
556 
557  CuMatrix<BaseFloat> temp_mat(cc.temp_rows, cc.temp_cols,
559 
560  // this if-statement handles breaking up the arguments
561  // and the computation into row-ranges if the temporary
562  // matrix would have been excessively large, and we've decided
563  // to give it fewer rows than the output (this saves
564  // memory). normally we won't take this if-statement
565  // so ignore it if you're trying to understand the framework.
566  if (cc.temp_rows != 0 && cc.temp_rows != input_rows) {
567  KALDI_ASSERT(cc.temp_rows % cc.num_images == 0);
568  int32 num_time_steps_per_chunk = cc.temp_rows / cc.num_images;
569  int32 num_extra_in = cc.num_t_in - cc.num_t_out;
570 
571  for (int32 t_start = 0; t_start < cc.num_t_out;
572  t_start += num_time_steps_per_chunk) {
573  int32 num_t_left = cc.num_t_out - t_start,
574  this_num_t_out = std::min<int32>(num_t_left,
575  num_time_steps_per_chunk),
576  this_num_t_in = this_num_t_out + num_extra_in;
577  CuSubMatrix<BaseFloat> input_part(input, t_start * cc.num_images,
578  this_num_t_in * cc.num_images,
579  0, input.NumCols());
580  CuSubMatrix<BaseFloat> output_part(*output, t_start * cc.num_images,
581  this_num_t_out * cc.num_images,
582  0, output->NumCols());
583  CuSubMatrix<BaseFloat> temp_part(temp_mat, 0,
584  this_num_t_out * cc.num_images,
585  0, temp_mat.NumCols());
586  ConvolveForwardInternal(cc, input_part, params,
587  &temp_part, &output_part);
588  }
589  return;
590  }
591  ConvolveForwardInternal(cc, input, params, &temp_mat, output);
592 }
MatrixIndexT NumCols() const
Definition: cu-matrix.h:196
static void ConvolveForwardInternal(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &params, CuMatrixBase< BaseFloat > *temp_mat, CuMatrixBase< BaseFloat > *output)
Definition: convolution.cc:447
MatrixIndexT NumRows() const
Dimensions.
Definition: cu-matrix.h:195
#define KALDI_ERR
Definition: kaldi-error.h:127
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
void ConvolveForward(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &params, CuMatrixBase< BaseFloat > *output)
This does the forward computation of convolution.
Definition: convolution.cc:523
MatrixIndexT Stride() const
Definition: cu-matrix.h:197
const Real * Data() const
Return data pointer (const).
Definition: cu-matrix.h:625
static void kaldi::nnet3::time_height_convolution::ConvolveForwardInternal ( const ConvolutionComputation &  cc,
const CuMatrixBase< BaseFloat > &  input,
const CuMatrixBase< BaseFloat > &  params,
CuMatrixBase< BaseFloat > *  temp_mat,
CuMatrixBase< BaseFloat > *  output 
)
static

Definition at line 447 of file convolution.cc.

References CuMatrixBase< Real >::AddMatMat(), ConvolutionComputation::ConvolutionStep::columns, ConvolutionComputation::ConvolutionStep::columns_are_contiguous, CuMatrixBase< Real >::CopyCols(), CuMatrixBase< Real >::Data(), CuArray< T >::Dim(), ConvolutionComputation::ConvolutionStep::first_column, ConvolutionComputation::height_out, ConvolutionComputation::ConvolutionStep::input_time_shift, KALDI_ASSERT, kaldi::kNoTrans, kaldi::kTrans, ConvolutionComputation::num_filters_out, ConvolutionComputation::num_images, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), ConvolutionComputation::ConvolutionStep::params_start_col, ConvolutionComputation::steps, and CuMatrixBase< Real >::Stride().

Referenced by ConvolveForward().

452  {
453  KALDI_ASSERT(temp_mat->Stride() == temp_mat->NumCols());
454 
455  // num_t_out supersedes cc.num_t_out (they'll only be different in
456  // cases where we are doing the computation in pieces to save memory).
457  int32 input_rows = input.NumRows(),
458  output_rows = output->NumRows();
459 
460  KALDI_ASSERT(output_rows <= input_rows &&
461  input_rows % cc.num_images == 0 &&
462  output_rows % cc.num_images == 0);
463 
464  int32 num_steps = cc.steps.size();
465  for (int32 s = 0; s < num_steps; s++) {
466  const ConvolutionComputation::ConvolutionStep &step = cc.steps[s];
467  int32 input_row_start = step.input_time_shift * cc.num_images;
468  // note: 'input_part' will normally be almost all of 'input', perhaps
469  // minus one or two time steps at the start or end.
470  CuSubMatrix<BaseFloat> input_part(input,
471  input_row_start, output_rows,
472  0, input.NumCols());
473  int32 temp_num_cols = step.columns.Dim(),
474  param_cols = temp_num_cols / cc.height_out;
475  CuSubMatrix<BaseFloat> params_part(params,
476  0, params.NumRows(),
477  step.params_start_col,
478  param_cols);
479  CuSubMatrix<BaseFloat> output_reshaped(
480  output->Data(), output_rows * cc.height_out,
481  cc.num_filters_out, cc.num_filters_out);
482  if (!step.columns_are_contiguous ||
483  temp_num_cols != input.NumCols()) {
484  // In most cases we will take this branch, where we have to copy the input
485  // to a temporary matrix. (however, different steps may require different
486  // num-cols of the temporary matrix, so we create sub-parts of 'temp_mat'.
487 
488  // We create the sub-matrix 'temp_mat_part' in a lower-level way, using
489  // pointers, because we need to ensure that its num-cols and the stride
490  // are the same (this is necessary so that we can do reshaping in
491  // ConvolutionReshapedMultiply()).
492  CuSubMatrix<BaseFloat> temp_mat_part(temp_mat->Data(),
493  temp_mat->NumRows(),
494  temp_num_cols, temp_num_cols);
495  if (!step.columns_are_contiguous) {
496  // we're doing a column mapping.
497  temp_mat_part.CopyCols(input_part, step.columns);
498  } else {
499  // we're just taking a sub-matrix of the input matrix, but we still need
500  // to make a copy because we need the stride == num-cols (so that the
501  // reshaping will work).
502  temp_mat_part.CopyFromMat(input_part.ColRange(step.first_column,
503  step.columns.Dim()));
504  }
505  CuSubMatrix<BaseFloat> temp_mat_part_reshaped(
506  temp_mat_part.Data(), temp_mat_part.NumRows() * cc.height_out,
507  temp_num_cols / cc.height_out, temp_num_cols / cc.height_out);
508 
509  output_reshaped.AddMatMat(1.0, temp_mat_part_reshaped, kNoTrans,
510  params_part, kTrans, 1.0);
511  } else {
512  CuSubMatrix<BaseFloat> input_reshaped(
513  input_part.Data(), input_part.NumRows() * cc.height_out,
514  input_part.NumCols() / cc.height_out,
515  input_part.NumCols() / cc.height_out);
516 
517  output_reshaped.AddMatMat(1.0, input_reshaped, kNoTrans,
518  params_part, kTrans, 1.0);
519  }
520  }
521 }
MatrixIndexT NumCols() const
Definition: cu-matrix.h:196
MatrixIndexT NumRows() const
Dimensions.
Definition: cu-matrix.h:195
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
MatrixIndexT Stride() const
Definition: cu-matrix.h:197
const Real * Data() const
Return data pointer (const).
Definition: cu-matrix.h:625
void kaldi::nnet3::time_height_convolution::ConvolveForwardSimple ( const ConvolutionModel model,
const std::vector< Index > &  input_indexes,
const std::vector< Index > &  output_indexes,
const CuMatrixBase< BaseFloat > &  input_cu,
const CuMatrixBase< BaseFloat > &  params_cu,
CuMatrixBase< BaseFloat > *  output_cu 
)

Definition at line 218 of file convolution-test.cc.

References CuMatrixBase< Real >::CopyFromMat(), ConvolutionModel::height_in, ConvolutionModel::height_out, ConvolutionModel::height_subsample_out, kaldi::nnet3::kNoTime, kaldi::kNoTrans, ConvolutionModel::num_filters_in, ConvolutionModel::num_filters_out, MatrixBase< Real >::NumRows(), ConvolutionModel::offsets, and Index::t.

Referenced by TestRunningComputation().

224  {
225  // these loops will be very slow on GPU, so do it all on CPU.
226  Matrix<BaseFloat> input(input_cu), params(params_cu),
227  output(*output_cu);
228  std::unordered_map<Index, int32, IndexHasher> index_to_row;
229  int32 input_rows = input.NumRows(),
230  output_rows = output.NumRows();
231  for (int32 r_in = 0; r_in < input_rows; r_in++) {
232  if (input_indexes[r_in].t != kNoTime) {
233  index_to_row[input_indexes[r_in]] = r_in;
234  }
235  }
236  int32 num_offsets = model.offsets.size(),
237  num_filters_in = model.num_filters_in,
238  num_filters_out = model.num_filters_out,
239  height_in = model.height_in,
240  height_out = model.height_out,
241  height_subsample_out = model.height_subsample_out;
242  for (int32 r_out = 0; r_out < output_rows; r_out++) {
243  Index index_out = output_indexes[r_out];
244  if (index_out.t == kNoTime)
245  continue;
246  SubVector<BaseFloat> output_row(output, r_out);
247  for (int32 o = 0; o < num_offsets; o++) {
248  int32 time_offset = model.offsets[o].time_offset,
249  height_offset = model.offsets[o].height_offset;
250  Index index_in(index_out);
251  index_in.t += time_offset;
252  std::unordered_map<Index, int32, IndexHasher>::const_iterator iter =
253  index_to_row.find(index_in);
254  if (iter != index_to_row.end()) {
255  SubMatrix<BaseFloat> params_part(params, 0, params.NumRows(),
256  o * num_filters_in, num_filters_in);
257  int32 r_in = iter->second;
258  SubVector<BaseFloat> input_row(input, r_in);
259  for (int32 h_out_subsampled = 0;
260  h_out_subsampled < height_out;
261  h_out_subsampled++) {
262  int32 h_out = h_out_subsampled * height_subsample_out,
263  h_in = h_out + height_offset;
264  if (h_in < 0 || h_in >= height_in)
265  continue;
266  SubVector<BaseFloat> output_part(output_row,
267  h_out_subsampled * num_filters_out,
268  num_filters_out),
269  input_part(input_row, h_in * num_filters_in, num_filters_in);
270  output_part.AddMatVec(1.0, params_part, kNoTrans, input_part, 1.0);
271  }
272  }
273  }
274  }
275  output_cu->CopyFromMat(output);
276 }
void CopyFromMat(const MatrixBase< OtherReal > &src, MatrixTransposeType trans=kNoTrans)
Definition: cu-matrix.cc:337
struct Index is intended to represent the various indexes by which we number the rows of the matrices...
Definition: nnet-common.h:44
Sub-matrix representation.
Definition: kaldi-matrix.h:908
Represents a non-allocating general vector which can be defined as a sub-vector of higher-level vecto...
Definition: kaldi-vector.h:482
const int kNoTime
Definition: nnet-common.cc:554
static void kaldi::nnet3::time_height_convolution::CreateIndexes ( const std::vector< std::pair< int32, int32 > > &  n_x_pairs,
int32  t_start,
int32  t_step,
int32  num_t_values,
int32  reorder_t,
std::vector< Index > *  indexes 
)
static

Creates a vector of indexes with a regular structure, according to these specifications.

'n_x_pairs' is the list of (n,x) pairs to include; they will appear in this order.

't_start', 't_step' and 'num_t_values' define the set of 't' values to include (note: t_step >= 0; they will appear in the natural order).

If reorder_t == 1 (the normal case), then the order is simple: 't' has the higher stride, then (n, x). So we'll output first all (n, x) pairs for t_start, then all pairs for t_start + t_step, and so on.

If instead reorder_t > 1, then the order is a little different [note: we expect that num_t_values % reorder_t == 0). Consider, for example, reorder_t == 2. In that case the first block has the first two t values, the second block has the next two t values, and so on. And within each block, the 't' values have the smallest stride (of 1).

Definition at line 1509 of file convolution.cc.

References KALDI_ASSERT, and Index::n.

Referenced by GetIndexesForComputation().

1511  {
1512  KALDI_ASSERT(reorder_t >= 1 && num_t_values % reorder_t == 0 && t_step >= 0);
1513  if (t_step == 0) {
1514  KALDI_ASSERT(num_t_values == 1);
1515  t_step = 1;
1516  }
1517  int32 num_n_x_pairs = n_x_pairs.size();
1518  indexes->clear();
1519  indexes->reserve(num_n_x_pairs * num_t_values);
1520  int32 outer_t_step = t_step * reorder_t,
1521  t_end = t_start + (num_t_values * t_step);
1522  Index index;
1523  for (int32 t_block = t_start; t_block < t_end; t_block += outer_t_step) {
1524  for (int32 nx = 0; nx < num_n_x_pairs; nx++) {
1525  index.n = n_x_pairs[nx].first;
1526  index.x = n_x_pairs[nx].second;
1527  for (int32 t = t_block; t < t_block + outer_t_step; t += t_step) {
1528  index.t = t;
1529  indexes->push_back(index);
1530  }
1531  }
1532  }
1533  // we can remove the next assert after a while.
1534  KALDI_ASSERT(indexes->size() == num_n_x_pairs * num_t_values);
1535 }
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static int32 kaldi::nnet3::time_height_convolution::FindGcdOfDifferences ( std::vector< int32 > &  vec)
static

Definition at line 1459 of file convolution.cc.

References kaldi::Gcd(), and rnnlm::i.

Referenced by RegularizeTList().

1459  {
1460  size_t size = vec.size();
1461  int32 ans = 0;
1462  for (size_t i = 0; i + 1 < size; i++) {
1463  int32 diff = vec[i+1] - vec[i];
1464  // diff should not be zero.
1465  ans = Gcd(ans, diff);
1466  }
1467  return ans;
1468 }
I Gcd(I m, I n)
Definition: kaldi-math.h:294
void GetComputationIo ( const std::vector< Index > &  input_indexes,
const std::vector< Index > &  output_indexes,
ConvolutionComputationIo *  io 
)

This function takes lists of input and output indexes to a computation (e.g.

as supplied to ReorderIndexes()), and figures out a regular structure for them (i.e. the smallest grid that will completely cover all the t,n pairs).

Definition at line 1558 of file convolution.cc.

References GetNxList(), GetTList(), kaldi::GetVerboseLevel(), KALDI_ASSERT, ConvolutionComputationIo::num_images, ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::num_t_out, RegularizeTList(), ConvolutionComputationIo::reorder_t_in, ConvolutionComputationIo::start_t_in, ConvolutionComputationIo::start_t_out, ConvolutionComputationIo::t_step_in, and ConvolutionComputationIo::t_step_out.

Referenced by CompileConvolutionComputation().

1561  {
1562  std::vector<std::pair<int32, int32> > n_x_pairs;
1563  GetNxList(input_indexes, &n_x_pairs);
1564  KALDI_ASSERT(!n_x_pairs.empty());
1565  io->num_images = n_x_pairs.size();
1566  if (GetVerboseLevel() >= 3) { // a debugging step.
1567  std::vector<std::pair<int32, int32> > n_x_pairs_2;
1568  GetNxList(output_indexes, &n_x_pairs_2);
1569  KALDI_ASSERT(n_x_pairs_2 == n_x_pairs);
1570  }
1571  std::vector<int32> t_values;
1572  GetTList(input_indexes, &t_values);
1573  RegularizeTList(t_values, &(io->start_t_in),
1574  &(io->t_step_in), &(io->num_t_in));
1575  GetTList(output_indexes, &t_values);
1576  RegularizeTList(t_values, &(io->start_t_out),
1577  &(io->t_step_out), &(io->num_t_out));
1578  io->reorder_t_in = 1;
1579 }
int32 GetVerboseLevel()
Definition: kaldi-error.h:69
static void RegularizeTList(std::vector< int32 > &t_values, int32 *start, int32 *step, int32 *num_values)
static void GetTList(const std::vector< Index > &indexes, std::vector< int32 > *t_values)
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static void GetNxList(const std::vector< Index > &indexes, std::vector< std::pair< int32, int32 > > *pairs)
void GetIndexesForComputation ( const ConvolutionComputationIo &  io,
const std::vector< Index > &  orig_input_indexes,
const std::vector< Index > &  orig_output_indexes,
std::vector< Index > *  input_indexes,
std::vector< Index > *  output_indexes 
)

This function computes the reordered and possibly padded indexes corresponding to the computation in 'io'.

Note: the computation may have undergone various manipulations (padding, etc.) after being obtained by the function GetComputationIo(). The original input and output indexes are needed because they dictate the set of (n, x) pairs; and because they determine when to use 'real' indexes and when to use 'blank' padding values (i.e. when to replace the t values in the indexes by kNoTime).

Definition at line 1582 of file convolution.cc.

References CreateIndexes(), GetNxList(), KALDI_ASSERT, ConvolutionComputationIo::num_images, ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::num_t_out, ConvolutionComputationIo::reorder_t_in, SetSomeIndexesBlank(), ConvolutionComputationIo::start_t_in, ConvolutionComputationIo::start_t_out, ConvolutionComputationIo::t_step_in, and ConvolutionComputationIo::t_step_out.

Referenced by CompileConvolutionComputation().

1587  {
1588  std::unordered_set<Index, IndexHasher> input_set, output_set;
1589  for (std::vector<Index>::const_iterator iter = orig_input_indexes.begin();
1590  iter != orig_input_indexes.end(); ++iter)
1591  input_set.insert(*iter);
1592  for (std::vector<Index>::const_iterator iter = orig_output_indexes.begin();
1593  iter != orig_output_indexes.end(); ++iter)
1594  output_set.insert(*iter);
1595  std::vector<std::pair<int32, int32> > n_x_pairs;
1596  GetNxList(orig_input_indexes, &n_x_pairs);
1597  KALDI_ASSERT(n_x_pairs.size() == io.num_images);
1598  CreateIndexes(n_x_pairs, io.start_t_in, io.t_step_in, io.num_t_in,
1599  io.reorder_t_in, input_indexes);
1600  SetSomeIndexesBlank(orig_input_indexes, input_indexes);
1601  CreateIndexes(n_x_pairs, io.start_t_out, io.t_step_out, io.num_t_out,
1602  1, output_indexes);
1603  SetSomeIndexesBlank(orig_output_indexes, output_indexes);
1604 }
static void CreateIndexes(const std::vector< std::pair< int32, int32 > > &n_x_pairs, int32 t_start, int32 t_step, int32 num_t_values, int32 reorder_t, std::vector< Index > *indexes)
Creates a vector of indexes with a regular structure, according to these specifications.
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static void SetSomeIndexesBlank(const std::vector< Index > &ref_indexes, std::vector< Index > *indexes)
This function modifies 'indexes' by, for any Indexes which was not present in 'ref_indexes', setting the 't' value to kNoTime.
static void GetNxList(const std::vector< Index > &indexes, std::vector< std::pair< int32, int32 > > *pairs)
static void kaldi::nnet3::time_height_convolution::GetNxList ( const std::vector< Index > &  indexes,
std::vector< std::pair< int32, int32 > > *  pairs 
)
static

Definition at line 1419 of file convolution.cc.

Referenced by GetComputationIo(), and GetIndexesForComputation().

1420  {
1421  // set of (n,x) pairs
1422  std::unordered_set<std::pair<int32, int32>, PairHasher<int32> > n_x_set;
1423 
1424  for (std::vector<Index>::const_iterator iter = indexes.begin();
1425  iter != indexes.end(); ++iter)
1426  n_x_set.insert(std::pair<int32, int32>(iter->n, iter->x));
1427  pairs->clear();
1428  pairs->reserve(n_x_set.size());
1429  for (std::unordered_set<std::pair<int32, int32>, PairHasher<int32> >::iterator
1430  iter = n_x_set.begin(); iter != n_x_set.end(); ++iter)
1431  pairs->push_back(*iter);
1432  std::sort(pairs->begin(), pairs->end());
1433 }
static void kaldi::nnet3::time_height_convolution::GetRandomConvolutionIndexes ( const ConvolutionModel model,
std::vector< Index > *  input_indexes,
std::vector< Index > *  output_indexes 
)
static

Definition at line 78 of file convolution-test.cc.

References ConvolutionModel::all_time_offsets, ConvolutionModel::Check(), rnnlm::i, kaldi::IsSortedAndUniq(), rnnlm::j, KALDI_ASSERT, Index::n, rnnlm::n, kaldi::RandInt(), ConvolutionModel::required_time_offsets, kaldi::SortAndUniq(), Index::t, and Index::x.

Referenced by UnitTestTimeHeightConvolutionCompile().

80  {
81  KALDI_ASSERT(model.Check());
82 
83  std::vector<std::pair<int32, int32> > n_x_pairs;
84  int32 num_n_x_pairs = RandInt(1, 3);
85  for (int32 i = 0; i < num_n_x_pairs; i++) {
86  int32 n = RandInt(0, 3), x = RandInt(0, 1);
87  n_x_pairs.push_back(std::pair<int32, int32>(n, x));
88  }
89  SortAndUniq(&n_x_pairs);
90  num_n_x_pairs = n_x_pairs.size();
91 
92 
93  // 'output_t_values' is the set of *possible* output
94  // t values; we'll later sub-sample from these.
95  std::vector<int32> output_t_values;
96 
97  {
98  int32 out_t_start = RandInt(-5, 5), out_t_step = RandInt(1, 3),
99  num_t_out = RandInt(1, 4);
100  for (int32 i = 0; i < num_t_out; i++)
101  output_t_values.push_back(out_t_start + i * out_t_step);
102  }
103 
104  input_indexes->clear();
105  output_indexes->clear();
106  for (size_t i = 0; i < n_x_pairs.size(); i++) {
107  std::vector<int32> chosen_output_t_values;
108  while (chosen_output_t_values.empty()) {
109  for (size_t j = 0; j < output_t_values.size(); j++)
110  if (RandInt(0, 1) != 0)
111  chosen_output_t_values.push_back(output_t_values[j]);
112  }
113  KALDI_ASSERT(IsSortedAndUniq(chosen_output_t_values));
114 
115  std::set<int32> required_input_t_values,
116  usable_input_t_values;
117  for (size_t j = 0; j < chosen_output_t_values.size(); j++) {
118  std::set<int32>::const_iterator iter;
119  int32 t_out = chosen_output_t_values[j];
120  for (iter = model.required_time_offsets.begin();
121  iter != model.required_time_offsets.end(); iter++) {
122  int32 offset = *iter;
123  required_input_t_values.insert(t_out + offset);
124  }
125  for (iter = model.all_time_offsets.begin();
126  iter != model.all_time_offsets.end(); iter++) {
127  int32 offset = *iter;
128  usable_input_t_values.insert(t_out + offset);
129  }
130  }
131 
132  // add to output_indexes
133  for (size_t j = 0; j < chosen_output_t_values.size(); j++) {
134  int32 t_out = chosen_output_t_values[j];
135  Index index;
136  index.n = n_x_pairs[i].first;
137  index.x = n_x_pairs[i].second;
138  index.t = t_out;
139  output_indexes->push_back(index);
140  }
141 
142  std::vector<int32> chosen_input_t_values(required_input_t_values.begin(),
143  required_input_t_values.end());
144  for (std::set<int32>::const_iterator iter = usable_input_t_values.begin();
145  iter != usable_input_t_values.end(); ++iter) {
146  int32 t = *iter;
147  if (RandInt(0, 1) == 0)
148  chosen_input_t_values.push_back(t);
149  }
150  SortAndUniq(&chosen_input_t_values);
151 
152  // add to input_indexes
153  for (size_t j = 0; j < chosen_input_t_values.size(); j++) {
154  int32 t_in = chosen_input_t_values[j];
155  Index index;
156  index.n = n_x_pairs[i].first;
157  index.x = n_x_pairs[i].second;
158  index.t = t_in;
159  input_indexes->push_back(index);
160  }
161  }
162 }
void SortAndUniq(std::vector< T > *vec)
Sorts and uniq's (removes duplicates) from a vector.
Definition: stl-utils.h:39
struct Index is intended to represent the various indexes by which we number the rows of the matrices...
Definition: nnet-common.h:44
struct rnnlm::@11::@12 n
bool Check(bool check_heights_used=true, bool allow_height_padding=true) const
Definition: convolution.cc:129
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
bool IsSortedAndUniq(const std::vector< T > &vec)
Returns true if the vector is sorted and contains each element only once.
Definition: stl-utils.h:63
int32 RandInt(int32 min_val, int32 max_val, struct RandomState *state)
Definition: kaldi-math.cc:94
static void kaldi::nnet3::time_height_convolution::GetRandomConvolutionModel ( ConvolutionModel model)
static

Definition at line 28 of file convolution-test.cc.

References ConvolutionModel::Check(), ConvolutionModel::ComputeDerived(), ConvolutionModel::height_in, ConvolutionModel::Offset::height_offset, ConvolutionModel::height_out, ConvolutionModel::height_subsample_out, rnnlm::i, ConvolutionModel::Info(), KALDI_WARN, ConvolutionModel::num_filters_in, ConvolutionModel::num_filters_out, ConvolutionModel::offsets, kaldi::RandInt(), ConvolutionModel::required_time_offsets, kaldi::SortAndUniq(), and ConvolutionModel::Offset::time_offset.

Referenced by UnitTestTimeHeightConvolutionCompile(), and UnitTestTimeHeightConvolutionIo().

28  {
29 start:
30  {
31  model->num_filters_in = RandInt(1, 10);
32  model->num_filters_out = RandInt(1, 10);
33  model->height_in = RandInt(1, 10);
34  int32 min_height_offset = RandInt(-2, 0),
35  max_height_offset = RandInt(0, 2),
36  min_time_offset = RandInt(-2, 0),
37  max_time_offset = RandInt(0, 2);
38 
39  model->height_out = RandInt(1, model->height_in);
40  model->height_subsample_out = 1;
41  if (RandInt(0, 1) == 0) {
42  if (model->height_out % 2 == 0) {
43  model->height_out /= 2;
44  model->height_subsample_out = 2;
45  } else if (model->height_out % 3 == 0) {
46  model->height_out /= 3;
47  model->height_subsample_out = 3;
48  }
49  }
50  std::vector<int32> all_time_offsets;
51  int32 max_offsets = RandInt(1, 10);
52  model->offsets.clear();
53  model->required_time_offsets.clear();
54  for (int32 i = 0; i < max_offsets; i++) {
56  o.time_offset = RandInt(min_time_offset, max_time_offset);
57  o.height_offset = RandInt(min_height_offset, max_height_offset);
58  all_time_offsets.push_back(o.time_offset);
59  model->offsets.push_back(o);
60  }
61  SortAndUniq(&(model->offsets));
62  SortAndUniq(&all_time_offsets);
63  std::random_shuffle(all_time_offsets.begin(), all_time_offsets.end());
64  int32 num_required_offsets = RandInt(1, all_time_offsets.size());
65  for (int32 i = 0; i < num_required_offsets; i++)
66  model->required_time_offsets.insert(all_time_offsets[i]);
67  model->ComputeDerived();
68  }
69  if (!model->Check()) {
70  KALDI_WARN << "Regenerating model because it didn't pass the check: "
71  << model->Info();
72  goto start;
73  }
74 }
void SortAndUniq(std::vector< T > *vec)
Sorts and uniq's (removes duplicates) from a vector.
Definition: stl-utils.h:39
#define KALDI_WARN
Definition: kaldi-error.h:130
bool Check(bool check_heights_used=true, bool allow_height_padding=true) const
Definition: convolution.cc:129
int32 RandInt(int32 min_val, int32 max_val, struct RandomState *state)
Definition: kaldi-math.cc:94
static void kaldi::nnet3::time_height_convolution::GetTList ( const std::vector< Index > &  indexes,
std::vector< int32 > *  t_values 
)
static

Definition at line 1438 of file convolution.cc.

References kaldi::nnet3::kNoTime.

Referenced by GetComputationIo().

1439  {
1440  // set of t values
1441  std::unordered_set<int32> t_set;
1442 
1443  for (std::vector<Index>::const_iterator iter = indexes.begin();
1444  iter != indexes.end(); ++iter)
1445  if (iter->t != kNoTime)
1446  t_set.insert(iter->t);
1447  t_values->clear();
1448  t_values->reserve(t_set.size());
1449  for (std::unordered_set<int32>::iterator iter = t_set.begin();
1450  iter != t_set.end(); ++iter)
1451  t_values->push_back(*iter);
1452  std::sort(t_values->begin(), t_values->end());
1453 }
const int kNoTime
Definition: nnet-common.cc:554
void MakeComputation ( const ConvolutionModel &  model,
ConvolutionComputationIo &  io,
const ConvolutionComputationOptions &  opts,
ConvolutionComputation *  computation 
)

Definition at line 1607 of file convolution.cc.

References ComputeTempMatrixSize(), ConvolutionModel::height_in, ConvolutionComputation::height_in, ConvolutionComputation::ConvolutionStep::height_map, ConvolutionModel::height_out, ConvolutionComputation::height_out, ConvolutionModel::height_subsample_out, ConvolutionComputation::ConvolutionStep::input_time_shift, kaldi::IsSortedAndUniq(), KALDI_ASSERT, ConvolutionModel::num_filters_in, ConvolutionComputation::num_filters_in, ConvolutionModel::num_filters_out, ConvolutionComputation::num_filters_out, ConvolutionComputation::num_images, ConvolutionComputationIo::num_images, ConvolutionComputation::num_t_in, ConvolutionComputationIo::num_t_in, ConvolutionComputation::num_t_out, ConvolutionComputationIo::num_t_out, ConvolutionModel::offsets, ConvolutionComputation::ConvolutionStep::params_start_col, ConvolutionComputationIo::reorder_t_in, ConvolutionComputationIo::start_t_in, ConvolutionComputationIo::start_t_out, ConvolutionComputation::steps, ConvolutionComputationIo::t_step_in, and ConvolutionComputationIo::t_step_out.

Referenced by CompileConvolutionComputation().

1610  {
1611  KALDI_ASSERT(io.t_step_in == io.t_step_out);
1612  computation->num_filters_in = model.num_filters_in;
1613  computation->num_filters_out = model.num_filters_out;
1614  computation->height_in = model.height_in;
1615  computation->height_out = model.height_out;
1616  computation->num_t_in = io.num_t_in;
1617  computation->num_t_out = io.num_t_out;
1618  computation->num_images = io.num_images;
1619  KALDI_ASSERT(io.reorder_t_in == 1);
1620  // first work out the steps of the computation, then
1621  // work out the dim of the temp matrix
1622 
1623  KALDI_ASSERT(IsSortedAndUniq(model.offsets));
1624  // Each distinct value of 'time_offset' in model.offsets
1625  // becomes one step of the computation.
1626 
1627  // if io.t_step_in was zero, use 1 (so divisions and the like will work as
1628  // expected).
1629  int32 t_step = std::max<int32>(1, io.t_step_in),
1630  num_t_extra = io.num_t_in - io.num_t_out;
1631 
1632  computation->steps.clear();
1633 
1634  int32 num_offsets = model.offsets.size(),
1635  cur_start_offset = 0, cur_end_offset = 0;
1636  for(; cur_start_offset < num_offsets; cur_start_offset = cur_end_offset) {
1637  cur_end_offset = cur_start_offset;
1638  while (cur_end_offset < num_offsets &&
1639  model.offsets[cur_end_offset].time_offset ==
1640  model.offsets[cur_start_offset].time_offset)
1641  cur_end_offset++;
1642  // we are processing the range of indexes into 'offsets'
1643  // from cur_start_offset to cur_end_offset - 1.
1644  int32 this_num_offsets = cur_end_offset - cur_start_offset;
1645  int32 time_offset = model.offsets[cur_start_offset].time_offset;
1646 
1647  ConvolutionComputation::ConvolutionStep step;
1648  // modified_time_offset will be used in working out the 'input_time_shift'
1649  // that determines which submatrix of the input matrix we'll use.
1650  // It equals the time-offset corrected for any time-difference between
1651  // the start of the output and of the input.
1652  int32 modified_time_offset = time_offset + io.start_t_out - io.start_t_in;
1653  KALDI_ASSERT(modified_time_offset >= 0 &&
1654  modified_time_offset % t_step == 0);
1655  step.input_time_shift = modified_time_offset / t_step;
1656  KALDI_ASSERT(step.input_time_shift <= num_t_extra);
1657  step.params_start_col = model.num_filters_in * cur_start_offset;
1658  step.height_map.clear();
1659  step.height_map.reserve(model.height_out * this_num_offsets);
1660  for (int32 h_out = 0;
1661  h_out < model.height_out * model.height_subsample_out;
1662  h_out += model.height_subsample_out) {
1663  for (int32 o = cur_start_offset; o < cur_end_offset; o++) {
1664  int32 this_height_offset = model.offsets[o].height_offset,
1665  h_in = h_out + this_height_offset;
1666  // by the time we call MakeComputation, the user should already have
1667  // called PadModelHeight, so there should be no need for zero padding on
1668  // the height axis, hence the following check. [we'll later modify the
1669  // resulting computation in UnPadModelHeight, and that's where
1670  // zero-padding gets taken account of.]
1671  KALDI_ASSERT(h_in >= 0 && h_in < model.height_in);
1672  step.height_map.push_back(h_in);
1673  }
1674  }
1675  computation->steps.push_back(step);
1676  }
1677  ComputeTempMatrixSize(opts, computation);
1678 }
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static void ComputeTempMatrixSize(const ConvolutionComputationOptions &opts, ConvolutionComputation *computation)
This function sets 'temp_rows' and 'temp_cols' in 'computation'.
Definition: convolution.cc:955
bool IsSortedAndUniq(const std::vector< T > &vec)
Returns true if the vector is sorted and contains each element only once.
Definition: stl-utils.h:63
void PadComputationInputTime ( const ConvolutionModel &  model,
ConvolutionComputationIo *  io 
)

This function extends the set of input indexes that the computation has, to account for any required zero-padding in the time dimension.

It reads model.all_time_offsets and model.time_offsets_modulus; and it may modify members start_t_in t_stride_in and num_t_in of *io.

This is stage 1 of compilation.

Definition at line 1050 of file convolution.cc.

References ConvolutionModel::all_time_offsets, kaldi::Gcd(), KALDI_ASSERT, ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::num_t_out, ConvolutionComputationIo::start_t_in, ConvolutionComputationIo::start_t_out, ConvolutionComputationIo::t_step_in, ConvolutionComputationIo::t_step_out, and ConvolutionModel::time_offsets_modulus.

Referenced by CompileConvolutionComputation().

1051  {
1052  if (model.time_offsets_modulus == 0) {
1053  // this can only happen if model->all_time_offsets.size() == 1,
1054  // and no padding could be required here. W return to avoid
1055  // special cases below in Gcd().
1056  return;
1057  }
1058  int32 min_time_offset = *model.all_time_offsets.begin(),
1059  max_time_offset = *model.all_time_offsets.rbegin();
1060 
1061  // it makes everything much simpler if we just enforce that the stride of the
1062  // input divides model.time_offsets_modulus and also the output stride.
1063  // (enforcing this may make the input stride smaller). This may in certain
1064  // very odd cases cause us to require more inputs [actually 'blanks'] than
1065  // we really need, but it avoids a lot of careful thought.
1066  int32 old_t_step_in = io->t_step_in;
1067  io->t_step_in = Gcd(io->t_step_in, model.time_offsets_modulus);
1068  if (io->t_step_out != 0)
1069  io->t_step_in = Gcd(io->t_step_in, io->t_step_out);
1070 
1071  // to ensure that we cover all the original input points, now that
1072  // we changed the stride we may need to increase num_t_in.
1073  io->num_t_in = 1 + (old_t_step_in * (io->num_t_in - 1)) / io->t_step_in;
1074 
1075  // by 'desired' we mean usable as an input, not necessarily
1076  // required in the sense of 'required_time_offsets'.
1077  int32 first_desired_input_t = io->start_t_out + min_time_offset;
1078  if (first_desired_input_t < io->start_t_in) {
1079  KALDI_ASSERT((io->start_t_in - first_desired_input_t) %
1080  io->t_step_in == 0);
1081  io->num_t_in += (io->start_t_in - first_desired_input_t) / io->t_step_in;
1082  io->start_t_in = first_desired_input_t;
1083  }
1084 
1085  int32 last_desired_input_t =
1086  io->start_t_out + (io->num_t_out - 1) * io->t_step_out + max_time_offset,
1087  last_input_t = io->start_t_in + (io->num_t_in - 1) * io->t_step_in;
1088  // if the following assert fails, it means we had provided more input than was
1089  // needed, which is not expected. This could cause problems later, in
1090  // AppendInputFrames().
1091  KALDI_ASSERT(last_desired_input_t >= last_input_t);
1092  if (last_desired_input_t > last_input_t) {
1093  KALDI_ASSERT((last_desired_input_t - last_input_t) %
1094  io->t_step_in == 0);
1095  io->num_t_in += (last_desired_input_t - last_input_t) / io->t_step_in;
1096  }
1097 }
I Gcd(I m, I n)
Definition: kaldi-math.h:294
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
void PadModelHeight ( const ConvolutionModel &  model,
ConvolutionModel *  model_padded 
)

This function takes a model that might require zero padding in the height dimension and outputs a model accepting a possibly-larger input dimension which does not require zero padding.

*model_padded may differ from 'model' in its height_in and its 'offsets' variable (the height-offsets need to be shifted if we pad at the bottom). We then work out the computation in terms of the model that doesn't need padding (which is easier), and later convert it back to work in the space where there is no padding.

This is stage 2 of compilation.

Definition at line 917 of file convolution.cc.

References ConvolutionModel::Check(), ConvolutionModel::height_in, ConvolutionModel::height_out, ConvolutionModel::height_subsample_out, rnnlm::i, KALDI_ASSERT, and ConvolutionModel::offsets.

Referenced by CompileConvolutionComputation().

918  {
919  *model_padded = model;
920  KALDI_ASSERT(!model.offsets.empty());
921  int32 min_height_offset = model.offsets[0].height_offset,
922  max_height_offset = model.offsets[0].height_offset,
923  num_offsets = model.offsets.size();
924  for (int32 i = 1; i < num_offsets; i++) {
925  min_height_offset = std::min<int32>(min_height_offset,
926  model.offsets[i].height_offset);
927  max_height_offset = std::max<int32>(max_height_offset,
928  model.offsets[i].height_offset);
929  }
930  int32 max_output_height = model.height_subsample_out * (model.height_out - 1),
931  max_required_input = max_height_offset + max_output_height,
932  min_required_input = min_height_offset + 0;
933  int32 bottom_padding = -min_required_input,
934  top_padding = max_required_input - (model.height_in - 1);
935  if (bottom_padding < 0)
936  bottom_padding = 0;
937  if (top_padding < 0)
938  top_padding = 0;
939  model_padded->height_in += bottom_padding + top_padding;
940  for (int32 i = 0; i < num_offsets; i++)
941  model_padded->offsets[i].height_offset += bottom_padding;
942 
943  // The reason why we say 'allow_height_padding = false' below is obvious--
944  // we've 'manually' padded by changing the model, so this modified model
945  // should not require height padding. The reason we set 'check_heights_used'
946  // is a little more non-obvious. The very lowest and hightest heights
947  // should always be used, but there may, in unusual models, be other heights
948  // that are not used. We found this in random testing.
949  KALDI_ASSERT(model_padded->Check(false, false));
950 }
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static int32 kaldi::nnet3::time_height_convolution::PrepareIoForAppending ( ConvolutionComputationIo *  io,
ConvolutionComputationIo *  io_appended 
)
static

Definition at line 1157 of file convolution.cc.

References KALDI_ASSERT, ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::num_t_out, ConvolutionComputationIo::reorder_t_in, ConvolutionComputationIo::t_step_in, and ConvolutionComputationIo::t_step_out.

Referenced by AppendInputFrames().

1158  {
1159  // first make sure that the output has nonzero stride (it would only have zero
1160  // stride if there was only one output time index, which is unusual). if
1161  // there's only one output time index we can set the stride to whatever we
1162  // want without affecting the list of output indexes.
1163  int32 ratio;
1164  if (io->t_step_out == 0) {
1165  KALDI_ASSERT(io->num_t_out == 1);
1166  io->t_step_out = io->t_step_in;
1167  }
1168  if (io->t_step_out == io->t_step_in) {
1169  // there is nothing to do; the output and input strides are the same.
1170  *io_appended = *io;
1171  ratio = 1;
1172  return ratio;
1173  }
1174  // Now, we ensured in PadComputationInputTime that if the output stride is
1175  // nonzero, then the input stride must divide the output stride; and if the
1176  // output stride was zero then we would have set it to the input stride just
1177  // above; and if both were zero we would have returned above. So we can just
1178  // assert that the input stride divides the output stride.
1179  KALDI_ASSERT(io->t_step_out % io->t_step_in == 0);
1180  ratio = io->t_step_out / io->t_step_in;
1181  // ratio says how many input indexes we have for each output index,
1182  // ignoring end effects. It is the number of input indexes we will
1183  // append together and 'pretend'
1184 
1185  // record this ratio in the 'input' I/O object, which we are also
1186  // modifying to record the extra required padding.
1187  io->reorder_t_in = ratio;
1188  if (io->num_t_in % ratio != 0) {
1189  // Round up the number of input frames to the nearest multiple (via
1190  // zero-padding) so we get an whole number of appended input frames.
1191  io->num_t_in += ratio - (io->num_t_in % ratio);
1192  }
1193 
1194  // OK, from this point we create the output io object.
1195  *io_appended = *io;
1196  io_appended->reorder_t_in = 1;
1197  io_appended->t_step_in = io->t_step_out;
1198  io_appended->num_t_in /= ratio;
1199  return ratio;
1200 }
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static void kaldi::nnet3::time_height_convolution::RegularizeTList ( std::vector< int32 > &  t_values,
int32 *  start,
int32 *  step,
int32 *  num_values 
)
static

Definition at line 1470 of file convolution.cc.

References FindGcdOfDifferences(), kaldi::IsSortedAndUniq(), and KALDI_ASSERT.

Referenced by GetComputationIo().

1473  {
1474  KALDI_ASSERT(!t_values.empty() && IsSortedAndUniq(t_values));
1475  *start = t_values[0];
1476  *step = FindGcdOfDifferences(t_values);
1477  if (*step == 0) {
1478  KALDI_ASSERT(t_values.size() == 1);
1479  *num_values = 1;
1480  } else {
1481  int32 last_value = t_values.back();
1482  *num_values = 1 + (last_value - *start) / *step;
1483  KALDI_ASSERT((last_value - *start) % *step == 0);
1484  }
1485 }
static int32 FindGcdOfDifferences(std::vector< int32 > &vec)
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
bool IsSortedAndUniq(const std::vector< T > &vec)
Returns true if the vector is sorted and contains each element only once.
Definition: stl-utils.h:63
static void kaldi::nnet3::time_height_convolution::ReverseColumnMapping ( const std::vector< int32 > &  columns,
int32  input_dim,
std::vector< std::vector< int32 > > *  backward_columns 
)
static

This function, used in ConvolutionComputation::ComputeDerived(), reverses a mapping that may not be unique.

'columns' is a column mapping where each member is either -1 (meaning, copy a zero), or a number between 0 and input_dim - 1.

Its output, 'backward_columns', is the reverse mapping, but it's a vector of vectors instead of just a vector because the mapping may have been many-to-one. Each element of 'backward_columns' will be of dimension input_dim. For each columns[i] = j such that j != -1, for some k we will have (*backward_columns)[k][j] = i.

Definition at line 43 of file convolution.cc.

References rnnlm::i, rnnlm::j, and KALDI_ASSERT.

Referenced by ConvolutionComputation::ComputeDerived().

46  {
47  int32 columns_dim = columns.size();
48  std::vector<std::vector<int32> > temp(input_dim);
49  for (int32 i = 0; i < columns_dim; i++) {
50  int32 j = columns[i];
51  KALDI_ASSERT(j >= -1 && j < input_dim);
52  if (j != -1)
53  temp[j].push_back(i);
54  }
55  // 'max_overlap' is the largest number of times that some j >= 0 appears in
56  // 'columns'.
57  int32 max_overlap = 0;
58  for (int32 j = 0; j < input_dim; j++)
59  max_overlap = std::max(max_overlap,
60  static_cast<int32>(temp[j].size()));
61  backward_columns->resize(max_overlap);
62  for (int32 k = 0; k < max_overlap; k++) {
63  (*backward_columns)[k].clear();
64  (*backward_columns)[k].resize(input_dim, -1);
65  }
66  for (int32 j = 0; j < input_dim; j++) {
67  for (int32 k = 0; k < static_cast<int32>(temp[j].size()); k++) {
68  int32 i = temp[j][k];
69  (*backward_columns)[k][j] = i;
70  }
71  }
72 }
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static int32 kaldi::nnet3::time_height_convolution::RoundDownToMultipleOf ( int32  i,
int32  n 
)
static

Definition at line 1102 of file convolution.cc.

References kaldi::DivideRoundingDown().

Referenced by AppendInputFrames().

1102  {
1103  return n * DivideRoundingDown(i, n);
1104 }
static int32 DivideRoundingDown(int32 a, int32 b)
Returns a / b, rounding towards negative infinity in all cases.
Definition: kaldi-math.h:284
struct rnnlm::@11::@12 n
static void kaldi::nnet3::time_height_convolution::SetSomeIndexesBlank ( const std::vector< Index > &  ref_indexes,
std::vector< Index > *  indexes 
)
static

This function modifies 'indexes' by, for any Indexes which was not present in 'ref_indexes', setting the 't' value to kNoTime.

This will cause the nnet3 framework to ignore such Indexes for certain purposes, it supresses certain error conditions that would otherwise happen from inserting unnecessary indexes into the input and output.

Definition at line 1544 of file convolution.cc.

References kaldi::nnet3::kNoTime.

Referenced by GetIndexesForComputation().

1545  {
1546  std::unordered_set<Index, IndexHasher> ref_set;
1547  for (std::vector<Index>::const_iterator iter = ref_indexes.begin();
1548  iter != ref_indexes.end(); ++iter)
1549  ref_set.insert(*iter);
1550 
1551  for (std::vector<Index>::iterator iter = indexes->begin();
1552  iter != indexes->end(); ++iter) {
1553  if (ref_set.count(*iter) == 0)
1554  iter->t = kNoTime;
1555  }
1556 }
const int kNoTime
Definition: nnet-common.cc:554
static void kaldi::nnet3::time_height_convolution::ShiftAllTimeOffsets ( int32  shift,
ConvolutionModel *  model 
)
static

Definition at line 1109 of file convolution.cc.

References ConvolutionModel::all_time_offsets, ConvolutionModel::offsets, and ConvolutionModel::required_time_offsets.

Referenced by AppendInputFrames().

1110  {
1111  { // shift 'offsets'.
1112  std::vector<ConvolutionModel::Offset>::iterator
1113  iter = model->offsets.begin(),
1114  end = model->offsets.end();
1115  for (; iter != end; ++iter)
1116  iter->time_offset += shift;
1117  }
1118  std::set<int32> temp;
1119  std::set<int32>::const_iterator iter;
1120  for (iter = model->required_time_offsets.begin();
1121  iter != model->required_time_offsets.end(); ++iter)
1122  temp.insert(*iter + shift);
1123  model->required_time_offsets.swap(temp);
1124  temp.clear();
1125  for (iter = model->all_time_offsets.begin();
1126  iter != model->all_time_offsets.end(); ++iter)
1127  temp.insert(*iter + shift);
1128  model->all_time_offsets.swap(temp);
1129 }
void kaldi::nnet3::time_height_convolution::TestComputationIo ( const ConvolutionComputation computation)

Definition at line 182 of file convolution-test.cc.

References KALDI_ASSERT, kaldi::RandInt(), ConvolutionComputation::Read(), and ConvolutionComputation::Write().

Referenced by UnitTestTimeHeightConvolutionCompile().

182  {
183  std::ostringstream os1, os2;
184  bool binary = (RandInt(0, 1) == 0);
185  computation.Write(os1, binary);
186  std::istringstream is(os1.str());
187  ConvolutionComputation computation2;
188  computation2.Read(is, binary);
189  computation2.Write(os2, binary);
190  KALDI_ASSERT(os1.str() == os2.str());
191  computation2.Check();
192 }
This struct represents the structure of a convolution computation.
Definition: convolution.h:252
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
void Write(std::ostream &os, bool binary) const
Definition: convolution.cc:282
int32 RandInt(int32 min_val, int32 max_val, struct RandomState *state)
Definition: kaldi-math.cc:94
void kaldi::nnet3::time_height_convolution::TestDataBackprop ( const ConvolutionModel conv_model,
const std::vector< Index > &  input_indexes,
const std::vector< Index > &  output_indexes,
const ConvolutionComputation computation 
)

Definition at line 308 of file convolution-test.cc.

References kaldi::ApproxEqual(), ConvolveBackwardData(), ConvolveForward(), ConvolutionModel::InputDim(), KALDI_ERR, KALDI_LOG, kaldi::kSetZero, kaldi::kStrideEqualNumCols, kaldi::kTrans, ConvolutionModel::OutputDim(), ConvolutionModel::ParamCols(), ConvolutionModel::ParamRows(), CuMatrixBase< Real >::SetRandn(), kaldi::TraceMatMat(), and ZeroBlankRows().

Referenced by UnitTestTimeHeightConvolutionCompile().

311  {
313  input_deriv(input_indexes.size(), conv_model.InputDim(),
315  input(input_indexes.size(), conv_model.InputDim(),
317  output(output_indexes.size(), conv_model.OutputDim(),
319  output_deriv(output_indexes.size(), conv_model.OutputDim(),
321  params(conv_model.ParamRows(), conv_model.ParamCols());
322 
323  input.SetRandn();
324  params.SetRandn();
325  output_deriv.SetRandn();
326 
327  ZeroBlankRows(output_indexes, &output_deriv);
328  ConvolveBackwardData(computation, params, output_deriv, &input_deriv);
329  ZeroBlankRows(input_indexes, &input_deriv);
330  ZeroBlankRows(input_indexes, &input);
331 
332  // define the objf as TraceMatMat(output_deriv, output, kTrans).
333  // we can work it out from the backpropagated data-derivative.
334  BaseFloat expected_objf = TraceMatMat(input_deriv, input, kTrans);
335 
336  ConvolveForward(computation, input, params, &output);
337  ZeroBlankRows(output_indexes, &output);
338 
339  BaseFloat observed_objf = TraceMatMat(output, output_deriv, kTrans);
340 
341  KALDI_LOG << "Expected objf = " << expected_objf
342  << ", observed objf = " << observed_objf;
343  if (!ApproxEqual(expected_objf, observed_objf, 0.1) &&
344  fabs(expected_objf) < 1.0) {
345  KALDI_ERR << "Difference in objf too large.";
346  }
347 }
float BaseFloat
Definition: kaldi-types.h:29
void ConvolveBackwardData(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &params, const CuMatrixBase< BaseFloat > &output_deriv, CuMatrixBase< BaseFloat > *input_deriv)
This does the part of the backward derivative computation of convolution, that propagates derivatives...
Definition: convolution.cc:681
void ZeroBlankRows(const std::vector< Index > &indexes, CuMatrix< BaseFloat > *matrix)
#define KALDI_ERR
Definition: kaldi-error.h:127
Real TraceMatMat(const MatrixBase< Real > &A, const MatrixBase< Real > &B, MatrixTransposeType trans)
We need to declare this here as it will be a friend function.
#define KALDI_LOG
Definition: kaldi-error.h:133
void ConvolveForward(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &params, CuMatrixBase< BaseFloat > *output)
This does the forward computation of convolution.
Definition: convolution.cc:523
static bool ApproxEqual(float a, float b, float relative_tolerance=0.001)
return abs(a - b) <= relative_tolerance * (abs(a)+abs(b)).
Definition: kaldi-math.h:262
void kaldi::nnet3::time_height_convolution::TestParamsBackprop ( const ConvolutionModel conv_model,
const std::vector< Index > &  input_indexes,
const std::vector< Index > &  output_indexes,
const ConvolutionComputation computation 
)

Definition at line 350 of file convolution-test.cc.

References kaldi::ApproxEqual(), ConvolveBackwardParams(), ConvolveForward(), ConvolutionModel::InputDim(), KALDI_ERR, KALDI_LOG, kaldi::kSetZero, kaldi::kStrideEqualNumCols, kaldi::kTrans, ConvolutionModel::OutputDim(), ConvolutionModel::ParamCols(), ConvolutionModel::ParamRows(), kaldi::RandInt(), CuMatrixBase< Real >::SetRandn(), kaldi::TraceMatMat(), and ZeroBlankRows().

Referenced by UnitTestTimeHeightConvolutionCompile().

353  {
355  input(input_indexes.size(), conv_model.InputDim(),
357  output(output_indexes.size(), conv_model.OutputDim(),
359  output_deriv(output_indexes.size(), conv_model.OutputDim(),
361  params(conv_model.ParamRows(), conv_model.ParamCols()),
362  params_deriv(conv_model.ParamRows(), conv_model.ParamCols());
363 
364  input.SetRandn();
365  params.SetRandn();
366  output_deriv.SetRandn();
367 
368  BaseFloat alpha = 0.5 * RandInt(1, 3);
369 
370  ZeroBlankRows(output_indexes, &output_deriv);
371  ZeroBlankRows(input_indexes, &input);
372 
373  ConvolveBackwardParams(computation, input, output_deriv, alpha,
374  &params_deriv);
375 
376  BaseFloat expected_objf = TraceMatMat(params_deriv, params, kTrans) / alpha;
377 
378  ConvolveForward(computation, input, params, &output);
379 
380  ZeroBlankRows(output_indexes, &output);
381 
382  BaseFloat observed_objf = TraceMatMat(output, output_deriv, kTrans);
383 
384  KALDI_LOG << "Expected objf = " << expected_objf
385  << ", observed objf = " << observed_objf;
386  if (!ApproxEqual(expected_objf, observed_objf, 0.1) &&
387  fabs(expected_objf) < 1.0) {
388  KALDI_ERR << "Difference in objf too large.";
389  }
390 }
void ConvolveBackwardParams(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &output_deriv, BaseFloat alpha, CuMatrixBase< BaseFloat > *params_deriv)
This does the part of the backward derivative computation of convolution, that computes derivatives w...
Definition: convolution.cc:839
float BaseFloat
Definition: kaldi-types.h:29
void ZeroBlankRows(const std::vector< Index > &indexes, CuMatrix< BaseFloat > *matrix)
#define KALDI_ERR
Definition: kaldi-error.h:127
Real TraceMatMat(const MatrixBase< Real > &A, const MatrixBase< Real > &B, MatrixTransposeType trans)
We need to declare this here as it will be a friend function.
#define KALDI_LOG
Definition: kaldi-error.h:133
void ConvolveForward(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &params, CuMatrixBase< BaseFloat > *output)
This does the forward computation of convolution.
Definition: convolution.cc:523
static bool ApproxEqual(float a, float b, float relative_tolerance=0.001)
return abs(a - b) <= relative_tolerance * (abs(a)+abs(b)).
Definition: kaldi-math.h:262
int32 RandInt(int32 min_val, int32 max_val, struct RandomState *state)
Definition: kaldi-math.cc:94
void kaldi::nnet3::time_height_convolution::TestRunningComputation ( const ConvolutionModel conv_model,
const std::vector< Index > &  input_indexes,
const std::vector< Index > &  output_indexes,
const ConvolutionComputation computation 
)

Definition at line 280 of file convolution-test.cc.

References ConvolveForward(), ConvolveForwardSimple(), ConvolutionModel::Info(), ConvolutionModel::InputDim(), KALDI_ERR, KALDI_LOG, kaldi::kSetZero, kaldi::kStrideEqualNumCols, ConvolutionModel::OutputDim(), ConvolutionModel::ParamCols(), ConvolutionModel::ParamRows(), CuMatrixBase< Real >::SetRandn(), and ZeroBlankRows().

Referenced by UnitTestTimeHeightConvolutionCompile().

283  {
284  CuMatrix<BaseFloat> input(input_indexes.size(), conv_model.InputDim(),
286  output(output_indexes.size(), conv_model.OutputDim(),
288  output2(output),
289  params(conv_model.ParamRows(), conv_model.ParamCols());
290  input.SetRandn();
291  params.SetRandn();
292  ZeroBlankRows(input_indexes, &input);
293  ConvolveForward(computation, input, params, &output);
294  ZeroBlankRows(output_indexes, &output);
295 
296  ConvolveForwardSimple(conv_model, input_indexes, output_indexes,
297  input, params, &output2);
298  KALDI_LOG << "Tested convolution for model: "
299  << conv_model.Info();
300  if (!output.ApproxEqual(output2, 0.001)) {
301  KALDI_LOG << "Output is: " << output;
302  KALDI_LOG << "Output2 is: " << output2;
303  KALDI_ERR << "Convolution test failure.";
304  }
305 }
void ZeroBlankRows(const std::vector< Index > &indexes, CuMatrix< BaseFloat > *matrix)
void ConvolveForwardSimple(const ConvolutionModel &model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const CuMatrixBase< BaseFloat > &input_cu, const CuMatrixBase< BaseFloat > &params_cu, CuMatrixBase< BaseFloat > *output_cu)
#define KALDI_ERR
Definition: kaldi-error.h:127
#define KALDI_LOG
Definition: kaldi-error.h:133
void ConvolveForward(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &params, CuMatrixBase< BaseFloat > *output)
This does the forward computation of convolution.
Definition: convolution.cc:523
static bool kaldi::nnet3::time_height_convolution::TimeValueInInput ( const ConvolutionComputationIo &  io,
int32  t 
)
static

Definition at line 1320 of file convolution.cc.

References ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::start_t_in, and ConvolutionComputationIo::t_step_in.

Referenced by CheckModelAndIo().

1321  {
1322  int32 t_step_in = std::max<int32>(1, io.t_step_in);
1323  return (t >= io.start_t_in &&
1324  t < io.start_t_in + (t_step_in * io.num_t_in) &&
1325  (t - io.start_t_in) % t_step_in == 0);
1326 }
void kaldi::nnet3::time_height_convolution::UnitTestTimeHeightConvolution ( )
void kaldi::nnet3::time_height_convolution::UnitTestTimeHeightConvolutionCompile ( )

Definition at line 394 of file convolution-test.cc.

References CompileConvolutionComputation(), GetRandomConvolutionIndexes(), GetRandomConvolutionModel(), rnnlm::i, KALDI_LOG, TestComputationIo(), TestDataBackprop(), TestParamsBackprop(), TestRunningComputation(), and kaldi::nnet3::WriteIndexVector().

Referenced by UnitTestTimeHeightConvolution().

394  {
395  for (int32 i = 0; i < 10; i++) {
396  KALDI_LOG << "iter = " << i;
397  // Create a ConvolutionModel
398  ConvolutionModel conv_model;
399  GetRandomConvolutionModel(&conv_model);
400  std::vector<Index> input_indexes, output_indexes;
401  GetRandomConvolutionIndexes(conv_model, &input_indexes, &output_indexes);
402 
404  ConvolutionComputation computation;
405  std::vector<Index> input_indexes_modified, output_indexes_modified;
406  CompileConvolutionComputation(conv_model, input_indexes, output_indexes,
407  opts, &computation,
408  &input_indexes_modified,
409  &output_indexes_modified);
410  TestComputationIo(computation);
411  TestRunningComputation(conv_model,
412  input_indexes_modified,
413  output_indexes_modified,
414  computation);
415  TestDataBackprop(conv_model,
416  input_indexes_modified,
417  output_indexes_modified,
418  computation);
419  TestParamsBackprop(conv_model,
420  input_indexes_modified,
421  output_indexes_modified,
422  computation);
423  std::ostringstream os;
424  os << "\nInput-indexes: ";
425  WriteIndexVector(os, false, input_indexes);
426  os << "\nInput-indexes-modified: ";
427  WriteIndexVector(os, false, input_indexes_modified);
428  os << "\nOutput-indexes: ";
429  WriteIndexVector(os, false, output_indexes);
430  os << "\nOutput-indexes-modified: ";
431  WriteIndexVector(os, false, output_indexes_modified);
432  KALDI_LOG << os.str();
433  }
434 }
void WriteIndexVector(std::ostream &os, bool binary, const std::vector< Index > &vec)
Definition: nnet-common.cc:126
This comment explains the basic framework used for everything related to time-height convolution...
Definition: convolution.h:125
static void GetRandomConvolutionIndexes(const ConvolutionModel &model, std::vector< Index > *input_indexes, std::vector< Index > *output_indexes)
void TestRunningComputation(const ConvolutionModel &conv_model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputation &computation)
void TestComputationIo(const ConvolutionComputation &computation)
void CompileConvolutionComputation(const ConvolutionModel &model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputationOptions &opts, ConvolutionComputation *computation, std::vector< Index > *input_indexes_modified, std::vector< Index > *output_indexes_modified)
This function does the compilation for a convolution computation; it's a wrapper for the functions be...
This struct represents the structure of a convolution computation.
Definition: convolution.h:252
void TestParamsBackprop(const ConvolutionModel &conv_model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputation &computation)
This struct contains options for compiling the convolutional computation.
Definition: convolution.h:362
static void GetRandomConvolutionModel(ConvolutionModel *model)
void TestDataBackprop(const ConvolutionModel &conv_model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputation &computation)
#define KALDI_LOG
Definition: kaldi-error.h:133
void kaldi::nnet3::time_height_convolution::UnitTestTimeHeightConvolutionIo ( )

Definition at line 165 of file convolution-test.cc.

References GetRandomConvolutionModel(), rnnlm::i, KALDI_ASSERT, KALDI_LOG, kaldi::RandInt(), ConvolutionModel::Read(), and ConvolutionModel::Write().

Referenced by UnitTestTimeHeightConvolution().

165  {
166  for (int32 i = 0; i < 10; i++) {
167  KALDI_LOG << "iter = " << i;
168  // Create a ConvolutionModel and test its I/O.
169  ConvolutionModel conv_model;
170  GetRandomConvolutionModel(&conv_model);
171  std::ostringstream os1, os2;
172  bool binary = (RandInt(0, 1) == 0);
173  conv_model.Write(os1, binary);
174  std::istringstream is(os1.str());
175  ConvolutionModel conv_model2;
176  conv_model2.Read(is, binary);
177  conv_model2.Write(os2, binary);
178  KALDI_ASSERT(os1.str() == os2.str() && conv_model2.Check());
179  }
180 }
This comment explains the basic framework used for everything related to time-height convolution...
Definition: convolution.h:125
void Write(std::ostream &os, bool binary) const
Definition: convolution.cc:224
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static void GetRandomConvolutionModel(ConvolutionModel *model)
#define KALDI_LOG
Definition: kaldi-error.h:133
int32 RandInt(int32 min_val, int32 max_val, struct RandomState *state)
Definition: kaldi-math.cc:94
void UnPadModelHeight ( const ConvolutionComputationOptions &  opts,
const ConvolutionModel &  model,
const ConvolutionModel &  model_padded,
ConvolutionComputation *  computation 
)

This function modifies, if necessary, a computation that has been built for the model 'model_padded', so that it can work for the original model 'model'.

This may involve modifying the members 'height_in', 'temp_cols', and the column-related members of the elements of the 'steps' array. View it as the reverse step for 'PadModelHeight'.

This function has to be aware that the computation will have been compiled after 'AppendInputFrames()' was called [this makes a difference in setups with subsampling], so the computation may have been built for input frames that were appended over several of the frames that 'model_padded' would require.

This is the reverse step for stage 2 of compilation (it's a transformation of the computation).

Definition at line 1000 of file convolution.cc.

References ConvolutionComputation::Check(), ConvolutionComputation::ComputeDerived(), ComputeTempMatrixSize(), ConvolutionModel::height_in, ConvolutionComputation::height_in, ConvolutionComputation::ConvolutionStep::height_map, ConvolutionModel::height_out, ConvolutionComputation::height_out, rnnlm::i, KALDI_ASSERT, ConvolutionModel::offsets, and ConvolutionComputation::steps.

Referenced by CompileConvolutionComputation().

1003  {
1004  // First work out how much padding was done in PadModelHeight().
1005  int32 bottom_padding = (model_padded.offsets[0].height_offset -
1006  model.offsets[0].height_offset),
1007  total_padding = model_padded.height_in - model.height_in,
1008  top_padding = total_padding - bottom_padding;
1009 
1010  int32 old_computation_height_in = computation->height_in;
1011  // The computation may have been built for the input appended over
1012  // several frames. Check that it is for an input height that's a multiple of
1013  // the model input height.
1014  KALDI_ASSERT(old_computation_height_in % model_padded.height_in == 0 &&
1015  computation->height_out == model.height_out);
1016 
1017  // 'ratio' is the same ratio from AppendInputFrames(), it's the number
1018  // of input frames in 'model' and 'model_padded' that get appended
1019  // to form a single frame in the computation.
1020  int32 num_steps = computation->steps.size(),
1021  unpadded_input_height = model.height_in,
1022  padded_input_height = model_padded.height_in,
1023  ratio = old_computation_height_in / padded_input_height;
1024 
1025  computation->height_in = ratio * unpadded_input_height;
1026  for (int32 s = 0; s < num_steps; s++) {
1027  ConvolutionComputation::ConvolutionStep &step = computation->steps[s];
1028  int32 height_map_size = step.height_map.size();
1029  for (int32 i = 0; i < height_map_size; i++) {
1030  int32 c = step.height_map[i];
1031  KALDI_ASSERT(c >= 0); // there should be no -1's in the padded computation.
1032  // below, h is the actual height in terms of the padded computation, and m
1033  // is an index that goes from zero to (num-appended-frames - 1).
1034  int32 h = c % padded_input_height,
1035  m = c / padded_input_height;
1036  KALDI_ASSERT(m < ratio);
1037  if (h < bottom_padding || h >= padded_input_height - top_padding) {
1038  step.height_map[i] = -1;
1039  } else {
1040  step.height_map[i] = (h - bottom_padding) + m * unpadded_input_height;
1041  }
1042  }
1043  }
1044  ComputeTempMatrixSize(opts, computation);
1045  computation->ComputeDerived();
1046  computation->Check();
1047 }
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static void ComputeTempMatrixSize(const ConvolutionComputationOptions &opts, ConvolutionComputation *computation)
This function sets 'temp_rows' and 'temp_cols' in 'computation'.
Definition: convolution.cc:955
static bool kaldi::nnet3::time_height_convolution::VectorIsContiguous ( const std::vector< int32 > &  vec)
static

Definition at line 76 of file convolution.cc.

References rnnlm::i, and KALDI_ASSERT.

Referenced by ConvolutionComputation::ComputeDerived(), and ComputeTempMatrixSize().

76  {
77  KALDI_ASSERT(!vec.empty());
78  int32 s = vec.size();
79  for (int32 i = 0; i + 1 < s; i++)
80  if (vec[i+1] != vec[i] + 1)
81  return false;
82  return true;
83 }
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
void kaldi::nnet3::time_height_convolution::ZeroBlankRows ( const std::vector< Index > &  indexes,
CuMatrix< BaseFloat > *  matrix 
)

Definition at line 198 of file convolution-test.cc.

References VectorBase< Real >::Data(), KALDI_ASSERT, kaldi::nnet3::kNoTime, kaldi::kUndefined, CuMatrixBase< Real >::MulRowsVec(), CuMatrixBase< Real >::NumRows(), VectorBase< Real >::Set(), and CuVector< Real >::Swap().

Referenced by TestDataBackprop(), TestParamsBackprop(), and TestRunningComputation().

199  {
200  KALDI_ASSERT(static_cast<int32>(indexes.size()) == matrix->NumRows());
201  int32 num_rows = matrix->NumRows();
202  if (num_rows == 0) return;
203  Vector<BaseFloat> mask(num_rows, kUndefined);
204  mask.Set(1.0);
205  const Index *indexes_ptr = &(indexes[0]);
206  BaseFloat *mask_ptr = mask.Data();
207  for (int32 r = 0; r < num_rows; r++) {
208  if (indexes_ptr[r].t == kNoTime)
209  mask_ptr[r] = 0.0;
210  }
211  CuVector<BaseFloat> cu_mask;
212  cu_mask.Swap(&mask);
213  matrix->MulRowsVec(cu_mask);
214 }
void Swap(Vector< Real > *vec)
Definition: cu-vector.cc:918
struct Index is intended to represent the various indexes by which we number the rows of the matrices...
Definition: nnet-common.h:44
void MulRowsVec(const CuVectorBase< Real > &scale)
scale i'th row by scale[i]
Definition: cu-matrix.cc:777
float BaseFloat
Definition: kaldi-types.h:29
MatrixIndexT NumRows() const
Dimensions.
Definition: cu-matrix.h:195
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
const int kNoTime
Definition: nnet-common.cc:554