All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
kaldi::nnet3::time_height_convolution Namespace Reference

Classes

struct  ConvolutionComputation
 This struct represents the structure of a convolution computation. More...
 
struct  ConvolutionComputationIo
 
struct  ConvolutionComputationOptions
 This struct contains options for compiling the convolutional computation. More...
 
struct  ConvolutionModel
 This comment explains the basic framework used for everything related to time-height convolution. More...
 

Functions

static void GetRandomConvolutionModel (ConvolutionModel *model)
 
static void GetRandomConvolutionIndexes (const ConvolutionModel &model, std::vector< Index > *input_indexes, std::vector< Index > *output_indexes)
 
void UnitTestTimeHeightConvolutionIo ()
 
void TestComputationIo (const ConvolutionComputation &computation)
 
void ZeroBlankRows (const std::vector< Index > &indexes, CuMatrix< BaseFloat > *matrix)
 
void ConvolveForwardSimple (const ConvolutionModel &model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const CuMatrixBase< BaseFloat > &input_cu, const CuMatrixBase< BaseFloat > &params_cu, CuMatrixBase< BaseFloat > *output_cu)
 
void TestRunningComputation (const ConvolutionModel &conv_model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputation &computation)
 
void TestDataBackprop (const ConvolutionModel &conv_model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputation &computation)
 
void TestParamsBackprop (const ConvolutionModel &conv_model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputation &computation)
 
void UnitTestTimeHeightConvolutionCompile ()
 
void UnitTestTimeHeightConvolution ()
 
static void ReverseColumnMapping (const std::vector< int32 > &columns, int32 input_dim, std::vector< std::vector< int32 > > *backward_columns)
 This function, used in ConvolutionComputation::ComputeDerived(), reverses a mapping that may not be unique. More...
 
static bool VectorIsContiguous (const std::vector< int32 > &vec)
 
static void ConvolveForwardInternal (const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &params, CuMatrixBase< BaseFloat > *temp_mat, CuMatrixBase< BaseFloat > *output)
 
void ConvolveForward (const ConvolutionComputation &conv_comp, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &params, CuMatrixBase< BaseFloat > *output)
 This does the forward computation of convolution. More...
 
static void ConvolveBackwardDataInternal (const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &params, const CuMatrixBase< BaseFloat > &output_deriv, CuMatrixBase< BaseFloat > *temp_mat, CuMatrixBase< BaseFloat > *input_deriv)
 
void ConvolveBackwardData (const ConvolutionComputation &conv_comp, const CuMatrixBase< BaseFloat > &params, const CuMatrixBase< BaseFloat > &output_deriv, CuMatrixBase< BaseFloat > *input_deriv)
 This does the part of the backward derivative computation of convolution, that propagates derivatives back to the input data. More...
 
static void ConvolveBackwardParamsInternal (const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &output_deriv, BaseFloat alpha, CuMatrixBase< BaseFloat > *temp_mat, CuMatrixBase< BaseFloat > *params_deriv)
 
void ConvolveBackwardParams (const ConvolutionComputation &conv_comp, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &output_deriv, BaseFloat alpha, CuMatrixBase< BaseFloat > *params_deriv)
 This does the part of the backward derivative computation of convolution, that computes derivatives w.r.t. More...
 
void PadModelHeight (const ConvolutionModel &model, ConvolutionModel *model_padded)
 This function takes a model that might require zero padding in the height dimension and outputs a model accepting a possibly-larger input dimension which does not require zero padding. More...
 
static void ComputeTempMatrixSize (const ConvolutionComputationOptions &opts, ConvolutionComputation *computation)
 This function sets 'temp_rows' and 'temp_cols' in 'computation'. More...
 
void UnPadModelHeight (const ConvolutionComputationOptions &opts, const ConvolutionModel &model, const ConvolutionModel &model_padded, ConvolutionComputation *computation)
 This function modifies, if necessary, a computation that has been built for the model 'model_padded', so that it can work for the original model 'model'. More...
 
void PadComputationInputTime (const ConvolutionModel &model, ConvolutionComputationIo *io)
 This function extends the set of input indexes that the computation has, to account for any required zero-padding in the time dimension. More...
 
static int32 RoundDownToMultipleOf (int32 i, int32 n)
 
static void ShiftAllTimeOffsets (int32 shift, ConvolutionModel *model)
 
static int32 PrepareIoForAppending (ConvolutionComputationIo *io, ConvolutionComputationIo *io_appended)
 
void AppendInputFrames (const ConvolutionModel &model, ConvolutionComputationIo *io, ConvolutionModel *model_appended, ConvolutionComputationIo *io_appended)
 This function takes an input model and I/O specification, and it modifies both of them if necessary to ensure that the output 'io_appended' object has the same input and output time strides (i.e. More...
 
static bool TimeValueInInput (const ConvolutionComputationIo &io, int32 t)
 
void CheckModelAndIo (const ConvolutionModel &model, const ConvolutionComputationIo &io, bool allow_extra_input=false)
 Check that this model and this I/O request are compatible in terms of required context, etc, and crash if not. More...
 
void CompileConvolutionComputation (const ConvolutionModel &model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputationOptions &opts, ConvolutionComputation *computation, std::vector< Index > *input_indexes_modified, std::vector< Index > *output_indexes_modified)
 This function does the compilation for a convolution computation; it's a wrapper for the functions below, which should not have to be called by the end user. More...
 
static int32 FindGcdOfDifferences (std::vector< int32 > &vec)
 
static void RegularizeTList (std::vector< int32 > &t_values, int32 *start, int32 *step, int32 *num_values)
 
static void CreateIndexes (const std::vector< std::pair< int32, int32 > > &n_x_pairs, int32 t_start, int32 t_step, int32 num_t_values, int32 reorder_t, std::vector< Index > *indexes)
 Creates a vector of indexes with a regular structure, according to these specifications. More...
 
static void SetSomeIndexesBlank (const std::vector< Index > &ref_indexes, std::vector< Index > *indexes)
 This function modifies 'indexes' by, for any Indexes which was not present in 'ref_indexes', setting the 't' value to kNoTime. More...
 
void GetComputationIo (const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, ConvolutionComputationIo *io)
 This function takes lists of input and output indexes to a computation (e.g. More...
 
void GetIndexesForComputation (const ConvolutionComputationIo &io, const std::vector< Index > &orig_input_indexes, const std::vector< Index > &orig_output_indexes, std::vector< Index > *input_indexes, std::vector< Index > *output_indexes)
 This function computes the reordered and possibly padded indexes corresponding to the computation in 'io'. More...
 
void MakeComputation (const ConvolutionModel &model, ConvolutionComputationIo &io, const ConvolutionComputationOptions &opts, ConvolutionComputation *computation)
 

Function Documentation

void AppendInputFrames ( const ConvolutionModel &  model,
ConvolutionComputationIo *  io,
ConvolutionModel *  model_appended,
ConvolutionComputationIo *  io_appended 
)

This function takes an input model and I/O specification, and it modifies both of them if necessary to ensure that the output 'io_appended' object has the same input and output time strides (i.e.

t_stride_in == t_stride_out). This is done by appending the input frames across several time values and viewing them as single frames of larger dimension.

The reason why 'io' is non-const is that it may be necessary to pad the number of input frames to ensure that the number of input frames is divisible by a multiple of t_stride_out / t_stride_in (if we pad the input frames, we pad to the right).

The model in 'model_appended' may have larger height_in, and different values of 'offsets' and derived variables thereof, versus the model in 'model'.

This is stage 3 of compilation.

Definition at line 1203 of file convolution.cc.

References ConvolutionModel::all_time_offsets, ConvolutionModel::Check(), ConvolutionModel::ComputeDerived(), ConvolutionModel::height_in, ConvolutionModel::Offset::height_offset, ConvolutionModel::height_out, ConvolutionModel::height_subsample_out, rnnlm::i, KALDI_ASSERT, ConvolutionModel::num_filters_in, ConvolutionModel::num_filters_out, ConvolutionModel::offsets, PrepareIoForAppending(), ConvolutionModel::required_time_offsets, RoundDownToMultipleOf(), ShiftAllTimeOffsets(), ConvolutionComputationIo::start_t_in, ConvolutionComputationIo::start_t_out, ConvolutionComputationIo::t_step_in, ConvolutionComputationIo::t_step_out, and ConvolutionModel::Offset::time_offset.

Referenced by CompileConvolutionComputation().

1206  {
1207  int32 ratio = PrepareIoForAppending(io, io_appended);
1208 
1209  if (ratio == 1) {
1210  // we are not doing any appending of frames.
1211  *model_appended = model;
1212  return;
1213  }
1214 
1215  // we also need the time-step of the output (which is also now the
1216  // time-step of the appended input).
1217  // We know that the time step is not zero, because in that case we would
1218  // have ratio == 1 and would have returned above.
1219  int32 time_step_out = io_appended->t_step_out;
1220  KALDI_ASSERT(time_step_out == io_appended->t_step_in && time_step_out != 0);
1221  int32 orig_time_step_in = io->t_step_in;
1222  KALDI_ASSERT(orig_time_step_in * ratio == time_step_out);
1223 
1224  // make sure the difference between first input and output frames is what we
1225  // expect, else something could go wrong here.
1226  int32 first_time_offset = *(model.all_time_offsets.begin());
1227  KALDI_ASSERT(io->start_t_in - io->start_t_out == first_time_offset);
1228 
1229  ConvolutionModel model_temp(model);
1230  // shift so that the first time offset is zero. this makes
1231  // the model conversion easier.
1232  ShiftAllTimeOffsets(-first_time_offset, &model_temp);
1233 
1234  model_appended->num_filters_in = model.num_filters_in;
1235  model_appended->num_filters_out = model.num_filters_out;
1236  model_appended->height_in = ratio * model.height_in;
1237  model_appended->height_out = model.height_out;
1238  model_appended->height_subsample_out = model.height_subsample_out;
1239  int32 num_offsets = model_temp.offsets.size(),
1240  old_height = model.height_in;
1241  model_appended->offsets.resize(num_offsets);
1242  model_appended->all_time_offsets.clear();
1243  for (int32 i = 0; i < num_offsets; i++) {
1244  const ConvolutionModel::Offset &old_offset = model_temp.offsets[i];
1245  ConvolutionModel::Offset &new_offset = model_appended->offsets[i];
1246  // The following two lines are important!! They are the core of how
1247  // we handle subsampling in this framework.
1248  new_offset.time_offset = RoundDownToMultipleOf(old_offset.time_offset,
1249  time_step_out);
1250  KALDI_ASSERT((old_offset.time_offset - new_offset.time_offset) %
1251  orig_time_step_in == 0);
1252  int32 row_offset = (old_offset.time_offset - new_offset.time_offset) /
1253  orig_time_step_in;
1254  new_offset.height_offset = old_offset.height_offset +
1255  row_offset * old_height;
1256  model_appended->all_time_offsets.insert(new_offset.time_offset);
1257  }
1258 
1259  // Because the 'appended' model will always be used after zero-padding on the
1260  // time axis, we can just pretend that all desired time-offsets are required.
1261  // It's a kind of free error-checking.
1262  model_appended->required_time_offsets = model_appended->all_time_offsets;
1263 
1264  // Undo the time-shifting that we did before.
1265  ShiftAllTimeOffsets(first_time_offset, model_appended);
1266 
1267  model_appended->ComputeDerived();
1268  KALDI_ASSERT(model_appended->Check(false, false));
1269 }
static int32 RoundDownToMultipleOf(int32 i, int32 n)
static int32 PrepareIoForAppending(ConvolutionComputationIo *io, ConvolutionComputationIo *io_appended)
static void ShiftAllTimeOffsets(int32 shift, ConvolutionModel *model)
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
void CheckModelAndIo ( const ConvolutionModel &  model,
const ConvolutionComputationIo &  io,
bool  allow_extra_input = false 
)

Check that this model and this I/O request are compatible in terms of required context, etc, and crash if not.

if allow_extra_input == false, this will crash if the input 'io' object has time values that would never be used because they are before/after the first/last desired time values.

Definition at line 1329 of file convolution.cc.

References ConvolutionModel::all_time_offsets, KALDI_ASSERT, KALDI_ERR, rnnlm::n, ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::num_t_out, kaldi::RandInt(), ConvolutionModel::required_time_offsets, ConvolutionComputationIo::start_t_in, ConvolutionComputationIo::start_t_out, ConvolutionComputationIo::t_step_in, ConvolutionComputationIo::t_step_out, and TimeValueInInput().

Referenced by CompileConvolutionComputation().

1331  {
1332  KALDI_ASSERT(io.num_t_in > 0 && io.num_t_out > 0 &&
1333  !model.required_time_offsets.empty() &&
1334  !model.all_time_offsets.empty());
1335  if (!allow_extra_input) {
1336  KALDI_ASSERT(io.start_t_in >= io.start_t_out +
1337  *model.all_time_offsets.begin());
1338  int32 last_t_in = io.start_t_in + io.t_step_in * (io.num_t_in - 1),
1339  last_t_out = io.start_t_out + io.t_step_out * (io.num_t_out - 1);
1340  KALDI_ASSERT(last_t_in <= last_t_out +
1341  *model.all_time_offsets.rbegin());
1342  }
1343 
1344  std::set<int32> input_times_to_check;
1345  for (int32 n = 0; n < std::min(5, io.num_t_out); n++) {
1346  int32 t_out = io.start_t_out +
1347  RandInt(0, io.num_t_out - 1) * io.t_step_out;
1348  for (std::set<int32>::const_iterator iter =
1349  model.required_time_offsets.begin();
1350  iter != model.required_time_offsets.end();
1351  ++iter) {
1352  int32 offset = *iter;
1353  input_times_to_check.insert(t_out + offset);
1354  }
1355  }
1356  for (std::set<int32>::const_iterator iter = input_times_to_check.begin();
1357  iter != input_times_to_check.end(); ++iter) {
1358  int32 t = *iter;
1359  if (!TimeValueInInput(io, t)) {
1360  KALDI_ERR << "Error checking model and IO: time " << t
1361  << " is required but not in the input.";
1362  }
1363  }
1364 }
struct rnnlm::@11::@12 n
#define KALDI_ERR
Definition: kaldi-error.h:127
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static bool TimeValueInInput(const ConvolutionComputationIo &io, int32 t)
int32 RandInt(int32 min_val, int32 max_val, struct RandomState *state)
Definition: kaldi-math.cc:94
void CompileConvolutionComputation ( const ConvolutionModel &  model,
const std::vector< Index > &  input_indexes,
const std::vector< Index > &  output_indexes,
const ConvolutionComputationOptions &  opts,
ConvolutionComputation *  computation,
std::vector< Index > *  input_indexes_modified,
std::vector< Index > *  output_indexes_modified 
)

This function does the compilation for a convolution computation; it's a wrapper for the functions below, which should not have to be called by the end user.

Parameters
[in]modelThe convolution model that this computation is for.
[in]input_indexesThe list of Indexes available at the input of the computation.
[in]output_indexesThe list of Indexes requested to be computed at the output of the computation. It is an error if all dependencies are not satisfied (specifically: for each Index (n,t,x) in 'output_indexes', the Index (n,t+time_offset,x) must be present in 'input_indexes' for each time_offset in model.required_time_offsets.
[out]computationIf non-NULL, the compiled computation will be written to this location.

Definition at line 1367 of file convolution.cc.

References AppendInputFrames(), CheckModelAndIo(), GetComputationIo(), GetIndexesForComputation(), MakeComputation(), PadComputationInputTime(), PadModelHeight(), and UnPadModelHeight().

Referenced by TimeHeightConvolutionComponent::PrecomputeIndexes(), TimeHeightConvolutionComponent::ReorderIndexes(), and UnitTestTimeHeightConvolutionCompile().

1374  {
1375 
1376  // stage zero [preparing the input and output in a regular grid.]
1377  ConvolutionComputationIo io;
1378  GetComputationIo(input_indexes, output_indexes, &io);
1379 
1380  CheckModelAndIo(model, io, false);
1381 
1382  // stage 1.
1383  PadComputationInputTime(model, &io);
1384 
1385  CheckModelAndIo(model, io, false);
1386 
1387  // stage 2.
1388  ConvolutionModel model_padded;
1389  PadModelHeight(model, &model_padded);
1390 
1391  CheckModelAndIo(model_padded, io, false);
1392 
1393  // stage 3.
1394  ConvolutionModel model_appended;
1395  ConvolutionComputationIo io_appended;
1396  // make a 'fake' model and io for possibly-appended input frames. 'io' is
1397  // non-const because we may need to pad with a few extra frames.
1398  AppendInputFrames(model_padded, &io,
1399  &model_appended, &io_appended);
1400 
1401  CheckModelAndIo(model_appended, io_appended, true);
1402 
1403  // stage 4.
1404  MakeComputation(model_appended, io_appended, opts, computation);
1405 
1406  // 'reverse' of stage 2. [stage 3 kind of does its own
1407  // 'reverse' by modifying its input IO object.]
1408  // The computation is still specified for the appended input,
1409  // but the execution code can figure that out itself.
1410  UnPadModelHeight(opts, model, model_padded, computation);
1411 
1412  GetIndexesForComputation(io, input_indexes, output_indexes,
1413  input_indexes_modified, output_indexes_modified);
1414 }
void GetIndexesForComputation(const ConvolutionComputationIo &io, const std::vector< Index > &orig_input_indexes, const std::vector< Index > &orig_output_indexes, std::vector< Index > *input_indexes, std::vector< Index > *output_indexes)
This function computes the reordered and possibly padded indexes corresponding to the computation in ...
void UnPadModelHeight(const ConvolutionComputationOptions &opts, const ConvolutionModel &model, const ConvolutionModel &model_padded, ConvolutionComputation *computation)
This function modifies, if necessary, a computation that has been built for the model 'model_padded'...
void MakeComputation(const ConvolutionModel &model, ConvolutionComputationIo &io, const ConvolutionComputationOptions &opts, ConvolutionComputation *computation)
void AppendInputFrames(const ConvolutionModel &model, ConvolutionComputationIo *io, ConvolutionModel *model_appended, ConvolutionComputationIo *io_appended)
This function takes an input model and I/O specification, and it modifies both of them if necessary t...
void PadModelHeight(const ConvolutionModel &model, ConvolutionModel *model_padded)
This function takes a model that might require zero padding in the height dimension and outputs a mod...
Definition: convolution.cc:918
void GetComputationIo(const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, ConvolutionComputationIo *io)
This function takes lists of input and output indexes to a computation (e.g.
void CheckModelAndIo(const ConvolutionModel &model, const ConvolutionComputationIo &io, bool allow_extra_input)
Check that this model and this I/O request are compatible in terms of required context, etc, and crash if not.
void PadComputationInputTime(const ConvolutionModel &model, ConvolutionComputationIo *io)
This function extends the set of input indexes that the computation has, to account for any required ...
static void kaldi::nnet3::time_height_convolution::ComputeTempMatrixSize ( const ConvolutionComputationOptions &  opts,
ConvolutionComputation *  computation 
)
static

This function sets 'temp_rows' and 'temp_cols' in 'computation'.

Definition at line 956 of file convolution.cc.

References ConvolutionComputation::height_in, ConvolutionComputation::ConvolutionStep::height_map, rnnlm::i, KALDI_WARN, ConvolutionComputationOptions::max_memory_mb, ConvolutionComputation::num_filters_in, ConvolutionComputation::num_images, ConvolutionComputation::num_t_out, ConvolutionComputation::steps, ConvolutionComputation::temp_cols, ConvolutionComputation::temp_rows, and VectorIsContiguous().

Referenced by MakeComputation(), and UnPadModelHeight().

957  {
958  int32 temp_rows = 0, temp_cols = 0;
959  for (size_t i = 0; i < computation->steps.size(); i++) {
960  const ConvolutionComputation::ConvolutionStep &step = computation->steps[i];
961  int32 height_map_size = step.height_map.size(),
962  this_num_cols = height_map_size * computation->num_filters_in;
963  bool columns_are_contiguous =
964  (step.height_map[0] != -1 && VectorIsContiguous(step.height_map));
965  bool need_temp_matrix = true;
966  if (columns_are_contiguous && step.height_map[0] == 0 &&
967  this_num_cols == computation->num_filters_in * computation->height_in) {
968  // the only situation in which we wouldn't need the temporary matrix
969  // for this step, is where the columns are all of the input matrix.
970  need_temp_matrix = false;
971  }
972  if (need_temp_matrix && this_num_cols > temp_cols)
973  temp_cols = this_num_cols;
974  }
975  if (temp_cols > 0) {
976  // work out how many rows the temporary matrix should have, taking
977  // into account the specified memory limit.
978  temp_rows = computation->num_t_out * computation->num_images;
979  BaseFloat num_megabytes = (4 * temp_rows * temp_cols) / 1000000.0,
980  megabyte_limit = opts.max_memory_mb;
981  // C++ rounds down; here, we want to round up so we add one.
982  int32 ratio = 1.0 + num_megabytes / megabyte_limit;
983 
984  // divide the number of time steps into 'ratio' pieces that are as equal as
985  // possible; round up when dividing, to make sure that new_temp_rows * ratio
986  // >= temp_rows so that we don't have a small leftover piece.
987  int32 new_num_t_out = (computation->num_t_out + ratio - 1) / ratio;
988  temp_rows = new_num_t_out * computation->num_images;
989  BaseFloat new_num_megabytes = (4 * temp_rows * temp_cols) / 1000000.0;
990  // make sure we're within the memory limit.
991  if (new_num_megabytes > megabyte_limit) {
992  KALDI_WARN << "Memory consumed in convolution is more than requested "
993  << "(maybe very long time sequence?)";
994  }
995  }
996  computation->temp_rows = temp_rows;
997  computation->temp_cols = temp_cols;
998 
999 }
static bool VectorIsContiguous(const std::vector< int32 > &vec)
Definition: convolution.cc:77
float BaseFloat
Definition: kaldi-types.h:29
#define KALDI_WARN
Definition: kaldi-error.h:130
void ConvolveBackwardData ( const ConvolutionComputation &  conv_comp,
const CuMatrixBase< BaseFloat > &  params,
const CuMatrixBase< BaseFloat > &  output_deriv,
CuMatrixBase< BaseFloat > *  input_deriv 
)

This does the part of the backward derivative computation of convolution, that propagates derivatives back to the input data.

See also ConvolveBackwardParams(), which is for the parameter derivative.

Parameters
[in]conv_compA struct that describes the convolution computation (should be the same as in the corresponding forward pass).
[in]paramsThe parameters used in the forward convolution. This should be of dimension num_filters_out by (X * num_filters_in), where X is the total number of pixels in the patches, which equals model.offsets.size() in the model for which the computation was compiled. E.g. for a regular 3x3 kernel, X would be 9.
[in]output_derivThe derivative of the objective function w.r.t. the output of the convolution. Should be of dimension conv_comp.num_t_out * conv_comp.num_images by conv_comp.height_out num_filters_out. It must satisfy output_deriv.NumCols() == output_deriv.Stride().
[out]input_derivIf non-NULL, the backpropagated derivative of the objective function w.r.t. the input will be *added to* this matrix. Should be the same dimension as the input to the original ConvolveForward() call.

Definition at line 682 of file convolution.cc.

References ConvolveBackwardDataInternal(), CuMatrixBase< Real >::Data(), ConvolutionComputation::height_in, ConvolutionComputation::height_out, KALDI_ASSERT, KALDI_ERR, kaldi::kSetZero, kaldi::kStrideEqualNumCols, ConvolutionComputation::num_filters_in, ConvolutionComputation::num_filters_out, ConvolutionComputation::num_images, ConvolutionComputation::num_t_in, ConvolutionComputation::num_t_out, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), CuMatrixBase< Real >::Stride(), ConvolutionComputation::temp_cols, and ConvolutionComputation::temp_rows.

Referenced by TimeHeightConvolutionComponent::Backprop(), and TestDataBackprop().

686  {
687  KALDI_ASSERT(input_deriv->NumCols() == input_deriv->Stride() &&
688  output_deriv.NumCols() == output_deriv.Stride());
689  KALDI_ASSERT(params.NumRows() == cc.num_filters_out);
690  KALDI_ASSERT(output_deriv.NumRows() == cc.num_t_out * cc.num_images &&
691  output_deriv.NumCols() == cc.height_out * cc.num_filters_out);
692  // the input might need to be reshaped but we can check its total size.
693  KALDI_ASSERT(input_deriv->NumRows() * input_deriv->NumCols() ==
694  cc.num_images * cc.num_t_in * cc.height_in * cc.num_filters_in);
695 
696  int32 input_rows = input_deriv->NumRows(),
697  required_input_rows = cc.num_images * cc.num_t_in;
698 
699  // this if-statement handles reshaping the input and recursing if there
700  // is subsampling.
701  if (input_rows != required_input_rows) {
702  if (input_rows % required_input_rows != 0)
703  KALDI_ERR << "Input matrix has wrong size."; // error in calling code.
704  // nr is a multiple of required_nr. Reshape the matrix.
705  // we already checked that its Stride() == NumCols();
706  int32 num_cols = input_deriv->NumCols(),
707  multiple = input_rows / required_input_rows,
708  new_num_cols = num_cols * multiple,
709  new_stride = new_num_cols;
710  CuSubMatrix<BaseFloat> input_deriv_reshaped(
711  input_deriv->Data(), required_input_rows,
712  new_num_cols, new_stride);
713  ConvolveBackwardData(cc, params, output_deriv, &input_deriv_reshaped);
714  return;
715  }
716 
717  CuMatrix<BaseFloat> temp_mat(cc.temp_rows, cc.temp_cols,
719 
720  // this if-statement handles breaking up the arguments
721  // and the computation into row-ranges if the temporary
722  // matrix would have been excessively large, and we've decided
723  // to give it fewer rows than the output (this saves
724  // memory). normally we won't take this if-statement
725  // so ignore it if you're trying to understand the framework.
726  if (cc.temp_rows != 0 && cc.temp_rows != input_rows) {
727  KALDI_ASSERT(cc.temp_rows % cc.num_images == 0);
728  int32 num_time_steps_per_chunk = cc.temp_rows / cc.num_images;
729  int32 num_extra_in = cc.num_t_in - cc.num_t_out;
730 
731  for (int32 t_start = 0; t_start < cc.num_t_out;
732  t_start += num_time_steps_per_chunk) {
733  int32 num_t_left = cc.num_t_out - t_start,
734  this_num_t_out = std::min<int32>(num_t_left,
735  num_time_steps_per_chunk),
736  this_num_t_in = this_num_t_out + num_extra_in;
737  CuSubMatrix<BaseFloat> input_deriv_part(
738  *input_deriv, t_start * cc.num_images,
739  this_num_t_in * cc.num_images,
740  0, input_deriv->NumCols());
741  CuSubMatrix<BaseFloat> output_deriv_part(
742  output_deriv, t_start * cc.num_images,
743  this_num_t_out * cc.num_images,
744  0, output_deriv.NumCols());
745  CuSubMatrix<BaseFloat> temp_part(
746  temp_mat, 0, this_num_t_out * cc.num_images,
747  0, temp_mat.NumCols());
748  ConvolveBackwardDataInternal(cc, params, output_deriv_part,
749  &temp_part, &input_deriv_part);
750  }
751  return;
752  }
753  ConvolveBackwardDataInternal(cc, params, output_deriv,
754  &temp_mat, input_deriv);
755 }
static void ConvolveBackwardDataInternal(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &params, const CuMatrixBase< BaseFloat > &output_deriv, CuMatrixBase< BaseFloat > *temp_mat, CuMatrixBase< BaseFloat > *input_deriv)
Definition: convolution.cc:603
MatrixIndexT NumCols() const
Definition: cu-matrix.h:196
void ConvolveBackwardData(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &params, const CuMatrixBase< BaseFloat > &output_deriv, CuMatrixBase< BaseFloat > *input_deriv)
This does the part of the backward derivative computation of convolution, that propagates derivatives...
Definition: convolution.cc:682
MatrixIndexT NumRows() const
Dimensions.
Definition: cu-matrix.h:195
#define KALDI_ERR
Definition: kaldi-error.h:127
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
MatrixIndexT Stride() const
Definition: cu-matrix.h:197
const Real * Data() const
Return data pointer (const).
Definition: cu-matrix.h:625
static void kaldi::nnet3::time_height_convolution::ConvolveBackwardDataInternal ( const ConvolutionComputation &  cc,
const CuMatrixBase< BaseFloat > &  params,
const CuMatrixBase< BaseFloat > &  output_deriv,
CuMatrixBase< BaseFloat > *  temp_mat,
CuMatrixBase< BaseFloat > *  input_deriv 
)
static

Definition at line 603 of file convolution.cc.

References CuMatrixBase< Real >::AddMatMat(), ConvolutionComputation::ConvolutionStep::backward_columns, ConvolutionComputation::ConvolutionStep::columns, ConvolutionComputation::ConvolutionStep::columns_are_contiguous, CuMatrixBase< Real >::Data(), CuArray< T >::Dim(), ConvolutionComputation::ConvolutionStep::first_column, ConvolutionComputation::height_out, rnnlm::i, ConvolutionComputation::ConvolutionStep::input_time_shift, KALDI_ASSERT, kaldi::kNoTrans, ConvolutionComputation::num_filters_out, ConvolutionComputation::num_images, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), ConvolutionComputation::ConvolutionStep::params_start_col, ConvolutionComputation::steps, and CuMatrixBase< Real >::Stride().

Referenced by ConvolveBackwardData().

608  {
609  KALDI_ASSERT(temp_mat->Stride() == temp_mat->NumCols());
610 
611  // num_t_out supersedes cc.num_t_out (they'll only be different in
612  // cases where we are doing the computation in pieces to save memory).
613  int32 input_rows = input_deriv->NumRows(),
614  output_rows = output_deriv.NumRows();
615 
616  KALDI_ASSERT(output_rows <= input_rows &&
617  input_rows % cc.num_images == 0 &&
618  output_rows % cc.num_images == 0);
619 
620  int32 num_steps = cc.steps.size();
621  for (int32 s = 0; s < num_steps; s++) {
622  const ConvolutionComputation::ConvolutionStep &step = cc.steps[s];
623  int32 input_row_start = step.input_time_shift * cc.num_images;
624  CuSubMatrix<BaseFloat> input_deriv_part(*input_deriv,
625  input_row_start, output_rows,
626  0, input_deriv->NumCols());
627  int32 temp_num_cols = step.columns.Dim(),
628  param_cols = temp_num_cols / cc.height_out;
629  CuSubMatrix<BaseFloat> params_part(params,
630  0, params.NumRows(),
631  step.params_start_col,
632  param_cols);
633  CuSubMatrix<BaseFloat> output_deriv_reshaped(
634  output_deriv.Data(), output_rows * cc.height_out,
635  cc.num_filters_out, cc.num_filters_out);
636 
637  if (!step.columns_are_contiguous ||
638  temp_num_cols != input_deriv->NumCols()) {
639  // In most cases we will take this branch, where we have to propagate the
640  // input-derivative via a temporary matrix. (however, different steps may
641  // require different num-cols of the temporary matrix, so we create
642  // sub-parts of 'temp_mat'.
643 
644  // We create the sub-matrix 'temp_mat_part' in a lower-level way, using
645  // pointers, because we need to ensure that its num-cols and the stride
646  // are the same (this is necessary so that we can do reshaping in
647  // ConvolutionReshapedMultiply()).
648  CuSubMatrix<BaseFloat> temp_mat_part(temp_mat->Data(),
649  temp_mat->NumRows(),
650  temp_num_cols, temp_num_cols),
651  temp_mat_part_reshaped(
652  temp_mat_part.Data(), temp_mat_part.NumRows() * cc.height_out,
653  temp_num_cols / cc.height_out, temp_num_cols / cc.height_out);
654 
655  temp_mat_part_reshaped.AddMatMat(1.0, output_deriv_reshaped, kNoTrans,
656  params_part, kNoTrans, 0.0);
657 
658  if (!step.columns_are_contiguous) {
659  for (size_t i = 0; i < step.backward_columns.size(); i++) {
660  input_deriv_part.AddCols(temp_mat_part, step.backward_columns[i]);
661  }
662  } else {
663  // we're just taking a sub-matrix of the input matrix, but we still need
664  // to make a copy because we need the stride == num-cols (so that the
665  // reshaping will work).
666  int32 num_cols = step.columns.Dim();
667  input_deriv_part.ColRange(step.first_column,
668  num_cols).AddMat(1.0, temp_mat_part);
669  }
670  } else {
671  CuSubMatrix<BaseFloat> input_deriv_reshaped(
672  input_deriv_part.Data(), input_deriv_part.NumRows() * cc.height_out,
673  input_deriv_part.NumCols() / cc.height_out,
674  input_deriv_part.NumCols() / cc.height_out);
675  input_deriv_reshaped.AddMatMat(1.0, output_deriv_reshaped, kNoTrans,
676  params_part, kNoTrans, 1.0);
677  }
678  }
679 }
MatrixIndexT NumCols() const
Definition: cu-matrix.h:196
MatrixIndexT NumRows() const
Dimensions.
Definition: cu-matrix.h:195
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
MatrixIndexT Stride() const
Definition: cu-matrix.h:197
const Real * Data() const
Return data pointer (const).
Definition: cu-matrix.h:625
void ConvolveBackwardParams ( const ConvolutionComputation &  conv_comp,
const CuMatrixBase< BaseFloat > &  input,
const CuMatrixBase< BaseFloat > &  output_deriv,
BaseFloat  alpha,
CuMatrixBase< BaseFloat > *  params_deriv 
)

This does the part of the backward derivative computation of convolution, that computes derivatives w.r.t.

the parameters. See also ConvolveBackwardData(), which computes derivatives w.r.t. the input data.

Parameters
[in]conv_compA struct that describes the computation that was performed in the forward pass.
[in]inputThe input to the original forward convolution. This should be of dimension (or should be reshapable to the dimension) conv_comp.num_t_in * conv_comp.num_images by conv_comp.height_in * num_filters_in. [highest-stride indexes come first in these multiplications]. It must satisfy input.NumCols() == input.Stride().
[in]output_derivThe derivative of the objective function w.r.t. the output of the convolution. Should be of dimension conv_comp.num_t_out * conv_comp.num_images by conv_comp.height_out num_filters_out. It must satisfy output_deriv.NumCols() == output_deriv.Stride().
[in]alphaThis scalar is multiplied into the derivative when we add to params_deriv, i.e. *params_deriv += alpha * derivative.
[out]params_derivThe derivative of the objective function w.r.t the parameters (the 'params' given to the ConvolveForward function) is *added* to this location. This matrix should be of dimension conv_comp.NumRows() by conv_comp.NumCols().

Definition at line 840 of file convolution.cc.

References ConvolveBackwardParamsInternal(), CuMatrixBase< Real >::Data(), ConvolutionComputation::height_in, ConvolutionComputation::height_out, KALDI_ASSERT, KALDI_ERR, kaldi::kStrideEqualNumCols, kaldi::kUndefined, ConvolutionComputation::num_filters_in, ConvolutionComputation::num_filters_out, ConvolutionComputation::num_images, ConvolutionComputation::num_t_in, ConvolutionComputation::num_t_out, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), CuMatrixBase< Real >::Stride(), ConvolutionComputation::temp_cols, and ConvolutionComputation::temp_rows.

Referenced by TestParamsBackprop(), TimeHeightConvolutionComponent::UpdateNaturalGradient(), and TimeHeightConvolutionComponent::UpdateSimple().

845  {
846  KALDI_ASSERT(input.NumCols() == input.Stride() &&
847  output_deriv.NumCols() == output_deriv.Stride());
848  KALDI_ASSERT(params_deriv->NumRows() == cc.num_filters_out);
849  KALDI_ASSERT(output_deriv.NumRows() == cc.num_t_out * cc.num_images &&
850  output_deriv.NumCols() == cc.height_out * cc.num_filters_out);
851  // the input might need to be reshaped but we can check its total size.
852  KALDI_ASSERT(input.NumRows() * input.NumCols() == cc.num_images *
853  cc.num_t_in * cc.height_in * cc.num_filters_in);
854 
855  int32 input_rows = input.NumRows(),
856  required_input_rows = cc.num_images * cc.num_t_in;
857 
858  // this if-statement handles reshaping the input and recursing if there
859  // is subsampling.
860  if (input_rows != required_input_rows) {
861  if (input_rows % required_input_rows != 0)
862  KALDI_ERR << "Input matrix has wrong size."; // error in calling code.
863  // nr is a multiple of required_nr. Reshape the matrix.
864  // we already checked that its Stride() == NumCols();
865  int32 num_cols = input.NumCols(),
866  multiple = input_rows / required_input_rows,
867  new_num_cols = num_cols * multiple,
868  new_stride = new_num_cols;
869  CuSubMatrix<BaseFloat> input_reshaped(
870  input.Data(), required_input_rows, new_num_cols, new_stride);
871  ConvolveBackwardParams(cc, input_reshaped, output_deriv, alpha,
872  params_deriv);
873  return;
874  }
875 
876  CuMatrix<BaseFloat> temp_mat(cc.temp_rows, cc.temp_cols,
878 
879  // this if-statement handles breaking up the arguments
880  // and the computation into row-ranges if the temporary
881  // matrix would have been excessively large, and we've decided
882  // to give it fewer rows than the output (this saves
883  // memory). normally we won't take this if-statement
884  // so ignore it if you're trying to understand the framework.
885  if (cc.temp_rows != 0 && cc.temp_rows != input_rows) {
886  KALDI_ASSERT(cc.temp_rows % cc.num_images == 0);
887  int32 num_time_steps_per_chunk = cc.temp_rows / cc.num_images;
888  int32 num_extra_in = cc.num_t_in - cc.num_t_out;
889 
890  for (int32 t_start = 0; t_start < cc.num_t_out;
891  t_start += num_time_steps_per_chunk) {
892  int32 num_t_left = cc.num_t_out - t_start,
893  this_num_t_out = std::min<int32>(num_t_left,
894  num_time_steps_per_chunk),
895  this_num_t_in = this_num_t_out + num_extra_in;
896  CuSubMatrix<BaseFloat> input_part(
897  input, t_start * cc.num_images,
898  this_num_t_in * cc.num_images,
899  0, input.NumCols());
900  CuSubMatrix<BaseFloat> output_deriv_part(
901  output_deriv, t_start * cc.num_images,
902  this_num_t_out * cc.num_images,
903  0, output_deriv.NumCols());
904  CuSubMatrix<BaseFloat> temp_part(temp_mat,
905  0, this_num_t_out * cc.num_images,
906  0, temp_mat.NumCols());
907  ConvolveBackwardParamsInternal(cc, input_part, output_deriv_part,
908  alpha, &temp_part, params_deriv);
909  }
910  return;
911  }
912  ConvolveBackwardParamsInternal(cc, input, output_deriv,
913  alpha, &temp_mat, params_deriv);
914 }
void ConvolveBackwardParams(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &output_deriv, BaseFloat alpha, CuMatrixBase< BaseFloat > *params_deriv)
This does the part of the backward derivative computation of convolution, that computes derivatives w...
Definition: convolution.cc:840
static void ConvolveBackwardParamsInternal(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &output_deriv, BaseFloat alpha, CuMatrixBase< BaseFloat > *temp_mat, CuMatrixBase< BaseFloat > *params_deriv)
Definition: convolution.cc:763
MatrixIndexT NumCols() const
Definition: cu-matrix.h:196
MatrixIndexT NumRows() const
Dimensions.
Definition: cu-matrix.h:195
#define KALDI_ERR
Definition: kaldi-error.h:127
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
MatrixIndexT Stride() const
Definition: cu-matrix.h:197
const Real * Data() const
Return data pointer (const).
Definition: cu-matrix.h:625
static void kaldi::nnet3::time_height_convolution::ConvolveBackwardParamsInternal ( const ConvolutionComputation &  cc,
const CuMatrixBase< BaseFloat > &  input,
const CuMatrixBase< BaseFloat > &  output_deriv,
BaseFloat  alpha,
CuMatrixBase< BaseFloat > *  temp_mat,
CuMatrixBase< BaseFloat > *  params_deriv 
)
static

Definition at line 763 of file convolution.cc.

References CuMatrixBase< Real >::AddMatMat(), ConvolutionComputation::ConvolutionStep::columns, ConvolutionComputation::ConvolutionStep::columns_are_contiguous, CuMatrixBase< Real >::CopyCols(), CuMatrixBase< Real >::Data(), CuArray< T >::Dim(), ConvolutionComputation::ConvolutionStep::first_column, ConvolutionComputation::height_out, ConvolutionComputation::ConvolutionStep::input_time_shift, KALDI_ASSERT, kaldi::kNoTrans, kaldi::kTrans, ConvolutionComputation::num_filters_out, ConvolutionComputation::num_images, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), ConvolutionComputation::ConvolutionStep::params_start_col, ConvolutionComputation::steps, and CuMatrixBase< Real >::Stride().

Referenced by ConvolveBackwardParams().

769  {
770  KALDI_ASSERT(temp_mat->Stride() == temp_mat->NumCols());
771 
772  // num_t_out supersedes cc.num_t_out (they'll only be different in
773  // cases where we are doing the computation in pieces to save memory).
774  int32 input_rows = input.NumRows(),
775  output_rows = output_deriv.NumRows();
776 
777  KALDI_ASSERT(output_rows <= input_rows &&
778  input_rows % cc.num_images == 0 &&
779  output_rows % cc.num_images == 0);
780 
781  int32 num_steps = cc.steps.size();
782  for (int32 s = 0; s < num_steps; s++) {
783  const ConvolutionComputation::ConvolutionStep &step = cc.steps[s];
784  int32 input_row_start = step.input_time_shift * cc.num_images;
785  // note: 'input_part' will normally be almost all of 'input', perhaps
786  // minus one or two time steps at the start or end.
787  CuSubMatrix<BaseFloat> input_part(input,
788  input_row_start, output_rows,
789  0, input.NumCols());
790  int32 temp_num_cols = step.columns.Dim(),
791  param_cols = temp_num_cols / cc.height_out;
792  CuSubMatrix<BaseFloat> params_deriv_part(*params_deriv,
793  0, params_deriv->NumRows(),
794  step.params_start_col,
795  param_cols);
796  CuSubMatrix<BaseFloat> output_deriv_reshaped(
797  output_deriv.Data(), output_rows * cc.height_out,
798  cc.num_filters_out, cc.num_filters_out);
799  if (!step.columns_are_contiguous ||
800  temp_num_cols != input.NumCols()) {
801  // In most cases we will take this branch, where we have to copy the input
802  // to a temporary matrix. (however, different steps may require different
803  // num-cols of the temporary matrix, so we create sub-parts of 'temp_mat'.
804 
805  // We create the sub-matrix 'temp_mat_part' in a lower-level way, using
806  // pointers, because we need to ensure that its num-cols and the stride
807  // are the same (this is necessary so that we can do reshaping in
808  // ConvolutionReshapedMultiply()).
809  CuSubMatrix<BaseFloat> temp_mat_part(temp_mat->Data(),
810  temp_mat->NumRows(),
811  temp_num_cols, temp_num_cols);
812  if (!step.columns_are_contiguous) {
813  // we're doing a column mapping.
814  temp_mat_part.CopyCols(input_part, step.columns);
815  } else {
816  // we're just taking a sub-matrix of the input matrix, but we still need
817  // to make a copy because we need the stride == num-cols (so that the
818  // reshaping will work).
819  temp_mat_part.CopyFromMat(input_part.ColRange(step.first_column,
820  step.columns.Dim()));
821  }
822  CuSubMatrix<BaseFloat> temp_mat_part_reshaped(
823  temp_mat_part.Data(), temp_mat_part.NumRows() * cc.height_out,
824  temp_num_cols / cc.height_out, temp_num_cols / cc.height_out);
825 
826  params_deriv_part.AddMatMat(alpha, output_deriv_reshaped, kTrans,
827  temp_mat_part_reshaped, kNoTrans, 1.0);
828  } else {
829  CuSubMatrix<BaseFloat> input_reshaped(
830  input_part.Data(), input_part.NumRows() * cc.height_out,
831  input_part.NumCols() / cc.height_out,
832  input_part.NumCols() / cc.height_out);
833 
834  params_deriv_part.AddMatMat(alpha, output_deriv_reshaped, kTrans,
835  input_reshaped, kNoTrans, 1.0);
836  }
837  }
838 }
MatrixIndexT NumCols() const
Definition: cu-matrix.h:196
MatrixIndexT NumRows() const
Dimensions.
Definition: cu-matrix.h:195
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
MatrixIndexT Stride() const
Definition: cu-matrix.h:197
const Real * Data() const
Return data pointer (const).
Definition: cu-matrix.h:625
void ConvolveForward ( const ConvolutionComputation &  conv_comp,
const CuMatrixBase< BaseFloat > &  input,
const CuMatrixBase< BaseFloat > &  params,
CuMatrixBase< BaseFloat > *  output 
)

This does the forward computation of convolution.

(note: this is convolution without a bias term; you have to handle that separately).

Parameters
[in]conv_compA struct that describes the computation to be performed.
[in]inputThe input to the convolution. This should be of dimension (or should be reshapable to the dimension) conv_comp.num_t_in * conv_comp.num_images by conv_comp.height_in * num_filters_in. [highest-stride indexes come first in these multiplications]. It must satisfy input.NumCols() == input.Stride().
[in]paramsThe parameters of the convolution. This should be of dimension conv_comp.ParamRows() by conv_comp.ParamCols().
[out]outputThe output of the convolution (this function adds to* the output). Should be of dimension conv_comp.num_t_out * conv_comp.num_images by conv_comp.height_out * num_filters_out. It must satisfy output.NumCols() == output.Stride().

Definition at line 524 of file convolution.cc.

References ConvolveForwardInternal(), CuMatrixBase< Real >::Data(), ConvolutionComputation::height_in, ConvolutionComputation::height_out, KALDI_ASSERT, KALDI_ERR, kaldi::kStrideEqualNumCols, kaldi::kUndefined, ConvolutionComputation::num_filters_in, ConvolutionComputation::num_filters_out, ConvolutionComputation::num_images, ConvolutionComputation::num_t_in, ConvolutionComputation::num_t_out, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), CuMatrixBase< Real >::Stride(), ConvolutionComputation::temp_cols, and ConvolutionComputation::temp_rows.

Referenced by TimeHeightConvolutionComponent::Propagate(), TestDataBackprop(), TestParamsBackprop(), and TestRunningComputation().

528  {
529  KALDI_ASSERT(input.NumCols() == input.Stride() &&
530  output->NumCols() == output->Stride());
531  KALDI_ASSERT(params.NumRows() == cc.num_filters_out);
532  KALDI_ASSERT(output->NumRows() == cc.num_t_out * cc.num_images &&
533  output->NumCols() == cc.height_out * cc.num_filters_out);
534  // the input might need to be reshaped but we can check its total size.
535  KALDI_ASSERT(input.NumRows() * input.NumCols() == cc.num_images *
536  cc.num_t_in * cc.height_in * cc.num_filters_in);
537 
538  int32 input_rows = input.NumRows(),
539  required_input_rows = cc.num_images * cc.num_t_in;
540 
541  // this if-statement handles reshaping the input and recursing if there
542  // is subsampling.
543  if (input_rows != required_input_rows) {
544  if (input_rows % required_input_rows != 0)
545  KALDI_ERR << "Input matrix has wrong size."; // error in calling code.
546  // nr is a multiple of required_nr. Reshape the matrix.
547  // we already checked that its Stride() == NumCols();
548  int32 num_cols = input.NumCols(),
549  multiple = input_rows / required_input_rows,
550  new_num_cols = num_cols * multiple,
551  new_stride = new_num_cols;
552  CuSubMatrix<BaseFloat> input_reshaped(
553  input.Data(), required_input_rows, new_num_cols, new_stride);
554  ConvolveForward(cc, input_reshaped, params, output);
555  return;
556  }
557 
558  CuMatrix<BaseFloat> temp_mat(cc.temp_rows, cc.temp_cols,
560 
561  // this if-statement handles breaking up the arguments
562  // and the computation into row-ranges if the temporary
563  // matrix would have been excessively large, and we've decided
564  // to give it fewer rows than the output (this saves
565  // memory). normally we won't take this if-statement
566  // so ignore it if you're trying to understand the framework.
567  if (cc.temp_rows != 0 && cc.temp_rows != input_rows) {
568  KALDI_ASSERT(cc.temp_rows % cc.num_images == 0);
569  int32 num_time_steps_per_chunk = cc.temp_rows / cc.num_images;
570  int32 num_extra_in = cc.num_t_in - cc.num_t_out;
571 
572  for (int32 t_start = 0; t_start < cc.num_t_out;
573  t_start += num_time_steps_per_chunk) {
574  int32 num_t_left = cc.num_t_out - t_start,
575  this_num_t_out = std::min<int32>(num_t_left,
576  num_time_steps_per_chunk),
577  this_num_t_in = this_num_t_out + num_extra_in;
578  CuSubMatrix<BaseFloat> input_part(input, t_start * cc.num_images,
579  this_num_t_in * cc.num_images,
580  0, input.NumCols());
581  CuSubMatrix<BaseFloat> output_part(*output, t_start * cc.num_images,
582  this_num_t_out * cc.num_images,
583  0, output->NumCols());
584  CuSubMatrix<BaseFloat> temp_part(temp_mat, 0,
585  this_num_t_out * cc.num_images,
586  0, temp_mat.NumCols());
587  ConvolveForwardInternal(cc, input_part, params,
588  &temp_part, &output_part);
589  }
590  return;
591  }
592  ConvolveForwardInternal(cc, input, params, &temp_mat, output);
593 }
MatrixIndexT NumCols() const
Definition: cu-matrix.h:196
static void ConvolveForwardInternal(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &params, CuMatrixBase< BaseFloat > *temp_mat, CuMatrixBase< BaseFloat > *output)
Definition: convolution.cc:448
MatrixIndexT NumRows() const
Dimensions.
Definition: cu-matrix.h:195
#define KALDI_ERR
Definition: kaldi-error.h:127
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
void ConvolveForward(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &params, CuMatrixBase< BaseFloat > *output)
This does the forward computation of convolution.
Definition: convolution.cc:524
MatrixIndexT Stride() const
Definition: cu-matrix.h:197
const Real * Data() const
Return data pointer (const).
Definition: cu-matrix.h:625
static void kaldi::nnet3::time_height_convolution::ConvolveForwardInternal ( const ConvolutionComputation &  cc,
const CuMatrixBase< BaseFloat > &  input,
const CuMatrixBase< BaseFloat > &  params,
CuMatrixBase< BaseFloat > *  temp_mat,
CuMatrixBase< BaseFloat > *  output 
)
static

Definition at line 448 of file convolution.cc.

References CuMatrixBase< Real >::AddMatMat(), ConvolutionComputation::ConvolutionStep::columns, ConvolutionComputation::ConvolutionStep::columns_are_contiguous, CuMatrixBase< Real >::CopyCols(), CuMatrixBase< Real >::Data(), CuArray< T >::Dim(), ConvolutionComputation::ConvolutionStep::first_column, ConvolutionComputation::height_out, ConvolutionComputation::ConvolutionStep::input_time_shift, KALDI_ASSERT, kaldi::kNoTrans, kaldi::kTrans, ConvolutionComputation::num_filters_out, ConvolutionComputation::num_images, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), ConvolutionComputation::ConvolutionStep::params_start_col, ConvolutionComputation::steps, and CuMatrixBase< Real >::Stride().

Referenced by ConvolveForward().

453  {
454  KALDI_ASSERT(temp_mat->Stride() == temp_mat->NumCols());
455 
456  // num_t_out supersedes cc.num_t_out (they'll only be different in
457  // cases where we are doing the computation in pieces to save memory).
458  int32 input_rows = input.NumRows(),
459  output_rows = output->NumRows();
460 
461  KALDI_ASSERT(output_rows <= input_rows &&
462  input_rows % cc.num_images == 0 &&
463  output_rows % cc.num_images == 0);
464 
465  int32 num_steps = cc.steps.size();
466  for (int32 s = 0; s < num_steps; s++) {
467  const ConvolutionComputation::ConvolutionStep &step = cc.steps[s];
468  int32 input_row_start = step.input_time_shift * cc.num_images;
469  // note: 'input_part' will normally be almost all of 'input', perhaps
470  // minus one or two time steps at the start or end.
471  CuSubMatrix<BaseFloat> input_part(input,
472  input_row_start, output_rows,
473  0, input.NumCols());
474  int32 temp_num_cols = step.columns.Dim(),
475  param_cols = temp_num_cols / cc.height_out;
476  CuSubMatrix<BaseFloat> params_part(params,
477  0, params.NumRows(),
478  step.params_start_col,
479  param_cols);
480  CuSubMatrix<BaseFloat> output_reshaped(
481  output->Data(), output_rows * cc.height_out,
482  cc.num_filters_out, cc.num_filters_out);
483  if (!step.columns_are_contiguous ||
484  temp_num_cols != input.NumCols()) {
485  // In most cases we will take this branch, where we have to copy the input
486  // to a temporary matrix. (however, different steps may require different
487  // num-cols of the temporary matrix, so we create sub-parts of 'temp_mat'.
488 
489  // We create the sub-matrix 'temp_mat_part' in a lower-level way, using
490  // pointers, because we need to ensure that its num-cols and the stride
491  // are the same (this is necessary so that we can do reshaping in
492  // ConvolutionReshapedMultiply()).
493  CuSubMatrix<BaseFloat> temp_mat_part(temp_mat->Data(),
494  temp_mat->NumRows(),
495  temp_num_cols, temp_num_cols);
496  if (!step.columns_are_contiguous) {
497  // we're doing a column mapping.
498  temp_mat_part.CopyCols(input_part, step.columns);
499  } else {
500  // we're just taking a sub-matrix of the input matrix, but we still need
501  // to make a copy because we need the stride == num-cols (so that the
502  // reshaping will work).
503  temp_mat_part.CopyFromMat(input_part.ColRange(step.first_column,
504  step.columns.Dim()));
505  }
506  CuSubMatrix<BaseFloat> temp_mat_part_reshaped(
507  temp_mat_part.Data(), temp_mat_part.NumRows() * cc.height_out,
508  temp_num_cols / cc.height_out, temp_num_cols / cc.height_out);
509 
510  output_reshaped.AddMatMat(1.0, temp_mat_part_reshaped, kNoTrans,
511  params_part, kTrans, 1.0);
512  } else {
513  CuSubMatrix<BaseFloat> input_reshaped(
514  input_part.Data(), input_part.NumRows() * cc.height_out,
515  input_part.NumCols() / cc.height_out,
516  input_part.NumCols() / cc.height_out);
517 
518  output_reshaped.AddMatMat(1.0, input_reshaped, kNoTrans,
519  params_part, kTrans, 1.0);
520  }
521  }
522 }
MatrixIndexT NumCols() const
Definition: cu-matrix.h:196
MatrixIndexT NumRows() const
Dimensions.
Definition: cu-matrix.h:195
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
MatrixIndexT Stride() const
Definition: cu-matrix.h:197
const Real * Data() const
Return data pointer (const).
Definition: cu-matrix.h:625
void kaldi::nnet3::time_height_convolution::ConvolveForwardSimple ( const ConvolutionModel model,
const std::vector< Index > &  input_indexes,
const std::vector< Index > &  output_indexes,
const CuMatrixBase< BaseFloat > &  input_cu,
const CuMatrixBase< BaseFloat > &  params_cu,
CuMatrixBase< BaseFloat > *  output_cu 
)

Definition at line 218 of file convolution-test.cc.

References CuMatrixBase< Real >::CopyFromMat(), ConvolutionModel::height_in, ConvolutionModel::height_out, ConvolutionModel::height_subsample_out, kaldi::nnet3::kNoTime, kaldi::kNoTrans, ConvolutionModel::num_filters_in, ConvolutionModel::num_filters_out, MatrixBase< Real >::NumRows(), ConvolutionModel::offsets, and Index::t.

Referenced by TestRunningComputation().

224  {
225  // these loops will be very slow on GPU, so do it all on CPU.
226  Matrix<BaseFloat> input(input_cu), params(params_cu),
227  output(*output_cu);
228  std::unordered_map<Index, int32, IndexHasher> index_to_row;
229  int32 input_rows = input.NumRows(),
230  output_rows = output.NumRows();
231  for (int32 r_in = 0; r_in < input_rows; r_in++) {
232  if (input_indexes[r_in].t != kNoTime) {
233  index_to_row[input_indexes[r_in]] = r_in;
234  }
235  }
236  int32 num_offsets = model.offsets.size(),
237  num_filters_in = model.num_filters_in,
238  num_filters_out = model.num_filters_out,
239  height_in = model.height_in,
240  height_out = model.height_out,
241  height_subsample_out = model.height_subsample_out;
242  for (int32 r_out = 0; r_out < output_rows; r_out++) {
243  Index index_out = output_indexes[r_out];
244  if (index_out.t == kNoTime)
245  continue;
246  SubVector<BaseFloat> output_row(output, r_out);
247  for (int32 o = 0; o < num_offsets; o++) {
248  int32 time_offset = model.offsets[o].time_offset,
249  height_offset = model.offsets[o].height_offset;
250  Index index_in(index_out);
251  index_in.t += time_offset;
252  std::unordered_map<Index, int32, IndexHasher>::const_iterator iter =
253  index_to_row.find(index_in);
254  if (iter != index_to_row.end()) {
255  SubMatrix<BaseFloat> params_part(params, 0, params.NumRows(),
256  o * num_filters_in, num_filters_in);
257  int32 r_in = iter->second;
258  SubVector<BaseFloat> input_row(input, r_in);
259  for (int32 h_out_subsampled = 0;
260  h_out_subsampled < height_out;
261  h_out_subsampled++) {
262  int32 h_out = h_out_subsampled * height_subsample_out,
263  h_in = h_out + height_offset;
264  if (h_in < 0 || h_in >= height_in)
265  continue;
266  SubVector<BaseFloat> output_part(output_row,
267  h_out_subsampled * num_filters_out,
268  num_filters_out),
269  input_part(input_row, h_in * num_filters_in, num_filters_in);
270  output_part.AddMatVec(1.0, params_part, kNoTrans, input_part, 1.0);
271  }
272  }
273  }
274  }
275  output_cu->CopyFromMat(output);
276 }
void CopyFromMat(const MatrixBase< OtherReal > &src, MatrixTransposeType trans=kNoTrans)
Definition: cu-matrix.cc:337
struct Index is intended to represent the various indexes by which we number the rows of the matrices...
Definition: nnet-common.h:44
Sub-matrix representation.
Definition: kaldi-matrix.h:908
Represents a non-allocating general vector which can be defined as a sub-vector of higher-level vecto...
Definition: kaldi-vector.h:482
const int kNoTime
Definition: nnet-common.cc:554
static void kaldi::nnet3::time_height_convolution::CreateIndexes ( const std::vector< std::pair< int32, int32 > > &  n_x_pairs,
int32  t_start,
int32  t_step,
int32  num_t_values,
int32  reorder_t,
std::vector< Index > *  indexes 
)
static

Creates a vector of indexes with a regular structure, according to these specifications.

'n_x_pairs' is the list of (n,x) pairs to include; they will appear in this order.

't_start', 't_step' and 'num_t_values' define the set of 't' values to include (note: t_step >= 0; they will appear in the natural order).

If reorder_t == 1 (the normal case), then the order is simple: 't' has the higher stride, then (n, x). So we'll output first all (n, x) pairs for t_start, then all pairs for t_start + t_step, and so on.

If instead reorder_t > 1, then the order is a little different [note: we expect that num_t_values % reorder_t == 0). Consider, for example, reorder_t == 2. In that case the first block has the first two t values, the second block has the next two t values, and so on. And within each block, the 't' values have the smallest stride (of 1).

Definition at line 1470 of file convolution.cc.

References KALDI_ASSERT, and Index::n.

Referenced by GetIndexesForComputation().

1472  {
1473  KALDI_ASSERT(reorder_t >= 1 && num_t_values % reorder_t == 0 && t_step >= 0);
1474  if (t_step == 0) {
1475  KALDI_ASSERT(num_t_values == 1);
1476  t_step = 1;
1477  }
1478  int32 num_n_x_pairs = n_x_pairs.size();
1479  indexes->clear();
1480  indexes->reserve(num_n_x_pairs * num_t_values);
1481  int32 outer_t_step = t_step * reorder_t,
1482  t_end = t_start + (num_t_values * t_step);
1483  Index index;
1484  for (int32 t_block = t_start; t_block < t_end; t_block += outer_t_step) {
1485  for (int32 nx = 0; nx < num_n_x_pairs; nx++) {
1486  index.n = n_x_pairs[nx].first;
1487  index.x = n_x_pairs[nx].second;
1488  for (int32 t = t_block; t < t_block + outer_t_step; t += t_step) {
1489  index.t = t;
1490  indexes->push_back(index);
1491  }
1492  }
1493  }
1494  // we can remove the next assert after a while.
1495  KALDI_ASSERT(indexes->size() == num_n_x_pairs * num_t_values);
1496 }
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static int32 kaldi::nnet3::time_height_convolution::FindGcdOfDifferences ( std::vector< int32 > &  vec)
static

Definition at line 1420 of file convolution.cc.

References kaldi::Gcd(), and rnnlm::i.

Referenced by RegularizeTList().

1420  {
1421  size_t size = vec.size();
1422  int32 ans = 0;
1423  for (size_t i = 0; i + 1 < size; i++) {
1424  int32 diff = vec[i+1] - vec[i];
1425  // diff should not be zero.
1426  ans = Gcd(ans, diff);
1427  }
1428  return ans;
1429 }
I Gcd(I m, I n)
Definition: kaldi-math.h:294
void GetComputationIo ( const std::vector< Index > &  input_indexes,
const std::vector< Index > &  output_indexes,
ConvolutionComputationIo *  io 
)

This function takes lists of input and output indexes to a computation (e.g.

as supplied to ReorderIndexes()), and figures out a regular structure for them (i.e. the smallest grid that will completely cover all the t,n pairs). This function ignores any 't' values that are kNoTime.

Definition at line 1519 of file convolution.cc.

References kaldi::nnet3::GetNxList(), kaldi::nnet3::GetTList(), kaldi::GetVerboseLevel(), KALDI_ASSERT, ConvolutionComputationIo::num_images, ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::num_t_out, RegularizeTList(), ConvolutionComputationIo::reorder_t_in, ConvolutionComputationIo::start_t_in, ConvolutionComputationIo::start_t_out, ConvolutionComputationIo::t_step_in, and ConvolutionComputationIo::t_step_out.

Referenced by CompileConvolutionComputation(), and RestrictedAttentionComponent::GetComputationStructure().

1522  {
1523  std::vector<std::pair<int32, int32> > n_x_pairs;
1524  GetNxList(input_indexes, &n_x_pairs);
1525  KALDI_ASSERT(!n_x_pairs.empty());
1526  io->num_images = n_x_pairs.size();
1527  if (GetVerboseLevel() >= 3) { // a debugging step.
1528  std::vector<std::pair<int32, int32> > n_x_pairs_2;
1529  GetNxList(output_indexes, &n_x_pairs_2);
1530  KALDI_ASSERT(n_x_pairs_2 == n_x_pairs);
1531  }
1532  std::vector<int32> t_values;
1533  GetTList(input_indexes, &t_values);
1534  RegularizeTList(t_values, &(io->start_t_in),
1535  &(io->t_step_in), &(io->num_t_in));
1536  GetTList(output_indexes, &t_values);
1537  RegularizeTList(t_values, &(io->start_t_out),
1538  &(io->t_step_out), &(io->num_t_out));
1539  io->reorder_t_in = 1;
1540 }
void GetTList(const std::vector< Index > &indexes, std::vector< int32 > *t_values)
This function outputs a sorted, unique list of the 't' values that are encountered in the provided li...
int32 GetVerboseLevel()
Definition: kaldi-error.h:69
static void RegularizeTList(std::vector< int32 > &t_values, int32 *start, int32 *step, int32 *num_values)
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
void GetNxList(const std::vector< Index > &indexes, std::vector< std::pair< int32, int32 > > *pairs)
This function outputs a unique, lexicographically sorted list of the pairs of (n, x) values that are ...
void GetIndexesForComputation ( const ConvolutionComputationIo &  io,
const std::vector< Index > &  orig_input_indexes,
const std::vector< Index > &  orig_output_indexes,
std::vector< Index > *  input_indexes,
std::vector< Index > *  output_indexes 
)

This function computes the reordered and possibly padded indexes corresponding to the computation in 'io'.

Note: the computation may have undergone various manipulations (padding, etc.) after being obtained by the function GetComputationIo(). The original input and output indexes are needed because they dictate the set of (n, x) pairs; and because they determine when to use 'real' indexes and when to use 'blank' padding values (i.e. when to replace the t values in the indexes by kNoTime).

Definition at line 1543 of file convolution.cc.

References CreateIndexes(), kaldi::nnet3::GetNxList(), KALDI_ASSERT, ConvolutionComputationIo::num_images, ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::num_t_out, ConvolutionComputationIo::reorder_t_in, SetSomeIndexesBlank(), ConvolutionComputationIo::start_t_in, ConvolutionComputationIo::start_t_out, ConvolutionComputationIo::t_step_in, and ConvolutionComputationIo::t_step_out.

Referenced by CompileConvolutionComputation().

1548  {
1549  std::unordered_set<Index, IndexHasher> input_set, output_set;
1550  for (std::vector<Index>::const_iterator iter = orig_input_indexes.begin();
1551  iter != orig_input_indexes.end(); ++iter)
1552  input_set.insert(*iter);
1553  for (std::vector<Index>::const_iterator iter = orig_output_indexes.begin();
1554  iter != orig_output_indexes.end(); ++iter)
1555  output_set.insert(*iter);
1556  std::vector<std::pair<int32, int32> > n_x_pairs;
1557  GetNxList(orig_input_indexes, &n_x_pairs);
1558  KALDI_ASSERT(n_x_pairs.size() == io.num_images);
1559  CreateIndexes(n_x_pairs, io.start_t_in, io.t_step_in, io.num_t_in,
1560  io.reorder_t_in, input_indexes);
1561  SetSomeIndexesBlank(orig_input_indexes, input_indexes);
1562  CreateIndexes(n_x_pairs, io.start_t_out, io.t_step_out, io.num_t_out,
1563  1, output_indexes);
1564  SetSomeIndexesBlank(orig_output_indexes, output_indexes);
1565 }
static void CreateIndexes(const std::vector< std::pair< int32, int32 > > &n_x_pairs, int32 t_start, int32 t_step, int32 num_t_values, int32 reorder_t, std::vector< Index > *indexes)
Creates a vector of indexes with a regular structure, according to these specifications.
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static void SetSomeIndexesBlank(const std::vector< Index > &ref_indexes, std::vector< Index > *indexes)
This function modifies 'indexes' by, for any Indexes which was not present in 'ref_indexes', setting the 't' value to kNoTime.
void GetNxList(const std::vector< Index > &indexes, std::vector< std::pair< int32, int32 > > *pairs)
This function outputs a unique, lexicographically sorted list of the pairs of (n, x) values that are ...
static void kaldi::nnet3::time_height_convolution::GetRandomConvolutionIndexes ( const ConvolutionModel model,
std::vector< Index > *  input_indexes,
std::vector< Index > *  output_indexes 
)
static

Definition at line 78 of file convolution-test.cc.

References ConvolutionModel::all_time_offsets, ConvolutionModel::Check(), rnnlm::i, kaldi::IsSortedAndUniq(), rnnlm::j, KALDI_ASSERT, Index::n, rnnlm::n, kaldi::RandInt(), ConvolutionModel::required_time_offsets, kaldi::SortAndUniq(), Index::t, and Index::x.

Referenced by UnitTestTimeHeightConvolutionCompile().

80  {
81  KALDI_ASSERT(model.Check());
82 
83  std::vector<std::pair<int32, int32> > n_x_pairs;
84  int32 num_n_x_pairs = RandInt(1, 3);
85  for (int32 i = 0; i < num_n_x_pairs; i++) {
86  int32 n = RandInt(0, 3), x = RandInt(0, 1);
87  n_x_pairs.push_back(std::pair<int32, int32>(n, x));
88  }
89  SortAndUniq(&n_x_pairs);
90  num_n_x_pairs = n_x_pairs.size();
91 
92 
93  // 'output_t_values' is the set of *possible* output
94  // t values; we'll later sub-sample from these.
95  std::vector<int32> output_t_values;
96 
97  {
98  int32 out_t_start = RandInt(-5, 5), out_t_step = RandInt(1, 3),
99  num_t_out = RandInt(1, 4);
100  for (int32 i = 0; i < num_t_out; i++)
101  output_t_values.push_back(out_t_start + i * out_t_step);
102  }
103 
104  input_indexes->clear();
105  output_indexes->clear();
106  for (size_t i = 0; i < n_x_pairs.size(); i++) {
107  std::vector<int32> chosen_output_t_values;
108  while (chosen_output_t_values.empty()) {
109  for (size_t j = 0; j < output_t_values.size(); j++)
110  if (RandInt(0, 1) != 0)
111  chosen_output_t_values.push_back(output_t_values[j]);
112  }
113  KALDI_ASSERT(IsSortedAndUniq(chosen_output_t_values));
114 
115  std::set<int32> required_input_t_values,
116  usable_input_t_values;
117  for (size_t j = 0; j < chosen_output_t_values.size(); j++) {
118  std::set<int32>::const_iterator iter;
119  int32 t_out = chosen_output_t_values[j];
120  for (iter = model.required_time_offsets.begin();
121  iter != model.required_time_offsets.end(); iter++) {
122  int32 offset = *iter;
123  required_input_t_values.insert(t_out + offset);
124  }
125  for (iter = model.all_time_offsets.begin();
126  iter != model.all_time_offsets.end(); iter++) {
127  int32 offset = *iter;
128  usable_input_t_values.insert(t_out + offset);
129  }
130  }
131 
132  // add to output_indexes
133  for (size_t j = 0; j < chosen_output_t_values.size(); j++) {
134  int32 t_out = chosen_output_t_values[j];
135  Index index;
136  index.n = n_x_pairs[i].first;
137  index.x = n_x_pairs[i].second;
138  index.t = t_out;
139  output_indexes->push_back(index);
140  }
141 
142  std::vector<int32> chosen_input_t_values(required_input_t_values.begin(),
143  required_input_t_values.end());
144  for (std::set<int32>::const_iterator iter = usable_input_t_values.begin();
145  iter != usable_input_t_values.end(); ++iter) {
146  int32 t = *iter;
147  if (RandInt(0, 1) == 0)
148  chosen_input_t_values.push_back(t);
149  }
150  SortAndUniq(&chosen_input_t_values);
151 
152  // add to input_indexes
153  for (size_t j = 0; j < chosen_input_t_values.size(); j++) {
154  int32 t_in = chosen_input_t_values[j];
155  Index index;
156  index.n = n_x_pairs[i].first;
157  index.x = n_x_pairs[i].second;
158  index.t = t_in;
159  input_indexes->push_back(index);
160  }
161  }
162 }
void SortAndUniq(std::vector< T > *vec)
Sorts and uniq's (removes duplicates) from a vector.
Definition: stl-utils.h:39
struct Index is intended to represent the various indexes by which we number the rows of the matrices...
Definition: nnet-common.h:44
struct rnnlm::@11::@12 n
bool Check(bool check_heights_used=true, bool allow_height_padding=true) const
Definition: convolution.cc:130
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
bool IsSortedAndUniq(const std::vector< T > &vec)
Returns true if the vector is sorted and contains each element only once.
Definition: stl-utils.h:63
int32 RandInt(int32 min_val, int32 max_val, struct RandomState *state)
Definition: kaldi-math.cc:94
static void kaldi::nnet3::time_height_convolution::GetRandomConvolutionModel ( ConvolutionModel model)
static

Definition at line 28 of file convolution-test.cc.

References ConvolutionModel::Check(), ConvolutionModel::ComputeDerived(), ConvolutionModel::height_in, ConvolutionModel::Offset::height_offset, ConvolutionModel::height_out, ConvolutionModel::height_subsample_out, rnnlm::i, ConvolutionModel::Info(), KALDI_WARN, ConvolutionModel::num_filters_in, ConvolutionModel::num_filters_out, ConvolutionModel::offsets, kaldi::RandInt(), ConvolutionModel::required_time_offsets, kaldi::SortAndUniq(), and ConvolutionModel::Offset::time_offset.

Referenced by UnitTestTimeHeightConvolutionCompile(), and UnitTestTimeHeightConvolutionIo().

28  {
29 start:
30  {
31  model->num_filters_in = RandInt(1, 10);
32  model->num_filters_out = RandInt(1, 10);
33  model->height_in = RandInt(1, 10);
34  int32 min_height_offset = RandInt(-2, 0),
35  max_height_offset = RandInt(0, 2),
36  min_time_offset = RandInt(-2, 0),
37  max_time_offset = RandInt(0, 2);
38 
39  model->height_out = RandInt(1, model->height_in);
40  model->height_subsample_out = 1;
41  if (RandInt(0, 1) == 0) {
42  if (model->height_out % 2 == 0) {
43  model->height_out /= 2;
44  model->height_subsample_out = 2;
45  } else if (model->height_out % 3 == 0) {
46  model->height_out /= 3;
47  model->height_subsample_out = 3;
48  }
49  }
50  std::vector<int32> all_time_offsets;
51  int32 max_offsets = RandInt(1, 10);
52  model->offsets.clear();
53  model->required_time_offsets.clear();
54  for (int32 i = 0; i < max_offsets; i++) {
56  o.time_offset = RandInt(min_time_offset, max_time_offset);
57  o.height_offset = RandInt(min_height_offset, max_height_offset);
58  all_time_offsets.push_back(o.time_offset);
59  model->offsets.push_back(o);
60  }
61  SortAndUniq(&(model->offsets));
62  SortAndUniq(&all_time_offsets);
63  std::random_shuffle(all_time_offsets.begin(), all_time_offsets.end());
64  int32 num_required_offsets = RandInt(1, all_time_offsets.size());
65  for (int32 i = 0; i < num_required_offsets; i++)
66  model->required_time_offsets.insert(all_time_offsets[i]);
67  model->ComputeDerived();
68  }
69  if (!model->Check()) {
70  KALDI_WARN << "Regenerating model because it didn't pass the check: "
71  << model->Info();
72  goto start;
73  }
74 }
void SortAndUniq(std::vector< T > *vec)
Sorts and uniq's (removes duplicates) from a vector.
Definition: stl-utils.h:39
#define KALDI_WARN
Definition: kaldi-error.h:130
bool Check(bool check_heights_used=true, bool allow_height_padding=true) const
Definition: convolution.cc:130
int32 RandInt(int32 min_val, int32 max_val, struct RandomState *state)
Definition: kaldi-math.cc:94
void MakeComputation ( const ConvolutionModel &  model,
ConvolutionComputationIo &  io,
const ConvolutionComputationOptions &  opts,
ConvolutionComputation *  computation 
)

Definition at line 1568 of file convolution.cc.

References ComputeTempMatrixSize(), ConvolutionModel::height_in, ConvolutionComputation::height_in, ConvolutionComputation::ConvolutionStep::height_map, ConvolutionModel::height_out, ConvolutionComputation::height_out, ConvolutionModel::height_subsample_out, ConvolutionComputation::ConvolutionStep::input_time_shift, kaldi::IsSortedAndUniq(), KALDI_ASSERT, ConvolutionModel::num_filters_in, ConvolutionComputation::num_filters_in, ConvolutionModel::num_filters_out, ConvolutionComputation::num_filters_out, ConvolutionComputation::num_images, ConvolutionComputationIo::num_images, ConvolutionComputation::num_t_in, ConvolutionComputationIo::num_t_in, ConvolutionComputation::num_t_out, ConvolutionComputationIo::num_t_out, ConvolutionModel::offsets, ConvolutionComputation::ConvolutionStep::params_start_col, ConvolutionComputationIo::reorder_t_in, ConvolutionComputationIo::start_t_in, ConvolutionComputationIo::start_t_out, ConvolutionComputation::steps, ConvolutionComputationIo::t_step_in, and ConvolutionComputationIo::t_step_out.

Referenced by CompileConvolutionComputation().

1571  {
1572  KALDI_ASSERT(io.t_step_in == io.t_step_out);
1573  computation->num_filters_in = model.num_filters_in;
1574  computation->num_filters_out = model.num_filters_out;
1575  computation->height_in = model.height_in;
1576  computation->height_out = model.height_out;
1577  computation->num_t_in = io.num_t_in;
1578  computation->num_t_out = io.num_t_out;
1579  computation->num_images = io.num_images;
1580  KALDI_ASSERT(io.reorder_t_in == 1);
1581  // first work out the steps of the computation, then
1582  // work out the dim of the temp matrix
1583 
1584  KALDI_ASSERT(IsSortedAndUniq(model.offsets));
1585  // Each distinct value of 'time_offset' in model.offsets
1586  // becomes one step of the computation.
1587 
1588  // if io.t_step_in was zero, use 1 (so divisions and the like will work as
1589  // expected).
1590  int32 t_step = std::max<int32>(1, io.t_step_in),
1591  num_t_extra = io.num_t_in - io.num_t_out;
1592 
1593  computation->steps.clear();
1594 
1595  int32 num_offsets = model.offsets.size(),
1596  cur_start_offset = 0, cur_end_offset = 0;
1597  for(; cur_start_offset < num_offsets; cur_start_offset = cur_end_offset) {
1598  cur_end_offset = cur_start_offset;
1599  while (cur_end_offset < num_offsets &&
1600  model.offsets[cur_end_offset].time_offset ==
1601  model.offsets[cur_start_offset].time_offset)
1602  cur_end_offset++;
1603  // we are processing the range of indexes into 'offsets'
1604  // from cur_start_offset to cur_end_offset - 1.
1605  int32 this_num_offsets = cur_end_offset - cur_start_offset;
1606  int32 time_offset = model.offsets[cur_start_offset].time_offset;
1607 
1608  ConvolutionComputation::ConvolutionStep step;
1609  // modified_time_offset will be used in working out the 'input_time_shift'
1610  // that determines which submatrix of the input matrix we'll use.
1611  // It equals the time-offset corrected for any time-difference between
1612  // the start of the output and of the input.
1613  int32 modified_time_offset = time_offset + io.start_t_out - io.start_t_in;
1614  KALDI_ASSERT(modified_time_offset >= 0 &&
1615  modified_time_offset % t_step == 0);
1616  step.input_time_shift = modified_time_offset / t_step;
1617  KALDI_ASSERT(step.input_time_shift <= num_t_extra);
1618  step.params_start_col = model.num_filters_in * cur_start_offset;
1619  step.height_map.clear();
1620  step.height_map.reserve(model.height_out * this_num_offsets);
1621  for (int32 h_out = 0;
1622  h_out < model.height_out * model.height_subsample_out;
1623  h_out += model.height_subsample_out) {
1624  for (int32 o = cur_start_offset; o < cur_end_offset; o++) {
1625  int32 this_height_offset = model.offsets[o].height_offset,
1626  h_in = h_out + this_height_offset;
1627  // by the time we call MakeComputation, the user should already have
1628  // called PadModelHeight, so there should be no need for zero padding on
1629  // the height axis, hence the following check. [we'll later modify the
1630  // resulting computation in UnPadModelHeight, and that's where
1631  // zero-padding gets taken account of.]
1632  KALDI_ASSERT(h_in >= 0 && h_in < model.height_in);
1633  step.height_map.push_back(h_in);
1634  }
1635  }
1636  computation->steps.push_back(step);
1637  }
1638  ComputeTempMatrixSize(opts, computation);
1639 }
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static void ComputeTempMatrixSize(const ConvolutionComputationOptions &opts, ConvolutionComputation *computation)
This function sets 'temp_rows' and 'temp_cols' in 'computation'.
Definition: convolution.cc:956
bool IsSortedAndUniq(const std::vector< T > &vec)
Returns true if the vector is sorted and contains each element only once.
Definition: stl-utils.h:63
void PadComputationInputTime ( const ConvolutionModel &  model,
ConvolutionComputationIo *  io 
)

This function extends the set of input indexes that the computation has, to account for any required zero-padding in the time dimension.

It reads model.all_time_offsets and model.time_offsets_modulus; and it may modify members start_t_in t_stride_in and num_t_in of *io.

This is stage 1 of compilation.

Definition at line 1051 of file convolution.cc.

References ConvolutionModel::all_time_offsets, kaldi::Gcd(), KALDI_ASSERT, ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::num_t_out, ConvolutionComputationIo::start_t_in, ConvolutionComputationIo::start_t_out, ConvolutionComputationIo::t_step_in, ConvolutionComputationIo::t_step_out, and ConvolutionModel::time_offsets_modulus.

Referenced by CompileConvolutionComputation().

1052  {
1053  if (model.time_offsets_modulus == 0) {
1054  // this can only happen if model->all_time_offsets.size() == 1,
1055  // and no padding could be required here. W return to avoid
1056  // special cases below in Gcd().
1057  return;
1058  }
1059  int32 min_time_offset = *model.all_time_offsets.begin(),
1060  max_time_offset = *model.all_time_offsets.rbegin();
1061 
1062  // it makes everything much simpler if we just enforce that the stride of the
1063  // input divides model.time_offsets_modulus and also the output stride.
1064  // (enforcing this may make the input stride smaller). This may in certain
1065  // very odd cases cause us to require more inputs [actually 'blanks'] than
1066  // we really need, but it avoids a lot of careful thought.
1067  int32 old_t_step_in = io->t_step_in;
1068  io->t_step_in = Gcd(io->t_step_in, model.time_offsets_modulus);
1069  if (io->t_step_out != 0)
1070  io->t_step_in = Gcd(io->t_step_in, io->t_step_out);
1071 
1072  // to ensure that we cover all the original input points, now that
1073  // we changed the stride we may need to increase num_t_in.
1074  io->num_t_in = 1 + (old_t_step_in * (io->num_t_in - 1)) / io->t_step_in;
1075 
1076  // by 'desired' we mean usable as an input, not necessarily
1077  // required in the sense of 'required_time_offsets'.
1078  int32 first_desired_input_t = io->start_t_out + min_time_offset;
1079  if (first_desired_input_t < io->start_t_in) {
1080  KALDI_ASSERT((io->start_t_in - first_desired_input_t) %
1081  io->t_step_in == 0);
1082  io->num_t_in += (io->start_t_in - first_desired_input_t) / io->t_step_in;
1083  io->start_t_in = first_desired_input_t;
1084  }
1085 
1086  int32 last_desired_input_t =
1087  io->start_t_out + (io->num_t_out - 1) * io->t_step_out + max_time_offset,
1088  last_input_t = io->start_t_in + (io->num_t_in - 1) * io->t_step_in;
1089  // if the following assert fails, it means we had provided more input than was
1090  // needed, which is not expected. This could cause problems later, in
1091  // AppendInputFrames().
1092  KALDI_ASSERT(last_desired_input_t >= last_input_t);
1093  if (last_desired_input_t > last_input_t) {
1094  KALDI_ASSERT((last_desired_input_t - last_input_t) %
1095  io->t_step_in == 0);
1096  io->num_t_in += (last_desired_input_t - last_input_t) / io->t_step_in;
1097  }
1098 }
I Gcd(I m, I n)
Definition: kaldi-math.h:294
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
void PadModelHeight ( const ConvolutionModel &  model,
ConvolutionModel *  model_padded 
)

This function takes a model that might require zero padding in the height dimension and outputs a model accepting a possibly-larger input dimension which does not require zero padding.

*model_padded may differ from 'model' in its height_in and its 'offsets' variable (the height-offsets need to be shifted if we pad at the bottom). We then work out the computation in terms of the model that doesn't need padding (which is easier), and later convert it back to work in the space where there is no padding.

This is stage 2 of compilation.

Definition at line 918 of file convolution.cc.

References ConvolutionModel::Check(), ConvolutionModel::height_in, ConvolutionModel::height_out, ConvolutionModel::height_subsample_out, rnnlm::i, KALDI_ASSERT, and ConvolutionModel::offsets.

Referenced by CompileConvolutionComputation().

919  {
920  *model_padded = model;
921  KALDI_ASSERT(!model.offsets.empty());
922  int32 min_height_offset = model.offsets[0].height_offset,
923  max_height_offset = model.offsets[0].height_offset,
924  num_offsets = model.offsets.size();
925  for (int32 i = 1; i < num_offsets; i++) {
926  min_height_offset = std::min<int32>(min_height_offset,
927  model.offsets[i].height_offset);
928  max_height_offset = std::max<int32>(max_height_offset,
929  model.offsets[i].height_offset);
930  }
931  int32 max_output_height = model.height_subsample_out * (model.height_out - 1),
932  max_required_input = max_height_offset + max_output_height,
933  min_required_input = min_height_offset + 0;
934  int32 bottom_padding = -min_required_input,
935  top_padding = max_required_input - (model.height_in - 1);
936  if (bottom_padding < 0)
937  bottom_padding = 0;
938  if (top_padding < 0)
939  top_padding = 0;
940  model_padded->height_in += bottom_padding + top_padding;
941  for (int32 i = 0; i < num_offsets; i++)
942  model_padded->offsets[i].height_offset += bottom_padding;
943 
944  // The reason why we say 'allow_height_padding = false' below is obvious--
945  // we've 'manually' padded by changing the model, so this modified model
946  // should not require height padding. The reason we set 'check_heights_used'
947  // is a little more non-obvious. The very lowest and hightest heights
948  // should always be used, but there may, in unusual models, be other heights
949  // that are not used. We found this in random testing.
950  KALDI_ASSERT(model_padded->Check(false, false));
951 }
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static int32 kaldi::nnet3::time_height_convolution::PrepareIoForAppending ( ConvolutionComputationIo *  io,
ConvolutionComputationIo *  io_appended 
)
static

Definition at line 1158 of file convolution.cc.

References KALDI_ASSERT, ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::num_t_out, ConvolutionComputationIo::reorder_t_in, ConvolutionComputationIo::t_step_in, and ConvolutionComputationIo::t_step_out.

Referenced by AppendInputFrames().

1159  {
1160  // first make sure that the output has nonzero stride (it would only have zero
1161  // stride if there was only one output time index, which is unusual). if
1162  // there's only one output time index we can set the stride to whatever we
1163  // want without affecting the list of output indexes.
1164  int32 ratio;
1165  if (io->t_step_out == 0) {
1166  KALDI_ASSERT(io->num_t_out == 1);
1167  io->t_step_out = io->t_step_in;
1168  }
1169  if (io->t_step_out == io->t_step_in) {
1170  // there is nothing to do; the output and input strides are the same.
1171  *io_appended = *io;
1172  ratio = 1;
1173  return ratio;
1174  }
1175  // Now, we ensured in PadComputationInputTime that if the output stride is
1176  // nonzero, then the input stride must divide the output stride; and if the
1177  // output stride was zero then we would have set it to the input stride just
1178  // above; and if both were zero we would have returned above. So we can just
1179  // assert that the input stride divides the output stride.
1180  KALDI_ASSERT(io->t_step_out % io->t_step_in == 0);
1181  ratio = io->t_step_out / io->t_step_in;
1182  // ratio says how many input indexes we have for each output index,
1183  // ignoring end effects. It is the number of input indexes we will
1184  // append together and 'pretend'
1185 
1186  // record this ratio in the 'input' I/O object, which we are also
1187  // modifying to record the extra required padding.
1188  io->reorder_t_in = ratio;
1189  if (io->num_t_in % ratio != 0) {
1190  // Round up the number of input frames to the nearest multiple (via
1191  // zero-padding) so we get an whole number of appended input frames.
1192  io->num_t_in += ratio - (io->num_t_in % ratio);
1193  }
1194 
1195  // OK, from this point we create the output io object.
1196  *io_appended = *io;
1197  io_appended->reorder_t_in = 1;
1198  io_appended->t_step_in = io->t_step_out;
1199  io_appended->num_t_in /= ratio;
1200  return ratio;
1201 }
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static void kaldi::nnet3::time_height_convolution::RegularizeTList ( std::vector< int32 > &  t_values,
int32 *  start,
int32 *  step,
int32 *  num_values 
)
static

Definition at line 1431 of file convolution.cc.

References FindGcdOfDifferences(), kaldi::IsSortedAndUniq(), and KALDI_ASSERT.

Referenced by GetComputationIo().

1434  {
1435  KALDI_ASSERT(!t_values.empty() && IsSortedAndUniq(t_values));
1436  *start = t_values[0];
1437  *step = FindGcdOfDifferences(t_values);
1438  if (*step == 0) {
1439  KALDI_ASSERT(t_values.size() == 1);
1440  *num_values = 1;
1441  } else {
1442  int32 last_value = t_values.back();
1443  *num_values = 1 + (last_value - *start) / *step;
1444  KALDI_ASSERT((last_value - *start) % *step == 0);
1445  }
1446 }
static int32 FindGcdOfDifferences(std::vector< int32 > &vec)
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
bool IsSortedAndUniq(const std::vector< T > &vec)
Returns true if the vector is sorted and contains each element only once.
Definition: stl-utils.h:63
static void kaldi::nnet3::time_height_convolution::ReverseColumnMapping ( const std::vector< int32 > &  columns,
int32  input_dim,
std::vector< std::vector< int32 > > *  backward_columns 
)
static

This function, used in ConvolutionComputation::ComputeDerived(), reverses a mapping that may not be unique.

'columns' is a column mapping where each member is either -1 (meaning, copy a zero), or a number between 0 and input_dim - 1.

Its output, 'backward_columns', is the reverse mapping, but it's a vector of vectors instead of just a vector because the mapping may have been many-to-one. Each element of 'backward_columns' will be of dimension input_dim. For each columns[i] = j such that j != -1, for some k we will have (*backward_columns)[k][j] = i.

Definition at line 44 of file convolution.cc.

References rnnlm::i, rnnlm::j, and KALDI_ASSERT.

Referenced by ConvolutionComputation::ComputeDerived().

47  {
48  int32 columns_dim = columns.size();
49  std::vector<std::vector<int32> > temp(input_dim);
50  for (int32 i = 0; i < columns_dim; i++) {
51  int32 j = columns[i];
52  KALDI_ASSERT(j >= -1 && j < input_dim);
53  if (j != -1)
54  temp[j].push_back(i);
55  }
56  // 'max_overlap' is the largest number of times that some j >= 0 appears in
57  // 'columns'.
58  int32 max_overlap = 0;
59  for (int32 j = 0; j < input_dim; j++)
60  max_overlap = std::max(max_overlap,
61  static_cast<int32>(temp[j].size()));
62  backward_columns->resize(max_overlap);
63  for (int32 k = 0; k < max_overlap; k++) {
64  (*backward_columns)[k].clear();
65  (*backward_columns)[k].resize(input_dim, -1);
66  }
67  for (int32 j = 0; j < input_dim; j++) {
68  for (int32 k = 0; k < static_cast<int32>(temp[j].size()); k++) {
69  int32 i = temp[j][k];
70  (*backward_columns)[k][j] = i;
71  }
72  }
73 }
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static int32 kaldi::nnet3::time_height_convolution::RoundDownToMultipleOf ( int32  i,
int32  n 
)
static

Definition at line 1103 of file convolution.cc.

References kaldi::DivideRoundingDown().

Referenced by AppendInputFrames().

1103  {
1104  return n * DivideRoundingDown(i, n);
1105 }
static int32 DivideRoundingDown(int32 a, int32 b)
Returns a / b, rounding towards negative infinity in all cases.
Definition: kaldi-math.h:284
struct rnnlm::@11::@12 n
static void kaldi::nnet3::time_height_convolution::SetSomeIndexesBlank ( const std::vector< Index > &  ref_indexes,
std::vector< Index > *  indexes 
)
static

This function modifies 'indexes' by, for any Indexes which was not present in 'ref_indexes', setting the 't' value to kNoTime.

This will cause the nnet3 framework to ignore such Indexes for certain purposes, it supresses certain error conditions that would otherwise happen from inserting unnecessary indexes into the input and output.

Definition at line 1505 of file convolution.cc.

References kaldi::nnet3::kNoTime.

Referenced by GetIndexesForComputation().

1506  {
1507  std::unordered_set<Index, IndexHasher> ref_set;
1508  for (std::vector<Index>::const_iterator iter = ref_indexes.begin();
1509  iter != ref_indexes.end(); ++iter)
1510  ref_set.insert(*iter);
1511 
1512  for (std::vector<Index>::iterator iter = indexes->begin();
1513  iter != indexes->end(); ++iter) {
1514  if (ref_set.count(*iter) == 0)
1515  iter->t = kNoTime;
1516  }
1517 }
const int kNoTime
Definition: nnet-common.cc:554
static void kaldi::nnet3::time_height_convolution::ShiftAllTimeOffsets ( int32  shift,
ConvolutionModel *  model 
)
static

Definition at line 1110 of file convolution.cc.

References ConvolutionModel::all_time_offsets, ConvolutionModel::offsets, and ConvolutionModel::required_time_offsets.

Referenced by AppendInputFrames().

1111  {
1112  { // shift 'offsets'.
1113  std::vector<ConvolutionModel::Offset>::iterator
1114  iter = model->offsets.begin(),
1115  end = model->offsets.end();
1116  for (; iter != end; ++iter)
1117  iter->time_offset += shift;
1118  }
1119  std::set<int32> temp;
1120  std::set<int32>::const_iterator iter;
1121  for (iter = model->required_time_offsets.begin();
1122  iter != model->required_time_offsets.end(); ++iter)
1123  temp.insert(*iter + shift);
1124  model->required_time_offsets.swap(temp);
1125  temp.clear();
1126  for (iter = model->all_time_offsets.begin();
1127  iter != model->all_time_offsets.end(); ++iter)
1128  temp.insert(*iter + shift);
1129  model->all_time_offsets.swap(temp);
1130 }
void kaldi::nnet3::time_height_convolution::TestComputationIo ( const ConvolutionComputation computation)

Definition at line 182 of file convolution-test.cc.

References KALDI_ASSERT, kaldi::RandInt(), ConvolutionComputation::Read(), and ConvolutionComputation::Write().

Referenced by UnitTestTimeHeightConvolutionCompile().

182  {
183  std::ostringstream os1, os2;
184  bool binary = (RandInt(0, 1) == 0);
185  computation.Write(os1, binary);
186  std::istringstream is(os1.str());
187  ConvolutionComputation computation2;
188  computation2.Read(is, binary);
189  computation2.Write(os2, binary);
190  KALDI_ASSERT(os1.str() == os2.str());
191  computation2.Check();
192 }
This struct represents the structure of a convolution computation.
Definition: convolution.h:252
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
void Write(std::ostream &os, bool binary) const
Definition: convolution.cc:283
int32 RandInt(int32 min_val, int32 max_val, struct RandomState *state)
Definition: kaldi-math.cc:94
void kaldi::nnet3::time_height_convolution::TestDataBackprop ( const ConvolutionModel conv_model,
const std::vector< Index > &  input_indexes,
const std::vector< Index > &  output_indexes,
const ConvolutionComputation computation 
)

Definition at line 308 of file convolution-test.cc.

References kaldi::ApproxEqual(), ConvolveBackwardData(), ConvolveForward(), ConvolutionModel::InputDim(), KALDI_ERR, KALDI_LOG, kaldi::kSetZero, kaldi::kStrideEqualNumCols, kaldi::kTrans, ConvolutionModel::OutputDim(), ConvolutionModel::ParamCols(), ConvolutionModel::ParamRows(), CuMatrixBase< Real >::SetRandn(), kaldi::TraceMatMat(), and ZeroBlankRows().

Referenced by UnitTestTimeHeightConvolutionCompile().

311  {
313  input_deriv(input_indexes.size(), conv_model.InputDim(),
315  input(input_indexes.size(), conv_model.InputDim(),
317  output(output_indexes.size(), conv_model.OutputDim(),
319  output_deriv(output_indexes.size(), conv_model.OutputDim(),
321  params(conv_model.ParamRows(), conv_model.ParamCols());
322 
323  input.SetRandn();
324  params.SetRandn();
325  output_deriv.SetRandn();
326 
327  ZeroBlankRows(output_indexes, &output_deriv);
328  ConvolveBackwardData(computation, params, output_deriv, &input_deriv);
329  ZeroBlankRows(input_indexes, &input_deriv);
330  ZeroBlankRows(input_indexes, &input);
331 
332  // define the objf as TraceMatMat(output_deriv, output, kTrans).
333  // we can work it out from the backpropagated data-derivative.
334  BaseFloat expected_objf = TraceMatMat(input_deriv, input, kTrans);
335 
336  ConvolveForward(computation, input, params, &output);
337  ZeroBlankRows(output_indexes, &output);
338 
339  BaseFloat observed_objf = TraceMatMat(output, output_deriv, kTrans);
340 
341  KALDI_LOG << "Expected objf = " << expected_objf
342  << ", observed objf = " << observed_objf;
343  if (!ApproxEqual(expected_objf, observed_objf, 0.1) &&
344  fabs(expected_objf) < 1.0) {
345  KALDI_ERR << "Difference in objf too large.";
346  }
347 }
float BaseFloat
Definition: kaldi-types.h:29
void ConvolveBackwardData(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &params, const CuMatrixBase< BaseFloat > &output_deriv, CuMatrixBase< BaseFloat > *input_deriv)
This does the part of the backward derivative computation of convolution, that propagates derivatives...
Definition: convolution.cc:682
void ZeroBlankRows(const std::vector< Index > &indexes, CuMatrix< BaseFloat > *matrix)
#define KALDI_ERR
Definition: kaldi-error.h:127
Real TraceMatMat(const MatrixBase< Real > &A, const MatrixBase< Real > &B, MatrixTransposeType trans)
We need to declare this here as it will be a friend function.
#define KALDI_LOG
Definition: kaldi-error.h:133
void ConvolveForward(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &params, CuMatrixBase< BaseFloat > *output)
This does the forward computation of convolution.
Definition: convolution.cc:524
static bool ApproxEqual(float a, float b, float relative_tolerance=0.001)
return abs(a - b) <= relative_tolerance * (abs(a)+abs(b)).
Definition: kaldi-math.h:262
void kaldi::nnet3::time_height_convolution::TestParamsBackprop ( const ConvolutionModel conv_model,
const std::vector< Index > &  input_indexes,
const std::vector< Index > &  output_indexes,
const ConvolutionComputation computation 
)

Definition at line 350 of file convolution-test.cc.

References kaldi::ApproxEqual(), ConvolveBackwardParams(), ConvolveForward(), ConvolutionModel::InputDim(), KALDI_ERR, KALDI_LOG, kaldi::kSetZero, kaldi::kStrideEqualNumCols, kaldi::kTrans, ConvolutionModel::OutputDim(), ConvolutionModel::ParamCols(), ConvolutionModel::ParamRows(), kaldi::RandInt(), CuMatrixBase< Real >::SetRandn(), kaldi::TraceMatMat(), and ZeroBlankRows().

Referenced by UnitTestTimeHeightConvolutionCompile().

353  {
355  input(input_indexes.size(), conv_model.InputDim(),
357  output(output_indexes.size(), conv_model.OutputDim(),
359  output_deriv(output_indexes.size(), conv_model.OutputDim(),
361  params(conv_model.ParamRows(), conv_model.ParamCols()),
362  params_deriv(conv_model.ParamRows(), conv_model.ParamCols());
363 
364  input.SetRandn();
365  params.SetRandn();
366  output_deriv.SetRandn();
367 
368  BaseFloat alpha = 0.5 * RandInt(1, 3);
369 
370  ZeroBlankRows(output_indexes, &output_deriv);
371  ZeroBlankRows(input_indexes, &input);
372 
373  ConvolveBackwardParams(computation, input, output_deriv, alpha,
374  &params_deriv);
375 
376  BaseFloat expected_objf = TraceMatMat(params_deriv, params, kTrans) / alpha;
377 
378  ConvolveForward(computation, input, params, &output);
379 
380  ZeroBlankRows(output_indexes, &output);
381 
382  BaseFloat observed_objf = TraceMatMat(output, output_deriv, kTrans);
383 
384  KALDI_LOG << "Expected objf = " << expected_objf
385  << ", observed objf = " << observed_objf;
386  if (!ApproxEqual(expected_objf, observed_objf, 0.1) &&
387  fabs(expected_objf) < 1.0) {
388  KALDI_ERR << "Difference in objf too large.";
389  }
390 }
void ConvolveBackwardParams(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &output_deriv, BaseFloat alpha, CuMatrixBase< BaseFloat > *params_deriv)
This does the part of the backward derivative computation of convolution, that computes derivatives w...
Definition: convolution.cc:840
float BaseFloat
Definition: kaldi-types.h:29
void ZeroBlankRows(const std::vector< Index > &indexes, CuMatrix< BaseFloat > *matrix)
#define KALDI_ERR
Definition: kaldi-error.h:127
Real TraceMatMat(const MatrixBase< Real > &A, const MatrixBase< Real > &B, MatrixTransposeType trans)
We need to declare this here as it will be a friend function.
#define KALDI_LOG
Definition: kaldi-error.h:133
void ConvolveForward(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &params, CuMatrixBase< BaseFloat > *output)
This does the forward computation of convolution.
Definition: convolution.cc:524
static bool ApproxEqual(float a, float b, float relative_tolerance=0.001)
return abs(a - b) <= relative_tolerance * (abs(a)+abs(b)).
Definition: kaldi-math.h:262
int32 RandInt(int32 min_val, int32 max_val, struct RandomState *state)
Definition: kaldi-math.cc:94
void kaldi::nnet3::time_height_convolution::TestRunningComputation ( const ConvolutionModel conv_model,
const std::vector< Index > &  input_indexes,
const std::vector< Index > &  output_indexes,
const ConvolutionComputation computation 
)

Definition at line 280 of file convolution-test.cc.

References ConvolveForward(), ConvolveForwardSimple(), ConvolutionModel::Info(), ConvolutionModel::InputDim(), KALDI_ERR, KALDI_LOG, kaldi::kSetZero, kaldi::kStrideEqualNumCols, ConvolutionModel::OutputDim(), ConvolutionModel::ParamCols(), ConvolutionModel::ParamRows(), CuMatrixBase< Real >::SetRandn(), and ZeroBlankRows().

Referenced by UnitTestTimeHeightConvolutionCompile().

283  {
284  CuMatrix<BaseFloat> input(input_indexes.size(), conv_model.InputDim(),
286  output(output_indexes.size(), conv_model.OutputDim(),
288  output2(output),
289  params(conv_model.ParamRows(), conv_model.ParamCols());
290  input.SetRandn();
291  params.SetRandn();
292  ZeroBlankRows(input_indexes, &input);
293  ConvolveForward(computation, input, params, &output);
294  ZeroBlankRows(output_indexes, &output);
295 
296  ConvolveForwardSimple(conv_model, input_indexes, output_indexes,
297  input, params, &output2);
298  KALDI_LOG << "Tested convolution for model: "
299  << conv_model.Info();
300  if (!output.ApproxEqual(output2, 0.001)) {
301  KALDI_LOG << "Output is: " << output;
302  KALDI_LOG << "Output2 is: " << output2;
303  KALDI_ERR << "Convolution test failure.";
304  }
305 }
void ZeroBlankRows(const std::vector< Index > &indexes, CuMatrix< BaseFloat > *matrix)
void ConvolveForwardSimple(const ConvolutionModel &model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const CuMatrixBase< BaseFloat > &input_cu, const CuMatrixBase< BaseFloat > &params_cu, CuMatrixBase< BaseFloat > *output_cu)
#define KALDI_ERR
Definition: kaldi-error.h:127
#define KALDI_LOG
Definition: kaldi-error.h:133
void ConvolveForward(const ConvolutionComputation &cc, const CuMatrixBase< BaseFloat > &input, const CuMatrixBase< BaseFloat > &params, CuMatrixBase< BaseFloat > *output)
This does the forward computation of convolution.
Definition: convolution.cc:524
static bool kaldi::nnet3::time_height_convolution::TimeValueInInput ( const ConvolutionComputationIo &  io,
int32  t 
)
static

Definition at line 1321 of file convolution.cc.

References ConvolutionComputationIo::num_t_in, ConvolutionComputationIo::start_t_in, and ConvolutionComputationIo::t_step_in.

Referenced by CheckModelAndIo().

1322  {
1323  int32 t_step_in = std::max<int32>(1, io.t_step_in);
1324  return (t >= io.start_t_in &&
1325  t < io.start_t_in + (t_step_in * io.num_t_in) &&
1326  (t - io.start_t_in) % t_step_in == 0);
1327 }
void kaldi::nnet3::time_height_convolution::UnitTestTimeHeightConvolution ( )
void kaldi::nnet3::time_height_convolution::UnitTestTimeHeightConvolutionCompile ( )

Definition at line 394 of file convolution-test.cc.

References CompileConvolutionComputation(), GetRandomConvolutionIndexes(), GetRandomConvolutionModel(), rnnlm::i, KALDI_LOG, TestComputationIo(), TestDataBackprop(), TestParamsBackprop(), TestRunningComputation(), and kaldi::nnet3::WriteIndexVector().

Referenced by UnitTestTimeHeightConvolution().

394  {
395  for (int32 i = 0; i < 10; i++) {
396  KALDI_LOG << "iter = " << i;
397  // Create a ConvolutionModel
398  ConvolutionModel conv_model;
399  GetRandomConvolutionModel(&conv_model);
400  std::vector<Index> input_indexes, output_indexes;
401  GetRandomConvolutionIndexes(conv_model, &input_indexes, &output_indexes);
402 
404  ConvolutionComputation computation;
405  std::vector<Index> input_indexes_modified, output_indexes_modified;
406  CompileConvolutionComputation(conv_model, input_indexes, output_indexes,
407  opts, &computation,
408  &input_indexes_modified,
409  &output_indexes_modified);
410  TestComputationIo(computation);
411  TestRunningComputation(conv_model,
412  input_indexes_modified,
413  output_indexes_modified,
414  computation);
415  TestDataBackprop(conv_model,
416  input_indexes_modified,
417  output_indexes_modified,
418  computation);
419  TestParamsBackprop(conv_model,
420  input_indexes_modified,
421  output_indexes_modified,
422  computation);
423  std::ostringstream os;
424  os << "\nInput-indexes: ";
425  WriteIndexVector(os, false, input_indexes);
426  os << "\nInput-indexes-modified: ";
427  WriteIndexVector(os, false, input_indexes_modified);
428  os << "\nOutput-indexes: ";
429  WriteIndexVector(os, false, output_indexes);
430  os << "\nOutput-indexes-modified: ";
431  WriteIndexVector(os, false, output_indexes_modified);
432  KALDI_LOG << os.str();
433  }
434 }
void WriteIndexVector(std::ostream &os, bool binary, const std::vector< Index > &vec)
Definition: nnet-common.cc:126
This comment explains the basic framework used for everything related to time-height convolution...
Definition: convolution.h:125
static void GetRandomConvolutionIndexes(const ConvolutionModel &model, std::vector< Index > *input_indexes, std::vector< Index > *output_indexes)
void TestRunningComputation(const ConvolutionModel &conv_model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputation &computation)
void TestComputationIo(const ConvolutionComputation &computation)
void CompileConvolutionComputation(const ConvolutionModel &model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputationOptions &opts, ConvolutionComputation *computation, std::vector< Index > *input_indexes_modified, std::vector< Index > *output_indexes_modified)
This function does the compilation for a convolution computation; it's a wrapper for the functions be...
This struct represents the structure of a convolution computation.
Definition: convolution.h:252
void TestParamsBackprop(const ConvolutionModel &conv_model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputation &computation)
This struct contains options for compiling the convolutional computation.
Definition: convolution.h:362
static void GetRandomConvolutionModel(ConvolutionModel *model)
void TestDataBackprop(const ConvolutionModel &conv_model, const std::vector< Index > &input_indexes, const std::vector< Index > &output_indexes, const ConvolutionComputation &computation)
#define KALDI_LOG
Definition: kaldi-error.h:133
void kaldi::nnet3::time_height_convolution::UnitTestTimeHeightConvolutionIo ( )

Definition at line 165 of file convolution-test.cc.

References GetRandomConvolutionModel(), rnnlm::i, KALDI_ASSERT, KALDI_LOG, kaldi::RandInt(), ConvolutionModel::Read(), and ConvolutionModel::Write().

Referenced by UnitTestTimeHeightConvolution().

165  {
166  for (int32 i = 0; i < 10; i++) {
167  KALDI_LOG << "iter = " << i;
168  // Create a ConvolutionModel and test its I/O.
169  ConvolutionModel conv_model;
170  GetRandomConvolutionModel(&conv_model);
171  std::ostringstream os1, os2;
172  bool binary = (RandInt(0, 1) == 0);
173  conv_model.Write(os1, binary);
174  std::istringstream is(os1.str());
175  ConvolutionModel conv_model2;
176  conv_model2.Read(is, binary);
177  conv_model2.Write(os2, binary);
178  KALDI_ASSERT(os1.str() == os2.str() && conv_model2.Check());
179  }
180 }
This comment explains the basic framework used for everything related to time-height convolution...
Definition: convolution.h:125
void Write(std::ostream &os, bool binary) const
Definition: convolution.cc:225
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static void GetRandomConvolutionModel(ConvolutionModel *model)
#define KALDI_LOG
Definition: kaldi-error.h:133
int32 RandInt(int32 min_val, int32 max_val, struct RandomState *state)
Definition: kaldi-math.cc:94
void UnPadModelHeight ( const ConvolutionComputationOptions &  opts,
const ConvolutionModel &  model,
const ConvolutionModel &  model_padded,
ConvolutionComputation *  computation 
)

This function modifies, if necessary, a computation that has been built for the model 'model_padded', so that it can work for the original model 'model'.

This may involve modifying the members 'height_in', 'temp_cols', and the column-related members of the elements of the 'steps' array. View it as the reverse step for 'PadModelHeight'.

This function has to be aware that the computation will have been compiled after 'AppendInputFrames()' was called [this makes a difference in setups with subsampling], so the computation may have been built for input frames that were appended over several of the frames that 'model_padded' would require.

This is the reverse step for stage 2 of compilation (it's a transformation of the computation).

Definition at line 1001 of file convolution.cc.

References ConvolutionComputation::Check(), ConvolutionComputation::ComputeDerived(), ComputeTempMatrixSize(), ConvolutionModel::height_in, ConvolutionComputation::height_in, ConvolutionComputation::ConvolutionStep::height_map, ConvolutionModel::height_out, ConvolutionComputation::height_out, rnnlm::i, KALDI_ASSERT, ConvolutionModel::offsets, and ConvolutionComputation::steps.

Referenced by CompileConvolutionComputation().

1004  {
1005  // First work out how much padding was done in PadModelHeight().
1006  int32 bottom_padding = (model_padded.offsets[0].height_offset -
1007  model.offsets[0].height_offset),
1008  total_padding = model_padded.height_in - model.height_in,
1009  top_padding = total_padding - bottom_padding;
1010 
1011  int32 old_computation_height_in = computation->height_in;
1012  // The computation may have been built for the input appended over
1013  // several frames. Check that it is for an input height that's a multiple of
1014  // the model input height.
1015  KALDI_ASSERT(old_computation_height_in % model_padded.height_in == 0 &&
1016  computation->height_out == model.height_out);
1017 
1018  // 'ratio' is the same ratio from AppendInputFrames(), it's the number
1019  // of input frames in 'model' and 'model_padded' that get appended
1020  // to form a single frame in the computation.
1021  int32 num_steps = computation->steps.size(),
1022  unpadded_input_height = model.height_in,
1023  padded_input_height = model_padded.height_in,
1024  ratio = old_computation_height_in / padded_input_height;
1025 
1026  computation->height_in = ratio * unpadded_input_height;
1027  for (int32 s = 0; s < num_steps; s++) {
1028  ConvolutionComputation::ConvolutionStep &step = computation->steps[s];
1029  int32 height_map_size = step.height_map.size();
1030  for (int32 i = 0; i < height_map_size; i++) {
1031  int32 c = step.height_map[i];
1032  KALDI_ASSERT(c >= 0); // there should be no -1's in the padded computation.
1033  // below, h is the actual height in terms of the padded computation, and m
1034  // is an index that goes from zero to (num-appended-frames - 1).
1035  int32 h = c % padded_input_height,
1036  m = c / padded_input_height;
1037  KALDI_ASSERT(m < ratio);
1038  if (h < bottom_padding || h >= padded_input_height - top_padding) {
1039  step.height_map[i] = -1;
1040  } else {
1041  step.height_map[i] = (h - bottom_padding) + m * unpadded_input_height;
1042  }
1043  }
1044  }
1045  ComputeTempMatrixSize(opts, computation);
1046  computation->ComputeDerived();
1047  computation->Check();
1048 }
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
static void ComputeTempMatrixSize(const ConvolutionComputationOptions &opts, ConvolutionComputation *computation)
This function sets 'temp_rows' and 'temp_cols' in 'computation'.
Definition: convolution.cc:956
static bool kaldi::nnet3::time_height_convolution::VectorIsContiguous ( const std::vector< int32 > &  vec)
static

Definition at line 77 of file convolution.cc.

References rnnlm::i, and KALDI_ASSERT.

Referenced by ConvolutionComputation::ComputeDerived(), and ComputeTempMatrixSize().

77  {
78  KALDI_ASSERT(!vec.empty());
79  int32 s = vec.size();
80  for (int32 i = 0; i + 1 < s; i++)
81  if (vec[i+1] != vec[i] + 1)
82  return false;
83  return true;
84 }
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
void kaldi::nnet3::time_height_convolution::ZeroBlankRows ( const std::vector< Index > &  indexes,
CuMatrix< BaseFloat > *  matrix 
)

Definition at line 198 of file convolution-test.cc.

References VectorBase< Real >::Data(), KALDI_ASSERT, kaldi::nnet3::kNoTime, kaldi::kUndefined, CuMatrixBase< Real >::MulRowsVec(), CuMatrixBase< Real >::NumRows(), VectorBase< Real >::Set(), and CuVector< Real >::Swap().

Referenced by TestDataBackprop(), TestParamsBackprop(), and TestRunningComputation().

199  {
200  KALDI_ASSERT(static_cast<int32>(indexes.size()) == matrix->NumRows());
201  int32 num_rows = matrix->NumRows();
202  if (num_rows == 0) return;
203  Vector<BaseFloat> mask(num_rows, kUndefined);
204  mask.Set(1.0);
205  const Index *indexes_ptr = &(indexes[0]);
206  BaseFloat *mask_ptr = mask.Data();
207  for (int32 r = 0; r < num_rows; r++) {
208  if (indexes_ptr[r].t == kNoTime)
209  mask_ptr[r] = 0.0;
210  }
211  CuVector<BaseFloat> cu_mask;
212  cu_mask.Swap(&mask);
213  matrix->MulRowsVec(cu_mask);
214 }
void Swap(Vector< Real > *vec)
Definition: cu-vector.cc:918
struct Index is intended to represent the various indexes by which we number the rows of the matrices...
Definition: nnet-common.h:44
void MulRowsVec(const CuVectorBase< Real > &scale)
scale i'th row by scale[i]
Definition: cu-matrix.cc:777
float BaseFloat
Definition: kaldi-types.h:29
MatrixIndexT NumRows() const
Dimensions.
Definition: cu-matrix.h:195
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
const int kNoTime
Definition: nnet-common.cc:554