nnet3-discriminative-shuffle-egs.cc File Reference

#include "base/kaldi-common.h"
#include "util/common-utils.h"
#include "hmm/transition-model.h"
#include "nnet3/nnet-discriminative-example.h"

Include dependency graph for nnet3-discriminative-shuffle-egs.cc:

Go to the source code of this file.

Functions
int	main (int argc, char *argv[])

Function Documentation

main()

int main	(	int	argc,
		char *	argv[]
	)

Definition at line 26 of file nnet3-discriminative-shuffle-egs.cc.

References SequentialTableReader< Holder >::Done(), ParseOptions::GetArg(), rnnlm::i, KALDI_ASSERT, KALDI_LOG, SequentialTableReader< Holder >::Key(), SequentialTableReader< Holder >::Next(), ParseOptions::NumArgs(), ParseOptions::PrintUsage(), kaldi::RandInt(), ParseOptions::Read(), ParseOptions::Register(), SequentialTableReader< Holder >::Value(), and TableWriter< Holder >::Write().

                                 {
  try {
    using namespace kaldi;
    using namespace kaldi::nnet3;
    typedef kaldi::int32 int32;
    typedef kaldi::int64 int64;

    const char *usage =
        "Copy nnet3 discriminative training examples from the input to output,\n"
        "while randomly shuffling the order.  This program will keep all of the examples\n"
        "in memory at once, unless you use the --buffer-size option\n"
        "\n"
        "Usage:  nnet3-discriminative-shuffle-egs [options] <egs-rspecifier> <egs-wspecifier>\n"
        "\n"
        "nnet3-discriminative-shuffle-egs --srand=1 ark:train.egs ark:shuffled.egs\n";

    int32 srand_seed = 0;
    int32 buffer_size = 0;
    ParseOptions po(usage);
    po.Register("srand", &srand_seed, "Seed for random number generator ");
    po.Register("buffer-size", &buffer_size, "If >0, size of a buffer we use "
                "to do limited-memory partial randomization.  Otherwise, do "
                "full randomization.");

    po.Read(argc, argv);

    srand(srand_seed);

    if (po.NumArgs() != 2) {
      po.PrintUsage();
      exit(1);
    }

    std::string examples_rspecifier = po.GetArg(1),
        examples_wspecifier = po.GetArg(2);

    int64 num_done = 0;

    std::vector<std::pair<std::string, NnetDiscriminativeExample*> > egs;

    SequentialNnetDiscriminativeExampleReader example_reader(examples_rspecifier);
    NnetDiscriminativeExampleWriter example_writer(examples_wspecifier);
    if (buffer_size == 0) { // Do full randomization
      // Putting in an extra level of indirection here to avoid excessive
      // computation and memory demands when we have to resize the vector.

      for (; !example_reader.Done(); example_reader.Next())
        egs.push_back(std::pair<std::string, NnetDiscriminativeExample*>(
            example_reader.Key(),
            new NnetDiscriminativeExample(example_reader.Value())));

      std::random_shuffle(egs.begin(), egs.end());
    } else {
      KALDI_ASSERT(buffer_size > 0);
      egs.resize(buffer_size,
                 std::pair<std::string, NnetDiscriminativeExample*>("", NULL));
      for (; !example_reader.Done(); example_reader.Next()) {
        int32 index = RandInt(0, buffer_size - 1);
        if (egs[index].second == NULL) {
          egs[index] = std::pair<std::string, NnetDiscriminativeExample*>(
              example_reader.Key(),
              new NnetDiscriminativeExample(example_reader.Value()));
        } else {
          example_writer.Write(egs[index].first, *(egs[index].second));
          egs[index].first = example_reader.Key();
          *(egs[index].second) = example_reader.Value();
          num_done++;
        }
      }
    }
    for (size_t i = 0; i < egs.size(); i++) {
      if (egs[i].second != NULL) {
        example_writer.Write(egs[i].first, *(egs[i].second));
        delete egs[i].second;
        num_done++;
      }
    }

    KALDI_LOG << "Shuffled order of " << num_done
              << " neural-network training examples "
              << (buffer_size ? "using a buffer (partial randomization)" : "");

    return (num_done == 0 ? 1 : 0);
  } catch(const std::exception &e) {
    std::cerr << e.what() << '\n';
    return -1;
  }
}