subset-feats.cc File Reference

#include "base/kaldi-common.h"
#include "util/common-utils.h"
#include "matrix/kaldi-matrix.h"

Include dependency graph for subset-feats.cc:

Go to the source code of this file.

Functions
int32	CopyIncludedFeats (std::string filename, SequentialBaseFloatMatrixReader *kaldi_reader, BaseFloatMatrixWriter *kaldi_writer)
int32	CopyExcludedFeats (std::string filename, SequentialBaseFloatMatrixReader *kaldi_reader, BaseFloatMatrixWriter *kaldi_writer)
int	main (int argc, char *argv[])

Function Documentation

CopyExcludedFeats()

int32 CopyExcludedFeats	(	std::string	filename,
		SequentialBaseFloatMatrixReader *	kaldi_reader,
		BaseFloatMatrixWriter *	kaldi_writer
	)

Definition at line 57 of file subset-feats.cc.

References SequentialTableReader< Holder >::Done(), KALDI_ASSERT, KALDI_LOG, SequentialTableReader< Holder >::Key(), SequentialTableReader< Holder >::Next(), kaldi::SplitStringToVector(), Input::Stream(), SequentialTableReader< Holder >::Value(), and TableWriter< Holder >::Write().

Referenced by main().

                                                             {
  unordered_set<std::string, StringHasher> exclude_set;
  bool binary;
  Input ki(filename, &binary);
  KALDI_ASSERT(!binary);
  std::string line;
  while (std::getline(ki.Stream(), line)) {
    std::vector<std::string> split_line;
    SplitStringToVector(line, " \t\r", true, &split_line);
    KALDI_ASSERT(!split_line.empty() &&
        "Empty line encountered in input from --include option");
    exclude_set.insert(split_line[0]);
  }

  int32 num_total = 0;
  size_t num_success = 0;
  for (; !kaldi_reader->Done(); kaldi_reader->Next(), num_total++) {
    if (exclude_set.count(kaldi_reader->Key()) == 0) {
      kaldi_writer->Write(kaldi_reader->Key(), kaldi_reader->Value());
      num_success++;
    }
  }

  KALDI_LOG << " Wrote " << num_success << " out of " << num_total
            << " utterances.";
  return (num_success != 0 ? 0 : 1);
}

CopyIncludedFeats()

int32 CopyIncludedFeats	(	std::string	filename,
		SequentialBaseFloatMatrixReader *	kaldi_reader,
		BaseFloatMatrixWriter *	kaldi_writer
	)

Definition at line 27 of file subset-feats.cc.

References SequentialTableReader< Holder >::Done(), KALDI_ASSERT, KALDI_LOG, SequentialTableReader< Holder >::Key(), SequentialTableReader< Holder >::Next(), kaldi::SplitStringToVector(), Input::Stream(), SequentialTableReader< Holder >::Value(), and TableWriter< Holder >::Write().

Referenced by main().

                                                             {
  unordered_set<std::string, StringHasher> include_set;
  bool binary;
  Input ki(filename, &binary);
  KALDI_ASSERT(!binary);
  std::string line;
  while (std::getline(ki.Stream(), line)) {
    std::vector<std::string> split_line;
    SplitStringToVector(line, " \t\r", true, &split_line);
    KALDI_ASSERT(!split_line.empty() &&
        "Empty line encountered in input from --include option");
    include_set.insert(split_line[0]);
  }

  int32 num_total = 0;
  size_t num_success = 0;
  for (; !kaldi_reader->Done(); kaldi_reader->Next(), num_total++) {
    if (include_set.count(kaldi_reader->Key()) > 0) {
      kaldi_writer->Write(kaldi_reader->Key(), kaldi_reader->Value());
      num_success++;
    }
  }

  KALDI_LOG << " Wrote " << num_success << " out of " << num_total
            << " utterances.";
  return (num_success != 0 ? 0 : 1);
}

main()

int main	(	int	argc,
		char *	argv[]
	)

Definition at line 87 of file subset-feats.cc.

References CopyExcludedFeats(), CopyIncludedFeats(), SequentialTableReader< Holder >::Done(), ParseOptions::GetArg(), KALDI_ASSERT, KALDI_ERR, SequentialTableReader< Holder >::Key(), rnnlm::n, SequentialTableReader< Holder >::Next(), ParseOptions::NumArgs(), ParseOptions::PrintUsage(), ParseOptions::Read(), ParseOptions::Register(), SequentialTableReader< Holder >::Value(), and TableWriter< Holder >::Write().

                                 {
  try {
    using namespace kaldi;

    const char *usage =
        "Copy a subset of features (by default, the first n feature files)\n"
        "Usually used where only a small amount of data is needed\n"
        "Note: if you want a specific subset, it's usually best to\n"
        "filter the original .scp file with utils/filter_scp.pl\n"
        "(possibly with the --exclude option).  The --include and --exclude\n"
        "options of this program are intended for specialized uses.\n"
        "The --include and --exclude options are mutually exclusive, \n"
        "and both cause the --n option to be ignored.\n"
        "Usage: subset-feats [options] <in-rspecifier> <out-wspecifier>\n"
        "e.g.: subset-feats --n=10 ark:- ark:-\n"
        "or:  subset-feats --include=include_uttlist ark:- ark:-\n"
        "or:  subset-feats --exclude=exclude_uttlist ark:- ark:-\n"
        "See also extract-feature-segments, select-feats, subsample-feats\n";

    ParseOptions po(usage);

    int32 n = 10;
    std::string include_rxfilename;
    std::string exclude_rxfilename;
    po.Register("n", &n, "If nonnegative, copy the first n feature files.");
    po.Register("include", &include_rxfilename,
                        "Text file, the first field of each"
                        " line being interpreted as an "
                        "utterance-id whose features will be included");
    po.Register("exclude", &exclude_rxfilename,
                        "Text file, the first field of each "
                        "line being interpreted as an utterance-id"
                        " whose features will be excluded");

    po.Read(argc, argv);

    if (po.NumArgs() != 2) {
      po.PrintUsage();
      exit(1);
    }

    std::string rspecifier = po.GetArg(1);
    std::string wspecifier = po.GetArg(2);

    KALDI_ASSERT(n >= 0);

    BaseFloatMatrixWriter kaldi_writer(wspecifier);
    SequentialBaseFloatMatrixReader kaldi_reader(rspecifier);

    if (include_rxfilename != "") {
      if (n != 10) {
        KALDI_ERR << "Should not have both --include and --n option!";
      }
      if (exclude_rxfilename != "") {
        KALDI_ERR << "should not have both --exclude and --include option!";
      }
      return CopyIncludedFeats(include_rxfilename,
                               &kaldi_reader, &kaldi_writer);
    }
    else if (exclude_rxfilename != "") {
      if (n != 10) {
        KALDI_ERR << "Should not have both --exclude and --n option!";
      }
      return CopyExcludedFeats(exclude_rxfilename,
                               &kaldi_reader, &kaldi_writer);
    }

    if (n == 0) {
      KALDI_ERR << "Invalid option --n=0. Should be at least 1";
    }

    int32 k = 0;
    for (; !kaldi_reader.Done() && k < n; kaldi_reader.Next(), k++)
      kaldi_writer.Write(kaldi_reader.Key(), kaldi_reader.Value());

    return 0;
  } catch(const std::exception &e) {
    std::cerr << e.what();
    return -1;
  }
}