"Table types and related functions"

This group is for classes and functions relatied to Tables; see also "Implementation classes for Table types" and "Specific Table types", and for a description of the Table concept see The Table concept. More...

Classes
struct	WspecifierOptions
struct	RspecifierOptions
class	RandomAccessTableReader< Holder >
	Allows random access to a collection of objects in an archive or script file; see The Table concept. More...
class	SequentialTableReader< Holder >
	A templated class for reading objects sequentially from an archive or script file; see The Table concept. More...
class	TableWriter< Holder >
	A templated class for writing objects to an archive or script file; see The Table concept. More...
class	RandomAccessTableReaderMapped< Holder >
	This class is for when you are reading something in random access, but it may actually be stored per-speaker (or something similar) but the keys you're using are per utterance. More...

Typedefs
typedef std::vector< std::string >	KeyList

Enumerations
enum	WspecifierType { kNoWspecifier, kArchiveWspecifier, kScriptWspecifier, kBothWspecifier }
enum	RspecifierType { kNoRspecifier, kArchiveRspecifier, kScriptRspecifier }

Functions
WspecifierType	ClassifyWspecifier (const std::string &wspecifier, std::string *archive_wxfilename, std::string *script_wxfilename, WspecifierOptions *opts)
bool	ReadScriptFile (const std::string &rxfilename, bool warn, std::vector< std::pair< std::string, std::string > > *script_out)
bool	ReadScriptFile (std::istream &is, bool warn, std::vector< std::pair< std::string, std::string > > *script_out)
bool	WriteScriptFile (const std::string &wxfilename, const std::vector< std::pair< std::string, std::string > > &script)
bool	WriteScriptFile (std::ostream &os, const std::vector< std::pair< std::string, std::string > > &script)
RspecifierType	ClassifyRspecifier (const std::string &rspecifier, std::string *rxfilename, RspecifierOptions *opts)

Detailed Description

This group is for classes and functions relatied to Tables; see also "Implementation classes for Table types" and "Specific Table types", and for a description of the Table concept see The Table concept.

Typedef Documentation

KeyList

typedef std::vector<std::string> KeyList

Definition at line 54 of file kaldi-table.h.

Enumeration Type Documentation

RspecifierType

enum RspecifierType

Enumerator
kNoRspecifier
kArchiveRspecifier
kScriptRspecifier

Definition at line 219 of file kaldi-table.h.

                     {
  kNoRspecifier,
  kArchiveRspecifier,
  kScriptRspecifier
};

WspecifierType

enum WspecifierType

Enumerator
kNoWspecifier
kArchiveWspecifier
kScriptWspecifier
kBothWspecifier

Definition at line 106 of file kaldi-table.h.

                     {
  kNoWspecifier,
  kArchiveWspecifier,
  kScriptWspecifier,
  kBothWspecifier
};

Function Documentation

ClassifyRspecifier()

RspecifierType ClassifyRspecifier	(	const std::string &	rspecifier,
		std::string *	rxfilename,
		RspecifierOptions *	opts
	)

Definition at line 225 of file kaldi-table.cc.

References RspecifierOptions::background, RspecifierOptions::called_sorted, rnnlm::i, kaldi::kArchiveRspecifier, kaldi::kNoRspecifier, kaldi::kScriptRspecifier, RspecifierOptions::once, RspecifierOptions::permissive, RspecifierOptions::sorted, and kaldi::SplitStringToVector().

Referenced by kaldi::ClassifyRxfilename(), kaldi::ClassifyWxfilename(), main(), SequentialTableReaderScriptImpl< Holder >::Open(), RandomAccessTableReader< kaldi::TokenHolder >::Open(), SequentialTableReader< Holder >::Open(), SequentialTableReaderArchiveImpl< Holder >::Open(), RandomAccessTableReaderScriptImpl< Holder >::Open(), RandomAccessTableReaderArchiveImplBase< Holder >::Open(), kaldi::SplitArgOnEquals(), kaldi::TypeThreeUsage(), kaldi::TypeTwoUsage(), and kaldi::UnitTestClassifyRspecifier().

                                                           {
  // Examples
  // ark:rxfilename  ->  kArchiveRspecifier
  // scp:rxfilename  -> kScriptRspecifier
  //
  // We also allow the meaningless prefixes b, and t,
  // plus the options o (once), no (not-once),
  // s (sorted) and ns (not-sorted), p (permissive)
  // and np (not-permissive).
  // so the following would be valid:
  //
  // f, o, b, np, ark:rxfilename  ->  kArchiveRspecifier
  //
  // Examples:
  //
  // b, ark:rxfilename  ->  kArchiveRspecifier
  // t, ark:rxfilename  ->  kArchiveRspecifier
  // b, scp:rxfilename  -> kScriptRspecifier
  // t, no, s, scp:rxfilename  -> kScriptRspecifier
  // t, ns, scp:rxfilename  -> kScriptRspecifier

  // Improperly formed Rspecifiers will be classified as kNoRspecifier.

  if (rxfilename) rxfilename->clear();

  if (opts != NULL)
    *opts = RspecifierOptions();  // Make sure all the defaults are as in the
                                  // default constructor of the options class.

  size_t pos = rspecifier.find(':');
  if (pos == std::string::npos) return kNoRspecifier;

  if (isspace(*(rspecifier.rbegin()))) return kNoRspecifier;  // Trailing space
  // disallowed.

  std::string before_colon(rspecifier, 0, pos),
      after_colon(rspecifier, pos+1);

  std::vector<std::string> split_first_part;  // Split part before ':' on ', '.
  SplitStringToVector(before_colon, ", ", false, &split_first_part);  // false==
  // don't omit empty strings between commas.

  RspecifierType rs = kNoRspecifier;

  for (size_t i = 0; i < split_first_part.size(); i++) {
    const std::string &str = split_first_part[i];  // e.g. "b", "t", "f", "ark",
    // "scp".
    const char *c = str.c_str();
    if (!strcmp(c, "b"));  // Ignore this option.  It's so we can use the same
    // specifiers for rspecifiers and wspecifiers.
    else if (!strcmp(c, "t"));  // Ignore this option too.
    else if (!strcmp(c, "o")) {
      if (opts) opts->once = true;
    } else if (!strcmp(c, "no")) {
      if (opts) opts->once = false;
    } else if (!strcmp(c, "p")) {
      if (opts) opts->permissive = true;
    } else if (!strcmp(c, "np")) {
      if (opts) opts->permissive = false;
    } else if (!strcmp(c, "s")) {
      if (opts) opts->sorted = true;
    } else if (!strcmp(c, "ns")) {
      if (opts) opts->sorted = false;
    } else if (!strcmp(c, "cs")) {
      if (opts) opts->called_sorted = true;
    } else if (!strcmp(c, "ncs")) {
      if (opts) opts->called_sorted = false;
    } else if (!strcmp(c, "bg")) {
      if (opts) opts->background = true;
    } else if (!strcmp(c, "ark")) {
      if (rs == kNoRspecifier) rs = kArchiveRspecifier;
      else
        return kNoRspecifier;  // Repeated or combined ark and scp options
      // invalid.
    } else if (!strcmp(c, "scp")) {
      if (rs == kNoRspecifier) rs = kScriptRspecifier;
      else
        return kNoRspecifier;  // Repeated or combined ark and scp options
      // invalid.
    } else {
      return kNoRspecifier;  // Could not interpret this option.
    }
  }
  if ((rs == kArchiveRspecifier || rs == kScriptRspecifier)
     && rxfilename != NULL)
    *rxfilename = after_colon;
  return rs;
}

ClassifyWspecifier()

WspecifierType ClassifyWspecifier	(	const std::string &	wspecifier,
		std::string *	archive_wxfilename,
		std::string *	script_wxfilename,
		WspecifierOptions *	opts
	)

Definition at line 135 of file kaldi-table.cc.

References WspecifierOptions::binary, WspecifierOptions::flush, rnnlm::i, kaldi::kArchiveWspecifier, kaldi::kBothWspecifier, kaldi::kNoWspecifier, kaldi::kScriptWspecifier, WspecifierOptions::permissive, and kaldi::SplitStringToVector().

Referenced by kaldi::ClassifyRxfilename(), kaldi::ClassifyWxfilename(), main(), TableWriter< Holder >::Open(), TableWriterArchiveImpl< Holder >::Open(), TableWriterScriptImpl< Holder >::Open(), TableWriterBothImpl< Holder >::Open(), kaldi::TypeThreeUsage(), kaldi::TypeTwoUsage(), kaldi::UnitTestClassifyWspecifier(), and WspecifierOptions::WspecifierOptions().

                                                           {
  //  Examples:
  //  ark,t:wxfilename -> kArchiveWspecifier
  //  ark,b:wxfilename -> kArchiveWspecifier
  //  scp,t:rxfilename -> kScriptWspecifier
  //  scp,t:rxfilename -> kScriptWspecifier
  //  ark,scp,t:filename, wxfilename -> kBothWspecifier
  //  ark,scp:filename, wxfilename ->  kBothWspecifier
  //  Note we can include the flush option (f) or no-flush (nf)
  // anywhere: e.g.
  //  ark,scp,f:filename, wxfilename ->  kBothWspecifier
  // or:
  //  scp,t,nf:rxfilename -> kScriptWspecifier

  if (archive_wxfilename) archive_wxfilename->clear();
  if (script_wxfilename) script_wxfilename->clear();

  size_t pos = wspecifier.find(':');
  if (pos == std::string::npos) return kNoWspecifier;
  if (isspace(*(wspecifier.rbegin()))) return kNoWspecifier;  // Trailing space
  // disallowed.

  std::string before_colon(wspecifier, 0, pos), after_colon(wspecifier, pos+1);

  std::vector<std::string> split_first_part;  // Split part before ':' on ', '.
  SplitStringToVector(before_colon, ", ", false, &split_first_part);  // false==
  // don't omit empty strings between commas.

  WspecifierType ws = kNoWspecifier;

  if (opts != NULL)
    *opts = WspecifierOptions();  // Make sure all the defaults are as in the
                                  // default constructor of the options class.

  for (size_t i = 0; i < split_first_part.size(); i++) {
    const std::string &str = split_first_part[i];  // e.g. "b", "t", "f", "ark",
    // "scp".
    const char *c = str.c_str();
    if (!strcmp(c, "b")) {
      if (opts) opts->binary = true;
    } else if (!strcmp(c, "f")) {
      if (opts) opts->flush = true;
    } else if (!strcmp(c, "nf")) {
      if (opts) opts->flush = false;
    } else if (!strcmp(c, "t")) {
      if (opts) opts->binary = false;
    } else if (!strcmp(c, "p")) {
      if (opts) opts->permissive = true;
    } else if (!strcmp(c, "ark")) {
      if (ws == kNoWspecifier) ws = kArchiveWspecifier;
      else
        return kNoWspecifier;  // We do not allow "scp, ark", only "ark,
      // scp".
    } else if (!strcmp(c, "scp")) {
      if (ws == kNoWspecifier) ws = kScriptWspecifier;
      else if (ws == kArchiveWspecifier) ws = kBothWspecifier;
      else
        return kNoWspecifier;  // repeated "scp" option: invalid.
    } else {
      return kNoWspecifier;  // Could not interpret this option.
    }
  }

  switch (ws) {
    case kArchiveWspecifier:
      if (archive_wxfilename)
        *archive_wxfilename = after_colon;
      break;
    case kScriptWspecifier:
      if (script_wxfilename)
        *script_wxfilename = after_colon;
      break;
    case kBothWspecifier:
      pos = after_colon.find(',');  // first comma.
      if (pos == std::string::npos) return kNoWspecifier;
      if (archive_wxfilename)
        *archive_wxfilename = std::string(after_colon, 0, pos);
      if (script_wxfilename)
        *script_wxfilename = std::string(after_colon, pos+1);
      break;
    case kNoWspecifier: default: break;
  }
  return ws;
}

ReadScriptFile() [1/2]

bool ReadScriptFile	(	const std::string &	rxfilename,
		bool	warn,
		std::vector< std::pair< std::string, std::string > > *	script_out
	)

Definition at line 26 of file kaldi-table.cc.

References KALDI_WARN, Input::Open(), kaldi::PrintableRxfilename(), and Input::Stream().

Referenced by TableWriterScriptImpl< Holder >::Open(), RandomAccessTableReaderScriptImpl< Holder >::Open(), kaldi::UnitTestReadScriptFile(), kaldi::UnitTestTableSequentialInt32Script(), and WspecifierOptions::WspecifierOptions().

                                 {
  bool is_binary;
  Input input;

  if (!input.Open(rxfilename, &is_binary)) {
    if (warn) KALDI_WARN << "Error opening script file: " <<
                 PrintableRxfilename(rxfilename);
    return false;
  }
  if (is_binary) {
    if (warn) KALDI_WARN << "Error: script file appears to be binary: " <<
                 PrintableRxfilename(rxfilename);
    return false;
  }

  bool ans = ReadScriptFile(input.Stream(), warn, script_out);
  if (warn && !ans)
    KALDI_WARN << "[script file was: " << PrintableRxfilename(rxfilename) <<
                  "]";
  return ans;
}

ReadScriptFile() [2/2]

bool ReadScriptFile	(	std::istream &	is,
		bool	warn,
		std::vector< std::pair< std::string, std::string > > *	script_out
	)

Definition at line 51 of file kaldi-table.cc.

References KALDI_ASSERT, KALDI_WARN, line_number, and kaldi::SplitStringOnFirstSpace().

                                 {
  KALDI_ASSERT(script_out != NULL);
  std::string line;
  int line_number = 0;
  while (getline(is, line)) {
    line_number++;
    const char *c = line.c_str();
    if (*c == '\0') {
      if (warn)
        KALDI_WARN << "Empty " << line_number << "'th line in script file";
      return false;  // Empty line so invalid scp file format..
    }

    std::string key, rest;
    SplitStringOnFirstSpace(line, &key, &rest);

    if (key.empty() || rest.empty()) {
      if (warn)
        KALDI_WARN << "Invalid " << line_number << "'th line in script file"
                          <<":\"" << line << '"';
      return false;
    }
    script_out->resize(script_out->size()+1);
    script_out->back().first = key;
    script_out->back().second = rest;
  }
  return true;
}

WriteScriptFile() [1/2]

bool WriteScriptFile	(	std::ostream &	os,
		const std::vector< std::pair< std::string, std::string > > &	script
	)

Definition at line 83 of file kaldi-table.cc.

References kaldi::IsToken(), and KALDI_WARN.

Referenced by kaldi::UnitTestTableSequentialInt32Script(), kaldi::WriteScriptFile(), and WspecifierOptions::WspecifierOptions().

                              {
  if (!os.good()) {
    KALDI_WARN << "WriteScriptFile: attempting to write to invalid stream.";
    return false;
  }
  std::vector<std::pair<std::string, std::string> >::const_iterator iter;
  for (iter = script.begin(); iter != script.end(); ++iter) {
    if (!IsToken(iter->first)) {
      KALDI_WARN << "WriteScriptFile: using invalid token \"" << iter->first <<
                    '"';
      return false;
    }
    if (iter->second.find('\n') != std::string::npos ||
       (iter->second.length() != 0 &&
        (isspace(iter->second[0]) ||
         isspace(iter->second[iter->second.length()-1])))) {
      // second part contains newline or leading or trailing space.
      KALDI_WARN << "WriteScriptFile: attempting to write invalid line \"" <<
                    iter->second << '"';
      return false;
    }
    os << iter->first << ' ' << iter->second << '\n';
  }
  if (!os.good()) {
    KALDI_WARN << "WriteScriptFile: stream in error state.";
    return false;
  }
  return true;
}

WriteScriptFile() [2/2]

bool WriteScriptFile	(	const std::string &	wxfilename,
		const std::vector< std::pair< std::string, std::string > > &	script
	)

Definition at line 115 of file kaldi-table.cc.

References KALDI_ERR, Output::Open(), kaldi::PrintableWxfilename(), Output::Stream(), and kaldi::WriteScriptFile().

                              {
  Output output;
  if (!output.Open(wxfilename, false, false)) {  // false, false means not
    // binary, no binary-mode header.
    KALDI_ERR << "Error opening output stream for script file: "
              << PrintableWxfilename(wxfilename);
    return false;
  }
  if (!WriteScriptFile(output.Stream(), script)) {
    KALDI_ERR << "Error writing script file to stream "
              << PrintableWxfilename(wxfilename);
    return false;
  }
  return true;
}