This group contains the Input and Output classes, which are provided to open streams for reading and writing in Kaldi code; for an explanation of how this fits into the bigger picture of Kaldi I/O, see How to open files in Kaldi. More...

Classes
class	Output

class	Input

Enumerations
enum	OutputType { kNoOutput, kFileOutput, kStandardOutput, kPipeOutput }

enum	InputType { kNoInput, kFileInput, kStandardInput, kOffsetFileInput, kPipeInput }

Functions
OutputType	ClassifyWxfilename (const std::string &wxfilename)
	ClassifyWxfilename interprets filenames as follows: More...

InputType	ClassifyRxfilename (const std::string &rxfilename)
	ClassifyRxfilenames interprets filenames for reading as follows: More...

template<class C >
void	ReadKaldiObject (const std::string &filename, C *c)

template<>
void	ReadKaldiObject (const std::string &filename, Matrix< float > *m)

template<>
void	ReadKaldiObject (const std::string &filename, Matrix< double > *m)

template<class C >
void	WriteKaldiObject (const C &c, const std::string &filename, bool binary)

std::string	PrintableRxfilename (const std::string &rxfilename)
	PrintableRxfilename turns the rxfilename into a more human-readable form for error reporting, i.e. More...

std::string	PrintableWxfilename (const std::string &wxfilename)
	PrintableWxfilename turns the wxfilename into a more human-readable form for error reporting, i.e. More...

Detailed Description

This group contains the Input and Output classes, which are provided to open streams for reading and writing in Kaldi code; for an explanation of how this fits into the bigger picture of Kaldi I/O, see How to open files in Kaldi.

Enumeration Type Documentation

◆ InputType

enum InputType

Enumerator
kNoInput
kFileInput
kStandardInput
kOffsetFileInput
kPipeInput

Definition at line 105 of file kaldi-io.h.

                {
   kNoInput,
   kFileInput,
   kStandardInput,
   kOffsetFileInput,
   kPipeInput
 };

◆ OutputType

enum OutputType

Enumerator
kNoOutput
kFileOutput
kStandardOutput
kPipeOutput

Definition at line 89 of file kaldi-io.h.

                 {
   kNoOutput,
   kFileOutput,
   kStandardOutput,
   kPipeOutput
 };

Function Documentation

◆ ClassifyRxfilename()

InputType ClassifyRxfilename ( const std::string & rxfilename )

ClassifyRxfilenames interprets filenames for reading as follows:

kNoInput: invalid filenames (leading or trailing space, things that look like wspecifiers and rspecifiers or pipes to write to with trailing |.
kFileInput: normal filenames
kStandardInput: the empty string or "-"
kPipeInput: e.g. "gunzip -c /tmp/abc.gz |"
kOffsetFileInput: offsets into files, e.g. /some/filename:12970

Definition at line 138 of file kaldi-io.cc.

References kaldi::ClassifyRspecifier(), kaldi::ClassifyWspecifier(), rnnlm::d, KALDI_WARN, kaldi::kFileInput, kaldi::kNoInput, kaldi::kNoRspecifier, kaldi::kNoWspecifier, kaldi::kOffsetFileInput, kaldi::kPipeInput, and kaldi::kStandardInput.

Referenced by Input::OpenInternal(), and kaldi::UnitTestClassifyRxfilename().

                                                         {
   const char *c = filename.c_str();
   size_t length = filename.length();
   char first_char = c[0],
       last_char = (length == 0 ? '\0' : c[filename.length()-1]);
 
   // if 'filename' is "" or "-", return kStandardInput.
   if (length == 0 || (length == 1 && first_char == '-')) {
     return kStandardInput;
   } else if (first_char == '|') {
     return kNoInput;  // An output pipe like "|blah": not
                       // valid for input.
   } else if (last_char == '|') {
     return kPipeInput;
   } else if (isspace(first_char) || isspace(last_char)) {
     return kNoInput;  // We don't allow leading or trailing space in a filename.
   } else if ((first_char == 'a' || first_char == 's') &&
              strchr(c, ':') != NULL &&
             (ClassifyWspecifier(filename, NULL, NULL, NULL) != kNoWspecifier ||
              ClassifyRspecifier(filename, NULL, NULL) != kNoRspecifier)) {
     // e.g. ark:something or scp:something... this is almost certainly a
     // scripting error, so call it an error rather than treating it as a file.
     // In practice in modern kaldi scripts all (r,w)filenames begin with "ark"
     // or "scp", even though technically speaking options like "b", "t", "s" or
     // "cs" can appear before the ark or scp, like "b,ark".  For efficiency,
     // and because this code is really just a nicety to catch errors earlier
     // than they would otherwise be caught, we only call those extra functions
     // for filenames beginning with 'a' or 's'.
     return kNoInput;
   } else if (isdigit(last_char)) {
     const char *d = c + length - 1;
     while (isdigit(*d) && d > c) d--;
     if (*d == ':') return kOffsetFileInput;  // Filename is like
                                              // some_file:12345
     // otherwise it could still be a filename; continue to the next check.
   }
 
 
   // At this point it matched no other pattern so we assume a filename, but
   // we check for '|' as it's a common source of errors to have pipe
   // commands without the pipe in the right place.  Say that it can't be
   // classified in this case.
   if (strchr(c, '|') != NULL) {
     KALDI_WARN << "Trying to classify rxfilename with pipe symbol in the"
         " wrong place (pipe without | at the end?): " << filename;
     return kNoInput;
   }
   return kFileInput;  // It matched no other pattern: assume it's a filename.
 }

◆ ClassifyWxfilename()

OutputType ClassifyWxfilename ( const std::string & wxfilename )

ClassifyWxfilename interprets filenames as follows:

kNoOutput: invalid filenames (leading or trailing space, things that look like wspecifiers and rspecifiers or like pipes to read from with leading |.
kFileOutput: Normal filenames
kStandardOutput: The empty string or "-", interpreted as standard output
kPipeOutput: pipes, e.g. "| gzip -c > /tmp/abc.gz"

Definition at line 85 of file kaldi-io.cc.

References kaldi::ClassifyRspecifier(), kaldi::ClassifyWspecifier(), rnnlm::d, KALDI_WARN, kaldi::kFileOutput, kaldi::kNoOutput, kaldi::kNoRspecifier, kaldi::kNoWspecifier, kaldi::kPipeOutput, and kaldi::kStandardOutput.

Referenced by Output::Open(), TableWriterBothImpl< Holder >::Open(), kaldi::UnitTestClassifyWxfilename(), and Output::~Output().

                                                          {
   const char *c = filename.c_str();
   size_t length = filename.length();
   char first_char = c[0],
       last_char = (length == 0 ? '\0' : c[filename.length()-1]);
 
   // if 'filename' is "" or "-", return kStandardOutput.
   if (length == 0 || (length == 1 && first_char == '-'))
     return kStandardOutput;
   else if (first_char == '|') return kPipeOutput;  // An output pipe like "|blah".
   else if (isspace(first_char) || isspace(last_char) || last_char == '|') {
       return kNoOutput;  // Leading or trailing space: can't interpret this.
                          // Final '|' would represent an input pipe, not an
                          // output pipe.
   } else if ((first_char == 'a' || first_char == 's') &&
              strchr(c, ':') != NULL &&
              (ClassifyWspecifier(filename, NULL, NULL, NULL) != kNoWspecifier ||
               ClassifyRspecifier(filename, NULL, NULL) != kNoRspecifier)) {
     // e.g. ark:something or scp:something... this is almost certainly a
     // scripting error, so call it an error rather than treating it as a file.
     // In practice in modern kaldi scripts all (r,w)filenames begin with "ark"
     // or "scp", even though technically speaking options like "b", "t", "s" or
     // "cs" can appear before the ark or scp, like "b,ark".  For efficiency,
     // and because this code is really just a nicety to catch errors earlier
     // than they would otherwise be caught, we only call those extra functions
     // for filenames beginning with 'a' or 's'.
     return kNoOutput;
   } else if (isdigit(last_char)) {
     // This could be a file, but we have to see if it's an offset into a file
     // (like foo.ark:4314328), which is not allowed for writing (but is
     // allowed for reaching).  This eliminates some things which would be
     // valid UNIX filenames but are not allowed by Kaldi.  (Even if we allowed
     // such filenames for writing, we woudln't be able to correctly read them).
     const char *d = c + length - 1;
     while (isdigit(*d) && d > c) d--;
     if (*d == ':') return kNoOutput;
     // else it could still be a filename; continue to the next check.
   }
 
   // At this point it matched no other pattern so we assume a filename, but we
   // check for internal '|' as it's a common source of errors to have pipe
   // commands without the pipe in the right place.  Say that it can't be
   // classified.
   if (strchr(c, '|') != NULL) {
     KALDI_WARN << "Trying to classify wxfilename with pipe symbol in the"
         " wrong place (pipe without | at the beginning?): " <<
         filename;
     return kNoOutput;
   }
   return kFileOutput;  // It matched no other pattern: assume it's a filename.
 }

◆ PrintableRxfilename()

std::string PrintableRxfilename ( const std::string & rxfilename )

PrintableRxfilename turns the rxfilename into a more human-readable form for error reporting, i.e.

it does quoting and escaping and replaces "" or "-" with "standard input".

Definition at line 61 of file kaldi-io.cc.

References ParseOptions::Escape().

                                                            {
   if (rxfilename == "" || rxfilename == "-") {
     return "standard input";
   } else {
     // If this call to Escape later causes compilation issues,
     // just replace it with "return rxfilename"; it's only a
     // pretty-printing issue.
     return ParseOptions::Escape(rxfilename);
   }
 }

◆ PrintableWxfilename()

std::string PrintableWxfilename ( const std::string & wxfilename )

PrintableWxfilename turns the wxfilename into a more human-readable form for error reporting, i.e.

it does quoting and escaping and replaces "" or "-" with "standard output".

Definition at line 73 of file kaldi-io.cc.

References ParseOptions::Escape().

Referenced by main(), Output::Open(), Output::Output(), kaldi::TypeThreeUsage(), kaldi::TypeTwoUsage(), TableWriterArchiveImpl< Holder >::Write(), TableWriterScriptImpl< Holder >::Write(), TableWriterBothImpl< Holder >::Write(), fst::WriteFstKaldi(), kaldi::WriteKaldiObject(), kaldi::WriteScriptFile(), Output::~Output(), and PipeOutputImpl::~PipeOutputImpl().

                                                            {
   if (wxfilename == "" || wxfilename == "-") {
     return "standard output";
   } else {
     // If this call to Escape later causes compilation issues,
     // just replace it with "return wxfilename"; it's only a
     // pretty-printing issue.
     return ParseOptions::Escape(wxfilename);
   }
 }

◆ ReadKaldiObject() [1/3]

void kaldi::ReadKaldiObject	(	const std::string &	filename,
		C *	c
	)

Definition at line 239 of file kaldi-io.h.

References kaldi::ReadKaldiObject(), and Input::Stream().

                                               {
   bool binary_in;
   Input ki(filename, &binary_in);
   c->Read(ki.Stream(), binary_in);
 }

◆ ReadKaldiObject() [2/3]

void ReadKaldiObject	(	const std::string &	filename,
		Matrix< float > *	m
	)

Definition at line 832 of file kaldi-io.cc.

References kaldi::ExtractObjectRange(), kaldi::ExtractRangeSpecifier(), KALDI_ERR, Matrix< Real >::Read(), and Input::Stream().

Referenced by ComputeLogPosteriors(), ComputeScores(), ConvolutionComponent::Init(), OnlineIvectorExtractionInfo::Init(), AffineComponent::Init(), AffineComponentPreconditioned::Init(), AffineComponentPreconditionedOnline::Init(), PerElementScaleComponent::Init(), Convolutional1dComponent::Init(), NaturalGradientAffineComponent::InitFromConfig(), LinearComponent::InitFromConfig(), FixedScaleComponent::InitFromConfig(), FixedBiasComponent::InitFromConfig(), PerElementOffsetComponent::InitFromConfig(), FixedScaleComponent::InitFromString(), FixedBiasComponent::InitFromString(), main(), OnlineFeaturePipeline::OnlineFeaturePipeline(), OnlineNnet2FeaturePipeline::OnlineNnet2FeaturePipeline(), kaldi::ReadKaldiObject(), kaldi::ReadModels(), kaldi::RunPerSpeaker(), and kaldi::TypeThreeUsage().

                                                    {
   if (!filename.empty() && filename[filename.size() - 1] == ']') {
     // This filename seems to have a 'range'... like foo.ark:4312423[20:30].
     // (the bit in square brackets is the range).
     std::string rxfilename, range;
     if (!ExtractRangeSpecifier(filename, &rxfilename, &range)) {
       KALDI_ERR << "Could not make sense of possible range specifier in filename "
                 << "while reading matrix: " << filename;
     }
     Matrix<float> temp;
     bool binary_in;
     Input ki(rxfilename, &binary_in);
     temp.Read(ki.Stream(), binary_in);
     if (!ExtractObjectRange(temp, range, m)) {
       KALDI_ERR << "Error extracting range of object: " << filename;
     }
   } else {
     // The normal case, there is no range.
     bool binary_in;
     Input ki(filename, &binary_in);
     m->Read(ki.Stream(), binary_in);
   }
 }

◆ ReadKaldiObject() [3/3]

void ReadKaldiObject	(	const std::string &	filename,
		Matrix< double > *	m
	)

Definition at line 857 of file kaldi-io.cc.

References kaldi::ExtractObjectRange(), kaldi::ExtractRangeSpecifier(), KALDI_ERR, Matrix< Real >::Read(), and Input::Stream().

                                                     {
   if (!filename.empty() && filename[filename.size() - 1] == ']') {
     // This filename seems to have a 'range'... like foo.ark:4312423[20:30].
     // (the bit in square brackets is the range).
     std::string rxfilename, range;
     if (!ExtractRangeSpecifier(filename, &rxfilename, &range)) {
       KALDI_ERR << "Could not make sense of possible range specifier in filename "
                 << "while reading matrix: " << filename;
     }
     Matrix<double> temp;
     bool binary_in;
     Input ki(rxfilename, &binary_in);
     temp.Read(ki.Stream(), binary_in);
     if (!ExtractObjectRange(temp, range, m)) {
       KALDI_ERR << "Error extracting range of object: " << filename;
     }
   } else {
     // The normal case, there is no range.
     bool binary_in;
     Input ki(filename, &binary_in);
     m->Read(ki.Stream(), binary_in);
   }
 }

◆ WriteKaldiObject()

void kaldi::WriteKaldiObject	(	const C &	c,
		const std::string &	filename,
		bool	binary
	)

inline

Definition at line 257 of file kaldi-io.h.

References kaldi::PrintableRxfilename(), kaldi::PrintableWxfilename(), and Output::Stream().

Referenced by kaldi::BuildConstArpaLm(), main(), kaldi::TypeThreeUsage(), kaldi::TypeTwoUsage(), kaldi::nnet2::UnitTestAffineComponent(), kaldi::nnet2::UnitTestAffineComponentPreconditioned(), kaldi::nnet2::UnitTestAffineComponentPreconditionedOnline(), kaldi::nnet2::UnitTestConvolutional1dComponent(), kaldi::WriteAsKaldiVector(), and NnetLdaStatsAccumulator::WriteStats().

                                                              {
   Output ko(filename, binary);
   c.Write(ko.Stream(), binary);
 }

Classes

Enumerations

Functions