I/O related functions and classes

Various namespace-scope functions for I/O. More...

Classes
struct	HtkHeader
	A structure containing the HTK header. More...

Functions
template<typename Real >
bool	ReadHtk (std::istream &is, Matrix< Real > *M_ptr, HtkHeader *header_ptr)
	Extension of the HTK header. More...
template<typename Real >
bool	WriteHtk (std::ostream &os, const MatrixBase< Real > &M, HtkHeader htk_hdr)
template<class Real >
bool	WriteSphinx (std::ostream &os, const MatrixBase< Real > &M)
template<typename Real >
std::ostream &	operator<< (std::ostream &os, const MatrixBase< Real > &M)
template<typename Real >
std::istream &	operator>> (std::istream &is, MatrixBase< Real > &M)
template<typename Real >
std::istream &	operator>> (std::istream &is, Matrix< Real > &M)
template<typename Real >
bool	SameDim (const MatrixBase< Real > &M, const MatrixBase< Real > &N)
template<typename Real >
std::ostream &	operator<< (std::ostream &os, const VectorBase< Real > &rv)
template<typename Real >
std::istream &	operator>> (std::istream &in, VectorBase< Real > &v)
	Input from a C++ stream. More...
template<typename Real >
std::istream &	operator>> (std::istream &in, Vector< Real > &v)
	Input from a C++ stream. More...
template<typename Real >
std::ostream &	operator<< (std::ostream &out, const PackedMatrix< Real > &M)
template<typename Real >
std::istream &	operator>> (std::istream &is, PackedMatrix< Real > &M)

Detailed Description

Various namespace-scope functions for I/O.

end of "addtogroup matrix_group"

Output to a C++ stream.

Note that the << and >> operators are not recommended for normal use in Kaldi-style I/O (for that, use the Read and Write methods). The operators are mainly useful for logging and error output.

Non-binary by default (use Write for binary output).

Function Documentation

operator<<() [1/3]

std::ostream & operator<<	(	std::ostream &	os,
		const VectorBase< Real > &	rv
	)

Definition at line 30 of file kaldi-vector-inl.h.

                                                                      {
  rv.Write(os, false);
  return os;
}

operator<<() [2/3]

std::ostream & operator<<	(	std::ostream &	out,
		const PackedMatrix< Real > &	M
	)

Definition at line 181 of file packed-matrix.h.

                                                                        {
  M.Write(os, false);
  return os;
}

operator<<() [3/3]

std::ostream & operator<< ( std::ostream &  os, const MatrixBase< Real > &  M  )

inline

Definition at line 41 of file kaldi-matrix-inl.h.

                                                                              {
  M.Write(os, false);
  return os;
}

operator>>() [1/5]

std::istream & operator>>	(	std::istream &	in,
		VectorBase< Real > &	v
	)

Input from a C++ stream.

Will automatically read text or binary data from the stream.

Definition at line 36 of file kaldi-vector-inl.h.

References VectorBase< Real >::Read().

                                                               {
  rv.Read(is, false);
  return is;
}

operator>>() [2/5]

std::istream & operator>>	(	std::istream &	in,
		Vector< Real > &	v
	)

Input from a C++ stream.

Will automatically read text or binary data from the stream.

Definition at line 42 of file kaldi-vector-inl.h.

References VectorBase< Real >::AddVec(), and Vector< Real >::Read().

                                                           {
  rv.Read(is, false);
  return is;
}

operator>>() [3/5]

std::istream & operator>> ( std::istream &  is, Matrix< Real > &  M  )

inline

Definition at line 47 of file kaldi-matrix-inl.h.

References Matrix< Real >::Read().

Referenced by SubVector< Real >::operator=(), and kaldi::TraceMat().

                                                                    {
  M.Read(is, false);
  return is;
}

operator>>() [4/5]

std::istream & operator>> ( std::istream &  is, MatrixBase< Real > &  M  )

inline

Definition at line 54 of file kaldi-matrix-inl.h.

References MatrixBase< Real >::Read().

                                                                        {
  M.Read(is, false);
  return is;
}

operator>>() [5/5]

std::istream& kaldi::operator>>	(	std::istream &	is,
		PackedMatrix< Real > &	M
	)

Definition at line 187 of file packed-matrix.h.

                                                                 {
  M.Read(is, false);
  return is;
}

ReadHtk()

bool ReadHtk	(	std::istream &	is,
		Matrix< Real > *	M_ptr,
		HtkHeader *	header_ptr
	)

Extension of the HTK header.

Definition at line 2303 of file kaldi-matrix.cc.

References rnnlm::i, rnnlm::j, KALDI_ASSERT_IS_FLOATING_TYPE, KALDI_ERR, KALDI_SWAP2, KALDI_SWAP4, KALDI_VLOG, KALDI_WARN, kaldi::MachineIsLittleEndian(), MatrixBase< Real >::NumCols(), MatrixBase< Real >::NumRows(), kaldi::ReadHtk(), Matrix< Real >::Resize(), and MatrixBase< Real >::RowData().

Referenced by main(), HtkMatrixHolder::Read(), UnitTestHTKCompare1(), UnitTestHTKCompare2(), UnitTestHTKCompare3(), UnitTestHTKCompare4(), UnitTestHTKCompare5(), UnitTestHTKCompare6(), and kaldi::UnitTestHtkIo().

{
  // check instantiated with double or float.
  KALDI_ASSERT_IS_FLOATING_TYPE(Real);
  Matrix<Real> &M = *M_ptr;
  HtkHeader htk_hdr;

  // TODO(arnab): this fails if the HTK file has CRC cheksum or is compressed.
  is.read((char*)&htk_hdr, sizeof(htk_hdr));  // we're being really POSIX here!
  if (is.fail()) {
    KALDI_WARN << "Could not read header from HTK feature file ";
    return false;
  }

  KALDI_SWAP4(htk_hdr.mNSamples);
  KALDI_SWAP4(htk_hdr.mSamplePeriod);
  KALDI_SWAP2(htk_hdr.mSampleSize);
  KALDI_SWAP2(htk_hdr.mSampleKind);

  bool has_checksum = false;
  {
    // See HParm.h in HTK code for sources of these things.
    enum BaseParmKind{
      Waveform, Lpc, Lprefc, Lpcepstra, Lpdelcep,
      Irefc, Mfcc, Fbank, Melspec, User, Discrete, Plp, Anon };

    const int32 IsCompressed = 02000, HasChecksum = 010000, HasVq = 040000,
        Problem = IsCompressed | HasVq;
    int32 base_parm = htk_hdr.mSampleKind & (077);
    has_checksum = (base_parm & HasChecksum) != 0;
    htk_hdr.mSampleKind &= ~HasChecksum; // We don't support writing with
                                         // checksum so turn it off.
    if (htk_hdr.mSampleKind & Problem)
      KALDI_ERR << "Code to read HTK features does not support compressed "
          "features, or features with VQ.";
    if (base_parm == Waveform || base_parm == Irefc || base_parm == Discrete)
      KALDI_ERR << "Attempting to read HTK features from unsupported type "
          "(e.g. waveform or discrete features.";
  }

  KALDI_VLOG(3) << "HTK header: Num Samples: " << htk_hdr.mNSamples
                << "; Sample period: " << htk_hdr.mSamplePeriod
                << "; Sample size: " << htk_hdr.mSampleSize
                << "; Sample kind: " << htk_hdr.mSampleKind;

  M.Resize(htk_hdr.mNSamples, htk_hdr.mSampleSize / sizeof(float));

  MatrixIndexT i;
  MatrixIndexT j;
  if (sizeof(Real) == sizeof(float)) {
    for (i = 0; i< M.NumRows(); i++) {
      is.read((char*)M.RowData(i), sizeof(float)*M.NumCols());
      if (is.fail()) {
        KALDI_WARN << "Could not read data from HTK feature file ";
        return false;
      }
      if (MachineIsLittleEndian()) {
        MatrixIndexT C = M.NumCols();
        for (j = 0; j < C; j++) {
          KALDI_SWAP4((M(i, j)));  // The HTK standard is big-endian!
        }
      }
    }
  } else {
    float *pmem = new float[M.NumCols()];
    for (i = 0; i < M.NumRows(); i++) {
      is.read((char*)pmem, sizeof(float)*M.NumCols());
      if (is.fail()) {
        KALDI_WARN << "Could not read data from HTK feature file ";
        delete [] pmem;
        return false;
      }
      MatrixIndexT C = M.NumCols();
      for (j = 0; j < C; j++) {
        if (MachineIsLittleEndian())  // HTK standard is big-endian!
          KALDI_SWAP4(pmem[j]);
        M(i, j) = static_cast<Real>(pmem[j]);
      }
    }
    delete [] pmem;
  }
  if (header_ptr) *header_ptr = htk_hdr;
  if (has_checksum) {
    int16 checksum;
    is.read((char*)&checksum, sizeof(checksum));
    if (is.fail())
      KALDI_WARN << "Could not read checksum from HTK feature file ";
    // We ignore the checksum.
  }
  return true;
}

SameDim()

bool kaldi::SameDim	(	const MatrixBase< Real > &	M,
		const MatrixBase< Real > &	N
	)

Definition at line 1111 of file kaldi-matrix.h.

References MatrixBase< Real >::NumCols(), and MatrixBase< Real >::NumRows().

Referenced by Fmpe::AccStats(), CuMatrixBase< float >::AddDiagVecMat(), MatrixBase< float >::AddDiagVecMat(), CuMatrixBase< float >::AddMatDiagVec(), MatrixBase< float >::AddMatDiagVec(), CuMatrixBase< float >::AddMatMatElements(), kaldi::nnet3::attention::AttentionBackward(), RestrictedAttentionComponent::Backprop(), BatchNormComponent::Backprop(), LogSoftmaxComponent::Backprop(), GeneralDropoutComponent::Backprop(), SpecAugmentTimeMaskComponent::Backprop(), DropoutComponent::Backprop(), ScaleAndOffsetComponent::Backprop(), kaldi::cu::BackpropLstmNonlinearity(), RestrictedAttentionComponent::BackpropOneHead(), CuMatrixBase< float >::Ceiling(), MatrixBase< float >::Ceiling(), NnetUpdater::ComputeObjfAndDeriv(), LstmNonlinearityComponent::ConsolidateMemory(), kaldi::cu::CpuBackpropLstmNonlinearity(), CuMatrixBase< float >::DiffLogSoftmaxPerRow(), CuMatrixBase< float >::DiffSigmoid(), MatrixBase< float >::DiffSigmoid(), CuMatrixBase< float >::DiffSoftmaxPerRow(), MatrixBase< float >::DiffTanh(), kaldi::cu::EnsureNonzero(), CuMatrixBase< float >::Exp(), MatrixBase< float >::Exp(), CuMatrixBase< float >::ExpLimited(), MatrixBase< float >::ExpLimited(), CuMatrixBase< float >::ExpSpecial(), MatrixBase< float >::ExpSpecial(), CuMatrixBase< float >::Floor(), MatrixBase< float >::Floor(), CuMatrixBase< float >::Heaviside(), MatrixBase< float >::Heaviside(), CuMatrixBase< float >::Log(), MatrixBase< float >::Log(), CuMatrixBase< float >::LogSoftMaxPerRow(), kaldi::cu::NormalizePerRow(), CuMatrixBase< float >::Pow(), MatrixBase< float >::Pow(), CuMatrixBase< float >::PowAbs(), MatrixBase< float >::PowAbs(), kaldi::nnet2::PreconditionDirections(), BatchNormComponent::Propagate(), GeneralDropoutComponent::Propagate(), SpecAugmentTimeMaskComponent::Propagate(), kaldi::cu::RegularizeL1(), CuMatrixBase< float >::Sigmoid(), MatrixBase< float >::Sigmoid(), kaldi::SlidingWindowCmn(), CuMatrixBase< float >::SoftHinge(), MatrixBase< float >::SoftHinge(), CuMatrixBase< float >::SoftMaxPerRow(), CuMatrixBase< float >::Tanh(), MatrixBase< float >::Tanh(), kaldi::TypeOneUsage(), kaldi::TypeOneUsageAverage(), and Fmpe::Update().

                                                                   {
  return (M.NumRows() == N.NumRows() && M.NumCols() == N.NumCols());
}

WriteHtk()

bool WriteHtk	(	std::ostream &	os,
		const MatrixBase< Real > &	M,
		HtkHeader	htk_hdr
	)

Definition at line 2403 of file kaldi-matrix.cc.

References rnnlm::i, rnnlm::j, KALDI_ASSERT, KALDI_SWAP2, KALDI_SWAP4, KALDI_WARN, kaldi::MachineIsLittleEndian(), HtkHeader::mNSamples, HtkHeader::mSampleKind, HtkHeader::mSamplePeriod, HtkHeader::mSampleSize, MatrixBase< Real >::NumCols(), MatrixBase< Real >::NumRows(), MatrixBase< Real >::RowData(), and kaldi::WriteHtk().

Referenced by main(), UnitTestHTKCompare1(), UnitTestHTKCompare2(), UnitTestHTKCompare3(), UnitTestHTKCompare4(), UnitTestHTKCompare5(), UnitTestHTKCompare6(), kaldi::UnitTestHtkIo(), and HtkMatrixHolder::Write().

{
  KALDI_ASSERT(M.NumRows() == static_cast<MatrixIndexT>(htk_hdr.mNSamples));
  KALDI_ASSERT(M.NumCols() == static_cast<MatrixIndexT>(htk_hdr.mSampleSize) /
               static_cast<MatrixIndexT>(sizeof(float)));

  KALDI_SWAP4(htk_hdr.mNSamples);
  KALDI_SWAP4(htk_hdr.mSamplePeriod);
  KALDI_SWAP2(htk_hdr.mSampleSize);
  KALDI_SWAP2(htk_hdr.mSampleKind);

  os.write((char*)&htk_hdr, sizeof(htk_hdr));
  if (os.fail())  goto bad;

  MatrixIndexT i;
  MatrixIndexT j;
  if (sizeof(Real) == sizeof(float) && !MachineIsLittleEndian()) {
    for (i = 0; i< M.NumRows(); i++) {  // Unlikely to reach here ever!
      os.write((char*)M.RowData(i), sizeof(float)*M.NumCols());
      if (os.fail()) goto bad;
    }
  } else {
    float *pmem = new float[M.NumCols()];

    for (i = 0; i < M.NumRows(); i++) {
      const Real *rowData = M.RowData(i);
      for (j = 0;j < M.NumCols();j++)
        pmem[j] =  static_cast<float> ( rowData[j] );
      if (MachineIsLittleEndian())
        for (j = 0;j < M.NumCols();j++)
          KALDI_SWAP4(pmem[j]);
      os.write((char*)pmem, sizeof(float)*M.NumCols());
      if (os.fail()) {
        delete [] pmem;
        goto bad;
      }
    }
    delete [] pmem;
  }
  return true;
bad:
  KALDI_WARN << "Could not write to HTK feature file ";
  return false;
}

WriteSphinx()

bool WriteSphinx	(	std::ostream &	os,
		const MatrixBase< Real > &	M
	)

Definition at line 2455 of file kaldi-matrix.cc.

References rnnlm::i, rnnlm::j, KALDI_SWAP4, KALDI_WARN, kaldi::MachineIsLittleEndian(), MatrixBase< Real >::NumCols(), MatrixBase< Real >::NumRows(), MatrixBase< Real >::RowData(), and kaldi::WriteSphinx().

Referenced by main().

{
  // CMUSphinx mfc file header contains count of the floats, followed
  // by the data in float little endian format.

  int size = M.NumRows() * M.NumCols();
  os.write((char*)&size, sizeof(int));
  if (os.fail())  goto bad;

  MatrixIndexT i;
  MatrixIndexT j;
  if (sizeof(Real) == sizeof(float) && MachineIsLittleEndian()) {
    for (i = 0; i< M.NumRows(); i++) {  // Unlikely to reach here ever!
      os.write((char*)M.RowData(i), sizeof(float)*M.NumCols());
      if (os.fail()) goto bad;
    }
  } else {
    float *pmem = new float[M.NumCols()];

    for (i = 0; i < M.NumRows(); i++) {
      const Real *rowData = M.RowData(i);
      for (j = 0;j < M.NumCols();j++)
        pmem[j] =  static_cast<float> ( rowData[j] );
      if (!MachineIsLittleEndian())
        for (j = 0;j < M.NumCols();j++)
          KALDI_SWAP4(pmem[j]);
      os.write((char*)pmem, sizeof(float)*M.NumCols());
      if (os.fail()) {
        delete [] pmem;
        goto bad;
      }
    }
    delete [] pmem;
  }
  return true;
bad:
  KALDI_WARN << "Could not write to Sphinx feature file";
  return false;
}