I/O related functions and classes

Various namespace-scope functions for I/O. More...

Classes

struct  HtkHeader
 A structure containing the HTK header. More...
 

Functions

template<typename Real >
bool ReadHtk (std::istream &is, Matrix< Real > *M_ptr, HtkHeader *header_ptr)
 Extension of the HTK header. More...
 
template<typename Real >
bool WriteHtk (std::ostream &os, const MatrixBase< Real > &M, HtkHeader htk_hdr)
 
template<class Real >
bool WriteSphinx (std::ostream &os, const MatrixBase< Real > &M)
 
template<typename Real >
std::ostream & operator<< (std::ostream &os, const MatrixBase< Real > &M)
 
template<typename Real >
std::istream & operator>> (std::istream &is, MatrixBase< Real > &M)
 
template<typename Real >
std::istream & operator>> (std::istream &is, Matrix< Real > &M)
 
template<typename Real >
bool SameDim (const MatrixBase< Real > &M, const MatrixBase< Real > &N)
 
template<typename Real >
std::ostream & operator<< (std::ostream &os, const VectorBase< Real > &rv)
 
template<typename Real >
std::istream & operator>> (std::istream &in, VectorBase< Real > &v)
 Input from a C++ stream. More...
 
template<typename Real >
std::istream & operator>> (std::istream &in, Vector< Real > &v)
 Input from a C++ stream. More...
 
template<typename Real >
std::ostream & operator<< (std::ostream &out, const PackedMatrix< Real > &M)
 
template<typename Real >
std::istream & operator>> (std::istream &is, PackedMatrix< Real > &M)
 

Detailed Description

Various namespace-scope functions for I/O.

end of "addtogroup matrix_group"

Output to a C++ stream.

Note that the << and >> operators are not recommended for normal use in Kaldi-style I/O (for that, use the Read and Write methods). The operators are mainly useful for logging and error output.

Non-binary by default (use Write for binary output).

Function Documentation

◆ operator<<() [1/3]

std::ostream & operator<< ( std::ostream &  os,
const VectorBase< Real > &  rv 
)

Definition at line 30 of file kaldi-vector-inl.h.

30  {
31  rv.Write(os, false);
32  return os;
33 }

◆ operator<<() [2/3]

std::ostream & operator<< ( std::ostream &  out,
const PackedMatrix< Real > &  M 
)

Definition at line 181 of file packed-matrix.h.

181  {
182  M.Write(os, false);
183  return os;
184 }

◆ operator<<() [3/3]

std::ostream & operator<< ( std::ostream &  os,
const MatrixBase< Real > &  M 
)
inline

Definition at line 41 of file kaldi-matrix-inl.h.

41  {
42  M.Write(os, false);
43  return os;
44 }

◆ operator>>() [1/5]

std::istream & operator>> ( std::istream &  in,
VectorBase< Real > &  v 
)

Input from a C++ stream.

Will automatically read text or binary data from the stream.

Definition at line 36 of file kaldi-vector-inl.h.

References VectorBase< Real >::Read().

36  {
37  rv.Read(is, false);
38  return is;
39 }

◆ operator>>() [2/5]

std::istream & operator>> ( std::istream &  in,
Vector< Real > &  v 
)

Input from a C++ stream.

Will automatically read text or binary data from the stream.

Definition at line 42 of file kaldi-vector-inl.h.

References VectorBase< Real >::AddVec(), and Vector< Real >::Read().

42  {
43  rv.Read(is, false);
44  return is;
45 }

◆ operator>>() [3/5]

std::istream & operator>> ( std::istream &  is,
Matrix< Real > &  M 
)
inline

Definition at line 47 of file kaldi-matrix-inl.h.

References Matrix< Real >::Read().

Referenced by SubVector< Real >::operator=(), and kaldi::TraceMat().

47  {
48  M.Read(is, false);
49  return is;
50 }

◆ operator>>() [4/5]

std::istream & operator>> ( std::istream &  is,
MatrixBase< Real > &  M 
)
inline

Definition at line 54 of file kaldi-matrix-inl.h.

References MatrixBase< Real >::Read().

54  {
55  M.Read(is, false);
56  return is;
57 }

◆ operator>>() [5/5]

std::istream& kaldi::operator>> ( std::istream &  is,
PackedMatrix< Real > &  M 
)

Definition at line 187 of file packed-matrix.h.

187  {
188  M.Read(is, false);
189  return is;
190 }

◆ ReadHtk()

bool ReadHtk ( std::istream &  is,
Matrix< Real > *  M_ptr,
HtkHeader header_ptr 
)

Extension of the HTK header.

Definition at line 2303 of file kaldi-matrix.cc.

References rnnlm::i, rnnlm::j, KALDI_ASSERT_IS_FLOATING_TYPE, KALDI_ERR, KALDI_SWAP2, KALDI_SWAP4, KALDI_VLOG, KALDI_WARN, kaldi::MachineIsLittleEndian(), MatrixBase< Real >::NumCols(), MatrixBase< Real >::NumRows(), kaldi::ReadHtk(), Matrix< Real >::Resize(), and MatrixBase< Real >::RowData().

Referenced by main(), HtkMatrixHolder::Read(), UnitTestHTKCompare1(), UnitTestHTKCompare2(), UnitTestHTKCompare3(), UnitTestHTKCompare4(), UnitTestHTKCompare5(), UnitTestHTKCompare6(), and kaldi::UnitTestHtkIo().

2304 {
2305  // check instantiated with double or float.
2307  Matrix<Real> &M = *M_ptr;
2308  HtkHeader htk_hdr;
2309 
2310  // TODO(arnab): this fails if the HTK file has CRC cheksum or is compressed.
2311  is.read((char*)&htk_hdr, sizeof(htk_hdr)); // we're being really POSIX here!
2312  if (is.fail()) {
2313  KALDI_WARN << "Could not read header from HTK feature file ";
2314  return false;
2315  }
2316 
2317  KALDI_SWAP4(htk_hdr.mNSamples);
2318  KALDI_SWAP4(htk_hdr.mSamplePeriod);
2319  KALDI_SWAP2(htk_hdr.mSampleSize);
2320  KALDI_SWAP2(htk_hdr.mSampleKind);
2321 
2322  bool has_checksum = false;
2323  {
2324  // See HParm.h in HTK code for sources of these things.
2325  enum BaseParmKind{
2326  Waveform, Lpc, Lprefc, Lpcepstra, Lpdelcep,
2327  Irefc, Mfcc, Fbank, Melspec, User, Discrete, Plp, Anon };
2328 
2329  const int32 IsCompressed = 02000, HasChecksum = 010000, HasVq = 040000,
2330  Problem = IsCompressed | HasVq;
2331  int32 base_parm = htk_hdr.mSampleKind & (077);
2332  has_checksum = (base_parm & HasChecksum) != 0;
2333  htk_hdr.mSampleKind &= ~HasChecksum; // We don't support writing with
2334  // checksum so turn it off.
2335  if (htk_hdr.mSampleKind & Problem)
2336  KALDI_ERR << "Code to read HTK features does not support compressed "
2337  "features, or features with VQ.";
2338  if (base_parm == Waveform || base_parm == Irefc || base_parm == Discrete)
2339  KALDI_ERR << "Attempting to read HTK features from unsupported type "
2340  "(e.g. waveform or discrete features.";
2341  }
2342 
2343  KALDI_VLOG(3) << "HTK header: Num Samples: " << htk_hdr.mNSamples
2344  << "; Sample period: " << htk_hdr.mSamplePeriod
2345  << "; Sample size: " << htk_hdr.mSampleSize
2346  << "; Sample kind: " << htk_hdr.mSampleKind;
2347 
2348  M.Resize(htk_hdr.mNSamples, htk_hdr.mSampleSize / sizeof(float));
2349 
2350  MatrixIndexT i;
2351  MatrixIndexT j;
2352  if (sizeof(Real) == sizeof(float)) {
2353  for (i = 0; i< M.NumRows(); i++) {
2354  is.read((char*)M.RowData(i), sizeof(float)*M.NumCols());
2355  if (is.fail()) {
2356  KALDI_WARN << "Could not read data from HTK feature file ";
2357  return false;
2358  }
2359  if (MachineIsLittleEndian()) {
2360  MatrixIndexT C = M.NumCols();
2361  for (j = 0; j < C; j++) {
2362  KALDI_SWAP4((M(i, j))); // The HTK standard is big-endian!
2363  }
2364  }
2365  }
2366  } else {
2367  float *pmem = new float[M.NumCols()];
2368  for (i = 0; i < M.NumRows(); i++) {
2369  is.read((char*)pmem, sizeof(float)*M.NumCols());
2370  if (is.fail()) {
2371  KALDI_WARN << "Could not read data from HTK feature file ";
2372  delete [] pmem;
2373  return false;
2374  }
2375  MatrixIndexT C = M.NumCols();
2376  for (j = 0; j < C; j++) {
2377  if (MachineIsLittleEndian()) // HTK standard is big-endian!
2378  KALDI_SWAP4(pmem[j]);
2379  M(i, j) = static_cast<Real>(pmem[j]);
2380  }
2381  }
2382  delete [] pmem;
2383  }
2384  if (header_ptr) *header_ptr = htk_hdr;
2385  if (has_checksum) {
2386  int16 checksum;
2387  is.read((char*)&checksum, sizeof(checksum));
2388  if (is.fail())
2389  KALDI_WARN << "Could not read checksum from HTK feature file ";
2390  // We ignore the checksum.
2391  }
2392  return true;
2393 }
#define KALDI_SWAP2(a)
Definition: kaldi-utils.h:114
OfflineFeatureTpl< MfccComputer > Mfcc
Definition: feature-mfcc.h:147
OfflineFeatureTpl< PlpComputer > Plp
Definition: feature-plp.h:169
kaldi::int32 int32
int32 MatrixIndexT
Definition: matrix-common.h:98
OfflineFeatureTpl< FbankComputer > Fbank
int MachineIsLittleEndian()
Definition: kaldi-utils.h:83
#define KALDI_ERR
Definition: kaldi-error.h:147
#define KALDI_WARN
Definition: kaldi-error.h:150
#define KALDI_ASSERT_IS_FLOATING_TYPE(F)
Definition: kaldi-utils.h:137
#define KALDI_VLOG(v)
Definition: kaldi-error.h:156
#define KALDI_SWAP4(a)
Definition: kaldi-utils.h:107

◆ SameDim()

bool kaldi::SameDim ( const MatrixBase< Real > &  M,
const MatrixBase< Real > &  N 
)

Definition at line 1111 of file kaldi-matrix.h.

References MatrixBase< Real >::NumCols(), and MatrixBase< Real >::NumRows().

Referenced by Fmpe::AccStats(), CuMatrixBase< float >::AddDiagVecMat(), MatrixBase< float >::AddDiagVecMat(), CuMatrixBase< float >::AddMatDiagVec(), MatrixBase< float >::AddMatDiagVec(), CuMatrixBase< float >::AddMatMatElements(), kaldi::nnet3::attention::AttentionBackward(), RestrictedAttentionComponent::Backprop(), BatchNormComponent::Backprop(), LogSoftmaxComponent::Backprop(), GeneralDropoutComponent::Backprop(), SpecAugmentTimeMaskComponent::Backprop(), DropoutComponent::Backprop(), ScaleAndOffsetComponent::Backprop(), kaldi::cu::BackpropLstmNonlinearity(), RestrictedAttentionComponent::BackpropOneHead(), CuMatrixBase< float >::Ceiling(), MatrixBase< float >::Ceiling(), NnetUpdater::ComputeObjfAndDeriv(), LstmNonlinearityComponent::ConsolidateMemory(), kaldi::cu::CpuBackpropLstmNonlinearity(), CuMatrixBase< float >::DiffLogSoftmaxPerRow(), CuMatrixBase< float >::DiffSigmoid(), MatrixBase< float >::DiffSigmoid(), CuMatrixBase< float >::DiffSoftmaxPerRow(), MatrixBase< float >::DiffTanh(), kaldi::cu::EnsureNonzero(), CuMatrixBase< float >::Exp(), MatrixBase< float >::Exp(), CuMatrixBase< float >::ExpLimited(), MatrixBase< float >::ExpLimited(), CuMatrixBase< float >::ExpSpecial(), MatrixBase< float >::ExpSpecial(), CuMatrixBase< float >::Floor(), MatrixBase< float >::Floor(), CuMatrixBase< float >::Heaviside(), MatrixBase< float >::Heaviside(), CuMatrixBase< float >::Log(), MatrixBase< float >::Log(), CuMatrixBase< float >::LogSoftMaxPerRow(), kaldi::cu::NormalizePerRow(), CuMatrixBase< float >::Pow(), MatrixBase< float >::Pow(), CuMatrixBase< float >::PowAbs(), MatrixBase< float >::PowAbs(), kaldi::nnet2::PreconditionDirections(), BatchNormComponent::Propagate(), GeneralDropoutComponent::Propagate(), SpecAugmentTimeMaskComponent::Propagate(), kaldi::cu::RegularizeL1(), CuMatrixBase< float >::Sigmoid(), MatrixBase< float >::Sigmoid(), kaldi::SlidingWindowCmn(), CuMatrixBase< float >::SoftHinge(), MatrixBase< float >::SoftHinge(), CuMatrixBase< float >::SoftMaxPerRow(), CuMatrixBase< float >::Tanh(), MatrixBase< float >::Tanh(), kaldi::TypeOneUsage(), kaldi::TypeOneUsageAverage(), and Fmpe::Update().

1111  {
1112  return (M.NumRows() == N.NumRows() && M.NumCols() == N.NumCols());
1113 }

◆ WriteHtk()

bool WriteHtk ( std::ostream &  os,
const MatrixBase< Real > &  M,
HtkHeader  htk_hdr 
)

Definition at line 2403 of file kaldi-matrix.cc.

References rnnlm::i, rnnlm::j, KALDI_ASSERT, KALDI_SWAP2, KALDI_SWAP4, KALDI_WARN, kaldi::MachineIsLittleEndian(), HtkHeader::mNSamples, HtkHeader::mSampleKind, HtkHeader::mSamplePeriod, HtkHeader::mSampleSize, MatrixBase< Real >::NumCols(), MatrixBase< Real >::NumRows(), MatrixBase< Real >::RowData(), and kaldi::WriteHtk().

Referenced by main(), UnitTestHTKCompare1(), UnitTestHTKCompare2(), UnitTestHTKCompare3(), UnitTestHTKCompare4(), UnitTestHTKCompare5(), UnitTestHTKCompare6(), kaldi::UnitTestHtkIo(), and HtkMatrixHolder::Write().

2404 {
2405  KALDI_ASSERT(M.NumRows() == static_cast<MatrixIndexT>(htk_hdr.mNSamples));
2406  KALDI_ASSERT(M.NumCols() == static_cast<MatrixIndexT>(htk_hdr.mSampleSize) /
2407  static_cast<MatrixIndexT>(sizeof(float)));
2408 
2409  KALDI_SWAP4(htk_hdr.mNSamples);
2410  KALDI_SWAP4(htk_hdr.mSamplePeriod);
2411  KALDI_SWAP2(htk_hdr.mSampleSize);
2412  KALDI_SWAP2(htk_hdr.mSampleKind);
2413 
2414  os.write((char*)&htk_hdr, sizeof(htk_hdr));
2415  if (os.fail()) goto bad;
2416 
2417  MatrixIndexT i;
2418  MatrixIndexT j;
2419  if (sizeof(Real) == sizeof(float) && !MachineIsLittleEndian()) {
2420  for (i = 0; i< M.NumRows(); i++) { // Unlikely to reach here ever!
2421  os.write((char*)M.RowData(i), sizeof(float)*M.NumCols());
2422  if (os.fail()) goto bad;
2423  }
2424  } else {
2425  float *pmem = new float[M.NumCols()];
2426 
2427  for (i = 0; i < M.NumRows(); i++) {
2428  const Real *rowData = M.RowData(i);
2429  for (j = 0;j < M.NumCols();j++)
2430  pmem[j] = static_cast<float> ( rowData[j] );
2431  if (MachineIsLittleEndian())
2432  for (j = 0;j < M.NumCols();j++)
2433  KALDI_SWAP4(pmem[j]);
2434  os.write((char*)pmem, sizeof(float)*M.NumCols());
2435  if (os.fail()) {
2436  delete [] pmem;
2437  goto bad;
2438  }
2439  }
2440  delete [] pmem;
2441  }
2442  return true;
2443 bad:
2444  KALDI_WARN << "Could not write to HTK feature file ";
2445  return false;
2446 }
#define KALDI_SWAP2(a)
Definition: kaldi-utils.h:114
int32 MatrixIndexT
Definition: matrix-common.h:98
int MachineIsLittleEndian()
Definition: kaldi-utils.h:83
#define KALDI_WARN
Definition: kaldi-error.h:150
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:185
#define KALDI_SWAP4(a)
Definition: kaldi-utils.h:107

◆ WriteSphinx()

bool WriteSphinx ( std::ostream &  os,
const MatrixBase< Real > &  M 
)

Definition at line 2455 of file kaldi-matrix.cc.

References rnnlm::i, rnnlm::j, KALDI_SWAP4, KALDI_WARN, kaldi::MachineIsLittleEndian(), MatrixBase< Real >::NumCols(), MatrixBase< Real >::NumRows(), MatrixBase< Real >::RowData(), and kaldi::WriteSphinx().

Referenced by main().

2456 {
2457  // CMUSphinx mfc file header contains count of the floats, followed
2458  // by the data in float little endian format.
2459 
2460  int size = M.NumRows() * M.NumCols();
2461  os.write((char*)&size, sizeof(int));
2462  if (os.fail()) goto bad;
2463 
2464  MatrixIndexT i;
2465  MatrixIndexT j;
2466  if (sizeof(Real) == sizeof(float) && MachineIsLittleEndian()) {
2467  for (i = 0; i< M.NumRows(); i++) { // Unlikely to reach here ever!
2468  os.write((char*)M.RowData(i), sizeof(float)*M.NumCols());
2469  if (os.fail()) goto bad;
2470  }
2471  } else {
2472  float *pmem = new float[M.NumCols()];
2473 
2474  for (i = 0; i < M.NumRows(); i++) {
2475  const Real *rowData = M.RowData(i);
2476  for (j = 0;j < M.NumCols();j++)
2477  pmem[j] = static_cast<float> ( rowData[j] );
2478  if (!MachineIsLittleEndian())
2479  for (j = 0;j < M.NumCols();j++)
2480  KALDI_SWAP4(pmem[j]);
2481  os.write((char*)pmem, sizeof(float)*M.NumCols());
2482  if (os.fail()) {
2483  delete [] pmem;
2484  goto bad;
2485  }
2486  }
2487  delete [] pmem;
2488  }
2489  return true;
2490 bad:
2491  KALDI_WARN << "Could not write to Sphinx feature file";
2492  return false;
2493 }
int32 MatrixIndexT
Definition: matrix-common.h:98
int MachineIsLittleEndian()
Definition: kaldi-utils.h:83
#define KALDI_WARN
Definition: kaldi-error.h:150
#define KALDI_SWAP4(a)
Definition: kaldi-utils.h:107