All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
I/O related functions and classes

Various namespace-scope functions for I/O. More...

Classes

struct  HtkHeader
 A structure containing the HTK header. More...
 

Functions

template<typename Real >
bool ReadHtk (std::istream &is, Matrix< Real > *M_ptr, HtkHeader *header_ptr)
 Extension of the HTK header. More...
 
template<typename Real >
bool WriteHtk (std::ostream &os, const MatrixBase< Real > &M, HtkHeader htk_hdr)
 
template<class Real >
bool WriteSphinx (std::ostream &os, const MatrixBase< Real > &M)
 
template<typename Real >
std::ostream & operator<< (std::ostream &os, const MatrixBase< Real > &M)
 
template<typename Real >
std::istream & operator>> (std::istream &is, MatrixBase< Real > &M)
 
template<typename Real >
std::istream & operator>> (std::istream &is, Matrix< Real > &M)
 
template<typename Real >
bool SameDim (const MatrixBase< Real > &M, const MatrixBase< Real > &N)
 
template<typename Real >
std::ostream & operator<< (std::ostream &os, const VectorBase< Real > &rv)
 
template<typename Real >
std::istream & operator>> (std::istream &in, VectorBase< Real > &v)
 Input from a C++ stream. More...
 
template<typename Real >
std::istream & operator>> (std::istream &in, Vector< Real > &v)
 Input from a C++ stream. More...
 
template<typename Real >
std::ostream & operator<< (std::ostream &out, const PackedMatrix< Real > &M)
 
template<typename Real >
std::istream & operator>> (std::istream &is, PackedMatrix< Real > &M)
 

Detailed Description

Various namespace-scope functions for I/O.

end of "addtogroup matrix_group"

Output to a C++ stream.

Note that the << and >> operators are not recommended for normal use in Kaldi-style I/O (for that, use the Read and Write methods). The operators are mainly useful for logging and error output.

Non-binary by default (use Write for binary output).

Function Documentation

std::ostream & operator<< ( std::ostream &  os,
const VectorBase< Real > &  rv 
)

Definition at line 30 of file kaldi-vector-inl.h.

30  {
31  rv.Write(os, false);
32  return os;
33 }
std::ostream & operator<< ( std::ostream &  out,
const PackedMatrix< Real > &  M 
)

Definition at line 181 of file packed-matrix.h.

181  {
182  M.Write(os, false);
183  return os;
184 }
std::ostream & operator<< ( std::ostream &  os,
const MatrixBase< Real > &  M 
)
inline

Definition at line 41 of file kaldi-matrix-inl.h.

41  {
42  M.Write(os, false);
43  return os;
44 }
std::istream & operator>> ( std::istream &  in,
VectorBase< Real > &  v 
)

Input from a C++ stream.

Will automatically read text or binary data from the stream.

Definition at line 36 of file kaldi-vector-inl.h.

References VectorBase< Real >::Read().

36  {
37  rv.Read(is, false);
38  return is;
39 }
std::istream & operator>> ( std::istream &  in,
Vector< Real > &  v 
)

Input from a C++ stream.

Will automatically read text or binary data from the stream.

Definition at line 42 of file kaldi-vector-inl.h.

References Vector< Real >::Read().

42  {
43  rv.Read(is, false);
44  return is;
45 }
std::istream & operator>> ( std::istream &  is,
Matrix< Real > &  M 
)
inline

Definition at line 47 of file kaldi-matrix-inl.h.

References Matrix< Real >::Read().

47  {
48  M.Read(is, false);
49  return is;
50 }
std::istream & operator>> ( std::istream &  is,
MatrixBase< Real > &  M 
)
inline

Definition at line 54 of file kaldi-matrix-inl.h.

References MatrixBase< Real >::Read().

54  {
55  M.Read(is, false);
56  return is;
57 }
std::istream& kaldi::operator>> ( std::istream &  is,
PackedMatrix< Real > &  M 
)

Definition at line 187 of file packed-matrix.h.

187  {
188  M.Read(is, false);
189  return is;
190 }
bool ReadHtk ( std::istream &  is,
Matrix< Real > *  M_ptr,
HtkHeader *  header_ptr 
)

Extension of the HTK header.

Definition at line 2254 of file kaldi-matrix.cc.

References rnnlm::i, rnnlm::j, KALDI_ASSERT_IS_FLOATING_TYPE, KALDI_ERR, KALDI_SWAP2, KALDI_SWAP4, KALDI_VLOG, KALDI_WARN, kaldi::MachineIsLittleEndian(), HtkHeader::mNSamples, HtkHeader::mSampleKind, HtkHeader::mSamplePeriod, HtkHeader::mSampleSize, MatrixBase< Real >::NumCols(), MatrixBase< Real >::NumRows(), Matrix< Real >::Resize(), and MatrixBase< Real >::RowData().

Referenced by main(), HtkMatrixHolder::Read(), UnitTestHTKCompare1(), UnitTestHTKCompare2(), UnitTestHTKCompare3(), UnitTestHTKCompare4(), UnitTestHTKCompare5(), UnitTestHTKCompare6(), and kaldi::UnitTestHtkIo().

2255 {
2256  // check instantiated with double or float.
2258  Matrix<Real> &M = *M_ptr;
2259  HtkHeader htk_hdr;
2260 
2261  // TODO(arnab): this fails if the HTK file has CRC cheksum or is compressed.
2262  is.read((char*)&htk_hdr, sizeof(htk_hdr)); // we're being really POSIX here!
2263  if (is.fail()) {
2264  KALDI_WARN << "Could not read header from HTK feature file ";
2265  return false;
2266  }
2267 
2268  KALDI_SWAP4(htk_hdr.mNSamples);
2269  KALDI_SWAP4(htk_hdr.mSamplePeriod);
2270  KALDI_SWAP2(htk_hdr.mSampleSize);
2271  KALDI_SWAP2(htk_hdr.mSampleKind);
2272 
2273  bool has_checksum = false;
2274  {
2275  // See HParm.h in HTK code for sources of these things.
2276  enum BaseParmKind{
2277  Waveform, Lpc, Lprefc, Lpcepstra, Lpdelcep,
2278  Irefc, Mfcc, Fbank, Melspec, User, Discrete, Plp, Anon };
2279 
2280  const int32 IsCompressed = 02000, HasChecksum = 010000, HasVq = 040000,
2281  Problem = IsCompressed | HasVq;
2282  int32 base_parm = htk_hdr.mSampleKind & (077);
2283  has_checksum = (base_parm & HasChecksum) != 0;
2284  htk_hdr.mSampleKind &= ~HasChecksum; // We don't support writing with
2285  // checksum so turn it off.
2286  if (htk_hdr.mSampleKind & Problem)
2287  KALDI_ERR << "Code to read HTK features does not support compressed "
2288  "features, or features with VQ.";
2289  if (base_parm == Waveform || base_parm == Irefc || base_parm == Discrete)
2290  KALDI_ERR << "Attempting to read HTK features from unsupported type "
2291  "(e.g. waveform or discrete features.";
2292  }
2293 
2294  KALDI_VLOG(3) << "HTK header: Num Samples: " << htk_hdr.mNSamples
2295  << "; Sample period: " << htk_hdr.mSamplePeriod
2296  << "; Sample size: " << htk_hdr.mSampleSize
2297  << "; Sample kind: " << htk_hdr.mSampleKind;
2298 
2299  M.Resize(htk_hdr.mNSamples, htk_hdr.mSampleSize / sizeof(float));
2300 
2301  MatrixIndexT i;
2302  MatrixIndexT j;
2303  if (sizeof(Real) == sizeof(float)) {
2304  for (i = 0; i< M.NumRows(); i++) {
2305  is.read((char*)M.RowData(i), sizeof(float)*M.NumCols());
2306  if (is.fail()) {
2307  KALDI_WARN << "Could not read data from HTK feature file ";
2308  return false;
2309  }
2310  if (MachineIsLittleEndian()) {
2311  MatrixIndexT C = M.NumCols();
2312  for (j = 0; j < C; j++) {
2313  KALDI_SWAP4((M(i, j))); // The HTK standard is big-endian!
2314  }
2315  }
2316  }
2317  } else {
2318  float *pmem = new float[M.NumCols()];
2319  for (i = 0; i < M.NumRows(); i++) {
2320  is.read((char*)pmem, sizeof(float)*M.NumCols());
2321  if (is.fail()) {
2322  KALDI_WARN << "Could not read data from HTK feature file ";
2323  delete [] pmem;
2324  return false;
2325  }
2326  MatrixIndexT C = M.NumCols();
2327  for (j = 0; j < C; j++) {
2328  if (MachineIsLittleEndian()) // HTK standard is big-endian!
2329  KALDI_SWAP4(pmem[j]);
2330  M(i, j) = static_cast<Real>(pmem[j]);
2331  }
2332  }
2333  delete [] pmem;
2334  }
2335  if (header_ptr) *header_ptr = htk_hdr;
2336  if (has_checksum) {
2337  int16 checksum;
2338  is.read((char*)&checksum, sizeof(checksum));
2339  if (is.fail())
2340  KALDI_WARN << "Could not read checksum from HTK feature file ";
2341  // We ignore the checksum.
2342  }
2343  return true;
2344 }
#define KALDI_SWAP2(a)
Definition: kaldi-utils.h:110
OfflineFeatureTpl< MfccComputer > Mfcc
Definition: feature-mfcc.h:143
OfflineFeatureTpl< PlpComputer > Plp
Definition: feature-plp.h:167
int32 MatrixIndexT
Definition: matrix-common.h:96
OfflineFeatureTpl< FbankComputer > Fbank
int MachineIsLittleEndian()
Definition: kaldi-utils.h:79
#define KALDI_ERR
Definition: kaldi-error.h:127
#define KALDI_WARN
Definition: kaldi-error.h:130
#define KALDI_ASSERT_IS_FLOATING_TYPE(F)
Definition: kaldi-utils.h:133
#define KALDI_VLOG(v)
Definition: kaldi-error.h:136
#define KALDI_SWAP4(a)
Definition: kaldi-utils.h:103
bool kaldi::SameDim ( const MatrixBase< Real > &  M,
const MatrixBase< Real > &  N 
)

Definition at line 1065 of file kaldi-matrix.h.

References MatrixBase< Real >::NumCols(), and MatrixBase< Real >::NumRows().

Referenced by Fmpe::AccStats(), CuMatrixBase< Real >::AddDiagVecMat(), MatrixBase< Real >::AddDiagVecMat(), CuMatrixBase< Real >::AddMatDiagVec(), MatrixBase< Real >::AddMatDiagVec(), CuMatrixBase< Real >::AddMatMatElements(), CuMatrixBase< Real >::ApplyLogSoftMaxPerRow(), CuMatrixBase< Real >::ApplySoftMaxPerRow(), kaldi::nnet3::attention::AttentionBackward(), RestrictedAttentionComponent::Backprop(), LogSoftmaxComponent::Backprop(), DropoutComponent::Backprop(), ScaleAndOffsetComponent::Backprop(), BatchNormComponent::Backprop(), kaldi::cu::BackpropLstmNonlinearity(), RestrictedAttentionComponent::BackpropOneHead(), NnetUpdater::ComputeObjfAndDeriv(), kaldi::cu::CpuBackpropLstmNonlinearity(), CuMatrixBase< Real >::DiffLogSoftmaxPerRow(), CuMatrixBase< Real >::DiffSigmoid(), MatrixBase< Real >::DiffSigmoid(), CuMatrixBase< Real >::DiffSoftmaxPerRow(), MatrixBase< Real >::DiffTanh(), kaldi::cu::EnsureNonzero(), CuMatrixBase< Real >::Heaviside(), MatrixBase< Real >::Heaviside(), kaldi::cu::NormalizePerRow(), kaldi::nnet2::PreconditionDirections(), BatchNormComponent::Propagate(), kaldi::cu::RegularizeL1(), CuMatrixBase< Real >::Sigmoid(), MatrixBase< Real >::Sigmoid(), kaldi::SlidingWindowCmn(), CuMatrixBase< Real >::SoftHinge(), MatrixBase< Real >::SoftHinge(), CuMatrixBase< Real >::Tanh(), MatrixBase< Real >::Tanh(), kaldi::TypeOneUsage(), and Fmpe::Update().

1065  {
1066  return (M.NumRows() == N.NumRows() && M.NumCols() == N.NumCols());
1067 }
bool WriteHtk ( std::ostream &  os,
const MatrixBase< Real > &  M,
HtkHeader  htk_hdr 
)

Definition at line 2354 of file kaldi-matrix.cc.

References rnnlm::i, rnnlm::j, KALDI_ASSERT, KALDI_SWAP2, KALDI_SWAP4, KALDI_WARN, kaldi::MachineIsLittleEndian(), HtkHeader::mNSamples, HtkHeader::mSampleKind, HtkHeader::mSamplePeriod, HtkHeader::mSampleSize, MatrixBase< Real >::NumCols(), MatrixBase< Real >::NumRows(), and MatrixBase< Real >::RowData().

Referenced by main(), UnitTestHTKCompare1(), UnitTestHTKCompare2(), UnitTestHTKCompare3(), UnitTestHTKCompare4(), UnitTestHTKCompare5(), UnitTestHTKCompare6(), kaldi::UnitTestHtkIo(), and HtkMatrixHolder::Write().

2355 {
2356  KALDI_ASSERT(M.NumRows() == static_cast<MatrixIndexT>(htk_hdr.mNSamples));
2357  KALDI_ASSERT(M.NumCols() == static_cast<MatrixIndexT>(htk_hdr.mSampleSize) /
2358  static_cast<MatrixIndexT>(sizeof(float)));
2359 
2360  KALDI_SWAP4(htk_hdr.mNSamples);
2361  KALDI_SWAP4(htk_hdr.mSamplePeriod);
2362  KALDI_SWAP2(htk_hdr.mSampleSize);
2363  KALDI_SWAP2(htk_hdr.mSampleKind);
2364 
2365  os.write((char*)&htk_hdr, sizeof(htk_hdr));
2366  if (os.fail()) goto bad;
2367 
2368  MatrixIndexT i;
2369  MatrixIndexT j;
2370  if (sizeof(Real) == sizeof(float) && !MachineIsLittleEndian()) {
2371  for (i = 0; i< M.NumRows(); i++) { // Unlikely to reach here ever!
2372  os.write((char*)M.RowData(i), sizeof(float)*M.NumCols());
2373  if (os.fail()) goto bad;
2374  }
2375  } else {
2376  float *pmem = new float[M.NumCols()];
2377 
2378  for (i = 0; i < M.NumRows(); i++) {
2379  const Real *rowData = M.RowData(i);
2380  for (j = 0;j < M.NumCols();j++)
2381  pmem[j] = static_cast<float> ( rowData[j] );
2382  if (MachineIsLittleEndian())
2383  for (j = 0;j < M.NumCols();j++)
2384  KALDI_SWAP4(pmem[j]);
2385  os.write((char*)pmem, sizeof(float)*M.NumCols());
2386  if (os.fail()) {
2387  delete [] pmem;
2388  goto bad;
2389  }
2390  }
2391  delete [] pmem;
2392  }
2393  return true;
2394 bad:
2395  KALDI_WARN << "Could not write to HTK feature file ";
2396  return false;
2397 }
#define KALDI_SWAP2(a)
Definition: kaldi-utils.h:110
int32 MatrixIndexT
Definition: matrix-common.h:96
int MachineIsLittleEndian()
Definition: kaldi-utils.h:79
#define KALDI_WARN
Definition: kaldi-error.h:130
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
#define KALDI_SWAP4(a)
Definition: kaldi-utils.h:103
bool WriteSphinx ( std::ostream &  os,
const MatrixBase< Real > &  M 
)

Definition at line 2406 of file kaldi-matrix.cc.

References rnnlm::i, rnnlm::j, KALDI_SWAP4, KALDI_WARN, kaldi::MachineIsLittleEndian(), MatrixBase< Real >::NumCols(), MatrixBase< Real >::NumRows(), and MatrixBase< Real >::RowData().

Referenced by main().

2407 {
2408  // CMUSphinx mfc file header contains count of the floats, followed
2409  // by the data in float little endian format.
2410 
2411  int size = M.NumRows() * M.NumCols();
2412  os.write((char*)&size, sizeof(int));
2413  if (os.fail()) goto bad;
2414 
2415  MatrixIndexT i;
2416  MatrixIndexT j;
2417  if (sizeof(Real) == sizeof(float) && MachineIsLittleEndian()) {
2418  for (i = 0; i< M.NumRows(); i++) { // Unlikely to reach here ever!
2419  os.write((char*)M.RowData(i), sizeof(float)*M.NumCols());
2420  if (os.fail()) goto bad;
2421  }
2422  } else {
2423  float *pmem = new float[M.NumCols()];
2424 
2425  for (i = 0; i < M.NumRows(); i++) {
2426  const Real *rowData = M.RowData(i);
2427  for (j = 0;j < M.NumCols();j++)
2428  pmem[j] = static_cast<float> ( rowData[j] );
2429  if (!MachineIsLittleEndian())
2430  for (j = 0;j < M.NumCols();j++)
2431  KALDI_SWAP4(pmem[j]);
2432  os.write((char*)pmem, sizeof(float)*M.NumCols());
2433  if (os.fail()) {
2434  delete [] pmem;
2435  goto bad;
2436  }
2437  }
2438  delete [] pmem;
2439  }
2440  return true;
2441 bad:
2442  KALDI_WARN << "Could not write to Sphinx feature file";
2443  return false;
2444 }
int32 MatrixIndexT
Definition: matrix-common.h:96
int MachineIsLittleEndian()
Definition: kaldi-utils.h:79
#define KALDI_WARN
Definition: kaldi-error.h:130
#define KALDI_SWAP4(a)
Definition: kaldi-utils.h:103