All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
I/O related functions and classes

Various namespace-scope functions for I/O. More...

Classes

struct  HtkHeader
 A structure containing the HTK header. More...
 

Functions

template<typename Real >
bool ReadHtk (std::istream &is, Matrix< Real > *M_ptr, HtkHeader *header_ptr)
 Extension of the HTK header. More...
 
template<typename Real >
bool WriteHtk (std::ostream &os, const MatrixBase< Real > &M, HtkHeader htk_hdr)
 
template<class Real >
bool WriteSphinx (std::ostream &os, const MatrixBase< Real > &M)
 
template<typename Real >
std::ostream & operator<< (std::ostream &os, const MatrixBase< Real > &M)
 
template<typename Real >
std::istream & operator>> (std::istream &is, MatrixBase< Real > &M)
 
template<typename Real >
std::istream & operator>> (std::istream &is, Matrix< Real > &M)
 
template<typename Real >
bool SameDim (const MatrixBase< Real > &M, const MatrixBase< Real > &N)
 
template<typename Real >
std::ostream & operator<< (std::ostream &os, const VectorBase< Real > &rv)
 
template<typename Real >
std::istream & operator>> (std::istream &in, VectorBase< Real > &v)
 Input from a C++ stream. More...
 
template<typename Real >
std::istream & operator>> (std::istream &in, Vector< Real > &v)
 Input from a C++ stream. More...
 
template<typename Real >
std::ostream & operator<< (std::ostream &out, const PackedMatrix< Real > &M)
 
template<typename Real >
std::istream & operator>> (std::istream &is, PackedMatrix< Real > &M)
 

Detailed Description

Various namespace-scope functions for I/O.

end of "addtogroup matrix_group"

Output to a C++ stream.

Note that the << and >> operators are not recommended for normal use in Kaldi-style I/O (for that, use the Read and Write methods). The operators are mainly useful for logging and error output.

Non-binary by default (use Write for binary output).

Function Documentation

std::ostream & operator<< ( std::ostream &  os,
const VectorBase< Real > &  rv 
)

Definition at line 30 of file kaldi-vector-inl.h.

30  {
31  rv.Write(os, false);
32  return os;
33 }
std::ostream & operator<< ( std::ostream &  out,
const PackedMatrix< Real > &  M 
)

Definition at line 181 of file packed-matrix.h.

181  {
182  M.Write(os, false);
183  return os;
184 }
std::ostream & operator<< ( std::ostream &  os,
const MatrixBase< Real > &  M 
)
inline

Definition at line 41 of file kaldi-matrix-inl.h.

41  {
42  M.Write(os, false);
43  return os;
44 }
std::istream & operator>> ( std::istream &  in,
VectorBase< Real > &  v 
)

Input from a C++ stream.

Will automatically read text or binary data from the stream.

Definition at line 36 of file kaldi-vector-inl.h.

References VectorBase< Real >::Read().

36  {
37  rv.Read(is, false);
38  return is;
39 }
std::istream & operator>> ( std::istream &  in,
Vector< Real > &  v 
)

Input from a C++ stream.

Will automatically read text or binary data from the stream.

Definition at line 42 of file kaldi-vector-inl.h.

References Vector< Real >::Read().

42  {
43  rv.Read(is, false);
44  return is;
45 }
std::istream & operator>> ( std::istream &  is,
Matrix< Real > &  M 
)
inline

Definition at line 47 of file kaldi-matrix-inl.h.

References Matrix< Real >::Read().

47  {
48  M.Read(is, false);
49  return is;
50 }
std::istream & operator>> ( std::istream &  is,
MatrixBase< Real > &  M 
)
inline

Definition at line 54 of file kaldi-matrix-inl.h.

References MatrixBase< Real >::Read().

54  {
55  M.Read(is, false);
56  return is;
57 }
std::istream& kaldi::operator>> ( std::istream &  is,
PackedMatrix< Real > &  M 
)

Definition at line 187 of file packed-matrix.h.

187  {
188  M.Read(is, false);
189  return is;
190 }
bool ReadHtk ( std::istream &  is,
Matrix< Real > *  M_ptr,
HtkHeader *  header_ptr 
)

Extension of the HTK header.

Definition at line 2093 of file kaldi-matrix.cc.

References rnnlm::i, rnnlm::j, KALDI_ASSERT_IS_FLOATING_TYPE, KALDI_ERR, KALDI_SWAP2, KALDI_SWAP4, KALDI_VLOG, KALDI_WARN, kaldi::MachineIsLittleEndian(), HtkHeader::mNSamples, HtkHeader::mSampleKind, HtkHeader::mSamplePeriod, HtkHeader::mSampleSize, MatrixBase< Real >::NumCols(), MatrixBase< Real >::NumRows(), Matrix< Real >::Resize(), and MatrixBase< Real >::RowData().

Referenced by main(), HtkMatrixHolder::Read(), UnitTestHTKCompare1(), UnitTestHTKCompare2(), UnitTestHTKCompare3(), UnitTestHTKCompare4(), UnitTestHTKCompare5(), UnitTestHTKCompare6(), and kaldi::UnitTestHtkIo().

2094 {
2095  // check instantiated with double or float.
2097  Matrix<Real> &M = *M_ptr;
2098  HtkHeader htk_hdr;
2099 
2100  // TODO(arnab): this fails if the HTK file has CRC cheksum or is compressed.
2101  is.read((char*)&htk_hdr, sizeof(htk_hdr)); // we're being really POSIX here!
2102  if (is.fail()) {
2103  KALDI_WARN << "Could not read header from HTK feature file ";
2104  return false;
2105  }
2106 
2107  KALDI_SWAP4(htk_hdr.mNSamples);
2108  KALDI_SWAP4(htk_hdr.mSamplePeriod);
2109  KALDI_SWAP2(htk_hdr.mSampleSize);
2110  KALDI_SWAP2(htk_hdr.mSampleKind);
2111 
2112  bool has_checksum = false;
2113  {
2114  // See HParm.h in HTK code for sources of these things.
2115  enum BaseParmKind{
2116  Waveform, Lpc, Lprefc, Lpcepstra, Lpdelcep,
2117  Irefc, Mfcc, Fbank, Melspec, User, Discrete, Plp, Anon };
2118 
2119  const int32 IsCompressed = 02000, HasChecksum = 010000, HasVq = 040000,
2120  Problem = IsCompressed | HasVq;
2121  int32 base_parm = htk_hdr.mSampleKind & (077);
2122  has_checksum = (base_parm & HasChecksum) != 0;
2123  htk_hdr.mSampleKind &= ~HasChecksum; // We don't support writing with
2124  // checksum so turn it off.
2125  if (htk_hdr.mSampleKind & Problem)
2126  KALDI_ERR << "Code to read HTK features does not support compressed "
2127  "features, or features with VQ.";
2128  if (base_parm == Waveform || base_parm == Irefc || base_parm == Discrete)
2129  KALDI_ERR << "Attempting to read HTK features from unsupported type "
2130  "(e.g. waveform or discrete features.";
2131  }
2132 
2133  KALDI_VLOG(3) << "HTK header: Num Samples: " << htk_hdr.mNSamples
2134  << "; Sample period: " << htk_hdr.mSamplePeriod
2135  << "; Sample size: " << htk_hdr.mSampleSize
2136  << "; Sample kind: " << htk_hdr.mSampleKind;
2137 
2138  M.Resize(htk_hdr.mNSamples, htk_hdr.mSampleSize / sizeof(float));
2139 
2140  MatrixIndexT i;
2141  MatrixIndexT j;
2142  if (sizeof(Real) == sizeof(float)) {
2143  for (i = 0; i< M.NumRows(); i++) {
2144  is.read((char*)M.RowData(i), sizeof(float)*M.NumCols());
2145  if (is.fail()) {
2146  KALDI_WARN << "Could not read data from HTK feature file ";
2147  return false;
2148  }
2149  if (MachineIsLittleEndian()) {
2150  MatrixIndexT C = M.NumCols();
2151  for (j = 0; j < C; j++) {
2152  KALDI_SWAP4((M(i, j))); // The HTK standard is big-endian!
2153  }
2154  }
2155  }
2156  } else {
2157  float *pmem = new float[M.NumCols()];
2158  for (i = 0; i < M.NumRows(); i++) {
2159  is.read((char*)pmem, sizeof(float)*M.NumCols());
2160  if (is.fail()) {
2161  KALDI_WARN << "Could not read data from HTK feature file ";
2162  delete [] pmem;
2163  return false;
2164  }
2165  MatrixIndexT C = M.NumCols();
2166  for (j = 0; j < C; j++) {
2167  if (MachineIsLittleEndian()) // HTK standard is big-endian!
2168  KALDI_SWAP4(pmem[j]);
2169  M(i, j) = static_cast<Real>(pmem[j]);
2170  }
2171  }
2172  delete [] pmem;
2173  }
2174  if (header_ptr) *header_ptr = htk_hdr;
2175  if (has_checksum) {
2176  int16 checksum;
2177  is.read((char*)&checksum, sizeof(checksum));
2178  if (is.fail())
2179  KALDI_WARN << "Could not read checksum from HTK feature file ";
2180  // We ignore the checksum.
2181  }
2182  return true;
2183 }
#define KALDI_SWAP2(a)
Definition: kaldi-utils.h:110
OfflineFeatureTpl< MfccComputer > Mfcc
Definition: feature-mfcc.h:143
OfflineFeatureTpl< PlpComputer > Plp
Definition: feature-plp.h:167
int32 MatrixIndexT
Definition: matrix-common.h:96
OfflineFeatureTpl< FbankComputer > Fbank
int MachineIsLittleEndian()
Definition: kaldi-utils.h:79
#define KALDI_ERR
Definition: kaldi-error.h:127
#define KALDI_WARN
Definition: kaldi-error.h:130
#define KALDI_ASSERT_IS_FLOATING_TYPE(F)
Definition: kaldi-utils.h:133
#define KALDI_VLOG(v)
Definition: kaldi-error.h:136
#define KALDI_SWAP4(a)
Definition: kaldi-utils.h:103
bool kaldi::SameDim ( const MatrixBase< Real > &  M,
const MatrixBase< Real > &  N 
)

Definition at line 1031 of file kaldi-matrix.h.

References MatrixBase< Real >::NumCols(), and MatrixBase< Real >::NumRows().

Referenced by Fmpe::AccStats(), CuMatrixBase< Real >::AddDiagVecMat(), MatrixBase< Real >::AddDiagVecMat(), CuMatrixBase< Real >::AddMatDiagVec(), MatrixBase< Real >::AddMatDiagVec(), CuMatrixBase< Real >::AddMatMatElements(), CuMatrixBase< Real >::ApplyLogSoftMaxPerRow(), CuMatrixBase< Real >::ApplySoftMaxPerRow(), kaldi::nnet3::attention::AttentionBackward(), RestrictedAttentionComponent::Backprop(), LogSoftmaxComponent::Backprop(), DropoutComponent::Backprop(), BatchNormComponent::Backprop(), kaldi::cu::BackpropLstmNonlinearity(), RestrictedAttentionComponent::BackpropOneHead(), NnetUpdater::ComputeObjfAndDeriv(), kaldi::cu::CpuBackpropLstmNonlinearity(), CuMatrixBase< Real >::DiffLogSoftmaxPerRow(), CuMatrixBase< Real >::DiffSigmoid(), MatrixBase< Real >::DiffSigmoid(), CuMatrixBase< Real >::DiffSoftmaxPerRow(), MatrixBase< Real >::DiffTanh(), CuMatrixBase< Real >::Heaviside(), MatrixBase< Real >::Heaviside(), kaldi::cu::NormalizePerRow(), kaldi::nnet2::PreconditionDirections(), BatchNormComponent::Propagate(), kaldi::cu::RegularizeL1(), CuMatrixBase< Real >::Sigmoid(), MatrixBase< Real >::Sigmoid(), kaldi::SlidingWindowCmn(), CuMatrixBase< Real >::SoftHinge(), MatrixBase< Real >::SoftHinge(), CuMatrixBase< Real >::Tanh(), MatrixBase< Real >::Tanh(), kaldi::TypeOneUsage(), and Fmpe::Update().

1031  {
1032  return (M.NumRows() == N.NumRows() && M.NumCols() == N.NumCols());
1033 }
bool WriteHtk ( std::ostream &  os,
const MatrixBase< Real > &  M,
HtkHeader  htk_hdr 
)

Definition at line 2193 of file kaldi-matrix.cc.

References rnnlm::i, rnnlm::j, KALDI_ASSERT, KALDI_SWAP2, KALDI_SWAP4, KALDI_WARN, kaldi::MachineIsLittleEndian(), HtkHeader::mNSamples, HtkHeader::mSampleKind, HtkHeader::mSamplePeriod, HtkHeader::mSampleSize, MatrixBase< Real >::NumCols(), MatrixBase< Real >::NumRows(), and MatrixBase< Real >::RowData().

Referenced by main(), UnitTestHTKCompare1(), UnitTestHTKCompare2(), UnitTestHTKCompare3(), UnitTestHTKCompare4(), UnitTestHTKCompare5(), UnitTestHTKCompare6(), kaldi::UnitTestHtkIo(), and HtkMatrixHolder::Write().

2194 {
2195  KALDI_ASSERT(M.NumRows() == static_cast<MatrixIndexT>(htk_hdr.mNSamples));
2196  KALDI_ASSERT(M.NumCols() == static_cast<MatrixIndexT>(htk_hdr.mSampleSize) /
2197  static_cast<MatrixIndexT>(sizeof(float)));
2198 
2199  KALDI_SWAP4(htk_hdr.mNSamples);
2200  KALDI_SWAP4(htk_hdr.mSamplePeriod);
2201  KALDI_SWAP2(htk_hdr.mSampleSize);
2202  KALDI_SWAP2(htk_hdr.mSampleKind);
2203 
2204  os.write((char*)&htk_hdr, sizeof(htk_hdr));
2205  if (os.fail()) goto bad;
2206 
2207  MatrixIndexT i;
2208  MatrixIndexT j;
2209  if (sizeof(Real) == sizeof(float) && !MachineIsLittleEndian()) {
2210  for (i = 0; i< M.NumRows(); i++) { // Unlikely to reach here ever!
2211  os.write((char*)M.RowData(i), sizeof(float)*M.NumCols());
2212  if (os.fail()) goto bad;
2213  }
2214  } else {
2215  float *pmem = new float[M.NumCols()];
2216 
2217  for (i = 0; i < M.NumRows(); i++) {
2218  const Real *rowData = M.RowData(i);
2219  for (j = 0;j < M.NumCols();j++)
2220  pmem[j] = static_cast<float> ( rowData[j] );
2221  if (MachineIsLittleEndian())
2222  for (j = 0;j < M.NumCols();j++)
2223  KALDI_SWAP4(pmem[j]);
2224  os.write((char*)pmem, sizeof(float)*M.NumCols());
2225  if (os.fail()) {
2226  delete [] pmem;
2227  goto bad;
2228  }
2229  }
2230  delete [] pmem;
2231  }
2232  return true;
2233 bad:
2234  KALDI_WARN << "Could not write to HTK feature file ";
2235  return false;
2236 }
#define KALDI_SWAP2(a)
Definition: kaldi-utils.h:110
int32 MatrixIndexT
Definition: matrix-common.h:96
int MachineIsLittleEndian()
Definition: kaldi-utils.h:79
#define KALDI_WARN
Definition: kaldi-error.h:130
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
#define KALDI_SWAP4(a)
Definition: kaldi-utils.h:103
bool WriteSphinx ( std::ostream &  os,
const MatrixBase< Real > &  M 
)

Definition at line 2245 of file kaldi-matrix.cc.

References rnnlm::i, rnnlm::j, KALDI_SWAP4, KALDI_WARN, kaldi::MachineIsLittleEndian(), MatrixBase< Real >::NumCols(), MatrixBase< Real >::NumRows(), and MatrixBase< Real >::RowData().

Referenced by main().

2246 {
2247  // CMUSphinx mfc file header contains count of the floats, followed
2248  // by the data in float little endian format.
2249 
2250  int size = M.NumRows() * M.NumCols();
2251  os.write((char*)&size, sizeof(int));
2252  if (os.fail()) goto bad;
2253 
2254  MatrixIndexT i;
2255  MatrixIndexT j;
2256  if (sizeof(Real) == sizeof(float) && MachineIsLittleEndian()) {
2257  for (i = 0; i< M.NumRows(); i++) { // Unlikely to reach here ever!
2258  os.write((char*)M.RowData(i), sizeof(float)*M.NumCols());
2259  if (os.fail()) goto bad;
2260  }
2261  } else {
2262  float *pmem = new float[M.NumCols()];
2263 
2264  for (i = 0; i < M.NumRows(); i++) {
2265  const Real *rowData = M.RowData(i);
2266  for (j = 0;j < M.NumCols();j++)
2267  pmem[j] = static_cast<float> ( rowData[j] );
2268  if (!MachineIsLittleEndian())
2269  for (j = 0;j < M.NumCols();j++)
2270  KALDI_SWAP4(pmem[j]);
2271  os.write((char*)pmem, sizeof(float)*M.NumCols());
2272  if (os.fail()) {
2273  delete [] pmem;
2274  goto bad;
2275  }
2276  }
2277  delete [] pmem;
2278  }
2279  return true;
2280 bad:
2281  KALDI_WARN << "Could not write to Sphinx feature file";
2282  return false;
2283 }
int32 MatrixIndexT
Definition: matrix-common.h:96
int MachineIsLittleEndian()
Definition: kaldi-utils.h:79
#define KALDI_WARN
Definition: kaldi-error.h:130
#define KALDI_SWAP4(a)
Definition: kaldi-utils.h:103