kaldi-holder-inl.h
Go to the documentation of this file.
1 // util/kaldi-holder-inl.h
2 
3 // Copyright 2009-2011 Microsoft Corporation
4 // 2016 Xiaohui Zhang
5 
6 // See ../../COPYING for clarification regarding multiple authors
7 //
8 // Licensed under the Apache License, Version 2.0 (the "License");
9 // you may not use this file except in compliance with the License.
10 // You may obtain a copy of the License at
11 //
12 // http://www.apache.org/licenses/LICENSE-2.0
13 //
14 // THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 // KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
16 // WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
17 // MERCHANTABLITY OR NON-INFRINGEMENT.
18 // See the Apache 2 License for the specific language governing permissions and
19 // limitations under the License.
20 
21 
22 #ifndef KALDI_UTIL_KALDI_HOLDER_INL_H_
23 #define KALDI_UTIL_KALDI_HOLDER_INL_H_
24 
25 #include <algorithm>
26 #include <vector>
27 #include <utility>
28 #include <string>
29 
30 #include "base/kaldi-utils.h"
31 #include "util/kaldi-io.h"
32 #include "util/text-utils.h"
33 #include "matrix/kaldi-matrix.h"
34 
35 namespace kaldi {
36 
39 
40 
41 // KaldiObjectHolder is valid only for Kaldi objects with
42 // copy constructors, default constructors, and "normal"
43 // Kaldi Write and Read functions. E.g. it works for
44 // Matrix and Vector.
45 template<class KaldiType> class KaldiObjectHolder {
46  public:
47  typedef KaldiType T;
48 
49  KaldiObjectHolder(): t_(NULL) { }
50 
51  static bool Write(std::ostream &os, bool binary, const T &t) {
52  InitKaldiOutputStream(os, binary); // Puts binary header if binary mode.
53  try {
54  t.Write(os, binary);
55  return os.good();
56  } catch(const std::exception &e) {
57  KALDI_WARN << "Exception caught writing Table object. " << e.what();
58  return false; // Write failure.
59  }
60  }
61 
62  void Clear() {
63  if (t_) {
64  delete t_;
65  t_ = NULL;
66  }
67  }
68 
69  // Reads into the holder.
70  bool Read(std::istream &is) {
71  delete t_;
72  t_ = new T;
73  // Don't want any existing state to complicate the read function: get new
74  // object.
75  bool is_binary;
76  if (!InitKaldiInputStream(is, &is_binary)) {
77  KALDI_WARN << "Reading Table object, failed reading binary header\n";
78  return false;
79  }
80  try {
81  t_->Read(is, is_binary);
82  return true;
83  } catch(const std::exception &e) {
84  KALDI_WARN << "Exception caught reading Table object. " << e.what();
85  delete t_;
86  t_ = NULL;
87  return false;
88  }
89  }
90 
91  // Kaldi objects always have the stream open in binary mode for
92  // reading.
93  static bool IsReadInBinary() { return true; }
94 
95  T &Value() {
96  // code error if !t_.
97  if (!t_) KALDI_ERR << "KaldiObjectHolder::Value() called wrongly.";
98  return *t_;
99  }
100 
101  void Swap(KaldiObjectHolder<T> *other) {
102  // the t_ values are pointers so this is a shallow swap.
103  std::swap(t_, other->t_);
104  }
105 
107  const std::string &range) {
108  KALDI_ASSERT(other.t_ != NULL);
109  delete t_;
110  t_ = new T;
111  // this call will fail for most object types.
112  return ExtractObjectRange(*(other.t_), range, t_);
113  }
114 
115  ~KaldiObjectHolder() { delete t_; }
116  private:
118  T *t_;
119 };
120 
121 
122 // BasicHolder is valid for float, double, bool, and integer
123 // types. There will be a compile time error otherwise, because
124 // we make sure that the {Write, Read}BasicType functions do not
125 // get instantiated for other types.
126 
127 template<class BasicType> class BasicHolder {
128  public:
129  typedef BasicType T;
130 
131  BasicHolder(): t_(static_cast<T>(-1)) { }
132 
133  static bool Write(std::ostream &os, bool binary, const T &t) {
134  InitKaldiOutputStream(os, binary); // Puts binary header if binary mode.
135  try {
136  WriteBasicType(os, binary, t);
137  if (!binary) os << '\n'; // Makes output format more readable and
138  // easier to manipulate.
139  return os.good();
140  } catch(const std::exception &e) {
141  KALDI_WARN << "Exception caught writing Table object. " << e.what();
142  return false; // Write failure.
143  }
144  }
145 
146  void Clear() { }
147 
148  // Reads into the holder.
149  bool Read(std::istream &is) {
150  bool is_binary;
151  if (!InitKaldiInputStream(is, &is_binary)) {
152  KALDI_WARN << "Reading Table object [integer type], failed reading binary"
153  " header\n";
154  return false;
155  }
156  try {
157  int c;
158  if (!is_binary) { // This is to catch errors, the class would work
159  // without it..
160  // Eat up any whitespace and make sure it's not newline.
161  while (isspace((c = is.peek())) && c != static_cast<int>('\n')) {
162  is.get();
163  }
164  if (is.peek() == '\n') {
165  KALDI_WARN << "Found newline but expected basic type.";
166  return false; // This is just to catch a more-
167  // likely-than average type of error (empty line before the token),
168  // since ReadBasicType will eat it up.
169  }
170  }
171 
172  ReadBasicType(is, is_binary, &t_);
173 
174  if (!is_binary) { // This is to catch errors, the class would work
175  // without it..
176  // make sure there is a newline.
177  while (isspace((c = is.peek())) && c != static_cast<int>('\n')) {
178  is.get();
179  }
180  if (is.peek() != '\n') {
181  KALDI_WARN << "BasicHolder::Read, expected newline, got "
182  << CharToString(is.peek()) << ", position " << is.tellg();
183  return false;
184  }
185  is.get(); // Consume the newline.
186  }
187  return true;
188  } catch(const std::exception &e) {
189  KALDI_WARN << "Exception caught reading Table object. " << e.what();
190  return false;
191  }
192  }
193 
194  // Objects read/written with the Kaldi I/O functions always have the stream
195  // open in binary mode for reading.
196  static bool IsReadInBinary() { return true; }
197 
198  T &Value() {
199  return t_;
200  }
201 
202  void Swap(BasicHolder<T> *other) {
203  std::swap(t_, other->t_);
204  }
205 
206  bool ExtractRange(const BasicHolder<T> &other, const std::string &range) {
207  KALDI_ERR << "ExtractRange is not defined for this type of holder.";
208  return false;
209  }
210 
212  private:
214 
215  T t_;
216 };
217 
218 
224 template<class BasicType> class BasicVectorHolder {
225  public:
226  typedef std::vector<BasicType> T;
227 
229 
230  static bool Write(std::ostream &os, bool binary, const T &t) {
231  InitKaldiOutputStream(os, binary); // Puts binary header if binary mode.
232  try {
233  if (binary) { // need to write the size, in binary mode.
234  KALDI_ASSERT(static_cast<size_t>(static_cast<int32>(t.size())) ==
235  t.size());
236  // Or this Write routine cannot handle such a large vector.
237  // use int32 because it's fixed size regardless of compilation.
238  // change to int64 (plus in Read function) if this becomes a problem.
239  WriteBasicType(os, binary, static_cast<int32>(t.size()));
240  for (typename std::vector<BasicType>::const_iterator iter = t.begin();
241  iter != t.end(); ++iter)
242  WriteBasicType(os, binary, *iter);
243 
244  } else {
245  for (typename std::vector<BasicType>::const_iterator iter = t.begin();
246  iter != t.end(); ++iter)
247  WriteBasicType(os, binary, *iter);
248  os << '\n'; // Makes output format more readable and
249  // easier to manipulate. In text mode, this function writes something
250  // like "1 2 3\n".
251  }
252  return os.good();
253  } catch(const std::exception &e) {
254  KALDI_WARN << "Exception caught writing Table object (BasicVector). "
255  << e.what();
256  return false; // Write failure.
257  }
258  }
259 
260  void Clear() { t_.clear(); }
261 
262  // Reads into the holder.
263  bool Read(std::istream &is) {
264  t_.clear();
265  bool is_binary;
266  if (!InitKaldiInputStream(is, &is_binary)) {
267  KALDI_WARN << "Reading Table object [integer type], failed reading binary"
268  " header\n";
269  return false;
270  }
271  if (!is_binary) {
272  // In text mode, we terminate with newline.
273  std::string line;
274  getline(is, line); // this will discard the \n, if present.
275  if (is.fail()) {
276  KALDI_WARN << "BasicVectorHolder::Read, error reading line " <<
277  (is.eof() ? "[eof]" : "");
278  return false; // probably eof. fail in any case.
279  }
280  std::istringstream line_is(line);
281  try {
282  while (1) {
283  line_is >> std::ws; // eat up whitespace.
284  if (line_is.eof()) break;
285  BasicType bt;
286  ReadBasicType(line_is, false, &bt);
287  t_.push_back(bt);
288  }
289  return true;
290  } catch(const std::exception &e) {
291  KALDI_WARN << "BasicVectorHolder::Read, could not interpret line: "
292  << "'" << line << "'" << "\n" << e.what();
293  return false;
294  }
295  } else { // binary mode.
296  size_t filepos = is.tellg();
297  try {
298  int32 size;
299  ReadBasicType(is, true, &size);
300  t_.resize(size);
301  for (typename std::vector<BasicType>::iterator iter = t_.begin();
302  iter != t_.end();
303  ++iter) {
304  ReadBasicType(is, true, &(*iter));
305  }
306  return true;
307  } catch(...) {
308  KALDI_WARN << "BasicVectorHolder::Read, read error or unexpected data"
309  " at archive entry beginning at file position " << filepos;
310  return false;
311  }
312  }
313  }
314 
315  // Objects read/written with the Kaldi I/O functions always have the stream
316  // open in binary mode for reading.
317  static bool IsReadInBinary() { return true; }
318 
319  T &Value() { return t_; }
320 
322  t_.swap(other->t_);
323  }
324 
326  const std::string &range) {
327  KALDI_ERR << "ExtractRange is not defined for this type of holder.";
328  return false;
329  }
330 
332  private:
334  T t_;
335 };
336 
337 
343 template<class BasicType> class BasicVectorVectorHolder {
344  public:
345  typedef std::vector<std::vector<BasicType> > T;
346 
348 
349  static bool Write(std::ostream &os, bool binary, const T &t) {
350  InitKaldiOutputStream(os, binary); // Puts binary header if binary mode.
351  try {
352  if (binary) { // need to write the size, in binary mode.
353  KALDI_ASSERT(static_cast<size_t>(static_cast<int32>(t.size())) ==
354  t.size());
355  // Or this Write routine cannot handle such a large vector.
356  // use int32 because it's fixed size regardless of compilation.
357  // change to int64 (plus in Read function) if this becomes a problem.
358  WriteBasicType(os, binary, static_cast<int32>(t.size()));
359  for (typename std::vector<std::vector<BasicType> >::const_iterator
360  iter = t.begin();
361  iter != t.end(); ++iter) {
362  KALDI_ASSERT(static_cast<size_t>(static_cast<int32>(iter->size()))
363  == iter->size());
364  WriteBasicType(os, binary, static_cast<int32>(iter->size()));
365  for (typename std::vector<BasicType>::const_iterator
366  iter2 = iter->begin();
367  iter2 != iter->end(); ++iter2) {
368  WriteBasicType(os, binary, *iter2);
369  }
370  }
371  } else { // text mode...
372  // In text mode, we write out something like (for integers):
373  // "1 2 3 ; 4 5 ; 6 ; ; 7 8 9 ;\n"
374  // where the semicolon is a terminator, not a separator
375  // (a separator would cause ambiguity between an
376  // empty list, and a list containing a single empty list).
377  for (typename std::vector<std::vector<BasicType> >::const_iterator
378  iter = t.begin();
379  iter != t.end();
380  ++iter) {
381  for (typename std::vector<BasicType>::const_iterator
382  iter2 = iter->begin();
383  iter2 != iter->end(); ++iter2)
384  WriteBasicType(os, binary, *iter2);
385  os << "; ";
386  }
387  os << '\n';
388  }
389  return os.good();
390  } catch(const std::exception &e) {
391  KALDI_WARN << "Exception caught writing Table object. " << e.what();
392  return false; // Write failure.
393  }
394  }
395 
396  void Clear() { t_.clear(); }
397 
398  // Reads into the holder.
399  bool Read(std::istream &is) {
400  t_.clear();
401  bool is_binary;
402  if (!InitKaldiInputStream(is, &is_binary)) {
403  KALDI_WARN << "Failed reading binary header\n";
404  return false;
405  }
406  if (!is_binary) {
407  // In text mode, we terminate with newline.
408  try { // catching errors from ReadBasicType..
409  std::vector<BasicType> v; // temporary vector
410  while (1) {
411  int i = is.peek();
412  if (i == -1) {
413  KALDI_WARN << "Unexpected EOF";
414  return false;
415  } else if (static_cast<char>(i) == '\n') {
416  if (!v.empty()) {
417  KALDI_WARN << "No semicolon before newline (wrong format)";
418  return false;
419  } else {
420  is.get();
421  return true;
422  }
423  } else if (std::isspace(i)) {
424  is.get();
425  } else if (static_cast<char>(i) == ';') {
426  t_.push_back(v);
427  v.clear();
428  is.get();
429  } else { // some object we want to read...
430  BasicType b;
431  ReadBasicType(is, false, &b); // throws on error.
432  v.push_back(b);
433  }
434  }
435  } catch(const std::exception &e) {
436  KALDI_WARN << "BasicVectorVectorHolder::Read, read error. " << e.what();
437  return false;
438  }
439  } else { // binary mode.
440  size_t filepos = is.tellg();
441  try {
442  int32 size;
443  ReadBasicType(is, true, &size);
444  t_.resize(size);
445  for (typename std::vector<std::vector<BasicType> >::iterator
446  iter = t_.begin();
447  iter != t_.end();
448  ++iter) {
449  int32 size2;
450  ReadBasicType(is, true, &size2);
451  iter->resize(size2);
452  for (typename std::vector<BasicType>::iterator iter2 = iter->begin();
453  iter2 != iter->end();
454  ++iter2)
455  ReadBasicType(is, true, &(*iter2));
456  }
457  return true;
458  } catch(...) {
459  KALDI_WARN << "Read error or unexpected data at archive entry beginning"
460  " at file position " << filepos;
461  return false;
462  }
463  }
464  }
465 
466  // Objects read/written with the Kaldi I/O functions always have the stream
467  // open in binary mode for reading.
468  static bool IsReadInBinary() { return true; }
469 
470  T &Value() { return t_; }
471 
473  t_.swap(other->t_);
474  }
475 
477  const std::string &range) {
478  KALDI_ERR << "ExtractRange is not defined for this type of holder.";
479  return false;
480  }
481 
483  private:
485  T t_;
486 };
487 
488 
494 template<class BasicType> class BasicPairVectorHolder {
495  public:
496  typedef std::vector<std::pair<BasicType, BasicType> > T;
497 
499 
500  static bool Write(std::ostream &os, bool binary, const T &t) {
501  InitKaldiOutputStream(os, binary); // Puts binary header if binary mode.
502  try {
503  if (binary) { // need to write the size, in binary mode.
504  KALDI_ASSERT(static_cast<size_t>(static_cast<int32>(t.size())) ==
505  t.size());
506  // Or this Write routine cannot handle such a large vector.
507  // use int32 because it's fixed size regardless of compilation.
508  // change to int64 (plus in Read function) if this becomes a problem.
509  WriteBasicType(os, binary, static_cast<int32>(t.size()));
510  for (typename T::const_iterator iter = t.begin();
511  iter != t.end(); ++iter) {
512  WriteBasicType(os, binary, iter->first);
513  WriteBasicType(os, binary, iter->second);
514  }
515  } else { // text mode...
516  // In text mode, we write out something like (for integers):
517  // "1 2 ; 4 5 ; 6 7 ; 8 9 \n"
518  // where the semicolon is a separator, not a terminator.
519  for (typename T::const_iterator iter = t.begin();
520  iter != t.end();) {
521  WriteBasicType(os, binary, iter->first);
522  WriteBasicType(os, binary, iter->second);
523  ++iter;
524  if (iter != t.end())
525  os << "; ";
526  }
527  os << '\n';
528  }
529  return os.good();
530  } catch(const std::exception &e) {
531  KALDI_WARN << "Exception caught writing Table object. " << e.what();
532  return false; // Write failure.
533  }
534  }
535 
536  void Clear() { t_.clear(); }
537 
538  // Reads into the holder.
539  bool Read(std::istream &is) {
540  t_.clear();
541  bool is_binary;
542  if (!InitKaldiInputStream(is, &is_binary)) {
543  KALDI_WARN << "Reading Table object [integer type], failed reading binary"
544  " header\n";
545  return false;
546  }
547  if (!is_binary) {
548  // In text mode, we terminate with newline.
549  try { // catching errors from ReadBasicType..
550  std::vector<BasicType> v; // temporary vector
551  while (1) {
552  int i = is.peek();
553  if (i == -1) {
554  KALDI_WARN << "Unexpected EOF";
555  return false;
556  } else if (static_cast<char>(i) == '\n') {
557  if (t_.empty() && v.empty()) {
558  is.get();
559  return true;
560  } else if (v.size() == 2) {
561  t_.push_back(std::make_pair(v[0], v[1]));
562  is.get();
563  return true;
564  } else {
565  KALDI_WARN << "Unexpected newline, reading vector<pair<?> >; got "
566  << v.size() << " elements, expected 2.";
567  return false;
568  }
569  } else if (std::isspace(i)) {
570  is.get();
571  } else if (static_cast<char>(i) == ';') {
572  if (v.size() != 2) {
573  KALDI_WARN << "Wrong input format, reading vector<pair<?> >; got "
574  << v.size() << " elements, expected 2.";
575  return false;
576  }
577  t_.push_back(std::make_pair(v[0], v[1]));
578  v.clear();
579  is.get();
580  } else { // some object we want to read...
581  BasicType b;
582  ReadBasicType(is, false, &b); // throws on error.
583  v.push_back(b);
584  }
585  }
586  } catch(const std::exception &e) {
587  KALDI_WARN << "BasicPairVectorHolder::Read, read error. " << e.what();
588  return false;
589  }
590  } else { // binary mode.
591  size_t filepos = is.tellg();
592  try {
593  int32 size;
594  ReadBasicType(is, true, &size);
595  t_.resize(size);
596  for (typename T::iterator iter = t_.begin();
597  iter != t_.end();
598  ++iter) {
599  ReadBasicType(is, true, &(iter->first));
600  ReadBasicType(is, true, &(iter->second));
601  }
602  return true;
603  } catch(...) {
604  KALDI_WARN << "BasicVectorHolder::Read, read error or unexpected data"
605  " at archive entry beginning at file position " << filepos;
606  return false;
607  }
608  }
609  }
610 
611  // Objects read/written with the Kaldi I/O functions always have the stream
612  // open in binary mode for reading.
613  static bool IsReadInBinary() { return true; }
614 
615  T &Value() { return t_; }
616 
618  t_.swap(other->t_);
619  }
620 
622  const std::string &range) {
623  KALDI_ERR << "ExtractRange is not defined for this type of holder.";
624  return false;
625  }
626 
628  private:
630  T t_;
631 };
632 
633 
634 
635 
636 // We define a Token as a nonempty, printable, whitespace-free std::string.
637 // The binary and text formats here are the same (newline-terminated)
638 // and as such we don't bother with the binary-mode headers.
639 class TokenHolder {
640  public:
641  typedef std::string T;
642 
644 
645  static bool Write(std::ostream &os, bool, const T &t) { // ignore binary-mode
646  KALDI_ASSERT(IsToken(t));
647  os << t << '\n';
648  return os.good();
649  }
650 
651  void Clear() { t_.clear(); }
652 
653  // Reads into the holder.
654  bool Read(std::istream &is) {
655  is >> t_;
656  if (is.fail()) return false;
657  char c;
658  while (isspace(c = is.peek()) && c!= '\n') is.get();
659  if (is.peek() != '\n') {
660  KALDI_WARN << "TokenHolder::Read, expected newline, got char "
661  << CharToString(is.peek())
662  << ", at stream pos " << is.tellg();
663  return false;
664  }
665  is.get(); // get '\n'
666  return true;
667  }
668 
669 
670  // Since this is fundamentally a text format, read in text mode (would work
671  // fine either way, but doing it this way will exercise more of the code).
672  static bool IsReadInBinary() { return false; }
673 
674  T &Value() { return t_; }
675 
677 
678  void Swap(TokenHolder *other) {
679  t_.swap(other->t_);
680  }
681 
682  bool ExtractRange(const TokenHolder &other,
683  const std::string &range) {
684  KALDI_ERR << "ExtractRange is not defined for this type of holder.";
685  return false;
686  }
687 
688  private:
690  T t_;
691 };
692 
693 // A Token is a nonempty, whitespace-free std::string.
694 // Class TokenVectorHolder is a Holder class for vectors of these.
696  public:
697  typedef std::vector<std::string> T;
698 
700 
701  static bool Write(std::ostream &os, bool, const T &t) { // ignore binary-mode
702  for (std::vector<std::string>::const_iterator iter = t.begin();
703  iter != t.end();
704  ++iter) {
705  KALDI_ASSERT(IsToken(*iter)); // make sure it's whitespace-free,
706  // printable and nonempty.
707  os << *iter << ' ';
708  }
709  os << '\n';
710  return os.good();
711  }
712 
713  void Clear() { t_.clear(); }
714 
715 
716  // Reads into the holder.
717  bool Read(std::istream &is) {
718  t_.clear();
719 
720  // there is no binary/non-binary mode.
721 
722  std::string line;
723  getline(is, line); // this will discard the \n, if present.
724  if (is.fail()) {
725  KALDI_WARN << "BasicVectorHolder::Read, error reading line " << (is.eof()
726  ? "[eof]" : "");
727  return false; // probably eof. fail in any case.
728  }
729  const char *white_chars = " \t\n\r\f\v";
730  SplitStringToVector(line, white_chars, true, &t_); // true== omit
731  // empty strings e.g. between spaces.
732  return true;
733  }
734 
735  // Read in text format since it's basically a text-mode thing.. doesn't really
736  // matter, it would work either way since we ignore the extra '\r'.
737  static bool IsReadInBinary() { return false; }
738 
739  T &Value() { return t_; }
740 
741  void Swap(TokenVectorHolder *other) {
742  t_.swap(other->t_);
743  }
744 
745  bool ExtractRange(const TokenVectorHolder &other,
746  const std::string &range) {
747  KALDI_ERR << "ExtractRange is not defined for this type of holder.";
748  return false;
749  }
750 
751  private:
753  T t_;
754 };
755 
756 
758  public:
759  typedef std::pair<Matrix<BaseFloat>, HtkHeader> T;
760 
762 
763  static bool Write(std::ostream &os, bool binary, const T &t) {
764  if (!binary)
765  KALDI_ERR << "Non-binary HTK-format write not supported.";
766  bool ans = WriteHtk(os, t.first, t.second);
767  if (!ans)
768  KALDI_WARN << "Error detected writing HTK-format matrix.";
769  return ans;
770  }
771 
772  void Clear() { t_.first.Resize(0, 0); }
773 
774  // Reads into the holder.
775  bool Read(std::istream &is) {
776  bool ans = ReadHtk(is, &t_.first, &t_.second);
777  if (!ans) {
778  KALDI_WARN << "Error detected reading HTK-format matrix.";
779  return false;
780  }
781  return ans;
782  }
783 
784  // HTK-format matrices only read in binary.
785  static bool IsReadInBinary() { return true; }
786 
787  T &Value() { return t_; }
788 
789  void Swap(HtkMatrixHolder *other) {
790  t_.first.Swap(&(other->t_.first));
791  std::swap(t_.second, other->t_.second);
792  }
793 
794  bool ExtractRange(const HtkMatrixHolder &other,
795  const std::string &range) {
796  KALDI_ERR << "ExtractRange is not defined for this type of holder.";
797  return false;
798  }
799  // Default destructor.
800  private:
802  T t_;
803 };
804 
805 // SphinxMatrixHolder can be used to read and write feature files in
806 // CMU Sphinx format. 13-dimensional big-endian features are assumed.
807 // The ultimate reference is SphinxBase's source code (for example see
808 // feat_s2mfc_read() in src/libsphinxbase/feat/feat.c).
809 // We can't fully automate the detection of machine/feature file endianess
810 // mismatch here, because for this Sphinx relies on comparing the feature
811 // file's size with the number recorded in its header. We are working with
812 // streams, however(what happens if this is a Kaldi archive?). This should
813 // be no problem, because the usage help of Sphinx' "wave2feat" for example
814 // says that Sphinx features are always big endian.
815 // Note: the kFeatDim defaults to 13, see forward declaration in kaldi-holder.h
816 template<int kFeatDim> class SphinxMatrixHolder {
817  public:
819 
821 
822  void Clear() { feats_.Resize(0, 0); }
823 
824  // Writes Sphinx-format features
825  static bool Write(std::ostream &os, bool binary, const T &m) {
826  if (!binary) {
827  KALDI_WARN << "SphinxMatrixHolder can't write Sphinx features in text ";
828  return false;
829  }
830 
831  int32 size = m.NumRows() * m.NumCols();
832  if (MachineIsLittleEndian())
833  KALDI_SWAP4(size);
834  // write the header
835  os.write(reinterpret_cast<char*> (&size), sizeof(size));
836 
837  for (MatrixIndexT i = 0; i < m.NumRows(); i++) {
838  std::vector<float32> tmp(m.NumCols());
839  for (MatrixIndexT j = 0; j < m.NumCols(); j++) {
840  tmp[j] = static_cast<float32>(m(i, j));
841  if (MachineIsLittleEndian())
842  KALDI_SWAP4(tmp[j]);
843  }
844  os.write(reinterpret_cast<char*>(&(tmp[0])),
845  tmp.size() * 4);
846  }
847  return true;
848  }
849 
850  // Reads the features into a Kaldi Matrix
851  bool Read(std::istream &is) {
852  int32 nmfcc;
853 
854  is.read(reinterpret_cast<char*> (&nmfcc), sizeof(nmfcc));
855  if (MachineIsLittleEndian())
856  KALDI_SWAP4(nmfcc);
857  KALDI_VLOG(2) << "#feats: " << nmfcc;
858  int32 nfvec = nmfcc / kFeatDim;
859  if ((nmfcc % kFeatDim) != 0) {
860  KALDI_WARN << "Sphinx feature count is inconsistent with vector length ";
861  return false;
862  }
863 
864  feats_.Resize(nfvec, kFeatDim);
865  for (MatrixIndexT i = 0; i < feats_.NumRows(); i++) {
866  if (sizeof(BaseFloat) == sizeof(float32)) {
867  is.read(reinterpret_cast<char*> (feats_.RowData(i)),
868  kFeatDim * sizeof(float32));
869  if (!is.good()) {
870  KALDI_WARN << "Unexpected error/EOF while reading Sphinx features ";
871  return false;
872  }
873  if (MachineIsLittleEndian()) {
874  for (MatrixIndexT j = 0; j < kFeatDim; j++)
875  KALDI_SWAP4(feats_(i, j));
876  }
877  } else { // KALDI_DOUBLEPRECISION=1
878  float32 tmp[kFeatDim];
879  is.read(reinterpret_cast<char*> (tmp), sizeof(tmp));
880  if (!is.good()) {
881  KALDI_WARN << "Unexpected error/EOF while reading Sphinx features ";
882  return false;
883  }
884  for (MatrixIndexT j = 0; j < kFeatDim; j++) {
885  if (MachineIsLittleEndian())
886  KALDI_SWAP4(tmp[j]);
887  feats_(i, j) = static_cast<BaseFloat>(tmp[j]);
888  }
889  }
890  }
891 
892  return true;
893  }
894 
895  // Only read in binary
896  static bool IsReadInBinary() { return true; }
897 
898  T &Value() { return feats_; }
899 
900  void Swap(SphinxMatrixHolder *other) {
901  feats_.Swap(&(other->feats_));
902  }
903 
904  bool ExtractRange(const SphinxMatrixHolder &other,
905  const std::string &range) {
906  KALDI_ERR << "ExtractRange is not defined for this type of holder.";
907  return false;
908  }
909 
910  private:
913 };
914 
915 
917 
918 } // end namespace kaldi
919 
920 
921 
922 #endif // KALDI_UTIL_KALDI_HOLDER_INL_H_
This code computes Goodness of Pronunciation (GOP) and extracts phone-level pronunciation feature for...
Definition: chain.dox:20
bool InitKaldiInputStream(std::istream &is, bool *binary)
Initialize an opened stream for reading by detecting the binary header and.
Definition: io-funcs-inl.h:306
KaldiObjectHolder works for Kaldi objects that have the "standard" Read and Write functions...
bool ExtractRange(const BasicVectorHolder< BasicType > &other, const std::string &range)
bool ExtractRange(BasicVectorVectorHolder< BasicType > &other, const std::string &range)
static bool Write(std::ostream &os, bool, const T &t)
bool Read(std::istream &is)
void Swap(TokenVectorHolder *other)
MatrixIndexT NumCols() const
Returns number of columns (or zero for empty matrix).
Definition: kaldi-matrix.h:67
bool ExtractRange(const HtkMatrixHolder &other, const std::string &range)
void ReadBasicType(std::istream &is, bool binary, T *t)
ReadBasicType is the name of the read function for bool, integer types, and floating-point types...
Definition: io-funcs-inl.h:55
std::vector< BasicType > T
bool WriteHtk(std::ostream &os, const MatrixBase< Real > &M, HtkHeader htk_hdr)
A Holder for a vector of basic types, e.g.
bool ExtractRange(const BasicHolder< T > &other, const std::string &range)
static bool Write(std::ostream &os, bool binary, const T &t)
std::vector< std::pair< BasicType, BasicType > > T
void Swap(BasicHolder< T > *other)
bool ExtractRange(const BasicPairVectorHolder< BasicType > &other, const std::string &range)
bool Read(std::istream &is)
bool ExtractRange(const TokenHolder &other, const std::string &range)
void swap(basic_filebuf< CharT, Traits > &x, basic_filebuf< CharT, Traits > &y)
kaldi::int32 int32
bool Read(std::istream &is)
void Swap(BasicPairVectorHolder< BasicType > *other)
A class for reading/writing Sphinx format matrices.
bool IsToken(const std::string &token)
Returns true if "token" is nonempty, and all characters are printable and whitespace-free.
Definition: text-utils.cc:105
void Swap(BasicVectorHolder< BasicType > *other)
static bool Write(std::ostream &os, bool binary, const T &t)
bool Read(std::istream &is)
static bool IsReadInBinary()
static bool Write(std::ostream &os, bool binary, const T &m)
bool ExtractObjectRange(const GeneralMatrix &input, const std::string &range, GeneralMatrix *output)
GeneralMatrix is always of type BaseFloat.
Definition: kaldi-holder.cc:88
int32 MatrixIndexT
Definition: matrix-common.h:98
BasicHolder is valid for float, double, bool, and integer types.
bool Read(std::istream &is)
std::pair< Matrix< BaseFloat >, HtkHeader > T
static bool Write(std::ostream &os, bool binary, const T &t)
KALDI_DISALLOW_COPY_AND_ASSIGN(KaldiObjectHolder)
static bool Write(std::ostream &os, bool binary, const T &t)
void SplitStringToVector(const std::string &full, const char *delim, bool omit_empty_strings, std::vector< std::string > *out)
Split a string using any of the single character delimiters.
Definition: text-utils.cc:63
float float32
Definition: kaldi-types.h:53
bool Read(std::istream &is)
int MachineIsLittleEndian()
Definition: kaldi-utils.h:83
#define KALDI_ERR
Definition: kaldi-error.h:147
std::vector< std::string > T
void Swap(SphinxMatrixHolder *other)
#define KALDI_WARN
Definition: kaldi-error.h:150
BasicVectorVectorHolder is a Holder for a vector of vector of a basic type, e.g.
static bool IsReadInBinary()
static bool Write(std::ostream &os, bool binary, const T &t)
void Swap(BasicVectorVectorHolder< BasicType > *other)
std::vector< std::vector< BasicType > > T
bool ExtractRange(const KaldiObjectHolder< T > &other, const std::string &range)
std::string CharToString(const char &c)
Definition: kaldi-utils.cc:36
bool Read(std::istream &is)
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:185
static bool Write(std::ostream &os, bool, const T &t)
MatrixIndexT NumRows() const
Returns number of rows (or zero for empty matrix).
Definition: kaldi-matrix.h:64
BasicPairVectorHolder is a Holder for a vector of pairs of a basic type, e.g.
bool ExtractRange(const TokenVectorHolder &other, const std::string &range)
#define KALDI_VLOG(v)
Definition: kaldi-error.h:156
void WriteBasicType(std::ostream &os, bool binary, T t)
WriteBasicType is the name of the write function for bool, integer types, and floating-point types...
Definition: io-funcs-inl.h:34
void InitKaldiOutputStream(std::ostream &os, bool binary)
InitKaldiOutputStream initializes an opened stream for writing by writing an optional binary header a...
Definition: io-funcs-inl.h:291
bool Read(std::istream &is)
void Swap(KaldiObjectHolder< T > *other)
static bool IsReadInBinary()
bool ExtractRange(const SphinxMatrixHolder &other, const std::string &range)
static bool Write(std::ostream &os, bool binary, const T &t)
bool Read(std::istream &is)
bool ReadHtk(std::istream &is, Matrix< Real > *M_ptr, HtkHeader *header_ptr)
Extension of the HTK header.
A structure containing the HTK header.
Definition: kaldi-matrix.h:955
void Swap(HtkMatrixHolder *other)
#define KALDI_SWAP4(a)
Definition: kaldi-utils.h:107
void Swap(TokenHolder *other)