kaldi-io.h
Go to the documentation of this file.
1 // util/kaldi-io.h
2 
3 // Copyright 2009-2011 Microsoft Corporation; Jan Silovsky
4 // 2016 Xiaohui Zhang
5 
6 // See ../../COPYING for clarification regarding multiple authors
7 //
8 // Licensed under the Apache License, Version 2.0 (the "License");
9 // you may not use this file except in compliance with the License.
10 // You may obtain a copy of the License at
11 
12 // http://www.apache.org/licenses/LICENSE-2.0
13 
14 // THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 // KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
16 // WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
17 // MERCHANTABLITY OR NON-INFRINGEMENT.
18 // See the Apache 2 License for the specific language governing permissions and
19 // limitations under the License.
20 #ifndef KALDI_UTIL_KALDI_IO_H_
21 #define KALDI_UTIL_KALDI_IO_H_
22 
23 #ifdef _MSC_VER
24 # include <fcntl.h>
25 # include <io.h>
26 #endif
27 #include <cctype> // For isspace.
28 #include <limits>
29 #include <string>
30 #include "base/kaldi-common.h"
31 #include "matrix/kaldi-matrix.h"
32 
33 
34 namespace kaldi {
35 
36 class OutputImplBase; // Forward decl; defined in a .cc file
37 class InputImplBase; // Forward decl; defined in a .cc file
38 
41 
42 // The Output and Input classes handle stream-opening for "extended" filenames
43 // that include actual files, standard-input/standard-output, pipes, and
44 // offsets into actual files. They also handle reading and writing the
45 // binary-mode headers for Kaldi files, where applicable. The classes have
46 // versions of the Open routines that throw and do not throw, depending whether
47 // the calling code wants to catch the errors or not; there are also versions
48 // that write (or do not write) the Kaldi binary-mode header that says if it's
49 // binary mode. Generally files that contain Kaldi objects will have the header
50 // on, so we know upon reading them whether they have the header. So you would
51 // use the OpenWithHeader routines for these (or the constructor); but other
52 // types of objects (e.g. FSTs) would have files without a header so you would
53 // use OpenNoHeader.
54 
55 // We now document the types of extended filenames that we use.
56 //
57 // A "wxfilename" is an extended filename for writing. It can take three forms:
58 // (1) Filename: e.g. "/some/filename", "./a/b/c", "c:\Users\dpovey\My
59 // Documents\\boo"
60 // (whatever the actual file-system interprets)
61 // (2) Standard output: "" or "-"
62 // (3) A pipe: e.g. "| gzip -c > /tmp/abc.gz"
63 //
64 //
65 // A "rxfilename" is an extended filename for reading. It can take four forms:
66 // (1) An actual filename, whatever the file-system can read, e.g. "/my/file".
67 // (2) Standard input: "" or "-"
68 // (3) A pipe: e.g. "gunzip -c /tmp/abc.gz |"
69 // (4) An offset into a file, e.g.: "/mnt/blah/data/1.ark:24871"
70 // [these are created by the Table and TableWriter classes; I may also write
71 // a program that creates them for arbitrary files]
72 //
73 
74 
75 // Typical usage:
76 // ...
77 // bool binary;
78 // MyObject.Write(Output(some_filename, binary).Stream(), binary);
79 //
80 // ... more extensive example:
81 // {
82 // Output ko(some_filename, binary);
83 // MyObject1.Write(ko.Stream(), binary);
84 // MyObject2.Write(ko.Stream(), binary);
85 // }
86 
87 
88 
89 enum OutputType {
94 };
95 
103 OutputType ClassifyWxfilename(const std::string &wxfilename);
104 
105 enum InputType {
111 };
112 
121 InputType ClassifyRxfilename(const std::string &rxfilename);
122 
123 
124 class Output {
125  public:
126  // The normal constructor, provided for convenience.
127  // Equivalent to calling with default constructor then Open()
128  // with these arguments.
129  Output(const std::string &filename, bool binary, bool write_header = true);
130 
131  Output(): impl_(NULL) {}
132 
140  bool Open(const std::string &wxfilename, bool binary, bool write_header);
141 
142  inline bool IsOpen(); // return true if we have an open stream. Does not
143  // imply stream is good for writing.
144 
145  std::ostream &Stream(); // will throw if not open; else returns stream.
146 
147  // Close closes the stream. Calling Close is never necessary unless you
148  // want to avoid exceptions being thrown. There are times when calling
149  // Close will hurt efficiency (basically, when using offsets into files,
150  // and using the same Input object),
151  // but most of the time the user won't be doing this directly, it will
152  // be done in kaldi-table.{h, cc}, so you don't have to worry about it.
153  bool Close();
154 
155  // This will throw if stream could not be closed (to check error status,
156  // call Close()).
157  ~Output();
158 
159  private:
160  OutputImplBase *impl_; // non-NULL if open.
161  std::string filename_;
163 };
164 
165 
166 // bool binary_in;
167 // Input ki(some_filename, &binary_in);
168 // MyObject.Read(ki.Stream(), binary_in);
169 //
170 // ... more extensive example:
171 //
172 // {
173 // bool binary_in;
174 // Input ki(some_filename, &binary_in);
175 // MyObject1.Read(ki.Stream(), &binary_in);
176 // MyObject2.Write(ki.Stream(), &binary_in);
177 // }
178 // Note that to catch errors you need to use try.. catch.
179 // Input communicates errors by throwing exceptions.
180 
181 
182 // Input interprets four kinds of filenames:
183 // (1) Normal filenames
184 // (2) The empty string or "-", interpreted as standard output
185 // (3) A pipe: e.g. "gunzip -c /tmp/abc.gz |"
186 // (4) Offsets into [real] files, e.g. "/my/filename:12049"
187 // The last one has no correspondence in Output.
188 
189 
190 class Input {
191  public:
196  Input(const std::string &rxfilename, bool *contents_binary = NULL);
197 
198  Input(): impl_(NULL) {}
199 
200  // Open opens the stream for reading (the mode, where relevant, is binary; use
201  // OpenTextMode for text-mode, we made this a separate function rather than a
202  // boolean argument, to avoid confusion with Kaldi's text/binary distinction,
203  // since reading in the file system's text mode is unusual.) If
204  // contents_binary != NULL, it reads the binary-mode header and puts it in the
205  // "binary" variable. Returns true on success. If it returns false it will
206  // not be open. You may call Open even if it is already open; it will close
207  // the existing stream and reopen (however if closing the old stream failed it
208  // will throw).
209  inline bool Open(const std::string &rxfilename, bool *contents_binary = NULL);
210 
211  // As Open but (if the file system has text/binary modes) opens in text mode;
212  // you shouldn't ever have to use this as in Kaldi we read even text files in
213  // binary mode (and ignore the \r).
214  inline bool OpenTextMode(const std::string &rxfilename);
215 
216  // Return true if currently open for reading and Stream() will
217  // succeed. Does not guarantee that the stream is good.
218  inline bool IsOpen();
219 
220  // It is never necessary or helpful to call Close, except if
221  // you are concerned about to many filehandles being open.
222  // Close does not throw. It returns the exit code as int32
223  // in the case of a pipe [kPipeInput], and always zero otherwise.
224  int32 Close();
225 
226  // Returns the underlying stream. Throws if !IsOpen()
227  std::istream &Stream();
228 
229  // Destructor does not throw: input streams may legitimately fail so we
230  // don't worry about the status when we close them.
231  ~Input();
232  private:
233  bool OpenInternal(const std::string &rxfilename, bool file_binary,
234  bool *contents_binary);
237 };
238 
239 template <class C> void ReadKaldiObject(const std::string &filename,
240  C *c) {
241  bool binary_in;
242  Input ki(filename, &binary_in);
243  c->Read(ki.Stream(), binary_in);
244 }
245 
246 // Specialize the template for reading matrices, because we want to be able to
247 // support reading 'ranges' (row and column ranges), like foo.mat[10:20].
248 template <> void ReadKaldiObject(const std::string &filename,
249  Matrix<float> *m);
250 
251 
252 template <> void ReadKaldiObject(const std::string &filename,
253  Matrix<double> *m);
254 
255 
256 
257 template <class C> inline void WriteKaldiObject(const C &c,
258  const std::string &filename,
259  bool binary) {
260  Output ko(filename, binary);
261  c.Write(ko.Stream(), binary);
262 }
263 
267 std::string PrintableRxfilename(const std::string &rxfilename);
268 
272 std::string PrintableWxfilename(const std::string &wxfilename);
273 
275 
276 } // end namespace kaldi.
277 
278 #include "util/kaldi-io-inl.h"
279 
280 #endif // KALDI_UTIL_KALDI_IO_H_
This code computes Goodness of Pronunciation (GOP) and extracts phone-level pronunciation feature for...
Definition: chain.dox:20
InputType ClassifyRxfilename(const std::string &filename)
ClassifyRxfilenames interprets filenames for reading as follows:
Definition: kaldi-io.cc:138
InputImplBase * impl_
Definition: kaldi-io.h:235
kaldi::int32 int32
A class for storing matrices.
Definition: kaldi-matrix.h:823
void ReadKaldiObject(const std::string &filename, Matrix< float > *m)
Definition: kaldi-io.cc:832
std::istream & Stream()
Definition: kaldi-io.cc:826
std::ostream & Stream()
Definition: kaldi-io.cc:701
OutputType
Definition: kaldi-io.h:89
KALDI_DISALLOW_COPY_AND_ASSIGN(Output)
bool Open(const std::string &wxfilename, bool binary, bool write_header)
This opens the stream, with the given mode (binary or text).
Definition: kaldi-io.cc:707
InputType
Definition: kaldi-io.h:105
OutputType ClassifyWxfilename(const std::string &filename)
ClassifyWxfilename interprets filenames as follows:
Definition: kaldi-io.cc:85
void WriteKaldiObject(const C &c, const std::string &filename, bool binary)
Definition: kaldi-io.h:257
std::string PrintableRxfilename(const std::string &rxfilename)
PrintableRxfilename turns the rxfilename into a more human-readable form for error reporting...
Definition: kaldi-io.cc:61
std::string PrintableWxfilename(const std::string &wxfilename)
PrintableWxfilename turns the wxfilename into a more human-readable form for error reporting...
Definition: kaldi-io.cc:73
OutputImplBase * impl_
Definition: kaldi-io.h:160
std::string filename_
Definition: kaldi-io.h:161
bool Close()
Definition: kaldi-io.cc:677