1 // util/kaldi-holder.h
3 // Copyright 2009-2011 Microsoft Corporation
4 // 2016 Johns Hopkins University (author: Daniel Povey)
5 // 2016 Xiaohui Zhang
7 // See ../../COPYING for clarification regarding multiple authors
8 //
9 // Licensed under the Apache License, Version 2.0 (the "License");
10 // you may not use this file except in compliance with the License.
11 // You may obtain a copy of the License at
12 //
13 // http://www.apache.org/licenses/LICENSE-2.0
14 //
15 // THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 // KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
17 // WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
18 // MERCHANTABLITY OR NON-INFRINGEMENT.
19 // See the Apache 2 License for the specific language governing permissions and
20 // limitations under the License.
23 #ifndef KALDI_UTIL_KALDI_HOLDER_H_
24 #define KALDI_UTIL_KALDI_HOLDER_H_
26 #include <algorithm>
27 #include "util/kaldi-io.h"
28 #include "util/text-utils.h"
29 #include "matrix/kaldi-vector.h"
30 #include "matrix/sparse-matrix.h"
32 namespace kaldi {
35 // The Table class uses a Holder class to wrap objects, and make them behave
36 // in a "normalized" way w.r.t. reading and writing, so the Table class can
37 // be template-ized without too much trouble. Look below this
38 // comment (search for GenericHolder) to see what it looks like.
39 //
40 // Requirements of the holder class:
41 //
42 // They can only contain objects that can be read/written without external
43 // information; other objects cannot be stored in this type of archive.
44 //
45 // In terms of what functions it should have, see GenericHolder below.
46 // It is just for documentation.
47 //
48 // (1) Requirements of the Read and Write functions
49 //
50 // The Read and Write functions should have the property that in a longer
51 // file, if the Read function is started from where the Write function started
52 // writing, it should go to where the Write function stopped writing, in either
53 // text or binary mode (but it's OK if it doesn't eat up trailing space).
54 //
55 // [Desirable property: when writing in text mode the output should contain
56 // exactly one newline, at the end of the output; this makes it easier to
57 // manipulate]
58 //
59 // [Desirable property for classes: the output should just be a binary-mode
60 // header (if in binary mode and it's a Kaldi object, or no header
61 // othewise), and then the output of Object.Write(). This means that when
62 // written to individual files with the scp: type of wspecifier, we can
63 // read the individual files in the "normal" Kaldi way by reading the
64 // binary header and then the object.]
65 //
66 //
67 // The Write function takes a 'binary' argument. In general, each object will
68 // have two formats: text and binary. However, it's permitted to throw() if
69 // asked to read in the text format if there is none. The file will be open, if
70 // the file system has binary/text modes, in the corresponding mode. However,
71 // the object should have a file-mode in which it can read either text or binary
72 // output. It announces this via the static IsReadInBinary() function. This
73 // will generally be the binary mode and it means that where necessary, in text
74 // formats, we must ignore \r characters.
75 //
76 // Memory requirements: if it allocates memory, the destructor should
77 // free that memory. Copying and assignment of Holder objects may be
78 // disallowed as the Table code never does this.
81 /// GenericHolder serves to document the requirements of the Holder interface;
82 /// it's not intended to be used.
83 template<class SomeType> class GenericHolder {
84 public:
85 typedef SomeType T;
87 /// Must have a constructor that takes no arguments.
88 GenericHolder() { }
90 /// Write() writes this object of type T. Possibly also writes a binary-mode
91 /// header so that the Read function knows which mode to read in (since the
92 /// Read function does not get this information). It's a static member so we
93 /// can write those not inside this class (can use this function with Value()
94 /// to write from this class). The Write method may throw if it cannot write
95 /// the object in the given (binary/non-binary) mode. The holder object can
96 /// assume the stream has been opened in the given mode (where relevant). The
97 /// object can write the data how it likes.
98 static bool Write(std::ostream &os, bool binary, const T &t);
100 /// Reads into the holder. Must work out from the stream (which will be
101 /// opened on Windows in binary mode if the IsReadInBinary() function of this
102 /// class returns true, and text mode otherwise) whether the actual data is
103 /// binary or not (usually via reading the Kaldi binary-mode header).
104 /// We put the responsibility for reading the Kaldi binary-mode header in the
105 /// Read function (rather than making the binary mode an argument to this
106 /// function), so that for non-Kaldi binary files we don't have to write the
107 /// header, which would prevent the file being read by non-Kaldi programs
108 /// (e.g. if we write to individual files using an scp).
109 /// Read must deallocate any existing data we have here, if applicable (must
110 /// not assume the object was newly constructed).
111 /// Returns true on success.
112 /// If Read() returns false, the contents of this object and hence the value
113 /// returned by Value() may be undefined.
114 bool Read(std::istream &is);
116 /// IsReadInBinary() will return true if the object wants the file to be
117 /// opened in binary for reading (if the file system has binary/text modes),
118 /// and false otherwise. Static function. Kaldi objects always return true
119 /// as they always read in binary mode. Note that we must be able to read, in
120 /// this mode, objects written in both text and binary mode by Write (which
121 /// may mean ignoring "\r" characters). I doubt we will ever want this
122 /// function to return false.
123 static bool IsReadInBinary() { return true; }
125 /// Returns the value of the object held here. Will only
126 /// ever be called if Read() has been previously called and it returned
127 /// true (so OK to throw exception if no object was read).
128 const T &Value() const { return t_; } // if t is a pointer, would return *t_;
130 /// The Clear() function doesn't have to do anything. Its purpose is to
131 /// allow the object to free resources if they're no longer needed.
132 void Clear() { }
134 /// This swaps the objects held by *this and *other (preferably a shallow
135 /// swap). Note, this is just an example. The swap is with the *same type*
136 /// of holder, not with some nonexistent base-class (remember, GenericHolder is
137 /// an example for documentation, not a base-class).
138 void Swap(GenericHolder<T> *other) { std::swap(t_, other->t_); }
140 /// At the time of writing this will only do something meaningful
141 /// KaldiObjectHolder holding matrix objects, in order to extract a holder
142 /// holding a sub-matrix specified by 'range', e.g. [0:3,2:10], like in Matlab
143 /// but with zero-based indexing. It returns true with successful extraction
144 /// of the range, false if the range was invalid or outside the bounds of the
145 /// matrix. For other types of holder it just throws an error.
146 bool ExtractRange(const GenericHolder<T> &other, const std::string &range) {
147 KALDI_ERR << "ExtractRange is not defined for this type of holder.";
148 return false;
149 }
151 /// If the object held pointers, the destructor would free them.
152 ~GenericHolder() { }
154 private:
155 KALDI_DISALLOW_COPY_AND_ASSIGN(GenericHolder);
156 T t_; // t_ may alternatively be of type T*.
157 };
160 // See kaldi-holder-inl.h for examples of some actual Holder
161 // classes and templates.
164 // The following two typedefs should probably be in their own file, but they're
165 // here until there are enough of them to warrant their own header.
168 /// \addtogroup holders
169 /// @{
171 /// KaldiObjectHolder works for Kaldi objects that have the "standard" Read
172 /// and Write functions, and a copy constructor.
173 template<class KaldiType> class KaldiObjectHolder;
175 /// BasicHolder is valid for float, double, bool, and integer
176 /// types. There will be a compile time error otherwise, because
177 /// we make sure that the {Write, Read}BasicType functions do not
178 /// get instantiated for other types.
179 template<class BasicType> class BasicHolder;
182 // A Holder for a vector of basic types, e.g.
183 // std::vector<int32>, std::vector<float>, and so on.
184 // Note: a basic type is defined as a type for which ReadBasicType
185 // and WriteBasicType are implemented, i.e. integer and floating
186 // types, and bool.
187 template<class BasicType> class BasicVectorHolder;
190 // A holder for vectors of vectors of basic types, e.g.
191 // std::vector<std::vector<int32> >, and so on.
192 // Note: a basic type is defined as a type for which ReadBasicType
193 // and WriteBasicType are implemented, i.e. integer and floating
194 // types, and bool.
195 template<class BasicType> class BasicVectorVectorHolder;
197 // A holder for vectors of pairsof basic types, e.g.
198 // std::vector<std::pair<int32, int32> >, and so on.
199 // Note: a basic type is defined as a type for which ReadBasicType
200 // and WriteBasicType are implemented, i.e. integer and floating
201 // types, and bool. Text format is (e.g. for integers),
202 // "1 12 ; 43 61 ; 17 8 \n"
203 template<class BasicType> class BasicPairVectorHolder;
205 /// We define a Token (not a typedef, just a word) as a nonempty, printable,
206 /// whitespace-free std::string. The binary and text formats here are the same
207 /// (newline-terminated) and as such we don't bother with the binary-mode
208 /// headers.
209 class TokenHolder;
211 /// Class TokenVectorHolder is a Holder class for vectors of Tokens
212 /// (T == std::string).
213 class TokenVectorHolder;
215 /// A class for reading/writing HTK-format matrices.
216 /// T == std::pair<Matrix<BaseFloat>, HtkHeader>
217 class HtkMatrixHolder;
219 /// A class for reading/writing Sphinx format matrices.
220 template<int kFeatDim = 13> class SphinxMatrixHolder;
222 /// This templated function exists so that we can write .scp files with
223 /// 'object ranges' specified: the canonical example is a [first:last] range
224 /// of rows of a matrix, or [first-row:last-row,first-column,last-column]
225 /// of a matrix. We can also support [begin-time:end-time] of a wave
226 /// file. The string 'range' is whatever is in the square brackets; it is
227 /// parsed inside this function.
228 /// This function returns true if the partial object was successfully extracted,
229 /// and false if there was an error such as an invalid range.
230 /// The generic version of this function just fails; we overload the template
231 /// whenever we need it for a specific class.
232 template <class T>
233 bool ExtractObjectRange(const T &input, const std::string &range, T *output) {
234 KALDI_ERR << "Ranges not supported for objects of this type.";
235 return false;
236 }
238 /// The template is specialized with a version that actually does something,
239 /// for types Matrix<float> and Matrix<double>. We can later add versions of
240 /// this template for other types, such as Vector, which can meaningfully
241 /// have ranges extracted.
242 template <class Real>
243 bool ExtractObjectRange(const Matrix<Real> &input, const std::string &range,
244 Matrix<Real> *output);
246 /// The template is specialized types Vector<float> and Vector<double>.
247 template <class Real>
248 bool ExtractObjectRange(const Vector<Real> &input, const std::string &range,
249 Vector<Real> *output);
251 /// GeneralMatrix is always of type BaseFloat
252 bool ExtractObjectRange(const GeneralMatrix &input, const std::string &range,
253 GeneralMatrix *output);
255 /// CompressedMatrix is always of the type BaseFloat but it is more
256 /// efficient to provide template as it uses CompressedMatrix's own
257 /// conversion to Matrix<Real>
258 template <class Real>
259 bool ExtractObjectRange(const CompressedMatrix &input, const std::string &range,
260 Matrix<Real> *output);
262 // In SequentialTableReaderScriptImpl and RandomAccessTableReaderScriptImpl, for
263 // cases where the scp contained 'range specifiers' (things in square brackets
264 // identifying parts of objects like matrices), use this function to separate
265 // the input string 'rxfilename_with_range' (e.g "1.ark:100[1:2,2:10]") into the data_rxfilename
266 // (e.g. "1.ark:100") and the optional range specifier which will be everything
267 // inside the square brackets. It returns true if everything seems OK, and
268 // false if for example the string contained more than one '['. This function
269 // should only be called if 'line' ends in ']', otherwise it is an error.
270 bool ExtractRangeSpecifier(const std::string &rxfilename_with_range,
271 std::string *data_rxfilename,
272 std::string *range);
275 /// @} end "addtogroup holders"
278 } // end namespace kaldi
280 #include "util/kaldi-holder-inl.h"
282 #endif // KALDI_UTIL_KALDI_HOLDER_H_