RandomAccessTableReaderScriptImpl< Holder > Class Template Reference

#include <kaldi-table-inl.h>

Inheritance diagram for RandomAccessTableReaderScriptImpl< Holder >:

Collaboration diagram for RandomAccessTableReaderScriptImpl< Holder >:

Public Types
typedef Holder::T	T
Public Types inherited from RandomAccessTableReaderImplBase< Holder >
typedef Holder::T	T

Public Member Functions
	RandomAccessTableReaderScriptImpl ()
virtual bool	Open (const std::string &rspecifier)
virtual bool	IsOpen () const
virtual bool	Close ()
virtual bool	HasKey (const std::string &key)
virtual const T &	Value (const std::string &key)
virtual	~RandomAccessTableReaderScriptImpl ()
Public Member Functions inherited from RandomAccessTableReaderImplBase< Holder >
virtual	~RandomAccessTableReaderImplBase ()

Private Types
enum	{   kUninitialized, kNotReadScript, kNotHaveObject, kHaveObject,   kHaveRange }

Private Member Functions
virtual bool	HasKeyInternal (const std::string &key, bool preload)
bool	LookupKey (const std::string &key, size_t *script_offset)

Private Attributes
Input	input_
RspecifierOptions	opts_
std::string	rspecifier_
std::string	script_rxfilename_
std::string	key_
Holder	holder_
Holder	range_holder_
std::string	range_
std::string	data_rxfilename_
std::vector< std::pair< std::string, std::string > >	script_
size_t	last_found_
enum kaldi::RandomAccessTableReaderScriptImpl:: { ... }	state_

Detailed Description

template<class Holder>
class kaldi::RandomAccessTableReaderScriptImpl< Holder >

Definition at line 1593 of file kaldi-table-inl.h.

Member Typedef Documentation

T

typedef Holder::T T

Definition at line 1596 of file kaldi-table-inl.h.

Member Enumeration Documentation

anonymous enum

anonymous enum

private

Enumerator
kUninitialized
kNotReadScript
kNotHaveObject
kHaveObject
kHaveRange

Definition at line 1863 of file kaldi-table-inl.h.

       {
    //                   (*) is script_ set up?
    //                          (*) does holder_ contain an object?
    //                               (*) does range_holder_ contain and object?
    //
    //
    kUninitialized,  //    no    no    no
    kNotReadScript,  //    no    no    no
    kNotHaveObject,  //    yes   no    no
    kHaveObject,     //    yes   yes   no
    kHaveRange,      //    yes   yes   yes

    // If we are in a state where holder_ contains an object, it always contains
    // the object from 'key_', and the corresponding rxfilename is always
    // 'data_rxfilename_'.  If range_holder_ contains an object, it always
    // corresponds to the range 'range_' of the object in 'holder_', and always
    // corresponds to the current key.
  } state_;

Constructor & Destructor Documentation

RandomAccessTableReaderScriptImpl()

RandomAccessTableReaderScriptImpl ( )

inline

Definition at line 1598 of file kaldi-table-inl.h.

: last_found_(0), state_(kUninitialized) {}

~RandomAccessTableReaderScriptImpl()

virtual ~RandomAccessTableReaderScriptImpl ( )

inlinevirtual

Definition at line 1693 of file kaldi-table-inl.h.

{ }

Member Function Documentation

Close()

virtual bool Close ( )

inlinevirtual

Implements RandomAccessTableReaderImplBase< Holder >.

Definition at line 1652 of file kaldi-table-inl.h.

References TableWriter< Holder >::IsOpen(), and KALDI_ERR.

                       {
    if (!IsOpen())
      KALDI_ERR << "Close() called on RandomAccessTableReader that was not"
                   " open.";
    holder_.Clear();
    range_holder_.Clear();
    state_ = kUninitialized;
    last_found_ = 0;
    script_.clear();
    key_ = "";
    range_ = "";
    data_rxfilename_ = "";
    // This cannot fail because any errors of a "global" nature would have been
    // detected when we did Open().  With archives it's different.
    return true;
  }

HasKey()

virtual bool HasKey ( const std::string &  key )

inlinevirtual

Implements RandomAccessTableReaderImplBase< Holder >.

Definition at line 1669 of file kaldi-table-inl.h.

                                            {
    bool preload = opts_.permissive;
    // In permissive mode, we have to check that we can read
    // the scp entry before we assert that the key is there.
    return HasKeyInternal(key, preload);
  }

HasKeyInternal()

virtual bool HasKeyInternal ( const std::string &  key, bool  preload  )

inlineprivatevirtual

Definition at line 1703 of file kaldi-table-inl.h.

References kaldi::ExtractRangeSpecifier(), kaldi::IsToken(), KALDI_ASSERT, KALDI_ERR, KALDI_WARN, and kaldi::PrintableRxfilename().

                                                                  {
    switch (state_) {
      case kUninitialized: case kNotReadScript:
        KALDI_ERR << "HasKey called on RandomAccessTableReader object that is"
                     " not open.";
      case kHaveObject:
        if (key == key_ && range_.empty())
          return true;
        break;
      case kHaveRange:
        if (key == key_)
          return true;
        break;
      case kNotHaveObject: default: break;
    }
    KALDI_ASSERT(IsToken(key));
    size_t key_pos = 0;
    if (!LookupKey(key, &key_pos)) {
      return false;
    } else {
      if (!preload) {
        return true;  // we have the key, and were not asked to verify that the
                      // object could be read.
      } else {  // preload specified, so we have to attempt to pre-load the
                // object before returning.
        std::string data_rxfilename, range; // We will split
        // script_[key_pos].second (e.g. "1.ark:100[0:2]" into data_rxfilename
        // (e.g. "1.ark:100") and range (if any), e.g. "0:2".
        if (script_[key_pos].second[script_[key_pos].second.size()-1] == ']') {
          if(!ExtractRangeSpecifier(script_[key_pos].second,
                                    &data_rxfilename,
                                    &range)) {
            KALDI_ERR << "TableReader: failed to parse range in '"
                      << script_[key_pos].second << "'";
          }
        } else {
          data_rxfilename = script_[key_pos].second;
        }
        if (state_ == kHaveRange) {
          if (data_rxfilename_ == data_rxfilename && range_ == range) {
            // the odd situation where two keys had the same rxfilename and range:
            // just change the key and keep the object.
            key_ = key;
            return true;
          } else {
            range_holder_.Clear();
            state_ = kHaveObject;
          }
        }
        // OK, at this point the state will be kHaveObject or kNotHaveObject.
        if (state_ == kHaveObject) {
          if (data_rxfilename_ != data_rxfilename) {
            // clear out the object.
            state_ = kNotHaveObject;
            holder_.Clear();
          }
        }
        // At this point we can safely switch to the new key, data_rxfilename
        // and range, and we know that if we have an object, it will already be
        // the correct one.  The state is now kHaveObject or kNotHaveObject.
        key_ = key;
        data_rxfilename_ = data_rxfilename;
        range_ = range;
        if (state_ == kNotHaveObject) {
          // we need to read the object.
          if (!input_.Open(data_rxfilename)) {
            KALDI_WARN << "Error opening stream "
                       << PrintableRxfilename(data_rxfilename);
            return false;
          } else {
            if (holder_.Read(input_.Stream())) {
              state_ = kHaveObject;
            } else {
              KALDI_WARN << "Error reading object from "
                  "stream " << PrintableRxfilename(data_rxfilename);
              return false;
            }
          }
        }
        // At this point the state is kHaveObject.
        if (range.empty())
          return true;  // we're done: no range was requested.
        if (range_holder_.ExtractRange(holder_, range)) {
          state_ = kHaveRange;
          return true;
        } else {
          KALDI_WARN  << "Failed to load object from "
                      << PrintableRxfilename(data_rxfilename)
                      << "[" << range << "]";
          // leave state at kHaveObject.
          return false;
        }
      }
    }
  }

IsOpen()

virtual bool IsOpen ( ) const

inlinevirtual

Definition at line 1647 of file kaldi-table-inl.h.

                              {
    return  (state_ == kNotHaveObject || state_ == kHaveObject ||
             state_ == kHaveRange);
  }

LookupKey()

bool LookupKey ( const std::string &  key, size_t *  script_offset  )

inlineprivate

Definition at line 1802 of file kaldi-table-inl.h.

                                                              {
    // First, an optimization: if we're going consecutively, this will
    // make the lookup very fast.  Since we may call HasKey and then
    // Value(), which both may look up the key, we test if either the
    // current or next position are correct.
    if (last_found_ < script_.size() && script_[last_found_].first == key) {
      *script_offset = last_found_;
      return true;
    }
    last_found_++;
    if (last_found_ < script_.size() && script_[last_found_].first == key) {
      *script_offset = last_found_;
      return true;
    }
    std::pair<std::string, std::string> pr(key, "");  // Important that ""
    // compares less than or equal to any string, so lower_bound points to the
    // element that has the same key.
    typedef typename std::vector<std::pair<std::string, std::string> >
                     ::const_iterator IterType;
    IterType iter = std::lower_bound(script_.begin(), script_.end(), pr);
    if (iter != script_.end() && iter->first == key) {
      last_found_ = *script_offset = iter - script_.begin();
      return true;
    } else {
      return false;
    }
  }

Open()

virtual bool Open ( const std::string &  rspecifier )

inlinevirtual

Implements RandomAccessTableReaderImplBase< Holder >.

Definition at line 1600 of file kaldi-table-inl.h.

References kaldi::ClassifyRspecifier(), rnnlm::i, KALDI_ASSERT, KALDI_ERR, KALDI_WARN, kaldi::kScriptRspecifier, kaldi::PrintableRxfilename(), and kaldi::ReadScriptFile().

                                                 {
    switch (state_) {
      case kNotHaveObject: case kHaveObject: case kHaveRange:
        KALDI_ERR << " Opening already open RandomAccessTableReader:"
                     " call Close first.";
      case kUninitialized: case kNotReadScript:
        break;
    }
    rspecifier_ = rspecifier;
    RspecifierType rs = ClassifyRspecifier(rspecifier,
                                           &script_rxfilename_,
                                           &opts_);
    KALDI_ASSERT(rs == kScriptRspecifier);  // or wrongly called.
    KALDI_ASSERT(script_.empty());  // no way it could be nonempty at this point

    if (!ReadScriptFile(script_rxfilename_,
                        true,  // print any warnings
                        &script_)) {  // error reading script file or invalid
                                      // format
      state_ = kNotReadScript;
      return false;  // no need to print further warnings.  user gets the error.
    }

    rspecifier_ = rspecifier;
    // If opts_.sorted, the user has asserted that the keys are already sorted.
    // Although we could easily sort them, we want to let the user know of this
    // mistake.  This same mistake could have serious effects if used with an
    // archive rather than a script.
    if (!opts_.sorted)
      std::sort(script_.begin(), script_.end());
    for (size_t i = 0; i + 1 < script_.size(); i++) {
      if (script_[i].first.compare(script_[i+1].first) >= 0) {
        // script[i] not < script[i+1] in lexical order...
        bool same = (script_[i].first == script_[i+1].first);
        KALDI_WARN << "Script file " << PrintableRxfilename(script_rxfilename_)
                   << (same ? " contains duplicate key: " :
                       " is not sorted (remove s, option or add ns, option):"
                       " key is ") << script_[i].first;
        state_ = kNotReadScript;
        return false;
      }
    }
    state_ = kNotHaveObject;
    key_ = "";  // make sure we don't have a key set
    return true;
  }

Value()

virtual const T& Value ( const std::string &  key )

inlinevirtual

Implements RandomAccessTableReaderImplBase< Holder >.

Definition at line 1679 of file kaldi-table-inl.h.

References KALDI_ASSERT, and KALDI_ERR.

                                                {
    if (!HasKeyInternal(key, true)) // true == preload.
      KALDI_ERR << "Could not get item for key " << key
                << ", rspecifier is " << rspecifier_ << " [to ignore this, "
                 << "add the p, (permissive) option to the rspecifier.";
    KALDI_ASSERT(key_ == key);
    if (state_ == kHaveObject) {
      return holder_.Value();
    } else {
      KALDI_ASSERT(state_ == kHaveRange);
      return range_holder_.Value();
    }
  }

Member Data Documentation

data_rxfilename_

std::string data_rxfilename_

private

Definition at line 1849 of file kaldi-table-inl.h.

holder_

Holder holder_

private

Definition at line 1843 of file kaldi-table-inl.h.

input_

Input input_

private

Definition at line 1831 of file kaldi-table-inl.h.

key_

std::string key_

private

Definition at line 1839 of file kaldi-table-inl.h.

last_found_

size_t last_found_

private

Definition at line 1861 of file kaldi-table-inl.h.

opts_

RspecifierOptions opts_

private

Definition at line 1834 of file kaldi-table-inl.h.

range_

std::string range_

private

Definition at line 1847 of file kaldi-table-inl.h.

range_holder_

Holder range_holder_

private

Definition at line 1844 of file kaldi-table-inl.h.

rspecifier_

std::string rspecifier_

private

Definition at line 1835 of file kaldi-table-inl.h.

script_

std::vector<std::pair<std::string, std::string> > script_

private

Definition at line 1860 of file kaldi-table-inl.h.

script_rxfilename_

std::string script_rxfilename_

private

Definition at line 1837 of file kaldi-table-inl.h.

state_

enum { ... } state_

---

The documentation for this class was generated from the following file:

• util/kaldi-table-inl.h