MatrixBuffer Class Reference

A buffer for caching (utterance-key, feature-matrix) pairs. More...

#include <nnet-matrix-buffer.h>

Collaboration diagram for MatrixBuffer:

Public Member Functions
	MatrixBuffer ()
	~MatrixBuffer ()
void	Init (SequentialBaseFloatMatrixReader *reader, MatrixBufferOptions opts=MatrixBufferOptions())
bool	Done ()
void	Next ()
void	ResetLength ()
std::string	Key ()
Matrix< BaseFloat >	Value ()
size_t	SizeInBytes () const
	Total amount of features in the buffer (bytes),. More...
size_t	SizeInMegaBytes () const
	Total amount of features in the buffer (Mega-bytes),. More...
size_t	NumPairs () const
	Total number of (key,matrix) pairs in the buffer,. More...

Private Types
typedef std::pair< std::string, Matrix< BaseFloat > >	PairType
typedef std::list< PairType >	ListType
typedef std::map< size_t, ListType >	BufferType

Private Member Functions
void	Read ()
	fills the buffer, More...
void	DisposeValue ()
	removes 'current_' from data structure, More...

Private Attributes
SequentialBaseFloatMatrixReader *	reader_
BufferType	buffer_
	Buffer indexed by 'NumRows()',. More...
PairType *	current_
	The currently active (key,value) pair,. More...
MatrixBufferOptions	opts_
size_t	preferred_length_

Detailed Description

A buffer for caching (utterance-key, feature-matrix) pairs.

Typically, it reads 'matrix_buffer_size' megabytes of data, and returns records with similar number of speech frames through the standard Key(), Value(), Next(), Done() interface.

The preferred length is reset by ResetLength(). The buffer gets refilled after having less data than 50% of 'matrix_buffer_size'.

Definition at line 61 of file nnet-matrix-buffer.h.

Member Typedef Documentation

BufferType

typedef std::map<size_t, ListType> BufferType

private

Definition at line 116 of file nnet-matrix-buffer.h.

ListType

typedef std::list<PairType> ListType

private

Definition at line 115 of file nnet-matrix-buffer.h.

PairType

typedef std::pair<std::string,Matrix<BaseFloat> > PairType

private

Definition at line 114 of file nnet-matrix-buffer.h.

Constructor & Destructor Documentation

MatrixBuffer()

MatrixBuffer ( )

inline

Definition at line 63 of file nnet-matrix-buffer.h.

                :
    reader_(NULL),
    current_(NULL),
    preferred_length_(0)
  { }

~MatrixBuffer()

~MatrixBuffer ( )

inline

Definition at line 69 of file nnet-matrix-buffer.h.

  { }

Member Function Documentation

DisposeValue()

void DisposeValue ( )

private

removes 'current_' from data structure,

Definition at line 215 of file nnet-matrix-buffer.h.

References KALDI_ASSERT.

                                {
  // remove old 'Value()' matrix,
  if (current_ != NULL) {
    size_t r = current_->second.NumRows();
    KALDI_ASSERT(current_ == &(buffer_[r].front()));
    // remove the (key,value) pair,
    buffer_[r].pop_front();
    // eventually remove the 'NumRows()' key,
    if (buffer_[r].empty()) { buffer_.erase(r); }
    current_ = NULL;
  }
}

Done()

bool Done ( )

inline

Definition at line 81 of file nnet-matrix-buffer.h.

              {
    return (reader_->Done() && NumPairs() <= 1);
  }

Init()

void Init ( SequentialBaseFloatMatrixReader *  reader, MatrixBufferOptions  opts = MatrixBufferOptions()  )

inline

Definition at line 72 of file nnet-matrix-buffer.h.

References KALDI_ASSERT.

Referenced by main().

                                                              {
    KALDI_ASSERT(SizeInBytes() == 0);
    reader_ = reader;
    opts_ = opts;

    Read();
  }

Key()

std::string Key ( )

inline

Definition at line 91 of file nnet-matrix-buffer.h.

                  {
    return current_->first;
  }

Next()

void Next

(

)

Definition at line 126 of file nnet-matrix-buffer.h.

References rnnlm::i, and KALDI_ASSERT.

                        {
  KALDI_ASSERT(!buffer_.empty());

  // remove old 'Value()' matrix,
  DisposeValue();

  // start re-filling,
  if (SizeInMegaBytes() < 0.5 * opts_.matrix_buffer_size) {
    Read();
  }

  KALDI_ASSERT(!buffer_.empty());

  // randomly select 'length' present in the 'map',
  // (weighted by total #frames in the bin),
  if (preferred_length_ == 0) {
    int32 longest = (--buffer_.end())->first;
    // pre-fill the vector of 'keys',
    std::vector<int32> keys;
    BufferType::iterator it;
    for (it = buffer_.begin(); it != buffer_.end(); ++it) {
      int32 key = it->first; // i.e. NumRows() of matrices in the bin,
      int32 frames_in_bin = it->second.size() * key;
      for (int32 i = 0; i < frames_in_bin; i += longest) {
        keys.push_back(key); // keys are repeated,
      }
    }
    // choose the key,
    std::vector<int32>::iterator it2 = keys.begin();
    std::advance(it2, rand() % keys.size());
    preferred_length_ = (*it2);  // NumRows(), key of the 'map',
  }

  // select list by 'preferred_length_',
  BufferType::iterator it = buffer_.lower_bound(preferred_length_);
  if (it == buffer_.end()) { --it; } // or the last one,

  // take a front element 'ptr' from that list,
  current_ = &(it->second.front());
}

NumPairs()

size_t NumPairs

(

)

const

Total number of (key,matrix) pairs in the buffer,.

Definition at line 181 of file nnet-matrix-buffer.h.

                                    {
  size_t ans = 0;
  for (BufferType::const_iterator it = buffer_.begin(); it != buffer_.end(); ++it) {
    ans += it->second.size();
  }
  return ans;
}

Read()

void Read ( )

private

fills the buffer,

Definition at line 189 of file nnet-matrix-buffer.h.

References KALDI_LOG, and MatrixBase< Real >::NumRows().

                        {
  if (!reader_->Done())
    KALDI_LOG << "Read() started... Buffer size in MB: "
              << SizeInMegaBytes() << ", max " << opts_.matrix_buffer_size
              << ", having " << NumPairs() << " utterances.";
  for ( ; !reader_->Done(); reader_->Next()) {
    // see if we are full,
    if (SizeInMegaBytes() > opts_.matrix_buffer_size) {
      KALDI_LOG << "Read() finished... Buffer size in MB: "
                << SizeInMegaBytes() << ", max " << opts_.matrix_buffer_size
                << ", having " << NumPairs() << " utterances.";
      break;
    }
    // get matrix,
    const std::string& key = reader_->Key();
    const Matrix<BaseFloat>& mat = reader_->Value();
    size_t num_rows = mat.NumRows();
    // see if 'num_rows' already in keys,
    if (buffer_.find(num_rows) == buffer_.end()) {
      buffer_[num_rows] = ListType();  // add empty list,
    }
    // add matrix to the buffer,
    buffer_[num_rows].push_back(PairType(key, mat));
  }
}

ResetLength()

void ResetLength ( )

inline

Definition at line 87 of file nnet-matrix-buffer.h.

                     {
    preferred_length_ = 0;
  }

SizeInBytes()

size_t SizeInBytes

(

)

const

Total amount of features in the buffer (bytes),.

Definition at line 167 of file nnet-matrix-buffer.h.

                                       {
  size_t ans = 0;
  for (BufferType::const_iterator it = buffer_.begin(); it != buffer_.end(); ++it) {
    for (ListType::const_iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) {
      ans += it2->second.SizeInBytes();
    }
  }
  return ans;
}

SizeInMegaBytes()

size_t SizeInMegaBytes

(

)

const

Total amount of features in the buffer (Mega-bytes),.

Definition at line 177 of file nnet-matrix-buffer.h.

                                           {
  return (SizeInBytes() / (1024 * 1024));
}

Value()

Matrix<BaseFloat> Value ( )

inline

Definition at line 94 of file nnet-matrix-buffer.h.

                            {
    return current_->second;
  }

Member Data Documentation

buffer_

BufferType buffer_

private

Buffer indexed by 'NumRows()',.

Definition at line 117 of file nnet-matrix-buffer.h.

current_

PairType* current_

private

The currently active (key,value) pair,.

Definition at line 119 of file nnet-matrix-buffer.h.

opts_

MatrixBufferOptions opts_

private

Definition at line 121 of file nnet-matrix-buffer.h.

preferred_length_

size_t preferred_length_

private

Definition at line 123 of file nnet-matrix-buffer.h.

reader_

SequentialBaseFloatMatrixReader* reader_

private

Definition at line 112 of file nnet-matrix-buffer.h.

---

The documentation for this class was generated from the following file:

• nnet/nnet-matrix-buffer.h