This is a templated class for online feature extraction; it's templated on a class like MfccComputer or PlpComputer that does the basic feature extraction. More...

#include <online-feature.h>

Inheritance diagram for OnlineGenericBaseFeature< C >:

[legend]

Collaboration diagram for OnlineGenericBaseFeature< C >:

[legend]

Public Member Functions
virtual int32	Dim () const

virtual bool	IsLastFrame (int32 frame) const
	Returns true if this is the last frame. More...

virtual BaseFloat	FrameShiftInSeconds () const

virtual int32	NumFramesReady () const
	returns the feature dimension. More...

virtual void	GetFrame (int32 frame, VectorBase< BaseFloat > *feat)
	Gets the feature vector for this frame. More...

	OnlineGenericBaseFeature (const typename C::Options &opts)

virtual void	AcceptWaveform (BaseFloat sampling_rate, const VectorBase< BaseFloat > &waveform)
	This would be called from the application, when you get more wave data. More...

virtual void	InputFinished ()
	InputFinished() tells the class you won't be providing any more waveform. More...

Public Member Functions inherited from OnlineFeatureInterface
virtual void	GetFrames (const std::vector< int32 > &frames, MatrixBase< BaseFloat > *feats)
	This is like GetFrame() but for a collection of frames. More...

virtual	~OnlineFeatureInterface ()
	Virtual destructor. More...

Private Member Functions
void	ComputeFeatures ()

void	MaybeCreateResampler (BaseFloat sampling_rate)

Private Attributes
C	computer_

std::unique_ptr< LinearResample >	resampler_

FeatureWindowFunction	window_function_

RecyclingVector	features_

bool	input_finished_

BaseFloat	sampling_frequency_

int64	waveform_offset_

Vector< BaseFloat >	waveform_remainder_

Detailed Description

template<class C>
class kaldi::OnlineGenericBaseFeature< C >

This is a templated class for online feature extraction; it's templated on a class like MfccComputer or PlpComputer that does the basic feature extraction.

Definition at line 78 of file online-feature.h.

Constructor & Destructor Documentation

◆ OnlineGenericBaseFeature()

OnlineGenericBaseFeature ( const typename C::Options & opts )

explicit

Definition at line 70 of file online-feature.cc.

References KALDI_ASSERT.

                                   :
     computer_(opts), window_function_(computer_.GetFrameOptions()),
     features_(opts.frame_opts.max_feature_vectors),
     input_finished_(false), waveform_offset_(0) {
   // RE the following assert: search for ONLINE_IVECTOR_LIMIT in
   // online-ivector-feature.cc.
   // Casting to uint32, an unsigned type, means that -1 would be treated
   // as `very large`.
   KALDI_ASSERT(static_cast<uint32>(opts.frame_opts.max_feature_vectors) > 200);
 }

Member Function Documentation

◆ AcceptWaveform()

void AcceptWaveform	(	BaseFloat	sampling_rate,
		const VectorBase< BaseFloat > &	waveform
	)

virtual

This would be called from the application, when you get more wave data.

Note: the sampling_rate is typically only provided so the code can assert that it matches the sampling rate expected in the options.

Implements OnlineBaseFeature.

Definition at line 131 of file online-feature.cc.

References OnlineGenericBaseFeature< C >::ComputeFeatures(), VectorBase< Real >::Dim(), OnlineGenericBaseFeature< C >::input_finished_, KALDI_ERR, OnlineGenericBaseFeature< C >::MaybeCreateResampler(), VectorBase< Real >::Range(), OnlineGenericBaseFeature< C >::resampler_, Vector< Real >::Resize(), and OnlineGenericBaseFeature< C >::waveform_remainder_.

Referenced by kaldi::TestOnlineAppendFeature(), kaldi::TestOnlineMfcc(), kaldi::TestOnlinePlp(), and kaldi::TestOnlineTransform().

                                                                              {
   if (original_waveform.Dim() == 0)
     return;  // Nothing to do.
   if (input_finished_)
     KALDI_ERR << "AcceptWaveform called after InputFinished() was called.";
 
   Vector<BaseFloat> appended_wave;
   Vector<BaseFloat> resampled_wave;
 
   const VectorBase<BaseFloat> *waveform;
 
   MaybeCreateResampler(sampling_rate);
   if (resampler_ == nullptr) {
     waveform = &original_waveform;
   } else {
     resampler_->Resample(original_waveform, false, &resampled_wave);
     waveform = &resampled_wave;
   }
 
   appended_wave.Resize(waveform_remainder_.Dim() + waveform->Dim());
   if (waveform_remainder_.Dim() != 0)
     appended_wave.Range(0, waveform_remainder_.Dim())
         .CopyFromVec(waveform_remainder_);
   appended_wave.Range(waveform_remainder_.Dim(), waveform->Dim())
       .CopyFromVec(*waveform);
   waveform_remainder_.Swap(&appended_wave);
   ComputeFeatures();
 }

◆ ComputeFeatures()

void ComputeFeatures ( )

private

Definition at line 162 of file online-feature.cc.

References OnlineGenericBaseFeature< C >::computer_, VectorBase< Real >::CopyFromVec(), kaldi::ExtractWindow(), OnlineGenericBaseFeature< C >::features_, kaldi::FirstSampleOfFrame(), OnlineGenericBaseFeature< C >::input_finished_, KALDI_ASSERT, kaldi::kUndefined, kaldi::NumFrames(), RecyclingVector::PushBack(), RecyclingVector::Size(), OnlineGenericBaseFeature< C >::waveform_offset_, OnlineGenericBaseFeature< C >::waveform_remainder_, and OnlineGenericBaseFeature< C >::window_function_.

Referenced by OnlineGenericBaseFeature< C >::AcceptWaveform(), and OnlineGenericBaseFeature< C >::InputFinished().

                                                   {
   const FrameExtractionOptions &frame_opts = computer_.GetFrameOptions();
   int64 num_samples_total = waveform_offset_ + waveform_remainder_.Dim();
   int32 num_frames_old = features_.Size(),
       num_frames_new = NumFrames(num_samples_total, frame_opts,
                                  input_finished_);
   KALDI_ASSERT(num_frames_new >= num_frames_old);
 
   Vector<BaseFloat> window;
   bool need_raw_log_energy = computer_.NeedRawLogEnergy();
   for (int32 frame = num_frames_old; frame < num_frames_new; frame++) {
     BaseFloat raw_log_energy = 0.0;
     ExtractWindow(waveform_offset_, waveform_remainder_, frame,
                   frame_opts, window_function_, &window,
                   need_raw_log_energy ? &raw_log_energy : NULL);
     Vector<BaseFloat> *this_feature = new Vector<BaseFloat>(computer_.Dim(),
                                                             kUndefined);
     // note: this online feature-extraction code does not support VTLN.
     BaseFloat vtln_warp = 1.0;
     computer_.Compute(raw_log_energy, vtln_warp, &window, this_feature);
     features_.PushBack(this_feature);
   }
   // OK, we will now discard any portion of the signal that will not be
   // necessary to compute frames in the future.
   int64 first_sample_of_next_frame = FirstSampleOfFrame(num_frames_new,
                                                         frame_opts);
   int32 samples_to_discard = first_sample_of_next_frame - waveform_offset_;
   if (samples_to_discard > 0) {
     // discard the leftmost part of the waveform that we no longer need.
     int32 new_num_samples = waveform_remainder_.Dim() - samples_to_discard;
     if (new_num_samples <= 0) {
       // odd, but we'll try to handle it.
       waveform_offset_ += waveform_remainder_.Dim();
       waveform_remainder_.Resize(0);
     } else {
       Vector<BaseFloat> new_remainder(new_num_samples);
       new_remainder.CopyFromVec(waveform_remainder_.Range(samples_to_discard,
                                                           new_num_samples));
       waveform_offset_ += samples_to_discard;
       waveform_remainder_.Swap(&new_remainder);
     }
   }
 }

◆ Dim()

virtual int32 Dim ( ) const

inlinevirtual

Implements OnlineFeatureInterface.

Definition at line 83 of file online-feature.h.

Referenced by kaldi::TestOnlineTransform().

83 { return computer_.Dim(); }

kaldi::OnlineGenericBaseFeature::computer_

C computer_

Definition: online-feature.h:129

◆ FrameShiftInSeconds()

virtual BaseFloat FrameShiftInSeconds ( ) const

inlinevirtual

Implements OnlineFeatureInterface.

Definition at line 90 of file online-feature.h.

                                                 {
     return computer_.GetFrameOptions().frame_shift_ms / 1000.0f;
   }

◆ GetFrame()

void GetFrame	(	int32	frame,
		VectorBase< BaseFloat > *	feat
	)

virtual

Gets the feature vector for this frame.

Before calling this for a given frame, it is assumed that you called NumFramesReady() and it returned a number greater than "frame". Otherwise this call will likely crash with an assert failure. This function is not declared const, in case there is some kind of caching going on, but most of the time it shouldn't modify the class.

Implements OnlineFeatureInterface.

Definition at line 64 of file online-feature.cc.

References VectorBase< Real >::CopyFromVec().

                                                                         {
   feat->CopyFromVec(*(features_.At(frame)));
 };

◆ InputFinished()

void InputFinished ( )

virtual

InputFinished() tells the class you won't be providing any more waveform.

This will help flush out the last few frames of delta or LDA features (it will typically affect the return value of IsLastFrame.

Implements OnlineBaseFeature.

Definition at line 107 of file online-feature.cc.

References OnlineGenericBaseFeature< C >::ComputeFeatures(), VectorBase< Real >::Dim(), OnlineGenericBaseFeature< C >::input_finished_, VectorBase< Real >::Range(), OnlineGenericBaseFeature< C >::resampler_, Vector< Real >::Resize(), and OnlineGenericBaseFeature< C >::waveform_remainder_.

Referenced by kaldi::TestOnlineAppendFeature(), kaldi::TestOnlineMfcc(), kaldi::TestOnlinePlp(), and kaldi::TestOnlineTransform().

                                                 {
   if (resampler_ != nullptr) {
     // There may be a few samples left once we flush the resampler_ object, telling it
     // that the file has finished.  This should rarely make any difference.
     Vector<BaseFloat> appended_wave;
     Vector<BaseFloat> resampled_wave;
     resampler_->Resample(appended_wave, true, &resampled_wave);
 
     if (resampled_wave.Dim() != 0) {
       appended_wave.Resize(waveform_remainder_.Dim() +
                            resampled_wave.Dim());
       if (waveform_remainder_.Dim() != 0)
         appended_wave.Range(0, waveform_remainder_.Dim())
             .CopyFromVec(waveform_remainder_);
       appended_wave.Range(waveform_remainder_.Dim(), resampled_wave.Dim())
           .CopyFromVec(resampled_wave);
       waveform_remainder_.Swap(&appended_wave);
     }
   }
   input_finished_ = true;
   ComputeFeatures();
 }

◆ IsLastFrame()

virtual bool IsLastFrame ( int32 frame ) const

inlinevirtual

Returns true if this is the last frame.

Frame indices are zero-based, so the first frame is zero. IsLastFrame(-1) will return false, unless the file is empty (which is a case that I'm not sure all the code will handle, so be careful). This function may return false for some frame if we haven't yet decided to terminate decoding, but later true if we decide to terminate decoding. This function exists mainly to correctly handle end effects in feature extraction, and is not a mechanism to determine how many frames are in the decodable object (as it used to be, and for backward compatibility, still is, in the Decodable interface).

Implements OnlineFeatureInterface.

Definition at line 87 of file online-feature.h.

                                               {
     return input_finished_ && frame == NumFramesReady() - 1;
   }

◆ MaybeCreateResampler()

void MaybeCreateResampler ( BaseFloat sampling_rate )

private

Definition at line 84 of file online-feature.cc.

References OnlineGenericBaseFeature< C >::computer_, KALDI_ASSERT, KALDI_ERR, and OnlineGenericBaseFeature< C >::resampler_.

Referenced by OnlineGenericBaseFeature< C >::AcceptWaveform().

                              {
   BaseFloat expected_sampling_rate = computer_.GetFrameOptions().samp_freq;
 
   if (resampler_ != nullptr) {
     KALDI_ASSERT(resampler_->GetInputSamplingRate() == sampling_rate);
     KALDI_ASSERT(resampler_->GetOutputSamplingRate() == expected_sampling_rate);
   } else if (((sampling_rate > expected_sampling_rate) &&
               computer_.GetFrameOptions().allow_downsample) ||
              ((sampling_rate < expected_sampling_rate) &&
               computer_.GetFrameOptions().allow_upsample)) {
     resampler_.reset(new LinearResample(
         sampling_rate, expected_sampling_rate,
         std::min(sampling_rate / 2, expected_sampling_rate / 2), 6));
   } else if (sampling_rate != expected_sampling_rate) {
     KALDI_ERR << "Sampling frequency mismatch, expected "
               << expected_sampling_rate << ", got " << sampling_rate
               << "\nPerhaps you want to use the options "
                  "--allow_{upsample,downsample}";
   }
 }

◆ NumFramesReady()

virtual int32 NumFramesReady ( ) const

inlinevirtual

returns the feature dimension.

Returns the total number of frames, since the start of the utterance, that are now available. In an online-decoding context, this will likely increase with time as more data becomes available.

Implements OnlineFeatureInterface.

Definition at line 94 of file online-feature.h.

94 { return features_.Size(); }

kaldi::OnlineGenericBaseFeature::features_

RecyclingVector features_

Definition: online-feature.h:139

kaldi::RecyclingVector::Size

int Size() const

This method returns the size as if no "recycling" had happened, i.e.

Definition: online-feature.cc:59