All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
Xent Class Reference

#include <nnet-loss.h>

Inheritance diagram for Xent:
Collaboration diagram for Xent:

Public Member Functions

 Xent (LossOptions &opts)
 
 ~Xent ()
 
void Eval (const VectorBase< BaseFloat > &frame_weights, const CuMatrixBase< BaseFloat > &net_out, const CuMatrixBase< BaseFloat > &target, CuMatrix< BaseFloat > *diff)
 Evaluate cross entropy using target-matrix (supports soft labels),. More...
 
void Eval (const VectorBase< BaseFloat > &frame_weights, const CuMatrixBase< BaseFloat > &net_out, const Posterior &target, CuMatrix< BaseFloat > *diff)
 Evaluate cross entropy using target-posteriors (supports soft labels),. More...
 
std::string Report ()
 Generate string with error report,. More...
 
std::string ReportPerClass ()
 Generate string with per-class error report,. More...
 
BaseFloat AvgLoss ()
 Get loss value (frame average),. More...
 
- Public Member Functions inherited from LossItf
 LossItf (LossOptions &opts)
 
virtual ~LossItf ()
 

Private Attributes

CuVector< double > frames_
 
Vector< double > correct_
 
CuVector< double > xentropy_
 
CuVector< double > entropy_
 
double frames_progress_
 
double xentropy_progress_
 
double entropy_progress_
 
std::vector< float > loss_vec_
 
double elapsed_seconds_
 
CuVector< BaseFloatframe_weights_
 
CuVector< BaseFloattarget_sum_
 
CuMatrix< BaseFloattgt_mat_
 
CuMatrix< BaseFloatframes_aux_
 
CuMatrix< BaseFloatxentropy_aux_
 
CuMatrix< BaseFloatentropy_aux_
 
CuArray< int32 > max_id_out_
 
CuArray< int32 > max_id_tgt_
 

Additional Inherited Members

- Protected Attributes inherited from LossItf
LossOptions opts_
 
Timer timer_
 

Detailed Description

Definition at line 82 of file nnet-loss.h.

Constructor & Destructor Documentation

Xent ( LossOptions opts)
inline

Definition at line 84 of file nnet-loss.h.

84  :
85  LossItf(opts),
86  frames_progress_(0.0),
87  xentropy_progress_(0.0),
88  entropy_progress_(0.0),
89  elapsed_seconds_(0.0)
90  { }
double xentropy_progress_
Definition: nnet-loss.h:128
double elapsed_seconds_
Definition: nnet-loss.h:131
double entropy_progress_
Definition: nnet-loss.h:129
LossItf(LossOptions &opts)
Definition: nnet-loss.h:53
double frames_progress_
Definition: nnet-loss.h:127
~Xent ( )
inline

Definition at line 92 of file nnet-loss.h.

93  { }

Member Function Documentation

BaseFloat AvgLoss ( )
inlinevirtual

Get loss value (frame average),.

Implements LossItf.

Definition at line 114 of file nnet-loss.h.

References Xent::entropy_, Xent::frames_, CuVectorBase< Real >::Sum(), and Xent::xentropy_.

114  {
115  if (frames_.Sum() == 0) return 0.0;
116  return (xentropy_.Sum() - entropy_.Sum()) / frames_.Sum();
117  }
CuVector< double > frames_
Definition: nnet-loss.h:121
CuVector< double > xentropy_
Definition: nnet-loss.h:123
CuVector< double > entropy_
Definition: nnet-loss.h:124
Real Sum() const
Definition: cu-vector.cc:290
void Eval ( const VectorBase< BaseFloat > &  frame_weights,
const CuMatrixBase< BaseFloat > &  net_out,
const CuMatrixBase< BaseFloat > &  target,
CuMatrix< BaseFloat > *  diff 
)
virtual

Evaluate cross entropy using target-matrix (supports soft labels),.

Implements LossItf.

Definition at line 62 of file nnet-loss.cc.

References CuMatrixBase< Real >::Add(), CuVectorBase< Real >::AddColSumMat(), CuMatrixBase< Real >::AddMat(), CuVectorBase< Real >::AddRowSumMat(), CuMatrixBase< Real >::ApplyLog(), Xent::correct_, kaldi::nnet1::CountCorrectFramesWeighted(), VectorBase< Real >::Dim(), CuVectorBase< Real >::Dim(), Timer::Elapsed(), Xent::elapsed_seconds_, Xent::entropy_, Xent::entropy_aux_, Xent::entropy_progress_, CuMatrixBase< Real >::FindRowMaxId(), Xent::frame_weights_, Xent::frames_, Xent::frames_aux_, Xent::frames_progress_, KALDI_ASSERT, KALDI_ISFINITE, KALDI_LOG, LossOptions::loss_report_frames, Xent::loss_vec_, Xent::max_id_out_, Xent::max_id_tgt_, CuVectorBase< Real >::MulElements(), CuMatrixBase< Real >::MulElements(), CuMatrixBase< Real >::MulRowsVec(), CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), LossItf::opts_, CuVector< Real >::Resize(), Vector< Real >::Resize(), CuVectorBase< Real >::Sum(), VectorBase< Real >::Sum(), CuMatrixBase< Real >::Sum(), Xent::target_sum_, LossItf::timer_, Xent::xentropy_, Xent::xentropy_aux_, and Xent::xentropy_progress_.

Referenced by Xent::Eval(), and main().

65  {
66  // check inputs,
67  KALDI_ASSERT(net_out.NumCols() == targets.NumCols());
68  KALDI_ASSERT(net_out.NumRows() == targets.NumRows());
69  KALDI_ASSERT(net_out.NumRows() == frame_weights.Dim());
70 
71  KALDI_ASSERT(KALDI_ISFINITE(frame_weights.Sum()));
72  KALDI_ASSERT(KALDI_ISFINITE(net_out.Sum()));
73  KALDI_ASSERT(KALDI_ISFINITE(targets.Sum()));
74 
75  // buffer initialization,
76  int32 num_classes = targets.NumCols();
77  if (frames_.Dim() == 0) {
78  frames_.Resize(num_classes);
79  xentropy_.Resize(num_classes);
80  entropy_.Resize(num_classes);
81  correct_.Resize(num_classes);
82  }
83 
84  // get frame_weights to GPU,
85  frame_weights_ = frame_weights;
86 
87  // There may be frames for which the sum of targets is zero.
88  // This happens in multi-lingual training when the frame
89  // has target class in the softmax of another language.
90  // We 'switch-off' such frames by masking the 'frame_weights_',
91  target_sum_.Resize(targets.NumRows());
92  target_sum_.AddColSumMat(1.0, targets, 0.0);
94 
95  // compute derivative wrt. activations of last layer of neurons,
96  *diff = net_out;
97  diff->AddMat(-1.0, targets);
98  diff->MulRowsVec(frame_weights_); // weighting,
99 
100  // count frames per class,
101  frames_aux_ = targets;
103  frames_.AddRowSumMat(1.0, CuMatrix<double>(frames_aux_));
104 
105  // evaluate the frame-level classification,
106  net_out.FindRowMaxId(&max_id_out_); // find max in nn-output
107  targets.FindRowMaxId(&max_id_tgt_); // find max in targets
110 
111  // calculate cross_entropy (in GPU),
112  xentropy_aux_ = net_out; // y
113  xentropy_aux_.Add(1e-20); // avoid log(0)
114  xentropy_aux_.ApplyLog(); // log(y)
115  xentropy_aux_.MulElements(targets); // t*log(y)
116  xentropy_aux_.MulRowsVec(frame_weights_); // w*t*log(y)
117  xentropy_.AddRowSumMat(-1.0, CuMatrix<double>(xentropy_aux_));
118 
119  // caluculate entropy (in GPU),
120  entropy_aux_ = targets; // t
121  entropy_aux_.Add(1e-20); // avoid log(0)
122  entropy_aux_.ApplyLog(); // log(t)
123  entropy_aux_.MulElements(targets); // t*log(t)
124  entropy_aux_.MulRowsVec(frame_weights_); // w*t*log(t)
125  entropy_.AddRowSumMat(-1.0, CuMatrix<double>(entropy_aux_));
126 
127  // progressive loss reporting
128  if (opts_.loss_report_frames > 0) {
132 
135 
137  // loss value,
138  double progress_value =
140 
141  // time-related info (fps is weighted),
142  double time_now = timer_.Elapsed();
143  double fps = frames_progress_ / (time_now - elapsed_seconds_);
144  double elapsed_hours = time_now / 3600;
145  elapsed_seconds_ = time_now; // store,
146 
147  // print,
148  KALDI_LOG << "ProgressLoss[last "
149  << static_cast<int>(frames_progress_/100/3600) << "h of "
150  << static_cast<int>(frames_.Sum()/100/3600) << "h]: "
151  << progress_value << " (Xent)"
152  << ", fps=" << fps
153  << std::setprecision(3)
154  << ", elapsed " << elapsed_hours << "h";
155  // store,
156  loss_vec_.push_back(progress_value);
157  // reset,
158  frames_progress_ = 0;
159  xentropy_progress_ = 0.0;
160  entropy_progress_ = 0.0;
161  }
162  }
163 }
double xentropy_progress_
Definition: nnet-loss.h:128
void FindRowMaxId(CuArray< int32 > *id) const
Find the id of the maximal element for each row (resizes the 'id' array to the appropriate size)...
Definition: cu-matrix.cc:1813
CuVector< double > frames_
Definition: nnet-loss.h:121
double elapsed_seconds_
Definition: nnet-loss.h:131
CuMatrix< BaseFloat > frames_aux_
Definition: nnet-loss.h:139
Real Sum() const
Definition: cu-matrix.cc:2979
CuVector< BaseFloat > target_sum_
Definition: nnet-loss.h:135
void AddColSumMat(Real alpha, const CuMatrixBase< Real > &mat, Real beta=1.0)
Sum the columns of the matrix, add to vector.
Definition: cu-vector.cc:1234
#define KALDI_ISFINITE(x)
Definition: kaldi-math.h:74
std::vector< float > loss_vec_
Definition: nnet-loss.h:130
CuVector< double > xentropy_
Definition: nnet-loss.h:123
Real Sum() const
Returns sum of the elements.
void MulElements(const CuVectorBase< Real > &v)
Definition: cu-vector.cc:790
MatrixIndexT NumCols() const
Definition: cu-matrix.h:215
void Resize(MatrixIndexT length, MatrixResizeType resize_type=kSetZero)
Set vector to a specified size (can be zero).
void Resize(MatrixIndexT dim, MatrixResizeType t=kSetZero)
Allocate the memory.
Definition: cu-vector.cc:941
CuMatrix< BaseFloat > entropy_aux_
Definition: nnet-loss.h:141
void CountCorrectFramesWeighted(const CuArray< T > &hyp, const CuArray< T > &ref, const CuVectorBase< BaseFloat > &weights, Vector< double > *correct)
Helper function of Xent::Eval, calculates number of matching elemente in 'hyp', 'ref' weighted by 'we...
Definition: nnet-loss.cc:41
CuArray< int32 > max_id_out_
Definition: nnet-loss.h:144
Vector< double > correct_
Definition: nnet-loss.h:122
void MulRowsVec(const CuVectorBase< Real > &scale)
scale i'th row by scale[i]
Definition: cu-matrix.cc:779
double entropy_progress_
Definition: nnet-loss.h:129
void AddRowSumMat(Real alpha, const CuMatrixBase< Real > &mat, Real beta=1.0)
Sum the rows of the matrix, add to vector.
Definition: cu-vector.cc:1222
int32 loss_report_frames
Report loss value every 'report_interval' frames,.
Definition: nnet-loss.h:39
MatrixIndexT Dim() const
Dimensions.
Definition: cu-vector.h:68
MatrixIndexT NumRows() const
Dimensions.
Definition: cu-matrix.h:214
CuVector< double > entropy_
Definition: nnet-loss.h:124
CuVector< BaseFloat > frame_weights_
Definition: nnet-loss.h:134
CuMatrix< BaseFloat > xentropy_aux_
Definition: nnet-loss.h:140
void AddMat(Real alpha, const CuMatrixBase< Real > &A, MatrixTransposeType trans=kNoTrans)
*this += alpha * A
Definition: cu-matrix.cc:941
CuArray< int32 > max_id_tgt_
Definition: nnet-loss.h:145
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
void MulElements(const CuMatrixBase< Real > &A)
Multiply two matrices elementwise: C = C .* A.
Definition: cu-matrix.cc:654
double frames_progress_
Definition: nnet-loss.h:127
double Elapsed() const
Returns time in seconds.
Definition: timer.h:74
void Add(Real value)
Definition: cu-matrix.cc:548
#define KALDI_LOG
Definition: kaldi-error.h:133
MatrixIndexT Dim() const
Returns the dimension of the vector.
Definition: kaldi-vector.h:63
LossOptions opts_
Definition: nnet-loss.h:77
Real Sum() const
Definition: cu-vector.cc:290
void Eval ( const VectorBase< BaseFloat > &  frame_weights,
const CuMatrixBase< BaseFloat > &  net_out,
const Posterior target,
CuMatrix< BaseFloat > *  diff 
)
virtual

Evaluate cross entropy using target-posteriors (supports soft labels),.

Implements LossItf.

Definition at line 166 of file nnet-loss.cc.

References Xent::Eval(), KALDI_ASSERT, CuMatrixBase< Real >::NumCols(), CuMatrixBase< Real >::NumRows(), kaldi::nnet1::PosteriorToMatrix(), and Xent::tgt_mat_.

169  {
170  int32 num_frames = net_out.NumRows(),
171  num_pdf = net_out.NumCols();
172  KALDI_ASSERT(num_frames == post.size());
173 
174  // convert posterior to matrix,
175  PosteriorToMatrix(post, num_pdf, &tgt_mat_);
176 
177  // call the other eval function,
178  Eval(frame_weights, net_out, tgt_mat_, diff);
179 }
MatrixIndexT NumCols() const
Definition: cu-matrix.h:215
void PosteriorToMatrix(const Posterior &post, const int32 post_dim, CuMatrix< Real > *mat)
Wrapper of PosteriorToMatrix with CuMatrix argument.
Definition: nnet-utils.h:292
CuMatrix< BaseFloat > tgt_mat_
Definition: nnet-loss.h:138
MatrixIndexT NumRows() const
Dimensions.
Definition: cu-matrix.h:214
void Eval(const VectorBase< BaseFloat > &frame_weights, const CuMatrixBase< BaseFloat > &net_out, const CuMatrixBase< BaseFloat > &target, CuMatrix< BaseFloat > *diff)
Evaluate cross entropy using target-matrix (supports soft labels),.
Definition: nnet-loss.cc:62
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:169
std::string Report ( )
virtual

Generate string with error report,.

Implements LossItf.

Definition at line 182 of file nnet-loss.cc.

References Xent::correct_, Xent::entropy_, Xent::frames_, Xent::loss_vec_, CuVectorBase< Real >::Sum(), VectorBase< Real >::Sum(), and Xent::xentropy_.

Referenced by main().

182  {
183  double loss_value =
184  (xentropy_.Sum() - entropy_.Sum()) / frames_.Sum();
185  std::ostringstream oss;
186  oss << "AvgLoss: " << loss_value << " (Xent), "
187  << "[AvgXent: " << xentropy_.Sum() / frames_.Sum()
188  << ", AvgTargetEnt: " << entropy_.Sum() / frames_.Sum()
189  << "]" << std::endl;
190 
191  oss << "progress: [";
192  std::copy(loss_vec_.begin(), loss_vec_.end(),
193  std::ostream_iterator<float>(oss, " "));
194  oss << "]" << std::endl;
195 
196  double frame_accuracy = 100.0 * correct_.Sum() / frames_.Sum();
197  oss << "FRAME_ACCURACY >> " << frame_accuracy << "% <<" << std::endl;
198 
199  return oss.str();
200 }
CuVector< double > frames_
Definition: nnet-loss.h:121
std::vector< float > loss_vec_
Definition: nnet-loss.h:130
CuVector< double > xentropy_
Definition: nnet-loss.h:123
Real Sum() const
Returns sum of the elements.
Vector< double > correct_
Definition: nnet-loss.h:122
CuVector< double > entropy_
Definition: nnet-loss.h:124
Real Sum() const
Definition: cu-vector.cc:290
std::string ReportPerClass ( )

Generate string with per-class error report,.

Definition at line 203 of file nnet-loss.cc.

References CuVectorBase< Real >::Add(), CuVectorBase< Real >::AddVec(), CuVectorBase< Real >::ApplyPow(), Xent::correct_, Xent::entropy_, Xent::frames_, CuVectorBase< Real >::MulElements(), CuVectorBase< Real >::Scale(), and Xent::xentropy_.

Referenced by main().

203  {
204  std::ostringstream oss;
205  oss << "PER-CLASS PERFORMANCE:" << std::endl;
206  oss << "@@@ Frames per-class:" << frames_;
207  // get inverted counts,
208  CuVector<double> inv_frames(frames_);
209  inv_frames.Add(0.5); // avoid 0-frames,
210  inv_frames.ApplyPow(-1.0);
211  // loss, kl = xentropy-entropy,
212  CuVector<double> loss(xentropy_);
213  loss.AddVec(-1.0, entropy_);
214  loss.MulElements(inv_frames);
215  oss << "@@@ Loss per-class:" << loss;
216  // frame accuracy (assuming targets are binary),
217  CuVector<double> frm_accu(correct_);
218  frm_accu.MulElements(inv_frames);
219  frm_accu.Scale(100.0);
220  oss << "@@@ Frame-accuracy per-class:" << frm_accu;
221  //
222  return oss.str();
223 }
CuVector< double > frames_
Definition: nnet-loss.h:121
CuVector< double > xentropy_
Definition: nnet-loss.h:123
Vector< double > correct_
Definition: nnet-loss.h:122
CuVector< double > entropy_
Definition: nnet-loss.h:124

Member Data Documentation

Vector<double> correct_
private

Definition at line 122 of file nnet-loss.h.

Referenced by Xent::Eval(), Xent::Report(), and Xent::ReportPerClass().

double elapsed_seconds_
private

Definition at line 131 of file nnet-loss.h.

Referenced by Xent::Eval().

CuVector<double> entropy_
private

Definition at line 124 of file nnet-loss.h.

Referenced by Xent::AvgLoss(), Xent::Eval(), Xent::Report(), and Xent::ReportPerClass().

CuMatrix<BaseFloat> entropy_aux_
private

Definition at line 141 of file nnet-loss.h.

Referenced by Xent::Eval().

double entropy_progress_
private

Definition at line 129 of file nnet-loss.h.

Referenced by Xent::Eval().

CuVector<BaseFloat> frame_weights_
private

Definition at line 134 of file nnet-loss.h.

Referenced by Xent::Eval().

CuVector<double> frames_
private

Definition at line 121 of file nnet-loss.h.

Referenced by Xent::AvgLoss(), Xent::Eval(), Xent::Report(), and Xent::ReportPerClass().

CuMatrix<BaseFloat> frames_aux_
private

Definition at line 139 of file nnet-loss.h.

Referenced by Xent::Eval().

double frames_progress_
private

Definition at line 127 of file nnet-loss.h.

Referenced by Xent::Eval().

std::vector<float> loss_vec_
private

Definition at line 130 of file nnet-loss.h.

Referenced by Xent::Eval(), and Xent::Report().

CuArray<int32> max_id_out_
private

Definition at line 144 of file nnet-loss.h.

Referenced by Xent::Eval().

CuArray<int32> max_id_tgt_
private

Definition at line 145 of file nnet-loss.h.

Referenced by Xent::Eval().

CuVector<BaseFloat> target_sum_
private

Definition at line 135 of file nnet-loss.h.

Referenced by Xent::Eval().

CuMatrix<BaseFloat> tgt_mat_
private

Definition at line 138 of file nnet-loss.h.

Referenced by Xent::Eval().

CuVector<double> xentropy_
private

Definition at line 123 of file nnet-loss.h.

Referenced by Xent::AvgLoss(), Xent::Eval(), Xent::Report(), and Xent::ReportPerClass().

CuMatrix<BaseFloat> xentropy_aux_
private

Definition at line 140 of file nnet-loss.h.

Referenced by Xent::Eval().

double xentropy_progress_
private

Definition at line 128 of file nnet-loss.h.

Referenced by Xent::Eval().


The documentation for this class was generated from the following files: