nnet-update-parallel.cc
Go to the documentation of this file.
1 // nnet2/nnet-update-parallel.cc
2 
3 // Copyright 2012 Johns Hopkins University (author: Daniel Povey)
4 
5 // See ../../COPYING for clarification regarding multiple authors
6 //
7 // Licensed under the Apache License, Version 2.0 (the "License");
8 // you may not use this file except in compliance with the License.
9 // You may obtain a copy of the License at
10 //
11 // http://www.apache.org/licenses/LICENSE-2.0
12 //
13 // THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 // KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
15 // WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
16 // MERCHANTABLITY OR NON-INFRINGEMENT.
17 // See the Apache 2 License for the specific language governing permissions and
18 // limitations under the License.
19 
20 #include <numeric>
22 #include "nnet2/nnet-update.h"
23 #include "util/kaldi-thread.h"
24 
25 namespace kaldi {
26 namespace nnet2 {
27 
28 
30  public:
31  // This constructor is only called for a temporary object
32  // that we pass to the RunMultiThreaded function.
34  ExamplesRepository *repository,
35  double *tot_weight_ptr,
36  double *log_prob_ptr,
37  Nnet *nnet_to_update,
38  bool store_separate_gradients):
39  nnet_(nnet), repository_(repository),
40  nnet_to_update_(nnet_to_update),
41  nnet_to_update_orig_(nnet_to_update),
42  store_separate_gradients_(store_separate_gradients),
43  tot_weight_ptr_(tot_weight_ptr),
44  log_prob_ptr_(log_prob_ptr),
45  tot_weight_(0.0),
46  log_prob_(0.0) { }
47 
48  // The following constructor is called multiple times within
49  // the RunMultiThreaded template function.
51  MultiThreadable(other),
52  nnet_(other.nnet_),
53  repository_(other.repository_),
59  tot_weight_(0),
60  log_prob_(0.0) {
62  // To ensure correctness, we work on separate copies of the gradient
63  // object, which we'll sum at the end. This is used for exact gradient
64  // computation.
65  if (other.nnet_to_update_ != NULL) {
66  nnet_to_update_ = new Nnet(*(other.nnet_to_update_));
67  // our "nnet_to_update_" variable is a copy of the neural network
68  // we are to update (presumably a gradient). If we don't set these
69  // to zero we would end up adding multiple copies of the any initial
70  // gradient that "nnet_to_update_" contained when we initialize
71  // the first instance of the class.
72  nnet_to_update_->SetZero(true);
73  } else { // support case where we don't really need a gradient.
74  nnet_to_update_ = NULL;
75  }
76  }
77  }
78  // This does the main function of the class.
79  void operator () () {
80  std::vector<NnetExample> examples;
81  while (repository_->ProvideExamples(&examples)) {
82  // This is a function call to a function defined in
83  // nnet-update.h
84  double tot_loglike;
85  if (nnet_to_update_ != NULL)
86  tot_loglike = DoBackprop(nnet_, examples, nnet_to_update_);
87  else
88  tot_loglike = ComputeNnetObjf(nnet_, examples);
90  log_prob_ += tot_loglike;
91  KALDI_VLOG(4) << "Thread " << thread_id_ << " saw "
92  << tot_weight_ << " frames so far (weighted); likelihood "
93  << "per frame so far is " << (log_prob_ / tot_weight_);
94  examples.clear();
95  }
96  }
97 
100  // This branch is only taken if this instance of the class is
101  // one of the multiple instances allocated inside the RunMultiThreaded
102  // template function, *and* store_separate_gradients_ has been set to true.
103  // In the typical hogwild case, we don't do this.
105  delete nnet_to_update_;
106  }
109  }
110  private:
111  const Nnet &nnet_;
117  double *log_prob_ptr_;
118  double tot_weight_;
119  double log_prob_; // log-like times num frames.
120 };
121 
122 
123 #if HAVE_CUDA == 1
124 double DoBackpropSingleThreaded(const Nnet &nnet,
125  int32 minibatch_size,
126  SequentialNnetExampleReader *examples_reader,
127  double *tot_weight_out,
128  Nnet *nnet_to_update) {
129  double ans = 0.0, tot_weight = 0.0;
130  KALDI_ASSERT(minibatch_size > 0);
131  while (!examples_reader->Done()) {
132  std::vector<NnetExample> egs;
133  egs.reserve(minibatch_size);
134  while (egs.size() < minibatch_size && examples_reader->Done()) {
135  egs.push_back(examples_reader->Value());
136  examples_reader->Next();
137  }
138  ans += DoBackprop(nnet, egs, nnet_to_update);
139  tot_weight += TotalNnetTrainingWeight(egs);
140  }
141  *tot_weight_out = tot_weight;
142  return ans;
143 }
144 #endif
145 
146 
147 double DoBackpropParallel(const Nnet &nnet,
148  int32 minibatch_size,
149  SequentialNnetExampleReader *examples_reader,
150  double *tot_weight,
151  Nnet *nnet_to_update) {
152 #if HAVE_CUDA == 1
153  // Our GPU code won't work with multithreading; we do this
154  // to enable it to work with this code in the single-threaded
155  // case.
156  if (CuDevice::Instantiate().Enabled())
157  return DoBackpropSingleThreaded(nnet, minibatch_size, examples_reader,
158  tot_weight, nnet_to_update);
159 #endif
160 
161  ExamplesRepository repository; // handles parallel programming issues regarding
162  // the "examples" of data.
163  double tot_log_prob = 0.0;
164  *tot_weight = 0.0;
165 
166  // This function assumes you want the exact gradient, if
167  // nnet_to_update != &nnet.
168  const bool store_separate_gradients = (nnet_to_update != &nnet);
169 
170  DoBackpropParallelClass c(nnet, &repository, tot_weight,
171  &tot_log_prob, nnet_to_update,
172  store_separate_gradients);
173 
174  {
175  // The initialization of the following class spawns the threads that
176  // process the examples. They get re-joined in its destructor.
178 
179  std::vector<NnetExample> examples;
180  for (; !examples_reader->Done(); examples_reader->Next()) {
181  examples.push_back(examples_reader->Value());
182  if (examples.size() == minibatch_size)
183  repository.AcceptExamples(&examples);
184  }
185  if (!examples.empty()) // partial minibatch.
186  repository.AcceptExamples(&examples);
187  // Here, the destructor of "m" re-joins the threads, and
188  // does the summing of the gradients if we're doing gradient
189  // computation (i.e. &nnet != nnet_to_update). This gets
190  // done in the destructors of the objects of type
191  // DoBackpropParallelClass.
192  repository.ExamplesDone();
193  }
194  KALDI_LOG << "Did backprop on " << *tot_weight << " examples, average log-prob "
195  << "per frame is " << (tot_log_prob / *tot_weight);
196  KALDI_LOG << "[this line is to be parsed by a script:] log-prob-per-frame="
197  << (tot_log_prob / *tot_weight);
198  return tot_log_prob;
199 }
200 
201 
202 double DoBackpropSingleThreaded(const Nnet &nnet,
203  int32 minibatch_size,
204  const std::vector<NnetExample> &egs,
205  double *tot_weight,
206  Nnet *nnet_to_update) {
207  double ans = 0.0;
208  *tot_weight = TotalNnetTrainingWeight(egs);
209  for (size_t i = 0; i < egs.size(); i += minibatch_size) {
210  std::vector<NnetExample>::const_iterator end_iter =
211  (i + minibatch_size > egs.size() ? egs.end() :
212  egs.begin() + i + minibatch_size);
213  std::vector<NnetExample> this_egs(egs.begin() + i,
214  end_iter);
215  ans += DoBackprop(nnet, this_egs, nnet_to_update);
216  }
217  return ans;
218 }
219 
220 
221 double DoBackpropParallel(const Nnet &nnet,
222  int32 minibatch_size,
223  int32 num_threads,
224  const std::vector<NnetExample> &egs,
225  double *tot_weight,
226  Nnet *nnet_to_update) {
227  if (num_threads == 1) // support GPUs: special case for 1 thread.
228  return DoBackpropSingleThreaded(nnet, minibatch_size, egs,
229  tot_weight, nnet_to_update);
230 
231  ExamplesRepository repository; // handles parallel programming issues regarding
232  // the "examples" of data.
233  double tot_log_prob = 0.0;
234  *tot_weight = 0;
235  const bool store_separate_gradients = (nnet_to_update != &nnet);
236 
237  DoBackpropParallelClass c(nnet, &repository, tot_weight,
238  &tot_log_prob, nnet_to_update,
239  store_separate_gradients);
240 
241  {
242  // The initialization of the following class spawns the threads that
243  // process the examples. They get re-joined in its destructor.
244  MultiThreader<DoBackpropParallelClass> m(num_threads, c);
245 
246  int32 num_egs = egs.size();
247  for (int32 offset = 0; offset < num_egs; offset += minibatch_size) {
248  int32 this_minibatch_size = std::min(minibatch_size, num_egs - offset);
249 
250  // We waste a little time copying the examples here, but it's very minor.
251  std::vector<NnetExample> examples(egs.begin() + offset,
252  egs.begin() + offset + this_minibatch_size);
253 
254  repository.AcceptExamples(&examples);
255  }
256 
257  // Here, the destructor of "m" re-joins the threads, and
258  // does the summing of the gradients if we're doing gradient
259  // computation (i.e. &nnet != nnet_to_update). This gets
260  // done in the destructors of the objects of type
261  // DoBackpropParallelClass.
262  repository.ExamplesDone();
263  }
264  KALDI_VLOG(2) << "Did backprop on " << *tot_weight << " examples, average log-prob "
265  << "per frame is " << (tot_log_prob / *tot_weight);
266  return tot_log_prob;
267 }
268 
269 
270 } // namespace nnet2
271 } // namespace kaldi
This code computes Goodness of Pronunciation (GOP) and extracts phone-level pronunciation feature for...
Definition: chain.dox:20
double DoBackpropSingleThreaded(const Nnet &nnet, int32 minibatch_size, const std::vector< NnetExample > &egs, double *tot_weight, Nnet *nnet_to_update)
void AcceptExamples(std::vector< NnetExample > *examples)
The following function is called by the code that reads in the examples, with a batch of examples...
void AddNnet(const VectorBase< BaseFloat > &scales, const Nnet &other)
For each updatatable component, adds to it the corresponding element of "other" times the appropriate...
Definition: nnet-nnet.cc:576
void ExamplesDone()
The following function is called by the code that reads in the examples, when we&#39;re done reading exam...
int32 g_num_threads
Definition: kaldi-thread.cc:25
kaldi::int32 int32
double ComputeNnetObjf(const Nnet &nnet, const std::vector< NnetExample > &examples, double *tot_accuracy)
Computes objective function over a minibatch.
Definition: nnet-update.cc:258
double DoBackprop(const Nnet &nnet, const std::vector< NnetExample > &examples, Nnet *nnet_to_update, double *tot_accuracy)
This function computes the objective function and either updates the model or adds to parameter gradi...
Definition: nnet-update.cc:265
void SetZero(bool treat_as_gradient)
Definition: nnet-nnet.cc:151
This class stores neural net training examples to be used in multi-threaded training.
Definition: nnet-example.h:99
bool ProvideExamples(std::vector< NnetExample > *examples)
This function is called by the code that does the training.
A templated class for reading objects sequentially from an archive or script file; see The Table conc...
Definition: kaldi-table.h:287
DoBackpropParallelClass(const DoBackpropParallelClass &other)
BaseFloat TotalNnetTrainingWeight(const std::vector< NnetExample > &egs)
Returns the total weight summed over all the examples...
Definition: nnet-update.cc:248
DoBackpropParallelClass(const Nnet &nnet, ExamplesRepository *repository, double *tot_weight_ptr, double *log_prob_ptr, Nnet *nnet_to_update, bool store_separate_gradients)
#define KALDI_ASSERT(cond)
Definition: kaldi-error.h:185
This header provides functionality for sample-by-sample stochastic gradient descent and gradient comp...
#define KALDI_VLOG(v)
Definition: kaldi-error.h:156
#define KALDI_LOG
Definition: kaldi-error.h:153
double DoBackpropParallel(const Nnet &nnet, int32 minibatch_size, SequentialNnetExampleReader *examples_reader, double *tot_weight, Nnet *nnet_to_update)
This function is similar to "DoBackprop" in nnet-update.h This function computes the objective functi...