SvdApplier Class Reference

Collaboration diagram for SvdApplier:

Classes
struct	ModifiedComponentInfo

Public Member Functions
	SvdApplier (const std::string component_name_pattern, int32 bottleneck_dim, BaseFloat energy_threshold, BaseFloat shrinkage_threshold, Nnet *nnet)
void	ApplySvd ()

Private Member Functions
void	DecomposeComponents ()
int32	GetReducedDimension (const Vector< BaseFloat > &input_vector, int32 lower, int32 upper, BaseFloat min_val)
bool	DecomposeComponent (const std::string &component_name, const AffineComponent &affine, Component **component_a_out, Component **component_b_out)
void	ModifyTopology ()

Private Attributes
std::vector< int32 >	modification_index_
std::vector< ModifiedComponentInfo >	modified_component_info_
Nnet *	nnet_
int32	bottleneck_dim_
BaseFloat	energy_threshold_
BaseFloat	shrinkage_threshold_
std::string	component_name_pattern_

Detailed Description

Definition at line 663 of file nnet-utils.cc.

Constructor & Destructor Documentation

SvdApplier()

SvdApplier ( const std::string  component_name_pattern, int32  bottleneck_dim, BaseFloat  energy_threshold, BaseFloat  shrinkage_threshold, Nnet *  nnet  )

inline

Definition at line 665 of file nnet-utils.cc.

                        : nnet_(nnet),
                          bottleneck_dim_(bottleneck_dim),
                          energy_threshold_(energy_threshold),
                          shrinkage_threshold_(shrinkage_threshold),
                          component_name_pattern_(component_name_pattern) { }

Member Function Documentation

ApplySvd()

void ApplySvd ( )

inline

Definition at line 674 of file nnet-utils.cc.

References SvdApplier::bottleneck_dim_, SvdApplier::DecomposeComponents(), KALDI_LOG, SvdApplier::modified_component_info_, and SvdApplier::ModifyTopology().

Referenced by kaldi::nnet3::ReadEditConfig().

                  {
    DecomposeComponents();
    if (!modified_component_info_.empty())
      ModifyTopology();
    KALDI_LOG << "Decomposed " << modified_component_info_.size()
              << " components with SVD dimension " << bottleneck_dim_;
  }

DecomposeComponent()

bool DecomposeComponent ( const std::string &  component_name, const AffineComponent &  affine, Component **  component_a_out, Component **  component_b_out  )

inlineprivate

Definition at line 765 of file nnet-utils.cc.

References VectorBase< Real >::AddVec2(), AffineComponent::BiasParams(), SvdApplier::bottleneck_dim_, SvdApplier::energy_threshold_, SvdApplier::GetReducedDimension(), AffineComponent::InputDim(), KALDI_ASSERT, KALDI_LOG, kaldi::kCopyData, AffineComponent::LinearParams(), MatrixBase< Real >::MulColsVec(), AffineComponent::OutputDim(), Matrix< Real >::Resize(), UpdatableComponent::SetUpdatableConfigs(), SvdApplier::shrinkage_threshold_, and kaldi::SortSvd().

Referenced by SvdApplier::DecomposeComponents().

                                                       {
    int32 input_dim = affine.InputDim(), output_dim = affine.OutputDim();
    Matrix<BaseFloat> linear_params(affine.LinearParams());
    Vector<BaseFloat> bias_params(affine.BiasParams());
    int32 middle_dim = std::min<int32>(input_dim, output_dim);

    // note: 'linear_params' is of dimension output_dim by input_dim.
    Vector<BaseFloat> s(middle_dim);
    Matrix<BaseFloat> A(middle_dim, input_dim),
        B(output_dim, middle_dim);
    linear_params.Svd(&s, &B, &A);
    // make sure the singular values are sorted from greatest to least value.
    SortSvd(&s, &B, &A);
    Vector<BaseFloat> s2(s.Dim());
    s2.AddVec2(1.0, s);
    BaseFloat s2_sum_orig = s2.Sum();
    KALDI_ASSERT(energy_threshold_ < 1);
    KALDI_ASSERT(shrinkage_threshold_ < 1);
    if (energy_threshold_ > 0) {
      BaseFloat min_singular_sum = energy_threshold_ * s2_sum_orig;
      bottleneck_dim_ = GetReducedDimension(s2, 0, s2.Dim()-1, min_singular_sum);
    }
    SubVector<BaseFloat> this_part(s2, 0, bottleneck_dim_);
    BaseFloat s2_sum_reduced = this_part.Sum();
    BaseFloat shrinkage_ratio =
      static_cast<BaseFloat>(bottleneck_dim_ * (input_dim+output_dim))
      / static_cast<BaseFloat>(input_dim * output_dim);
    if (shrinkage_ratio > shrinkage_threshold_) {
      KALDI_LOG << "Shrinkage ratio " << shrinkage_ratio
                << " greater than threshold : " << shrinkage_threshold_
                << " Skipping SVD for this layer.";
      return false;
    }

    s.Resize(bottleneck_dim_, kCopyData);
    A.Resize(bottleneck_dim_, input_dim, kCopyData);
    B.Resize(output_dim, bottleneck_dim_, kCopyData);
    KALDI_LOG << "For component " << component_name
              << " singular value squared sum changed by "
              << (s2_sum_orig - s2_sum_reduced)
              << " (from " << s2_sum_orig << " to " << s2_sum_reduced << ")";
    KALDI_LOG << "For component " << component_name
              << " dimension reduced from "
              << " (" << input_dim << "," << output_dim << ")"
              << " to [(" << input_dim << "," << bottleneck_dim_
              << "), (" << bottleneck_dim_ << "," << output_dim <<")]";
    KALDI_LOG << "shrinkage ratio : " << shrinkage_ratio;

    // we'll divide the singular values equally between the two
    // parameter matrices.
    s.ApplyPow(0.5);
    A.MulRowsVec(s);
    B.MulColsVec(s);

    CuMatrix<BaseFloat> A_cuda(A), B_cuda(B);
    CuVector<BaseFloat> bias_params_cuda(bias_params);

    LinearComponent *component_a = new LinearComponent(A_cuda);
    NaturalGradientAffineComponent *component_b =
        new NaturalGradientAffineComponent(B_cuda, bias_params_cuda);
    // set the learning rates, max-change, and so on.
    component_a->SetUpdatableConfigs(affine);
    component_b->SetUpdatableConfigs(affine);
    *component_a_out = component_a;
    *component_b_out = component_b;
    return true;
  }

DecomposeComponents()

void DecomposeComponents ( )

inlineprivate

Definition at line 686 of file nnet-utils.cc.

References Nnet::AddComponent(), SvdApplier::bottleneck_dim_, SvdApplier::ModifiedComponentInfo::component_a_index, SvdApplier::ModifiedComponentInfo::component_b_index, SvdApplier::ModifiedComponentInfo::component_index, SvdApplier::ModifiedComponentInfo::component_name, SvdApplier::ModifiedComponentInfo::component_name_a, SvdApplier::ModifiedComponentInfo::component_name_b, SvdApplier::component_name_pattern_, SvdApplier::DecomposeComponent(), Nnet::GetComponent(), Nnet::GetComponentIndex(), Nnet::GetComponentName(), AffineComponent::InputDim(), KALDI_ERR, KALDI_LOG, KALDI_WARN, SvdApplier::modification_index_, SvdApplier::modified_component_info_, rnnlm::n, kaldi::nnet3::NameMatchesPattern(), SvdApplier::nnet_, Nnet::NumComponents(), and AffineComponent::OutputDim().

Referenced by SvdApplier::ApplySvd().

                             {
    int32 num_components = nnet_->NumComponents();
    modification_index_.resize(num_components, -1);
    for (int32 c = 0; c < num_components; c++) {
      Component *component = nnet_->GetComponent(c);
      std::string component_name = nnet_->GetComponentName(c);
      if (NameMatchesPattern(component_name.c_str(),
                             component_name_pattern_.c_str())) {
        AffineComponent *affine =  dynamic_cast<AffineComponent*>(component);
        if (affine == NULL) {
          KALDI_WARN << "Not decomposing component " << component_name
                     << " as it is not an AffineComponent.";
          continue;
        }
        int32 input_dim = affine->InputDim(),
            output_dim = affine->OutputDim();
        if (input_dim <= bottleneck_dim_ || output_dim <= bottleneck_dim_) {
          KALDI_WARN << "Not decomposing component " << component_name
                     << " with SVD to rank " << bottleneck_dim_
                     << " because its dimension is " << input_dim
                     << " -> " << output_dim;
          continue;
        }
        Component *component_a = NULL, *component_b = NULL;
        if (DecomposeComponent(component_name, *affine, &component_a, &component_b)) {
          size_t n = modified_component_info_.size();
          modification_index_[c] = n;
          modified_component_info_.resize(n + 1);
          ModifiedComponentInfo &info = modified_component_info_[n];
          info.component_index = c;
          info.component_name = component_name;
          info.component_name_a = component_name + "_a";
          info.component_name_b = component_name + "_b";
          if (nnet_->GetComponentIndex(info.component_name_a) >= 0)
            KALDI_ERR << "Neural network already has a component named "
                      << info.component_name_a;
          if (nnet_->GetComponentIndex(info.component_name_b) >= 0)
            KALDI_ERR << "Neural network already has a component named "
                      << info.component_name_b;
          info.component_a_index = nnet_->AddComponent(info.component_name_a,
                                                       component_a);
          info.component_b_index = nnet_->AddComponent(info.component_name_b,
                                                       component_b);
        }
      }
    }
    KALDI_LOG << "Converted " << modified_component_info_.size()
              << " components to FixedAffineComponent.";
  }

GetReducedDimension()

int32 GetReducedDimension ( const Vector< BaseFloat > &  input_vector, int32  lower, int32  upper, BaseFloat  min_val  )

inlineprivate

Definition at line 743 of file nnet-utils.cc.

References rnnlm::i.

Referenced by SvdApplier::DecomposeComponent().

                                                {
    BaseFloat sum = 0;
    int32 i = 0;
    for (i = lower; i <= upper; i++) {
        sum = sum + input_vector(i);
        if (sum >= min_val) break;
    }
    return (i+1);
  }

ModifyTopology()

void ModifyTopology ( )

inlineprivate

Definition at line 847 of file nnet-utils.cc.

References NetworkNode::component_index, SvdApplier::ModifiedComponentInfo::component_name_a, SvdApplier::ModifiedComponentInfo::component_name_b, NetworkNode::descriptor, NetworkNode::dim, NetworkNode::dim_offset, Nnet::GetComponentName(), Nnet::GetNode(), Nnet::GetNodeNames(), Nnet::IsComponentInputNode(), Nnet::IsComponentNode(), Nnet::IsInputNode(), Nnet::IsOutputNode(), KALDI_ASSERT, KALDI_ERR, kaldi::nnet3::kComponent, kaldi::nnet3::kDescriptor, kaldi::nnet3::kDimRange, kaldi::nnet3::kLinear, SvdApplier::modification_index_, SvdApplier::modified_component_info_, rnnlm::n, SvdApplier::nnet_, NetworkNode::node_index, NetworkNode::node_type, Nnet::NumNodes(), NetworkNode::objective_type, Nnet::ReadConfig(), Nnet::RemoveOrphanComponents(), Nnet::RemoveOrphanNodes(), NetworkNode::u, and Descriptor::WriteConfig().

Referenced by SvdApplier::ApplySvd().

                        {
    std::set<int32> nodes_to_modify;
    std::vector<std::string> node_names_orig = nnet_->GetNodeNames(),
        node_names_modified = node_names_orig;

    // The following loop sets up 'nodes_to_modify' and 'node_names_modified'.
    for (int32 n = 0; n < nnet_->NumNodes(); n++) {
      if (nnet_->IsComponentNode(n)) {
        NetworkNode &node = nnet_->GetNode(n);
        int32 component_index = node.u.component_index,
            modification_index = modification_index_[component_index];
        if (modification_index >= 0) {
          // This is a component-node for one of the components that we're
          // splitting in two.
          nodes_to_modify.insert(n);
          std::string node_name = node_names_orig[n],
              node_name_b = node_name + "_b";
          node_names_modified[n] = node_name_b;
        }
      }
    }


    // config_os is a stream to which we are printing lines that we'll later
    // read using nnet_->ReadConfig().
    std::ostringstream config_os;
    // The following loop writes to 'config_os'. The the code is modified from
    // the private function Nnet::GetAsConfigLine(), and from
    // Nnet::GetConfigLines().
    for (int32 n = 0; n < nnet_->NumNodes(); n++) {
      if (nnet_->IsComponentInputNode(n) || nnet_->IsInputNode(n)) {
        // component-input descriptor nodes aren't handled separately from their
        // associated components (we deal with them along with their
        // component-node); and input-nodes won't be affected so we don't have
        // to print anything.
        continue;
      }
      const NetworkNode &node = nnet_->GetNode(n);
      int32 c = node.u.component_index;  // 'c' will only be meaningful if the
                                         // node is a component-node.
      std::string node_name = node_names_orig[n];
      if (node.node_type == kComponent &&  modification_index_[c] >= 0) {
        ModifiedComponentInfo &info = modified_component_info_[
            modification_index_[c]];
        std::string node_name_a = node_name + "_a",
            node_name_b = node_name + "_b";
        // we print two component-nodes, the "a" an "b".  The original
        // one will later be removed when we call RemoveOrphanNodes().
        config_os << "component-node name=" << node_name_a << " component="
                  << info.component_name_a << " input=";
        nnet_->GetNode(n-1).descriptor.WriteConfig(config_os, node_names_modified);
        config_os << "\n";
        config_os << "component-node name=" << node_name_b << " component="
                  << info.component_name_b << " input=" << node_name_a << "\n";
      } else {
        // This code is modified from Nnet::GetAsConfigLine().  The key difference
        // is that we're using node_names_modified, which will replace all the
        // nodes we're splitting with their "b" versions.
        switch (node.node_type) {
          case kDescriptor:
            // assert that it's an output-descriptor, not one describing the input to
            // a component-node.
            KALDI_ASSERT(nnet_->IsOutputNode(n));
            config_os << "output-node name=" << node_name << " input=";
            node.descriptor.WriteConfig(config_os, node_names_modified);
            config_os << " objective=" << (node.u.objective_type == kLinear ?
                                           "linear" : "quadratic");
            break;
          case kComponent:
            config_os << "component-node name=" << node_name << " component="
                      << nnet_->GetComponentName(node.u.component_index)
                      << " input=";
            nnet_->GetNode(n-1).descriptor.WriteConfig(config_os,
                                                       node_names_modified);
            break;
          case kDimRange:
            config_os << "dim-range-node name=" << node_name << " input-node="
                      << node_names_modified[node.u.node_index]
                      << " dim-offset=" << node.dim_offset
                      << " dim=" << node.dim;
            break;
          default:
            KALDI_ERR << "Unexpected node type.";
        }
        config_os << "\n";
      }
    }
    std::istringstream config_is(config_os.str());
    nnet_->ReadConfig(config_is);
    nnet_->RemoveOrphanNodes();
    nnet_->RemoveOrphanComponents();
  }

Member Data Documentation

bottleneck_dim_

int32 bottleneck_dim_

private

Definition at line 967 of file nnet-utils.cc.

Referenced by SvdApplier::ApplySvd(), SvdApplier::DecomposeComponent(), and SvdApplier::DecomposeComponents().

component_name_pattern_

std::string component_name_pattern_

private

Definition at line 970 of file nnet-utils.cc.

Referenced by SvdApplier::DecomposeComponents().

energy_threshold_

BaseFloat energy_threshold_

private

Definition at line 968 of file nnet-utils.cc.

Referenced by SvdApplier::DecomposeComponent().

modification_index_

std::vector<int32> modification_index_

private

Definition at line 945 of file nnet-utils.cc.

Referenced by SvdApplier::DecomposeComponents(), and SvdApplier::ModifyTopology().

modified_component_info_

std::vector<ModifiedComponentInfo> modified_component_info_

private

Definition at line 963 of file nnet-utils.cc.

Referenced by SvdApplier::ApplySvd(), SvdApplier::DecomposeComponents(), and SvdApplier::ModifyTopology().

nnet_

Nnet* nnet_

private

Definition at line 966 of file nnet-utils.cc.

Referenced by ModelCollapser::Collapse(), ModelCollapser::CollapseComponentsAffine(), ModelCollapser::CollapseComponentsBatchnorm(), ModelCollapser::CollapseComponentsDropout(), ModelCollapser::CollapseComponentsScale(), SvdApplier::DecomposeComponents(), ModelCollapser::GetDiagonallyPreModifiedComponentIndex(), ModelCollapser::GetScaledComponentIndex(), SvdApplier::ModifyTopology(), ModelCollapser::OptimizeNode(), and ModelCollapser::ReplaceNodeInDescriptor().

shrinkage_threshold_

BaseFloat shrinkage_threshold_

private

Definition at line 969 of file nnet-utils.cc.

Referenced by SvdApplier::DecomposeComponent().

---

The documentation for this class was generated from the following file:

• nnet3/nnet-utils.cc