Skip to content

Commit

Permalink
[sycl] Add depth-wise policy (dmlc#10690)
Browse files Browse the repository at this point in the history 
Co-authored-by: Dmitry Razdoburdin <>
  • Loading branch information
@razdoburdin
razdoburdin authored Aug 13, 2024
1 parent b457d0d commit 773ded6
Show file tree
Hide file tree
Showing 3 changed files with 252 additions and 1 deletion.
158 changes: 157 additions & 1 deletion plugin/sycl/tree/hist_updater.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,162 @@ void HistUpdater<GradientSumT>::BuildLocalHistograms(
builder_monitor_.Stop("BuildLocalHistograms");
}

template<typename GradientSumT>
void HistUpdater<GradientSumT>::BuildNodeStats(
const common::GHistIndexMatrix &gmat,
RegTree *p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair) {
builder_monitor_.Start("BuildNodeStats");
for (auto const& entry : qexpand_depth_wise_) {
int nid = entry.nid;
this->InitNewNode(nid, gmat, gpair, *p_tree);
// add constraints
if (!(*p_tree)[nid].IsLeftChild() && !(*p_tree)[nid].IsRoot()) {
// it's a right child
auto parent_id = (*p_tree)[nid].Parent();
auto left_sibling_id = (*p_tree)[parent_id].LeftChild();
auto parent_split_feature_id = snode_host_[parent_id].best.SplitIndex();
tree_evaluator_.AddSplit(
parent_id, left_sibling_id, nid, parent_split_feature_id,
snode_host_[left_sibling_id].weight, snode_host_[nid].weight);
interaction_constraints_.Split(parent_id, parent_split_feature_id,
left_sibling_id, nid);
}
}
builder_monitor_.Stop("BuildNodeStats");
}

template<typename GradientSumT>
void HistUpdater<GradientSumT>::AddSplitsToTree(
const common::GHistIndexMatrix &gmat,
RegTree *p_tree,
int *num_leaves,
int depth,
std::vector<ExpandEntry>* nodes_for_apply_split,
std::vector<ExpandEntry>* temp_qexpand_depth) {
builder_monitor_.Start("AddSplitsToTree");
auto evaluator = tree_evaluator_.GetEvaluator();
for (auto const& entry : qexpand_depth_wise_) {
const auto lr = param_.learning_rate;
int nid = entry.nid;

if (snode_host_[nid].best.loss_chg < kRtEps ||
(param_.max_depth > 0 && depth == param_.max_depth) ||
(param_.max_leaves > 0 && (*num_leaves) == param_.max_leaves)) {
(*p_tree)[nid].SetLeaf(snode_host_[nid].weight * lr);
} else {
nodes_for_apply_split->push_back(entry);

NodeEntry<GradientSumT>& e = snode_host_[nid];
bst_float left_leaf_weight =
evaluator.CalcWeight(nid, GradStats<GradientSumT>{e.best.left_sum}) * lr;
bst_float right_leaf_weight =
evaluator.CalcWeight(nid, GradStats<GradientSumT>{e.best.right_sum}) * lr;
p_tree->ExpandNode(nid, e.best.SplitIndex(), e.best.split_value,
e.best.DefaultLeft(), e.weight, left_leaf_weight,
right_leaf_weight, e.best.loss_chg, e.stats.GetHess(),
e.best.left_sum.GetHess(), e.best.right_sum.GetHess());

int left_id = (*p_tree)[nid].LeftChild();
int right_id = (*p_tree)[nid].RightChild();
temp_qexpand_depth->push_back(ExpandEntry(left_id, p_tree->GetDepth(left_id)));
temp_qexpand_depth->push_back(ExpandEntry(right_id, p_tree->GetDepth(right_id)));
// - 1 parent + 2 new children
(*num_leaves)++;
}
}
builder_monitor_.Stop("AddSplitsToTree");
}


template<typename GradientSumT>
void HistUpdater<GradientSumT>::EvaluateAndApplySplits(
const common::GHistIndexMatrix &gmat,
RegTree *p_tree,
int *num_leaves,
int depth,
std::vector<ExpandEntry> *temp_qexpand_depth) {
EvaluateSplits(qexpand_depth_wise_, gmat, *p_tree);

std::vector<ExpandEntry> nodes_for_apply_split;
AddSplitsToTree(gmat, p_tree, num_leaves, depth,
&nodes_for_apply_split, temp_qexpand_depth);
ApplySplit(nodes_for_apply_split, gmat, p_tree);
}

// Split nodes to 2 sets depending on amount of rows in each node
// Histograms for small nodes will be built explicitly
// Histograms for big nodes will be built by 'Subtraction Trick'
// Exception: in distributed setting, we always build the histogram for the left child node
// and use 'Subtraction Trick' to built the histogram for the right child node.
// This ensures that the workers operate on the same set of tree nodes.
template <typename GradientSumT>
void HistUpdater<GradientSumT>::SplitSiblings(
const std::vector<ExpandEntry> &nodes,
std::vector<ExpandEntry> *small_siblings,
std::vector<ExpandEntry> *big_siblings,
RegTree *p_tree) {
builder_monitor_.Start("SplitSiblings");
for (auto const& entry : nodes) {
int nid = entry.nid;
RegTree::Node &node = (*p_tree)[nid];
if (node.IsRoot()) {
small_siblings->push_back(entry);
} else {
const int32_t left_id = (*p_tree)[node.Parent()].LeftChild();
const int32_t right_id = (*p_tree)[node.Parent()].RightChild();

if (nid == left_id && row_set_collection_[left_id ].Size() <
row_set_collection_[right_id].Size()) {
small_siblings->push_back(entry);
} else if (nid == right_id && row_set_collection_[right_id].Size() <=
row_set_collection_[left_id ].Size()) {
small_siblings->push_back(entry);
} else {
big_siblings->push_back(entry);
}
}
}
builder_monitor_.Stop("SplitSiblings");
}

template<typename GradientSumT>
void HistUpdater<GradientSumT>::ExpandWithDepthWise(
const common::GHistIndexMatrix &gmat,
RegTree *p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair) {
int num_leaves = 0;

// in depth_wise growing, we feed loss_chg with 0.0 since it is not used anyway
qexpand_depth_wise_.emplace_back(ExpandEntry::kRootNid,
p_tree->GetDepth(ExpandEntry::kRootNid));
++num_leaves;
for (int depth = 0; depth < param_.max_depth + 1; depth++) {
std::vector<int> sync_ids;
std::vector<ExpandEntry> temp_qexpand_depth;
SplitSiblings(qexpand_depth_wise_, &nodes_for_explicit_hist_build_,
&nodes_for_subtraction_trick_, p_tree);
hist_rows_adder_->AddHistRows(this, &sync_ids, p_tree);
BuildLocalHistograms(gmat, p_tree, gpair);
hist_synchronizer_->SyncHistograms(this, sync_ids, p_tree);
BuildNodeStats(gmat, p_tree, gpair);

EvaluateAndApplySplits(gmat, p_tree, &num_leaves, depth,
&temp_qexpand_depth);

// clean up
qexpand_depth_wise_.clear();
nodes_for_subtraction_trick_.clear();
nodes_for_explicit_hist_build_.clear();
if (temp_qexpand_depth.empty()) {
break;
} else {
qexpand_depth_wise_ = temp_qexpand_depth;
temp_qexpand_depth.clear();
}
}
}

template<typename GradientSumT>
void HistUpdater<GradientSumT>::ExpandWithLossGuide(
const common::GHistIndexMatrix& gmat,
Expand Down Expand Up @@ -326,7 +482,7 @@ void HistUpdater<GradientSumT>::InitData(
if (param_.grow_policy == xgboost::tree::TrainParam::kLossGuide) {
qexpand_loss_guided_.reset(new ExpandQueue(LossGuide));
} else {
LOG(WARNING) << "Depth-wise building is not yet implemented";
qexpand_depth_wise_.clear();
}
}
builder_monitor_.Stop("InitData");
Expand Down
31 changes: 31 additions & 0 deletions plugin/sycl/tree/hist_updater.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,36 @@ class HistUpdater {
const USMVector<GradientPair, MemoryType::on_device> &gpair,
const RegTree& tree);

// Split nodes to 2 sets depending on amount of rows in each node
// Histograms for small nodes will be built explicitly
// Histograms for big nodes will be built by 'Subtraction Trick'
void SplitSiblings(const std::vector<ExpandEntry>& nodes,
std::vector<ExpandEntry>* small_siblings,
std::vector<ExpandEntry>* big_siblings,
RegTree *p_tree);

void BuildNodeStats(const common::GHistIndexMatrix &gmat,
RegTree *p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair);

void EvaluateAndApplySplits(const common::GHistIndexMatrix &gmat,
RegTree *p_tree,
int *num_leaves,
int depth,
std::vector<ExpandEntry> *temp_qexpand_depth);

void AddSplitsToTree(
const common::GHistIndexMatrix &gmat,
RegTree *p_tree,
int *num_leaves,
int depth,
std::vector<ExpandEntry>* nodes_for_apply_split,
std::vector<ExpandEntry>* temp_qexpand_depth);

void ExpandWithDepthWise(const common::GHistIndexMatrix &gmat,
RegTree *p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair);

void BuildLocalHistograms(const common::GHistIndexMatrix &gmat,
RegTree *p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair);
Expand Down Expand Up @@ -180,6 +210,7 @@ class HistUpdater {
std::function<bool(ExpandEntry, ExpandEntry)>>;

std::unique_ptr<ExpandQueue> qexpand_loss_guided_;
std::vector<ExpandEntry> qexpand_depth_wise_;

enum DataLayout { kDenseDataZeroBased, kDenseDataOneBased, kSparseData };
DataLayout data_layout_;
Expand Down
64 changes: 64 additions & 0 deletions tests/cpp/plugin/test_sycl_hist_updater.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,13 @@ class TestHistUpdater : public HistUpdater<GradientSumT> {
const USMVector<GradientPair, MemoryType::on_device> &gpair) {
HistUpdater<GradientSumT>::ExpandWithLossGuide(gmat, p_tree, gpair);
}

auto TestExpandWithDepthWise(const common::GHistIndexMatrix& gmat,
DMatrix *p_fmat,
RegTree* p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair) {
HistUpdater<GradientSumT>::ExpandWithDepthWise(gmat, p_tree, gpair);
}
};

void GenerateRandomGPairs(::sycl::queue* qu, GradientPair* gpair_ptr, size_t num_rows, bool has_neg_hess) {
Expand Down Expand Up @@ -544,6 +551,55 @@ void TestHistUpdaterExpandWithLossGuide(const xgboost::tree::TrainParam& param)
}


template <typename GradientSumT>
void TestHistUpdaterExpandWithDepthWise(const xgboost::tree::TrainParam& param) {
const size_t num_rows = 3;
const size_t num_columns = 1;
const size_t n_bins = 16;

Context ctx;
ctx.UpdateAllowUnknown(Args{{"device", "sycl"}});

DeviceManager device_manager;
auto qu = device_manager.GetQueue(ctx.Device());

std::vector<float> data = {7, 3, 15};
auto p_fmat = GetDMatrixFromData(data, num_rows, num_columns);

DeviceMatrix dmat;
dmat.Init(qu, p_fmat.get());
common::GHistIndexMatrix gmat;
gmat.Init(qu, &ctx, dmat, n_bins);

std::vector<GradientPair> gpair_host = {{1, 2}, {3, 1}, {1, 1}};
USMVector<GradientPair, MemoryType::on_device> gpair(&qu, gpair_host);

RegTree tree;
FeatureInteractionConstraintHost int_constraints;
ObjInfo task{ObjInfo::kRegression};
std::unique_ptr<TreeUpdater> pruner{TreeUpdater::Create("prune", &ctx, &task)};
TestHistUpdater<GradientSumT> updater(&ctx, qu, param, std::move(pruner), int_constraints, p_fmat.get());
updater.SetHistSynchronizer(new BatchHistSynchronizer<GradientSumT>());
updater.SetHistRowsAdder(new BatchHistRowsAdder<GradientSumT>());
auto* row_set_collection = updater.TestInitData(gmat, gpair, *p_fmat, tree);

updater.TestExpandWithDepthWise(gmat, p_fmat.get(), &tree, gpair);

const auto& nodes = tree.GetNodes();
std::vector<float> ans(data.size());
for (size_t data_idx = 0; data_idx < data.size(); ++data_idx) {
size_t node_idx = 0;
while (!nodes[node_idx].IsLeaf()) {
node_idx = data[data_idx] < nodes[node_idx].SplitCond() ? nodes[node_idx].LeftChild() : nodes[node_idx].RightChild();
}
ans[data_idx] = nodes[node_idx].LeafValue();
}

ASSERT_NEAR(ans[0], -0.15, 1e-6);
ASSERT_NEAR(ans[1], -0.45, 1e-6);
ASSERT_NEAR(ans[2], -0.15, 1e-6);
}

TEST(SyclHistUpdater, Sampling) {
xgboost::tree::TrainParam param;
param.UpdateAllowUnknown(Args{{"subsample", "0.7"}});
Expand Down Expand Up @@ -620,4 +676,12 @@ TEST(SyclHistUpdater, ExpandWithLossGuide) {
TestHistUpdaterExpandWithLossGuide<double>(param);
}

TEST(SyclHistUpdater, ExpandWithDepthWise) {
xgboost::tree::TrainParam param;
param.UpdateAllowUnknown(Args{{"max_depth", "2"}});

TestHistUpdaterExpandWithDepthWise<float>(param);
TestHistUpdaterExpandWithDepthWise<double>(param);
}

} // namespace xgboost::sycl::tree

0 comments on commit 773ded6

Please sign in to comment.