From 87a23a9a53387f5e4a3f89678672b67532cf48c8 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Fri, 10 Nov 2023 15:48:18 +0100 Subject: [PATCH 1/5] Minor. --- src/expression_diff.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/expression_diff.cpp b/src/expression_diff.cpp index 3334595f7..5959efa4f 100644 --- a/src/expression_diff.cpp +++ b/src/expression_diff.cpp @@ -505,7 +505,7 @@ void diff_tensors_forward_impl( // will produce several times the same derivative, and thus // we need to store the derivatives in a dictionary in order // to prevent duplicates. For order-1 derivatives, no duplicate - // derivatives will be produced and thus we can use a plain vector + // derivatives will be produced and thus we can use a plain vector, // which can be quite a bit faster. using diff_map_t = fast_umap; using diff_vec_t = std::vector>; @@ -740,7 +740,7 @@ void diff_tensors_reverse_impl( // will produce several times the same derivative, and thus // we need to store the derivatives in a dictionary in order // to prevent duplicates. For order-1 derivatives, no duplicate - // derivatives will be produced and thus we can use a plain vector + // derivatives will be produced and thus we can use a plain vector, // which can be quite a bit faster. using diff_map_t = fast_umap; using diff_vec_t = std::vector>; From 99664eceb0f29cf60268ab24853b879a71f5e470 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Fri, 10 Nov 2023 22:07:57 +0100 Subject: [PATCH 2/5] Fix quadratic complexity when building a ffnn model. --- src/model/ffnn.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/model/ffnn.cpp b/src/model/ffnn.cpp index 4e5358021..435b8839b 100644 --- a/src/model/ffnn.cpp +++ b/src/model/ffnn.cpp @@ -19,6 +19,7 @@ #include #include +#include #include HEYOKA_BEGIN_NAMESPACE @@ -43,21 +44,28 @@ std::vector compute_layer(su32 layer_id, const std::vector retval(static_cast::size_type>(n_neurons_curr_layer), 0_dbl); + std::vector retval, tmp_sum; + retval.reserve(n_neurons_curr_layer); + for (su32 i = 0; i < n_neurons_curr_layer; ++i) { + // Clear the summation terms. + tmp_sum.clear(); + for (su32 j = 0; j < n_neurons_prev_layer; ++j) { // Add the weight and update the weight counter. - retval[i] += nn_wb[wcounter] * inputs[j]; + tmp_sum.push_back(nn_wb[wcounter] * inputs[j]); ++wcounter; } // Add the bias and update the counter. - retval[i] += nn_wb[bcounter + n_net_w]; + tmp_sum.push_back(nn_wb[bcounter + n_net_w]); ++bcounter; + // Activation function. - retval[i] = activation(retval[i]); + retval.push_back(activation(sum(tmp_sum))); } + return retval; } From 889e8e0baf00f0d57447c166572a16c64c13da54 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Fri, 10 Nov 2023 22:16:45 +0100 Subject: [PATCH 3/5] Coverage fixes. --- src/expression_diff.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/expression_diff.cpp b/src/expression_diff.cpp index 5959efa4f..f2b52111d 100644 --- a/src/expression_diff.cpp +++ b/src/expression_diff.cpp @@ -749,7 +749,10 @@ void diff_tensors_reverse_impl( // Helpers to ease the access to the active member of the local_diff variant. // NOTE: if used incorrectly, these will throw at runtime. - auto local_dmap = [&local_diff]() -> diff_map_t & { return std::get(local_diff); }; + // NOTE: currently local_dmap is never used because the heuristic + // for deciding between forward and reverse mode prevents reverse mode + // from being used for order > 1. + auto local_dmap = [&local_diff]() -> diff_map_t & { return std::get(local_diff); }; // LCOV_EXCL_LINE auto local_dvec = [&local_diff]() -> diff_vec_t & { return std::get(local_diff); }; // Cache the number of diff arguments. @@ -912,6 +915,7 @@ void diff_tensors_reverse_impl( local_dvec().emplace_back(tmp_v_idx, std::move(cur_der)); } else { + // LCOV_EXCL_START // Check if we already computed this derivative. if (const auto it = local_dmap().find(tmp_v_idx); it == local_dmap().end()) { // The derivative is new. If the diff argument is present in the @@ -926,6 +930,7 @@ void diff_tensors_reverse_impl( [[maybe_unused]] const auto [_, flag] = local_dmap().try_emplace(tmp_v_idx, std::move(cur_der)); assert(flag); } + // LCOV_EXCL_STOP } } @@ -938,7 +943,7 @@ void diff_tensors_reverse_impl( if (cur_order == 1u) { diff_map.insert(diff_map.end(), local_dvec().begin(), local_dvec().end()); } else { - diff_map.insert(diff_map.end(), local_dmap().begin(), local_dmap().end()); + diff_map.insert(diff_map.end(), local_dmap().begin(), local_dmap().end()); // LCOV_EXCL_LINE } } From 360af6185faadbd6324e87495bb3379717d4c29b Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Fri, 10 Nov 2023 22:18:37 +0100 Subject: [PATCH 4/5] Update changelog. --- doc/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/changelog.rst b/doc/changelog.rst index 7c605e168..9a8e42762 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -21,6 +21,9 @@ New Changes ~~~~~~~ +- Substantial speedups in the computation of first-order derivatives + with respect to many variables/parameters + (`#358 `__). - Substantial performance improvements in the computation of derivative tensors of large expressions with a high degree of internal redundancy From 2832c9e69da937a27ad9981776da7e651dc32091 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Fri, 10 Nov 2023 23:33:09 +0100 Subject: [PATCH 5/5] Dox fix. --- doc/advanced_tutorials.rst | 2 +- doc/basic_tutorials.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/advanced_tutorials.rst b/doc/advanced_tutorials.rst index ad1d9a419..02833671f 100644 --- a/doc/advanced_tutorials.rst +++ b/doc/advanced_tutorials.rst @@ -5,7 +5,7 @@ Advanced tutorials .. important:: - More :ref:`tutorials ` and :ref:`examples ` are available in the documentation + More tutorials and examples are available in the documentation of heyoka's `Python bindings `__. In this section we will show some of heyoka's more advanced functionalities, diff --git a/doc/basic_tutorials.rst b/doc/basic_tutorials.rst index 5c69e1a2f..951a8d6ef 100644 --- a/doc/basic_tutorials.rst +++ b/doc/basic_tutorials.rst @@ -5,7 +5,7 @@ Basic tutorials .. important:: - More :ref:`tutorials ` and :ref:`examples ` are available in the documentation + More tutorials and examples are available in the documentation of heyoka's `Python bindings `__. The code snippets in these tutorials assume the inclusion of the