From 82f3514c53acde7a1baa50f98d5f2cd31ca02a6e Mon Sep 17 00:00:00 2001 From: Vladislav Senin Date: Fri, 9 Feb 2024 20:54:33 +0300 Subject: [PATCH] fixed cyclic traversal of classes to measure memory consumption (#942) --- compiler/code-gen/declarations.cpp | 4 +- runtime/dummy-visitor-methods.h | 3 +- runtime/exception.h | 2 +- runtime/ffi.h | 8 +- runtime/job-workers/job-interface.cpp | 2 +- runtime/job-workers/job-interface.h | 6 +- runtime/memcache.h | 10 +- runtime/memory_usage.cpp | 21 --- runtime/memory_usage.h | 229 ++++++++++++++------------ runtime/mysql.h | 2 +- runtime/runtime.cmake | 1 - runtime/tl/rpc_function.h | 8 +- tests/phpt/memory_usage/11_shapes.php | 36 ++++ tests/phpt/memory_usage/2_classes.php | 42 +++++ 14 files changed, 222 insertions(+), 152 deletions(-) delete mode 100644 runtime/memory_usage.cpp create mode 100644 tests/phpt/memory_usage/11_shapes.php diff --git a/compiler/code-gen/declarations.cpp b/compiler/code-gen/declarations.cpp index e5a9b8929d..b046e2f750 100644 --- a/compiler/code-gen/declarations.cpp +++ b/compiler/code-gen/declarations.cpp @@ -932,7 +932,7 @@ void ClassDeclaration::compile_accept_visitor_methods(CodeGenerator &W, ClassPtr // for kphp_instance_cache_value_size statshouse metrics klass->need_instance_cache_visitors) { W << NL; - compile_accept_visitor(W, klass, "InstanceMemoryEstimateVisitor"); + compile_accept_visitor(W, klass, "CommonMemoryEstimateVisitor"); } if (klass->need_instance_cache_visitors) { @@ -1092,7 +1092,7 @@ void ClassMembersDefinition::compile(CodeGenerator &W) const { // for kphp_instance_cache_value_size statshouse metrics klass->need_instance_cache_visitors) { W << NL; - compile_generic_accept_instantiations(W, klass, "InstanceMemoryEstimateVisitor"); + compile_generic_accept_instantiations(W, klass, "CommonMemoryEstimateVisitor"); } if (klass->need_instance_cache_visitors) { diff --git a/runtime/dummy-visitor-methods.h b/runtime/dummy-visitor-methods.h index e93e70f38b..b874ab825e 100644 --- a/runtime/dummy-visitor-methods.h +++ b/runtime/dummy-visitor-methods.h @@ -11,7 +11,8 @@ struct DummyVisitorMethods { // for f$estimate_memory_usage() // set at compiler at deeply_require_instance_memory_estimate_visitor() - void accept(InstanceMemoryEstimateVisitor &) noexcept {} +// void accept(InstanceMemoryEstimateVisitor &) noexcept {} + void accept(CommonMemoryEstimateVisitor &) noexcept {} // for f$instance_to_array(), f$to_array_debug() // set at compiler at deeply_require_to_array_debug_visitor() void accept(ToArrayVisitor &) noexcept {} diff --git a/runtime/exception.h b/runtime/exception.h index 6be87997cd..21b2007394 100644 --- a/runtime/exception.h +++ b/runtime/exception.h @@ -36,7 +36,7 @@ struct C$Throwable : public refcountable_polymorphic_php_classes_virt<> { } } - virtual void accept(InstanceMemoryEstimateVisitor &visitor) noexcept { + virtual void accept(CommonMemoryEstimateVisitor &visitor) noexcept { generic_accept(visitor); } diff --git a/runtime/ffi.h b/runtime/ffi.h index d8f04bff0d..781865957c 100644 --- a/runtime/ffi.h +++ b/runtime/ffi.h @@ -43,7 +43,7 @@ struct CDataArray: public refcountable_php_classes> { } } - void accept(InstanceMemoryEstimateVisitor&) {} + void accept(CommonMemoryEstimateVisitor&) {} }; // Maybe CDataRef is enough for both field/array references, @@ -94,14 +94,14 @@ struct CDataPtr { c_value = reinterpret_cast(1); } - void accept(InstanceMemoryEstimateVisitor&) {} + void accept(CommonMemoryEstimateVisitor&) {} }; template struct CDataRef { T *c_value; - void accept(InstanceMemoryEstimateVisitor &visitor __attribute__((unused))) {} + void accept(CommonMemoryEstimateVisitor &visitor __attribute__((unused))) {} const char *get_class() const noexcept { return "FFI\\CDataRef"; } int get_hash() const noexcept { return -1965114283; } }; @@ -112,7 +112,7 @@ struct CDataArrayRef { T *data; int64_t len; - void accept(InstanceMemoryEstimateVisitor&) {} + void accept(CommonMemoryEstimateVisitor&) {} }; template diff --git a/runtime/job-workers/job-interface.cpp b/runtime/job-workers/job-interface.cpp index 1ff0efdf65..381e460d47 100644 --- a/runtime/job-workers/job-interface.cpp +++ b/runtime/job-workers/job-interface.cpp @@ -97,6 +97,6 @@ class_instance create_error_on_other_memory(int32_ return error; } -void C$KphpJobWorkerResponseError::accept(InstanceMemoryEstimateVisitor &visitor) noexcept { +void C$KphpJobWorkerResponseError::accept(CommonMemoryEstimateVisitor &visitor) noexcept { return generic_accept(visitor); } diff --git a/runtime/job-workers/job-interface.h b/runtime/job-workers/job-interface.h index ddfe0c3853..8a429591dc 100644 --- a/runtime/job-workers/job-interface.h +++ b/runtime/job-workers/job-interface.h @@ -20,7 +20,7 @@ class InstanceDeepDestroyVisitor; class ToArrayVisitor; -class InstanceMemoryEstimateVisitor; +class CommonMemoryEstimateVisitor; namespace job_workers { @@ -45,7 +45,7 @@ struct SendingInstanceBase : virtual abstract_refcountable_php_interface { virtual void accept(ToArrayVisitor &) noexcept {} - virtual void accept(InstanceMemoryEstimateVisitor &) noexcept {} + virtual void accept(CommonMemoryEstimateVisitor &) noexcept {} virtual size_t virtual_builtin_sizeof() const noexcept = 0; virtual SendingInstanceBase *virtual_builtin_clone() const noexcept = 0; @@ -108,7 +108,7 @@ struct C$KphpJobWorkerResponseError: public refcountable_polymorphic_php_classes return generic_accept(visitor); } - void accept(InstanceMemoryEstimateVisitor &visitor) noexcept; + void accept(CommonMemoryEstimateVisitor &visitor) noexcept; size_t virtual_builtin_sizeof() const noexcept { return sizeof(*this); diff --git a/runtime/memcache.h b/runtime/memcache.h index b11dddf0f6..fc0c030dfb 100644 --- a/runtime/memcache.h +++ b/runtime/memcache.h @@ -31,7 +31,7 @@ constexpr int64_t MEMCACHE_COMPRESSED = 2; struct C$Memcache : public abstract_refcountable_php_interface { public: - virtual void accept(InstanceMemoryEstimateVisitor &) = 0; + virtual void accept(CommonMemoryEstimateVisitor &) = 0; virtual const char *get_class() const = 0; virtual int32_t get_hash() const = 0; }; @@ -51,8 +51,8 @@ class C$McMemcache final : public refcountable_polymorphic_php_classes{}); } const char *get_class() const final { @@ -63,10 +63,6 @@ class C$McMemcache final : public refcountable_polymorphic_php_classes(vk::std_hash(vk::string_view(C$McMemcache::get_class()))); } - friend inline int32_t f$estimate_memory_usage(const C$McMemcache::host &) { - return 0; - } - virtual C$McMemcache* virtual_builtin_clone() const noexcept { return new C$McMemcache{*this}; } diff --git a/runtime/memory_usage.cpp b/runtime/memory_usage.cpp deleted file mode 100644 index 6a01c84ca9..0000000000 --- a/runtime/memory_usage.cpp +++ /dev/null @@ -1,21 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#include "runtime/memory_usage.h" - -int64_t f$estimate_memory_usage(const string &value) { - if (value.is_reference_counter(ExtraRefCnt::for_global_const) || value.is_reference_counter(ExtraRefCnt::for_instance_cache)) { - return 0; - } - return static_cast(value.estimate_memory_usage()); -} - -int64_t f$estimate_memory_usage(const mixed &value) { - if(value.is_string()) { - return f$estimate_memory_usage(value.as_string()); - } else if (value.is_array()) { - return f$estimate_memory_usage(value.as_array()); - } - return 0; -} diff --git a/runtime/memory_usage.h b/runtime/memory_usage.h index 111a1c09d6..e0307306d2 100644 --- a/runtime/memory_usage.h +++ b/runtime/memory_usage.h @@ -4,7 +4,16 @@ #pragma once -#include +#include +#include +#include +#include +#include +#include + +#include "common/mixin/not_copyable.h" +#include "common/type_traits/list_of_types.h" +#include "runtime/declarations.h" #include "runtime/kphp_core.h" #include "runtime/shape.h" @@ -14,152 +23,160 @@ struct CDataPtr; template struct CDataRef; -template -int64_t f$estimate_memory_usage(const CDataRef &ref __attribute__ ((unused))) { return sizeof(void*); } - -template -int64_t f$estimate_memory_usage(const CDataPtr &ptr __attribute__ ((unused))) { return 0; } - -int64_t f$estimate_memory_usage(const string &value); - -int64_t f$estimate_memory_usage(const mixed &value); - -template>> -int64_t f$estimate_memory_usage(const T &); - -template -int64_t f$estimate_memory_usage(const array &value); - -template -int64_t f$estimate_memory_usage(const Optional &value); +class CommonMemoryEstimateVisitor : vk::not_copyable { +public: + template + void operator()(const char *name [[maybe_unused]], T &&value) noexcept { + process(std::forward(value)); + auto &[size, clazz] = last_instance_element.top(); + if (clazz < static_cast(&value)) { + clazz = &value; + size = static_cast(sizeof(value)); + } + } -template -int64_t f$estimate_memory_usage(const std::tuple &value); + template + void process(const CDataRef &value [[maybe_unused]]) { + estimated_elements_memory_ += sizeof(void *); + } -template -int64_t f$estimate_memory_usage(const shape, T...> &value); + template + void process(const CDataPtr &value [[maybe_unused]]) {} -template -int64_t f$estimate_memory_usage(const class_instance &value); + void process(const string &value) { + if (value.is_reference_counter(ExtraRefCnt::for_global_const) || value.is_reference_counter(ExtraRefCnt::for_instance_cache)) { + return; + } + estimated_elements_memory_ += value.estimate_memory_usage(); + } -template{}>> -array f$get_global_vars_memory_stats(Int limit = 0); + void process(const mixed &value) { + if (value.is_string()) { + process(value.as_string()); + } + if (value.is_array()) { + process(value.as_array()); + } + } -template -int64_t f$estimate_memory_usage(const T &) { - return 0; -} + template>> + void process(const T &value [[maybe_unused]]) {} -template -int64_t f$estimate_memory_usage(const array &value) { - if (value.is_reference_counter(ExtraRefCnt::for_global_const) || value.is_reference_counter(ExtraRefCnt::for_instance_cache)) { - return 0; + template + void process(const array &value) { + if (value.is_reference_counter(ExtraRefCnt::for_global_const) || value.is_reference_counter(ExtraRefCnt::for_instance_cache)) { + return; + } + estimated_elements_memory_ += static_cast(value.estimate_memory_usage()); + for (auto it = value.cbegin(); it != value.cend(); ++it) { + process(it.get_key()); + process(it.get_value()); + } } - int64_t result = static_cast(value.estimate_memory_usage()); - for (auto it = value.cbegin(); it != value.cend(); ++it) { - result += f$estimate_memory_usage(it.get_key()) + f$estimate_memory_usage(it.get_value()); + + template + void process(const Optional &value) { + if (value.has_value()) { + process(value.val()); + } } - return result; -} -template -int64_t f$estimate_memory_usage(const Optional &value) { - return value.has_value() ? f$estimate_memory_usage(value.val()) : 0; -} + template + void process(const std::tuple &value) { + for_each(value, [this](const auto &v) { this->process(v); }); + } -namespace impl_ { + template + void process(const shape, T...> &value) { + (process(value.template get()), ...); + } -template -std::enable_if_t estimate_tuple_memory_usage(const std::tuple &) { - return 0; -} + template + void process(const class_instance &value) { + last_instance_element.push({}); + estimated_elements_memory_ += estimate_class_instance_memory_usage(value, std::is_empty{}); + last_instance_element.pop(); + } -template -std::enable_if_t estimate_tuple_memory_usage(const std::tuple &value) { - return estimate_tuple_memory_usage(value) + f$estimate_memory_usage(std::get(value)); -} + int64_t get_estimated_memory() const { + return estimated_elements_memory_; + } -} // namespace impl_ +private: + template + std::enable_if_t for_each(const std::tuple &value [[maybe_unused]], const Fn &func [[maybe_unused]]) {} -template -int64_t f$estimate_memory_usage(const std::tuple &value) { - return impl_::estimate_tuple_memory_usage(value); -} + template + std::enable_if_t for_each(const std::tuple &value, const Fn &func) { + func(std::get(value)); + for_each(value, func); + } -template -int64_t f$estimate_memory_usage(const shape, T...> &value) { - int64_t memory[] = {f$estimate_memory_usage(value.template get())...}; - return std::accumulate(std::begin(memory), std::end(memory), 0); -} + template + int64_t estimate_class_instance_memory_usage(const class_instance &value [[maybe_unused]], std::true_type) const { + return 0; + } -class InstanceMemoryEstimateVisitor { -public: template - void operator()(const char *, const T &value) { - estimated_elements_memory_ += f$estimate_memory_usage(value); - if (class_last_element_ < static_cast(&value)) { - class_last_element_ = &value; - last_element_size_ = static_cast(sizeof(value)); + int64_t estimate_class_instance_memory_usage(const class_instance &value, std::false_type) { + if (value.is_null() || value.is_reference_counter(ExtraRefCnt::for_instance_cache)) { + return 0; } - } - template - int64_t get_estimated_memory(const ClassType *obj) const { - return get_estimated_memory_impl(obj, std::is_polymorphic{}); + if (!processed_instances.insert(value.get()->get_instance_data_raw_ptr()).second) { + return 0; + } + value.get()->accept(*this); + return this->get_instance_estimated_memory_impl(value.get(), std::is_polymorphic{}); } -private: template - int64_t get_estimated_memory_impl(const ClassType *, std::false_type) const { - return static_cast(sizeof(ClassType)) + estimated_elements_memory_; + int64_t get_instance_estimated_memory_impl(const ClassType *obj [[maybe_unused]], std::false_type) const { + return sizeof(ClassType); } template - int64_t get_estimated_memory_impl(const ClassType *obj, std::true_type) const { + int64_t get_instance_estimated_memory_impl(const ClassType *obj, std::true_type) const { // TODO: virtual sizeof? // empty polymorphic class - if (!class_last_element_) { + const auto [size, clazz] = last_instance_element.top(); + if (clazz == nullptr) { // vtable + counter - return estimated_elements_memory_ + 16; + return 16; } - const uint8_t *class_begin = static_cast(dynamic_cast(obj)); - int64_t class_size = static_cast(class_last_element_) - class_begin; + const auto *class_begin = static_cast(dynamic_cast(obj)); + int64_t class_size = static_cast(clazz) - class_begin; php_assert(class_size > 0); - class_size += last_element_size_; + class_size += size; // all polymorphic classes are aligned by 8 (vtable ptr) class_size = (class_size + 7) & -8; - return class_size + estimated_elements_memory_; + return class_size; } + struct instance_element { + int64_t size = 0; + const void *clazz = nullptr; + }; + int64_t estimated_elements_memory_{0}; - const void *class_last_element_{nullptr}; - int64_t last_element_size_{0}; -}; -template -int64_t estimate_class_instance_memory_usage(const class_instance &, std::true_type) { - return 0; -} + dl::CriticalSectionGuard guard; + std::unordered_set processed_instances; + std::stack last_instance_element; +}; template -int64_t estimate_class_instance_memory_usage(const class_instance &value, std::false_type) { - if (value.is_null() || value.is_reference_counter(ExtraRefCnt::for_instance_cache)) { - return 0; - } - - InstanceMemoryEstimateVisitor visitor; - value.get()->accept(visitor); - return visitor.get_estimated_memory(value.get()); +int64_t f$estimate_memory_usage(const T &value) { + CommonMemoryEstimateVisitor visitor; + visitor.process(value); + return visitor.get_estimated_memory(); } -template -int64_t f$estimate_memory_usage(const class_instance &value) { - return estimate_class_instance_memory_usage(value, std::is_empty{}); -} +template{}>> +array f$get_global_vars_memory_stats(Int limit = 0); template -array f$get_global_vars_memory_stats(Int lower_bound) { +array f$get_global_vars_memory_stats(Int limit) { array get_global_vars_memory_stats_impl(int64_t) noexcept; - return get_global_vars_memory_stats_impl(lower_bound); + return get_global_vars_memory_stats_impl(limit); } diff --git a/runtime/mysql.h b/runtime/mysql.h index 694ed911ce..9c001fd53b 100644 --- a/runtime/mysql.h +++ b/runtime/mysql.h @@ -26,7 +26,7 @@ class C$mysqli : public refcountable_php_classes, private DummyVisitor using DummyVisitorMethods::accept; - void accept(InstanceMemoryEstimateVisitor &visitor) { + void accept(CommonMemoryEstimateVisitor &visitor) { visitor("", error); visitor("", query_results); visitor("", last_query_id); diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index 7bfc8d802f..f7e0ab8222 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -87,7 +87,6 @@ prepend(KPHP_RUNTIME_SOURCES ${BASE_DIR}/runtime/ math_functions.cpp mbstring.cpp memcache.cpp - memory_usage.cpp migration_php8.cpp misc.cpp mixed.cpp diff --git a/runtime/tl/rpc_function.h b/runtime/tl/rpc_function.h index abc4a5c1c0..65abbd832d 100644 --- a/runtime/tl/rpc_function.h +++ b/runtime/tl/rpc_function.h @@ -11,7 +11,7 @@ struct tl_func_base; class ToArrayVisitor; -class InstanceMemoryEstimateVisitor; +class CommonMemoryEstimateVisitor; class InstanceReferencesCountingVisitor; class InstanceDeepCopyVisitor; class InstanceDeepDestroyVisitor; @@ -26,7 +26,7 @@ struct C$VK$TL$RpcFunction : abstract_refcountable_php_interface { virtual int32_t get_hash() const { return static_cast(vk::std_hash(vk::string_view(C$VK$TL$RpcFunction::get_class()))); } virtual void accept(ToArrayVisitor &) noexcept {} - virtual void accept(InstanceMemoryEstimateVisitor &) noexcept {} + virtual void accept(CommonMemoryEstimateVisitor &) noexcept {} virtual void accept(InstanceReferencesCountingVisitor &) noexcept {} virtual void accept(InstanceDeepCopyVisitor &) noexcept {} virtual void accept(InstanceDeepDestroyVisitor &) noexcept {} @@ -45,7 +45,7 @@ struct C$VK$TL$RpcFunctionReturnResult : abstract_refcountable_php_interface { virtual int32_t get_hash() const { return static_cast(vk::std_hash(vk::string_view(C$VK$TL$RpcFunctionReturnResult::get_class()))); } virtual void accept(ToArrayVisitor &) noexcept {} - virtual void accept(InstanceMemoryEstimateVisitor &) noexcept {} + virtual void accept(CommonMemoryEstimateVisitor &) noexcept {} virtual void accept(InstanceReferencesCountingVisitor &) noexcept {} virtual void accept(InstanceDeepCopyVisitor &) noexcept {} virtual void accept(InstanceDeepDestroyVisitor &) noexcept {} @@ -62,7 +62,7 @@ struct C$VK$TL$RpcResponse : abstract_refcountable_php_interface { using X = class_instance; virtual void accept(ToArrayVisitor &) noexcept {} - virtual void accept(InstanceMemoryEstimateVisitor &) noexcept {} + virtual void accept(CommonMemoryEstimateVisitor &) noexcept {} virtual void accept(InstanceReferencesCountingVisitor &) noexcept {} virtual void accept(InstanceDeepCopyVisitor &) noexcept {} virtual void accept(InstanceDeepDestroyVisitor &) noexcept {} diff --git a/tests/phpt/memory_usage/11_shapes.php b/tests/phpt/memory_usage/11_shapes.php new file mode 100644 index 0000000000..ad78894e10 --- /dev/null +++ b/tests/phpt/memory_usage/11_shapes.php @@ -0,0 +1,36 @@ +@ok + 5, 'y' => "hello", 'z' => [7, 42]]); + var_dump(estimate_memory_usage($sh)); +} + +function test_shape_in_class() { + class A { + /** @var shape(x:int, y:string, z:int[]) */ + public $sh; + + function __construct() { + $this->sh = shape(['y' => 'y', 'x' => 2, 'z' => [1, 2, 3]]); + } + } + +#ifndef KPHP + var_dump(32); + return; +#endif + $a = new A(); + var_dump(estimate_memory_usage($a)); +} + +test_simple_shape(); +test_shape_in_class(); diff --git a/tests/phpt/memory_usage/2_classes.php b/tests/phpt/memory_usage/2_classes.php index fade728918..81561c4459 100644 --- a/tests/phpt/memory_usage/2_classes.php +++ b/tests/phpt/memory_usage/2_classes.php @@ -67,6 +67,48 @@ public function __construct($size) { var_dump(estimate_memory_usage($instance)); } +function test_class_with_cyclic_dependence() { + class ClassWithCyclicDependence { + public int $value = 0; + /** @var ClassWithCyclicDependence */ + public $child; + + public function __construct(int $value) { + $this->value = $value; + $this->child = $this; + } + } + +#ifndef KPHP + var_dump(24); + return; +#endif + $instance = new ClassWithCyclicDependence(42); + var_dump(estimate_memory_usage($instance)); +} + +function test_class_with_cyclic_dependence_in_dynamic_array() { + class ClassWithCyclicDependenceInArray { + public int $value = 0; // 8 + /** @var ClassWithCyclicDependenceInArray[] */ + public $childs = []; // 8 + + public function __construct(int $value) { + $this->value = $value; + $this->childs[] = $this; + } + } + +#ifndef KPHP + var_dump(72); + return; +#endif + $instance = new ClassWithCyclicDependenceInArray(42); + var_dump(estimate_memory_usage($instance)); +} + test_empty_class(); test_class_with_simple_fields(); test_class_with_dynamic_array(); +test_class_with_cyclic_dependence(); +test_class_with_cyclic_dependence_in_dynamic_array();