Per-cpu refcounters

Signed-off-by: Adam Rutkowski <[email protected]> Signed-off-by: Jan Musial <[email protected]> Signed-off-by: Michal Mielewczyk <[email protected]> Signed-off-by: Rafal Stefanowski <[email protected]>
Open-CAS · Feb 6, 2025 · 53ee7c1 · 53ee7c1
1 parent c200c24
commit 53ee7c1
Show file tree

Hide file tree

Showing 21 changed files with 320 additions and 204 deletions.
diff --git a/src/utils/utils_refcnt.c → env/posix/ocf_env_refcnt.c b/src/utils/utils_refcnt.c → env/posix/ocf_env_refcnt.c
@@ -4,49 +4,54 @@
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
-#include "../utils/utils_refcnt.h"
+#include "ocf_env_refcnt.h"
 
-void ocf_refcnt_init(struct ocf_refcnt *rc)
+int env_refcnt_init(struct env_refcnt *rc, const char *name, size_t name_len)
 {
 	env_atomic_set(&rc->counter, 0);
 	env_atomic_set(&rc->freeze, 0);
 	env_atomic_set(&rc->callback, 0);
 	rc->cb = NULL;
+
+	return 0;
 }
 
-int ocf_refcnt_dec(struct ocf_refcnt *rc)
+void env_refcnt_deinit(struct env_refcnt *rc)
+{
+
+}
+
+void env_refcnt_dec(struct env_refcnt *rc)
 {
 	int val = env_atomic_dec_return(&rc->counter);
 	ENV_BUG_ON(val < 0);
 
 	if (!val && env_atomic_cmpxchg(&rc->callback, 1, 0))
 		rc->cb(rc->priv);
-
-	return val;
 }
 
-int ocf_refcnt_inc(struct ocf_refcnt  *rc)
+bool env_refcnt_inc(struct env_refcnt  *rc)
 {
 	int val;
 
 	if (!env_atomic_read(&rc->freeze)) {
 		val = env_atomic_inc_return(&rc->counter);
 		if (!env_atomic_read(&rc->freeze))
-			return  val;
+			return !!val;
 		else
-			ocf_refcnt_dec(rc);
+			env_refcnt_dec(rc);
 	}
 
 	return 0;
 }
 
 
-void ocf_refcnt_freeze(struct ocf_refcnt *rc)
+void env_refcnt_freeze(struct env_refcnt *rc)
 {
 	env_atomic_inc(&rc->freeze);
 }
 
-void ocf_refcnt_register_zero_cb(struct ocf_refcnt *rc, ocf_refcnt_cb_t cb,
+void env_refcnt_register_zero_cb(struct env_refcnt *rc, env_refcnt_cb_t cb,
 		void *priv)
 {
 	ENV_BUG_ON(!env_atomic_read(&rc->freeze));
@@ -56,21 +61,21 @@ void ocf_refcnt_register_zero_cb(struct ocf_refcnt *rc, ocf_refcnt_cb_t cb,
 	rc->cb = cb;
 	rc->priv = priv;
 	env_atomic_set(&rc->callback, 1);
-	ocf_refcnt_dec(rc);
+	env_refcnt_dec(rc);
 }
 
-void ocf_refcnt_unfreeze(struct ocf_refcnt *rc)
+void env_refcnt_unfreeze(struct env_refcnt *rc)
 {
 	int val = env_atomic_dec_return(&rc->freeze);
 	ENV_BUG_ON(val < 0);
 }
 
-bool ocf_refcnt_frozen(struct ocf_refcnt *rc)
+bool env_refcnt_frozen(struct env_refcnt *rc)
 {
 	return !!env_atomic_read(&rc->freeze);
 }
 
-bool ocf_refcnt_zeroed(struct ocf_refcnt *rc)
+bool env_refcnt_zeroed(struct env_refcnt *rc)
 {
 	return (env_atomic_read(&rc->counter) == 0);
 }
diff --git a/env/posix/ocf_env_refcnt.h b/env/posix/ocf_env_refcnt.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright(c) 2019-2021 Intel Corporation
+ * Copyright(c) 2024-2025 Huawei Technologies
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef __OCF_ENV_REFCNT_H__
+#define __OCF_ENV_REFCNT_H__
+
+#include "ocf_env.h"
+
+typedef void (*env_refcnt_cb_t)(void *priv);
+
+struct env_refcnt
+{
+	env_atomic counter __attribute__((aligned(64)));
+	env_atomic freeze;
+	env_atomic callback;
+	env_refcnt_cb_t cb;
+	void *priv;
+};
+
+/* Initialize reference counter */
+int env_refcnt_init(struct env_refcnt *rc, const char *name, size_t name_len);
+
+void env_refcnt_deinit(struct env_refcnt *rc);
+
+/* Try to increment counter. Returns true if successfull, false
+ * if counter is frozen */
+bool env_refcnt_inc(struct env_refcnt  *rc);
+
+/* Decrement reference counter */
+void env_refcnt_dec(struct env_refcnt *rc);
+
+/* Disallow incrementing of underlying counter - attempts to increment counter
+ * will be failing until env_refcnt_unfreeze is calleed.
+ * It's ok to call freeze multiple times, in which case counter is frozen
+ * until all freeze calls are offset by a corresponding unfreeze.*/
+void env_refcnt_freeze(struct env_refcnt *rc);
+
+/* Cancel the effect of single env_refcnt_freeze call */
+void env_refcnt_unfreeze(struct env_refcnt *rc);
+
+bool env_refcnt_frozen(struct env_refcnt *rc);
+
+bool env_refcnt_zeroed(struct env_refcnt *rc);
+
+/* Register callback to be called when reference counter drops to 0.
+ * Must be called after counter is frozen.
+ * Cannot be called until previously regsitered callback had fired. */
+void env_refcnt_register_zero_cb(struct env_refcnt *rc, env_refcnt_cb_t cb,
+		void *priv);
+
+#endif // __OCF_ENV_REFCNT_H__
diff --git a/src/cleaning/acp_structs.h b/src/cleaning/acp_structs.h
@@ -1,11 +1,12 @@
 /*
  * Copyright(c) 2012-2021 Intel Corporation
+ * Copyright(c) 2023-2025 Huawei Technologies Co., Ltd.
  * SPDX-License-Identifier: BSD-3-Clause
  */
 #ifndef __CLEANING_AGGRESSIVE_STRUCTS_H__
 #define __CLEANING_AGGRESSIVE_STRUCTS_H__
 
-#include "../utils/utils_cleaner.h"
+#include "ocf_env_headers.h"
 
 /* TODO: remove acp metadata */
 struct acp_cleaning_policy_meta {

diff --git a/src/cleaning/cleaning.h b/src/cleaning/cleaning.h
@@ -1,5 +1,6 @@
 /*
  * Copyright(c) 2012-2022 Intel Corporation
+ * Copyright(c) 2023-2025 Huawei Technologies Co., Ltd.
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -9,8 +10,8 @@
 #include "alru_structs.h"
 #include "nop_structs.h"
 #include "acp_structs.h"
+#include "ocf_env_refcnt.h"
 #include "ocf/ocf_cleaner.h"
-#include "../utils/utils_refcnt.h"
 
 #define CLEANING_POLICY_CONFIG_BYTES 256
 #define CLEANING_POLICY_TYPE_MAX 4
@@ -40,7 +41,7 @@ struct cleaning_policy_meta {
 };
 
 struct ocf_cleaner {
-	struct ocf_refcnt refcnt __attribute__((aligned(64)));
+	struct env_refcnt refcnt;
 	ocf_cleaning_t policy;
 	void *cleaning_policy_context;
 	ocf_queue_t io_queue;

diff --git a/src/cleaning/cleaning_ops.h b/src/cleaning/cleaning_ops.h
@@ -1,5 +1,6 @@
 /*
  * Copyright(c) 2012-2022 Intel Corporation
+ * Copyright(c) 2023-2025 Huawei Technologies Co., Ltd.
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -9,7 +10,7 @@
 #include "../metadata/metadata_superblock.h"
 #include "../metadata/metadata_structs.h"
 #include "../ocf_cache_priv.h"
-#include "../utils/utils_refcnt.h"
+#include "ocf_env_refcnt.h"
 
 struct cleaning_policy_ops {
 	void (*setup)(ocf_cache_t cache);
@@ -125,7 +126,7 @@ static inline int ocf_cleaning_add_core(ocf_cache_t cache,
 	ocf_cleaning_t policy;
 	int result = 0;
 
-	if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt)))
+	if (unlikely(!env_refcnt_inc(&cache->cleaner.refcnt)))
 		return -OCF_ERR_NO_LOCK;
 
 	policy = cache->cleaner.policy;
@@ -138,7 +139,7 @@ static inline int ocf_cleaning_add_core(ocf_cache_t cache,
 	result = cleaning_policy_ops[policy].add_core(cache, core_id);
 
 unlock:
-	ocf_refcnt_dec(&cache->cleaner.refcnt);
+	env_refcnt_dec(&cache->cleaner.refcnt);
 
 	return result;
 }
@@ -148,7 +149,7 @@ static inline void ocf_cleaning_remove_core(ocf_cache_t cache,
 {
 	ocf_cleaning_t policy;
 
-	if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt)))
+	if (unlikely(!env_refcnt_inc(&cache->cleaner.refcnt)))
 		return;
 
 	policy = cache->cleaner.policy;
@@ -161,15 +162,15 @@ static inline void ocf_cleaning_remove_core(ocf_cache_t cache,
 	cleaning_policy_ops[policy].remove_core(cache, core_id);
 
 unlock:
-	ocf_refcnt_dec(&cache->cleaner.refcnt);
+	env_refcnt_dec(&cache->cleaner.refcnt);
 }
 
 static inline void ocf_cleaning_init_cache_block(ocf_cache_t cache,
 		ocf_cache_line_t cache_line)
 {
 	ocf_cleaning_t policy;
 
-	if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt)))
+	if (unlikely(!env_refcnt_inc(&cache->cleaner.refcnt)))
 		return;
 
 	policy = cache->cleaner.policy;
@@ -181,15 +182,15 @@ static inline void ocf_cleaning_init_cache_block(ocf_cache_t cache,
 	cleaning_policy_ops[policy].init_cache_block(cache, cache_line);
 
 unlock:
-	ocf_refcnt_dec(&cache->cleaner.refcnt);
+	env_refcnt_dec(&cache->cleaner.refcnt);
 }
 
 static inline void ocf_cleaning_purge_cache_block(ocf_cache_t cache,
 		ocf_cache_line_t cache_line)
 {
 	ocf_cleaning_t policy;
 
-	if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt)))
+	if (unlikely(!env_refcnt_inc(&cache->cleaner.refcnt)))
 		return;
 
 	policy = cache->cleaner.policy;
@@ -201,15 +202,15 @@ static inline void ocf_cleaning_purge_cache_block(ocf_cache_t cache,
 	cleaning_policy_ops[policy].purge_cache_block(cache, cache_line);
 
 unlock:
-	ocf_refcnt_dec(&cache->cleaner.refcnt);
+	env_refcnt_dec(&cache->cleaner.refcnt);
 }
 
 static inline void ocf_cleaning_purge_range(ocf_cache_t cache,
 		ocf_core_id_t core_id, uint64_t start_byte, uint64_t end_byte)
 {
 	ocf_cleaning_t policy;
 
-	if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt)))
+	if (unlikely(!env_refcnt_inc(&cache->cleaner.refcnt)))
 		return;
 
 	policy = cache->cleaner.policy;
@@ -222,15 +223,15 @@ static inline void ocf_cleaning_purge_range(ocf_cache_t cache,
 			end_byte);
 
 unlock:
-	ocf_refcnt_dec(&cache->cleaner.refcnt);
+	env_refcnt_dec(&cache->cleaner.refcnt);
 }
 
 static inline void ocf_cleaning_set_hot_cache_line(ocf_cache_t cache,
 		ocf_cache_line_t cache_line)
 {
 	ocf_cleaning_t policy;
 
-	if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt)))
+	if (unlikely(!env_refcnt_inc(&cache->cleaner.refcnt)))
 		return;
 
 	policy = cache->cleaner.policy;
@@ -242,7 +243,7 @@ static inline void ocf_cleaning_set_hot_cache_line(ocf_cache_t cache,
 	cleaning_policy_ops[policy].set_hot_cache_line(cache, cache_line);
 
 unlock:
-	ocf_refcnt_dec(&cache->cleaner.refcnt);
+	env_refcnt_dec(&cache->cleaner.refcnt);
 }
 
 static inline int ocf_cleaning_set_param(ocf_cache_t cache,
@@ -274,7 +275,7 @@ static inline void ocf_cleaning_perform_cleaning(ocf_cache_t cache,
 {
 	ocf_cleaning_t policy;
 
-	if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt))) {
+	if (unlikely(!env_refcnt_inc(&cache->cleaner.refcnt))) {
 		cmpl(&cache->cleaner, 1000);
 		return;
 	}
@@ -283,14 +284,14 @@ static inline void ocf_cleaning_perform_cleaning(ocf_cache_t cache,
 	ENV_BUG_ON(policy >= ocf_cleaning_max);
 
 	if (unlikely(!cleaning_policy_ops[policy].perform_cleaning)) {
-		ocf_refcnt_dec(&cache->cleaner.refcnt);
+		env_refcnt_dec(&cache->cleaner.refcnt);
 		cmpl(&cache->cleaner, 1000);
 		return;
 	}
 
 	cleaning_policy_ops[policy].perform_cleaning(cache, cmpl);
 
-	ocf_refcnt_dec(&cache->cleaner.refcnt);
+	env_refcnt_dec(&cache->cleaner.refcnt);
 }
 
 static inline const char *ocf_cleaning_get_name(ocf_cleaning_t policy)

diff --git a/src/engine/cache_engine.c b/src/engine/cache_engine.c
@@ -1,6 +1,6 @@
 /*
  * Copyright(c) 2012-2022 Intel Corporation
- * Copyright(c) 2024 Huawei Technologies
+ * Copyright(c) 2024-2025 Huawei Technologies
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -22,7 +22,6 @@
 #include "engine_flush.h"
 #include "engine_discard.h"
 #include "../utils/utils_user_part.h"
-#include "../utils/utils_refcnt.h"
 #include "../ocf_request.h"
 #include "../metadata/metadata.h"
 #include "../ocf_space.h"