[PATCH v2] linux-gen: pool: optimize freeing events to a single pool

platform/linux-generic/include/odp_buffer_internal.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2013-2018 Linaro Limited
- * Copyright (c) 2019-2024 Nokia
+ * Copyright (c) 2019-2025 Nokia
  */
 
 /**
@@ -22,11 +22,13 @@ extern "C" {
 #include <odp/api/buffer.h>
 #include <odp/api/debug.h>
 #include <odp/api/align.h>
-#include <odp_config_internal.h>
 #include <odp/api/byteorder.h>
 #include <odp/api/thread.h>
 #include <odp/api/event.h>
+
+#include <odp_config_internal.h>
 #include <odp_event_internal.h>
+#include <odp_pool_internal.h>
 
 #include <stddef.h>
 #include <stdint.h>
@@ -65,6 +67,11 @@ static inline uint32_t _odp_buffer_index(odp_buffer_t buf)
 	return _odp_buf_hdr(buf)->event_hdr.index.event;
 }
 
+static inline void _odp_buffer_free_sp(const odp_buffer_t buf[], int num)
+{
+	_odp_event_free_sp((_odp_event_hdr_t **)(uintptr_t)buf, num);
+}
+
 #ifdef __cplusplus
 }
 #endif

platform/linux-generic/include/odp_pool_internal.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2013-2018 Linaro Limited
- * Copyright (c) 2019-2022 Nokia
+ * Copyright (c) 2019-2025 Nokia
  */
 
 /**
@@ -21,7 +21,6 @@ extern "C" {
 #include <odp/api/ticketlock.h>
 #include <odp/api/align.h>
 
-#include <odp_buffer_internal.h>
 #include <odp_event_internal.h>
 #include <odp_config_internal.h>
 #include <odp_ring_ptr_internal.h>
@@ -202,11 +201,12 @@ static inline _odp_event_hdr_t *_odp_event_hdr_from_index_u32(uint32_t u32)
 odp_event_t _odp_event_alloc(pool_t *pool);
 int _odp_event_alloc_multi(pool_t *pool, _odp_event_hdr_t *event_hdr[], int num);
 void _odp_event_free_multi(_odp_event_hdr_t *event_hdr[], int num_free);
+void _odp_event_free_sp(_odp_event_hdr_t *event_hdr[], int num);
 int _odp_event_is_valid(odp_event_t event);
 
 static inline void _odp_event_free(odp_event_t event)
 {
-	_odp_event_free_multi((_odp_event_hdr_t **)&event, 1);
+	_odp_event_free_sp((_odp_event_hdr_t **)&event, 1);
 }
 
 odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,

platform/linux-generic/include/odp_timer_internal.h

@@ -63,4 +63,9 @@ static inline uint64_t timer_run(int dec)
 	return UINT64_MAX;
 }
 
+static inline void _odp_timeout_free_sp(odp_timeout_t tmo[], int num)
+{
+	_odp_event_free_sp((_odp_event_hdr_t **)(uintptr_t)tmo, num);
+}
+
 #endif

platform/linux-generic/odp_event.c

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2015-2018 Linaro Limited
- * Copyright (c) 2020-2024 Nokia
+ * Copyright (c) 2020-2025 Nokia
  */
 
 #include <odp/autoheader_external.h>
@@ -19,6 +19,7 @@
 #include <odp_debug_internal.h>
 #include <odp_packet_internal.h>
 #include <odp_event_internal.h>
+#include <odp_timer_internal.h>
 #include <odp_event_validation_internal.h>
 #include <odp_event_vector_internal.h>
 
@@ -127,6 +128,41 @@ static inline void event_free_multi(const odp_event_t event[], int num, odp_even
 	}
 }
 
+static inline void event_free_sp(const odp_event_t event[], int num, odp_event_type_t type,
+				 _odp_ev_id_t id)
+{
+	switch (type) {
+	case ODP_EVENT_BUFFER:
+		_odp_buffer_validate_multi((odp_buffer_t *)(uintptr_t)event, num, id);
+		_odp_buffer_free_sp((odp_buffer_t *)(uintptr_t)event, num);
+		break;
+	case ODP_EVENT_PACKET:
+		_odp_packet_validate_multi((odp_packet_t *)(uintptr_t)event, num, id);
+		odp_packet_free_sp((odp_packet_t *)(uintptr_t)event, num);
+		break;
+	case ODP_EVENT_PACKET_VECTOR:
+		packet_vector_free_full_multi((odp_packet_vector_t *)(uintptr_t)event, num);
+		break;
+	case ODP_EVENT_TIMEOUT:
+		_odp_timeout_free_sp((odp_timeout_t *)(uintptr_t)event, num);
+		break;
+	case ODP_EVENT_IPSEC_STATUS:
+		ipsec_status_free_multi((ipsec_status_t *)(uintptr_t)event, num);
+		break;
+	case ODP_EVENT_PACKET_TX_COMPL:
+		packet_tx_compl_free_multi((odp_packet_tx_compl_t *)(uintptr_t)event, num);
+		break;
+	case ODP_EVENT_DMA_COMPL:
+		dma_compl_free_multi((odp_dma_compl_t *)(uintptr_t)event, num);
+		break;
+	case ODP_EVENT_ML_COMPL:
+		ml_compl_free_multi((odp_ml_compl_t *)(uintptr_t)event, num);
+		break;
+	default:
+		_ODP_ABORT("Invalid event type: %d\n", type);
+	}
+}
+
 void odp_event_free_multi(const odp_event_t event[], int num)
 {
 	const odp_event_t *burst_start;
@@ -170,7 +206,7 @@ void odp_event_free_sp(const odp_event_t event[], int num)
 			_ODP_ASSERT(_odp_event_pool(event[i]) == pool);
 	}
 
-	event_free_multi(event, num, odp_event_type(event[0]), _ODP_EV_EVENT_FREE_SP);
+	event_free_sp(event, num, odp_event_type(event[0]), _ODP_EV_EVENT_FREE_SP);
 }
 
 uint64_t odp_event_to_u64(odp_event_t hdl)

platform/linux-generic/odp_packet.c

@@ -1,10 +1,11 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2013-2018 Linaro Limited
- * Copyright (c) 2019-2024 Nokia
+ * Copyright (c) 2019-2025 Nokia
  */
 
 #include <odp/autoheader_external.h>
 
+#include <odp/api/buffer.h>
 #include <odp/api/byteorder.h>
 #include <odp/api/hash.h>
 #include <odp/api/hints.h>
@@ -383,7 +384,7 @@ static inline odp_packet_hdr_t *alloc_segments(pool_t *pool, int num)
 
 	if (odp_unlikely(ret != num)) {
 		if (ret > 0)
-			_odp_event_free_multi((_odp_event_hdr_t **)pkt_hdr, ret);
+			_odp_event_free_sp((_odp_event_hdr_t **)pkt_hdr, ret);
 
 		return NULL;
 	}
@@ -465,13 +466,12 @@ static inline int is_multi_ref(uint32_t ref_cnt)
 	return (ref_cnt > 1);
 }
 
-static inline void packet_free_multi(odp_packet_hdr_t *hdr[], int num)
+static inline int skip_references(odp_packet_hdr_t *hdr[], int num)
 {
-	int i;
 	uint32_t ref_cnt;
 	int num_ref = 0;
 
-	for (i = 0; i < num; i++) {
+	for (int i = 0; i < num; i++) {
 		/* Zero when reference API has not been used */
 		ref_cnt = segment_ref(hdr[i]);
 
@@ -489,12 +489,25 @@ static inline void packet_free_multi(odp_packet_hdr_t *hdr[], int num)
 			hdr[i - num_ref] = hdr[i];
 	}
 
-	num -= num_ref;
+	return num_ref;
+}
+
+static inline void packet_free_multi(odp_packet_hdr_t *hdr[], int num)
+{
+	num -= skip_references(hdr, num);
 
 	if (odp_likely(num))
 		_odp_event_free_multi((_odp_event_hdr_t **)(uintptr_t)hdr, num);
 }
 
+static inline void packet_free_sp(odp_packet_hdr_t *hdr[], int num)
+{
+	num -= skip_references(hdr, num);
+
+	if (odp_likely(num))
+		_odp_event_free_sp((_odp_event_hdr_t **)(uintptr_t)hdr, num);
+}
+
 static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num)
 {
 	int i;
@@ -603,7 +616,7 @@ static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt,
 			_odp_event_hdr_t **p;
 
 			p = (_odp_event_hdr_t **)&pkt_hdr[num_buf - num_free];
-			_odp_event_free_multi(p, num_free);
+			_odp_event_free_sp(p, num_free);
 		}
 
 		if (num == 0)
@@ -695,23 +708,16 @@ void odp_packet_free(odp_packet_t pkt)
 	_ODP_ASSERT(segment_ref(pkt_hdr) > 0);
 
 	if (odp_likely(num_seg == 1))
-		packet_free_multi(&pkt_hdr, 1);
+		packet_free_sp(&pkt_hdr, 1);
 	else
 		free_all_segments(pkt_hdr, num_seg);
 }
 
-static inline void packet_free_multi_ev(const odp_packet_t pkt[], int num, _odp_ev_id_t id)
+static inline int free_segmented(const odp_packet_t pkt[], int num, odp_packet_hdr_t *pkt_hdrs[])
 {
-	if (odp_unlikely(!num))
-		return;
-
-	odp_packet_hdr_t *pkt_hdrs[num];
-	int i;
 	int num_freed = 0;
 
-	_odp_packet_validate_multi(pkt, num, id);
-
-	for (i = 0; i < num; i++) {
+	for (int i = 0; i < num; i++) {
 		odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]);
 		int num_seg = pkt_hdr->seg_count;
 
@@ -726,18 +732,42 @@ static inline void packet_free_multi_ev(const odp_packet_t pkt[], int num, _odp_
 		pkt_hdrs[i - num_freed] = pkt_hdr;
 	}
 
-	if (odp_likely(num - num_freed))
-		packet_free_multi(pkt_hdrs, num - num_freed);
+	return num_freed;
 }
 
 void odp_packet_free_multi(const odp_packet_t pkt[], int num)
 {
-	packet_free_multi_ev(pkt, num, _ODP_EV_PACKET_FREE_MULTI);
+	if (odp_unlikely(num < 1))
+		return;
+
+	odp_packet_hdr_t *pkt_hdrs[num];
+
+	_odp_packet_validate_multi(pkt, num, _ODP_EV_PACKET_FREE_MULTI);
+
+	num -= free_segmented(pkt, num, pkt_hdrs);
+	if (odp_likely(num))
+		packet_free_multi(pkt_hdrs, num);
 }
 
 void odp_packet_free_sp(const odp_packet_t pkt[], int num)
 {
-	packet_free_multi_ev(pkt, num, _ODP_EV_PACKET_FREE_SP);
+	if (odp_unlikely(num < 1))
+		return;
+
+	odp_packet_hdr_t *pkt_hdrs[num];
+
+	_odp_packet_validate_multi(pkt, num, _ODP_EV_PACKET_FREE_SP);
+
+	if (ODP_DEBUG) {
+		const odp_pool_t pool = odp_packet_pool(pkt[0]);
+
+		for (int i = 1; i < num; i++)
+			_ODP_ASSERT(odp_packet_pool(pkt[i]) == pool);
+	}
+
+	num -= free_segmented(pkt, num, pkt_hdrs);
+	if (odp_likely(num))
+		packet_free_sp(pkt_hdrs, num);
 }
 
 uint32_t odp_packet_reset_max_len(odp_packet_t pkt)

platform/linux-generic/odp_pool.c

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2013-2018 Linaro Limited
- * Copyright (c) 2019-2023 Nokia
+ * Copyright (c) 2019-2025 Nokia
  */
 
 #include <odp/api/align.h>
@@ -1424,6 +1424,13 @@ void _odp_event_free_multi(_odp_event_hdr_t *event_hdr[], int num_total)
 	}
 }
 
+void _odp_event_free_sp(_odp_event_hdr_t *event_hdr[], int num)
+{
+	_ODP_ASSERT(num > 0);
+
+	event_free_to_pool(_odp_pool_entry(event_hdr[0]->pool), event_hdr, num);
+}
+
 odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl)
 {
 	odp_buffer_t buf;
@@ -1474,7 +1481,7 @@ void odp_buffer_free(odp_buffer_t buf)
 {
 	_odp_buffer_validate(buf, _ODP_EV_BUFFER_FREE);
 
-	_odp_event_free_multi((_odp_event_hdr_t **)&buf, 1);
+	_odp_event_free(odp_buffer_to_event(buf));
 }
 
 void odp_buffer_free_multi(const odp_buffer_t buf[], int num)

platform/linux-generic/pktio/ipc.c

@@ -5,6 +5,7 @@
 
 #include <odp/api/deprecated.h>
 #include <odp/api/hints.h>
+#include <odp/api/pool.h>
 #include <odp/api/system_info.h>
 
 #include <odp_debug_internal.h>

test/performance/odp_bench_packet.c

@@ -1842,10 +1842,11 @@ int main(int argc, char *argv[])
 		exit(EXIT_FAILURE);
 	}
 
-	/* At least 2 x TEST_REPEAT_COUNT packets required */
-	pkt_num = (gbl_args->appl.burst_size > 2) ?
+	/* At least 2 x TEST_REPEAT_COUNT packets are required for the tests and depending on the
+	 * implementation extra packets may be required for packet references. */
+	pkt_num = (gbl_args->appl.burst_size > 4) ?
 			gbl_args->appl.burst_size * TEST_REPEAT_COUNT :
-			2 * TEST_REPEAT_COUNT;
+			4 * TEST_REPEAT_COUNT;
 
 	if (capa.pkt.max_num && capa.pkt.max_num < pkt_num) {
 		ODPH_ERR("Error: packet pool size not supported.\n");