From 2150add64592a7815f28d3862771bb5c5c184771 Mon Sep 17 00:00:00 2001 From: ferrol aderholdt Date: Wed, 16 Oct 2024 10:29:48 -0700 Subject: [PATCH 1/7] API: add explicit memory mapping --- src/ucc/api/ucc.h | 80 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/src/ucc/api/ucc.h b/src/ucc/api/ucc.h index 38a6d1fd3f..2ed645ef95 100644 --- a/src/ucc/api/ucc.h +++ b/src/ucc/api/ucc.h @@ -2220,5 +2220,85 @@ ucc_status_t ucc_ee_wait(ucc_ee_h ee, ucc_ev_t *ev); */ ucc_status_t ucc_collective_triggered_post(ucc_ee_h ee, ucc_ev_t *ee_event); +typedef void * ucc_mem_map_mem_h; + +/** + * @ingroup UCC_DATATYPE + */ +enum ucc_mem_map_flags { + UCC_MEM_MAP_EXPORT = 0, /*!< Indicate @ref ucc_mem_map() should export + memory handles from TLs used by context */ + UCC_MEM_MAP_IMPORT = 1 /*!< Indicate @ref ucc_mem_map() should import + memory handles from user memory handle */ +}; + +/** + * @ingroup UCC_CONTEXT + * @brief Routine registers or maps memory for use in future collectives. + * + * This routine maps memory a user-specified memory segment with a @ref + * ucc_context_t. The segment is considered "mapped" with the context until + * either the user calls @ref ucc_mem_unmap or @ref ucc_context_destroy(). A + * handle to the mapped memory is provided in memh. If the flag + * UCC_MEM_MAP_EXPORT is used, the memory will be mapped and memory + * handles from TLs will be generated and stored in the memh. If the flag + * UCC_MEM_MAP_IMPORT is used, the user must provide a valid memh, otherwise + * behavior is undefined. + * + * @params [in] context Context mapped memory is associated with + * @params [in] flags flags dictating the behavior of the routine + * @params [in] params parameters indicating the address and length of + * memory to map + * @params [inout] *memh Handle for the registered memory + * + * @return Error code as defined by @ref ucc_status_t. + */ + +ucc_status_t ucc_mem_map(ucc_context_t context, + ucc_mem_map_flags flags, + ucc_mem_map_params params, + ucc_mem_map_mem_h *memh); + +/** + * @ingroup UCC_CONTEXT + * @brief Routine exchanges mapped memory with peers to enable future + * collectives. + * + * This routine performs an nonblocking exchange of mapped memory with peers + * within the same context on which it is mapped. This is a collective routine + * and must be called by all members of the ucc_context on which the memory + * is mapped. + * + * @params [in] *memh Handle of the registered memory + * + * @return Error code as defined by @ref ucc_status_t. + */ +ucc_status_t ucc_mem_exchange_post(ucc_mem_map_mem_h *memh); + +/** + * @ingroup UCC_CONTEXT + * @brief Routine tests for the completion of a memory exchange on a context. + * + * This routine tests for the completion of a memory exchange on a context. + * + * @params [in] *memh Handle of the registered memory + * + * @return Error code as defined by @ref ucc_status_t. + */ +ucc_status_t ucc_mem_exchange_test(ucc_mem_map_mem_h *memh); + +/** + * @ingroup UCC_CONTEXT + * @brief Routine unmaps memory from a context + * + * This routine unmaps memory and all resources associated with the memory + * from a context. The memh object is freed and cannot be reused. + * + * @params [in] *memh Handle of the registered memory + * + * @return Error code as defined by @ref ucc_status_t. + */ +ucc_status_t ucc_mem_unmap(ucc_mem_map_mem_h *memh); + END_C_DECLS #endif From dd2c696ab9744244cd73bc440eb9d4929c59d3eb Mon Sep 17 00:00:00 2001 From: ferrol aderholdt Date: Fri, 1 Nov 2024 10:12:43 -0700 Subject: [PATCH 2/7] REVIEW: address feedback --- src/ucc/api/ucc.h | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/src/ucc/api/ucc.h b/src/ucc/api/ucc.h index 2ed645ef95..23fba5704d 100644 --- a/src/ucc/api/ucc.h +++ b/src/ucc/api/ucc.h @@ -2258,35 +2258,6 @@ ucc_status_t ucc_mem_map(ucc_context_t context, ucc_mem_map_flags flags, ucc_mem_map_params params, ucc_mem_map_mem_h *memh); - -/** - * @ingroup UCC_CONTEXT - * @brief Routine exchanges mapped memory with peers to enable future - * collectives. - * - * This routine performs an nonblocking exchange of mapped memory with peers - * within the same context on which it is mapped. This is a collective routine - * and must be called by all members of the ucc_context on which the memory - * is mapped. - * - * @params [in] *memh Handle of the registered memory - * - * @return Error code as defined by @ref ucc_status_t. - */ -ucc_status_t ucc_mem_exchange_post(ucc_mem_map_mem_h *memh); - -/** - * @ingroup UCC_CONTEXT - * @brief Routine tests for the completion of a memory exchange on a context. - * - * This routine tests for the completion of a memory exchange on a context. - * - * @params [in] *memh Handle of the registered memory - * - * @return Error code as defined by @ref ucc_status_t. - */ -ucc_status_t ucc_mem_exchange_test(ucc_mem_map_mem_h *memh); - /** * @ingroup UCC_CONTEXT * @brief Routine unmaps memory from a context From c6c02b5e8b35a85b093b0562effa191530eb5030 Mon Sep 17 00:00:00 2001 From: ferrol aderholdt Date: Fri, 1 Nov 2024 10:30:17 -0700 Subject: [PATCH 3/7] CODESTYLE: fix code style --- src/ucc/api/ucc.h | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/ucc/api/ucc.h b/src/ucc/api/ucc.h index 23fba5704d..8ec8a5832f 100644 --- a/src/ucc/api/ucc.h +++ b/src/ucc/api/ucc.h @@ -2220,30 +2220,29 @@ ucc_status_t ucc_ee_wait(ucc_ee_h ee, ucc_ev_t *ev); */ ucc_status_t ucc_collective_triggered_post(ucc_ee_h ee, ucc_ev_t *ee_event); -typedef void * ucc_mem_map_mem_h; +typedef void *ucc_mem_map_mem_h; /** * @ingroup UCC_DATATYPE */ -enum ucc_mem_map_flags { +typedef enum { UCC_MEM_MAP_EXPORT = 0, /*!< Indicate @ref ucc_mem_map() should export memory handles from TLs used by context */ - UCC_MEM_MAP_IMPORT = 1 /*!< Indicate @ref ucc_mem_map() should import + UCC_MEM_MAP_IMPORT = 1 /*!< Indicate @ref ucc_mem_map() should import memory handles from user memory handle */ -}; +} ucc_mem_map_flags_t; /** * @ingroup UCC_CONTEXT * @brief Routine registers or maps memory for use in future collectives. * - * This routine maps memory a user-specified memory segment with a @ref - * ucc_context_t. The segment is considered "mapped" with the context until - * either the user calls @ref ucc_mem_unmap or @ref ucc_context_destroy(). A - * handle to the mapped memory is provided in memh. If the flag - * UCC_MEM_MAP_EXPORT is used, the memory will be mapped and memory - * handles from TLs will be generated and stored in the memh. If the flag - * UCC_MEM_MAP_IMPORT is used, the user must provide a valid memh, otherwise - * behavior is undefined. + * This routine maps a user-specified memory segment with a ucc_context_h. The + * segment is considered "mapped" with the context until either the user calls + * @ref ucc_mem_unmap or @ref ucc_context_destroy(). A handle to the mapped + * memory is provided in memh. If the flag UCC_MEM_MAP_EXPORT is used, the + * memory will be mapped and memory handles from TLs will be generated and + * stored in the memh. If the flag UCC_MEM_MAP_IMPORT is used, the user must + * provide a valid memh, otherwise behavior is undefined. * * @params [in] context Context mapped memory is associated with * @params [in] flags flags dictating the behavior of the routine @@ -2254,10 +2253,8 @@ enum ucc_mem_map_flags { * @return Error code as defined by @ref ucc_status_t. */ -ucc_status_t ucc_mem_map(ucc_context_t context, - ucc_mem_map_flags flags, - ucc_mem_map_params params, - ucc_mem_map_mem_h *memh); +ucc_status_t ucc_mem_map(ucc_context_h context, ucc_mem_map_flags_t flags, + ucc_mem_map_params_t params, ucc_mem_map_mem_h *memh); /** * @ingroup UCC_CONTEXT * @brief Routine unmaps memory from a context From fe7f690ea7103ed2ee8f19f359c0cdc44adfa98f Mon Sep 17 00:00:00 2001 From: ferrol aderholdt Date: Fri, 1 Nov 2024 11:01:54 -0700 Subject: [PATCH 4/7] REVIEW: remove ref --- src/ucc/api/ucc.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ucc/api/ucc.h b/src/ucc/api/ucc.h index 8ec8a5832f..0d218265ad 100644 --- a/src/ucc/api/ucc.h +++ b/src/ucc/api/ucc.h @@ -2226,9 +2226,9 @@ typedef void *ucc_mem_map_mem_h; * @ingroup UCC_DATATYPE */ typedef enum { - UCC_MEM_MAP_EXPORT = 0, /*!< Indicate @ref ucc_mem_map() should export + UCC_MEM_MAP_EXPORT = 0, /*!< Indicate ucc_mem_map() should export memory handles from TLs used by context */ - UCC_MEM_MAP_IMPORT = 1 /*!< Indicate @ref ucc_mem_map() should import + UCC_MEM_MAP_IMPORT = 1 /*!< Indicate ucc_mem_map() should import memory handles from user memory handle */ } ucc_mem_map_flags_t; @@ -2238,7 +2238,7 @@ typedef enum { * * This routine maps a user-specified memory segment with a ucc_context_h. The * segment is considered "mapped" with the context until either the user calls - * @ref ucc_mem_unmap or @ref ucc_context_destroy(). A handle to the mapped + * ucc_mem_unmap or ucc_context_destroy(). A handle to the mapped * memory is provided in memh. If the flag UCC_MEM_MAP_EXPORT is used, the * memory will be mapped and memory handles from TLs will be generated and * stored in the memh. If the flag UCC_MEM_MAP_IMPORT is used, the user must @@ -2250,7 +2250,7 @@ typedef enum { * memory to map * @params [inout] *memh Handle for the registered memory * - * @return Error code as defined by @ref ucc_status_t. + * @return Error code as defined by ucc_status_t. */ ucc_status_t ucc_mem_map(ucc_context_h context, ucc_mem_map_flags_t flags, @@ -2264,7 +2264,7 @@ ucc_status_t ucc_mem_map(ucc_context_h context, ucc_mem_map_flags_t flags, * * @params [in] *memh Handle of the registered memory * - * @return Error code as defined by @ref ucc_status_t. + * @return Error code as defined by ucc_status_t. */ ucc_status_t ucc_mem_unmap(ucc_mem_map_mem_h *memh); From f85f87ac822fb66ddd74e56ffabf0321ccc4c385 Mon Sep 17 00:00:00 2001 From: ferrol aderholdt Date: Fri, 1 Nov 2024 11:20:03 -0700 Subject: [PATCH 5/7] REVIEW: address feedback --- src/ucc/api/ucc.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ucc/api/ucc.h b/src/ucc/api/ucc.h index 0d218265ad..d06cbeea02 100644 --- a/src/ucc/api/ucc.h +++ b/src/ucc/api/ucc.h @@ -2244,11 +2244,11 @@ typedef enum { * stored in the memh. If the flag UCC_MEM_MAP_IMPORT is used, the user must * provide a valid memh, otherwise behavior is undefined. * - * @params [in] context Context mapped memory is associated with - * @params [in] flags flags dictating the behavior of the routine - * @params [in] params parameters indicating the address and length of + * @param [in] context Context mapped memory is associated with + * @param [in] flags flags dictating the behavior of the routine + * @param [in] params parameters indicating the address and length of * memory to map - * @params [inout] *memh Handle for the registered memory + * @param [inout] *memh Handle for the registered memory * * @return Error code as defined by ucc_status_t. */ @@ -2262,7 +2262,7 @@ ucc_status_t ucc_mem_map(ucc_context_h context, ucc_mem_map_flags_t flags, * This routine unmaps memory and all resources associated with the memory * from a context. The memh object is freed and cannot be reused. * - * @params [in] *memh Handle of the registered memory + * @param [in] *memh Handle of the registered memory * * @return Error code as defined by ucc_status_t. */ From be38a7fa07e97f9b354350e8823231fdaf71376c Mon Sep 17 00:00:00 2001 From: ferrol aderholdt Date: Wed, 11 Dec 2024 09:49:06 -0800 Subject: [PATCH 6/7] REVIEW: address feedback --- src/ucc/api/ucc.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/ucc/api/ucc.h b/src/ucc/api/ucc.h index d06cbeea02..498b9e6378 100644 --- a/src/ucc/api/ucc.h +++ b/src/ucc/api/ucc.h @@ -2237,8 +2237,9 @@ typedef enum { * @brief Routine registers or maps memory for use in future collectives. * * This routine maps a user-specified memory segment with a ucc_context_h. The - * segment is considered "mapped" with the context until either the user calls - * ucc_mem_unmap or ucc_context_destroy(). A handle to the mapped + * segment is considered "mapped" with the context until the user calls + * ucc_mem_unmap. It is the user's responsibility to unmap all mapped segments + * prior to calling ucc_context_destroy(). A handle to the mapped * memory is provided in memh. If the flag UCC_MEM_MAP_EXPORT is used, the * memory will be mapped and memory handles from TLs will be generated and * stored in the memh. If the flag UCC_MEM_MAP_IMPORT is used, the user must @@ -2255,12 +2256,14 @@ typedef enum { ucc_status_t ucc_mem_map(ucc_context_h context, ucc_mem_map_flags_t flags, ucc_mem_map_params_t params, ucc_mem_map_mem_h *memh); + /** * @ingroup UCC_CONTEXT * @brief Routine unmaps memory from a context * - * This routine unmaps memory and all resources associated with the memory - * from a context. The memh object is freed and cannot be reused. + * This is a collective routine that unmaps memory and all resources + * associated with the memory from a context. The memh object is freed and + * cannot be reused. * * @param [in] *memh Handle of the registered memory * From 41c840f7cd67cd7f55ec3c6271b0b558419283f3 Mon Sep 17 00:00:00 2001 From: ferrol aderholdt Date: Fri, 13 Dec 2024 11:07:55 -0800 Subject: [PATCH 7/7] REVIEW: address feedback --- src/ucc/api/ucc.h | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/src/ucc/api/ucc.h b/src/ucc/api/ucc.h index 498b9e6378..90a30e4d68 100644 --- a/src/ucc/api/ucc.h +++ b/src/ucc/api/ucc.h @@ -1798,9 +1798,12 @@ enum ucc_coll_args_field { UCC_COLL_ARGS_FIELD_TAG = UCC_BIT(1), UCC_COLL_ARGS_FIELD_CB = UCC_BIT(2), UCC_COLL_ARGS_FIELD_GLOBAL_WORK_BUFFER = UCC_BIT(3), - UCC_COLL_ARGS_FIELD_ACTIVE_SET = UCC_BIT(4) + UCC_COLL_ARGS_FIELD_ACTIVE_SET = UCC_BIT(4), + UCC_COLL_ARGS_FIELD_MEM_MAP_MEMH = UCC_BIT(5) }; +typedef void *ucc_mem_map_mem_h; + /** * @ingroup UCC_COLLECTIVES * @@ -1873,6 +1876,26 @@ typedef struct ucc_coll_args { int64_t stride; uint64_t size; } active_set; + union { + ucc_mem_map_mem_h local_memh; /*!< Memory handle of locally + mapped memory for the src + buffer */ + ucc_mem_map_mem_h *global_memh; /*!< Array of memory handles + for the src buffer + corresponding to all + participating processes + in the collective */ + } src_memh; + union { + ucc_mem_map_mem_h local_memh; /*!< Memory handle of locally + mapped memory for the dst + buffer */ + ucc_mem_map_mem_h *global_memh; /*!< Array of memory handles + for the dst buffer + corresponding to all + participating processes + in the collective */ + } dst_memh; } ucc_coll_args_t; /** @@ -2220,8 +2243,6 @@ ucc_status_t ucc_ee_wait(ucc_ee_h ee, ucc_ev_t *ev); */ ucc_status_t ucc_collective_triggered_post(ucc_ee_h ee, ucc_ev_t *ee_event); -typedef void *ucc_mem_map_mem_h; - /** * @ingroup UCC_DATATYPE */ @@ -2236,12 +2257,12 @@ typedef enum { * @ingroup UCC_CONTEXT * @brief Routine registers or maps memory for use in future collectives. * - * This routine maps a user-specified memory segment with a ucc_context_h. The - * segment is considered "mapped" with the context until the user calls - * ucc_mem_unmap. It is the user's responsibility to unmap all mapped segments - * prior to calling ucc_context_destroy(). A handle to the mapped - * memory is provided in memh. If the flag UCC_MEM_MAP_EXPORT is used, the - * memory will be mapped and memory handles from TLs will be generated and + * This local routine maps a user-specified memory segment with a + * ucc_context_h. The segment is considered "mapped" with the context until + * the user calls ucc_mem_unmap. It is the user's responsibility to unmap all + * mapped segments prior to calling ucc_context_destroy(). A handle to the + * mapped memory is provided in memh. If the flag UCC_MEM_MAP_EXPORT is used, + * the memory will be mapped and memory handles from TLs will be generated and * stored in the memh. If the flag UCC_MEM_MAP_IMPORT is used, the user must * provide a valid memh, otherwise behavior is undefined. * @@ -2261,7 +2282,7 @@ ucc_status_t ucc_mem_map(ucc_context_h context, ucc_mem_map_flags_t flags, * @ingroup UCC_CONTEXT * @brief Routine unmaps memory from a context * - * This is a collective routine that unmaps memory and all resources + * This is a local routine that unmaps memory and all resources * associated with the memory from a context. The memh object is freed and * cannot be reused. *