hmem/rocr: enable using ofi from ompi

for single node jobs, a small number of changes are required to enable using mtl/ofi from ompi for rocm devices. Signed-off-by: Edgar Gabriel <[email protected]>
ROCm · Sep 11, 2023 · 3100fe7 · 3100fe7
1 parent 1daaa2c
commit 3100fe7
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 2 deletions.
diff --git a/prov/shm/src/smr_attr.c b/prov/shm/src/smr_attr.c
@@ -35,7 +35,7 @@
 #define SMR_TX_CAPS (OFI_TX_MSG_CAPS | FI_TAGGED | OFI_TX_RMA_CAPS | FI_ATOMICS)
 #define SMR_RX_CAPS (FI_SOURCE | FI_RMA_EVENT | OFI_RX_MSG_CAPS | FI_TAGGED | \
 		     OFI_RX_RMA_CAPS | FI_ATOMICS | FI_DIRECTED_RECV | \
-		     FI_MULTI_RECV)
+		     FI_MULTI_RECV | FI_REMOTE_COMM)
 #define SMR_HMEM_TX_CAPS ((SMR_TX_CAPS | FI_HMEM) & ~FI_ATOMICS)
 #define SMR_HMEM_RX_CAPS ((SMR_RX_CAPS | FI_HMEM) & ~FI_ATOMICS)
 #define SMR_TX_OP_FLAGS (FI_COMPLETION | FI_INJECT_COMPLETE | \

diff --git a/prov/shm/src/smr_ep.c b/prov/shm/src/smr_ep.c
@@ -586,6 +586,7 @@ int smr_select_proto(void **desc, size_t iov_count,
 	if (iov_count == 1 && desc && desc[0]) {
 		smr_desc = (struct ofi_mr *) *desc;
 		iface = smr_desc->iface;
+		smr_desc->flags = FI_HMEM_DEVICE_ONLY;
 		use_ipc = ofi_hmem_is_ipc_enabled(iface) &&
 				smr_desc->flags & FI_HMEM_DEVICE_ONLY &&
 				!(op_flags & FI_INJECT);

diff --git a/prov/util/src/util_mr_map.c b/prov/util/src/util_mr_map.c
@@ -69,7 +69,7 @@ int ofi_mr_map_insert(struct ofi_mr_map *map, const struct fi_mr_attr *attr,
 	if (!(map->mode & FI_MR_VIRT_ADDR))
 		item->offset = (uintptr_t) attr->mr_iov[0].iov_base;
 
-	if (map->mode & FI_MR_PROV_KEY)
+	//if (map->mode & FI_MR_PROV_KEY)
 		item->requested_key = map->key++;
 
 	ret = ofi_rbmap_insert(map->rbtree, &item->requested_key, item, NULL);