Port verbs layer to verbs-linux-arm8 (#3837)

* getInfiCmiChunkThread(): Check for empty PCQueue.

Be more informative about ibv_reg_mr() failures.

* Add verbs-linux-arm8 port.

---------

Co-authored-by: Eric Bohm <[email protected]>
Co-authored-by: Ritvik Rao <[email protected]>

src/arch/verbs-linux-arm8/conv-mach-ibud.h

@@ -0,0 +1,23 @@
+#undef CMK_USE_IBUD
+#define CMK_USE_IBUD				1
+
+#undef CMK_USE_IBVERBS
+#define CMK_USE_IBVERBS             1
+
+// FIXME: See if I need to include any of these flags
+#undef CMK_NETPOLL
+#define CMK_NETPOLL						1
+
+#undef CMK_MALLOC_USE_GNU_MALLOC
+#define CMK_MALLOC_USE_GNU_MALLOC                          0
+
+#undef CMK_MALLOC_USE_OS_BUILTIN
+#define CMK_MALLOC_USE_OS_BUILTIN                          1
+
+#undef CMK_IMMEDIATE_MSG
+#define CMK_IMMEDIATE_MSG       0
+
+#undef CMK_DISABLE_SYNC
+#define CMK_DISABLE_SYNC       1
+
+

src/arch/verbs-linux-arm8/conv-mach-ibud.sh

@@ -0,0 +1,18 @@
+#default ibverbs path for openib
+if test -z "$CMK_INCDIR"
+then
+  #openib-1.1
+  if test -f /opt/ofed/include/infiniband/verbs.h
+  then
+    CMK_INCDIR="-I/opt/ofed/include/"
+    CMK_LIBDIR="-L/opt/ofed/lib64"
+	fi
+  if test -f /usr/local/ofed/include/infiniband/verbs.h
+  then
+    CMK_INCDIR="-I/usr/local/ofed/include/"
+    CMK_LIBDIR="-L/usr/local/ofed/lib64"
+	fi
+fi
+
+CMK_LIBS="$CMK_LIBS -libverbs"
+

src/arch/verbs-linux-arm8/conv-mach-pxshm.h

@@ -0,0 +1,3 @@
+#undef CMK_USE_PXSHM
+#define CMK_USE_PXSHM 			1
+#define PXSHM_LOCK                      1

src/arch/verbs-linux-arm8/conv-mach-pxshm.sh

@@ -0,0 +1,2 @@
+CMK_SYSLIBS="$CMK_SYSLIBS -lrt -lpthread"
+

src/arch/verbs-linux-arm8/conv-mach-smp.h

@@ -0,0 +1,20 @@
+
+#define CMK_SMP						   1
+
+
+#undef CMK_SHARED_VARS_UNAVAILABLE
+#undef CMK_SHARED_VARS_POSIX_THREADS_SMP
+#define CMK_SHARED_VARS_UNAVAILABLE                        0
+#define CMK_SHARED_VARS_POSIX_THREADS_SMP                  1
+
+#undef CMK_MALLOC_USE_GNU_MALLOC
+#undef CMK_MALLOC_USE_OS_BUILTIN
+#define CMK_MALLOC_USE_GNU_MALLOC                          0
+#define CMK_MALLOC_USE_OS_BUILTIN                          1
+
+/*#define CMK_MMAP_PROBE                                     1 */
+
+/*#define CMK_PCQUEUE_LOCK				   1 */
+/*#define CMK_USE_MFENCE 					   1 */
+/* Replaced by CMK_NOT_USE_TLS_THREAD as default */
+/*#define CMK_USE_TLS_THREAD                                 1*/

src/arch/verbs-linux-arm8/conv-mach-smp.sh

@@ -0,0 +1,3 @@
+CMK_DEFS="$CMK_DEFS -D_REENTRANT"
+CMK_LIBS="-lpthread $CMK_LIBS "
+CMK_SMP='1'

src/arch/verbs-linux-arm8/conv-mach.h

@@ -0,0 +1,72 @@
+
+#ifndef _CONV_MACH_H
+#define _CONV_MACH_H
+
+#define CMK_USE_IBVERBS				1
+#define CMK_NETPOLL         1
+
+#define CMK_ARM                                   1
+
+#define CMK_ASYNC_NOT_NEEDED                               0
+#define CMK_ASYNC_USE_FIOASYNC_AND_FIOSETOWN               0
+#define CMK_ASYNC_USE_FIOASYNC_AND_SIOCSPGRP               0
+#define CMK_ASYNC_USE_FIOSSAIOSTAT_AND_FIOSSAIOOWN         0
+#define CMK_ASYNC_USE_F_SETFL_AND_F_SETOWN                 1
+
+#define CMK_DLL_CC  "g++ -shared -O3 -o "
+
+#define CMK_GETPAGESIZE_AVAILABLE                          1
+
+#define CMK_MALLOC_USE_GNU_MALLOC                          0
+#define CMK_MALLOC_USE_OS_BUILTIN                          1
+
+#define CMK_MEMORY_PAGESIZE                                4096
+#define CMK_MEMORY_PROTECTABLE                             0
+
+
+#define CMK_SSH_IS_A_COMMAND                               1
+#define CMK_SSH_NOT_NEEDED                                 0
+
+#define CMK_SHARED_VARS_UNAVAILABLE                        1
+
+#define CMK_THREADS_USE_CONTEXT                            0
+#define CMK_THREADS_USE_FCONTEXT                           1
+#define CMK_THREADS_USE_PTHREADS                           0
+#define CMK_THREADS_ARE_WIN32_FIBERS                       0
+
+#define CMK_SIGNAL_NOT_NEEDED                              0
+#define CMK_SIGNAL_USE_SIGACTION                           0
+#define CMK_SIGNAL_USE_SIGACTION_WITH_RESTART              1
+
+#define CMK_THREADS_REQUIRE_NO_CPV                         0
+#define CMK_THREADS_COPY_STACK                             0
+
+#define CMK_TIMER_USE_RDTSC                                0
+#define CMK_TIMER_USE_GETRUSAGE                            1
+#define CMK_TIMER_USE_SPECIAL                              0
+#define CMK_TIMER_USE_TIMES                                0
+
+
+#define CMK_64BIT    1
+
+#define CMK_WHEN_PROCESSOR_IDLE_BUSYWAIT                   0
+#define CMK_WHEN_PROCESSOR_IDLE_USLEEP                     1
+
+
+#define CMK_DEBUG_MODE					   0 
+#define CMK_WEB_MODE                                       1
+
+#define CMK_LBDB_ON					   1
+
+/*
+#undef CMK_IMMEDIATE_MSG
+#define CMK_IMMEDIATE_MSG       0
+*/
+
+#undef CMK_DISABLE_SYNC
+#define CMK_DISABLE_SYNC       1
+
+#undef CMK_IBVERBS_FAST_START
+#define CMK_IBVERBS_FAST_START                         0
+
+#endif

src/arch/verbs-linux-arm8/conv-mach.sh

@@ -0,0 +1,24 @@
+. $CHARMINC/cc-gcc.sh
+
+#CMK_DEFS="$CMK_DEFS -DHAVE_USR_INCLUDE_MALLOC_H=1 "
+CMK_XIOPTS=''
+CMK_LIBS="$CMK_LIBS -libverbs"
+CMK_CXX_OPTIMIZE='-O3'
+
+CMK_QT='generic64-light'
+
+#default ibverbs path for openib
+if test -z "$CMK_INCDIR"
+then
+  #openib-1.1
+  if test -f /opt/ofed/include/infiniband/verbs.h
+  then
+    CMK_INCDIR='-I/opt/ofed/include/'
+    CMK_LIBDIR='-L/opt/ofed/lib64'
+	fi
+  if test -f /usr/local/ofed/include/infiniband/verbs.h
+  then
+    CMK_INCDIR='-I/usr/local/ofed/include/'
+    CMK_LIBDIR='-L/usr/local/ofed/lib64'
+	fi
+fi

src/arch/verbs/machine-ibverbs.C

@@ -2540,7 +2540,9 @@ static inline void *getInfiCmiChunkThread(int dataSize){
 			if(!PCQueueEmpty(queuePool[CmiMyRank()][i])){
 				for(j = 0; j < PCQueueLength(queuePool[CmiMyRank()][i]); j++){
 					pointer = (void *)PCQueuePop(queuePool[CmiMyRank()][i]);
-					infi_CmiFreeDirect(pointer);	
+					// Check because queue might actually be empty
+					if(pointer != NULL)
+					    infi_CmiFreeDirect(pointer);
 				}
 			}
 		}	
@@ -2570,8 +2572,12 @@ static inline void *getInfiCmiChunkThread(int dataSize){
 		hdr = (CmiChunkHeader *)res;
 
 		key = ibv_reg_mr(context->pd,res,(allocSize+sizeof(CmiChunkHeader))*count,IBV_ACCESS_REMOTE_READ | IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
-	        if(key == NULL)
+                if(key == NULL) {
+                    CmiError("Failed to allocate %d messages in pool %d\n", count, poolIdx);
+                    CmiError("Consider increasing IBVBlockAllocRatio (currently %d) or IBVBlockThreshold (currently %d)\n",
+                              blockAllocRatio, blockThreshold);
                     CmiAbort("ibv_reg_mr failed to pin memory\n");
+		}
 #if CMK_IBVERBS_STATS
 		numCurReg++;
 		numReg++;
@@ -2672,7 +2678,12 @@ static inline void *getInfiCmiChunk(int dataSize){
                 hdr = (CmiChunkHeader *)res;
 
                 key = ibv_reg_mr(context->pd,res,(allocSize+sizeof(CmiChunkHeader))*count,IBV_ACCESS_REMOTE_READ | IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
-                CmiAssert(key != NULL);
+                if(key == NULL) {
+                    CmiError("Failed to allocate %d messages in pool %d\n", count, poolIdx);
+                    CmiError("Consider increasing IBVBlockAllocRatio (currently %d) or IBVBlockThreshold (currently %d)\n",
+                              blockAllocRatio, blockThreshold);
+                    CmiAbort("ibv_reg_mr failed to pin memory\n");
+		}
 #if CMK_IBVERBS_STATS
 		numCurReg++;
 		numReg++;