c: add asan instrumentation for mmap/munmap/mremap

TEST
tarantool · Jul 19, 2024 · 8610ab1 · 8610ab1
1 parent 01f4586
commit 8610ab1
Show file tree

Hide file tree

Showing 4 changed files with 429 additions and 8 deletions.
diff --git a/.github/workflows/sanitizers-testing.yml b/.github/workflows/sanitizers-testing.yml
@@ -34,17 +34,22 @@ jobs:
         # XXX: Let's start with only Linux/x86_64
         BUILDTYPE: [Debug, Release]
         CC: [gcc-10, clang-11]
+        ALLOCATOR: [SYSMALLOC, LUAJIT_ALLOCATOR]
         include:
           - BUILDTYPE: Debug
             CMAKEFLAGS: -DCMAKE_BUILD_TYPE=Debug -DLUA_USE_ASSERT=ON -DLUA_USE_APICHECK=ON
           - BUILDTYPE: Release
             CMAKEFLAGS: -DCMAKE_BUILD_TYPE=RelWithDebInfo
+          - ALLOCATOR: SYSMALLOC
+            ASANFLAGS: -DLUAJIT_USE_ASAN=ON -DLUAJIT_USE_SYSMALLOC=ON
+          - ALLOCATOR: LUAJIT_ALLOCATOR
+            ASANFLAGS: -DLUAJIT_USE_ASAN_HARDENING=ON
     runs-on: [self-hosted, regular, Linux, x86_64]
     name: >
       LuaJIT with ASan and UBSan (Linux/x86_64)
       ${{ matrix.BUILDTYPE }}
       CC:${{ matrix.CC }}
-      GC64:ON SYSMALLOC:ON
+      GC64:ON ALLOCATOR=${{ matrix.ALLOCATOR }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -71,8 +76,7 @@ jobs:
           -G Ninja
           ${{ matrix.CMAKEFLAGS }}
           -DLUAJIT_ENABLE_GC64=ON
-          -DLUAJIT_USE_ASAN=ON
-          -DLUAJIT_USE_SYSMALLOC=ON
+          ${{ matrix.ASANFLAGS }}
           -DLUAJIT_USE_UBSAN=ON
       - name: build
         run: cmake --build . --parallel

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -291,10 +291,10 @@ option(LUAJIT_USE_ASAN "Build LuaJIT with AddressSanitizer" OFF)
 if(LUAJIT_USE_ASAN)
   if(NOT LUAJIT_USE_SYSMALLOC)
     message(WARNING
-      "Unfortunately, internal LuaJIT memory allocator is not instrumented yet,"
-      " so to find any memory errors it's better to build LuaJIT with system"
-      " provided memory allocator (i.e. run CMake configuration phase with"
-      " -DLUAJIT_USE_SYSMALLOC=ON)."
+      "Run CMake configuration phase with -DLUAJIT_USE_SYSMALLOC=ON"
+      "to use system memory allocator or replace -DLUAJIT_USE_ASAN=ON"
+      "to -DLUAJIT_USE_ASAN_HARDENING=ON to use internal LuaJIT memory"
+      "allocator."
     )
   endif()
   # Use all recommendations described in AddressSanitize docs:
@@ -309,6 +309,30 @@ if(LUAJIT_USE_ASAN)
   )
 endif()
 
+# Same as LUAJIT_USE_ASAN, but using the internal LuaJIT memory
+# allocator instrumented with ASAN.
+option(LUAJIT_USE_ASAN_HARDENING "Build LuaJIT with an internal allocator with integrated AddressSanitizer" OFF)
+if(LUAJIT_USE_ASAN_HARDENING)
+  if(NOT LUAJIT_ENABLE_GC64)
+    message(FATAL_ERROR
+      "ASAN only with GC64."
+    )
+  endif()
+  # Use all recommendations described in AddressSanitize docs:
+  # https://clang.llvm.org/docs/AddressSanitizer.html.
+  AppendFlags(CMAKE_C_FLAGS
+    # Enable hints for AddressSanitizer (see src/lj_str.c).
+    -DLUAJIT_USE_ASAN
+    # Enable ASAN instrumentation of internal LuaJIT memory allocator
+    # see (src/lj_alloc.c)
+    -DLUAJIT_USE_ASAN_HARDENING
+    # XXX: To get nicer stack traces in error messages.
+    -fno-omit-frame-pointer
+    # Enable AddressSanitizer support.
+    -fsanitize=address
+  )
+endif()
+
 option(LUAJIT_USE_UBSAN "Build LuaJIT with UndefinedBehaviorSanitizer" OFF)
 if(LUAJIT_USE_UBSAN)
   # Use all needed checks from the UndefinedBehaviorSanitizer