Rework Docker Image build (#741)

* Try reworking Docker image * Fix some old behavior for building abi libraries I don't think the result is actually usable... * Remove most Python2 references Still left some references to python2 for helper scripts since they aren't tested (afaik) in CI * Fix CI * Remove old CMakeLists.txt and remove support for old TRAILOFBITS_LIBRARIES method of building * Update README
lifting-bits · Mar 5, 2021 · 339800e · 339800e
1 parent e42f45f
commit 339800e
Show file tree

Hide file tree

Showing 9 changed files with 106 additions and 426 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -25,7 +25,7 @@ jobs:
     needs: [VersionFile]
     strategy:
       matrix:
-        llvm: ["900", "1000", "1100"]
+        llvm: ["9", "10", "11"]
         ubuntu: ["20.04", "18.04"]
     steps:
     - uses: actions/checkout@v2
@@ -37,6 +37,10 @@ jobs:
       run: |
         docker run --rm docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --version
         docker run --rm --entrypoint=mcsema-disass docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --help
+
+        docker run --rm -v $(pwd)/tests/test_suite_generator/generated/prebuilt_cfg/amd64/linux/cfg:/work docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --arch amd64 --os linux --cfg /work/hello_world --output /work/hello_world.bc
+        docker run --rm -v $(pwd)/tests/test_suite_generator/generated/prebuilt_cfg/x86/linux/cfg:/work docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --arch amd64 --os linux --cfg /work/hello_world --output /work/hello_world.bc
+
     - name: Push Image for LLVM ${{ matrix.llvm }} on ${{ matrix.ubuntu }}
       if: github.event_name == 'push' && github.ref == 'refs/heads/master'
       run: |
@@ -45,23 +49,12 @@ jobs:
         docker login -u "${DOCKER_HUB_USER}" -p "${DOCKER_HUB_TOKEN}"
         docker tag docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest trailofbits/mcsema:llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64
         docker push trailofbits/mcsema:llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64
+
       env:
         GH_USER: ${{ github.actor }}
         GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         DOCKER_HUB_USER: ${{  secrets.DOCKER_HUB_USER  }}
         DOCKER_HUB_TOKEN: ${{  secrets.DOCKER_HUB_TOKEN  }}
-    - name: Compress Artifacts LLVM ${{ matrix.llvm }} on ${{ matrix.ubuntu }}
-      if: github.event_name == 'push' && github.ref == 'refs/heads/master'
-      run: |
-        mkdir -p artifacts
-        cp -v scripts/emit-artifact.sh ./artifacts/
-        docker run -v "$(pwd)/artifacts:/out" --rm --entrypoint /bin/bash docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest -c "/out/emit-artifact.sh /out/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64.tar.xz"
-        ls -l artifacts/
-    - uses: actions/upload-artifact@v1
-      if: github.event_name == 'push' && github.ref == 'refs/heads/master'
-      with:
-        name: mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64.tar.xz
-        path: artifacts/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64.tar.xz
   CreateRelease:
     needs: [Docker_Linux, VersionFile]
     runs-on: ubuntu-latest

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -13,27 +13,22 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-if (NOT DEFINED ENV{TRAILOFBITS_LIBRARIES})
-  message(STATUS "Using new vcpkg build system")
-  include(CMakeLists_vcpkg.txt)
-  return()
-endif()
+# Find remill first because its config has useful dependency-finding info that
+# needs to be found before the CMake `project` declaration
+find_package(remill COMPONENTS VCPKG_DEPS QUIET)
 
-project(mcsema)
-cmake_minimum_required(VERSION 3.14)
+include(cmake/vcpkg_helper.cmake)
 
-# remill
-if("${PLATFORM_NAME}" STREQUAL "windows")
-  set(REMILL_FINDPACKAGE_HINTS HINTS "${CMAKE_INSTALL_PREFIX}/remill/lib")
-endif()
+project(mcsema C CXX ASM)
+
+include(GNUInstallDirs)
+cmake_minimum_required(VERSION 3.14)
 
-find_package(remill REQUIRED ${REMILL_FINDPACKAGE_HINTS})
 
 include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/settings.cmake")
 include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/utils.cmake")
 include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/BCCompiler.cmake")
 include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/ccache.cmake")
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules")
 
 configureCcache()
 FindAndSelectClangCompiler()
@@ -54,17 +49,23 @@ if(NOT DEFINED WIN32)
   set(PROJECT_CXXFLAGS
     ${GLOBAL_CXXFLAGS}
     -Wconversion
-    -pedantic 
-    -Wno-unreachable-code-return 
+    -pedantic
+    -Wno-unreachable-code-return
   )
 endif()
 
 #
 # libraries
 #
 
+find_package(remill CONFIG REQUIRED)
+list(APPEND PROJECT_LIBRARIES remill_settings remill)
+get_target_property(REMILL_INCLUDE_LOCATION remill_settings INTERFACE_INCLUDE_DIRECTORIES)
+
 # protobuf
-find_package(Protobuf REQUIRED)
+# Compatibility since we use older protobuf CMake functions
+set(protobuf_MODULE_COMPATIBLE ON CACHE BOOL "" FORCE)
+find_package(Protobuf CONFIG REQUIRED)
 list(APPEND PROJECT_LIBRARIES ${Protobuf_LIBRARIES})
 list(APPEND PROJECT_INCLUDEDIRECTORIES ${Protobuf_INCLUDE_DIR})
 list(APPEND PROJECT_DEFINITIONS "GOOGLE_PROTOBUF_NO_RTTI")
@@ -140,32 +141,15 @@ add_custom_command(
 list(APPEND PROJECT_INCLUDEDIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR})
 
 add_dependencies(${MCSEMA_LIFT}
-  semantics
   protobuf_python_module_ida)
 
 #
 # libraries
 #
 
-# remill
-if(NOT TARGET remill)
-  if("${PLATFORM_NAME}" STREQUAL "windows")
-    set(REMILL_FINDPACKAGE_HINTS HINTS "${CMAKE_INSTALL_PREFIX}/remill/lib")
-  endif()
-
-  find_package(remill REQUIRED ${REMILL_FINDPACKAGE_HINTS})
-endif()
-
-list(APPEND PROJECT_LIBRARIES remill)
-
 # anvill
-if(NOT TARGET anvill-${REMILL_LLVM_VERSION})
-  find_package(anvill REQUIRED)
-  if(NOT anvill_FOUND)
-    message(FATAL_ERROR "McSema depends upon Anvill being installed")
-  endif()
-endif()
-list(APPEND PROJECT_LIBRARIES anvill-${REMILL_LLVM_VERSION})
+find_package(anvill CONFIG REQUIRED)
+list(APPEND PROJECT_LIBRARIES anvill)
 
 # mcsema-disass
 
@@ -191,7 +175,7 @@ set(MCSEMA_PYTHON_SOURCES
 if(MCSEMA_INSTALL_PYTHON3_LIBS)
   add_custom_target(build_mcsema_disass_python3
     DEPENDS ${MCSEMA_PYTHON_SOURCES})
-    
+
   add_custom_command(
       TARGET build_mcsema_disass_python3 POST_BUILD
       COMMAND which python3 && python3 setup.py build --force