diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b2a3da0ae..3d4daa092 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -25,7 +25,7 @@ jobs:
needs: [VersionFile]
strategy:
matrix:
- llvm: ["900", "1000", "1100"]
+ llvm: ["9", "10", "11"]
ubuntu: ["20.04", "18.04"]
steps:
- uses: actions/checkout@v2
@@ -37,6 +37,10 @@ jobs:
run: |
docker run --rm docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --version
docker run --rm --entrypoint=mcsema-disass docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --help
+
+ docker run --rm -v $(pwd)/tests/test_suite_generator/generated/prebuilt_cfg/amd64/linux/cfg:/work docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --arch amd64 --os linux --cfg /work/hello_world --output /work/hello_world.bc
+ docker run --rm -v $(pwd)/tests/test_suite_generator/generated/prebuilt_cfg/x86/linux/cfg:/work docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --arch amd64 --os linux --cfg /work/hello_world --output /work/hello_world.bc
+
- name: Push Image for LLVM ${{ matrix.llvm }} on ${{ matrix.ubuntu }}
if: github.event_name == 'push' && github.ref == 'refs/heads/master'
run: |
@@ -45,23 +49,12 @@ jobs:
docker login -u "${DOCKER_HUB_USER}" -p "${DOCKER_HUB_TOKEN}"
docker tag docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest trailofbits/mcsema:llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64
docker push trailofbits/mcsema:llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64
+
env:
GH_USER: ${{ github.actor }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }}
DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }}
- - name: Compress Artifacts LLVM ${{ matrix.llvm }} on ${{ matrix.ubuntu }}
- if: github.event_name == 'push' && github.ref == 'refs/heads/master'
- run: |
- mkdir -p artifacts
- cp -v scripts/emit-artifact.sh ./artifacts/
- docker run -v "$(pwd)/artifacts:/out" --rm --entrypoint /bin/bash docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest -c "/out/emit-artifact.sh /out/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64.tar.xz"
- ls -l artifacts/
- - uses: actions/upload-artifact@v1
- if: github.event_name == 'push' && github.ref == 'refs/heads/master'
- with:
- name: mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64.tar.xz
- path: artifacts/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64.tar.xz
CreateRelease:
needs: [Docker_Linux, VersionFile]
runs-on: ubuntu-latest
diff --git a/CMakeLists.txt b/CMakeLists.txt
index dcddc8075..551b0ed48 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -13,27 +13,22 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
-if (NOT DEFINED ENV{TRAILOFBITS_LIBRARIES})
- message(STATUS "Using new vcpkg build system")
- include(CMakeLists_vcpkg.txt)
- return()
-endif()
+# Find remill first because its config has useful dependency-finding info that
+# needs to be found before the CMake `project` declaration
+find_package(remill COMPONENTS VCPKG_DEPS QUIET)
-project(mcsema)
-cmake_minimum_required(VERSION 3.14)
+include(cmake/vcpkg_helper.cmake)
-# remill
-if("${PLATFORM_NAME}" STREQUAL "windows")
- set(REMILL_FINDPACKAGE_HINTS HINTS "${CMAKE_INSTALL_PREFIX}/remill/lib")
-endif()
+project(mcsema C CXX ASM)
+
+include(GNUInstallDirs)
+cmake_minimum_required(VERSION 3.14)
-find_package(remill REQUIRED ${REMILL_FINDPACKAGE_HINTS})
include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/settings.cmake")
include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/utils.cmake")
include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/BCCompiler.cmake")
include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/ccache.cmake")
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules")
configureCcache()
FindAndSelectClangCompiler()
@@ -54,8 +49,8 @@ if(NOT DEFINED WIN32)
set(PROJECT_CXXFLAGS
${GLOBAL_CXXFLAGS}
-Wconversion
- -pedantic
- -Wno-unreachable-code-return
+ -pedantic
+ -Wno-unreachable-code-return
)
endif()
@@ -63,8 +58,14 @@ endif()
# libraries
#
+find_package(remill CONFIG REQUIRED)
+list(APPEND PROJECT_LIBRARIES remill_settings remill)
+get_target_property(REMILL_INCLUDE_LOCATION remill_settings INTERFACE_INCLUDE_DIRECTORIES)
+
# protobuf
-find_package(Protobuf REQUIRED)
+# Compatibility since we use older protobuf CMake functions
+set(protobuf_MODULE_COMPATIBLE ON CACHE BOOL "" FORCE)
+find_package(Protobuf CONFIG REQUIRED)
list(APPEND PROJECT_LIBRARIES ${Protobuf_LIBRARIES})
list(APPEND PROJECT_INCLUDEDIRECTORIES ${Protobuf_INCLUDE_DIR})
list(APPEND PROJECT_DEFINITIONS "GOOGLE_PROTOBUF_NO_RTTI")
@@ -140,32 +141,15 @@ add_custom_command(
list(APPEND PROJECT_INCLUDEDIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR})
add_dependencies(${MCSEMA_LIFT}
- semantics
protobuf_python_module_ida)
#
# libraries
#
-# remill
-if(NOT TARGET remill)
- if("${PLATFORM_NAME}" STREQUAL "windows")
- set(REMILL_FINDPACKAGE_HINTS HINTS "${CMAKE_INSTALL_PREFIX}/remill/lib")
- endif()
-
- find_package(remill REQUIRED ${REMILL_FINDPACKAGE_HINTS})
-endif()
-
-list(APPEND PROJECT_LIBRARIES remill)
-
# anvill
-if(NOT TARGET anvill-${REMILL_LLVM_VERSION})
- find_package(anvill REQUIRED)
- if(NOT anvill_FOUND)
- message(FATAL_ERROR "McSema depends upon Anvill being installed")
- endif()
-endif()
-list(APPEND PROJECT_LIBRARIES anvill-${REMILL_LLVM_VERSION})
+find_package(anvill CONFIG REQUIRED)
+list(APPEND PROJECT_LIBRARIES anvill)
# mcsema-disass
@@ -191,7 +175,7 @@ set(MCSEMA_PYTHON_SOURCES
if(MCSEMA_INSTALL_PYTHON3_LIBS)
add_custom_target(build_mcsema_disass_python3
DEPENDS ${MCSEMA_PYTHON_SOURCES})
-
+
add_custom_command(
TARGET build_mcsema_disass_python3 POST_BUILD
COMMAND which python3 && python3 setup.py build --force
diff --git a/CMakeLists_vcpkg.txt b/CMakeLists_vcpkg.txt
deleted file mode 100644
index 551b0ed48..000000000
--- a/CMakeLists_vcpkg.txt
+++ /dev/null
@@ -1,275 +0,0 @@
-# Copyright (c) 2020 Trail of Bits, Inc.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see .
-
-# Find remill first because its config has useful dependency-finding info that
-# needs to be found before the CMake `project` declaration
-find_package(remill COMPONENTS VCPKG_DEPS QUIET)
-
-include(cmake/vcpkg_helper.cmake)
-
-project(mcsema C CXX ASM)
-
-include(GNUInstallDirs)
-cmake_minimum_required(VERSION 3.14)
-
-
-include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/settings.cmake")
-include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/utils.cmake")
-include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/BCCompiler.cmake")
-include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/ccache.cmake")
-
-configureCcache()
-FindAndSelectClangCompiler()
-
-set(MCSEMA_SOURCE_DIR "${PROJECT_SOURCE_DIR}")
-
-option(MCSEMA_ENABLE_RUNTIME "Should runtimes for re-compilation of bitcode be produced?" ON)
-
-find_package(Python3 QUIET)
-if(Python3_FOUND)
- option(MCSEMA_INSTALL_PYTHON3_LIBS "Install Python 3 libraries" ON)
-else()
- option(MCSEMA_INSTALL_PYTHON3_LIBS "Install Python 3 libraries")
-endif()
-
-# warnings and compiler settings
-if(NOT DEFINED WIN32)
- set(PROJECT_CXXFLAGS
- ${GLOBAL_CXXFLAGS}
- -Wconversion
- -pedantic
- -Wno-unreachable-code-return
- )
-endif()
-
-#
-# libraries
-#
-
-find_package(remill CONFIG REQUIRED)
-list(APPEND PROJECT_LIBRARIES remill_settings remill)
-get_target_property(REMILL_INCLUDE_LOCATION remill_settings INTERFACE_INCLUDE_DIRECTORIES)
-
-# protobuf
-# Compatibility since we use older protobuf CMake functions
-set(protobuf_MODULE_COMPATIBLE ON CACHE BOOL "" FORCE)
-find_package(Protobuf CONFIG REQUIRED)
-list(APPEND PROJECT_LIBRARIES ${Protobuf_LIBRARIES})
-list(APPEND PROJECT_INCLUDEDIRECTORIES ${Protobuf_INCLUDE_DIR})
-list(APPEND PROJECT_DEFINITIONS "GOOGLE_PROTOBUF_NO_RTTI")
-
-#
-# protobuf file generation
-#
-
-# this function can't be told where to store the output files! we have to add the whole binary directory
-# to the include directories (or change it and lose compatibility with the system libraries)
-protobuf_generate_cpp(PROJECT_PROTOBUFSOURCEFILES
- PROJECT_PROTOBUFHEADERFILES "${CMAKE_CURRENT_SOURCE_DIR}/mcsema/CFG/CFG.proto"
-)
-
-list(APPEND PROJECT_INCLUDEDIRECTORIES ${CMAKE_CURRENT_BINARY_DIR})
-
-protobuf_generate_python(PROJECT_PROTOBUFPYTHONMODULE
- "${CMAKE_CURRENT_SOURCE_DIR}/mcsema/CFG/CFG.proto"
-)
-
-add_custom_target(protobuf_python_module_ida
- DEPENDS ${PROJECT_PROTOBUFPYTHONMODULE}
-)
-
-# disable -Werror on these file since they have been generated
-set_source_files_properties(${PROJECT_PROTOBUFSOURCEFILES} PROPERTIES
- COMPILE_FLAGS "-Wno-sign-conversion -Wno-shorten-64-to-32 -Wno-conversion"
-)
-
-set_source_files_properties(${PROJECT_PROTOBUFHEADERFILES} PROPERTIES
- COMPILE_FLAGS "-Wno-sign-conversion -Wno-shorten-64-to-32 -Wno-conversion"
-)
-
-#
-# target settings
-#
-
-set(MCSEMA_LIFT mcsema-lift-${REMILL_LLVM_VERSION})
-
-# for version information
-add_subdirectory(mcsema/Version)
-
-add_executable(${MCSEMA_LIFT}
- ${PROJECT_PROTOBUFSOURCEFILES}
-
- mcsema/Arch/Arch.cpp
-
- mcsema/CFG/CFG.cpp
-
- mcsema/BC/Callback.cpp
- mcsema/BC/External.cpp
- mcsema/BC/Function.cpp
- mcsema/BC/Instruction.cpp
- mcsema/BC/Legacy.cpp
- mcsema/BC/Lift.cpp
- mcsema/BC/Optimize.cpp
- mcsema/BC/Segment.cpp
- mcsema/BC/Util.cpp
-
- tools/mcsema_lift/Lift.cpp
-)
-
-
-# Copy mcsema-disass in
-add_custom_command(
- TARGET protobuf_python_module_ida POST_BUILD
- DEPENDS {PROJECT_PROTOBUFPYTHONMODULE}
- COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_PROTOBUFPYTHONMODULE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/mcsema_disass/ida7
-)
-
-# this is needed for the #include directives with absolutes paths to work correctly; it must
-# also be set to PUBLIC since mcsema-lift includes some files directly
-list(APPEND PROJECT_INCLUDEDIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR})
-
-add_dependencies(${MCSEMA_LIFT}
- protobuf_python_module_ida)
-
-#
-# libraries
-#
-
-# anvill
-find_package(anvill CONFIG REQUIRED)
-list(APPEND PROJECT_LIBRARIES anvill)
-
-# mcsema-disass
-
-set(MCSEMA_PYTHON_SOURCES
- tools/setup.py
- tools/mcsema_disass/__init__.py
- tools/mcsema_disass/__main__.py
- tools/mcsema_disass/ida7/__init__.py
- tools/mcsema_disass/ida7/anvill_compat.py
- tools/mcsema_disass/ida7/arm_util.py
- tools/mcsema_disass/ida7/CFG_pb2.py
- tools/mcsema_disass/ida7/disass.py
- tools/mcsema_disass/ida7/exception.py
- tools/mcsema_disass/ida7/flow.py
- tools/mcsema_disass/ida7/get_cfg.py
- tools/mcsema_disass/ida7/refs.py
- tools/mcsema_disass/ida7/segment.py
- tools/mcsema_disass/ida7/table.py
- tools/mcsema_disass/ida7/util.py
- tools/mcsema_disass/ida7/x86_util.py
-)
-
-if(MCSEMA_INSTALL_PYTHON3_LIBS)
- add_custom_target(build_mcsema_disass_python3
- DEPENDS ${MCSEMA_PYTHON_SOURCES})
-
- add_custom_command(
- TARGET build_mcsema_disass_python3 POST_BUILD
- COMMAND which python3 && python3 setup.py build --force
- COMMENT "Building McSema Python 3 mcsema-disass"
- WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/tools")
-
- add_dependencies(${MCSEMA_LIFT}
- build_mcsema_disass_python3)
-
- add_dependencies(build_mcsema_disass_python3
- protobuf_python_module_ida)
-endif()
-
-#
-# target settings
-#
-
-target_link_libraries(${MCSEMA_LIFT} PRIVATE ${PROJECT_LIBRARIES} McSemaVersion)
-target_include_directories(${MCSEMA_LIFT} SYSTEM PUBLIC ${PROJECT_INCLUDEDIRECTORIES})
-target_compile_definitions(${MCSEMA_LIFT} PUBLIC ${PROJECT_DEFINITIONS})
-target_compile_options(${MCSEMA_LIFT} PRIVATE ${PROJECT_CXXFLAGS})
-
-if("${CMAKE_HOST_SYSTEM_PROCESSOR}" STREQUAL "x86_64" AND MCSEMA_ENABLE_RUNTIME)
- add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/mcsema/Arch/X86/Runtime)
-endif()
-
-#TODO(artem): this may need some logic to select only ABIs compatible with current os/arch
-#TODO(kumarak): Disable the ABI libraries build till we don't have script to automate the generation of `ABI_libc.h`. Use pre-build library to test the --abi_libraries flag
-function(GetABILibraryList)
- file(GLOB abi_library_list RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/mcsema/OS" "${CMAKE_CURRENT_SOURCE_DIR}/mcsema/OS/*")
- set(GetABILibraryList_Output ${abi_library_list} PARENT_SCOPE)
-endfunction()
-
-if(NOT MCSEMA_ABI_LIBRARY_LIST_INITIALIZED)
- GetABILibraryList()
- set(MCSEMA_DISABLED_ABI_LIBRARIES ${GetABILibraryList_Output} CACHE
- STRING "A semicolon-separated list of disabled ABI libraries"
- )
-
- set(MCSEMA_ABI_LIBRARY_LIST_INITIALIZED ON CACHE INTERNAL
- "Whether the abi library list has been initialized or not"
- )
-endif()
-
-message(STATUS "Disabled ABI libraries: ${MCSEMA_DISABLED_ABI_LIBRARIES}")
-message(STATUS "You can change this setting with -DMCSEMA_DISABLED_ABI_LIBRARIES:STRING=\"Name1;Name2\"")
-
-GetABILibraryList()
-foreach(abi_library_name ${GetABILibraryList_Output})
- set(abi_library_path "mcsema/OS/${abi_library_name}")
-
- list(FIND MCSEMA_DISABLED_ABI_LIBRARIES "${abi_library_name}" abi_lib_index)
- if(NOT ${abi_lib_index} EQUAL -1)
- message(STATUS "Skipping ABI library: ${abi_library_path}")
- continue()
- endif()
-
- message(STATUS "Adding ABI library: ${abi_library_path}")
- add_subdirectory("${abi_library_path}")
-endforeach()
-
-if(DEFINED WIN32)
- set(install_folder "${CMAKE_INSTALL_PREFIX}/mcsema")
-else()
- set(install_folder "${CMAKE_INSTALL_PREFIX}")
-endif()
-
-install(
- TARGETS ${MCSEMA_LIFT}
- RUNTIME DESTINATION "${install_folder}/bin"
- LIBRARY DESTINATION "${install_folder}/lib"
-)
-
-# target for dyninst frontend
-if (DEFINED BUILD_MCSEMA_DYNINST_DISASS)
- if(${BUILD_MCSEMA_DYNINST_DISASS} EQUAL 1)
- add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/tools/mcsema_disass/dyninst)
- endif()
-endif()
-
-set(python_package_installer "${CMAKE_CURRENT_SOURCE_DIR}/tools/setup_launcher")
-if(DEFINED WIN32)
- string(REPLACE "/" "\\" python_package_install_path "${install_folder}")
- set(python_package_installer3 "${python_package_installer}.bat")
- set(optional_interpreter "cmd.exe /C")
-else()
- set(python_package_install_path "${install_folder}")
- set(python_package_installer3 "${python_package_installer}_py3.sh")
-endif()
-
-if(MCSEMA_INSTALL_PYTHON3_LIBS)
- install(CODE
- "execute_process(COMMAND ${optional_interpreter} \"${python_package_installer3}\" \"${python_package_install_path}\"\nWORKING_DIRECTORY \"${PROJECT_SOURCE_DIR}/tools\"\nRESULT_VARIABLE exit_code3)\n if(NOT exit_code3 EQUAL 0)\n message(FATAL_ERROR \"Failed to install the Python 3 package\")\n endif()")
-endif()
-
-add_subdirectory(examples)
-
diff --git a/Dockerfile b/Dockerfile
index 039a4b6ea..5bff01984 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,9 +1,9 @@
-ARG LLVM_VERSION=1000
+ARG LLVM_VERSION=11
ARG ARCH=amd64
ARG UBUNTU_VERSION=18.04
ARG DISTRO_BASE=ubuntu${UBUNTU_VERSION}
ARG BUILD_BASE=ubuntu:${UBUNTU_VERSION}
-ARG LIBRARIES=/opt/trailofbits/libraries
+ARG LIBRARIES=/opt/trailofbits
# Using this file:
# 1. Clone the mcsema repo https://github.com/lifting-bits/mcsema
@@ -18,40 +18,44 @@ FROM ${BUILD_BASE} as base
ARG UBUNTU_VERSION
ARG LIBRARIES
RUN apt-get update && \
- apt-get install -qqy --no-install-recommends python3 python3-pip python3-setuptools python3-six zlib1g curl ca-certificates && \
- if [ "${UBUNTU_VERSION}" = "18.04" ] ; then \
- apt-get install -qqy --no-install-recommends libtinfo5 ; \
- else \
- apt-get install -qqy --no-install-recommends libtinfo6 ; \
- fi && \
+ apt-get install -qqy --no-install-recommends python3 python3-pip python3-setuptools python3-six python3.8 zlib1g curl ca-certificates && \
rm -rf /var/lib/apt/lists/*
-# Will copy anvill installation from here
-FROM trailofbits/anvill:llvm${LLVM_VERSION}-${DISTRO_BASE}-${ARCH} as anvill
-
-
# Build-time dependencies go here
-FROM trailofbits/cxx-common:llvm${LLVM_VERSION}-${DISTRO_BASE}-${ARCH} as deps
+FROM trailofbits/cxx-common-vcpkg-builder-ubuntu:${UBUNTU_VERSION} as deps
+ARG UBUNTU_VERSION
+ARG ARCH
+ARG LLVM_VERSION
ARG LIBRARIES
RUN apt-get update && \
- apt-get install -qqy python3 python3-pip libc6-dev wget liblzma-dev zlib1g-dev libtinfo-dev curl git build-essential ninja-build libselinux1-dev libbsd-dev ccache && \
+ apt-get install -qqy python3 python3-pip libc6-dev wget liblzma-dev zlib1g-dev curl git build-essential ninja-build libselinux1-dev libbsd-dev ccache pixz xz-utils make rpm && \
if [ "$(uname -m)" = "x86_64" ]; then dpkg --add-architecture i386 && apt-get update && apt-get install -qqy gcc-multilib g++-multilib zip zlib1g-dev:i386; fi && \
rm -rf /var/lib/apt/lists/* && \
pip3 install ccsyspath
-COPY --from=anvill /opt/trailofbits/remill /opt/trailofbits/remill
-COPY --from=anvill /opt/trailofbits/anvill /opt/trailofbits/anvill
+# Build dependencies
+RUN git clone --depth=1 --branch master https://github.com/lifting-bits/remill.git && \
+ cd remill && \
+ ./scripts/build.sh --llvm-version ${LLVM_VERSION} --prefix ${LIBRARIES} --download-dir /tmp
-ENV PATH="${LIBRARIES}/llvm/bin:${LIBRARIES}/cmake/bin:${LIBRARIES}/protobuf/bin:${PATH}" \
- CC="${LIBRARIES}/llvm/bin/clang" \
- CXX="${LIBRARIES}/llvm/bin/clang++" \
- TRAILOFBITS_LIBRARIES="${LIBRARIES}"
+# Make this a separate RUN because the build script above downloads a lot
+RUN cd remill && \
+ cmake --build remill-build --target install -- -j "$(nproc)" && \
+ cd ../ && \
+ git clone --depth=1 --branch master https://github.com/lifting-bits/anvill.git && \
+ mkdir -p anvill/build && cd anvill/build && \
+ cmake -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_INSTALL_PREFIX=${LIBRARIES} -Dremill_DIR=${LIBRARIES}/lib/cmake/remill -DVCPKG_ROOT=/tmp/vcpkg_ubuntu-${UBUNTU_VERSION}_llvm-${LLVM_VERSION}_${ARCH} .. && \
+ cmake --build . --target install -- -j "$(nproc)"
WORKDIR /mcsema
# Source code build
FROM deps as build
+ARG UBUNTU_VERSION
+ARG ARCH
+ARG LLVM_VERSION
+ARG LIBRARIES
COPY . ./
@@ -59,36 +63,35 @@ COPY . ./
# version directory since we don't know exactly which Python3 version Ubutnu
# ships with to set the environment variable PYTHONPATH in dist image
RUN mkdir -p ./build && cd ./build && \
- cmake -G Ninja -DCMAKE_PREFIX_PATH="/opt/trailofbits/remill;/opt/trailofbits/anvill" -DMCSEMA_DISABLED_ABI_LIBRARIES:STRING="" -DCMAKE_VERBOSE_MAKEFILE=True -DCMAKE_INSTALL_PREFIX=/opt/trailofbits/mcsema .. && \
- cmake --build . --target install && \
- mv "/opt/trailofbits/mcsema/lib/python$(python3 --version 2>&1 | awk '{ print $2 }' | cut -d '.' -f 1-2)" /opt/trailofbits/mcsema/lib/python3
+ cmake -G Ninja -Danvill_DIR=${LIBRARIES}/lib/cmake/anvill -Dremill_DIR=${LIBRARIES}/lib/cmake/remill -DMCSEMA_DISABLED_ABI_LIBRARIES:STRING="" -DCMAKE_VERBOSE_MAKEFILE=True -DVCPKG_ROOT=/tmp/vcpkg_ubuntu-${UBUNTU_VERSION}_llvm-${LLVM_VERSION}_${ARCH} -DCMAKE_INSTALL_PREFIX=${LIBRARIES} .. && \
+ cmake --build . --target install
+RUN mv ${LIBRARIES}/lib/python3.* ${LIBRARIES}/lib/python3
# WORKDIR tests/test_suite_generator
# RUN mkdir -p build && \
# cd build && \
-# cmake -DMCSEMALIFT_PATH=/opt/trailofbits/mcsema/bin \
+# cmake -DMCSEMALIFT_PATH=/opt/trailofbits/bin \
# -DMCSEMA_PREBUILT_CFG_PATH="$(pwd)/../generated/prebuilt_cfg/" \
-# -DMCSEMADISASS_PATH=/opt/trailofbits/mcsema/bin \
+# -DMCSEMADISASS_PATH=/opt/trailofbits/bin \
# .. && \
# cmake --build . --target install
-#
+#
# RUN cd test_suite && \
-# PATH="/opt/trailofbits/mcsema/bin:${PATH}" python3 start.py
+# PATH="/opt/trailofbits/bin:${PATH}" python3 start.py
FROM base as dist
+ARG LIBRARIES
ARG LLVM_VERSION
# Allow for mounting of local folder
RUN mkdir -p /mcsema/local
-COPY --from=build /opt/trailofbits/remill /opt/trailofbits/remill
-COPY --from=build /opt/trailofbits/anvill /opt/trailofbits/anvill
-COPY --from=build /opt/trailofbits/mcsema /opt/trailofbits/mcsema
-COPY scripts/docker-lifter-entrypoint.sh /opt/trailofbits/mcsema
+COPY --from=build ${LIBRARIES} ${LIBRARIES}
+COPY scripts/docker-lifter-entrypoint.sh ${LIBRARIES}
ENV LLVM_VERSION=llvm${LLVM_VERSION} \
- PATH="/opt/trailofbits/mcsema/bin:${PATH}" \
- PYTHONPATH="/opt/trailofbits/mcsema/lib/python3/site-packages"
-ENTRYPOINT ["/opt/trailofbits/mcsema/docker-lifter-entrypoint.sh"]
+ PATH="${LIBRARIES}/bin:${PATH}" \
+ PYTHONPATH="${LIBRARIES}/lib/python3/site-packages"
+ENTRYPOINT ["/opt/trailofbits/docker-lifter-entrypoint.sh"]
################################
# Left to reader to install #
diff --git a/README.md b/README.md
index 14d4ba3bd..bc2188c66 100644
--- a/README.md
+++ b/README.md
@@ -135,6 +135,9 @@ sudo apt-get install \
cmake \
python3 python3-pip python3-virtualenv \
wget \
+ xz-utils pixz \
+ clang \
+ rpm \
build-essential \
gcc-multilib g++-multilib \
libtinfo-dev \
@@ -144,24 +147,6 @@ sudo apt-get install \
ccache
```
-##### Ubuntu 14.04 and Ubuntu 16.04
-
-```bash
-sudo apt-get install realpath
-```
-
-##### Ubuntu 18.04
-
-```bash
-sudo apt-get install -qqy --no-install-recommends libtinfo5
-```
-
-##### Ubuntu 20.04
-
-```bash
-sudo apt-get install -qqy --no-install-recommends libtinfo6
-```
-
#### macOS pre-requisites
Download and install the [Homebrew](https://brew.sh) package manager.
@@ -206,47 +191,44 @@ git clone --depth 1 --single-branch --branch master https://github.com/lifting-b
git clone --depth 1 --single-branch --branch master https://github.com/lifting-bits/anvill.git
git clone --depth 1 --single-branch --branch master https://github.com/lifting-bits/mcsema.git
-if [ -z "${VIRTUAL_ENV}" ]
-then
- # no virtualenv; global install for all users requires sudo
- function make_install { ; sudo make install ; }
-else
- # found a virtualenv; local install does not need root
- function make_install { ; make install ; }
-fi
+export CC="$(which clang)"
+export CXX="$(which clang++)"
# Download cxx-common, build Remill.
-./remill/scripts/build.sh --llvm-version 9.0
-cd remill-build
-make_install
-
-export TRAILOFBITS_LIBRARIES=`pwd`/remill-build/libraries
+./remill/scripts/build.sh --llvm-version 9 --download-dir ./
+pushd remill-build
+sudo cmake --build . --target install
+popd
# Build and install Anvill
mkdir anvill-build
pushd anvill-build
-${TRAILOFBITS_LIBRARIES}/cmake/bin/cmake ../anvill
-make
-make_install
+# Set VCPKG_ROOT to whatever directory the remill script downloaded
+cmake -DVCPKG_ROOT=$(pwd)/../vcpkg_ubuntu-20.04_llvm-9_amd64 ../anvill
+sudo cmake --build . --target install
popd
# Build and install McSema
mkdir mcsema-build
pushd mcsema-build
-${TRAILOFBITS_LIBRARIES}/cmake/bin/cmake ../mcsema
-make
-make_install
+# Set VCPKG_ROOT to whatever directory the remill script downloaded
+cmake -DVCPKG_ROOT=$(pwd)/../vcpkg_ubuntu-20.04_llvm-9_amd64 ../mcsema
+sudo cmake --build . --target install
+
+# NOTE: Might need to do this for Python package
+pip install ../mcsema/tools
+
popd
```
-Once installed, you may use `mcsema-disass` for disassembling binaries, and `mcsema-lift-4.0` for lifting the disassembled binaries. If you specified `--llvm-version 3.6` to the `build.sh` script, then you would use `mcsema-lift-3.6`.
+Once installed, you may use `mcsema-disass` for disassembling binaries, and `mcsema-lift-9.0` for lifting the disassembled binaries. If you specified `--llvm-version 9` to the `build.sh` script, then you would use `mcsema-lift-9.0`.
#### Step 3: Verifying Your McSema Installation
-Step 2 specified `--llvm-version 9.0` to Remill's `build.sh` script. This means
-that Remill, Anvill, and McSema have all been built against a copy of LLVM 9.0.
+Step 2 specified `--llvm-version 9` to Remill's `build.sh` script. This means
+that Remill, Anvill, and McSema have all been built against a copy of LLVM 9.
To enable you to use multiple LLVM versions simultaneously, we suffix our binaries
-with the LLVM version. Thus, you may use `mcsema-lift-9.0` to lift to LLVM 9.0 bitcode.
+with the LLVM version. Thus, you may use `mcsema-lift-9.0` to lift to LLVM 9 bitcode.
Try running `mcsema-lift-9.0 --version` to see if McSema has been installed.
diff --git a/mcsema/OS/Linux/X86/CMakeLists.txt b/mcsema/OS/Linux/X86/CMakeLists.txt
index 7d44c72fe..201b78643 100644
--- a/mcsema/OS/Linux/X86/CMakeLists.txt
+++ b/mcsema/OS/Linux/X86/CMakeLists.txt
@@ -53,31 +53,34 @@ function(add_runtime_helper target_name source_file address_bit_size)
)
endfunction()
+find_package(Clang CONFIG REQUIRED)
+get_target_property(CLANG_EXE clang LOCATION)
+
add_custom_command(
OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.c ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.h
DEPENDS ${source_file}
- COMMAND env TRAILOFBITS_LIBRARIES=${CXX_COMMON_REPOSITORY_ROOT} python3 ${CMAKE_CURRENT_SOURCE_DIR}/../generate_abi_wrapper.py
- --arch "amd64" --type "c" --input ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.pph --output ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.c
+ COMMAND env CLANG_EXE="${CLANG_EXE}" python3 ${CMAKE_CURRENT_SOURCE_DIR}/../generate_abi_wrapper.py
+ --arch "amd64" --type "c" --input ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.pph --output ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.c
)
#add_custom_command(
# OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc_x86.c ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.h
# DEPENDS ${source_file}
-# COMMAND env TRAILOFBITS_LIBRARIES=${CXX_COMMON_REPOSITORY_ROOT} python3 ${CMAKE_CURRENT_SOURCE_DIR}/../generate_abi_wrapper.py
+# COMMAND env CLANG_EXE="${CLANG_EXE}" python3 ${CMAKE_CURRENT_SOURCE_DIR}/../generate_abi_wrapper.py
# --arch "x86" --type "c" --input ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.pph --output ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc_x86.c
#)
#add_custom_command(
# OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libcpp_amd64.c
# DEPENDS ${source_file}
-# COMMAND env TRAILOFBITS_LIBRARIES=${CXX_COMMON_REPOSITORY_ROOT} python ${CMAKE_CURRENT_SOURCE_DIR}/../generate_abi_wrapper.py
-# --arch "amd64" --type "cpp" --input ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.pph --output ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libcpp_amd64.c
+# COMMAND env CLANG_EXE="${CLANG_EXE}" python ${CMAKE_CURRENT_SOURCE_DIR}/../generate_abi_wrapper.py
+# --arch "amd64" --type "cpp" --input ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.pph --output ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libcpp_amd64.c
#)
#add_custom_command(
# OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libcpp_x86.c
# DEPENDS ${source_file}
-# COMMAND env TRAILOFBITS_LIBRARIES=${CXX_COMMON_REPOSITORY_ROOT} python ${CMAKE_CURRENT_SOURCE_DIR}/../generate_abi_wrapper.py
+# COMMAND env CLANG_EXE="${CLANG_EXE}" python ${CMAKE_CURRENT_SOURCE_DIR}/../generate_abi_wrapper.py
# --arch "x86" --type "cpp" --input ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.pph --output ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libcpp_x86.c
#)
diff --git a/mcsema/OS/Linux/generate_abi_wrapper.py b/mcsema/OS/Linux/generate_abi_wrapper.py
index a89dfed44..4f1e3854d 100644
--- a/mcsema/OS/Linux/generate_abi_wrapper.py
+++ b/mcsema/OS/Linux/generate_abi_wrapper.py
@@ -20,8 +20,8 @@
from collections import defaultdict
-tob_path = os.environ['TRAILOFBITS_LIBRARIES']
-cc_path = tob_path + "/llvm/bin/clang"
+cc_path = os.environ['CLANG_EXE']
+assert cc_path, "Please specify environment variable 'CLANG_EXE' with path to clang executable"
try:
import ccsyspath
diff --git a/scripts/docker-lifter-entrypoint.sh b/scripts/docker-lifter-entrypoint.sh
index a0a503616..5719866a3 100755
--- a/scripts/docker-lifter-entrypoint.sh
+++ b/scripts/docker-lifter-entrypoint.sh
@@ -18,7 +18,7 @@
# Needed to process multiple arguments to docker image
V=""
-case ${LLVM_VERSION} in
+case ${LLVM_VERSION} in
llvm35*)
V=3.5
;;
@@ -35,28 +35,28 @@ case ${LLVM_VERSION} in
V=3.9
;;
# There is an llvm401 that we treat as 4.0
- llvm40*)
+ llvm4*)
V=4.0
;;
- llvm50*)
+ llvm5*)
V=5.0
;;
- llvm60*)
+ llvm6*)
V=6.0
;;
- llvm70*)
+ llvm7*)
V=7.0
;;
- llvm80*)
+ llvm8*)
V=8.0
;;
- llvm90*)
+ llvm9*)
V=9.0
;;
- llvm100*)
+ llvm10*)
V=10.0
;;
- llvm110*)
+ llvm11*)
V=11.0
;;
*)
diff --git a/scripts/emit-artifact.sh b/scripts/emit-artifact.sh
deleted file mode 100755
index 6d92dd153..000000000
--- a/scripts/emit-artifact.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-
-# This script is run inside Github Actions CI to create an archive of a fully-built project
-
-# install pixz for parallel xz
-apt-get update
-apt-get install -yqq pixz
-# compress /opt/trailofbits/{mcsema,remill} and emit it to $1
-echo "Compressing to: ${1}"
-tar -Ipixz -cf "${1}" -C /opt/trailofbits mcsema remill
\ No newline at end of file