diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2a3da0ae..3d4daa092 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: needs: [VersionFile] strategy: matrix: - llvm: ["900", "1000", "1100"] + llvm: ["9", "10", "11"] ubuntu: ["20.04", "18.04"] steps: - uses: actions/checkout@v2 @@ -37,6 +37,10 @@ jobs: run: | docker run --rm docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --version docker run --rm --entrypoint=mcsema-disass docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --help + + docker run --rm -v $(pwd)/tests/test_suite_generator/generated/prebuilt_cfg/amd64/linux/cfg:/work docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --arch amd64 --os linux --cfg /work/hello_world --output /work/hello_world.bc + docker run --rm -v $(pwd)/tests/test_suite_generator/generated/prebuilt_cfg/x86/linux/cfg:/work docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --arch amd64 --os linux --cfg /work/hello_world --output /work/hello_world.bc + - name: Push Image for LLVM ${{ matrix.llvm }} on ${{ matrix.ubuntu }} if: github.event_name == 'push' && github.ref == 'refs/heads/master' run: | @@ -45,23 +49,12 @@ jobs: docker login -u "${DOCKER_HUB_USER}" -p "${DOCKER_HUB_TOKEN}" docker tag docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest trailofbits/mcsema:llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64 docker push trailofbits/mcsema:llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64 + env: GH_USER: ${{ github.actor }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }} DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }} - - name: Compress Artifacts LLVM ${{ matrix.llvm }} on ${{ matrix.ubuntu }} - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - run: | - mkdir -p artifacts - cp -v scripts/emit-artifact.sh ./artifacts/ - docker run -v "$(pwd)/artifacts:/out" --rm --entrypoint /bin/bash docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest -c "/out/emit-artifact.sh /out/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64.tar.xz" - ls -l artifacts/ - - uses: actions/upload-artifact@v1 - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - with: - name: mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64.tar.xz - path: artifacts/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64.tar.xz CreateRelease: needs: [Docker_Linux, VersionFile] runs-on: ubuntu-latest diff --git a/CMakeLists.txt b/CMakeLists.txt index dcddc8075..551b0ed48 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,27 +13,22 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -if (NOT DEFINED ENV{TRAILOFBITS_LIBRARIES}) - message(STATUS "Using new vcpkg build system") - include(CMakeLists_vcpkg.txt) - return() -endif() +# Find remill first because its config has useful dependency-finding info that +# needs to be found before the CMake `project` declaration +find_package(remill COMPONENTS VCPKG_DEPS QUIET) -project(mcsema) -cmake_minimum_required(VERSION 3.14) +include(cmake/vcpkg_helper.cmake) -# remill -if("${PLATFORM_NAME}" STREQUAL "windows") - set(REMILL_FINDPACKAGE_HINTS HINTS "${CMAKE_INSTALL_PREFIX}/remill/lib") -endif() +project(mcsema C CXX ASM) + +include(GNUInstallDirs) +cmake_minimum_required(VERSION 3.14) -find_package(remill REQUIRED ${REMILL_FINDPACKAGE_HINTS}) include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/settings.cmake") include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/utils.cmake") include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/BCCompiler.cmake") include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/ccache.cmake") -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules") configureCcache() FindAndSelectClangCompiler() @@ -54,8 +49,8 @@ if(NOT DEFINED WIN32) set(PROJECT_CXXFLAGS ${GLOBAL_CXXFLAGS} -Wconversion - -pedantic - -Wno-unreachable-code-return + -pedantic + -Wno-unreachable-code-return ) endif() @@ -63,8 +58,14 @@ endif() # libraries # +find_package(remill CONFIG REQUIRED) +list(APPEND PROJECT_LIBRARIES remill_settings remill) +get_target_property(REMILL_INCLUDE_LOCATION remill_settings INTERFACE_INCLUDE_DIRECTORIES) + # protobuf -find_package(Protobuf REQUIRED) +# Compatibility since we use older protobuf CMake functions +set(protobuf_MODULE_COMPATIBLE ON CACHE BOOL "" FORCE) +find_package(Protobuf CONFIG REQUIRED) list(APPEND PROJECT_LIBRARIES ${Protobuf_LIBRARIES}) list(APPEND PROJECT_INCLUDEDIRECTORIES ${Protobuf_INCLUDE_DIR}) list(APPEND PROJECT_DEFINITIONS "GOOGLE_PROTOBUF_NO_RTTI") @@ -140,32 +141,15 @@ add_custom_command( list(APPEND PROJECT_INCLUDEDIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}) add_dependencies(${MCSEMA_LIFT} - semantics protobuf_python_module_ida) # # libraries # -# remill -if(NOT TARGET remill) - if("${PLATFORM_NAME}" STREQUAL "windows") - set(REMILL_FINDPACKAGE_HINTS HINTS "${CMAKE_INSTALL_PREFIX}/remill/lib") - endif() - - find_package(remill REQUIRED ${REMILL_FINDPACKAGE_HINTS}) -endif() - -list(APPEND PROJECT_LIBRARIES remill) - # anvill -if(NOT TARGET anvill-${REMILL_LLVM_VERSION}) - find_package(anvill REQUIRED) - if(NOT anvill_FOUND) - message(FATAL_ERROR "McSema depends upon Anvill being installed") - endif() -endif() -list(APPEND PROJECT_LIBRARIES anvill-${REMILL_LLVM_VERSION}) +find_package(anvill CONFIG REQUIRED) +list(APPEND PROJECT_LIBRARIES anvill) # mcsema-disass @@ -191,7 +175,7 @@ set(MCSEMA_PYTHON_SOURCES if(MCSEMA_INSTALL_PYTHON3_LIBS) add_custom_target(build_mcsema_disass_python3 DEPENDS ${MCSEMA_PYTHON_SOURCES}) - + add_custom_command( TARGET build_mcsema_disass_python3 POST_BUILD COMMAND which python3 && python3 setup.py build --force diff --git a/CMakeLists_vcpkg.txt b/CMakeLists_vcpkg.txt deleted file mode 100644 index 551b0ed48..000000000 --- a/CMakeLists_vcpkg.txt +++ /dev/null @@ -1,275 +0,0 @@ -# Copyright (c) 2020 Trail of Bits, Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# Find remill first because its config has useful dependency-finding info that -# needs to be found before the CMake `project` declaration -find_package(remill COMPONENTS VCPKG_DEPS QUIET) - -include(cmake/vcpkg_helper.cmake) - -project(mcsema C CXX ASM) - -include(GNUInstallDirs) -cmake_minimum_required(VERSION 3.14) - - -include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/settings.cmake") -include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/utils.cmake") -include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/BCCompiler.cmake") -include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/ccache.cmake") - -configureCcache() -FindAndSelectClangCompiler() - -set(MCSEMA_SOURCE_DIR "${PROJECT_SOURCE_DIR}") - -option(MCSEMA_ENABLE_RUNTIME "Should runtimes for re-compilation of bitcode be produced?" ON) - -find_package(Python3 QUIET) -if(Python3_FOUND) - option(MCSEMA_INSTALL_PYTHON3_LIBS "Install Python 3 libraries" ON) -else() - option(MCSEMA_INSTALL_PYTHON3_LIBS "Install Python 3 libraries") -endif() - -# warnings and compiler settings -if(NOT DEFINED WIN32) - set(PROJECT_CXXFLAGS - ${GLOBAL_CXXFLAGS} - -Wconversion - -pedantic - -Wno-unreachable-code-return - ) -endif() - -# -# libraries -# - -find_package(remill CONFIG REQUIRED) -list(APPEND PROJECT_LIBRARIES remill_settings remill) -get_target_property(REMILL_INCLUDE_LOCATION remill_settings INTERFACE_INCLUDE_DIRECTORIES) - -# protobuf -# Compatibility since we use older protobuf CMake functions -set(protobuf_MODULE_COMPATIBLE ON CACHE BOOL "" FORCE) -find_package(Protobuf CONFIG REQUIRED) -list(APPEND PROJECT_LIBRARIES ${Protobuf_LIBRARIES}) -list(APPEND PROJECT_INCLUDEDIRECTORIES ${Protobuf_INCLUDE_DIR}) -list(APPEND PROJECT_DEFINITIONS "GOOGLE_PROTOBUF_NO_RTTI") - -# -# protobuf file generation -# - -# this function can't be told where to store the output files! we have to add the whole binary directory -# to the include directories (or change it and lose compatibility with the system libraries) -protobuf_generate_cpp(PROJECT_PROTOBUFSOURCEFILES - PROJECT_PROTOBUFHEADERFILES "${CMAKE_CURRENT_SOURCE_DIR}/mcsema/CFG/CFG.proto" -) - -list(APPEND PROJECT_INCLUDEDIRECTORIES ${CMAKE_CURRENT_BINARY_DIR}) - -protobuf_generate_python(PROJECT_PROTOBUFPYTHONMODULE - "${CMAKE_CURRENT_SOURCE_DIR}/mcsema/CFG/CFG.proto" -) - -add_custom_target(protobuf_python_module_ida - DEPENDS ${PROJECT_PROTOBUFPYTHONMODULE} -) - -# disable -Werror on these file since they have been generated -set_source_files_properties(${PROJECT_PROTOBUFSOURCEFILES} PROPERTIES - COMPILE_FLAGS "-Wno-sign-conversion -Wno-shorten-64-to-32 -Wno-conversion" -) - -set_source_files_properties(${PROJECT_PROTOBUFHEADERFILES} PROPERTIES - COMPILE_FLAGS "-Wno-sign-conversion -Wno-shorten-64-to-32 -Wno-conversion" -) - -# -# target settings -# - -set(MCSEMA_LIFT mcsema-lift-${REMILL_LLVM_VERSION}) - -# for version information -add_subdirectory(mcsema/Version) - -add_executable(${MCSEMA_LIFT} - ${PROJECT_PROTOBUFSOURCEFILES} - - mcsema/Arch/Arch.cpp - - mcsema/CFG/CFG.cpp - - mcsema/BC/Callback.cpp - mcsema/BC/External.cpp - mcsema/BC/Function.cpp - mcsema/BC/Instruction.cpp - mcsema/BC/Legacy.cpp - mcsema/BC/Lift.cpp - mcsema/BC/Optimize.cpp - mcsema/BC/Segment.cpp - mcsema/BC/Util.cpp - - tools/mcsema_lift/Lift.cpp -) - - -# Copy mcsema-disass in -add_custom_command( - TARGET protobuf_python_module_ida POST_BUILD - DEPENDS {PROJECT_PROTOBUFPYTHONMODULE} - COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_PROTOBUFPYTHONMODULE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/mcsema_disass/ida7 -) - -# this is needed for the #include directives with absolutes paths to work correctly; it must -# also be set to PUBLIC since mcsema-lift includes some files directly -list(APPEND PROJECT_INCLUDEDIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}) - -add_dependencies(${MCSEMA_LIFT} - protobuf_python_module_ida) - -# -# libraries -# - -# anvill -find_package(anvill CONFIG REQUIRED) -list(APPEND PROJECT_LIBRARIES anvill) - -# mcsema-disass - -set(MCSEMA_PYTHON_SOURCES - tools/setup.py - tools/mcsema_disass/__init__.py - tools/mcsema_disass/__main__.py - tools/mcsema_disass/ida7/__init__.py - tools/mcsema_disass/ida7/anvill_compat.py - tools/mcsema_disass/ida7/arm_util.py - tools/mcsema_disass/ida7/CFG_pb2.py - tools/mcsema_disass/ida7/disass.py - tools/mcsema_disass/ida7/exception.py - tools/mcsema_disass/ida7/flow.py - tools/mcsema_disass/ida7/get_cfg.py - tools/mcsema_disass/ida7/refs.py - tools/mcsema_disass/ida7/segment.py - tools/mcsema_disass/ida7/table.py - tools/mcsema_disass/ida7/util.py - tools/mcsema_disass/ida7/x86_util.py -) - -if(MCSEMA_INSTALL_PYTHON3_LIBS) - add_custom_target(build_mcsema_disass_python3 - DEPENDS ${MCSEMA_PYTHON_SOURCES}) - - add_custom_command( - TARGET build_mcsema_disass_python3 POST_BUILD - COMMAND which python3 && python3 setup.py build --force - COMMENT "Building McSema Python 3 mcsema-disass" - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/tools") - - add_dependencies(${MCSEMA_LIFT} - build_mcsema_disass_python3) - - add_dependencies(build_mcsema_disass_python3 - protobuf_python_module_ida) -endif() - -# -# target settings -# - -target_link_libraries(${MCSEMA_LIFT} PRIVATE ${PROJECT_LIBRARIES} McSemaVersion) -target_include_directories(${MCSEMA_LIFT} SYSTEM PUBLIC ${PROJECT_INCLUDEDIRECTORIES}) -target_compile_definitions(${MCSEMA_LIFT} PUBLIC ${PROJECT_DEFINITIONS}) -target_compile_options(${MCSEMA_LIFT} PRIVATE ${PROJECT_CXXFLAGS}) - -if("${CMAKE_HOST_SYSTEM_PROCESSOR}" STREQUAL "x86_64" AND MCSEMA_ENABLE_RUNTIME) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/mcsema/Arch/X86/Runtime) -endif() - -#TODO(artem): this may need some logic to select only ABIs compatible with current os/arch -#TODO(kumarak): Disable the ABI libraries build till we don't have script to automate the generation of `ABI_libc.h`. Use pre-build library to test the --abi_libraries flag -function(GetABILibraryList) - file(GLOB abi_library_list RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/mcsema/OS" "${CMAKE_CURRENT_SOURCE_DIR}/mcsema/OS/*") - set(GetABILibraryList_Output ${abi_library_list} PARENT_SCOPE) -endfunction() - -if(NOT MCSEMA_ABI_LIBRARY_LIST_INITIALIZED) - GetABILibraryList() - set(MCSEMA_DISABLED_ABI_LIBRARIES ${GetABILibraryList_Output} CACHE - STRING "A semicolon-separated list of disabled ABI libraries" - ) - - set(MCSEMA_ABI_LIBRARY_LIST_INITIALIZED ON CACHE INTERNAL - "Whether the abi library list has been initialized or not" - ) -endif() - -message(STATUS "Disabled ABI libraries: ${MCSEMA_DISABLED_ABI_LIBRARIES}") -message(STATUS "You can change this setting with -DMCSEMA_DISABLED_ABI_LIBRARIES:STRING=\"Name1;Name2\"") - -GetABILibraryList() -foreach(abi_library_name ${GetABILibraryList_Output}) - set(abi_library_path "mcsema/OS/${abi_library_name}") - - list(FIND MCSEMA_DISABLED_ABI_LIBRARIES "${abi_library_name}" abi_lib_index) - if(NOT ${abi_lib_index} EQUAL -1) - message(STATUS "Skipping ABI library: ${abi_library_path}") - continue() - endif() - - message(STATUS "Adding ABI library: ${abi_library_path}") - add_subdirectory("${abi_library_path}") -endforeach() - -if(DEFINED WIN32) - set(install_folder "${CMAKE_INSTALL_PREFIX}/mcsema") -else() - set(install_folder "${CMAKE_INSTALL_PREFIX}") -endif() - -install( - TARGETS ${MCSEMA_LIFT} - RUNTIME DESTINATION "${install_folder}/bin" - LIBRARY DESTINATION "${install_folder}/lib" -) - -# target for dyninst frontend -if (DEFINED BUILD_MCSEMA_DYNINST_DISASS) - if(${BUILD_MCSEMA_DYNINST_DISASS} EQUAL 1) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/tools/mcsema_disass/dyninst) - endif() -endif() - -set(python_package_installer "${CMAKE_CURRENT_SOURCE_DIR}/tools/setup_launcher") -if(DEFINED WIN32) - string(REPLACE "/" "\\" python_package_install_path "${install_folder}") - set(python_package_installer3 "${python_package_installer}.bat") - set(optional_interpreter "cmd.exe /C") -else() - set(python_package_install_path "${install_folder}") - set(python_package_installer3 "${python_package_installer}_py3.sh") -endif() - -if(MCSEMA_INSTALL_PYTHON3_LIBS) - install(CODE - "execute_process(COMMAND ${optional_interpreter} \"${python_package_installer3}\" \"${python_package_install_path}\"\nWORKING_DIRECTORY \"${PROJECT_SOURCE_DIR}/tools\"\nRESULT_VARIABLE exit_code3)\n if(NOT exit_code3 EQUAL 0)\n message(FATAL_ERROR \"Failed to install the Python 3 package\")\n endif()") -endif() - -add_subdirectory(examples) - diff --git a/Dockerfile b/Dockerfile index 039a4b6ea..5bff01984 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ -ARG LLVM_VERSION=1000 +ARG LLVM_VERSION=11 ARG ARCH=amd64 ARG UBUNTU_VERSION=18.04 ARG DISTRO_BASE=ubuntu${UBUNTU_VERSION} ARG BUILD_BASE=ubuntu:${UBUNTU_VERSION} -ARG LIBRARIES=/opt/trailofbits/libraries +ARG LIBRARIES=/opt/trailofbits # Using this file: # 1. Clone the mcsema repo https://github.com/lifting-bits/mcsema @@ -18,40 +18,44 @@ FROM ${BUILD_BASE} as base ARG UBUNTU_VERSION ARG LIBRARIES RUN apt-get update && \ - apt-get install -qqy --no-install-recommends python3 python3-pip python3-setuptools python3-six zlib1g curl ca-certificates && \ - if [ "${UBUNTU_VERSION}" = "18.04" ] ; then \ - apt-get install -qqy --no-install-recommends libtinfo5 ; \ - else \ - apt-get install -qqy --no-install-recommends libtinfo6 ; \ - fi && \ + apt-get install -qqy --no-install-recommends python3 python3-pip python3-setuptools python3-six python3.8 zlib1g curl ca-certificates && \ rm -rf /var/lib/apt/lists/* -# Will copy anvill installation from here -FROM trailofbits/anvill:llvm${LLVM_VERSION}-${DISTRO_BASE}-${ARCH} as anvill - - # Build-time dependencies go here -FROM trailofbits/cxx-common:llvm${LLVM_VERSION}-${DISTRO_BASE}-${ARCH} as deps +FROM trailofbits/cxx-common-vcpkg-builder-ubuntu:${UBUNTU_VERSION} as deps +ARG UBUNTU_VERSION +ARG ARCH +ARG LLVM_VERSION ARG LIBRARIES RUN apt-get update && \ - apt-get install -qqy python3 python3-pip libc6-dev wget liblzma-dev zlib1g-dev libtinfo-dev curl git build-essential ninja-build libselinux1-dev libbsd-dev ccache && \ + apt-get install -qqy python3 python3-pip libc6-dev wget liblzma-dev zlib1g-dev curl git build-essential ninja-build libselinux1-dev libbsd-dev ccache pixz xz-utils make rpm && \ if [ "$(uname -m)" = "x86_64" ]; then dpkg --add-architecture i386 && apt-get update && apt-get install -qqy gcc-multilib g++-multilib zip zlib1g-dev:i386; fi && \ rm -rf /var/lib/apt/lists/* && \ pip3 install ccsyspath -COPY --from=anvill /opt/trailofbits/remill /opt/trailofbits/remill -COPY --from=anvill /opt/trailofbits/anvill /opt/trailofbits/anvill +# Build dependencies +RUN git clone --depth=1 --branch master https://github.com/lifting-bits/remill.git && \ + cd remill && \ + ./scripts/build.sh --llvm-version ${LLVM_VERSION} --prefix ${LIBRARIES} --download-dir /tmp -ENV PATH="${LIBRARIES}/llvm/bin:${LIBRARIES}/cmake/bin:${LIBRARIES}/protobuf/bin:${PATH}" \ - CC="${LIBRARIES}/llvm/bin/clang" \ - CXX="${LIBRARIES}/llvm/bin/clang++" \ - TRAILOFBITS_LIBRARIES="${LIBRARIES}" +# Make this a separate RUN because the build script above downloads a lot +RUN cd remill && \ + cmake --build remill-build --target install -- -j "$(nproc)" && \ + cd ../ && \ + git clone --depth=1 --branch master https://github.com/lifting-bits/anvill.git && \ + mkdir -p anvill/build && cd anvill/build && \ + cmake -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_INSTALL_PREFIX=${LIBRARIES} -Dremill_DIR=${LIBRARIES}/lib/cmake/remill -DVCPKG_ROOT=/tmp/vcpkg_ubuntu-${UBUNTU_VERSION}_llvm-${LLVM_VERSION}_${ARCH} .. && \ + cmake --build . --target install -- -j "$(nproc)" WORKDIR /mcsema # Source code build FROM deps as build +ARG UBUNTU_VERSION +ARG ARCH +ARG LLVM_VERSION +ARG LIBRARIES COPY . ./ @@ -59,36 +63,35 @@ COPY . ./ # version directory since we don't know exactly which Python3 version Ubutnu # ships with to set the environment variable PYTHONPATH in dist image RUN mkdir -p ./build && cd ./build && \ - cmake -G Ninja -DCMAKE_PREFIX_PATH="/opt/trailofbits/remill;/opt/trailofbits/anvill" -DMCSEMA_DISABLED_ABI_LIBRARIES:STRING="" -DCMAKE_VERBOSE_MAKEFILE=True -DCMAKE_INSTALL_PREFIX=/opt/trailofbits/mcsema .. && \ - cmake --build . --target install && \ - mv "/opt/trailofbits/mcsema/lib/python$(python3 --version 2>&1 | awk '{ print $2 }' | cut -d '.' -f 1-2)" /opt/trailofbits/mcsema/lib/python3 + cmake -G Ninja -Danvill_DIR=${LIBRARIES}/lib/cmake/anvill -Dremill_DIR=${LIBRARIES}/lib/cmake/remill -DMCSEMA_DISABLED_ABI_LIBRARIES:STRING="" -DCMAKE_VERBOSE_MAKEFILE=True -DVCPKG_ROOT=/tmp/vcpkg_ubuntu-${UBUNTU_VERSION}_llvm-${LLVM_VERSION}_${ARCH} -DCMAKE_INSTALL_PREFIX=${LIBRARIES} .. && \ + cmake --build . --target install +RUN mv ${LIBRARIES}/lib/python3.* ${LIBRARIES}/lib/python3 # WORKDIR tests/test_suite_generator # RUN mkdir -p build && \ # cd build && \ -# cmake -DMCSEMALIFT_PATH=/opt/trailofbits/mcsema/bin \ +# cmake -DMCSEMALIFT_PATH=/opt/trailofbits/bin \ # -DMCSEMA_PREBUILT_CFG_PATH="$(pwd)/../generated/prebuilt_cfg/" \ -# -DMCSEMADISASS_PATH=/opt/trailofbits/mcsema/bin \ +# -DMCSEMADISASS_PATH=/opt/trailofbits/bin \ # .. && \ # cmake --build . --target install -# +# # RUN cd test_suite && \ -# PATH="/opt/trailofbits/mcsema/bin:${PATH}" python3 start.py +# PATH="/opt/trailofbits/bin:${PATH}" python3 start.py FROM base as dist +ARG LIBRARIES ARG LLVM_VERSION # Allow for mounting of local folder RUN mkdir -p /mcsema/local -COPY --from=build /opt/trailofbits/remill /opt/trailofbits/remill -COPY --from=build /opt/trailofbits/anvill /opt/trailofbits/anvill -COPY --from=build /opt/trailofbits/mcsema /opt/trailofbits/mcsema -COPY scripts/docker-lifter-entrypoint.sh /opt/trailofbits/mcsema +COPY --from=build ${LIBRARIES} ${LIBRARIES} +COPY scripts/docker-lifter-entrypoint.sh ${LIBRARIES} ENV LLVM_VERSION=llvm${LLVM_VERSION} \ - PATH="/opt/trailofbits/mcsema/bin:${PATH}" \ - PYTHONPATH="/opt/trailofbits/mcsema/lib/python3/site-packages" -ENTRYPOINT ["/opt/trailofbits/mcsema/docker-lifter-entrypoint.sh"] + PATH="${LIBRARIES}/bin:${PATH}" \ + PYTHONPATH="${LIBRARIES}/lib/python3/site-packages" +ENTRYPOINT ["/opt/trailofbits/docker-lifter-entrypoint.sh"] ################################ # Left to reader to install # diff --git a/README.md b/README.md index 14d4ba3bd..bc2188c66 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,9 @@ sudo apt-get install \ cmake \ python3 python3-pip python3-virtualenv \ wget \ + xz-utils pixz \ + clang \ + rpm \ build-essential \ gcc-multilib g++-multilib \ libtinfo-dev \ @@ -144,24 +147,6 @@ sudo apt-get install \ ccache ``` -##### Ubuntu 14.04 and Ubuntu 16.04 - -```bash -sudo apt-get install realpath -``` - -##### Ubuntu 18.04 - -```bash -sudo apt-get install -qqy --no-install-recommends libtinfo5 -``` - -##### Ubuntu 20.04 - -```bash -sudo apt-get install -qqy --no-install-recommends libtinfo6 -``` - #### macOS pre-requisites Download and install the [Homebrew](https://brew.sh) package manager. @@ -206,47 +191,44 @@ git clone --depth 1 --single-branch --branch master https://github.com/lifting-b git clone --depth 1 --single-branch --branch master https://github.com/lifting-bits/anvill.git git clone --depth 1 --single-branch --branch master https://github.com/lifting-bits/mcsema.git -if [ -z "${VIRTUAL_ENV}" ] -then - # no virtualenv; global install for all users requires sudo - function make_install { ; sudo make install ; } -else - # found a virtualenv; local install does not need root - function make_install { ; make install ; } -fi +export CC="$(which clang)" +export CXX="$(which clang++)" # Download cxx-common, build Remill. -./remill/scripts/build.sh --llvm-version 9.0 -cd remill-build -make_install - -export TRAILOFBITS_LIBRARIES=`pwd`/remill-build/libraries +./remill/scripts/build.sh --llvm-version 9 --download-dir ./ +pushd remill-build +sudo cmake --build . --target install +popd # Build and install Anvill mkdir anvill-build pushd anvill-build -${TRAILOFBITS_LIBRARIES}/cmake/bin/cmake ../anvill -make -make_install +# Set VCPKG_ROOT to whatever directory the remill script downloaded +cmake -DVCPKG_ROOT=$(pwd)/../vcpkg_ubuntu-20.04_llvm-9_amd64 ../anvill +sudo cmake --build . --target install popd # Build and install McSema mkdir mcsema-build pushd mcsema-build -${TRAILOFBITS_LIBRARIES}/cmake/bin/cmake ../mcsema -make -make_install +# Set VCPKG_ROOT to whatever directory the remill script downloaded +cmake -DVCPKG_ROOT=$(pwd)/../vcpkg_ubuntu-20.04_llvm-9_amd64 ../mcsema +sudo cmake --build . --target install + +# NOTE: Might need to do this for Python package +pip install ../mcsema/tools + popd ``` -Once installed, you may use `mcsema-disass` for disassembling binaries, and `mcsema-lift-4.0` for lifting the disassembled binaries. If you specified `--llvm-version 3.6` to the `build.sh` script, then you would use `mcsema-lift-3.6`. +Once installed, you may use `mcsema-disass` for disassembling binaries, and `mcsema-lift-9.0` for lifting the disassembled binaries. If you specified `--llvm-version 9` to the `build.sh` script, then you would use `mcsema-lift-9.0`. #### Step 3: Verifying Your McSema Installation -Step 2 specified `--llvm-version 9.0` to Remill's `build.sh` script. This means -that Remill, Anvill, and McSema have all been built against a copy of LLVM 9.0. +Step 2 specified `--llvm-version 9` to Remill's `build.sh` script. This means +that Remill, Anvill, and McSema have all been built against a copy of LLVM 9. To enable you to use multiple LLVM versions simultaneously, we suffix our binaries -with the LLVM version. Thus, you may use `mcsema-lift-9.0` to lift to LLVM 9.0 bitcode. +with the LLVM version. Thus, you may use `mcsema-lift-9.0` to lift to LLVM 9 bitcode. Try running `mcsema-lift-9.0 --version` to see if McSema has been installed. diff --git a/mcsema/OS/Linux/X86/CMakeLists.txt b/mcsema/OS/Linux/X86/CMakeLists.txt index 7d44c72fe..201b78643 100644 --- a/mcsema/OS/Linux/X86/CMakeLists.txt +++ b/mcsema/OS/Linux/X86/CMakeLists.txt @@ -53,31 +53,34 @@ function(add_runtime_helper target_name source_file address_bit_size) ) endfunction() +find_package(Clang CONFIG REQUIRED) +get_target_property(CLANG_EXE clang LOCATION) + add_custom_command( OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.c ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.h DEPENDS ${source_file} - COMMAND env TRAILOFBITS_LIBRARIES=${CXX_COMMON_REPOSITORY_ROOT} python3 ${CMAKE_CURRENT_SOURCE_DIR}/../generate_abi_wrapper.py - --arch "amd64" --type "c" --input ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.pph --output ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.c + COMMAND env CLANG_EXE="${CLANG_EXE}" python3 ${CMAKE_CURRENT_SOURCE_DIR}/../generate_abi_wrapper.py + --arch "amd64" --type "c" --input ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.pph --output ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.c ) #add_custom_command( # OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc_x86.c ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.h # DEPENDS ${source_file} -# COMMAND env TRAILOFBITS_LIBRARIES=${CXX_COMMON_REPOSITORY_ROOT} python3 ${CMAKE_CURRENT_SOURCE_DIR}/../generate_abi_wrapper.py +# COMMAND env CLANG_EXE="${CLANG_EXE}" python3 ${CMAKE_CURRENT_SOURCE_DIR}/../generate_abi_wrapper.py # --arch "x86" --type "c" --input ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.pph --output ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc_x86.c #) #add_custom_command( # OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libcpp_amd64.c # DEPENDS ${source_file} -# COMMAND env TRAILOFBITS_LIBRARIES=${CXX_COMMON_REPOSITORY_ROOT} python ${CMAKE_CURRENT_SOURCE_DIR}/../generate_abi_wrapper.py -# --arch "amd64" --type "cpp" --input ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.pph --output ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libcpp_amd64.c +# COMMAND env CLANG_EXE="${CLANG_EXE}" python ${CMAKE_CURRENT_SOURCE_DIR}/../generate_abi_wrapper.py +# --arch "amd64" --type "cpp" --input ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.pph --output ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libcpp_amd64.c #) #add_custom_command( # OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libcpp_x86.c # DEPENDS ${source_file} -# COMMAND env TRAILOFBITS_LIBRARIES=${CXX_COMMON_REPOSITORY_ROOT} python ${CMAKE_CURRENT_SOURCE_DIR}/../generate_abi_wrapper.py +# COMMAND env CLANG_EXE="${CLANG_EXE}" python ${CMAKE_CURRENT_SOURCE_DIR}/../generate_abi_wrapper.py # --arch "x86" --type "cpp" --input ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libc.pph --output ${CMAKE_CURRENT_SOURCE_DIR}/ABI_libcpp_x86.c #) diff --git a/mcsema/OS/Linux/generate_abi_wrapper.py b/mcsema/OS/Linux/generate_abi_wrapper.py index a89dfed44..4f1e3854d 100644 --- a/mcsema/OS/Linux/generate_abi_wrapper.py +++ b/mcsema/OS/Linux/generate_abi_wrapper.py @@ -20,8 +20,8 @@ from collections import defaultdict -tob_path = os.environ['TRAILOFBITS_LIBRARIES'] -cc_path = tob_path + "/llvm/bin/clang" +cc_path = os.environ['CLANG_EXE'] +assert cc_path, "Please specify environment variable 'CLANG_EXE' with path to clang executable" try: import ccsyspath diff --git a/scripts/docker-lifter-entrypoint.sh b/scripts/docker-lifter-entrypoint.sh index a0a503616..5719866a3 100755 --- a/scripts/docker-lifter-entrypoint.sh +++ b/scripts/docker-lifter-entrypoint.sh @@ -18,7 +18,7 @@ # Needed to process multiple arguments to docker image V="" -case ${LLVM_VERSION} in +case ${LLVM_VERSION} in llvm35*) V=3.5 ;; @@ -35,28 +35,28 @@ case ${LLVM_VERSION} in V=3.9 ;; # There is an llvm401 that we treat as 4.0 - llvm40*) + llvm4*) V=4.0 ;; - llvm50*) + llvm5*) V=5.0 ;; - llvm60*) + llvm6*) V=6.0 ;; - llvm70*) + llvm7*) V=7.0 ;; - llvm80*) + llvm8*) V=8.0 ;; - llvm90*) + llvm9*) V=9.0 ;; - llvm100*) + llvm10*) V=10.0 ;; - llvm110*) + llvm11*) V=11.0 ;; *) diff --git a/scripts/emit-artifact.sh b/scripts/emit-artifact.sh deleted file mode 100755 index 6d92dd153..000000000 --- a/scripts/emit-artifact.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -# This script is run inside Github Actions CI to create an archive of a fully-built project - -# install pixz for parallel xz -apt-get update -apt-get install -yqq pixz -# compress /opt/trailofbits/{mcsema,remill} and emit it to $1 -echo "Compressing to: ${1}" -tar -Ipixz -cf "${1}" -C /opt/trailofbits mcsema remill \ No newline at end of file