Skip to content

Commit

Permalink
[wasm] add wasm-wasi build (#10)
Browse files Browse the repository at this point in the history
  • Loading branch information
makslevental authored Nov 14, 2024
1 parent 10299ab commit 40b284c
Show file tree
Hide file tree
Showing 6 changed files with 158 additions and 23 deletions.
32 changes: 22 additions & 10 deletions .github/actions/setup_base/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ inputs:
os:
required: true
description: ''
arch:
required: true
description: ''
cache-key:
required: true
description: ''
Expand Down Expand Up @@ -119,20 +122,29 @@ runs:
fi
echo "python3_command=$python3_command" >> $GITHUB_ENV
- name: Setup Emscripten
if: contains(inputs.arch, 'wasm')
uses: mymindstorm/setup-emsdk@v14
with:
version: 3.1.71

- name: "Set CMake/ccache env vars"
shell: bash
run: |
echo "CCACHE_DIR=${{ steps.canonicalize-cache-dir.outputs.cache-dir }}/ccache" >> $GITHUB_ENV
echo "CCACHE_COMPILERCHECK=string:$($CC --version | head -n 1)" >> $GITHUB_ENV
echo $CCACHE_COMPILERCHECK
echo "CCACHE_MAXSIZE=700M" >> $GITHUB_ENV
echo "CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros" >> $GITHUB_ENV
echo "CCACHE_CPP2=true" >> $GITHUB_ENV
echo "CCACHE_UMASK=002" >> $GITHUB_ENV
echo "CCACHE_DIR=${{ steps.canonicalize-cache-dir.outputs.cache-dir }}/ccache" >> $GITHUB_ENV
echo "CCACHE_COMPILERCHECK=string:$($CC --version | head -n 1)" >> $GITHUB_ENV
echo $CCACHE_COMPILERCHECK
echo "CCACHE_MAXSIZE=700M" >> $GITHUB_ENV
echo "CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros" >> $GITHUB_ENV
echo "CCACHE_CPP2=true" >> $GITHUB_ENV
echo "CCACHE_UMASK=002" >> $GITHUB_ENV
echo "CMAKE_C_COMPILER_LAUNCHER=ccache" >> $GITHUB_ENV
echo "CMAKE_CXX_COMPILER_LAUNCHER=ccache" >> $GITHUB_ENV
echo "CMAKE_C_COMPILER_LAUNCHER=ccache" >> $GITHUB_ENV
echo "CMAKE_CXX_COMPILER_LAUNCHER=ccache" >> $GITHUB_ENV
echo "Python3_EXECUTABLE=$(which $python3_command)" >> $GITHUB_ENV
if [[ "${{ inputs.arch }}" != "wasm32-wasi" ]]; then
echo "CMAKE_EXE_LINKER_FLAGS_INIT=-fuse-ld=lld" >> $GITHUB_ENV
echo "CMAKE_SHARED_LINKER_FLAGS_INIT=-fuse-ld=lld" >> $GITHUB_ENV
echo "CMAKE_MODULE_LINKER_FLAGS_INIT=-fuse-ld=lld" >> $GITHUB_ENV
echo "Python3_EXECUTABLE=$(which $python3_command)" >> $GITHUB_ENV
fi
24 changes: 19 additions & 5 deletions .github/workflows/build_llvm.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,15 +44,24 @@ jobs:
runs-on: "ubuntu-22.04"
container: "quay.io/pypa/manylinux_2_28_x86_64"
os: "almalinux"
arch: x86_64
- name: "wasm_wasi"
runs-on: "ubuntu-22.04"
container: "quay.io/pypa/manylinux_2_28_x86_64"
os: "almalinux"
arch: "wasm32-wasi"
- name: "windows_x86_64"
runs-on: "windows-2019"
os: "windows"
arch: x86_64
- name: "macos_arm64"
runs-on: "macos-14"
os: "macos"
arch: arm64
- name: "macos_x86_64"
runs-on: "macos-13"
os: "macos"
arch: x86_64

runs-on: ${{ matrix.runs-on }}

Expand Down Expand Up @@ -86,6 +95,7 @@ jobs:
cache-key: ${{ env.cache-key }}
restore-key: "mlir_${{ matrix.name }}_clang"
os: ${{ matrix.os }}
arch: ${{ matrix.arch }}

- name: "Build LLVM and MLIR"
id: build
Expand All @@ -97,12 +107,16 @@ jobs:
ccache -z
$python3_command -m pip install -r third_party/llvm-project/mlir/python/requirements.txt
build_tools/cmake/build_llvm.sh
ccache -s
echo "*********************** SMOKE TEST *********************************"
"$LLVM_INSTALL_DIR/bin/mlir-tblgen" --version
echo "*********************** SMOKE TEST *********************************"
if [[ "${{ matrix.arch }}" == "wasm32-wasi" ]]; then
build_tools/build_llvm_wasi.sh
else
build_tools/build_llvm.sh
echo "*********************** SMOKE TEST *********************************"
"$LLVM_INSTALL_DIR/bin/mlir-tblgen" --version
echo "*********************** SMOKE TEST *********************************"
fi
ccache -s
pushd $LLVM_SOURCE_DIR && LLVM_SHA_SHORT=$(git rev-parse --short HEAD) && popd
tar -czf mlir_${{ matrix.name }}_$(date +'%Y%m%d')_$LLVM_SHA_SHORT.tar.gz -C "$LLVM_INSTALL_DIR/.." llvm-install
Expand Down
7 changes: 2 additions & 5 deletions build_tools/cmake/build_llvm.sh → build_tools/build_llvm.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/usr/bin/env bash

TD="$(cd $(dirname $0) && pwd)"
REPO_ROOT="$(cd $TD/../.. && pwd)"
REPO_ROOT="$(cd $TD/.. && pwd)"

LLVM_SOURCE_DIR="${LLVM_SOURCE_DIR:-${REPO_ROOT}/third_party/llvm-project}"
LLVM_BUILD_DIR="${LLVM_BUILD_DIR:-${REPO_ROOT}/llvm-build}"
Expand All @@ -19,9 +19,6 @@ echo "LLVM_SOURCE_DIR=${LLVM_SOURCE_DIR}"
echo "LLVM_BUILD_DIR=${LLVM_BUILD_DIR}"
echo "LLVM_INSTALL_DIR=${LLVM_INSTALL_DIR}"

command="$1"
shift

python3_command=""
if (command -v python3 &> /dev/null); then
python3_command="python3"
Expand All @@ -42,7 +39,7 @@ cmake_options=(
-DLLVM_TARGETS_TO_BUILD="${LLVM_TARGETS_TO_BUILD:-host}"
-DCMAKE_BUILD_TYPE=Release
-DPython3_EXECUTABLE="$Python3_EXECUTABLE"
-C "$TD/llvm_cache.cmake"
-C "$TD/cmake/llvm_cache.cmake"
-DCMAKE_INSTALL_PREFIX="${LLVM_INSTALL_DIR}"
)

Expand Down
62 changes: 62 additions & 0 deletions build_tools/build_llvm_wasi.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#!/usr/bin/env bash

TD="$(cd $(dirname $0) && pwd)"
REPO_ROOT="$(cd $TD/.. && pwd)"

LLVM_SOURCE_DIR="${LLVM_SOURCE_DIR:-${REPO_ROOT}/third_party/llvm-project}"
LLVM_BUILD_DIR="${LLVM_BUILD_DIR:-${REPO_ROOT}/llvm-build}"
LLVM_INSTALL_DIR="${LLVM_INSTALL_DIR:-${REPO_ROOT}/llvm-install}"

mkdir -p $LLVM_BUILD_DIR
mkdir -p $LLVM_INSTALL_DIR

LLVM_SOURCE_DIR="$(realpath "${LLVM_SOURCE_DIR}")"
LLVM_BUILD_DIR="$(realpath "${LLVM_BUILD_DIR}")"
LLVM_INSTALL_DIR="$(realpath "${LLVM_INSTALL_DIR}")"
EMSDK="$(realpath "${EMSDK}")"

echo "Paths canonicalized as:"
echo "LLVM_SOURCE_DIR=${LLVM_SOURCE_DIR}"
echo "LLVM_BUILD_DIR=${LLVM_BUILD_DIR}"
echo "LLVM_INSTALL_DIR=${LLVM_INSTALL_DIR}"

$EMSDK/emsdk activate
source $EMSDK/emsdk_env.sh
export CCACHE_COMPILERCHECK="string:$(emcc --version | head -n 1)"

set -euo pipefail

echo "*********************** BUILDING LLVM *********************************"

# hack to emit html wrappers
# https://stackoverflow.com/a/75596433/9045206
sed -i.bak 's/CMAKE_EXECUTABLE_SUFFIX ".js"/CMAKE_EXECUTABLE_SUFFIX ".html"/g' "$EMSDK/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake"

cmake_options=(
-GNinja
-S "${LLVM_SOURCE_DIR}/llvm"
-B "${LLVM_BUILD_DIR}"
# optimize for size
-DCMAKE_C_FLAGS="-Os"
-DCMAKE_CXX_FLAGS="-Os"
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_EXE_LINKER_FLAGS="-sSTANDALONE_WASM=1 -sWASM=1 -sWASM_BIGINT=1 -sEXPORT_ALL=0 -sEXPORTED_RUNTIME_METHODS=cwrap,ccall,getValue,setValue,writeAsciiToMemory,wasmTable -lembind"
-DCMAKE_INSTALL_PREFIX="${LLVM_INSTALL_DIR}"
-DCMAKE_SYSTEM_NAME=Emscripten
-DCMAKE_TOOLCHAIN_FILE="$EMSDK/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake"
-DCROSS_TOOLCHAIN_FLAGS_NATIVE="-DCMAKE_C_COMPILER=$CC;-DCMAKE_CXX_COMPILER=$CXX"
-C "$TD/cmake/llvm_wasm_cache.cmake"
)

echo "Source Directory: ${LLVM_SOURCE_DIR}"
echo "Build Directory: ${LLVM_BUILD_DIR}"
echo "CMake Options: ${cmake_options[*]}"

cmake "${cmake_options[@]}"
cmake --build "${LLVM_BUILD_DIR}" \
--target install-mlirdevelopment-distribution

sed -i.bak 's/CMAKE_EXECUTABLE_SUFFIX ".html"/CMAKE_EXECUTABLE_SUFFIX ".js"/g' "$EMSDK/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake"

# wasi files aren't installed for some reason
cp "${LLVM_BUILD_DIR}"/bin/* "${LLVM_INSTALL_DIR}/bin"
7 changes: 4 additions & 3 deletions build_tools/cmake/llvm_cache.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ set(MLIR_ENABLE_SPIRV_CPU_RUNNER ON CACHE BOOL "")

# space savings

set(LLVM_BUILD_DOCS OFF CACHE BOOL "")
set(LLVM_ENABLE_OCAMLDOC OFF CACHE BOOL "")
set(LLVM_ENABLE_BINDINGS OFF CACHE BOOL "")
set(LLVM_BUILD_BENCHMARKS OFF CACHE BOOL "")
set(LLVM_BUILD_EXAMPLES OFF CACHE BOOL "")
set(LLVM_ENABLE_LIBCXX OFF CACHE BOOL "")
Expand All @@ -37,6 +40,7 @@ set(LLVM_ENABLE_RTTI ON CACHE BOOL "")
set(LLVM_ENABLE_TERMINFO OFF CACHE BOOL "")
# When exceptions are disabled, unwind tables are large and useless
set(LLVM_ENABLE_UNWIND_TABLES OFF CACHE BOOL "")
set(LLVM_ENABLE_CRASH_OVERRIDES OFF CACHE BOOL "")
set(LLVM_ENABLE_Z3_SOLVER OFF CACHE BOOL "")
set(LLVM_ENABLE_ZLIB OFF CACHE BOOL "")
set(LLVM_ENABLE_ZSTD OFF CACHE BOOL "")
Expand All @@ -59,16 +63,13 @@ set(LLVM_INSTALL_TOOLCHAIN_ONLY OFF CACHE BOOL "")

set(LLVM_DISTRIBUTIONS MlirDevelopment CACHE STRING "")
set(LLVM_MlirDevelopment_DISTRIBUTION_COMPONENTS
# these are needed basically for execution engine
# check LINK_COMPONENTS there
llvm-config
llvm-headers
llvm-libraries
cmake-exports
FileCheck
not
MLIRPythonModules
MLIRPythonModules
# triggers LLVMMlirDevelopmentExports.cmake
mlirdevelopment-cmake-exports
# triggers MLIRMlirDevelopmentTargets.cmake
Expand Down
49 changes: 49 additions & 0 deletions build_tools/cmake/llvm_wasm_cache.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
set(LLVM_ENABLE_PROJECTS "mlir;llvm" CACHE STRING "")
set(LLVM_BUILD_TOOLS ON CACHE BOOL "")
set(LLVM_INCLUDE_TOOLS ON CACHE BOOL "")
set(LLVM_TARGETS_TO_BUILD "WebAssembly" CACHE STRING "")
set(LLVM_TARGET_ARCH "wasm32-wasi" CACHE STRING "")
set(LLVM_DEFAULT_TARGET_TRIPLE "wasm32-wasi" CACHE STRING "")

set(LLVM_BUILD_DOCS OFF CACHE BOOL "")
set(LLVM_ENABLE_BACKTRACES OFF CACHE BOOL "")
set(LLVM_ENABLE_BINDINGS OFF CACHE BOOL "")
set(LLVM_ENABLE_CRASH_OVERRIDES OFF CACHE BOOL "")
set(LLVM_ENABLE_LIBEDIT OFF CACHE BOOL "")
set(LLVM_ENABLE_LIBPFM OFF CACHE BOOL "")
set(LLVM_ENABLE_LIBXML2 OFF CACHE BOOL "")
set(LLVM_ENABLE_OCAMLDOC OFF CACHE BOOL "")
set(LLVM_ENABLE_PIC OFF CACHE BOOL "")
set(LLVM_ENABLE_THREADS OFF CACHE BOOL "")
set(LLVM_ENABLE_UNWIND_TABLES OFF CACHE BOOL "")
set(LLVM_ENABLE_ZLIB OFF CACHE BOOL "")
set(LLVM_ENABLE_ZSTD OFF CACHE BOOL "")
set(LLVM_INCLUDE_BENCHMARKS OFF CACHE BOOL "")
set(LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "")
set(LLVM_INCLUDE_TESTS OFF CACHE BOOL "")
set(LLVM_INCLUDE_UTILS OFF CACHE BOOL "")

### Distributions ###

set(LLVM_INSTALL_TOOLCHAIN_ONLY OFF CACHE BOOL "")

set(LLVM_DISTRIBUTIONS MlirDevelopment CACHE STRING "")
set(LLVM_MlirDevelopment_DISTRIBUTION_COMPONENTS
llvm-config
llvm-headers
llvm-libraries
cmake-exports
opt
# triggers LLVMMlirDevelopmentExports.cmake
mlirdevelopment-cmake-exports
# triggers MLIRMlirDevelopmentTargets.cmake
mlir-mlirdevelopment-cmake-exports
# triggers MLIRConfig.cmake and etc
mlir-cmake-exports
mlir-headers
mlir-libraries
mlir-opt
mlir-reduce
mlir-tblgen
mlir-translate
CACHE STRING "")

0 comments on commit 40b284c

Please sign in to comment.