From d84332a995e4ab0dc89e6d8b8d671c2f01947b33 Mon Sep 17 00:00:00 2001 From: edouardparis Date: Thu, 14 Nov 2024 09:58:35 +0100 Subject: [PATCH] fix the reproducible build to include the workspace change --- contrib/release/release.sh | 12 +- contrib/reproducible/docker/docker-build.sh | 36 +++--- contrib/reproducible/docker/macos_cmd.sh | 25 ++-- contrib/reproducible/docker/windows_cmd.sh | 1 + contrib/reproducible/guix/build.sh | 39 +++--- contrib/reproducible/guix/guix-build.sh | 128 +++++++------------- contrib/reproducible/guix/manifest.scm | 42 +++---- 7 files changed, 117 insertions(+), 166 deletions(-) diff --git a/contrib/release/release.sh b/contrib/release/release.sh index 698dc9a5d..8e4e6e61d 100755 --- a/contrib/release/release.sh +++ b/contrib/release/release.sh @@ -38,13 +38,13 @@ TARGET_DIR="$BUILD_DIR" ./contrib/reproducible/docker/docker-build.sh ( cd "$BUILD_DIR" create_dir "$LINUX_DIR_NAME" - cp "$BUILD_DIR/release/lianad" "$BUILD_DIR/release/liana-cli" "$BUILD_DIR/gui/release/liana-gui" ../README.md "$LINUX_DIR_NAME" + cp "$BUILD_DIR/release/lianad" "$BUILD_DIR/release/liana-cli" "$BUILD_DIR/release/liana-gui" ../README.md "$LINUX_DIR_NAME" tar -czf "$LINUX_ARCHIVE" "$LINUX_DIR_NAME" cp "$LINUX_ARCHIVE" "$RELEASE_DIR" unzip ../contrib/release/debian/package.zip sed -i "s/VERSION_PLACEHOLDER/$VERSION/g" ./package/DEBIAN/control - cp "$BUILD_DIR/release/lianad" "$BUILD_DIR/release/liana-cli" "$BUILD_DIR/gui/release/liana-gui" ../README.md ./package/usr/bin/ + cp "$BUILD_DIR/release/lianad" "$BUILD_DIR/release/liana-cli" "$BUILD_DIR/release/liana-gui" ../README.md ./package/usr/bin/ DIRNAME="liana_$VERSION-1_amd64" mv ./package "$DIRNAME" dpkg-deb -Zxz --build "$DIRNAME" @@ -55,23 +55,23 @@ TARGET_DIR="$BUILD_DIR" ./contrib/reproducible/docker/docker-build.sh ( cd "$BUILD_DIR" create_dir "$WINDOWS_DIR_NAME" - cp "$BUILD_DIR/gui/x86_64-pc-windows-gnu/release/liana-gui.exe" ../README.md "$WINDOWS_DIR_NAME" + cp "$BUILD_DIR/x86_64-pc-windows-gnu/release/liana-gui.exe" ../README.md "$WINDOWS_DIR_NAME" zip -r "$WINDOWS_ARCHIVE" "$WINDOWS_DIR_NAME" cp "$WINDOWS_ARCHIVE" "$RELEASE_DIR" - cp "$BUILD_DIR/gui/x86_64-pc-windows-gnu/release/liana-gui.exe" "$RELEASE_DIR/$LIANA_PREFIX.exe" + cp "$BUILD_DIR/x86_64-pc-windows-gnu/release/liana-gui.exe" "$RELEASE_DIR/$LIANA_PREFIX.exe" ) # Create the MacOS archive and a zipped application bundle of liana-gui. ( cd "$BUILD_DIR" create_dir "$MAC_DIR_NAME" - cp "$BUILD_DIR/x86_64-apple-darwin/release/lianad" "$BUILD_DIR/x86_64-apple-darwin/release/liana-cli" "$BUILD_DIR/gui/x86_64-apple-darwin/release/liana-gui" ../README.md "$MAC_DIR_NAME" + cp "$BUILD_DIR/x86_64-apple-darwin/release/lianad" "$BUILD_DIR/x86_64-apple-darwin/release/liana-cli" "$BUILD_DIR/x86_64-apple-darwin/release/liana-gui" ../README.md "$MAC_DIR_NAME" tar -czf "$MAC_ARCHIVE" "$MAC_DIR_NAME" cp "$MAC_ARCHIVE" "$RELEASE_DIR" unzip ../contrib/release/macos/Liana.app.zip sed -i "s/VERSION_PLACEHOLDER/$VERSION/g" ./Liana.app/Contents/Info.plist - cp "$BUILD_DIR/gui/x86_64-apple-darwin/release/liana-gui" ./Liana.app/Contents/MacOS/Liana + cp "$BUILD_DIR/x86_64-apple-darwin/release/liana-gui" ./Liana.app/Contents/MacOS/Liana zip -ry Liana-noncodesigned.zip Liana.app cp ./Liana-noncodesigned.zip "$RELEASE_DIR/" diff --git a/contrib/reproducible/docker/docker-build.sh b/contrib/reproducible/docker/docker-build.sh index f16b941b1..2e42de529 100755 --- a/contrib/reproducible/docker/docker-build.sh +++ b/contrib/reproducible/docker/docker-build.sh @@ -11,15 +11,18 @@ XCODE_SHASUM="28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0" # Build (only) the Liana GUI on Windows. docker build . -t liana_cross_win -f contrib/reproducible/docker/windows.Dockerfile docker run --rm -ti \ - -v "$TARGET_DIR/gui":/liana/target \ + -v "$TARGET_DIR":/liana/target \ -v "$PWD/contrib/reproducible/docker":/liana/docker \ - -v "$PWD/gui/Cargo.toml":/liana/Cargo.toml \ - -v "$PWD/gui/Cargo.lock":/liana/Cargo.lock \ - -v "$PWD/gui/src":/liana/src \ - -v "$PWD/gui/ui/Cargo.toml":/liana/ui/Cargo.toml \ - -v "$PWD/gui/ui/Cargo.lock":/liana/ui/Cargo.lock \ - -v "$PWD/gui/ui/src":/liana/ui/src \ - -v "$PWD/gui/ui/static":/liana/ui/static \ + -v "$PWD/Cargo.toml":/liana/Cargo.toml \ + -v "$PWD/Cargo.lock":/liana/Cargo.lock \ + -v "$PWD/liana/Cargo.toml":/liana/liana/Cargo.toml \ + -v "$PWD/liana/src":/liana/liana/src \ + -v "$PWD/liana-gui/Cargo.toml":/liana/liana-gui/Cargo.toml \ + -v "$PWD/liana-gui/src":/liana/liana-gui/src \ + -v "$PWD/liana-ui/Cargo.toml":/liana/liana-ui/Cargo.toml \ + -v "$PWD/liana-ui/src":/liana/liana-ui/src \ + -v "$PWD/liana-ui/static":/liana/liana-ui/static \ + -v "$PWD/fuzz/Cargo.toml":/liana/fuzz/Cargo.toml \ liana_cross_win @@ -33,18 +36,17 @@ fi docker build . -t liana_cross_mac -f contrib/reproducible/docker/macos.Dockerfile docker run -ti \ -v "$TARGET_DIR":/liana/target \ - -v "$TARGET_DIR/gui":/liana/gui/target \ -v "$PWD/contrib/reproducible/docker":/liana/docker \ -v "$PWD/Cargo.toml":/liana/Cargo.toml \ -v "$PWD/Cargo.lock":/liana/Cargo.lock \ - -v "$PWD/src":/liana/src \ - -v "$PWD/gui/Cargo.toml":/liana/gui/Cargo.toml \ - -v "$PWD/gui/Cargo.lock":/liana/gui/Cargo.lock \ - -v "$PWD/gui/src":/liana/gui/src \ - -v "$PWD/gui/ui/Cargo.toml":/liana/gui/ui/Cargo.toml \ - -v "$PWD/gui/ui/Cargo.lock":/liana/gui/ui/Cargo.lock \ - -v "$PWD/gui/ui/src":/liana/gui/ui/src \ - -v "$PWD/gui/ui/static":/liana/gui/ui/static \ + -v "$PWD/liana/Cargo.toml":/liana/liana/Cargo.toml \ + -v "$PWD/liana/src":/liana/liana/src \ + -v "$PWD/liana-gui/Cargo.toml":/liana/liana-gui/Cargo.toml \ + -v "$PWD/liana-gui/src":/liana/liana-gui/src \ + -v "$PWD/liana-ui/Cargo.toml":/liana/liana-ui/Cargo.toml \ + -v "$PWD/liana-ui/src":/liana/liana-ui/src \ + -v "$PWD/liana-ui/static":/liana/liana-ui/static \ + -v "$PWD/fuzz/Cargo.toml":/liana/fuzz/Cargo.toml \ -v "$XCODE_PATH":"/liana/$XCODE_FILENAME" \ -e XCODE_PATH="/liana/$XCODE_FILENAME" \ liana_cross_mac diff --git a/contrib/reproducible/docker/macos_cmd.sh b/contrib/reproducible/docker/macos_cmd.sh index 89b915e54..dd45f1498 100755 --- a/contrib/reproducible/docker/macos_cmd.sh +++ b/contrib/reproducible/docker/macos_cmd.sh @@ -29,25 +29,18 @@ cd .. # Finally build the projects using the toolchain just created. alias cargo="/liana/rust-1.71.1-x86_64-unknown-linux-gnu/cargo/bin/cargo" -PATH="$PATH:$PWD/osxcross/target/bin/" \ - CC=o64-clang \ - CXX=o64-clang++ \ - RUSTFLAGS="$RUSTFLAGS -Clinker=o64-clang" \ - cargo rustc \ +for package_name in "liana" "liana-gui"; do + PATH="$PATH:$PWD/osxcross/target/bin/" \ + CC=o64-clang \ + CXX=o64-clang++ \ + RUSTFLAGS="$RUSTFLAGS -Clinker=o64-clang" \ + cargo rustc \ + -p "$package_name" \ --target x86_64-apple-darwin \ --release - -cd gui/ -PATH="$PATH:$PWD/../osxcross/target/bin/" \ - CC=o64-clang \ - CXX=o64-clang++ \ - RUSTFLAGS="$RUSTFLAGS -Clinker=o64-clang" \ - cargo rustc \ - --target x86_64-apple-darwin \ - --release -cd .. +done # Avoid having to get root on the host to remove the target dir. -chmod -R a+rw target/ gui/target +chmod -R a+rw target/ set +xe diff --git a/contrib/reproducible/docker/windows_cmd.sh b/contrib/reproducible/docker/windows_cmd.sh index 14a8a2aa7..b087a7cb1 100755 --- a/contrib/reproducible/docker/windows_cmd.sh +++ b/contrib/reproducible/docker/windows_cmd.sh @@ -11,6 +11,7 @@ set -xe alias cargo="/liana/rust-1.71.1-x86_64-unknown-linux-gnu/cargo/bin/cargo" RUSTFLAGS="-Clink-arg=-Wl,--no-insert-timestamp" \ cargo rustc \ + -p liana-gui \ --release \ --target x86_64-pc-windows-gnu diff --git a/contrib/reproducible/guix/build.sh b/contrib/reproducible/guix/build.sh index 482632a51..9af2d6624 100755 --- a/contrib/reproducible/guix/build.sh +++ b/contrib/reproducible/guix/build.sh @@ -1,6 +1,6 @@ -# ========================================================================== -# The script ran within the GUIX container to build the Liana daemon or GUI. -# ========================================================================== +# =========================================================================== +# The script ran within the GUIX container to build the Liana daemon and GUI. +# =========================================================================== set -ex @@ -24,24 +24,25 @@ branch = "patch-0.12.3" replace-with = "vendored_sources" EOF +ls -la .cargo/config.toml + +export CARGO_HOME="/liana/.cargo" + # We need to set RUSTC_BOOTSTRAP=1 as a workaround to be able to use unstable # features in the GUI dependencies -RUSTC_BOOTSTRAP=1 cargo -vvv \ - --color always \ - --frozen \ - --offline \ - rustc \ - --jobs "$JOBS" \ - --release \ - --target-dir "/out" - -if [ "$IS_GUI" = "1" ]; then - BIN_NAMES="liana-gui" -else - BIN_NAMES="lianad liana-cli" -fi - -for bin_name in $BIN_NAMES; do +for package_name in "liana" "liana-gui"; do + RUSTC_BOOTSTRAP=1 cargo -vvv \ + --color always \ + --frozen \ + --offline \ + rustc \ + -p "$package_name" \ + --jobs "$JOBS" \ + --release \ + --target-dir "/out" +done + +for bin_name in "liana-gui" "lianad" "liana-cli"; do # Assume 64bits. Even bitcoind doesn't ship 32bits binaries for x86. # FIXME: is there a cleaner way than using patchelf for this? patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 "/out/release/$bin_name" diff --git a/contrib/reproducible/guix/guix-build.sh b/contrib/reproducible/guix/guix-build.sh index a2e385102..ceeaaf214 100755 --- a/contrib/reproducible/guix/guix-build.sh +++ b/contrib/reproducible/guix/guix-build.sh @@ -48,95 +48,55 @@ fi # across time. time_machine() { guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=059d38dc3f8b087f4a42df586daeb05761ee18d7 \ - --cores="$JOBS" \ - --keep-failed \ - --fallback \ - -- "$@" + --commit=059d38dc3f8b087f4a42df586daeb05761ee18d7 \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + -- "$@" } -# Build both the daemon (at the root of the repository) and the GUI (in gui/) -for project_folder in "" "gui"; do - PROJECT_ROOT="$PWD/$project_folder" - PROJECT_VENDOR_DIR="$VENDOR_DIR/$project_folder" - PROJECT_OUT_DIR="$OUT_DIR/$project_folder" - PROJECT_PATCHES_ROOT="$PWD/contrib/reproducible/guix/liana-patches/$project_folder" - - project_needs_patches() { - test "$(ls -A1q "$PROJECT_PATCHES_ROOT" |grep patch)" - } - - maybe_create_dir "$PROJECT_OUT_DIR" - - # Pull the sources of our dependencies before building them in the container. - if ! [ -d "$PROJECT_VENDOR_DIR" ]; then - # Download the dependencies - ( cd "$project_folder" && $CARGO_BIN vendor "$PROJECT_VENDOR_DIR" ) - - # Patch some dependencies sources if needed for this project - if project_needs_patches; then - ( - cd "$PROJECT_VENDOR_DIR" - for patch_file in $(ls "$PROJECT_PATCHES_ROOT"); do - patch -p1 < "$PROJECT_PATCHES_ROOT/$patch_file" - done - ) - - # Some of the checksums will be incorrect. Instead of cherry-picking remove them - # altogether, since they aren't useful anyways (see comment below). - for dep in $(ls "$PROJECT_VENDOR_DIR"); do - echo "{\"files\":{}}" > "$PROJECT_VENDOR_DIR/$dep/.cargo-checksum.json" - done - fi - fi +# Build both the daemon (at the root of the repository) and the GUI +PROJECT_ROOT="$PWD" +PROJECT_VENDOR_DIR="$VENDOR_DIR" +PROJECT_OUT_DIR="$OUT_DIR" - cp "$PROJECT_ROOT/Cargo.lock" "$BUILD_ROOT/Cargo.lock" - if project_needs_patches; then - # Remove the checksums from the Cargo.lock. In the container `cargo rustc` would compare - # them against the .cargo-checksum.json to make sure they weren't tampered with since they - # where vendored. But we just removed the checksums from the .cargo-checksum.json. - # There is little point in checking integrity between the above vendor step and now anyways. - # What matters is checking integrity after downloading the crates from the internet and - # `cargo vendor` does that already. - sed -i '/checksum/d' "$BUILD_ROOT/Cargo.lock" - fi +maybe_create_dir "$PROJECT_OUT_DIR" - # FIXME: find a cleaner way to get the binary name, or get rid of patchelf entirely - # Note: we also rely on it in manifest.scm - if [ "$project_folder" = "" ]; then - IS_GUI=0 - elif [ "$project_folder" = "gui" ]; then - IS_GUI=1 - else - echo "Can't determine binary name" - exit 1 - fi +# Pull the sources of our dependencies before building them in the container. +if ! [ -d "$PROJECT_VENDOR_DIR" ]; then + # Download the dependencies + ($CARGO_BIN vendor "$PROJECT_VENDOR_DIR" ) +fi - # Bootstrap a reproducible environment as specified by the manifest in an isolated - # container, and build the project. - # NOTE: it looks like "--rebuild-cache" is necessary for the IS_GUI variable to - # be taken into account when building the container (otherwise the GUI container could - # miss some dependencies). - # TODO: only expose the "ui" folder for GUI builds - IS_GUI="$IS_GUI" time_machine shell --no-cwd \ - --expose="$PROJECT_ROOT/src=/liana/src" \ - --expose="$PROJECT_ROOT/Cargo.toml=/liana/Cargo.toml" \ - --expose="$BUILD_ROOT/Cargo.lock=/liana/Cargo.lock" \ - --expose="$PWD/gui/ui/src=/liana/ui/src" \ - --expose="$PWD/gui/ui/Cargo.toml=/liana/ui/Cargo.toml" \ - --expose="$PWD/gui/ui/static=/liana/ui/static" \ - --expose="$PWD/contrib/reproducible/guix/build.sh=/liana/build.sh" \ - --expose="$PROJECT_VENDOR_DIR=/vendor" \ - --share="$PROJECT_OUT_DIR=/out" \ - --cores="$JOBS" \ - --container \ - --pure \ - --fallback \ - --rebuild-cache \ - -m $PWD/contrib/reproducible/guix/manifest.scm \ - -- env CC=gcc VENDOR_DIR="$PROJECT_VENDOR_DIR" TARGET_DIR="$PROJECT_OUT_DIR" IS_GUI="$IS_GUI" JOBS="$JOBS" \ - /bin/sh -c "cd /liana && ./build.sh" -done +cp "$PROJECT_ROOT/Cargo.lock" "$BUILD_ROOT/Cargo.lock" + +# Bootstrap a reproducible environment as specified by the manifest in an isolated +# container, and build the project. +# NOTE: it looks like "--rebuild-cache" is necessary for the IS_GUI variable to +# be taken into account when building the container (otherwise the GUI container could +# miss some dependencies). +time_machine shell --no-cwd \ + --expose="$PWD/Cargo.toml=/liana/Cargo.toml" \ + --expose="$BUILD_ROOT/Cargo.lock=/liana/Cargo.lock" \ + --expose="$PWD/liana/src=/liana/liana/src" \ + --expose="$PWD/liana/Cargo.toml=/liana/liana/Cargo.toml" \ + --expose="$PWD/liana-gui/Cargo.toml=/liana/liana-gui/Cargo.toml" \ + --expose="$PWD/liana-gui/src=/liana/liana-gui/src" \ + --expose="$PWD/liana-ui/src=/liana/liana-ui/src" \ + --expose="$PWD/liana-ui/Cargo.toml=/liana/liana-ui/Cargo.toml" \ + --expose="$PWD/liana-ui/static=/liana/liana-ui/static" \ + --expose="$PWD/fuzz/Cargo.toml=/liana/fuzz/Cargo.toml" \ + --expose="$PWD/contrib/reproducible/guix/build.sh=/liana/build.sh" \ + --expose="$PROJECT_VENDOR_DIR=/vendor" \ + --share="$PROJECT_OUT_DIR=/out" \ + --cores="$JOBS" \ + --container \ + --pure \ + --fallback \ + --rebuild-cache \ + -m $PWD/contrib/reproducible/guix/manifest.scm \ + -- env CC=gcc VENDOR_DIR="$PROJECT_VENDOR_DIR" TARGET_DIR="$PROJECT_OUT_DIR" IS_GUI="$IS_GUI" JOBS="$JOBS" \ + /bin/sh -c "cd /liana && ./build.sh" set +ex diff --git a/contrib/reproducible/guix/manifest.scm b/contrib/reproducible/guix/manifest.scm index fa8e39781..46d89637e 100644 --- a/contrib/reproducible/guix/manifest.scm +++ b/contrib/reproducible/guix/manifest.scm @@ -170,27 +170,21 @@ FILE-NAME found in ./patches relative to the current file." ;; END of the newer rustc versions copied over from the current Guix master. -(let ((is_gui (getenv "IS_GUI"))) - (concatenate-manifests - (list - (specifications->manifest - (append - (list - "rust:cargo" - "coreutils-minimal" - "patchelf" - "gcc-toolchain@10.3.0") - ;; Additional dependencies for building the GUI. - (if - (string=? is_gui "1") - (list "pkg-config" - "eudev" - "fontconfig") - '()))) - ;; The GUI's MSRV is 1.70 and the daemon's 1.63. We just use the same rustc version for - ;; both. - ;; FIXME: be able to compile against a specified glibc (or musl) instead of having to - ;; resort to backporting the newer rustc releases here. Also have proper Guix packages - ;; for the two projects. - (packages->manifest - `(,rust-1.71))))) +(concatenate-manifests + (list + (specifications->manifest + (list + "rust:cargo" + "coreutils-minimal" + "patchelf" + "gcc-toolchain@10.3.0" + "pkg-config" + "eudev" + "fontconfig")) + ;; The GUI's MSRV is 1.70 and the daemon's 1.63. We just use the same rustc version for + ;; both. + ;; FIXME: be able to compile against a specified glibc (or musl) instead of having to + ;; resort to backporting the newer rustc releases here. Also have proper Guix packages + ;; for the two projects. + (packages->manifest + `(,rust-1.71))))