Problem: zstd build failed when cross-compilied on macos (#947)

* Problem: zstd build failed when cross-compilied on macos Solution: - vendor the zstd nix expression which has the fix * Update nix/build_overlay.nix Signed-off-by: yihuang <[email protected]> * Update nix/build_overlay.nix Signed-off-by: yihuang <[email protected]> --------- Signed-off-by: yihuang <[email protected]>
crypto-org-chain · Apr 28, 2023 · 04346d0 · 04346d0
1 parent 4e11862
commit 04346d0
Show file tree

Hide file tree

Showing 5 changed files with 188 additions and 16 deletions.
diff --git a/flake.nix b/flake.nix
@@ -27,6 +27,7 @@
           pkgs = import nixpkgs {
             inherit system;
             overlays = [
+              (import ./nix/build_overlay.nix)
               gomod2nix.overlays.default
               self.overlay
             ];
@@ -71,14 +72,6 @@
             | gzip -9 > $out
         '';
         bundle-win-exe = drv: final.callPackage ./nix/bundle-win-exe.nix { cronosd = drv; };
-        rocksdb = final.callPackage ./nix/rocksdb.nix { };
-        go_1_20 = prev.go_1_20.overrideAttrs (prev: rec {
-          version = "1.20.3";
-          src = final.fetchurl {
-            url = "https://go.dev/dl/go${version}.src.tar.gz";
-            hash = "sha256-5Ee0mM3lAhXE92GeUSSw/E4l+10W6kcnHEfyeOeqdjo=";
-          };
-        });
       } // (with final;
         let
           matrix = lib.cartesianProductOfSets {

diff --git a/nix/build_overlay.nix b/nix/build_overlay.nix
@@ -0,0 +1,15 @@
+# some basic overlays nessesary for the build
+final: super: {
+  # include the fix: https://github.com/NixOS/nixpkgs/pull/211515
+  zstd = final.callPackage ./zstd.nix {
+    cmake = final.buildPackages.cmakeMinimal;
+  };
+  rocksdb = final.callPackage ./rocksdb.nix { };
+  go_1_20 = super.go_1_20.overrideAttrs (prev: rec {
+    version = "1.20.3";
+    src = final.fetchurl {
+      url = "https://go.dev/dl/go${version}.src.tar.gz";
+      hash = "sha256-5Ee0mM3lAhXE92GeUSSw/E4l+10W6kcnHEfyeOeqdjo=";
+    };
+  });
+}
diff --git a/nix/default.nix b/nix/default.nix
@@ -12,14 +12,8 @@ import sources.nixpkgs {
       hermes = pkgs.callPackage ./hermes.nix { src = sources.ibc-rs; };
     })
     (import "${sources.gomod2nix}/overlay.nix")
+    (import ./build_overlay.nix)
     (pkgs: prev: {
-      go_1_20 = prev.go_1_20.overrideAttrs (prev: rec {
-        version = "1.20.3";
-        src = pkgs.fetchurl {
-          url = "https://go.dev/dl/go${version}.src.tar.gz";
-          hash = "sha256-5Ee0mM3lAhXE92GeUSSw/E4l+10W6kcnHEfyeOeqdjo=";
-        };
-      });
       go = pkgs.go_1_20;
       test-env = pkgs.callPackage ./testenv.nix { };
       lint-ci = pkgs.writeShellScriptBin "lint-ci" ''
@@ -35,7 +29,6 @@ import sources.nixpkgs {
       chain-maind-zemu = pkgs.callPackage ../. {
         ledger_zemu = true;
       };
-      rocksdb = pkgs.callPackage ./rocksdb.nix { };
     })
   ];
   config = { };

diff --git a/nix/playtests-darwin.patch b/nix/playtests-darwin.patch
@@ -0,0 +1,40 @@
+--- a/tests/playTests.sh
++++ b/tests/playTests.sh
+@@ -112,29 +112,19 @@ case "$OS" in
+ esac
+
+ case "$UNAME" in
+-  Darwin) MD5SUM="md5 -r" ;;
+-  FreeBSD) MD5SUM="gmd5sum" ;;
+-  NetBSD) MD5SUM="md5 -n" ;;
+-  OpenBSD) MD5SUM="md5" ;;
+   *) MD5SUM="md5sum" ;;
+ esac
+
+ MTIME="stat -c %Y"
+-case "$UNAME" in
+-    Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;;
+-esac
+
+ assertSameMTime() {
+     MT1=$($MTIME "$1")
+     MT2=$($MTIME "$2")
+     echo MTIME $MT1 $MT2
+     [ "$MT1" = "$MT2" ] || die "mtime on $1 doesn't match mtime on $2 ($MT1 != $MT2)"
+ }
+
+ GET_PERMS="stat -c %a"
+-case "$UNAME" in
+-    Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;;
+-esac
+
+ assertFilePermissions() {
+     STAT1=$($GET_PERMS "$1")
+@@ -967,7 +957,6 @@ $MD5SUM dirTestDict/* > tmph1
+ zstd -f --rm dirTestDict/* -D tmpDictC
+ zstd -d --rm dirTestDict/*.zst -D tmpDictC  # note : use internal checksum by default
+ case "$UNAME" in
+-  Darwin) println "md5sum -c not supported on OS-X : test skipped" ;;  # not compatible with OS-X's md5
+   *) $MD5SUM -c tmph1 ;;
+ esac
+ rm -rf dirTestDict
diff --git a/nix/zstd.nix b/nix/zstd.nix
@@ -0,0 +1,131 @@
+{ lib, stdenv, fetchFromGitHub, cmake, bash, gnugrep
+, fixDarwinDylibNames
+, file
+, fetchpatch
+, legacySupport ? false
+, static ? stdenv.hostPlatform.isStatic
+# these need to be ran on the host, thus disable when cross-compiling
+, buildContrib ? stdenv.hostPlatform == stdenv.buildPlatform
+, doCheck ? stdenv.hostPlatform == stdenv.buildPlatform
+, nix-update-script
+
+# for passthru.tests
+, libarchive
+, rocksdb
+, arrow-cpp
+, libzip
+, curl
+, python3Packages
+, haskellPackages
+}:
+
+stdenv.mkDerivation rec {
+  pname = "zstd";
+  version = "1.5.5";
+
+  src = fetchFromGitHub {
+    owner = "facebook";
+    repo = "zstd";
+    rev = "v${version}";
+    sha256 = "sha256-tHHHIsQU7vJySrVhJuMKUSq11MzkmC+Pcsj00uFJdnQ=";
+  };
+
+  nativeBuildInputs = [ cmake ]
+   ++ lib.optional stdenv.isDarwin fixDarwinDylibNames;
+  buildInputs = lib.optional stdenv.hostPlatform.isUnix bash;
+
+  patches = [
+    # This patches makes sure we do not attempt to use the MD5 implementation
+    # of the host platform when running the tests
+    ./playtests-darwin.patch
+  ];
+
+  postPatch = lib.optionalString (!static) ''
+    substituteInPlace build/cmake/CMakeLists.txt \
+      --replace 'message(SEND_ERROR "You need to build static library to build tests")' ""
+    substituteInPlace build/cmake/tests/CMakeLists.txt \
+      --replace 'libzstd_static' 'libzstd_shared'
+    sed -i \
+      "1aexport ${lib.optionalString stdenv.isDarwin "DY"}LD_LIBRARY_PATH=$PWD/build_/lib" \
+      tests/playTests.sh
+  '';
+
+  LDFLAGS = lib.optionalString stdenv.hostPlatform.isRiscV "-latomic";
+
+  cmakeFlags = lib.attrsets.mapAttrsToList
+    (name: value: "-DZSTD_${name}:BOOL=${if value then "ON" else "OFF"}") {
+      BUILD_SHARED = !static;
+      BUILD_STATIC = static;
+      BUILD_CONTRIB = buildContrib;
+      PROGRAMS_LINK_SHARED = !static;
+      LEGACY_SUPPORT = legacySupport;
+      BUILD_TESTS = doCheck;
+    };
+
+  cmakeDir = "../build/cmake";
+  dontUseCmakeBuildDir = true;
+  preConfigure = ''
+    mkdir -p build_ && cd $_
+  '';
+
+  nativeCheckInputs = [ file ];
+  inherit doCheck;
+  checkPhase = ''
+    runHook preCheck
+    # Patch shebangs for playTests
+    patchShebangs ../programs/zstdgrep
+    ctest -R playTests # The only relatively fast test.
+    runHook postCheck
+  '';
+
+  preInstall = ''
+    mkdir -p $bin/bin
+    substituteInPlace ../programs/zstdgrep \
+      --replace ":-grep" ":-${gnugrep}/bin/grep" \
+      --replace ":-zstdcat" ":-$bin/bin/zstdcat"
+
+    substituteInPlace ../programs/zstdless \
+      --replace "zstdcat" "$bin/bin/zstdcat"
+  '' + lib.optionalString buildContrib (
+    ''
+      cp contrib/pzstd/pzstd $bin/bin/pzstd
+    '' + lib.optionalString stdenv.isDarwin ''
+      install_name_tool -change @rpath/libzstd.1.dylib $out/lib/libzstd.1.dylib $bin/bin/pzstd
+    ''
+  );
+
+  outputs = [ "bin" "dev" ]
+    ++ lib.optional stdenv.hostPlatform.isUnix "man"
+    ++ [ "out" ];
+
+  passthru = {
+    updateScript = nix-update-script { };
+    tests = {
+      inherit libarchive rocksdb arrow-cpp;
+      libzip = libzip.override { withZstd = true; };
+      curl = curl.override { zstdSupport = true; };
+      python-zstd = python3Packages.zstd;
+      haskell-zstd = haskellPackages.zstd;
+      haskell-hs-zstd = haskellPackages.hs-zstd;
+    };
+  };
+
+  meta = with lib; {
+    description = "Zstandard real-time compression algorithm";
+    longDescription = ''
+      Zstd, short for Zstandard, is a fast lossless compression algorithm,
+      targeting real-time compression scenarios at zlib-level compression
+      ratio. Zstd can also offer stronger compression ratio at the cost of
+      compression speed. Speed/ratio trade-off is configurable by small
+      increment, to fit different situations. Note however that decompression
+      speed is preserved and remain roughly the same at all settings, a
+      property shared by most LZ compression algorithms, such as zlib.
+    '';
+    homepage = "https://facebook.github.io/zstd/";
+    changelog = "https://github.com/facebook/zstd/blob/v${version}/CHANGELOG";
+    license = with licenses; [ bsd3 ]; # Or, at your opinion, GPL-2.0-only.
+
+    platforms = platforms.all;
+    maintainers = with maintainers; [ orivej ];
+  };
+}