Merge #234

234: Make mbedtls windows ready, force CLANG on sgx, rearrange CI scripts r=[arai-fortanix] a=kwakkel1000

We have fixes in CLANG for SGX, so now we force that you use CLANG if you compile for SGX.
We can now compile mbedtls on windows (with CI checked)
CI scripts are restructured for better understanding.

Co-authored-by: Gijs Kwakkel <[email protected]>
Co-authored-by: Yuxiang Cao <[email protected]>
Co-authored-by: YxC <[email protected]>
Co-authored-by: YX Cao <[email protected]>

.github/workflows/test.yml

@@ -28,20 +28,23 @@ jobs:
             target: x86_64-unknown-linux-gnu
             os: ubuntu-20.04
           - rust: stable
-            target: aarch64-unknown-linux-musl
+            target: x86_64-fortanix-unknown-sgx
             os: ubuntu-20.04
           - rust: stable
-            target: x86_64-fortanix-unknown-sgx
+            target: x86_64-pc-windows-msvc
+            os: windows-latest
+          - rust: stable
+            target: aarch64-unknown-linux-musl
             os: ubuntu-20.04
+          - rust: stable
+            target: x86_64-apple-darwin
+            os: macos-latest
           - rust: beta
             target: x86_64-unknown-linux-gnu
             os: ubuntu-20.04
           - rust: nightly
             target: x86_64-unknown-linux-gnu
             os: ubuntu-20.04
-          - rust: stable
-            target: x86_64-apple-darwin
-            os: macos-latest
 
     runs-on: ${{ matrix.os }}
 
@@ -54,26 +57,35 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y qemu-user
 
+    # python version is read from .python-version
+    - name: Setup python
+      uses: actions/setup-python@v4
+
+    - name: Install python dependencies
+      run: python3 -m pip install -r mbedtls-sys/vendor/scripts/basic.requirements.txt
+
+    - name: Cache Dependencies
+      uses: Swatinem/rust-cache@988c164c3d0e93c4dbab36aaf5bbeb77425b2894
+      with:
+        key: ${{ matrix.rust }}
+
     - name: Setup Rust toolchain
       uses: actions-rs/toolchain@v1
       with:
         toolchain: ${{ matrix.rust }}
         target: ${{ matrix.target }}
         override: true
 
-    - name: Cache Dependencies
-      uses: Swatinem/rust-cache@988c164c3d0e93c4dbab36aaf5bbeb77425b2894
-      with:
-        key: ${{ matrix.rust }}
-
     - name: Run tests
       run: |
-        ./ct.sh
+        ./ci_tools.sh
+        ./ci.sh
       env:
         TRAVIS_RUST_VERSION: ${{ matrix.rust }}
         TARGET: ${{ matrix.target }}
         ZLIB_INSTALLED: ${{ matrix.target == 'x86_64-unknown-linux-gnu' && 'true' || '' }}
         AES_NI_SUPPORT: ${{ matrix.target == 'x86_64-unknown-linux-gnu' && 'true' || '' }}
+      shell: bash
   ci-success:
     name: ci
     if: always()

.python-version

@@ -0,0 +1 @@
+3.8.10

ci.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+set -ex
+cd "$(dirname "$0")"
+
+repo_root=$(readlink -f $(dirname "${BASH_SOURCE[0]}"))
+
+if [ -z $TRAVIS_RUST_VERSION ]; then
+    echo "Expected TRAVIS_RUST_VERSION to be set in env"
+    exit 1
+fi
+
+# According to `mbedtls-sys/vendor/README.md`, need to install needed pkgs
+python -m venv venv
+source venv/bin/activate || source venv/Scripts/activate
+python -m pip install -r ./mbedtls-sys/vendor/scripts/basic.requirements.txt
+
+# Test logic start from here
+export CFLAGS_x86_64_fortanix_unknown_sgx="-isystem/usr/include/x86_64-linux-gnu -mlvi-hardening -mllvm -x86-experimental-lvi-inline-asm-hardening"
+export CC_x86_64_fortanix_unknown_sgx=clang-11
+export CC_aarch64_unknown_linux_musl=/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc
+export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc
+export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER=qemu-aarch64
+
+cd "${repo_root}/mbedtls"
+case "$TRAVIS_RUST_VERSION" in
+    stable|beta|nightly)
+        # Install the rust toolchain
+        rustup default $TRAVIS_RUST_VERSION
+        rustup target add --toolchain $TRAVIS_RUST_VERSION $TARGET
+        printenv
+
+        # The SGX target cannot be run under test like a ELF binary
+        if [ "$TARGET" != "x86_64-fortanix-unknown-sgx" ]; then
+            # make sure that explicitly providing the default target works
+            cargo nextest run --target $TARGET --release
+            cargo nextest run --features dsa --target $TARGET
+            cargo nextest run --features async-rt,tls13 --target $TARGET
+
+            # If AES-NI is supported, test the feature
+            if [ -n "$AES_NI_SUPPORT" ]; then
+                cargo nextest run --features force_aesni_support,tls13 --target $TARGET
+            fi
+            # no_std tests only are able to run on x86 platform
+            if [ "$TARGET" == "x86_64-unknown-linux-gnu" ] || [ "$TARGET" == "x86_64-apple-darwin" ] || [[ "$TARGET" =~ ^x86_64-pc-windows- ]]; then
+                cargo nextest run --no-default-features --features no_std_deps,rdrand,time --target $TARGET
+                cargo nextest run --no-default-features --features no_std_deps --target $TARGET
+            fi
+
+        else
+            cargo +$TRAVIS_RUST_VERSION test --no-run --target=$TARGET
+            cargo +$TRAVIS_RUST_VERSION test --no-default-features --features dsa,force_aesni_support,mpi_force_c_code,rdrand,std,time,tls13 --no-run --target=$TARGET
+        fi
+        ;;
+    *)
+        # Default case: If TRAVIS_RUST_VERSION does not match any of the above
+        echo "Unknown version $TRAVIS_RUST_VERSION"
+        exit 1
+        ;;
+esac

ci_tools.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+set -ex
+cd "$(dirname "$0")"
+
+repo_root=$(readlink -f $(dirname "${BASH_SOURCE[0]}"))
+
+# Setup dependencies and tools
+
+# checks if a file has a specific sha512 hash
+check_sha512() {
+    local hash="$1"
+    local file="$2"
+    local platform=$(uname)
+    case $platform in
+        Linux)
+            sha512sum -c <<< "$hash *$file"
+            ;;
+        Darwin)
+            shasum -a 512 -c <<< "$hash *$file"
+            ;;
+        MINGW64_NT-*)
+            sha512sum -c <<< "$hash *$file"
+            ;;
+        *)
+            echo "Unsupported platform '$platfom'"
+            exit 1
+            ;;
+    esac
+}
+
+# function for downloading pre-built `cargo-nextest` on various platforms
+download_cargo_nextest() {
+    local platform="$1"
+    local cargo_nextest_hash="$2"
+    local url="$3"
+    echo "Check if need to download pre-built $platform 'cargo-nextest'"
+    if ! check_sha512 "${cargo_nextest_hash}" "${CARGO_HOME:-$HOME/.cargo}/bin/cargo-nextest"; then
+        case $platform in
+            MINGW64-*)
+                curl -LsSf "$url" -o temp.zip && unzip -d "${CARGO_HOME:-$HOME/.cargo}/bin" temp.zip && rm temp.zip
+                ;;
+            *)
+                curl -LsSf "$url" | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
+                ;;
+        esac
+        check_sha512 "${cargo_nextest_hash}" "${CARGO_HOME:-$HOME/.cargo}/bin/cargo-nextest"
+    fi
+}
+
+aarch64_cross_toolchain_hash=c8ee0e7fd58f5ec6811e3cec5fcdd8fc47cb2b49fb50e9d7717696ddb69c812547b5f389558f62dfbf9db7d6ad808a5a515cc466b8ea3e9ab3daeb20ba1adf33
+# save to directory that will be cached
+aarch64_cross_toolchain_save_path=${repo_root}/target/aarch64-linux-musl-cross.tgz
+if [ "$TARGET" == "aarch64-unknown-linux-musl" ]; then
+    if ! check_sha512 ${aarch64_cross_toolchain_hash} ${aarch64_cross_toolchain_save_path}; then
+        wget https://more.musl.cc/10-20210301/x86_64-linux-musl/aarch64-linux-musl-cross.tgz -O ${aarch64_cross_toolchain_save_path}
+        check_sha512 ${aarch64_cross_toolchain_hash} ${aarch64_cross_toolchain_save_path}
+    fi
+    tar -xf ${aarch64_cross_toolchain_save_path} -C /tmp;
+fi
+
+# download pre-built `cargo-nextest`
+kernel=$(uname)
+architecture=$(uname -m)
+case "$kernel-$architecture" in
+  Linux-x86_64 | Linux-amd64)
+    download_cargo_nextest "amd64" "d22ce5799f3056807fd0cd8223a290c7153a5f084d5ab931fce755c2cabd33f79c0f75542eb724fe07a7ca083f415ec1f84edc46584b06df43d97a0ff91018da" "https://get.nexte.st/0.9.52/linux"
+    ;;
+  Linux-arm64)
+    download_cargo_nextest "arm64" "cff3297c84560de8693e7f887fcf6cf33ab0036e27a9debf2b0a0832094555335f34dc30d0f9d1128ce8472dcb4594a3cf33be2357b19dcc94269b58090cc1a9" "https://get.nexte.st/0.9.52/linux-arm"
+    ;;
+  Darwin-x86_64)
+    download_cargo_nextest "Darwin-amd64" "0bb8b77ce019de3d06ee6b7382d830ed67309f187781e0de3866a0635879b494c7db48d55eee7553cfaa0bfca59abd8f8540a6d81ed703f06f9c81514d20073d" "https://get.nexte.st/0.9.52/mac"
+    ;;
+  MINGW64_NT-*-x86_64)
+    download_cargo_nextest "MINGW64-amd64" "3ffd504a4ef0b4b5e988457e6c525e62bd030d46b8f303f1c4e83a9a8ba89aef34bb239e23f391d1dddb75bea6ff074499153b2c71b06338a05d74916408de9c" "https://get.nexte.st/0.9.52/windows"
+    ;;
+  *)
+    echo "Unknown platform '$kernel-$architecture'"
+    exit 1
+    ;;
+esac