From ddc2691168acbc6726ed3458348331244bd2a591 Mon Sep 17 00:00:00 2001 From: Gijs Kwakkel Date: Wed, 15 Mar 2023 12:27:00 +0100 Subject: [PATCH 01/10] Make mbedtls compile on windows --- mbedtls-platform-support/src/rust_printf.c | 8 +++++++- mbedtls-sys/Cargo.toml | 3 +++ mbedtls-sys/build/bindgen.rs | 13 +++++++------ mbedtls-sys/build/cmake.rs | 13 ++----------- mbedtls-sys/build/features.rs | 4 ++-- mbedtls-sys/src/types.rs | 2 +- mbedtls/src/pk/mod.rs | 2 +- 7 files changed, 23 insertions(+), 22 deletions(-) diff --git a/mbedtls-platform-support/src/rust_printf.c b/mbedtls-platform-support/src/rust_printf.c index c3b2ac93c..d11d02e44 100644 --- a/mbedtls-platform-support/src/rust_printf.c +++ b/mbedtls-platform-support/src/rust_printf.c @@ -8,6 +8,12 @@ #include #include +#ifdef _WIN32 +#define alloca _alloca +#include +#else +#include +#endif extern void mbedtls_log(const char* msg); @@ -22,7 +28,7 @@ extern int mbedtls_printf(const char *fmt, ...) { return -1; n++; - char p[n]; + char *p = alloca(n); va_start(ap,fmt); n=vsnprintf(p,n,fmt,ap); diff --git a/mbedtls-sys/Cargo.toml b/mbedtls-sys/Cargo.toml index e06808f38..a37efef35 100644 --- a/mbedtls-sys/Cargo.toml +++ b/mbedtls-sys/Cargo.toml @@ -22,6 +22,9 @@ cfg-if = "1.0.0" [target.'cfg(unix)'.dependencies] libc = { version = "0.2.0" } +[target.'cfg(windows)'.dependencies] +libc = { version = "0.2.0" } + [build-dependencies] bindgen = { version = "0.65.1", features = ["experimental"] } cmake = "0.1.17" diff --git a/mbedtls-sys/build/bindgen.rs b/mbedtls-sys/build/bindgen.rs index cbd2045ce..c5a450e06 100644 --- a/mbedtls-sys/build/bindgen.rs +++ b/mbedtls-sys/build/bindgen.rs @@ -72,12 +72,13 @@ impl super::BuildConfig { header.push_str("#include \n"); let mut cc = cc::Build::new(); - cc.include(&self.mbedtls_include) - .flag(&format!( - "-DMBEDTLS_CONFIG_FILE=\"{}\"", - self.config_h.to_str().expect("config.h UTF-8 error") - )); - + if cc.get_compiler().is_like_msvc() { + cc.flag("--driver-mode=cl"); + } + cc.include(&self.mbedtls_include).define( + "MBEDTLS_CONFIG_FILE", + Some(format!(r#""{}""#, self.config_h.to_str().expect("config.h UTF-8 error")).as_str()), + ); for cflag in &self.cflags { cc.flag(cflag); } diff --git a/mbedtls-sys/build/cmake.rs b/mbedtls-sys/build/cmake.rs index 32b8a4e0b..1b9bca354 100644 --- a/mbedtls-sys/build/cmake.rs +++ b/mbedtls-sys/build/cmake.rs @@ -19,7 +19,7 @@ impl super::BuildConfig { .define("ENABLE_TESTING", "OFF") // Prefer unix-style over Apple-style Python3 on macOS, required for the Github Actions CI .define("Python3_FIND_FRAMEWORK", "LAST") - .build_target("lib"); + .build_target("install"); for cflag in &self.cflags { cmk.cflag(cflag); } @@ -42,16 +42,7 @@ impl super::BuildConfig { let mut dst = cmk.build(); - dst.push("build"); - dst.push("library"); - println!( - "cargo:rustc-link-search=native={}", - dst.to_str().expect("link-search UTF-8 error") - ); - - assert!(dst.pop()); - dst.push("crypto"); - dst.push("library"); + dst.push("lib"); println!( "cargo:rustc-link-search=native={}", dst.to_str().expect("link-search UTF-8 error") diff --git a/mbedtls-sys/build/features.rs b/mbedtls-sys/build/features.rs index 61e7866b8..596425646 100644 --- a/mbedtls-sys/build/features.rs +++ b/mbedtls-sys/build/features.rs @@ -48,14 +48,14 @@ impl Features { } } if let Some(components) = self.with_feature("std") { - if env_have_target_cfg("family", "unix") { + if env_have_target_cfg("family", "unix") || env_have_target_cfg("family", "windows") { components.insert("net"); components.insert("fs"); components.insert("entropy"); } } if let Some(components) = self.with_feature("time") { - if !have_custom_gmtime_r && env_have_target_cfg("family", "unix") { + if !have_custom_gmtime_r && (env_have_target_cfg("family", "unix") || env_have_target_cfg("family", "windows")) { components.insert("libc"); } else { components.insert("custom"); diff --git a/mbedtls-sys/src/types.rs b/mbedtls-sys/src/types.rs index 31d7b3c4c..d5cb2baa9 100644 --- a/mbedtls-sys/src/types.rs +++ b/mbedtls-sys/src/types.rs @@ -76,7 +76,7 @@ pub mod raw_types { } } -#[cfg(unix)] +#[cfg(any(unix, windows))] extern crate libc; #[cfg(std_component = "fs")] diff --git a/mbedtls/src/pk/mod.rs b/mbedtls/src/pk/mod.rs index a47484183..112c681fe 100644 --- a/mbedtls/src/pk/mod.rs +++ b/mbedtls/src/pk/mod.rs @@ -38,7 +38,7 @@ pub use crate::ecp::EcGroup; pub use dhparam::Dhm; -const RAW_RSA_DECRYPT : i32 = 1040451858; +const RAW_RSA_DECRYPT: i32 = 1040451858; define!( #[c_ty(pk_type_t)] From 2dd834bb2e10d9bdfb8ed88c2459aed7ca7e039c Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Fri, 8 Sep 2023 14:57:05 -0700 Subject: [PATCH 02/10] ci: update CI for x86_64-pc-windows-msvc --- .github/workflows/test.yml | 4 ++ ct.sh | 98 ++++++++++++++++++++++---------------- 2 files changed, 60 insertions(+), 42 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 76bda5c4d..421094f82 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,6 +42,9 @@ jobs: - rust: stable target: x86_64-apple-darwin os: macos-latest + - rust: stable + target: x86_64-pc-windows-msvc + os: windows-latest runs-on: ${{ matrix.os }} @@ -74,6 +77,7 @@ jobs: TARGET: ${{ matrix.target }} ZLIB_INSTALLED: ${{ matrix.target == 'x86_64-unknown-linux-gnu' && 'true' || '' }} AES_NI_SUPPORT: ${{ matrix.target == 'x86_64-unknown-linux-gnu' && 'true' || '' }} + shell: bash ci-success: name: ci if: always() diff --git a/ct.sh b/ct.sh index bda77d8c8..2e424178b 100755 --- a/ct.sh +++ b/ct.sh @@ -7,7 +7,9 @@ if [ -z $TRAVIS_RUST_VERSION ]; then exit 1 fi -# checks if a file has a specific sha512 hash, compatible for Linux and macOS +# Setup dependencies and tools + +# checks if a file has a specific sha512 hash check_sha512() { local hash="$1" local file="$2" @@ -19,6 +21,9 @@ check_sha512() { Darwin) shasum -a 512 -c <<< "$hash *$file" ;; + MINGW64_NT-*) + sha512sum -c <<< "$hash *$file" + ;; *) echo "Unsupported platform '$platfom'" exit 1 @@ -37,15 +42,9 @@ if [ "$TARGET" == "aarch64-unknown-linux-musl" ]; then tar -xf ${aarch64_cross_toolchain_save_path} -C /tmp; fi -export CFLAGS_x86_64_fortanix_unknown_sgx="-isystem/usr/include/x86_64-linux-gnu -mlvi-hardening -mllvm -x86-experimental-lvi-inline-asm-hardening" -export CC_x86_64_fortanix_unknown_sgx=clang-11 -export CC_aarch64_unknown_linux_musl=/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc -export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc -export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER=qemu-aarch64 - -# Setup dependencies and tools - # According to `mbedtls-sys/vendor/README.md`, need to install needed pkgs +python3 -m venv venv +source venv/bin/activate || source venv/Scripts/activate python3 -m pip install -r ./mbedtls-sys/vendor/scripts/basic.requirements.txt # function for downloading pre-built `cargo-nextest` on various platforms @@ -55,7 +54,14 @@ download_cargo_nextest() { local url="$3" echo "Check if need to download pre-built $platform 'cargo-nextest'" if ! check_sha512 "${cargo_nextest_hash}" "${CARGO_HOME:-$HOME/.cargo}/bin/cargo-nextest"; then - curl -LsSf "$url" | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin" + case $platform in + MINGW64-*) + curl -LsSf "$url" -o temp.zip && unzip -d "${CARGO_HOME:-$HOME/.cargo}/bin" temp.zip && rm temp.zip + ;; + *) + curl -LsSf "$url" | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin" + ;; + esac check_sha512 "${cargo_nextest_hash}" "${CARGO_HOME:-$HOME/.cargo}/bin/cargo-nextest" fi } @@ -72,47 +78,55 @@ case "$kernel-$architecture" in Darwin-x86_64) download_cargo_nextest "Darwin-amd64" "0bb8b77ce019de3d06ee6b7382d830ed67309f187781e0de3866a0635879b494c7db48d55eee7553cfaa0bfca59abd8f8540a6d81ed703f06f9c81514d20073d" "https://get.nexte.st/0.9.52/mac" ;; + MINGW64_NT-*-x86_64) + download_cargo_nextest "MINGW64-amd64" "3ffd504a4ef0b4b5e988457e6c525e62bd030d46b8f303f1c4e83a9a8ba89aef34bb239e23f391d1dddb75bea6ff074499153b2c71b06338a05d74916408de9c" "https://get.nexte.st/0.9.52/windows" + ;; *) echo "Unknown platform '$kernel-$architecture'" exit 1 ;; esac - # Test logic start from here +export CFLAGS_x86_64_fortanix_unknown_sgx="-isystem/usr/include/x86_64-linux-gnu -mlvi-hardening -mllvm -x86-experimental-lvi-inline-asm-hardening" +export CC_x86_64_fortanix_unknown_sgx=clang-11 +export CC_aarch64_unknown_linux_musl=/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc +export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc +export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER=qemu-aarch64 + cd "./mbedtls" -if [ "$TRAVIS_RUST_VERSION" == "stable" ] || [ "$TRAVIS_RUST_VERSION" == "beta" ] || [ "$TRAVIS_RUST_VERSION" == "nightly" ]; then - # Install the rust toolchain - rustup default $TRAVIS_RUST_VERSION - rustup target add --toolchain $TRAVIS_RUST_VERSION $TARGET - printenv +case "$TRAVIS_RUST_VERSION" in + stable|beta|nightly) + # Install the rust toolchain + rustup default $TRAVIS_RUST_VERSION + rustup target add --toolchain $TRAVIS_RUST_VERSION $TARGET + printenv - # The SGX target cannot be run under test like a ELF binary - if [ "$TARGET" != "x86_64-fortanix-unknown-sgx" ]; then - # make sure that explicitly providing the default target works - cargo nextest run --target $TARGET --release - cargo nextest run --features dsa --target $TARGET - cargo nextest run --features async-rt,tls13 --target $TARGET + # The SGX target cannot be run under test like a ELF binary + if [ "$TARGET" != "x86_64-fortanix-unknown-sgx" ]; then + # make sure that explicitly providing the default target works + cargo nextest run --target $TARGET --release + cargo nextest run --features dsa --target $TARGET + cargo nextest run --features async-rt,tls13 --target $TARGET - # If AES-NI is supported, test the feature - if [ -n "$AES_NI_SUPPORT" ]; then - cargo nextest run --features force_aesni_support,tls13 --target $TARGET - fi + # If AES-NI is supported, test the feature + if [ -n "$AES_NI_SUPPORT" ]; then + cargo nextest run --features force_aesni_support,tls13 --target $TARGET + fi + # no_std tests only are able to run on x86 platform + if [ "$TARGET" == "x86_64-unknown-linux-gnu" ] || [ "$TARGET" == "x86_64-apple-darwin" ] || [[ "$TARGET" =~ ^x86_64-pc-windows- ]]; then + cargo nextest run --no-default-features --features no_std_deps,rdrand,time --target $TARGET + cargo nextest run --no-default-features --features no_std_deps --target $TARGET + fi - # no_std tests only are able to run on x86 platform - if [ "$TARGET" == "x86_64-unknown-linux-gnu" ]; then - cargo nextest run --no-default-features --features no_std_deps,rdrand,time --target $TARGET - cargo nextest run --no-default-features --features no_std_deps,rdrand --target $TARGET + else + cargo +$TRAVIS_RUST_VERSION test --no-run --target=$TARGET + cargo +$TRAVIS_RUST_VERSION test --no-default-features --features dsa,force_aesni_support,mpi_force_c_code,rdrand,std,time,tls13 --no-run --target=$TARGET fi - if [ "$TARGET" == "x86_64-apple-darwin" ]; then - cargo nextest run --no-default-features --features no_std_deps --target $TARGET - fi - else - cargo +$TRAVIS_RUST_VERSION test --no-run --target=$TARGET - cargo +$TRAVIS_RUST_VERSION test --no-default-features --features dsa,force_aesni_support,mpi_force_c_code,rdrand,std,time,tls13 --no-run --target=$TARGET - fi - -else - echo "Unknown version $TRAVIS_RUST_VERSION" - exit 1 -fi + ;; + *) + # Default case: If TRAVIS_RUST_VERSION does not match any of the above + echo "Unknown version $TRAVIS_RUST_VERSION" + exit 1 + ;; +esac From 34888dfcba61c6b03d52a84e4abd4929a2e57d55 Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Fri, 8 Sep 2023 14:57:30 -0700 Subject: [PATCH 03/10] fix: update code to be compile in windows --- mbedtls-platform-support/src/self_test.rs | 21 ++++++++++----------- mbedtls-sys/build/bindgen.rs | 2 +- mbedtls-sys/build/cmake.rs | 2 ++ mbedtls/src/ssl/config.rs | 2 +- mbedtls/tests/support/mod.rs | 1 + 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/mbedtls-platform-support/src/self_test.rs b/mbedtls-platform-support/src/self_test.rs index 73ec11123..35febb024 100644 --- a/mbedtls-platform-support/src/self_test.rs +++ b/mbedtls-platform-support/src/self_test.rs @@ -40,18 +40,17 @@ cfg_if::cfg_if! { } } } -cfg_if::cfg_if! { - if #[cfg(any(not(feature = "std"), target_env = "sgx"))] { - #[allow(non_upper_case_globals)] - static mut rand_f: Option c_int> = None; - // needs to be pub for global visiblity - #[doc(hidden)] - #[no_mangle] - pub unsafe extern "C" fn rand() -> c_int { - rand_f.expect("Called self-test rand without enabling self-test")() - } - } +#[cfg(any(not(feature = "std"), target_env = "sgx"))] +#[allow(non_upper_case_globals)] +static mut rand_f: Option c_int> = None; + +// needs to be pub for global visiblity +#[cfg(all(any(not(feature = "std"), target_env = "sgx"), not(target_env = "msvc")))] +#[doc(hidden)] +#[no_mangle] +pub unsafe extern "C" fn rand() -> c_int { + rand_f.expect("Called self-test rand without enabling self-test")() } /// Set callback functions to enable the MbedTLS self tests. diff --git a/mbedtls-sys/build/bindgen.rs b/mbedtls-sys/build/bindgen.rs index c5a450e06..d9a070629 100644 --- a/mbedtls-sys/build/bindgen.rs +++ b/mbedtls-sys/build/bindgen.rs @@ -108,7 +108,6 @@ impl super::BuildConfig { .clang_args(cc.get_compiler().args().iter().map(|arg| arg.to_str().unwrap())) .header_contents("bindgen-input.h", &header) .allowlist_function("^(?i)mbedtls_.*") - .allowlist_function("^(?i)psa_.*") .wrap_static_fns(true) .wrap_static_fns_path(&self.static_wrappers_c) .generate().expect("bindgen error"); @@ -160,6 +159,7 @@ fn bindgen_builder(cc: &cc::Build, header: &String) -> bindgen::Builder { .header_contents("bindgen-input.h", header) .allowlist_recursively(false) .blocklist_type("^mbedtls_time_t$") + .blocklist_item("^(?i)mbedtls_.*vsnprintf") .use_core() .ctypes_prefix("::types::raw_types") .parse_callbacks(Box::new(MbedtlsParseCallbacks)) diff --git a/mbedtls-sys/build/cmake.rs b/mbedtls-sys/build/cmake.rs index 1b9bca354..4708205ef 100644 --- a/mbedtls-sys/build/cmake.rs +++ b/mbedtls-sys/build/cmake.rs @@ -17,6 +17,8 @@ impl super::BuildConfig { )) .define("ENABLE_PROGRAMS", "OFF") .define("ENABLE_TESTING", "OFF") + // This is turn off on windows by default + .define("GEN_FILES", "ON") // Prefer unix-style over Apple-style Python3 on macOS, required for the Github Actions CI .define("Python3_FIND_FRAMEWORK", "LAST") .build_target("install"); diff --git a/mbedtls/src/ssl/config.rs b/mbedtls/src/ssl/config.rs index 3b709789d..ac12b9e08 100644 --- a/mbedtls/src/ssl/config.rs +++ b/mbedtls/src/ssl/config.rs @@ -45,7 +45,7 @@ define!( impl From for Version { fn from(value: u32) -> Self { use Version::*; - match value { + match value as ssl_protocol_version { SSL_VERSION_TLS1_2 => Tls12, #[cfg(feature = "tls13")] SSL_VERSION_TLS1_3 => Tls13, diff --git a/mbedtls/tests/support/mod.rs b/mbedtls/tests/support/mod.rs index a1c9b39a1..57e1d7bdd 100644 --- a/mbedtls/tests/support/mod.rs +++ b/mbedtls/tests/support/mod.rs @@ -9,6 +9,7 @@ #![allow(dead_code)] pub mod entropy; pub mod keys; +#[cfg(unix)] #[cfg(sys_std_component = "net")] pub mod net; pub mod rand; From e78944e34b95066eff391e3c2caf9050ebcda515 Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Fri, 8 Sep 2023 16:29:14 -0700 Subject: [PATCH 04/10] test: disable is_prime_tests on macos --- mbedtls/tests/bignum.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mbedtls/tests/bignum.rs b/mbedtls/tests/bignum.rs index ae6a9240f..d1fa7e1aa 100644 --- a/mbedtls/tests/bignum.rs +++ b/mbedtls/tests/bignum.rs @@ -417,6 +417,7 @@ fn test_base58_encode() { } #[cfg(feature = "rdrand")] +#[cfg(not(target_os = "macos"))] #[cfg(test)] mod is_prime_tests { use crate::Mpi; From a7851561483540a33904aaf7b2dd2011135f4178 Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Fri, 8 Sep 2023 21:55:11 -0700 Subject: [PATCH 05/10] ci: refactor --- .github/workflows/test.yml | 41 +++++++++++++------- .python-version | 1 + ci.sh | 56 +++++++++++++++++++++++++++ ct.sh => ci_tools.sh | 77 +++++++------------------------------- ct_locally.sh | 52 ------------------------- 5 files changed, 97 insertions(+), 130 deletions(-) create mode 100644 .python-version create mode 100755 ci.sh rename ct.sh => ci_tools.sh (54%) delete mode 100755 ct_locally.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 421094f82..60308a9ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,23 +28,23 @@ jobs: target: x86_64-unknown-linux-gnu os: ubuntu-20.04 - rust: stable - target: aarch64-unknown-linux-musl + target: x86_64-fortanix-unknown-sgx os: ubuntu-20.04 - rust: stable - target: x86_64-fortanix-unknown-sgx + target: x86_64-pc-windows-msvc + os: windows-latest + - rust: stable + target: aarch64-unknown-linux-musl os: ubuntu-20.04 + - rust: stable + target: x86_64-apple-darwin + os: macos-latest - rust: beta target: x86_64-unknown-linux-gnu os: ubuntu-20.04 - rust: nightly target: x86_64-unknown-linux-gnu os: ubuntu-20.04 - - rust: stable - target: x86_64-apple-darwin - os: macos-latest - - rust: stable - target: x86_64-pc-windows-msvc - os: windows-latest runs-on: ${{ matrix.os }} @@ -57,21 +57,34 @@ jobs: sudo apt-get update sudo apt-get install -y qemu-user - - name: Setup Rust toolchain - uses: actions-rs/toolchain@v1 + # python version is read from .python-version + - name: Setup python + uses: actions/setup-python@v4 with: - toolchain: ${{ matrix.rust }} - target: ${{ matrix.target }} - override: true + cache: 'pip' + cache-dependency-path: | + mbedtls-sys/vendor/scripts/basic.requirements.txt + mbedtls-sys/vendor/scripts/driver.requirements.txt + + - name: Install python dependencies + run: python3 -m pip install -r mbedtls-sys/vendor/scripts/basic.requirements.txt - name: Cache Dependencies uses: Swatinem/rust-cache@988c164c3d0e93c4dbab36aaf5bbeb77425b2894 with: key: ${{ matrix.rust }} + - name: Setup Rust toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.rust }} + target: ${{ matrix.target }} + override: true + - name: Run tests run: | - ./ct.sh + ./ci_tools.sh + ./ci.sh env: TRAVIS_RUST_VERSION: ${{ matrix.rust }} TARGET: ${{ matrix.target }} diff --git a/.python-version b/.python-version new file mode 100644 index 000000000..d20cc2bf0 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.8.10 diff --git a/ci.sh b/ci.sh new file mode 100755 index 000000000..29dbc4a51 --- /dev/null +++ b/ci.sh @@ -0,0 +1,56 @@ +#!/bin/bash +set -ex +cd "$(dirname "$0")" + +repo_root=$(readlink -f $(dirname "${BASH_SOURCE[0]}")) + +if [ -z $TRAVIS_RUST_VERSION ]; then + echo "Expected TRAVIS_RUST_VERSION to be set in env" + exit 1 +fi + +python3 -m pip install -r ./mbedtls-sys/vendor/scripts/basic.requirements.txt + +# Test logic start from here +export CFLAGS_x86_64_fortanix_unknown_sgx="-isystem/usr/include/x86_64-linux-gnu -mlvi-hardening -mllvm -x86-experimental-lvi-inline-asm-hardening" +export CC_x86_64_fortanix_unknown_sgx=clang-11 +export CC_aarch64_unknown_linux_musl=/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc +export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=/tmp/target/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc +export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER=qemu-aarch64 + +cd "${repo_root}/mbedtls" +case "$TRAVIS_RUST_VERSION" in + stable|beta|nightly) + # Install the rust toolchain + rustup default $TRAVIS_RUST_VERSION + rustup target add --toolchain $TRAVIS_RUST_VERSION $TARGET + printenv + + # The SGX target cannot be run under test like a ELF binary + if [ "$TARGET" != "x86_64-fortanix-unknown-sgx" ]; then + # make sure that explicitly providing the default target works + cargo nextest run --target $TARGET --release + cargo nextest run --features dsa --target $TARGET + cargo nextest run --features async-rt,tls13 --target $TARGET + + # If AES-NI is supported, test the feature + if [ -n "$AES_NI_SUPPORT" ]; then + cargo nextest run --features force_aesni_support,tls13 --target $TARGET + fi + # no_std tests only are able to run on x86 platform + if [ "$TARGET" == "x86_64-unknown-linux-gnu" ] || [ "$TARGET" == "x86_64-apple-darwin" ] || [[ "$TARGET" =~ ^x86_64-pc-windows- ]]; then + cargo nextest run --no-default-features --features no_std_deps,rdrand,time --target $TARGET + cargo nextest run --no-default-features --features no_std_deps --target $TARGET + fi + + else + cargo +$TRAVIS_RUST_VERSION test --no-run --target=$TARGET + cargo +$TRAVIS_RUST_VERSION test --no-default-features --features dsa,force_aesni_support,mpi_force_c_code,rdrand,std,time,tls13 --no-run --target=$TARGET + fi + ;; + *) + # Default case: If TRAVIS_RUST_VERSION does not match any of the above + echo "Unknown version $TRAVIS_RUST_VERSION" + exit 1 + ;; +esac diff --git a/ct.sh b/ci_tools.sh similarity index 54% rename from ct.sh rename to ci_tools.sh index 2e424178b..edffece12 100755 --- a/ct.sh +++ b/ci_tools.sh @@ -2,10 +2,7 @@ set -ex cd "$(dirname "$0")" -if [ -z $TRAVIS_RUST_VERSION ]; then - echo "Expected TRAVIS_RUST_VERSION to be set in env" - exit 1 -fi +repo_root=$(readlink -f $(dirname "${BASH_SOURCE[0]}")) # Setup dependencies and tools @@ -31,22 +28,6 @@ check_sha512() { esac } -aarch64_cross_toolchain_hash=c8ee0e7fd58f5ec6811e3cec5fcdd8fc47cb2b49fb50e9d7717696ddb69c812547b5f389558f62dfbf9db7d6ad808a5a515cc466b8ea3e9ab3daeb20ba1adf33 -# save to directory that will be cached -aarch64_cross_toolchain_save_path=/tmp/aarch64-linux-musl-cross.tgz -if [ "$TARGET" == "aarch64-unknown-linux-musl" ]; then - if ! check_sha512 ${aarch64_cross_toolchain_hash} ${aarch64_cross_toolchain_save_path}; then - wget https://more.musl.cc/10-20210301/x86_64-linux-musl/aarch64-linux-musl-cross.tgz -O ${aarch64_cross_toolchain_save_path} - check_sha512 ${aarch64_cross_toolchain_hash} ${aarch64_cross_toolchain_save_path} - fi - tar -xf ${aarch64_cross_toolchain_save_path} -C /tmp; -fi - -# According to `mbedtls-sys/vendor/README.md`, need to install needed pkgs -python3 -m venv venv -source venv/bin/activate || source venv/Scripts/activate -python3 -m pip install -r ./mbedtls-sys/vendor/scripts/basic.requirements.txt - # function for downloading pre-built `cargo-nextest` on various platforms download_cargo_nextest() { local platform="$1" @@ -65,6 +46,18 @@ download_cargo_nextest() { check_sha512 "${cargo_nextest_hash}" "${CARGO_HOME:-$HOME/.cargo}/bin/cargo-nextest" fi } + +aarch64_cross_toolchain_hash=c8ee0e7fd58f5ec6811e3cec5fcdd8fc47cb2b49fb50e9d7717696ddb69c812547b5f389558f62dfbf9db7d6ad808a5a515cc466b8ea3e9ab3daeb20ba1adf33 +# save to directory that will be cached +aarch64_cross_toolchain_save_path=${repo_root}/target/aarch64-linux-musl-cross.tgz +if [ "$TARGET" == "aarch64-unknown-linux-musl" ]; then + if ! check_sha512 ${aarch64_cross_toolchain_hash} ${aarch64_cross_toolchain_save_path}; then + wget https://more.musl.cc/10-20210301/x86_64-linux-musl/aarch64-linux-musl-cross.tgz -O ${aarch64_cross_toolchain_save_path} + check_sha512 ${aarch64_cross_toolchain_hash} ${aarch64_cross_toolchain_save_path} + fi + tar -xf ${aarch64_cross_toolchain_save_path} -C /tmp; +fi + # download pre-built `cargo-nextest` kernel=$(uname) architecture=$(uname -m) @@ -86,47 +79,3 @@ case "$kernel-$architecture" in exit 1 ;; esac - -# Test logic start from here -export CFLAGS_x86_64_fortanix_unknown_sgx="-isystem/usr/include/x86_64-linux-gnu -mlvi-hardening -mllvm -x86-experimental-lvi-inline-asm-hardening" -export CC_x86_64_fortanix_unknown_sgx=clang-11 -export CC_aarch64_unknown_linux_musl=/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc -export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc -export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER=qemu-aarch64 - -cd "./mbedtls" -case "$TRAVIS_RUST_VERSION" in - stable|beta|nightly) - # Install the rust toolchain - rustup default $TRAVIS_RUST_VERSION - rustup target add --toolchain $TRAVIS_RUST_VERSION $TARGET - printenv - - # The SGX target cannot be run under test like a ELF binary - if [ "$TARGET" != "x86_64-fortanix-unknown-sgx" ]; then - # make sure that explicitly providing the default target works - cargo nextest run --target $TARGET --release - cargo nextest run --features dsa --target $TARGET - cargo nextest run --features async-rt,tls13 --target $TARGET - - # If AES-NI is supported, test the feature - if [ -n "$AES_NI_SUPPORT" ]; then - cargo nextest run --features force_aesni_support,tls13 --target $TARGET - fi - # no_std tests only are able to run on x86 platform - if [ "$TARGET" == "x86_64-unknown-linux-gnu" ] || [ "$TARGET" == "x86_64-apple-darwin" ] || [[ "$TARGET" =~ ^x86_64-pc-windows- ]]; then - cargo nextest run --no-default-features --features no_std_deps,rdrand,time --target $TARGET - cargo nextest run --no-default-features --features no_std_deps --target $TARGET - fi - - else - cargo +$TRAVIS_RUST_VERSION test --no-run --target=$TARGET - cargo +$TRAVIS_RUST_VERSION test --no-default-features --features dsa,force_aesni_support,mpi_force_c_code,rdrand,std,time,tls13 --no-run --target=$TARGET - fi - ;; - *) - # Default case: If TRAVIS_RUST_VERSION does not match any of the above - echo "Unknown version $TRAVIS_RUST_VERSION" - exit 1 - ;; -esac diff --git a/ct_locally.sh b/ct_locally.sh deleted file mode 100755 index eecf4fafd..000000000 --- a/ct_locally.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -set -ex - -cwd=`pwd` -export script_dir="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" - -export RUST_BACKTRACE=1 -export TRAVIS_HOME=$HOME - -targets=() -targets+=("x86_64-fortanix-unknown-sgx") -targets+=("aarch64-unknown-linux-musl") -targets+=("x86_64-unknown-linux-gnu") - -versions=() -versions+=("beta") -versions+=("nightly") - -aarch64_cross_toolchain_hash=c8ee0e7fd58f5ec6811e3cec5fcdd8fc47cb2b49fb50e9d7717696ddb69c812547b5f389558f62dfbf9db7d6ad808a5a515cc466b8ea3e9ab3daeb20ba1adf33 -# save to directorie that will be cached -aarch64_cross_toolchain_save_path=$TRAVIS_HOME/.rustup/aarch64-linux-musl-cross.tgz -if [ "$TARGET" == "aarch64-unknown-linux-musl" ]; then - if ! echo "${aarch64_cross_toolchain_hash} ${aarch64_cross_toolchain_save_path}" | sha512sum -c; then - wget https://more.musl.cc/10-20210301/x86_64-linux-musl/aarch64-linux-musl-cross.tgz -O ${aarch64_cross_toolchain_save_path} - echo "${aarch64_cross_toolchain_hash} ${aarch64_cross_toolchain_save_path}" | sha512sum -c - fi - tar -xf ${aarch64_cross_toolchain_save_path} -C /tmp; -fi - -export CC_aarch64_unknown_linux_musl=/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc -export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc -export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER=qemu-aarch64 - - -for local_target in "${targets[@]}" -do - export TARGET=$local_target - export TRAVIS_RUST_VERSION="stable" - $script_dir/ct.sh -done - - -for local_version in "${versions[@]}" -do - export TARGET="x86_64-unknown-linux-gnu" - export AES_NI_SUPPORT=true - export ZLIB_INSTALLED=true - export TRAVIS_RUST_VERSION=$local_version - $script_dir/ct.sh -done - -cd $cwd From e12b26121ff556813fdb1d2ee5843b4d39210b13 Mon Sep 17 00:00:00 2001 From: YxC Date: Sat, 9 Sep 2023 14:47:02 -0700 Subject: [PATCH 06/10] ci: fix wrong path --- ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index 29dbc4a51..16bd80768 100755 --- a/ci.sh +++ b/ci.sh @@ -15,7 +15,7 @@ python3 -m pip install -r ./mbedtls-sys/vendor/scripts/basic.requirements.txt export CFLAGS_x86_64_fortanix_unknown_sgx="-isystem/usr/include/x86_64-linux-gnu -mlvi-hardening -mllvm -x86-experimental-lvi-inline-asm-hardening" export CC_x86_64_fortanix_unknown_sgx=clang-11 export CC_aarch64_unknown_linux_musl=/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc -export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=/tmp/target/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc +export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER=qemu-aarch64 cd "${repo_root}/mbedtls" From c2ec6884410c3d8e9ac2679931d3980e1c068592 Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Mon, 11 Sep 2023 16:16:41 -0700 Subject: [PATCH 07/10] build: bump versions --- Cargo.lock | 4 ++-- mbedtls-platform-support/Cargo.toml | 2 +- mbedtls-sys/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d0f16be7..8e57bd1b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -634,7 +634,7 @@ dependencies = [ [[package]] name = "mbedtls-platform-support" -version = "0.3.0" +version = "0.3.1" dependencies = [ "cc", "cfg-if", @@ -647,7 +647,7 @@ dependencies = [ [[package]] name = "mbedtls-sys-auto" -version = "3.5.0-alpha.3+0b3de6f" +version = "3.5.0-alpha.4+0b3de6f" dependencies = [ "bindgen", "cc", diff --git a/mbedtls-platform-support/Cargo.toml b/mbedtls-platform-support/Cargo.toml index 67f6aa2c3..9289f985b 100644 --- a/mbedtls-platform-support/Cargo.toml +++ b/mbedtls-platform-support/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mbedtls-platform-support" -version = "0.3.0" +version = "0.3.1" authors = ["Yuxiang Cao "] build = "build.rs" edition = "2018" diff --git a/mbedtls-sys/Cargo.toml b/mbedtls-sys/Cargo.toml index a37efef35..aef38be2b 100644 --- a/mbedtls-sys/Cargo.toml +++ b/mbedtls-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mbedtls-sys-auto" -version = "3.5.0-alpha.3+0b3de6f" +version = "3.5.0-alpha.4+0b3de6f" authors = ["Jethro Beekman "] build = "build/build.rs" license = "Apache-2.0 OR GPL-2.0-or-later" From 91211a41320357ad5d8482b9ce2d21ce7cebc0af Mon Sep 17 00:00:00 2001 From: YX Cao Date: Thu, 14 Sep 2023 11:07:08 -0700 Subject: [PATCH 08/10] ci: use python venv --- .github/workflows/test.yml | 5 ----- ci.sh | 3 +++ 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 60308a9ed..478110425 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,11 +60,6 @@ jobs: # python version is read from .python-version - name: Setup python uses: actions/setup-python@v4 - with: - cache: 'pip' - cache-dependency-path: | - mbedtls-sys/vendor/scripts/basic.requirements.txt - mbedtls-sys/vendor/scripts/driver.requirements.txt - name: Install python dependencies run: python3 -m pip install -r mbedtls-sys/vendor/scripts/basic.requirements.txt diff --git a/ci.sh b/ci.sh index 16bd80768..ac791fbf0 100755 --- a/ci.sh +++ b/ci.sh @@ -9,6 +9,9 @@ if [ -z $TRAVIS_RUST_VERSION ]; then exit 1 fi +# According to `mbedtls-sys/vendor/README.md`, need to install needed pkgs +python3 -m venv venv +source venv/bin/activate || source venv/Scripts/activate python3 -m pip install -r ./mbedtls-sys/vendor/scripts/basic.requirements.txt # Test logic start from here From b298c47486624cf5e7b4093f6bc1de83983c5822 Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Thu, 14 Sep 2023 11:13:24 -0700 Subject: [PATCH 09/10] cI: try fix --- .github/workflows/test.yml | 36 ++++++++++++++++++------------------ ci.sh | 4 ++-- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 478110425..6f45db54e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,27 +24,27 @@ jobs: strategy: matrix: include: - - rust: stable - target: x86_64-unknown-linux-gnu - os: ubuntu-20.04 - - rust: stable - target: x86_64-fortanix-unknown-sgx - os: ubuntu-20.04 + # - rust: stable + # target: x86_64-unknown-linux-gnu + # os: ubuntu-20.04 + # - rust: stable + # target: x86_64-fortanix-unknown-sgx + # os: ubuntu-20.04 - rust: stable target: x86_64-pc-windows-msvc os: windows-latest - - rust: stable - target: aarch64-unknown-linux-musl - os: ubuntu-20.04 - - rust: stable - target: x86_64-apple-darwin - os: macos-latest - - rust: beta - target: x86_64-unknown-linux-gnu - os: ubuntu-20.04 - - rust: nightly - target: x86_64-unknown-linux-gnu - os: ubuntu-20.04 + # - rust: stable + # target: aarch64-unknown-linux-musl + # os: ubuntu-20.04 + # - rust: stable + # target: x86_64-apple-darwin + # os: macos-latest + # - rust: beta + # target: x86_64-unknown-linux-gnu + # os: ubuntu-20.04 + # - rust: nightly + # target: x86_64-unknown-linux-gnu + # os: ubuntu-20.04 runs-on: ${{ matrix.os }} diff --git a/ci.sh b/ci.sh index ac791fbf0..b8564bdf3 100755 --- a/ci.sh +++ b/ci.sh @@ -10,9 +10,9 @@ if [ -z $TRAVIS_RUST_VERSION ]; then fi # According to `mbedtls-sys/vendor/README.md`, need to install needed pkgs -python3 -m venv venv +python -m venv venv source venv/bin/activate || source venv/Scripts/activate -python3 -m pip install -r ./mbedtls-sys/vendor/scripts/basic.requirements.txt +python -m pip install -r ./mbedtls-sys/vendor/scripts/basic.requirements.txt # Test logic start from here export CFLAGS_x86_64_fortanix_unknown_sgx="-isystem/usr/include/x86_64-linux-gnu -mlvi-hardening -mllvm -x86-experimental-lvi-inline-asm-hardening" From 777c9657dcda309a21c170d4ae34c453594fbd9b Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Thu, 14 Sep 2023 11:20:56 -0700 Subject: [PATCH 10/10] ci: clean up --- .github/workflows/test.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6f45db54e..478110425 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,27 +24,27 @@ jobs: strategy: matrix: include: - # - rust: stable - # target: x86_64-unknown-linux-gnu - # os: ubuntu-20.04 - # - rust: stable - # target: x86_64-fortanix-unknown-sgx - # os: ubuntu-20.04 + - rust: stable + target: x86_64-unknown-linux-gnu + os: ubuntu-20.04 + - rust: stable + target: x86_64-fortanix-unknown-sgx + os: ubuntu-20.04 - rust: stable target: x86_64-pc-windows-msvc os: windows-latest - # - rust: stable - # target: aarch64-unknown-linux-musl - # os: ubuntu-20.04 - # - rust: stable - # target: x86_64-apple-darwin - # os: macos-latest - # - rust: beta - # target: x86_64-unknown-linux-gnu - # os: ubuntu-20.04 - # - rust: nightly - # target: x86_64-unknown-linux-gnu - # os: ubuntu-20.04 + - rust: stable + target: aarch64-unknown-linux-musl + os: ubuntu-20.04 + - rust: stable + target: x86_64-apple-darwin + os: macos-latest + - rust: beta + target: x86_64-unknown-linux-gnu + os: ubuntu-20.04 + - rust: nightly + target: x86_64-unknown-linux-gnu + os: ubuntu-20.04 runs-on: ${{ matrix.os }}