ci: More parallelization and caching (#2124)

* ci: Try and use artifacts to cache prebuilt NSS * needs * actions/cache * Fix * Fix * no sccache * sccache is killing our cache * Set env * Env * NSS_PREBUILT * Check if set * Cache on self-hosted runner * Fixes * Fixes * Fixes * Run fuzz in parallel * Invert * fuzz-bench * SCCACHE_INSTALLED && build -> check * Fixes * Fixes * Don't update rustup * Compile less
mozilla · Sep 18, 2024 · b780e53 · b780e53
1 parent 0e40d36
commit b780e53
Show file tree

Hide file tree

Showing 8 changed files with 125 additions and 45 deletions.
diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml
@@ -61,26 +61,29 @@ runs:
 
     - name: Use sccache
       # Apparently the action can't be installed twice in the same workflow, so check if
-      # it's already installed by checking if the RUSTC_WRAPPER environment variable is set
+      # it's already installed by checking if the SCCACHE_ENABLED environment variable is set
       # (which every "use" of this action needs to therefore set)
-      if: env.RUSTC_WRAPPER != 'sccache'
+      #
+      # Also, only enable sscache on our self-hosted runner, because the GitHub cache limit
+      # is too small for this to be effective there.
+      if: env.SCCACHE_ENABLED != '1' && env.BUILD_NSS == '1' && runner.environment != 'github-hosted'
       uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4
 
     - name: Enable sscache
+      if: env.BUILD_NSS == '1' && runner.environment != 'github-hosted'
       shell: bash
       run: |
+        echo "SCCACHE_ENABLED=1" >> "$GITHUB_ENV"
         if [ "${{ runner.os }}" != "Windows" ]; then
           # TODO: Figure out how to make this work on Windows
           echo "SCCACHE_CC=sccache cc" >> "$GITHUB_ENV"
           echo "SCCACHE_CXX=sccache c++" >> "$GITHUB_ENV"
         fi
         echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV"
         echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV"
-        if [ "$GITHUB_WORKFLOW" ]; then
+        if [ "${{ runner.environment }}" == "github-hosted" ]; then
           echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV"
         fi
-        echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV"
-        echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV"
 
     - name: Checkout NSS
       if: env.BUILD_NSS == '1'
@@ -96,6 +99,34 @@ runs:
         repository: nss-dev/nspr
         path: nspr
 
+    - name: Get head revisions
+      if: env.BUILD_NSS == '1'
+      shell: bash
+      run: |
+        NSS_HEAD=$(git -C nss rev-parse HEAD)
+        NSPR_HEAD=$(git -C nspr rev-parse HEAD)
+        echo "NSS_HEAD=$NSS_HEAD" >> "$GITHUB_ENV"
+        echo "NSPR_HEAD=$NSPR_HEAD" >> "$GITHUB_ENV"
+
+    - name: Cache NSS
+      id: cache
+      if: env.BUILD_NSS == '1' && runner.environment == 'github-hosted'
+      uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+      with:
+        path: dist
+        key: nss-${{ runner.os }}-${{ inputs.type }}-${{ env.NSS_HEAD }}-${{ env.NSPR_HEAD }}
+
+    - name: Check if build is needed
+      if: env.BUILD_NSS == '1' && runner.environment == 'github-hosted'
+      shell: bash
+      run: |
+        if [ "${{ steps.cache.outputs.cache-hit }}" == "true" ]; then
+          echo "Using cached prebuilt NSS"
+          echo "BUILD_NSS=0" >> "$GITHUB_ENV"
+        else
+          echo "Building NSS from source"
+        fi
+
     - name: Install build dependencies (Linux)
       shell: bash
       if: runner.os == 'Linux' && env.BUILD_NSS == '1' && runner.environment == 'github-hosted'
@@ -143,6 +174,21 @@ runs:
         # See https://github.com/ilammy/msvc-dev-cmd#name-conflicts-with-shell-bash
         rm /usr/bin/link.exe || true
 
+    - name: Set up environment
+      shell: bash
+      run: |
+        NSS_TARGET="${{ inputs.type }}"
+        echo "NSS_TARGET=$NSS_TARGET" >> "$GITHUB_ENV"
+        NSS_OUT="$NSS_DIR/../dist/$NSS_TARGET"
+        echo "LD_LIBRARY_PATH=$NSS_OUT/lib" >> "$GITHUB_ENV"
+        echo "DYLD_FALLBACK_LIBRARY_PATH=$NSS_OUT/lib" >> "$GITHUB_ENV"
+        echo "$NSS_OUT/lib" >> "$GITHUB_PATH"
+        echo "NSS_DIR=$NSS_DIR" >> "$GITHUB_ENV"
+        echo "NSS_PREBUILT=1" >> "$GITHUB_ENV"
+      env:
+        NSS_DIR: ${{ github.workspace }}/nss
+        NSPR_DIR: ${{ github.workspace }}/nspr
+
     - name: Build
       shell: bash
       if: env.BUILD_NSS == '1'
@@ -154,15 +200,5 @@ runs:
           OPT="-o"
           [ "${{ runner.os }}" != "Windows" ] && export CFLAGS="-ggdb3 -fno-omit-frame-pointer"
         fi
-        NSS_TARGET="${{ inputs.type }}"
-        echo "NSS_TARGET=$NSS_TARGET" >> "$GITHUB_ENV"
-        NSS_OUT="$NSS_DIR/../dist/$NSS_TARGET"
-        echo "LD_LIBRARY_PATH=$NSS_OUT/lib" >> "$GITHUB_ENV"
-        echo "DYLD_FALLBACK_LIBRARY_PATH=$NSS_OUT/lib" >> "$GITHUB_ENV"
-        echo "$NSS_OUT/lib" >> "$GITHUB_PATH"
-        echo "NSS_DIR=$NSS_DIR" >> "$GITHUB_ENV"
         [ "$SCCACHE_CC" ] && [ "$SCCACHE_CXX" ] && export CC="$SCCACHE_CC" CXX="$SCCACHE_CXX"
         $NSS_DIR/build.sh -g -Ddisable_tests=1 $OPT --static
-      env:
-        NSS_DIR: ${{ github.workspace }}/nss
-        NSPR_DIR: ${{ github.workspace }}/nspr
diff --git a/.github/actions/rust/action.yml b/.github/actions/rust/action.yml
@@ -21,11 +21,6 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: Upgrade rustup (MacOS)
-      shell: bash
-      if: runner.os == 'MacOS'
-      run: brew update && brew upgrade rustup
-
     - name: Install Rust
       uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17 # master
       with:
@@ -35,21 +30,24 @@ runs:
 
     - name: Use sccache
       # Apparently the action can't be installed twice in the same workflow, so check if
-      # it's already installed by checking if the RUSTC_WRAPPER environment variable is set
+      # it's already installed by checking if the SCCACHE_ENABLED environment variable is set
       # (which every "use" of this action needs to therefore set)
-      if: env.RUSTC_WRAPPER != 'sccache'
+      #
+      # Also, only enable sscache on our self-hosted runner, because the GitHub cache limit
+      # is too small for this to be effective there.
+      if: env.SCCACHE_ENABLED != '1' && runner.environment != 'github-hosted'
       uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4
 
     - name: Enable sscache
+      if: runner.environment != 'github-hosted'
       shell: bash
       run: |
-        echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV"
-        echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV"
-        if [ "$GITHUB_WORKFLOW" ]; then
-          echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV"
-        fi
+        echo "SCCACHE_ENABLED=1" >> "$GITHUB_ENV"
         echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV"
         echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV"
+        if [ "${{ runner.environment }}" == "github-hosted" ]; then
+          echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV"
+        fi
 
     - name: Set up MSVC (Windows)
       if: runner.os == 'Windows'

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
@@ -58,8 +58,8 @@ jobs:
 
       - name: Build neqo
         run: |
-          cargo "+$TOOLCHAIN" bench --features bench --no-run
-          cargo "+$TOOLCHAIN" build --release
+          cargo "+$TOOLCHAIN" bench --workspace --features bench --no-run
+          cargo "+$TOOLCHAIN" build --release --bin neqo-client --bin neqo-server
 
       - name: Build msquic
         run: |

diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
@@ -47,8 +47,8 @@ jobs:
       - uses: ./.github/actions/rust
         with:
           version: ${{ matrix.rust-toolchain }}
-          components: clippy, llvm-tools-preview
-          tools: cargo-llvm-cov, cargo-nextest, cargo-hack, cargo-fuzz
+          components: ${{ matrix.rust-toolchain == 'stable' && 'llvm-tools-preview' || '' }}
+          tools: ${{ matrix.rust-toolchain == 'stable' && 'cargo-llvm-cov, ' || '' }} cargo-nextest
           token: ${{ secrets.GITHUB_TOKEN }}
 
       - id: nss-version
@@ -58,20 +58,19 @@ jobs:
         with:
           minimum-version: ${{ steps.nss-version.outputs.minimum }}
 
-      - name: Build
+      - name: Check
         run: |
           # shellcheck disable=SC2086
-          cargo +${{ matrix.rust-toolchain }} build $BUILD_TYPE --all-targets --features ci
-          # Check that the fuzz targets also build
-          if [ ${{ startsWith(matrix.rust-toolchain, 'nightly') && 'nightly' }} == 'nightly' ]; then
-            cargo +${{ matrix.rust-toolchain }} fuzz check
-          fi
+          cargo +${{ matrix.rust-toolchain }} check $BUILD_TYPE --all-targets --features ci
 
       - name: Run tests and determine coverage
         run: |
           # shellcheck disable=SC2086
-          RUST_LOG=trace cargo +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --features ci --no-fail-fast --lcov --output-path lcov.info
-          cargo +${{ matrix.rust-toolchain }} bench --features bench --no-run
+          if [ "${{ matrix.rust-toolchain }}" == "stable" ]; then
+            RUST_LOG=trace cargo +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --features ci --no-fail-fast --lcov --output-path lcov.info
+          else
+            RUST_LOG=trace cargo +${{ matrix.rust-toolchain }} nextest run $BUILD_TYPE --features ci --no-fail-fast
+          fi
 
       - name: Run client/server transfer
         run: |

diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml
@@ -27,7 +27,7 @@ jobs:
       - uses: ./.github/actions/rust
         with:
           components: clippy
-          tools: cargo-hack, cargo-fuzz
+          tools: cargo-hack
           token: ${{ secrets.GITHUB_TOKEN }}
 
       - id: nss-version

diff --git a/.github/workflows/fuzz-bench.yml b/.github/workflows/fuzz-bench.yml
@@ -0,0 +1,39 @@
+name: Fuzz & Bench
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: ["main"]
+    paths-ignore: ["*.md", "*.png", "*.svg", "LICENSE-*"]
+  merge_group:
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  fuzz-bench:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: ./.github/actions/rust
+        with:
+          version: nightly
+          tools: cargo-fuzz
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - id: nss-version
+        run: echo "minimum=$(cat neqo-crypto/min_version.txt)" >> "$GITHUB_OUTPUT"
+
+      - uses: ./.github/actions/nss
+        with:
+          minimum-version: ${{ steps.nss-version.outputs.minimum }}
+
+      # Check that the fuzz and bench targets build
+      - run: cargo fuzz check
+      - run: cargo bench --features bench --no-run
diff --git a/Cargo.toml b/Cargo.toml
@@ -40,6 +40,11 @@ nursery = { level = "warn", priority = -1 }
 pedantic = { level = "warn", priority = -1 }
 multiple_crate_versions = "allow"
 
+# Optimize build dependencies, because bindgen and proc macros / style
+# compilation take more to run than to build otherwise.
+[profile.dev.build-override]
+opt-level = 1
+
 [profile.release]
 lto = "fat"
 

diff --git a/neqo-crypto/build.rs b/neqo-crypto/build.rs
@@ -109,14 +109,14 @@ fn get_bash() -> PathBuf {
     )
 }
 
-fn build_nss(dir: PathBuf) {
+fn build_nss(dir: PathBuf, nsstarget: &str) {
     let mut build_nss = vec![
         String::from("./build.sh"),
         String::from("-Ddisable_tests=1"),
         // Generate static libraries in addition to shared libraries.
         String::from("--static"),
     ];
-    if !is_debug() {
+    if nsstarget == "Release" {
         build_nss.push(String::from("-o"));
     }
     if let Ok(d) = env::var("NSS_JOBS") {
@@ -317,15 +317,18 @@ fn setup_standalone(nss: &str) -> Vec<String> {
         "The NSS_DIR environment variable is expected to be an absolute path."
     );
 
-    build_nss(nss.clone());
-
     // $NSS_DIR/../dist/
     let nssdist = nss.parent().unwrap().join("dist");
     println!("cargo:rerun-if-env-changed=NSS_TARGET");
     let nsstarget = env::var("NSS_TARGET")
         .unwrap_or_else(|_| fs::read_to_string(nssdist.join("latest")).unwrap());
-    let nsstarget = nssdist.join(nsstarget.trim());
 
+    // If NSS_PREBUILT is set, we assume that the NSS libraries are already built.
+    if env::var("NSS_PREBUILT").is_err() {
+        build_nss(nss, &nsstarget);
+    }
+
+    let nsstarget = nssdist.join(nsstarget.trim());
     let includes = get_includes(&nsstarget, &nssdist);
 
     let nsslibdir = nsstarget.join("lib");