CI: uses shippable toolchains

ghostery · Sep 7, 2023 · 088e0c6 · 088e0c6
1 parent 6b72ca2
commit 088e0c6
Show file tree

Hide file tree

Showing 6 changed files with 34 additions and 108 deletions.
diff --git a/build/Linux.dockerfile b/build/Linux.dockerfile
@@ -1,14 +1,19 @@
 FROM ua-build-base
 
-ENV PERFHERDER_EXTRA_OPTIONS="opt asan" \
-    MOZ_AUTOMATION_PACKAGE_TESTS="1"
+ENV MOZ_AUTOMATION_PACKAGE_TESTS="1" \
+    MOZ_ENABLE_FULL_SYMBOLS="1"
+
+RUN wget -nv -O /builds/worker/fetches/binutils.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-binutils/binutils.tar.zst && \
+    cd /builds/worker/fetches/ && \
+    tar -xf binutils.tar.zst && \
+    rm binutils.tar.zst
 
 RUN wget -nv -O /builds/worker/fetches/clang.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-clang-16/clang.tar.zst && \
     cd /builds/worker/fetches/ && \
     tar -xf clang.tar.zst && \
     rm clang.tar.zst
 
-RUN wget -nv -O /builds/worker/fetches/rustc.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-rust-dev/rustc.tar.zst && \
+RUN wget -nv -O /builds/worker/fetches/rustc.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-rust-1.71/rustc.tar.zst && \
     cd /builds/worker/fetches/ && \
     tar -xf rustc.tar.zst && \
     rm rustc.tar.zst
@@ -28,16 +33,6 @@ RUN wget -nv -O /builds/worker/fetches/dump_syms.tar.zst https://ghostery-user-a
     tar -xf dump_syms.tar.zst && \
     rm dump_syms.tar.zst
 
-RUN wget -nv -O /builds/worker/fetches/llvm-symbolizer.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-llvm-symbolizer-16/llvm-symbolizer.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf llvm-symbolizer.tar.zst && \
-    rm llvm-symbolizer.tar.zst
-
-RUN wget -nv -O /builds/worker/fetches/sccache.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-sccache/sccache.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf sccache.tar.zst && \
-    rm sccache.tar.zst
-
 RUN wget -nv -O /builds/worker/fetches/nasm.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-nasm/nasm.tar.zst && \
     cd /builds/worker/fetches/ && \
     tar -xf nasm.tar.zst && \

diff --git a/build/MacOSARM.dockerfile b/build/MacOSARM.dockerfile
@@ -1,23 +1,18 @@
 FROM ua-build-base
 
 ENV MOZ_AUTOMATION_PACKAGE_TESTS="1" \
-    PERFHERDER_EXTRA_OPTIONS="aarch64-asan-fuzzing"
+    PERFHERDER_EXTRA_OPTIONS="aarch64"
+
+RUN wget -nv -O /builds/worker/fetches/cctools.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-cctools-port/cctools.tar.zst && \
+    cd /builds/worker/fetches/ && \
+    tar -xf cctools.tar.zst && \
+    rm cctools.tar.zst
 
 RUN wget -nv -O /builds/worker/fetches/clang.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-clang-16/clang.tar.zst && \
     cd /builds/worker/fetches/ && \
     tar -xf clang.tar.zst && \
     rm clang.tar.zst
 
-RUN wget -nv -O /builds/worker/fetches/sccache.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-sccache/sccache.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf sccache.tar.zst && \
-    rm sccache.tar.zst
-
-RUN wget -nv -O /builds/worker/fetches/llvm-symbolizer.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/macosx64-aarch64-llvm-symbolizer-16/llvm-symbolizer.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf llvm-symbolizer.tar.zst && \
-    rm llvm-symbolizer.tar.zst
-
 RUN wget -nv -O /builds/worker/fetches/sysroot-wasm32-wasi.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/sysroot-wasm32-wasi-clang-16/sysroot-wasm32-wasi.tar.zst && \
     cd /builds/worker/fetches/ && \
     tar -xf sysroot-wasm32-wasi.tar.zst && \
@@ -83,4 +78,4 @@ ENV MOZ_FETCHES_DIR=/builds/worker/fetches/ \
 
 COPY configs /builds/worker/configs
 
-WORKDIR $WORKSPACE
+WORKDIR $WORKSPACE
diff --git a/build/MacOSX.dockerfile b/build/MacOSX.dockerfile
@@ -1,23 +1,17 @@
 FROM ua-build-base
 
-ENV MOZ_AUTOMATION_PACKAGE_TESTS="1" \
-    PERFHERDER_EXTRA_OPTIONS="asan-fuzzing"
+ENV MOZ_AUTOMATION_PACKAGE_TESTS="1"
+
+RUN wget -nv -O /builds/worker/fetches/cctools.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-cctools-port/cctools.tar.zst && \
+    cd /builds/worker/fetches/ && \
+    tar -xf cctools.tar.zst && \
+    rm cctools.tar.zst
 
 RUN wget -nv -O /builds/worker/fetches/clang.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-clang-16/clang.tar.zst && \
     cd /builds/worker/fetches/ && \
     tar -xf clang.tar.zst && \
     rm clang.tar.zst
 
-RUN wget -nv -O /builds/worker/fetches/sccache.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-sccache/sccache.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf sccache.tar.zst && \
-    rm sccache.tar.zst
-
-RUN wget -nv -O /builds/worker/fetches/llvm-symbolizer.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/macosx64-llvm-symbolizer-16/llvm-symbolizer.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf llvm-symbolizer.tar.zst && \
-    rm llvm-symbolizer.tar.zst
-
 RUN wget -nv -O /builds/worker/fetches/sysroot-wasm32-wasi.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/sysroot-wasm32-wasi-clang-16/sysroot-wasm32-wasi.tar.zst && \
     cd /builds/worker/fetches/ && \
     tar -xf sysroot-wasm32-wasi.tar.zst && \
@@ -83,4 +77,4 @@ ENV MOZ_FETCHES_DIR=/builds/worker/fetches/ \
 
 COPY configs /builds/worker/configs
 
-WORKDIR $WORKSPACE
+WORKDIR $WORKSPACE
diff --git a/build/Windows.dockerfile b/build/Windows.dockerfile
@@ -1,7 +1,6 @@
 FROM ua-build-base
 
-ENV PERFHERDER_EXTRA_OPTIONS="opt asan" \
-    MOZ_AUTOMATION_PACKAGE_TESTS="1"
+ENV MOZ_AUTOMATION_PACKAGE_TESTS="1"
 
 RUN /builds/worker/bin/fetch-content static-url \
     --sha256 5c076f87ba64d82f11513f4af0ceb07246a3540aa3c72ca3ffc2d53971fa56e3 \
@@ -42,11 +41,6 @@ RUN wget -nv -O /builds/worker/fetches/cbindgen.tar.zst https://ghostery-user-ag
     tar -xf cbindgen.tar.zst && \
     rm cbindgen.tar.zst
 
-RUN wget -nv -O /builds/worker/fetches/sccache.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-sccache/sccache.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf sccache.tar.zst && \
-    rm sccache.tar.zst
-
 RUN wget -nv -O /builds/worker/fetches/dump_syms.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-dump_syms/dump_syms.tar.zst && \
     cd /builds/worker/fetches/ && \
     tar -xf dump_syms.tar.zst && \
@@ -77,11 +71,6 @@ RUN wget -nv -O /builds/worker/fetches/sysroot-wasm32-wasi.tar.zst https://ghost
     tar -xf sysroot-wasm32-wasi.tar.zst && \
     rm sysroot-wasm32-wasi.tar.zst
 
-RUN wget -nv -O /builds/worker/fetches/llvm-symbolizer.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/win64-llvm-symbolizer-16/llvm-symbolizer.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf llvm-symbolizer.tar.zst && \
-    rm llvm-symbolizer.tar.zst
-
 ADD --chown=worker:worker makecab.exe /builds/worker/fetches/
 
 ENV MOZ_FETCHES_DIR=/builds/worker/fetches/ \

diff --git a/build/WindowsARM.dockerfile b/build/WindowsARM.dockerfile
@@ -1,7 +1,9 @@
 FROM ua-build-base
 
 ENV PERFHERDER_EXTRA_OPTIONS="aarch64" \
-    MOZ_AUTOMATION_PACKAGE_TESTS="1"
+    MOZ_ARTIFACT_TASK="[object Object]" \
+    MOZ_ARTIFACT_TASK_WIN32_OPT="[object Object]" \
+    USE_ARTIFACT="1"
 
 RUN /builds/worker/bin/fetch-content static-url \
     --sha256 5c076f87ba64d82f11513f4af0ceb07246a3540aa3c72ca3ffc2d53971fa56e3 \
@@ -12,71 +14,21 @@ RUN /builds/worker/bin/fetch-content static-url \
     unzip upx-3.95-win64.zip && \
     rm upx-3.95-win64.zip
 
-RUN wget -nv -O /builds/worker/fetches/clang.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-clang-16/clang.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf clang.tar.zst && \
-    rm clang.tar.zst
-
-RUN wget -nv -O /builds/worker/fetches/rustc.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-rust-cross-1.71/rustc.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf rustc.tar.zst && \
-    rm rustc.tar.zst
-
-RUN wget -nv -O /builds/worker/fetches/rust-size.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-rust-size/rust-size.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf rust-size.tar.zst && \
-    rm rust-size.tar.zst
-
-RUN wget -nv -O /builds/worker/fetches/nasm.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-nasm/nasm.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf nasm.tar.zst && \
-    rm nasm.tar.zst
-
 RUN wget -nv -O /builds/worker/fetches/node.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-node-16/node.tar.zst && \
     cd /builds/worker/fetches/ && \
     tar -xf node.tar.zst && \
     rm node.tar.zst
 
-RUN wget -nv -O /builds/worker/fetches/cbindgen.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-cbindgen/cbindgen.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf cbindgen.tar.zst && \
-    rm cbindgen.tar.zst
-
-RUN wget -nv -O /builds/worker/fetches/sccache.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-sccache/sccache.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf sccache.tar.zst && \
-    rm sccache.tar.zst
-
-RUN wget -nv -O /builds/worker/fetches/dump_syms.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-dump_syms/dump_syms.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf dump_syms.tar.zst && \
-    rm dump_syms.tar.zst
-
 RUN wget -nv -O /builds/worker/fetches/wine.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-wine/wine.tar.zst && \
     cd /builds/worker/fetches/ && \
     tar -xf wine.tar.zst && \
     rm wine.tar.zst
 
-RUN wget -nv -O /builds/worker/fetches/winchecksec.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/linux64-winchecksec/winchecksec.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf winchecksec.tar.zst && \
-    rm winchecksec.tar.zst
-
 RUN wget -nv -O /builds/worker/fetches/nsis.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/nsis/nsis.tar.zst && \
     cd /builds/worker/fetches/ && \
     tar -xf nsis.tar.zst && \
     rm nsis.tar.zst
 
-RUN wget -nv -O /builds/worker/fetches/sysroot-x86_64-linux-gnu.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/sysroot-x86_64-linux-gnu/sysroot-x86_64-linux-gnu.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf sysroot-x86_64-linux-gnu.tar.zst && \
-    rm sysroot-x86_64-linux-gnu.tar.zst
-
-RUN wget -nv -O /builds/worker/fetches/sysroot-wasm32-wasi.tar.zst https://ghostery-user-agent-cache-public.s3.amazonaws.com/toolchains/117.0/sysroot-wasm32-wasi-clang-16/sysroot-wasm32-wasi.tar.zst && \
-    cd /builds/worker/fetches/ && \
-    tar -xf sysroot-wasm32-wasi.tar.zst && \
-    rm sysroot-wasm32-wasi.tar.zst
-
 ADD --chown=worker:worker makecab.exe /builds/worker/fetches/
 
 ENV MOZ_FETCHES_DIR=/builds/worker/fetches/ \

diff --git a/fern/core/docker.js b/fern/core/docker.js
@@ -163,7 +163,7 @@ async function generate(artifactBaseDir) {
   const buildConfigs = [
     {
       name: "Linux",
-      key: "linux64-asan/opt",
+      key: "linux64-shippable/opt",
       buildPath: path.join(
         root,
         "mozilla-release",
@@ -175,7 +175,7 @@ async function generate(artifactBaseDir) {
     },
     {
       name: "Windows",
-      key: "win64-asan/opt",
+      key: "win64-shippable/opt",
       buildPath: path.join(
         root,
         "mozilla-release",
@@ -187,7 +187,7 @@ async function generate(artifactBaseDir) {
     },
     {
       name: "WindowsARM",
-      key: "win64-aarch64/opt",
+      key: "build-win64-aarch64-shippable-no-eme/opt",
       arch: "arm64",
       buildPath: path.join(
         root,
@@ -200,7 +200,7 @@ async function generate(artifactBaseDir) {
     },
     {
       name: "MacOSX",
-      key: "macosx64-asan-fuzzing/opt",
+      key: "macosx64-x64-shippable/opt",
       buildPath: path.join(
         root,
         "mozilla-release",
@@ -212,7 +212,7 @@ async function generate(artifactBaseDir) {
     },
     {
       name: "MacOSARM",
-      key: "macosx64-aarch64-asan-fuzzing/opt",
+      key: "macosx64-aarch64-shippable/opt",
       arch: "arm64",
       buildPath: path.join(
         root,
@@ -267,12 +267,13 @@ async function generate(artifactBaseDir) {
 
   const releaseFetches = {};
   for (const jobType of jobTypes) {
+    const url = `https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-release.shippable.revision.${release.hash}.firefox.${jobType}`;
     const releaseTaskId = await fetch(
-      `https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-release.revision.${release.hash}.firefox.${jobType}`
+      url
     ).then(async (res) => {
       if (!res.ok) {
         throw new Error(
-          `Failed to find Taskcluster Task for release ${releaseLabel} ${jobType}: ${res.status}: ${res.statusText}`,
+          `Failed to find Taskcluster Task for release ${releaseLabel} ${jobType}: ${url} - ${res.status}: ${res.statusText}`,
         );
       }