ci: Add github debs update (#33)

Signed-off-by: Ryan Northey <[email protected]>

.bazelrc

@@ -3,19 +3,26 @@ common --noenable_bzlmod
 common --color=yes
 common --action_env=APT_ROOT=apt
 common --host_action_env=APT_ROOT=apt
+common --action_env=GITHUB_TOKEN
+common --host_action_env=GITHUB_TOKEN
 
 common:ci --noshow_progress
 common:ci --noshow_loading_progress
 common:ci --test_output=errors
 common:ci --//:aptly-custom=//:.aptly-ci-override
 common:ci --action_env=APT_ROOT=/opt/build/cache
 common:ci --host_action_env=APT_ROOT=/opt/build/cache
+# common:ci --sandbox_writable_path=/opt/build
+common:ci --jobs=HOST_CPUS
 
 common:debs-ci --config=ci
 common:debs-ci --//debs:excludes=//debs:excludes.txt
-# common:debs-ci --//debs:token=//debs:token.txt
+common:debs-ci --//debs:token=//debs:token.txt
 
 common:publish-ci --config=debs-ci
 common:publish-ci --//tools/tarball:target=//:html
 common:publish-ci --//tools/tarball:overwrite=//tools/tarball:true
 common:publish-ci --//debs:signing-token=//debs:signing-token.txt
+
+common:debug-bazel --announce_rc
+common:debug-bazel -s

.github/workflows/update.yml

@@ -0,0 +1,44 @@
+name: Update
+
+permissions:
+  contents: read
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: >-
+    ${{ github.event.inputs.head_ref || github.run_id }}
+
+
+jobs:
+  update:
+    runs-on: ubuntu-24.04
+    if: github.repository_owner == 'envoyproxy'
+    steps:
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  # v4.1.7
+
+    - run: |
+        mkdir ./debs/import
+        gsutil -mq rsync gs://envoy-apt-cache/debs ./debs/import/
+        ls -alh debs/import/
+
+    - run: |
+        . ./build-repository.sh
+        import_public_key
+        touch debs/excludes.txt
+        echo ${{ github.token }} > debs/token.txt
+        GNUPG_HOME="$(realpath ~/.gnupg)"
+        time bazel build "--sandbox_writable_path=${GNUPG_HOME}" --config=debs-ci //debs
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+
+    - uses: envoyproxy/toolshed/gh-actions/gcp/[email protected]
+      name: Setup GCP (cache)
+      if: ${{ github.event_name != 'pull_request' }}
+      with:
+        key: ${{ secrets.GCS_CACHE_KEY }}
+    - run: |
+        gsutil -mq rsync bazel-bin/debs/debs gs://envoy-apt-cache/debs
+      if: ${{ github.event_name != 'pull_request' }}

build-repository.sh

@@ -27,6 +27,7 @@ underline () {
 }
 
 import_public_key () {
+    gpg --list-keys > /dev/null 2>&1
     touch ~/.gnupg/trustedkeys.gpg
     echo -e "$(underline $(bold "Import maintainers public key: checksum verification"))"
     gpg --import envoy-maintainers-public.key

debs/BUILD

@@ -1,4 +1,5 @@
 load("@aspect_bazel_lib//lib:jq.bzl", "jq")
+load(":debs.bzl", "fetch_debs")
 
 MAINTAINER = "Envoy maintainers <[email protected]>"
 
@@ -98,12 +99,13 @@ genrule(
     name = "published_checksums",
     outs = ["published_checksums.txt"],
     cmd = """
-    $(location //tools/fetch) $(location :deb_checksum_downloads) --output=json > $@
+    $(location //tools/fetch) $(location :deb_checksum_downloads) --token-path=$(location :token) --output=json > $@
     """,
-    tools = [
-        "//tools/fetch",
+    srcs = [
         ":deb_checksum_downloads",
+        ":token",
     ],
+    tools = ["//tools/fetch"],
 )
 
 jq(
@@ -130,27 +132,11 @@ jq(
     visibility = ["//visibility:public"],
 )
 
-genrule(
+fetch_debs(
     name = "debs",
-    outs = ["debs.tar.gz"],
-    cmd = """
-    $(location //tools/fetch) $(location :debs_downloads) \
-        --concurrency 4 \
-        --excludes=$(location :excludes) \
-        --token-path=$(location :token) \
-        --extract-downloads \
-        --output-path=$@
-    if [[ ! -e $@ ]]; then
-        touch $@
-    fi
-    """,
-    tools = [
-        "//tools/fetch",
-        ":debs_downloads",
-        ":excludes",
-        ":token",
-    ],
-    visibility = ["//visibility:public"],
+    downloads = ":debs_downloads",
+    token = ":token",
+    excludes = ":excludes",
 )
 
 PUBLISH_ENV = {
@@ -205,6 +191,7 @@ genrule(
         ":publish",
         ":signing-token",
     ],
+    tags = ["no-remote"],
     visibility = ["//visibility:public"],
 )
 

debs/debs.bzl

@@ -0,0 +1,48 @@
+
+def _fetch_impl(ctx):
+    output_dir = ctx.actions.declare_directory("debs")
+    inputs = depset(
+        [ctx.executable.fetch,
+         ctx.file.downloads,
+         ctx.file.excludes,
+         ctx.file.token],
+        transitive = [ctx.attr.fetch[DefaultInfo].files],
+    )
+    ctx.actions.run(
+        executable = ctx.executable.fetch,
+        inputs = inputs,
+        outputs = [output_dir],
+        arguments = [
+            ctx.file.downloads.path,
+            "--concurrency 4",
+            "--excludes=%s" % ctx.file.excludes.path,
+            "--token-path=%s" % ctx.file.token.path,
+            "--extract-downloads",
+            "--output-path=%s" % output_dir.path,
+            "--output=dir",
+            # "-ldebug",
+            # "-vdebug",
+        ],
+        mnemonic = "FetchDebs",
+    )
+    return [DefaultInfo(files = depset([output_dir]))]
+
+fetch_debs = rule(
+    implementation = _fetch_impl,
+    attrs = {
+        "downloads": attr.label(
+            mandatory = True,
+            allow_single_file = True,
+        ),
+        "fetch": attr.label(
+            default = "//tools/fetch",
+            executable = True,
+            cfg = "host",
+        ),
+        "token": attr.label(
+            mandatory = True,
+            allow_single_file = True,
+        ),
+        "excludes": attr.label(allow_single_file = True),
+    },
+)

debs/publish.sh

@@ -40,7 +40,9 @@ create_dirs () {
 }
 
 unpack_debs () {
-    if [[ -s "$DEBS" ]]; then
+    if [[ -d "$DEBS" ]]; then
+        return 0
+    elif [[ -s "$DEBS" ]]; then
         tar xf "$DEBS" -C "$DEBS_ROOT"
     fi
 }