Merge branch 'master' into osx-arm-deployment-target

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+### [2.13.2](https://www.github.com/bioconda/bioconda-utils/compare/v2.13.1...v2.13.2) (2024-03-27)
+
+
+### Bug Fixes
+
+* update anaconda-client=1.12 to fix uploads ([#970](https://www.github.com/bioconda/bioconda-utils/issues/970)) ([06dcacc](https://www.github.com/bioconda/bioconda-utils/commit/06dcacca60c17c1a97770e14ae6348ac7acbe9a6))
+
+### [2.13.1](https://www.github.com/bioconda/bioconda-utils/compare/v2.13.0...v2.13.1) (2024-03-27)
+
+
+### Bug Fixes
+
+* detect errors when uploading artifacts ([#968](https://www.github.com/bioconda/bioconda-utils/issues/968)) ([d0a79cd](https://www.github.com/bioconda/bioconda-utils/commit/d0a79cdd5aeed30c4da88e2135329d66b336832f))
+
 ## [2.13.0](https://www.github.com/bioconda/bioconda-utils/compare/v2.12.0...v2.13.0) (2024-03-22)
 
 

bioconda_utils/artifacts.py

@@ -1,5 +1,6 @@
 
 
+from enum import Enum
 import glob
 import os
 import re
@@ -20,7 +21,14 @@
 IMAGE_RE = re.compile(r"(.+)(?::|%3A)(.+)\.tar\.gz$")
 
 
-def upload_pr_artifacts(config, repo, git_sha, dryrun=False, mulled_upload_target=None, label=None, artifact_source="azure") -> bool:
+class UploadResult(Enum):
+    SUCCESS = 1
+    FAILURE = 2
+    NO_ARTIFACTS = 3
+    NO_PR = 4
+
+
+def upload_pr_artifacts(config, repo, git_sha, dryrun=False, mulled_upload_target=None, label=None, artifact_source="azure") -> UploadResult:
     _config = utils.load_config(config)
     repodata = utils.RepoData()
 
@@ -32,26 +40,33 @@ def upload_pr_artifacts(config, repo, git_sha, dryrun=False, mulled_upload_targe
     prs = commit.get_pulls()
     if not prs:
         # no PR found for the commit
-        return True
+        return UploadResult.NO_PR
     pr = prs[0]
-    artifacts = set(fetch_artifacts(pr, artifact_source))
+    artifacts = set(fetch_artifacts(pr, artifact_source, repo))
     if not artifacts:
         # no artifacts found, fail and rebuild packages
         logger.info("No artifacts found.")
-        return False
+        return UploadResult.NO_ARTIFACTS
     else:
+        success = []
         for artifact in artifacts:
             with tempfile.TemporaryDirectory() as tmpdir:
                 # download the artifact
                 if artifact_source == "azure":
                     artifact_path = os.path.join(tmpdir, os.path.basename(artifact))
-                    download_artifact(artifact, artifact_path)
+                    download_artifact(artifact, artifact_path, artifact_source)
                     zipfile.ZipFile(artifact_path).extractall(tmpdir)
                 elif artifact_source == "circleci":
                     artifact_dir = os.path.join(tmpdir, *(artifact.split("/")[-4:-1]))
                     artifact_path = os.path.join(tmpdir, artifact_dir, os.path.basename(artifact))
-                    Path(artifact_dir).mkdir(parents=True, exist_ok=True) 
-                    download_artifact(artifact, artifact_path)
+                    Path(artifact_dir).mkdir(parents=True, exist_ok=True)
+                    download_artifact(artifact, artifact_path, artifact_source)
+                elif artifact_source == "github-actions":
+                    artifact_dir = os.path.join(tmpdir, "artifacts")
+                    artifact_path = os.path.join(artifact_dir, os.path.basename(artifact))
+                    Path(artifact_dir).mkdir(parents=True, exist_ok=True)
+                    download_artifact(artifact, artifact_path, artifact_source)
+                    zipfile.ZipFile(artifact_path).extractall(artifact_dir)
 
                 # get all the contained packages and images and upload them
                 platform_patterns = [repodata.platform2subdir(repodata.native_platform())]
@@ -67,7 +82,7 @@ def upload_pr_artifacts(config, repo, git_sha, dryrun=False, mulled_upload_targe
                         else:
                             logger.info(f"Uploading {pkg} to anaconda.org.")
                             # upload the package
-                            anaconda_upload(pkg, label=label)
+                            success.append(anaconda_upload(pkg, label=label))
 
                 if mulled_upload_target:
                     quay_login = os.environ['QUAY_LOGIN']
@@ -90,25 +105,35 @@ def upload_pr_artifacts(config, repo, git_sha, dryrun=False, mulled_upload_targe
                         else:
                             # upload the image
                             logger.info(f"Uploading {img} to {target}.")
-                            skopeo_upload(fixed_img_name, target, creds=quay_login)
-        return True
+                            success.append(skopeo_upload(fixed_img_name, target, creds=quay_login))
+        if all(success):
+            return UploadResult.SUCCESS
+        else:
+            return UploadResult.FAILURE
 
 
 @backoff.on_exception(
     backoff.expo,
     requests.exceptions.RequestException
 )
-def download_artifact(url, to_path):
+def download_artifact(url, to_path, artifact_source):
     logger.info(f"Downloading artifact {url}.")
-    resp = requests.get(url, stream=True, allow_redirects=True)
+    headers = {}
+    if artifact_source == "github-actions":
+        token = os.environ.get("GITHUB_TOKEN")
+        if not token:
+            logger.critical("GITHUB_TOKEN required to download GitHub Actions artifacts")
+            exit(1)
+        headers = {"Authorization": f"token {token}"}
+    resp = requests.get(url, stream=True, allow_redirects=True, headers=headers)
     resp.raise_for_status()
     with open(to_path, "wb") as f:
         for chunk in resp.iter_content(chunk_size=1024):
             if chunk:
                 f.write(chunk)
 
 
-def fetch_artifacts(pr, artifact_source):
+def fetch_artifacts(pr, artifact_source, repo):
     """
     Fetch artifacts from a PR.
 
@@ -143,6 +168,13 @@ def fetch_artifacts(pr, artifact_source):
             # Circle CI builds
             artifact_url = get_circleci_artifacts(check_run, platform)
             yield from artifact_url
+        elif (
+            artifact_source == "github-actions" and
+            check_run.app.slug == "github-actions"
+        ):
+            # GitHubActions builds
+            artifact_url = get_gha_artifacts(check_run, platform, repo)
+            yield from artifact_url
 
 
 def get_azure_artifacts(check_run):
@@ -185,3 +217,18 @@ def get_circleci_artifacts(check_run, platform):
                             continue
                         else:
                             yield artifact_url
+
+def parse_gha_build_id(url: str) -> str:
+    # Get workflow run id from URL
+    return re.search("runs/(\d+)/", url).group(1)
+
+def get_gha_artifacts(check_run, platform, repo):
+    gha_workflow_id = parse_gha_build_id(check_run.details_url)
+    if (gha_workflow_id) :
+        # The workflow run is different from the check run
+        run = repo.get_workflow_run(int(gha_workflow_id))
+        artifacts = run.get_artifacts()
+        for artifact in artifacts:
+            # This URL is valid for 1 min and requires a token
+            artifact_url = artifact.archive_download_url
+            yield artifact_url

bioconda_utils/bioconda_utils-requirements.txt

@@ -26,7 +26,7 @@ pyopenssl>=22.1      # Stay compatible with cryptography
 conda-forge-pinning=2023.05.06.13.08.41
 
 # tools
-anaconda-client=1.6.*  # anaconda_upload
+anaconda-client=1.12.*  # anaconda_upload
 involucro=1.1.*        # mulled test and container build
 skopeo=1.11.*          # docker upload
 git=2.*                # well - git

bioconda_utils/cli.py

@@ -9,7 +9,7 @@
 import warnings
 from bioconda_utils import bulk
 
-from bioconda_utils.artifacts import upload_pr_artifacts
+from bioconda_utils.artifacts import UploadResult, upload_pr_artifacts
 from bioconda_utils.skiplist import Skiplist
 from bioconda_utils.build_failure import BuildFailureRecord, collect_build_failure_dataframe
 warnings.filterwarnings("ignore", message="numpy.dtype size changed")
@@ -524,7 +524,7 @@ def build(recipe_folder, config, packages="*", git_range=None, testonly=False,
 @arg('--dryrun', action='store_true', help='''Do not actually upload anything.''')
 @arg('--fallback', choices=['build', 'ignore'], default='build', help="What to do if no artifacts are found in the PR.")
 @arg('--quay-upload-target', help="Provide a quay.io target to push docker images to.")
-@arg('--artifact-source', choices=['azure', 'circleci'], default='azure', help="Application hosting build artifacts (e.g., Azure or Circle CI).")
+@arg('--artifact-source', choices=['azure', 'circleci','github-actions'], default='azure', help="Application hosting build artifacts (e.g., Azure, Circle CI, or GitHub Actions).")
 @enable_logging()
 def handle_merged_pr(
     recipe_folder,
@@ -538,10 +538,12 @@ def handle_merged_pr(
 ):
     label = os.getenv('BIOCONDA_LABEL', None) or None
 
-    success = upload_pr_artifacts(
-        config, repo, git_range[1], dryrun=dryrun, mulled_upload_target=quay_upload_target, label=label, artifact_source=artifact_source
+    res = upload_pr_artifacts(
+        config, repo, git_range[1], dryrun=dryrun,
+        mulled_upload_target=quay_upload_target, label=label,
+        artifact_source=artifact_source
     )
-    if not success and fallback == 'build':
+    if res == UploadResult.NO_ARTIFACTS and fallback == 'build':
         success = build(
             recipe_folder,
             config,
@@ -551,6 +553,8 @@ def handle_merged_pr(
             mulled_test=True,
             label=label,
         )
+    else:
+        success = res != UploadResult.FAILURE
     exit(0 if success else 1)
 
 @recipe_folder_and_config()