diff --git a/.github/workflows/daily-remote-tests-issue-template.md b/.github/workflows/daily-remote-tests-issue-template.md new file mode 100644 index 0000000000..ca0acfea9a --- /dev/null +++ b/.github/workflows/daily-remote-tests-issue-template.md @@ -0,0 +1,8 @@ +--- +title: Daily GitHub Actions fail for remote tests on {{ date | date('ddd, MMMM Do YYYY') }} +assignees: nguyenv, johnkerl +labels: bug +--- + +See run for more details: +https://github.com/{{ env.GITHUB_REPOSITORY }}/actions/runs/{{ env.GITHUB_RUN_ID }} diff --git a/.github/workflows/daily-test-build-issue-template.md b/.github/workflows/daily-test-build-issue-template.md index b6c19770c4..efb48190c5 100644 --- a/.github/workflows/daily-test-build-issue-template.md +++ b/.github/workflows/daily-test-build-issue-template.md @@ -1,5 +1,5 @@ --- -title: Daily GitHub Actions Build Fail on {{ date | date('ddd, MMMM Do YYYY') }} +title: Daily GitHub Actions build fail on {{ date | date('ddd, MMMM Do YYYY') }} assignees: nguyenv, ryan-williams, johnkerl labels: bug --- diff --git a/.github/workflows/libtiledb-ci.yml b/.github/workflows/libtiledb-ci.yml index 87f7b118fe..9bbfa562fc 100644 --- a/.github/workflows/libtiledb-ci.yml +++ b/.github/workflows/libtiledb-ci.yml @@ -1,18 +1,15 @@ name: libTileDB-SOMA CodeCov on: -# XXX TEMP -# pull_request: -# paths-ignore: -# - "apis/python/**" -# - "apis/r/**" -# - ".pre-commit-config.yaml" -# # XXX TEMP -# - ".github/workflows/python-remote-storage.yml" -# push: -# branches: -# - main -# - 'release-*' + pull_request: + paths-ignore: + - "apis/python/**" + - "apis/r/**" + - ".pre-commit-config.yaml" + push: + branches: + - main + - 'release-*' workflow_dispatch: jobs: diff --git a/.github/workflows/libtiledbsoma-asan-ci.yml b/.github/workflows/libtiledbsoma-asan-ci.yml index 45e1271024..631e080601 100644 --- a/.github/workflows/libtiledbsoma-asan-ci.yml +++ b/.github/workflows/libtiledbsoma-asan-ci.yml @@ -1,18 +1,16 @@ name: libtiledbsoma ASAN on: -# XXX TEMP -# pull_request: -# paths-ignore: -# - "apis/python/**" -# - "apis/r/**" -# - ".pre-commit-config.yaml" -# # XXX TEMP -# - ".github/workflows/python-remote-storage.yml" -# push: -# branches: -# - main -# - 'release-*' + pull_request: + paths-ignore: + - "apis/python/**" + - "apis/r/**" + - ".pre-commit-config.yaml" + - ".github/workflows/python-remote-storage.yml" + push: + branches: + - main + - 'release-*' workflow_dispatch: jobs: diff --git a/.github/workflows/python-ci-minimal.yml b/.github/workflows/python-ci-minimal.yml index 81d214e777..7168102607 100644 --- a/.github/workflows/python-ci-minimal.yml +++ b/.github/workflows/python-ci-minimal.yml @@ -8,19 +8,18 @@ name: TileDB-SOMA Python CI (Minimal) # To test the full matrix on a working branch, invoke ./python-ci-full.yml from # https://github.com/single-cell-data/TileDB-SOMA/actions/workflows/python-ci-full.yml on: - # XXX TEMP -# pull_request: -# branches: -# - main -# - 'release-*' -# paths: -# - '**' -# - '!**.md' -# - '!apis/r/**' -# - '!docs/**' -# - '!.github/**' -# - '.github/workflows/python-ci-minimal.yml' -# - '.github/workflows/python-ci-single.yml' + pull_request: + branches: + - main + - 'release-*' + paths: + - '**' + - '!**.md' + - '!apis/r/**' + - '!docs/**' + - '!.github/**' + - '.github/workflows/python-ci-minimal.yml' + - '.github/workflows/python-ci-single.yml' workflow_dispatch: jobs: diff --git a/.github/workflows/python-dependency-variation.yml b/.github/workflows/python-dependency-variation.yml index 3f2c654d1f..db6417013d 100644 --- a/.github/workflows/python-dependency-variation.yml +++ b/.github/workflows/python-dependency-variation.yml @@ -1,16 +1,15 @@ name: TileDB-SOMA Python CI with varying dependencies on: -# XXX TEMP -# push: -# branches: -# - main -# - 'release-*' -# pull_request: -# paths-ignore: -# - '**.md' -# - 'apis/r/**' -# - 'docs/**' + push: + branches: + - main + - 'release-*' + pull_request: + paths-ignore: + - '**.md' + - 'apis/r/**' + - 'docs/**' workflow_dispatch: jobs: diff --git a/.github/workflows/python-remote-storage.yml b/.github/workflows/python-remote-storage.yml index 8c5b8e8664..eec8ccd5f6 100644 --- a/.github/workflows/python-remote-storage.yml +++ b/.github/workflows/python-remote-storage.yml @@ -1,13 +1,18 @@ name: TileDB-SOMA Python CI (remote storage) on: - workflow_dispatch: + # Not for regular use, but you can uncomment this when putting up PRs on this + # file. Just remember to take it back out. There's no need to run + # remote-storage tests on every single PR push to our repo. + # + # pull_request: # - # Not for regular use: - # TEMP - pull_request: + # Allows for 'as needed' manual trigger: + workflow_dispatch: # - # TODO: a nightly cron + # Use a regular nighly build as well (time is UTC): + schedule: + - cron: "25 5 * * *" env: # Don't name this "TILEDB_REST_TOKEN" since that will map into a core @@ -28,8 +33,8 @@ jobs: # in GitHub Actions as of 2025-02-06. - name: linux os: ubuntu-24.04 - # TODO: also on 3.12. But 3.9 is higher-pri, until we drop support - # for it. (Note our main CI tests across a broader set of Python + # TO DO: also on 3.12. But 3.9 is higher-pri, until we drop support + # for it. (Note our main CI run tests across a broader set of Python # versions.) python_version: 3.9 cc: gcc-13 @@ -102,3 +107,18 @@ jobs: - name: Run pytests for Python shell: bash run: python -m pytest apis/python/remote_tests -v --durations=20 --maxfail=50 + + # File a bug report if anything fails, but don't file tickets for manual runs + # -- only for scheduled ones. + create_issue_on_fail: + needs: [ci] + if: (failure() || cancelled()) && github.event_name != 'workflow_dispatch' + steps: + - name: Checkout TileDB-SOMA `main` + uses: actions/checkout@v2 + - name: Create Issue if Build Fails + uses: JasonEtco/create-an-issue@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + filename: .github/workflows/daily-remote-tests-issue-template.md diff --git a/.github/workflows/r-python-interop-testing.yml b/.github/workflows/r-python-interop-testing.yml index b4c2f2110b..db0c32a4e9 100644 --- a/.github/workflows/r-python-interop-testing.yml +++ b/.github/workflows/r-python-interop-testing.yml @@ -1,21 +1,20 @@ name: TileDB-SOMA R-Python interop testing on: -# XXX TEMP -# pull_request: -# paths-ignore: -# # XXX TEMP -# - ".github/workflows/python-remote-storage.yml" -# # TODO: leave this enabled for pre-merge signal for now. At some point we may want to go back to -# # only having this signal post-merge. -# #paths: -# # - "apis/python/**" -# # - "apis/r/**" -# # - "apis/system/**" -# push: -# branches: -# - main -# - "release-*" + pull_request: + paths-ignore: + # XXX TEMP + - ".github/workflows/python-remote-storage.yml" + # TODO: leave this enabled for pre-merge signal for now. At some point we may want to go back to + # only having this signal post-merge. + #paths: + # - "apis/python/**" + # - "apis/r/**" + # - "apis/system/**" + push: + branches: + - main + - "release-*" workflow_dispatch: jobs: diff --git a/apis/python/remote_tests/README.md b/apis/python/remote_tests/README.md index 1870871e26..0729d3dc37 100644 --- a/apis/python/remote_tests/README.md +++ b/apis/python/remote_tests/README.md @@ -1,5 +1,7 @@ # How to run these tests +These are nominally a nightly cron. However, you can run them manually if you want. + ``` export TILEDB_REST_TOKEN="..." # Get the token for the Saas `unittest` user unsetTILEDB_REST_PAYER_NAMESPACE # If you have that set diff --git a/apis/python/remote_tests/test_99_todo.py b/apis/python/remote_tests/test_99_todo.py deleted file mode 100644 index 17682f3803..0000000000 --- a/apis/python/remote_tests/test_99_todo.py +++ /dev/null @@ -1,59 +0,0 @@ -## ================================================================ -### UDFs -# def remote_obs_schema(exp_uri): -# import tiledbsoma -# exp = tiledbsoma.Experiment.open(exp_uri) -# return exp.obs.schema -# import tiledb.cloud -# import tiledb.cloud.udf -# tiledb.cloud.udf.exec( -# remote_obs_schema, -# soma_pbmc3k_uri, -# ) -# -# def remote_query(exp_uri): -# import tiledbsoma -# exp = tiledbsoma.Experiment.open(exp_uri) -# -# query = tiledbsoma.ExperimentAxisQuery( -# experiment=exp, -# measurement_name="RNA", -# obs_query=tiledbsoma.AxisQuery( -# value_filter="n_genes_by_counts > 1000", -# ), -# var_query=tiledbsoma.AxisQuery( -# value_filter="n_cells_by_counts > 100", -# ), -# ) -# -# return (query.n_obs, query.n_vars) -# tiledb.cloud.udf.exec( -# remote_query, soma_pbmc3k_uri, -# ) -# -## ================================================================ -## Collection-mapper test -# from tiledb.cloud.taskgraphs import client_executor as executor -# soco_uri = 'tiledb://TileDB-Inc/stack-small-soco-staging' -# res = tiledb.cloud.udf.exec( -# 'TileDB-Inc/soma_experiment_collection_mapper', -# soco_uri=soco_uri, -# measurement_name="RNA", -# X_layer_name="data", -# # callback = lambda x: x.obs.shape, -# # callback = lambda x: x, -# callback = lambda adata: [adata.obs.shape, adata.var.shape, adata.X.shape], -# # callback = lambda adata: adata.var, -# args_dict={}, -# reducer = lambda x: x, -# obs_attrs = ['obs_id', 'cell_type', 'is_primary_data'], -# var_attrs = ['var_id', 'means'], -# ) -# dag = executor.LocalExecutor(res, namespace = "TileDB-Inc") -# dag.visualize() -##%%time -# dag.execute() -# dag.wait() -# dag.node("output").result() - -# * Show, upgrade, resize