diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 378f420..b408b60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,37 +1,64 @@ -# Automatically build multi-architectural tagged container images and push them to DockerHub -# https://github.com/FNNDSC/cookiecutter-chrisapp/wiki/Automatic-Builds +# Continuous integration testing for ChRIS Plugin. +# https://github.com/FNNDSC/python-chrisapp-template/wiki/Continuous-Integration # -# - targeted platforms: x86_64, PowerPC64, ARM64 -# - master is built as fnndsc/pl-dircopy:latest -# - tagged commits are built as fnndsc/pl-dircopy: -# - tagged commits are also uploaded to chrisstore.co -# -# In order to use this workflow, see -# https://github.com/FNNDSC/cookiecutter-chrisapp/wiki/Automatic-Builds#steps-to-enable +# - on push and PR: run pytest +# - on push to main: build and push container images as ":latest" +# - on push to semver tag: build and push container image with tag and +# upload plugin description to https://chrisstore.co -name: ci +name: build on: push: - # we have to guess what the name of the default branch is - branches: [ main, master ] - tags: [ "[0-9]+.[0-9]+.[0-9]+*" ] + branches: [ main ] + tags: + - "v?[0-9]+.[0-9]+.[0-9]+*" pull_request: - branches: [ main, master ] + branches: [ main ] jobs: + test: + name: Unit tests + if: false # delete this line to enable automatic testing + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - uses: docker/setup-buildx-action@v2 + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Build + uses: docker/build-push-action@v3 + with: + build-args: extras_require=dev + context: . + load: true + push: false + tags: "localhost/local/app:dev" + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + - name: Run pytest + run: | + docker run -v "$GITHUB_WORKSPACE:/app:ro" -w /app localhost/local/app:dev \ + pytest -o cache_dir=/tmp/pytest + build: name: Build + if: github.event_name == 'push' || github.event_name == 'release' + # needs: [ test ] # uncomment to require passing tests runs-on: ubuntu-22.04 + steps: - name: Stop docker run: sudo systemctl stop docker - - name: Clean docker data run: | - sudo rm -rf /var/lib/docker - sudo mkdir /var/lib/docker - + sudo rm -rf /var/lib/docker + sudo mkdir /var/lib/docker - name: Maximize build space uses: easimon/maximize-build-space@6ae56c86ea8db291ae39f62352a412c36ab8179b with: @@ -43,52 +70,46 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' remove-docker-images: 'false' - - name: Start docker run: sudo systemctl start docker - - - name: Decide image tags - id: info - shell: python + - name: Get git tag + id: git_info + if: startsWith(github.ref, 'refs/tags/') + run: echo "tag=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT + - name: Get project info + id: determine + env: + git_tag: ${{ steps.git_info.outputs.tag }} run: | - import os - import itertools - - registries = ['docker.io', 'ghcr.io'] - repos = ['${{ github.repository }}'] - if '${{ github.ref_type }}' == 'branch': - version = 'unknown' - tags = ['latest'] - elif '${{ github.ref_type }}' == 'tag': - tag = '${{ github.ref_name }}' - version = tag[1:] if tag.startswith('v') else tag - tags = ['latest', version] - else: - tags = [] + repo="${GITHUB_REPOSITORY,,}" # to lower case + # if build triggered by tag, use tag name + tag="${git_tag:-latest}" - def join_tag(t): - registry, repo, tag = t - return f'{registry}/{repo}:{tag}'.lower() + # if tag is a version number prefixed by 'v', remove the 'v' + if [[ "$tag" =~ ^v[0-9].* ]]; then + tag="${tag:1}" + fi - product = itertools.product(registries, repos, tags) - tags_csv = ','.join(map(join_tag, product)) - outputs = { - 'tags_csv' : tags_csv, - 'push' : 'true' if tags_csv else 'false', - 'local_tag': join_tag(('localhost', '${{ github.repository }}', 'latest')), - 'one_tag': join_tag(('ghcr.io', repos[0], version)) - } - with open(os.environ['GITHUB_OUTPUT'], 'a') as out: - for k, v in outputs.items(): - out.write(f'{k}={v}\n') + dock_image=$repo:$tag + echo $dock_image + echo "dock_image=$dock_image" >> $GITHUB_OUTPUT + echo "repo=$repo" >> $GITHUB_OUTPUT - uses: actions/checkout@v3 # QEMU is used for non-x86_64 builds - - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-qemu-action@v2 # buildx adds additional features to docker build - - uses: docker/setup-buildx-action@v3 + - uses: docker/setup-buildx-action@v2 with: driver-opts: network=host + # cache slightly improves rebuild time + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- # Here, we want to do the docker build twice: # The first build pushes to our local registry for testing. @@ -99,55 +120,84 @@ jobs: with: context: . file: ./Dockerfile - tags: ${{ steps.info.outputs.local_tag }} + tags: localhost/${{ steps.determine.outputs.dock_image }} load: true - cache-from: type=gha + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + # If you have a directory called examples/incoming/ and examples/outgoing/, then + # run your ChRIS plugin with no parameters, and asser that it creates all the files + # which are expected. File contents are not compared. + - name: Run examples + id: run_examples + run: | + if ! [ -d 'examples/incoming/' ] || ! [ -d 'examples/outgoing/' ]; then + echo "No examples." + exit 0 + fi + + dock_image=localhost/${{ steps.determine.outputs.dock_image }} + output_dir=$(mktemp -d) + cmd=$(docker image inspect -f '{{ (index .Config.Cmd 0) }}' $dock_image) + docker run --rm -u "$(id -u):$(id -g)" \ + -v "$PWD/examples/incoming:/incoming:ro" \ + -v "$output_dir:/outgoing:rw" \ + $dock_image $cmd /incoming /outgoing + + for expected_file in $(find examples/outgoing -type f); do + fname="${expected_file##*/}" + out_path="$output_dir/$fname" + printf "Checking output %s exists..." "$out_path" + if [ -f "$out_path" ]; then + echo "ok" + else + echo "not found" + exit 1 + fi + done - name: Login to DockerHub - if: (github.event_name == 'push' || github.event_name == 'release') && contains(steps.info.outputs.tags_csv, 'docker.io') - uses: docker/login-action@v3 + id: dockerhub_login + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Login to GitHub Container Registry - if: (github.event_name == 'push' || github.event_name == 'release') && contains(steps.info.outputs.tags_csv, 'ghcr.io') - uses: docker/login-action@v3 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push uses: docker/build-push-action@v3 - if: (github.event_name == 'push' || github.event_name == 'release') + if: github.event_name == 'push' || github.event_name == 'release' with: context: . file: ./Dockerfile - tags: ${{ steps.info.outputs.tags_csv }} - platforms: linux/amd64,linux/arm64,linux/ppc64le - push: ${{ steps.info.outputs.push }} - cache-to: type=gha,mode=max + tags: | + docker.io/${{ steps.determine.outputs.dock_image }} + ghcr.io/${{ steps.determine.outputs.dock_image }} + # if non-x86_84 architectures are supported, add them here + platforms: linux/amd64 #,linux/arm64,linux/ppc64le + push: true + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache - - name: Create plugin description + - name: Get plugin meta + id: pluginmeta run: | - docker run --rm ${{ steps.info.outputs.local_tag }} fshack.py --json \ - | jq '. + {"name": "pl-fshack", "public_repo": "${{ github.server_url }}/${{ github.repository }}", "dock_image": "${{ steps.info.outputs.one_tag }}"}' > ./description.json - - name: Upload ChRIS Plugin - id: upload - if: github.ref_type == 'tag' - uses: FNNDSC/upload-chris-plugin@v1 - with: - description_file: ./description.json - username: ${{ secrets.CHRISPROJECT_USERNAME }} - password: ${{ secrets.CHRISPROJECT_PASSWORD }} - chris_url: https://cube.chrisproject.org/api/v1/ - compute_names: NERC + repo=${{ steps.determine.outputs.repo }} + dock_image=${{ steps.determine.outputs.dock_image }} + docker run --rm localhost/$dock_image chris_plugin_info > /tmp/description.json + jq < /tmp/description.json # pretty print in log + echo "title=$(jq -r '.title' < /tmp/description.json)" >> $GITHUB_OUTPUT - name: Update DockerHub description - if: steps.upload.outcome == 'success' uses: peter-evans/dockerhub-description@v3 continue-on-error: true # it is not crucial that this works with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - short-description: ${{ steps.upload.outputs.title }} - readme-filepath: ./README.rst \ No newline at end of file + short-description: ${{ steps.pluginmeta.outputs.title }} + readme-filepath: ./README.md + repository: ${{ steps.determine.outputs.repo }} \ No newline at end of file