From e7d8cbfd0bddae4c483c3a93b78f643f1f9f1d53 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Mon, 30 Sep 2024 16:38:25 -0700 Subject: [PATCH] Update GitHub Actions workflows --- .github/workflows/build.yml | 44 ++++++++++++++++ .github/workflows/compliance.yaml | 85 +++++++++++++++++++++++++++++++ .github/workflows/hugo.yml | 5 -- .github/workflows/nextflow.yaml | 34 +++---------- .github/workflows/s3.yaml | 52 +++++++++++++++++++ .github/workflows/tests.yaml | 64 ++++++++++++----------- 6 files changed, 222 insertions(+), 62 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/compliance.yaml create mode 100644 .github/workflows/s3.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..cf39df736 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,44 @@ +name: Funnel Build and Cache + +on: + workflow_call: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v5 + with: + go-version: 1.21 + + - name: Check out code + uses: actions/checkout@v2 + + - name: Cache Funnel binary + uses: actions/cache@v3 + with: + path: ./funnel + key: ${{ runner.os }}-funnel-bin-${{ hashFiles('**/go.sum') }}-${{ github.ref }} + restore-keys: | + ${{ runner.os }}-funnel-bin-${{ github.ref }} + ${{ runner.os }}-funnel-bin- + + - name: Build Funnel (if cache doesn't exist) + run: | + if [ ! -f ./funnel ]; then + make build + fi + + - name: Cache Funnel binary (after build) + uses: actions/cache@v3 + with: + path: ./funnel + key: ${{ runner.os }}-funnel-bin-${{ hashFiles('**/go.sum') }}-${{ github.ref }} + + - name: Upload Funnel binary as artifact + uses: actions/upload-artifact@v4 + with: + name: funnel + path: funnel diff --git a/.github/workflows/compliance.yaml b/.github/workflows/compliance.yaml new file mode 100644 index 000000000..0c7676e0a --- /dev/null +++ b/.github/workflows/compliance.yaml @@ -0,0 +1,85 @@ +# Workflow for running the TES compliance suite against Funnel +# +# This includes the following steps: +# 1. Build Funnel and store the resulting binary artifact +# 2. Install tes-compliance-suite and run against every version of TES simultaneously +# 3. start-report-deployment: Send a dispatch to the funnel-compliance repository to generate and publish +# the tes-compliance-suite report to https://ohsu-comp-bio.github.io/funnel-compliance/ +# +# Optionally debug via SSH +# Ref: https://fleetdm.com/engineering/tips-for-github-actions-usability +# +# To use this step uncomment and place anywhere in the build steps. The build will pause on this step and +# output a ssh address associated with the Github action worker. Helpful for debugging build steps and +# and intermediary files/artifacts. +# +# - name: "Debug: Package dependancies for tmate (CentOS)" +# run: | +# yum install -y xz +# ln -s /bin/true /bin/apt-get +# +# - name: Setup tmate session +# uses: mxschmitt/action-tmate@v3 + +name: Compliance Test + +on: + push: + pull_request: + +jobs: + build: + uses: ./.github/workflows/build.yml + + compliance: + strategy: + fail-fast: false + matrix: + version: [1.0.0, 1.1.0] + db: ["boltdb", "mongodb"] + compute: ["local", "kubernetes"] + storage: ["local", "s3"] + needs: build + runs-on: ubuntu-latest + steps: + # Required to access the 'tests/mongo.config.yml' file + # Perhaps uploading it as an artifact would be more efficient? + - name: Check out code + uses: actions/checkout@v2 + + - uses: actions/download-artifact@v4 + with: + name: funnel + + - name: Start Funnel server + run: | + touch config.yml + + if [ ${{ matrix.db }} = "mongodb" ]; then + make start-mongodb + cat `pwd`/tests/mongo.config.yml >> config.yml + fi + + if [ ${{ matrix.storage }} = "s3" ]; then + docker run -d -p 9000:9000 --name minio \ + -e "MINIO_ROOT_USER=minioadmin" \ + -e "MINIO_ROOT_PASSWORD=minioadmin" \ + -v /tmp/data:/data \ + -v /tmp/config:/root/.minio \ + minio/minio server /data + cat `pwd`/tests/s3.config.yml >> config.yml + fi + + chmod +x funnel + ./funnel server run --config `pwd`/config.yml &> funnel.logs & + + - name: Run OpenAPI Test Runner + run: | + git clone https://github.com/elixir-cloud-aai/openapi-test-runner + cd openapi-test-runner + python3 -m venv venv + source venv/bin/activate + pip install -r requirements.txt + python setup.py install + openapi-test-runner report --version "${{ matrix.version }}" --server "http://localhost:8000/" + diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index 81986c025..0e3d40fc5 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -59,11 +59,6 @@ jobs: --baseURL "${{ steps.pages.outputs.base_url }}/" \ --source website \ --destination public - - # Run pagefind to add search functionality - # Reference: https://pagefind.app/docs/ - npx -y pagefind --site public - - name: Upload artifact uses: actions/upload-pages-artifact@v3 with: diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index 3273306e6..1b04b0180 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -2,49 +2,31 @@ name: Nextflow Test on: push: + pull_request: jobs: build: - runs-on: ubuntu-latest - steps: - - name: Set up Go 1.x - uses: actions/setup-go@v5 - with: - go-version: 1.21 - - - name: Check out code - uses: actions/checkout@v2 + uses: ./.github/workflows/build.yml - - name: Build Funnel (if cache does not exist) - run: make build - - - name: Store Funnel - uses: actions/upload-artifact@v4 - with: - name: funnelBin - path: funnel - nextflow: - runs-on: ubuntu-latest needs: build + runs-on: ubuntu-latest steps: - name: Download Funnel - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: - name: funnelBin - path: funnel + name: funnel - name: Start Funnel - working-directory: run: | - cd funnel/ chmod +x ./funnel ./funnel server --LocalStorage.AllowedDirs $HOME run & - name: Install Nextflow run: | - curl -s https://get.nextflow.io | bash - chmod +x nextflow + git clone https://github.com/nextflow-io/nextflow + cd nextflow + make compile - name: Install nf-canary and GA4GH-TES plugin run: | diff --git a/.github/workflows/s3.yaml b/.github/workflows/s3.yaml new file mode 100644 index 000000000..1855fd519 --- /dev/null +++ b/.github/workflows/s3.yaml @@ -0,0 +1,52 @@ +# Credit: rhnvrm +# Adapted from: https://rohanverma.net/blog/2021/02/09/minio-github-actions/ + +name: S3 Integration Test + +on: + push: + pull_request: + +jobs: + build: + uses: ./.github/workflows/build.yml + + s3Test: + needs: build + runs-on: ubuntu-latest + steps: + - name: Setup minio + run: | + docker run -d -p 9000:9000 --name minio \ + -e "MINIO_ROOT_USER=minioadmin" \ + -e "MINIO_ROOT_PASSWORD=minioadmin" \ + -v /tmp/data:/data \ + -v /tmp/config:/root/.minio \ + minio/minio server /data + + - uses: actions/download-artifact@v4 + with: + name: funnel + + - name: Start Funnel server + run: | + cat < config.yml + LocalStorage: + Disabled: true + AmazonS3: + Disabled: true + GoogleStorage: + Disabled: true + HTTPStorage: + Disabled: true + FTPStorage: + Disabled: true + GenericS3: + - Disabled: false + Endpoint: "localhost:9000" + Key: "minioadmin" + Secret: "minioadmin" + EOF + chmod +x funnel + ./funnel server run --config `pwd`/config.yml &> funnel.logs & + ./funnel task run examples/s3-test.yml diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a9911e64b..911b61aab 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -2,6 +2,7 @@ name: Go Tests on: push: + pull_request: jobs: lint: @@ -11,33 +12,31 @@ jobs: - uses: actions/setup-go@v3 with: go-version: 1.21 + - uses: actions/checkout@v3 + - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: version: latest - # TODO: Re-enable all linters - args: --timeout 3m --verbose -D unused -D errcheck -D staticcheck -D govet -D gosimple - - build: - runs-on: ubuntu-latest - steps: - - name: Set up Go 1.x - uses: actions/setup-go@v5 - with: - go-version: 1.21 - - - name: Check out code - uses: actions/checkout@v2 - - - name: Build - run: make build - - - name: Store funnel - uses: actions/upload-artifact@v4 - with: - name: funnelBin - path: funnel + # Matches the "primary" golangci-lint command in the Makefile + args: | + --timeout 3m --disable-all --enable=govet --enable=gofmt --enable=goimports --enable=misspell \ + --skip-dirs "vendor" \ + --skip-dirs "webdash" \ + --skip-dirs "cmd/webdash" \ + --skip-dirs "funnel-work-dir" \ + -e '.*bundle.go' -e ".*pb.go" -e ".*pb.gw.go" \ + ./... + + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + version: latest + # Matches the "termdash" golangci-lint command in the Makefile + args: | + --timeout 3m --disable-all --enable=vet --enable=gofmt --enable=goimports --enable=misspell \ + ./cmd/termdash/... unitTest: runs-on: ubuntu-latest @@ -52,6 +51,9 @@ jobs: - name: Unit Tests run: make test-verbose + build: + uses: ./.github/workflows/build.yml + mongoTest: runs-on: ubuntu-latest needs: build @@ -65,9 +67,9 @@ jobs: uses: actions/checkout@v2 - name: Download funnel bin - uses: actions/download-artifact@v4.1.7 + uses: actions/download-artifact@v4 with: - name: funnelBin + name: funnel - name: MongoTest run: | @@ -88,9 +90,9 @@ jobs: uses: actions/checkout@v2 - name: Download funnel bin - uses: actions/download-artifact@v4.1.7 + uses: actions/download-artifact@v4 with: - name: funnelBin + name: funnel - name: Badger Test run: | chmod +x funnel @@ -108,9 +110,9 @@ jobs: uses: actions/checkout@v2 - name: Download funnel bin - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: - name: funnelBin + name: funnel - name: Slurm Test run: | @@ -129,9 +131,9 @@ jobs: uses: actions/checkout@v2 - name: Download funnel bin - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: - name: funnelBin + name: funnel - name: S3 Test run: | @@ -139,4 +141,4 @@ jobs: make start-generic-s3 sleep 10 make test-generic-s3 - \ No newline at end of file +