chore(test): split out test jobs into their own workflows

newrelic · May 28, 2024 · 24feefc · 24feefc
1 parent 0d9cb9e
commit 24feefc
Show file tree

Hide file tree

Showing 7 changed files with 250 additions and 148 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
diff --git a/.github/workflows/test_integration.yml b/.github/workflows/test_integration.yml
@@ -0,0 +1,85 @@
+name: Integration Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+  # schedule:
+  #   # Cron executes at 0800, 1200 1600 (8am, 12pm, 4pm)
+  #   - cron: "0 8,12,16 * * 1-5"
+
+jobs:
+  test-integration:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.21.x
+
+      # - name: Add GOBIN to PATH
+      #   run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+      #   shell: bash
+
+      # - name: Checkout code
+      #   uses: actions/checkout@v4
+
+      # - name: Cache deps
+      #   uses: actions/cache@v4
+      #   with:
+      #     path: ~/go/pkg/mod
+      #     key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+      #     restore-keys: |
+      #       ${{ runner.os }}-go-
+
+      # - name: Integration Tests
+      #   if: github.event.pull_request.head.repo.full_name == github.repository
+      #   run: make test-integration cover-report
+      #   env:
+      #     NEW_RELIC_ACCOUNT_ID: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
+      #     NEW_RELIC_SUBACCOUNT_ID: ${{ secrets.NEW_RELIC_SUBACCOUNT_ID }}
+      #     NEW_RELIC_ADMIN_API_KEY: ${{ secrets.NEW_RELIC_ADMIN_API_KEY }}
+      #     NEW_RELIC_API_KEY: ${{ secrets.NEW_RELIC_API_KEY }}
+      #     NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
+      #     NEW_RELIC_REGION: ${{ secrets.NEW_RELIC_REGION }}
+      #     NEW_RELIC_INSIGHTS_INSERT_KEY: ${{ secrets.NEW_RELIC_INSIGHTS_INSERT_KEY }}
+      #     NR_ACC_TESTING: ${{ secrets.NR_ACC_TESTING }}
+      #     INTEGRATION_TESTING_AWS_ARN: ${{ secrets.INTEGRATION_TESTING_AWS_ARN }}
+      #     INTEGRATION_TESTING_GCP_ACCOUNT_NAME: ${{ secrets.INTEGRATION_TESTING_GCP_ACCOUNT_NAME }}
+      #     INTEGRATION_TESTING_GCP_PROJECT_ID: ${{ secrets.INTEGRATION_TESTING_GCP_PROJECT_ID }}
+      #     INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID: ${{secrets.INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID}}
+      #     INTEGRATION_TESTING_AZURE_APPLICATION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_APPLICATION_ID }}
+      #     INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID }}
+      #     INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID }}
+      #     INTEGRATION_TESTING_AZURE_TENANT_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_TENANT_ID }}
+
+      # - name: Create test failure report
+      #   id: testFailureReport
+      #   run: |
+      #     go install github.com/mfridman/tparse@latest
+      #     make test-failure-report
+      #     tparse -file=coverage/integration.failures -format=markdown >> "$GITHUB_OUTPUT"
+
+      # - name: Send test failures report to Slack
+      #   id: slack
+      #   uses: slackapi/[email protected]
+      #   with:
+      #     # Uses Slack's Block Kit to build the message
+      #     payload: |
+      #       {
+      #         "text": "New Relic Terraform Provider Test Failures",
+      #         "blocks": [
+      #           {
+      #             "type": "section",
+      #             "text": {
+      #               "type": "mrkdwn",
+      #               "text": "${{ steps.testFailureReport.outputs }}"
+      #             }
+      #           }
+      #         ]
+      #       }
+      #   env:
+      #     SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+      #     SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+
diff --git a/.github/workflows/test_lint.yml b/.github/workflows/test_lint.yml
@@ -0,0 +1,47 @@
+name: Lint Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.21.x
+
+      - name: Add GOBIN to PATH
+        run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+        shell: bash
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Full git history
+        run: |
+          git fetch --prune --unshallow
+
+      - name: Cache deps
+        uses: actions/cache@v4
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Lint
+        run: make lint
+
+      - name: Check links
+        uses: gaurav-nelson/github-action-markdown-link-check@v1
+        with:
+          use-quiet-mode: 'yes'
+          use-verbose-mode: 'yes'
+          config-file: '.markdownlinkcheck.json'
+          folder-path: 'website/docs'
+          file-extension: '.markdown'
diff --git a/.github/workflows/test_unit.yml b/.github/workflows/test_unit.yml
@@ -0,0 +1,94 @@
+name: Unit Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test-unit:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.21.x
+
+      - name: Add GOBIN to PATH
+        run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+        shell: bash
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Cache deps
+        uses: actions/cache@v4
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Unit Tests
+        id: unit-tests
+        run: |
+          make test-unit cover-report
+
+      - name: Create test failure report
+        id: test-failure-report
+        if: ${{ failure() }}
+        run: |
+          go install github.com/mfridman/tparse@latest
+          make test-failure-report
+          tparse -file=coverage/unit.failures -format=markdown > coverage/unit.md
+
+      - name: Prepare multiline test report
+        id: payload
+        shell: bash
+        if: ${{ always() }}
+        run: |
+          cat coverage/unit.md
+          result=$(cat coverage/unit.md)
+
+          echo "Result: $result"
+
+          echo 'UNIT_TEST_FAILURES<<EOF'
+          $result
+          echo EOF
+
+      - name: Failure report
+        id: failure-report
+        if: ${{ always() }}
+        run: |
+          echo "${{ join(steps.test-failure-report.outputs.*, '\n') }}"
+
+      # - name: Report unit test coverage via Codecov
+      #   uses: codecov/codecov-action@v3
+      #   with:
+      #     files: ./coverage/coverage.out
+
+      - name: Send test failures report to Slack
+        id: slack
+        if: ${{ always() }}
+        uses: slackapi/[email protected]
+        with:
+          # Uses Slack's Block Kit to build the message
+          payload: |
+            {
+              "text": "New Relic Terraform Provider Test Failures",
+              "blocks": [
+                {
+                  "type": "context",
+                  "elements": [
+                    {
+                      "type": "plain_text",
+                      "text": ${{ toJSON(steps.test-failure-report.outputs.UNIT_TEST_FAILURES) }}
+                    }
+                  ]
+                }
+              ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
diff --git a/build/test.mk b/build/test.mk
@@ -25,16 +25,21 @@ test-only: test-unit test-integration
 test-unit: tools
 	@echo "=== $(PROJECT_NAME) === [ test-unit        ]: running unit tests..."
 	@mkdir -p $(COVERAGE_DIR)
-	@$(TEST_RUNNER) -f testname --junitfile $(COVERAGE_DIR)/unit.xml --packages "$(GO_PKGS)" \
+	@$(TEST_RUNNER) -f testname --junitfile $(COVERAGE_DIR)/unit.xml --packages "$(GO_PKGS)" --jsonfile $(COVERAGE_DIR)/unit.report \
 		-- -v -parallel 10 -tags=unit $(TEST_ARGS) -covermode=$(COVERMODE) -coverprofile $(COVERAGE_DIR)/unit.tmp
 
 test-integration: tools
 	@echo "=== $(PROJECT_NAME) === [ test-integration ]: running integration tests..."
 	@mkdir -p $(COVERAGE_DIR)
-	@TF_ACC=1 $(TEST_RUNNER) -f testname --junitfile $(COVERAGE_DIR)/integration.xml --rerun-fails=1 --packages "$(GO_PKGS)" \
-		-- -v -parallel 8 -tags=integration $(TEST_ARGS) -covermode=$(COVERMODE) -coverprofile $(COVERAGE_DIR)/integration.tmp \
+	@TF_ACC=1 $(TEST_RUNNER) -f testname --junitfile $(COVERAGE_DIR)/integration.xml --rerun-fails=1 --packages "$(GO_PKGS)" --jsonfile $(COVERAGE_DIR)/integration.report \
+		-- -v -parallel 10 -tags=integration $(TEST_ARGS) -covermode=$(COVERMODE) -coverprofile $(COVERAGE_DIR)/integration.tmp \
 		   -timeout 120m -ldflags=$(LDFLAGS_TEST)
 
+test-failure-report: tools
+	@echo "=== $(PROJECT_NAME) === [ test-report-failed ]: generating report..."
+	@mkdir -p $(COVERAGE_DIR)
+	./scripts/test-failure-report.sh
+
 #
 # Coverage
 #