From 24feefc77c66a0335857a4fab1454e2fdd2305d3 Mon Sep 17 00:00:00 2001 From: Sander Blue Date: Mon, 29 Apr 2024 16:02:18 -0500 Subject: [PATCH 01/12] chore(test): split out test jobs into their own workflows --- .github/workflows/test.yml | 144 ------------------------- .github/workflows/test_integration.yml | 85 +++++++++++++++ .github/workflows/test_lint.yml | 47 ++++++++ .github/workflows/test_unit.yml | 94 ++++++++++++++++ build/test.mk | 11 +- newrelic/provider_unit_test.go | 2 +- scripts/test-failure-report.sh | 15 +++ 7 files changed, 250 insertions(+), 148 deletions(-) delete mode 100644 .github/workflows/test.yml create mode 100644 .github/workflows/test_integration.yml create mode 100644 .github/workflows/test_lint.yml create mode 100644 .github/workflows/test_unit.yml create mode 100755 scripts/test-failure-report.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 7bb0d8219..000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,144 +0,0 @@ -name: Testing - -on: - workflow_dispatch: - push: - branches: [main] - pull_request: - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - name: Install Go - uses: actions/setup-go@v5 - with: - go-version: 1.21.x - - - name: Add GOBIN to PATH - run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - shell: bash - - - name: Checkout code - uses: actions/checkout@v4 - - - name: Full git history - run: | - git fetch --prune --unshallow - - - name: Cache deps - uses: actions/cache@v4 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - - name: Lint - run: make lint - - - name: Check links - uses: gaurav-nelson/github-action-markdown-link-check@v1 - with: - use-quiet-mode: 'yes' - use-verbose-mode: 'yes' - config-file: '.markdownlinkcheck.json' - folder-path: 'website/docs' - file-extension: '.markdown' - - test-unit: - runs-on: ubuntu-latest - steps: - - name: Install Go - uses: actions/setup-go@v5 - with: - go-version: 1.21.x - - - name: Add GOBIN to PATH - run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - shell: bash - - - name: Checkout code - uses: actions/checkout@v4 - - - name: Cache deps - uses: actions/cache@v4 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - - name: Unit Tests - run: make test-unit cover-report - - - name: Report unit test coverage via Codecov - uses: codecov/codecov-action@v3 - with: - files: ./coverage/coverage.out - - - # - name: New Relic JUnit Reporter - # if: github.event.pull_request.head.repo.full_name == github.repository - # uses: newrelic/junit-reporter-action@v0.1.1 - # with: - # accountId: ${{ secrets.NEW_RELIC_ACCOUNT_ID }} - # insertApiKey: ${{ secrets.NEW_RELIC_INSIGHTS_INSERT_KEY }} - # testOutputPath: coverage/unit.xml - - test-integration: - runs-on: ubuntu-latest - steps: - - name: Install Go - uses: actions/setup-go@v5 - with: - go-version: 1.21.x - - - name: Add GOBIN to PATH - run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - shell: bash - - - name: Checkout code - uses: actions/checkout@v4 - - - name: Cache deps - uses: actions/cache@v4 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - - name: Integration Tests - if: github.event.pull_request.head.repo.full_name == github.repository - run: make test-integration cover-report - env: - NEW_RELIC_ACCOUNT_ID: ${{ secrets.NEW_RELIC_ACCOUNT_ID }} - NEW_RELIC_SUBACCOUNT_ID: ${{ secrets.NEW_RELIC_SUBACCOUNT_ID }} - NEW_RELIC_ADMIN_API_KEY: ${{ secrets.NEW_RELIC_ADMIN_API_KEY }} - NEW_RELIC_API_KEY: ${{ secrets.NEW_RELIC_API_KEY }} - NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} - NEW_RELIC_REGION: ${{ secrets.NEW_RELIC_REGION }} - NEW_RELIC_INSIGHTS_INSERT_KEY: ${{ secrets.NEW_RELIC_INSIGHTS_INSERT_KEY }} - NR_ACC_TESTING: ${{ secrets.NR_ACC_TESTING }} - INTEGRATION_TESTING_AWS_ARN: ${{ secrets.INTEGRATION_TESTING_AWS_ARN }} - INTEGRATION_TESTING_GCP_ACCOUNT_NAME: ${{ secrets.INTEGRATION_TESTING_GCP_ACCOUNT_NAME }} - INTEGRATION_TESTING_GCP_PROJECT_ID: ${{ secrets.INTEGRATION_TESTING_GCP_PROJECT_ID }} - INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID: ${{secrets.INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID}} - INTEGRATION_TESTING_AZURE_APPLICATION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_APPLICATION_ID }} - INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID }} - INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID }} - INTEGRATION_TESTING_AZURE_TENANT_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_TENANT_ID }} - - - name: Report integration test coverage via Codecov - uses: codecov/codecov-action@v3 - with: - files: ./coverage/coverage.out - - # - name: New Relic JUnit Reporter - # if: github.event.pull_request.head.repo.full_name == github.repository - # uses: newrelic/junit-reporter-action@v0.1.1 - # with: - # accountId: ${{ secrets.NEW_RELIC_ACCOUNT_ID }} - # insertApiKey: ${{ secrets.NEW_RELIC_INSIGHTS_INSERT_KEY }} - # testOutputPath: coverage/integration.xml diff --git a/.github/workflows/test_integration.yml b/.github/workflows/test_integration.yml new file mode 100644 index 000000000..716b6883d --- /dev/null +++ b/.github/workflows/test_integration.yml @@ -0,0 +1,85 @@ +name: Integration Tests + +on: + workflow_dispatch: + push: + branches: [main] + pull_request: + # schedule: + # # Cron executes at 0800, 1200 1600 (8am, 12pm, 4pm) + # - cron: "0 8,12,16 * * 1-5" + +jobs: + test-integration: + runs-on: ubuntu-latest + steps: + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.21.x + + # - name: Add GOBIN to PATH + # run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + # shell: bash + + # - name: Checkout code + # uses: actions/checkout@v4 + + # - name: Cache deps + # uses: actions/cache@v4 + # with: + # path: ~/go/pkg/mod + # key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + # restore-keys: | + # ${{ runner.os }}-go- + + # - name: Integration Tests + # if: github.event.pull_request.head.repo.full_name == github.repository + # run: make test-integration cover-report + # env: + # NEW_RELIC_ACCOUNT_ID: ${{ secrets.NEW_RELIC_ACCOUNT_ID }} + # NEW_RELIC_SUBACCOUNT_ID: ${{ secrets.NEW_RELIC_SUBACCOUNT_ID }} + # NEW_RELIC_ADMIN_API_KEY: ${{ secrets.NEW_RELIC_ADMIN_API_KEY }} + # NEW_RELIC_API_KEY: ${{ secrets.NEW_RELIC_API_KEY }} + # NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} + # NEW_RELIC_REGION: ${{ secrets.NEW_RELIC_REGION }} + # NEW_RELIC_INSIGHTS_INSERT_KEY: ${{ secrets.NEW_RELIC_INSIGHTS_INSERT_KEY }} + # NR_ACC_TESTING: ${{ secrets.NR_ACC_TESTING }} + # INTEGRATION_TESTING_AWS_ARN: ${{ secrets.INTEGRATION_TESTING_AWS_ARN }} + # INTEGRATION_TESTING_GCP_ACCOUNT_NAME: ${{ secrets.INTEGRATION_TESTING_GCP_ACCOUNT_NAME }} + # INTEGRATION_TESTING_GCP_PROJECT_ID: ${{ secrets.INTEGRATION_TESTING_GCP_PROJECT_ID }} + # INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID: ${{secrets.INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID}} + # INTEGRATION_TESTING_AZURE_APPLICATION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_APPLICATION_ID }} + # INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID }} + # INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID }} + # INTEGRATION_TESTING_AZURE_TENANT_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_TENANT_ID }} + + # - name: Create test failure report + # id: testFailureReport + # run: | + # go install github.com/mfridman/tparse@latest + # make test-failure-report + # tparse -file=coverage/integration.failures -format=markdown >> "$GITHUB_OUTPUT" + + # - name: Send test failures report to Slack + # id: slack + # uses: slackapi/slack-github-action@v1.26 + # with: + # # Uses Slack's Block Kit to build the message + # payload: | + # { + # "text": "New Relic Terraform Provider Test Failures", + # "blocks": [ + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "${{ steps.testFailureReport.outputs }}" + # } + # } + # ] + # } + # env: + # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + # SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + diff --git a/.github/workflows/test_lint.yml b/.github/workflows/test_lint.yml new file mode 100644 index 000000000..32251bb1c --- /dev/null +++ b/.github/workflows/test_lint.yml @@ -0,0 +1,47 @@ +name: Lint Tests + +on: + workflow_dispatch: + push: + branches: [main] + pull_request: + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.21.x + + - name: Add GOBIN to PATH + run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + shell: bash + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Full git history + run: | + git fetch --prune --unshallow + + - name: Cache deps + uses: actions/cache@v4 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Lint + run: make lint + + - name: Check links + uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + use-quiet-mode: 'yes' + use-verbose-mode: 'yes' + config-file: '.markdownlinkcheck.json' + folder-path: 'website/docs' + file-extension: '.markdown' diff --git a/.github/workflows/test_unit.yml b/.github/workflows/test_unit.yml new file mode 100644 index 000000000..60ad540fb --- /dev/null +++ b/.github/workflows/test_unit.yml @@ -0,0 +1,94 @@ +name: Unit Tests + +on: + workflow_dispatch: + push: + branches: [main] + pull_request: + +jobs: + test-unit: + runs-on: ubuntu-latest + steps: + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.21.x + + - name: Add GOBIN to PATH + run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + shell: bash + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Cache deps + uses: actions/cache@v4 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Unit Tests + id: unit-tests + run: | + make test-unit cover-report + + - name: Create test failure report + id: test-failure-report + if: ${{ failure() }} + run: | + go install github.com/mfridman/tparse@latest + make test-failure-report + tparse -file=coverage/unit.failures -format=markdown > coverage/unit.md + + - name: Prepare multiline test report + id: payload + shell: bash + if: ${{ always() }} + run: | + cat coverage/unit.md + result=$(cat coverage/unit.md) + + echo "Result: $result" + + echo 'UNIT_TEST_FAILURES< coverage/integration.failures +fi + +if ! [ -f ./coverage/unit.report ]; then + echo "File coverage/unit.report does not exist. Skipping." +else + grep 'FAIL:\|FAIL\|Error:\|error:' coverage/unit.report > coverage/unit.failures +fi + From 092ca5a46e4af8a756e18d6e602db69bad670910 Mon Sep 17 00:00:00 2001 From: Sander Blue Date: Thu, 23 May 2024 13:57:35 -0500 Subject: [PATCH 02/12] chore: testing drift detection script --- .github/workflows/test_unit.yml | 71 +++++++++++++++++++-------------- scripts/detect-state-drift.js | 54 +++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 29 deletions(-) create mode 100755 scripts/detect-state-drift.js diff --git a/.github/workflows/test_unit.yml b/.github/workflows/test_unit.yml index 60ad540fb..60cebe634 100644 --- a/.github/workflows/test_unit.yml +++ b/.github/workflows/test_unit.yml @@ -35,60 +35,73 @@ jobs: run: | make test-unit cover-report - - name: Create test failure report - id: test-failure-report - if: ${{ failure() }} - run: | - go install github.com/mfridman/tparse@latest - make test-failure-report - tparse -file=coverage/unit.failures -format=markdown > coverage/unit.md - - - name: Prepare multiline test report - id: payload - shell: bash - if: ${{ always() }} - run: | - cat coverage/unit.md - result=$(cat coverage/unit.md) - - echo "Result: $result" - - echo 'UNIT_TEST_FAILURES< { + const fs = require('fs'); + + // Read the text from the file + fs.readFile('coverage/unit.report', 'utf8', (err, data) => { + if (err) { + console.error('Error reading file:', err); + return; + } + + // Split the text into individual lines + const lines = data.trim().split('\n'); + + // Parse each line and convert it to JSON + const jsonData = lines.map(line => { + try { + return JSON.parse(line); + } catch (error) { + console.error('Error parsing line:', error); + return null; + } + }); + + // Print the resulting JSON array + // console.log(jsonData); + + const report = jsonData.filter(data => { + return data.Output ? data.Output.includes('error: After applying this test step, the plan was not empty') : false; + }).map(t => t.Output.trim()); + + console.log(report); + + let msg = 'error: After applying this test step, the plan was not empty'; + if (report.length > 0) { + msg = `'${report.join('\n')}'`; + } + + core.setOutput('failed_tests_with_drift', msg); + + // // Convert JSON data to string + // const jsonString = JSON.stringify(failedTests, null, 2); + + // // Write the JSON data to a file + // fs.writeFile('output.json', jsonString, 'utf8', err => { + // if (err) { + // console.error('Error writing file:', err); + // return; + // } + // console.log('JSON data has been written to output.json'); + // }); + }); +}; From da19f1faa0ef8ade52cc20d5e9a854ebeda6eb43 Mon Sep 17 00:00:00 2001 From: Sander Blue Date: Thu, 23 May 2024 15:51:32 -0500 Subject: [PATCH 03/12] feat(automation): run integration tests and report drift --- .github/workflows/test_integration.yml | 131 ++++++++++---------- .github/workflows/test_integration_cron.yml | 101 +++++++++++++++ .github/workflows/test_unit.yml | 65 +--------- build/test.mk | 5 - newrelic/provider_unit_test.go | 2 +- scripts/detect-state-drift.js | 17 +-- 6 files changed, 171 insertions(+), 150 deletions(-) create mode 100644 .github/workflows/test_integration_cron.yml diff --git a/.github/workflows/test_integration.yml b/.github/workflows/test_integration.yml index 716b6883d..e817d2f1d 100644 --- a/.github/workflows/test_integration.yml +++ b/.github/workflows/test_integration.yml @@ -5,9 +5,40 @@ on: push: branches: [main] pull_request: - # schedule: - # # Cron executes at 0800, 1200 1600 (8am, 12pm, 4pm) - # - cron: "0 8,12,16 * * 1-5" + workflow_call: + secrets: + NEW_RELIC_ACCOUNT_ID: + required: true + NEW_RELIC_SUBACCOUNT_ID: + required: true + NEW_RELIC_ADMIN_API_KEY: + required: true + NEW_RELIC_API_KEY: + required: true + NEW_RELIC_LICENSE_KEY: + required: true + NEW_RELIC_REGION: + required: true + NEW_RELIC_INSIGHTS_INSERT_KEY: + required: true + NR_ACC_TESTING: + required: true + INTEGRATION_TESTING_AWS_ARN: + required: true + INTEGRATION_TESTING_GCP_ACCOUNT_NAME: + required: true + INTEGRATION_TESTING_GCP_PROJECT_ID: + required: true + INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID: + required: true + INTEGRATION_TESTING_AZURE_APPLICATION_ID: + required: true + INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID: + required: true + INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID: + required: true + INTEGRATION_TESTING_AZURE_TENANT_ID: + required: true jobs: test-integration: @@ -18,68 +49,38 @@ jobs: with: go-version: 1.21.x - # - name: Add GOBIN to PATH - # run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - # shell: bash + - name: Add GOBIN to PATH + run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + shell: bash - # - name: Checkout code - # uses: actions/checkout@v4 + - name: Checkout code + uses: actions/checkout@v4 - # - name: Cache deps - # uses: actions/cache@v4 - # with: - # path: ~/go/pkg/mod - # key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - # restore-keys: | - # ${{ runner.os }}-go- - - # - name: Integration Tests - # if: github.event.pull_request.head.repo.full_name == github.repository - # run: make test-integration cover-report - # env: - # NEW_RELIC_ACCOUNT_ID: ${{ secrets.NEW_RELIC_ACCOUNT_ID }} - # NEW_RELIC_SUBACCOUNT_ID: ${{ secrets.NEW_RELIC_SUBACCOUNT_ID }} - # NEW_RELIC_ADMIN_API_KEY: ${{ secrets.NEW_RELIC_ADMIN_API_KEY }} - # NEW_RELIC_API_KEY: ${{ secrets.NEW_RELIC_API_KEY }} - # NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} - # NEW_RELIC_REGION: ${{ secrets.NEW_RELIC_REGION }} - # NEW_RELIC_INSIGHTS_INSERT_KEY: ${{ secrets.NEW_RELIC_INSIGHTS_INSERT_KEY }} - # NR_ACC_TESTING: ${{ secrets.NR_ACC_TESTING }} - # INTEGRATION_TESTING_AWS_ARN: ${{ secrets.INTEGRATION_TESTING_AWS_ARN }} - # INTEGRATION_TESTING_GCP_ACCOUNT_NAME: ${{ secrets.INTEGRATION_TESTING_GCP_ACCOUNT_NAME }} - # INTEGRATION_TESTING_GCP_PROJECT_ID: ${{ secrets.INTEGRATION_TESTING_GCP_PROJECT_ID }} - # INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID: ${{secrets.INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID}} - # INTEGRATION_TESTING_AZURE_APPLICATION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_APPLICATION_ID }} - # INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID }} - # INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID }} - # INTEGRATION_TESTING_AZURE_TENANT_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_TENANT_ID }} - - # - name: Create test failure report - # id: testFailureReport - # run: | - # go install github.com/mfridman/tparse@latest - # make test-failure-report - # tparse -file=coverage/integration.failures -format=markdown >> "$GITHUB_OUTPUT" - - # - name: Send test failures report to Slack - # id: slack - # uses: slackapi/slack-github-action@v1.26 - # with: - # # Uses Slack's Block Kit to build the message - # payload: | - # { - # "text": "New Relic Terraform Provider Test Failures", - # "blocks": [ - # { - # "type": "section", - # "text": { - # "type": "mrkdwn", - # "text": "${{ steps.testFailureReport.outputs }}" - # } - # } - # ] - # } - # env: - # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - # SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + - name: Cache deps + uses: actions/cache@v4 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Integration Tests + if: github.event.pull_request.head.repo.full_name == github.repository + run: make test-integration cover-report + env: + NEW_RELIC_ACCOUNT_ID: ${{ secrets.NEW_RELIC_ACCOUNT_ID }} + NEW_RELIC_SUBACCOUNT_ID: ${{ secrets.NEW_RELIC_SUBACCOUNT_ID }} + NEW_RELIC_ADMIN_API_KEY: ${{ secrets.NEW_RELIC_ADMIN_API_KEY }} + NEW_RELIC_API_KEY: ${{ secrets.NEW_RELIC_API_KEY }} + NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} + NEW_RELIC_REGION: ${{ secrets.NEW_RELIC_REGION }} + NEW_RELIC_INSIGHTS_INSERT_KEY: ${{ secrets.NEW_RELIC_INSIGHTS_INSERT_KEY }} + NR_ACC_TESTING: ${{ secrets.NR_ACC_TESTING }} + INTEGRATION_TESTING_AWS_ARN: ${{ secrets.INTEGRATION_TESTING_AWS_ARN }} + INTEGRATION_TESTING_GCP_ACCOUNT_NAME: ${{ secrets.INTEGRATION_TESTING_GCP_ACCOUNT_NAME }} + INTEGRATION_TESTING_GCP_PROJECT_ID: ${{ secrets.INTEGRATION_TESTING_GCP_PROJECT_ID }} + INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID: ${{secrets.INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID}} + INTEGRATION_TESTING_AZURE_APPLICATION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_APPLICATION_ID }} + INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID }} + INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID }} + INTEGRATION_TESTING_AZURE_TENANT_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_TENANT_ID }} diff --git a/.github/workflows/test_integration_cron.yml b/.github/workflows/test_integration_cron.yml new file mode 100644 index 000000000..95b979610 --- /dev/null +++ b/.github/workflows/test_integration_cron.yml @@ -0,0 +1,101 @@ +name: Integration Tests Cron Job + +on: + workflow_dispatch: # this is just to test the functionality, remove before merging!!!!!!!!!!!! + # schedule: + # # Cron executes at 0800, 1200 1600 (8am, 12pm, 4pm) + # - cron: "0 8,12,16 * * 1-5" + +jobs: + test-integration: + # if: github.ref == 'refs/heads/main' + uses: newrelic/terraform-provider-newrelic/.github/workflows/test_integration.yml@feat/integration-test-cron-job + secrets: + NEW_RELIC_ACCOUNT_ID: ${{ secrets.NEW_RELIC_ACCOUNT_ID }} + NEW_RELIC_SUBACCOUNT_ID: ${{ secrets.NEW_RELIC_SUBACCOUNT_ID }} + NEW_RELIC_ADMIN_API_KEY: ${{ secrets.NEW_RELIC_ADMIN_API_KEY }} + NEW_RELIC_API_KEY: ${{ secrets.NEW_RELIC_API_KEY }} + NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} + NEW_RELIC_REGION: ${{ secrets.NEW_RELIC_REGION }} + NEW_RELIC_INSIGHTS_INSERT_KEY: ${{ secrets.NEW_RELIC_INSIGHTS_INSERT_KEY }} + NR_ACC_TESTING: ${{ secrets.NR_ACC_TESTING }} + INTEGRATION_TESTING_AWS_ARN: ${{ secrets.INTEGRATION_TESTING_AWS_ARN }} + INTEGRATION_TESTING_GCP_ACCOUNT_NAME: ${{ secrets.INTEGRATION_TESTING_GCP_ACCOUNT_NAME }} + INTEGRATION_TESTING_GCP_PROJECT_ID: ${{ secrets.INTEGRATION_TESTING_GCP_PROJECT_ID }} + INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID: ${{secrets.INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID}} + INTEGRATION_TESTING_AZURE_APPLICATION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_APPLICATION_ID }} + INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID }} + INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID }} + INTEGRATION_TESTING_AZURE_TENANT_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_TENANT_ID }} + + detect-drift: + runs-on: ubuntu-latest + steps: + - name: Check check + run: ls + + - name: Detect drift + if: ${{ always() }} + uses: actions/github-script@v7 + id: detect-state-drift + with: + script: | + const script = require('./scripts/detect-state-drift.js') + await script({core}) + + - name: Send report to Slack + id: slack + if: ${{ steps.detect-state-drift.outputs.drift_report != '' }} + uses: slackapi/slack-github-action@v1.26 + with: + # Uses Slack's Block Kit to build the message + # https://app.slack.com/block-kit-builder + payload: | + { + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":warning: *Terraform Provider | Drift Detected*" + } + }, + { + "type": "divider" + }, + { + "type": "context", + "elements": [ + { + "type": "plain_text", + "text": ${{ toJSON(steps.detect-state-drift.outputs.failed_tests_with_drift) }} + } + ] + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "More information can be viewed in the job summary." + }, + "accessory": { + "type": "button", + "text": { + "type": "plain_text", + "text": "Job Summary", + "emoji": true + }, + "value": "View Job Summary", + "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", + "action_id": "button-action" + } + }, + { + "type": "divider" + } + ] + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + diff --git a/.github/workflows/test_unit.yml b/.github/workflows/test_unit.yml index 60cebe634..af6f811b6 100644 --- a/.github/workflows/test_unit.yml +++ b/.github/workflows/test_unit.yml @@ -35,73 +35,10 @@ jobs: run: | make test-unit cover-report - - name: Detect drift - if: ${{ always() }} - uses: actions/github-script@v7 - id: detect-state-drift - with: - script: | - const script = require('./scripts/detect-state-drift.js') - await script({core}) - # - name: Report unit test coverage via Codecov # uses: codecov/codecov-action@v3 # with: # files: ./coverage/coverage.out - - name: Send report to Slack - id: slack - if: ${{ always() }} - uses: slackapi/slack-github-action@v1.26 - with: - # Uses Slack's Block Kit to build the message - # https://app.slack.com/block-kit-builder - payload: | - { - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": ":warning: *Terraform Provider | Drift Detected*" - } - }, - { - "type": "divider" - }, - { - "type": "context", - "elements": [ - { - "type": "plain_text", - "text": ${{ toJSON(steps.detect-state-drift.outputs.failed_tests_with_drift) }} - } - ] - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "More information can be viewed in the job summary." - }, - "accessory": { - "type": "button", - "text": { - "type": "plain_text", - "text": "Job Summary", - "emoji": true - }, - "value": "View Job Summary", - "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", - "action_id": "button-action" - } - }, - { - "type": "divider" - } - ] - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + diff --git a/build/test.mk b/build/test.mk index 7c560af0a..138a7dd51 100644 --- a/build/test.mk +++ b/build/test.mk @@ -35,11 +35,6 @@ test-integration: tools -- -v -parallel 10 -tags=integration $(TEST_ARGS) -covermode=$(COVERMODE) -coverprofile $(COVERAGE_DIR)/integration.tmp \ -timeout 120m -ldflags=$(LDFLAGS_TEST) -test-failure-report: tools - @echo "=== $(PROJECT_NAME) === [ test-report-failed ]: generating report..." - @mkdir -p $(COVERAGE_DIR) - ./scripts/test-failure-report.sh - # # Coverage # diff --git a/newrelic/provider_unit_test.go b/newrelic/provider_unit_test.go index 65a34e8b0..67d14760a 100644 --- a/newrelic/provider_unit_test.go +++ b/newrelic/provider_unit_test.go @@ -28,7 +28,7 @@ func TestBuildUserAgentStringWithDefaultServiceName(t *testing.T) { tfUA := "HashiCorp Terraform/1.3.5 (+https://www.terraform.io) Terraform Plugin SDK/2.10.1" result := buildUserAgentString(tfUA, getUserAgentServiceName(), ProviderVersion) - require.Contains(t, result, " terraform-provider-newrelic/dev---fixthisbcitsatest") + require.Contains(t, result, " terraform-provider-newrelic/dev") } func TestBuildUserAgentStringWithCustomServiceName(t *testing.T) { diff --git a/scripts/detect-state-drift.js b/scripts/detect-state-drift.js index 082199991..d51ec8ee3 100755 --- a/scripts/detect-state-drift.js +++ b/scripts/detect-state-drift.js @@ -4,7 +4,7 @@ module.exports = async ({ const fs = require('fs'); // Read the text from the file - fs.readFile('coverage/unit.report', 'utf8', (err, data) => { + fs.readFile('coverage/integration.report', 'utf8', (err, data) => { if (err) { console.error('Error reading file:', err); return; @@ -23,7 +23,6 @@ module.exports = async ({ } }); - // Print the resulting JSON array // console.log(jsonData); const report = jsonData.filter(data => { @@ -37,18 +36,6 @@ module.exports = async ({ msg = `'${report.join('\n')}'`; } - core.setOutput('failed_tests_with_drift', msg); - - // // Convert JSON data to string - // const jsonString = JSON.stringify(failedTests, null, 2); - - // // Write the JSON data to a file - // fs.writeFile('output.json', jsonString, 'utf8', err => { - // if (err) { - // console.error('Error writing file:', err); - // return; - // } - // console.log('JSON data has been written to output.json'); - // }); + core.setOutput('drift_report', msg); }); }; From 3f8027632e67266f89622d224996d2526ae4d7f7 Mon Sep 17 00:00:00 2001 From: Sander Blue Date: Tue, 28 May 2024 13:03:19 -0500 Subject: [PATCH 04/12] chore(tests): minor adjustment for entity (app) creation lag --- newrelic/data_source_newrelic_entity_integration_test.go | 3 +++ newrelic/provider_test.go | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/newrelic/data_source_newrelic_entity_integration_test.go b/newrelic/data_source_newrelic_entity_integration_test.go index 8fc28ed34..800f047d8 100644 --- a/newrelic/data_source_newrelic_entity_integration_test.go +++ b/newrelic/data_source_newrelic_entity_integration_test.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" "testing" + "time" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" @@ -37,6 +38,8 @@ func TestAccNewRelicSingleQuotedEntityData_Basic(t *testing.T) { resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { testAccSingleQuotedPreCheck(t) + + time.Sleep(2 * time.Second) }, Providers: testAccProviders, Steps: []resource.TestStep{ diff --git a/newrelic/provider_test.go b/newrelic/provider_test.go index e9421e831..77a9d912a 100644 --- a/newrelic/provider_test.go +++ b/newrelic/provider_test.go @@ -171,6 +171,8 @@ func testAccCreateEntity(t *testing.T, name string) { t.Fatalf("Unable to find application entity: %s", retryErr) } + // We have to give time for the async nature of the entity creation to complete + time.Sleep(1 * time.Second) } func testAccPreCheckEnvVars(t *testing.T) { @@ -237,7 +239,7 @@ func testAccApplicationsCleanup(t *testing.T) { if err == nil { deletedAppCount++ - t.Logf("deleted application %d (%d/%d)", app.ID, deletedAppCount, len(applications)) + t.Logf("deleted application %s (%d/%d)", app.Name, deletedAppCount, len(applications)) } } } From f32aa6e33aad0d38e7f6fb02e7bb1b2df5cdf039 Mon Sep 17 00:00:00 2001 From: Sander Blue Date: Tue, 28 May 2024 13:39:55 -0500 Subject: [PATCH 05/12] chore(tests): avoid polluting test environment during user agent unit tests --- newrelic/provider_unit_test.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/newrelic/provider_unit_test.go b/newrelic/provider_unit_test.go index 67d14760a..6eb9a7f9d 100644 --- a/newrelic/provider_unit_test.go +++ b/newrelic/provider_unit_test.go @@ -23,8 +23,6 @@ func TestGenerateNameForIntegrationTestResource(t *testing.T) { } func TestBuildUserAgentStringWithDefaultServiceName(t *testing.T) { - t.Parallel() - tfUA := "HashiCorp Terraform/1.3.5 (+https://www.terraform.io) Terraform Plugin SDK/2.10.1" result := buildUserAgentString(tfUA, getUserAgentServiceName(), ProviderVersion) @@ -32,8 +30,6 @@ func TestBuildUserAgentStringWithDefaultServiceName(t *testing.T) { } func TestBuildUserAgentStringWithCustomServiceName(t *testing.T) { - t.Parallel() - // In a real scenario, this is set via -ldflags at compile time. // This would be "pulumi" or some other third-party service in a real scenario. UserAgentServiceName = "test" @@ -47,15 +43,11 @@ func TestBuildUserAgentStringWithCustomServiceName(t *testing.T) { } func TestGetUserAgentServiceNameDefault(t *testing.T) { - t.Parallel() - result := getUserAgentServiceName() require.Equal(t, "terraform-provider-newrelic", result) } func TestGetUserAgentServiceNameCustom(t *testing.T) { - t.Parallel() - // In a real scenario, this is set via -ldflags at compile time. UserAgentServiceName = "test" From 095679c50b915c6307b3c7fcecfb9cc130db8d22 Mon Sep 17 00:00:00 2001 From: Sander Blue Date: Tue, 28 May 2024 13:55:52 -0500 Subject: [PATCH 06/12] chore(tests): improve test efficiency, no need to run testAccPreCheck() here --- newrelic/resource_newrelic_alert_policy_channel_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/newrelic/resource_newrelic_alert_policy_channel_test.go b/newrelic/resource_newrelic_alert_policy_channel_test.go index ec6e55849..ee4a39ee4 100644 --- a/newrelic/resource_newrelic_alert_policy_channel_test.go +++ b/newrelic/resource_newrelic_alert_policy_channel_test.go @@ -18,7 +18,6 @@ func TestAccNewRelicAlertPolicyChannel_Basic(t *testing.T) { rName := acctest.RandString(5) resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, Providers: testAccProviders, CheckDestroy: testAccCheckNewRelicAlertPolicyChannelDestroy, Steps: []resource.TestStep{ @@ -51,7 +50,6 @@ func TestAccNewRelicAlertPolicyChannel_ChannelOrder(t *testing.T) { rName := acctest.RandString(5) resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, Providers: testAccProviders, CheckDestroy: testAccCheckNewRelicAlertPolicyChannelDestroy, Steps: []resource.TestStep{ @@ -71,7 +69,6 @@ func TestAccNewRelicAlertPolicyChannel_MutipleChannels(t *testing.T) { rName := acctest.RandString(5) resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, Providers: testAccProviders, CheckDestroy: testAccCheckNewRelicAlertPolicyChannelDestroy, Steps: []resource.TestStep{ @@ -103,7 +100,6 @@ func TestAccNewRelicAlertPolicyChannel_AlertPolicyNotFound(t *testing.T) { rName := fmt.Sprintf("tf-test-%s", acctest.RandString(5)) resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, Providers: testAccProviders, CheckDestroy: testAccCheckNewRelicAlertPolicyChannelDestroy, Steps: []resource.TestStep{ @@ -125,7 +121,6 @@ func TestAccNewRelicAlertPolicyChannel_AlertChannelNotFound(t *testing.T) { rName := fmt.Sprintf("tf-test-%s", acctest.RandString(5)) resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, Providers: testAccProviders, CheckDestroy: testAccCheckNewRelicAlertPolicyChannelDestroy, Steps: []resource.TestStep{ From f51ebdac563ada5bcbfef9d51ec5de71315d8682 Mon Sep 17 00:00:00 2001 From: Sander Blue Date: Tue, 28 May 2024 14:47:22 -0500 Subject: [PATCH 07/12] chore(tests): account for breaking API change, js_config was removed from browser application response --- ...newrelic_agent_application_browser_integration_test.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/newrelic/resource_newrelic_agent_application_browser_integration_test.go b/newrelic/resource_newrelic_agent_application_browser_integration_test.go index 4de544729..b2009f894 100644 --- a/newrelic/resource_newrelic_agent_application_browser_integration_test.go +++ b/newrelic/resource_newrelic_agent_application_browser_integration_test.go @@ -39,9 +39,6 @@ func TestAccNewRelicAgentApplicationBrowser(t *testing.T) { ), Check: resource.ComposeTestCheckFunc( testAccCheckNewRelicAgentApplicationBrowserExists(resourceName), - resource.TestCheckResourceAttrSet(resourceName, "js_config"), - resource.TestCheckResourceAttrSet(resourceName, "application_id"), - testAccCheckJsConfigNestedAttributes(resourceName, "js_config", []string{"init", "info", "loader_config"}), ), }, // Test: Import @@ -93,12 +90,13 @@ func testAccCheckNewRelicAgentApplicationBrowserExists(n string) resource.TestCh client := testAccProvider.Meta().(*ProviderConfig).NewClient // Provide a minimal delay to allow for the entity to be indexed. - time.Sleep(2 * time.Second) + time.Sleep(3 * time.Second) result, err := client.Entities.GetEntity(common.EntityGUID(rs.Primary.ID)) if err != nil { return err } - if result != nil { + + if result != nil && *result != nil { if string((*result).GetGUID()) != rs.Primary.ID { return fmt.Errorf("the browser agent application was not found %v - %v", (*result).GetGUID(), rs.Primary.ID) } From ec95f28964ef806fa3c272a96c042cdf2c17467a Mon Sep 17 00:00:00 2001 From: Sander Blue Date: Tue, 28 May 2024 15:17:16 -0500 Subject: [PATCH 08/12] chore: detecting drift via multiple error substrings --- .github/workflows/test_integration_cron.yml | 71 ++++++++++++------- ...rce_newrelic_notifications_channel_test.go | 5 +- scripts/detect-state-drift.js | 24 +++++-- 3 files changed, 65 insertions(+), 35 deletions(-) diff --git a/.github/workflows/test_integration_cron.yml b/.github/workflows/test_integration_cron.yml index 95b979610..cb68e5e08 100644 --- a/.github/workflows/test_integration_cron.yml +++ b/.github/workflows/test_integration_cron.yml @@ -1,38 +1,57 @@ name: Integration Tests Cron Job on: - workflow_dispatch: # this is just to test the functionality, remove before merging!!!!!!!!!!!! + workflow_dispatch: + push: + branches: [feat/integration-test-cron-job] # schedule: # # Cron executes at 0800, 1200 1600 (8am, 12pm, 4pm) # - cron: "0 8,12,16 * * 1-5" jobs: test-integration: - # if: github.ref == 'refs/heads/main' - uses: newrelic/terraform-provider-newrelic/.github/workflows/test_integration.yml@feat/integration-test-cron-job - secrets: - NEW_RELIC_ACCOUNT_ID: ${{ secrets.NEW_RELIC_ACCOUNT_ID }} - NEW_RELIC_SUBACCOUNT_ID: ${{ secrets.NEW_RELIC_SUBACCOUNT_ID }} - NEW_RELIC_ADMIN_API_KEY: ${{ secrets.NEW_RELIC_ADMIN_API_KEY }} - NEW_RELIC_API_KEY: ${{ secrets.NEW_RELIC_API_KEY }} - NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} - NEW_RELIC_REGION: ${{ secrets.NEW_RELIC_REGION }} - NEW_RELIC_INSIGHTS_INSERT_KEY: ${{ secrets.NEW_RELIC_INSIGHTS_INSERT_KEY }} - NR_ACC_TESTING: ${{ secrets.NR_ACC_TESTING }} - INTEGRATION_TESTING_AWS_ARN: ${{ secrets.INTEGRATION_TESTING_AWS_ARN }} - INTEGRATION_TESTING_GCP_ACCOUNT_NAME: ${{ secrets.INTEGRATION_TESTING_GCP_ACCOUNT_NAME }} - INTEGRATION_TESTING_GCP_PROJECT_ID: ${{ secrets.INTEGRATION_TESTING_GCP_PROJECT_ID }} - INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID: ${{secrets.INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID}} - INTEGRATION_TESTING_AZURE_APPLICATION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_APPLICATION_ID }} - INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID }} - INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID }} - INTEGRATION_TESTING_AZURE_TENANT_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_TENANT_ID }} - - detect-drift: runs-on: ubuntu-latest steps: - - name: Check check - run: ls + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.21.x + + - name: Add GOBIN to PATH + run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + shell: bash + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Cache deps + uses: actions/cache@v4 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Integration Tests + run: make test-integration cover-report + continue-on-error: true + env: + NEW_RELIC_ACCOUNT_ID: ${{ secrets.NEW_RELIC_ACCOUNT_ID }} + NEW_RELIC_SUBACCOUNT_ID: ${{ secrets.NEW_RELIC_SUBACCOUNT_ID }} + NEW_RELIC_ADMIN_API_KEY: ${{ secrets.NEW_RELIC_ADMIN_API_KEY }} + NEW_RELIC_API_KEY: ${{ secrets.NEW_RELIC_API_KEY }} + NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} + NEW_RELIC_REGION: ${{ secrets.NEW_RELIC_REGION }} + NEW_RELIC_INSIGHTS_INSERT_KEY: ${{ secrets.NEW_RELIC_INSIGHTS_INSERT_KEY }} + NR_ACC_TESTING: ${{ secrets.NR_ACC_TESTING }} + INTEGRATION_TESTING_AWS_ARN: ${{ secrets.INTEGRATION_TESTING_AWS_ARN }} + INTEGRATION_TESTING_GCP_ACCOUNT_NAME: ${{ secrets.INTEGRATION_TESTING_GCP_ACCOUNT_NAME }} + INTEGRATION_TESTING_GCP_PROJECT_ID: ${{ secrets.INTEGRATION_TESTING_GCP_PROJECT_ID }} + INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID: ${{secrets.INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID}} + INTEGRATION_TESTING_AZURE_APPLICATION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_APPLICATION_ID }} + INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID }} + INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID }} + INTEGRATION_TESTING_AZURE_TENANT_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_TENANT_ID }} - name: Detect drift if: ${{ always() }} @@ -45,7 +64,6 @@ jobs: - name: Send report to Slack id: slack - if: ${{ steps.detect-state-drift.outputs.drift_report != '' }} uses: slackapi/slack-github-action@v1.26 with: # Uses Slack's Block Kit to build the message @@ -68,7 +86,7 @@ jobs: "elements": [ { "type": "plain_text", - "text": ${{ toJSON(steps.detect-state-drift.outputs.failed_tests_with_drift) }} + "text": "${{ toJSON(steps.detect-state-drift.outputs.failed_tests_with_drift) }}" } ] }, @@ -98,4 +116,3 @@ jobs: env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - diff --git a/newrelic/resource_newrelic_notifications_channel_test.go b/newrelic/resource_newrelic_notifications_channel_test.go index 2b9d910c2..7967659c6 100644 --- a/newrelic/resource_newrelic_notifications_channel_test.go +++ b/newrelic/resource_newrelic_notifications_channel_test.go @@ -89,10 +89,7 @@ func TestNewRelicNotificationChannel_WebhookPropertyError(t *testing.T) { # Test error for missing property key = url ` - destinationPropsAttr := `property { - key = "url" - value = "https://webhook.site/" - } + destinationPropsAttr := ` ` resource.ParallelTest(t, resource.TestCase{ diff --git a/scripts/detect-state-drift.js b/scripts/detect-state-drift.js index d51ec8ee3..a098903b8 100755 --- a/scripts/detect-state-drift.js +++ b/scripts/detect-state-drift.js @@ -26,16 +26,32 @@ module.exports = async ({ // console.log(jsonData); const report = jsonData.filter(data => { - return data.Output ? data.Output.includes('error: After applying this test step, the plan was not empty') : false; - }).map(t => t.Output.trim()); + const driftDetectedSubStrings = [ + 'the plan was not empty', + 'expected an error but got none', + ]; + + if (!data.Output) { + return false; + } - console.log(report); + let driftDetected = false + for (let i = 0; i < driftDetectedSubStrings.length; i++) { + if (data.Output.includes(driftDetectedSubStrings[i])) { + return true; + } + } - let msg = 'error: After applying this test step, the plan was not empty'; + return driftDetected + }).map(t => t.Output.trim()); + + let msg = 'No drift detected.'; if (report.length > 0) { msg = `'${report.join('\n')}'`; } + console.log('drift_report:', msg); + core.setOutput('drift_report', msg); }); }; From 279536e12144f7c7388f25bf639a0cd9296972ee Mon Sep 17 00:00:00 2001 From: Sander Blue Date: Wed, 29 May 2024 09:45:01 -0500 Subject: [PATCH 09/12] chore: continue testing --- .github/workflows/test_integration_cron.yml | 4 ++-- scripts/detect-state-drift.js | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_integration_cron.yml b/.github/workflows/test_integration_cron.yml index cb68e5e08..40913e5e5 100644 --- a/.github/workflows/test_integration_cron.yml +++ b/.github/workflows/test_integration_cron.yml @@ -75,7 +75,7 @@ jobs: "type": "section", "text": { "type": "mrkdwn", - "text": ":warning: *Terraform Provider | Drift Detected*" + "text": ":warning: *Terraform Provider | Potential Drift Detected*" } }, { @@ -86,7 +86,7 @@ jobs: "elements": [ { "type": "plain_text", - "text": "${{ toJSON(steps.detect-state-drift.outputs.failed_tests_with_drift) }}" + "text": ${{ toJSON(steps.detect-state-drift.outputs.drift_report) }} } ] }, diff --git a/scripts/detect-state-drift.js b/scripts/detect-state-drift.js index a098903b8..fa93e1dbe 100755 --- a/scripts/detect-state-drift.js +++ b/scripts/detect-state-drift.js @@ -23,8 +23,6 @@ module.exports = async ({ } }); - // console.log(jsonData); - const report = jsonData.filter(data => { const driftDetectedSubStrings = [ 'the plan was not empty', @@ -47,7 +45,7 @@ module.exports = async ({ let msg = 'No drift detected.'; if (report.length > 0) { - msg = `'${report.join('\n')}'`; + msg = `'${report.join('\n\n')}'`; } console.log('drift_report:', msg); From 0dc77535cc5aefecdac575a9a2fff95037fa526f Mon Sep 17 00:00:00 2001 From: Sander Blue Date: Wed, 29 May 2024 12:41:46 -0500 Subject: [PATCH 10/12] chore: remove test trigger that was used during development --- .github/workflows/test_integration_cron.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_integration_cron.yml b/.github/workflows/test_integration_cron.yml index 40913e5e5..f006d999a 100644 --- a/.github/workflows/test_integration_cron.yml +++ b/.github/workflows/test_integration_cron.yml @@ -2,11 +2,9 @@ name: Integration Tests Cron Job on: workflow_dispatch: - push: - branches: [feat/integration-test-cron-job] - # schedule: - # # Cron executes at 0800, 1200 1600 (8am, 12pm, 4pm) - # - cron: "0 8,12,16 * * 1-5" + schedule: + # Cron executes at 0800, 1200 1600 (8am, 12pm, 4pm) + - cron: "0 8,12,16 * * 1-5" jobs: test-integration: From 7b5281bd521e27ec564ac136c6ab9ea0d7b4f20b Mon Sep 17 00:00:00 2001 From: Sander Blue Date: Wed, 29 May 2024 12:49:13 -0500 Subject: [PATCH 11/12] chore: add codecov report back to PRs --- .github/workflows/test_unit.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_unit.yml b/.github/workflows/test_unit.yml index af6f811b6..eb0d7e249 100644 --- a/.github/workflows/test_unit.yml +++ b/.github/workflows/test_unit.yml @@ -35,10 +35,10 @@ jobs: run: | make test-unit cover-report - # - name: Report unit test coverage via Codecov - # uses: codecov/codecov-action@v3 - # with: - # files: ./coverage/coverage.out + - name: Report test coverage via Codecov + uses: codecov/codecov-action@v4 + with: + files: ./coverage/coverage.out From b1a7e77269919662dcd9447263fb7e6f18224df9 Mon Sep 17 00:00:00 2001 From: Sander Blue Date: Wed, 29 May 2024 13:12:39 -0500 Subject: [PATCH 12/12] chore: remove workflow_call since we don't use it --- .github/workflows/test_integration.yml | 39 ++++---------------------- .github/workflows/test_unit.yml | 3 -- 2 files changed, 5 insertions(+), 37 deletions(-) diff --git a/.github/workflows/test_integration.yml b/.github/workflows/test_integration.yml index e817d2f1d..31070d062 100644 --- a/.github/workflows/test_integration.yml +++ b/.github/workflows/test_integration.yml @@ -5,40 +5,6 @@ on: push: branches: [main] pull_request: - workflow_call: - secrets: - NEW_RELIC_ACCOUNT_ID: - required: true - NEW_RELIC_SUBACCOUNT_ID: - required: true - NEW_RELIC_ADMIN_API_KEY: - required: true - NEW_RELIC_API_KEY: - required: true - NEW_RELIC_LICENSE_KEY: - required: true - NEW_RELIC_REGION: - required: true - NEW_RELIC_INSIGHTS_INSERT_KEY: - required: true - NR_ACC_TESTING: - required: true - INTEGRATION_TESTING_AWS_ARN: - required: true - INTEGRATION_TESTING_GCP_ACCOUNT_NAME: - required: true - INTEGRATION_TESTING_GCP_PROJECT_ID: - required: true - INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID: - required: true - INTEGRATION_TESTING_AZURE_APPLICATION_ID: - required: true - INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID: - required: true - INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID: - required: true - INTEGRATION_TESTING_AZURE_TENANT_ID: - required: true jobs: test-integration: @@ -84,3 +50,8 @@ jobs: INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID }} INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID }} INTEGRATION_TESTING_AZURE_TENANT_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_TENANT_ID }} + + - name: Report test coverage via Codecov + uses: codecov/codecov-action@v4 + with: + files: ./coverage/coverage.out diff --git a/.github/workflows/test_unit.yml b/.github/workflows/test_unit.yml index eb0d7e249..f0cbf6dd5 100644 --- a/.github/workflows/test_unit.yml +++ b/.github/workflows/test_unit.yml @@ -39,6 +39,3 @@ jobs: uses: codecov/codecov-action@v4 with: files: ./coverage/coverage.out - - -