Skip to content

Commit

Permalink
feat(automation): run integration tests and report drift
Browse files Browse the repository at this point in the history
  • Loading branch information
@sanderblue
sanderblue committed May 28, 2024
1 parent 092ca5a commit da19f1f
Show file tree
Hide file tree
Showing 6 changed files with 171 additions and 150 deletions.
131 changes: 66 additions & 65 deletions .github/workflows/test_integration.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,40 @@ on:
push:
branches: [main]
pull_request:
# schedule:
# # Cron executes at 0800, 1200 1600 (8am, 12pm, 4pm)
# - cron: "0 8,12,16 * * 1-5"
workflow_call:
secrets:
NEW_RELIC_ACCOUNT_ID:
required: true
NEW_RELIC_SUBACCOUNT_ID:
required: true
NEW_RELIC_ADMIN_API_KEY:
required: true
NEW_RELIC_API_KEY:
required: true
NEW_RELIC_LICENSE_KEY:
required: true
NEW_RELIC_REGION:
required: true
NEW_RELIC_INSIGHTS_INSERT_KEY:
required: true
NR_ACC_TESTING:
required: true
INTEGRATION_TESTING_AWS_ARN:
required: true
INTEGRATION_TESTING_GCP_ACCOUNT_NAME:
required: true
INTEGRATION_TESTING_GCP_PROJECT_ID:
required: true
INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID:
required: true
INTEGRATION_TESTING_AZURE_APPLICATION_ID:
required: true
INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID:
required: true
INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID:
required: true
INTEGRATION_TESTING_AZURE_TENANT_ID:
required: true

jobs:
test-integration:
Expand All @@ -18,68 +49,38 @@ jobs:
with:
go-version: 1.21.x

# - name: Add GOBIN to PATH
# run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
# shell: bash
- name: Add GOBIN to PATH
run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
shell: bash

# - name: Checkout code
# uses: actions/checkout@v4
- name: Checkout code
uses: actions/checkout@v4

# - name: Cache deps
# uses: actions/cache@v4
# with:
# path: ~/go/pkg/mod
# key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
# restore-keys: |
# ${{ runner.os }}-go-

# - name: Integration Tests
# if: github.event.pull_request.head.repo.full_name == github.repository
# run: make test-integration cover-report
# env:
# NEW_RELIC_ACCOUNT_ID: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
# NEW_RELIC_SUBACCOUNT_ID: ${{ secrets.NEW_RELIC_SUBACCOUNT_ID }}
# NEW_RELIC_ADMIN_API_KEY: ${{ secrets.NEW_RELIC_ADMIN_API_KEY }}
# NEW_RELIC_API_KEY: ${{ secrets.NEW_RELIC_API_KEY }}
# NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
# NEW_RELIC_REGION: ${{ secrets.NEW_RELIC_REGION }}
# NEW_RELIC_INSIGHTS_INSERT_KEY: ${{ secrets.NEW_RELIC_INSIGHTS_INSERT_KEY }}
# NR_ACC_TESTING: ${{ secrets.NR_ACC_TESTING }}
# INTEGRATION_TESTING_AWS_ARN: ${{ secrets.INTEGRATION_TESTING_AWS_ARN }}
# INTEGRATION_TESTING_GCP_ACCOUNT_NAME: ${{ secrets.INTEGRATION_TESTING_GCP_ACCOUNT_NAME }}
# INTEGRATION_TESTING_GCP_PROJECT_ID: ${{ secrets.INTEGRATION_TESTING_GCP_PROJECT_ID }}
# INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID: ${{secrets.INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID}}
# INTEGRATION_TESTING_AZURE_APPLICATION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_APPLICATION_ID }}
# INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID }}
# INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID }}
# INTEGRATION_TESTING_AZURE_TENANT_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_TENANT_ID }}

# - name: Create test failure report
# id: testFailureReport
# run: |
# go install github.com/mfridman/tparse@latest
# make test-failure-report
# tparse -file=coverage/integration.failures -format=markdown >> "$GITHUB_OUTPUT"

# - name: Send test failures report to Slack
# id: slack
# uses: slackapi/[email protected]
# with:
# # Uses Slack's Block Kit to build the message
# payload: |
# {
# "text": "New Relic Terraform Provider Test Failures",
# "blocks": [
# {
# "type": "section",
# "text": {
# "type": "mrkdwn",
# "text": "${{ steps.testFailureReport.outputs }}"
# }
# }
# ]
# }
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
# SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
- name: Cache deps
uses: actions/cache@v4
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Integration Tests
if: github.event.pull_request.head.repo.full_name == github.repository
run: make test-integration cover-report
env:
NEW_RELIC_ACCOUNT_ID: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
NEW_RELIC_SUBACCOUNT_ID: ${{ secrets.NEW_RELIC_SUBACCOUNT_ID }}
NEW_RELIC_ADMIN_API_KEY: ${{ secrets.NEW_RELIC_ADMIN_API_KEY }}
NEW_RELIC_API_KEY: ${{ secrets.NEW_RELIC_API_KEY }}
NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
NEW_RELIC_REGION: ${{ secrets.NEW_RELIC_REGION }}
NEW_RELIC_INSIGHTS_INSERT_KEY: ${{ secrets.NEW_RELIC_INSIGHTS_INSERT_KEY }}
NR_ACC_TESTING: ${{ secrets.NR_ACC_TESTING }}
INTEGRATION_TESTING_AWS_ARN: ${{ secrets.INTEGRATION_TESTING_AWS_ARN }}
INTEGRATION_TESTING_GCP_ACCOUNT_NAME: ${{ secrets.INTEGRATION_TESTING_GCP_ACCOUNT_NAME }}
INTEGRATION_TESTING_GCP_PROJECT_ID: ${{ secrets.INTEGRATION_TESTING_GCP_PROJECT_ID }}
INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID: ${{secrets.INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID}}
INTEGRATION_TESTING_AZURE_APPLICATION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_APPLICATION_ID }}
INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID }}
INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID }}
INTEGRATION_TESTING_AZURE_TENANT_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_TENANT_ID }}
101 changes: 101 additions & 0 deletions .github/workflows/test_integration_cron.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
name: Integration Tests Cron Job

on:
workflow_dispatch: # this is just to test the functionality, remove before merging!!!!!!!!!!!!
# schedule:
# # Cron executes at 0800, 1200 1600 (8am, 12pm, 4pm)
# - cron: "0 8,12,16 * * 1-5"

jobs:
test-integration:
# if: github.ref == 'refs/heads/main'
uses: newrelic/terraform-provider-newrelic/.github/workflows/test_integration.yml@feat/integration-test-cron-job
secrets:
NEW_RELIC_ACCOUNT_ID: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
NEW_RELIC_SUBACCOUNT_ID: ${{ secrets.NEW_RELIC_SUBACCOUNT_ID }}
NEW_RELIC_ADMIN_API_KEY: ${{ secrets.NEW_RELIC_ADMIN_API_KEY }}
NEW_RELIC_API_KEY: ${{ secrets.NEW_RELIC_API_KEY }}
NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
NEW_RELIC_REGION: ${{ secrets.NEW_RELIC_REGION }}
NEW_RELIC_INSIGHTS_INSERT_KEY: ${{ secrets.NEW_RELIC_INSIGHTS_INSERT_KEY }}
NR_ACC_TESTING: ${{ secrets.NR_ACC_TESTING }}
INTEGRATION_TESTING_AWS_ARN: ${{ secrets.INTEGRATION_TESTING_AWS_ARN }}
INTEGRATION_TESTING_GCP_ACCOUNT_NAME: ${{ secrets.INTEGRATION_TESTING_GCP_ACCOUNT_NAME }}
INTEGRATION_TESTING_GCP_PROJECT_ID: ${{ secrets.INTEGRATION_TESTING_GCP_PROJECT_ID }}
INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID: ${{secrets.INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID}}
INTEGRATION_TESTING_AZURE_APPLICATION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_APPLICATION_ID }}
INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID }}
INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID }}
INTEGRATION_TESTING_AZURE_TENANT_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_TENANT_ID }}

detect-drift:
runs-on: ubuntu-latest
steps:
- name: Check check
run: ls

- name: Detect drift
if: ${{ always() }}
uses: actions/github-script@v7
id: detect-state-drift
with:
script: |
const script = require('./scripts/detect-state-drift.js')
await script({core})
- name: Send report to Slack
id: slack
if: ${{ steps.detect-state-drift.outputs.drift_report != '' }}
uses: slackapi/[email protected]
with:
# Uses Slack's Block Kit to build the message
# https://app.slack.com/block-kit-builder
payload: |
{
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ":warning: *Terraform Provider | Drift Detected*"
}
},
{
"type": "divider"
},
{
"type": "context",
"elements": [
{
"type": "plain_text",
"text": ${{ toJSON(steps.detect-state-drift.outputs.failed_tests_with_drift) }}
}
]
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "More information can be viewed in the job summary."
},
"accessory": {
"type": "button",
"text": {
"type": "plain_text",
"text": "Job Summary",
"emoji": true
},
"value": "View Job Summary",
"url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
"action_id": "button-action"
}
},
{
"type": "divider"
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

65 changes: 1 addition & 64 deletions .github/workflows/test_unit.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,73 +35,10 @@ jobs:
run: |
make test-unit cover-report
- name: Detect drift
if: ${{ always() }}
uses: actions/github-script@v7
id: detect-state-drift
with:
script: |
const script = require('./scripts/detect-state-drift.js')
await script({core})
# - name: Report unit test coverage via Codecov
# uses: codecov/codecov-action@v3
# with:
# files: ./coverage/coverage.out

- name: Send report to Slack
id: slack
if: ${{ always() }}
uses: slackapi/[email protected]
with:
# Uses Slack's Block Kit to build the message
# https://app.slack.com/block-kit-builder
payload: |
{
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ":warning: *Terraform Provider | Drift Detected*"
}
},
{
"type": "divider"
},
{
"type": "context",
"elements": [
{
"type": "plain_text",
"text": ${{ toJSON(steps.detect-state-drift.outputs.failed_tests_with_drift) }}
}
]
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "More information can be viewed in the job summary."
},
"accessory": {
"type": "button",
"text": {
"type": "plain_text",
"text": "Job Summary",
"emoji": true
},
"value": "View Job Summary",
"url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
"action_id": "button-action"
}
},
{
"type": "divider"
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK


5 changes: 0 additions & 5 deletions build/test.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,6 @@ test-integration: tools
-- -v -parallel 10 -tags=integration $(TEST_ARGS) -covermode=$(COVERMODE) -coverprofile $(COVERAGE_DIR)/integration.tmp \
-timeout 120m -ldflags=$(LDFLAGS_TEST)

test-failure-report: tools
@echo "=== $(PROJECT_NAME) === [ test-report-failed ]: generating report..."
@mkdir -p $(COVERAGE_DIR)
./scripts/test-failure-report.sh

#
# Coverage
#
Expand Down
2 changes: 1 addition & 1 deletion newrelic/provider_unit_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ func TestBuildUserAgentStringWithDefaultServiceName(t *testing.T) {
tfUA := "HashiCorp Terraform/1.3.5 (+https://www.terraform.io) Terraform Plugin SDK/2.10.1"

result := buildUserAgentString(tfUA, getUserAgentServiceName(), ProviderVersion)
require.Contains(t, result, " terraform-provider-newrelic/dev---fixthisbcitsatest")
require.Contains(t, result, " terraform-provider-newrelic/dev")
}

func TestBuildUserAgentStringWithCustomServiceName(t *testing.T) {
Expand Down
17 changes: 2 additions & 15 deletions scripts/detect-state-drift.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ module.exports = async ({
const fs = require('fs');

// Read the text from the file
fs.readFile('coverage/unit.report', 'utf8', (err, data) => {
fs.readFile('coverage/integration.report', 'utf8', (err, data) => {
if (err) {
console.error('Error reading file:', err);
return;
Expand All @@ -23,7 +23,6 @@ module.exports = async ({
}
});

// Print the resulting JSON array
// console.log(jsonData);

const report = jsonData.filter(data => {
Expand All @@ -37,18 +36,6 @@ module.exports = async ({
msg = `'${report.join('\n')}'`;
}

core.setOutput('failed_tests_with_drift', msg);

// // Convert JSON data to string
// const jsonString = JSON.stringify(failedTests, null, 2);

// // Write the JSON data to a file
// fs.writeFile('output.json', jsonString, 'utf8', err => {
// if (err) {
// console.error('Error writing file:', err);
// return;
// }
// console.log('JSON data has been written to output.json');
// });
core.setOutput('drift_report', msg);
});
};

0 comments on commit da19f1f

Please sign in to comment.