Add test_examples CI job to test against p/examples test suite (#3065)

Increase test coverage for changes by extending testing to the existing test suite in pulumi/examples. The approach is basically to - check out p/examples in addition to p/pulumi-azure-native - import the test suite from there, via a Go `replace` directive - run as regular ProgramTests with the same config than the repo's own tests, to inject the locally built provider and SDKs.
pulumi · Feb 26, 2024 · b7e9ef9 · b7e9ef9
1 parent ab7eaf1
commit b7e9ef9
Show file tree

Hide file tree

Showing 11 changed files with 471 additions and 2,339 deletions.
diff --git a/.github/actions/schema-check/action.yml b/.github/actions/schema-check/action.yml
@@ -16,7 +16,7 @@ runs:
   using: "composite"
   steps:
       - name: Install Schema Tools
-        uses: jaxxstorm/action-install-gh-release@v1.9.0
+        uses: jaxxstorm/action-install-gh-release@v1.11.0
         with:
           repo: pulumi/schema-tools
           tag: v0.6.0

diff --git a/.github/actions/test-setup/action.yml b/.github/actions/test-setup/action.yml
@@ -0,0 +1,74 @@
+name: Test setup
+description: Installs dependencies and downloads previously built provider and SDKs to get everything ready for running e2e tests
+
+inputs:
+  language:
+    description: The programming language for which to set up
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Languages & Frameworks
+      uses: ./.github/actions/install
+
+    - run: make ensure
+      shell: bash
+
+    - name: Prerequisites artifact restore
+      uses: ./.github/actions/prerequisites-artifact-restore
+
+    - name: Mark prerequisites as up-to-date
+      run: |
+        make prebuild
+        make --touch codegen schema provider
+      shell: bash
+
+    - name: Download Go SDK artifact
+      if: ${{ inputs.language == 'go' }}
+      uses: actions/download-artifact@v4
+      with:
+        name: pulumi-azure-native-sdk.tar.gz
+        path: ${{ github.workspace}}/sdk/
+
+    - name: UnTar Go SDK artifact
+      if: ${{ inputs.language == 'go' }}
+      run: |
+        mkdir -p ${{github.workspace}}/sdk/pulumi-azure-native-sdk
+        tar -zxf ${{ github.workspace}}/sdk/pulumi-azure-native-sdk.tar.gz -C ${{github.workspace}}/sdk/pulumi-azure-native-sdk
+      shell: bash
+
+    - name: Download ${{ inputs.language  }} SDK
+      if: ${{ inputs.language != 'go' }}
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ inputs.language  }}-sdk.tar.gz
+        path: ${{ github.workspace}}/sdk/
+
+    - name: Uncompress ${{ inputs.language  }} SDK
+      if: ${{ inputs.language != 'go' }}
+      run: tar -zxf ${{github.workspace}}/sdk/${{ inputs.language  }}.tar.gz -C ${{github.workspace}}/sdk/${{ inputs.language  }}
+      shell: bash
+
+    - name: Update path
+      run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH"
+      shell: bash
+
+    - name: Install Node dependencies
+      run: yarn global add typescript
+      shell: bash
+
+    - name: Install Python deps
+      run: |-
+        pip3 install virtualenv==20.0.23
+        pip3 install pipenv
+      shell: bash
+
+    - name: Install dependencies
+      run: make install_${{ inputs.language}}_sdk
+      shell: bash
+
+    - name: Install gotestfmt
+      uses: jaxxstorm/[email protected]
+      with:
+        tag: v2.5.0
+        repo: GoTestTools/gotestfmt
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -47,7 +47,7 @@ jobs:
     name: Build binaries and schema
     steps:
       - name: Checkout Repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ inputs.ref }}
           submodules: true
@@ -94,12 +94,10 @@ jobs:
           - dotnet
           - go
           - java
-    name: Build & Test SDKs
-    permissions:
-      id-token: write # required for OIDC auth
+    name: Build SDKs
     steps:
       - name: Checkout Repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ inputs.ref }}
           submodules: true
@@ -126,48 +124,13 @@ jobs:
       - name: Check worktree clean
         uses: ./.github/actions/check-worktree-clean
 
-      - name: Update path
-        run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH"
-
-      - name: Install Node dependencies
-        run: yarn global add typescript
-
-      - name: Install Python deps
-        run: |-
-          pip3 install virtualenv==20.0.23
-          pip3 install pipenv
-
-      - name: Install dependencies
-        run: make install_${{ matrix.language}}_sdk
-
-      - name: Install gotestfmt
-        uses: jaxxstorm/[email protected]
-        with:
-          tag: v2.4.0
-          repo: GoTestTools/gotestfmt
-
-      - name: Run tests
-        if: ${{ ! inputs.short_test }}
-        env:
-          # specifying this id will cause the OIDC test(s) to run against this AD application
-          OIDC_ARM_CLIENT_ID: ${{ inputs.oidc_arm_client_id }}
-        run: |
-          set -euo pipefail
-          cd examples && go test -v -json -cover -timeout 2h -tags=${{ matrix.language }} -parallel 16 . 2>&1 | tee /tmp/gotest.log | gotestfmt
-
-      - name: Run short tests
-        if: inputs.short_test
-        run: |
-          set -euo pipefail
-          cd examples && go test -v -json -cover -timeout 15m -short -tags=${{ matrix.language }} -parallel 16 . 2>&1 | tee /tmp/gotest.log | gotestfmt
-
       - name: Tar SDK folder
         if: ${{ matrix.language != 'go' }}
         run: tar -zcf sdk/${{ matrix.language }}.tar.gz -C sdk/${{ matrix.language }} .
 
       - name: Upload artifacts
         if: ${{ matrix.language != 'go' }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.language  }}-sdk.tar.gz
           path: ${{ github.workspace}}/sdk/${{ matrix.language }}.tar.gz
@@ -178,7 +141,7 @@ jobs:
         run: tar -zcf sdk/pulumi-azure-native-sdk.tar.gz -C sdk/pulumi-azure-native-sdk .
 
       - name: Upload split Go artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ matrix.language == 'go' }}
         with:
           name: pulumi-azure-native-sdk.tar.gz
@@ -193,13 +156,107 @@ jobs:
           fields: repo,commit,author,action
           status: ${{ job.status }}
 
+  test_sdks:
+    needs: build_sdks
+    # Use big runner for dotnet and nodejs because we need more memory and more compute, respectively
+    runs-on: ${{ (matrix.language == 'dotnet' || matrix.language == 'nodejs' || matrix.language == 'go') && 'pulumi-ubuntu-8core' || 'ubuntu-latest' }}
+    strategy:
+      fail-fast: true
+      matrix:
+        language:
+          - nodejs
+          - python
+          - dotnet
+          - go
+          - java
+    name: Test SDKs
+    permissions:
+      id-token: write # required for OIDC auth
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          submodules: true
+
+      - name: Setup
+        uses: ./.github/actions/test-setup
+        with:
+          language: ${{ matrix.language }}
+
+      - name: Run tests
+        if: ${{ ! inputs.short_test }}
+        env:
+          # specifying this id will cause the OIDC test(s) to run against this AD application
+          OIDC_ARM_CLIENT_ID: ${{ inputs.oidc_arm_client_id }}
+        run: |
+          set -euo pipefail
+          cd examples && go test -v -json -cover -timeout 2h -tags=${{ matrix.language }} -skip TestPulumiExamples -parallel 16 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+
+      - name: Run short tests
+        if: inputs.short_test
+        run: |
+          set -euo pipefail
+          cd examples && go test -v -json -cover -timeout 15m -short -tags=${{ matrix.language }} -skip TestPulumiExamples -parallel 16 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+
+  test_examples:
+    needs: build_sdks
+    runs-on: ubuntu-latest
+    name: Test pulumi/examples
+    strategy:
+      fail-fast: true
+      matrix:
+        language:
+          - nodejs
+          - python
+          - dotnet
+          - go
+          - java
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          submodules: true
+      - name: Checkout p/examples
+        uses: actions/checkout@v4
+        with:
+          repository: pulumi/examples
+          ref: master
+          path: p-examples
+
+      - name: Setup
+        uses: ./.github/actions/test-setup
+        with:
+          language: ${{ matrix.language }}
+
+      - name: Free up disk space
+        run: |-
+          df -h
+          rm -rf /opt/hostedtoolcache/CodeQL
+          rm -rf /opt/hostedtoolcache/go/1.19.*
+          rm -rf /opt/hostedtoolcache/go/1.20.*
+          sudo apt clean
+          df -h
+
+      - name: Run pulumi/examples tests
+        if: ${{ !inputs.short_test }}
+        env:
+          # specifying this id will cause the OIDC test(s) to run against this AD application
+          OIDC_ARM_CLIENT_ID: ${{ inputs.oidc_arm_client_id }}
+        run: |
+          set -euo pipefail
+          cd examples && \
+            go mod edit -replace github.com/pulumi/examples/misc/test=../p-examples/misc/test/ && \
+            go test -v -json -cover -timeout 2h -tags=${{ matrix.language }} -run TestPulumiExamples -parallel 16 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+          
   test_provider:
     runs-on: ubuntu-latest
     name: Test Provider
     needs: prerequisites
     steps:
       - name: Checkout Repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ inputs.ref }}
           submodules: true
@@ -210,9 +267,9 @@ jobs:
           skip_dotnet_and_java: "true"
 
       - name: Install gotestfmt
-        uses: jaxxstorm/action-install-gh-release@v1.9.0
+        uses: jaxxstorm/action-install-gh-release@v1.11.0
         with:
-          tag: v2.4.0
+          tag: v2.5.0
           repo: GoTestTools/gotestfmt
 
       - run: make ensure
@@ -243,7 +300,7 @@ jobs:
     needs: prerequisites
     steps:
       - name: Checkout Repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
           ref: ${{ inputs.ref }}
@@ -269,7 +326,7 @@ jobs:
         run: make dist --jobs=2
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: dist
           path: dist

diff --git a/.github/workflows/nightly-sdk-generation.yml b/.github/workflows/nightly-sdk-generation.yml
@@ -98,7 +98,7 @@ jobs:
           source_branch: generate-sdk/${{ github.run_id }}-${{ github.run_number }}
 
       - name: Install Schema Tools
-        uses: jaxxstorm/action-install-gh-release@v1.9.0
+        uses: jaxxstorm/action-install-gh-release@v1.11.0
         with:
           repo: pulumi/schema-tools
 

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -78,7 +78,7 @@ jobs:
         run: make dist --jobs=2
 
       - name: Install Schema Tools
-        uses: jaxxstorm/action-install-gh-release@v1.9.0
+        uses: jaxxstorm/action-install-gh-release@v1.11.0
         with:
           repo: pulumi/schema-tools
 

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -47,7 +47,7 @@ jobs:
     if: ${{ !contains(github.ref_name,'-') }}
     steps:
       - name: Install pulumictl
-        uses: jaxxstorm/action-install-gh-release@v1.9.0
+        uses: jaxxstorm/action-install-gh-release@v1.11.0
         with:
           repo: pulumi/pulumictl
       - name: Dispatch Event

diff --git a/.gitignore b/.gitignore
@@ -42,3 +42,5 @@ pulumi-azure-native-sdk
 sdk/python/venv
 
 DEBUG_CODEGEN_EXAMPLE_HCL
+
+p-examples/
diff --git a/examples/examples_dotnet_test.go b/examples/examples_dotnet_test.go
@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	pexamples "github.com/pulumi/examples/misc/test/definitions"
 	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
 )
 
@@ -39,6 +40,19 @@ func TestAccSql(t *testing.T) {
 	integration.ProgramTest(t, &test)
 }
 
+func TestPulumiExamples(t *testing.T) {
+	for _, example := range pexamples.GetTestsByTags(pexamples.AzureNativeProvider, pexamples.CS) {
+		t.Run(example.Dir, func(t *testing.T) {
+			test := getCsharpBaseOptions(t).
+				With(example.Options).
+				With(integration.ProgramTestOptions{
+					Dir: filepath.Join(getCwd(t), pulumiExamplesPath, example.Dir),
+				})
+			integration.ProgramTest(t, &test)
+		})
+	}
+}
+
 func getCsharpBaseOptions(t *testing.T) integration.ProgramTestOptions {
 	base := getBaseOptions(t)
 	baseCsharp := base.With(integration.ProgramTestOptions{

diff --git a/examples/examples_test.go b/examples/examples_test.go
@@ -11,6 +11,8 @@ import (
 	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
 )
 
+const pulumiExamplesPath = "../p-examples"
+
 func getLocation(t *testing.T) string {
 	azureLocation := os.Getenv("ARM_LOCATION")
 	if azureLocation == "" {
Original file line number	Diff line number	Diff line change
Expand Up		@@ -42,3 +42,5 @@ pulumi-azure-native-sdk
		sdk/python/venv

		DEBUG_CODEGEN_EXAMPLE_HCL

		p-examples/