Reorganize examples integration tests into long and short based on ru…

…n times (#3095)

This PR was motivated by #3078 because this new test has a run time of
~24m, too long for PRs. That motivated me to collect data on the run
times of all our e2e tests. Until now, about half of the tests were
classified as "long", but with no clear pattern.

The data shows that two thirds of the tests take 7 minutes or less, so
we picked that as a cut-off. Those tests are now classified as "short".
PRs will now only run the short tests, leading to faster builds with
still two thirds of the previous test coverage.

.github/workflows/acceptance-test.yml

@@ -52,7 +52,7 @@ jobs:
     with:
       ref: ${{ github.ref }}
       version: ${{ inputs.version || needs.version.outputs.version }}
-      short_test: ${{ inputs.short_test || false }}
+      short_test: ${{ (github.event_name == 'pull_request' || inputs.short_test) || false }}
       retention_days: 30
     secrets: inherit
 

.github/workflows/build-test.yml

@@ -145,7 +145,7 @@ jobs:
           repo: GoTestTools/gotestfmt
 
       - name: Run tests
-        if: "!inputs.short_test"
+        if: ${{ ! inputs.short_test }}
         env:
           # specifying this id will cause the OIDC test(s) to run against this AD application
           OIDC_ARM_CLIENT_ID: ${{ inputs.oidc_arm_client_id }}

examples/examples_dotnet_test.go

@@ -31,7 +31,6 @@ func TestAccSimpleDotnet(t *testing.T) {
 }
 
 func TestAccSql(t *testing.T) {
-	skipIfShort(t)
 	test := getCsharpBaseOptions(t).
 		With(integration.ProgramTestOptions{
 			Dir: filepath.Join(getCwd(t), "cs-sql"),

examples/examples_nodejs_keyvault_test.go

@@ -12,7 +12,6 @@ import (
 )
 
 func TestAccKeyVaultTs(t *testing.T) {
-	skipIfShort(t)
 	test := getJSBaseOptions(t).
 		With(integration.ProgramTestOptions{
 			Dir: filepath.Join(getCwd(t), "keyvault"),
@@ -22,8 +21,6 @@ func TestAccKeyVaultTs(t *testing.T) {
 }
 
 func TestAccKeyVaultTs_OICD(t *testing.T) {
-	skipIfShort(t)
-
 	oidcClientId := os.Getenv("OIDC_ARM_CLIENT_ID")
 	if oidcClientId == "" {
 		t.Skip("Skipping OIDC test without OIDC_ARM_CLIENT_ID")

examples/examples_nodejs_test.go

@@ -45,7 +45,7 @@ func TestAccSimpleTs(t *testing.T) {
 	integration.ProgramTest(t, &test)
 }
 
-func TestDestroyTs(t *testing.T) {
+func TestStaticWebsiteDestroyTs(t *testing.T) {
 	// Tests eventually-consistent deletion handling.
 	skipIfShort(t)
 	test := getJSBaseOptions(t).
@@ -57,7 +57,6 @@ func TestDestroyTs(t *testing.T) {
 }
 
 func TestImportTs(t *testing.T) {
-	skipIfShort(t)
 	test := getJSBaseOptions(t).
 		With(integration.ProgramTestOptions{
 			Dir: filepath.Join(getCwd(t), "import"),
@@ -120,6 +119,7 @@ func TestSecretsTs(t *testing.T) {
 }
 
 func TestTimeSeriesTs(t *testing.T) {
+	skipIfShort(t)
 	test := getJSBaseOptions(t).
 		With(integration.ProgramTestOptions{
 			Dir: filepath.Join(getCwd(t), "timeseries"),
@@ -129,6 +129,7 @@ func TestTimeSeriesTs(t *testing.T) {
 }
 
 func TestPublicIpUpdateTs(t *testing.T) {
+	skipIfShort(t)
 	test := getJSBaseOptions(t).
 		With(integration.ProgramTestOptions{
 			Dir:         filepath.Join(getCwd(t), "public-ip-update"),
@@ -198,7 +199,6 @@ func TestStorageAccountNetworkRule(t *testing.T) {
 }
 
 func TestAccKeyVaultAccessPoliciesTs(t *testing.T) {
-	skipIfShort(t)
 	test := getJSBaseOptions(t).
 		With(integration.ProgramTestOptions{
 			ExpectRefreshChanges: false,