chore: split cypress tests for different needs in different environme…

…nts (#3663)

* chore: split cypress tests for different needs in different environments

* chore: correct logic

* chore: better cypress test suite names

* chore: slight naming and remve percy variables from integration tests

* chore: comment context"

* chore: whoops

.github/workflows/on_deployment_status_cypress.yml

@@ -42,20 +42,38 @@ jobs:
         with:
           node-version: 20.5.x
 
-      - name: Percy Batch ID
-        run: echo "${{ needs.percy-batch-id.outputs.result }}"
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
 
-      - name: Print ref
-        run: echo "${{ github.ref }}"
+      # Run cypress on preview deployments as integration tests, but without VRT
+      # We only want to run VRT on demand, not on every preview deployment
+      # The deployment_status event though, has no PR related context, so we can't conditionally run VRT
+      # So we run VRT on every merge to main, and on every PR (on_pull_request_cypress) if requested via a label
+      - name: Cypress run website preview
+        uses: cypress-io/github-action@v4
+        if: github.event.production_environment == false
+        with:
+          record: true
+          parallel: true
+          group: "Paste Docsite integration tests"
+          config: baseUrl=${{ github.event.deployment_status.environment_url }}
+          # enabled percy to run cypress for VRT
+          command-prefix: "percy exec --parallel -- npx"
+          # this config excludes the VRT tests. Those are run on the pull_request event if requested
+          config-file: cypress.config.e2e.ts
+        env:
+          CYPRESS_BASE_URL: ${{ github.event.deployment_status.environment_url }}
 
-      # Run cypress on merge to main aong with VRT to create our new baelines
+      # Run cypress on merge to main aong with VRT to create our new baselines
       - name: Cypress run with site VRT on main
         uses: cypress-io/github-action@v4
-        if: contains(github.ref, 'main')
+        if: github.event.production_environment
         with:
           record: true
           parallel: true
-          group: "Paste Actions with VRT Parallel"
+          group: "Paste Docsite VRT Baseline Creation"
           config: baseUrl=${{ github.event.deployment_status.environment_url }}
           # enabled percy to run cypress for VRT
           command-prefix: "percy exec --parallel -- npx"

.github/workflows/on_merge_to_main.yml

@@ -55,6 +55,9 @@ jobs:
         run: yarn prerelease
         env:
           NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
+          # needed to generate website data files for the website linting
+          AIRTABLE_APIKEY: ${{ secrets.AIRTABLE_APIKEY }}
+          AIRTABLE_BASEID: ${{ secrets.AIRTABLE_BASEID }}
 
       - name: Setup CI Git User
         run: |
@@ -138,3 +141,16 @@ jobs:
           STORYBOOK_GITHUB_SHA: ${{ github.sha }}
           # https://github.com/storybookjs/builder-vite/issues/409#issuecomment-1199236279
           NODE_OPTIONS: --max-old-space-size=6144
+
+  # run this job first to create a single timestamped id for all percy batches on parallel machines
+  percy-batch-id:
+    runs-on: ubuntu-latest
+    # persist job results to other jobs in the workflow
+    outputs:
+      result: ${{ steps.percy-batch-id.outputs.result }}
+    steps:
+      # persist step results to other steps in the job
+      - name: Create Percy Batch ID
+        id: percy-batch-id
+        # adding a timestamp makes the batch id more unique for re-runs
+        run: echo "result=${{ github.run_id }}-$(date +%s)" >> $GITHUB_OUTPUT

.github/workflows/on_pull_request.yml

@@ -107,7 +107,7 @@ jobs:
         run: yarn build
 
       - name: Generate website data files
-        run: yarn workspace @twilio-paste/website fetch:data
+        run: yarn generate:website-data
 
       - name: Run eslint
         run: yarn lint

.github/workflows/on_pull_request_cypress.yml

@@ -19,9 +19,10 @@ jobs:
         run: echo "result=${{ github.run_id }}-$(date +%s)" >> $GITHUB_OUTPUT
 
   cypress:
-    name: Website tests
+    name: Website vrt
     runs-on: ubuntu-latest
     needs: percy-batch-id
+    if: ${{ contains(github.event.pull_request.labels.*.name, '🕵🏻‍♀️ Run website visual regression') }}
     strategy:
       fail-fast: false
       matrix:
@@ -84,29 +85,20 @@ jobs:
       - name: Percy Batch ID
         run: echo "${{ needs.percy-batch-id.outputs.result }}"
 
-      - name: Cypress run
-        uses: cypress-io/github-action@v4
-        if: ${{ !contains(github.event.pull_request.labels.*.name, '🕵🏻‍♀️ Run website visual regression') }}
-        with:
-          record: true
-          parallel: true
-          start: yarn start:website
-          wait-on: "http://localhost:3000"
-          group: "Paste Actions Parallel"
-          config: baseUrl=http://localhost:3000
-
+      # run cypress via a config that only runs the percy tests
       - name: Cypress run with site VRT
         uses: cypress-io/github-action@v4
-        if: ${{ contains(github.event.pull_request.labels.*.name, '🕵🏻‍♀️ Run website visual regression') }}
         with:
           record: true
           parallel: true
           start: yarn start:website
           wait-on: "http://localhost:3000"
-          group: "Paste Actions with VRT Parallel"
+          group: "Paste Docsite VRT"
           config: baseUrl=http://localhost:3000
           # enabled percy to run cypress for VRT
           command-prefix: "percy exec --parallel -- npx"
+          # this only includes the tests that run percy snapshots, so we can only call them when we need to
+          config-file: cypress.config.vrt.ts
         env:
           USE_CYPRESS_VRT: true
           PERCY_PARALLEL_NONCE: ${{ needs.percy-batch-id.outputs.result }}

cypress.config.e2e.ts

@@ -0,0 +1,33 @@
+import { defineConfig } from "cypress";
+
+// eslint-disable-next-line import/no-default-export
+// Cypress config to only run the site e2e tests, excluding the VRT tests.
+export default defineConfig({
+  e2e: {
+    baseUrl: "http://localhost:3000",
+    env: {
+      USE_CYPRESS_VRT: process.env.USE_CYPRESS_VRT,
+      CYPRESS_BASE_URL: process.env.CYPRESS_BASE_URL,
+    },
+    viewportWidth: 1440,
+    viewportHeight: 1440,
+    projectId: "pe4h41",
+    retries: {
+      runMode: 2,
+      openMode: 2,
+    },
+    video: false,
+    blockHosts: [
+      "*.google-analytics.com",
+      "*.codesandbox.io",
+      "codesandbox.io",
+      "*.loom.com",
+      "*.youtube.com",
+      "*.github.com",
+      "*.googletagmanager.com",
+    ],
+    specPattern: ["cypress/**/*.spec.ts"],
+    excludeSpecPattern: ["cypress/integration/sitemap-vrt/**/*"],
+    defaultCommandTimeout: 6000,
+  },
+});

cypress.config.ts

@@ -1,6 +1,7 @@
 import { defineConfig } from "cypress";
 
 // eslint-disable-next-line import/no-default-export
+// Cypress config to run all test include the site vrt tests via Percy
 export default defineConfig({
   e2e: {
     baseUrl: "http://localhost:3000",

cypress.config.vrt.ts

@@ -0,0 +1,32 @@
+import { defineConfig } from "cypress";
+
+// eslint-disable-next-line import/no-default-export
+// Cypress config to only run the site vrt tests via Percy, excluding the e2e tests.
+export default defineConfig({
+  e2e: {
+    baseUrl: "http://localhost:3000",
+    env: {
+      USE_CYPRESS_VRT: process.env.USE_CYPRESS_VRT,
+      CYPRESS_BASE_URL: process.env.CYPRESS_BASE_URL,
+    },
+    viewportWidth: 1440,
+    viewportHeight: 1440,
+    projectId: "pe4h41",
+    retries: {
+      runMode: 2,
+      openMode: 2,
+    },
+    video: false,
+    blockHosts: [
+      "*.google-analytics.com",
+      "*.codesandbox.io",
+      "codesandbox.io",
+      "*.loom.com",
+      "*.youtube.com",
+      "*.github.com",
+      "*.googletagmanager.com",
+    ],
+    specPattern: ["cypress/integration/sitemap-vrt/**/*.spec.ts"],
+    defaultCommandTimeout: 6000,
+  },
+});

package.json

@@ -49,8 +49,9 @@
     "generate:embeddings": "yarn nx run @twilio-paste/website:generate:embeddings",
     "generate:embeddings:refresh": "yarn nx run @twilio-paste/website:generate:embeddings:refresh",
     "generate:db-types": "yarn nx run @twilio-paste/backend:generate:db-types",
+    "generate:website-data": "yarn workspace @twilio-paste/website fetch:data",
     "pre-push": "concurrently \"yarn:lint\" \"yarn:test\" \"yarn:format\" \"yarn:type-check\"",
-    "prerelease": "yarn build && yarn lint && yarn test",
+    "prerelease": "yarn build && generate:website-data && yarn lint && yarn test",
     "release": "yarn changeset publish",
     "release:next": "yarn lerna publish -m 'chore(release): pre release' --conventional-commits --canary --preid beta --dist-tag next",
     "version": "yarn changeset version && yarn install --mode=update-lockfile",

tsconfig.json

@@ -36,6 +36,8 @@
     "./.jest/@types/index.d.ts",
     "tools/**/*",
     "cypress.config.ts",
+    "cypress.config.vrt.ts",
+    "cypress.config.e2e.ts",
     "vite.config.ts"
   ],
   "exclude": [