Add support for visual regression tests (#384)

Co-authored-by: Balázs Reé <[email protected]>
Co-authored-by: Victor Fernandez de Alba <[email protected]>

.github/workflows/acceptance.yml

@@ -6,7 +6,6 @@ env:
   CYPRESS_RETRIES: 2
 
 jobs:
-
   acceptance:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
     runs-on: ubuntu-latest
@@ -178,11 +177,11 @@ jobs:
         if: failure()
         with:
           name: cypress-screenshots-acceptance
-          path: acceptance/cypress/screenshots
+          path: cypress/screenshots
 
       # Upload Cypress videos
       - uses: actions/upload-artifact@v3
         if: failure()
         with:
           name: cypress-videos-acceptance
-          path: acceptance/cypress/videos
+          path: cypress/videos

.github/workflows/visual-acceptance.yaml

@@ -0,0 +1,112 @@
+name: Visual acceptance tests
+on: [push, pull_request]
+
+# # Unkomment if you want a scheduled run
+#
+# on:
+#   workflow_dispatch:
+#   schedule:
+#     - cron:  '0 */12 * * *'
+
+env:
+  node-version: 20.x
+
+jobs:
+  visual:
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
+    runs-on: ubuntu-latest
+    env:
+      CYPRESS_A11Y: true
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Use Node.js ${{ env.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.node-version }}
+
+      - name: Enable corepack
+        run: corepack enable
+
+      - name: Get pnpm store directory
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
+
+      - uses: actions/cache@v4
+        name: Setup pnpm cache
+        with:
+          path: ${{ env.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Cache Cypress Binary
+        id: cache-cypress-binary
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/Cypress
+          key: binary-${{ env.node-version }}-${{ hashFiles('pnpm-lock.yaml') }}
+
+      - name: Install dependencies
+        run: make install
+
+      - name: Install Cypress if not in cache
+        if: steps.cache-cypress-binary.outputs.cache-hit != 'true'
+        working-directory: core/packages/volto
+        run: make cypress-install
+
+      - uses: JarvusInnovations/background-action@v1
+        name: Start Servers
+        with:
+          run: |
+            make ci-acceptance-server-visual-start &
+            make ci-acceptance-frontend-visual-start &
+          # your step-level and job-level environment variables are available to your commands as-is
+          # npm install will count towards the wait-for timeout
+          # whenever possible, move unrelated scripts to a different step
+          # to background multiple processes: add & to the end of the command
+
+          wait-on: |
+            http-get://localhost:8080/Plone
+            http://localhost:3000
+          # IMPORTANT: to use environment variables in wait-on, you must use this form: ${{ env.VAR }}
+          # See wait-on section below for all resource types and prefixes
+
+          tail: true # true = stderr,stdout
+          # This will allow you to monitor the progress live
+
+          log-output-resume: stderr
+          # Eliminates previosuly output stderr log entries from post-run output
+
+          wait-for: 10m
+
+          log-output: stderr,stdout # same as true
+
+          log-output-if: failure
+          # failure = exit-early or timeout
+
+          # working-directory: backend
+
+      - run: make ci-acceptance-test-visual
+
+      # Upload Cypress screenshots
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: cypress-screenshots-visual-acceptance
+          path: cypress/screenshots
+
+      # Upload Cypress videos
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: cypress-videos-visual-acceptance
+          path: cypress/videos
+
+      # Upload Cypress visual testing screenshots
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: cypress-image-snapshots-visual-acceptance
+          path: cypress/__image_snapshots__

Makefile

@@ -19,6 +19,8 @@ GREEN=`tput setaf 2`
 RESET=`tput sgr0`
 YELLOW=`tput setaf 3`
 
+NODEBIN = ./node_modules/.bin
+
 PLONE_VERSION=6
 DOCKER_IMAGE=ghcr.io/kitconcept/voltolighttheme:latest
 DOCKER_IMAGE_ACCEPTANCE=plone/server-acceptance:${PLONE_VERSION}
@@ -149,3 +151,67 @@ acceptance-a11y-test: ## Start a11y Cypress in interactive mode
 .PHONY: ci-acceptance-a11y-test
 ci-acceptance-a11y-test: ## Run a11y cypress tests in headless mode for CI
 	CYPRESS_a11y=1 CYPRESS_API_PATH=http://localhost:8080/Plone pnpm exec cypress run --config specPattern=$(CURRENT_DIR)'/cypress/tests/a11y/**/*.{js,jsx,ts,tsx}'
+
+### Visual acceptance tests ###
+#
+# Useful env vars:
+#
+# cypress_cumulativeReport: cumulative report file, by default `cumulative.report`
+# cypress_cumulativeSummaryReport: cumulative summary report file, by default adding `-summary.report`
+#                                  to the end of the cumulative file
+#                                  They are placed into `frontend/cypress/config` and
+#                                  cannot be outside
+#                                  of Cypress due to its virtual filesystem.
+#
+# Env vars in support of CI runs:
+#
+# cypress_baseUrl: url of server to run the test against
+#                  e.g. cypress_baseUrl=https://plone-redaktion.de
+#
+#
+
+.PHONY: acceptance-test-visual
+acceptance-test-visual: ## Start visual Cypress Acceptance Tests
+	NODE_ENV=production CYPRESS_a11y=1 $(NODEBIN)/cypress open --config-file cypress/config/cypress.visual.config.js
+
+# Running the visual tests in headless, cumulative mode will SKIP tests that have run previously, and
+# only run the still failing tests HOWEVER the results are not collected in interactive mode due to
+# inherent limitations of Cypress. But if you run the tests in cumulative headless mode first,
+# then this command can be used to run ONLY the failing tests in interactive mode.
+.PHONY: acceptance-test-visual-cumulative
+acceptance-test-visual-cumulative: ## Start visual Cypress Acceptance Tests with cumulative filtering
+	@echo WARNING: cumulative results are NOT collected in interactive mode, use headless mode for collection
+	NODE_ENV=production CYPRESS_a11y=1 cypress_enableCumulative=true $(NODEBIN)/cypress open --config-file cypress/config/cypress.visual.config.js
+
+.PHONY: ci-acceptance-test-visual
+ci-acceptance-test-visual: ## Start visual Cypress Acceptance Tests in headless mode
+	NODE_ENV=production CYPRESS_a11y=1 $(NODEBIN)/cypress run --browser firefox --config-file cypress/config/cypress.visual.config.js
+
+# Running the visual tests in headless, cumulative mode will SKIP tests that have run previously, and
+# only run the still failing tests.
+.PHONY: ci-acceptance-test-visual-cumulative
+ci-acceptance-test-visual-cumulative: ## Start visual Cypress Acceptance Tests in headless mode
+	NODE_ENV=production CYPRESS_a11y=1 cypress_enableCumulative=true $(NODEBIN)/cypress run --browser firefox --config-file cypress/config/cypress.visual.config.js
+
+# Automatically update all changed visual snapshots
+.PHONY: ci-acceptance-test-visual-update
+ci-acceptance-test-visual-update: ## Start visual Cypress Acceptance Tests in headless mode, always pass and update images
+	NODE_ENV=production CYPRESS_a11y=1 cypress_pluginVisualRegressionUpdateImages=true $(NODEBIN)/cypress run --browser firefox --config-file cypress/config/cypress.visual.config.js
+
+# Automatically update all changed visual snapshots, and
+# running the visual tests in headless, cumulative mode will SKIP tests that have run previously, and
+# only run the still failing tests
+.PHONY: ci-acceptance-test-visual-update-cumulative
+ci-acceptance-test-visual-update-cumulative: ## Start visual Cypress Acceptance Tests in headless mode, always pass and update images, with cumulative results
+	NODE_ENV=production CYPRESS_a11y=1 cypress_enableCumulative=true cypress_pluginVisualRegressionUpdateImages=true $(NODEBIN)/cypress run --browser firefox --config-file cypress/config/cypress.visual.config.js
+
+.PHONY: summarize-cumulative-state
+summarize-cumulative-state: ## Summarize cumulative state into ...-summary.report
+	pnpm summarize-cumulative-state
+
+
+.PHONY: ci-acceptance-server-visual-start
+ci-acceptance-server-visual-start: ci-acceptance-a11y-backend-start
+
+.PHONY: ci-acceptance-frontend-visual-start
+ci-acceptance-frontend-visual-start: acceptance-a11y-frontend-prod-start

cypress.config.js

@@ -1,4 +1,7 @@
 const { defineConfig } = require('cypress');
+const {
+  initPlugin: initVisualRegressionPlugin,
+} = require('@frsource/cypress-plugin-visual-regression-diff/dist/plugins');
 
 module.exports = defineConfig({
   viewportWidth: 1280,
@@ -9,6 +12,9 @@ module.exports = defineConfig({
       runMode: 2,
     },
     setupNodeEvents(on, config) {
+      // visual regression testing must be initialized
+      // when the plugin is installed
+      initVisualRegressionPlugin(on, config);
       on('task', {
         table(message) {
           console.table(message);

cypress/.gitignore

@@ -0,0 +1,11 @@
+# Only store the #0.png images, ignore #1.png, #2.png etc...
+# Also ignore *.actual.png, *.diff.png
+!/__*_snapshots__/**
+/__*_snapshots__/*.png
+!/__*_snapshots__/*#0.png
+
+# Cumulative report outputs must be in the cypress
+# filesystem, but we want to avoid checking them in
+/config/cumulative*.report
+
+/screenshots