Accessibility tests (#12900)

* WIP: Accessibility e2e test suite and support

* Add e2e job for accessibility tests

* Fix a11y tests

* Remove test

* Fix lint issues

* Add report upload

* Upload a11y screenshots

* Remove failure gate on a11y screenshots

* Test debugging

* Fix lint

* Fix setup issue

* Tweak tests

* Create a11y report

* Fix screenshot folder

* Build out more tests to cover a11y

.github/workflows/test.yaml

@@ -142,7 +142,65 @@ jobs:
           name: ${{github.run_number}}-${{github.run_attempt}}-screenshots-${{ matrix.role.tag }}+${{ matrix.features[0] }}
           path: cypress/screenshots
 
+  a11y-test:
+    if: "!contains( github.event.pull_request.labels.*.name, 'ci/skip-e2e')"
+    needs: e2e-ui-build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: Setup env
+        uses: ./.github/actions/setup
 
+      # Installing fixed version of Chrome since latest version does not work (128 didn't work)
+      # Leaving this here again in case we need to pin to a specific Chrome version in the future
+      - name: Install Chrome 127
+        run: |
+          sudo apt-get install -y wget libu2f-udev
+          cd /tmp
+          wget -q http://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_127.0.6533.72-1_amd64.deb
+          sudo dpkg -i google-chrome-stable_127.0.6533.72-1_amd64.deb
+          sudo apt-get install -y --allow-downgrades ./google-chrome-stable_127.0.6533.72-1_amd64.deb
+          google-chrome --version
+
+      - name: Download e2e build
+        uses:  actions/download-artifact@v4
+        with:
+          name: ${{ env.E2E_BUILD_DIST_NAME }}
+          path: ${{ env.E2E_BUILD_DIST_DIR }}
+      - name: Download e2e build ember
+        uses:  actions/download-artifact@v4
+        with:
+          name: ${{ env.E2E_BUILD_DIST_EMBER_NAME }}
+          path: ${{ env.E2E_BUILD_DIST_EMBER_DIR }}
+
+      - name: Run Rancher
+        run: yarn e2e:docker
+
+      - name: Setup Rancher and user
+        run: |
+          yarn e2e:prod
+        env: 
+          GREP_TAGS: "@adminUserSetup+@accessibility --@jenkins"
+          TEST_USERNAME: admin
+          TEST_ONLY: setup
+      - name: Run user tests
+        run: |
+          yarn e2e:prod
+          [ "$BUILD_DASHBOARD" != "false" ] || exit 0
+        env: 
+          TEST_SKIP: setup
+          GREP_TAGS: "@adminUser+@accessibility --@jenkins"
+          TEST_USERNAME: admin
+          TEST_A11Y: true
+
+      - name: Upload report
+        uses: actions/upload-artifact@v4
+        with:
+          name: accessibility-report
+          path: cypress/accessibility
+
   unit-test:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -90,6 +90,8 @@ sw.*
 # Cypress e2e testing
 cypress/videos
 cypress/screenshots
+cypress/accessibility
+setupTestEnv.sh
 
 # Storybook
 storybook-static/

cypress.config.ts

@@ -3,6 +3,7 @@ import { defineConfig } from 'cypress';
 import { removeDirectory } from 'cypress-delete-downloads-folder';
 import { getSpecPattern } from '@/scripts/cypress';
 import websocketTasks from './cypress/support/utils/webSocket-utils';
+import path from 'path';
 
 // Required for env vars to be available in cypress
 require('dotenv').config();
@@ -11,13 +12,17 @@ require('dotenv').config();
  * VARIABLES
  */
 const hasCoverage = (process.env.TEST_INSTRUMENT === 'true') || false; // Add coverage if instrumented
-const testDirs = ['priority', 'components', 'setup', 'pages', 'navigation', 'global-ui', 'features', 'extensions'];
+let testDirs = ['priority', 'components', 'setup', 'pages', 'navigation', 'global-ui', 'features', 'extensions'];
 const skipSetup = process.env.TEST_SKIP?.includes('setup');
 const baseUrl = (process.env.TEST_BASE_URL || 'https://localhost:8005').replace(/\/$/, '');
 const DEFAULT_USERNAME = 'admin';
 const username = process.env.TEST_USERNAME || DEFAULT_USERNAME;
 const apiUrl = process.env.API || (baseUrl.endsWith('/dashboard') ? baseUrl.split('/').slice(0, -1).join('/') : baseUrl);
 
+if (process.env.TEST_A11Y) {
+  testDirs = ['accessibility'];
+}
+
 /**
  * LOGS:
  * Summary of the environment variables that we have detected (or are going ot use)
@@ -97,6 +102,8 @@ export default defineConfig({
     azureClientSecret:   process.env.AZURE_CLIENT_SECRET,
     customNodeIp:        process.env.CUSTOM_NODE_IP,
     customNodeKey:       process.env.CUSTOM_NODE_KEY,
+    accessibility:       !!process.env.TEST_A11Y, // Are we running accessibility tests?
+    a11yFolder:          path.join('.', 'cypress', 'accessibility'),
     gkeServiceAccount:   process.env.GKE_SERVICE_ACCOUNT,
   },
   e2e: {
@@ -106,6 +113,12 @@ export default defineConfig({
       require('@cypress/code-coverage/task')(on, config);
       require('@cypress/grep/src/plugin')(config);
       // For more info: https://www.npmjs.com/package/cypress-delete-downloads-folder
+
+      // Load Accessibility plugin if configured
+      if (process.env.TEST_A11Y) {
+        require('./cypress/support/plugins/accessibility').default(on, config);
+      }
+
       on('task', { removeDirectory });
       websocketTasks(on, config);
 

cypress/e2e/po/pages/account-api-keys.po.ts

@@ -56,6 +56,10 @@ export default class AccountPagePo extends PagePo {
     return this.applyButton().click();
   }
 
+  cancel(): Cypress.Chainable {
+    return this.self().get('button[type="reset"]').click();
+  }
+
   currentPassword(): PasswordPo {
     return new PasswordPo('[data-testid="account__current_password"]');
   }

cypress/e2e/tests/accessibility/shell.spec.ts

@@ -0,0 +1,95 @@
+import { LoginPagePo } from '@/cypress/e2e/po/pages/login-page.po';
+import HomePagePo from '@/cypress/e2e/po/pages/home.po';
+import AboutPagePo from '@/cypress/e2e/po/pages/about.po';
+import PreferencesPagePo from '@/cypress/e2e/po/pages/preferences.po';
+import UserMenuPo from '@/cypress/e2e/po/side-bars/user-menu.po';
+import AccountPagePo from '@/cypress/e2e/po/pages/account-api-keys.po';
+import CreateKeyPagePo from '@/cypress/e2e/po/pages/account-api-keys-create_key.po';
+
+describe('Shell a11y testing', { tags: ['@adminUser', '@accessibility'] }, () => {
+  it('login page', () => {
+    const loginPage = new LoginPagePo();
+
+    loginPage.goTo();
+    loginPage.waitForPage();
+    cy.injectAxe();
+    loginPage.username().set('test user');
+
+    cy.checkPageAccessibility();
+  });
+
+  it('locale selector', () => {
+    const loginPage = new LoginPagePo();
+
+    loginPage.goTo();
+    loginPage.waitForPage();
+    cy.injectAxe();
+    cy.get('[data-testid="locale-selector"]').click();
+    cy.checkPageAccessibility();
+  });
+
+  describe('Logged in', { testIsolation: 'off' }, () => {
+    const aboutPage = new AboutPagePo();
+    const prefPage = new PreferencesPagePo();
+    const userMenu = new UserMenuPo();
+
+    before(() => {
+      cy.login();
+    });
+
+    it('home page', () => {
+      HomePagePo.goToAndWaitForGet();
+      cy.injectAxe();
+
+      cy.checkPageAccessibility();
+    });
+
+    it('about page', () => {
+      AboutPagePo.navTo();
+      aboutPage.waitForPage();
+
+      cy.checkPageAccessibility();
+    });
+
+    it('preferences page', () => {
+      userMenu.clickMenuItem('Preferences');
+      userMenu.isClosed();
+      prefPage.waitForPage();
+      prefPage.checkIsCurrentPage();
+      prefPage.title();
+      cy.injectAxe();
+
+      cy.checkPageAccessibility();
+    });
+
+    describe('account', () => {
+      const accountPage = new AccountPagePo();
+      const createKeyPage = new CreateKeyPagePo();
+
+      it('account page', () => {
+        userMenu.clickMenuItem('Account & API Keys');
+        accountPage.waitForPage();
+        cy.injectAxe();
+        accountPage.checkIsCurrentPage();
+
+        cy.checkPageAccessibility();
+      });
+
+      it('change password dialog', () => {
+        accountPage.changePassword();
+
+        cy.checkElementAccessibility('.change-password-modal');
+
+        accountPage.cancel();
+      });
+
+      it('create API key', () => {
+        accountPage.create();
+        createKeyPage.waitForPage();
+        createKeyPage.isCurrentPage();
+
+        cy.checkPageAccessibility();
+      });
+    });
+  });
+});

cypress/e2e/tests/setup/rancher-setup.spec.ts

@@ -6,7 +6,7 @@ import { serverUrlLocalhostCases, urlWithTrailingForwardSlash, httpUrl, nonUrlCa
 
 // Cypress or the GrepTags avoid to run multiples times the same test for each tag used.
 // This is a temporary solution till initialization is not handled as a test
-describe('Rancher setup', { tags: ['@adminUserSetup', '@standardUserSetup', '@setup', '@components', '@navigation', '@charts', '@explorer', '@explorer2', '@extensions', '@fleet', '@generic', '@globalSettings', '@manager', '@userMenu', '@usersAndAuths', '@elemental', '@vai', '@virtualizationMgmt'] }, () => {
+describe('Rancher setup', { tags: ['@adminUserSetup', '@standardUserSetup', '@setup', '@components', '@navigation', '@charts', '@explorer', '@explorer2', '@extensions', '@fleet', '@generic', '@globalSettings', '@manager', '@userMenu', '@usersAndAuths', '@elemental', '@vai', '@virtualizationMgmt', '@accessibility'] }, () => {
   const rancherSetupLoginPage = new RancherSetupLoginPagePo();
   const rancherSetupConfigurePage = new RancherSetupConfigurePage();
   const homePage = new HomePagePo();

cypress/globals.d.ts

@@ -175,6 +175,16 @@ declare global {
        * Check if the vai FF is enabled
        */
       isVaiCacheEnabled(): Chainable<boolean>;
+
+      /**
+       * Run an accessibility check on the current page or the specified element
+       */
+      checkPageAccessibility(description?: string);
+
+      /**
+       * Run an accessibility check on the specified element
+       */
+      checkElementAccessibility(selector: any, description?: string);
     }
   }
 }