From 1e2f96aaad564650ea3e095ee9da0d2a0d28b51a Mon Sep 17 00:00:00 2001 From: Brian Muenzenmeyer Date: Sat, 28 Oct 2023 04:27:49 -0500 Subject: [PATCH] Add Lighthouse CI (#6047) * add lighthouse preview * add more than one url * remove unneeded step * use other vercel action * remove unnecessary condition * checkout forked branch again * increase vercel timeout * use correct filename * update pull-request comment * tun assertions * add more than one url a different way * format lighthouse output * fix typo * set locale in urls * remove unused config * format result * use same comment on final result * make valid cjs module * comment todo * formatting scores * revert longer timeout * formatting * increase vercel timeout afterall * add more comment * change to ESM * add /en/about page * Revert "change to ESM" This reverts commit db8b02a0b652da95cc9c686b7693ef79901e96d2. * add previous releases page * condensed output * add blog * cleanup * do not run on push * simplify comment, trigger change * troubleshoot why links output is empty * testing lighthouse * chore: simplify code, add tests * use renamed function * increase vercel preview timeout --- .github/workflows/lighthouse.yml | 117 ++++++++++++++++++++ .lighthouserc.json | 18 +++ scripts/lighthouse/__tests__/index.test.mjs | 69 ++++++++++++ scripts/lighthouse/index.mjs | 50 +++++++++ 4 files changed, 254 insertions(+) create mode 100644 .github/workflows/lighthouse.yml create mode 100644 .lighthouserc.json create mode 100644 scripts/lighthouse/__tests__/index.test.mjs create mode 100644 scripts/lighthouse/index.mjs diff --git a/.github/workflows/lighthouse.yml b/.github/workflows/lighthouse.yml new file mode 100644 index 0000000000000..dbdca14eda005 --- /dev/null +++ b/.github/workflows/lighthouse.yml @@ -0,0 +1,117 @@ +# Security Notes +# This workflow uses `pull_request_target`, so will run against all PRs automatically (without approval), be careful with allowing any user-provided code to be run here +# Only selected Actions are allowed within this repository. Please refer to (https://github.com/nodejs/nodejs.org/settings/actions) +# for the full list of available actions. If you want to add a new one, please reach out a maintainer with Admin permissions. +# REVIEWERS, please always double-check security practices before merging a PR that contains Workflow changes!! +# AUTHORS, please only use actions with explicit SHA references, and avoid using `@master` or `@main` references or `@version` tags. +# MERGE QUEUE NOTE: This Workflow does not run on `merge_group` trigger, as this Workflow is not required for Merge Queue's + +name: Lighthouse + +on: + pull_request_target: + branches: + - main + types: + - labeled + +defaults: + run: + # This ensures that the working directory is the root of the repository + working-directory: ./ + +permissions: + contents: read + actions: read + # This permission is required by `thollander/actions-comment-pull-request` + pull-requests: write + +jobs: + lighthouse-ci: + # We want to skip our lighthouse analysis on Dependabot PRs + if: startsWith(github.event.pull_request.head.ref, 'dependabot/') == false + + name: Lighthouse Report + runs-on: ubuntu-latest + + steps: + - name: Git Checkout + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + # Since we checkout the HEAD of the current Branch, if the Pull Request comes from a Fork + # we want to clone the fork's repository instead of the base repository + # this allows us to have the correct history tree of the perspective of the Pull Request's branch + # If the Workflow is running on `merge_group` or `push` events it fallsback to the base repository + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + # We checkout the branch itself instead of a specific SHA (Commit) as we want to ensure that this Workflow + # is always running with the latest `ref` (changes) of the Pull Request's branch + # If the Workflow is running on `merge_group` or `push` events it fallsback to `github.ref` which will often be `main` + # or the merge_group `ref` + ref: ${{ github.event.pull_request.head.ref || github.ref }} + + - name: Add Comment to PR + # Signal that a lighthouse run is about to start + uses: thollander/actions-comment-pull-request@d61db783da9abefc3437960d0cce08552c7c004f # v2.4.2 + with: + message: | + Running Lighthouse audit... + # Used later to edit the existing comment + comment_tag: 'lighthouse_audit' + + - name: Capture Vercel Preview + uses: patrickedqvist/wait-for-vercel-preview@dca4940010f36d2d44caa487087a09b57939b24a # v1.3.1 + id: vercel_preview_url + with: + token: ${{ secrets.GITHUB_TOKEN }} + max_timeout: 90 + + - name: Git Checkout + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + # By default Git Checkout on `pull-request-target` will checkout + # the `default` branch of the Pull Request. We want to checkout + # the actual branch of the Pull Request. + ref: ${{ github.event.pull_request.head.ref }} + + - name: Audit Preview URL with Lighthouse + # Conduct the lighthouse audit + id: lighthouse_audit + uses: treosh/lighthouse-ci-action@03becbfc543944dd6e7534f7ff768abb8a296826 # v10.1.0 + with: + # Defines the settings and assertions to audit + configPath: './.lighthouserc.json' + # These URLS capture critical pages / site functionality. + urls: | + ${{ steps.vercel_preview_url.outputs.url }}/en + ${{ steps.vercel_preview_url.outputs.url }}/en/about + ${{ steps.vercel_preview_url.outputs.url }}/en/about/previous-releases + ${{ steps.vercel_preview_url.outputs.url }}/en/download + ${{ steps.vercel_preview_url.outputs.url }}/en/blog + uploadArtifacts: true # save results as a action artifacts + temporaryPublicStorage: true # upload lighthouse report to the temporary storage + + - name: Format Lighthouse Score + # Transform the audit results into a single, friendlier output + id: format_lighthouse_score + uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1 + env: + # using env as input to our script + # see https://github.com/actions/github-script#use-env-as-input + LIGHTHOUSE_RESULT: ${{ steps.lighthouse_audit.outputs.manifest }} + LIGHTHOUSE_LINKS: ${{ steps.lighthouse_audit.outputs.links }} + VERCEL_PREVIEW_URL: ${{ steps.vercel_preview_url.outputs.url }} + with: + # Run as a separate file so we do not have to inline all of our formatting logic. + # See https://github.com/actions/github-script#run-a-separate-file for more info. + script: | + const { formatLighthouseResults } = await import('${{github.workspace}}/scripts/lighthouse/index.mjs') + await formatLighthouseResults({core}) + + - name: Add Comment to PR + # Replace the previous message with our formatted lighthouse results + uses: thollander/actions-comment-pull-request@d61db783da9abefc3437960d0cce08552c7c004f # v2.4.2 + with: + # Reference the previously created comment + comment_tag: 'lighthouse_audit' + message: | + ${{ steps.format_lighthouse_score.outputs.comment }} diff --git a/.lighthouserc.json b/.lighthouserc.json new file mode 100644 index 0000000000000..eb1c22031ae30 --- /dev/null +++ b/.lighthouserc.json @@ -0,0 +1,18 @@ +{ + "ci": { + "collect": { + "numberOfRuns": 1, + "settings": { + "preset": "desktop" + } + }, + "assert": { + "assertions": { + "categories:performance": ["warn", { "minScore": 0.9 }], + "categories:accessibility": ["warn", { "minScore": 0.9 }], + "categories:best-practices": ["warn", { "minScore": 0.9 }], + "categories:seo": ["warn", { "minScore": 0.9 }] + } + } + } +} diff --git a/scripts/lighthouse/__tests__/index.test.mjs b/scripts/lighthouse/__tests__/index.test.mjs new file mode 100644 index 0000000000000..d472be30490ba --- /dev/null +++ b/scripts/lighthouse/__tests__/index.test.mjs @@ -0,0 +1,69 @@ +import { formatLighthouseResults } from '..'; + +describe('formatLighthouseResults', () => { + const MOCK_VERCEL_PREVIEW_URL = `https://some.vercel.preview.url`; + + const MOCK_LIGHTHOUSE_RESULT = `[ + { + "url": "${MOCK_VERCEL_PREVIEW_URL}/en", + "isRepresentativeRun": true, + "summary": { "performance": 0.99, "accessibility": 0.98, "best-practices": 1, "seo": 0.96, "pwa": 0.71 } + }, + { + "url": "${MOCK_VERCEL_PREVIEW_URL}/en/download", + "isRepresentativeRun": true, + "summary": { "performance": 0.49, "accessibility": 0.75, "best-practices": 1, "seo": 0.90, "pwa": 0.71 } + } + ]`; + + const MOCK_LIGHTHOUSE_LINKS = `{ + "${MOCK_VERCEL_PREVIEW_URL}/en": "fake.url/to/result/1", + "${MOCK_VERCEL_PREVIEW_URL}/en/download" : "fake.url/to/result/2" + }`; + + let mockCore, originalEnv; + + beforeEach(() => { + mockCore = { setOutput: jest.fn() }; + originalEnv = process.env; + process.env = { + ...process.env, + LIGHTHOUSE_RESULT: MOCK_LIGHTHOUSE_RESULT, + LIGHTHOUSE_LINKS: MOCK_LIGHTHOUSE_LINKS, + VERCEL_PREVIEW_URL: MOCK_VERCEL_PREVIEW_URL, + }; + }); + + afterEach(() => { + process.env = originalEnv; + }); + + it('formats preview urls correctly', () => { + formatLighthouseResults({ core: mockCore }); + + const expectations = [ + expect.stringContaining(`[/en](${MOCK_VERCEL_PREVIEW_URL}/en)`), + expect.stringContaining( + `[/en/download](${MOCK_VERCEL_PREVIEW_URL}/en/download)` + ), + ]; + + expectations.forEach(expectation => { + expect(mockCore.setOutput).toBeCalledWith('comment', expectation); + }); + }); + + it('formats stoplight colors correctly', () => { + formatLighthouseResults({ core: mockCore }); + + const expectations = [ + expect.stringContaining(`🟢 90`), + expect.stringContaining(`🟠 75`), + expect.stringContaining(`🔴 49`), + ]; + + expectations.forEach(expectation => { + expect(mockCore.setOutput).toBeCalledWith('comment', expectation); + }); + }); +}); diff --git a/scripts/lighthouse/index.mjs b/scripts/lighthouse/index.mjs new file mode 100644 index 0000000000000..df25285d162b8 --- /dev/null +++ b/scripts/lighthouse/index.mjs @@ -0,0 +1,50 @@ +'use strict'; + +const stoplight = res => (res >= 90 ? '🟢' : res >= 75 ? '🟠' : '🔴'); +const normalizeScore = res => Math.round(res * 100); +const formatScore = res => { + const normalizedScore = normalizeScore(res); + return `${stoplight(normalizedScore)} ${normalizedScore}`; +}; + +/** + * `core` is in scope from https://github.com/actions/github-script + */ +export const formatLighthouseResults = ({ core }) => { + // this will be the shape of https://github.com/treosh/lighthouse-ci-action#manifest + const results = JSON.parse(process.env.LIGHTHOUSE_RESULT); + + // this will be the shape of https://github.com/treosh/lighthouse-ci-action#links + const links = JSON.parse(process.env.LIGHTHOUSE_LINKS); + + // start creating our markdown table + const header = [ + 'Lighthouse Results', + 'URL | Performance | Accessibility | Best Practices | SEO | Report', + '| - | - | - | - | - | - |', + ]; + + // map over each url result, formatting and linking to the output + const urlResults = results.map(({ url, summary }) => { + // make the tested link as a markdown link, without the long-generated host + const shortPreviewLink = `[${url.replace( + process.env.VERCEL_PREVIEW_URL, + '' + )}](${url})`; + + // make each formatted score from our lighthouse properties + const performanceScore = formatScore(summary.performance); + const accessibilityScore = formatScore(summary.accessibility); + const bestPracticesScore = formatScore(summary['best-practices']); + const seoScore = formatScore(summary.seo); + + // create the markdown table row + return `${shortPreviewLink} | ${performanceScore} | ${accessibilityScore} | ${bestPracticesScore} | ${seoScore} | [🔗](${links[url]})`; + }); + + // join the header and the rows together + const finalResults = [...header, ...urlResults].join('\n'); + + // return our output to the github action + core.setOutput('comment', finalResults); +};