Update test set-up to leverage playwright when able to (vercel#28634)

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
momolarson · Sep 13, 2021 · a92a5ca · a92a5ca
1 parent fd2af14
commit a92a5ca
Show file tree

Hide file tree

Showing 484 changed files with 8,021 additions and 6,863 deletions.
diff --git a/.github/actions/next-stats-action/src/prepare/repo-setup.js b/.github/actions/next-stats-action/src/prepare/repo-setup.js
@@ -104,7 +104,7 @@ module.exports = (actionInfo) => {
       // to the correct versions
       for (const pkgName of pkgDatas.keys()) {
         const { pkg, pkgPath } = pkgDatas.get(pkgName)
-        await exec(`cd ${pkgPath} && yarn pack -f ${pkg}-packed.tgz`)
+        await exec(`cd ${pkgPath} && yarn pack -f ${pkg}-packed.tgz`, true)
       }
       return pkgPaths
     },

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
@@ -95,7 +95,6 @@ jobs:
     env:
       NEXT_TELEMETRY_DISABLED: 1
       NEXT_TEST_JOB: 1
-      HEADLESS: true
     steps:
       - uses: actions/cache@v2
         if: ${{needs.build.outputs.docsChange != 'docs only change'}}
@@ -107,14 +106,87 @@ jobs:
       - run: node run-tests.js --type unit
         if: ${{needs.build.outputs.docsChange != 'docs only change'}}
 
+  testDev:
+    name: Test Development
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      NEXT_TELEMETRY_DISABLED: 1
+      NEXT_TEST_JOB: 1
+    steps:
+      - run: echo ${{needs.build.outputs.docsChange}}
+
+      # https://github.com/actions/virtual-environments/issues/1187
+      - name: tune linux network
+        run: sudo ethtool -K eth0 tx off rx off
+
+      - uses: actions/cache@v2
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+        id: restore-build
+        with:
+          path: ./*
+          key: ${{ github.sha }}
+
+      - run: npm i -g [email protected] && npx playwright install-deps
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+
+      # TODO: remove after we fix watchpack watching too much
+      - run: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+
+      # docker run is used here to fix jest snapshots failing inside of the
+      # bare GitHub actions environment for the acceptance tests
+      - run: docker run --ipc=host -e NEXT_TEST_JOB=1 -e NEXT_TELEMETRY_DISABLED=1 -v $(pwd):/next.js mcr.microsoft.com/playwright:focal /bin/bash -c 'cd /next.js && node run-tests.js --type development'
+        name: Run test/development
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+
+      - run: NEXT_TEST_MODE=dev node run-tests.js --type e2e
+        name: Run test/e2e (dev)
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+
+  testProd:
+    name: Test Production
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      NEXT_TELEMETRY_DISABLED: 1
+      NEXT_TEST_JOB: 1
+    steps:
+      - run: echo ${{needs.build.outputs.docsChange}}
+
+      # https://github.com/actions/virtual-environments/issues/1187
+      - name: tune linux network
+        run: sudo ethtool -K eth0 tx off rx off
+
+      - uses: actions/cache@v2
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+        id: restore-build
+        with:
+          path: ./*
+          key: ${{ github.sha }}
+
+      - run: npm i -g [email protected] && npx playwright install-deps
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+
+      # TODO: remove after we fix watchpack watching too much
+      - run: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+
+      - run: node run-tests.js --type production
+        name: Run test/production
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+
+      - run: NEXT_TEST_MODE=start node run-tests.js --type e2e
+        name: Run test/e2e (production)
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+
   testIntegration:
     name: Test Integration
     runs-on: ubuntu-latest
     needs: build
     env:
       NEXT_TELEMETRY_DISABLED: 1
       NEXT_TEST_JOB: 1
-      HEADLESS: true
     strategy:
       fail-fast: false
       matrix:
@@ -133,11 +205,14 @@ jobs:
           path: ./*
           key: ${{ github.sha }}
 
+      - run: npm i -g [email protected] && npx playwright install-deps
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+
       # TODO: remove after we fix watchpack watching too much
       - run: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
         if: ${{needs.build.outputs.docsChange != 'docs only change'}}
 
-      - run: xvfb-run node run-tests.js --timings -g ${{ matrix.group }}/6 -c 3
+      - run: xvfb-run node run-tests.js --timings -g ${{ matrix.group }}/6
         if: ${{needs.build.outputs.docsChange != 'docs only change'}}
 
   testElectron:
@@ -147,7 +222,6 @@ jobs:
     env:
       NEXT_TELEMETRY_DISABLED: 1
       NEXT_TEST_JOB: 1
-      HEADLESS: true
       TEST_ELECTRON: 1
     steps:
       - uses: actions/cache@v2
@@ -187,7 +261,16 @@ jobs:
   testsPass:
     name: thank you, next
     runs-on: ubuntu-latest
-    needs: [lint, checkPrecompiled, testIntegration, testUnit, testYarnPnP]
+    needs:
+      [
+        lint,
+        checkPrecompiled,
+        testIntegration,
+        testUnit,
+        testYarnPnP,
+        testDev,
+        testProd,
+      ]
     steps:
       - run: exit 0
 
@@ -198,7 +281,6 @@ jobs:
     env:
       NEXT_TELEMETRY_DISABLED: 1
       NEXT_TEST_JOB: 1
-      HEADLESS: true
       NEXT_PRIVATE_TEST_WEBPACK4_MODE: 1
 
     steps:
@@ -213,15 +295,16 @@ jobs:
           path: ./*
           key: ${{ github.sha }}
 
-      - run: xvfb-run node run-tests.js test/integration/{basic,fallback-modules,link-ref,production,async-modules,font-optimization,ssr-ctx}/test/index.test.js test/acceptance/*.test.js
+      # docker run is used here to fix jest snapshots failing inside of the
+      # bare GitHub actions environment for the acceptance tests
+      - run: docker run --ipc=host -e NEXT_PRIVATE_TEST_WEBPACK4_MODE=1 -e NEXT_TEST_JOB=1 -e NEXT_TELEMETRY_DISABLED=1 -v $(pwd):/next.js mcr.microsoft.com/playwright:focal /bin/bash -c 'cd /next.js && node run-tests.js --type development test/development/acceptance/{ReactRefresh,ReactRefreshLogBox-app-doc,ReactRefreshLogBox-scss,ReactRefreshLogBox,ReactRefreshLogBoxMisc,ReactRefreshRegression,ReactRefreshRequire}.test.ts test/development/basic/*.test.ts && node run-tests.js test/integration/{fallback-modules,link-ref,production,async-modules,font-optimization,ssr-ctx}/test/index.test.js'
         if: ${{needs.build.outputs.docsChange != 'docs only change'}}
 
   testFirefox:
     name: Test Firefox (production)
     runs-on: ubuntu-latest
     needs: build
     env:
-      HEADLESS: true
       BROWSER_NAME: 'firefox'
       NEXT_TELEMETRY_DISABLED: 1
     steps:
@@ -231,7 +314,9 @@ jobs:
         with:
           path: ./*
           key: ${{ github.sha }}
-      - run: node run-tests.js -c 1 test/integration/production/test/index.test.js
+      - run: npx playwright install-deps && npx playwright install firefox
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+      - run: node run-tests.js test/integration/production/test/index.test.js
         if: ${{needs.build.outputs.docsChange != 'docs only change'}}
 
   testSafari:
@@ -256,6 +341,9 @@ jobs:
         with:
           path: ./*
           key: ${{ github.sha }}
+
+      # TODO: use macos runner so that we can use playwright to test against
+      # PRs instead of only running on canary?
       - run: '[[ -z "$BROWSERSTACK_ACCESS_KEY" ]] && echo "Skipping for PR" || node run-tests.js -c 1 test/integration/production/test/index.test.js'
         if: ${{needs.build.outputs.docsChange != 'docs only change'}}
 
@@ -282,7 +370,11 @@ jobs:
         with:
           path: ./*
           key: ${{ github.sha }}
-      - run: '[[ -z "$BROWSERSTACK_ACCESS_KEY" ]] && echo "Skipping for PR" || node run-tests.js -c 1 test/integration/production-nav/test/index.test.js'
+
+      - run: '[[ -z "$BROWSERSTACK_ACCESS_KEY" ]] && echo "Skipping for PR" || npm i -g [email protected]'
+        if: ${{needs.build.outputs.docsChange != 'docs only change'}}
+
+      - run: '[[ -z "$BROWSERSTACK_ACCESS_KEY" ]] && echo "Skipping for PR" || node run-tests.js test/integration/production-nav/test/index.test.js'
         if: ${{needs.build.outputs.docsChange != 'docs only change'}}
 
   publishRelease:

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -87,6 +87,11 @@ stages:
               key: $(Build.SourceVersion)
               path: $(System.DefaultWorkingDirectory)
             displayName: Cache Build
+
+          - script: |
+              npm i -g [email protected]
+            displayName: 'Install selenium node'
+
           - script: |
               node run-tests.js -c 1 test/integration/production/test/index.test.js test/integration/css-client-nav/test/index.test.js test/integration/rewrites-has-condition/test/index.test.js
             displayName: 'Run tests'

diff --git a/contributing.md b/contributing.md
@@ -56,16 +56,9 @@ If you need to clean the project for any reason, use `yarn clean`.
 
 ## Testing
 
-Make sure you have `chromedriver` installed, and it should match your Chrome version.
-You can install it with:
+See the [testing readme](./test/readme.md) for information on writing tests.
 
-- `apt install chromedriver` on Ubuntu/Debian
-- `brew install --cask chromedriver` on Mac OS X
-- `chocolatey install chromedriver` on Windows
-
-- Or manually download the version that matches your installed chrome version (if there's no match, download a version under it, but not above) from the [chromedriver repo](https://chromedriver.storage.googleapis.com/index.html) and add the binary to `<next-repo>/node_modules/.bin`
-
-You may also have to [install Rust](https://www.rust-lang.org/tools/install) and build our native packages to see all tests pass locally. We check in binaries for the most common targets and those required for CI so that most people don't have to, but if you do not see a binary for your target in `packages/next/native`, you can build it by running `yarn --cwd packages/next build-native`. If you are working on the Rust code and you need to build the binaries for ci, you can manually trigger [the workflow](https://github.com/vercel/next.js/actions/workflows/build_native.yml) to build and commit with the "Run workflow" button.
+You may have to [install Rust](https://www.rust-lang.org/tools/install) and build our native packages to see all tests pass locally. We check in binaries for the most common targets and those required for CI so that most people don't have to, but if you do not see a binary for your target in `packages/next/native`, you can build it by running `yarn --cwd packages/next build-native`. If you are working on the Rust code and you need to build the binaries for ci, you can manually trigger [the workflow](https://github.com/vercel/next.js/actions/workflows/build_native.yml) to build and commit with the "Run workflow" button.
 
 ### Running tests
 
@@ -79,12 +72,6 @@ If you would like to run the tests in headless mode (with the browser windows hi
 yarn testheadless
 ```
 
-If you would like to use a specific Chrome/Chromium binary to run tests you can specify it with
-
-```sh
-CHROME_BIN='path/to/chrome/bin' yarn testonly
-```
-
 Running a specific test suite inside of the `test/integration` directory:
 
 ```sh

diff --git a/jest.config.js b/jest.config.js
@@ -1,18 +1,16 @@
 module.exports = {
   testMatch: ['**/*.test.js', '**/*.test.ts', '**/*.test.tsx'],
+  setupFilesAfterEnv: ['<rootDir>/jest-setup-after-env.ts'],
   verbose: true,
   rootDir: 'test',
   modulePaths: ['<rootDir>/lib'],
-  globalSetup: '<rootDir>/jest-global-setup.js',
-  globalTeardown: '<rootDir>/jest-global-teardown.js',
-  setupFilesAfterEnv: ['<rootDir>/jest-setup-after-env.js'],
-  testEnvironment: '<rootDir>/jest-environment.js',
   transformIgnorePatterns: ['/node_modules/', '/next[/\\\\]dist/'],
   transform: {
     '.+\\.(t|j)sx?$': [
       // this matches our SWC options used in https://github.com/vercel/next.js/blob/canary/packages/next/taskfile-swc.js
       '@swc/jest',
       {
+        sourceMaps: 'inline',
         module: {
           type: 'commonjs',
         },

diff --git a/package.json b/package.json
@@ -54,6 +54,7 @@
     "@types/fs-extra": "8.1.0",
     "@types/http-proxy": "1.17.3",
     "@types/jest": "24.0.13",
+    "@types/selenium-webdriver": "4.0.15",
     "@types/sharp": "0.28.4",
     "@types/string-hash": "1.1.1",
     "@typescript-eslint/eslint-plugin": "4.29.1",
@@ -67,7 +68,6 @@
     "amphtml-validator": "1.0.33",
     "async-sema": "3.0.1",
     "browserslist": "^4.14.7",
-    "browserstack-local": "1.4.0",
     "cheerio": "0.22.0",
     "clone": "2.1.2",
     "cookie": "0.4.1",
@@ -108,6 +108,7 @@
     "npm-run-all": "4.1.5",
     "nprogress": "0.2.0",
     "pixrem": "5.0.0",
+    "playwright-chromium": "1.14.1",
     "pnpm": "5.14.3",
     "postcss-nested": "4.2.1",
     "postcss-pseudoelements": "5.0.0",
@@ -124,14 +125,13 @@
     "react-ssr-prepass": "1.0.8",
     "release": "6.3.0",
     "request-promise-core": "1.1.2",
+    "resolve-from": "5.0.0",
     "rimraf": "2.6.3",
     "seedrandom": "3.0.5",
-    "selenium-standalone": "6.18.0",
-    "selenium-webdriver": "4.0.0-alpha.7",
+    "selenium-webdriver": "4.0.0-beta.4",
     "shell-quote": "1.7.2",
     "sqlite": "4.0.22",
     "sqlite3": "5.0.2",
-    "styled-components": "5.1.0",
     "styled-jsx-plugin-postcss": "3.0.2",
     "tailwindcss": "1.1.3",
     "taskr": "1.1.0",

diff --git a/packages/next/cli/next-dev.ts b/packages/next/cli/next-dev.ts
@@ -71,19 +71,26 @@ const nextDev: cliCommand = (argv) => {
     }
   }
 
-  const port =
+  let port: number =
     args['--port'] || (process.env.PORT && parseInt(process.env.PORT)) || 3000
 
+  // we allow the server to use a random port while testing
+  // instead of attempting to find a random port and then hope
+  // it doesn't become occupied before we leverage it
+  if (process.env.__NEXT_RAND_PORT) {
+    port = 0
+  }
+
   // We do not set a default host value here to prevent breaking
   // some set-ups that rely on listening on other interfaces
   const host = args['--hostname']
-  const appUrl = `http://${
-    !host || host === '0.0.0.0' ? 'localhost' : host
-  }:${port}`
 
   startServer({ dir, dev: true, isNextDevCommand: true }, port, host)
-    .then(async (app) => {
-      startedDevelopmentServer(appUrl, `${host || '0.0.0.0'}:${port}`)
+    .then(async ({ app, actualPort }) => {
+      const appUrl = `http://${
+        !host || host === '0.0.0.0' ? 'localhost' : host
+      }:${actualPort}`
+      startedDevelopmentServer(appUrl, `${host || '0.0.0.0'}:${actualPort}`)
       // Start preflight after server is listening and ignore errors:
       preflight().catch(() => {})
       // Finalize server bootup:

diff --git a/packages/next/cli/next-start.ts b/packages/next/cli/next-start.ts
@@ -49,13 +49,20 @@ const nextStart: cliCommand = (argv) => {
   }
 
   const dir = resolve(args._[0] || '.')
-  const port =
+  let port: number =
     args['--port'] || (process.env.PORT && parseInt(process.env.PORT)) || 3000
   const host = args['--hostname'] || '0.0.0.0'
-  const appUrl = `http://${host === '0.0.0.0' ? 'localhost' : host}:${port}`
+
+  if (process.env.__NEXT_RAND_PORT) {
+    port = 0
+  }
+
   startServer({ dir }, port, host)
-    .then(async (app) => {
-      Log.ready(`started server on ${host}:${port}, url: ${appUrl}`)
+    .then(async ({ app, actualPort }) => {
+      const appUrl = `http://${
+        host === '0.0.0.0' ? 'localhost' : host
+      }:${actualPort}`
+      Log.ready(`started server on ${host}:${actualPort}, url: ${appUrl}`)
       await app.prepare()
     })
     .catch((err) => {

diff --git a/packages/next/compiled/babel-packages/packages-bundle.js b/packages/next/compiled/babel-packages/packages-bundle.js
diff --git a/packages/next/compiled/babel/bundle.js b/packages/next/compiled/babel/bundle.js
diff --git a/packages/next/server/lib/start-server.ts b/packages/next/server/lib/start-server.ts
@@ -19,5 +19,9 @@ export default async function start(
   })
   // It's up to caller to run `app.prepare()`, so it can notify that the server
   // is listening before starting any intensive operations.
-  return app
+  const addr = srv.address()
+  return {
+    app,
+    actualPort: addr && typeof addr === 'object' ? addr.port : port,
+  }
 }