♻️ use playwright for e2e

.gitignore

@@ -17,3 +17,7 @@ browserstack.err
 !.yarn/releases
 !.yarn/sdks
 !.yarn/versions
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

package.json

@@ -32,11 +32,13 @@
   },
   "devDependencies": {
     "@jsdevtools/coverage-istanbul-loader": "3.0.5",
+    "@playwright/test": "1.49.0",
     "@types/chrome": "0.0.287",
     "@types/connect-busboy": "1.0.3",
     "@types/cors": "2.8.17",
     "@types/express": "4.17.21",
     "@types/jasmine": "3.10.18",
+    "@types/node": "22.10.2",
     "@typescript-eslint/eslint-plugin": "8.16.0",
     "@typescript-eslint/parser": "8.16.0",
     "@wdio/browserstack-service": "8.40.6",

playwright.config.ts

@@ -0,0 +1,80 @@
+import { defineConfig, devices } from '@playwright/test'
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// import dotenv from 'dotenv';
+// import path from 'path';
+// dotenv.config({ path: path.resolve(__dirname, '.env') });
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './test/e2e/scenario',
+  testMatch: '**/init.scenario.ts',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://127.0.0.1:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+
+    {
+      name: 'firefox',
+      use: { ...devices['Desktop Firefox'] },
+    },
+
+    {
+      name: 'webkit',
+      use: { ...devices['Desktop Safari'] },
+    },
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: { ...devices['Pixel 5'] },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: { ...devices['iPhone 12'] },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+    // },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  // webServer: {
+  //   command: 'npm run start',
+  //   url: 'http://127.0.0.1:3000',
+  //   reuseExistingServer: !process.env.CI,
+  // },
+})

test/e2e/lib/framework/createTest.ts

@@ -1,8 +1,11 @@
 import type { LogsInitConfiguration } from '@datadog/browser-logs'
 import type { RumInitConfiguration } from '@datadog/browser-rum-core'
 import { DefaultPrivacyLevel } from '@datadog/browser-rum'
+import type { BrowserContext, Page } from '@playwright/test'
+import { test, expect } from '@playwright/test'
 import { getRunId } from '../../../envUtils'
-import { deleteAllCookies, getBrowserName, withBrowserLogs } from '../helpers/browser'
+import type { BrowserLog } from '../helpers/browser'
+import { BrowserLogsManager, deleteAllCookies } from '../helpers/browser'
 import { APPLICATION_ID, CLIENT_TOKEN } from '../helpers/configuration'
 import { validateRumFormat } from '../helpers/validation'
 import { IntakeRegistry } from './intakeRegistry'
@@ -43,9 +46,15 @@ interface TestContext {
   crossOriginUrl: string
   intakeRegistry: IntakeRegistry
   servers: Servers
+  page: Page
+  browserContext: BrowserContext
+  withBrowserLogs: (cb: (logs: BrowserLog[]) => void) => void
+  flushBrowserLogs: () => void
+  flushEvents: () => Promise<void>
+  deleteAllCookies: () => Promise<void>
 }
 
-type TestRunner = (testContext: TestContext) => Promise<void>
+type TestRunner = (testContext: TestContext) => Promise<void> | void
 
 class TestBuilder {
   private rumConfiguration: RumInitConfiguration | undefined = undefined
@@ -132,7 +141,7 @@ class TestBuilder {
     }
 
     if (this.alsoRunWithRumSlim) {
-      describe(this.title, () => {
+      test.describe(this.title, () => {
         declareTestsForSetups('rum', setups, setupOptions, runner)
         declareTestsForSetups(
           'rum-slim',
@@ -155,11 +164,6 @@ class TestBuilder {
   }
 }
 
-interface ItResult {
-  getFullName(): string
-}
-declare function it(expectation: string, assertion?: jasmine.ImplementationCallback, timeout?: number): ItResult
-
 function declareTestsForSetups(
   title: string,
   setups: Array<{ factory: SetupFactory; name?: string }>,
@@ -178,53 +182,82 @@ function declareTestsForSetups(
 }
 
 function declareTest(title: string, setupOptions: SetupOptions, factory: SetupFactory, runner: TestRunner) {
-  const spec = it(title, async () => {
-    log(`Start '${spec.getFullName()}' in ${getBrowserName()}`)
-    setupOptions.context.test_name = spec.getFullName()
+  test(title, async ({ page, context, browserName }) => {
+    const title = test.info().titlePath.join(' > ')
+    log(`Start '${title}' in ${browserName}`)
+
+    setupOptions.context.test_name = title
 
     const servers = await getTestServers()
+    const browserLogs = new BrowserLogsManager()
 
-    const testContext = createTestContext(servers, setupOptions)
+    const testContext = createTestContext(servers, page, context, browserLogs, setupOptions)
     servers.intake.bindServerApp(createIntakeServerApp(testContext.intakeRegistry))
 
     const setup = factory(setupOptions, servers)
     servers.base.bindServerApp(createMockServerApp(servers, setup))
     servers.crossOrigin.bindServerApp(createMockServerApp(servers, setup))
 
-    await setUpTest(testContext)
+    await setUpTest(browserLogs, testContext)
 
     try {
       await runner(testContext)
     } finally {
       await tearDownTest(testContext)
-      log(`End '${spec.getFullName()}'`)
+      log(`End '${title}'`)
     }
   })
 }
 
-function createTestContext(servers: Servers, { basePath }: SetupOptions): TestContext {
+function createTestContext(
+  servers: Servers,
+  page: Page,
+  browserContext: BrowserContext,
+  browserLogsManager: BrowserLogsManager,
+  { basePath }: SetupOptions
+): TestContext {
   return {
     baseUrl: servers.base.url + basePath,
     crossOriginUrl: servers.crossOrigin.url,
     intakeRegistry: new IntakeRegistry(),
     servers,
+    page,
+    browserContext,
+    withBrowserLogs: (cb: (logs: BrowserLog[]) => void) => {
+      cb(browserLogsManager.get())
+      browserLogsManager.clear()
+    },
+    flushBrowserLogs: () => {
+      browserLogsManager.clear()
+    },
+    flushEvents: () => flushEvents(page),
+    deleteAllCookies: () => deleteAllCookies(browserContext),
   }
 }
 
-async function setUpTest({ baseUrl }: TestContext) {
-  await browser.url(baseUrl)
+async function setUpTest(browserLogsManager: BrowserLogsManager, { baseUrl, page, browserContext }: TestContext) {
+  browserContext.on('console', (msg) =>
+    browserLogsManager.add({
+      level: msg.type() as BrowserLog['level'],
+      message: msg.text(),
+      source: 'console',
+      timestamp: Date.now(),
+    })
+  )
+
+  await page.goto(baseUrl)
   await waitForServersIdle()
 }
 
-async function tearDownTest({ intakeRegistry }: TestContext) {
+async function tearDownTest({ intakeRegistry, withBrowserLogs, flushEvents, deleteAllCookies }: TestContext) {
   await flushEvents()
   expect(intakeRegistry.telemetryErrorEvents).toEqual([])
   validateRumFormat(intakeRegistry.rumEvents)
-  await withBrowserLogs((logs) => {
+  withBrowserLogs((logs) => {
     logs.forEach((browserLog) => {
       log(`Browser ${browserLog.source}: ${browserLog.level} ${browserLog.message}`)
     })
-    expect(logs.filter((l) => (l as any).level === 'SEVERE')).toEqual([])
+    expect(logs.filter((log) => log.level === 'error')).toEqual([])
   })
   await deleteAllCookies()
 }

test/e2e/lib/framework/flushEvents.ts

@@ -1,8 +1,9 @@
+import type { Page } from '@playwright/test'
 import { getTestServers, waitForServersIdle } from './httpServers'
 import { waitForRequests } from './waitForRequests'
 
-export async function flushEvents() {
-  await waitForRequests()
+export async function flushEvents(page: Page) {
+  await waitForRequests(page)
 
   const servers = await getTestServers()
 
@@ -20,6 +21,6 @@ export async function flushEvents() {
   // The issue mainly occurs with local e2e tests (not browserstack), because the network latency is
   // very low (same machine), so the request resolves very quickly. In real life conditions, this
   // issue is mitigated, because requests will likely take a few milliseconds to reach the server.
-  await browser.url(`${servers.base.url}/ok?duration=200`)
+  await page.goto(`${servers.base.url}/ok?duration=200`)
   await waitForServersIdle()
 }

test/e2e/lib/framework/logger.ts

@@ -1,8 +1,10 @@
 import * as fs from 'fs'
 import { inspect } from 'util'
 
-const logsPath = (browser.options as WebdriverIO.Config & { logsPath: string }).logsPath
-const stream: { write(s: string): void } = logsPath ? fs.createWriteStream(logsPath, { flags: 'a' }) : process.stdout
+const logsPath = undefined // (browser.options as WebdriverIO.Config & { logsPath: string }).logsPath
+const stream: { write(s: string): void } = logsPath
+  ? fs.createWriteStream(logsPath, { flags: 'a' })
+  : { write: () => {} } // TODO: why do we need to log to stdout?
 
 export function log(...args: any[]) {
   const prefix = `[${process.pid}] ${new Date().toISOString()}`

test/e2e/lib/framework/pageSetups.ts

@@ -21,10 +21,10 @@ export interface SetupOptions {
 
 export type SetupFactory = (options: SetupOptions, servers: Servers) => string
 
-const isBrowserStack =
-  'services' in browser.options &&
-  browser.options.services &&
-  browser.options.services.some((service) => (Array.isArray(service) ? service[0] : service) === 'browserstack')
+const isBrowserStack = false
+// 'services' in browser.options &&
+// browser.options.services &&
+// browser.options.services.some((service) => (Array.isArray(service) ? service[0] : service) === 'browserstack')
 
 const isContinuousIntegration = Boolean(process.env.CI_JOB_ID)
 

test/e2e/lib/framework/waitForRequests.ts

@@ -1,3 +1,4 @@
+import type { Page } from '@playwright/test'
 import { waitForServersIdle } from './httpServers'
 
 /**
@@ -10,11 +11,14 @@ import { waitForServersIdle } from './httpServers'
  * As a workaround, this function delays the `waitForServersIdle()` call by doing a browser
  * roundtrip, ensuring requests have plenty of time to reach the local server.
  */
-export async function waitForRequests() {
-  await browser.executeAsync((done) =>
-    setTimeout(() => {
-      done(undefined)
-    }, 200)
+export async function waitForRequests(page: Page) {
+  await page.evaluate(
+    () =>
+      new Promise((resolve) => {
+        setTimeout(() => {
+          resolve(undefined)
+        }, 200)
+      })
   )
   await waitForServersIdle()
 }

test/e2e/lib/helpers/browser.ts

@@ -1,4 +1,5 @@
 import * as os from 'os'
+import type { BrowserContext } from '@playwright/test'
 
 // To keep tests sane, ensure we got a fixed list of possible platforms and browser names.
 const validPlatformNames = ['windows', 'macos', 'linux', 'ios', 'android'] as const
@@ -56,39 +57,33 @@ function includes<T>(list: readonly T[], item: unknown): item is T {
   return list.includes(item as any)
 }
 
-interface BrowserLog {
-  level: string
+export interface BrowserLog {
+  level: 'log' | 'debug' | 'info' | 'error' | 'warning'
   message: string
   source: string
   timestamp: number
 }
 
-export async function withBrowserLogs(fn: (logs: BrowserLog[]) => void) {
-  // browser.getLogs is not defined when using a remote webdriver service. We should find an
-  // alternative at some point.
-  // https://github.com/webdriverio/webdriverio/issues/4275
-  if (browser.getLogs) {
-    const logs = (await browser.getLogs('browser')) as BrowserLog[]
-    fn(logs)
+export class BrowserLogsManager {
+  private logs: BrowserLog[] = []
+
+  add(log: BrowserLog) {
+    this.logs.push(log)
   }
-}
 
-export async function flushBrowserLogs() {
-  await withBrowserLogs(() => {
-    // Ignore logs
-  })
+  get() {
+    return this.logs
+  }
+
+  clear() {
+    this.logs = []
+  }
 }
 
+// TODO, see if we can use the browser context to clear cookies or we should keep the previous hack
 // wdio method does not work for some browsers
-export function deleteAllCookies() {
-  return browser.execute(() => {
-    const cookies = document.cookie.split(';')
-    for (const cookie of cookies) {
-      const eqPos = cookie.indexOf('=')
-      const name = eqPos > -1 ? cookie.substr(0, eqPos) : cookie
-      document.cookie = `${name}=;expires=Thu, 01 Jan 1970 00:00:00 GMT;samesite=strict`
-    }
-  })
+export function deleteAllCookies(context: BrowserContext) {
+  return context.clearCookies()
 }
 
 export function setCookie(name: string, value: string, expiresDelay: number = 0) {