diff --git a/package-lock.json b/package-lock.json index 07a6a52726c30..3ef67e7e84443 100644 --- a/package-lock.json +++ b/package-lock.json @@ -127,22 +127,6 @@ "node": ">=6.0.0" } }, - "node_modules/@anthropic-ai/sdk": { - "version": "0.33.1", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.33.1.tgz", - "integrity": "sha512-VrlbxiAdVRGuKP2UQlCnsShDHJKWepzvfRCkZMpU+oaUdKLpOfmylLMRojGrAgebV+kDtPjewCVP0laHXg+vsA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7" - } - }, "node_modules/@babel/cli": { "version": "7.26.4", "resolved": "https://registry.npmjs.org/@babel/cli/-/cli-7.26.4.tgz", @@ -1638,8 +1622,8 @@ "resolved": "packages/playwright-ct-vue", "link": true }, - "node_modules/@playwright/experimental-tools": { - "resolved": "packages/playwright-tools", + "node_modules/@playwright/mcp": { + "resolved": "packages/playwright-mcp", "link": true }, "node_modules/@playwright/test": { @@ -2078,17 +2062,6 @@ "undici-types": "~5.26.4" } }, - "node_modules/@types/node-fetch": { - "version": "2.6.12", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz", - "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.0" - } - }, "node_modules/@types/prop-types": { "version": "15.7.14", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.14.tgz", @@ -2574,19 +2547,6 @@ "dev": true, "license": "ISC" }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "dev": true, - "license": "MIT", - "dependencies": { - "event-target-shim": "^5.0.0" - }, - "engines": { - "node": ">=6.5" - } - }, "node_modules/accepts": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", @@ -2643,19 +2603,6 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, - "node_modules/agentkeepalive": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", - "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -2926,13 +2873,6 @@ "node": ">= 0.4" } }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true, - "license": "MIT" - }, "node_modules/available-typed-arrays": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", @@ -3335,19 +3275,6 @@ "node": ">=0.1.90" } }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/commander": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz", @@ -3689,16 +3616,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -4492,16 +4409,6 @@ "node": ">= 0.6" } }, - "node_modules/event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/eventsource": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.5.tgz", @@ -4823,43 +4730,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/form-data": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.2.tgz", - "integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==", - "dev": true, - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/form-data-encoder": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", - "dev": true, - "license": "MIT" - }, - "node_modules/formdata-node": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", - "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "node-domexception": "1.0.0", - "web-streams-polyfill": "4.0.0-beta.3" - }, - "engines": { - "node": ">= 12.20" - } - }, "node_modules/formidable": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz", @@ -5369,16 +5239,6 @@ "node": ">=10.19.0" } }, - "node_modules/humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.0.0" - } - }, "node_modules/iconv-lite": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.2.tgz", @@ -6529,47 +6389,6 @@ "node": ">= 0.6" } }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "dev": true, - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "node_modules/node-releases": { "version": "2.0.19", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", @@ -6801,37 +6620,6 @@ "wrappy": "1" } }, - "node_modules/openai": { - "version": "4.85.1", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.85.1.tgz", - "integrity": "sha512-jkX2fntHljUvSH3MkWh4jShl10oNkb+SsCj4auKlbu2oF4KWAnmHLNR5EpnUHK1ZNW05Rp0fjbJzYwQzMsH8ZA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7" - }, - "bin": { - "openai": "bin/cli" - }, - "peerDependencies": { - "ws": "^8.18.0", - "zod": "^3.23.8" - }, - "peerDependenciesMeta": { - "ws": { - "optional": true - }, - "zod": { - "optional": true - } - } - }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -8345,13 +8133,6 @@ "node": ">=0.6" } }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "dev": true, - "license": "MIT" - }, "node_modules/trace-viewer": { "resolved": "packages/trace-viewer", "link": true @@ -9264,34 +9045,6 @@ "resolved": "packages/web", "link": true }, - "node_modules/web-streams-polyfill": { - "version": "4.0.0-beta.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", - "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "dev": true, - "license": "BSD-2-Clause" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dev": true, - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -10308,6 +10061,20 @@ "node": ">=18" } }, + "packages/playwright-mcp": { + "name": "@playwright/mcp", + "version": "0.0.1", + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.52.0-next" + }, + "devDependencies": { + "@modelcontextprotocol/sdk": "^1.6.1" + }, + "engines": { + "node": ">=18" + } + }, "packages/playwright-test": { "name": "@playwright/test", "version": "1.52.0-next", @@ -10325,6 +10092,7 @@ "packages/playwright-tools": { "name": "@playwright/experimental-tools", "version": "0.0.0", + "extraneous": true, "license": "Apache-2.0", "dependencies": { "playwright": "1.52.0-next" diff --git a/packages/playwright-tools/.npmignore b/packages/playwright-mcp/.npmignore similarity index 100% rename from packages/playwright-tools/.npmignore rename to packages/playwright-mcp/.npmignore diff --git a/packages/playwright-tools/README.md b/packages/playwright-mcp/README.md similarity index 100% rename from packages/playwright-tools/README.md rename to packages/playwright-mcp/README.md diff --git a/packages/playwright-mcp/package.json b/packages/playwright-mcp/package.json new file mode 100644 index 0000000000000..e07f13961daa3 --- /dev/null +++ b/packages/playwright-mcp/package.json @@ -0,0 +1,33 @@ +{ + "name": "@playwright/mcp", + "private": true, + "version": "0.0.1", + "description": "Playwright Tools for MCP", + "repository": { + "type": "git", + "url": "git+https://github.com/microsoft/playwright.git" + }, + "homepage": "https://playwright.dev", + "engines": { + "node": ">=18" + }, + "author": { + "name": "Microsoft Corporation" + }, + "license": "Apache-2.0", + "exports": { + "./servers/server": "./lib/servers/server.js", + "./servers/screenshot": "./lib/servers/screenshot.js", + "./servers/snapshot": "./lib/servers/snapshot.js", + "./tools/common": "./lib/tools/common.js", + "./tools/screenshot": "./lib/tools/screenshot.js", + "./tools/snapshot": "./lib/tools/snapshot.js", + "./package.json": "./package.json" + }, + "dependencies": { + "playwright": "1.52.0-next" + }, + "devDependencies": { + "@modelcontextprotocol/sdk": "^1.6.1" + } +} diff --git a/packages/playwright-tools/browser.js b/packages/playwright-mcp/src/servers/screenshot.ts similarity index 59% rename from packages/playwright-tools/browser.js rename to packages/playwright-mcp/src/servers/screenshot.ts index d0475e384b5aa..2f9eefae743d2 100644 --- a/packages/playwright-tools/browser.js +++ b/packages/playwright-mcp/src/servers/screenshot.ts @@ -14,6 +14,22 @@ * limitations under the License. */ -const { schema, call, snapshot } = require('./lib/tools/browser'); +import { Server } from './server'; +import { navigate, wait, pressKey } from '../tools/common'; +import { screenshot, moveMouse, click, drag, type } from '../tools/screenshot'; -module.exports = { schema, call, snapshot }; +const server = new Server({ + name: 'Playwright screenshot-based browser server', + version: '0.0.1', + tools: [ + navigate, + screenshot, + moveMouse, + click, + drag, + type, + pressKey, + wait, + ] +}); +server.start(); diff --git a/packages/playwright-mcp/src/servers/server.ts b/packages/playwright-mcp/src/servers/server.ts new file mode 100644 index 0000000000000..c81de2a3f9151 --- /dev/null +++ b/packages/playwright-mcp/src/servers/server.ts @@ -0,0 +1,96 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Server as MCPServer } from '@modelcontextprotocol/sdk/server/index.js'; +import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; +import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js'; +import * as playwright from 'playwright'; + +import type { Tool } from '../tools/common'; + +export class Server { + private _server: MCPServer; + private _tools: Tool[]; + private _page: playwright.Page | undefined; + + constructor(options: { name: string, version: string, tools: Tool[] }) { + const { name, version, tools } = options; + this._server = new MCPServer({ name, version }, { capabilities: { tools: {} } }); + this._tools = tools; + + this._server.setRequestHandler(ListToolsRequestSchema, async () => { + return { tools: tools.map(tool => tool.schema) }; + }); + + this._server.setRequestHandler(CallToolRequestSchema, async request => { + const page = await this._openPage(); + + const tool = this._tools.find(tool => tool.schema.name === request.params.name); + if (!tool) { + return { + content: [{ type: 'text', text: `Tool "${request.params.name}" not found` }], + isError: true, + }; + } + + try { + const result = await tool.handle({ page }, request.params.arguments); + return result; + } catch (error) { + return { + content: [{ type: 'text', text: String(error) }], + isError: true, + }; + } + }); + + this._setupExitWatchdog(); + } + + start() { + const transport = new StdioServerTransport(); + void this._server.connect(transport); + } + + private async _createBrowser(): Promise { + if (process.env.PLAYWRIGHT_WS_ENDPOINT) { + return await playwright.chromium.connect( + process.env.PLAYWRIGHT_WS_ENDPOINT + ); + } + return await playwright.chromium.launch({ headless: false }); + } + + private async _openPage(): Promise { + if (!this._page) { + const browser = await this._createBrowser(); + const context = await browser.newContext(); + this._page = await context.newPage(); + } + return this._page; + } + + private _setupExitWatchdog() { + process.stdin.on('close', async () => { + this._server.close(); + // eslint-disable-next-line no-restricted-properties + setTimeout(() => process.exit(0), 15000); + await this._page?.context()?.browser()?.close(); + // eslint-disable-next-line no-restricted-properties + process.exit(0); + }); + } +} diff --git a/packages/playwright-tools/types.d.ts b/packages/playwright-mcp/src/servers/snapshot.ts similarity index 61% rename from packages/playwright-tools/types.d.ts rename to packages/playwright-mcp/src/servers/snapshot.ts index 5fe61526c2c50..71b6b478b9997 100644 --- a/packages/playwright-tools/types.d.ts +++ b/packages/playwright-mcp/src/servers/snapshot.ts @@ -14,14 +14,21 @@ * limitations under the License. */ -import type playwright from 'playwright'; +import { Server } from './server'; +import { wait, pressKey } from '../tools/common'; +import { navigate, snapshot, click, hover, type } from '../tools/snapshot'; -export type JSONSchemaType = string | number | boolean | JSONSchemaObject | JSONSchemaArray | null; -interface JSONSchemaObject { [key: string]: JSONSchemaType; } -interface JSONSchemaArray extends Array {} - -export type ToolDeclaration = { - name: string; - description: string; - parameters: any; -}; +const server = new Server({ + name: 'Playwright snapshot-based browser server', + version: '0.0.1', + tools: [ + navigate, + snapshot, + click, + hover, + type, + pressKey, + wait, + ] +}); +server.start(); diff --git a/packages/playwright-mcp/src/tools/common.ts b/packages/playwright-mcp/src/tools/common.ts new file mode 100644 index 0000000000000..214b2e997e79f --- /dev/null +++ b/packages/playwright-mcp/src/tools/common.ts @@ -0,0 +1,124 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { waitForCompletion } from '../utils'; + +import type * as playwright from 'playwright'; +import type { ImageContent, TextContent } from '@modelcontextprotocol/sdk/types'; + +export type ToolContext = { + page: playwright.Page; +}; + +export type ToolSchema = { + name: string; + description: string; + inputSchema: Record; +}; + +export type ToolResult = { + content: (ImageContent | TextContent)[]; + isError?: boolean; +}; + +export type Tool = { + schema: ToolSchema; + handle: (context: ToolContext, params?: Record) => Promise; +}; + +export const navigate: Tool = { + schema: { + name: 'navigate', + description: 'Navigate to a URL', + inputSchema: { + type: 'object', + properties: { + url: { + type: 'string', + description: 'URL to navigate to', + }, + }, + } + }, + + handle: async (context, params) => { + await waitForCompletion(context.page, async () => { + await context.page.goto(params!.url as string); + }); + return { + content: [{ + type: 'text', + text: `Navigated to ${params!.url}`, + }], + }; + } +}; + +export const wait: Tool = { + schema: { + name: 'wait', + description: `Wait for given amount of time to see if the page updates. Use it after action if you think page is not ready yet`, + inputSchema: { + type: 'object', + properties: { + time: { + type: 'integer', + description: 'Time to wait in seconds', + }, + }, + required: ['time'], + } + }, + + handle: async (context, params) => { + await context.page.waitForTimeout(Math.min(10000, params!.time as number * 1000)); + return { + content: [{ + type: 'text', + text: `Waited for ${params!.time} seconds`, + }], + }; + } +}; + +export const pressKey: Tool = { + schema: { + name: 'press_key', + description: 'Press a key', + inputSchema: { + type: 'object', + properties: { + key: { + type: 'string', + description: 'Name of the key to press or a character to generate, such as `ArrowLeft` or `a`', + }, + }, + required: ['key'], + } + }, + + handle: async (context, params) => { + await waitForCompletion(context.page, async () => { + await context.page.keyboard.press(params!.key as string); + }); + return { + content: [{ + type: 'text', + text: `Pressed key ${params!.key}`, + }], + }; + } +}; diff --git a/packages/playwright-mcp/src/tools/screenshot.ts b/packages/playwright-mcp/src/tools/screenshot.ts new file mode 100644 index 0000000000000..155f39a130edf --- /dev/null +++ b/packages/playwright-mcp/src/tools/screenshot.ts @@ -0,0 +1,143 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Tool } from './common'; +import { waitForCompletion } from '../utils'; + +export const screenshot: Tool = { + schema: { + name: 'screenshot', + description: 'Take a screenshot of the current page', + inputSchema: { + type: 'object', + properties: {}, + } + }, + + handle: async context => { + const screenshot = await context.page.screenshot({ type: 'jpeg', quality: 50, scale: 'css' }); + return { + content: [{ type: 'image', data: screenshot.toString('base64'), mimeType: 'image/jpeg' }], + }; + } +}; + +export const moveMouse: Tool = { + schema: { + name: 'move_mouse', + description: 'Move mouse to a given position', + inputSchema: { + type: 'object', + properties: { + x: { + type: 'number', + description: 'X coordinate', + }, + y: { + type: 'number', + description: 'Y coordinate', + }, + }, + required: ['x', 'y'], + } + }, + + handle: async (context, params) => { + await context.page.mouse.move(params!.x as number, params!.y as number); + return { + content: [{ type: 'text', text: `Moved mouse to (${params!.x}, ${params!.y})` }], + }; + } +}; + +export const click: Tool = { + schema: { + name: 'click', + description: 'Click left mouse button', + inputSchema: { + type: 'object', + properties: {}, + } + }, + + handle: async context => { + await waitForCompletion(context.page, async () => { + await context.page.mouse.down(); + await context.page.mouse.up(); + }); + return { + content: [{ type: 'text', text: 'Clicked mouse' }], + }; + } +}; + +export const drag: Tool = { + schema: { + name: 'drag', + description: 'Drag left mouse button', + inputSchema: { + type: 'object', + properties: { + x: { + type: 'number', + description: 'X coordinate', + }, + y: { + type: 'number', + description: 'Y coordinate', + }, + }, + required: ['x', 'y'], + } + }, + + handle: async (context, params) => { + await waitForCompletion(context.page, async () => { + await context.page.mouse.down(); + await context.page.mouse.move(params!.x as number, params!.y as number); + await context.page.mouse.up(); + }); + return { + content: [{ type: 'text', text: `Dragged mouse to (${params!.x}, ${params!.y})` }], + }; + } +}; + +export const type: Tool = { + schema: { + name: 'type', + description: 'Type text', + inputSchema: { + type: 'object', + properties: { + text: { + type: 'string', + description: 'Text to type', + }, + }, + required: ['text'], + } + }, + + handle: async (context, params) => { + await waitForCompletion(context.page, async () => { + await context.page.keyboard.type(params!.text as string); + }); + return { + content: [{ type: 'text', text: `Typed text "${params!.text}"` }], + }; + } +}; diff --git a/packages/playwright-mcp/src/tools/snapshot.ts b/packages/playwright-mcp/src/tools/snapshot.ts new file mode 100644 index 0000000000000..22cf2eea533c2 --- /dev/null +++ b/packages/playwright-mcp/src/tools/snapshot.ts @@ -0,0 +1,148 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { waitForCompletion } from '../utils'; + +import type * as playwright from 'playwright'; +import type { Tool, ToolContext, ToolResult } from './common'; + +const elementIdProperty = { + elementId: { + type: 'number', + description: 'Target element', + } +}; + +export const snapshot: Tool = { + schema: { + name: 'snapshot', + description: 'Capture accessibility snapshot of the current page, this is better than screenshot', + inputSchema: { + type: 'object', + properties: {}, + } + }, + + handle: async context => { + return await captureAriaSnapshot(context.page); + } +}; + +export const navigate: Tool = { + schema: { + name: 'navigate', + description: 'Navigate to a URL', + inputSchema: { + type: 'object', + properties: { + url: { + type: 'string', + description: 'URL to navigate to', + }, + }, + } + }, + + handle: async (context, params) => { + return runAndCaptureSnapshot(context, () => context.page.goto(params!.url)); + } +}; + +export const click: Tool = { + schema: { + name: 'click', + description: 'Perform click on a web page', + inputSchema: { + type: 'object', + properties: { + ...elementIdProperty, + }, + required: ['elementId'], + } + }, + + handle: async (context, params) => { + const locator = elementIdLocator(context.page, params!); + return runAndCaptureSnapshot(context, () => locator.click()); + } +}; + +export const hover: Tool = { + schema: { + name: 'hover', + description: 'Hover over element on page', + inputSchema: { + type: 'object', + properties: { + ...elementIdProperty, + }, + required: ['elementId'], + } + }, + + handle: async (context, params) => { + const locator = elementIdLocator(context.page, params!); + return runAndCaptureSnapshot(context, () => locator.hover()); + } +}; + +export const type: Tool = { + schema: { + name: 'type', + description: 'Type text into editable element', + inputSchema: { + type: 'object', + properties: { + ...elementIdProperty, + text: { + type: 'string', + description: 'Text to enter', + }, + submit: { + type: 'boolean', + description: 'Whether to submit entered text (press Enter after)' + } + }, + required: ['elementId', 'text'], + } + }, + + handle: async (context, params) => { + const locator = elementIdLocator(context.page, params!); + return await runAndCaptureSnapshot(context, async () => { + locator.fill(params!.text as string); + if (params!.submit) + await locator.press('Enter'); + }); + } +}; + +function elementIdLocator(page: playwright.Page, params: Record): playwright.Locator { + return page.locator(`internal:aria-id=${params.elementId}`); +} + +async function runAndCaptureSnapshot(context: ToolContext, callback: () => Promise): Promise { + const page = context.page; + await waitForCompletion(page, () => callback()); + return captureAriaSnapshot(page); +} + +async function captureAriaSnapshot(page: playwright.Page): Promise { + const snapshot = await page.locator('html').ariaSnapshot({ _id: true } as any); + return { + content: [{ type: 'text', text: `# Current page snapshot\n${snapshot}` }], + }; +} diff --git a/packages/playwright-tools/src/tools/utils.ts b/packages/playwright-mcp/src/utils.ts similarity index 77% rename from packages/playwright-tools/src/tools/utils.ts rename to packages/playwright-mcp/src/utils.ts index 71c49e4fe19d0..4a89d7466e73a 100644 --- a/packages/playwright-tools/src/tools/utils.ts +++ b/packages/playwright-mcp/src/utils.ts @@ -1,12 +1,11 @@ /** - * Copyright 2017 Google Inc. All rights reserved. - * Modifications copyright (c) Microsoft Corporation. + * Copyright (c) Microsoft Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,20 +14,19 @@ * limitations under the License. */ -import { ManualPromise } from 'playwright-core/lib/utils'; +import type * as playwright from 'playwright'; -import type playwright from 'playwright'; - -export async function waitForNetwork(page: playwright.Page, callback: () => Promise): Promise { +export async function waitForCompletion(page: playwright.Page, callback: () => Promise): Promise { const requests = new Set(); let frameNavigated = false; - const waitBarrier = new ManualPromise(); + let waitCallback: () => void = () => {}; + const waitBarrier = new Promise(f => { waitCallback = f; }); const requestListener = (request: playwright.Request) => requests.add(request); const requestFinishedListener = (request: playwright.Request) => { requests.delete(request); if (!requests.size) - waitBarrier.resolve(); + waitCallback(); }; const frameNavigateListener = (frame: playwright.Frame) => { @@ -38,13 +36,13 @@ export async function waitForNetwork(page: playwright.Page, callback: () => P dispose(); clearTimeout(timeout); void frame.waitForLoadState('load').then(() => { - waitBarrier.resolve(); + waitCallback(); }); }; const onTimeout = () => { dispose(); - waitBarrier.resolve(); + waitCallback(); }; page.on('request', requestListener); @@ -62,7 +60,7 @@ export async function waitForNetwork(page: playwright.Page, callback: () => P try { const result = await callback(); if (!requests.size && !frameNavigated) - waitBarrier.resolve(); + waitCallback(); await waitBarrier; await page.evaluate(() => new Promise(f => setTimeout(f, 1000))); return result; diff --git a/packages/playwright-tools/browser.d.ts b/packages/playwright-tools/browser.d.ts deleted file mode 100644 index e06589e947538..0000000000000 --- a/packages/playwright-tools/browser.d.ts +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import type playwright from 'playwright'; -import { ToolDeclaration, JSONSchemaType } from './types'; - -export type ToolResult = { - error?: string; - code: Array; - snapshot: string; -} - -export type ToolCall = (page: playwright.Page, tool: string, parameters: { [key: string]: JSONSchemaType; }) => Promise; - -export const schema: ToolDeclaration[]; -export const call: ToolCall; -export const snapshot: (page) => Promise; diff --git a/packages/playwright-tools/computer-20241022.d.ts b/packages/playwright-tools/computer-20241022.d.ts deleted file mode 100644 index ce0f821d6d380..0000000000000 --- a/packages/playwright-tools/computer-20241022.d.ts +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import type playwright from 'playwright'; -import { JSONSchemaType } from './types'; - -export type ToolResult = { - output?: string; - error?: string; - base64_image?: string; -}; - -export type ToolCall = (page: playwright.Page, tool: string, parameters: { [key: string]: JSONSchemaType; }) => Promise; - -export const call: ToolCall; diff --git a/packages/playwright-tools/computer-20241022.js b/packages/playwright-tools/computer-20241022.js deleted file mode 100644 index 030d0e57e2918..0000000000000 --- a/packages/playwright-tools/computer-20241022.js +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const { call } = require('./lib/tools/computer-20241022'); - -module.exports = { call }; diff --git a/packages/playwright-tools/package.json b/packages/playwright-tools/package.json deleted file mode 100644 index 2140066a65670..0000000000000 --- a/packages/playwright-tools/package.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "name": "@playwright/experimental-tools", - "private": true, - "version": "0.0.0", - "description": "Playwright Tools for AI", - "repository": { - "type": "git", - "url": "git+https://github.com/microsoft/playwright.git" - }, - "homepage": "https://playwright.dev", - "engines": { - "node": ">=18" - }, - "author": { - "name": "Microsoft Corporation" - }, - "license": "Apache-2.0", - "exports": { - "./browser": { - "types": "./browser.d.ts", - "default": "./browser.js" - }, - "./computer-20241022": { - "types": "./computer-20241022.d.ts", - "default": "./computer-20241022.js" - }, - "./package.json": "./package.json" - }, - "dependencies": { - "playwright": "1.52.0-next" - }, - "devDependencies": { - "@anthropic-ai/sdk": "^0.33.1", - "@modelcontextprotocol/sdk": "^1.6.1", - "openai": "^4.79.1" - } -} diff --git a/packages/playwright-tools/src/examples/browser-anthropic.ts b/packages/playwright-tools/src/examples/browser-anthropic.ts deleted file mode 100644 index d7dd750a188b1..0000000000000 --- a/packages/playwright-tools/src/examples/browser-anthropic.ts +++ /dev/null @@ -1,143 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* eslint-disable no-console */ - -import Anthropic from '@anthropic-ai/sdk'; -import browser from '@playwright/experimental-tools/browser'; -import dotenv from 'dotenv'; -import playwright from 'playwright'; - -dotenv.config(); - -const anthropic = new Anthropic(); - -export const system = ` -You are a web tester. - - -- Perform test according to the provided checklist -- Use browser tools to perform actions on web page -- Never ask questions, always perform a best guess action -- Use one tool at a time, wait for its result before proceeding. -- When ready use "reportResult" tool to report result -`; - -const reportTool: Anthropic.Tool = { - name: 'reportResult', - description: 'Submit test result', - input_schema: { - type: 'object', - properties: { - 'success': { type: 'boolean', description: 'Whether test passed' }, - 'result': { type: 'string', description: 'Result of the test if some information has been requested' }, - 'error': { type: 'string', description: 'Error message if test failed' } - }, - required: ['success'] - } -}; - -type Message = Anthropic.Beta.Messages.BetaMessageParam & { - history: Anthropic.Beta.Messages.BetaMessageParam['content'] -}; - -async function anthropicAgentLoop(page: playwright.Page, task: string) { - // Convert them into tools for Anthropic. - const pageTools: Anthropic.Tool[] = browser.schema.map(tool => { - return { - name: tool.name, - description: tool.description, - input_schema: tool.parameters as any, - }; - }); - - // Add report tool. - const tools = [reportTool, ...pageTools]; - - const history: Message[] = [{ - role: 'user', - history: `Task: ${task}`, - content: `Task: ${task}\n\n${await browser.snapshot(page)}`, - }]; - - // Run agentic loop, cap steps. - for (let i = 0; i < 50; i++) { - const response = await anthropic.messages.create({ - model: 'claude-3-5-sonnet-20241022', - max_tokens: 1024, - temperature: 0, - tools, - system, - messages: toAnthropicMessages(history), - }); - history.push({ role: 'assistant', content: response.content, history: response.content }); - - const toolUse = response.content.find(block => block.type === 'tool_use'); - if (!toolUse) { - history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' }); - continue; - } - - if (toolUse.name === 'reportResult') { - console.log(toolUse.input); - return; - } - - // Run the Playwright tool. - const { error, snapshot, code } = await browser.call(page, toolUse.name, toolUse.input as any); - if (code.length) - console.log(code.join('\n')); - - // Report the result. - const resultText = error ? `Error: ${error}\n` : 'Done\n'; - history.push({ - role: 'user', - content: [{ - type: 'tool_result', - tool_use_id: toolUse.id, - content: [{ type: 'text', text: resultText + snapshot }], - }], - history: [{ - type: 'tool_result', - tool_use_id: toolUse.id, - content: [{ type: 'text', text: resultText }], - }], - }); - } -} - -function toAnthropicMessages(messages: Message[]): Anthropic.Beta.Messages.BetaMessageParam[] { - return messages.map((message, i) => { - if (i === messages.length - 1) - return { ...message, history: undefined }; - return { ...message, content: message.history, history: undefined }; - }); -} - -async function main() { - const browser = await playwright.chromium.launch({ headless: false }); - const page = await browser.newPage(); - await anthropicAgentLoop(page, ` - - Go to http://github.com/microsoft - - Search for "playwright" repository - - Navigate to it - - Switch into the Issues tab - - Report 3 first issues - `); - await browser.close(); -} - -void main(); diff --git a/packages/playwright-tools/src/examples/browser-openai.ts b/packages/playwright-tools/src/examples/browser-openai.ts deleted file mode 100644 index c3f1752d78544..0000000000000 --- a/packages/playwright-tools/src/examples/browser-openai.ts +++ /dev/null @@ -1,157 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* eslint-disable no-console */ - -import browser from '@playwright/experimental-tools/browser'; -import dotenv from 'dotenv'; -import OpenAI from 'openai'; -import playwright from 'playwright'; - -import type { ChatCompletionMessageParam, ChatCompletionTool } from 'openai/resources'; - -dotenv.config(); - -const openai = new OpenAI(); - -export const system = ` -You are a web tester. - -to -- Perform test according to the provided checklist -- Use browser tools to perform actions on web page -- Never ask questions, always perform a best guess action -- When ready use "reportResult" tool to report result -- You can only make one tool call at a time. -`; - -type Message = ChatCompletionMessageParam & { - history: any -}; - -const reportTool: ChatCompletionTool = { - type: 'function', - function: { - name: 'reportResult', - description: 'Submit test result', - parameters: { - type: 'object', - properties: { - success: { type: 'boolean', description: 'Whether test passed' }, - result: { type: 'string', description: 'Result of the test if requested' }, - error: { type: 'string', description: 'Error if test failed' }, - }, - required: ['success'], - additionalProperties: false, - }, - } -}; - -async function openAIAgentLoop(page: playwright.Page, task: string) { - const pageTools: ChatCompletionTool[] = browser.schema.map(tool => ({ - type: 'function', - function: { - name: tool.name, - description: tool.description, - parameters: { - ...tool.parameters, - additionalProperties: false, - }, - } - })); - - const tools = [reportTool, ...pageTools]; - - const history: Message[] = [ - { - role: 'system', content: system, history: system - }, - { - role: 'user', - history: `Task: ${task}`, - content: `Task: ${task}\n\n${await browser.snapshot(page)}`, - } - ]; - - // Run agentic loop, cap steps. - for (let i = 0; i < 50; i++) { - const completion = await openai.chat.completions.create({ - model: 'gpt-4o', - messages: toOpenAIMessages(history), - tools, - tool_choice: 'required', - store: true, - }); - - const toolCalls = completion.choices[0]?.message?.tool_calls; - if (!toolCalls || toolCalls.length !== 1 || toolCalls[0].type !== 'function') { - history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' }); - continue; - } - - const toolCall = toolCalls[0]; - if (toolCall.function.name === 'reportResult') { - console.log(JSON.parse(toolCall.function.arguments)); - return; - } - - history.push({ ...completion.choices[0].message, history: null }); - - // Run the Playwright tool. - const params = JSON.parse(toolCall.function.arguments); - const { error, snapshot, code } = await browser.call(page, toolCall.function.name, params); - if (code.length) - console.log(code.join('\n')); - - if (toolCall.function.name === 'log') - return; - - // Report the result. - const resultText = error ? `Error: ${error}\n` : 'Done\n'; - history.push({ - role: 'tool', - tool_call_id: toolCall.id, - content: resultText + snapshot, - history: resultText, - }); - } -} - -function toOpenAIMessages(messages: Message[]): ChatCompletionMessageParam[] { - return messages.map((message, i) => { - const copy: Message = { ...message }; - delete copy.history; - if (i === messages.length - 1) - return copy; - copy.content = message.history; - return copy; - }); -} - -async function main() { - const browser = await playwright.chromium.launch({ headless: false }); - const page = await browser.newPage(); - await openAIAgentLoop(page, ` - - Go to http://github.com/microsoft - - Search for "playwright" repository - - Navigate to it - - Switch into the Issues tab - - Report 3 first issues - `); - await browser.close(); -} - -void main(); diff --git a/packages/playwright-tools/src/examples/computer-20241022-anthropic.ts b/packages/playwright-tools/src/examples/computer-20241022-anthropic.ts deleted file mode 100644 index 2d8ee4b7f98f1..0000000000000 --- a/packages/playwright-tools/src/examples/computer-20241022-anthropic.ts +++ /dev/null @@ -1,150 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* eslint-disable no-console */ - -import Anthropic from '@anthropic-ai/sdk'; -import computer from '@playwright/experimental-tools/computer-20241022'; -import dotenv from 'dotenv'; -import playwright from 'playwright'; - -import type { BetaImageBlockParam, BetaTextBlockParam } from '@anthropic-ai/sdk/resources/beta/messages/messages'; -import type { ToolResult } from '@playwright/experimental-tools/computer-20241022'; - -dotenv.config(); - -const anthropic = new Anthropic(); - -export const system = ` -You are a web tester. - - -- Perform test according to the provided checklist -- Use browser tools to perform actions on web page -- Never ask questions, always perform a best guess action -- Use one tool at a time, wait for its result before proceeding. -- When ready use "reportResult" tool to report result -`; - -const computerTool: Anthropic.Beta.BetaToolUnion = { - type: 'computer_20241022', - name: 'computer', - display_width_px: 1920, - display_height_px: 1080, - display_number: 1, -}; - -const reportTool: Anthropic.Tool = { - name: 'reportResult', - description: 'Submit test result', - input_schema: { - type: 'object', - properties: { - 'success': { type: 'boolean', description: 'Whether test passed' }, - 'result': { type: 'string', description: 'Result of the test if some information has been requested' }, - 'error': { type: 'string', description: 'Error message if test failed' } - }, - required: ['success'] - } -}; - -type Message = Anthropic.Beta.Messages.BetaMessageParam & { - history: Anthropic.Beta.Messages.BetaMessageParam['content'] -}; - -async function anthropicAgentLoop(page: playwright.Page, task: string) { - // Add report tool. - const tools = [reportTool, computerTool]; - - const history: Message[] = [{ - role: 'user', - history: `Task: ${task}`, - content: `Task: ${task}`, - }]; - - // Run agentic loop, cap steps. - for (let i = 0; i < 50; i++) { - const response = await anthropic.beta.messages.create({ - model: 'claude-3-5-sonnet-20241022', - max_tokens: 1024, - temperature: 0, - tools, - system, - messages: toAnthropicMessages(history), - betas: ['computer-use-2024-10-22'], - }); - - history.push({ role: 'assistant', content: response.content, history: response.content }); - - const toolUse = response.content.find(block => block.type === 'tool_use'); - if (!toolUse) { - history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' }); - continue; - } - - if (toolUse.name === 'reportResult') { - console.log(toolUse.input); - return; - } - - const result: ToolResult = await computer.call(page, toolUse.name, toolUse.input as any); - const contentEntry: BetaTextBlockParam | BetaImageBlockParam = result.base64_image ? { - type: 'image', - source: { type: 'base64', media_type: 'image/jpeg', data: result.base64_image } - } : { - type: 'text', - text: result.output || '', - }; - history.push({ - role: 'user', - content: [{ - type: 'tool_result', - tool_use_id: toolUse.id, - content: [contentEntry], - }], - history: [{ - type: 'tool_result', - tool_use_id: toolUse.id, - content: [{ type: 'text', text: '' }], - }], - }); - } -} - -function toAnthropicMessages(messages: Message[]): Anthropic.Beta.Messages.BetaMessageParam[] { - return messages.map((message, i) => { - if (i === messages.length - 1) - return { ...message, history: undefined }; - return { ...message, content: message.history, history: undefined }; - }); -} - -const githubTask = ` - - Search for "playwright" repository - - Navigate to it - - Switch into the Issues tab - - Report 3 first issues -`; - -async function main() { - const browser = await playwright.chromium.launch({ headless: false }); - const page = await browser.newPage(); - await page.goto('http://github.com/microsoft'); - await anthropicAgentLoop(page, githubTask); - await browser.close(); -} - -void main(); diff --git a/packages/playwright-tools/src/examples/mcp.ts b/packages/playwright-tools/src/examples/mcp.ts deleted file mode 100644 index e5e39e1c229a0..0000000000000 --- a/packages/playwright-tools/src/examples/mcp.ts +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Server } from '@modelcontextprotocol/sdk/server/index.js'; -import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; -import { - CallToolRequestSchema, - ListToolsRequestSchema, -} from '@modelcontextprotocol/sdk/types.js'; -import * as playwright from 'playwright'; -import browser from '@playwright/experimental-tools/browser'; - -const server = new Server( - { - name: 'MCP Server for Playwright', - version: '0.0.1', - }, - { - capabilities: { - tools: {}, - }, - } -); - -server.setRequestHandler(ListToolsRequestSchema, async () => { - return { - tools: browser.schema.map(tool => ({ - name: tool.name, - description: tool.description, - inputSchema: tool.parameters, - })), - }; -}); - -async function createBrowser(): Promise { - if (process.env.PLAYWRIGHT_WS_ENDPOINT) { - return await playwright.chromium.connect( - process.env.PLAYWRIGHT_WS_ENDPOINT - ); - } - return await playwright.chromium.launch({ headless: false }); -} - -async function getPage(): Promise { - if (!page) { - const browser = await createBrowser(); - const context = await browser.newContext(); - page = await context.newPage(); - } - return page; -} - -let page: playwright.Page | undefined; - -async function main() { - server.setRequestHandler(CallToolRequestSchema, async request => { - const page = await getPage(); - const response = await browser.call( - page, - request.params.name, - request.params.arguments as any - ); - const content: { type: string; text: string }[] = []; - if (response.error) - content.push({ type: 'text', text: response.error }); - if (response.snapshot) - content.push({ type: 'text', text: response.snapshot }); - return { - content, - isError: response.error ? true : false, - }; - }); - - process.stdin.on('close', async () => { - server.close(); - // eslint-disable-next-line no-restricted-properties - setTimeout(() => process.exit(0), 15000); - await page?.context()?.browser()?.close(); - // eslint-disable-next-line no-restricted-properties - process.exit(0); - }); - - await server.connect(new StdioServerTransport()); -} - -void main(); diff --git a/packages/playwright-tools/src/tools/browser.ts b/packages/playwright-tools/src/tools/browser.ts deleted file mode 100644 index 1ca7fb074a755..0000000000000 --- a/packages/playwright-tools/src/tools/browser.ts +++ /dev/null @@ -1,150 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * Modifications copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { waitForNetwork } from './utils'; - -import type { ToolResult } from '../../browser'; -import type { JSONSchemaType, ToolDeclaration } from '../../types'; -import type playwright from 'playwright'; - - -type LocatorEx = playwright.Locator & { - _generateLocatorString: () => Promise; -}; - -const intentProperty = { - intent: { - type: 'string', - description: 'Intent behind this particular action. Used as a comment.', - } -}; - -const elementIdProperty = { - elementId: { - type: 'number', - description: 'Target element', - } -}; - -export const schema: ToolDeclaration[] = [ - { - name: 'navigate', - description: 'Navigate to a URL', - parameters: { - type: 'object', - properties: { - ...intentProperty, - url: { - type: 'string', - description: 'URL to navigate to', - }, - }, - required: ['intent', 'elementId'], - } - }, - { - name: 'click', - description: 'Perform click on a web page', - parameters: { - type: 'object', - properties: { - ...intentProperty, - ...elementIdProperty, - }, - required: ['intent', 'elementId'], - } - }, - { - name: 'enterText', - description: 'Enter text into editable element', - parameters: { - type: 'object', - properties: { - ...intentProperty, - ...elementIdProperty, - text: { - type: 'string', - description: 'Text to enter', - }, - submit: { - type: 'boolean', - description: 'Whether to submit entered text (press Enter after)' - } - }, - required: ['intent', 'elementId', 'text'], - } - }, - { - name: 'wait', - description: `Wait for given amount of time to see if the page updates. Use it after action if you think page is not ready yet`, - parameters: { - type: 'object', - properties: { - ...intentProperty, - time: { - type: 'integer', - description: 'Time to wait in seconds', - }, - }, - required: ['intent', 'time'], - } - }, -]; - -export async function call(page: playwright.Page, toolName: string, params: Record): Promise { - const code: string[] = []; - try { - await waitForNetwork(page, async () => { - await performAction(page, toolName, params, code); - }); - } catch (e) { - return { error: e.message, snapshot: await snapshot(page), code }; - } - return { snapshot: await snapshot(page), code }; -} - -export async function snapshot(page: playwright.Page) { - const params = { _id: true } as any; - return `\n${await page.locator('body').ariaSnapshot(params)}\n`; -} - -async function performAction(page: playwright.Page, toolName: string, params: Record, code: string[]) { - const locator = elementLocator(page, params); - code.push((params.intent as string).split('\n').map(line => `// ${line}`).join('\n')); - if (toolName === 'navigate') { - code.push(`await page.goto(${JSON.stringify(params.url)})`); - await page.goto(params.url as string); - } else if (toolName === 'wait') { - await page.waitForTimeout(Math.min(10000, params.time as number * 1000)); - } else if (toolName === 'click') { - code.push(`await page.${await locator._generateLocatorString()}.click()`); - await locator.click(); - } else if (toolName === 'enterText') { - code.push(`await page.${await locator._generateLocatorString()}.click()`); - await locator.click(); - code.push(`await page.${await locator._generateLocatorString()}.fill(${JSON.stringify(params.text)})`); - await locator.fill(params.text as string); - if (params.submit) { - code.push(`await page.${await locator._generateLocatorString()}.press("Enter")`); - await locator.press('Enter'); - } - } -} - -function elementLocator(page: playwright.Page, params: any): LocatorEx { - return page.locator(`internal:aria-id=${params.elementId}`) as LocatorEx; -} diff --git a/packages/playwright-tools/src/tools/computer-20241022.ts b/packages/playwright-tools/src/tools/computer-20241022.ts deleted file mode 100644 index 2a7574c485aaf..0000000000000 --- a/packages/playwright-tools/src/tools/computer-20241022.ts +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * Modifications copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the 'License'); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an 'AS IS' BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { waitForNetwork } from './utils'; - -import type { ToolResult } from '../../computer-20241022'; -import type { JSONSchemaType } from '../../types'; -import type playwright from 'playwright'; - - -export async function call(page: playwright.Page, toolName: string, input: Record): Promise { - if (toolName !== 'computer') - throw new Error('Unsupported tool'); - return await waitForNetwork(page, async () => { - return await performAction(page, toolName, input); - }); -} - -type PageState = { - x: number; - y: number; -}; - -const pageStateSymbol = Symbol('pageState'); - -function pageState(page: playwright.Page): PageState { - if (!(page as any)[pageStateSymbol]) - (page as any)[pageStateSymbol] = { x: 0, y: 0 }; - return (page as any)[pageStateSymbol]; -} - -async function performAction(page: playwright.Page, toolName: string, input: Record): Promise { - const state = pageState(page); - const { action } = input as { action: string }; - if (action === 'screenshot') { - const screenshot = await page.screenshot({ type: 'jpeg', quality: 50, scale: 'css' }); - return { - output: 'Screenshot', - base64_image: screenshot.toString('base64'), - }; - } - if (action === 'mouse_move') { - const { coordinate } = input as { coordinate: [number, number] }; - state.x = coordinate[0]; - state.y = coordinate[1]; - await page.mouse.move(state.x, state.y); - return { output: 'Mouse moved' }; - } - if (action === 'left_click') { - await page.mouse.down(); - await page.mouse.up(); - return { output: 'Left clicked' }; - } - if (action === 'left_click_drag') { - await page.mouse.down(); - const { coordinate } = input as { coordinate: [number, number] }; - state.x = coordinate[0]; - state.y = coordinate[1]; - await page.mouse.move(state.x, state.y); - await page.mouse.up(); - return { output: 'Left dragged' }; - } - if (action === 'right_click') { - await page.mouse.down({ button: 'right' }); - await page.mouse.up({ button: 'right' }); - return { output: 'Right clicked' }; - } - if (action === 'double_click') { - await page.mouse.down(); - await page.mouse.up(); - await page.mouse.down(); - await page.mouse.up(); - return { output: 'Double clicked' }; - } - if (action === 'middle_click') { - await page.mouse.down({ button: 'middle' }); - await page.mouse.up({ button: 'middle' }); - return { output: 'Middle clicked' }; - } - if (action === 'key') { - const { text } = input as { text: string }; - await page.keyboard.press(xToPlaywright(text)); - return { output: 'Text typed' }; - } - if (action === 'cursor_position') - return { output: `X=${state.x},Y=${state.y}` }; - throw new Error('Unimplemented tool: ' + toolName); -} - -const xToPlaywrightKeyMap = new Map([ - ['BackSpace', 'Backspace'], - ['Tab', 'Tab'], - ['Return', 'Enter'], - ['Escape', 'Escape'], - ['space', ' '], - ['Delete', 'Delete'], - ['Home', 'Home'], - ['End', 'End'], - ['Left', 'ArrowLeft'], - ['Up', 'ArrowUp'], - ['Right', 'ArrowRight'], - ['Down', 'ArrowDown'], - ['Insert', 'Insert'], - ['Page_Up', 'PageUp'], - ['Page_Down', 'PageDown'], - ['F1', 'F1'], - ['F2', 'F2'], - ['F3', 'F3'], - ['F4', 'F4'], - ['F5', 'F5'], - ['F6', 'F6'], - ['F7', 'F7'], - ['F8', 'F8'], - ['F9', 'F9'], - ['F10', 'F10'], - ['F11', 'F11'], - ['F12', 'F12'], - ['Shift_L', 'Shift'], - ['Shift_R', 'Shift'], - ['Control_L', 'Control'], - ['Control_R', 'Control'], - ['Alt_L', 'Alt'], - ['Alt_R', 'Alt'], - ['Super_L', 'Meta'], - ['Super_R', 'Meta'], -]); - -const xToPlaywrightModifierMap = new Map([ - ['alt', 'Alt'], - ['control', 'Control'], - ['meta', 'Meta'], - ['shift', 'Shift'], -]); - - -const xToPlaywright = (key: string) => { - const tokens = key.split('+'); - if (tokens.length === 1) - return xToPlaywrightKeyMap.get(key) || key; - if (tokens.length === 2) { - const modifier = xToPlaywrightModifierMap.get(tokens[0]); - const key = xToPlaywrightKeyMap.get(tokens[1]) || tokens[1]; - return modifier + '+' + key; - } - throw new Error('Invalid key: ' + key); -}; diff --git a/utils/workspace.js b/utils/workspace.js index ef03a1fc107af..cba0021c17e83 100755 --- a/utils/workspace.js +++ b/utils/workspace.js @@ -174,7 +174,7 @@ const workspace = new Workspace(ROOT_PATH, [ }), new PWPackage({ name: '@playwright/experimental-tools', - path: path.join(ROOT_PATH, 'packages', 'playwright-tools'), + path: path.join(ROOT_PATH, 'packages', 'playwright-mcp'), files: LICENCE_FILES, }), new PWPackage({