From 616a015251bb30e29b98bb59d429e735f8702547 Mon Sep 17 00:00:00 2001 From: chenhaoli Date: Wed, 22 Jan 2025 02:32:59 +0800 Subject: [PATCH 1/5] feat(renderer): share provider --- src/main/store/types.ts | 2 + src/main/window/index.ts | 2 +- .../src/components/ChatInput/index.tsx | 88 ++++++++++++++----- src/renderer/src/pages/settings/index.tsx | 12 ++- src/renderer/src/utils/share.ts | 25 ++++++ 5 files changed, 104 insertions(+), 25 deletions(-) create mode 100644 src/renderer/src/utils/share.ts diff --git a/src/main/store/types.ts b/src/main/store/types.ts index a063a95..f3080c8 100644 --- a/src/main/store/types.ts +++ b/src/main/store/types.ts @@ -63,4 +63,6 @@ export type LocalStore = { vlmApiKey: string; vlmModelName: string; screenshotScale: number; // 0.1 ~ 1.0 + // Add Share Provider Configuration + shareEndpoint?: string; }; diff --git a/src/main/window/index.ts b/src/main/window/index.ts index 7faca0f..aaa8de9 100644 --- a/src/main/window/index.ts +++ b/src/main/window/index.ts @@ -94,7 +94,7 @@ export function createSettingsWindow( console.log('mainWindowBounds', mainWindowBounds); const width = 480; - const height = 600; + const height = 700; let x, y; if (mainWindowBounds) { diff --git a/src/renderer/src/components/ChatInput/index.tsx b/src/renderer/src/components/ChatInput/index.tsx index 491ac1a..e9c4882 100644 --- a/src/renderer/src/components/ChatInput/index.tsx +++ b/src/renderer/src/components/ChatInput/index.tsx @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ import { Box, Button, Flex, HStack, Spinner, VStack } from '@chakra-ui/react'; +import { useToast } from '@chakra-ui/react'; import React, { forwardRef, useEffect, useRef } from 'react'; import { FaPaperPlane, FaStop, FaTrash } from 'react-icons/fa'; import { LuScreenShare } from 'react-icons/lu'; @@ -15,6 +16,7 @@ import { ComputerUseUserData } from '@ui-tars/shared/types/data'; import { useRunAgent } from '@renderer/hooks/useRunAgent'; import { useStore } from '@renderer/hooks/useStore'; import { reportHTMLContent } from '@renderer/utils/html'; +import { uploadAndShare } from '@renderer/utils/share'; import reportHTMLUrl from '@resources/report.html?url'; @@ -24,11 +26,13 @@ const ChatInput = forwardRef((_props, _ref) => { instructions: savedInstructions, messages, restUserData, + settings, } = useStore(); const [localInstructions, setLocalInstructions] = React.useState( savedInstructions ?? '', ); + const toast = useToast(); const { run } = useRunAgent(); const textareaRef = useRef(null); @@ -75,33 +79,71 @@ const ChatInput = forwardRef((_props, _ref) => { ?.value || ''; const handleShare = async () => { - const response = await fetch(reportHTMLUrl); - const html = await response.text(); + try { + const response = await fetch(reportHTMLUrl); + const html = await response.text(); - const userData = { - ...restUserData, - status, - conversations: messages, - } as ComputerUseUserData; + const userData = { + ...restUserData, + status, + conversations: messages, + } as ComputerUseUserData; - const htmlContent = reportHTMLContent(html, [userData]); + const htmlContent = reportHTMLContent(html, [userData]); - // create Blob object - const blob = new Blob([htmlContent], { type: 'text/html' }); + if (settings?.shareEndpoint) { + try { + const { url } = await uploadAndShare( + htmlContent, + settings.shareEndpoint, + ); + // Copy link to clipboard + await navigator.clipboard.writeText(url); + toast({ + title: 'Share link copied to clipboard!', + status: 'success', + position: 'top', + duration: 2000, + isClosable: true, + variant: 'ui-tars-success', + }); + return; + } catch (error) { + console.error('Share failed:', error); + toast({ + title: 'Failed to share', + description: + error instanceof Error ? error.message : JSON.stringify(error), + status: 'error', + position: 'top', + duration: 3000, + isClosable: true, + }); + } + } - // create download link - const url = window.URL.createObjectURL(blob); - const a = document.createElement('a'); - a.href = url; - a.download = `report-${Date.now()}.html`; - - // trigger download - document.body.appendChild(a); - a.click(); - - // clean up - document.body.removeChild(a); - window.URL.revokeObjectURL(url); + // If shareEndpoint is not configured or the upload fails, fall back to downloading the file + const blob = new Blob([htmlContent], { type: 'text/html' }); + const url = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `report-${Date.now()}.html`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + window.URL.revokeObjectURL(url); + } catch (error) { + console.error('Share failed:', error); + toast({ + title: 'Failed to generate share content', + description: + error instanceof Error ? error.message : JSON.stringify(error), + status: 'error', + position: 'top', + duration: 3000, + isClosable: true, + }); + } }; const handleClearMessages = () => { diff --git a/src/renderer/src/pages/settings/index.tsx b/src/renderer/src/pages/settings/index.tsx index eb1662e..b1246d6 100644 --- a/src/renderer/src/pages/settings/index.tsx +++ b/src/renderer/src/pages/settings/index.tsx @@ -201,6 +201,16 @@ const Settings = () => { /> + + Share Provider Endpoint (Optional) + + + )}
From a0f8c4afb4dcf9558b7b4a4174a170eb8bcec0eb Mon Sep 17 00:00:00 2001 From: chenhaoli Date: Sun, 26 Jan 2025 09:29:19 +0800 Subject: [PATCH 3/5] chore: rename `shareEndpoint` to `reportStorageEndpoint` --- src/main/store/types.ts | 3 +-- src/renderer/src/components/ChatInput/index.tsx | 12 ++++++------ src/renderer/src/pages/settings/index.tsx | 14 ++++++++------ src/renderer/src/utils/share.ts | 6 +++--- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/main/store/types.ts b/src/main/store/types.ts index f3080c8..0850a2a 100644 --- a/src/main/store/types.ts +++ b/src/main/store/types.ts @@ -63,6 +63,5 @@ export type LocalStore = { vlmApiKey: string; vlmModelName: string; screenshotScale: number; // 0.1 ~ 1.0 - // Add Share Provider Configuration - shareEndpoint?: string; + reportStorageEndpoint?: string; }; diff --git a/src/renderer/src/components/ChatInput/index.tsx b/src/renderer/src/components/ChatInput/index.tsx index 533e8d8..87746b3 100644 --- a/src/renderer/src/components/ChatInput/index.tsx +++ b/src/renderer/src/components/ChatInput/index.tsx @@ -16,7 +16,7 @@ import { ComputerUseUserData } from '@ui-tars/shared/types/data'; import { useRunAgent } from '@renderer/hooks/useRunAgent'; import { useStore } from '@renderer/hooks/useStore'; import { reportHTMLContent } from '@renderer/utils/html'; -import { uploadAndShare } from '@renderer/utils/share'; +import { uploadAndShare, uploadReport } from '@renderer/utils/share'; import reportHTMLUrl from '@resources/report.html?url'; @@ -116,16 +116,16 @@ const ChatInput = forwardRef((_props, _ref) => { const htmlContent = reportHTMLContent(html, [userData]); - if (settings?.shareEndpoint) { + if (settings?.reportStorageEndpoint) { try { - const { url } = await uploadAndShare( + const { url } = await uploadReport( htmlContent, - settings.shareEndpoint, + settings.reportStorageEndpoint, ); // Copy link to clipboard await navigator.clipboard.writeText(url); toast({ - title: 'Share link copied to clipboard!', + title: 'Report link copied to clipboard!', status: 'success', position: 'top', duration: 2000, @@ -136,7 +136,7 @@ const ChatInput = forwardRef((_props, _ref) => { } catch (error) { console.error('Share failed:', error); toast({ - title: 'Failed to share', + title: 'Failed to upload report', description: error instanceof Error ? error.message : JSON.stringify(error), status: 'error', diff --git a/src/renderer/src/pages/settings/index.tsx b/src/renderer/src/pages/settings/index.tsx index b1246d6..8c84c4c 100644 --- a/src/renderer/src/pages/settings/index.tsx +++ b/src/renderer/src/pages/settings/index.tsx @@ -202,12 +202,14 @@ const Settings = () => { - Share Provider Endpoint (Optional) + + Report Storage Provider Endpoint (Optional) + @@ -247,4 +249,4 @@ const Settings = () => { export default Settings; -export { Settings as Component }; \ No newline at end of file +export { Settings as Component }; diff --git a/src/renderer/src/utils/share.ts b/src/renderer/src/utils/share.ts index da276cb..a4ca6a4 100644 --- a/src/renderer/src/utils/share.ts +++ b/src/renderer/src/utils/share.ts @@ -1,4 +1,4 @@ -export async function uploadAndShare( +export async function uploadReport( htmlContent: string, endpoint: string, ): Promise<{ url: string }> { @@ -19,7 +19,7 @@ export async function uploadAndShare( return await response.json(); } catch (error) { throw new Error( - `Failed to upload file: ${error instanceof Error ? error.message : JSON.stringify(error)}`, + `Failed to upload report: ${error instanceof Error ? error.message : JSON.stringify(error)}`, ); } -} +} \ No newline at end of file From 9129beee8258f6eb982364970580439f6933f9bb Mon Sep 17 00:00:00 2001 From: chenhaoli Date: Sun, 26 Jan 2025 15:39:32 +0800 Subject: [PATCH 4/5] feat: implement UTIO (UI-TARS Insights and Observation) --- package.json | 1 + packages/utio/package.json | 30 ++++++ packages/utio/src/index.ts | 27 ++++++ packages/utio/src/types.ts | 41 ++++++++ packages/utio/tsconfig.json | 19 ++++ packages/utio/tsup.config.ts | 16 ++++ pnpm-lock.yaml | 85 ++++++++--------- src/main/agent/index.ts | 6 ++ src/main/main.ts | 15 +++ src/main/services/utio.ts | 95 +++++++++++++++++++ src/main/store/types.ts | 1 + src/main/window/index.ts | 2 +- src/preload/index.d.ts | 8 -- src/preload/index.ts | 6 ++ .../src/components/ChatInput/index.tsx | 21 +++- src/renderer/src/pages/settings/index.tsx | 12 ++- src/renderer/src/typings/preload.d.ts | 2 +- 17 files changed, 329 insertions(+), 58 deletions(-) create mode 100644 packages/utio/package.json create mode 100644 packages/utio/src/index.ts create mode 100644 packages/utio/src/types.ts create mode 100644 packages/utio/tsconfig.json create mode 100644 packages/utio/tsup.config.ts create mode 100644 src/main/services/utio.ts delete mode 100644 src/preload/index.d.ts diff --git a/package.json b/package.json index b065ef7..3e09b7b 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "@electron-toolkit/utils": "^3.0.0", "@ui-tars/action-parser": "workspace:*", "@ui-tars/shared": "workspace:*", + "@ui-tars/utio": "workspace:*", "async-retry": "^1.3.3", "big.js": "^6.2.2", "dotenv": "^16.4.7", diff --git a/packages/utio/package.json b/packages/utio/package.json new file mode 100644 index 0000000..5b8d825 --- /dev/null +++ b/packages/utio/package.json @@ -0,0 +1,30 @@ +{ + "name": "@ui-tars/utio", + "version": "1.0.0", + "description": "UTIO (UI-TARS Insights and Observation)", + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "scripts": { + "prepare": "npm run build", + "dev": "tsup --watch", + "build": "tsup", + "prepack": "npm run build" + }, + "keywords": [ + "UI-TARS" + ], + "license": "Apache-2.0", + "publishConfig": { + "registry": "https://registry.npmjs.org/" + }, + "files": [ + "dist", + "src" + ], + "dependencies": {}, + "devDependencies": { + "tsup": "^8.3.5", + "typescript": "^5.7.2" + } +} diff --git a/packages/utio/src/index.ts b/packages/utio/src/index.ts new file mode 100644 index 0000000..c52a7f9 --- /dev/null +++ b/packages/utio/src/index.ts @@ -0,0 +1,27 @@ +import { UTIOPayload, UTIOType } from './types'; + +export type { UTIOPayload }; + +export class UTIO { + constructor(private readonly endpoint: string) {} + + async send(data: UTIOPayload): Promise { + if (!this.endpoint) return; + + try { + const response = await fetch(this.endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(data), + }); + + if (!response.ok) { + throw new Error(`UTIO upload failed with status: ${response.status}`); + } + } catch (error) { + // Silent fail + } + } +} diff --git a/packages/utio/src/types.ts b/packages/utio/src/types.ts new file mode 100644 index 0000000..4fb24ef --- /dev/null +++ b/packages/utio/src/types.ts @@ -0,0 +1,41 @@ +/** + * Copyright (c) 2025 Bytedance, Inc. and its affiliates. + * SPDX-License-Identifier: Apache-2.0 + */ + +export type UTIOType = 'appLaunched' | 'sendInstruction' | 'shareReport'; + +export interface UTIOBasePayload { + instruction: string; +} + +export type UTIOPayloadMap = { + /** + * The application is opened + */ + appLaunched: { + type: 'appLaunched'; + platform: string; + osVersion: string; + screenWidth: number; + screenHeight: number; + }; + /** + * User sent instruction + */ + sendInstruction: { + type: 'sendInstruction'; + instruction: string; + }; + /** + * Share report + */ + shareReport: UTIOBasePayload & { + type: 'shareReport'; + lastScreenshot?: string; + report?: string; + instruction: string; + }; +}; + +export type UTIOPayload = UTIOPayloadMap[T]; diff --git a/packages/utio/tsconfig.json b/packages/utio/tsconfig.json new file mode 100644 index 0000000..5f8bd0f --- /dev/null +++ b/packages/utio/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "esnext", + "module": "esnext", + "moduleResolution": "Bundler", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "./dist", + "strict": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "isolatedModules": true, + "skipLibCheck": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/utio/tsup.config.ts b/packages/utio/tsup.config.ts new file mode 100644 index 0000000..b72ad97 --- /dev/null +++ b/packages/utio/tsup.config.ts @@ -0,0 +1,16 @@ +/** + * Copyright (c) 2025 Bytedance, Inc. and its affiliates. + * SPDX-License-Identifier: Apache-2.0 + */ +import { defineConfig } from 'tsup'; + +export default defineConfig((options) => { + return { + entry: ['src/**/*.ts'], + format: ['esm', 'cjs'], + dts: true, + clean: true, + bundle: false, + outDir: 'dist', + }; +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c455c43..7f040a3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,6 +29,9 @@ importers: '@ui-tars/shared': specifier: workspace:* version: link:packages/shared + '@ui-tars/utio': + specifier: workspace:* + version: link:packages/utio async-retry: specifier: ^1.3.3 version: 1.3.3 @@ -302,6 +305,15 @@ importers: specifier: ^5.7.2 version: 5.7.2 + packages/utio: + devDependencies: + tsup: + specifier: ^8.3.5 + version: 8.3.5(jiti@2.4.2)(postcss@8.4.49)(tsx@4.19.2)(typescript@5.7.2)(yaml@2.6.1) + typescript: + specifier: ^5.7.2 + version: 5.7.2 + packages: '@actions/core@1.11.1': @@ -1708,8 +1720,8 @@ packages: resolution: {integrity: sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==} engines: {node: '>=6.0.0'} - '@jridgewell/source-map@0.3.6': - resolution: {integrity: sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ==} + '@jridgewell/source-map@0.3.5': + resolution: {integrity: sha512-UTYAUj/wviwdsMfzoSJspJxbkH5o1snzwX0//0ENX1u/55kkZZkcTZP6u9bwKGkv+dkk9at4m1Cpt0uY80kcpQ==} '@jridgewell/sourcemap-codec@1.5.0': resolution: {integrity: sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==} @@ -4048,9 +4060,8 @@ packages: resolution: {integrity: sha512-6xwYfHbajpoF0xLW+iwLkhwgvLoZDfjYfoFNu8ftMoXINzwuymNLd9u/KmwtdT2GbR+/Cz66otEGEVVUHX9QLQ==} engines: {node: '>=10.13.0'} - ip-address@9.0.5: - resolution: {integrity: sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==} - engines: {node: '>= 12'} + ip@2.0.0: + resolution: {integrity: sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==} ipaddr.js@1.9.1: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} @@ -4315,9 +4326,6 @@ packages: resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} hasBin: true - jsbn@1.1.0: - resolution: {integrity: sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==} - jsdom@20.0.3: resolution: {integrity: sha512-SYhBvTh89tTfCD/CRdSOm13mOBa42iTaTyfyEWBdKcGdPxPtLFBXuHR8XHb33YNYaP+lLbmSvBTsnoesCNJEsQ==} engines: {node: '>=14'} @@ -4479,6 +4487,7 @@ packages: lodash.isequal@4.5.0: resolution: {integrity: sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==} + deprecated: This package is deprecated. Use require('node:util').isDeepStrictEqual instead. lodash.isplainobject@4.0.6: resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} @@ -4782,10 +4791,6 @@ packages: resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} engines: {node: '>= 0.6'} - negotiator@0.6.4: - resolution: {integrity: sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==} - engines: {node: '>= 0.6'} - new-github-issue-url@0.2.1: resolution: {integrity: sha512-md4cGoxuT4T4d/HDOXbrUHkTKrp/vp+m3aOA7XXVYwNsUNMK49g3SQicTSeV5GIz/5QVGAeYRAOlyp9OvlgsYA==} engines: {node: '>=10'} @@ -5046,8 +5051,8 @@ packages: resolution: {integrity: sha512-1Y1A//QUXEZK7YKz+rD9WydcE1+EuPr6ZBgKecAB8tmoW6UFv0NREVJe1p+jRxtThkcbbKkfwIbWJe/IeE6m2Q==} engines: {node: '>=0.10.0'} - parse5@7.2.1: - resolution: {integrity: sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==} + parse5@7.1.2: + resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} parseurl@1.3.3: resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} @@ -5269,8 +5274,8 @@ packages: proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} - psl@1.15.0: - resolution: {integrity: sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==} + psl@1.9.0: + resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} pump@3.0.2: resolution: {integrity: sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==} @@ -5852,9 +5857,9 @@ packages: resolution: {integrity: sha512-Fgl0YPZ902wEsAyiQ+idGd1A7rSFx/ayC1CQVMw5P+EQx2V0SgpGtf6OKFhVjPflPUl9YMmEOnmfjCdMUsygww==} engines: {node: '>= 10'} - socks@2.8.3: - resolution: {integrity: sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==} - engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} + socks@2.7.1: + resolution: {integrity: sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==} + engines: {node: '>= 10.13.0', npm: '>= 3.0.0'} source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} @@ -6174,8 +6179,8 @@ packages: resolution: {integrity: sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ==} engines: {node: '>=10'} - tough-cookie@4.1.4: - resolution: {integrity: sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==} + tough-cookie@4.1.3: + resolution: {integrity: sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==} engines: {node: '>=6'} tr46@0.0.3: @@ -8542,7 +8547,7 @@ snapshots: '@jridgewell/set-array@1.2.1': {} - '@jridgewell/source-map@0.3.6': + '@jridgewell/source-map@0.3.5': dependencies: '@jridgewell/gen-mapping': 0.3.8 '@jridgewell/trace-mapping': 0.3.25 @@ -11411,10 +11416,7 @@ snapshots: interpret@3.1.1: {} - ip-address@9.0.5: - dependencies: - jsbn: 1.1.0 - sprintf-js: 1.1.3 + ip@2.0.0: {} ipaddr.js@1.9.1: {} @@ -11674,8 +11676,6 @@ snapshots: dependencies: argparse: 2.0.1 - jsbn@1.1.0: {} - jsdom@20.0.3: dependencies: abab: 2.0.6 @@ -11693,10 +11693,10 @@ snapshots: https-proxy-agent: 5.0.1 is-potential-custom-element-name: 1.0.1 nwsapi: 2.2.16 - parse5: 7.2.1 + parse5: 7.1.2 saxes: 6.0.0 symbol-tree: 3.2.4 - tough-cookie: 4.1.4 + tough-cookie: 4.1.3 w3c-xmlserializer: 4.0.0 webidl-conversions: 7.0.0 whatwg-encoding: 2.0.0 @@ -11968,7 +11968,7 @@ snapshots: minipass-fetch: 2.1.2 minipass-flush: 1.0.5 minipass-pipeline: 1.2.4 - negotiator: 0.6.4 + negotiator: 0.6.3 promise-retry: 2.0.1 socks-proxy-agent: 7.0.0 ssri: 9.0.1 @@ -12166,8 +12166,6 @@ snapshots: negotiator@0.6.3: {} - negotiator@0.6.4: {} - new-github-issue-url@0.2.1: {} nice-try@1.0.5: {} @@ -12455,7 +12453,7 @@ snapshots: parse-passwd@1.0.0: {} - parse5@7.2.1: + parse5@7.1.2: dependencies: entities: 4.5.0 optional: true @@ -12611,9 +12609,7 @@ snapshots: proxy-from-env@1.1.0: {} - psl@1.15.0: - dependencies: - punycode: 2.3.1 + psl@1.9.0: optional: true pump@3.0.2: @@ -13251,13 +13247,13 @@ snapshots: dependencies: agent-base: 6.0.2 debug: 4.4.0 - socks: 2.8.3 + socks: 2.7.1 transitivePeerDependencies: - supports-color - socks@2.8.3: + socks@2.7.1: dependencies: - ip-address: 9.0.5 + ip: 2.0.0 smart-buffer: 4.2.0 source-map-js@1.2.1: {} @@ -13300,7 +13296,8 @@ snapshots: sprintf-js@1.0.3: {} - sprintf-js@1.1.3: {} + sprintf-js@1.1.3: + optional: true ssri@9.0.1: dependencies: @@ -13496,7 +13493,7 @@ snapshots: terser@5.37.0: dependencies: - '@jridgewell/source-map': 0.3.6 + '@jridgewell/source-map': 0.3.5 acorn: 8.14.0 commander: 2.20.3 source-map-support: 0.5.21 @@ -13583,9 +13580,9 @@ snapshots: '@tokenizer/token': 0.3.0 ieee754: 1.2.1 - tough-cookie@4.1.4: + tough-cookie@4.1.3: dependencies: - psl: 1.15.0 + psl: 1.9.0 punycode: 2.3.1 universalify: 0.2.0 url-parse: 1.5.10 diff --git a/src/main/agent/index.ts b/src/main/agent/index.ts index 6f7e8a3..68e9269 100644 --- a/src/main/agent/index.ts +++ b/src/main/agent/index.ts @@ -15,9 +15,12 @@ import { ScreenshotResult, StatusEnum } from '@ui-tars/shared/types'; import { ComputerUseUserData, Conversation } from '@ui-tars/shared/types/data'; import { ShareVersion } from '@ui-tars/shared/types/share'; import sleep from '@ui-tars/shared/utils/sleep'; +import { UTIO } from '@ui-tars/utio'; import { logger } from '@main/logger'; +import { UTIOService } from '../services/utio'; +import { store } from '../store/create'; import { markClickPosition } from '../utils/image'; import { Desktop } from './device'; import { VLM, VlmRequest } from './llm/base'; @@ -111,6 +114,9 @@ export class ComputerUseAgent { const { config, logger } = this; const { abortController, device, vlm, instruction } = config; + // Send instruction data to UTIO + await UTIOService.getInstance().sendInstruction(instruction); + // init this.mode = VlmModeEnum.Agent; this.conversations = [ diff --git a/src/main/main.ts b/src/main/main.ts index aa70c06..f017cd4 100644 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -17,6 +17,7 @@ import { createSettingsWindow, } from '@main/window/index'; +import { UTIOService } from './services/utio'; import { store } from './store/create'; import { createTray } from './tray'; @@ -92,6 +93,9 @@ const initializeApp = async () => { // Tray await createTray(); + // Send app launched event + await UTIOService.getInstance().appLaunched(); + const launcherWindowIns = LauncherWindow.getInstance(); globalShortcut.register('Alt+T', () => { @@ -130,6 +134,15 @@ const initializeApp = async () => { logger.info('initializeApp end'); }; +/** + * Register IPC handlers + */ +const registerIPCHandlers = () => { + ipcMain.handle('utio:shareReport', async (_, params) => { + await UTIOService.getInstance().shareReport(params); + }); +}; + /** * Add event listeners... */ @@ -156,6 +169,8 @@ app await initializeApp(); + registerIPCHandlers(); + logger.info('app.whenReady end'); }) .catch(console.log); diff --git a/src/main/services/utio.ts b/src/main/services/utio.ts new file mode 100644 index 0000000..bf318ac --- /dev/null +++ b/src/main/services/utio.ts @@ -0,0 +1,95 @@ +import os from 'node:os'; + +import { screen } from 'electron'; + +import { UTIO, UTIOPayload } from '@ui-tars/utio'; + +import { logger } from '../logger'; +import { store } from '../store/create'; + +export class UTIOService { + private static instance: UTIOService; + private utio: UTIO | null = null; + + static getInstance(): UTIOService { + if (!UTIOService.instance) { + UTIOService.instance = new UTIOService(); + } + return UTIOService.instance; + } + + private getEndpoint(): string | undefined { + const endpoint = store.getState().getSetting('utioEndpoint'); + logger.debug('[UTIO] endpoint:', endpoint); + return endpoint; + } + + private ensureUTIO() { + const endpoint = this.getEndpoint(); + if (endpoint && !this.utio) { + this.utio = new UTIO(endpoint); + } + return this.utio; + } + + async appLaunched() { + try { + const utio = this.ensureUTIO(); + if (utio) { + const primaryDisplay = screen.getPrimaryDisplay(); + const { width, height } = primaryDisplay.size; + const payload: UTIOPayload<'appLaunched'> = { + type: 'appLaunched', + platform: process.platform, + osVersion: os.release(), + screenWidth: width, + screenHeight: height, + }; + + logger.debug('[UTIO] payload:', payload); + await utio.send(payload); + } + } catch (error) { + logger.error('[UTIO] error:', error); + throw error; + } + } + + async sendInstruction(instruction: string) { + try { + const utio = this.ensureUTIO(); + if (utio) { + const payload: UTIOPayload<'sendInstruction'> = { + type: 'sendInstruction', + instruction, + }; + logger.debug('[UTIO] payload:', payload); + await utio.send(payload); + } + } catch (error) { + logger.error('[UTIO] error:', error); + throw error; + } + } + + async shareReport(params: { + instruction: string; + lastScreenshot?: string; + report?: string; + }) { + try { + const utio = this.ensureUTIO(); + if (utio) { + const payload: UTIOPayload<'shareReport'> = { + type: 'shareReport', + ...params, + }; + logger.debug('[UTIO] payload:', payload); + await utio.send(payload); + } + } catch (error) { + logger.error('[UTIO] error:', error); + throw error; + } + } +} diff --git a/src/main/store/types.ts b/src/main/store/types.ts index 0850a2a..3bd8662 100644 --- a/src/main/store/types.ts +++ b/src/main/store/types.ts @@ -64,4 +64,5 @@ export type LocalStore = { vlmModelName: string; screenshotScale: number; // 0.1 ~ 1.0 reportStorageEndpoint?: string; + utioEndpoint?: string; }; diff --git a/src/main/window/index.ts b/src/main/window/index.ts index aaa8de9..eb13372 100644 --- a/src/main/window/index.ts +++ b/src/main/window/index.ts @@ -94,7 +94,7 @@ export function createSettingsWindow( console.log('mainWindowBounds', mainWindowBounds); const width = 480; - const height = 700; + const height = 760; let x, y; if (mainWindowBounds) { diff --git a/src/preload/index.d.ts b/src/preload/index.d.ts deleted file mode 100644 index aab28a6..0000000 --- a/src/preload/index.d.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { ElectronAPI } from '@electron-toolkit/preload'; - -declare global { - interface Window { - electron: ElectronAPI; - api: unknown; - } -} diff --git a/src/preload/index.ts b/src/preload/index.ts index 4cd982c..112a305 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -5,6 +5,8 @@ import { IpcRendererEvent, contextBridge, ipcRenderer } from 'electron'; import { preloadZustandBridge } from 'zutron/preload'; +import type { UTIOPayload } from '@ui-tars/utio'; + import type { AppState } from '@main/store/types'; export type Channels = 'ipc-example'; @@ -33,6 +35,10 @@ const electronHandler = { maximize: () => ipcRenderer.invoke('maximize-window'), close: () => ipcRenderer.invoke('close-window'), }, + utio: { + shareReport: (params: UTIOPayload<'shareReport'>) => + ipcRenderer.invoke('utio:shareReport', params), + }, }; // Initialize Zutron bridge diff --git a/src/renderer/src/components/ChatInput/index.tsx b/src/renderer/src/components/ChatInput/index.tsx index 87746b3..c72a22c 100644 --- a/src/renderer/src/components/ChatInput/index.tsx +++ b/src/renderer/src/components/ChatInput/index.tsx @@ -16,7 +16,7 @@ import { ComputerUseUserData } from '@ui-tars/shared/types/data'; import { useRunAgent } from '@renderer/hooks/useRunAgent'; import { useStore } from '@renderer/hooks/useStore'; import { reportHTMLContent } from '@renderer/utils/html'; -import { uploadAndShare, uploadReport } from '@renderer/utils/share'; +import { uploadReport } from '@renderer/utils/share'; import reportHTMLUrl from '@resources/report.html?url'; @@ -116,13 +116,15 @@ const ChatInput = forwardRef((_props, _ref) => { const htmlContent = reportHTMLContent(html, [userData]); + let reportUrl: string | undefined; + if (settings?.reportStorageEndpoint) { try { const { url } = await uploadReport( htmlContent, settings.reportStorageEndpoint, ); - // Copy link to clipboard + reportUrl = url; await navigator.clipboard.writeText(url); toast({ title: 'Report link copied to clipboard!', @@ -132,7 +134,6 @@ const ChatInput = forwardRef((_props, _ref) => { isClosable: true, variant: 'ui-tars-success', }); - return; } catch (error) { console.error('Share failed:', error); toast({ @@ -147,6 +148,20 @@ const ChatInput = forwardRef((_props, _ref) => { } } + // Send UTIO data through IPC + if (settings?.utioEndpoint) { + const lastScreenshot = messages + .filter((m) => m.screenshotBase64) + .pop()?.screenshotBase64; + + await window.electron.utio.shareReport({ + type: 'shareReport', + instruction: lastHumanMessage, + lastScreenshot, + report: reportUrl, + }); + } + // If shareEndpoint is not configured or the upload fails, fall back to downloading the file const blob = new Blob([htmlContent], { type: 'text/html' }); const url = window.URL.createObjectURL(blob); diff --git a/src/renderer/src/pages/settings/index.tsx b/src/renderer/src/pages/settings/index.tsx index 8c84c4c..93d1870 100644 --- a/src/renderer/src/pages/settings/index.tsx +++ b/src/renderer/src/pages/settings/index.tsx @@ -203,7 +203,7 @@ const Settings = () => { - Report Storage Provider Endpoint (Optional) + Report Storage Endpoint { /> + + UTIO Endpoint + + + + + + + + + Preset URL + setRemoteUrl(e.target.value)} + placeholder="https://example.com/preset.yaml" + /> + + + Auto update on startup + setAutoUpdate(e.target.checked)} + /> + + + + + + + + + + + + + ); +} diff --git a/src/renderer/src/pages/settings/index.tsx b/src/renderer/src/pages/settings/index.tsx index 93d1870..4ca91f1 100644 --- a/src/renderer/src/pages/settings/index.tsx +++ b/src/renderer/src/pages/settings/index.tsx @@ -9,6 +9,7 @@ import { FormControl, FormLabel, HStack, + IconButton, Input, Select, Spinner, @@ -21,7 +22,8 @@ import { useToast, } from '@chakra-ui/react'; import { Field, Form, Formik } from 'formik'; -import { useLayoutEffect } from 'react'; +import { useLayoutEffect, useState } from 'react'; +import { IoAdd } from 'react-icons/io5'; import { useDispatch } from 'zutron'; import { VlmProvider } from '@main/store/types'; @@ -29,8 +31,11 @@ import { VlmProvider } from '@main/store/types'; import { useStore } from '@renderer/hooks/useStore'; import { isWindows } from '@renderer/utils/os'; +import PresetImport from './PresetImport'; + const Settings = () => { const { settings, thinking } = useStore(); + const [isPresetModalOpen, setPresetModalOpen] = useState(false); const toast = useToast(); const dispatch = useDispatch(window.zutron); @@ -73,7 +78,25 @@ const Settings = () => { }); }; - console.log('initialValues', settings); + const handleUpdatePreset = async () => { + try { + await dispatch('UPDATE_PRESET_FROM_REMOTE'); + toast({ + title: 'Preset updated successfully', + status: 'success', + duration: 2000, + }); + } catch (error) { + toast({ + title: 'Failed to update preset', + description: error.message, + status: 'error', + duration: 3000, + }); + } + }; + + const isPresetEnabled = !!settings?.presetSource; return ( @@ -89,11 +112,49 @@ const Settings = () => { General + + {settings?.presetSource?.type === 'remote' && ( + + )} + } + aria-label="Import Preset" + variant="ghost" + onClick={() => setPresetModalOpen(true)} + /> + + {settings?.presetSource && ( + + + Settings managed by {settings.presetSource.type} preset + {settings.presetSource.url && + ` from ${settings.presetSource.url}`} + {settings.presetSource.lastUpdated && + ` (Last updated: ${new Date(settings.presetSource.lastUpdated).toLocaleString()})`} + + + + )} + {settings ? ( {({ values = {}, setFieldValue }) => ( @@ -253,6 +314,10 @@ const Settings = () => { + setPresetModalOpen(false)} + /> ); };