feat(o-gui-agent): support ChromeUI gui operation on AIO sandbox (#1383)

Author: ZhaoHeh

multimodal/gui-agent/operator-aio/.gitignore

@@ -0,0 +1,2 @@
+# Dumps directory
+dumps/

multimodal/gui-agent/operator-aio/README.md

@@ -0,0 +1,46 @@
+# @gui-agent/operator-aio
+
+AIO (All-in-One) operator for GUI Agent that provides comprehensive computer control capabilities.
+
+## Features
+
+- Remote computer control
+- Mouse operations (click, drag, scroll)
+- Keyboard input and hotkeys
+- Screenshot capture
+- Cross-platform support
+
+## Installation
+
+```bash
+npm install @gui-agent/operator-aio
+```
+
+## Usage
+
+```typescript
+import { AioOperator } from '@gui-agent/operator-aio';
+
+// Create AIO operator instance
+const operator = await AioOperator.create();
+
+// Take screenshot
+const screenshot = await operator.screenshot();
+
+// Execute actions
+const result = await operator.execute({
+  parsedPrediction: {
+    action_type: 'click',
+    action_inputs: {
+      start_box: '[100, 100, 200, 200]'
+    }
+  },
+  screenWidth: 1920,
+  screenHeight: 1080,
+  scaleFactor: 1
+});
+```
+
+## License
+
+Apache-2.0

multimodal/gui-agent/operator-aio/examples/test-runner.ts

@@ -0,0 +1,212 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { AIOHybridOperator } from '../src/AIOHybridOperator';
+import { StatusEnum } from '@ui-tars/sdk';
+// @ts-ignore - Module resolution issue: TypeScript cannot resolve this module with current 'Bundler' moduleResolution setting
+// The type exists at '/node_modules/@ui-tars/sdk/dist/core.d.ts' but requires 'node16', 'nodenext', or 'bundler' moduleResolution
+import type { ExecuteParams } from '@ui-tars/sdk/core';
+import * as fs from 'fs';
+import * as path from 'path';
+
+// 配置真实的baseURL
+// 在文件顶部添加
+import 'dotenv/config';
+
+const CONFIG = {
+  baseURL: process.env.AIO_BASE_URL || 'http://localhost:8080', // 您的真实URL
+  timeout: 10000,
+};
+
+async function testAIOHybridOperator() {
+  console.log('🚀 开始测试 AIOHybridOperator...');
+  console.log('配置:', CONFIG);
+
+  try {
+    // 1. 创建操作器实例
+    console.log('\n📦 创建 AIOHybridOperator 实例...');
+    const operator = await AIOHybridOperator.create(CONFIG);
+    console.log('✅ 实例创建成功');
+
+    // 2. 测试截图功能
+    console.log('\n📸 测试截图功能...');
+    const screenshot = await operator.screenshot();
+
+    // 创建dumps目录
+    const dumpsDir = path.join(__dirname, 'dumps');
+    if (!fs.existsSync(dumpsDir)) {
+      fs.mkdirSync(dumpsDir, { recursive: true });
+    }
+
+    // 保存截图
+    if (screenshot.base64) {
+      const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+      const filename = `screenshot-${timestamp}.png`;
+      const filepath = path.join(dumpsDir, filename);
+
+      // 将base64转换为buffer并保存
+      const base64Data = screenshot.base64.replace(/^data:image\/png;base64,/, '');
+      const buffer = Buffer.from(base64Data, 'base64');
+      fs.writeFileSync(filepath, buffer);
+
+      console.log('截图已保存:', filepath);
+    }
+
+    console.log('截图结果:', {
+      base64Length: screenshot.base64?.length || 0,
+      scaleFactor: screenshot.scaleFactor,
+      hasBase64: !!screenshot.base64,
+    });
+    console.log('✅ 截图功能正常');
+
+    // 3. 测试各种动作执行
+    const testCases = [
+      {
+        name: '点击动作',
+        params: {
+          parsedPrediction: {
+            action_type: 'click',
+            action_inputs: {
+              start_box: '[100, 200, 150, 250]',
+            },
+            reflection: '测试点击',
+            thought: '执行点击操作',
+            prediction: 'click action',
+            factors: [1000, 1000],
+          },
+          screenWidth: 1920,
+          screenHeight: 1080,
+          scaleFactor: 1,
+          prediction: 'click action',
+          factors: [1000, 1000],
+        } as ExecuteParams,
+      },
+      {
+        name: '输入文本',
+        params: {
+          parsedPrediction: {
+            action_type: 'type',
+            action_inputs: {
+              content: 'Hello World\n',
+            },
+            reflection: '测试输入',
+            thought: '输入测试文本',
+            prediction: 'type action',
+            factors: [1000, 1000],
+          },
+          screenWidth: 1920,
+          screenHeight: 1080,
+          scaleFactor: 1,
+          prediction: 'type action',
+          factors: [1000, 1000],
+        } as ExecuteParams,
+      },
+      {
+        name: '快捷键',
+        params: {
+          parsedPrediction: {
+            action_type: 'hotkey',
+            action_inputs: {
+              key: 'Ctrl+C',
+            },
+            reflection: '测试快捷键',
+            thought: '执行复制快捷键',
+            prediction: 'hotkey action',
+            factors: [1000, 1000],
+          },
+          screenWidth: 1920,
+          screenHeight: 1080,
+          scaleFactor: 1,
+          prediction: 'hotkey action',
+          factors: [1000, 1000],
+        } as ExecuteParams,
+      },
+      {
+        name: '滚动操作',
+        params: {
+          parsedPrediction: {
+            action_type: 'scroll',
+            action_inputs: {
+              start_box: '[500, 500, 600, 600]',
+              direction: 'down',
+            },
+            reflection: '测试滚动',
+            thought: '向下滚动',
+            prediction: 'scroll action',
+            factors: [1000, 1000],
+          },
+          screenWidth: 1920,
+          screenHeight: 1080,
+          scaleFactor: 1,
+          prediction: 'scroll action',
+          factors: [1000, 1000],
+        } as ExecuteParams,
+      },
+      {
+        name: '等待操作',
+        params: {
+          parsedPrediction: {
+            action_type: 'wait',
+            action_inputs: {},
+            reflection: '测试等待',
+            thought: '等待5秒',
+            prediction: 'wait action',
+            factors: [1000, 1000],
+          },
+          screenWidth: 1920,
+          screenHeight: 1080,
+          scaleFactor: 1,
+          prediction: 'wait action',
+          factors: [1000, 1000],
+        } as ExecuteParams,
+      },
+    ];
+
+    console.log('\n🎯 开始测试各种动作执行...');
+    for (const testCase of testCases) {
+      console.log(`\n测试: ${testCase.name}`);
+      try {
+        const result = await operator.execute(testCase.params);
+        console.log('执行结果:', JSON.stringify(result));
+        console.log(`✅ ${testCase.name} 执行成功`);
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        console.error(`❌ ${testCase.name} 执行失败:`, errorMessage);
+      }
+    }
+
+    // 4. 验证动作空间定义
+    console.log('\n📋 验证动作空间定义...');
+    const actionSpaces = AIOHybridOperator.MANUAL.ACTION_SPACES;
+    console.log('动作空间数量:', actionSpaces.length);
+    console.log('动作空间列表:');
+    actionSpaces.forEach((action, index) => {
+      console.log(`  ${index + 1}. ${action}`);
+    });
+    console.log('✅ 动作空间验证完成');
+
+    console.log('\n🎉 所有测试完成！');
+  } catch (error) {
+    console.error('❌ 测试过程中发生错误:', error);
+    const errorStack = error instanceof Error ? error.stack : String(error);
+    console.error('错误详情:', errorStack);
+    process.exit(1);
+  }
+}
+
+// 运行测试
+if (require.main === module) {
+  testAIOHybridOperator()
+    .then(() => {
+      console.log('\n✨ 测试脚本执行完成');
+      process.exit(0);
+    })
+    .catch((error) => {
+      console.error('\n💥 测试脚本执行失败:', error);
+      process.exit(1);
+    });
+}
+
+export { testAIOHybridOperator };

multimodal/gui-agent/operator-aio/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@gui-agent/operator-aio",
+  "description": "AIO (All-in-One) operator for GUI Agent",
+  "version": "0.3.0-beta.7",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/bytedance/UI-TARS-desktop"
+  },
+  "bugs": {
+    "url": "https://github.com/bytedance/UI-TARS-desktop/issues"
+  },
+  "keywords": [
+    "AI",
+    "Core",
+    "SDK",
+    "Operator",
+    "UI-TARS",
+    "AIO"
+  ],
+  "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "scripts": {
+    "dev": "rslib build --watch",
+    "build": "rslib build",
+    "build:watch": "rslib build --watch",
+    "test": "vitest",
+    "example": "npx tsx examples/test-runner.ts"
+  },
+  "license": "Apache-2.0",
+  "files": [
+    "dist"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  },
+  "dependencies": {
+    "@agent-infra/browser": "0.1.1",
+    "@agent-infra/logger": "0.0.2-beta.2",
+    "@agent-infra/sandbox": "0.0.2-beta7",
+    "@agent-infra/media-utils": "0.1.5",
+    "@ui-tars/sdk": "1.2.3",
+    "@ui-tars/shared": "1.2.3"
+  },
+  "devDependencies": {
+    "@rslib/core": "0.10.0",
+    "dotenv": "^16.3.1",
+    "ts-node": "^10.9.2",
+    "tsx": "^4.19.2",
+    "typescript": "^5.7.2",
+    "vitest": "^3.0.2"
+  }
+}

multimodal/gui-agent/operator-aio/rslib.config.ts

@@ -0,0 +1,41 @@
+/**
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { defineConfig } from '@rslib/core';
+
+const BANNER = `/**
+* Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+* SPDX-License-Identifier: Apache-2.0
+*/`;
+
+export default defineConfig({
+  source: {
+    entry: {
+      index: ['src/**'],
+      // examples: ['examples/**'], // Prevent static files in examples/dumps from being bundled into dist, so this line is commented out
+    },
+  },
+  lib: [
+    {
+      format: 'esm',
+      syntax: 'es2021',
+      bundle: false,
+      autoExternal: false,
+      dts: true,
+      banner: { js: BANNER },
+    },
+    {
+      format: 'cjs',
+      syntax: 'es2021',
+      bundle: false,
+      dts: true,
+      banner: { js: BANNER },
+    },
+  ],
+  output: {
+    target: 'node',
+    cleanDistPath: false,
+    sourceMap: true,
+  },
+});