diff --git a/prompts/e2e.md b/prompts/e2e.md index e32ec5d0..e718d027 100644 --- a/prompts/e2e.md +++ b/prompts/e2e.md @@ -4,7 +4,7 @@ Your action history is: {history} Please provide the next action based on the screenshot and your action history. You should do careful reasoning before providing the action. Your action space includes: -- Name: click, Parameters: target_element (a high-level description of the UI element to click), bbox (an **absolute** bounding box of the target element). +- Name: click, Parameters: target_element (a high-level description of the UI element to click), bbox (an **absolute** bounding box of the target element,[x1, y1, x2, y2]). - Name: swipe, Parameters: direction (one of UP, DOWN, LEFT, RIGHT). - Name: input, Parameters: text (the text to input). - Name: wait, Parameters: (no parameters, will wait for 1 second). diff --git a/prompts/e2e_nohistory_qwen3.md b/prompts/e2e_nohistory_qwen3.md new file mode 100644 index 00000000..b86c1296 --- /dev/null +++ b/prompts/e2e_nohistory_qwen3.md @@ -0,0 +1,11 @@ + +You are a phone-use AI agent. Now your task is "{task}". +Please provide the next action based on the screenshot. You should do careful reasoning before providing the action. +Your action space includes: +- Name: click, Parameters: target_element (a high-level description of the UI element to click), bbox (an bounding box of the target element,[x1, y1, x2, y2]). +- Name: swipe, Parameters: direction (one of UP, DOWN, LEFT, RIGHT), start_coords (the starting absolute coordinate [x, y]), end_coords (the ending absolute coordinate [x, y]). +- Name: input, Parameters: text (the text to input). +- Name: wait, Parameters: (no parameters, will wait for 1 second). +- Name: done, Parameters: status (the completion status of the current task, one of `success', `suspended` and `failed`). +Your output should be a JSON object with the following format: +{{"reasoning": "Your reasoning here", "action": "The next action (one of click, input, swipe, wait, done)", "parameters": {{"param1": "value1","param2": "value2", ...}}}} \ No newline at end of file diff --git a/prompts/e2e_qwen3.md b/prompts/e2e_qwen3.md new file mode 100644 index 00000000..3d2d7ca4 --- /dev/null +++ b/prompts/e2e_qwen3.md @@ -0,0 +1,13 @@ + +You are a phone-use AI agent. Now your task is "{task}". +Your action history is: +{history} +Please provide the next action based on the screenshot and your action history. You should do careful reasoning before providing the action. +Your action space includes: +- Name: click, Parameters: target_element (a high-level description of the UI element to click), bbox (an bounding box of the target element,[x1, y1, x2, y2]). +- Name: swipe, Parameters: direction (one of UP, DOWN, LEFT, RIGHT), start_coords (the starting absolute coordinate [x, y]), end_coords (the ending absolute coordinate [x, y]). +- Name: input, Parameters: text (the text to input). +- Name: wait, Parameters: (no parameters, will wait for 1 second). +- Name: done, Parameters: status (the completion status of the current task, one of `success`, `suspended` and `failed`). +Your output should be a JSON object with the following format: +{{"reasoning": "Your reasoning here", "action": "The next action (one of click, input, swipe, wait, done)", "parameters": {{"param1": "value1","param2": "value2", ...}}}} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ca5ad6fd..f5a95459 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,4 +41,4 @@ llama-index llama-index-embeddings-huggingface pandas hmdriver2 -mem0ai +mem0ai \ No newline at end of file diff --git a/runner/RUNNER_README.md b/runner/RUNNER_README.md new file mode 100644 index 00000000..b4e0d965 --- /dev/null +++ b/runner/RUNNER_README.md @@ -0,0 +1,130 @@ +# 统一 GUI Agent Runner 框架使用指南 + +这是一个统一的 GUI Agent 任务执行框架，支持接入多种模型（MobiAgent, UI-TARS, Qwen, AutoGLM 等）并执行移动端自动化任务。该框架提供了统一的接口、参数管理、动作规范化以及执行过程可视化功能。 + +## 快速开始 + +### 1. 执行单个任务 + +可以通过 `run.py` 直接执行单条任务指令。建议参考 `examples/` 目录下的脚本。 + +```bash +# 使用 MobiAgent (HarmonyOS) +python run.py --provider mobiagent --task "在淘宝上搜索电动牙刷" --device-type Harmony + +# 使用 UI-TARS (带可视化绘制) +python run.py --provider uitars --task "在微信发送消息给张三，说我的AI助手很厉害" --draw --api-base http://localhost:8000/v1 + +# 使用 Qwen VLM +python run.py --provider qwen --task "在微博查看热搜" --api-base http://localhost:8080/v1 +``` + +### 2. 批量执行任务 + +支持从 JSON 文件中读取任务列表进行批量测试。 + +```bash +# 执行通用任务列表 +python run.py --provider mobiagent --task-file task.json + +# 执行 MobiFlow 格式任务列表 +python run.py --provider uitars --task-file task_mobiflow.json +``` + +--- + +## 📂 项目结构 + +```text +runner/ +├── run.py # 统一执行入口，处理核心命令行逻辑 +├── base_task.py # 基础任务抽象类，定义统一接口和动作映射 +├── task_manager.py # 任务管理器，负责 Provider 实例化 +├── device.py # 设备抽象层 (支持 Android 和 Harmony) +├── providers/ # 模型适配器目录 +│ ├── mobiagent/ # MobiAgent 相关适配逻辑 +│ ├── uitars/ # UI-TARS 适配器 +│ ├── qwen/ # Qwen-VL 适配器 +│ └── autoglm/ # AutoGLM 适配器 +├── examples/ # 各模型的启动脚本示例 +└── results/ # 默认执行结果输出路径 +``` + +--- + +## 🛠️ 参数说明 + +### 1. 基础参数 +| 参数 | 说明 | 默认值 | 可选值 | +| :--- | :--- | :--- | :--- | +| `--provider` | 模型提供者 | `mobiagent` | `uitars`, `qwen(或其他任意VLM模型)`, `autoglm` | +| `--device-type`| 设备系统类型 | `Android` | `Android`, `Harmony` | +| `--device-id` | 设备 ID 或 IP | None | - | +| `--max-steps` | 单个任务最大执行步数 | 40 | - | +| `--log-level` | 日志详细程度 | `INFO` | `DEBUG`, `INFO`, `WARNING`, `ERROR` | +| `--draw` | 是否在截图上绘制操作可视化 | `False` | - | + +### 2. 任务参数 +| 参数 | 说明 | +| :--- | :--- | +| `--task` | 单个任务描述字符串 | +| `--task-file` | 任务列表 JSON 文件路径 | +| `--output-dir` | 结果输出根目录 (默认 `results`) | + +### 3. 通用模型参数 +框架对常见的模型调用参数进行了统一，各 Provider 会映射到各自的底层实现。 +| 参数 | 说明 | +| :--- | :--- | +| `--api-base` | 模型服务基础 URL | +| `--api-key` | API 密钥 | +| `--model` | 模型名称 | +| `--temperature` | 生成温度 | + +### 4. 特定 Provider 参数 +- **MobiAgent**: + - `--service-ip`: 服务主机 IP + - `--decider-port`: Decider 端口 (默认 8000) + - `--grounder-port`: Grounder 端口 (默认 8001) + - `--planner-port`: Planner 端口 (默认 8002) + - `--enable-planning`: 启用任务规划模式 + - `--use-e2e`: 启用端到端执行模式 +- **UI-TARS**: + - `--step-delay`: 每步操作间的延迟 (秒) + +--- + +## 📊 输出结果 + +每个任务执行后会在 `output-dir` 下生成以时间戳命名的目录，包含： + +- `1.jpg, 2.jpg...`: 每一步的屏幕截图。 +- `1_draw.jpg...`: 标注了动作（如点击红点、滑动箭头）的可视化截图（需开启 `--draw`）。 +- `1.xml` 或 `1.json`: 每一步的 UI 结构树。 +- `actions.json`: 完整的动作序列记录，包含坐标和参数。 +- `react.json`: 模型每一步推导的思维过程 (Reasoning)。 + +--- + +## 🧩 接入新模型 (Provider) + +接入新模型只需遵循以下步骤： + +1. **创建适配器**: + 在 `providers/` 下创建子目录，实现一个继承自 `base_task.py:BaseTask` 的类。 + - 实现 `execute_step(self, step_index)`：接收当前步数，返回动作序列列表。 + - 或者实现 `_execute_task(self)`：如果你希望自己接管整个任务循环。 + +2. **动作规范化**: + 在 `BaseTask` 中已定义了统一的动作映射表 `ACTION_TYPE_ALIASES`。如果新模型的动作名称不同，只需在 `base_task.py` 中添加映射即可，例如将 `tap` 映射为 `click`。 + +3. **注册 Provider**: + 在 `task_manager.py` 的 `_get_task_map` 方法中添加你的模型名称与类的映射。 + +4. **添加配置**: + 在 `run.py` 的 `PROVIDER_DEFAULTS` 中添加该模型的默认 API 地址和模型名。 + +--- + +## 🙏 感谢 + +感谢 **[23japhone](https://github.com/23japhone)** 的测试框架参考。 diff --git a/runner/base_task.py b/runner/base_task.py new file mode 100644 index 00000000..b38f1817 --- /dev/null +++ b/runner/base_task.py @@ -0,0 +1,950 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + +import os +import os +import math +import shutil +import json +import time +import logging +from abc import ABC, abstractmethod +from typing import Dict, List, Optional, Tuple +import re +import textwrap +from PIL import Image, ImageDraw, ImageFont + +# 使用模块级别的logger（级别由 setup_logging() 统一配置） +logger = logging.getLogger(__name__) + +# 绘图常量 +RED_DOT_COLOR = (255, 0, 0, 200) +SCROLL_COLOR = (0, 122, 255, 210) +SCROLL_WIDTH = 12 +STRIP_BG_COLOR = "white" +STRIP_ACCENT_COLOR = "#0062ff" + +ACTION_LABELS = { + "open_app": "Open App", + "click": "Tap", + "longclick": "Long Press", + "doubleclick": "Double Tap", + "scroll": "Scroll", + "input_text": "Input Text", + "back": "Back", + "home": "Home", + "retry": "Retry", + "wait": "Wait", +} + +# 统一动作类型映射表：将各模型特定的动作名称映射为统一格式 +# 格式: 原始名称 -> 规范名称 +ACTION_TYPE_ALIASES = { + # ==================== 点击相关 ==================== + 'click': 'click', + 'tap': 'click', + 'left_single': 'click', + 'right_single': 'click', + 'hover': 'click', + + # ==================== 长按相关 ==================== + 'longclick': 'longclick', + 'long_press': 'longclick', + 'long_click': 'longclick', + + # ==================== 双击相关 ==================== + 'doubleclick': 'doubleclick', + 'double_tap': 'doubleclick', + 'double_click': 'doubleclick', + 'left_double': 'doubleclick', + + # ==================== 输入相关 ==================== + 'input_text': 'input_text', + 'type': 'input_text', + 'input': 'input_text', + 'set_text': 'input_text', + + # ==================== 滑动相关 ==================== + 'scroll': 'scroll', + 'swipe': 'scroll', + 'drag': 'scroll', + + # ==================== 启动应用 ==================== + 'open_app': 'open_app', + 'open': 'open_app', + 'start_app': 'open_app', + 'launch': 'open_app', + + # ==================== 系统按键 ==================== + 'home': 'home', + 'press_home': 'home', + 'key_home': 'home', + 'back': 'back', + 'press_back': 'back', + 'key_back': 'back', + + # ==================== 等待 ==================== + 'wait': 'wait', + + # ==================== 任务终止 ==================== + 'done': 'done', + 'finished': 'done', + 'complete': 'done', + + # ==================== 重试/失败 ==================== + 'retry': 'retry', + 'fail': 'retry', + 'error': 'retry', + + # ==================== AutoGLM 特有动作 ==================== + 'take_over': 'wait', # 用户接管，暂时映射为等待 + 'note': 'wait', # 记录页面，暂不实现 + 'call_api': 'wait', # 调用API，暂不实现 + 'interact': 'wait', # 用户交互，暂时映射为等待 +} + +class BaseTask(ABC): + """ + 基础任务类，定义统一的任务执行接口 + 所有模型适配器都需要继承此类并实现抽象方法 + + 支持两种执行模式： + 1. 步骤循环模式：框架控制循环，子类实现execute_step返回单步动作 + 2. 一次性执行模式：子类实现_execute_task完成整个任务（用于已有循环逻辑的模型） + """ + + def __init__( + self, + task_description: str, + device, + data_dir: str, + device_type: str = "Android", + max_steps: int = 40, + max_retries: int = 3, + use_step_loop: bool = False, + draw: bool = False, + enable_planning: bool = False, + **kwargs + ): + """ + 初始化基础任务 + + Args: + task_description: 任务描述 + device: 设备对象（AndroidDevice或HarmonyDevice） + data_dir: 数据保存目录 + device_type: 设备类型 ("Android" 或 "Harmony") + max_steps: 最大步骤数 + max_retries: 最大重试次数 + use_step_loop: 是否使用步骤循环模式 + draw: 是否绘制操作可视化 + enable_planning: 是否启用任务规划（可分析APP、结合experience和profile等） + **kwargs: 其他参数 + """ + self.task_description = task_description + self.original_task_description = task_description # 保存原始任务描述 + self.device = device + self.data_dir = data_dir + self.device_type = device_type + self.max_steps = max_steps + self.max_retries = max_retries + self.use_step_loop = use_step_loop + self.draw = draw + self.enable_planning = enable_planning + + # 确保数据目录存在 + os.makedirs(self.data_dir, exist_ok=True) + + self.actions = [] + self.reacts = [] + self.step_count = 0 + self.retry_count = 0 + self.total_time = 0.0 + + # planning 相关属性 + self.app_name = None + self.package_name = None + self.planning_info = {} # 存储 planning 返回的其他信息 + + logger.info(f"初始化任务: {task_description}") + logger.info(f"数据目录: {data_dir}") + logger.info(f"设备类型: {device_type}") + logger.info(f"使用步骤循环: {use_step_loop}") + logger.info(f"绘制可视化: {draw}") + logger.info(f"启用规划: {enable_planning}") + + def execute(self) -> Dict: + """ + 执行任务的主流程 + + Returns: + 包含任务执行结果的字典 + """ + logger.info(f"开始执行任务: {self.task_description}") + start_time = time.time() + + try: + # 如果启用 planning，先进行任务规划 + if self.enable_planning: + logger.info("执行 planning...") + try: + planning_result = self._plan_task() + if planning_result: + # 更新任务描述（如果 planning 返回了优化后的描述） + if 'task_description' in planning_result: + self.task_description = planning_result['task_description'] + logger.info(f"任务描述已优化: {self.task_description}") + + # 更新 APP 信息 + if 'app_name' in planning_result: + self.app_name = planning_result['app_name'] + logger.info(f"目标 APP: {self.app_name}") + + if 'package_name' in planning_result: + self.package_name = planning_result['package_name'] + logger.info(f"包名: {self.package_name}") + + # 保存其他 planning 信息 + self.planning_info = planning_result + + # 如果需要启动应用 + if self.package_name: + logger.info(f"启动应用: {self.package_name}") + self.device.app_start(self.package_name) + except Exception as e: + logger.warning(f"planning 失败，继续使用原始任务: {e}") + + logger.info(60*"=") + logger.info(f"step_count: {self.step_count}") + logger.info(60*"=") + if self.use_step_loop: + # 步骤循环模式：框架控制循环 + result = self._execute_with_step_loop() + + else: + # 一次性执行模式：子类自己控制循环 + result = self._execute_task() + logger.info(60*"=") + + # 如果启动了应用，执行完成后停止应用 + if self.enable_planning and self.package_name: + try: + logger.info(f"停止应用: {self.package_name}") + self.device.app_stop(self.package_name) + except Exception as e: + logger.warning(f"停止应用失败: {e}") + + self.total_time = time.time() - start_time + + # 保存结果 + self._save_results(result) + + logger.info(f"任务已完成，耗时{self.total_time:.2f}秒") + return result + + except Exception as e: + logger.error(f"任务执行失败: {e}", exc_info=True) + self.total_time = time.time() - start_time + result = { + "status": "error", + "error": str(e), + "step_count": self.step_count, + "total_time": self.total_time + } + self._save_results(result) + return result + + def _execute_with_step_loop(self) -> Dict: + """ + 使用步骤循环模式执行任务 + + Returns: + 任务执行结果字典 + """ + logger.info("使用步骤循环模式") + + while True: + if self.step_count >= self.max_steps: + logger.warning(f"达到最大步数: {self.max_steps}") + return { + "status": "max_steps_reached", + "step_count": self.step_count, + "message": f"任务达到最大步数 ({self.max_steps})" + } + + self.step_count += 1 + logger.info(f"\n{'='*50}") + logger.info(f"Step {self.step_count}/{self.max_steps}") + logger.info(f"{'='*50}") + + try: + screenshot_path = self._save_screenshot(self.step_count) + hierarchy_path = self._save_hierarchy(self.step_count) + except Exception as e: + logger.error(f"保存状态失败: {e}") + + try: + step_start_time = time.time() + action_seq = self.execute_step(self.step_count) + step_elapsed_time = time.time() - step_start_time + + logger.info(f"Step {self.step_count} 动作: {json.dumps(action_seq, ensure_ascii=False)}") + + # 统一规范化动作类型 + action_seq = [self._normalize_action(a) for a in action_seq] + logger.debug(f"规范化后动作: {json.dumps(action_seq, ensure_ascii=False)}") + + should_continue = self._execute_action_seq(action_seq) + + if self.draw: + try: + self._draw_actions_on_image(self.step_count, action_seq) + except Exception as e: + logger.warning(f"绘制失败: {e}") + + if not should_continue: + status = "completed" + for action in action_seq: + if action.get("type") == "done": + status = action.get("params", {}).get("status", "success") + break + elif action.get("type") == "retry" and self.retry_count >= self.max_retries: + status = "failed" + break + + return { + "status": status, + "step_count": self.step_count, + "message": f"Step {self.step_count} 任务完成" + } + + time.sleep(2) + + if hasattr(self, 'reflect_action') and callable(getattr(self, 'reflect_action', None)): + try: + reflect_start_time = time.time() + self.reflect_action(self.step_count) + step_elapsed_time += time.time() - reflect_start_time + except Exception as e: + logger.warning(f"反思失败: {e}") + + except Exception as e: + logger.error(f"Step {self.step_count} 失败: {e}", exc_info=True) + return { + "status": "error", + "step_count": self.step_count, + "error": str(e) + } + + # ... (rest of methods unchanged until _save_results needed access to PIL?) + # 绘制方法将在类末尾实现 + + def _execute_task(self) -> Dict: + """ + 子类一次性任务执行逻辑（用于已有循环逻辑的模型） + + Returns: + 包含任务执行结果的字典 + """ + raise NotImplementedError( + "Either implement _execute_task() for one-time execution " + "or implement execute_step() with use_step_loop=True" + ) + + def execute_step(self, step_index: int) -> List[Dict]: + """ + 子类单步执行逻辑（用于步骤循环模式） + + Args: + step_index: 当前步骤索引（从1开始） + + Returns: + 动作序列列表，每个动作是一个字典，包含： + - type: 动作类型（click, type, swipe, done等） + - params: 动作参数 + """ + raise NotImplementedError( + "Either implement execute_step() with use_step_loop=True " + "or implement _execute_task() for one-time execution" + ) + + def _normalize_action(self, action: Dict) -> Dict: + """ + 规范化动作字典，将各模型特定的动作名称映射为统一格式 + + Args: + action: 原始动作字典，包含 type 和 params + + Returns: + 规范化后的动作字典 + """ + if not isinstance(action, dict): + return action + + action_type = action.get('type', '').lower() + params = action.get('params', {}) + + # 映射动作类型 + normalized_type = ACTION_TYPE_ALIASES.get(action_type, action_type) + + # 规范化参数名称 + normalized_params = self._normalize_params(normalized_type, params) + + return {'type': normalized_type, 'params': normalized_params} + + def _normalize_params(self, action_type: str, params: Dict) -> Dict: + """ + 规范化参数格式，统一坐标参数名称 + + Args: + action_type: 规范化后的动作类型 + params: 原始参数字典 + + Returns: + 规范化后的参数字典 + """ + if not isinstance(params, dict): + return params + + result = dict(params) + + # 点击/长按/双击动作的坐标参数统一 + if action_type in ['click', 'longclick', 'doubleclick']: + # 将 'points' 格式转换为 'coordinate' 格式 + if 'points' in result and 'coordinate' not in result: + points = result['points'] + if isinstance(points, list) and len(points) >= 2: + result['coordinate'] = [points[0], points[1]] + + # 将 'position_x'/'position_y' 格式转换为 'coordinate' 格式 + if 'position_x' in result and 'position_y' in result and 'coordinate' not in result: + result['coordinate'] = [result['position_x'], result['position_y']] + + # 滑动动作的坐标参数统一 + if action_type == 'scroll': + # 将 'start'/'end' 格式转换为 'start_coordinate'/'end_coordinate' + if 'start' in result and 'start_coordinate' not in result: + result['start_coordinate'] = result.pop('start') + if 'end' in result and 'end_coordinate' not in result: + result['end_coordinate'] = result.pop('end') + + # 输入动作的文本参数统一 + if action_type == 'input_text': + # 将 'content' 格式转换为 'text' 格式 + if 'content' in result and 'text' not in result: + result['text'] = result.pop('content') + + return result + + def _execute_action_seq(self, action_seq: List[Dict]) -> bool: + """ + 执行动作序列 + + Args: + action_seq: 动作序列列表 + + Returns: + 是否继续执行（True 继续，False 停止） + """ + for action in action_seq: + action_type = action.get('type') + params = action.get('params', {}) + + if action_type == 'retry': + self.retry_count += 1 + logger.warning(f"重试次数: {self.retry_count}/{self.max_retries}") + if self.retry_count >= self.max_retries: + logger.error(f"达到最大重试次数 ({self.max_retries})") + return False + continue + else: + self.retry_count = 0 + + if action_type == 'done': + status = params.get('status', 'completed') + logger.info(f"任务完成，状态: {status}") + return False + + try: + self._perform_action(action_type, params) + self._add_action( + action_type=action_type, + step_index=self.step_count, + **params + ) + except Exception as e: + logger.error(f"执行动作 {action_type} 失败: {e}") + raise + + return True + + def _perform_action(self, action_type: str, params: Dict): + """ + 执行具体动作 + + Args: + action_type: 动作类型 + params: 动作参数 + """ + logger.info(f"执行 {action_type}：{params}") + + action_type = action_type.lower() + + if action_type in ['click', 'tap']: + if 'coordinate' in params: + x, y = params['coordinate'] + elif 'position_x' in params and 'position_y' in params: + x, y = params['position_x'], params['position_y'] + elif 'points' in params: + x, y = params['points'] + else: + raise ValueError("Click 动作缺少坐标信息") + self.device.click(x, y) + + elif action_type in ['long_press', 'long_click', 'longclick']: + if 'coordinate' in params: + x, y = params['coordinate'] + elif 'position_x' in params and 'position_y' in params: + x, y = params['position_x'], params['position_y'] + elif 'points' in params: + x, y = params['points'] + else: + raise ValueError("Long press action missing coordinate information") + self.device.long_click(x, y) + + elif action_type in ['double_tap', 'double_click', 'doubleclick']: + if 'coordinate' in params: + x, y = params['coordinate'] + elif 'position_x' in params and 'position_y' in params: + x, y = params['position_x'], params['position_y'] + elif 'points' in params: + x, y = params['points'] + else: + raise ValueError("Double tap action missing coordinate information") + self.device.double_click(x, y) + + elif action_type in ['type', 'input', 'set_text', 'input_text']: + text = params.get('text', '') + self.device.input(text) + # 尝试在输入后直接回车确定、搜索 + time.sleep(0.2) + self.device.keyevent('ENTER') + + elif action_type in ['swipe', 'scroll']: + if 'direction' in params: + direction = params['direction'] + scale = params.get('scale', 0.5) + self.device.swipe(direction, scale) + elif 'start_coordinate' in params and 'end_coordinate' in params: + start = params['start_coordinate'] + end = params['end_coordinate'] + self.device.swipe_with_coords(start[0], start[1], end[0], end[1]) + elif 'points' in params and len(params['points']) == 4: + x1, y1, x2, y2 = params['points'] + self.device.swipe_with_coords(x1, y1, x2, y2) + else: + raise ValueError("Swipe action missing coordinate information") + + elif action_type == 'back': + self.device.keyevent('BACK') + + elif action_type == 'home': + self.device.keyevent('HOME') + + elif action_type in ['open', 'start_app', 'launch', 'open_app']: + app_name = params.get('app_name') + package_name = params.get('package_name') + if app_name: + self.device.start_app(app_name) + elif package_name: + self.device.app_start(package_name) + + elif action_type == 'wait': + seconds = params.get('seconds', 1) + time.sleep(seconds) + + else: + logger.warning(f"未知的动作类型: {action_type}") + + def reflect_action(self, step_index: int): + """ + 可选的反思方法，子类可以实现以改进后续步骤 + + Args: + step_index: 当前步骤索引 + """ + pass + + def _plan_task(self) -> Optional[Dict]: + """ + 任务规划方法，子类可以实现以进行任务分析和优化 + + 此方法在任务执行前调用，可以用于： + - 分析任务描述，确定需要使用的 APP + - 结合历史经验 (experience) 优化任务描述 + - 使用用户画像 (profile) 个性化任务 + - 返回任务执行所需的额外信息 + + Returns: + 包含规划结果的字典，可能包含： + - task_description: 优化后的任务描述 + - app_name: 目标应用名称 + - package_name: 应用包名 + - 其他任务执行所需的信息 + + 如果不需要 planning 或 planning 失败，返回 None + + Example: + { + "task_description": "在淘宝搜索苹果手机并查看价格", + "app_name": "淘宝", + "package_name": "com.taobao.taobao", + "experience_used": True + } + """ + # 默认实现：不进行 planning + # 子类可以覆盖此方法实现具体的 planning 逻辑 + logger.debug("Default _plan_task implementation: no planning performed") + return None + + def _save_screenshot(self, step_index: int) -> str: + """保存截图""" + screenshot_path = os.path.join(self.data_dir, f"{step_index}.jpg") + self.device.screenshot(screenshot_path) + return screenshot_path + + def _save_hierarchy(self, step_index: int) -> str: + """保存 UI 层级结构""" + try: + hierarchy = self.device.dump_hierarchy() + + if self.device_type == "Android": + hierarchy_path = os.path.join(self.data_dir, f"{step_index}.xml") + with open(hierarchy_path, "w", encoding="utf-8") as f: + f.write(hierarchy) + else: + hierarchy_path = os.path.join(self.data_dir, f"{step_index}.json") + if isinstance(hierarchy, str): + try: + hierarchy_json = json.loads(hierarchy) + except json.JSONDecodeError: + hierarchy_json = {"raw": hierarchy} + else: + hierarchy_json = hierarchy + + with open(hierarchy_path, "w", encoding="utf-8") as f: + json.dump(hierarchy_json, f, ensure_ascii=False, indent=2) + + return hierarchy_path + + except Exception as e: + logger.error(f"保存 Step {step_index} 的 UI 层级失败: {e}") + return "" + + def _save_results(self, result: Dict): + """保存任务执行结果""" + actions_data = { + "original_task_description": self.original_task_description, + "task_description": self.task_description, + "device_type": self.device_type, + "action_count": len(self.actions), + "actions": self.actions, + "total_time": self.total_time, + "status": result.get("status", "unknown") + } + + if self.enable_planning: + actions_data["planning_enabled"] = True + if self.app_name: + actions_data["app_name"] = self.app_name + if self.package_name: + actions_data["package_name"] = self.package_name + if self.planning_info: + actions_data["planning_info"] = self.planning_info + + actions_path = os.path.join(self.data_dir, "actions.json") + with open(actions_path, "w", encoding="utf-8") as f: + json.dump(actions_data, f, ensure_ascii=False, indent=2) + + react_path = os.path.join(self.data_dir, "react.json") + with open(react_path, "w", encoding="utf-8") as f: + json.dump(self.reacts, f, ensure_ascii=False, indent=2) + + logger.info(f"结果已保存到 {self.data_dir}") + + def _add_action(self, action_type: str, step_index: int, **kwargs): + """添加动作记录""" + action = { + "type": action_type, + "action_index": step_index, + **kwargs + } + self.actions.append(action) + + def _add_react(self, reasoning: str, action: str, parameters: Dict, step_index: int): + """添加推理记录""" + react = { + "reasoning": reasoning, + "function": { + "name": action, + "parameters": parameters + }, + "action_index": step_index + } + self.reacts.append(react) + + # ---------------------------- + # 绘制方法 + # ---------------------------- + + def _draw_actions_on_image(self, step_index: int, action_seq: List[Dict]): + """绘制动作到截图上""" + src_path = os.path.join(self.data_dir, f"{step_index}.jpg") + dst_path = os.path.join(self.data_dir, f"{step_index}_draw.jpg") + + if not os.path.exists(src_path): + logger.warning(f"源截图不存在: {src_path}") + return + + try: + image = Image.open(src_path) + # 使用原始图像大小进行绘制 + width, height = image.size + modified = False + + # 因为 action_seq 可能包含多个动作，我们将绘制所有可视化动作 + # 并为描述条使用最后一个动作（或聚合） + + last_action_desc = None + + for action in action_seq: + action_type = action.get('type', '').lower() + params = action.get('params', {}) + + # 规范化类型 + if action_type == 'tap': action_type = 'click' + if action_type == 'long_press': action_type = 'longclick' + if action_type == 'double_tap': action_type = 'doubleclick' + if action_type == 'input': action_type = 'input_text' + if action_type in ['launch', 'start_app', 'open']: action_type = 'open_app' + + # 提取坐标 + tap_point = None + gesture = None + + if 'points' in params: + pts = params['points'] + if len(pts) == 2: + tap_point = (pts[0], pts[1]) + elif len(pts) == 4: + gesture = ((pts[0], pts[1]), (pts[2], pts[3])) + elif 'coordinate' in params: + tap_point = (params['coordinate'][0], params['coordinate'][1]) + elif 'start_coordinate' in params and 'end_coordinate' in params: + gesture = ( + (params['start_coordinate'][0], params['start_coordinate'][1]), + (params['end_coordinate'][0], params['end_coordinate'][1]) + ) + elif 'direction' in params: + w, h = width, height + cx, cy = w // 2, h // 2 + d = 300 + direction = params['direction'].lower() + if direction == 'up': gesture = ((cx, cy+d//2), (cx, cy-d//2)) + elif direction == 'down': gesture = ((cx, cy-d//2), (cx, cy+d//2)) + elif direction == 'left': gesture = ((cx+d//2, cy), (cx-d//2, cy)) + elif direction == 'right': gesture = ((cx-d//2, cy), (cx+d//2, cy)) + + # 绘制动作 + if action_type in ['click', 'longclick', 'doubleclick'] and tap_point: + image = self._draw_tap_overlay(image, tap_point) + modified = True + + elif action_type in ['scroll', 'swipe'] and gesture: + image = self._draw_scroll_overlay(image, gesture) + modified = True + + # 准备动作描述 + payload = params.get('text') or params.get('app_name') or str(tap_point or gesture or "") + + detail = f"Action payload: {payload}" + action_label = ACTION_LABELS.get(action_type, action_type.replace("_", " ").title()) + + if action_type in ['click', 'longclick', "doubleclick"]: + verb = action_label + detail = f"{verb} at {tap_point}" if tap_point else f"{verb}" + elif action_type == 'open_app': + detail = f"Open app: {payload or 'Unknown'}" + elif action_type in ['scroll', 'swipe']: + if gesture: + detail = f"Scroll from {gesture[0]} to {gesture[1]}" + else: + detail = f"Scroll {params.get('direction', 'unknown')}" + elif action_type == 'input_text': + detail = f"Input text: {payload}" + elif action_type == 'back': + detail = "Go back to previous screen" + elif action_type == 'home': + detail = "Return to the home screen" + elif action_type == 'wait': + detail = f"Wait for {params.get('seconds', 1)} seconds" + + last_action_desc = { + "label": action_label, + "detail": detail + } + + # 如果有动作记录，则保存 + if last_action_desc: + # 添加描述条 + image = self._add_description_strip(image, last_action_desc) + modified = True + + if modified: + image.convert("RGB").save(dst_path) + logger.info(f"保存可视化图像：{dst_path}") + else: + shutil.copy(src_path, dst_path) + + except Exception as e: + logger.warning(f"绘制动作失败: {e}") + + def _draw_tap_overlay(self, image: Image.Image, coordinates: tuple, radius: int = 20) -> Image.Image: + base = image.convert("RGBA") + overlay = Image.new("RGBA", base.size, (0, 0, 0, 0)) + draw = ImageDraw.Draw(overlay) + x, y = coordinates + left_up = (x - radius, y - radius) + right_down = (x + radius, y + radius) + draw.ellipse([left_up, right_down], fill=RED_DOT_COLOR) + return Image.alpha_composite(base, overlay) + + def _draw_scroll_overlay(self, image: Image.Image, gesture: tuple) -> Image.Image: + base = image.convert("RGBA") + overlay = Image.new("RGBA", base.size, (0, 0, 0, 0)) + draw = ImageDraw.Draw(overlay) + start, end = gesture + draw.line([start, end], fill=SCROLL_COLOR, width=SCROLL_WIDTH) + self._draw_arrow_head(draw, start, end, SCROLL_COLOR) + draw.ellipse( + [ + (start[0] - 15, start[1] - 15), + (start[0] + 15, start[1] + 15) + ], + outline=SCROLL_COLOR, + width=6 + ) + return Image.alpha_composite(base, overlay) + + def _draw_arrow_head(self, draw, start, end, color, size=40): + dx = end[0] - start[0] + dy = end[1] - start[1] + length = math.hypot(dx, dy) + if length == 0: + return + ux, uy = dx / length, dy / length + left = (end[0] - ux * size - uy * size / 2, end[1] - uy * size + ux * size / 2) + right = (end[0] - ux * size + uy * size / 2, end[1] - uy * size - ux * size / 2) + draw.polygon([end, left, right], fill=color) + + def _load_font(self, font_size: int) -> ImageFont.ImageFont: + """加载字体，优先使用 msyh.ttf，失败则使用默认字体""" + font_path = os.path.join(os.path.dirname(__file__),"..", "msyh.ttf") + possible_paths = [ + font_path , + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", + "/System/Library/Fonts/Helvetica.ttc" + ] + + for p in possible_paths: + if os.path.exists(p): + try: + return ImageFont.truetype(p, font_size) + except: + continue + + # 降级使用默认字体 + try: + return ImageFont.load_default() + except: + return None + + def _calculate_characters_per_line(self, image_width: int, font: ImageFont.ImageFont) -> int: + """计算一行能放入的字符数""" + if not font: return 50 + sample_text = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + try: + total_width = 0 + for char in sample_text: + bbox = font.getbbox(char) + total_width += bbox[2] - bbox[0] + average_char_width = max(1, total_width // len(sample_text)) + return max(1, image_width // average_char_width) + except: + # 默认字体降级处理 + return image_width // 15 + + def _add_description_strip(self, image: Image.Image, action_desc: Dict, font_size: int = 40, line_spacing: int = 10) -> Image.Image: + """在图像下方添加动作描述条""" + image = image.convert("RGB") + width, height = image.size + + font = self._load_font(font_size) + characters_per_line = self._calculate_characters_per_line(width, font) + + segments = [ + {"text": f"Action: {action_desc['label']}", "color": "red"}, + {"text": f"Detail: {action_desc['detail']}", "color": "black"} + ] + + wrapped_lines = [] + wrapper = textwrap.TextWrapper(width=characters_per_line, break_long_words=True, break_on_hyphens=False) + + for segment in segments: + lines = segment["text"].splitlines() or [segment["text"]] + for line in lines: + wrapped = wrapper.wrap(line) or [line] + for wrapped_line in wrapped: + wrapped_lines.append((wrapped_line, segment["color"])) + + # 计算文本高度 + text_height = 0 + dummy_draw = ImageDraw.Draw(image) + + for line, _ in wrapped_lines: + if font: + bbox = dummy_draw.textbbox((0, 0), line, font=font) + h_line = bbox[3] - bbox[1] + else: + h_line = 20 + + text_height += h_line + line_spacing + + strip_height = text_height + 20 + accent_height = 10 + + new_image = Image.new("RGB", (width, height + accent_height + strip_height), STRIP_BG_COLOR) + new_image.paste(image, (0, 0)) + draw = ImageDraw.Draw(new_image) + draw.rectangle([(0, height), (width, height + accent_height)], fill=STRIP_ACCENT_COLOR) + + y = height + accent_height + 10 + for line, color in wrapped_lines: + if font: + bbox = draw.textbbox((0, 0), line, font=font) + text_width = bbox[2] - bbox[0] + h_line = bbox[3] - bbox[1] + else: + text_width = len(line) * 8 + h_line = 20 + + x = (width - text_width) // 2 + + if font: + draw.text((x, y), line, font=font, fill=color) + else: + draw.text((x, y), line, fill=color) + + y += h_line + line_spacing + + return new_image + diff --git a/runner/config.json b/runner/config.json new file mode 100644 index 00000000..13fc1a72 --- /dev/null +++ b/runner/config.json @@ -0,0 +1,27 @@ +{ + "mobiagent": { + "service_ip": "localhost", + "decider_port": 8000, + "grounder_port": 8001, + "planner_port": 8002, + "use_qwen3": true, + "use_e2e": true, + "use_experience": false, + "use_graphrag": false + }, + "uitars": { + "model_base_url": "http://localhost:8000/v1", + "model_name": "UI-TARS-7B-SFT", + "temperature": 0.7, + "step_delay": 2.0 + }, + "device": { + "type": "Android", + "device_id": null + }, + "execution": { + "max_steps": 40, + "log_level": "INFO", + "output_dir": "results" + } +} diff --git a/runner/device.py b/runner/device.py new file mode 100644 index 00000000..10358039 --- /dev/null +++ b/runner/device.py @@ -0,0 +1,388 @@ +""" +设备抽象层 +提供 Android 和 HarmonyOS 设备的统一接口 +""" +import time +import base64 +from abc import ABC, abstractmethod +import uiautomator2 as u2 +from hmdriver2.driver import Driver + + +class Device(ABC): + """设备抽象基类""" + + @abstractmethod + def start_app(self, app): + """启动应用（通过应用名）""" + pass + + @abstractmethod + def app_start(self, package_name): + """启动应用（通过包名）""" + pass + + @abstractmethod + def app_stop(self, package_name): + """停止应用""" + pass + + @abstractmethod + def screenshot(self, path): + """截图""" + pass + + @abstractmethod + def click(self, x, y): + """点击坐标""" + pass + + @abstractmethod + def long_click(self, x, y): + """长按坐标""" + pass + + @abstractmethod + def double_click(self, x, y): + """双击坐标""" + pass + + @abstractmethod + def input(self, text): + """输入文本""" + pass + + @abstractmethod + def swipe(self, direction): + """滑动（方向：up/down/left/right）""" + pass + + @abstractmethod + def swipe_with_coords(self, start_x, start_y, end_x, end_y): + """使用坐标滑动""" + pass + + @abstractmethod + def keyevent(self, key): + """按键事件""" + pass + + @abstractmethod + def dump_hierarchy(self): + """获取UI层级""" + pass + + +class AndroidDevice(Device): + """Android 设备实现""" + + def __init__(self, adb_endpoint=None): + super().__init__() + if adb_endpoint: + self.d = u2.connect(adb_endpoint) + else: + # 默认连接第一个设备 + self.d = u2.connect() + + # 常用应用的包名映射 + self.app_package_names = { + # 旅行出行类 + "携程旅行": "ctrip.android.view", + "携程": "ctrip.android.view", + "同程旅行": "com.tongcheng.android", + "同程": "com.tongcheng.android", + "飞猪": "com.taobao.trip", + "去哪儿": "com.Qunar", + "华住会": "com.htinns", + "滴滴出行": "com.sdu.didi.psnger", + "高德地图": "com.autonavi.minimap", + + # 生活服务类 + "饿了么": "me.ele", + "支付宝": "com.eg.android.AlipayGphone", + "美团": "com.sankuai.meituan", + "大众点评": "com.dianping.v1", + + # 购物电商类 + "淘宝": "com.taobao.taobao", + "京东": "com.jingdong.app.mall", + "拼多多": "com.xunmeng.pinduoduo", + "华为商城": "com.vmall.client", + "闲鱼": "com.taobao.idlefish", # 修正原"咸鱼"错别字 + + # 社交通讯类 + "微信": "com.tencent.mm", + "QQ": "com.tencent.mobileqq", + "新浪微博": "com.sina.weibo", + "微博": "com.sina.weibo", + "小红书": "com.xingin.xhs", + + # 影音娱乐类 + "bilibili": "tv.danmaku.bili", + "哔哩哔哩": "tv.danmaku.bili", + "爱奇艺": "com.qiyi.video", + "腾讯视频": "com.tencent.qqlive", + "优酷": "com.youku.phone", + "QQ音乐": "com.tencent.qqmusic", + "网易云音乐": "com.netease.cloudmusic", + "酷狗音乐": "com.kugou.android", + "抖音": "com.ss.android.ugc.aweme", + "快手": "com.smile.gifmaker", + "今日头条": "com.ss.android.article.news", + "知乎": "com.zhihu.android", + "华为视频": "com.huawei.himovie", + "华为音乐": "com.huawei.music", + + # 系统工具类 + "浏览器": "com.microsoft.emmx", + "华为应用市场": "com.huawei.appmarket", + "备忘录": "com.huawei.notepad" + } + + def start_app(self, app): + """通过应用名启动应用""" + package_name = self.app_package_names.get(app) + if not package_name: + raise ValueError(f"App '{app}' is not registered with a package name.") + self.d.app_start(package_name, stop=True) + time.sleep(3) + if not self.d.app_wait(package_name, timeout=10): + raise RuntimeError(f"Failed to start app '{app}' with package '{package_name}'") + + def app_start(self, package_name): + """通过包名启动应用""" + self.d.app_start(package_name, stop=True) + time.sleep(1) + if not self.d.app_wait(package_name, timeout=10): + raise RuntimeError(f"Failed to start package '{package_name}'") + + def app_stop(self, package_name): + """停止应用""" + self.d.app_stop(package_name) + + def screenshot(self, path): + """截图""" + self.d.screenshot(path) + + def click(self, x, y): + """点击坐标""" + self.d.click(x, y) + time.sleep(0.5) + + def long_click(self, x, y): + """长按坐标""" + self.d.long_click(x, y) + time.sleep(0.5) + + def double_click(self, x, y): + """双击坐标""" + self.d.double_click(x, y) + time.sleep(0.5) + + def clear_input(self): + """清除输入框内容""" + self.d.shell(['input', 'keyevent', 'KEYCODE_MOVE_END']) + self.d.shell(['input', 'keyevent', 'KEYCODE_MOVE_HOME']) + self.d.shell(['input', 'keyevent', 'KEYCODE_DEL']) + + def input(self, text): + """输入文本（使用 ADB Keyboard）""" + current_ime = self.d.current_ime() + # 切换到 ADB Keyboard + self.d.shell(['settings', 'put', 'secure', 'default_input_method', 'com.android.adbkeyboard/.AdbIME']) + time.sleep(0.5) + # 清除现有文本 + self.d.shell(['am', 'broadcast', '-a', 'ADB_CLEAR_TEXT']) + time.sleep(0.2) + # 输入文本（使用 base64 编码支持中文） + charsb64 = base64.b64encode(text.encode('utf-8')).decode('utf-8') + self.d.shell(['am', 'broadcast', '-a', 'ADB_INPUT_B64', '--es', 'msg', charsb64]) + time.sleep(0.5) + # 恢复原输入法 + self.d.shell(['settings', 'put', 'secure', 'default_input_method', current_ime]) + time.sleep(0.5) + + def swipe(self, direction, scale=0.5): + """滑动（方向：up/down/left/right）""" + if direction.lower() == "up": + self.d.swipe(0.5, 0.7, 0.5, 0.3,duration=0.5) + elif direction.lower() == "down": + self.d.swipe(0.5, 0.3, 0.5, 0.7,duration=0.5) + elif direction.lower() == "left": + self.d.swipe(0.7, 0.5, 0.3, 0.5,duration=0.5) + elif direction.lower() == "right": + self.d.swipe(0.3, 0.5, 0.7, 0.5,duration=0.5) + + def swipe_with_coords(self, start_x, start_y, end_x, end_y): + """使用绝对坐标滑动""" + self.d.swipe(start_x, start_y, end_x, end_y,duration=0.5) + + def keyevent(self, key): + """按键事件""" + self.d.keyevent(key) + + def dump_hierarchy(self): + """获取UI层级（XML格式）""" + return self.d.dump_hierarchy() + + +class HarmonyDevice(Device): + """HarmonyOS 设备实现""" + + def __init__(self): + super().__init__() + self.d = Driver() + + # 常用应用的包名映射 + self.app_package_names = { + # 旅行出行类 + "携程旅行": "com.ctrip.harmonynext", + "携程": "com.ctrip.harmonynext", + "飞猪旅行": "com.fliggy.hmos", + "飞猪": "com.fliggy.hmos", + "同程旅行": "com.tongcheng.hmos", + "同程": "com.tongcheng.hmos", + "航旅纵横": "com.umetrip.hm.app", + "慧通差旅": "com.smartcom.itravelhm", + "滴滴出行": "com.sdu.didi.hmos.psnger", + + # 生活服务类 + "饿了么": "me.ele.eleme", + "美团": "com.sankuai.hmeituan", + "美团外卖": "com.meituan.takeaway", + "大众点评": "com.sankuai.dianping", + "支付宝": "com.alipay.mobile.client", + "微信": "com.tencent.wechat", + "天气": "com.huawei.hmsapp.totemweather", + "什么值得买": "com.smzdm.client.hmos", + + # 购物电商类 + "淘宝": "com.taobao.taobao4hmos", + "京东": "com.jd.hm.mall", + "闲鱼": "com.taobao.idlefish4ohos", + "拼多多": "com.xunmeng.pinduoduo.hos", + "华为商城": "com.huawei.hmos.vmall", + "高德地图": "com.amap.hmapp", + + # 社交娱乐类 + "知乎": "com.zhihu.hmos", + "哔哩哔哩": "yylx.danmaku.bili", + "小红书": "com.xingin.xhs_hos", + "微博": "com.sina.weibo.stage", + "QQ音乐": "com.tencent.hm.qqmusic", + "豆包": "com.larus.nova.hm", + "懂车帝": "com.ss.dcar.auto", + + # 华为系统/应用类 + "电子邮件": "com.huawei.hmos.email", + "图库": "com.huawei.hmos.photos", + "日历": "com.huawei.hmos.calendar", + "心声社区": "com.huawei.it.hmxinsheng", + "信息": "com.ohos.mms", + "文件管理": "com.huawei.hmos.files", + "运动健康": "com.huawei.hmos.health", + "智慧生活": "com.huawei.hmos.ailife", + "WeLink": "com.huawei.it.welink", + "设置": "com.huawei.hmos.settings", + "浏览器": "com.huawei.hmos.browser", + "华为阅读": "com.huawei.hmsapp.books", + + # 其他类 + "PowerAgent": "com.example.osagent" + } + + def start_app(self, app): + """通过应用名启动应用""" + package_name = self.app_package_names.get(app) + if not package_name: + raise ValueError(f"App '{app}' is not registered with a package name.") + self.d.start_app(package_name) + time.sleep(2) + + def app_start(self, package_name): + """通过包名启动应用（强制启动）""" + self.d.force_start_app(package_name) + time.sleep(1.5) + + def app_stop(self, package_name): + """停止应用""" + self.d.stop_app(package_name) + + def screenshot(self, path): + """截图""" + self.d.screenshot(path) + + def click(self, x, y): + """点击坐标""" + self.d.click(x, y) + time.sleep(0.5) + + def long_click(self, x, y): + """长按坐标""" + self.d.long_click(x, y) + time.sleep(0.5) + + def double_click(self, x, y): + """双击坐标""" + self.d.double_click(x, y) + time.sleep(0.5) + + def input(self, text): + """输入文本""" + # 触发键盘 + self.d.shell("uitest uiInput keyEvent 2072 2017") + # 清除旧内容 + self.d.press_key(2071) + # 输入新文本 + self.d.input_text(text) + + def swipe(self, direction, scale=0.5): + """滑动（方向：up/down/left/right）""" + if direction.lower() == "up": + self.d.swipe(0.5, 0.7, 0.5, 0.3, speed=2000) + elif direction.lower() == "down": + self.d.swipe(0.5, 0.3, 0.5, 0.7, speed=2000) + elif direction.lower() == "left": + self.d.swipe(0.7, 0.5, 0.3, 0.5, speed=2000) + elif direction.lower() == "right": + self.d.swipe(0.3, 0.5, 0.7, 0.5, speed=2000) + + def swipe_with_coords(self, start_x, start_y, end_x, end_y): + """使用绝对坐标滑动""" + self.d.swipe(start_x, start_y, end_x, end_y, speed=2000) + + def keyevent(self, key): + from hmdriver2.proto import KeyCode + """按键事件""" + if key == "HOME": + self.d.go_home() + elif key == "BACK": + self.d.go_back() + elif key == "RECENTS": + self.d.press_key(KeyCode.ENTER) + + + # self.d.press_key(key) + + def dump_hierarchy(self): + """获取UI层级（JSON格式）""" + return self.d.dump_hierarchy() + + +def create_device(device_type, adb_endpoint=None): + """ + 创建设备实例的工厂方法 + + Args: + device_type: "Android" 或 "Harmony" + adb_endpoint: Android 设备的 ADB 端点（可选） + + Returns: + Device 实例 + """ + if device_type.lower() == "android": + return AndroidDevice(adb_endpoint) + elif device_type.lower() == "harmony": + return HarmonyDevice() + else: + raise ValueError(f"Unsupported device type: {device_type}") diff --git a/runner/examples/run_autoglm.sh b/runner/examples/run_autoglm.sh new file mode 100755 index 00000000..1c41afaa --- /dev/null +++ b/runner/examples/run_autoglm.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# 示例: 使用 AutoGLM 执行任务 + +python run.py \ + --provider autoglm \ + --api-base http://localhost:9003/v1 \ + --model autoglm-phone-9b \ + --max-steps 20 \ + --output-dir results \ + --draw \ + --task "打开小红书搜索博主影视飓风并查看他的主页第一条内容" \ + --device-type Harmony diff --git a/runner/examples/run_qwen.sh b/runner/examples/run_qwen.sh new file mode 100755 index 00000000..bad65f31 --- /dev/null +++ b/runner/examples/run_qwen.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +python run.py \ + --provider qwen \ + --output-dir results \ + --qwen-api-base http://localhost:8080/v1 \ + --max-steps 20 \ + --draw \ + --task "帮我在微博浏览查看央视新闻今天发了什么，查看前2条微博" \ No newline at end of file diff --git a/runner/examples/run_single_mobiagent-e2e.sh b/runner/examples/run_single_mobiagent-e2e.sh new file mode 100755 index 00000000..8bc29445 --- /dev/null +++ b/runner/examples/run_single_mobiagent-e2e.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# 示例1: 使用MobiAgent执行单个任务 + +python run.py \ + --provider mobiagent \ + --service-ip localhost \ + --decider-port 9002 \ + --grounder-port 9002 \ + --planner-port 8080 \ + --max-steps 30 \ + --output-dir results \ + --enable-planning \ + --use-e2e \ + --task "在淘宝上搜索电动牙刷，选最畅销的那款" \ + --device-type Harmony \ No newline at end of file diff --git a/runner/examples/run_single_mobiagent.sh b/runner/examples/run_single_mobiagent.sh new file mode 100755 index 00000000..8116b7c5 --- /dev/null +++ b/runner/examples/run_single_mobiagent.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# 示例1: 使用MobiAgent执行单个任务 + +python run.py \ + --provider mobiagent_step \ + --task "在淘宝上搜索电动牙刷，选最畅销的那款并加入购物车" \ + --service-ip localhost \ + --decider-port 9002 \ + --grounder-port 9002 \ + --planner-port 8080 \ + --enable-planning \ + --max-steps 30 \ + --draw \ + --output-dir results diff --git a/runner/examples/run_single_mobiagent_no_planning.sh b/runner/examples/run_single_mobiagent_no_planning.sh new file mode 100644 index 00000000..380c74e1 --- /dev/null +++ b/runner/examples/run_single_mobiagent_no_planning.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# 示例: 使用MobiAgent执行任务（不启用planning，需手动在任务描述中指定APP） + +python run.py \ + --provider mobiagent \ + --api-base http://localhost:9002/v1 \ + --max-steps 30 \ + --output-dir results \ + --task "在淘宝上搜索电动牙刷，选最畅销的那款" \ No newline at end of file diff --git a/runner/examples/run_uitars_batch.sh b/runner/examples/run_uitars_batch.sh new file mode 100644 index 00000000..1c1bb4b1 --- /dev/null +++ b/runner/examples/run_uitars_batch.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# 示例4: 使用UI-TARS批量执行MobiFlow任务 + +python run.py \ + --provider uitars \ + --task-file mobiagent/task_mobiflow.json \ + --model-url http://localhost:8000/v1 \ + --model-name UI-TARS-7B-SFT \ + --max-steps 25 \ + --output-dir results/uitars_batch \ + --log-level INFO diff --git a/runner/examples/run_uitars_single.sh b/runner/examples/run_uitars_single.sh new file mode 100644 index 00000000..8e5c06af --- /dev/null +++ b/runner/examples/run_uitars_single.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# 示例: 使用 UI-TARS 执行任务 + +python run.py \ + --provider uitars \ + --api-base http://localhost:9003/v1 \ + --model UI-TARS-1.5-7B \ + --max-steps 25 \ + --output-dir results \ + --draw \ + --task "在淘宝上搜索电动牙刷" diff --git a/runner/mobiagent/mobiagent.py b/runner/mobiagent/mobiagent.py index 2fd20939..f5705756 100644 --- a/runner/mobiagent/mobiagent.py +++ b/runner/mobiagent/mobiagent.py @@ -68,6 +68,10 @@ def input(self, text): def swipe(self, direction): pass + @abstractmethod + def swipe_with_coords(self, start_x, start_y, end_x, end_y): + pass + @abstractmethod def keyevent(self, key): pass @@ -155,7 +159,19 @@ def input(self, text): def swipe(self, direction, scale=0.5): # self.d.swipe_ext(direction, scale) - self.d.swipe_ext(direction=direction, scale=scale) + # self.d.swipe_ext(direction=direction, scale=scale) + if direction.lower() == "up": + self.d.swipe(0.5,0.7,0.5,0.3) + elif direction.lower() == "down": + self.d.swipe(0.5,0.3,0.5,0.7) + elif direction.lower() == "left": + self.d.swipe(0.7,0.5,0.3,0.5) + elif direction.lower() == "right": + self.d.swipe(0.3,0.5,0.7,0.5) + + def swipe_with_coords(self, start_x, start_y, end_x, end_y): + """Swipe from (start_x, start_y) to (end_x, end_y)""" + self.d.swipe(start_x, start_y, end_x, end_y) def keyevent(self, key): self.d.keyevent(key) @@ -250,6 +266,12 @@ def swipe(self, direction, scale=0.5): elif direction.lower() == "right": self.d.swipe(0.3,0.5,0.7,0.5,speed=2000) + def swipe_with_coords(self, start_x, start_y, end_x, end_y): + """Swipe from (start_x, start_y) to (end_x, end_y)""" + # Convert absolute coordinates to normalized coordinates if needed + # For Harmony Device, swipe expects coordinates in format (x, y, x, y) + self.d.swipe(start_x, start_y, end_x, end_y, speed=2000) + def keyevent(self, key): self.d.press_key(key) @@ -264,6 +286,7 @@ def dump_hierarchy(self): decider_model = "" grounder_model = "" + # 全局偏好提取器 preference_extractor = None def init(service_ip, decider_port, grounder_port, planner_port, enable_user_profile=False, use_graphrag=False): @@ -294,8 +317,6 @@ def init(service_ip, decider_port, grounder_port, planner_port, enable_user_prof # 截图缩放比例 factor = 0.5 - - def parse_json_response(response_str: str, is_guided_decoding: bool = True) -> dict: """ 解析 JSON 响应，支持 guided decoding 和普通模式 @@ -419,7 +440,7 @@ def convert_qwen3_coordinates_to_absolute(bbox_or_coords, img_width, img_height, y = int(y / 1000 * img_height) return [x, y] -def create_swipe_visualization(data_dir, image_index, direction): +def create_swipe_visualization(data_dir, image_index, direction, start_x=None, start_y=None, end_x=None, end_y=None): """为滑动动作创建可视化图像""" try: # 读取原始截图 @@ -433,24 +454,29 @@ def create_swipe_visualization(data_dir, image_index, direction): height, width = img.shape[:2] - # 根据方向计算箭头起点和终点 - center_x, center_y = width // 2, height // 2 - arrow_length = min(width, height) // 4 - - if direction == "up": - start_point = (center_x, center_y + arrow_length // 2) - end_point = (center_x, center_y - arrow_length // 2) - elif direction == "down": - start_point = (center_x, center_y - arrow_length // 2) - end_point = (center_x, center_y + arrow_length // 2) - elif direction == "left": - start_point = (center_x + arrow_length // 2, center_y) - end_point = (center_x - arrow_length // 2, center_y) - elif direction == "right": - start_point = (center_x - arrow_length // 2, center_y) - end_point = (center_x + arrow_length // 2, center_y) + # 如果提供了具体坐标，使用具体坐标；否则根据方向计算 + if start_x is not None and start_y is not None and end_x is not None and end_y is not None: + start_point = (int(start_x), int(start_y)) + end_point = (int(end_x), int(end_y)) else: - return + # 根据方向计算箭头起点和终点 + center_x, center_y = width // 2, height // 2 + arrow_length = min(width, height) // 4 + + if direction == "up": + start_point = (center_x, center_y + arrow_length // 2) + end_point = (center_x, center_y - arrow_length // 2) + elif direction == "down": + start_point = (center_x, center_y - arrow_length // 2) + end_point = (center_x, center_y + arrow_length // 2) + elif direction == "left": + start_point = (center_x + arrow_length // 2, center_y) + end_point = (center_x - arrow_length // 2, center_y) + elif direction == "right": + start_point = (center_x - arrow_length // 2, center_y) + end_point = (center_x + arrow_length // 2, center_y) + else: + return # 绘制箭头 cv2.arrowedLine(img, start_point, end_point, (255, 0, 0), 8, tipLength=0.3) # 蓝色箭头 @@ -527,16 +553,21 @@ def robust_json_loads(s): logging.error(f"原始内容: {s[:300]}...") raise ValueError(f"无法解析 JSON 响应: 响应格式不正确") -def task_in_app(app, old_task, task, device, data_dir, bbox_flag=True, use_qwen3=True, device_type="Android"): +def task_in_app(app, old_task, task, device, data_dir, bbox_flag=True, use_qwen3=True, device_type="Android", use_e2e=False): history = [] actions = [] reacts = [] + if use_e2e: + # 在e2e模式下使用e2e_qwen3.md，否则使用decider_v2.md + decider_prompt_template = load_prompt("e2e_qwen3.md") + logging.info("Using e2e mode with e2e_qwen3.md") - if use_qwen3: + elif use_qwen3: grounder_prompt_template_bbox = load_prompt("grounder_qwen3_bbox.md") grounder_prompt_template_no_bbox = load_prompt("grounder_qwen3_coordinates.md") - # decider_prompt_template = load_prompt("decider_qwen3.md") + decider_prompt_template = load_prompt("decider_v2.md") + else: grounder_prompt_template_bbox = load_prompt("grounder_bbox.md") grounder_prompt_template_no_bbox = load_prompt("grounder_coordinates.md") @@ -557,9 +588,8 @@ def task_in_app(app, old_task, task, device, data_dir, bbox_flag=True, use_qwen3 task=task, history=history_str ) - logging.info(f"Decider prompt: \n{decider_prompt}") + # logging.info(f"Decider prompt: \n{decider_prompt}") - decider_start_time = time.time() # --- 修改 API 调用 --- # vLLM 将会强制输出一个符合 ActionPlan 结构的 JSON 字符串 # 若响应超时或者返回结果解析失败，则进行重试 @@ -583,6 +613,7 @@ def task_in_app(app, old_task, task, device, data_dir, bbox_flag=True, use_qwen3 timeout=30, max_tokens=256, + response_format={"type": "json_object"} ).choices[0].message.content decider_end_time = time.time() logging.info(f"[evaluation] Decider time taken: {decider_end_time - decider_start_time} seconds") @@ -628,15 +659,24 @@ def task_in_app(app, old_task, task, device, data_dir, bbox_flag=True, use_qwen3 pass # 根据设备类型保存hierarchy + if device_type == "Android": logging.info("Dumping UI hierarchy...") - hierarchy = device.dump_hierarchy() + try: + hierarchy = device.dump_hierarchy() + except Exception as e: + logging.error(f"Failed to dump UI hierarchy: {e}") + hierarchy = "" # Android设备保存为XML格式 hierarchy_path = os.path.join(data_dir, f"{image_index}.xml") with open(hierarchy_path, "w", encoding="utf-8") as f: f.write(hierarchy) else: - hierarchy = device.dump_hierarchy() + try: + hierarchy = device.dump_hierarchy() + except Exception as e: + logging.error(f"Failed to dump UI hierarchy: {e}") + hierarchy = {} # Harmony设备保存为JSON格式 hierarchy_path = os.path.join(data_dir, f"{image_index}.json") try: @@ -665,56 +705,80 @@ def task_in_app(app, old_task, task, device, data_dir, bbox_flag=True, use_qwen3 if action == "click": reasoning = decider_response["reasoning"] target_element = decider_response["parameters"]["target_element"] - grounder_prompt = (grounder_prompt_template_bbox if bbox_flag else grounder_prompt_template_no_bbox).format(reasoning=reasoning, description=target_element) - - # 重试5次获取grounder响应，同时调整temperature - temperature = 0.0 - for attempt in range(5): - try: - grounder_start_time = time.time() - grounder_response_str = grounder_client.chat.completions.create( - model=grounder_model, - messages=[ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{screenshot_resize}"}}, - {"type": "text", "text": grounder_prompt}, - ] - } - ], - temperature=temperature, - timeout=30, - max_tokens=128, - ).choices[0].message.content - grounder_end_time = time.time() - logging.info(f"[evaluation] Grounder time taken: {grounder_end_time - grounder_start_time} seconds") - logging.info(f"Grounder response: \n{grounder_response_str}") - # grounder_response = json.loads(grounder_response_str) - grounder_response = parse_json_response(grounder_response_str) - break # 成功获取响应，跳出重试循环 - except Exception as e: - temperature = 0.1 + attempt * 0.1 # 每次重试时增加温度，增加多样性 - logging.error(f"Grounder 调用失败: {e}, 正在重试 temperature={temperature}...") - time.sleep(2) - - if(bbox_flag): - # 直接尝试获取含有bbox的字段，而不要求完全匹配 - bbox = None - for key in grounder_response: - if key.lower() in ["bbox", "bbox_2d", "bbox-2d", "bbox_2D", "bbox2d", "bbox_2009"]: - bbox = grounder_response[key] - break + + # e2e模式：直接从decider获取bbox，不调用grounder + if use_e2e: + bbox = decider_response["parameters"]["bbox"] + if bbox is None: + logging.error("E2E mode: bbox not found in decider response") + raise ValueError("E2E mode requires bbox in decider response") - - # 如果使用 Qwen3 模型，进行坐标转换 + logging.info(f"E2E mode: Using bbox directly from decider: {bbox}") + # 使用 Qwen3 模型进行坐标转换 if use_qwen3: bbox = convert_qwen3_coordinates_to_absolute(bbox, img.width, img.height, is_bbox=True) - x1, y1, x2, y2 = bbox - else: - x1, y1, x2, y2 = [int(coord/factor) for coord in bbox] + x1, y1, x2, y2 = bbox + else: + # 非e2e模式：调用grounder获取bbox + grounder_prompt = (grounder_prompt_template_bbox if bbox_flag else grounder_prompt_template_no_bbox).format(reasoning=reasoning, description=target_element) - + # 重试5次获取grounder响应，同时调整temperature + temperature = 0.0 + for attempt in range(5): + try: + grounder_start_time = time.time() + grounder_response_str = grounder_client.chat.completions.create( + model=grounder_model, + messages=[ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{screenshot_resize}"}}, + {"type": "text", "text": grounder_prompt}, + ] + } + ], + temperature=temperature, + timeout=30, + max_tokens=128, + response_format={"type": "json_object"} + ).choices[0].message.content + grounder_end_time = time.time() + logging.info(f"[evaluation] Grounder time taken: {grounder_end_time - grounder_start_time} seconds") + logging.info(f"Grounder response: \n{grounder_response_str}") + grounder_response = parse_json_response(grounder_response_str) + break # 成功获取响应，跳出重试循环 + except Exception as e: + temperature = 0.1 + attempt * 0.1 # 每次重试时增加温度，增加多样性 + logging.error(f"Grounder 调用失败: {e}, 正在重试 temperature={temperature}...") + time.sleep(2) + + if(bbox_flag): + # 直接尝试获取含有bbox的字段，而不要求完全匹配 + bbox = None + for key in grounder_response: + if key.lower() in ["bbox", "bbox_2d", "bbox-2d", "bbox_2D", "bbox2d", "bbox_2009"]: + bbox = grounder_response[key] + break + + + # 如果使用 Qwen3 模型，进行坐标转换 + if use_qwen3: + bbox = convert_qwen3_coordinates_to_absolute(bbox, img.width, img.height, is_bbox=True) + x1, y1, x2, y2 = bbox + else: + x1, y1, x2, y2 = [int(coord/factor) for coord in bbox] + + else: + coordinates = grounder_response["coordinates"] + if use_qwen3: + coordinates = convert_qwen3_coordinates_to_absolute(coordinates, img.width, img.height, is_bbox=False) + x, y = coordinates + else: + x, y = [int(coord / factor) for coord in coordinates] + + # 通用的click处理逻辑（e2e和非e2e都使用） + if bbox_flag or use_e2e: print(f"Clicking on bbox: [{x1}, {y1}, {x2}, {y2}]") print(f"Image size: width={img.width}, height={img.height}") print(f"Adjusted bbox: [{x1}, {y1}, {x2}, {y2}]") @@ -758,14 +822,8 @@ def task_in_app(app, old_task, task, device, data_dir, bbox_flag=True, use_qwen3 # 保存带点击点的图像 click_point_path = os.path.join(data_dir, f"{image_index}_click_point.jpg") cv2.imwrite(click_point_path, cv2image) - else: - coordinates = grounder_response["coordinates"] - if use_qwen3: - coordinates = convert_qwen3_coordinates_to_absolute(coordinates, img.width, img.height, is_bbox=False) - x, y = coordinates - else: - x, y = [int(coord / factor) for coord in coordinates] + # 非bbox_flag的情况（使用coordinates） device.click(x, y) actions.append({ "type": "click", @@ -790,45 +848,81 @@ def task_in_app(app, old_task, task, device, data_dir, bbox_flag=True, use_qwen3 elif action == "swipe": direction = decider_response["parameters"]["direction"] direction = direction.upper() - - if direction == "DOWN": - device.swipe(direction.lower(), 0.4) - # record the swipe as an action (index only) - actions.append({ - "type": "swipe", - "press_position_x": None, - "press_position_y": None, - "release_position_x": None, - "release_position_y": None, - "direction": direction.lower(), - "action_index": image_index - }) - - history.append(decider_response_str) - - # 为向下滑动创建可视化 - create_swipe_visualization(data_dir, image_index, direction.lower()) - continue - - if direction in ["UP", "LEFT", "RIGHT"]: - device.swipe(direction.lower(), 0.4) - actions.append({ - "type": "swipe", - "press_position_x": None, - "press_position_y": None, - "release_position_x": None, - "release_position_y": None, - "direction": direction.lower(), - "action_index": image_index - }) - - history.append(decider_response_str) + + # e2e模式：尝试获取起始和结束坐标 + if use_e2e: + start_coords = decider_response["parameters"].get("start_coords") + end_coords = decider_response["parameters"].get("end_coords") - # 为滑动创建可视化 - create_swipe_visualization(data_dir, image_index, direction.lower()) - + if start_coords and end_coords: + # 进行坐标转换（如果需要） + if use_qwen3: + start_coords = convert_qwen3_coordinates_to_absolute(start_coords, img.width, img.height, is_bbox=False) + end_coords = convert_qwen3_coordinates_to_absolute(end_coords, img.width, img.height, is_bbox=False) + + start_x, start_y = start_coords + end_x, end_y = end_coords + + logging.info(f"E2E mode: swipe from [{start_x}, {start_y}] to [{end_x}, {end_y}]") + device.swipe_with_coords(start_x, start_y, end_x, end_y) + + actions.append({ + "type": "swipe", + "press_position_x": start_x, + "press_position_y": start_y, + "release_position_x": end_x, + "release_position_y": end_y, + "direction": direction.lower(), + "action_index": image_index + }) + history.append(decider_response_str) + create_swipe_visualization(data_dir, image_index, direction.lower(), start_x, start_y, end_x, end_y) + else: + logging.warning("E2E mode: start_coords or end_coords not found, falling back to direction-based swipe") + # 回退到方向based swipe + if direction == "DOWN": + device.swipe(direction.lower(), 0.4) + press_position_x = img.width * 0.3 + press_position_y = img.height * 0.5 + release_position_x = img.width * 0.7 + release_position_y = img.height * 0.5 + elif direction in ["UP", "LEFT", "RIGHT"]: + device.swipe(direction.lower(), 0.4) + + else: + raise ValueError(f"Unknown swipe direction: {direction}") + + actions.append({ + "type": "swipe", + "press_position_x": None, + "press_position_y": None, + "release_position_x": None, + "release_position_y": None, + "direction": direction.lower(), + "action_index": image_index + }) + history.append(decider_response_str) + create_swipe_visualization(data_dir, image_index, direction.lower()) else: - raise ValueError(f"Unknown swipe direction: {direction}") + if direction in ["DOWN", "UP", "LEFT", "RIGHT"]: + device.swipe(direction.lower(), 0.4) + actions.append({ + "type": "swipe", + "press_position_x": None, + "press_position_y": None, + "release_position_x": None, + "release_position_y": None, + "direction": direction.lower(), + "action_index": image_index + }) + + history.append(decider_response_str) + + # 为滑动创建可视化 + create_swipe_visualization(data_dir, image_index, direction.lower()) + + else: + raise ValueError(f"Unknown swipe direction: {direction}") elif action == "wait": print("Waiting for a while...") actions.append({ @@ -942,7 +1036,7 @@ def get_app_package_name(task_description, use_graphrag=False, device_type="Andr return app_name, package_name, final_desc -def execute_single_task(task_description, device, data_dir, use_experience, use_graphrag, current_device_type, use_qwen3_model): +def execute_single_task(task_description, device, data_dir, use_experience, use_graphrag, current_device_type, use_qwen3_model, use_e2e=False): """ 执行单个任务的通用函数 @@ -954,6 +1048,7 @@ def execute_single_task(task_description, device, data_dir, use_experience, use_ use_graphrag: 是否使用GraphRAG current_device_type: 设备类型 use_qwen3_model: 是否使用Qwen3模型 + use_e2e: 是否使用e2e模式（skip grounder调用） """ # 调用 planner 获取应用名称和包名 logging.info(f"Calling planner to get app_name and package_name") @@ -972,7 +1067,7 @@ def execute_single_task(task_description, device, data_dir, use_experience, use_ logging.info(f"Starting task in app: {app_name} (package: {package_name})") device.app_start(package_name) - task_in_app(app_name, task_description, new_task_description, device, data_dir, True, use_qwen3_model, current_device_type) + task_in_app(app_name, task_description, new_task_description, device, data_dir, True, use_qwen3_model, current_device_type, use_e2e) logging.info(f"Stopping app: {app_name} (package: {package_name})") device.app_stop(package_name) @@ -993,6 +1088,7 @@ def execute_single_task(task_description, device, data_dir, use_experience, use_ parser.add_argument("--use_experience", action="store_true", default=False, help="Whether to use experience (use planner for task rewriting) (default: False)") parser.add_argument("--data_dir", type=str, default=None, help="Directory to save data (default: ./data relative to script location)") parser.add_argument("--task_file", type=str, default=None, help="Path to task.json file (default: ./task.json relative to script location)") + parser.add_argument("--e2e", action="store_true", default=False, help="Enable e2e mode: use e2e_qwen3.md as decider prompt and return coordinates directly from decider (default: False)") args = parser.parse_args() # 使用命令行参数初始化 @@ -1029,6 +1125,7 @@ def execute_single_task(task_description, device, data_dir, use_experience, use_ logging.info(f"Use Qwen3 model: {use_qwen3_model}") logging.info(f"Use experience (planner task rewriting): {use_experience}") logging.info(f"Device type: {current_device_type}") + logging.info(f"Use E2E mode: {args.e2e}") # 配置数据保存目录 if args.data_dir: data_base_dir = args.data_dir @@ -1070,7 +1167,7 @@ def execute_single_task(task_description, device, data_dir, use_experience, use_ os.makedirs(data_dir, exist_ok=True) logging.info(f"Processing task {task_index} of {app_name_from_file}/{task_type}: {task_description}") - execute_single_task(task_description, device, data_dir, use_experience, use_graphrag, current_device_type, use_qwen3_model) + execute_single_task(task_description, device, data_dir, use_experience, use_graphrag, current_device_type, use_qwen3_model, args.e2e) else: # 旧格式：简单任务列表 existing_dirs = [d for d in os.listdir(data_base_dir) if os.path.isdir(os.path.join(data_base_dir, d)) and d.isdigit()] @@ -1082,7 +1179,7 @@ def execute_single_task(task_description, device, data_dir, use_experience, use_ os.makedirs(data_dir, exist_ok=True) task_description = task_item - execute_single_task(task_description, device, data_dir, use_experience, use_graphrag, current_device_type, use_qwen3_model) + execute_single_task(task_description, device, data_dir, use_experience, use_graphrag, current_device_type, use_qwen3_model, args.e2e) # 等待所有偏好提取任务完成 if preference_extractor and hasattr(preference_extractor, 'executor'): diff --git a/runner/mobiagent/task.json b/runner/mobiagent/task.json index 818947aa..4f842b47 100644 --- a/runner/mobiagent/task.json +++ b/runner/mobiagent/task.json @@ -1,27 +1,81 @@ [ - "在淘宝上搜索电动牙刷，选最畅销的那款", - "用淘宝搜一下苹果笔记本电脑", - "用淘宝查一下哪个蓝牙耳机最便宜", - "在淘宝上搜索男士休闲夹克", - "用淘宝查一下哪款空气净化器销量第一", - "用淘宝帮我找一下耐克运动鞋，然后放进购物车", - "在淘宝上将香奈儿口红加入购物车", - "用淘宝帮我搜一下小天鹅洗衣机，再放进购物车", - "用淘宝找一下荣耀Magic6手机，并加到购物车", - "淘宝查找iPad Pro，然后把它放进购物车", - "用淘宝把最便宜的蓝牙音箱加到购物车", - "淘宝将天猫自营的佳能单反相机加入购物车", - "淘宝请将销量最高的那款跑步机放进购物车", - "淘宝请将售价最高的那款奢侈品手表添加到购物车", - "在淘宝上，将销量最高的华为Mate 70加入购物车", - "用淘宝将L码的优衣库短袖T恤添加到购物车", - "用淘宝将M码的运动鞋加入购物车", - "在淘宝上，将12号的足球加入购物车", - "淘宝选中黑色的iPhone 14 Pro，然后加到购物车里", - "在淘宝上将1米的Anker USB-C数据线加入购物车", - "在淘宝上，将价格最低的64GB蓝色iPhone 15加入购物车", - "打开淘宝，找到大号的北欧风地毯，并把销量最高的那款放进购物车", - "用淘宝找到金色32GB优盘里价格最高的那一款并加购", - "在淘宝上，将销量最高的粉色AirPods保护壳加到购物车里", - "在淘宝上，将售价最便宜的2米Type-C快充线加入购物车" + "打开饿了么给我找一杯1点点的金桔柠檬加入购物车", + "在饿了么帮我点3份老乡鸡的西红柿炒蛋", + "饿了么里帮我点2份蜜雪冰城的葡萄冰美式，要少冰，七分糖", + + "在淘宝上购买销量第一的M码男士卫衣", + "淘宝中买一个价格最低的16GB+512GB黑色的小米15Ultra手机", + "帮我在淘宝买将型号为42码的阿迪达斯跑鞋", + "淘宝中买销量最高，价格在100-200元之间的男士夹克", + "帮我在淘宝买一个150到300元左右的电风扇", + "帮我在淘宝上买一个300到600元左右的智能手表", + "在淘宝买一台75英寸 4K超清的索尼电视", + "帮我在淘宝购买销量最高的蓝牙耳机", + "淘宝中购买价格最低的16GB+512GB白色的小米15Ultra手机", + + "用京东帮我买一台1.5匹新一级能效的格力空调", + "帮我在京东上买一个300到600元左右的智能手表", + "帮我在京东买一个150到300元左右的电风扇", + "在京东买一台75英寸 4K超清的索尼电视", + "帮我在京东中购买AirPodspro，选择销量最高的", + "帮我在京东购买销量最高的蓝牙耳机", + "京东中购买价格最低的16GB+512GB白色的小米15Ultra手机", + "在京东购买型号为40码的阿迪达斯跑鞋", + "京东中查找价格在100-200元之间的销量最高的男士夹克并购买", + + "用哔哩哔哩查找视频当我在假期打开英雄联盟", + "用b站浏览“老师好我叫何同学”的个人主页", + "请你在B站帮我到影视飓风的iPhone17测评视频下评论，无限进步！抽我抽我！", + + "飞猪查询上海的豫园附近的酒店价格，⼊住时间为12⽉20⽇到12⽉22⽇", + "飞猪查询成都春熙路附近的全季酒店大床房价格，入住时间为12月13日至12月15日", + "飞猪查询从北京到上海的机票，出发时间为12月20日18:00-24:00之间", + "飞猪查询12月13日从昆明到大连的机票，航班到达时间为18:00-24:00之间", + + "携程查询上海的豫园附近的酒店价格，⼊住时间为12⽉20⽇到12月22⽇", + "用携程查询成都春熙路附近的全季酒店大床房价格，入住时间为12月13日至12月15日", + "用携程查询从上海到北京的机票，出发时间为12月20日18:00-24:00之间", + "用携程帮我搜索桂林到天津的飞机票，要求12月20号出发，在晚上18点到24点之间到达", + + "打开饿了么给我找一杯1点点的金桔柠檬加入购物车", + "在饿了么帮我点3份老乡鸡的西红柿炒蛋", + "饿了么里帮我点2份蜜雪冰城的葡萄冰美式，要少冰，七分糖", + + "在淘宝上把销量第一的M码男士卫衣添加到购物车", + "淘宝中将价格最低的16GB+512GB黑色的小米14Ultra手机加入购物车", + "淘宝中将型号为42码的阿迪达斯跑鞋加入购物车", + "淘宝中查找价格在100-200元之间的男士夹克，选择销量最高的加入购物车", + + "在京东中搜索AirPodspro，选择销量最高的商品加入购物车", + "京东中将价格最低的16GB+512GB白色的小米14Ultra手机加入购物车", + "京东中将型号为40码的阿迪达斯跑鞋加入购物车", + "京东中查找价格在100-200元之间的男士夹克，选择销量最高的加入购物车", + + "在小红书里查看博主“王冰冰”的最新笔记。", + "在小红书里关注博主酸酸甜甜就是我", + "在小红书里给博主“影视飓风”的最新视频点赞并收藏", + + "请你进入淘宝，搜索影石Insta360店铺，选择第一个店铺，进入后在店铺内搜索instago360B，选择默认配置并加入购物车。", + "请你进入淘宝，搜索影石Insta360店铺，进入后在店铺内搜索instago360B，选择默认配置并加入购物车。", + "请你进入淘宝，搜索影石Insta360店铺，选择店铺标签，选择第一个店铺，进入后在店铺内搜索instago360B，选择默认配置并立即购买。", + + "帮我用淘宝购买70~150元左右的台灯，并下单支付", + "帮我用淘宝购买200~300元左右的蓝牙耳机，并下单支付", + "帮我用淘宝购买200~400元左右的电动牙刷，并下单支付", + "帮我用淘宝购买1000~1500元左右的烤箱，并下单支付", + "帮我用淘宝购买80~120元左右的充电宝，并下单支付", + "帮我用淘宝购买600~900元左右的行李箱，并下单支付", + "帮我用淘宝购买30~60元左右的手机支架，并下单支付", + "帮我用淘宝购买500~800元左右的机械键盘，并下单支付", + "帮我用淘宝购买50~100元左右的运动水杯，并下单支付", + + "帮我用淘宝购买70~150元左右的台灯", + "帮我用淘宝购买200~300元左右的蓝牙耳机", + "帮我用淘宝购买200~400元左右的电动牙刷", + "帮我用淘宝购买1000~1500元左右的烤箱", + "帮我用淘宝购买80~120元左右的充电宝", + "帮我用淘宝购买600~900元左右的行李箱", + "帮我用淘宝购买30~60元左右的手机支架", + "帮我用淘宝购买500~800元左右的机械键盘", + "帮我用淘宝购买50~100元左右的运动水杯" ] \ No newline at end of file diff --git a/runner/mobiagent/task_mobiflow.json b/runner/mobiagent/task_mobiflow.json new file mode 100644 index 00000000..7615d30c --- /dev/null +++ b/runner/mobiagent/task_mobiflow.json @@ -0,0 +1,463 @@ +[ + { + "app": "bilibili", + "type": "type1", + "tasks": [ + "在B站搜一下“巴黎奥运会开幕式”", + "在B站搜一下“三伏天避暑指南”", + "在B站搜一下“华为Mate70爆料”", + "在B站搜一下“科目三舞蹈原版”", + "在B站搜一下“深夜泡面番推荐”" + ] + }, + { + "app": "bilibili", + "type": "type2", + "tasks": [ + "在B站播放《进击的巨人》最终季", + "在B站播放LPL夏季赛总决赛", + "在B站播放周杰伦演唱会4K修复版", + "在B站播放《星空》游戏实况", + "在B站播放《流浪地球3》预告片" + ] + }, + { + "app": "bilibili", + "type": "type3", + "tasks": [ + "在B站搜一下UP主老番茄", + "在B站搜一下UP主老师好我叫何同学", + "在B站搜一下UP主罗翔说刑法", + "在B站搜一下UP主小潮院长", + "在B站搜一下UP主木鱼水心" + ] + }, + { + "app": "bilibili", + "type": "type4", + "tasks": [ + "在B站进入UP主半佛仙人的主页", + "在B站进入UP主花少北的主页", + "在B站进入UP主敖厂长的主页", + "在B站进入UP主某幻君的主页", + "在B站进入UP主大祥哥来啦的主页" + ] + }, + { + "app": "bilibili", + "type": "type5", + "tasks": [ + "B站搜索up主麻薯波比呀的《马斯克组建美国党》视频", + "B站搜索up主一岸舟的武林外传视频", + "B站搜索up主老师好我叫何同学的5G测评视频", + "B站搜索up主罗翔说刑法的视频《拐卖男友去电诈》", + "B站搜索up主小潮院长的《走到哪算哪》视频" + ] + }, + { + "app": "bilibili", + "type": "type6", + "tasks": [ + "搜索up主'天师道的白山正'的《中国奇谭》解析，并播放", + "搜索up主峡谷混血家的《与顶级混子的极限拉扯》，并播放", + "搜索up主影视飓风的视频《一位粉丝想看到自己奔跑的样子》，并播放", + "搜索up主花少北的《双人成行》实况，并播放", + "搜索up主罗翔的视频我们为什么要读书，并播放" + ] + }, + { + "app": "bilibili", + "type": "type7", + "tasks": [ + "关注up主逍遥散人", + "关注up主麻薯波比呀", + "关注up主漫游会议室", + "关注up主认真的阿真", + "关注up主帅农鸟哥" + ] + }, + { + "app": "bilibili", + "type": "type8", + "tasks": [ + "在B站视频《黑神话：悟空》实机演示下评论“画质炸裂，期待发售！”", + "在B站视频《原神：高清重置—5周年》下评论“感谢UP，已经转发给表弟”", + "在B站视频《相机大战》下评论“截图当壁纸了，太美了”", + "在B站视频《ai理解的春晚是什么样的2》下评论“小姐姐跳得比原版还齐”", + "在B站视频《法考经验贴》下评论“收藏了，明年必过！”" + ] + }, + { + "app": "bilibili", + "type": "type9", + "tasks": [ + "在up主老番茄的视频《驯 虫 高 手》下评论“番茄别回头，我是你粉丝！”", + "在up主老师好我叫何同学5G测评下评论“这期科普太硬核，已三连”", + "在up主罗翔说刑法的视频《我们为什么要读书》下评论“法外狂徒张三又出现了”", + "在up主噢呼w的视频《敢 杀 我 的 马？！》下评论“申遗成功！”", + "在up主影视飓风的视频《一位粉丝想看到自己奔跑的样子》下评论“现在看到的是自己”" + ] + }, + { + "app": "bilibili", + "type": "type10", + "tasks": [ + "搜索up主全一女团的《开业兔子舞》解析，播放后点赞", + "搜索up主三个大比兜啊的《谁家手办跑出来了》，播放后点赞", + "搜索up主冷淡熊的《曾经我也想过自刎归天！》，播放后点赞", + "搜索up主逍遥散人的《双人成行》实况，播放后点赞", + "搜索up主何同学的《有多快？5G在日常使用中的真实体验》，播放后点赞" + ] + }, + { + "app": "淘宝", + "type": "type1", + "tasks": [ + "淘宝搜索最畅销的机械键盘", + "淘宝中搜索价格最低的智能手环", + "淘宝中搜索戴森的吹风机", + "淘宝中搜索销量最高的儿童安全座椅", + "淘宝中搜索价格最高的空气炸锅" + ] + }, + { + "app": "淘宝", + "type": "type2", + "tasks": [ + "淘宝中选择销量最高的机械键盘，并选择第一个", + "淘宝中查找一款价格最低的智能手环，并选择第一个", + "淘宝中查找最便宜的品牌为戴森的吹风机，并选择第一个", + "淘宝中搜索销量最高的儿童安全座椅，并选择第一个", + "淘宝中搜索价格最高的空气炸锅，并选择第一个" + ] + }, + { + "app": "淘宝", + "type": "type3", + "tasks": [ + "淘宝中搜索电动牙刷，加入购物车", + "淘宝中搜索智能手环加入购物车", + "淘宝中将iPhone17Pro加入购物车", + "淘宝中将一件防晒衣，加入购物车", + "淘宝中将冰丝凉席三件套加入购物车" + ] + }, + { + "app": "淘宝", + "type": "type4", + "tasks": [ + "淘宝中将销量最高的智能门锁加入购物车", + "淘宝中将价格最低的儿童安全座椅加入购物车", + "淘宝中将品牌为戴森的V15吸尘器加入购物车", + "淘宝中将销量最高的空气炸锅加入购物车", + "淘宝中将价格最高的骨传导耳机加入购物车" + ] + }, + { + "app": "淘宝", + "type": "type5", + "tasks": [ + "淘宝中将型号为M码的冰丝防晒裤加入购物车", + "淘宝中将长度为1.5米的绿联USB-C快充线加入购物车", + "淘宝中将型号为XL码的李宁速干运动男短袖加入购物车", + "淘宝中将型号为L码的户外冲锋衣加入购物车", + "淘宝中将型号为42码的阿迪达斯跑鞋加入购物车" + ] + }, + { + "app": "淘宝", + "type": "type6", + "tasks": [ + "淘宝中将销量最高型号为42码的耐克跑鞋加入购物车", + "淘宝中将销量最高颜色为白色的破壁机加入购物车", + "淘宝中将价格最低型号为L码的始祖鸟冲锋衣加入购物车", + "淘宝中将销量最高的16GB+512GB黑色的小米15手机加入购物车" + ] + }, + { + "app": "网易云音乐", + "type": "type1", + "tasks": [ + "网易云音乐中帮我搜索周深的歌曲", + "网易云音乐中找一下张惠妹的歌曲", + "网易云音乐中帮我搜索陈奕迅的歌曲", + "网易云音乐中帮我搜索Taylor Swift的歌曲", + "网易云音乐中帮我搜一下陈楚生的歌曲" + ] + }, + { + "app": "网易云音乐", + "type": "type2", + "tasks": [ + "网易云音乐中播放林俊杰的《江南》", + "网易云音乐中播放歌曲稻香", + "网易云音乐中播放邓紫棋的歌曲", + "网易云音乐中播放陈奕迅的《十年》", + "网易云音乐中播放周深的《大鱼》" + ] + }, + { + "app": "携程", + "type": "type1", + "tasks": [ + "携程中搜索北京到上海的飞机票", + "携程中搜索广州到深圳的火车票", + "携程中搜索成都到重庆的飞机票", + "携程中搜索杭州到南京的火车票", + "携程中搜索西安到郑州的飞机票" + ] + }, + { + "app": "携程", + "type": "type2", + "tasks": [ + "携程中搜索2026年1月25日上海到北京的火车票", + "携程中搜索2026年1月30日深圳到广州的飞机票", + "携程中搜索2026年2月5日重庆到成都的火车票", + "携程中搜索2026年2月15日郑州到西安的火车票", + "携程中搜索2026年2月10日南京到杭州的飞机票" + ] + }, + { + "app": "携程", + "type": "type3", + "tasks": [ + "携程中搜索2026年1月26日北京到广州、出发时间08:00-12:00的航班", + "携程中搜索2026年1月28日上海到杭州、出发时间14:00-18:00的火车票", + "携程中搜索2026年2月2日广州到成都、出发时间06:00-10:00的航班", + "携程中搜索2026年2月12日杭州到北京、出发时间16:00-20:00的航班", + "携程中搜索2026年2月8日成都到西安、出发时间10:00-14:00的火车票" + ] + }, + { + "app": "携程", + "type": "type4", + "tasks": [ + "携程中搜索2026年1月27日北京到上海、到达时间12:00-16:00的火车票", + "携程中搜索2026年2月1日上海到成都、到达时间18:00-22:00的航班", + "携程中搜索2026年2月7日深圳到广州、到达时间13:00-17:00的航班", + "携程中搜索2026年2月4日武汉到北京、到达时间09:00-11:00的火车票", + "携程中搜索2026年2月14日成都到杭州、到达时间20:00-23:59的火车票" + ] + }, + { + "app": "携程", + "type": "type6", + "tasks": [ + "在携程里帮我找一下附近全季酒店", + "携程中帮我查询一下附近桔子酒店的相关信息", + "帮我在携程上找一找汉庭酒店", + "携程里帮我搜一下亚朵酒店", + "在携程中帮我查寻一下希尔顿酒店" + ] + }, + { + "app": "携程", + "type": "type7", + "tasks": [ + "携程中帮我查一下上海外滩附近的全季酒店", + "在携程里找一找广州珠江新城周边的亚朵酒店", + "携程上帮我查询一下深圳福田CBD附近的如家酒店", + "帮我在携程中查寻一下成都春熙路周边的希尔顿酒店", + "携程里帮我搜一下杭州西湖景区附近的桔子酒店" + ] + }, + { + "app": "携程", + "type": "type8", + "tasks": [ + "携程中帮我预定一间上海外滩附近亚朵酒店的双床房", + "携程中帮我找一间广州珠江新城周边全季酒店的大床房", + "携程中帮我预定一间深圳福田CBD附近汉庭酒店的双床房", + "携程中帮我找一间成都春熙路周边桔子水晶酒店的高级大床房", + "携程中帮我预定一间杭州西湖景区附近希尔顿酒店的大床房" + ] + }, + { + "app": "携程", + "type": "type9", + "tasks": [ + "携程中帮我预定一间上海外滩附近评分最高的全季酒店双床房", + "携程中帮我找一间广州珠江新城周边价格最低的汉庭酒店大床房", + "携程中帮我预定一间深圳福田CBD距离最近的桔子水晶酒店双床房", + "携程中帮我找一间成都春熙路周边价格最低的希尔顿酒店大床房", + "携程中帮我预定一间杭州西湖景区附近评价最高的亚朵酒店高级大床房" + ] + }, + { + "app": "小红书", + "type": "type1", + "tasks": [ + "小红书关注博主阿喵", + "小红书关注博主抽象一坨", + "小红书关注博主侯绿萝", + "小红书关注博主人名网", + "小红书关注博主酸酸甜甜就是我" + ] + }, + { + "app": "小红书", + "type": "type2", + "tasks": [ + "进入小红书博主混子哥边画边讲的主页", + "进入小红书博主李福贵的主页", + "进入小红书博主许二木的主页", + "进入小红书博主阿喵的主页", + "进入小红书博主抽象一坨的主页" + ] + }, + { + "app": "小红书", + "type": "type3", + "tasks": [ + "小红书搜索博主赵露思，查看他的第一个笔记", + "小红书搜索博主张曼玉Maggie，查看她的第一个内容", + "小红书搜索博主木梓蓝，查看他第一个文章", + "小红书搜索博主混子哥边画边讲，查看他第一个文章", + "小红书搜索博主李福贵，查看他的第一个笔记" + ] + }, + { + "app": "小红书", + "type": "type4", + "tasks": [ + "在小红书里搜索创意摄影", + "在小红书里搜索风景园林科普", + "在小红书里搜索手工沙发制作", + "在小红书里搜索朗读经典文学", + "在小红书里搜索国庆出游" + ] + }, + { + "app": "小红书", + "type": "type5", + "tasks": [ + "在小红书里搜索平替好物推荐，并点击查看第一个结果", + "在小红书里搜索家居收纳改造，并点击查看第一个结果", + "在小红书里搜索明星穿搭分享，并点击查看第一个结果", + "在小红书里搜索美食制作教程，并点击查看第一个结果", + "在小红书里搜索宠物日常趣事，并点击查看第一个结果" + ] + }, + { + "app": "小红书", + "type": "type6", + "tasks": [ + "小红书搜索博主苏打气泡水的《交大的粉黛子开》", + "小红书搜索博主马俊达Mars的中式美学笔记", + "小红书搜索博主人民网的视频‘睡了吗？今晚吃陕西宝鸡的腊汁肉夹馍。’", + "小红书搜索博主阿好好好的视频 带你走进史铁生的心理世界", + "小红书搜索博主平替姐的‘细数中国制造’笔记" + ] + }, + { + "app": "小红书", + "type": "type7", + "tasks": [ + "查看小红书博主桂非妃的挑战用一张纸换XX视频", + "查看小红书博主王冰冰的穿上正装变成大人模样笔记", + "查看小红书博主佳减乘除的在公司摆一天摊能赚多少钱内容", + "查看小红书博主豆皮一点都不皮的需要电子宠物吗视频", + "查看小红书博主飓风课堂的小红书正式官宣笔记" + ] + }, + { + "app": "高德", + "type": "type1", + "tasks": [ + "高德开车导航至上海迪士尼乐园", + "高德从当前位置开车导航至上海外滩", + "高德公交导航至上海科技馆", + "高德公交导航至上海虹桥站", + "高德公交导航至上海复旦大学邯郸校区" + ] + }, + { + "app": "高德", + "type": "type2", + "tasks": [ + "高德打车前往上海陆家嘴金融中心", + "高德打车前往上海南京东路步行街", + "高德打车至上海静安寺商圈", + "高德打车前往上海徐汇滨江绿地", + "高德打车至上海普陀区环球港购物中心" + ] + }, + { + "app": "高德", + "type": "type3", + "tasks": [ + "高德从上海浦东国际机场打车至上海外滩华尔道夫酒店", + "高德从上海虹桥国际机场T2航站楼打车至人民广场", + "高德从上海火车站南广场打车至上海杨浦区五角场万达广场", + "高德从上海迪士尼乐园停车场打车至上海浦东新区世纪公园", + "高德从上海闵行区莘庄地铁站打车至上海松江区欢乐谷景区" + ] + }, + { + "app": "饿了么", + "type": "type1", + "tasks": [ + "饿了么帮我搜索麦当劳", + "饿了么帮我搜索喜茶", + "饿了么帮我搜索老乡鸡", + "饿了么帮我搜索瑞幸", + "饿了么帮我搜索肯德基" + ] + }, + { + "app": "饿了么", + "type": "type2", + "tasks": [ + "饿了么中帮我点经典香辣鸡腿堡", + "饿了么中帮我点杨枝甘露多肉葡萄", + "饿了么中帮我点农家小炒肉盖饭", + "饿了么中帮我点生椰拿铁", + "饿了么中帮我点老北京鸡肉卷" + ] + }, + { + "app": "饿了么", + "type": "type3", + "tasks": [ + "在饿了么帮我点麦当劳的板烧鸡腿堡", + "在饿了么帮我点喜茶的清爽芭乐提", + "在饿了么帮我点老乡鸡的梅菜扣肉饭", + "在饿了么帮我点瑞幸的生椰拿铁", + "在饿了么帮我点肯德基的吮指原味鸡" + ] + }, + { + "app": "饿了么", + "type": "type4", + "tasks": [ + "在饿了么帮我点2份麦当劳的麦辣鸡翅", + "在饿了么帮我点1杯喜茶的芒芒甘露", + "在饿了么帮我点3份老乡鸡的西红柿炒蛋", + "在饿了么帮我点2杯瑞幸的橙C美式", + "在饿了么帮我点1份肯德基的香辣鸡腿汉堡" + ] + }, + { + "app": "饿了么", + "type": "type5", + "tasks": [ + "在饿了么点麦当劳的可乐，要中杯", + "在饿了么点喜茶的芒芒甘露，要常规杯、少糖、少冰", + "帮我点蜜雪冰城的芋圆葡萄，要少冰，5分糖，加珍珠", + "在饿了么帮我点瑞幸的小黄油拿铁，要大杯、冰、少甜", + "在饿了么帮我点肯德基的薯条，要大份" + ] + }, + { + "app": "饿了么", + "type": "type6", + "tasks": [ + "饿了么里帮我点2杯麦当劳的雪碧，要中杯、去冰、正常甜度", + "饿了么里帮我点3杯喜茶的多肉葡萄，要大杯、少糖、常温", + "饿了么里帮我点2份蜜雪冰城的葡萄冰美式，要少冰，七分糖", + "饿了么里帮我点1杯瑞幸的生椰拿铁，要大杯、热、不加糖", + "饿了么里帮我点4份肯德基的蛋挞，要原味" + ] + } +] \ No newline at end of file diff --git a/runner/providers/__init__.py b/runner/providers/__init__.py new file mode 100644 index 00000000..9e195745 --- /dev/null +++ b/runner/providers/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + +""" +Provider包，包含不同模型的任务适配器 +""" + +from .uitars.uitars_task import UITARSTask +from .mobiagent.mobile_task import MobiAgentStepTask + +__all__ = [ + 'MobiAgentTask', + 'UITARSTask', + 'MobiAgentStepTask', +] diff --git a/runner/providers/autoglm/__init__.py b/runner/providers/autoglm/__init__.py new file mode 100644 index 00000000..4a3a158e --- /dev/null +++ b/runner/providers/autoglm/__init__.py @@ -0,0 +1,5 @@ +"""AutoGLM Provider for MobiAgent Runner.""" + +from providers.autoglm.autoglm_task import AutoGLMTask + +__all__ = ["AutoGLMTask"] diff --git a/runner/providers/autoglm/action_parser.py b/runner/providers/autoglm/action_parser.py new file mode 100644 index 00000000..3f2460c0 --- /dev/null +++ b/runner/providers/autoglm/action_parser.py @@ -0,0 +1,159 @@ +"""Action parser for AutoGLM model responses.""" + +import ast +import re +from typing import Any, Dict, Tuple + + +def parse_action(response: str) -> Dict[str, Any]: + """ + Parse action from model response. + + Args: + response: Raw response string from the model. + + Returns: + Parsed action dictionary with format: + { + "_metadata": "do" or "finish", + "action": "Tap" | "Type" | ..., + "element": [x, y], # for Tap/Long Press/Double Tap + "text": "...", # for Type + "start": [x, y], # for Swipe + "end": [x, y], # for Swipe + "app": "...", # for Launch + "message": "...", # for finish + } + + Raises: + ValueError: If the response cannot be parsed. + """ + try: + response = response.strip() + + # ==================== 处理简写形式 ==================== + # [Back]、[Home] 等简写形式 + shorthand_map = { + '[back]': {"_metadata": "do", "action": "Back"}, + '[home]': {"_metadata": "do", "action": "Home"}, + } + # '[wait]': {"_metadata": "do", "action": "Wait", "duration": "2 seconds"}, + response_lower = response.lower() + if response_lower in shorthand_map: + return shorthand_map[response_lower] + + # 处理包含简写的响应：如 "[Back]do(action="Back")" + for shorthand, action_dict in shorthand_map.items(): + if response_lower.startswith(shorthand): + remaining = response[len(shorthand):].strip() + if not remaining: # 只有简写 + return action_dict + response = remaining # 继续解析剩余部分 + break + + # ==================== 处理 Take_over 的各种格式 ==================== + # 格式1: do(action="Take_over", message="...") + # 格式2: [{'type': 'Take_over", message="..."} + # 格式3: Take_over(message="...") + # if 'take_over' in response_lower or "type': 'take_over" in response_lower: + # # 尝试提取 message + # message = "User intervention required" + # if 'message=' in response: + # try: + # msg_start = response.find('message=') + len('message=') + # if msg_start < len(response) and response[msg_start] in ['"', "'"]: + # quote = response[msg_start] + # msg_end = response.find(quote, msg_start + 1) + # if msg_end > msg_start: + # message = response[msg_start + 1:msg_end] + # except Exception: + # pass + # return {"_metadata": "do", "action": "Take_over", "message": message} + + # ==================== 标准格式解析 ==================== + # Handle Type action with special text parsing + if response.startswith('do(action="Type"') or response.startswith( + 'do(action="Type_Name"' + ): + text = response.split("text=", 1)[1][1:-2] + action = {"_metadata": "do", "action": "Type", "text": text} + return action + + elif response.startswith("do"): + # Use AST parsing for safety + try: + # Escape special characters + response = response.replace('\n', '\\n') + response = response.replace('\r', '\\r') + response = response.replace('\t', '\\t') + + tree = ast.parse(response, mode="eval") + if not isinstance(tree.body, ast.Call): + raise ValueError("Expected a function call") + + call = tree.body + action = {"_metadata": "do"} + for keyword in call.keywords: + key = keyword.arg + value = ast.literal_eval(keyword.value) + action[key] = value + + return action + except (SyntaxError, ValueError) as e: + raise ValueError(f"Failed to parse do() action: {e}") + + elif response.startswith("finish"): + action = { + "_metadata": "finish", + "message": response.replace("finish(message=", "")[1:-2], + } + return action + + else: + raise ValueError(f"Failed to parse action: {response}") + + except Exception as e: + raise ValueError(f"Failed to parse action: {e}") + + +def parse_response(content: str) -> Tuple[str, str]: + """ + Parse the model response into thinking and action parts. + + Parsing rules: + 1. If content contains 'finish(message=', everything before is thinking, + everything from 'finish(message=' onwards is action. + 2. If rule 1 doesn't apply but content contains 'do(action=', + everything before is thinking, everything from 'do(action=' onwards is action. + 3. Fallback: If content contains '', use legacy parsing with XML tags. + 4. Otherwise, return empty thinking and full content as action. + + Args: + content: Raw response content. + + Returns: + Tuple of (thinking, action). + """ + # Rule 1: Check for finish(message= + if "finish(message=" in content: + parts = content.split("finish(message=", 1) + thinking = parts[0].strip() + action = "finish(message=" + parts[1] + return thinking, action + + # Rule 2: Check for do(action= + if "do(action=" in content: + parts = content.split("do(action=", 1) + thinking = parts[0].strip() + action = "do(action=" + parts[1] + return thinking, action + + # Rule 3: Fallback to legacy XML tag parsing + if "" in content: + parts = content.split("", 1) + thinking = parts[0].replace("", "").replace("", "").strip() + action = parts[1].replace("", "").strip() + return thinking, action + + # Rule 4: No markers found, return content as action + return "", content diff --git a/runner/providers/autoglm/autoglm_task.py b/runner/providers/autoglm/autoglm_task.py new file mode 100644 index 00000000..dbbe0846 --- /dev/null +++ b/runner/providers/autoglm/autoglm_task.py @@ -0,0 +1,416 @@ +""" +AutoGLM Task Adapter for MobiAgent Runner Framework. + +This adapter integrates AutoGLM model into the unified execution framework. +Reference: Open-AutoGLM/phone_agent/agent.py +""" + +import os +import sys +import time +import json +import base64 +import logging +from typing import Dict, List, Optional, Any +from PIL import Image + +from openai import OpenAI + +# 使用模块级别的logger（级别由 setup_logging() 统一配置） +logger = logging.getLogger(__name__) + +# 添加父目录到路径以导入base_task +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../..')) +from base_task import BaseTask + +from providers.autoglm.prompts import SYSTEM_PROMPT +from providers.autoglm.action_parser import parse_action, parse_response + + +class AutoGLMTask(BaseTask): + """ + AutoGLM 任务适配器（步骤循环模式） + + 使用 AutoGLM 模型进行手机自动化任务。 + 支持的动作包括：Launch, Tap, Type, Swipe, Back, Home, Long Press, Double Tap, Wait, finish + + 坐标系统：使用相对坐标 (0-1000)，自动转换为绝对屏幕坐标 + """ + + def __init__( + self, + task_description: str, + device, + data_dir: str, + device_type: str = "Android", + max_steps: int = 30, + # 统一参数 + api_base: str = None, + api_key: str = None, + model: str = None, + temperature: float = None, + # AutoGLM 专属参数 + max_tokens: int = 3000, + top_p: float = 0.85, + frequency_penalty: float = 0.2, + # 向后兼容的旧参数 + model_base_url: str = None, + model_name: str = None, + **kwargs + ): + """ + 初始化 AutoGLM 任务 + + Args: + task_description: 任务描述 + device: 设备对象 + data_dir: 数据保存目录 + device_type: 设备类型，目前仅支持 Android + max_steps: 最大步骤数 + api_base: 模型服务基础URL + api_key: API密钥 + model: 模型名称 + temperature: 生成温度 + max_tokens: 最大输出token数 + top_p: Top-p采样参数 + frequency_penalty: 频率惩罚参数 + """ + super().__init__( + task_description=task_description, + device=device, + data_dir=data_dir, + device_type=device_type, + max_steps=max_steps, + use_step_loop=True, # 使用步骤循环模式 + **kwargs + ) + + # 处理参数优先级: 新参数 > 旧参数 > 默认值 + self.api_base = api_base or model_base_url or "http://localhost:8000/v1" + self.model_name = model or model_name or "autoglm-phone-9b" + self.api_key = api_key or "EMPTY" + self.temperature = temperature if temperature is not None else 0.0 + self.max_tokens = max_tokens + self.top_p = top_p + self.frequency_penalty = frequency_penalty + + # 初始化 OpenAI 客户端 + try: + self.client = OpenAI( + base_url=self.api_base, + api_key=self.api_key + ) + logger.info(f"AutoGLM client initialized: {self.model_name} at {self.api_base}") + except Exception as e: + logger.error(f"Failed to initialize OpenAI client: {e}") + raise + + # 对话上下文 + self._context: List[Dict[str, Any]] = [] + + logger.info("AutoGLMTask initialized") + + def execute_step(self, step_index: int) -> List[Dict]: + """ + 执行单步操作 + + Args: + step_index: 当前步骤索引 + + Returns: + 动作序列列表 + """ + # 获取截图路径 + screenshot_path = os.path.join(self.data_dir, f"{step_index}.jpg") + + # 读取截图 + try: + with open(screenshot_path, "rb") as f: + image_data = f.read() + encoded_image = base64.b64encode(image_data).decode('utf-8') + + # 获取图像尺寸 + with Image.open(screenshot_path) as img: + screen_width, screen_height = img.size + except Exception as e: + logger.error(f"Failed to read screenshot: {e}") + return [{"type": "retry", "params": {"error": str(e)}}] + + # 获取当前应用信息（尝试获取） + try: + current_app = self._get_current_app() + except Exception: + current_app = "Unknown" + + # 构建消息 + is_first = len(self._context) == 0 + + if is_first: + # 第一步：添加系统提示和用户任务 + self._context.append({ + "role": "system", + "content": SYSTEM_PROMPT + }) + + screen_info = json.dumps({"current_app": current_app}, ensure_ascii=False) + text_content = f"{self.task_description}\n\n** Screen Info **\n\n{screen_info}" + + self._context.append({ + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"}}, + {"type": "text", "text": text_content} + ] + }) + else: + # 后续步骤：添加当前屏幕观察 + screen_info = json.dumps({"current_app": current_app}, ensure_ascii=False) + text_content = f"** Screen Info **\n\n{screen_info}" + + self._context.append({ + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"}}, + {"type": "text", "text": text_content} + ] + }) + + # 调用模型 + try: + logger.info(f"Calling AutoGLM model (step {step_index})...") + start_time = time.time() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=self._context, + max_tokens=self.max_tokens, + temperature=self.temperature, + top_p=self.top_p, + frequency_penalty=self.frequency_penalty, + stream=False + ) + + raw_content = response.choices[0].message.content + elapsed_time = time.time() - start_time + logger.info(60*'-') + logger.info(f"Model response ({elapsed_time:.2f}s): {raw_content}") + logger.info(60*'-') + + except Exception as e: + logger.error(f"Model request failed: {e}") + return [{"type": "retry", "params": {"error": str(e)}}] + + # 解析响应 + try: + thinking, action_str = parse_response(raw_content) + logger.info(f"Thinking: {thinking[:100]}...") + logger.info(f"Action string: {action_str}") + + action = parse_action(action_str) + logger.info(f"Parsed action: {json.dumps(action, ensure_ascii=False)}") + + except ValueError as e: + logger.error(f"Failed to parse response: {e}") + # 尝试将整个响应作为finish + action = {"_metadata": "finish", "message": raw_content} + + # 从context中移除图像以节省空间 + if len(self._context) > 0: + last_msg = self._context[-1] + if isinstance(last_msg.get("content"), list): + self._context[-1]["content"] = [ + item for item in last_msg["content"] + if item.get("type") == "text" + ] + + # 添加助手响应到上下文 + self._context.append({ + "role": "assistant", + "content": f"{thinking}{action_str}" + }) + + # 记录推理过程 + self._add_react( + reasoning=thinking, + action=action.get("action", action.get("_metadata", "unknown")), + parameters=action, + step_index=step_index + ) + + # 转换为 runner 标准格式 + return self._convert_to_runner_actions(action, screen_width, screen_height) + + def _get_current_app(self) -> str: + """获取当前应用名称""" + try: + # 使用 adb 获取当前焦点窗口 + import subprocess + result = subprocess.run( + ["adb", "shell", "dumpsys", "window"], + capture_output=True, + text=True, + encoding="utf-8" + ) + output = result.stdout + + for line in output.split("\n"): + if "mCurrentFocus" in line or "mFocusedApp" in line: + # 简单返回包名 + return line.strip() + + return "System Home" + except Exception: + return "Unknown" + + def _convert_to_runner_actions( + self, + action: Dict[str, Any], + screen_width: int, + screen_height: int + ) -> List[Dict]: + """ + 将 AutoGLM 动作转换为 runner 标准格式 + + Args: + action: AutoGLM 解析后的动作 + screen_width: 屏幕宽度 + screen_height: 屏幕高度 + + Returns: + runner 标准动作序列 + """ + action_type = action.get("_metadata") + + # 处理 finish 动作 + if action_type == "finish": + return [{ + "type": "done", + "params": { + "status": "success", + "message": action.get("message", "Task completed") + } + }] + + if action_type != "do": + logger.warning(f"Unknown action metadata: {action_type}") + return [{"type": "retry", "params": {}}] + + action_name = action.get("action", "") + + # Launch -> open_app + if action_name == "Launch": + app_name = action.get("app", "") + return [{ + "type": "open_app", + "params": {"app_name": app_name} + }] + + # Tap -> click + elif action_name == "Tap": + element = action.get("element", [500, 500]) + x, y = self._convert_relative_to_absolute(element, screen_width, screen_height) + return [{ + "type": "click", + "params": {"coordinate": [x, y]} + }] + + # Type / Type_Name -> input_text + elif action_name in ["Type", "Type_Name"]: + text = action.get("text", "") + return [{ + "type": "input_text", + "params": {"text": text} + }] + + # Swipe -> scroll + elif action_name == "Swipe": + start = action.get("start", [500, 700]) + end = action.get("end", [500, 300]) + start_x, start_y = self._convert_relative_to_absolute(start, screen_width, screen_height) + end_x, end_y = self._convert_relative_to_absolute(end, screen_width, screen_height) + return [{ + "type": "scroll", + "params": { + "start_coordinate": [start_x, start_y], + "end_coordinate": [end_x, end_y] + } + }] + + # Back -> back + elif action_name == "Back": + return [{"type": "back", "params": {}}] + + # Home -> home + elif action_name == "Home": + return [{"type": "home", "params": {}}] + + # Double Tap -> doubleclick + elif action_name == "Double Tap": + element = action.get("element", [500, 500]) + x, y = self._convert_relative_to_absolute(element, screen_width, screen_height) + return [{ + "type": "doubleclick", + "params": {"coordinate": [x, y]} + }] + + # Long Press -> longclick + elif action_name == "Long Press": + element = action.get("element", [500, 500]) + x, y = self._convert_relative_to_absolute(element, screen_width, screen_height) + return [{ + "type": "longclick", + "params": {"coordinate": [x, y]} + }] + + # Wait -> wait + elif action_name == "Wait": + duration_str = action.get("duration", "2 seconds") + try: + duration = float(duration_str.replace("seconds", "").strip()) + except ValueError: + duration = 2.0 + return [{ + "type": "wait", + "params": {"seconds": duration} + }] + + # Take_over -> wait (需要用户介入，暂时作为等待处理) + elif action_name == "Take_over": + logger.warning(f"Take_over requested: {action.get('message', '')}") + return [{ + "type": "wait", + "params": {"seconds": 5} + }] + + # Note / Call_API / Interact -> wait (暂不实现具体功能) + elif action_name in ["Note", "Call_API", "Interact"]: + logger.info(f"Action {action_name} treated as wait") + return [{ + "type": "wait", + "params": {"seconds": 1} + }] + + else: + logger.warning(f"Unknown action: {action_name}") + return [{"type": "retry", "params": {}}] + + def _convert_relative_to_absolute( + self, + element: List[int], + screen_width: int, + screen_height: int + ) -> tuple: + """ + 将相对坐标 (0-1000) 转换为绝对像素坐标 + + Args: + element: [x, y] 相对坐标 (0-1000) + screen_width: 屏幕宽度 + screen_height: 屏幕高度 + + Returns: + (x, y) 绝对坐标 + """ + x = int(element[0] / 1000 * screen_width) + y = int(element[1] / 1000 * screen_height) + return x, y diff --git a/runner/providers/autoglm/prompts.py b/runner/providers/autoglm/prompts.py new file mode 100644 index 00000000..2438e48a --- /dev/null +++ b/runner/providers/autoglm/prompts.py @@ -0,0 +1,68 @@ +"""System prompt for AutoGLM model.""" + +from datetime import datetime + +# 生成日期信息 +today = datetime.today() +weekday_names = ["星期一", "星期二", "星期三", "星期四", "星期五", "星期六", "星期日"] +weekday = weekday_names[today.weekday()] +formatted_date = today.strftime("%Y年%m月%d日") + " " + weekday + +SYSTEM_PROMPT = ( + "今天的日期是: " + + formatted_date + + """ +你是一个智能体分析专家，可以根据操作历史和当前状态图执行一系列操作来完成任务。 +你必须严格按照要求输出以下格式： +{think} +{action} + +其中： +- {think} 是对你为什么选择这个操作的简短推理说明。 +- {action} 是本次执行的具体操作指令，必须严格遵循下方定义的指令格式。 + +操作指令及其作用如下： +- do(action="Launch", app="xxx") + Launch是启动目标app的操作，这比通过主屏幕导航更快。此操作完成后，您将自动收到结果状态的截图。 +- do(action="Tap", element=[x,y]) + Tap是点击操作，点击屏幕上的特定点。可用此操作点击按钮、选择项目、从主屏幕打开应用程序，或与任何可点击的用户界面元素进行交互。坐标系统从左上角 (0,0) 开始到右下角（999,999)结束。此操作完成后，您将自动收到结果状态的截图。 +- do(action="Tap", element=[x,y], message="重要操作") + 基本功能同Tap，点击涉及财产、支付、隐私等敏感按钮时触发。 +- do(action="Type", text="xxx") + Type是输入操作，在当前聚焦的输入框中输入文本。使用此操作前，请确保输入框已被聚焦（先点击它）。输入的文本将像使用键盘输入一样输入。 +- do(action="Type_Name", text="xxx") + Type_Name是输入人名的操作，基本功能同Type。 +- do(action="Interact") + Interact是当有多个满足条件的选项时而触发的交互操作，询问用户如何选择。 +- do(action="Swipe", start=[x1,y1], end=[x2,y2]) + Swipe是滑动操作，通过从起始坐标拖动到结束坐标来执行滑动手势。可用于滚动内容、在屏幕之间导航、下拉通知栏以及项目栏或进行基于手势的导航。坐标系统从左上角 (0,0) 开始到右下角（999,999)结束。 +- do(action="Note", message="True") + 记录当前页面内容以便后续总结。 +- do(action="Call_API", instruction="xxx") + 总结或评论当前页面或已记录的内容。 +- do(action="Long Press", element=[x,y]) + Long Press是长按操作，在屏幕上的特定点长按指定时间。可用于触发上下文菜单、选择文本或激活长按交互。坐标系统从左上角 (0,0) 开始到右下角（999,999)结束。 +- do(action="Double Tap", element=[x,y]) + Double Tap在屏幕上的特定点快速连续点按两次。使用此操作可以激活双击交互，如缩放、选择文本或打开项目。坐标系统从左上角 (0,0) 开始到右下角（999,999)结束。 +- do(action="Take_over", message="xxx") + Take_over是接管操作，表示在登录和验证阶段需要用户协助。 +- do(action="Back") + 导航返回到上一个屏幕或关闭当前对话框。相当于按下 Android 的返回按钮。 +- do(action="Home") + Home是回到系统桌面的操作，相当于按下 Android 主屏幕按钮。 +- do(action="Wait", duration="x seconds") + 等待页面加载，x为需要等待多少秒。 +- finish(message="xxx") + finish是结束任务的操作，表示准确完整完成任务，message是终止信息。 + +必须遵循的规则： +1. 在执行任何操作前，先检查当前app是否是目标app，如果不是，先执行 Launch。 +2. 如果进入到了无关页面，先执行 Back。如果执行Back后页面没有变化，请点击页面左上角的返回键进行返回，或者右上角的X号关闭。 +3. 如果页面未加载出内容，最多连续 Wait 三次，否则执行 Back重新进入。 +4. 如果页面显示网络问题，需要重新加载，请点击重新加载。 +5. 如果当前页面找不到目标联系人、商品、店铺等信息，可以尝试 Swipe 滑动查找。 +6. 遇到价格区间、时间区间等筛选条件，如果没有完全符合的，可以放宽要求。 +7. 在做小红书总结类任务时一定要筛选图文笔记。 +8. 在结束任务前请一定要仔细检查任务是否完整准确的完成，如果出现错选、漏选、多选的情况，请返回之前的步骤进行纠正。 +""" +) diff --git a/runner/providers/mobiagent/.env.example b/runner/providers/mobiagent/.env.example new file mode 100644 index 00000000..184e8edc --- /dev/null +++ b/runner/providers/mobiagent/.env.example @@ -0,0 +1,14 @@ +# MobiAgent配置示例 +# 复制此文件为.env并修改配置 + +# 服务配置 +MOBIAGENT_SERVICE_IP=localhost +MOBIAGENT_DECIDER_PORT=8000 +MOBIAGENT_GROUNDER_PORT=8001 + +# 模型配置 +MOBIAGENT_DECIDER_MODEL=qwen-vl +MOBIAGENT_GROUNDER_MODEL=qwen-vl + +# 模式配置 +MOBIAGENT_USE_E2E=true diff --git a/runner/providers/mobiagent/__init__.py b/runner/providers/mobiagent/__init__.py new file mode 100644 index 00000000..5f8fcdaa --- /dev/null +++ b/runner/providers/mobiagent/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + +""" +MobiAgent模块，支持框架单步循环模式 +""" + +from .mobile_task import MobiAgentStepTask + +__all__ = ['MobiAgentStepTask'] diff --git a/runner/providers/mobiagent/load_md_prompt.py b/runner/providers/mobiagent/load_md_prompt.py new file mode 100644 index 00000000..00357cd5 --- /dev/null +++ b/runner/providers/mobiagent/load_md_prompt.py @@ -0,0 +1,21 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + +import os + +def load_prompt(md_name, prompt_dir=None): + """从markdown文件加载prompt模板 + + Args: + md_name: markdown文件名 + prompt_dir: prompt目录路径（可选，默认为当前文件的prompts子目录） + """ + if prompt_dir is None: + current_dir = os.path.dirname(os.path.abspath(__file__)) + prompt_dir = os.path.join(current_dir, "prompts") + + prompt_file = os.path.join(prompt_dir, md_name) + + with open(prompt_file, "r", encoding="utf-8") as f: + content = f.read() + content = content.replace("````markdown", "").replace("````", "") + return content.strip() diff --git a/runner/providers/mobiagent/mobile_task.py b/runner/providers/mobiagent/mobile_task.py new file mode 100644 index 00000000..19a16fbd --- /dev/null +++ b/runner/providers/mobiagent/mobile_task.py @@ -0,0 +1,707 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + +import os +import sys +import json +import time +import logging +import base64 +import io +from typing import Dict, List, Optional +from PIL import Image +from openai import OpenAI + +# 使用模块级别的logger（级别由 setup_logging() 统一配置） +logger = logging.getLogger(__name__) + +# 添加父目录到路径以导入base_task +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../..')) +from base_task import BaseTask + +from .load_md_prompt import load_prompt + + +class MobiAgentStepTask(BaseTask): + """ + MobiAgent任务适配器（单步循环模式） + 使用框架的步骤循环，每次execute_step返回一步的动作 + """ + + def __init__( + self, + task_description: str, + device, + data_dir: str, + device_type: str = "Android", + max_steps: int = 40, + max_retries: int = 3, + api_base: str = None, + service_ip: str = "localhost", + decider_port: int = 8000, + grounder_port: int = 8001, + planner_port: int = 8080, + enable_planning: bool = False, + use_e2e: bool = True, + decider_model: str = "MobiMind-1.5-4B", + grounder_model: str = "MobiMind-1.5-4B", + planner_model: str = "Qwen3-VL-30B-A3B-Instruct", + use_experience: bool = False, + **kwargs + ): + """ + 初始化MobiAgent任务 + + Args: + task_description: 任务描述 + device: 设备对象 + data_dir: 数据保存目录 + device_type: 设备类型 + max_steps: 最大步骤数 + max_retries: 最大重试次数 + service_ip: 服务IP地址 + decider_port: Decider模型端口 + grounder_port: Grounder模型端口 + planner_port: Planner模型端口 + enable_planning: 是否启用任务规划 + use_e2e: 是否使用端到端模式 + decider_model: Decider模型名称 + grounder_model: Grounder模型名称 + planner_model: Planner模型名称 + use_experience: 是否使用经验（调用planner改写任务） + """ + super().__init__( + task_description=task_description, + device=device, + data_dir=data_dir, + device_type=device_type, + max_steps=max_steps, + max_retries=max_retries, + use_step_loop=True, # 启用步骤循环模式 + enable_planning=enable_planning, + **kwargs + ) + + # 配置服务URL + if api_base is not None and service_ip is None: + if api_base.startswith("http://") or api_base.startswith("https://"): + self.api_base = api_base + else: + logger.error("Invalid API base URL: %s", api_base) + logger.error("API base URL must start with http:// or https://") + return + decider_base_url = self.api_base + grounder_base_url = self.api_base + planner_base_url = self.api_base + logger.info("Using API base URL: %s", self.api_base) + elif service_ip is not None and (service_ip.startswith("http://") or service_ip.startswith("https://")): + decider_base_url = f"{service_ip}:{decider_port}/v1" + grounder_base_url = f"{service_ip}:{grounder_port}/v1" + planner_base_url = f"{service_ip}:{planner_port}/v1" + logger.info("Using service IP: %s", service_ip) + else: + decider_base_url = f"http://{service_ip}:{decider_port}/v1" + grounder_base_url = f"http://{service_ip}:{grounder_port}/v1" + planner_base_url = f"http://{service_ip}:{planner_port}/v1" + + # 初始化客户端 + self.decider_client = OpenAI( + api_key="0", + base_url=decider_base_url, + ) + + self.grounder_client = OpenAI( + api_key="0", + base_url=grounder_base_url, + ) + + self.planner_client = OpenAI( + api_key="0", + base_url=planner_base_url, + ) + + self.decider_model = decider_model + self.grounder_model = grounder_model + self.planner_model = planner_model + self.use_e2e = use_e2e + self.use_experience = use_experience + + # 加载prompt模板 + prompt_dir = os.path.join(os.path.dirname(__file__), "prompts") + if use_e2e: + self.decider_prompt_template = load_prompt("e2e_qwen3.md", prompt_dir) + logging.info("MobiAgent initialized with e2e mode") + else: + self.decider_prompt_template = load_prompt("decider_v2.md", prompt_dir) + self.grounder_prompt_template_bbox = load_prompt("grounder_qwen3_bbox.md", prompt_dir) + self.grounder_prompt_template_no_bbox = load_prompt("grounder_qwen3_coordinates.md", prompt_dir) + logger.info("MobiAgent initialized with decider+grounder mode") + + # 历史记录 + self.history = [] + + # APP包名映射表 + self._init_app_package_mapping() + + logger.info("MobiAgentStepTask initialized") + + def _init_app_package_mapping(self): + """初始化APP名称到包名的映射""" + self.android_app_packages = { + "携程": "ctrip.android.view", + "同程": "com.tongcheng.android", + "同程旅行": "com.tongcheng.android", + "飞猪": "com.taobao.trip", + "去哪儿": "com.Qunar", + "华住会": "com.htinns", + "饿了么": "me.ele", + "支付宝": "com.eg.android.AlipayGphone", + "淘宝": "com.taobao.taobao", + "京东": "com.jingdong.app.mall", + "美团": "com.sankuai.meituan", + "美团外卖": "com.sankuai.meituan.takeoutnew", + "滴滴出行": "com.sdu.didi.psnger", + "微信": "com.tencent.mm", + "微博": "com.sina.weibo", + "华为商城": "com.vmall.client", + "华为视频": "com.huawei.himovie", + "华为音乐": "com.huawei.music", + "华为应用市场": "com.huawei.appmarket", + "拼多多": "com.xunmeng.pinduoduo", + "大众点评": "com.dianping.v1", + "小红书": "com.xingin.xhs", + "浏览器": "com.microsoft.emmx", + "QQ": "com.tencent.mobileqq", + "知乎": "com.zhihu.android", + "QQ音乐": "com.tencent.qqmusic", + "网易云音乐": "com.netease.cloudmusic", + "酷狗音乐": "com.kugou.android", + "抖音": "com.ss.android.ugc.aweme", + "快手": "com.smile.gifmaker", + "哔哩哔哩": "tv.danmaku.bili", + "爱奇艺": "com.qiyi.video", + "腾讯视频": "com.tencent.qqlive", + "优酷": "com.youku.phone", + "高德地图": "com.autonavi.minimap", + "百度地图": "com.baidu.BaiduMap", + "闲鱼": "com.taobao.idlefish" + } + + self.harmony_app_packages = { + "携程": "com.ctrip.harmonynext", + "携程旅行": "com.ctrip.harmonynext", + "飞猪": "com.fliggy.hmos", + "飞猪旅行": "com.fliggy.hmos", + "IntelliOS": "ohos.hongmeng.intellios", + "同程": "com.tongcheng.hmos", + "饿了么": "me.ele.eleme", + "知乎": "com.zhihu.hmos", + "哔哩哔哩": "yylx.danmaku.bili", + "微信": "com.tencent.wechat", + "小红书": "com.xingin.xhs_hos", + "QQ音乐": "com.tencent.hm.qqmusic", + "高德地图": "com.amap.hmapp", + "淘宝": "com.taobao.taobao4hmos", + "微博": "com.sina.weibo.stage", + "京东": "com.jd.hm.mall", + "天气": "com.huawei.hmsapp.totemweather", + "什么值得买": "com.smzdm.client.hmos", + "闲鱼": "com.taobao.idlefish4ohos", + "慧通差旅": "com.smartcom.itravelhm", + "PowerAgent": "com.example.osagent", + "航旅纵横": "com.umetrip.hm.app", + "滴滴出行": "com.sdu.didi.hmos.psnger", + "电子邮件": "com.huawei.hmos.email", + "图库": "com.huawei.hmos.photos", + "日历": "com.huawei.hmos.calendar", + "心声社区": "com.huawei.it.hmxinsheng", + "信息": "com.ohos.mms", + "文件管理": "com.huawei.hmos.files", + "运动健康": "com.huawei.hmos.health", + "智慧生活": "com.huawei.hmos.ailife", + "豆包": "com.larus.nova.hm", + "WeLink": "com.huawei.it.welink", + "设置": "com.huawei.hmos.settings", + "懂车帝": "com.ss.dcar.auto", + "美团外卖": "com.meituan.takeaway", + "大众点评": "com.sankuai.dianping", + "美团": "com.sankuai.hmeituan", + "浏览器": "com.huawei.hmos.browser", + "拼多多": "com.xunmeng.pinduoduo.hos" + } + + def _get_package_name(self, app_name: str) -> Optional[str]: + """根据APP名称获取包名""" + if self.device_type == "Android": + return self.android_app_packages.get(app_name) + elif self.device_type == "Harmony": + return self.harmony_app_packages.get(app_name) + return None + + def _plan_task(self) -> Optional[Dict]: + """ + 任务规划：调用 planner 服务分析任务并确定目标应用 + + Returns: + 包含 task_description, app_name, package_name 等信息的字典 + """ + try: + # 加载 planner prompt 模板（根据设备类型选择） + prompt_dir = os.path.join(os.path.dirname(__file__), "prompts") + + # 根据设备类型选择对应的 prompt + if self.device_type == "Android": + prompt_file = "planner_android.md" + elif self.device_type == "Harmony": + prompt_file = "planner_harmony.md" + else: + logger.warning(f"Unknown device type: {self.device_type}, using Android prompt") + prompt_file = "planner_android.md" + + planner_prompt_path = os.path.join(prompt_dir, prompt_file) + if os.path.exists(planner_prompt_path): + planner_prompt_template = load_prompt(prompt_file, prompt_dir) + else: + # 使用默认的 planner prompt 模板 + logger.warning(f"Planner prompt {prompt_file} not found, using default") + planner_prompt_template = self._get_default_planner_prompt() + + # 构建 prompt + prompt = planner_prompt_template.format( + task_description=self.original_task_description, + experience_content="(No experience available)" # 可以后续集成经验检索 + ) + + logger.info(f"Calling planner service for {self.device_type} device...") + start_time = time.time() + + # 调用 planner 服务 + response_str = self.planner_client.chat.completions.create( + model=self.planner_model, + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": prompt}], + } + ], + temperature=0.1, + timeout=30, + ).choices[0].message.content + + elapsed_time = time.time() - start_time + logger.info(f"Planner time: {elapsed_time:.2f}s") + logger.info(f"Planner response: {response_str}") + + # 解析响应 + response_json = self._parse_planner_response(response_str) + + if response_json is None: + logger.warning("Failed to parse planner response") + return None + + # 提取信息 + app_name = response_json.get("app_name") + final_task_desc = response_json.get("final_task_description", self.original_task_description) + + if not app_name: + logger.warning("Planner response missing app_name") + return None + + # 本地匹配包名 + package_name = self._get_package_name(app_name) + + if not package_name: + logger.warning(f"Package name not found for app: {app_name}") + # 尝试常见的变体 + if app_name.endswith("旅行"): + alternative_name = app_name[:-2] + package_name = self._get_package_name(alternative_name) + if package_name: + logger.info(f"Found package using alternative name: {alternative_name}") + + if not package_name: + logger.error(f"Cannot find package name for app: {app_name}") + return None + + result = { + "app_name": app_name, + "package_name": package_name + } + + # 如果启用 experience，使用 planner 优化后的任务描述 + if self.use_experience: + result["task_description"] = final_task_desc + + logger.info(f"Planning result: App={app_name}, Package={package_name}") + return result + + except Exception as e: + logger.error(f"Planning failed: {e}", exc_info=True) + return None + + def _parse_planner_response(self, response_str: str) -> Optional[Dict]: + """解析 planner 响应（只包含 app_name 和 final_task_description）""" + import re + + # 移除可能的代码块标记 + response_str = response_str.strip() + if response_str.startswith("```json"): + response_str = response_str[7:] + elif response_str.startswith("```"): + response_str = response_str[3:] + + if response_str.endswith("```"): + response_str = response_str[:-3] + + response_str = response_str.strip() + + # 尝试直接解析 + try: + return json.loads(response_str) + except json.JSONDecodeError: + pass + + # 尝试匹配 JSON 代码块 + pattern = re.compile(r"```json\s*(.*?)\s*```", re.DOTALL) + match = pattern.search(response_str) + + if match: + json_str = match.group(1).strip() + try: + return json.loads(json_str) + except json.JSONDecodeError: + pass + + # 尝试提取 JSON 对象 + start_idx = response_str.find('{') + if start_idx != -1: + end_idx = response_str.rfind('}') + if end_idx > start_idx: + json_str = response_str[start_idx:end_idx+1] + try: + return json.loads(json_str) + except json.JSONDecodeError: + pass + + logger.error(f"Failed to parse planner JSON response") + return None + + def _get_default_planner_prompt(self) -> str: + """获取默认的 planner prompt 模板""" + return """You are a mobile task planner. Given a user task description, determine which app should be used and provide a refined task description. + +Task: {task_description} + +Experience: {experience_content} + +Please respond in JSON format with the following structure: +{{ + "app_name": "", + "final_task_description": "" +}} + +Important: Return ONLY the JSON object, no additional text or markdown formatting.""" + + def execute_step(self, step_index: int) -> List[Dict]: + """ + 执行单步操作 + + Args: + step_index: 当前步骤索引 + + Returns: + 动作序列列表 + """ + # 获取当前截图 + screenshot_path = os.path.join(self.data_dir, f"{step_index}.jpg") + + # 读取截图并转换为base64 + try: + with open(screenshot_path, "rb") as f: + image_data = f.read() + encoded_image = base64.b64encode(image_data).decode('utf-8') + img = Image.open(io.BytesIO(image_data)) + except Exception as e: + logger.error(f"Failed to read screenshot: {e}") + return [{"type": "retry", "params": {}}] + + # 构建历史记录字符串 + if len(self.history) == 0: + history_str = "(No history)" + else: + history_str = "\\n".join(f"{idx}. {h}" for idx, h in enumerate(self.history, 1)) + + # 构建decider prompt + decider_prompt = self.decider_prompt_template.format( + task=self.task_description, + history=history_str + ) + + # 调用decider获取决策 + decider_response = self._call_decider(encoded_image, decider_prompt) + + if decider_response is None: + logger.error("Decider调用失败，返回重试操作") + return [{"type": "retry", "params": {}}] + + # 将响应添加到历史记录 + self.history.append(json.dumps(decider_response, ensure_ascii=False)) + + # 解析操作 + action = decider_response.get("action") + parameters = decider_response.get("parameters", {}) + reasoning = decider_response.get("reasoning", "") + + logger.info(f"Action: {action}, Reasoning: {reasoning}") + + # 记录推理过程 + self._add_react( + reasoning=reasoning, + action=action, + parameters=parameters, + step_index=step_index + ) + + # 根据操作类型构建动作序列 + return self._build_action_sequence(action, parameters, reasoning, encoded_image, img) + + def _call_decider(self, encoded_image: str, prompt: str, max_attempts: int = 5) -> Optional[Dict]: + """调用Decider模型""" + temperature = 0.1 + + for attempt in range(max_attempts): + try: + start_time = time.time() + response_str = self.decider_client.chat.completions.create( + model=self.decider_model, + messages=[ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"}}, + {"type": "text", "text": prompt}, + ] + } + ], + temperature=temperature, + timeout=30, + max_tokens=256, + response_format={"type": "json_object"} + ).choices[0].message.content + + elapsed_time = time.time() - start_time + logger.info(f"Decider time: {elapsed_time:.2f}s") + logger.info(f"Decider response: {response_str}") + + return self._parse_json_response(response_str) + + except Exception as e: + temperature = 0.1 + attempt * 0.1 + logger.error(f"Decider调用失败 (attempt {attempt+1}/{max_attempts}): {e}") + if attempt < max_attempts - 1: + time.sleep(2) + + return None + + def _call_grounder(self, encoded_image: str, prompt: str, max_attempts: int = 5) -> Optional[Dict]: + """调用Grounder模型""" + temperature = 0.0 + + for attempt in range(max_attempts): + try: + start_time = time.time() + response_str = self.grounder_client.chat.completions.create( + model=self.grounder_model, + messages=[ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"}}, + {"type": "text", "text": prompt}, + ] + } + ], + temperature=temperature, + timeout=30, + max_tokens=128, + response_format={"type": "json_object"} + ).choices[0].message.content + + elapsed_time = time.time() - start_time + logger.info(f"Grounder time: {elapsed_time:.2f}s") + logger.info(f"Grounder response: {response_str}") + + return self._parse_json_response(response_str) + + except Exception as e: + temperature = 0.1 + attempt * 0.1 + logger.error(f"Grounder调用失败 (attempt {attempt+1}/{max_attempts}): {e}") + if attempt < max_attempts - 1: + time.sleep(2) + + return None + + def _build_action_sequence( + self, + action: str, + parameters: Dict, + reasoning: str, + encoded_image: str, + img: Image.Image + ) -> List[Dict]: + """构建动作序列""" + + if action == "done": + status = parameters.get("status", "success") + return [{ + "type": "done", + "params": {"status": status} + }] + + elif action == "click": + coords = self._get_click_coordinates(parameters, reasoning, encoded_image, img) + if coords is None: + return [{"type": "retry", "params": {}}] + + return [{ + "type": "click", + "params": {"points": coords} + }] + + elif action == "input": + text = parameters.get("text", "") + return [{ + "type": "input", + "params": {"text": text} + }] + + elif action == "swipe": + direction = parameters.get("direction") + start_coords = parameters.get("start_coords") + end_coords = parameters.get("end_coords") + + if start_coords and end_coords: + start = self._convert_qwen3_coordinates(start_coords, img.width, img.height, is_bbox=False) + end = self._convert_qwen3_coordinates(end_coords, img.width, img.height, is_bbox=False) + return [{ + "type": "swipe", + "params": {"points": [start[0], start[1], end[0], end[1]]} + }] + elif direction: + return [{ + "type": "swipe", + "params": {"direction": direction, "scale": 0.5} + }] + + elif action == "wait": + duration = parameters.get("duration", 2) + # 框架已经有等待逻辑，这里返回空操作 + logger.info(f"Waiting {duration}s...") + return [] + + else: + logger.warning(f"Unknown action: {action}") + return [{"type": "retry", "params": {}}] + + def _get_click_coordinates( + self, + parameters: Dict, + reasoning: str, + encoded_image: str, + img: Image.Image + ) -> Optional[List[int]]: + """获取点击坐标""" + + if self.use_e2e: + # E2E模式：从decider直接获取bbox + bbox = parameters.get("bbox") + if bbox is None: + logger.error("E2E mode: bbox not found") + return None + + bbox = self._convert_qwen3_coordinates(bbox, img.width, img.height, is_bbox=True) + x1, y1, x2, y2 = bbox + return [(x1 + x2) // 2, (y1 + y2) // 2] + + else: + # 非E2E模式：调用grounder + target_element = parameters.get("target_element", "") + bbox_flag = True + + if bbox_flag: + grounder_prompt = self.grounder_prompt_template_bbox.format( + reasoning=reasoning, + description=target_element + ) + else: + grounder_prompt = self.grounder_prompt_template_no_bbox.format( + reasoning=reasoning, + description=target_element + ) + + grounder_response = self._call_grounder(encoded_image, grounder_prompt) + + if grounder_response is None: + logger.error("Grounder调用失败") + return None + + if bbox_flag: + # 获取bbox + bbox = None + for key in grounder_response: + if key.lower() in ["bbox", "bbox_2d", "bbox-2d", "bbox_2d", "bbox2d"]: + bbox = grounder_response[key] + break + + if bbox is None: + logger.error("Grounder response missing bbox") + return None + + bbox = self._convert_qwen3_coordinates(bbox, img.width, img.height, is_bbox=True) + x1, y1, x2, y2 = bbox + return [(x1 + x2) // 2, (y1 + y2) // 2] + else: + # 获取坐标点 + coordinates = grounder_response.get("coordinates") + if coordinates is None: + logger.error("Grounder response missing coordinates") + return None + + return self._convert_qwen3_coordinates(coordinates, img.width, img.height, is_bbox=False) + + def _convert_qwen3_coordinates(self, coords, img_width: int, img_height: int, is_bbox: bool = True): + """转换Qwen3坐标（0-1000范围）到绝对坐标""" + if is_bbox: + x1, y1, x2, y2 = coords + abs_x1 = int(x1 / 1000.0 * img_width) + abs_y1 = int(y1 / 1000.0 * img_height) + abs_x2 = int(x2 / 1000.0 * img_width) + abs_y2 = int(y2 / 1000.0 * img_height) + return [abs_x1, abs_y1, abs_x2, abs_y2] + else: + x, y = coords + abs_x = int(x / 1000.0 * img_width) + abs_y = int(y / 1000.0 * img_height) + return [abs_x, abs_y] + + def _parse_json_response(self, response_str: str) -> Optional[Dict]: + """解析JSON响应，支持代码块包裹的格式""" + import re + + # 移除可能的代码块标记 + response_str = response_str.strip() + if response_str.startswith("```json"): + response_str = response_str[7:] + elif response_str.startswith("```"): + response_str = response_str[3:] + + if response_str.endswith("```"): + response_str = response_str[:-3] + + response_str = response_str.strip() + + try: + return json.loads(response_str) + except json.JSONDecodeError as e: + logger.error(f"JSON解析失败: {e}") + logger.error(f"Response: {response_str}") + return None diff --git a/runner/providers/mobiagent/prompts/decider_v2.md b/runner/providers/mobiagent/prompts/decider_v2.md new file mode 100644 index 00000000..e5f3fe37 --- /dev/null +++ b/runner/providers/mobiagent/prompts/decider_v2.md @@ -0,0 +1,13 @@ + +You are a phone-use AI agent. Now your task is "{task}". +Your action history is: +{history} +Please provide the next action based on the screenshot and your action history. You should do careful reasoning before providing the action. +Your action space includes: +- Name: click, Parameters: target_element (a high-level description of the UI element to click). +- Name: swipe, Parameters: direction (one of UP, DOWN, LEFT, RIGHT). +- Name: input, Parameters: text (the text to input). +- Name: wait, Parameters: (no parameters, will wait for 1 second). +- Name: done, Parameters: status (the completion status of the current task, one of `success', `suspended` and `failed`). +Your output should be a JSON object with the following format: +{{"reasoning": "Your reasoning here", "action": "The next action (one of click, input, swipe, wait, done)", "parameters": {{"param1": "value1", ...}}}} \ No newline at end of file diff --git a/runner/providers/mobiagent/prompts/e2e_qwen3.md b/runner/providers/mobiagent/prompts/e2e_qwen3.md new file mode 100644 index 00000000..6f1c7e65 --- /dev/null +++ b/runner/providers/mobiagent/prompts/e2e_qwen3.md @@ -0,0 +1,13 @@ + +You are a phone-use AI agent. Now your task is "{task}". +Your action history is: +{history} +Please provide the next action based on the screenshot and your action history. You should do careful reasoning before providing the action. +Your action space includes: +- Name: click, Parameters: target_element (a high-level description of the UI element to click), bbox (an bounding box of the target element,[x1, y1, x2, y2]). +- Name: swipe, Parameters: direction (one of UP, DOWN, LEFT, RIGHT), start_coords (the starting absolute coordinate [x, y]), end_coords (the ending absolute coordinate [x, y]). +- Name: input, Parameters: text (the text to input). +- Name: wait, Parameters: (no parameters, will wait for 1 second). +- Name: done, Parameters: status (the completion status of the current task, one of `success', `suspended` and `failed`). +Your output should be a JSON object with the following format: +{{"reasoning": "Your reasoning here", "action": "The next action (one of click, input, swipe, wait, done)", "parameters": {{"param1": "value1","param2": "value2", ...}}}} \ No newline at end of file diff --git a/runner/providers/mobiagent/prompts/grounder_qwen3_bbox.md b/runner/providers/mobiagent/prompts/grounder_qwen3_bbox.md new file mode 100644 index 00000000..60397d31 --- /dev/null +++ b/runner/providers/mobiagent/prompts/grounder_qwen3_bbox.md @@ -0,0 +1,5 @@ + +Based on user's intent and the description of the target UI element, locate the element in the screenshot. +User's intent: {reasoning} +Target element's description: {description} +Report the bbox coordinates in JSON format. \ No newline at end of file diff --git a/runner/providers/mobiagent/prompts/grounder_qwen3_coordinates.md b/runner/providers/mobiagent/prompts/grounder_qwen3_coordinates.md new file mode 100644 index 00000000..2f956c1f --- /dev/null +++ b/runner/providers/mobiagent/prompts/grounder_qwen3_coordinates.md @@ -0,0 +1,5 @@ + +Based on user's intent and the description of the target UI element, locate the element in the screenshot. +User's intent: {reasoning} +Target element's description: {description} +Report the point coordinates in JSON format. \ No newline at end of file diff --git a/runner/providers/mobiagent/prompts/planner_android.md b/runner/providers/mobiagent/prompts/planner_android.md new file mode 100644 index 00000000..e49c7ead --- /dev/null +++ b/runner/providers/mobiagent/prompts/planner_android.md @@ -0,0 +1,72 @@ +# 移动任务规划器 (Android) + +你是一个 Android 设备的移动任务规划助手。你的职责是分析用户任务描述并确定： +1. 应该使用哪个移动应用来完成任务 +2. 一个经过优化的任务描述 + +## 任务描述 +{task_description} + +## 可用经验 +{experience_content} + +## 你的任务 +基于任务描述和可用的经验，提供： +1. **app_name**: 最适合此任务的移动应用名称（中文） +2. **final_task_description**: 一个清晰且可执行的优化任务描述 + +## 可用的 Android 应用 +- 淘宝 +- 京东 +- 拼多多 +- 美团 +- 美团外卖 +- 大众点评 +- 饿了么 +- 支付宝 +- 微信 +- QQ +- 微博 +- 小红书 +- 抖音 +- 快手 +- 哔哩哔哩 +- 爱奇艺 +- 腾讯视频 +- 优酷 +- 携程 +- 同程旅行 +- 飞猪 +- 去哪儿 +- 滴滴出行 +- 高德地图 +- 百度地图 +- 知乎 +- QQ音乐 +- 网易云音乐 +- 酷狗音乐 +- 闲鱼 +- 华为商城 +- 华为音乐 +- 华为视频 +- 华为应用市场 +- 华住会 +- 浏览器 + +## 输出格式 +只返回 JSON 对象（不要包含任何额外文本、解释或 markdown 代码块）： + +{{ + "app_name": "<应用名称（中文）>", + "final_task_description": "<优化后的任务描述>" +}} + +## 指南 +1. 根据任务需求选择最合适的应用 +2. 保持优化后的任务描述清晰、具体且可执行 +3. 如果任务中提到了特定应用，使用该应用 +4. 如果没有提到特定应用，选择最合适的应用 +5. 优化后的描述应保持用户的原始意图，同时更加精确 +6. 只返回 JSON 对象，不要包含 markdown 格式或额外文本 + +现在请分析任务并以 JSON 格式提供你的响应。 diff --git a/runner/providers/mobiagent/prompts/planner_harmony.md b/runner/providers/mobiagent/prompts/planner_harmony.md new file mode 100644 index 00000000..d0bd6350 --- /dev/null +++ b/runner/providers/mobiagent/prompts/planner_harmony.md @@ -0,0 +1,76 @@ +# 移动任务规划器 (Harmony) + +你是一个 Harmony 设备的移动任务规划助手。你的职责是分析用户任务描述并确定： +1. 应该使用哪个移动应用来完成任务 +2. 一个经过优化的任务描述 + +## 任务描述 +{task_description} + +## 可用经验 +{experience_content} + +## 你的任务 +基于任务描述和可用的经验，提供： +1. **app_name**: 最适合此任务的移动应用名称（中文） +2. **final_task_description**: 一个清晰且可执行的优化任务描述 + +## 可用的 Harmony 应用 +- 淘宝 +- 京东 +- 拼多多 +- 美团 +- 美团外卖 +- 大众点评 +- 饿了么 +- 微信 +- 微博 +- 小红书 +- 哔哩哔哩 +- 携程 +- 携程旅行 +- 飞猪 +- 飞猪旅行 +- 同程 +- 滴滴出行 +- 高德地图 +- 知乎 +- QQ音乐 +- 闲鱼 +- 天气 +- 什么值得买 +- 慧通差旅 +- 航旅纵横 +- 电子邮件 +- 图库 +- 日历 +- 心声社区 +- 信息 +- 文件管理 +- 运动健康 +- 智慧生活 +- 豆包 +- WeLink +- 设置 +- 懂车帝 +- 浏览器 +- IntelliOS +- PowerAgent + +## 输出格式 +只返回 JSON 对象（不要包含任何额外文本、解释或 markdown 代码块）： + +{{ + "app_name": "<应用名称（中文）>", + "final_task_description": "<优化后的任务描述>" +}} + +## 指南 +1. 根据任务需求选择最合适的应用 +2. 保持优化后的任务描述清晰、具体且可执行 +3. 如果任务中提到了特定应用，使用该应用 +4. 如果没有提到特定应用，选择最合适的应用 +5. 优化后的描述应保持用户的原始意图，同时更加精确 +6. 只返回 JSON 对象，不要包含 markdown 格式或额外文本 + +现在请分析任务并以 JSON 格式提供你的响应。 diff --git a/runner/providers/qwen/prompts.py b/runner/providers/qwen/prompts.py new file mode 100644 index 00000000..c8e408f3 --- /dev/null +++ b/runner/providers/qwen/prompts.py @@ -0,0 +1,67 @@ +SYSTEM_PROMPT = """ +## 角色定位 +你是一个拥有视觉能力的专业 AI 移动端助手。你通过分析手机屏幕截图（原始图及带索引标注的 UI 元素图）来理解当前状态，并通过调用预定义的操作序列来协助用户完成任务。 + +## 任务执行原则 +1. **多模态观察**：在决定操作前，请先仔细比对原始截图与标注截图，识别关键图标、文字标签及其对应的索引。 +2. **逐步推理**： + - **观察**：当前屏幕上有什么？（例如：在什么 App 里，有哪些按钮，是否有弹窗）。 + - **分析**：当前状态距离任务目标还有多远？上一条操作是否生效？ + - **计划**：下一步应该做什么，以及为什么要执行这个动作。 +3. **容错与重试**：如果当前页面没出现预期元素，尝试滑动寻找或检查是否需要点击“返回”。 +4. **准确性**：仅操作标注图中存在的索引。如果目标元素未被标注，请尝试通过坐标滑动或点击其父级容器。 + +## 动作空间 (Action Space) +你可以执行以下操作： + +1. **CLICK [index]**：点击索引为 `index` 的 UI 元素。 +2. **INPUT [index] **：在索引为 `index` 的输入框中输入指定文本（会自动先执行点击聚焦）。 +3. **SWIPE [start_x, start_y] to [end_x, end_y]**： + - 坐标范围 0-1000（如 [500, 500] 表示屏幕中心）。 + - 向上滚动以查看更多内容：`SWIPE [500, 800] to [500, 200]`。 +4. **LONG PRESS [index]**：长按索引为 `index` 的元素。 +5. **KEY_BACK**：模拟物理返回键,返回上一页。 +6. **KEY_HOME**：返回手机主屏幕。 +7. **OPEN_APP **：从主屏或搜索中打开指定应用。 +8. **WAIT**：当前页面正在加载（如进度条、转圈），等待一会儿。 +9. **DONE**：任务结束，完成/失败/挂起（需要人工介入），并简要总结结果。 + +## 输出格式 +你必须以严格的 JSON 格式输出，不要包含任何额外的 Markdown 文本或解释： +{ + "reasoning": "描述当前屏幕的关键信息及状态。基于历史记录和当前观测，分析为什么要执行下一步。", + "action": "ACTION_TYPE", + "params": { + "index": null, // 适用于 CLICK, LONG PRESS + "text": "", // 适用于 INPUT + "start": [x, y], // 适用于 SWIPE + "end": [x, y], // 适用于 SWIPE + "app_name": "" // 适用于 OPEN_APP + "status": "success/failed/suspended" // 适用于 DONE"，分别表示完成/失败/挂起（需要人工介入） + } +} +params的字段说明： +- 如果动作为 CLICK 或 LONG PRESS，`params` 仅包含 `index`。 +- 如果动作为 INPUT，`params` 仅包含 `text`。 +- 如果动作为 SWIPE，`params` 仅包含 `start` 和 `end`。 +- 如果动作为 OPEN_APP，`params` 仅包含 `app_name`。 +- 如果动作为 DONE，`params` 仅包含 `status`。 +""" + +USER_PROMPT_TEMPLATE = """ +### 任务目标 +{task_description} + +### 历史操作记录 (由远及近) +{history} + +### 当前环境观测 +- **标注层级**：当前提供 1 张原始截图和 {layer_count} 层视觉标注图。 +- **当前状态**：请结合历史记录，判断你当前所处的页面。 + +### 要求 +1. 检查上一条指令是否执行成功（对比截图差异）。 +2. 如果任务已完成，请使用 DONE 动作。 +3. 如果陷入死循环（如反复点击同一按钮），请尝试不同的路径或 KEY_BACK。 +4. 请输出 JSON。 +""" \ No newline at end of file diff --git a/runner/providers/qwen/qwen_task.py b/runner/providers/qwen/qwen_task.py new file mode 100644 index 00000000..7cba3f1c --- /dev/null +++ b/runner/providers/qwen/qwen_task.py @@ -0,0 +1,347 @@ + +import os +import time +import json +import base64 +import logging +import io +import re +from typing import Dict, List, Optional +from PIL import Image + +from base_task import BaseTask +from providers.qwen.utils import process_screenshot +from providers.qwen.prompts import SYSTEM_PROMPT, USER_PROMPT_TEMPLATE +import sys + +# 使用模块级别的logger（级别由 setup_logging() 统一配置） +logger = logging.getLogger(__name__) + +# base64编码图像的辅助函数 +def image_to_base64(image): + buffered = io.BytesIO() + image.save(buffered, format="JPEG") + return base64.b64encode(buffered.getvalue()).decode('utf-8') + +class QwenTask(BaseTask): + """ + Qwen VLM任务适配器（支持Qwen3-VL、Gemini等通用VLLM） + 使用Set-of-Marks (SoM)方法进行元素交互 + """ + + def __init__( + self, + task_description: str, + device, + data_dir: str, + device_type: str = "Android", + use_step_loop: bool = True, + # 统一参数 + api_base: str = None, + api_key: str = "", + model: str = None, + temperature: float = None, + # 向后兼容的旧参数 + model_name: str = None, + **kwargs + ): + super().__init__( + task_description=task_description, + device=device, + data_dir=data_dir, + device_type=device_type, + use_step_loop=use_step_loop, + **kwargs + ) + + # 处理参数优先级: 新参数 > 旧参数 > 默认值 + self.model_name = model if model is not None else model_name + self.api_base = api_base + self.api_key = api_key + self.temperature = temperature if temperature is not None else 0.0 + + # 初始化OpenAI客户端 + try: + from openai import OpenAI + self.client = OpenAI( + base_url=self.api_base, + api_key=self.api_key or "EMPTY" + ) + logger.info(f"Initialized OpenAI client for {self.model_name} at {self.api_base}") + except ImportError: + logger.error("OpenAI package not found. Please install it: pip install openai") + raise + + def execute_step(self, step_index: int) -> List[Dict]: + """ + 执行单一步骤：观察 -> 思考 -> 执行 + """ + logger.info(f"使用{self.model_name}执行第{step_index}步") + + current_screenshot_path = os.path.join(self.data_dir, f"{step_index}.jpg") + + # 验证截图是否存在 + if not os.path.exists(current_screenshot_path): + logger.warning(f"截图不存在于{current_screenshot_path}，正在重新获取") + self.device.screenshot(current_screenshot_path) + + # 处理SoM标注 + logger.info("处理截图以标注UI元素(SoM)...") + bounds_list, layer_images = process_screenshot(current_screenshot_path) + layer_count = len(layer_images) + logger.info(f"生成了{layer_count}层SoM标注图层") + + # 保存调试用的图层图像 + for idx, img in enumerate(layer_images): + layer_path = os.path.join(self.data_dir, f"{step_index}_layer_{idx+1}.jpg") + img.convert("RGB").save(layer_path) + + # 构造提示词 + # 加载历史记录 + history_str = self._format_history() + + user_prompt = USER_PROMPT_TEMPLATE.format( + task_description=self.task_description, + history=history_str, + layer_count=layer_count + ) + + messages = [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": []} + ] + + # 添加文本内容 + messages[1]["content"].append({"type": "text", "text": user_prompt}) + + # 添加原始截图 + screenshot_b64 = image_to_base64(Image.open(current_screenshot_path)) + messages[1]["content"].append({ + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"} + }) + + # 添加图层标注图像 + for i, img in enumerate(layer_images): + layer_b64 = image_to_base64(img) + messages[1]["content"].append({"type": "text", "text": f"第{i+1}层:"}) + messages[1]["content"].append({ + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{layer_b64}"} + }) + + # 模型推理 + try: + logger.info(f"[{self.model_name}] 正在发送请求到VLLM... (超时: 60s)") + start_time = time.time() + completion = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + temperature=0.0, + max_tokens=1024, + timeout=60.0, + response_format={"type": "json_object"} + ) + elapsed = time.time() - start_time + logger.info(f"[{self.model_name}] VLLM响应已接收，耗时{elapsed:.2f}s") + + response_content = completion.choices[0].message.content + logger.info(f"[{self.model_name}] 原始响应: {response_content}") + + except Exception as e: + logger.error(f"[{self.model_name}] VLLM请求失败: {e}") + return [{"type": "error", "error": str(e)}] + + # 解析响应 + try: + action_data = self._parse_json_response(response_content) + logger.info(f"解析的动作: {action_data}") + + # 映射动作到runner的动作格式 + return self._map_to_runner_actions(action_data, bounds_list, step_index) + + except Exception as e: + logger.error(f"解析响应失败: {e}") + return [{"type": "error", "error": str(e)}] + + def _format_history(self) -> str: + if not self.actions: + return "(暂无历史记录)" + + formatted = [] + for i, act in enumerate(self.actions[-5:]): + act_type = act.get("type", "unknown") + formatted.append(f"步骤 {act.get('action_index', '?')}: {act_type}") + + return "\n".join(formatted) + + def _parse_json_response(self, content: str) -> Dict: + # 从代码块中提取JSON（如果存在） + json_match = re.search(r"```json\s*(.*?)\s*```", content, re.DOTALL) + if json_match: + json_str = json_match.group(1) + else: + json_str = content + + json_str = json_str.strip() + + return json.loads(json_str) + + def _map_to_runner_actions(self, action_data: Dict, bounds_list: List, step_index: int) -> List[Dict]: + action_type = action_data.get("action", "").upper().strip() + params = action_data.get("params", {}) + reasoning = action_data.get("reasoning", "") + + # 获取屏幕尺寸以进行坐标转换 + current_screenshot_path = os.path.join(self.data_dir, f"{step_index}.jpg") + screen_width, screen_height = 1080, 1920 + if os.path.exists(current_screenshot_path): + with Image.open(current_screenshot_path) as img: + screen_width, screen_height = img.size + logger.info(f"屏幕尺寸: {screen_width}x{screen_height}") + + # 规范化动作类型别名 + action_type_map = { + "TAP": "CLICK", + "LONG_PRESS": "LONG PRESS", + "SCROLL": "SWIPE", + "TYPE": "INPUT", + "INPUT_TEXT": "INPUT", + "OPEN": "OPEN_APP", + "BACK": "KEY_BACK", + "HOME": "KEY_HOME", + } + action_type = action_type_map.get(action_type, action_type) + + logger.info(f"动作类型: {action_type}, 参数: {params}") + + # 保存推理过程 + self._add_react(reasoning, action_type, params, step_index) + + runner_actions = [] + + # 将相对坐标(0-1000)转换为绝对屏幕坐标的辅助函数 + def convert_coordinates(coord_list): + """将[x, y]从1000x1000相对坐标转换为绝对屏幕坐标""" + if not coord_list or len(coord_list) != 2: + return coord_list + rel_x, rel_y = coord_list + abs_x = int(rel_x * screen_width / 1000) + abs_y = int(rel_y * screen_height / 1000) + logger.info(f"坐标转换: [{rel_x}, {rel_y}] (相对) -> [{abs_x}, {abs_y}] (绝对)") + return [abs_x, abs_y] + + if action_type in ["CLICK", "LONG PRESS"]: + index = params.get("index") + coordinate = params.get("coordinate") or params.get("position") + + # 情形1: 使用元素索引 + if index is not None and coordinate is None: + if 0 <= index < len(bounds_list): + bounds = bounds_list[index] + # 计算中心坐标 + center_x = (bounds[0] + bounds[2]) // 2 + center_y = (bounds[1] + bounds[3]) // 2 + + runner_action = { + "type": "click" if action_type == "CLICK" else "long_press", + "params": { + "coordinate": [center_x, center_y], + "index": index, + "bounds": bounds, + "reasoning": reasoning + } + } + runner_actions.append(runner_action) + else: + logger.warning(f"边界列表大小为{len(bounds_list)}，索引{index}无效") + runner_actions.append({"type": "fail", "params": {"reason": f"无效的元素索引{index}"}}) + + # 情形2: 使用直接坐标 + elif coordinate is not None: + abs_coordinate = convert_coordinates(coordinate) + runner_action = { + "type": "click" if action_type == "CLICK" else "long_press", + "params": { + "coordinate": abs_coordinate, + "reasoning": reasoning + } + } + runner_actions.append(runner_action) + + else: + logger.warning(f"CLICK/LONG PRESS动作缺少索引和坐标") + runner_actions.append({"type": "fail", "params": {"reason": "缺少点击动作的索引或坐标"}}) + + elif action_type == "INPUT": + text = params.get("text", "") + runner_actions.append({ + "type": "input", + "params": {"text": text, "reasoning": reasoning} + }) + + elif action_type == "SWIPE": + start = params.get("start") + end = params.get("end") + if start and end: + # 将相对坐标转换为绝对坐标 + abs_start = convert_coordinates(start) + abs_end = convert_coordinates(end) + runner_actions.append({ + "type": "swipe", + "params": { + "start_coordinate": abs_start, + "end_coordinate": abs_end, + "reasoning": reasoning + } + }) + else: + runner_actions.append({"type": "fail", "params": {"reason": "缺少滑动坐标"}}) + + elif action_type == "OPEN_APP": + app_name = params.get("app_name") + if app_name: + runner_actions.append({ + "type": "open_app", + "params": {"app_name": app_name, "reasoning": reasoning} + }) + logger.info(f"打开应用: {app_name}") + else: + logger.warning("OPEN_APP 动作缺少应用名称") + runner_actions.append({"type": "fail", "params": {"reason": "缺少应用名称"}}) + + elif action_type == "KEY_BACK": + runner_actions.append({ + "type": "back", + "params": {"reasoning": reasoning} + }) + logger.info("执行返回动作") + + elif action_type == "KEY_HOME": + runner_actions.append({ + "type": "home", + "params": {"reasoning": reasoning} + }) + logger.info("执行主屏动作") + + elif action_type == "WAIT": + duration = params.get("duration", 2) + runner_actions.append({ + "type": "wait", + "params": {"seconds": duration, "reasoning": reasoning} + }) + logger.info(f"等待 {duration} 秒") + + elif action_type == "DONE": + status = params.get("status", "success") + runner_actions.append({ + "type": "done", + "params": {"status": status, "reasoning": reasoning} + }) + logger.info(f"任务完成，状态: {status}") + + else: + logger.warning(f"未知的动作类型: {action_type}") + runner_actions.append({"type": "fail", "params": {"reason": f"未知的动作{action_type}"}}) + + return runner_actions diff --git a/runner/providers/qwen/utils.py b/runner/providers/qwen/utils.py new file mode 100644 index 00000000..d0035635 --- /dev/null +++ b/runner/providers/qwen/utils.py @@ -0,0 +1,133 @@ +import os +import sys +from PIL import Image, ImageDraw, ImageFont +from pathlib import Path + +# 导入utils.parse_omni中的extract_all_bounds函数 +current_file = Path(__file__).resolve() +project_root = current_file.parent.parent.parent.parent +if str(project_root) not in sys.path: + sys.path.append(str(project_root)) + +try: + from utils.parse_omni import extract_all_bounds +except ImportError: + print("警告: 无法导入utils.parse_omni，extract_all_bounds将不可用") + extract_all_bounds = None + +font_path = project_root / "msyh.ttf" + +def check_text_overlap(text_rect1, text_rect2): + """检查两个文本矩形是否重叠""" + x1, y1, x2, y2 = text_rect1 + x3, y3, x4, y4 = text_rect2 + + if x2 < x3 or x4 < x1 or y2 < y3 or y4 < y1: + return False + return True + +def draw_bounds_on_screenshot(image_or_path, layer, output_path=None): + """ + 在截图上绘制边界框 + Args: + image_or_path: PIL Image或图像路径 + layer: (index, bounds, text_rect)的列表 + output_path: 如果提供，将图像保存到此处 + Returns: + 绘制了边界框的PIL Image + """ + if isinstance(image_or_path, str): + image = Image.open(image_or_path).convert('RGB') + else: + image = image_or_path.copy().convert('RGB') + + draw = ImageDraw.Draw(image) + + try: + font = ImageFont.truetype(str(font_path), 40) + except Exception: + font = ImageFont.load_default() + + for index, bounds, text_rect in layer: + left, top, right, bottom = bounds + draw.rectangle([left, top, right, bottom], outline='red', width=5) + + text = str(index) + text_x, text_y, _, _ = text_rect + + draw.rectangle(text_rect, fill='red', outline='red', width=1) + draw.text((text_x, text_y), text, fill='white', font=font) + + if output_path: + image.save(output_path) + + return image + +def assign_bounds_to_layers(image_path, bounds_list): + """ + 将边界框分配到不同的层以避免文本重叠 + Returns: + 层的列表，每个层是(index, bounds, text_rect)的列表 + """ + image = Image.open(image_path) + draw = ImageDraw.Draw(image) + + try: + font = ImageFont.truetype(str(font_path), 40) + except Exception: + font = ImageFont.load_default() + + layers = [] + + for index, bounds in enumerate(bounds_list): + left, top, right, bottom = bounds + + text = str(index) + bbox = draw.textbbox((0, 0), text, font=font) + text_width = bbox[2] - bbox[0] + text_height = bbox[3] - bbox[1] + + text_x = left + text_y = top + text_rect = (text_x, text_y, text_x + text_width + 5, text_y + text_height + 15) + + placed = False + for layer in layers: + can_place = True + for _, existing_bounds, existing_text_rect in layer: + if (check_text_overlap(bounds, existing_bounds) or + check_text_overlap(text_rect, existing_bounds) or + check_text_overlap(bounds, existing_text_rect) or + check_text_overlap(text_rect, existing_text_rect)): + can_place = False + break + + if can_place: + layer.append((index, bounds, text_rect)) + placed = True + break + + if not placed: + layers.append([(index, bounds, text_rect)]) + + return layers + +def process_screenshot(screenshot_path): + """ + 处理截图：提取边界框并生成图层 + Returns: + (bounds_list, layer_images) + layer_images是PIL Image的列表 + """ + if extract_all_bounds is None: + return [], [] + + bounds_list = extract_all_bounds(screenshot_path) + layers_data = assign_bounds_to_layers(screenshot_path, bounds_list) + + layer_images = [] + for layer in layers_data: + img = draw_bounds_on_screenshot(screenshot_path, layer) + layer_images.append(img) + + return bounds_list, layer_images diff --git a/runner/providers/uitars/ui_tars_helper.py b/runner/providers/uitars/ui_tars_helper.py new file mode 100644 index 00000000..0e8af573 --- /dev/null +++ b/runner/providers/uitars/ui_tars_helper.py @@ -0,0 +1,219 @@ +import re +import logging +from typing import Dict, List, Tuple, Optional, Any + +# 使用模块级别的logger（级别由 setup_logging() 统一配置） +logger = logging.getLogger(__name__) + +# 内置提示词模板 +PROMPT_TEMPLATE = """You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. + +## Output Format +``` +Thought: ... +Action: ... +``` + +## Action Space + +click(point='x1 y1') +long_press(point='x1 y1') +type(content='') #If you want to submit your input, use "\\n" at the end of `content`. +scroll(point='x1 y1', direction='down or up or right or left') +drag(start_point='x1 y1', end_point='x2 y2') +press_home() +press_back() +finished(content='xxx') # Use escape characters \\', \\", and \\n in content part to ensure we can parse the content in normal python string format. + +## Note +- Use {language} in `Thought` part. +- Write a small plan and finally summarize your next action (with its target element) in one sentence in `Thought` part. +- To open an app, use click() to tap on the app icon you can see in the screenshot, don't use open_app(). +- Always look for app icons, buttons, or UI elements in the current screenshot and click on them. + +## User Instruction +{instruction}""" + +class UITarsHelper: + @staticmethod + def parse_response(response: str, image_width: int, image_height: int, model_name: str = "UI-TARS-1.5-7B") -> Tuple[str, str, List[Dict]]: + """ + 解析模型响应 + Returns: + (thought, raw_action, parsed_actions_list) + """ + try: + # 由ui_tars添加到路径中 + import ui_tars.action_parser as ui_tars_parser + except ImportError: + logger.error("未能导入utils(UI-TARS)。请确保sys.path正确。") + raise + + # 计算smart resize尺寸 + smart_height, smart_width = ui_tars_parser.smart_resize( + image_height, image_width, + factor=ui_tars_parser.IMAGE_FACTOR + ) + + # 使用官方解析 + actions = ui_tars_parser.parse_action_to_structure_output( + response, + factor=ui_tars_parser.IMAGE_FACTOR, + origin_resized_height=smart_height, + origin_resized_width=smart_width, + model_type="qwen25vl" + ) + + if not actions: + logger.warning("未解析到结构化动作。") + return "", response, [] + + action = actions[0] + thought = action.get('thought', '') + raw_action = response.split("Action:")[-1].strip().split('\n')[0] if "Action:" in response else response + + # 转换为PyAutoGUI代码（用于日志） + pyautogui_code = ui_tars_parser.parsing_response_to_pyautogui_code( + action, image_height, image_width + ) + logger.debug(f"PyAutoGUI代码: {pyautogui_code}") + + # 转换为内部格式 + internal_action = UITarsHelper._convert_action_to_internal(action, pyautogui_code, image_width, image_height, model_name) + + return thought, raw_action, [internal_action] + + @staticmethod + def _convert_action_to_internal(action: Dict, pyautogui_code: str, width: int, height: int, model_name: str) -> Dict: + """ + 转换为框架的标准动作格式 {type: ..., params: ...} + """ + action_type = action.get('action_type', '') + action_inputs = action.get('action_inputs', {}) + raw_text = action.get('text', '') + + # 从原始文本解析(x,y)的辅助函数，支持多种格式 + def extract_coords(text, *keys): + for key in keys: + # 模式1: key='(x,y)' + match = re.search(f"{key}=['\"]\((\d+),(\d+)\)['\"]", text) + if match: return int(match.group(1)), int(match.group(2)) + match = re.search(f"{key}=\((\d+),(\d+)\)", text) + if match: return int(match.group(1)), int(match.group(2)) + + # 模式2: key='x y' + match = re.search(f"{key}=['\"](\d+)\s+(\d+)['\"]", text) + if match: return int(match.group(1)), int(match.group(2)) + return None + + # UI-TARS-1.5-7B特殊处理 + # 使用返回的绝对坐标 + if model_name == "UI-TARS-1.5-7B": + # 点击/长按 + if action_type in ["click", "left_single", "left_double", "right_single", "hover", "long_press"]: + coords = extract_coords(raw_text, "start_box", "point") + if coords: + logger.info(f"解析的绝对坐标: {coords}") + return {'type': 'click', 'params': {'points': [coords[0], coords[1]]}} + + # 拖拽/滑动 + elif action_type == "drag": + start = extract_coords(raw_text, "start_box", "start_point") + end = extract_coords(raw_text, "end_box", "end_point") + if start and end: + return { + 'type': 'swipe', + 'params': {'points': [start[0], start[1], end[0], end[1]]} + } + + # 滚动 + elif action_type == "scroll": + direction = action_inputs.get('direction', 'down') + coords = extract_coords(raw_text, "start_box", "point") + params = {'direction': direction} + if coords: + pass + return {'type': 'scroll', 'params': params} + + # 输入/类型 + elif action_type == "type": + content = action_inputs.get('content', '') + return {'type': 'input', 'params': {'text': content}} + + # 完成 + elif action_type == "finished": + content = action_inputs.get('content', 'success') + return {'type': 'done', 'params': {'status': 'success', 'message': content}} + + # 首页/返回 + elif action_type == "press_home": + return {'type': 'home', 'params': {}} + elif action_type == "press_back": + return {'type': 'back', 'params': {}} + + # 其他模型或7B解析未命中的备选方案 + + # 内部映射结果 + result_type = None + result_params = {} + + if pyautogui_code.strip() == "DONE": + content = action_inputs.get('content', 'success') + return {'type': 'done', 'params': {'status': 'success', 'message': content}} + + if action_type in ["click", "long_press"]: + click_match = re.search(r'pyautogui\.click\((\d+(?:\.\d+)?), (\d+(?:\.\d+)?)', pyautogui_code) + if click_match: + x = float(click_match.group(1)) + y = float(click_match.group(2)) + + if model_name == "UI-TARS-1.5": + if 0 <= x <= 1 and 0 <= y <= 1: + x = int(x * width) + y = int(y * height) + else: + x, y = int(x), int(y) + elif 0 <= x <= 1 and 0 <= y <= 1: + x = int(x * width) + y = int(y * height) + else: + x, y = int(x), int(y) + + result_type = 'click' + result_params = {'points': [int(x), int(y)]} + + elif action_type == "type": + content = action_inputs.get('content', '') + result_type = 'input' + result_params = {'text': content} + + elif action_type == "press_home": + result_type = 'home' + elif action_type == "press_back": + result_type = 'back' + elif action_type == "finished": + result_type = 'done' + result_params = {'status': 'success'} + + elif action_type == "scroll": + scroll_match = re.search(r'pyautogui\.scroll\((-?\d+)', pyautogui_code) + direction = 'up' + if scroll_match: + val = int(scroll_match.group(1)) + direction = 'up' if val < 0 else 'down' + result_type = 'swipe' + result_params = {'direction': direction} + elif action_type == "drag": + move_match = re.search(r'pyautogui\.moveTo\((\d+(?:\.\d+)?), (\d+(?:\.\d+)?)', pyautogui_code) + drag_match = re.search(r'pyautogui\.dragTo\((\d+(?:\.\d+)?), (\d+(?:\.\d+)?)', pyautogui_code) + if move_match and drag_match: + sx = float(move_match.group(1)) + sy = float(move_match.group(2)) + ex = float(drag_match.group(1)) + ey = float(drag_match.group(2)) + result_type = 'swipe' + result_params = {'points': [int(sx), int(sy), int(ex), int(ey)]} + if result_type: + return {'type': result_type, 'params': result_params} + + return {'type': 'wait', 'params': {}} diff --git a/runner/providers/uitars/uitars_task.py b/runner/providers/uitars/uitars_task.py new file mode 100644 index 00000000..9241d29b --- /dev/null +++ b/runner/providers/uitars/uitars_task.py @@ -0,0 +1,170 @@ +import sys +import os +import time +import base64 +import json +import logging +from typing import Dict, List, Optional +from openai import OpenAI + +# 使用模块级别的logger（级别由 setup_logging() 统一配置） +logger = logging.getLogger(__name__) + +# 导入UI-TARS所需的模块 +runner_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +agent_root = os.path.dirname(os.path.dirname(os.path.dirname(runner_dir))) + +from base_task import BaseTask +from .ui_tars_helper import UITarsHelper, PROMPT_TEMPLATE + +class UITARSTask(BaseTask): + """ + UI-TARS任务适配器（独立实现） + """ + + def __init__( + self, + task_description: str, + device, + data_dir: str, + device_type: str = "Android", + max_steps: int = 40, + # 统一参数 + api_base: str = None, + model: str = "UI-TARS-1.5-7B", + temperature: float = 0.0, + # UI-TARS 专属参数 + step_delay: float = 2.0, + device_ip: Optional[str] = None, + # 向后兼容的旧参数 + model_base_url: str = None, + model_name: str = None, + **kwargs + ): + super().__init__( + task_description=task_description, + device=device, + data_dir=data_dir, + device_type=device_type, + max_steps=max_steps, + use_step_loop=True, + **kwargs + ) + + # 处理参数优先级: 新参数 > 旧参数 > 默认值 + self.model_base_url = api_base or model_base_url or "http://localhost:8000/v1" + self.model_name = model or model_name or "UI-TARS-1.5-7B" + self.temperature = temperature if temperature is not None else 0.0 + self.step_delay = step_delay + # 初始化OpenAI客户端 + try: + self.client = OpenAI( + base_url=self.model_base_url, + api_key="EMPTY" + ) + logger.info(f"已连接到模型服务: {self.model_base_url}") + except Exception as e: + logger.error(f"初始化OpenAI客户端失败: {e}") + raise + + # UI-TARS特定的历史记录 + self.history = [] + + logger.info("UITARSTask (步骤循环模式)已初始化") + + def execute_step(self, step_index: int) -> List[Dict]: + """ + 执行单一步骤 + """ + screenshot_path = os.path.join(self.data_dir, f"{step_index}.jpg") + + # 读取截图 + if not os.path.exists(screenshot_path): + self.device.screenshot(screenshot_path) + + with open(screenshot_path, "rb") as f: + image_data = base64.b64encode(f.read()).decode('utf-8') + image_data_url = f"data:image/jpeg;base64,{image_data}" + + # 获取图像尺寸 + try: + from PIL import Image + with Image.open(screenshot_path) as img: + width, height = img.size + except Exception as e: + logger.warning(f"获取图像尺寸失败: {e}") + width, height = 1080, 2340 + + # 构建消息 + messages = self._build_messages(image_data_url) + + # 调用模型 + try: + logger.info("调用UI-TARS模型...") + start_time = time.time() + chat_completion = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + temperature=self.temperature, + max_tokens=512, + stream=False + ) + response = chat_completion.choices[0].message.content + logger.info(f"模型响应 ({time.time() - start_time:.2f}s): {response}") + except Exception as e: + logger.error(f"模型调用失败: {e}") + return [{"type": "retry", "params": {"error": str(e)}}] + + # 使用辅助函数解析响应 + try: + thought, raw_action, actions = UITarsHelper.parse_response( + response, width, height, model_name=self.model_name + ) + + # 记录历史 + self.history.append({ + "thought": thought, + "raw_action": raw_action + }) + + # 添加推理记录 + self._add_react(thought, raw_action, {}, step_index) + + return actions + + except Exception as e: + logger.error(f"解析失败: {e}") + return [{"type": "retry", "params": {"error": f"解析失败: {e}"}}] + + def _build_messages(self, image_data_url: str) -> List[Dict]: + """为模型构建消息""" + system_prompt = PROMPT_TEMPLATE.format( + language="Chinese", + instruction=self.task_description + ) + + messages = [ + {"role": "user", "content": system_prompt} + ] + + # 添加历史记录 + # 格式: + # User: ... + # Assistant: Thought: ... \n Action: ... + # ... + # User: <当前图像> + + for record in self.history: + if record['thought'] and record['raw_action']: + content = f"Thought: {record['thought']}\nAction: {record['raw_action']}" + messages.append({"role": "assistant", "content": content}) + + # 添加当前观察 + messages.append({ + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": image_data_url}} + ] + }) + + return messages diff --git a/runner/run.py b/runner/run.py new file mode 100644 index 00000000..b79c52fe --- /dev/null +++ b/runner/run.py @@ -0,0 +1,585 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + +""" +统一的任务执行入口 +支持多种模型（MobiAgent, UI-TARS等） +""" + +import os +import sys +import json +import logging +import argparse +import time +from datetime import datetime +from typing import Dict, List, Optional + +# 添加当前目录到路径 +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from task_manager import TaskManager +from device import create_device, AndroidDevice, HarmonyDevice + + +def setup_logging(log_level: str = "INFO"): + """ + 设置日志 - 配置根logger和所有模块logger + + Args: + log_level: 日志级别 ('DEBUG', 'INFO', 'WARNING', 'ERROR') + """ + numeric_level = getattr(logging, log_level.upper(), None) + if not isinstance(numeric_level, int): + numeric_level = logging.INFO + + # 清除已有的handlers以避免重复 + root = logging.getLogger() + if root.handlers: + for handler in root.handlers[:]: + root.removeHandler(handler) + + # 设置根logger级别 + root.setLevel(numeric_level) + + # 创建日志格式化器 + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + + # 添加标准输出处理器 + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(numeric_level) + console_handler.setFormatter(formatter) + root.addHandler(console_handler) + + # 添加文件处理器 + try: + file_handler = logging.FileHandler('runner.log', encoding='utf-8') + file_handler.setLevel(numeric_level) + file_handler.setFormatter(formatter) + root.addHandler(file_handler) + except Exception as e: + print(f"Warning: Failed to setup file logging: {e}") + + # 配置所有已知模块的logger + module_loggers = [ + 'task_manager', + 'base_task', + 'device', + 'providers.mobiagent.mobile_task', + 'providers.mobiagent.load_md_prompt', + 'providers.qwen.qwen_task', + 'providers.qwen.utils', + 'providers.uitars.uitars_task', + 'providers.uitars.ui_tars_helper', + 'providers.autoglm.autoglm_task', + ] + + for logger_name in module_loggers: + module_logger = logging.getLogger(logger_name) + module_logger.setLevel(numeric_level) + + sys.stdout.flush() + + +def load_tasks(task_file: str) -> List: + """ + 从文件加载任务列表 + + Args: + task_file: 任务文件路径 + + Returns: + 任务列表 + """ + with open(task_file, 'r', encoding='utf-8') as f: + tasks = json.load(f) + return tasks + + +# Provider 默认配置 +PROVIDER_DEFAULTS = { + 'mobiagent': { + 'api_base': 'http://localhost:8000/v1', + 'model': 'MobiMind-1.5-4B', + 'temperature': 0.1, + }, + 'mobiagent_step': { + 'api_base': 'http://localhost:8000/v1', + 'model': 'MobiMind-1.5-4B', + 'temperature': 0.1, + }, + 'uitars': { + 'api_base': 'http://localhost:8000/v1', + 'model': 'UI-TARS-1.5-7B', + 'temperature': 0.0, + }, + 'qwen': { + 'api_base': 'http://localhost:8080/v1', + 'model': 'Qwen3-VL-30B-A3B-Instruct', + 'temperature': 0.0, + }, + 'autoglm': { + 'api_base': 'http://localhost:8000/v1', + 'model': 'autoglm-phone-9b', + 'temperature': 0.0, + }, +} + + +def parse_args(): + """解析命令行参数""" + parser = argparse.ArgumentParser( + description='统一的GUI Agent任务执行器', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +示例: + # 使用 MobiAgent 执行单任务 + python run.py --provider mobiagent --task "打开淘宝搜索手机" --api-base http://localhost:8000/v1 + + # 使用 UI-TARS 执行任务 + python run.py --provider uitars --task "打开微信" --api-base http://localhost:8000/v1 --model UI-TARS-1.5-7B + + # 使用 Qwen VLM 执行任务 + python run.py --provider qwen --task "在微博查看新闻" --api-base http://localhost:8080/v1 +""" + ) + + # ==================== 基础参数 ==================== + parser.add_argument('--provider', type=str, default='mobiagent_step', + choices=['mobiagent', 'mobiagent_step', 'uitars', 'qwen', 'autoglm'], + help='模型提供者 (默认: mobiagent_step, mobiagent是mobiagent_step的别名)') + parser.add_argument('--device-type', type=str, default='Android', + choices=['Android', 'Harmony'], + help='设备类型 (默认: Android)') + parser.add_argument('--device-id', type=str, default=None, + help='设备ID或IP地址') + parser.add_argument('--max-steps', type=int, default=40, + help='最大步骤数 (默认: 40)') + parser.add_argument('--log-level', type=str, default='INFO', + choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'], + help='日志级别 (默认: INFO)') + + # ==================== 任务相关 ==================== + parser.add_argument('--task-file', type=str, default=None, + help='任务文件路径 (task.json 或 task_mobiflow.json)') + parser.add_argument('--task', type=str, default=None, + help='单个任务描述（直接指定任务）') + parser.add_argument('--output-dir', type=str, default='results', + help='结果输出目录 (默认: results)') + + # ==================== 通用模型参数 ==================== + parser.add_argument('--api-base', type=str, default=None, + help='模型服务基础URL (通用参数, 默认按provider自动设置)') + parser.add_argument('--api-key', type=str, default='', + help='API密钥 (通用参数, 无需验证时可留空)') + parser.add_argument('--model', type=str, default=None, + help='模型名称 (通用参数, 默认按provider自动设置)') + parser.add_argument('--temperature', type=float, default=None, + help='生成温度 (通用参数, 默认按provider自动设置)') + + # ==================== MobiAgent 专属参数 ==================== + mobiagent_group = parser.add_argument_group('MobiAgent 专属参数') + mobiagent_group.add_argument('--service-ip', type=str, default='localhost', + help='MobiAgent服务IP (默认: localhost)') + mobiagent_group.add_argument('--decider-port', type=int, default=8000, + help='Decider端口 (默认: 8000)') + mobiagent_group.add_argument('--grounder-port', type=int, default=8001, + help='Grounder端口 (默认: 8001)') + mobiagent_group.add_argument('--planner-port', type=int, default=8080, + help='Planner端口 (默认: 8080)') + mobiagent_group.add_argument('--planner-model', type=str, default='Qwen3-VL-30B-A3B-Instruct', + help='Planner模型名称 (默认: Qwen3-VL-30B-A3B-Instruct)') + mobiagent_group.add_argument('--enable-planning', action='store_true', default=False, + help='启用任务规划（自动分析APP和优化任务描述）') + mobiagent_group.add_argument('--use-e2e', action='store_true', default=True, + help='使用端到端模式 (默认: True)') + mobiagent_group.add_argument('--decider-model', type=str, default='MobiMind-1.5-4B', + help='Decider模型名称 (默认: MobiMind-1.5-4B)') + mobiagent_group.add_argument('--grounder-model', type=str, default='MobiMind-1.5-4B', + help='Grounder模型名称 (默认: MobiMind-1.5-4B)') + mobiagent_group.add_argument('--use-experience', action='store_true', default=False, + help='使用经验') + + # ==================== UI-TARS 专属参数 ==================== + uitars_group = parser.add_argument_group('UI-TARS 专属参数') + uitars_group.add_argument('--step-delay', type=float, default=2.0, + help='步骤延迟秒数 (默认: 2.0)') + + # ==================== 向后兼容的旧参数 (deprecated) ==================== + compat_group = parser.add_argument_group('向后兼容参数 (deprecated, 建议使用通用参数)') + compat_group.add_argument('--model-url', type=str, default=None, + help='[deprecated] 模型服务地址, 请使用 --api-base') + compat_group.add_argument('--model-name', type=str, default=None, + help='[deprecated] 模型名称, 请使用 --model') + compat_group.add_argument('--qwen-api-key', type=str, default=None, + help='[deprecated] Qwen API密钥, 请使用 --api-key') + compat_group.add_argument('--qwen-api-base', type=str, default=None, + help='[deprecated] Qwen API地址, 请使用 --api-base') + compat_group.add_argument('--qwen-model', type=str, default="Qwen3-VL-30B-A3B-Instruct", + help='[deprecated] Qwen模型名称, 请使用 --model') + + # ==================== 可视化参数 ==================== + parser.add_argument('--draw', action='store_true', default=False, + help='是否在截图上绘制操作可视化 (默认: False)') + + args = parser.parse_args() + + # 处理 provider 别名 + if args.provider == 'mobiagent': + args.provider = 'mobiagent_step' + + # 获取 provider 默认值 + defaults = PROVIDER_DEFAULTS.get(args.provider, {}) + + # 合并向后兼容参数到统一参数 + # api-base 优先级: --api-base > --model-url > --qwen-api-base > provider默认值 + if args.api_base is None: + args.api_base = args.model_url or args.qwen_api_base or defaults.get('api_base') + + # model 优先级: --model > --model-name > --qwen-model > provider默认值 + if args.model is None: + args.model = args.model_name or args.qwen_model or defaults.get('model') + + # api-key 优先级: --api-key > --qwen-api-key + if not args.api_key and args.qwen_api_key: + args.api_key = args.qwen_api_key + + # temperature 使用 provider 默认值 + if args.temperature is None: + args.temperature = defaults.get('temperature', 0.0) + + return args + + +def create_device(device_type: str, device_id: Optional[str] = None): + """ + 创建设备对象（使用独立的device模块） + + Args: + device_type: 设备类型 + device_id: 设备ID或IP + + Returns: + 设备对象 + """ + from device import create_device as factory_create_device + device = factory_create_device(device_type, adb_endpoint=device_id) + logging.info(f"已连接到 {device_type} 设备") + return device + + +def execute_single_task( + provider: str, + task_description: str, + device, + output_dir: str, + device_type: str, + args, + app_name: Optional[str] = None, + task_type: Optional[str] = None +) -> Dict: + """ + 执行单个任务 + + Args: + provider: 模型提供者 + task_description: 任务描述 + device: 设备对象 + output_dir: 输出目录 + device_type: 设备类型 + args: 命令行参数 + app_name: APP名称 (可选) + task_type: 任务类型 (可选) + + Returns: + 执行结果字典 + """ + # 创建任务特定的输出目录 + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + # 清理任务描述中的特殊字符 + safe_task = "".join(c if c.isalnum() or c in (' ', '-', '_') else '_' + for c in task_description)[:50] + + if app_name and task_type: + task_dir = os.path.join(output_dir, provider, app_name, task_type, f"{timestamp}_{safe_task}") + else: + task_dir = os.path.join(output_dir, provider, f"{timestamp}_{safe_task}") + + os.makedirs(task_dir, exist_ok=True) + + logging.info(f"=" * 60) + logging.info(f"开始执行任务: {task_description}") + logging.info(f"Provider: {provider}") + logging.info(f"输出目录: {task_dir}") + if app_name and task_type: + logging.info(f"App: {app_name}, Type: {task_type}") + logging.info(f"=" * 60) + + # 准备kwargs参数 - 使用统一的参数命名 + kwargs = { + # 通用参数 + "api_base": args.api_base, + "api_key": args.api_key, + "model": args.model, + "temperature": args.temperature, + } + + if provider == "mobiagent_step": + kwargs.update({ + "service_ip": args.service_ip, + "decider_port": args.decider_port, + "grounder_port": args.grounder_port, + "planner_port": args.planner_port, + "enable_planning": args.enable_planning, + "use_e2e": args.use_e2e, + "decider_model": args.decider_model, + "grounder_model": args.grounder_model, + "planner_model": args.planner_model, + "use_experience": args.use_experience, + }) + elif provider == "uitars": + kwargs.update({ + "step_delay": args.step_delay, + "device_ip": args.device_id, + # 向后兼容: 同时传递旧参数名 + "model_base_url": args.api_base, + "model_name": args.model, + }) + elif provider == "qwen": + kwargs.update({ + # 向后兼容: 同时传递旧参数名 + "model_name": args.model, + }) + + # 创建任务管理器 + try: + task_manager = TaskManager( + provider=provider, + task_description=task_description, + device=device, + data_dir=task_dir, + device_type=device_type, + max_steps=args.max_steps, + draw=args.draw, + **kwargs + ) + + # 执行任务 + start_time = time.time() + result = task_manager.execute() + elapsed_time = time.time() - start_time + + result["elapsed_time"] = elapsed_time + result["task_description"] = task_description + result["output_dir"] = task_dir + + logging.info(f"任务完成! 状态: {result.get('status', 'unknown')}") + logging.info(f"耗时: {elapsed_time:.2f}秒") + logging.info(f"步数: {result.get('step_count', 0)}") + + return result + + except Exception as e: + logging.error(f"任务执行失败: {e}", exc_info=True) + return { + "status": "error", + "error": str(e), + "task_description": task_description, + "output_dir": task_dir + } + + +def execute_batch_tasks( + provider: str, + tasks: List, + device, + output_dir: str, + device_type: str, + args +) -> Dict: + """ + 批量执行任务 + + Args: + provider: 模型提供者 + tasks: 任务列表 + device: 设备对象 + output_dir: 输出目录 + device_type: 设备类型 + args: 命令行参数 + + Returns: + 批量执行结果字典 + """ + results = [] + success_count = 0 + fail_count = 0 + error_count = 0 + + total_tasks = len(tasks) if isinstance(tasks, list) else sum( + len(app_data.get("tasks", [])) for app_data in tasks + ) + + logging.info(f"开始批量执行 {total_tasks} 个任务") + + # 处理不同格式的任务文件 + # 判断是否为MobiFlow格式(多APP多任务) + is_mobiflow = False + if isinstance(tasks, list) and len(tasks) > 0 and isinstance(tasks[0], dict): + if "tasks" in tasks[0] and isinstance(tasks[0]["tasks"], list): + is_mobiflow = True + + if is_mobiflow: + # MobiFlow格式任务 + task_list = [] + for app_data in tasks: + app_name = app_data.get("app", "unknown") + task_type = app_data.get("type", "unknown") + for task_desc in app_data.get("tasks", []): + task_list.append({ + "app": app_name, + "type": task_type, + "description": task_desc + }) + else: + # 简单列表格式或其他格式 + if isinstance(tasks, dict): + task_list = [] + logging.warning("未知的任务格式") + else: + task_list = tasks + + for idx, task_item in enumerate(task_list, 1): + # 提取任务描述和元数据 + app_name = None + task_type = None + + if isinstance(task_item, str): + task_description = task_item + elif isinstance(task_item, dict): + task_description = task_item.get("description", + task_item.get("task", str(task_item))) + app_name = task_item.get("app") + task_type = task_item.get("type") + else: + task_description = str(task_item) + + logging.info(f"\n[{idx}/{total_tasks}] 执行任务: {task_description}") + + result = execute_single_task( + provider=provider, + task_description=task_description, + device=device, + output_dir=output_dir, + device_type=device_type, + args=args, + app_name=app_name, + task_type=task_type + ) + + # 统计结果 + status = result.get("status", "unknown") + if status == "success" or result.get("success", False): + success_count += 1 + elif status == "failed": + fail_count += 1 + else: + error_count += 1 + + results.append(result) + + # 任务间休息 + if idx < total_tasks: + logging.info("等待3秒后执行下一个任务...") + time.sleep(3) + + # 生成汇总报告 + summary = { + "total_tasks": total_tasks, + "success_count": success_count, + "fail_count": fail_count, + "error_count": error_count, + "success_rate": success_count / total_tasks if total_tasks > 0 else 0, + "results": results + } + + # 保存汇总报告 + summary_path = os.path.join(output_dir, provider, "summary.json") + os.makedirs(os.path.dirname(summary_path), exist_ok=True) + with open(summary_path, 'w', encoding='utf-8') as f: + json.dump(summary, f, ensure_ascii=False, indent=2) + + logging.info(f"\n{'='*60}") + logging.info(f"批量任务执行完成!") + logging.info(f"总任务数: {total_tasks}") + logging.info(f"成功: {success_count} ({success_count/total_tasks*100:.1f}%)") + logging.info(f"失败: {fail_count}") + logging.info(f"错误: {error_count}") + logging.info(f"汇总报告: {summary_path}") + logging.info(f"{'='*60}") + + return summary + + +def main(): + """主函数""" + args = parse_args() + + # 设置日志 + setup_logging(args.log_level) + + logging.info("=" * 60) + logging.info("统一GUI Agent任务执行器") + logging.info(f"Provider: {args.provider}") + logging.info(f"Device Type: {args.device_type}") + logging.info("=" * 60) + + # 创建设备 + try: + device = create_device(args.device_type, args.device_id) + logging.info(f"设备连接成功: {args.device_type}") + except Exception as e: + logging.error(f"设备连接失败: {e}") + return 1 + + # 确定任务 + if args.task: + # 单个任务 + result = execute_single_task( + provider=args.provider, + task_description=args.task, + device=device, + output_dir=args.output_dir, + device_type=args.device_type, + args=args + ) + return 0 if result.get("status") != "error" else 1 + + elif args.task_file: + # 批量任务 + if not os.path.exists(args.task_file): + logging.error(f"任务文件不存在: {args.task_file}") + return 1 + + tasks = load_tasks(args.task_file) + summary = execute_batch_tasks( + provider=args.provider, + tasks=tasks, + device=device, + output_dir=args.output_dir, + device_type=args.device_type, + args=args + ) + return 0 if summary["error_count"] == 0 else 1 + + else: + logging.error("请指定 --task 或 --task-file 参数") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/runner/task_manager.py b/runner/task_manager.py new file mode 100644 index 00000000..5662b824 --- /dev/null +++ b/runner/task_manager.py @@ -0,0 +1,140 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + +import logging +from typing import Dict, Optional + + +class TaskManager: + """ + 任务管理器，负责根据provider类型创建和执行相应的任务 + """ + + def __init__( + self, + provider: str, + task_description: str, + device, + data_dir: str, + device_type: str = "Android", + max_steps: int = 40, + draw: bool = False, + **kwargs + ): + """ + 初始化任务管理器 + + Args: + provider: 模型提供者名称 ("mobiagent", "uitars", 等) + task_description: 任务描述 + device: 设备对象 + data_dir: 数据保存目录 + device_type: 设备类型 + max_steps: 最大步骤数 + draw: 是否在截图上绘制操作 + log_level: 日志级别 + **kwargs: 传递给具体任务类的其他参数 + """ + self.provider = provider + self.task_description = task_description + self.device = device + self.data_dir = data_dir + self.device_type = device_type + self.max_steps = max_steps + self.kwargs = kwargs + + # 导入provider + self.task_map = self._get_task_map() + + # 创建任务实例 + if provider not in self.task_map: + raise ValueError( + f"Unknown provider: {provider}. " + f"Available providers: {list(self.task_map.keys())}" + ) + + task_class = self.task_map[provider] + self.task = task_class( + task_description=task_description, + device=device, + data_dir=data_dir, + device_type=device_type, + max_steps=max_steps, + draw=draw, + **kwargs + ) + + logging.info(f"TaskManager initialized with provider: {provider}") + + def _get_task_map(self) -> Dict: + """ + 获取provider到任务类的映射 + + Returns: + provider名称到任务类的字典 + """ + try: + # from providers.mobiagent_task import MobiAgentTask # Legacy, removed + from providers.uitars.uitars_task import UITARSTask + from providers.mobiagent.mobile_task import MobiAgentStepTask + from providers.qwen.qwen_task import QwenTask + from providers.autoglm.autoglm_task import AutoGLMTask + + task_map = { + "mobiagent": MobiAgentStepTask, # 别名 + "mobiagent_step": MobiAgentStepTask, + "uitars": UITARSTask, + "qwen": QwenTask, + "autoglm": AutoGLMTask, + } + + return task_map + + except ImportError as e: + logging.warning(f"Failed to import some providers: {e}") + # 返回部分可用的providers + task_map = {} + + try: + from providers.mobiagent.mobile_task import MobiAgentStepTask + task_map["mobiagent_step"] = MobiAgentStepTask + except ImportError: + pass + try: + from providers.uitars.uitars_task import UITARSTask + task_map["uitars"] = UITARSTask + except ImportError: + pass + + try: + from providers.qwen.qwen_task import QwenTask + task_map["qwen"] = QwenTask + except ImportError: + pass + + return task_map + + def execute(self) -> Dict: + """ + 执行任务 + + Returns: + 任务执行结果字典 + """ + logging.info(f"Executing task with provider: {self.provider}") + result = self.task.execute() + return result + + def get_task_info(self) -> Dict: + """ + 获取任务信息 + + Returns: + 任务信息字典 + """ + return { + "provider": self.provider, + "task_description": self.task_description, + "device_type": self.device_type, + "max_steps": self.max_steps, + "data_dir": self.data_dir + } diff --git a/utils/local_experience.py b/utils/local_experience.py index 5d2dbda9..3fd472c5 100644 --- a/utils/local_experience.py +++ b/utils/local_experience.py @@ -176,7 +176,8 @@ def extract_full_description(self, result): # 确保 result 是字符串 result_str = str(result) - print(f"提取 full_experience 字段内容: {result_str}") + # For debug + # print(f"提取 full_experience 字段内容: {result_str}") # 方法 1: 使用正则表达式查找单任务格式的 JSON 对象 single_task_pattern = r'\{"name":\s*"[^"]*",\s*"description":\s*"[^"]*",\s*"full_experience":\s*"(?:[^"\\]|\\.)*"\}' single_matches = re.findall(single_task_pattern, result_str) diff --git a/utils/parse_omni.py b/utils/parse_omni.py index 19833c04..e5f6dbe5 100644 --- a/utils/parse_omni.py +++ b/utils/parse_omni.py @@ -1,10 +1,11 @@ from utils.omni_utils import get_som_labeled_img, check_ocr_box, get_yolo_model from PIL import Image import torch - +import os device = "cuda" if torch.cuda.is_available() else "cpu" -detect_model_path='./weights/icon_detect/model.pt' -caption_model_path='./weights/icon_caption_florence' +current_file = os.path.dirname(os.path.abspath(__file__)) +detect_model_path=os.path.join(current_file,"..","weights/icon_detect/model.pt") +caption_model_path=os.path.join(current_file,'..','weights/icon_caption_florence') som_model = get_yolo_model(detect_model_path) som_model.to(device)