feat(web): surface frontend tool call activity

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+- WebUI 展示前台与后台工具调用的有界生命周期，按工具调用 ID 合并状态，并默认不保留
+  原始参数与结果内容。
+
 - 桌面设置保留通过环境配置选择的通用 ACP 后台，并显示实际可用状态；
   应用其他设置不再将 `AGENT_PROTOCOL=acp` 静默覆盖为 `none`。
 

docs/contract.md

@@ -258,6 +258,7 @@ conversation surface.
 | server → client | `audio.delta`, `audio.done`, `playback.clear` | Audio playback stream and cancellation |
 | server → client | `response.started`, `response.interrupted` | Response lifecycle keyed by `responseId` |
 | server → client | `transcript.delta`, `transcript.final`, `transcript.discard` | User and assistant transcript lifecycle |
+| server → client | `tool.call` | Bounded foreground/backend Tool Call lifecycle for presentation; it is not an execution request and clients must not infer internal state from it |
 | server → client | `task.*` | Optional background Task snapshots, progress, authorization, and completion |
 | server → client | `agent.activity`, `client.state`, `error` | Foreground activity hints, the temporary 5.x client-state migration alias, and errors |
 

docs/contract.zh.md

@@ -228,6 +228,7 @@ await orb.load()
 | 服务端 → 客户端 | `audio.delta`、`audio.done`、`playback.clear` | 播放音频流及清除指令 |
 | 服务端 → 客户端 | `response.started`、`response.interrupted` | 以 `responseId` 标识的回复生命周期 |
 | 服务端 → 客户端 | `transcript.delta`、`transcript.final`、`transcript.discard` | 用户与助手转写生命周期 |
+| 服务端 → 客户端 | `tool.call` | 有界的前台/后台工具调用生命周期，仅用于展示；它不是执行请求，客户端不得据此推断内部状态 |
 | 服务端 → 客户端 | `task.*` | 可选的后台 Task 快照、进度、授权与完成事件 |
 | 服务端 → 客户端 | `agent.activity`、`client.state`、`error` | 前台活动提示、临时保留的 5.x Client State 迁移别名与错误 |
 

docs/gateway-protocol.md

@@ -277,7 +277,7 @@ Use OpenAI Realtime terminology where the semantics match:
 | `response.output_audio_transcript.delta` / `.done` | G→C | Assistant transcript |
 | `response.done` | G→C | Final response state; cancellation is `response.status = "cancelled"` |
 
-Gateway extensions include `turn.started`, `transcript.discard`, `playback.clear`, and playback receipts. `input_file` is a Gateway content-part extension, not an OpenAI Realtime standard part.
+Gateway extensions include `turn.started`, `transcript.discard`, `tool.call`, `playback.clear`, and playback receipts. `tool.call` is a bounded, presentation-only lifecycle event for a Realtime Tool Call; it is not an execution request, is not replayed, and clients must not infer private Gateway or backend state from it. `input_file` is a Gateway content-part extension, not an OpenAI Realtime standard part.
 
 User input is authoritative user intent and opens or supersedes a user turn. Client semantic events never impersonate user input.
 

docs/gateway-protocol.zh.md

@@ -249,7 +249,7 @@ Gateway 采用扁平的 OpenAI Realtime 风格信封：
 | `response.output_audio_transcript.delta` / `.done` | G→C | 助手转写 |
 | `response.done` | G→C | 回复最终状态；取消使用 `response.status = "cancelled"` |
 
-Gateway 扩展包括 `turn.started`、`transcript.discard`、`playback.clear` 和播放回执。`input_file` 是 Gateway content part 扩展，不属于 OpenAI Realtime 标准字段。
+Gateway 扩展包括 `turn.started`、`transcript.discard`、`tool.call`、`playback.clear` 和播放回执。`tool.call` 是用于展示 Realtime 工具调用的有界生命周期事件，不是执行请求，不参与回放，客户端不得据此推断 Gateway 或后台的私有状态。`input_file` 是 Gateway content part 扩展，不属于 OpenAI Realtime 标准字段。
 
 用户输入代表明确的用户意图，会开启或替代用户轮次。Client 语义事件不能伪装成用户输入。
 

web/src/App.jsx

@@ -13,6 +13,12 @@ import {
   upsertAssistantTranscript,
   upsertUserTranscript,
 } from './message-order.js'
+import {
+  cancelToolCallsForResponse,
+  displayToolName,
+  mergeToolCallDebug,
+  toolCallFromGatewayEvent,
+} from './tool-call-debug.js'
 import MessageContent from './MessageContent.jsx'
 import MultimodalComposer from './composer/MultimodalComposer.jsx'
 import TaskArtifacts from './TaskArtifacts.jsx'
@@ -209,6 +215,7 @@ export default function App() {
   }))
   const [waitingForVoice, setWaitingForVoice] = useState(false)
   const [messages, setMessages] = useState([])
+  const [toolCalls, setToolCalls] = useState([])
   const [activity, setActivity] = useState(t('正在检查后台 Agent'))
   const [frontend, setFrontend] = useState({ label: 'Realtime Agent' })
   const [modelStatus, setModelStatus] = useState(() => realtimeModelStatus())
@@ -390,7 +397,7 @@ export default function App() {
     if (container && stickToBottom.current) {
       container.scrollTop = container.scrollHeight
     }
-  }, [messages, agentTasks])
+  }, [messages, agentTasks, toolCalls])
 
   useEffect(() => () => {
     taskDismissTimers.current.forEach(timer => clearTimeout(timer))
@@ -456,6 +463,7 @@ export default function App() {
         id,
         content: event.content,
         turnId: event.turnId,
+        createdAt: Date.now(),
         final,
       }))
     if (final) noteInteraction()
@@ -475,11 +483,25 @@ export default function App() {
       taskIds: event.taskIds,
       origin: event.origin,
       citations: event.citations,
+      createdAt: Date.now(),
       final,
     }))
   }, [])
 
   const onRealtimeEvent = useCallback(event => {
+    const toolCall = toolCallFromGatewayEvent(event, currentTurnId.current)
+    if (toolCall) {
+      setToolCalls(items => mergeToolCallDebug(items, {
+        ...toolCall,
+        createdAt: Date.now(),
+      }))
+      if (
+        ['received', 'running'].includes(toolCall.status)
+        && (!toolCall.turnId || toolCall.turnId === currentTurnId.current)
+      ) {
+        setActivity(t('正在执行工具'))
+      }
+    }
     const animationEvent = spriteAnimationEventForGatewayEvent(event)
     if (animationEvent) {
       triggerSpriteAnimation(animationEvent)
@@ -492,6 +514,11 @@ export default function App() {
     }
     if (event.type === 'gateway.disconnected') {
       setActivity(t('qwen-audio-agent Gateway 已断开，正在重连'))
+      setToolCalls(items => items.map(toolCall => (
+        ['completed', 'failed', 'cancelled'].includes(toolCall.status)
+          ? toolCall
+          : { ...toolCall, status: 'cancelled' }
+      )))
       setAgentTasks(items => items.map(task => (
         [
           'queued',
@@ -613,6 +640,7 @@ export default function App() {
           ? { ...message, interrupted: true, live: false }
           : message
       )))
+      setToolCalls(items => cancelToolCallsForResponse(items, event.responseId))
     }
     if (event.type === 'task.scheduled') {
       const task = event.task
@@ -1061,6 +1089,7 @@ export default function App() {
     localStorage.setItem('qwen-audio-agent.session', next)
     setSessionId(next)
     setMessages([])
+    setToolCalls([])
     setAgentTasks([])
     currentTurnId.current = ''
     activeVoiceResponse.current = ''
@@ -1094,8 +1123,8 @@ export default function App() {
   }
 
   const turns = useMemo(
-    () => buildConversationTurns(messages, agentTasks),
-    [messages, agentTasks],
+    () => buildConversationTurns(messages, agentTasks, toolCalls),
+    [messages, agentTasks, toolCalls],
   )
 
   const beginOrbDrag = event => {
@@ -1331,6 +1360,36 @@ export default function App() {
     </div>}
   </aside>
 
+  const renderToolCall = toolCall => {
+    const surface = toolCall.surface === 'backend' ? t('后台工具') : t('前台工具')
+    const status = {
+      received: t('工具执行中'),
+      running: t('工具执行中'),
+      completed: t('工具已完成'),
+      failed: t('工具失败'),
+      cancelled: t('工具已取消'),
+    }[toolCall.status] || t('工具执行中')
+    const duration = Number.isFinite(toolCall.durationMs)
+      ? toolCall.durationMs < 1_000
+        ? t('{ms} 毫秒', { ms: Math.max(0, Math.round(toolCall.durationMs)) })
+        : t('{seconds} 秒', {
+            seconds: Number((toolCall.durationMs / 1_000).toFixed(1)),
+          })
+      : ''
+    return <aside
+      key={`tool-call:${toolCall.callId}`}
+      className={`tool-call-activity ${toolCall.surface} ${toolCall.status}`}
+      aria-label={`${surface} ${displayToolName(toolCall.name)} · ${status}`}
+      title={toolCall.name}
+    >
+      <span className="tool-call-indicator" aria-hidden="true" />
+      <div>
+        <b>{displayToolName(toolCall.name)}</b>
+        <small>{[surface, status, duration].filter(Boolean).join(' · ')}</small>
+      </div>
+    </aside>
+  }
+
   const renderMessage = message => <article
     key={message.id}
     className={`${message.role}${message.companion ? ' companion' : ''}`}
@@ -1466,7 +1525,11 @@ export default function App() {
           key={turn.id}
           className={`conversation-turn${turn.standalone ? ' standalone' : ''}`}
         >
-          {turn.beforeActivities.map(renderMessage)}
+          {turn.beforeEvents.map(item => (
+            item.type === 'tool-call'
+              ? renderToolCall(item.value)
+              : renderMessage(item.value)
+          ))}
           {turn.tasks.map(renderTask)}
           {turn.afterActivities.map(renderMessage)}
         </section>)}

web/src/i18n.js

@@ -40,6 +40,14 @@ const translations = {
   '正在继续处理': 'Resuming',
   '正在准备回复': 'Preparing reply',
   '处理完成': 'Completed',
+  '前台工具': 'Frontend tool',
+  '后台工具': 'Backend tool',
+  '工具执行中': 'Tool running',
+  '工具已完成': 'Tool completed',
+  '工具失败': 'Tool failed',
+  '工具已取消': 'Tool cancelled',
+  '{ms} 毫秒': '{ms}ms',
+  '{seconds} 秒': '{seconds}s',
   '后台失败：{error}': 'Background task failed: {error}',
   '已取消': 'Cancelled',
   '已创建新会话': 'New session created',

web/src/message-order.js

@@ -80,6 +80,7 @@ export function upsertUserTranscript(items, {
   id,
   content,
   turnId,
+  createdAt,
   final = false,
 }) {
   const normalized = normalizeTranscript(content)
@@ -89,6 +90,7 @@ export function upsertUserTranscript(items, {
     role: 'user',
     content: normalized,
     turnId,
+    ...(Number.isFinite(createdAt) && createdAt > 0 ? { createdAt } : {}),
     voice: true,
     final,
     live: !final,
@@ -100,7 +102,15 @@ export function upsertUserTranscript(items, {
   // utterance that may replace it.
   if (items[index].final) return items
   const next = [...items]
-  next[index] = { ...next[index], ...message }
+  next[index] = {
+    ...next[index],
+    ...message,
+    ...(
+      Number.isFinite(next[index].createdAt) && next[index].createdAt > 0
+        ? { createdAt: next[index].createdAt }
+        : {}
+    ),
+  }
   return next
 }
 
@@ -112,6 +122,7 @@ export function upsertAssistantTranscript(items, {
   taskIds,
   origin,
   citations,
+  createdAt,
   final = false,
 }) {
   const index = items.findIndex(item => item.id === id)
@@ -125,6 +136,7 @@ export function upsertAssistantTranscript(items, {
       taskIds,
       origin,
       ...(citations?.length ? { citations } : {}),
+      ...(Number.isFinite(createdAt) && createdAt > 0 ? { createdAt } : {}),
       live: !final,
     })
   }
@@ -140,6 +152,13 @@ export function upsertAssistantTranscript(items, {
     taskIds: taskIds || existing.taskIds,
     origin: origin || existing.origin,
     ...(citations?.length ? { citations } : {}),
+    ...(
+      Number.isFinite(existing.createdAt) && existing.createdAt > 0
+        ? { createdAt: existing.createdAt }
+        : Number.isFinite(createdAt) && createdAt > 0
+          ? { createdAt }
+          : {}
+    ),
     live: !final,
   }
   return next
@@ -151,19 +170,25 @@ export function discardUserTranscript(items, turnId) {
   return items.filter(item => item.id !== id || item.final)
 }
 
-export function buildConversationTimeline(messages, tasks) {
-  return buildConversationTurns(messages, tasks).flatMap(turn => [
-    ...turn.beforeActivities.map(value => ({ type: 'message', value })),
+export function buildConversationTimeline(messages, tasks, toolCalls = []) {
+  return buildConversationTurns(messages, tasks, toolCalls).flatMap(turn => [
+    ...turn.beforeEvents,
     ...turn.tasks.map(value => ({ type: 'task', value })),
     ...turn.afterActivities.map(value => ({ type: 'message', value })),
   ])
 }
 
-export function buildConversationTurns(messages, tasks) {
+export function buildConversationTurns(messages, tasks, toolCalls = []) {
   const turns = []
   const byTurnId = new Map()
   const createTurn = (id, standalone = false) => {
-    const turn = { id, standalone, messages: [], tasks: [] }
+    const turn = {
+      id,
+      standalone,
+      messages: [],
+      tasks: [],
+      toolCalls: [],
+    }
     turns.push(turn)
     if (!standalone) byTurnId.set(id, turn)
     return turn
@@ -187,6 +212,14 @@ export function buildConversationTurns(messages, tasks) {
     turn.tasks.push(task)
   })
 
+  toolCalls.forEach(toolCall => {
+    const id = toolCall.turnId || `tool:${toolCall.callId}`
+    const turn = toolCall.turnId
+      ? byTurnId.get(id) || createTurn(id)
+      : createTurn(id, true)
+    turn.toolCalls.push(toolCall)
+  })
+
   const orderedTurns = turns.filter(turn => turn.messages.length)
   const taskOnlyTurns = turns
     .filter(turn => !turn.messages.length)
@@ -213,9 +246,32 @@ export function buildConversationTurns(messages, tasks) {
         message.taskId,
       ].some(taskId => taskIds.has(taskId))
     )
+    const beforeActivities = turn.messages.filter(message => !afterTaskCard(message))
+    const beforeEvents = [
+      ...beforeActivities.map((value, index) => ({
+        type: 'message',
+        value,
+        order: index,
+      })),
+      ...turn.toolCalls.map((value, index) => ({
+        type: 'tool-call',
+        value,
+        order: beforeActivities.length + index,
+      })),
+    ].sort((left, right) => {
+      const leftTime = Number(left.value.createdAt)
+      const rightTime = Number(right.value.createdAt)
+      const leftHasTime = Number.isFinite(leftTime) && leftTime > 0
+      const rightHasTime = Number.isFinite(rightTime) && rightTime > 0
+      if (leftHasTime && rightHasTime && leftTime !== rightTime) {
+        return leftTime - rightTime
+      }
+      return left.order - right.order
+    }).map(({ type, value }) => ({ type, value }))
     return {
       ...turn,
-      beforeActivities: turn.messages.filter(message => !afterTaskCard(message)),
+      beforeActivities,
+      beforeEvents,
       afterActivities: turn.messages.filter(afterTaskCard),
     }
   })
@@ -225,6 +281,7 @@ function turnChronologicalTime(turn) {
   const itemTimes = [
     ...turn.messages.map(item => Number(item.createdAt || 0)),
     ...turn.tasks.map(item => Number(item.createdAt || 0)),
+    ...turn.toolCalls.map(item => Number(item.createdAt || 0)),
   ].filter(value => value > 0)
   if (itemTimes.length) return Math.min(...itemTimes)
   const timestamp = turnTimestamp(turn.id)