From 98689b1ef34ab368a6314bc1fd972ce5c22aa80f Mon Sep 17 00:00:00 2001 From: alectimison-maker Date: Tue, 4 Aug 2026 22:30:26 +0800 Subject: [PATCH 1/5] feat(compact): expose upload_file --- src/chrome/src/agent/tools.js | 25 +++++++++++++++++- src/firefox/src/agent/tools.js | 24 +++++++++++++++++- test/run.js | 46 +++++++++++++++++++++++++++++++++- 3 files changed, 92 insertions(+), 3 deletions(-) diff --git a/src/chrome/src/agent/tools.js b/src/chrome/src/agent/tools.js index 89f88ec8e..9ebc3d767 100644 --- a/src/chrome/src/agent/tools.js +++ b/src/chrome/src/agent/tools.js @@ -1266,6 +1266,25 @@ const WATCH_BEEP_TOOL = { }, }; +function compactUploadFileTool(tool) { + return { + ...tool, + function: { + ...tool.function, + description: 'Attach a user-provided file directly to an existing file input without clicking the page upload control. Use attachmentId from the current user-attachment notice, or an absolute filePath the user supplied. This proves only local page attachment, not remote upload or submission. Use the exact selector for the intended input; never guess a generic input[type="file"] selector when multiple inputs exist. If the widget creates its input lazily, make one guarded click on its add-files control, then retry upload_file with the exact selector.', + parameters: { + type: 'object', + properties: { + selector: { type: 'string', description: 'Exact CSS selector for the intended file input.' }, + attachmentId: { type: 'string', description: 'Opaque id from the current user-attachment notice. Never guess an id.' }, + filePath: { type: 'string', description: 'Absolute local path explicitly supplied by the user. Optional when attachmentId is given.' }, + }, + required: ['selector'], + }, + }, + }; +} + /** * Get tools filtered by mode. * @@ -1285,7 +1304,9 @@ export function getToolsForMode(mode, opts = {}) { } else if (devCompactBlocked) { base = []; } else if (tier === 'compact') { - base = AGENT_TOOLS.filter(t => COMPACT_TOOL_NAMES.has(t.function.name)); + base = AGENT_TOOLS + .filter(t => COMPACT_TOOL_NAMES.has(t.function.name)) + .map(t => (t.function.name === 'upload_file' ? compactUploadFileTool(t) : t)); } else if (tier === 'mid') { base = AGENT_TOOLS.filter(t => MID_TOOL_NAMES.has(t.function.name)); } else { @@ -1702,6 +1723,7 @@ export const COMPACT_TOOL_NAMES = new Set([ 'click', 'type_text', 'press_keys', 'navigate', 'new_tab', 'wait_for_element', 'fetch_url', + 'upload_file', 'scratchpad_write', 'progress_update', 'progress_read', 'clarify', 'done', ]); @@ -1747,6 +1769,7 @@ TOOLS — use ONLY these: - new_tab({url}): Open a URL in a background tab for user reference. It does not activate or retarget the current run, so never use it as a site-permission workaround. - wait_for_element({selector}): Wait for an element to appear. - fetch_url({url}): Fetch a URL for its content. +- upload_file({selector, attachmentId|filePath}): Attach a user-provided file directly to an existing file input; do not click the page upload control first. Use the exact selector and never guess generic input[type="file"] when multiple inputs exist. If the widget creates its input lazily, make one guarded initializer click, re-read the page, then retry with the exact selector. Verify the page shows the attachment before submitting. - scratchpad_write({text}): Save notes that persist across steps. - progress_update({items}) / progress_read({status}): Structured progress ledger for the active repeated item/action task. On GitHub stargazers, only "Follow USER" buttons are follow targets when following is allowed by the task; "Unfollow USER" means skip/already followed unless the ledger shows acted. - done({summary, outcome}): Signal success, partial progress, or a failed blocker. diff --git a/src/firefox/src/agent/tools.js b/src/firefox/src/agent/tools.js index 966bb4942..8d4eeffaa 100644 --- a/src/firefox/src/agent/tools.js +++ b/src/firefox/src/agent/tools.js @@ -993,6 +993,7 @@ export const COMPACT_TOOL_NAMES = new Set([ 'click', 'type_text', 'press_keys', 'navigate', 'new_tab', 'wait_for_element', 'fetch_url', + 'upload_file', 'scratchpad_write', 'progress_update', 'progress_read', 'clarify', 'done', ]); @@ -1114,6 +1115,24 @@ const WATCH_BEEP_TOOL = { }, }; +function compactUploadFileTool(tool) { + return { + ...tool, + function: { + ...tool.function, + description: 'Attach a user-provided file directly to an existing file input without clicking the page upload control. Use attachmentId from the current user-attachment notice, or omit it to open WebBrain\'s file picker. This proves only local page attachment, not remote upload or submission. Use the exact selector for the intended input; never guess a generic input[type="file"] selector when multiple inputs exist. If the widget creates its input lazily, make one guarded click on its add-files control, then retry upload_file with the exact selector.', + parameters: { + type: 'object', + properties: { + selector: { type: 'string', description: 'Exact CSS selector for the intended file input.' }, + attachmentId: { type: 'string', description: 'Opaque id from the current user-attachment notice. Omit only to ask the user through WebBrain\'s file picker; never guess an id.' }, + }, + required: ['selector'], + }, + }, + }; +} + /** * Get tools filtered by mode. * @@ -1132,7 +1151,9 @@ export function getToolsForMode(mode, opts = {}) { } else if (devCompactBlocked) { base = []; } else if (tier === 'compact') { - base = AGENT_TOOLS.filter(t => COMPACT_TOOL_NAMES.has(t.function.name)); + base = AGENT_TOOLS + .filter(t => COMPACT_TOOL_NAMES.has(t.function.name)) + .map(t => (t.function.name === 'upload_file' ? compactUploadFileTool(t) : t)); } else if (tier === 'mid') { base = AGENT_TOOLS.filter(t => MID_TOOL_NAMES.has(t.function.name)); } else { @@ -1230,6 +1251,7 @@ TOOLS - use only these: - new_tab({url}): Open a URL in a background tab for user reference. It does not activate or retarget the current run, so never use it as a site-permission workaround. - wait_for_element({selector}): Wait for an element to appear. - fetch_url({url}): Fetch other URLs for reading only; do not use it to re-read the active tab. +- upload_file({selector, attachmentId?}): Attach a user-provided file directly to an existing file input; do not click the page upload control first. Use the current attachmentId or omit it for WebBrain's picker. Use the exact selector and never guess generic input[type="file"] when multiple inputs exist. If the widget creates its input lazily, make one guarded initializer click, re-read the page, then retry with the exact selector. Verify the page shows the attachment before submitting. - scratchpad_write({text}): Save notes that persist across steps. - progress_update({items}) / progress_read({status}): Structured progress ledger for the active repeated item/action task. On GitHub stargazers, only "Follow USER" buttons are follow targets when following is allowed by the task; "Unfollow USER" means skip/already followed unless the ledger shows acted. - clarify({question, options?}): Ask the user only when materially blocked or ambiguous. Unanswered clarifies auto-select options[0] after timeout (source=timeout is not user approval for high-risk steps; source=auto Instant is intentional auto-approve). diff --git a/test/run.js b/test/run.js index 8c24ea4a2..00c1c93e6 100644 --- a/test/run.js +++ b/test/run.js @@ -12785,6 +12785,7 @@ test('getToolsForMode: compact mode restricts act tools in both browsers', () => [...compactNames].sort(), ); assert.ok(compactNamesActual.includes('done'), `[${label}] compact mode must keep done`); + assert.ok(compactNamesActual.includes('upload_file'), `[${label}] compact mode must expose upload_file`); for (const excluded of ['resize_window', 'download_social_media', 'solve_captcha']) { assert.equal(compactNamesActual.includes(excluded), false, `[${label}] compact mode must omit ${excluded}`); } @@ -12792,6 +12793,49 @@ test('getToolsForMode: compact mode restricts act tools in both browsers', () => } }); +test('compact Act exposes a direct-upload-only file workflow in both browsers', () => { + for (const [label, getTools, prompt] of [ + ['chrome', getToolsForModeCh, SYSTEM_PROMPT_ACT_COMPACT_CH], + ['firefox', getToolsForModeFx, SYSTEM_PROMPT_ACT_COMPACT_FX], + ]) { + const askNames = getTools('ask').map(tool => tool.function.name); + const compactTools = getTools('act', { tier: 'compact' }); + const compactNames = compactTools.map(tool => tool.function.name); + const upload = compactTools.find(tool => tool.function.name === 'upload_file'); + const fullUpload = getTools('act').find(tool => tool.function.name === 'upload_file'); + + assert.ok(upload, `[${label}] compact Act must expose upload_file`); + assert.equal(askNames.includes('upload_file'), false, `[${label}] Ask must remain read-only`); + for (const unavailable of [ + 'download_files', + 'download_resource_from_page', + 'list_downloads', + 'read_downloaded_file', + ]) { + assert.equal(compactNames.includes(unavailable), false, `[${label}] compact Act exposed ${unavailable}`); + } + assert.match(upload.function.description, /directly to an existing file input/i); + assert.match(upload.function.description, /without clicking the page upload control/i); + assert.match(upload.function.description, /exact selector/i); + assert.match(upload.function.description, /one guarded click/i); + assert.doesNotMatch(upload.function.description, /download_files|list_downloads|downloadId/i); + assert.ok(upload.function.parameters.properties.attachmentId, `[${label}] compact upload must accept attachmentId`); + assert.equal(upload.function.parameters.properties.downloadId, undefined, `[${label}] compact upload must hide downloadId`); + assert.ok(fullUpload.function.parameters.properties.downloadId, `[${label}] full upload must retain downloadId`); + assert.match(prompt, /upload_file[\s\S]{0,180}do not click the page upload control/i); + assert.match(prompt, /upload_file[\s\S]{0,360}exact selector/i); + assert.match(prompt, /upload_file[\s\S]{0,420}one guarded initializer click/i); + assert.doesNotMatch(prompt, /download_files|list_downloads|downloadId/i); + + if (label === 'chrome') { + assert.ok(upload.function.parameters.properties.filePath, 'chrome: compact upload must accept a user-supplied filePath'); + } else { + assert.equal(upload.function.parameters.properties.filePath, undefined, 'firefox: compact upload must not invent filePath support'); + assert.match(upload.function.description, /WebBrain's file picker/i); + } + } +}); + test('getToolsForMode: mode/tier redesign exposes the intended normal and Dev tools', () => { for (const [label, getTools] of [ ['chrome', getToolsForModeCh], @@ -50221,7 +50265,7 @@ test('user attachment upload guidance follows the active tier tool catalog', () ]); for (const [mode, tier, shouldAdvertiseUpload] of [ - ['act', 'compact', false], + ['act', 'compact', true], ['act', 'mid', true], ['act', 'full', true], ['ask', 'full', false], From e09c54e32c8b951f253aeee46457923aac7652c5 Mon Sep 17 00:00:00 2001 From: Emre Sokullu Date: Wed, 5 Aug 2026 15:53:17 +0300 Subject: [PATCH 2/5] fix(compact): make upload_file recoverable and attachment-only An ambiguous selector latches upload_file until get_interactive_elements returns a verified file-input selector, and nothing else clears it. That tool was not in the compact catalog, so one ambiguous selector left compact unable to upload for the rest of the page's life while the handler kept asking for a tool the model had not been given. Compact now carries get_interactive_elements, and the tool description and prompt bullet name it as the recovery path. A test asserts the coupling for every tier that ships upload_file. Compact also no longer advertises filePath. It has no download tools, so the only file it can legitimately reach is the one the user attached to this run; a path could only come from the model inventing one, and on Chrome filePath is a CDP-backed read of any local file into an untrusted page's input. Hidden parameters are now deleted from the base schema rather than rebuilt, so a future upload_file parameter still reaches compact. Prompt assertions moved off character-distance regexes onto the upload_file bullet itself. Co-Authored-By: Claude Opus 5 --- src/chrome/src/agent/tools.js | 24 ++++++++++-- src/firefox/src/agent/tools.js | 21 +++++++++-- test/run.js | 68 ++++++++++++++++++++++++++++++---- 3 files changed, 98 insertions(+), 15 deletions(-) diff --git a/src/chrome/src/agent/tools.js b/src/chrome/src/agent/tools.js index 9ebc3d767..d0ba232f2 100644 --- a/src/chrome/src/agent/tools.js +++ b/src/chrome/src/agent/tools.js @@ -1266,18 +1266,28 @@ const WATCH_BEEP_TOOL = { }, }; +// Compact has no download tools, so the only file it can legitimately reach is +// the one the user attached to this run. Dropping downloadId and filePath is +// therefore not just prompt economy: filePath is a CDP-backed read of any local +// path into an untrusted page's input, and compact omits the full-tier guidance +// that exists to stop the model inventing one. Deleting the keys rather than +// rebuilding `parameters` keeps any future base parameter reaching compact. +const COMPACT_UPLOAD_HIDDEN_PARAMS = ['downloadId', 'filePath']; + function compactUploadFileTool(tool) { + const properties = { ...tool.function.parameters.properties }; + for (const key of COMPACT_UPLOAD_HIDDEN_PARAMS) delete properties[key]; return { ...tool, function: { ...tool.function, - description: 'Attach a user-provided file directly to an existing file input without clicking the page upload control. Use attachmentId from the current user-attachment notice, or an absolute filePath the user supplied. This proves only local page attachment, not remote upload or submission. Use the exact selector for the intended input; never guess a generic input[type="file"] selector when multiple inputs exist. If the widget creates its input lazily, make one guarded click on its add-files control, then retry upload_file with the exact selector.', + description: 'Attach a user-provided file directly to an existing file input without clicking the page upload control. Use attachmentId from the current user-attachment notice. This proves only local page attachment, not remote upload or submission. Use the exact selector for the intended input; never guess a generic input[type="file"] selector when multiple inputs exist. If the selector is ambiguous, call get_interactive_elements and use the exact selector on the intended file-input record before retrying. If the widget creates its input lazily, make one guarded click on its add-files control, then retry upload_file with the exact selector.', parameters: { - type: 'object', + ...tool.function.parameters, properties: { + ...properties, selector: { type: 'string', description: 'Exact CSS selector for the intended file input.' }, attachmentId: { type: 'string', description: 'Opaque id from the current user-attachment notice. Never guess an id.' }, - filePath: { type: 'string', description: 'Absolute local path explicitly supplied by the user. Optional when attachmentId is given.' }, }, required: ['selector'], }, @@ -1719,6 +1729,11 @@ export const COMPACT_TOOL_NAMES = new Set([ 'get_accessibility_tree', 'read_page', 'scroll', 'get_window_info', 'extract_data', 'get_selection', 'find_text', + // get_interactive_elements is the only tool that returns a verified, unique + // CSS selector for a file input, so upload_file's ambiguous-selector + // recovery is built on it. It stays in the compact catalog for as long as + // upload_file does; without it an ambiguous selector is unrecoverable. + 'get_interactive_elements', 'click_ax', 'set_checked', 'type_ax', 'set_field', 'click', 'type_text', 'press_keys', 'navigate', 'new_tab', 'wait_for_element', @@ -1756,6 +1771,7 @@ TOOLS — use ONLY these: - get_window_info: Read window/viewport size. - scroll: Scroll up/down. - extract_data: Get tables, headings, images. +- get_interactive_elements: List interactive elements with exact CSS selectors. Use it when you need a selector rather than a ref_id — above all to find the intended file input before upload_file, and to recover after an ambiguous upload selector. - click_ax({ref_id}): Click by ref_id from the tree. PREFERRED. - set_checked({ref_id, checked}): Idempotently set and verify a native checkbox. Never toggle checkboxes repeatedly with click_ax. - type_ax({ref_id, text}): Type into a field by ref_id. @@ -1769,7 +1785,7 @@ TOOLS — use ONLY these: - new_tab({url}): Open a URL in a background tab for user reference. It does not activate or retarget the current run, so never use it as a site-permission workaround. - wait_for_element({selector}): Wait for an element to appear. - fetch_url({url}): Fetch a URL for its content. -- upload_file({selector, attachmentId|filePath}): Attach a user-provided file directly to an existing file input; do not click the page upload control first. Use the exact selector and never guess generic input[type="file"] when multiple inputs exist. If the widget creates its input lazily, make one guarded initializer click, re-read the page, then retry with the exact selector. Verify the page shows the attachment before submitting. +- upload_file({selector, attachmentId}): Attach a user-provided file directly to an existing file input; do not click the page upload control first. Use the exact selector from get_interactive_elements and never guess generic input[type="file"] when multiple inputs exist. If the selector is ambiguous, call get_interactive_elements and retry with the exact selector it returns. If the widget creates its input lazily, make one guarded initializer click, re-read the page, then retry with the exact selector. Verify the page shows the attachment before submitting. - scratchpad_write({text}): Save notes that persist across steps. - progress_update({items}) / progress_read({status}): Structured progress ledger for the active repeated item/action task. On GitHub stargazers, only "Follow USER" buttons are follow targets when following is allowed by the task; "Unfollow USER" means skip/already followed unless the ledger shows acted. - done({summary, outcome}): Signal success, partial progress, or a failed blocker. diff --git a/src/firefox/src/agent/tools.js b/src/firefox/src/agent/tools.js index 8d4eeffaa..f0b807922 100644 --- a/src/firefox/src/agent/tools.js +++ b/src/firefox/src/agent/tools.js @@ -989,6 +989,11 @@ export const COMPACT_TOOL_NAMES = new Set([ 'get_accessibility_tree', 'read_page', 'scroll', 'get_window_info', 'extract_data', 'get_selection', 'find_text', + // get_interactive_elements is the only tool that returns a verified, unique + // CSS selector for a file input, so upload_file's ambiguous-selector + // recovery is built on it. It stays in the compact catalog for as long as + // upload_file does; without it an ambiguous selector is unrecoverable. + 'get_interactive_elements', 'click_ax', 'set_checked', 'type_ax', 'set_field', 'click', 'type_text', 'press_keys', 'navigate', 'new_tab', 'wait_for_element', @@ -1115,15 +1120,24 @@ const WATCH_BEEP_TOOL = { }, }; +// Compact has no download tools, so the only file it can legitimately reach is +// the one the user attached to this run, or one the user picks themselves. +// Deleting the keys rather than rebuilding `parameters` keeps any future base +// parameter reaching compact. Firefox never had filePath (no CDP). +const COMPACT_UPLOAD_HIDDEN_PARAMS = ['downloadId', 'filePath']; + function compactUploadFileTool(tool) { + const properties = { ...tool.function.parameters.properties }; + for (const key of COMPACT_UPLOAD_HIDDEN_PARAMS) delete properties[key]; return { ...tool, function: { ...tool.function, - description: 'Attach a user-provided file directly to an existing file input without clicking the page upload control. Use attachmentId from the current user-attachment notice, or omit it to open WebBrain\'s file picker. This proves only local page attachment, not remote upload or submission. Use the exact selector for the intended input; never guess a generic input[type="file"] selector when multiple inputs exist. If the widget creates its input lazily, make one guarded click on its add-files control, then retry upload_file with the exact selector.', + description: 'Attach a user-provided file directly to an existing file input without clicking the page upload control. Use attachmentId from the current user-attachment notice, or omit it to open WebBrain\'s file picker. This proves only local page attachment, not remote upload or submission. Use the exact selector for the intended input; never guess a generic input[type="file"] selector when multiple inputs exist. If the selector is ambiguous, call get_interactive_elements and use the exact selector on the intended file-input record before retrying. If the widget creates its input lazily, make one guarded click on its add-files control, then retry upload_file with the exact selector.', parameters: { - type: 'object', + ...tool.function.parameters, properties: { + ...properties, selector: { type: 'string', description: 'Exact CSS selector for the intended file input.' }, attachmentId: { type: 'string', description: 'Opaque id from the current user-attachment notice. Omit only to ask the user through WebBrain\'s file picker; never guess an id.' }, }, @@ -1238,6 +1252,7 @@ TOOLS - use only these: - get_window_info: Read window/viewport size. - scroll: Scroll up/down. - extract_data: Get tables, headings, images, or links. +- get_interactive_elements: List interactive elements with exact CSS selectors. Use it when you need a selector rather than a ref_id — above all to find the intended file input before upload_file, and to recover after an ambiguous upload selector. - click_ax({ref_id}): Click by ref_id from the tree. Preferred. - set_checked({ref_id, checked}): Idempotently set and verify a native checkbox. Never toggle checkboxes repeatedly with click_ax. - type_ax({ref_id, text}): Type into a field by ref_id. @@ -1251,7 +1266,7 @@ TOOLS - use only these: - new_tab({url}): Open a URL in a background tab for user reference. It does not activate or retarget the current run, so never use it as a site-permission workaround. - wait_for_element({selector}): Wait for an element to appear. - fetch_url({url}): Fetch other URLs for reading only; do not use it to re-read the active tab. -- upload_file({selector, attachmentId?}): Attach a user-provided file directly to an existing file input; do not click the page upload control first. Use the current attachmentId or omit it for WebBrain's picker. Use the exact selector and never guess generic input[type="file"] when multiple inputs exist. If the widget creates its input lazily, make one guarded initializer click, re-read the page, then retry with the exact selector. Verify the page shows the attachment before submitting. +- upload_file({selector, attachmentId?}): Attach a user-provided file directly to an existing file input; do not click the page upload control first. Use the current attachmentId or omit it for WebBrain's picker. Use the exact selector from get_interactive_elements and never guess generic input[type="file"] when multiple inputs exist. If the selector is ambiguous, call get_interactive_elements and retry with the exact selector it returns. If the widget creates its input lazily, make one guarded initializer click, re-read the page, then retry with the exact selector. Verify the page shows the attachment before submitting. - scratchpad_write({text}): Save notes that persist across steps. - progress_update({items}) / progress_read({status}): Structured progress ledger for the active repeated item/action task. On GitHub stargazers, only "Follow USER" buttons are follow targets when following is allowed by the task; "Unfollow USER" means skip/already followed unless the ledger shows acted. - clarify({question, options?}): Ask the user only when materially blocked or ambiguous. Unanswered clarifies auto-select options[0] after timeout (source=timeout is not user approval for high-risk steps; source=auto Instant is intentional auto-approve). diff --git a/test/run.js b/test/run.js index 00c1c93e6..6ca6e5dfd 100644 --- a/test/run.js +++ b/test/run.js @@ -12820,18 +12820,70 @@ test('compact Act exposes a direct-upload-only file workflow in both browsers', assert.match(upload.function.description, /one guarded click/i); assert.doesNotMatch(upload.function.description, /download_files|list_downloads|downloadId/i); assert.ok(upload.function.parameters.properties.attachmentId, `[${label}] compact upload must accept attachmentId`); - assert.equal(upload.function.parameters.properties.downloadId, undefined, `[${label}] compact upload must hide downloadId`); assert.ok(fullUpload.function.parameters.properties.downloadId, `[${label}] full upload must retain downloadId`); - assert.match(prompt, /upload_file[\s\S]{0,180}do not click the page upload control/i); - assert.match(prompt, /upload_file[\s\S]{0,360}exact selector/i); - assert.match(prompt, /upload_file[\s\S]{0,420}one guarded initializer click/i); + + // Compact can reach only the file the user attached to this run: it has no + // download tools to produce a downloadId, and no way to learn a local path + // that the model did not invent. + for (const hidden of ['downloadId', 'filePath']) { + assert.equal( + upload.function.parameters.properties[hidden], + undefined, + `[${label}] compact upload must hide ${hidden}`, + ); + } + assert.deepEqual( + Object.keys(upload.function.parameters.properties).sort(), + ['attachmentId', 'selector'], + `[${label}] compact upload must expose exactly selector + attachmentId`, + ); + + // An ambiguous selector latches upload_file until get_interactive_elements + // returns a verified file-input selector, so the recovery tool has to be in + // the catalog the model is given. + assert.ok( + compactNames.includes('get_interactive_elements'), + `[${label}] compact Act must expose upload_file's ambiguity-recovery tool`, + ); + assert.match(upload.function.description, /get_interactive_elements/); + + // Assert on the prompt's own upload_file bullet rather than on character + // distances, so rewording the neighbouring bullets cannot break these. + const uploadLine = prompt.split('\n').find(line => line.startsWith('- upload_file(')); + assert.ok(uploadLine, `[${label}] compact prompt must document upload_file`); + assert.match(uploadLine, /do not click the page upload control/i); + assert.match(uploadLine, /exact selector/i); + assert.match(uploadLine, /one guarded initializer click/i); + assert.match(uploadLine, /get_interactive_elements/); + assert.ok( + prompt.split('\n').some(line => line.startsWith('- get_interactive_elements')), + `[${label}] compact prompt must document the recovery tool it points at`, + ); assert.doesNotMatch(prompt, /download_files|list_downloads|downloadId/i); - if (label === 'chrome') { - assert.ok(upload.function.parameters.properties.filePath, 'chrome: compact upload must accept a user-supplied filePath'); - } else { - assert.equal(upload.function.parameters.properties.filePath, undefined, 'firefox: compact upload must not invent filePath support'); + if (label === 'firefox') { assert.match(upload.function.description, /WebBrain's file picker/i); + } else { + assert.doesNotMatch(uploadLine, /filePath/); + } + } +}); + +test('every tier exposing upload_file also exposes its ambiguity-recovery tool', () => { + // upload_file latches on an ambiguous selector and only a + // get_interactive_elements response carrying a verified file-input selector + // clears it (Agent._clearUploadSelectorRecoveryAfterInspection). A tier that + // ships upload_file without it can never recover: the handler keeps + // returning recoveryRequired:'get_interactive_elements' and the model has no + // way to satisfy it until the page is replaced. + for (const [label, getTools] of [['chrome', getToolsForModeCh], ['firefox', getToolsForModeFx]]) { + for (const tier of ['compact', 'mid', 'full']) { + const names = new Set(getTools('act', { tier }).map(tool => tool.function.name)); + if (!names.has('upload_file')) continue; + assert.ok( + names.has('get_interactive_elements'), + `[${label}] act/${tier} exposes upload_file without get_interactive_elements, so an ambiguous selector is unrecoverable`, + ); } } }); From 384efbacc3422678cbd2bae8f586d4e1091d9c63 Mon Sep 17 00:00:00 2001 From: Emre Sokullu Date: Wed, 5 Aug 2026 15:59:59 +0300 Subject: [PATCH 3/5] test: cover the upload retry that recovery enables The ambiguity test proved the latch clears but stopped there. Assert the step it exists for: after get_interactive_elements supplies a unique selector, the corrected upload dispatches and leaves the latch clear. Co-Authored-By: Claude Opus 5 --- test/run.js | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/test/run.js b/test/run.js index 6ca6e5dfd..22985dfec 100644 --- a/test/run.js +++ b/test/run.js @@ -50872,6 +50872,20 @@ test('upload_file prefers a valid downloadId and falls back to filePath for an i true, ); assert.equal(agent._uploadSelectorRecoveryRequired.has(42), false); + + // The retry the recovery exists to enable: once the inspection has supplied + // a unique selector, the corrected upload must actually dispatch. Without + // this the latch could clear and still leave uploads wedged. + selectorMatches = ['input-501']; + const recoveredRetry = await agent.executeTool(42, 'upload_file', { + selector: 'input[type=file]:not([accept])', + downloadId: 9123, + }); + assert.equal(recoveredRetry.success, true, 'a corrected selector must upload after recovery'); + assert.equal(recoveredRetry.file, realPath); + assert.equal(recoveredRetry.attachmentState, 'input_attached'); + assert.equal(agent._uploadSelectorRecoveryRequired.has(42), false, 'a successful retry must leave the latch clear'); + agent._uploadSelectorRecoveryRequired.set(42, 2); agent._clearRunLoopState(42); assert.equal(agent._uploadSelectorRecoveryRequired.has(42), false, 'run cleanup must clear upload recovery'); @@ -50880,8 +50894,8 @@ test('upload_file prefers a valid downloadId and falls back to filePath for an i assert.equal(agent._uploadSelectorRecoveryRequired.has(42), false, 'navigation cleanup must clear upload recovery'); assert.deepEqual( releasedGroups, - ['upload-query-1', 'upload-query-2', 'upload-query-3', 'upload-query-4'], - 'early upload failures must release selector handles', + ['upload-query-1', 'upload-query-2', 'upload-query-3', 'upload-query-4', 'upload-query-5'], + 'early upload failures and the post-recovery retry must release selector handles', ); } finally { if (originalChrome === undefined) delete globalThis.chrome; From b0b8f0a03426d8da350b05da920b749eb8268afe Mon Sep 17 00:00:00 2001 From: Barack Sokullu Date: Thu, 6 Aug 2026 12:17:25 +0300 Subject: [PATCH 4/5] fix(compact): make upload_file self-targeting --- src/chrome/src/agent/agent.js | 224 +++++++++++++++- src/chrome/src/agent/permission-gate.js | 2 + src/chrome/src/agent/tools.js | 21 +- src/chrome/src/content/content.js | 34 +++ src/firefox/src/agent/agent.js | 218 +++++++++++++++- src/firefox/src/agent/permission-gate.js | 2 + src/firefox/src/agent/tools.js | 21 +- src/firefox/src/content/content.js | 34 +++ test/run.js | 319 ++++++++++++++++++++--- 9 files changed, 804 insertions(+), 71 deletions(-) diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index 1d3494ac2..3bcbedde4 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -415,6 +415,8 @@ export class Agent extends LoopDetector { this._clickAxCdpFallbacks = new Map(); // tabId -> Set(documentToken|ref_id), one trusted fallback per document target this._lastAxScopes = new Map(); // tabId -> { documentToken, pageUrl }, captured by the latest AX read this._uploadSelectorRecoveryRequired = new Map(); // tabId -> prior ambiguous match count; cleared only by inspection/navigation/cleanup + this._compactUploadTargets = new Map(); // tabId -> { pageUrl, targets: Map(targetId, internal candidate) } + this._compactUploadTargetCounter = 0; // Productive browsing often mixes reads and scrolling, so exact-call loop // detection cannot tell when the agent already has enough evidence to // answer. Track long observation-only streaks and remind it to deliver a @@ -1670,6 +1672,7 @@ export class Agent extends LoopDetector { _clearPageLoopState(tabId) { super._clearPageLoopState(tabId); this._uploadSelectorRecoveryRequired.delete(tabId); + this._compactUploadTargets.delete(tabId); this.deliveryObservationStreaks.delete(tabId); this.bulkApiMutationClicks.delete(tabId); this.bulkApiMutationHints.delete(tabId); @@ -1694,6 +1697,162 @@ export class Agent extends LoopDetector { return true; } + _toolResultTrustName(name, result) { + return name === 'upload_file' && result?.discoveryOnly + ? 'get_file_input_targets' + : name; + } + + async _readCompactUploadFileInputs(tabId) { + const response = await this.executeTool(tabId, 'get_file_input_targets', {}); + if (!Array.isArray(response)) { + return { + ok: false, + error: response?.error || 'Could not inspect this page for file inputs.', + }; + } + const fileInputs = response.filter(element => ( + element?.tag === 'input' + && String(element.type || '').toLowerCase() === 'file' + )); + return { + ok: true, + fileInputs, + usable: fileInputs.filter(element => ( + typeof element.selector === 'string' && element.selector.trim().length > 0 + )), + }; + } + + _compactUploadTargetKey(element) { + return JSON.stringify([ + String(element?.selector || ''), + String(element?.id || ''), + String(element?.name || ''), + element?.accept == null ? null : String(element.accept), + element?.multiple === true, + element?.inShadowDOM === true, + ]); + } + + async _publishCompactUploadTargets(tabId, inventory, prefix = '') { + this._compactUploadTargets.delete(tabId); + if (!inventory?.ok) { + return { + success: false, + dispatched: false, + noDispatch: true, + discoveryOnly: true, + requiresTarget: true, + error: `${prefix}${inventory?.error || 'Could not inspect this page for file inputs.'}`, + }; + } + + const maxCandidates = 12; + const pageUrl = await this._currentUrl(tabId); + const targets = new Map(); + const candidates = inventory.usable.slice(0, maxCandidates).map((element, index) => { + const targetId = `file_target_${(++this._compactUploadTargetCounter).toString(36)}`; + targets.set(targetId, { + selector: element.selector.trim(), + key: this._compactUploadTargetKey(element), + }); + const label = String(element.text || element.name || element.id || `File input ${index + 1}`) + .replace(/[\r\n]+/g, ' ') + .trim() + .slice(0, 100); + return { + targetId, + label: label || `File input ${index + 1}`, + ...(element.name ? { name: String(element.name).slice(0, 100) } : {}), + ...(element.accept != null ? { accept: String(element.accept).slice(0, 200) } : {}), + multiple: element.multiple === true, + inShadowDOM: element.inShadowDOM === true, + }; + }); + + if (targets.size) this._compactUploadTargets.set(tabId, { pageUrl, targets }); + const candidateCount = inventory.fileInputs.length; + const addressableCount = inventory.usable.length; + if (!candidates.length) { + const foundButUnsafe = candidateCount > 0; + return { + success: false, + dispatched: false, + noDispatch: true, + discoveryOnly: true, + requiresTarget: true, + candidateCount, + addressableCount, + candidates: [], + initializerSuggested: !foundButUnsafe, + error: `${prefix}${foundButUnsafe + ? 'File inputs were found, but none had a verified unique target. Re-read the page and expose the intended upload widget before repeating upload_file without targetId.' + : 'No file input is currently available. If the upload widget creates one lazily, make one guarded click on its add-files control, re-read the page, then repeat upload_file without targetId.'}`, + }; + } + + return { + success: false, + dispatched: false, + noDispatch: true, + discoveryOnly: true, + requiresTarget: true, + candidateCount, + addressableCount, + candidates, + truncated: addressableCount > candidates.length, + error: `${prefix}Choose the intended file input from candidates, then retry upload_file with the same attachmentId and that exact targetId. Never guess or modify a targetId.`, + }; + } + + async _discoverCompactUploadTargets(tabId, prefix = '') { + return this._publishCompactUploadTargets( + tabId, + await this._readCompactUploadFileInputs(tabId), + prefix, + ); + } + + async _resolveCompactUploadTarget(tabId, targetId) { + const normalizedTargetId = typeof targetId === 'string' ? targetId.trim() : ''; + const state = this._compactUploadTargets.get(tabId); + const saved = normalizedTargetId ? state?.targets?.get(normalizedTargetId) : null; + if (!saved) { + return { + ok: false, + result: await this._discoverCompactUploadTargets( + tabId, + 'That targetId is missing, expired, or was not returned by the latest discovery. ', + ), + }; + } + + const pageUrl = await this._currentUrl(tabId); + const inventory = await this._readCompactUploadFileInputs(tabId); + const current = inventory?.ok + ? inventory.usable.find(element => ( + element.selector.trim() === saved.selector + && this._compactUploadTargetKey(element) === saved.key + )) + : null; + if (this._normalizeUrl(pageUrl) !== this._normalizeUrl(state.pageUrl) || !current) { + return { + ok: false, + result: await this._publishCompactUploadTargets( + tabId, + inventory, + 'The page changed and that targetId expired. ', + ), + }; + } + + // Compact target handles are one-use. A retry must rediscover so a page + // that consumed or replaced the input cannot receive a stale attachment. + this._compactUploadTargets.delete(tabId); + return { ok: true, selector: saved.selector }; + } + _rememberAxScope(tabId, documentToken, pageUrl = '') { const next = { documentToken: String(documentToken || ''), @@ -4248,7 +4407,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d fnName, fnArgs, onUpdate, - { completionBatchStartState }, + { completionBatchStartState, promptTier }, ); const toolResult = this._normalizeToolResult(fnName, rawToolResult, missingResponseOutcomeUnknown); const inspectFormValidationAfter = formValidationCandidate @@ -4639,7 +4798,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // Wrap page-derived results as untrusted DATA BEFORE appending any of // our own trusted notes (the loop nudge), so the nudge stays outside the // box and is read as an instruction, not data. - let resultContent = this._wrapUntrusted(fnName, this._limitToolResult(toolResult)); + const resultTrustName = this._toolResultTrustName(fnName, toolResult); + let resultContent = this._wrapUntrusted(resultTrustName, this._limitToolResult(toolResult)); if (toolResult?.errorCode === 'chrome_protected_page') { resultContent += '\n[TRUSTED RUNTIME ROUTING: Chrome blocks extension DOM/debugger access on this dashboard. Do not call another DOM, accessibility, wait, script, iframe, WebMCP, or upload_file tool here. Continue manually in the dashboard.]'; onUpdate('warning', { message: 'Chrome-protected dashboard detected; DOM automation is unavailable.' }); @@ -16961,7 +17121,46 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (name === 'upload_file') { args = args || {}; - if (this._uploadSelectorRecoveryRequired.has(tabId)) { + const compactUpload = ( + executionContext?.promptTier || this._resolvePromptTier() + ) === 'compact'; + let attachmentPayload = null; + if (compactUpload) { + if ( + args.selector != null + || args.downloadId != null + || args.filePath != null + ) { + return { + success: false, + dispatched: false, + noDispatch: true, + denied: true, + error: 'Compact upload_file accepts only attachmentId and a targetId returned by its own discovery phase. Do not provide selector, downloadId, or filePath.', + }; + } + if (args.attachmentId == null || !String(args.attachmentId).trim()) { + return { + success: false, + dispatched: false, + noDispatch: true, + error: 'Compact Chrome upload_file requires attachmentId from the current user-attachment notice.', + }; + } + const resolvedAttachment = this._resolveUserAttachment(tabId, args.attachmentId); + if (!resolvedAttachment.ok) return { success: false, error: resolvedAttachment.error }; + attachmentPayload = resolvedAttachment; + if (args.targetId == null || !String(args.targetId).trim()) { + return await this._discoverCompactUploadTargets(tabId); + } + const resolvedTarget = await this._resolveCompactUploadTarget(tabId, args.targetId); + if (!resolvedTarget.ok) return resolvedTarget.result; + args = { + attachmentId: String(args.attachmentId), + selector: resolvedTarget.selector, + }; + } + if (!compactUpload && this._uploadSelectorRecoveryRequired.has(tabId)) { return { success: false, dispatched: false, @@ -16972,8 +17171,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d error: 'A previous upload selector matched multiple file inputs. Call get_interactive_elements now and use the exact selector returned on the intended file-input record before retrying upload_file; do not guess another selector variant.', }; } - let attachmentPayload = null; - if (args.attachmentId != null) { + if (!attachmentPayload && args.attachmentId != null) { if (args.downloadId != null || (typeof args.filePath === 'string' && args.filePath.trim())) { return { success: false, error: 'upload_file accepts only one source when attachmentId is used. Remove downloadId/filePath and retry with the current attachmentId.' }; } @@ -17025,9 +17223,21 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d uploadQuery = await cdpClient.querySelectorPierce(tabId, args.selector); const objectIds = uploadQuery?.objectIds || []; if (objectIds.length === 0) { + if (compactUpload) { + return await this._discoverCompactUploadTargets( + tabId, + 'The selected file input changed before attachment. ', + ); + } return { success: false, error: `File input not found for selector "${args.selector}". Re-inspect the page with get_interactive_elements or get_accessibility_tree to find the real (some upload widgets hide it until you click their "add files" button first).` }; } if (objectIds.length > 1) { + if (compactUpload) { + return await this._discoverCompactUploadTargets( + tabId, + 'The selected file input became ambiguous before attachment. ', + ); + } this._uploadSelectorRecoveryRequired.set(tabId, objectIds.length); return { success: false, @@ -18777,6 +18987,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const actionMap = { 'read_page': 'get_page_info_cdp', 'get_interactive_elements': 'get_interactive_elements_cdp', + // Internal only: Compact upload_file turns these selectors into opaque, + // one-use targetIds before exposing the bounded candidate list. + 'get_file_input_targets': 'get_file_input_targets', // Accessibility-tree path (preferred). Ported from Claude for Chrome — // flat indented text output with persistent WeakRef-backed ref_ids. 'get_accessibility_tree': 'get_accessibility_tree', @@ -18833,6 +19046,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d name === 'type_text' || name === 'type_ax' || name === 'set_field' || name === 'press_keys' || name === 'scroll' || name === 'get_accessibility_tree' || name === 'get_interactive_elements' || + name === 'get_file_input_targets' || name === 'extract_data' || name === 'inspect_element_styles' || name === 'wait_for_element' || name === 'wait_for_stable' || name === 'get_selection' || name === 'find_text' diff --git a/src/chrome/src/agent/permission-gate.js b/src/chrome/src/agent/permission-gate.js index 706f7b713..50808bcad 100644 --- a/src/chrome/src/agent/permission-gate.js +++ b/src/chrome/src/agent/permission-gate.js @@ -59,6 +59,8 @@ export const UNTRUSTED_CONTENT_TOOLS = new Set([ 'read_page', 'get_accessibility_tree', 'get_interactive_elements', + // Hidden Compact-upload discovery returns page-authored file-input labels. + 'get_file_input_targets', 'get_shadow_dom', 'shadow_dom_query', 'get_frames', diff --git a/src/chrome/src/agent/tools.js b/src/chrome/src/agent/tools.js index d0ba232f2..2be977d56 100644 --- a/src/chrome/src/agent/tools.js +++ b/src/chrome/src/agent/tools.js @@ -1270,9 +1270,10 @@ const WATCH_BEEP_TOOL = { // the one the user attached to this run. Dropping downloadId and filePath is // therefore not just prompt economy: filePath is a CDP-backed read of any local // path into an untrusted page's input, and compact omits the full-tier guidance -// that exists to stop the model inventing one. Deleting the keys rather than -// rebuilding `parameters` keeps any future base parameter reaching compact. -const COMPACT_UPLOAD_HIDDEN_PARAMS = ['downloadId', 'filePath']; +// that exists to stop the model inventing one. Compact also replaces selector +// with an opaque targetId returned by upload_file's own discovery phase, so a +// small model never has to choose another inspection tool or construct CSS. +const COMPACT_UPLOAD_HIDDEN_PARAMS = ['selector', 'downloadId', 'filePath']; function compactUploadFileTool(tool) { const properties = { ...tool.function.parameters.properties }; @@ -1281,15 +1282,15 @@ function compactUploadFileTool(tool) { ...tool, function: { ...tool.function, - description: 'Attach a user-provided file directly to an existing file input without clicking the page upload control. Use attachmentId from the current user-attachment notice. This proves only local page attachment, not remote upload or submission. Use the exact selector for the intended input; never guess a generic input[type="file"] selector when multiple inputs exist. If the selector is ambiguous, call get_interactive_elements and use the exact selector on the intended file-input record before retrying. If the widget creates its input lazily, make one guarded click on its add-files control, then retry upload_file with the exact selector.', + description: 'Attach the current user-provided file through a two-step Compact workflow. First call with attachmentId only: this is read-only and returns opaque targetId choices for the page\'s file inputs. Then call again with the same attachmentId and one returned targetId. Never invent or modify a targetId. This proves only local page attachment, not remote upload or submission. If no file input exists because a widget creates it lazily, make one guarded click on its add-files control, then repeat the discovery call.', parameters: { ...tool.function.parameters, properties: { ...properties, - selector: { type: 'string', description: 'Exact CSS selector for the intended file input.' }, attachmentId: { type: 'string', description: 'Opaque id from the current user-attachment notice. Never guess an id.' }, + targetId: { type: 'string', description: 'Opaque file-input target returned by a prior upload_file discovery call in this run. Never guess or modify it.' }, }, - required: ['selector'], + required: ['attachmentId'], }, }, }; @@ -1729,11 +1730,6 @@ export const COMPACT_TOOL_NAMES = new Set([ 'get_accessibility_tree', 'read_page', 'scroll', 'get_window_info', 'extract_data', 'get_selection', 'find_text', - // get_interactive_elements is the only tool that returns a verified, unique - // CSS selector for a file input, so upload_file's ambiguous-selector - // recovery is built on it. It stays in the compact catalog for as long as - // upload_file does; without it an ambiguous selector is unrecoverable. - 'get_interactive_elements', 'click_ax', 'set_checked', 'type_ax', 'set_field', 'click', 'type_text', 'press_keys', 'navigate', 'new_tab', 'wait_for_element', @@ -1771,7 +1767,6 @@ TOOLS — use ONLY these: - get_window_info: Read window/viewport size. - scroll: Scroll up/down. - extract_data: Get tables, headings, images. -- get_interactive_elements: List interactive elements with exact CSS selectors. Use it when you need a selector rather than a ref_id — above all to find the intended file input before upload_file, and to recover after an ambiguous upload selector. - click_ax({ref_id}): Click by ref_id from the tree. PREFERRED. - set_checked({ref_id, checked}): Idempotently set and verify a native checkbox. Never toggle checkboxes repeatedly with click_ax. - type_ax({ref_id, text}): Type into a field by ref_id. @@ -1785,7 +1780,7 @@ TOOLS — use ONLY these: - new_tab({url}): Open a URL in a background tab for user reference. It does not activate or retarget the current run, so never use it as a site-permission workaround. - wait_for_element({selector}): Wait for an element to appear. - fetch_url({url}): Fetch a URL for its content. -- upload_file({selector, attachmentId}): Attach a user-provided file directly to an existing file input; do not click the page upload control first. Use the exact selector from get_interactive_elements and never guess generic input[type="file"] when multiple inputs exist. If the selector is ambiguous, call get_interactive_elements and retry with the exact selector it returns. If the widget creates its input lazily, make one guarded initializer click, re-read the page, then retry with the exact selector. Verify the page shows the attachment before submitting. +- upload_file({attachmentId, targetId?}): Two steps: first call with the current attachmentId only to discover file inputs; then call again with the same attachmentId and one returned targetId. Never guess a targetId. If discovery finds no input because the widget creates it lazily, make one guarded initializer click and repeat discovery. Verify the page shows the attachment before submitting. - scratchpad_write({text}): Save notes that persist across steps. - progress_update({items}) / progress_read({status}): Structured progress ledger for the active repeated item/action task. On GitHub stargazers, only "Follow USER" buttons are follow targets when following is allowed by the task; "Unfollow USER" means skip/already followed unless the ledger shows acted. - done({summary, outcome}): Signal success, partial progress, or a failed blocker. diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js index 5ab829a32..23dfd3134 100644 --- a/src/chrome/src/content/content.js +++ b/src/chrome/src/content/content.js @@ -2938,6 +2938,39 @@ return ''; } + // Internal Compact-upload discovery. This is intentionally not a model + // tool: it returns only file inputs, and the agent replaces each selector + // with a run-scoped opaque targetId before the result reaches the model. + function getFileInputTargets() { + const targets = []; + const seen = new Set(); + const visit = (root, inShadowDOM = false) => { + try { + root.querySelectorAll('input').forEach(el => { + if (!(el instanceof HTMLInputElement) || el.type !== 'file' || seen.has(el)) return; + seen.add(el); + const selector = _uniqueFileInputSelector(el); + targets.push({ + tag: 'input', + type: 'file', + text: _siteInteractionText(el).slice(0, 100), + id: el.id || '', + name: el.name || '', + accept: el.getAttribute('accept'), + multiple: el.hasAttribute('multiple'), + inShadowDOM, + ...(selector ? { selector } : {}), + }); + }); + root.querySelectorAll('*').forEach(host => { + if (host.shadowRoot) visit(host.shadowRoot, true); + }); + } catch {} + }; + visit(document); + return targets; + } + function getInteractiveElementsFull() { return queryInteractiveFull().map((c, i) => { const el = c.el; @@ -3668,6 +3701,7 @@ 'get_page_info_cdp': () => getPageInfoFull(msg.params || {}), 'get_interactive_elements': () => getInteractiveElements(), 'get_interactive_elements_cdp': () => getInteractiveElementsFull(), + 'get_file_input_targets': () => getFileInputTargets(), 'click': () => clickElement(msg.params || {}), 'consume_file_picker_guard': () => consumeFilePickerGuard(msg.params?.guardId), 'type': () => typeText(msg.params || {}), diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index 08f95c55d..41c08d1a6 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -368,6 +368,8 @@ export class Agent extends LoopDetector { this.strictSecretMode = false; this._lastAxScopes = new Map(); // tabId -> { documentToken, pageUrl }, captured by the latest AX read this._uploadSelectorRecoveryRequired = new Map(); // tabId -> prior ambiguous match count; cleared only by inspection/navigation/cleanup + this._compactUploadTargets = new Map(); // tabId -> { pageUrl, targets: Map(targetId, internal candidate) } + this._compactUploadTargetCounter = 0; // Productive browsing often mixes reads and scrolling, so exact-call loop // detection cannot tell when the agent already has enough evidence to // answer. Track long observation-only streaks and remind it to deliver a @@ -1702,6 +1704,7 @@ export class Agent extends LoopDetector { _clearPageLoopState(tabId) { super._clearPageLoopState(tabId); this._uploadSelectorRecoveryRequired.delete(tabId); + this._compactUploadTargets.delete(tabId); this.deliveryObservationStreaks.delete(tabId); this.bulkApiMutationClicks.delete(tabId); this.bulkApiMutationHints.delete(tabId); @@ -1726,6 +1729,162 @@ export class Agent extends LoopDetector { return true; } + _toolResultTrustName(name, result) { + return name === 'upload_file' && result?.discoveryOnly + ? 'get_file_input_targets' + : name; + } + + async _readCompactUploadFileInputs(tabId) { + const response = await this.executeTool(tabId, 'get_file_input_targets', {}); + if (!Array.isArray(response)) { + return { + ok: false, + error: response?.error || 'Could not inspect this page for file inputs.', + }; + } + const fileInputs = response.filter(element => ( + element?.tag === 'input' + && String(element.type || '').toLowerCase() === 'file' + )); + return { + ok: true, + fileInputs, + usable: fileInputs.filter(element => ( + typeof element.selector === 'string' && element.selector.trim().length > 0 + )), + }; + } + + _compactUploadTargetKey(element) { + return JSON.stringify([ + String(element?.selector || ''), + String(element?.id || ''), + String(element?.name || ''), + element?.accept == null ? null : String(element.accept), + element?.multiple === true, + element?.inShadowDOM === true, + ]); + } + + async _publishCompactUploadTargets(tabId, inventory, prefix = '') { + this._compactUploadTargets.delete(tabId); + if (!inventory?.ok) { + return { + success: false, + dispatched: false, + noDispatch: true, + discoveryOnly: true, + requiresTarget: true, + error: `${prefix}${inventory?.error || 'Could not inspect this page for file inputs.'}`, + }; + } + + const maxCandidates = 12; + const pageUrl = await this._currentUrl(tabId); + const targets = new Map(); + const candidates = inventory.usable.slice(0, maxCandidates).map((element, index) => { + const targetId = `file_target_${(++this._compactUploadTargetCounter).toString(36)}`; + targets.set(targetId, { + selector: element.selector.trim(), + key: this._compactUploadTargetKey(element), + }); + const label = String(element.text || element.name || element.id || `File input ${index + 1}`) + .replace(/[\r\n]+/g, ' ') + .trim() + .slice(0, 100); + return { + targetId, + label: label || `File input ${index + 1}`, + ...(element.name ? { name: String(element.name).slice(0, 100) } : {}), + ...(element.accept != null ? { accept: String(element.accept).slice(0, 200) } : {}), + multiple: element.multiple === true, + inShadowDOM: element.inShadowDOM === true, + }; + }); + + if (targets.size) this._compactUploadTargets.set(tabId, { pageUrl, targets }); + const candidateCount = inventory.fileInputs.length; + const addressableCount = inventory.usable.length; + if (!candidates.length) { + const foundButUnsafe = candidateCount > 0; + return { + success: false, + dispatched: false, + noDispatch: true, + discoveryOnly: true, + requiresTarget: true, + candidateCount, + addressableCount, + candidates: [], + initializerSuggested: !foundButUnsafe, + error: `${prefix}${foundButUnsafe + ? 'File inputs were found, but none had a verified unique target. Re-read the page and expose the intended upload widget before repeating upload_file without targetId.' + : 'No file input is currently available. If the upload widget creates one lazily, make one guarded click on its add-files control, re-read the page, then repeat upload_file without targetId.'}`, + }; + } + + return { + success: false, + dispatched: false, + noDispatch: true, + discoveryOnly: true, + requiresTarget: true, + candidateCount, + addressableCount, + candidates, + truncated: addressableCount > candidates.length, + error: `${prefix}Choose the intended file input from candidates, then retry upload_file with that exact targetId and the current attachmentId, or omit attachmentId on that second call for WebBrain's picker. Never guess or modify a targetId.`, + }; + } + + async _discoverCompactUploadTargets(tabId, prefix = '') { + return this._publishCompactUploadTargets( + tabId, + await this._readCompactUploadFileInputs(tabId), + prefix, + ); + } + + async _resolveCompactUploadTarget(tabId, targetId) { + const normalizedTargetId = typeof targetId === 'string' ? targetId.trim() : ''; + const state = this._compactUploadTargets.get(tabId); + const saved = normalizedTargetId ? state?.targets?.get(normalizedTargetId) : null; + if (!saved) { + return { + ok: false, + result: await this._discoverCompactUploadTargets( + tabId, + 'That targetId is missing, expired, or was not returned by the latest discovery. ', + ), + }; + } + + const pageUrl = await this._currentUrl(tabId); + const inventory = await this._readCompactUploadFileInputs(tabId); + const current = inventory?.ok + ? inventory.usable.find(element => ( + element.selector.trim() === saved.selector + && this._compactUploadTargetKey(element) === saved.key + )) + : null; + if (this._normalizeUrl(pageUrl) !== this._normalizeUrl(state.pageUrl) || !current) { + return { + ok: false, + result: await this._publishCompactUploadTargets( + tabId, + inventory, + 'The page changed and that targetId expired. ', + ), + }; + } + + // Compact target handles are one-use. A retry must rediscover so a page + // that consumed or replaced the input cannot receive a stale attachment. + this._compactUploadTargets.delete(tabId); + return { ok: true, selector: saved.selector }; + } + _rememberAxScope(tabId, documentToken, pageUrl = '') { const next = { documentToken: String(documentToken || ''), @@ -3801,6 +3960,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const _toolStart = Date.now(); const rawToolResult = await this.executeTool(tabId, fnName, fnArgs, onUpdate, { completionBatchStartState, + promptTier, }); const toolResult = this._normalizeToolResult(fnName, rawToolResult, missingResponseOutcomeUnknown); const inspectFormValidationAfter = formValidationCandidate @@ -4186,7 +4346,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // Wrap page-derived results as untrusted DATA BEFORE appending any of // our own trusted notes (the loop nudge), so the nudge stays outside the // box and is read as an instruction, not data. - let resultContent = this._wrapUntrusted(fnName, this._limitToolResult(toolResult)); + const resultTrustName = this._toolResultTrustName(fnName, toolResult); + let resultContent = this._wrapUntrusted(resultTrustName, this._limitToolResult(toolResult)); if (captchaGateDecision?.status === 'solve_required') { resultContent += '\n[TRUSTED CAPTCHA GATE: A supported verification challenge is active. Call solve_captcha once now. Do not dismiss or close the dialog, click Continue/Submit, or use another page-changing tool until solve_captcha returns.]'; onUpdate('warning', { message: 'Supported verification challenge detected; solve_captcha is required.' }); @@ -13802,7 +13963,47 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (name === 'upload_file') { const UPLOAD_MAX_BYTES = 25 * 1024 * 1024; try { - if (this._uploadSelectorRecoveryRequired.has(tabId)) { + args = args || {}; + const compactUpload = ( + executionContext?.promptTier || this._resolvePromptTier() + ) === 'compact'; + let compactAttachmentPayload = null; + if (compactUpload) { + if ( + args.selector != null + || args.downloadId != null + || args.filePath != null + ) { + return { + success: false, + dispatched: false, + noDispatch: true, + denied: true, + error: 'Compact upload_file accepts only attachmentId and a targetId returned by its own discovery phase. Do not provide selector, downloadId, or filePath.', + }; + } + if (args.attachmentId != null) { + const resolvedAttachment = this._resolveUserAttachment( + tabId, + args.attachmentId, + UPLOAD_MAX_BYTES, + ); + if (!resolvedAttachment.ok) { + return { success: false, error: resolvedAttachment.error }; + } + compactAttachmentPayload = resolvedAttachment; + } + if (args.targetId == null || !String(args.targetId).trim()) { + return await this._discoverCompactUploadTargets(tabId); + } + const resolvedTarget = await this._resolveCompactUploadTarget(tabId, args.targetId); + if (!resolvedTarget.ok) return resolvedTarget.result; + args = { + ...(args.attachmentId != null ? { attachmentId: String(args.attachmentId) } : {}), + selector: resolvedTarget.selector, + }; + } + if (!compactUpload && this._uploadSelectorRecoveryRequired.has(tabId)) { return { success: false, dispatched: false, @@ -13828,7 +14029,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (args.downloadId != null) { return { success: false, error: 'upload_file accepts only one source. Remove downloadId and retry with the current attachmentId.' }; } - const resolved = this._resolveUserAttachment(tabId, args.attachmentId, UPLOAD_MAX_BYTES); + const resolved = compactAttachmentPayload + || this._resolveUserAttachment(tabId, args.attachmentId, UPLOAD_MAX_BYTES); if (!resolved.ok) return { success: false, error: resolved.error }; ({ base64, filename, mimeType } = resolved); } else if (args.downloadId != null) { @@ -14052,6 +14254,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } const res = results && results[0]; if (!res || !res.success) { + if (compactUpload && res?.dispatched !== true) { + return await this._discoverCompactUploadTargets( + tabId, + 'The selected file input changed before attachment. ', + ); + } if (res?.ambiguous) this._uploadSelectorRecoveryRequired.set(tabId, Number(res.matchCount) || 0); return { success: false, @@ -14938,6 +15146,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const actionMap = { 'read_page': 'get_page_info_cdp', 'get_interactive_elements': 'get_interactive_elements_cdp', + // Internal only: Compact upload_file turns these selectors into opaque, + // one-use targetIds before exposing the bounded candidate list. + 'get_file_input_targets': 'get_file_input_targets', 'get_shadow_dom': 'get_shadow_dom', 'get_frames': 'get_frames', 'click': 'click', @@ -15028,6 +15239,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d name === 'press_keys' || name === 'scroll' || name === 'hover' || name === 'drag_drop' || name === 'get_accessibility_tree' || name === 'get_interactive_elements' || + name === 'get_file_input_targets' || name === 'extract_data' || name === 'inspect_element_styles' || name === 'wait_for_element' || name === 'wait_for_stable' || name === 'get_selection' || name === 'find_text' || name === 'execute_js' diff --git a/src/firefox/src/agent/permission-gate.js b/src/firefox/src/agent/permission-gate.js index a957fb4bc..c55190b7e 100644 --- a/src/firefox/src/agent/permission-gate.js +++ b/src/firefox/src/agent/permission-gate.js @@ -57,6 +57,8 @@ export const UNTRUSTED_CONTENT_TOOLS = new Set([ 'read_page', 'get_accessibility_tree', 'get_interactive_elements', + // Hidden Compact-upload discovery returns page-authored file-input labels. + 'get_file_input_targets', 'get_shadow_dom', 'shadow_dom_query', 'get_frames', diff --git a/src/firefox/src/agent/tools.js b/src/firefox/src/agent/tools.js index f0b807922..3f5707d4e 100644 --- a/src/firefox/src/agent/tools.js +++ b/src/firefox/src/agent/tools.js @@ -989,11 +989,6 @@ export const COMPACT_TOOL_NAMES = new Set([ 'get_accessibility_tree', 'read_page', 'scroll', 'get_window_info', 'extract_data', 'get_selection', 'find_text', - // get_interactive_elements is the only tool that returns a verified, unique - // CSS selector for a file input, so upload_file's ambiguous-selector - // recovery is built on it. It stays in the compact catalog for as long as - // upload_file does; without it an ambiguous selector is unrecoverable. - 'get_interactive_elements', 'click_ax', 'set_checked', 'type_ax', 'set_field', 'click', 'type_text', 'press_keys', 'navigate', 'new_tab', 'wait_for_element', @@ -1122,9 +1117,10 @@ const WATCH_BEEP_TOOL = { // Compact has no download tools, so the only file it can legitimately reach is // the one the user attached to this run, or one the user picks themselves. -// Deleting the keys rather than rebuilding `parameters` keeps any future base -// parameter reaching compact. Firefox never had filePath (no CDP). -const COMPACT_UPLOAD_HIDDEN_PARAMS = ['downloadId', 'filePath']; +// Compact replaces selector with an opaque targetId returned by upload_file's +// own discovery phase, so a small model never has to choose another inspection +// tool or construct CSS. Firefox never had filePath (no CDP). +const COMPACT_UPLOAD_HIDDEN_PARAMS = ['selector', 'downloadId', 'filePath']; function compactUploadFileTool(tool) { const properties = { ...tool.function.parameters.properties }; @@ -1133,15 +1129,15 @@ function compactUploadFileTool(tool) { ...tool, function: { ...tool.function, - description: 'Attach a user-provided file directly to an existing file input without clicking the page upload control. Use attachmentId from the current user-attachment notice, or omit it to open WebBrain\'s file picker. This proves only local page attachment, not remote upload or submission. Use the exact selector for the intended input; never guess a generic input[type="file"] selector when multiple inputs exist. If the selector is ambiguous, call get_interactive_elements and use the exact selector on the intended file-input record before retrying. If the widget creates its input lazily, make one guarded click on its add-files control, then retry upload_file with the exact selector.', + description: 'Attach a file through a two-step Compact workflow. First call without targetId: this is read-only and returns opaque targetId choices for the page\'s file inputs. Then call again with one returned targetId and the current attachmentId, or omit attachmentId on that second call to open WebBrain\'s user-controlled picker. Never invent or modify a targetId. This proves only local page attachment, not remote upload or submission. If no file input exists because a widget creates it lazily, make one guarded click on its add-files control, then repeat the discovery call.', parameters: { ...tool.function.parameters, properties: { ...properties, - selector: { type: 'string', description: 'Exact CSS selector for the intended file input.' }, attachmentId: { type: 'string', description: 'Opaque id from the current user-attachment notice. Omit only to ask the user through WebBrain\'s file picker; never guess an id.' }, + targetId: { type: 'string', description: 'Opaque file-input target returned by a prior upload_file discovery call in this run. Never guess or modify it.' }, }, - required: ['selector'], + required: [], }, }, }; @@ -1252,7 +1248,6 @@ TOOLS - use only these: - get_window_info: Read window/viewport size. - scroll: Scroll up/down. - extract_data: Get tables, headings, images, or links. -- get_interactive_elements: List interactive elements with exact CSS selectors. Use it when you need a selector rather than a ref_id — above all to find the intended file input before upload_file, and to recover after an ambiguous upload selector. - click_ax({ref_id}): Click by ref_id from the tree. Preferred. - set_checked({ref_id, checked}): Idempotently set and verify a native checkbox. Never toggle checkboxes repeatedly with click_ax. - type_ax({ref_id, text}): Type into a field by ref_id. @@ -1266,7 +1261,7 @@ TOOLS - use only these: - new_tab({url}): Open a URL in a background tab for user reference. It does not activate or retarget the current run, so never use it as a site-permission workaround. - wait_for_element({selector}): Wait for an element to appear. - fetch_url({url}): Fetch other URLs for reading only; do not use it to re-read the active tab. -- upload_file({selector, attachmentId?}): Attach a user-provided file directly to an existing file input; do not click the page upload control first. Use the current attachmentId or omit it for WebBrain's picker. Use the exact selector from get_interactive_elements and never guess generic input[type="file"] when multiple inputs exist. If the selector is ambiguous, call get_interactive_elements and retry with the exact selector it returns. If the widget creates its input lazily, make one guarded initializer click, re-read the page, then retry with the exact selector. Verify the page shows the attachment before submitting. +- upload_file({attachmentId?, targetId?}): Two steps: first call without targetId to discover file inputs; then call again with one returned targetId and the current attachmentId, or omit attachmentId on the second call for WebBrain's picker. Never guess a targetId. If discovery finds no input because the widget creates it lazily, make one guarded initializer click and repeat discovery. Verify the page shows the attachment before submitting. - scratchpad_write({text}): Save notes that persist across steps. - progress_update({items}) / progress_read({status}): Structured progress ledger for the active repeated item/action task. On GitHub stargazers, only "Follow USER" buttons are follow targets when following is allowed by the task; "Unfollow USER" means skip/already followed unless the ledger shows acted. - clarify({question, options?}): Ask the user only when materially blocked or ambiguous. Unanswered clarifies auto-select options[0] after timeout (source=timeout is not user approval for high-risk steps; source=auto Instant is intentional auto-approve). diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js index 2c5f70053..2042a0c97 100644 --- a/src/firefox/src/content/content.js +++ b/src/firefox/src/content/content.js @@ -926,6 +926,39 @@ return ''; } + // Internal Compact-upload discovery. This is intentionally not a model + // tool: it returns only file inputs, and the agent replaces each selector + // with a run-scoped opaque targetId before the result reaches the model. + function getFileInputTargets() { + const targets = []; + const seen = new Set(); + const visit = (root, inShadowDOM = false) => { + try { + root.querySelectorAll('input').forEach(el => { + if (!(el instanceof HTMLInputElement) || el.type !== 'file' || seen.has(el)) return; + seen.add(el); + const selector = _uniqueFileInputSelector(el); + targets.push({ + tag: 'input', + type: 'file', + text: _siteInteractionText(el).slice(0, 100), + id: el.id || '', + name: el.name || '', + accept: el.getAttribute('accept'), + multiple: el.hasAttribute('multiple'), + inShadowDOM, + ...(selector ? { selector } : {}), + }); + }); + root.querySelectorAll('*').forEach(host => { + if (host.shadowRoot) visit(host.shadowRoot, true); + }); + } catch {} + }; + visit(document); + return targets; + } + window.__wb_resolve_click_target_for_submit_probe = function resolveClickTargetForSubmitProbe(params = {}) { if (params?.index == null) return null; const index = Number(params.index); @@ -3019,6 +3052,7 @@ 'get_page_info_cdp': () => getPageInfoFull(msg.params || {}), 'get_interactive_elements': () => getInteractiveElements(), 'get_interactive_elements_cdp': () => getInteractiveElementsFull(), + 'get_file_input_targets': () => getFileInputTargets(), 'click': () => clickElement(msg.params || {}), 'consume_file_picker_guard': () => consumeFilePickerGuard(msg.params?.guardId), 'type': () => typeText(msg.params || {}), diff --git a/test/run.js b/test/run.js index 22985dfec..08f4131dd 100644 --- a/test/run.js +++ b/test/run.js @@ -12793,7 +12793,7 @@ test('getToolsForMode: compact mode restricts act tools in both browsers', () => } }); -test('compact Act exposes a direct-upload-only file workflow in both browsers', () => { +test('compact Act exposes a self-targeting upload workflow without a general selector tool', () => { for (const [label, getTools, prompt] of [ ['chrome', getToolsForModeCh, SYSTEM_PROMPT_ACT_COMPACT_CH], ['firefox', getToolsForModeFx, SYSTEM_PROMPT_ACT_COMPACT_FX], @@ -12814,18 +12814,23 @@ test('compact Act exposes a direct-upload-only file workflow in both browsers', ]) { assert.equal(compactNames.includes(unavailable), false, `[${label}] compact Act exposed ${unavailable}`); } - assert.match(upload.function.description, /directly to an existing file input/i); - assert.match(upload.function.description, /without clicking the page upload control/i); - assert.match(upload.function.description, /exact selector/i); + assert.match(upload.function.description, /two-step Compact workflow/i); + assert.match(upload.function.description, /read-only/i); + assert.match(upload.function.description, /targetId choices/i); + assert.match(upload.function.description, /Never invent or modify a targetId/i); assert.match(upload.function.description, /one guarded click/i); assert.doesNotMatch(upload.function.description, /download_files|list_downloads|downloadId/i); + assert.doesNotMatch(upload.function.description, /get_interactive_elements|CSS selector/i); assert.ok(upload.function.parameters.properties.attachmentId, `[${label}] compact upload must accept attachmentId`); + assert.ok(upload.function.parameters.properties.targetId, `[${label}] compact upload must accept its own targetId`); assert.ok(fullUpload.function.parameters.properties.downloadId, `[${label}] full upload must retain downloadId`); + assert.ok(fullUpload.function.parameters.properties.selector, `[${label}] full upload must retain selector`); + assert.equal(fullUpload.function.parameters.properties.targetId, undefined, `[${label}] full upload must not gain compact targetId`); // Compact can reach only the file the user attached to this run: it has no // download tools to produce a downloadId, and no way to learn a local path // that the model did not invent. - for (const hidden of ['downloadId', 'filePath']) { + for (const hidden of ['selector', 'downloadId', 'filePath']) { assert.equal( upload.function.parameters.properties[hidden], undefined, @@ -12834,56 +12839,105 @@ test('compact Act exposes a direct-upload-only file workflow in both browsers', } assert.deepEqual( Object.keys(upload.function.parameters.properties).sort(), - ['attachmentId', 'selector'], - `[${label}] compact upload must expose exactly selector + attachmentId`, + ['attachmentId', 'targetId'], + `[${label}] compact upload must expose exactly attachmentId + targetId`, ); - - // An ambiguous selector latches upload_file until get_interactive_elements - // returns a verified file-input selector, so the recovery tool has to be in - // the catalog the model is given. - assert.ok( + assert.equal( compactNames.includes('get_interactive_elements'), - `[${label}] compact Act must expose upload_file's ambiguity-recovery tool`, + false, + `[${label}] compact upload discovery must not expose the general selector tool`, ); - assert.match(upload.function.description, /get_interactive_elements/); // Assert on the prompt's own upload_file bullet rather than on character // distances, so rewording the neighbouring bullets cannot break these. const uploadLine = prompt.split('\n').find(line => line.startsWith('- upload_file(')); assert.ok(uploadLine, `[${label}] compact prompt must document upload_file`); - assert.match(uploadLine, /do not click the page upload control/i); - assert.match(uploadLine, /exact selector/i); + assert.match(uploadLine, /Two steps/i); + assert.match(uploadLine, /targetId/i); + assert.match(uploadLine, /Never guess a targetId/i); assert.match(uploadLine, /one guarded initializer click/i); - assert.match(uploadLine, /get_interactive_elements/); - assert.ok( + assert.doesNotMatch(uploadLine, /selector|get_interactive_elements/i); + assert.equal( prompt.split('\n').some(line => line.startsWith('- get_interactive_elements')), - `[${label}] compact prompt must document the recovery tool it points at`, + false, + `[${label}] compact prompt must not advertise the removed general selector tool`, ); assert.doesNotMatch(prompt, /download_files|list_downloads|downloadId/i); if (label === 'firefox') { - assert.match(upload.function.description, /WebBrain's file picker/i); + assert.match(upload.function.description, /user-controlled picker/i); + assert.deepEqual(upload.function.parameters.required, []); } else { - assert.doesNotMatch(uploadLine, /filePath/); + assert.deepEqual(upload.function.parameters.required, ['attachmentId']); } } }); -test('every tier exposing upload_file also exposes its ambiguity-recovery tool', () => { - // upload_file latches on an ambiguous selector and only a - // get_interactive_elements response carrying a verified file-input selector - // clears it (Agent._clearUploadSelectorRecoveryAfterInspection). A tier that - // ships upload_file without it can never recover: the handler keeps - // returning recoveryRequired:'get_interactive_elements' and the model has no - // way to satisfy it until the page is replaced. +test('upload targeting is tier-scoped: compact self-discovers while mid/full keep selectors', () => { for (const [label, getTools] of [['chrome', getToolsForModeCh], ['firefox', getToolsForModeFx]]) { - for (const tier of ['compact', 'mid', 'full']) { - const names = new Set(getTools('act', { tier }).map(tool => tool.function.name)); - if (!names.has('upload_file')) continue; - assert.ok( - names.has('get_interactive_elements'), - `[${label}] act/${tier} exposes upload_file without get_interactive_elements, so an ambiguous selector is unrecoverable`, - ); + const compact = getTools('act', { tier: 'compact' }); + const compactNames = new Set(compact.map(tool => tool.function.name)); + assert.equal(compactNames.has('upload_file'), true, `[${label}] compact must keep upload_file`); + assert.equal(compactNames.has('get_interactive_elements'), false, `[${label}] compact must avoid the overlapping inspection tool`); + + for (const tier of ['mid', 'full']) { + const tools = getTools('act', { tier }); + const names = new Set(tools.map(tool => tool.function.name)); + const upload = tools.find(tool => tool.function.name === 'upload_file'); + assert.equal(names.has('upload_file'), true, `[${label}] act/${tier} must keep upload_file`); + assert.equal(names.has('get_interactive_elements'), true, `[${label}] act/${tier} must keep selector recovery`); + assert.ok(upload.function.parameters.properties.selector, `[${label}] act/${tier} lost selector`); + assert.equal(upload.function.parameters.properties.targetId, undefined, `[${label}] act/${tier} gained compact targetId`); + assert.deepEqual(upload.function.parameters.required, ['selector'], `[${label}] act/${tier} selector contract changed`); + } + } +}); + +test('Compact upload discovery uses a hidden file-input-only page action', async () => { + for (const [label, AgentClass, getTools, untrustedTools] of [ + ['chrome', AgentCh, getToolsForModeCh, UNTRUSTED_CONTENT_TOOLS_CH], + ['firefox', AgentFx, getToolsForModeFx, UNTRUSTED_CONTENT_TOOLS], + ]) { + const agent = Object.create(AgentClass.prototype); + let invocation = null; + agent.executeTool = async (tabId, name, args) => { + invocation = { tabId, name, args }; + return [ + { tag: 'input', type: 'file', selector: '#avatar' }, + { tag: 'button', type: 'button', selector: '#submit' }, + ]; + }; + + const inventory = await agent._readCompactUploadFileInputs(42); + assert.deepEqual(invocation, { + tabId: 42, + name: 'get_file_input_targets', + args: {}, + }, `[${label}] Compact discovery must call the narrow internal action`); + assert.equal(inventory.ok, true); + assert.deepEqual(inventory.fileInputs, [{ tag: 'input', type: 'file', selector: '#avatar' }]); + assert.deepEqual(inventory.usable, [{ tag: 'input', type: 'file', selector: '#avatar' }]); + assert.equal(untrustedTools.has('get_file_input_targets'), true); + assert.equal( + agent._toolResultTrustName('upload_file', { discoveryOnly: true }), + 'get_file_input_targets', + `[${label}] page-derived discovery labels must use the untrusted boundary`, + ); + assert.equal( + agent._toolResultTrustName('upload_file', { success: true }), + 'upload_file', + `[${label}] ordinary Mid/Full upload results must retain their existing trust classification`, + ); + + for (const [mode, tiers] of [['ask', ['full']], ['act', ['compact', 'mid', 'full']], ['dev', ['compact', 'mid', 'full']]]) { + for (const tier of tiers) { + const names = getTools(mode, { tier }).map(tool => tool.function.name); + assert.equal( + names.includes('get_file_input_targets'), + false, + `[${label}] ${mode}/${tier} must not expose the internal discovery action`, + ); + } } } }); @@ -50785,7 +50839,7 @@ test('upload_file prefers a valid downloadId and falls back to filePath for an i const agent = new AgentCh({}); const args = { selector: 'input[type=file]', downloadId: 9123, filePath: stalePath }; - const result = await agent.executeTool(42, 'upload_file', args); + const result = await agent.executeTool(42, 'upload_file', args, null, { promptTier: 'mid' }); assert.equal(result.success, true); assert.equal(result.file, realPath); @@ -50904,6 +50958,197 @@ test('upload_file prefers a valid downloadId and falls back to filePath for an i } }); +test('Compact Chrome upload_file discovers opaque targets, rejects hidden full-tier inputs, and attaches on retry', async () => { + const originalCdp = { + attach: cdpClientCh.attach, + querySelectorPierce: cdpClientCh.querySelectorPierce, + releaseObjectGroup: cdpClientCh.releaseObjectGroup, + setFileInputData: cdpClientCh.setFileInputData, + getFileInputFiles: cdpClientCh.getFileInputFiles, + }; + let cdpQueries = 0; + let attachedPayload = null; + try { + cdpClientCh.attach = async () => ({ attached: true }); + cdpClientCh.querySelectorPierce = async (_tabId, selector) => { + cdpQueries++; + assert.equal(selector, '#resume-upload', 'the model-facing targetId must resolve to the internal verified selector'); + return { objectIds: ['input-501'], objectGroup: 'compact-upload-query' }; + }; + cdpClientCh.releaseObjectGroup = async () => {}; + cdpClientCh.setFileInputData = async (_tabId, objectId, payload) => { + assert.equal(objectId, 'input-501'); + attachedPayload = payload; + return { success: true, dispatched: true }; + }; + cdpClientCh.getFileInputFiles = async () => [{ name: 'resume.pdf', size: 42 }]; + + const agent = new AgentCh({}); + agent._currentUrl = async () => 'https://example.com/apply'; + agent._resolveUserAttachment = (_tabId, attachmentId) => ({ + ok: true, + attachmentId, + filename: 'resume.pdf', + mimeType: 'application/pdf', + base64: 'JVBERi0=', + }); + agent._readCompactUploadFileInputs = async () => ({ + ok: true, + fileInputs: [ + { + tag: 'input', + type: 'file', + selector: '#resume-upload', + text: 'Resume', + name: 'resume', + accept: '.pdf', + multiple: false, + inShadowDOM: false, + }, + ], + usable: [ + { + tag: 'input', + type: 'file', + selector: '#resume-upload', + text: 'Resume', + name: 'resume', + accept: '.pdf', + multiple: false, + inShadowDOM: false, + }, + ], + }); + + const denied = await agent.executeTool(42, 'upload_file', { + attachmentId: 'att_1', + selector: '#resume-upload', + }, null, { promptTier: 'compact' }); + assert.equal(denied.denied, true); + assert.equal(denied.noDispatch, true); + assert.match(denied.error, /only attachmentId and a targetId/i); + + const discovery = await agent.executeTool(42, 'upload_file', { + attachmentId: 'att_1', + }, null, { promptTier: 'compact' }); + assert.equal(discovery.success, false); + assert.equal(discovery.discoveryOnly, true); + assert.equal(discovery.requiresTarget, true); + assert.equal(discovery.dispatched, false); + assert.equal(discovery.candidates.length, 1); + assert.equal(discovery.candidates[0].label, 'Resume'); + assert.equal(discovery.candidates[0].accept, '.pdf'); + assert.equal(typeof discovery.candidates[0].targetId, 'string'); + assert.equal('selector' in discovery.candidates[0], false, 'Compact must never expose CSS to the model'); + assert.equal(cdpQueries, 0, 'discovery must not attach or query through the upload mutation path'); + + const result = await agent.executeTool(42, 'upload_file', { + attachmentId: 'att_1', + targetId: discovery.candidates[0].targetId, + }, null, { promptTier: 'compact' }); + assert.equal(result.success, true); + assert.equal(result.file, 'resume.pdf'); + assert.equal(result.attachmentState, 'input_attached'); + assert.equal(cdpQueries, 1); + assert.equal(attachedPayload.filename, 'resume.pdf'); + assert.equal(agent._compactUploadTargets.has(42), false, 'targetId must be one-use'); + + const stale = await agent.executeTool(42, 'upload_file', { + attachmentId: 'att_1', + targetId: discovery.candidates[0].targetId, + }, null, { promptTier: 'compact' }); + assert.equal(stale.discoveryOnly, true); + assert.match(stale.error, /missing, expired/i); + assert.equal(cdpQueries, 1, 'a stale targetId must fail before attachment'); + + agent._clearPageLoopState(42); + assert.equal(agent._compactUploadTargets.has(42), false, 'navigation cleanup must clear compact targets'); + } finally { + Object.assign(cdpClientCh, originalCdp); + } +}); + +test('Compact Firefox upload_file discovers before opening its picker and attaches only to a returned targetId', async () => { + const originalBrowser = globalThis.browser; + const scripts = []; + let pickerEvent = null; + try { + globalThis.browser = { + tabs: { + async executeScript(_tabId, details) { + scripts.push(details.code); + if (details.code.includes('WebBrain file attachment settle probe')) { + return [{ attachmentState: 'input_attached' }]; + } + return [{ success: true, dispatched: true, file: 'resume.pdf', size: 4, attachmentState: 'input_attached' }]; + }, + }, + }; + + const candidate = { + tag: 'input', + type: 'file', + selector: '#resume-upload', + text: 'Resume', + name: 'resume', + accept: '.pdf', + multiple: false, + inShadowDOM: false, + }; + const agent = new AgentFx({}); + agent._currentUrl = async () => 'https://example.com/apply'; + agent._readCompactUploadFileInputs = async () => ({ + ok: true, + fileInputs: [candidate], + usable: [candidate], + }); + + const denied = await agent.executeTool(42, 'upload_file', { + selector: '#resume-upload', + }, null, { promptTier: 'compact' }); + assert.equal(denied.denied, true); + assert.equal(denied.noDispatch, true); + + const discovery = await agent.executeTool( + 42, + 'upload_file', + {}, + (evt, data) => { if (evt === 'upload_picker') pickerEvent = data; }, + { promptTier: 'compact' }, + ); + assert.equal(discovery.discoveryOnly, true); + assert.equal(discovery.candidates.length, 1); + assert.equal(pickerEvent, null, 'read-only discovery must not open Firefox\'s picker'); + assert.equal(scripts.length, 0, 'discovery must not inject attachment code'); + + const uploadPromise = agent.executeTool( + 42, + 'upload_file', + { targetId: discovery.candidates[0].targetId }, + (evt, data) => { if (evt === 'upload_picker') pickerEvent = data; }, + { promptTier: 'compact' }, + ); + await new Promise(resolve => setTimeout(resolve, 10)); + assert.ok(pickerEvent?.pickerId, 'the picker should open only after a valid targetId is selected'); + agent.submitUploadPickerResponse(42, pickerEvent.pickerId, { + base64: 'JVBERg==', + name: 'resume.pdf', + type: 'application/pdf', + size: 4, + }); + const result = await uploadPromise; + assert.equal(result.success, true); + assert.equal(result.file, 'resume.pdf'); + assert.equal(result.attachmentState, 'input_attached'); + assert.equal(scripts.length, 2); + assert.match(scripts[0], /const selector = "#resume-upload"/); + assert.equal(agent._compactUploadTargets.has(42), false, 'targetId must be one-use'); + } finally { + if (originalBrowser === undefined) delete globalThis.browser; + else globalThis.browser = originalBrowser; + } +}); + test('upload_file schema accepts downloadId and no longer hard-requires filePath (firefox)', () => { const tools = getToolsForModeFx('act', {}); const up = tools.find(t => t.function?.name === 'upload_file'); @@ -50966,7 +51211,7 @@ test('Firefox upload_file injects the exact user attachment bytes without re-fet const result = await agent.executeTool(42, 'upload_file', { selector: 'input[type=file]', attachmentId, - }); + }, null, { promptTier: 'mid' }); assert.equal(result.success, true); assert.equal(result.attachmentId, attachmentId); From 1eec0528f2656a41db45eabe9280885fcc45ec88 Mon Sep 17 00:00:00 2001 From: Barack Sokullu Date: Thu, 6 Aug 2026 12:40:37 +0300 Subject: [PATCH 5/5] test(compact): enforce upload helper parity --- src/chrome/src/agent/agent.js | 4 +++- src/firefox/src/agent/agent.js | 4 +++- test/run.js | 28 ++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index e4f589358..31db1ab1c 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -1747,6 +1747,7 @@ export class Agent extends LoopDetector { return true; } + // COMPACT_UPLOAD_TARGET_HELPERS_START _toolResultTrustName(name, result) { return name === 'upload_file' && result?.discoveryOnly ? 'get_file_input_targets' @@ -1852,7 +1853,7 @@ export class Agent extends LoopDetector { addressableCount, candidates, truncated: addressableCount > candidates.length, - error: `${prefix}Choose the intended file input from candidates, then retry upload_file with the same attachmentId and that exact targetId. Never guess or modify a targetId.`, + error: `${prefix}Choose the intended file input from candidates, then retry upload_file with that exact targetId and the same attachmentId if one was provided. Never guess or modify a targetId.`, }; } @@ -1902,6 +1903,7 @@ export class Agent extends LoopDetector { this._compactUploadTargets.delete(tabId); return { ok: true, selector: saved.selector }; } + // COMPACT_UPLOAD_TARGET_HELPERS_END _rememberAxScope(tabId, documentToken, pageUrl = '') { const next = { diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index 5713caf9c..ba01ebd85 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -1821,6 +1821,7 @@ export class Agent extends LoopDetector { return true; } + // COMPACT_UPLOAD_TARGET_HELPERS_START _toolResultTrustName(name, result) { return name === 'upload_file' && result?.discoveryOnly ? 'get_file_input_targets' @@ -1926,7 +1927,7 @@ export class Agent extends LoopDetector { addressableCount, candidates, truncated: addressableCount > candidates.length, - error: `${prefix}Choose the intended file input from candidates, then retry upload_file with that exact targetId and the current attachmentId, or omit attachmentId on that second call for WebBrain's picker. Never guess or modify a targetId.`, + error: `${prefix}Choose the intended file input from candidates, then retry upload_file with that exact targetId and the same attachmentId if one was provided. Never guess or modify a targetId.`, }; } @@ -1976,6 +1977,7 @@ export class Agent extends LoopDetector { this._compactUploadTargets.delete(tabId); return { ok: true, selector: saved.selector }; } + // COMPACT_UPLOAD_TARGET_HELPERS_END _rememberAxScope(tabId, documentToken, pageUrl = '') { const next = { diff --git a/test/run.js b/test/run.js index 024346a07..a6e6f4ea9 100644 --- a/test/run.js +++ b/test/run.js @@ -12996,6 +12996,34 @@ test('Compact upload discovery uses a hidden file-input-only page action', async } }); +test('Compact upload target helpers stay byte-identical across Chrome and Firefox', () => { + const startMarker = ' // COMPACT_UPLOAD_TARGET_HELPERS_START'; + const endMarker = ' // COMPACT_UPLOAD_TARGET_HELPERS_END'; + const helperBlock = (browser) => { + const source = fs.readFileSync(path.join(ROOT, `src/${browser}/src/agent/agent.js`), 'utf8'); + const start = source.indexOf(startMarker); + const end = source.indexOf(endMarker, start + startMarker.length); + assert.ok(start >= 0 && end > start, `${browser}: Compact upload helper parity markers are missing or reversed`); + assert.equal( + source.indexOf(startMarker, start + startMarker.length), + -1, + `${browser}: Compact upload helper start marker must be unique`, + ); + assert.equal( + source.indexOf(endMarker, end + endMarker.length), + -1, + `${browser}: Compact upload helper end marker must be unique`, + ); + return source.slice(start, end + endMarker.length); + }; + + assert.equal( + helperBlock('chrome'), + helperBlock('firefox'), + 'Chrome and Firefox Compact upload target helpers must remain byte-identical', + ); +}); + test('getToolsForMode: mode/tier redesign exposes the intended normal and Dev tools', () => { for (const [label, getTools] of [ ['chrome', getToolsForModeCh],