Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@
import inspect
import json

from agentplatform.agent_engines.templates.adk import AdkApp
from fastapi import FastAPI, HTTPException, Request, encoders, responses
from vertexai.agent_engines.templates.adk import AdkApp
Comment thread
anaelledrai marked this conversation as resolved.
Resolved

from app.app_utils import services

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,51 +14,12 @@

"""Defines visual Search Results Agent powered by Gemini Computer Use."""

import copy
from typing import Any

from google.adk.agents import LlmAgent
from google.adk.models.llm_request import LlmRequest

from ...shared_libraries import constants
from ...tools.browser_computer import get_computer_use_toolset
from . import prompt

_TOOL_NAME_ALIASES: list[tuple[str, tuple[str, ...]]] = [
(
"click_at",
(
"click",
"double_click",
"triple_click",
"right_click",
"middle_click",
),
),
("type_text_at", ("type",)),
("hover_at", ("move", "mouse_down", "mouse_up")),
("scroll_at", ("scroll",)),
("current_state", ("take_screenshot",)),
("key_combination", ("press_key", "hotkey", "key_down", "key_up")),
]


async def adapt_computer_use_tools_callback(
callback_context: Any,
llm_request: LlmRequest,
) -> None:
"""Registers modern Gemini 3.x action names as renamed copies of legacy tools."""
for method_name, aliases in _TOOL_NAME_ALIASES:
tool = llm_request.tools_dict.get(method_name)
if tool is None:
continue
for alias in aliases:
if alias not in llm_request.tools_dict:
aliased = copy.copy(tool)
aliased.name = alias
llm_request.tools_dict[alias] = aliased


search_results_agent = LlmAgent(
model=constants.MODEL,
name="search_results_agent",
Expand All @@ -67,6 +28,5 @@ async def adapt_computer_use_tools_callback(
tools=[
get_computer_use_toolset(),
],
before_model_callback=adapt_computer_use_tools_callback,
output_key="observed_search_results",
)
Original file line number Diff line number Diff line change
Expand Up @@ -243,17 +243,30 @@ async def open_web_browser(self) -> ComputerState:
self._url = _format_url(self._url or DEFAULT_INITIAL_SEARCH_URL)
return await self.current_state()

async def click_at(self, x: int = 0, y: int = 0) -> ComputerState:
async def click_at(self, x: int, y: int) -> ComputerState:
return await self.current_state()

async def hover_at(self, x: int = 0, y: int = 0) -> ComputerState:
async def hover_at(self, x: int, y: int) -> ComputerState:
return await self.current_state()

async def type_text(
self,
text: str,
press_enter: bool = False,
) -> ComputerState:
return await self.type_text_at(
x=0,
y=0,
text=text,
press_enter=press_enter,
clear_before_typing=False,
)

async def type_text_at(
self,
text: str = "",
x: int | None = None,
y: int | None = None,
x: int,
y: int,
text: str,
press_enter: bool = True,
clear_before_typing: bool = True,
) -> ComputerState:
Expand All @@ -264,20 +277,20 @@ async def type_text_at(
return await self.current_state()

async def scroll_document(
self, direction: Literal["up", "down", "left", "right"] = "down"
self, direction: Literal["up", "down", "left", "right"]
) -> ComputerState:
return await self.current_state()

async def scroll_at(
self,
direction: Literal["up", "down", "left", "right"] = "down",
x: int | None = None,
y: int | None = None,
x: int,
y: int,
direction: Literal["up", "down", "left", "right"],
magnitude: int = DEFAULT_SCROLL_OFFSET,
) -> ComputerState:
return await self.current_state()

async def wait(self, seconds: int = 1) -> ComputerState:
async def wait(self, seconds: int) -> ComputerState:
return await self.current_state()

async def go_back(self) -> ComputerState:
Expand Down Expand Up @@ -312,17 +325,11 @@ async def navigate(self, url: str) -> ComputerState:
self._visit(formatted_url)
return await self.current_state()

async def key_combination(
self, keys: list[str] | None = None, key: str | None = None
) -> ComputerState:
async def key_combination(self, keys: list[str]) -> ComputerState:
return await self.current_state()

async def drag_and_drop(
self,
x: int = 0,
y: int = 0,
destination_x: int = 0,
destination_y: int = 0,
self, x: int, y: int, destination_x: int, destination_y: int
) -> ComputerState:
return await self.current_state()

Expand Down Expand Up @@ -405,7 +412,7 @@ async def open_web_browser(self) -> ComputerState:
await self._page.goto(target_url, wait_until="domcontentloaded")
return await self.current_state()

async def click_at(self, x: int = 0, y: int = 0) -> ComputerState:
async def click_at(self, x: int, y: int) -> ComputerState:
await self._ensure_browser()
if self._page:
await self._page.mouse.click(x, y)
Expand All @@ -417,27 +424,19 @@ async def click_at(self, x: int = 0, y: int = 0) -> ComputerState:
pass
return await self.current_state()

async def hover_at(self, x: int = 0, y: int = 0) -> ComputerState:
async def hover_at(self, x: int, y: int) -> ComputerState:
await self._ensure_browser()
if self._page:
await self._page.mouse.move(x, y)
return await self.current_state()

async def type_text_at(
async def type_text(
Comment thread
anaelledrai marked this conversation as resolved.
Resolved
self,
text: str = "",
x: int | None = None,
y: int | None = None,
press_enter: bool = True,
clear_before_typing: bool = True,
text: str,
press_enter: bool = False,
) -> ComputerState:
await self._ensure_browser()
if self._page:
if x is not None and y is not None:
await self._page.mouse.click(x, y)
if clear_before_typing:
await self._page.keyboard.press("Control+A")
await self._page.keyboard.press("Backspace")
await self._page.keyboard.type(text)
if press_enter:
await self._page.keyboard.press("Enter")
Expand All @@ -449,8 +448,24 @@ async def type_text_at(
pass
return await self.current_state()

async def type_text_at(
self,
x: int,
y: int,
text: str,
press_enter: bool = True,
clear_before_typing: bool = True,
) -> ComputerState:
await self._ensure_browser()
if self._page:
await self._page.mouse.click(x, y)
if clear_before_typing:
await self._page.keyboard.press("Control+A")
await self._page.keyboard.press("Backspace")
return await self.type_text(text=text, press_enter=press_enter)

async def scroll_document(
self, direction: Literal["up", "down", "left", "right"] = "down"
self, direction: Literal["up", "down", "left", "right"]
) -> ComputerState:
await self._ensure_browser()
if self._page:
Expand All @@ -463,21 +478,20 @@ async def scroll_document(

async def scroll_at(
self,
direction: Literal["up", "down", "left", "right"] = "down",
x: int | None = None,
y: int | None = None,
x: int,
y: int,
direction: Literal["up", "down", "left", "right"],
magnitude: int = DEFAULT_SCROLL_OFFSET,
) -> ComputerState:
await self._ensure_browser()
if self._page:
if x is not None and y is not None:
await self._page.mouse.move(x, y)
await self._page.mouse.move(x, y)
delta_x, delta_y = _calculate_scroll_deltas(direction, magnitude)
await self._page.mouse.wheel(delta_x, delta_y)
await asyncio.sleep(POST_SCROLL_WAIT_SECONDS)
return await self.current_state()

async def wait(self, seconds: int = 1) -> ComputerState:
async def wait(self, seconds: int) -> ComputerState:
await self._ensure_browser()
await asyncio.sleep(min(seconds, MAX_WAIT_SECONDS))
return await self.current_state()
Expand Down Expand Up @@ -532,21 +546,14 @@ async def navigate(self, url: str) -> ComputerState:
await self._page.goto(formatted_url, wait_until="domcontentloaded")
return await self.current_state()

async def key_combination(
self, keys: list[str] | None = None, key: str | None = None
) -> ComputerState:
async def key_combination(self, keys: list[str]) -> ComputerState:
await self._ensure_browser()
combo = keys or ([key] if key else [])
if self._page and combo:
await self._page.keyboard.press("+".join(combo))
if self._page and keys:
await self._page.keyboard.press("+".join(keys))
return await self.current_state()

async def drag_and_drop(
self,
x: int = 0,
y: int = 0,
destination_x: int = 0,
destination_y: int = 0,
self, x: int, y: int, destination_x: int, destination_y: int
) -> ComputerState:
await self._ensure_browser()
if self._page:
Expand Down
22 changes: 18 additions & 4 deletions contrib/python/brand-search-optimization/deployment/deploy.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,12 @@

"""Deployment script for Brand Search Optimization agent."""

import os

import vertexai
from absl import app, flags
from vertexai import agent_engines
from vertexai.preview.reasoning_engines import AdkApp
from vertexai.agent_engines.templates.adk import AdkApp

from app.agent import root_agent
from app.shared_libraries import constants
Expand All @@ -43,8 +45,8 @@ def create(env_vars: dict) -> None:
remote_agent = agent_engines.create(
adk_app,
requirements=[
"google-adk>=2.6.0,<3.0.0",
"google-cloud-aiplatform[agent_engines]>=1.93.0",
"google-adk>=2.13.0,<3.0.0",
Comment thread
anaelledrai marked this conversation as resolved.
Resolved
"google-cloud-aiplatform[agent_engines]>=1.156.0,<3.0.0",
"pydantic>=2.10.0",
"requests",
"python-dotenv",
Expand All @@ -68,7 +70,13 @@ def delete(resource_id: str) -> None:

def main(argv: list[str]) -> None:
project_id = FLAGS.project_id if FLAGS.project_id else constants.PROJECT
location = FLAGS.location if FLAGS.location else constants.LOCATION
# Reasoning Engine requires a regional location (e.g., us-central1) and
# rejects 'global' as lightweight-only.
location = (
FLAGS.location
or os.getenv("GOOGLE_CLOUD_AGENT_ENGINE_LOCATION")
or "us-central1"
)
bucket = FLAGS.bucket if FLAGS.bucket else constants.STAGING_BUCKET
env_vars = {}

Expand All @@ -89,6 +97,12 @@ def main(argv: list[str]) -> None:
env_vars["DISABLE_WEB_DRIVER"] = (
"1" if constants.DISABLE_WEB_DRIVER else "0"
)
if constants.MODEL:
env_vars["MODEL"] = constants.MODEL
# Forward model endpoint location (e.g. 'global' for Gemini 3.5 models)
# to the remote Reasoning Engine instance.
if constants.LOCATION:
env_vars["GOOGLE_CLOUD_LOCATION"] = constants.LOCATION

vertexai.init(
project=project_id,
Expand Down
7 changes: 4 additions & 3 deletions contrib/python/brand-search-optimization/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ readme = "README.md"
requires-python = ">=3.11,<3.13"

dependencies = [
"google-adk[gcp,otel-gcp]<3.0.0,>=2.6.0",
"google-adk[gcp,otel-gcp]<3.0.0,>=2.13.0",
"google-cloud-bigquery>=3.25.0",
"google-genai>=1.9.0",
"pillow>=11.0.0",
Expand All @@ -20,13 +20,13 @@ dependencies = [
"gcsfs>=2024.11.0",
"opentelemetry-resourcedetector-gcp<=1.12.0a0",
"joserfc>=1.6.8",
"google-cloud-aiplatform[agent-engines]>=1.156.0,<2.0.0",
"google-cloud-aiplatform[agent-engines]>=1.156.0,<3.0.0",
]

[dependency-groups]
dev = [
"agent-starter-pack>=0.32.0",
"google-adk[eval]>=2.6.0",
"google-adk[eval]>=2.13.0",
"mypy>=1.17.0",
"pytest>=8.3.5",
"pytest-asyncio>=0.26.0",
Expand All @@ -46,6 +46,7 @@ exclude = ["^deployment/", "^\\.venv/", "__pycache__"]

[tool.pytest.ini_options]
pythonpath = "."
testpaths = ["tests"]

[tool.hatch.build.targets.wheel]
packages = ["app"]
Expand Down
Loading
Loading