Skip to content

Commit c647170

Browse files
bobleerbowen628
andcommitted
feat: add Browser scene, web-element context and relay server improvements (GCWing#163)
- Add embedded Browser scene with Tauri webview integration and inspector - Introduce WebElementContext type for referencing web page elements in chat - Improve ChatInput focus management and context tag insertion reliability - Harden relay server: bind to 127.0.0.1, redesign landing page, fix deps - Add browser-related Tauri capabilities and API commands Co-authored-by: bowen628 <bowen628@noreply.gitcode.com>
1 parent aef9095 commit c647170

43 files changed

Lines changed: 2243 additions & 96 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ similar = "2.5"
9696
urlencoding = "2.1"
9797

9898
# Tauri (desktop only)
99-
tauri = { version = "2", features = [] }
99+
tauri = { version = "2", features = ["unstable"] }
100100
tauri-plugin-opener = "2"
101101
tauri-plugin-dialog = "2.6"
102102
tauri-plugin-fs = "2"
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"$schema": "../gen/schemas/desktop-schema.json",
3+
"identifier": "browser-webview",
4+
"description": "Minimal permissions for embedded browser webviews to emit events back to the host",
5+
"webviews": ["embedded-browser-*"],
6+
"local": true,
7+
"remote": {
8+
"urls": ["https://*", "http://*"]
9+
},
10+
"permissions": [
11+
"core:event:allow-emit"
12+
]
13+
}

src/apps/desktop/capabilities/default.json

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,15 @@
1111
"core:event:allow-listen",
1212
"core:event:allow-emit",
1313
"core:window:default",
14+
"core:webview:default",
15+
"core:webview:allow-create-webview",
16+
"core:webview:allow-set-webview-position",
17+
"core:webview:allow-set-webview-size",
18+
"core:webview:allow-set-webview-focus",
19+
"core:webview:allow-reparent",
20+
"core:webview:allow-webview-show",
21+
"core:webview:allow-webview-hide",
22+
"core:webview:allow-webview-close",
1423
"core:window:allow-create",
1524
"core:window:allow-set-focus",
1625
"core:window:allow-set-always-on-top",
Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
//! Browser API — commands for the embedded browser feature.
2+
3+
use serde::Deserialize;
4+
use tauri::Manager;
5+
6+
#[derive(Debug, Deserialize)]
7+
#[serde(rename_all = "camelCase")]
8+
pub struct WebviewEvalRequest {
9+
pub label: String,
10+
pub script: String,
11+
}
12+
13+
#[tauri::command]
14+
pub async fn browser_webview_eval(
15+
app: tauri::AppHandle,
16+
request: WebviewEvalRequest,
17+
) -> Result<(), String> {
18+
let webview = app
19+
.get_webview(&request.label)
20+
.ok_or_else(|| format!("Webview not found: {}", request.label))?;
21+
22+
webview
23+
.eval(&request.script)
24+
.map_err(|e| format!("eval failed: {e}"))
25+
}
26+
27+
// #region agent log
28+
#[derive(Debug, Deserialize)]
29+
#[serde(rename_all = "camelCase")]
30+
pub struct WebviewLabelRequest {
31+
pub label: String,
32+
}
33+
34+
/// Pull debug logs from the injected inspector script.
35+
/// The script stores logs in `window.__bitfun_debug_logs`.
36+
/// We eval a script that writes them into URL hash, then read the URL.
37+
#[tauri::command]
38+
pub async fn browser_pull_debug_logs(
39+
app: tauri::AppHandle,
40+
request: WebviewLabelRequest,
41+
) -> Result<String, String> {
42+
let webview = app
43+
.get_webview(&request.label)
44+
.ok_or_else(|| format!("Webview not found: {}", request.label))?;
45+
46+
webview
47+
.eval(
48+
r#"(function(){
49+
var logs = window.__bitfun_debug_logs || [];
50+
window.__bitfun_debug_logs = [];
51+
try {
52+
var hash = '__BFDEBUG__' + encodeURIComponent(JSON.stringify(logs));
53+
history.replaceState(null, '', '#' + hash);
54+
} catch(e) {
55+
history.replaceState(null, '', '#__BFDEBUG__ERR_' + encodeURIComponent(String(e)));
56+
}
57+
})()"#,
58+
)
59+
.map_err(|e| format!("eval failed: {e}"))?;
60+
61+
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
62+
63+
let url = webview.url().map_err(|e| format!("url failed: {e}"))?;
64+
let fragment = url.fragment().unwrap_or("");
65+
66+
if fragment.starts_with("__BFDEBUG__") {
67+
let encoded = &fragment[11..];
68+
let decoded = urlencoding::decode(encoded)
69+
.map(|s| s.into_owned())
70+
.unwrap_or_else(|_| format!("decode_error:{}", encoded));
71+
72+
webview
73+
.eval("try { history.replaceState(null, '', location.pathname + location.search); } catch(e) {}")
74+
.ok();
75+
76+
Ok(decoded)
77+
} else {
78+
Ok(format!("no_debug_marker_in_fragment:{}", &fragment.chars().take(100).collect::<String>()))
79+
}
80+
}
81+
// #endregion

src/apps/desktop/src/api/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
33
pub mod agentic_api;
44
pub mod ai_memory_api;
5+
pub mod browser_api;
56
pub mod ai_rules_api;
67
pub mod app_state;
78
pub mod btw_api;

src/apps/desktop/src/lib.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -633,6 +633,9 @@ pub async fn run() {
633633
api::miniapp_api::miniapp_dialog_message,
634634
api::miniapp_api::miniapp_import_from_path,
635635
api::miniapp_api::miniapp_sync_from_fs,
636+
// Browser API
637+
api::browser_api::browser_webview_eval,
638+
api::browser_api::browser_pull_debug_logs,
636639
])
637640
.run(tauri::generate_context!());
638641
if let Err(e) = run_result {

src/apps/relay-server/Cargo.toml

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,9 @@ tracing = "0.1"
3434
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
3535

3636
# Utilities
37-
chrono = { workspace = true }
38-
dashmap = { workspace = true }
39-
rand = { workspace = true }
40-
base64 = { workspace = true }
41-
sha2 = { workspace = true }
37+
uuid = { version = "1.0", features = ["v4", "serde"] }
38+
chrono = { version = "0.4", features = ["serde", "clock"] }
39+
dashmap = "5.5"
40+
rand = "0.8"
41+
base64 = "0.21"
42+
sha2 = "0.10"

src/apps/relay-server/docker-compose.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ services:
66
container_name: bitfun-relay
77
restart: unless-stopped
88
ports:
9-
- "9700:9700"
9+
- "${RELAY_HOST_BIND_IP:-0.0.0.0}:9700:9700"
1010
environment:
1111
- RELAY_PORT=9700
1212
- RELAY_STATIC_DIR=/app/static

src/apps/relay-server/restart.sh

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ set -euo pipefail
66

77
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
88
CONTAINER_NAME="bitfun-relay"
9+
RELAY_HOST_BIND_IP="127.0.0.1"
910

1011
usage() {
1112
cat <<'EOF'
@@ -65,13 +66,14 @@ cd "$SCRIPT_DIR"
6566

6667
if container_running; then
6768
echo "Relay service is running. Restarting it..."
68-
docker compose restart
69+
RELAY_HOST_BIND_IP="$RELAY_HOST_BIND_IP" docker compose up -d --force-recreate
6970
else
7071
echo "Relay service is not running. Starting it instead..."
71-
docker compose up -d
72+
RELAY_HOST_BIND_IP="$RELAY_HOST_BIND_IP" docker compose up -d
7273
fi
7374

7475
echo ""
7576
echo "Relay service is ready."
77+
echo "Relay endpoint: http://127.0.0.1:9700"
7678
echo "Check status: docker compose ps"
7779
echo "View logs: docker compose logs -f relay-server"

src/apps/relay-server/start.sh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ set -euo pipefail
66

77
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
88
CONTAINER_NAME="bitfun-relay"
9+
RELAY_HOST_BIND_IP="127.0.0.1"
910

1011
usage() {
1112
cat <<'EOF'
@@ -77,9 +78,10 @@ else
7778
echo "Relay service is not created yet. Creating and starting it..."
7879
fi
7980

80-
docker compose up -d
81+
RELAY_HOST_BIND_IP="$RELAY_HOST_BIND_IP" docker compose up -d
8182

8283
echo ""
8384
echo "Relay service started."
85+
echo "Relay endpoint: http://127.0.0.1:9700"
8486
echo "Check status: docker compose ps"
8587
echo "View logs: docker compose logs -f relay-server"

0 commit comments

Comments
 (0)