From 5e6c36faac04264303b1375235a26095fa8e8c06 Mon Sep 17 00:00:00 2001 From: lorenzomicheli Date: Mon, 5 Oct 2026 08:20:18 +0000 Subject: [PATCH 1/6] android: expose GPUI's accessibility tree to TalkBack GPUI 0.3.7 builds an AccessKit tree and hands it to PlatformWindow::a11y_init / a11y_tree_update, which the Android window left as no-ops. Wire them to accesskit_android's InjectingAdapter, which installs an AccessibilityDelegate on the existing host View, so hosts need no Java changes. accesskit_android 0.7.5 is the newest release on accesskit 0.24 (the version gpui-pre 0.3.7 uses). It raises accessibility events without checking AccessibilityManager.isEnabled(), and Android throws on the UI thread when no service is listening, so updates are skipped while accessibility is off (0.9 has this check but needs accesskit 0.25). Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.toml | 3 + src/android/accessibility.rs | 191 +++++++++++++++++++++++++++++++++++ src/android/jni.rs | 2 + src/android/mod.rs | 1 + src/android/window.rs | 8 ++ 5 files changed, 205 insertions(+) create mode 100644 src/android/accessibility.rs diff --git a/Cargo.toml b/Cargo.toml index 10f46d8..d159823 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,6 +119,9 @@ android_logger = "0.15" # Android uses Cosmic Text and system fonts; font-kit pulls in desktop Fontconfig. gpui_wgpu = { package = "gpui-pre-wgpu", version = "=0.3.8" } +# TalkBack: exposes GPUI's AccessKit tree (accesskit 0.24, as gpui-pre). +accesskit_android = { version = "0.7.5", features = ["embedded-dex"] } + # rustc-hash is required by the PlatformKeyboardMapper trait signature rustc-hash = "2" uuid = { version = "1", features = ["v5"] } diff --git a/src/android/accessibility.rs b/src/android/accessibility.rs new file mode 100644 index 0000000..dbbd00f --- /dev/null +++ b/src/android/accessibility.rs @@ -0,0 +1,191 @@ +//! Screen-reader support: hands GPUI's AccessKit tree to TalkBack. +//! +//! GPUI builds an AccessKit tree every frame once a screen reader is active and passes +//! it to `PlatformWindow::a11y_tree_update`. `accesskit_android`'s [`InjectingAdapter`] +//! exposes that tree as virtual `AccessibilityNodeInfo`s by installing an +//! `AccessibilityDelegate` on an existing `View`, so hosts need no Java changes. +//! +//! Node bounds are in surface pixels, so the `View` must line up with GPUI's surface: +//! the decor view on the `android-activity` path (the native surface fills the window) +//! and the content view on the host-driven path (it holds the `SurfaceView`). +//! +//! There is one GPUI window per Activity, so the state is global. A recreated Activity +//! has new views; [`super::jni::set_host_activity`] moves the adapter onto them. +//! +//! Android has no "screen reader stopped" callback for this adapter, so once TalkBack +//! has been used GPUI keeps building the tree until the process ends. Updates are +//! dropped while accessibility is off: this adapter version raises events without +//! checking, and Android throws on the UI thread when nobody is listening +//! (`accesskit_android` 0.9 checks, but needs `accesskit` 0.25). + +use std::sync::{Arc, Mutex}; + +use accesskit_android::{ + jni::{ + objects::{GlobalRef, JObject}, + JNIEnv, JavaVM, + }, + InjectingAdapter, +}; +use gpui::{ + accesskit::{ActionHandler, ActionRequest, ActivationHandler, TreeUpdate}, + A11yCallbacks, +}; + +static BRIDGE: Mutex> = Mutex::new(None); + +/// `android.R.id.content`. +const CONTENT_VIEW_ID: i32 = 0x0102_0002; + +struct Bridge { + callbacks: Handlers, + adapter: Option, +} + +struct Adapter { + inner: InjectingAdapter, + vm: JavaVM, + /// `android.view.accessibility.AccessibilityManager`. + manager: GlobalRef, +} + +// SAFETY: `InjectingAdapter` holds a `JavaVM`, global/weak references and a +// `&'static` global class reference, all valid on any thread; its methods attach to +// the JVM and post UI work to the host `View` themselves. +unsafe impl Send for Adapter {} + +/// GPUI's callbacks, shared by every adapter a recreated Activity gets. +#[derive(Clone)] +struct Handlers(Arc>); + +impl ActivationHandler for Handlers { + fn request_initial_tree(&mut self) -> Option { + (self.0.lock().expect("poisoned").activation)() + } +} + +impl ActionHandler for Handlers { + fn do_action(&mut self, request: ActionRequest) { + (self.0.lock().expect("poisoned").action)(request) + } +} + +/// `PlatformWindow::a11y_init`, on the GPUI thread. +pub(crate) fn init(callbacks: A11yCallbacks) { + let mut bridge = BRIDGE.lock().expect("poisoned"); + bridge + .insert(Bridge { + callbacks: Handlers(Arc::new(Mutex::new(callbacks))), + adapter: None, + }) + .attach(); +} + +/// `PlatformWindow::a11y_tree_update`, on the GPUI thread that called [`init`]. +pub(crate) fn update(tree: TreeUpdate) { + let mut bridge = BRIDGE.lock().expect("poisoned"); + let Some(adapter) = bridge.as_mut().and_then(|b| b.adapter.as_mut()) else { + return; + }; + if adapter.is_accessibility_enabled() { + adapter.inner.update_if_active(|| tree); + } +} + +/// A new Activity was registered; move the adapter onto its views. +pub(crate) fn activity_changed() { + if let Some(bridge) = BRIDGE.lock().expect("poisoned").as_mut() { + bridge.attach(); + } +} + +impl Bridge { + fn attach(&mut self) { + // Dropping the old adapter removes its delegate from the old view. + self.adapter = None; + let callbacks = self.callbacks.clone(); + match with_host_view(|env, view| { + let manager = accessibility_manager(env, view)?; + Ok(Adapter { + inner: InjectingAdapter::new(env, view, callbacks.clone(), callbacks), + vm: env.get_java_vm()?, + manager, + }) + }) { + Ok(adapter) => self.adapter = Some(adapter), + Err(err) => log::warn!("accessibility: cannot attach to the host view: {err}"), + } + } +} + +impl Adapter { + fn is_accessibility_enabled(&self) -> bool { + let enabled = self.vm.get_env().and_then(|mut env| { + env.call_method(&self.manager, "isEnabled", "()Z", &[])?.z() + }); + enabled.unwrap_or_else(|err| { + log::warn!("accessibility: AccessibilityManager.isEnabled failed: {err}"); + false + }) + } +} + +fn accessibility_manager( + env: &mut JNIEnv, + view: &JObject, +) -> accesskit_android::jni::errors::Result { + let context = env + .call_method(view, "getContext", "()Landroid/content/Context;", &[])? + .l()?; + let name = env.new_string("accessibility")?; + let manager = env + .call_method( + &context, + "getSystemService", + "(Ljava/lang/String;)Ljava/lang/Object;", + &[(&name).into()], + )? + .l()?; + env.new_global_ref(manager) +} + +fn with_host_view( + f: impl FnOnce(&mut JNIEnv, &JObject) -> accesskit_android::jni::errors::Result, +) -> Result { + let vm = super::jni::java_vm(); + let activity = super::jni::activity_as_ptr(); + if vm.is_null() || activity.is_null() { + return Err("JavaVM or Activity not available".into()); + } + // SAFETY: both pointers come from the running JVM; the Activity is a global + // reference that `super::jni` keeps alive. + let vm = unsafe { JavaVM::from_raw(vm.cast()) }.map_err(|e| e.to_string())?; + let activity = unsafe { JObject::from_raw(activity.cast()) }; + // `InjectingAdapter::update_if_active` expects the calling thread to stay attached. + let mut env = vm + .attach_current_thread_permanently() + .map_err(|e| e.to_string())?; + env.with_local_frame(8, |env| { + let window = env + .call_method(&activity, "getWindow", "()Landroid/view/Window;", &[])? + .l()?; + let mut view = env + .call_method(&window, "getDecorView", "()Landroid/view/View;", &[])? + .l()?; + if super::jni::android_app().is_none() { + view = env + .call_method( + &view, + "findViewById", + "(I)Landroid/view/View;", + &[CONTENT_VIEW_ID.into()], + )? + .l()?; + } + if view.is_null() { + return Ok(Err("the Activity has no content view".into())); + } + Ok::<_, accesskit_android::jni::errors::Error>(Ok(f(env, &view)?)) + }) + .map_err(|e| e.to_string())? +} diff --git a/src/android/jni.rs b/src/android/jni.rs index c190e49..8a717f4 100644 --- a/src/android/jni.rs +++ b/src/android/jni.rs @@ -317,6 +317,8 @@ pub fn set_host_activity(env: &mut jni::Env<'_>, activity: &JObject<'_>) -> Resu let mut slot = HOST_ACTIVITY.lock().expect("poisoned"); slot.previous = slot.current.take(); slot.current = Some(global); + drop(slot); + super::accessibility::activity_changed(); Ok(()) } diff --git a/src/android/mod.rs b/src/android/mod.rs index 1811cc4..eefff4d 100644 --- a/src/android/mod.rs +++ b/src/android/mod.rs @@ -149,6 +149,7 @@ pub fn size(width: T, height: T) -> Size { // ── sub-modules ────────────────────────────────────────────────────────────── +mod accessibility; pub mod dispatcher; pub mod display; mod document_picker; diff --git a/src/android/window.rs b/src/android/window.rs index efdcee0..6b343da 100644 --- a/src/android/window.rs +++ b/src/android/window.rs @@ -1547,6 +1547,14 @@ impl PlatformWindow for AndroidPlatformWindow { } } + fn a11y_init(&self, callbacks: gpui::A11yCallbacks) { + super::accessibility::init(callbacks); + } + + fn a11y_tree_update(&self, tree_update: gpui::accesskit::TreeUpdate) { + super::accessibility::update(tree_update); + } + fn prompt( &self, _level: PromptLevel, From d368f10dc4a81ed7f0022ddc72e5b63745a0039c Mon Sep 17 00:00:00 2001 From: lorenzomicheli Date: Mon, 5 Oct 2026 08:28:05 +0000 Subject: [PATCH 2/6] android: narrow accessibility bridge visibility to pub(super) Only the sibling window and jni modules call it, matching document_picker and text_input. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/android/accessibility.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/android/accessibility.rs b/src/android/accessibility.rs index dbbd00f..73e9a2b 100644 --- a/src/android/accessibility.rs +++ b/src/android/accessibility.rs @@ -71,7 +71,7 @@ impl ActionHandler for Handlers { } /// `PlatformWindow::a11y_init`, on the GPUI thread. -pub(crate) fn init(callbacks: A11yCallbacks) { +pub(super) fn init(callbacks: A11yCallbacks) { let mut bridge = BRIDGE.lock().expect("poisoned"); bridge .insert(Bridge { @@ -82,7 +82,7 @@ pub(crate) fn init(callbacks: A11yCallbacks) { } /// `PlatformWindow::a11y_tree_update`, on the GPUI thread that called [`init`]. -pub(crate) fn update(tree: TreeUpdate) { +pub(super) fn update(tree: TreeUpdate) { let mut bridge = BRIDGE.lock().expect("poisoned"); let Some(adapter) = bridge.as_mut().and_then(|b| b.adapter.as_mut()) else { return; @@ -93,7 +93,7 @@ pub(crate) fn update(tree: TreeUpdate) { } /// A new Activity was registered; move the adapter onto its views. -pub(crate) fn activity_changed() { +pub(super) fn activity_changed() { if let Some(bridge) = BRIDGE.lock().expect("poisoned").as_mut() { bridge.attach(); } From 7eac56abfabcec6200750b3166ccf0e32e1d99d7 Mon Sep 17 00:00:00 2001 From: lorenzomicheli Date: Mon, 5 Oct 2026 08:36:27 +0000 Subject: [PATCH 3/6] android: drop a redundant unsafe impl Send InjectingAdapter is already Send (its JavaVM, WeakRef, global JClass and Arc> fields all are in jni 0.21), so the bridge state can live in the static Mutex without it. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/android/accessibility.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/android/accessibility.rs b/src/android/accessibility.rs index 73e9a2b..d287282 100644 --- a/src/android/accessibility.rs +++ b/src/android/accessibility.rs @@ -49,11 +49,6 @@ struct Adapter { manager: GlobalRef, } -// SAFETY: `InjectingAdapter` holds a `JavaVM`, global/weak references and a -// `&'static` global class reference, all valid on any thread; its methods attach to -// the JVM and post UI work to the host `View` themselves. -unsafe impl Send for Adapter {} - /// GPUI's callbacks, shared by every adapter a recreated Activity gets. #[derive(Clone)] struct Handlers(Arc>); From a34b878c0ac9107a83ab062007c79447ee2a83dc Mon Sep 17 00:00:00 2001 From: lorenzomicheli Date: Mon, 5 Oct 2026 14:34:24 +0000 Subject: [PATCH 4/6] android: let a later screen reader activate GPUI again Once accessibility turns off, tell GPUI to stop building trees and swap in an adapter that is not yet active. Before, the adapter stayed active after TalkBack was turned off, so turning it back on never re-activated GPUI and TalkBack read a stale tree until something repainted; GPUI also kept building trees nobody read. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/android/accessibility.rs | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/android/accessibility.rs b/src/android/accessibility.rs index d287282..a8a695e 100644 --- a/src/android/accessibility.rs +++ b/src/android/accessibility.rs @@ -12,11 +12,11 @@ //! There is one GPUI window per Activity, so the state is global. A recreated Activity //! has new views; [`super::jni::set_host_activity`] moves the adapter onto them. //! -//! Android has no "screen reader stopped" callback for this adapter, so once TalkBack -//! has been used GPUI keeps building the tree until the process ends. Updates are -//! dropped while accessibility is off: this adapter version raises events without -//! checking, and Android throws on the UI thread when nobody is listening -//! (`accesskit_android` 0.9 checks, but needs `accesskit` 0.25). +//! This adapter version has no "screen reader stopped" callback and raises events +//! without checking whether anyone listens, which Android answers by throwing on the +//! UI thread (`accesskit_android` 0.9 checks, but needs `accesskit` 0.25). So each +//! update first asks `AccessibilityManager.isEnabled()`; once it is off, GPUI is told +//! to stop building trees and a fresh adapter waits for the next screen reader. use std::sync::{Arc, Mutex}; @@ -79,12 +79,20 @@ pub(super) fn init(callbacks: A11yCallbacks) { /// `PlatformWindow::a11y_tree_update`, on the GPUI thread that called [`init`]. pub(super) fn update(tree: TreeUpdate) { let mut bridge = BRIDGE.lock().expect("poisoned"); - let Some(adapter) = bridge.as_mut().and_then(|b| b.adapter.as_mut()) else { + let Some(bridge) = bridge.as_mut() else { + return; + }; + let Some(adapter) = bridge.adapter.as_mut() else { return; }; if adapter.is_accessibility_enabled() { adapter.inner.update_if_active(|| tree); + return; } + // The screen reader has gone. Stop GPUI building trees, and swap in an adapter + // that is not yet active, so the next screen reader activates GPUI again. + (bridge.callbacks.0.lock().expect("poisoned").deactivation)(); + bridge.attach(); } /// A new Activity was registered; move the adapter onto its views. @@ -115,9 +123,10 @@ impl Bridge { impl Adapter { fn is_accessibility_enabled(&self) -> bool { - let enabled = self.vm.get_env().and_then(|mut env| { - env.call_method(&self.manager, "isEnabled", "()Z", &[])?.z() - }); + let enabled = self + .vm + .get_env() + .and_then(|mut env| env.call_method(&self.manager, "isEnabled", "()Z", &[])?.z()); enabled.unwrap_or_else(|err| { log::warn!("accessibility: AccessibilityManager.isEnabled failed: {err}"); false From 844a555dd390b38719540c8603dde4e66b8d233b Mon Sep 17 00:00:00 2001 From: lorenzomicheli Date: Mon, 5 Oct 2026 14:50:08 +0000 Subject: [PATCH 5/6] android: let touch exploration reach the accessibility delegate On the android-activity (NativeActivity) path every motion event goes to the native input queue first, and hover events were reported as handled even though GPUI skips them. With TalkBack on, touch exploration arrives as hover events, so the decor view's accessibility delegate never saw them and TalkBack found nothing under the finger. Report them unhandled so Android passes them on, as touches on platform views already are. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/android/jni.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/android/jni.rs b/src/android/jni.rs index 8a717f4..71f6922 100644 --- a/src/android/jni.rs +++ b/src/android/jni.rs @@ -490,6 +490,18 @@ fn process_input_events(app: &AndroidApp) { } }; + // GPUI does not use hover on mobile. With a screen reader on, + // touch exploration arrives as hover events, which the + // accessibility delegate on the decor view has to see. + if matches!( + action, + MotionAction::HoverEnter + | MotionAction::HoverMove + | MotionAction::HoverExit + ) { + return android_activity::InputStatus::Unhandled; + } + if hits_platform_view { log::debug!( "process_input_events: touch hits platform view, skipping GPUI dispatch", From d7e7a7341a06b25efa6b96947f4c1245640c1fa4 Mon Sep 17 00:00:00 2001 From: lorenzomicheli Date: Fri, 9 Oct 2026 07:22:43 +0000 Subject: [PATCH 6/6] android: let the active window own the screen reader Since #26 each host Activity has its own window, and going back re-attaches an earlier one. The bridge kept the last window's callbacks and view, so the returning window got no screen reader. Each window now keeps its callbacks, takes the adapter over when it becomes active, and the window it replaces is told to stop building trees. Co-Authored-By: Claude Opus 5.5 --- src/android/accessibility.rs | 42 +++++++++++++++++++++++++++--------- src/android/window.rs | 18 ++++++++++++++-- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/src/android/accessibility.rs b/src/android/accessibility.rs index a8a695e..f6a4793 100644 --- a/src/android/accessibility.rs +++ b/src/android/accessibility.rs @@ -9,8 +9,9 @@ //! the decor view on the `android-activity` path (the native surface fills the window) //! and the content view on the host-driven path (it holds the `SurfaceView`). //! -//! There is one GPUI window per Activity, so the state is global. A recreated Activity -//! has new views; [`super::jni::set_host_activity`] moves the adapter onto them. +//! Only the active window's Activity is on screen, so one adapter serves the active +//! window, and a window that becomes active takes it over. A recreated Activity has +//! new views; [`super::jni::set_host_activity`] moves the adapter onto them. //! //! This adapter version has no "screen reader stopped" callback and raises events //! without checking whether anyone listens, which Android answers by throwing on the @@ -49,9 +50,19 @@ struct Adapter { manager: GlobalRef, } -/// GPUI's callbacks, shared by every adapter a recreated Activity gets. +/// One window's GPUI callbacks, shared by every adapter its Activity gets. #[derive(Clone)] -struct Handlers(Arc>); +pub(super) struct Handlers(Arc>); + +impl Handlers { + pub(super) fn new(callbacks: A11yCallbacks) -> Self { + Self(Arc::new(Mutex::new(callbacks))) + } + + fn deactivate(&self) { + (self.0.lock().expect("poisoned").deactivation)(); + } +} impl ActivationHandler for Handlers { fn request_initial_tree(&mut self) -> Option { @@ -65,21 +76,32 @@ impl ActionHandler for Handlers { } } -/// `PlatformWindow::a11y_init`, on the GPUI thread. -pub(super) fn init(callbacks: A11yCallbacks) { +/// Point the screen reader at this window, on the GPUI thread. Called from +/// `PlatformWindow::a11y_init` and when the window becomes active again. +pub(super) fn init(callbacks: &Handlers) { let mut bridge = BRIDGE.lock().expect("poisoned"); + if let Some(previous) = bridge.as_ref() { + if Arc::ptr_eq(&previous.callbacks.0, &callbacks.0) { + return; + } + // The previous window is off screen; stop it building trees. + previous.callbacks.deactivate(); + } bridge .insert(Bridge { - callbacks: Handlers(Arc::new(Mutex::new(callbacks))), + callbacks: callbacks.clone(), adapter: None, }) .attach(); } /// `PlatformWindow::a11y_tree_update`, on the GPUI thread that called [`init`]. -pub(super) fn update(tree: TreeUpdate) { +pub(super) fn update(callbacks: &Handlers, tree: TreeUpdate) { let mut bridge = BRIDGE.lock().expect("poisoned"); - let Some(bridge) = bridge.as_mut() else { + let Some(bridge) = bridge + .as_mut() + .filter(|bridge| Arc::ptr_eq(&bridge.callbacks.0, &callbacks.0)) + else { return; }; let Some(adapter) = bridge.adapter.as_mut() else { @@ -91,7 +113,7 @@ pub(super) fn update(tree: TreeUpdate) { } // The screen reader has gone. Stop GPUI building trees, and swap in an adapter // that is not yet active, so the next screen reader activates GPUI again. - (bridge.callbacks.0.lock().expect("poisoned").deactivation)(); + bridge.callbacks.deactivate(); bridge.attach(); } diff --git a/src/android/window.rs b/src/android/window.rs index 6b343da..a6e862f 100644 --- a/src/android/window.rs +++ b/src/android/window.rs @@ -397,6 +397,9 @@ pub struct AndroidWindow { /// present an empty image (a black screen). One `force_render` frame /// repopulates it. force_render_once: Arc, + /// GPUI's accessibility callbacks, once a screen reader has been asked for. The + /// screen reader follows whichever window is active. + a11y: std::sync::Mutex>, } // SAFETY: `WindowState` is protected by a `Mutex`. @@ -476,6 +479,7 @@ impl AndroidWindow { id, active: Arc::new(std::sync::atomic::AtomicBool::new(true)), force_render_once: Arc::new(std::sync::atomic::AtomicBool::new(true)), + a11y: Default::default(), })) } @@ -510,6 +514,7 @@ impl AndroidWindow { id: ((width as u64) << 32) | (height as u64), active: Arc::new(std::sync::atomic::AtomicBool::new(false)), force_render_once: Arc::new(std::sync::atomic::AtomicBool::new(true)), + a11y: Default::default(), }) } @@ -986,6 +991,11 @@ impl AndroidWindow { } } log::info!("AndroidWindow::set_active({}) — done", active); + if active { + if let Some(handlers) = self.a11y.lock().expect("poisoned").clone() { + super::accessibility::init(&handlers); + } + } } if active { // A vsync callback posted before the app left the foreground may @@ -1548,11 +1558,15 @@ impl PlatformWindow for AndroidPlatformWindow { } fn a11y_init(&self, callbacks: gpui::A11yCallbacks) { - super::accessibility::init(callbacks); + let handlers = super::accessibility::Handlers::new(callbacks); + *self.window.a11y.lock().expect("poisoned") = Some(handlers.clone()); + super::accessibility::init(&handlers); } fn a11y_tree_update(&self, tree_update: gpui::accesskit::TreeUpdate) { - super::accessibility::update(tree_update); + if let Some(handlers) = self.window.a11y.lock().expect("poisoned").clone() { + super::accessibility::update(&handlers, tree_update); + } } fn prompt(