From bb2a2ecf16a6e651d31c99f29b214b02e5e36510 Mon Sep 17 00:00:00 2001 From: Jehu Date: Tue, 25 Aug 2026 09:14:56 +0200 Subject: [PATCH 1/2] Make the interactive Browser honor configured keyboard layouts The private Browser Xvfb always started with the US XKB layout, so non-US keyboards could not type their printed characters in the interactive Browser window: German Mac users could not type "@" via Option+L, and umlauts, brackets, and AltGr/Option symbols landed on wrong keys or vanished. Add keyboard_layout / keyboard_variant Browser settings (validated as safe XKB tokens, empty keeps US behavior). Browser startup applies the layout to the private display with setxkbmap and pins it on the Xpra shadow server (--keyboard-sync=no, --keyboard-layout, --keyboard-variant) so connecting clients cannot override the server layout. Layout changes flow through browser_runtime_config and restart internal runtimes. The fallback canvas input path also stopped discarding Alt/AltGr combinations that produce printable characters, and the settings UI offers free-text layout/variant fields linking to the official xkeyboard-config list. --- plugins/_browser/AGENTS.md | 1 + plugins/_browser/default_config.yaml | 6 + plugins/_browser/helpers/config.py | 16 +++ plugins/_browser/helpers/interactive_view.py | 54 +++++++++ .../_browser/webui/browser-config-store.js | 10 ++ plugins/_browser/webui/browser-store.js | 5 +- plugins/_browser/webui/config.html | 31 +++++ tests/test_browser_agent_regressions.py | 109 ++++++++++++++++++ 8 files changed, 230 insertions(+), 2 deletions(-) diff --git a/plugins/_browser/AGENTS.md b/plugins/_browser/AGENTS.md index ce97ff7dad..240cb0c81b 100644 --- a/plugins/_browser/AGENTS.md +++ b/plugins/_browser/AGENTS.md @@ -48,6 +48,7 @@ - Annotation voice input reuses Whisper STT's configured draft/send delivery mode and shared microphone state. - Internal-browser proxy settings map directly to Playwright's persistent-context proxy option, never to Bring Your Own Browser, and changes must restart active internal runtimes. - Run internal Chromium headful through Patchright on the private virtual display; do not add user-agent or header spoofing on top of the patched driver. +- Browser keyboard layout settings (`keyboard_layout`/`keyboard_variant`, e.g. `de`/`mac`) apply the configured XKB layout to the private browser display with setxkbmap and pin it on the Xpra shadow server so non-US keyboards type their printed characters; layout changes flow through `browser_runtime_config` and restart internal runtimes. - Browser startup and on-demand launch must converge on the Chromium revision declared by Patchright; let its installer select the host architecture rather than hardcoding x64 or ARM downloads. - `hooks.prepare_playwright_cache()` owns reconciliation of the pinned Patchright package and Chromium binary so repository self-updates and fresh images use the same setup path. - Browser startup must install the shared virtual-desktop route hook itself; do not make Browser depend on the Desktop plugin being enabled. diff --git a/plugins/_browser/default_config.yaml b/plugins/_browser/default_config.yaml index e15d3d880e..9f7dd057aa 100644 --- a/plugins/_browser/default_config.yaml +++ b/plugins/_browser/default_config.yaml @@ -49,3 +49,9 @@ host_browser_selection: "" # Optional _model_config preset used by Browser-owned model helpers. # Empty uses the effective Main Model. model_preset: "" + +# XKB keyboard layout applied to the interactive Browser display so typed keys +# match the physical keyboard. German Mac example: keyboard_layout: "de", +# keyboard_variant: "mac" (Option+L then types "@"). Empty keeps US behavior. +keyboard_layout: "" +keyboard_variant: "" diff --git a/plugins/_browser/helpers/config.py b/plugins/_browser/helpers/config.py index f000a6f3cf..eecd970037 100644 --- a/plugins/_browser/helpers/config.py +++ b/plugins/_browser/helpers/config.py @@ -22,6 +22,8 @@ PROXY_BYPASS_KEY = "proxy_bypass" PROXY_USERNAME_KEY = "proxy_username" PROXY_PASSWORD_KEY = "proxy_password" +KEYBOARD_LAYOUT_KEY = "keyboard_layout" +KEYBOARD_VARIANT_KEY = "keyboard_variant" RUNTIME_BACKENDS = {"container", "host_required"} BROWSER_TAB_SCOPES = {"per_context", "shared"} HOST_BROWSER_PRIVACY_POLICIES = {"enforce_local", "warn", "allow"} @@ -100,6 +102,12 @@ def _normalize_int(value: Any, *, default: int, minimum: int, maximum: int) -> i return max(minimum, min(maximum, number)) +def _normalize_xkb_token(value: Any) -> str: + return "".join( + ch for ch in str(value or "").strip().lower() if ch.isalnum() or ch in {"_", "-"} + )[:32] + + def _normalize_choice(value: Any, *, allowed: set[str], default: str) -> str: normalized = str(value or "").strip().lower().replace("-", "_") if normalized in allowed: @@ -125,6 +133,10 @@ def _model_config_summary(config: dict[str, Any] | None) -> str: def normalize_browser_config(settings: dict[str, Any] | None) -> dict[str, Any]: raw = settings if isinstance(settings, dict) else {} extension_paths = _normalize_extension_paths(raw.get("extension_paths", [])) + keyboard_layout = _normalize_xkb_token(raw.get(KEYBOARD_LAYOUT_KEY, "")) + keyboard_variant = ( + _normalize_xkb_token(raw.get(KEYBOARD_VARIANT_KEY, "")) if keyboard_layout else "" + ) return { "extension_paths": extension_paths, DEFAULT_HOMEPAGE_KEY: _normalize_default_homepage( @@ -165,6 +177,8 @@ def normalize_browser_config(settings: dict[str, Any] | None) -> dict[str, Any]: PROXY_BYPASS_KEY: str(raw.get(PROXY_BYPASS_KEY, "") or "").strip()[:4096], PROXY_USERNAME_KEY: str(raw.get(PROXY_USERNAME_KEY, "") or "")[:1024], PROXY_PASSWORD_KEY: str(raw.get(PROXY_PASSWORD_KEY, "") or "")[:4096], + KEYBOARD_LAYOUT_KEY: keyboard_layout, + KEYBOARD_VARIANT_KEY: keyboard_variant, MODEL_PRESET_KEY: _normalize_model_preset(raw.get(MODEL_PRESET_KEY, "")), } @@ -177,6 +191,8 @@ def browser_runtime_config(settings: dict[str, Any] | None) -> dict[str, Any]: PROXY_BYPASS_KEY: config[PROXY_BYPASS_KEY], PROXY_USERNAME_KEY: config[PROXY_USERNAME_KEY], PROXY_PASSWORD_KEY: config[PROXY_PASSWORD_KEY], + KEYBOARD_LAYOUT_KEY: config[KEYBOARD_LAYOUT_KEY], + KEYBOARD_VARIANT_KEY: config[KEYBOARD_VARIANT_KEY], } diff --git a/plugins/_browser/helpers/interactive_view.py b/plugins/_browser/helpers/interactive_view.py index a603633d0f..6689e55bbb 100644 --- a/plugins/_browser/helpers/interactive_view.py +++ b/plugins/_browser/helpers/interactive_view.py @@ -20,6 +20,21 @@ START_TIMEOUT_SECONDS = 15.0 +def keyboard_options() -> dict[str, str]: + """Resolve the configured XKB keyboard layout for browser displays.""" + from plugins._browser.helpers.config import ( + KEYBOARD_LAYOUT_KEY, + KEYBOARD_VARIANT_KEY, + get_browser_config, + ) + + config = get_browser_config() + return { + "layout": str(config.get(KEYBOARD_LAYOUT_KEY, "") or "").strip(), + "variant": str(config.get(KEYBOARD_VARIANT_KEY, "") or "").strip(), + } + + def collect_status() -> dict[str, Any]: binaries = { name: shutil.which(name) or "" @@ -114,6 +129,7 @@ def ensure_display(self) -> str: self._xvfb = process self.display = int(display_number) + self._apply_keyboard_layout() self.resize(self.width, self.height) return self.display_name @@ -190,6 +206,43 @@ def close(self) -> None: self._stop_locked() shutil.rmtree(self.state_dir, ignore_errors=True) + def _apply_keyboard_layout(self) -> None: + """Apply the configured XKB layout to this private display.""" + if self.display is None: + return + options = keyboard_options() + if not options["layout"]: + return + setxkbmap = shutil.which("setxkbmap") + if not setxkbmap: + return + command = [setxkbmap, "-display", self.display_name, "-layout", options["layout"]] + if options["variant"]: + command.extend(["-variant", options["variant"]]) + try: + subprocess.run( + command, + check=False, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=5, + ) + except (OSError, subprocess.TimeoutExpired): + pass + + def _keyboard_xpra_args(self) -> list[str]: + options = keyboard_options() + if not options["layout"]: + return [] + args = [ + "--keyboard-sync=no", + "--keyboard-layout", options["layout"], + ] + if options["variant"]: + args.extend(["--keyboard-variant", options["variant"]]) + return args + def _start_xpra(self, xpra: str) -> None: self.port = self._free_port() runtime_dir = self.state_dir / "runtime" @@ -227,6 +280,7 @@ def _start_xpra(self, xpra: str) -> None: "--encoding=auto", "--quality=90", "--speed=90", + *self._keyboard_xpra_args(), f"--bind-tcp=127.0.0.1:{self.port}", f"--socket-dir={socket_dir}", f"--log-dir={self.state_dir}", diff --git a/plugins/_browser/webui/browser-config-store.js b/plugins/_browser/webui/browser-config-store.js index 8220f9e33e..0c941674c0 100644 --- a/plugins/_browser/webui/browser-config-store.js +++ b/plugins/_browser/webui/browser-config-store.js @@ -41,6 +41,8 @@ function ensureConfig(config) { config.proxy_bypass = String(config.proxy_bypass || "").trim(); config.proxy_username = String(config.proxy_username || ""); config.proxy_password = String(config.proxy_password || ""); + config.keyboard_layout = normalizeXkbToken(config.keyboard_layout); + config.keyboard_variant = normalizeXkbToken(config.keyboard_variant); config.host_browser_privacy_policy = normalizeChoice( config.host_browser_privacy_policy, HOST_PRIVACY_POLICIES, @@ -68,6 +70,14 @@ function normalizeInt(value, fallback, minimum, maximum) { return Math.max(minimum, Math.min(maximum, number)); } +function normalizeXkbToken(value) { + return String(value || "") + .trim() + .toLowerCase() + .replace(/[^a-z0-9_-]/g, "") + .slice(0, 32); +} + function normalizeRuntimeBackend(value) { const normalized = String(value || "").trim().toLowerCase().replace(/-/g, "_"); if (normalized === "host_when_available") return "host_required"; diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js index 1b59f68388..e85ac38f1e 100644 --- a/plugins/_browser/webui/browser-store.js +++ b/plugins/_browser/webui/browser-store.js @@ -2908,10 +2908,11 @@ const model = { if (this.annotating) return; const contextId = this.normalizeContextId(this.activeBrowserContextId || this.contextId); if (!contextId || !this.activeBrowserId) return; - if (event.ctrlKey || event.metaKey || event.altKey) return; + const printable = event.key && event.key.length === 1; + const altGrText = printable && event.altKey && !event.metaKey; + if ((event.ctrlKey || event.metaKey || event.altKey) && !altGrText) return; if (isLocalEditableTarget(event?.target)) return; event.preventDefault(); - const printable = event.key && event.key.length === 1; await websocket.emit("browser_viewer_input", { context_id: contextId, browser_id: this.activeBrowserId, diff --git a/plugins/_browser/webui/config.html b/plugins/_browser/webui/config.html index 765e562887..326b513b69 100644 --- a/plugins/_browser/webui/config.html +++ b/plugins/_browser/webui/config.html @@ -243,6 +243,37 @@ + + + +