diff --git a/config/scripts/verify-computer-native.mjs b/config/scripts/verify-computer-native.mjs index ce3f0b58e31..750d425ef14 100644 --- a/config/scripts/verify-computer-native.mjs +++ b/config/scripts/verify-computer-native.mjs @@ -33,6 +33,12 @@ const checks = [ args: ['native/computer-use-linux/runtime_render_test.py'], enabled: true }, + { + name: 'Linux keyboard synthesis tests', + command: 'python3', + args: ['native/computer-use-linux/runtime_keyboard_test.py'], + enabled: true + }, { name: 'native provider argument guardrails', run: verifyNativeArgumentGuardrails, diff --git a/native/computer-use-linux/runtime.py b/native/computer-use-linux/runtime.py index fecce949aaa..26b9ba43674 100644 --- a/native/computer-use-linux/runtime.py +++ b/native/computer-use-linux/runtime.py @@ -962,11 +962,14 @@ def press_key(raw): return if Gdk is None: raise RuntimeError("GDK is required for non-character key synthesis") - Atspi.generate_keyboard_event(Gdk.keyval_from_name(name), None, Atspi.KeySynthType.PRESSRELEASE) + Atspi.generate_keyboard_event(Gdk.keyval_from_name(name), None, Atspi.KeySynthType.SYM) def hotkey(raw): key_spec = re.sub(r"(?i)commandorcontrol|cmdorctrl", "ctrl", str(raw)) + if "+" in key_spec: + # xdotool treats an upper-case letter as its shifted keysym, so ctrl+A would press ctrl+shift+a. + key_spec = "+".join(part.lower() if len(part) == 1 else part for part in key_spec.split("+")) xdotool = shutil.which("xdotool") if xdotool: subprocess.run([xdotool, "key", key_spec], check=True) diff --git a/native/computer-use-linux/runtime_keyboard_test.py b/native/computer-use-linux/runtime_keyboard_test.py new file mode 100644 index 00000000000..22e6eef31d4 --- /dev/null +++ b/native/computer-use-linux/runtime_keyboard_test.py @@ -0,0 +1,71 @@ +import importlib.util +import sys +import types +import unittest +from pathlib import Path +from unittest import mock + + +class FakeKeySynthType: + PRESSRELEASE = "pressrelease" + SYM = "sym" + STRING = "string" + + +KEYSYMS = {"Return": 0xFF0D, "Tab": 0xFF09, "Escape": 0xFF1B, "Left": 0xFF51, "Down": 0xFF54} + + +def load_runtime(): + gi = types.ModuleType("gi") + repository = types.ModuleType("gi.repository") + gi.require_version = lambda *_: None + repository.Atspi = types.SimpleNamespace(KeySynthType=FakeKeySynthType, generate_keyboard_event=None) + repository.Gdk = types.SimpleNamespace(keyval_from_name=KEYSYMS.__getitem__) + repository.GdkPixbuf = types.SimpleNamespace() + gi.repository = repository + sys.modules["gi"] = gi + sys.modules["gi.repository"] = repository + + path = Path(__file__).with_name("runtime.py") + spec = importlib.util.spec_from_file_location("orca_linux_runtime_keyboard_test", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +runtime = load_runtime() + + +class RuntimeKeyboardTest(unittest.TestCase): + def setUp(self): + self.events = [] + patcher = mock.patch.object( + runtime.Atspi, "generate_keyboard_event", lambda *args: self.events.append(args) + ) + patcher.start() + self.addCleanup(patcher.stop) + + def test_named_keys_synthesize_the_keysym(self): + for raw, keysym in (("enter", 0xFF0D), ("Tab", 0xFF09), ("esc", 0xFF1B), ("left", 0xFF51), ("Down", 0xFF54)): + self.events.clear() + runtime.press_key(raw) + self.assertEqual(self.events, [(keysym, None, FakeKeySynthType.SYM)], raw) + + def run_hotkey(self, raw): + with mock.patch.object(runtime.shutil, "which", return_value="/usr/bin/xdotool"), mock.patch.object( + runtime.subprocess, "run" + ) as run: + runtime.hotkey(raw) + return run.call_args.args[0] + + def test_hotkey_lowercases_letters_so_xdotool_does_not_add_shift(self): + self.assertEqual(self.run_hotkey("CmdOrCtrl+A"), ["/usr/bin/xdotool", "key", "ctrl+a"]) + self.assertEqual(self.run_hotkey("ctrl+shift+T"), ["/usr/bin/xdotool", "key", "ctrl+shift+t"]) + + def test_hotkey_keeps_named_keys(self): + self.assertEqual(self.run_hotkey("alt+Tab"), ["/usr/bin/xdotool", "key", "alt+Tab"]) + + +if __name__ == "__main__": + unittest.main()