From ae05b25530aab563c28f69350ac2ecb40d826ced Mon Sep 17 00:00:00 2001 From: Z3u$ <26085706+zeuslcf@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:46:40 -0400 Subject: [PATCH] fix(mobile): deliver choice taps as separate keys and drop stale ones (#1315) The option buttons shown when an agent asks a question worked only some of the time. An arrow-menu pick went out as one write (N arrows plus CR), which the gateway hands to the pane as one input, so the TUI read it as a single chunk and could miss the move or the Enter. The arrows were always CSI, ignoring DECCKM. Buttons stayed tappable until the next parse, so a second tap, or a tap after the prompt was answered on the desktop, typed a stray digit or Enter into whatever came next. Taps while reconnecting were dropped without a word. Each key is now its own write, 15ms apart with 50ms before Enter (the desktop composer's KEY_GAP and SETTLE), with ESC O A/B in application cursor mode. Each choice set gets an identity (question, options, selection, buffer row); after a tap the buttons dim and take no other tap until the screen shows a different set or 2s pass, and a tap whose set is no longer on screen is dropped. A tap that cannot be sent buzzes, shows the banner and leaves the buttons live. A numbered run that does not start at 1 is accepted when the agent's selection mark is in it, so a scrolled menu still gets buttons. Detection and key building move to mobile/src/choices.ts as pure functions. Co-authored-by: Morpheus Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 11 +++ mobile/src/choices.ts | 128 ++++++++++++++++++++++++++++++++++ mobile/src/main.ts | 159 ++++++++++++++++++++++++------------------ mobile/src/style.css | 5 ++ 4 files changed, 237 insertions(+), 66 deletions(-) create mode 100644 mobile/src/choices.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ed18ee3..8147b391 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 again or Paste code instead, and a windowed scan that fails on Android is retried with the scanner's full-screen camera (#1211). +- **Tapping an agent's choice on the phone picks it reliably** — In the mobile + app, a choice in an arrow menu (such as "trust this folder?") is now typed as + separate keys with short pauses, the way the desktop composer types, and in + the arrow form the program asked for. Sent all at once, the agent could miss + the move or the Enter. After one tap the buttons dim and take no second tap + until the question changes, and a tap on buttons the screen has already moved + past is dropped rather than typing a stray digit or Enter into the next + prompt. A tap while the pane is reconnecting buzzes and says it wasn't sent, + where before it did nothing. A numbered menu scrolled so its first options + are off screen now shows its buttons too. + - **A restored Codex session survives a second restart** — After a restart brings a Codex conversation back, the pane now remembers which session it resumed right away. It used to learn the id only from Codex's first new diff --git a/mobile/src/choices.ts b/mobile/src/choices.ts new file mode 100644 index 00000000..fa9129b3 --- /dev/null +++ b/mobile/src/choices.ts @@ -0,0 +1,128 @@ +// An agent's choices read off the screen, and the keys that pick one. Pure: +// rows of text in, choices and keystrokes out, so the reading and the typing +// can be checked without a terminal. + +/** One choice. `n` is typed to pick it; `steps`, where given, means it is + * picked by moving that many places down (negative: up) and pressing Enter. */ +export type Choice = { n: string; label: string; steps?: number }; + +/** The choices on screen, with an identity: the same question, options, + * selection and place on screen give the same `id`, anything else another. */ +export type ChoiceSet = { id: string; choices: Choice[] }; + +/** One write to the pane, `delay` ms after the one before. */ +export type Keystroke = { data: string; delay: number }; + +/** Between keys an agent must read one at a time (the desktop composer's + * KEY_GAP, src/terminal/composer.rs). */ +export const KEY_GAP_MS = 15; +/** Before the Enter that ends them (the composer's SETTLE): in the same read + * as the keys before it, a TUI can take it as part of them. */ +export const SETTLE_MS = 50; + +/** The keys that pick `choice`, each its own write. Arrows follow the + * pane's cursor-key mode (DECCKM): `ESC O A` when it asked for application + * mode, `ESC [ A` otherwise. */ +export function choiceKeys(choice: Choice, appCursor: boolean): Keystroke[] { + if (choice.steps === undefined) return [{ data: choice.n, delay: 0 }]; + const arrow = `\x1b${appCursor ? "O" : "["}${choice.steps > 0 ? "B" : "A"}`; + const keys: Keystroke[] = []; + for (let i = 0; i < Math.abs(choice.steps); i++) keys.push({ data: arrow, delay: i === 0 ? 0 : KEY_GAP_MS }); + keys.push({ data: "\r", delay: keys.length ? SETTLE_MS : 0 }); + return keys; +} + +/** FNV-1a, 32 bits, in base 36: enough to tell two choice sets apart. */ +function hash(text: string) { + let h = 0x811c9dc5; + for (let i = 0; i < text.length; i++) { + h ^= text.charCodeAt(i); + h = Math.imul(h, 0x01000193); + } + return (h >>> 0).toString(36); +} + +/** The nearest non-blank row above `row`: the question, as near as the + * screen says it. */ +function above(rows: string[], row: number) { + for (let y = row - 1; y >= 0; y--) if (rows[y].trim()) return rows[y].trim(); + return ""; +} + +function identify(kind: string, rows: string[], base: number, first: number, choices: Choice[]): ChoiceSet { + const at = choices.findIndex((c) => c.steps === 0); + const id = hash([kind, base + first, at, above(rows, first), ...choices.map((c) => `${c.n} ${c.label}`)].join("\n")); + return { id, choices }; +} + +/** A menu picked with the arrows, no numbers — an agent's "trust this + * folder?" — said by the hint under it. Its options are the lines above, at + * the selected one's indent. */ +function arrowMenu(rows: string[], base: number): ChoiceSet | null { + let hint = -1; + for (let y = rows.length - 1; y >= 0 && hint < 0; y--) if (/Enter to (confirm|select)/i.test(rows[y])) hint = y; + if (hint <= 0) return null; + let y = hint - 1; + while (y >= 0 && !rows[y].trim()) y--; + const end = y; + for (; y >= 0 && rows[y].trim(); y--); + const block = rows.slice(y + 1, end + 1); + const sel = block.findIndex((l) => /^\s*❯\s/.test(l)); + if (sel < 0 || /^\s*❯\s+\d{1,2}[.)]\s/.test(block[sel])) return null; + const col = block[sel].indexOf("❯"); + const options = block + .map((l, i) => ({ l, i })) + .filter(({ l, i }) => i === sel || (l.slice(0, col + 2).trim() === "" && l[col + 2] !== " ")); + if (options.length < 2 || options.length > 9) return null; + const at = options.findIndex((o) => o.i === sel); + const choices = options.map((o, k) => ({ n: String(k + 1), label: o.l.replace(/^\s*❯?\s*/, ""), steps: k - at })); + return identify("arrows", rows, base, y + 1 + options[0].i, choices); +} + +/** The last run of consecutively numbered lines ("❯ 1. Yes", "2. No, and + * tell Claude…"), box edges and the cursor mark stripped. A run from 1, or + * one from further down whose first lines have scrolled off, if the + * selection mark is in it: an agent's own cursor, not a numbered list in its + * output. */ +function numberedMenu(rows: string[], base: number): ChoiceSet | null { + let found: { first: number; choices: Choice[] } | null = null; + // From 1: lines between that are not the next number (a label wrapped, a + // description under it) are passed over, as they always were. + let run: Choice[] = []; + let runFirst = 0; + // From anywhere: broken by any number out of turn, kept only if marked. + let loose: Choice[] = []; + let looseFirst = 0; + let marked = false; + for (let y = 0; y < rows.length; y++) { + const edged = rows[y].replace(/^[\s│┃|]+|[\s│┃|]+$/g, ""); + const mark = /^[❯›]/.test(edged); + const m = /^(\d{1,2})[.)]\s+(.+)$/.exec(edged.replace(/^[❯›>▸●◉○]\s*/, "")); + if (!m) continue; + const n = Number(m[1]); + const choice = { n: m[1], label: m[2] }; + if (n === 1) { + run = []; + runFirst = y; + } + if (n === run.length + 1) run.push(choice); + if (run.length >= 2 && n === run.length) found = { first: runFirst, choices: [...run] }; + if (loose.length && n === Number(loose[loose.length - 1].n) + 1) { + loose.push(choice); + marked ||= mark; + } else { + loose = [choice]; + looseFirst = y; + marked = mark; + } + if (loose.length >= 2 && marked && loose[0].n !== "1") found = { first: looseFirst, choices: [...loose] }; + } + return found && identify("numbers", rows, base, found.first, found.choices); +} + +/** The choices on screen. `rows` is the pane's screen, top to bottom, and + * `base` the buffer line of its first row; numbered lines count only while + * the agent is waiting. */ +export function findChoices(rows: string[], base: number, waiting: boolean): ChoiceSet | null { + return arrowMenu(rows, base) ?? (waiting ? numberedMenu(rows, base) : null); +} diff --git a/mobile/src/main.ts b/mobile/src/main.ts index 7eecb951..37ee1743 100644 --- a/mobile/src/main.ts +++ b/mobile/src/main.ts @@ -11,7 +11,7 @@ import * as scanner from "@tauri-apps/plugin-barcode-scanner"; import { getVersion, onBackButtonPress } from "@tauri-apps/api/app"; import { listen } from "@tauri-apps/api/event"; -import { impactFeedback, selectionFeedback } from "@tauri-apps/plugin-haptics"; +import { impactFeedback, notificationFeedback, selectionFeedback } from "@tauri-apps/plugin-haptics"; import * as api from "./api"; import { MAX_UPLOAD } from "./api"; @@ -28,6 +28,8 @@ import type { WorkspaceView, } from "./api"; import { agentLook, icon } from "./icons"; +import { choiceKeys, findChoices } from "./choices"; +import type { Choice, ChoiceSet } from "./choices"; import logoUrl from "./assets/logo.svg?url"; const app = document.getElementById("app")!; @@ -121,9 +123,15 @@ function sentence(text: string) { } /** A tap felt under the finger: `key` for a key or a button that types, - * `tick` for a control passing a point (a swipe opening, a toggle). */ -function feel(kind: "key" | "tick") { - (kind === "key" ? impactFeedback("light") : selectionFeedback()).catch(() => {}); + * `tick` for a control passing a point (a swipe opening, a toggle), `miss` + * for a tap that did not do what it says. */ +function feel(kind: "key" | "tick" | "miss") { + (kind === "key" + ? impactFeedback("light") + : kind === "miss" + ? notificationFeedback("error") + : selectionFeedback() + ).catch(() => {}); } /** Why a machine could not be reached, in words, for under a title that @@ -1240,6 +1248,11 @@ const SLOW_CONNECT_MS = 10_000; * network changed — are back well within it. */ const QUIET_MS = 3_000; +/** How long an agent's choices, once one is tapped, take no other tap while + * the screen still shows them: time for the answer to land and the screen to + * move on. Past it, a set still there takes taps again. */ +const SPENT_MS = 2_000; + function hostScreen(host: Host, direction: "push" | "pop" = "pop") { remember("last.host", host.id); go(direction, () => { @@ -4179,73 +4192,87 @@ function terminalScreen(host: Host, place: Place, pane: PaneView, title: string, }; const clearBanner = () => banner.replaceChildren(); - // The choices on screen, read off the bottom of the pane when the agent - // is waiting: the last run of lines numbered from 1, as Claude Code and - // Codex draw them ("❯ 1. Yes", "2. No, and tell Claude…"), box edges and - // the cursor mark stripped. + // The choices on screen (choices.ts): an arrow menu over its "Enter to + // confirm" hint, or, while the agent waits, its numbered options. The + // whole screen is read: the terminal here can be taller than the pane, + // with the agent's menu well above its bottom rows. + const scanChoices = (): ChoiceSet | null => { + const buf = term.buffer.active; + const rows: string[] = []; + for (let y = buf.baseY; y < buf.baseY + term.rows; y++) rows.push(buf.getLine(y)?.translateToString(true) ?? ""); + return findChoices(rows, buf.baseY, agentWaiting); + }; + // The set the buttons are for, and one already answered: its buttons + // stay dimmed and take no second tap until the screen shows another set, + // or for a while if it never does (the answer went nowhere). + let shownSet = ""; + let spent: { id: string; timer: number } | null = null; + // An answer still being typed: its own arrows move the selection, which + // redraws the set, and that is not the screen moving on. + let answering = false; + const unspend = () => { + if (spent) clearTimeout(spent.timer); + spent = null; + answers.classList.remove("spent"); + }; + const spend = (id: string) => { + unspend(); + spent = { id, timer: window.setTimeout(unspend, SPENT_MS) }; + answers.classList.add("spent"); + }; + // An answer that could not be sent says so — a buzz, and the banner — + // and leaves its buttons to tap again. + const unsent = () => { + unspend(); + feel("miss"); + if (!banner.hasChildNodes()) showBanner("Not connected, so that answer wasn't sent."); + }; + // Each key its own write, apart, as the desktop's composer types them: a + // TUI reads arrows and Enter that land in one read as something else. + const typeKeys = async (keys: { data: string; delay: number }[]) => { + for (const { data, delay } of keys) { + if (delay) await new Promise((done) => window.setTimeout(done, delay)); + if (!alive || !(await input(data))) return false; + } + return true; + }; + const pick = async (set: ChoiceSet, choice: Choice) => { + if (spent || answering) return; + // The screen moved on since the buttons were drawn — answered at the + // desktop, or another question — so this tap is not for what is there. + if (scanChoices()?.id !== set.id) { + feel("miss"); + readChoices(); + return; + } + if (handle === null || !live) { + speak(); + unsent(); + return; + } + spend(set.id); + feel("key"); + term.scrollToBottom(); + showCursor(); + answering = true; + const delivered = await typeKeys(choiceKeys(choice, term.modes.applicationCursorKeysMode)); + answering = false; + if (!alive) return; + if (!delivered) unsent(); + readChoices(); + }; const readChoices = () => { - // `n` is typed to pick it; `keys`, where given, is sent instead. - const found: { n: string; label: string; keys?: string }[] = []; - // A menu picked with the arrows, no numbers — an agent's "trust this - // folder?" — said by the hint under it. Its options are the lines - // above, at the selected one's indent; picking one moves to it and - // presses Enter. - { - const buf = term.buffer.active; - const rows: string[] = []; - for (let y = buf.baseY; y < buf.baseY + term.rows; y++) rows.push(buf.getLine(y)?.translateToString(true) ?? ""); - let hint = -1; - for (let y = rows.length - 1; y >= 0 && hint < 0; y--) if (/Enter to (confirm|select)/i.test(rows[y])) hint = y; - if (hint > 0) { - let y = hint - 1; - while (y >= 0 && !rows[y].trim()) y--; - const block: string[] = []; - for (; y >= 0 && rows[y].trim(); y--) block.unshift(rows[y]); - const sel = block.findIndex((l) => /^\s*❯\s/.test(l)); - if (sel >= 0 && !/^\s*❯\s+\d{1,2}[.)]\s/.test(block[sel])) { - const col = block[sel].indexOf("❯"); - const options = block - .map((l, i) => ({ l, i })) - .filter(({ l, i }) => i === sel || (l.slice(0, col + 2).trim() === "" && l[col + 2] !== " ")); - const at = options.findIndex((o) => o.i === sel); - if (options.length >= 2 && options.length <= 9) - options.forEach((o, k) => { - const step = k > at ? "\x1b[B" : "\x1b[A"; - found.push({ - n: String(k + 1), - label: o.l.replace(/^\s*❯?\s*/, ""), - keys: step.repeat(Math.abs(k - at)) + "\r", - }); - }); - } - } - } - if (agentWaiting && !found.length) { - // The whole screen: the terminal here can be taller than the pane, - // with the agent's menu well above its bottom rows. - const buf = term.buffer.active; - const bottom = buf.baseY + term.rows; - let run: typeof found = []; - for (let y = buf.baseY; y < bottom; y++) { - const text = (buf.getLine(y)?.translateToString(true) ?? "") - .replace(/^[\s│┃|]+|[\s│┃|]+$/g, "") - .replace(/^[❯›>▸●◉○]\s*/, ""); - const m = /^(\d{1,2})[.)]\s+(.+)$/.exec(text); - if (!m) continue; - if (m[1] === "1") run = []; - if (Number(m[1]) === run.length + 1) run.push({ n: m[1], label: m[2] }); - if (run.length >= 2) found.splice(0, found.length, ...run); - } - } - const shown = answers.dataset.shown ?? ""; - const now = found.map((c) => `${c.n}${c.label}`).join("\n"); - if (shown === now) return; - answers.dataset.shown = now; + const set = scanChoices(); + const id = set?.id ?? ""; + if (spent && spent.id !== id && !answering) unspend(); + if (shownSet === id) return; + shownSet = id; + const found = set?.choices ?? []; answers.replaceChildren( ...found.map((c) => { const b = h("button", { class: "answer" }, h("b", {}, c.n), h("span", {}, c.label)); b.onpointerdown = (e) => e.preventDefault(); - b.onclick = () => (feel("key"), send(c.keys ?? c.n)); + b.onclick = () => set && void pick(set, c); return b; }), ); diff --git a/mobile/src/style.css b/mobile/src/style.css index 6dba2414..62493f4d 100644 --- a/mobile/src/style.css +++ b/mobile/src/style.css @@ -2136,6 +2136,11 @@ html[data-nav="pop"]::view-transition-new(root) { opacity: 0.6; } +/* Answered: the set takes no second tap until the screen moves on. */ +.answers.spent .answer { + opacity: 0.45; +} + .suggestion { flex: none; max-width: 78%;