fix(mobile): deliver choice taps as separate keys and drop stale ones (#1315)

The option buttons shown when an agent asks a question worked only some
of the time. An arrow-menu pick went out as one write (N arrows plus CR),
which the gateway hands to the pane as one input, so the TUI read it as
a single chunk and could miss the move or the Enter. The arrows were
always CSI, ignoring DECCKM. Buttons stayed tappable until the next
parse, so a second tap, or a tap after the prompt was answered on the
desktop, typed a stray digit or Enter into whatever came next. Taps while
reconnecting were dropped without a word.

Each key is now its own write, 15ms apart with 50ms before Enter (the
desktop composer's KEY_GAP and SETTLE), with ESC O A/B in application
cursor mode. Each choice set gets an identity (question, options,
selection, buffer row); after a tap the buttons dim and take no other
tap until the screen shows a different set or 2s pass, and a tap whose
set is no longer on screen is dropped. A tap that cannot be sent buzzes,
shows the banner and leaves the buttons live. A numbered run that does
not start at 1 is accepted when the agent's selection mark is in it, so
a scrolled menu still gets buttons.

Detection and key building move to mobile/src/choices.ts as pure
functions.

Co-authored-by: Morpheus <morpheus@Morpheuss-MacBook-Pro.local>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Z3u$
2026-11-14 20:46:40 +08:00
committed by GitHub
co-authored by Morpheus Claude Opus 5.5
parent b6abbd2272
commit ae05b25530
4 changed files with 237 additions and 66 deletions
+11
View File
@@ -24,6 +24,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
again or Paste code instead, and a windowed scan that fails on Android is
retried with the scanner's full-screen camera (#1211).
- **Tapping an agent's choice on the phone picks it reliably** — In the mobile
app, a choice in an arrow menu (such as "trust this folder?") is now typed as
separate keys with short pauses, the way the desktop composer types, and in
the arrow form the program asked for. Sent all at once, the agent could miss
the move or the Enter. After one tap the buttons dim and take no second tap
until the question changes, and a tap on buttons the screen has already moved
past is dropped rather than typing a stray digit or Enter into the next
prompt. A tap while the pane is reconnecting buzzes and says it wasn't sent,
where before it did nothing. A numbered menu scrolled so its first options
are off screen now shows its buttons too.
- **A restored Codex session survives a second restart** — After a restart
brings a Codex conversation back, the pane now remembers which session it
resumed right away. It used to learn the id only from Codex's first new
+128
View File
@@ -0,0 +1,128 @@
// An agent's choices read off the screen, and the keys that pick one. Pure:
// rows of text in, choices and keystrokes out, so the reading and the typing
// can be checked without a terminal.
/** One choice. `n` is typed to pick it; `steps`, where given, means it is
* picked by moving that many places down (negative: up) and pressing Enter. */
export type Choice = { n: string; label: string; steps?: number };
/** The choices on screen, with an identity: the same question, options,
* selection and place on screen give the same `id`, anything else another. */
export type ChoiceSet = { id: string; choices: Choice[] };
/** One write to the pane, `delay` ms after the one before. */
export type Keystroke = { data: string; delay: number };
/** Between keys an agent must read one at a time (the desktop composer's
* KEY_GAP, src/terminal/composer.rs). */
export const KEY_GAP_MS = 15;
/** Before the Enter that ends them (the composer's SETTLE): in the same read
* as the keys before it, a TUI can take it as part of them. */
export const SETTLE_MS = 50;
/** The keys that pick `choice`, each its own write. Arrows follow the
* pane's cursor-key mode (DECCKM): `ESC O A` when it asked for application
* mode, `ESC [ A` otherwise. */
export function choiceKeys(choice: Choice, appCursor: boolean): Keystroke[] {
if (choice.steps === undefined) return [{ data: choice.n, delay: 0 }];
const arrow = `\x1b${appCursor ? "O" : "["}${choice.steps > 0 ? "B" : "A"}`;
const keys: Keystroke[] = [];
for (let i = 0; i < Math.abs(choice.steps); i++) keys.push({ data: arrow, delay: i === 0 ? 0 : KEY_GAP_MS });
keys.push({ data: "\r", delay: keys.length ? SETTLE_MS : 0 });
return keys;
}
/** FNV-1a, 32 bits, in base 36: enough to tell two choice sets apart. */
function hash(text: string) {
let h = 0x811c9dc5;
for (let i = 0; i < text.length; i++) {
h ^= text.charCodeAt(i);
h = Math.imul(h, 0x01000193);
}
return (h >>> 0).toString(36);
}
/** The nearest non-blank row above `row`: the question, as near as the
* screen says it. */
function above(rows: string[], row: number) {
for (let y = row - 1; y >= 0; y--) if (rows[y].trim()) return rows[y].trim();
return "";
}
function identify(kind: string, rows: string[], base: number, first: number, choices: Choice[]): ChoiceSet {
const at = choices.findIndex((c) => c.steps === 0);
const id = hash([kind, base + first, at, above(rows, first), ...choices.map((c) => `${c.n} ${c.label}`)].join("\n"));
return { id, choices };
}
/** A menu picked with the arrows, no numbers — an agent's "trust this
* folder?" — said by the hint under it. Its options are the lines above, at
* the selected one's indent. */
function arrowMenu(rows: string[], base: number): ChoiceSet | null {
let hint = -1;
for (let y = rows.length - 1; y >= 0 && hint < 0; y--) if (/Enter to (confirm|select)/i.test(rows[y])) hint = y;
if (hint <= 0) return null;
let y = hint - 1;
while (y >= 0 && !rows[y].trim()) y--;
const end = y;
for (; y >= 0 && rows[y].trim(); y--);
const block = rows.slice(y + 1, end + 1);
const sel = block.findIndex((l) => /^\s*❯\s/.test(l));
if (sel < 0 || /^\s*❯\s+\d{1,2}[.)]\s/.test(block[sel])) return null;
const col = block[sel].indexOf("❯");
const options = block
.map((l, i) => ({ l, i }))
.filter(({ l, i }) => i === sel || (l.slice(0, col + 2).trim() === "" && l[col + 2] !== " "));
if (options.length < 2 || options.length > 9) return null;
const at = options.findIndex((o) => o.i === sel);
const choices = options.map((o, k) => ({ n: String(k + 1), label: o.l.replace(/^\s*❯?\s*/, ""), steps: k - at }));
return identify("arrows", rows, base, y + 1 + options[0].i, choices);
}
/** The last run of consecutively numbered lines ("❯ 1. Yes", "2. No, and
* tell Claude…"), box edges and the cursor mark stripped. A run from 1, or
* one from further down whose first lines have scrolled off, if the
* selection mark is in it: an agent's own cursor, not a numbered list in its
* output. */
function numberedMenu(rows: string[], base: number): ChoiceSet | null {
let found: { first: number; choices: Choice[] } | null = null;
// From 1: lines between that are not the next number (a label wrapped, a
// description under it) are passed over, as they always were.
let run: Choice[] = [];
let runFirst = 0;
// From anywhere: broken by any number out of turn, kept only if marked.
let loose: Choice[] = [];
let looseFirst = 0;
let marked = false;
for (let y = 0; y < rows.length; y++) {
const edged = rows[y].replace(/^[\s│┃|]+|[\s│┃|]+$/g, "");
const mark = /^[❯›]/.test(edged);
const m = /^(\d{1,2})[.)]\s+(.+)$/.exec(edged.replace(/^[❯›>▸●◉○]\s*/, ""));
if (!m) continue;
const n = Number(m[1]);
const choice = { n: m[1], label: m[2] };
if (n === 1) {
run = [];
runFirst = y;
}
if (n === run.length + 1) run.push(choice);
if (run.length >= 2 && n === run.length) found = { first: runFirst, choices: [...run] };
if (loose.length && n === Number(loose[loose.length - 1].n) + 1) {
loose.push(choice);
marked ||= mark;
} else {
loose = [choice];
looseFirst = y;
marked = mark;
}
if (loose.length >= 2 && marked && loose[0].n !== "1") found = { first: looseFirst, choices: [...loose] };
}
return found && identify("numbers", rows, base, found.first, found.choices);
}
/** The choices on screen. `rows` is the pane's screen, top to bottom, and
* `base` the buffer line of its first row; numbered lines count only while
* the agent is waiting. */
export function findChoices(rows: string[], base: number, waiting: boolean): ChoiceSet | null {
return arrowMenu(rows, base) ?? (waiting ? numberedMenu(rows, base) : null);
}
+93 -66
View File
@@ -11,7 +11,7 @@ import * as scanner from "@tauri-apps/plugin-barcode-scanner";
import { getVersion, onBackButtonPress } from "@tauri-apps/api/app";
import { listen } from "@tauri-apps/api/event";
import { impactFeedback, selectionFeedback } from "@tauri-apps/plugin-haptics";
import { impactFeedback, notificationFeedback, selectionFeedback } from "@tauri-apps/plugin-haptics";
import * as api from "./api";
import { MAX_UPLOAD } from "./api";
@@ -28,6 +28,8 @@ import type {
WorkspaceView,
} from "./api";
import { agentLook, icon } from "./icons";
import { choiceKeys, findChoices } from "./choices";
import type { Choice, ChoiceSet } from "./choices";
import logoUrl from "./assets/logo.svg?url";
const app = document.getElementById("app")!;
@@ -121,9 +123,15 @@ function sentence(text: string) {
}
/** A tap felt under the finger: `key` for a key or a button that types,
* `tick` for a control passing a point (a swipe opening, a toggle). */
function feel(kind: "key" | "tick") {
(kind === "key" ? impactFeedback("light") : selectionFeedback()).catch(() => {});
* `tick` for a control passing a point (a swipe opening, a toggle), `miss`
* for a tap that did not do what it says. */
function feel(kind: "key" | "tick" | "miss") {
(kind === "key"
? impactFeedback("light")
: kind === "miss"
? notificationFeedback("error")
: selectionFeedback()
).catch(() => {});
}
/** Why a machine could not be reached, in words, for under a title that
@@ -1240,6 +1248,11 @@ const SLOW_CONNECT_MS = 10_000;
* network changed — are back well within it. */
const QUIET_MS = 3_000;
/** How long an agent's choices, once one is tapped, take no other tap while
* the screen still shows them: time for the answer to land and the screen to
* move on. Past it, a set still there takes taps again. */
const SPENT_MS = 2_000;
function hostScreen(host: Host, direction: "push" | "pop" = "pop") {
remember("last.host", host.id);
go(direction, () => {
@@ -4179,73 +4192,87 @@ function terminalScreen(host: Host, place: Place, pane: PaneView, title: string,
};
const clearBanner = () => banner.replaceChildren();
// The choices on screen, read off the bottom of the pane when the agent
// is waiting: the last run of lines numbered from 1, as Claude Code and
// Codex draw them ("❯ 1. Yes", "2. No, and tell Claude…"), box edges and
// the cursor mark stripped.
// The choices on screen (choices.ts): an arrow menu over its "Enter to
// confirm" hint, or, while the agent waits, its numbered options. The
// whole screen is read: the terminal here can be taller than the pane,
// with the agent's menu well above its bottom rows.
const scanChoices = (): ChoiceSet | null => {
const buf = term.buffer.active;
const rows: string[] = [];
for (let y = buf.baseY; y < buf.baseY + term.rows; y++) rows.push(buf.getLine(y)?.translateToString(true) ?? "");
return findChoices(rows, buf.baseY, agentWaiting);
};
// The set the buttons are for, and one already answered: its buttons
// stay dimmed and take no second tap until the screen shows another set,
// or for a while if it never does (the answer went nowhere).
let shownSet = "";
let spent: { id: string; timer: number } | null = null;
// An answer still being typed: its own arrows move the selection, which
// redraws the set, and that is not the screen moving on.
let answering = false;
const unspend = () => {
if (spent) clearTimeout(spent.timer);
spent = null;
answers.classList.remove("spent");
};
const spend = (id: string) => {
unspend();
spent = { id, timer: window.setTimeout(unspend, SPENT_MS) };
answers.classList.add("spent");
};
// An answer that could not be sent says so — a buzz, and the banner —
// and leaves its buttons to tap again.
const unsent = () => {
unspend();
feel("miss");
if (!banner.hasChildNodes()) showBanner("Not connected, so that answer wasn't sent.");
};
// Each key its own write, apart, as the desktop's composer types them: a
// TUI reads arrows and Enter that land in one read as something else.
const typeKeys = async (keys: { data: string; delay: number }[]) => {
for (const { data, delay } of keys) {
if (delay) await new Promise((done) => window.setTimeout(done, delay));
if (!alive || !(await input(data))) return false;
}
return true;
};
const pick = async (set: ChoiceSet, choice: Choice) => {
if (spent || answering) return;
// The screen moved on since the buttons were drawn — answered at the
// desktop, or another question — so this tap is not for what is there.
if (scanChoices()?.id !== set.id) {
feel("miss");
readChoices();
return;
}
if (handle === null || !live) {
speak();
unsent();
return;
}
spend(set.id);
feel("key");
term.scrollToBottom();
showCursor();
answering = true;
const delivered = await typeKeys(choiceKeys(choice, term.modes.applicationCursorKeysMode));
answering = false;
if (!alive) return;
if (!delivered) unsent();
readChoices();
};
const readChoices = () => {
// `n` is typed to pick it; `keys`, where given, is sent instead.
const found: { n: string; label: string; keys?: string }[] = [];
// A menu picked with the arrows, no numbers — an agent's "trust this
// folder?" — said by the hint under it. Its options are the lines
// above, at the selected one's indent; picking one moves to it and
// presses Enter.
{
const buf = term.buffer.active;
const rows: string[] = [];
for (let y = buf.baseY; y < buf.baseY + term.rows; y++) rows.push(buf.getLine(y)?.translateToString(true) ?? "");
let hint = -1;
for (let y = rows.length - 1; y >= 0 && hint < 0; y--) if (/Enter to (confirm|select)/i.test(rows[y])) hint = y;
if (hint > 0) {
let y = hint - 1;
while (y >= 0 && !rows[y].trim()) y--;
const block: string[] = [];
for (; y >= 0 && rows[y].trim(); y--) block.unshift(rows[y]);
const sel = block.findIndex((l) => /^\s*❯\s/.test(l));
if (sel >= 0 && !/^\s*❯\s+\d{1,2}[.)]\s/.test(block[sel])) {
const col = block[sel].indexOf("❯");
const options = block
.map((l, i) => ({ l, i }))
.filter(({ l, i }) => i === sel || (l.slice(0, col + 2).trim() === "" && l[col + 2] !== " "));
const at = options.findIndex((o) => o.i === sel);
if (options.length >= 2 && options.length <= 9)
options.forEach((o, k) => {
const step = k > at ? "\x1b[B" : "\x1b[A";
found.push({
n: String(k + 1),
label: o.l.replace(/^\s*❯?\s*/, ""),
keys: step.repeat(Math.abs(k - at)) + "\r",
});
});
}
}
}
if (agentWaiting && !found.length) {
// The whole screen: the terminal here can be taller than the pane,
// with the agent's menu well above its bottom rows.
const buf = term.buffer.active;
const bottom = buf.baseY + term.rows;
let run: typeof found = [];
for (let y = buf.baseY; y < bottom; y++) {
const text = (buf.getLine(y)?.translateToString(true) ?? "")
.replace(/^[\s│┃|]+|[\s│┃|]+$/g, "")
.replace(/^[❯›>▸●◉○]\s*/, "");
const m = /^(\d{1,2})[.)]\s+(.+)$/.exec(text);
if (!m) continue;
if (m[1] === "1") run = [];
if (Number(m[1]) === run.length + 1) run.push({ n: m[1], label: m[2] });
if (run.length >= 2) found.splice(0, found.length, ...run);
}
}
const shown = answers.dataset.shown ?? "";
const now = found.map((c) => `${c.n}${c.label}`).join("\n");
if (shown === now) return;
answers.dataset.shown = now;
const set = scanChoices();
const id = set?.id ?? "";
if (spent && spent.id !== id && !answering) unspend();
if (shownSet === id) return;
shownSet = id;
const found = set?.choices ?? [];
answers.replaceChildren(
...found.map((c) => {
const b = h("button", { class: "answer" }, h("b", {}, c.n), h("span", {}, c.label));
b.onpointerdown = (e) => e.preventDefault();
b.onclick = () => (feel("key"), send(c.keys ?? c.n));
b.onclick = () => set && void pick(set, c);
return b;
}),
);
+5
View File
@@ -2136,6 +2136,11 @@ html[data-nav="pop"]::view-transition-new(root) {
opacity: 0.6;
}
/* Answered: the set takes no second tap until the screen moves on. */
.answers.spent .answer {
opacity: 0.45;
}
.suggestion {
flex: none;
max-width: 78%;