fix(runtime): require prompt shape before a credential refusal

The auth-verb x auth-flow pair rule fired on any two auth-ish lines in the
live window with no terminator and no position requirement, so an agent
summarising auth work ("Added two-factor authentication, the authenticator
app tests pass") refused the next prompt and, because the reason is
unconditional, published an idle agent as `permission` to every status reader.

The pair rule now demands the shape of a request for input: an action-flow
marker owning the bottom row as an unfinished clause, or an inquirer-style
`? ...authenticate?` dialog header. Topic wording (2fa, authenticator app,
device code) no longer carries the bottom row on its own. The bare-label rule
(`password:`) is likewise restricted to the bottom row, so printed k8s
manifests and form templates pass.

Widening `paste code here` and anchoring on the bottom row also closes the two
device-code flows the guard was missing: `gh auth login` and Claude `/login`.

Corpus: 25 must-refuse (23 existing + the 2 former false negatives), 50
must-allow (32 existing + the 18 confirmed false positives).
This commit is contained in:
Neil
2026-09-10 23:51:07 -07:00
parent 3d4aeef4ec
commit 833d9f0c08
2 changed files with 211 additions and 16 deletions
@@ -85,7 +85,22 @@ const LIVE_CREDENTIAL_SURFACES: readonly (readonly [string, string[]])[] = [
['access token ask', ['Provide your access token:']],
['otp ask', ['Type your OTP:']],
['bare credentials label', ['credentials:']],
['device code ask', ['Enter device code:']]
['device code ask', ['Enter device code:']],
[
'gh auth login device code',
[
'! First copy your one-time code: 1A2B-3C4D',
'Press Enter to open github.com in your browser...'
]
],
[
'claude /login paste-code screen',
[
"Browser didn't open? Use the url below to sign in:",
'https://claude.ai/oauth/authorize?code=true',
'Paste code here if prompted >'
]
]
]
const LEGITIMATE_AGENT_SCREENS: readonly (readonly [string, string[]])[] = [
@@ -244,6 +259,132 @@ const LEGITIMATE_AGENT_SCREENS: readonly (readonly [string, string[]])[] = [
'',
'› Ask Codex to do anything'
]
],
// An agent SUMMARISING auth work it just finished, with its own composer on the last row.
// These pair an auth verb with an auth-flow phrase, which is the shape that used to match
// with no prompt terminator and no position requirement at all.
[
'codex summarising two-factor work',
[
'• I implemented two-factor authentication for the login flow.',
' The authenticator app now generates a 6-digit code.',
'› Ask Codex to do anything'
]
],
[
'claude summarising two-factor work',
[
'· Added two-factor authentication. Tests for the authenticator app pass.',
'✳ Claude Code',
'> '
]
],
[
'codex summarising a sign-in button',
[
'• Added a Sign in with Google button; it logs the authorization result.',
'› Ask Codex to do anything'
]
],
[
'codex reporting an auth error it hit',
[
' └ ERROR: authentication required. Please sign in with the CLI.',
'› Ask Codex to do anything'
]
],
[
'agent asking whether MFA is wanted',
['· Should the app require MFA, or is authentication via password enough?', '> ']
],
[
'codex summarising a device-code flow it built',
[
'• The OAuth flow now shows a device code and waits for authentication.',
'› Ask Codex to do anything'
]
],
[
'rg hit on auth documentation',
[' └ docs/auth.md:12: Users authenticate with the authenticator app.', '> ']
],
[
'rg hits on a login component',
[
' └ src/Login.tsx:31: <button>Sign in with GitHub</button>',
' └ src/Login.tsx:44: // authorization code exchange',
'› Ask Codex to do anything'
]
],
[
'codex reporting MFA tests passing',
['• All MFA tests pass; authentication is wired end to end.', '› Ask Codex to do anything']
],
[
'codex quoting a build failure',
[
"• The build failed: 'authorization required'. You need to log in with `vercel login`.",
'› Ask Codex to do anything'
]
],
[
'rg hit on a readme auth section',
[
' └ docs/auth.md:3: ## Authentication',
' Users sign in with GitHub or an authenticator app.',
'> '
]
],
[
'claude summarising an oauth change',
[
'· Done — the OAuth login now requires authentication via the device code flow.',
'✳ Claude Code',
'> '
]
],
[
'gemini summarising SSO work',
['✦ Added SSO. Users authenticate with Okta; the sign in with SAML path is tested.', '◇ ']
],
[
'opencode wrapping an auth summary',
[
'Added requireAuth middleware. Unauthenticated requests get 401; sign in with the',
'token endpoint returns a JWT.',
'❯ '
]
],
[
'stack trace over a codex composer',
[
'Error: authentication required',
' at signInWithToken (auth.ts:22)',
'› Ask Codex to do anything'
]
],
[
'shell deploy failure',
[
'Running deploy...',
'ERROR: authentication required',
'Please sign in with the CLI and retry.',
'exit code 1'
]
],
// Printed config whose bare `password:` label is not the screen's bottom row.
[
'printed kubernetes secret manifest',
['kind: Secret', 'stringData:', ' password:', '› Ask Codex to do anything']
],
[
'printed signup form template',
[
'• The signup form now has these fields:',
' email:',
' password:',
'› Ask Codex to do anything'
]
]
]
@@ -17,6 +17,13 @@ import { startOfLastNonBlankLines } from './terminal-wait-tail-window'
* text into a credential field; a false positive only delays a prompt until the
* dialog is answered, and the wait poll re-evaluates the tail continuously, so
* a refusal clears on its own.
*
* That asymmetry is not a licence to match auth wording anywhere in the window:
* the refusal is unconditional, so a false positive also pins an idle agent to
* `permission` in the agent-status store. Every rule here therefore demands
* prompt *shape* — a terminator, or the bottom row of the screen — because an
* agent narrating auth work reads as prose and leaves its own composer caret on
* the last row.
*/
// Why bounded: an answered credential prompt stays in scrollback, and agents
@@ -32,6 +39,7 @@ const CREDENTIAL_NOUN_SOURCE =
'password|passphrase|api[ -]?keys?|access[ -]tokens?|auth(?:orization)?[ -]tokens?|bearer tokens?|personal access tokens?|secret keys?|client secrets?|one[- ]time (?:code|password)|otp|verification codes?|authentication codes?|security codes?|2fa codes?|device codes?|credentials?'
const CREDENTIAL_NOUN_RE = new RegExp(CREDENTIAL_NOUN_SOURCE, 'gi')
const CREDENTIAL_NOUN_ANYWHERE_RE = new RegExp(CREDENTIAL_NOUN_SOURCE, 'i')
// Why a vendor slot: real prompts read "enter your Anthropic API key" and
// "paste your personal access token", not just "enter your API key".
@@ -54,12 +62,29 @@ const CLAUSE_TERMINATED_RE = /[:›❯»>_]\s*$/
const SUDO_PASSWORD_RE = /^[^a-z0-9]{0,8}\[sudo\]\s+password for\b/i
const GIT_CREDENTIAL_RE = /^[^a-z0-9]{0,8}(?:username|password) for ['"]?[a-z][a-z0-9+.-]*:\/\//i
// Why two markers: a lone auth verb is narration. A device-code or sign-in
// dialog always pairs the verb with a flow marker in the same live window.
const AUTH_VERB_RE =
/\b(?:sign[ -]?in|signin|log[ -]?in|authenticate|authorized?|authorization|authentication)\b/i
const AUTH_FLOW_RE =
/\b(?:sign[ -]?in with|log[ -]?in with|authenticate with|authentication required|authorization required|sign[ -]?in required|login required|enter (?:the )?code|device code|verification code|waiting for (?:authentication|authorization|you to)|open (?:this|the following) url|press enter to (?:open|sign)|paste (?:it|the code) (?:here|below)|two[ -]factor|2fa|authenticator app|mfa|\d-digit code)\b/i
// Wording only a dialog addressing the user uses.
const AUTH_ACTION_FLOW_SOURCE =
'sign[ -]?in with|log[ -]?in with|authenticate with|authentication required|authorization required|sign[ -]?in required|login required|enter (?:the )?code|waiting for (?:authentication|authorization|you to)|open (?:this|the following) url|press enter to (?:open|sign)|paste (?:it|(?:the |your )?code) (?:here|below)'
// Wording equally at home in a dialog and in narration about auth work.
const AUTH_TOPIC_FLOW_SOURCE =
'device code|verification code|two[ -]factor|2fa|authenticator app|mfa|\\d-digit code'
const AUTH_ACTION_FLOW_RE = new RegExp(`\\b(?:${AUTH_ACTION_FLOW_SOURCE})\\b`, 'i')
const AUTH_FLOW_RE = new RegExp(
`\\b(?:${AUTH_ACTION_FLOW_SOURCE}|${AUTH_TOPIC_FLOW_SOURCE})\\b`,
'i'
)
// An inquirer-style question row. Prose never starts with a bare `?`, so a `?`
// row asking about auth is a dialog header even when its options wrap below it.
const AUTH_QUESTION_ROW_RE = /^\?\s+\S/
// A finished sentence, i.e. narration. A lone `.`/`!`/`?` ends a clause; `...`
// and `…` are progress wording ("opening browser...") and are not sentences.
const NARRATION_ROW_RE = /(?:^|[^.])\.(?:\s|$)|[!?]\s*$/
/**
* Cheap superset of everything `findCredentialPromptIndex` can match, tested
@@ -68,11 +93,17 @@ const AUTH_FLOW_RE =
*/
export const TERMINAL_CREDENTIAL_PROMPT_SENTINEL_RE = new RegExp(
`(?:(?:${CREDENTIAL_NOUN_SOURCE}|username for)[^\\n]{0,64}[:?>›❯»_]\\s*$)|` +
`(?:${AUTH_FLOW_RE.source})`,
`(?:${AUTH_FLOW_RE.source})|` +
`(?:^\\s*\\?\\s+[^\\n]{0,120}(?:${AUTH_VERB_RE.source}))`,
'im'
)
function isCredentialPromptLine(line: string): boolean {
/**
* Whether `line` is itself a credential prompt. `isLastRow` gates the bare-label
* form (`password:`) because a label with no ask verb is also how printed k8s
* manifests and form templates read; only the screen's bottom row is a prompt.
*/
function isCredentialPromptLine(line: string, isLastRow: boolean): boolean {
if (line.length > MAX_CREDENTIAL_LINE_LENGTH) {
return false
}
@@ -93,7 +124,10 @@ function isCredentialPromptLine(line: string): boolean {
if (!terminated) {
continue
}
if (CREDENTIAL_ASK_PREFIX_RE.test(prefix) || CREDENTIAL_LABEL_PREFIX_RE.test(prefix)) {
if (CREDENTIAL_ASK_PREFIX_RE.test(prefix)) {
return true
}
if (isLastRow && CREDENTIAL_LABEL_PREFIX_RE.test(prefix)) {
return true
}
}
@@ -104,24 +138,44 @@ function isCredentialPromptLine(line: string): boolean {
export function findCredentialPromptIndex(normalized: string): number | null {
const windowStart = startOfLastNonBlankLines(normalized, CREDENTIAL_TAIL_LINES)
const tail = normalized.slice(windowStart)
const raws = tail.split('\n')
// Why: dialogs are drawn inside box rules, which otherwise glue the frame to the wording.
const lines = raws.map((raw) => raw.replace(/[\u2500-\u257f]+/g, ' ').trim())
const lastRow = lines.findLastIndex((line) => line.length > 0)
let offset = 0
let promptIndex: number | null = null
let sawAuthVerb = false
let sawAuthFlow = false
for (const raw of tail.split('\n')) {
// Why: dialogs are drawn inside box rules, which otherwise glue the frame to the wording.
const line = raw.replace(/[\u2500-\u257f]+/g, ' ').trim()
if (isCredentialPromptLine(line)) {
let sawCredentialNoun = false
let sawAuthQuestion = false
for (let index = 0; index < lines.length; index += 1) {
const line = lines[index]
if (isCredentialPromptLine(line, index === lastRow)) {
promptIndex = windowStart + offset
}
if (line.length <= MAX_CREDENTIAL_LINE_LENGTH) {
sawAuthVerb = sawAuthVerb || AUTH_VERB_RE.test(line)
sawAuthFlow = sawAuthFlow || AUTH_FLOW_RE.test(line)
sawCredentialNoun = sawCredentialNoun || CREDENTIAL_NOUN_ANYWHERE_RE.test(line)
sawAuthQuestion =
sawAuthQuestion ||
(AUTH_QUESTION_ROW_RE.test(line) && (AUTH_VERB_RE.test(line) || AUTH_FLOW_RE.test(line)))
}
offset += raw.length + 1
offset += raws[index].length + 1
}
if (promptIndex !== null) {
return promptIndex
}
return sawAuthVerb && sawAuthFlow ? windowStart : null
// A flow marker alone is narration ("added the 2fa tests"). It is a live auth
// surface only when it owns the bottom row as an unfinished request, or when
// an interactive auth question drew the screen.
const bottomRow = lastRow === -1 ? '' : lines[lastRow]
const bottomRowAsks =
AUTH_ACTION_FLOW_RE.test(bottomRow) ||
(AUTH_FLOW_RE.test(bottomRow) && CLAUSE_TERMINATED_RE.test(bottomRow))
const authFlowOwnsBottom =
bottomRow.length <= MAX_CREDENTIAL_LINE_LENGTH &&
bottomRowAsks &&
!NARRATION_ROW_RE.test(bottomRow) &&
(sawAuthVerb || sawCredentialNoun)
return authFlowOwnsBottom || sawAuthQuestion ? windowStart : null
}