fix(mobile): harden grouped Claude question answers

This commit is contained in:
Merge Sim
2026-09-06 10:31:06 -07:00
parent 585cfb90da
commit f011ba1f59
8 changed files with 367 additions and 70 deletions
@@ -0,0 +1,82 @@
import { createElement } from 'react'
import { act, create, type ReactTestRenderer } from 'react-test-renderer'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { MobileNativeChatQuestion } from './MobileNativeChatQuestion'
vi.mock('react-native', () => ({
Pressable: 'Pressable',
StyleSheet: { create: (styles: unknown) => styles, hairlineWidth: 1 },
Text: 'Text',
TextInput: 'TextInput',
View: 'View'
}))
vi.mock('lucide-react-native', () => ({
ArrowUp: 'ArrowUp',
Check: 'Check',
CircleHelp: 'CircleHelp'
}))
describe('MobileNativeChatQuestion', () => {
let renderer: ReactTestRenderer | null = null
afterEach(() => {
act(() => renderer?.unmount())
renderer = null
})
it('submits the selected duplicate-label row by position', async () => {
const onAnswer = vi.fn(async () => true)
await act(async () => {
renderer = create(
createElement(MobileNativeChatQuestion, {
question: {
question: 'Pick regions',
options: ['Region', 'Region'],
multiSelect: true,
allowOther: false,
optionTokens: ['first-token', 'second-token']
},
onAnswer
})
)
})
const choices = renderer.root.findAllByProps({ accessibilityRole: 'checkbox' })
await act(async () => choices[1]!.props.onPress())
const submit = renderer.root.findByProps({ accessibilityLabel: 'Submit selected options' })
await act(async () => submit.props.onPress())
expect(onAnswer).toHaveBeenCalledWith('second-token')
})
it('submits structured multi-select choices together with other text', async () => {
const onAnswer = vi.fn(async () => true)
await act(async () => {
renderer = create(
createElement(MobileNativeChatQuestion, {
question: {
question: 'Pick regions',
options: ['us-east', 'eu-west'],
multiSelect: true,
allowOther: true,
optionTokens: ['east-token', 'west-token'],
freeTextToken: 'other-token'
},
onAnswer
})
)
})
const choices = renderer.root.findAllByProps({ accessibilityRole: 'checkbox' })
await act(async () => choices[0]!.props.onPress())
const input = renderer.root.findByType('TextInput')
await act(async () => input.props.onChangeText('ap-south'))
const submit = renderer.root.findByProps({ accessibilityLabel: 'Submit selected options' })
await act(async () => submit.props.onPress())
expect(onAnswer).toHaveBeenCalledWith('east-token, other-token:ap-south')
})
})
+50 -16
View File
@@ -4,6 +4,8 @@ import { ArrowUp, Check, CircleHelp } from 'lucide-react-native'
import { colors, radii, spacing, typography } from '../theme/mobile-theme'
import {
formatQuestionAnswer,
formatQuestionAnswerByIndexes,
formatQuestionAnswerWithOtherByIndexes,
formatQuestionFreeTextAnswer,
type MobileChatQuestion
} from './mobile-native-chat-question'
@@ -18,7 +20,7 @@ type Props = {
* the user answer freely (the escape hatch) when the heuristic misreads the
* options or none apply. */
export function MobileNativeChatQuestion({ question, onAnswer }: Props): React.JSX.Element {
const [selected, setSelected] = useState<string[]>([])
const [selectedOptionIndexes, setSelectedOptionIndexes] = useState<number[]>([])
const [freeText, setFreeText] = useState('')
const [sending, setSending] = useState(false)
const sendingRef = useRef(false)
@@ -27,9 +29,11 @@ export function MobileNativeChatQuestion({ question, onAnswer }: Props): React.J
const hasOptions = question.options.length > 0
const trimmedFreeText = freeText.trim()
const toggle = (option: string): void => {
setSelected((prev) =>
prev.includes(option) ? prev.filter((o) => o !== option) : [...prev, option]
const toggle = (optionIndex: number): void => {
setSelectedOptionIndexes((prev) =>
prev.includes(optionIndex)
? prev.filter((index) => index !== optionIndex)
: [...prev, optionIndex]
)
}
@@ -53,28 +57,43 @@ export function MobileNativeChatQuestion({ question, onAnswer }: Props): React.J
}
const submitMulti = async (): Promise<void> => {
if (selected.length === 0) {
if (selectedOptionIndexes.length === 0) {
return
}
await sendAnswer(formatQuestionAnswer(question, selected))
const answer =
question.freeTextToken && trimmedFreeText.length > 0
? formatQuestionAnswerWithOtherByIndexes(question, selectedOptionIndexes, trimmedFreeText)
: formatQuestionAnswerByIndexes(question, selectedOptionIndexes)
if (await sendAnswer(answer)) {
setFreeText('')
}
}
const submitFreeText = async (): Promise<void> => {
if (trimmedFreeText.length === 0) {
return
}
if (await sendAnswer(formatQuestionFreeTextAnswer(question, trimmedFreeText))) {
const answer =
question.multiSelect && question.freeTextToken && selectedOptionIndexes.length > 0
? formatQuestionAnswerWithOtherByIndexes(question, selectedOptionIndexes, trimmedFreeText)
: formatQuestionFreeTextAnswer(question, trimmedFreeText)
if (await sendAnswer(answer)) {
setFreeText('')
}
}
const canSubmitMulti = selected.length > 0 && !sending
const canSubmitMulti = selectedOptionIndexes.length > 0 && !sending
const canSendFreeText = allowOther && trimmedFreeText.length > 0 && !sending
// Stable keys for option rows even if an agent repeats a label.
const optionRows = useMemo(
() => question.options.map((label, index) => ({ label, key: `${index}:${label}` })),
[question.options]
() =>
question.options.map((label, index) => ({
label,
description: question.optionDescriptions?.[index],
key: `${index}:${label}`
})),
[question.optionDescriptions, question.options]
)
return (
@@ -86,8 +105,8 @@ export function MobileNativeChatQuestion({ question, onAnswer }: Props): React.J
{hasOptions ? (
<View style={styles.options}>
{optionRows.map(({ label, key }, optIndex) => {
const isSelected = selected.includes(label)
{optionRows.map(({ label, description, key }, optIndex) => {
const isSelected = selectedOptionIndexes.includes(optIndex)
return (
<Pressable
key={key}
@@ -99,7 +118,7 @@ export function MobileNativeChatQuestion({ question, onAnswer }: Props): React.J
pressed && styles.pressed
]}
onPress={() =>
question.multiSelect ? toggle(label) : answerSingle(label, optIndex)
question.multiSelect ? toggle(optIndex) : answerSingle(label, optIndex)
}
>
{question.multiSelect ? (
@@ -107,7 +126,14 @@ export function MobileNativeChatQuestion({ question, onAnswer }: Props): React.J
{isSelected ? <Check size={13} color={colors.bgBase} strokeWidth={3} /> : null}
</View>
) : null}
<Text style={styles.optionText}>{label}</Text>
<View style={styles.optionBody}>
<Text style={styles.optionText}>{label}</Text>
{description ? (
<Text style={styles.optionDescription} numberOfLines={2}>
{description}
</Text>
) : null}
</View>
</Pressable>
)
})}
@@ -126,7 +152,7 @@ export function MobileNativeChatQuestion({ question, onAnswer }: Props): React.J
disabled={!canSubmitMulti}
>
<Text style={[styles.submitText, !canSubmitMulti && styles.submitTextDisabled]}>
Submit{selected.length > 0 ? ` (${selected.length})` : ''}
Submit{selectedOptionIndexes.length > 0 ? ` (${selectedOptionIndexes.length})` : ''}
</Text>
</Pressable>
) : null}
@@ -207,11 +233,19 @@ const styles = StyleSheet.create({
optionSelected: {
borderColor: colors.accentBlue
},
optionText: {
optionBody: {
flex: 1,
gap: 2
},
optionText: {
color: colors.textPrimary,
fontSize: typography.bodySize + 1
},
optionDescription: {
color: colors.textMuted,
fontSize: typography.metaSize,
lineHeight: typography.metaSize + 5
},
checkbox: {
width: 20,
height: 20,
@@ -13,6 +13,8 @@ export type MobileChatQuestion = {
* parallel to `options`. Null where the option was a plain bullet. Used to
* echo the exact choice the agent listed back to the terminal. */
optionTokens: (string | null)[]
/** Per-option secondary text from structured prompts, parallel to `options`. */
optionDescriptions?: (string | undefined)[]
/** Opaque prefix used when free-text answers must target a specific prompt. */
freeTextToken?: string
}
@@ -130,6 +132,48 @@ export function parseAgentQuestion(text: string): MobileChatQuestion | null {
}
}
function formatQuestionOptionAtIndex(question: MobileChatQuestion, index: number): string | null {
if (!Number.isInteger(index) || index < 0 || index >= question.options.length) {
return null
}
const label = question.options[index]
if (label == null || label.trim().length === 0) {
return null
}
const token = question.optionTokens[index]
return token != null && token.length > 0 ? token : label
}
function formatQuestionAnswerPartsByIndexes(
question: MobileChatQuestion,
selectedIndexes: number[]
): string[] {
return selectedIndexes
.map((index) => formatQuestionOptionAtIndex(question, index))
.filter((part): part is string => part != null && part.trim().length > 0)
}
export function formatQuestionAnswerByIndexes(
question: MobileChatQuestion,
selectedIndexes: number[]
): string {
const parts = formatQuestionAnswerPartsByIndexes(question, selectedIndexes)
return parts.join(question.multiSelect ? ', ' : ' ')
}
export function formatQuestionAnswerWithOtherByIndexes(
question: MobileChatQuestion,
selectedIndexes: number[],
text: string
): string {
const parts = formatQuestionAnswerPartsByIndexes(question, selectedIndexes)
const other = formatQuestionFreeTextAnswer(question, text)
if (other.length > 0) {
parts.push(other)
}
return parts.join(question.multiSelect ? ', ' : ' ')
}
/**
* Build the text to send to the agent terminal for the selected option(s).
* Convention: echo the option's leading marker (number/letter) when the list had
@@ -150,8 +194,7 @@ export function formatQuestionAnswer(question: MobileChatQuestion, selected: str
// Free-text / unknown entry: pass the user's text straight through.
return label
}
const token = question.optionTokens[index]
return token != null && token.length > 0 ? token : label
return formatQuestionOptionAtIndex(question, index) ?? label
})
return parts.join(question.multiSelect ? ', ' : ' ')
@@ -20,7 +20,7 @@ function groupedPrompt(): StructuredQuestionItem {
id: 'q1',
question: 'Which database?',
multiSelect: false,
options: [{ id: 'q1:choice-1', label: 'Postgres' }],
options: [{ id: 'q1:choice-1', label: 'Postgres', description: 'Durable server' }],
freeTextQuestionId: 'q1'
},
{
@@ -42,6 +42,7 @@ describe('structured question projection for grouped Claude prompts', () => {
expect(projected?.question).not.toBe('2 grouped questions from Claude')
expect(projected?.options).toEqual(['Postgres'])
expect(projected?.optionDescriptions).toEqual(['Durable server'])
expect(projected?.optionTokens.filter(Boolean)).toHaveLength(1)
})
})
@@ -161,9 +161,11 @@ export function projectStructuredQuestion(
groupedQuestionPromptKey(prompt.itemId, prompt.revision)
)
}
const optionDescriptions = prompt.body.options.map((option) => option.description)
return {
question: prompt.body.question,
options: prompt.body.options.map((option) => option.label),
...(optionDescriptions.some(Boolean) ? { optionDescriptions } : {}),
multiSelect: false,
allowOther: Boolean(prompt.body.freeTextQuestionId),
optionTokens: prompt.body.options.map((option) =>
@@ -123,6 +123,22 @@ describe('mobile structured grouped questions', () => {
).toEqual([{ questionId: 'q1', optionIds: [], other: 'DuckDB' }])
})
it('keeps selected options and other text for grouped multi-select answers', () => {
const questions = [SECOND]
const only = projectGroupedQuestion(questions, null, PROMPT_KEY)!
const result = advanceGroupedQuestion({
response: `${tapOption(only, 0)}, ${formatQuestionFreeTextAnswer(only, 'ap-south')}`,
questions,
draft: null,
promptKey: PROMPT_KEY
})
expect(
decodeAgentSessionQuestionAnswers(result?.kind === 'submit' ? result.optionId : '')
).toEqual([{ questionId: 'q2', optionIds: ['q2:choice-1'], other: 'ap-south' }])
})
it('gives each step a distinct card key so a selection cannot carry into the next question', () => {
// The view keys MobileNativeChatQuestion by this value; an identical key would reuse the
// mounted card and submit step 1's checkboxes as step 2's answer. Claude can legitimately ask
@@ -169,6 +185,61 @@ describe('mobile structured grouped questions', () => {
).toBeNull()
})
it('refuses an option token rendered for a superseded prompt revision', () => {
const stale = projectGroupedQuestion([question()], null, groupedQuestionPromptKey('item-1', 2))!
expect(
advanceGroupedQuestion({
response: tapOption(stale, 0),
questions: [question()],
draft: null,
promptKey: PROMPT_KEY
})
).toBeNull()
})
it('refuses free text rendered for a superseded prompt revision', () => {
const stale = projectGroupedQuestion([question()], null, groupedQuestionPromptKey('item-1', 2))!
expect(
advanceGroupedQuestion({
response: formatQuestionFreeTextAnswer(stale, 'stale answer'),
questions: [question()],
draft: null,
promptKey: PROMPT_KEY
})
).toBeNull()
})
it('rejects a multi-select response when one selected token is malformed', () => {
const questions = [SECOND]
const only = projectGroupedQuestion(questions, null, PROMPT_KEY)!
expect(
advanceGroupedQuestion({
response: `${tapOption(only, 0)}, not-a-grouped-token`,
questions,
draft: null,
promptKey: PROMPT_KEY
})
).toBeNull()
})
it('rejects a multi-select response when one selected token belongs to another prompt', () => {
const questions = [SECOND]
const current = projectGroupedQuestion(questions, null, PROMPT_KEY)!
const stale = projectGroupedQuestion(questions, null, groupedQuestionPromptKey('item-1', 2))!
expect(
advanceGroupedQuestion({
response: `${tapOption(current, 0)}, ${tapOption(stale, 1)}`,
questions,
draft: null,
promptKey: PROMPT_KEY
})
).toBeNull()
})
it('refuses an empty multi-select rather than sending a group the host would reject', () => {
const questions = [SECOND]
const only = projectGroupedQuestion(questions, null, PROMPT_KEY)!
@@ -26,8 +26,8 @@ export type GroupedQuestionAdvance =
const GROUPED_TOKEN_PREFIX = 'structured-grouped-question:'
type GroupedTokenPayload =
| { kind: 'option'; questionId: string; optionId: string }
| { kind: 'free-text'; questionId: string }
| { kind: 'option'; promptKey: string; questionId: string; optionId: string }
| { kind: 'free-text'; promptKey: string; questionId: string }
export function groupedQuestionPromptKey(itemId: string, revision: number): string {
return `${itemId}:${revision}`
@@ -45,14 +45,19 @@ function decodeGroupedToken(value: string): GroupedTokenPayload | null {
const decoded = JSON.parse(
decodeURIComponent(value.slice(GROUPED_TOKEN_PREFIX.length))
) as Record<string, unknown>
if (typeof decoded.questionId !== 'string') {
if (typeof decoded.promptKey !== 'string' || typeof decoded.questionId !== 'string') {
return null
}
if (decoded.kind === 'option' && typeof decoded.optionId === 'string') {
return { kind: 'option', questionId: decoded.questionId, optionId: decoded.optionId }
return {
kind: 'option',
promptKey: decoded.promptKey,
questionId: decoded.questionId,
optionId: decoded.optionId
}
}
if (decoded.kind === 'free-text') {
return { kind: 'free-text', questionId: decoded.questionId }
return { kind: 'free-text', promptKey: decoded.promptKey, questionId: decoded.questionId }
}
} catch {
return null
@@ -60,7 +65,11 @@ function decodeGroupedToken(value: string): GroupedTokenPayload | null {
return null
}
function decodeGroupedFreeTextAnswer(value: string): { questionId: string; answer: string } | null {
function decodeGroupedFreeTextAnswer(value: string): {
promptKey: string
questionId: string
answer: string
} | null {
if (!value.startsWith(GROUPED_TOKEN_PREFIX)) {
return null
}
@@ -70,9 +79,18 @@ function decodeGroupedFreeTextAnswer(value: string): { questionId: string; answe
return null
}
const payload = decodeGroupedToken(value.slice(0, separator))
return payload?.kind === 'free-text'
? { questionId: payload.questionId, answer: decodeURIComponent(value.slice(separator + 1)) }
: null
if (payload?.kind !== 'free-text') {
return null
}
try {
return {
promptKey: payload.promptKey,
questionId: payload.questionId,
answer: decodeURIComponent(value.slice(separator + 1))
}
} catch {
return null
}
}
/** Answers already collected for this exact prompt revision; a stale draft counts as none. */
@@ -95,17 +113,30 @@ export function projectGroupedQuestion(
return null
}
const heading = question.header ? `${question.header}: ${question.question}` : question.question
const optionDescriptions = question.options.map((option) => option.description)
return {
question:
questions.length > 1 ? `${heading} (${answered + 1} of ${questions.length})` : heading,
options: question.options.map((option) => option.label),
...(optionDescriptions.some(Boolean) ? { optionDescriptions } : {}),
multiSelect: question.multiSelect,
allowOther: Boolean(question.freeTextQuestionId),
optionTokens: question.options.map((option) =>
encodeGroupedToken({ kind: 'option', questionId: question.id, optionId: option.id })
encodeGroupedToken({
kind: 'option',
promptKey,
questionId: question.id,
optionId: option.id
})
),
...(question.freeTextQuestionId
? { freeTextToken: encodeGroupedToken({ kind: 'free-text', questionId: question.id }) }
? {
freeTextToken: encodeGroupedToken({
kind: 'free-text',
promptKey,
questionId: question.id
})
}
: {})
}
}
@@ -113,33 +144,56 @@ export function projectGroupedQuestion(
/** Read one step's answer out of what the question card sent back. */
function answerFromResponse(
response: string,
question: AgentJournalQuestion
question: AgentJournalQuestion,
promptKey: string
): AgentSessionQuestionAnswer | null {
const freeText = decodeGroupedFreeTextAnswer(response)
if (freeText) {
const other = freeText.answer.trim()
return freeText.questionId === question.id && other
? { questionId: question.id, optionIds: [], other }
: null
// Multi-select submits comma-joined parts; tokens and free text are encoded, so the separator is stable.
const optionIds: string[] = []
let other: string | undefined
for (const part of response.split(', ')) {
const trimmed = part.trim()
const freeText = decodeGroupedFreeTextAnswer(trimmed)
if (freeText) {
const answer = freeText.answer.trim()
if (
freeText.promptKey !== promptKey ||
freeText.questionId !== question.id ||
answer.length === 0 ||
other !== undefined
) {
return null
}
other = answer
continue
}
const payload = decodeGroupedToken(trimmed)
if (
payload?.kind !== 'option' ||
payload.promptKey !== promptKey ||
payload.questionId !== question.id
) {
return null
}
optionIds.push(payload.optionId)
}
// Multi-select submits the selected option tokens as one comma-joined string; option tokens are
// percent-encoded, so no token can contain the separator.
const optionIds = response
.split(', ')
.map((part) => decodeGroupedToken(part.trim()))
.flatMap((payload) =>
payload?.kind === 'option' && payload.questionId === question.id ? [payload.optionId] : []
)
const offered = new Set(question.options.map((option) => option.id))
return optionIds.length > 0 && optionIds.every((optionId) => offered.has(optionId))
? { questionId: question.id, optionIds }
: null
if (optionIds.some((optionId) => !offered.has(optionId))) {
return null
}
if (other && !question.freeTextQuestionId) {
return null
}
const answerCount = optionIds.length + (other ? 1 : 0)
if (answerCount === 0 || (!question.multiSelect && answerCount !== 1)) {
return null
}
return { questionId: question.id, optionIds, ...(other ? { other } : {}) }
}
/**
* Fold one answer into the draft. Returns `advance` while questions remain and `submit` with the
* encoded group once the last one lands; null when the response does not answer the current step,
* so the caller can fall through to the flat single-question path.
* encoded group once the last one lands; null when the response does not answer this prompt step.
*/
export function advanceGroupedQuestion(args: {
response: string
@@ -152,7 +206,7 @@ export function advanceGroupedQuestion(args: {
if (!question) {
return null
}
const answer = answerFromResponse(args.response, question)
const answer = answerFromResponse(args.response, question, args.promptKey)
if (!answer) {
return null
}
@@ -65,22 +65,36 @@ export function useMobileStructuredPromptResponses(args: {
const respondQuestion = useCallback(
async (answer: string): Promise<boolean> => {
const prompt = stateRef.current.items.find(pendingStructuredQuestion) ?? null
const grouped = prompt?.body.questions
? advanceGroupedQuestion({
response: answer,
questions: prompt.body.questions,
draft: groupedDraft,
promptKey: groupedQuestionPromptKey(prompt.itemId, prompt.revision)
})
: null
if (grouped?.kind === 'advance') {
setCollected({ sessionKey, draft: grouped.draft })
return true
if (prompt?.body.questions) {
const grouped = advanceGroupedQuestion({
response: answer,
questions: prompt.body.questions,
draft: groupedDraft,
promptKey: groupedQuestionPromptKey(prompt.itemId, prompt.revision)
})
if (!grouped) {
return false
}
if (grouped.kind === 'advance') {
setCollected({ sessionKey, draft: grouped.draft })
return true
}
const result = await mutate<AgentSessionPromptResult>(
'agentSession.respondToQuestion',
'agentSession.respondTo:question',
{ itemId: prompt.itemId, expectedRevision: prompt.revision, optionId: grouped.optionId }
)
if (result.status !== 'rejected') {
// The group left the phone; a retry must start from the first question, not a stale tail.
setCollected(null)
}
if (result.status === 'unknown') {
onSendError('Answer unconfirmed — check chat before retrying')
return false
}
return result.status === 'accepted'
}
const target =
grouped && prompt
? { itemId: prompt.itemId, expectedRevision: prompt.revision, optionId: grouped.optionId }
: structuredQuestionResponseTarget(answer, prompt)
const target = structuredQuestionResponseTarget(answer, prompt)
if (!target) {
return false
}
@@ -89,10 +103,6 @@ export function useMobileStructuredPromptResponses(args: {
'agentSession.respondTo:question',
target
)
if (grouped && result.status !== 'rejected') {
// The group left the phone; a retry must start from the first question, not a stale tail.
setCollected(null)
}
if (result.status === 'unknown') {
onSendError('Answer unconfirmed — check chat before retrying')
return false