From 32e258d7fc42f098fd077d4c8fc6eca2dfeddc4b Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:31:12 -0700 Subject: [PATCH] feat(mobile): show attached images as removable thumbnails in native chat Attaching a picture in mobile native chat gave no visible feedback: the image uploaded to the host and its path was pasted into the hidden PTY, so nothing on screen ever showed it was attached. Defer the TUI paste to send time and hold picked images in composer state, rendered as a thumbnail tray above the input (spinner while uploading, X to remove, image-only sends allowed). On send, each ready image's path is pasted bracketed into the agent TUI right before the message text, matching the desktop composer-attachment model. The terminal live-input surface keeps the immediate paste, where the TUI's own input line is the feedback. --- .../app/h/[hostId]/session/[worktreeId].tsx | 38 ++- .../src/session/MobileNativeChatComposer.tsx | 225 ++++++++++++------ .../src/session/MobileNativeChatOverlay.tsx | 13 +- mobile/src/session/MobileNativeChatView.tsx | 7 + .../mobile-image-source-picker.test.ts | 7 +- .../src/session/mobile-image-source-picker.ts | 7 +- .../session/use-mobile-image-attachment.ts | 29 ++- .../use-mobile-native-chat-image-send.test.ts | 133 +++++++++++ .../use-mobile-native-chat-image-send.ts | 95 ++++++++ ...-mobile-native-chat-pending-images.test.ts | 171 +++++++++++++ .../use-mobile-native-chat-pending-images.ts | 178 ++++++++++++++ 11 files changed, 816 insertions(+), 87 deletions(-) create mode 100644 mobile/src/session/use-mobile-native-chat-image-send.test.ts create mode 100644 mobile/src/session/use-mobile-native-chat-image-send.ts create mode 100644 mobile/src/session/use-mobile-native-chat-pending-images.test.ts create mode 100644 mobile/src/session/use-mobile-native-chat-pending-images.ts diff --git a/mobile/app/h/[hostId]/session/[worktreeId].tsx b/mobile/app/h/[hostId]/session/[worktreeId].tsx index dee14cdd387..63da7c98ed3 100644 --- a/mobile/app/h/[hostId]/session/[worktreeId].tsx +++ b/mobile/app/h/[hostId]/session/[worktreeId].tsx @@ -176,6 +176,8 @@ import { import type { MobileNewTabAgentOption } from '../../../../src/session/mobile-new-tab-agent-options' import { loadMobileNewTabAgentOptions } from '../../../../src/session/mobile-new-tab-agent-loader' import { useMobileImageAttachment } from '../../../../src/session/use-mobile-image-attachment' +import { useMobileNativeChatPendingImages } from '../../../../src/session/use-mobile-native-chat-pending-images' +import { useMobileNativeChatImageSend } from '../../../../src/session/use-mobile-native-chat-image-send' import { useMobileAttachmentInputLeaseGate } from '../../../../src/session/use-mobile-attachment-input-lease-gate' import { useMobileTerminalPaste } from '../../../../src/session/use-mobile-terminal-paste' import { useTerminalLiveInputModePreference } from '../../../../src/session/use-terminal-live-input-mode-preference' @@ -3637,6 +3639,37 @@ export default function SessionScreen() { onError: triggerError }) + // Native-chat attachments defer the TUI paste to send time so the composer + // thumbnail stays removable; the terminal surface keeps the immediate paste + // (the TUI's own input line is the visible feedback there). + const { + pendingChatImages, + attachPendingChatImage, + removePendingChatImage, + getReadyChatImages, + consumePendingChatImages + } = useMobileNativeChatPendingImages({ + client, + activeHandle, + canAttach: canSend, + connState, + getConnectionId: getActiveWorktreeConnectionId, + showToast, + onSuccess: triggerSelection, + onError: triggerError + }) + + const sendNativeChatWithImages = useMobileNativeChatImageSend({ + clientRef, + activeHandleRef, + deviceTokenRef, + inputLeaseReadyRef: nativeChatInputLeaseReadyRef, + sendText: nativeChatController.handleNativeChatSend, + getReadyImages: getReadyChatImages, + consumeImages: consumePendingChatImages, + onSendError: showNativeChatSendError + }) + // Why: refresh canPaste on mount, AppState active, after paste. useEffect(() => { let mounted = true @@ -4687,8 +4720,11 @@ export default function SessionScreen() { ))} void attachImage('library')} + onAttachImage={() => void attachPendingChatImage('library')} isAttaching={isAttaching} + onSend={sendNativeChatWithImages} + pendingImages={pendingChatImages} + onRemovePendingImage={removePendingChatImage} onMicPress={handleDictationToggle} micActive={dictation.isRecording} dictationMode={dictationMode} diff --git a/mobile/src/session/MobileNativeChatComposer.tsx b/mobile/src/session/MobileNativeChatComposer.tsx index ba02fe3d44b..0c71ad1c9a3 100644 --- a/mobile/src/session/MobileNativeChatComposer.tsx +++ b/mobile/src/session/MobileNativeChatComposer.tsx @@ -1,6 +1,7 @@ import { useEffect, useMemo, useRef, useState } from 'react' import { ActivityIndicator, + Image, Pressable, ScrollView, StyleSheet, @@ -8,8 +9,9 @@ import { TextInput, View } from 'react-native' -import { ArrowUp, ImagePlus, Mic, Square } from 'lucide-react-native' +import { ArrowUp, ImagePlus, Mic, Square, X } from 'lucide-react-native' import { colors, radii, spacing, typography } from '../theme/mobile-theme' +import type { MobileNativeChatPendingImage } from './use-mobile-native-chat-pending-images' import { applyAutocomplete, detectAutocompleteTrigger, @@ -30,6 +32,7 @@ const SLASH_COMMANDS = [ ] const NO_FILE_PATHS: string[] = [] +const NO_PENDING_IMAGES: MobileNativeChatPendingImage[] = [] type Props = { /** Controlled composer text — owned by the parent so dictation can write to it. */ @@ -38,6 +41,9 @@ type Props = { onSend: (text: string) => Promise onAttachImage?: () => void isAttaching?: boolean + /** Picked-but-unsent images shown as removable thumbnails above the input. */ + pendingImages?: readonly MobileNativeChatPendingImage[] + onRemovePendingImage?: (id: string) => void onMicPress?: () => void micActive?: boolean /** Dictation trigger style — 'hold' uses press-in/out, 'toggle' uses tap. */ @@ -56,6 +62,8 @@ export function MobileNativeChatComposer({ onSend, onAttachImage, isAttaching = false, + pendingImages = NO_PENDING_IMAGES, + onRemovePendingImage, onMicPress, micActive = false, dictationMode = 'toggle', @@ -76,7 +84,15 @@ export function MobileNativeChatComposer({ const sendingRef = useRef(false) const [sending, setSending] = useState(false) const trimmed = value.trim() - const canSend = trimmed.length > 0 && !disabled && !sending && !isAttaching + const readyImageCount = pendingImages.filter((image) => image.status === 'ready').length + const uploadingImages = pendingImages.length > readyImageCount + // Image-only sends are allowed; uploads must finish so send can't race them. + const canSend = + (trimmed.length > 0 || readyImageCount > 0) && + !disabled && + !sending && + !isAttaching && + !uploadingImages const trigger = useMemo(() => detectAutocompleteTrigger(value, cursor), [value, cursor]) const suggestions = useMemo(() => { @@ -145,72 +161,106 @@ export function MobileNativeChatComposer({ ) : null} - - {onAttachImage ? ( - [styles.iconButton, pressed && styles.pressed]} - onPress={onAttachImage} - disabled={isAttaching || disabled} - > - {isAttaching ? ( - - ) : ( - - )} - + + {pendingImages.length > 0 ? ( + + {pendingImages.map((image) => ( + + + {image.status === 'uploading' ? ( + + + + ) : null} + {onRemovePendingImage ? ( + [styles.attachmentRemove, pressed && styles.pressed]} + onPress={() => onRemovePendingImage(image.id)} + hitSlop={8} + accessibilityLabel="Remove attached image" + > + + + ) : null} + + ))} + ) : null} - { - setCursor(e.nativeEvent.selection.end) - setPendingSelection(null) - }} - placeholder={placeholder} - placeholderTextColor={colors.textMuted} - selectionColor={colors.accentBlue} - multiline - editable={!disabled} - textAlignVertical="top" - /> - {onMicPress ? ( + + {onAttachImage ? ( + [styles.iconButton, pressed && styles.pressed]} + onPress={onAttachImage} + disabled={isAttaching || disabled} + > + {isAttaching ? ( + + ) : ( + + )} + + ) : null} + { + setCursor(e.nativeEvent.selection.end) + setPendingSelection(null) + }} + placeholder={placeholder} + placeholderTextColor={colors.textMuted} + selectionColor={colors.accentBlue} + multiline + editable={!disabled} + textAlignVertical="top" + /> + {onMicPress ? ( + [styles.iconButton, pressed && styles.pressed]} + // Hold mode is walkie-talkie (press-in/out); toggle mode taps. + onPress={dictationMode === 'hold' ? undefined : onMicPress} + onPressIn={dictationMode === 'hold' ? onMicPressIn : undefined} + onPressOut={dictationMode === 'hold' ? onMicPressOut : undefined} + disabled={disabled} + > + {micActive ? ( + + ) : ( + + )} + + ) : null} [styles.iconButton, pressed && styles.pressed]} - // Hold mode is walkie-talkie (press-in/out); toggle mode taps. - onPress={dictationMode === 'hold' ? undefined : onMicPress} - onPressIn={dictationMode === 'hold' ? onMicPressIn : undefined} - onPressOut={dictationMode === 'hold' ? onMicPressOut : undefined} - disabled={disabled} + accessibilityLabel="Send message" + style={({ pressed }) => [ + styles.sendButton, + !canSend && styles.sendButtonDisabled, + pressed && canSend && styles.pressed + ]} + onPress={handleSend} + disabled={!canSend} > - {micActive ? ( - - ) : ( - - )} + - ) : null} - [ - styles.sendButton, - !canSend && styles.sendButtonDisabled, - pressed && canSend && styles.pressed - ]} - onPress={handleSend} - disabled={!canSend} - > - - + ) @@ -239,15 +289,54 @@ const styles = StyleSheet.create({ fontFamily: typography.monoFamily, fontSize: typography.metaSize }, + // Hairline + panel background live on the surface so the attachment tray and + // input bar read as one composer block (no divider between them). + composerSurface: { + borderTopWidth: StyleSheet.hairlineWidth, + borderTopColor: colors.borderSubtle, + backgroundColor: colors.bgPanel + }, + attachmentTray: { + flexDirection: 'row', + flexWrap: 'wrap', + gap: spacing.sm, + paddingHorizontal: spacing.md, + paddingTop: spacing.sm + }, + attachmentTile: { + width: 88, + height: 88, + borderRadius: radii.card, + overflow: 'hidden', + backgroundColor: colors.bgRaised + }, + attachmentImage: { + width: '100%', + height: '100%' + }, + attachmentUploadingOverlay: { + ...StyleSheet.absoluteFillObject, + alignItems: 'center', + justifyContent: 'center', + backgroundColor: 'rgba(17, 17, 17, 0.45)' + }, + attachmentRemove: { + position: 'absolute', + top: spacing.xs, + right: spacing.xs, + width: 24, + height: 24, + borderRadius: 12, + alignItems: 'center', + justifyContent: 'center', + backgroundColor: 'rgba(17, 17, 17, 0.72)' + }, bar: { flexDirection: 'row', alignItems: 'flex-end', gap: spacing.sm, paddingHorizontal: spacing.md, - paddingVertical: spacing.sm, - borderTopWidth: StyleSheet.hairlineWidth, - borderTopColor: colors.borderSubtle, - backgroundColor: colors.bgPanel + paddingVertical: spacing.sm }, input: { flex: 1, diff --git a/mobile/src/session/MobileNativeChatOverlay.tsx b/mobile/src/session/MobileNativeChatOverlay.tsx index 6acf2d305f6..4cb13fa3213 100644 --- a/mobile/src/session/MobileNativeChatOverlay.tsx +++ b/mobile/src/session/MobileNativeChatOverlay.tsx @@ -1,11 +1,17 @@ import { StyleSheet, View } from 'react-native' import { MobileNativeChatView, type MobileNativeChatInputLockReason } from './MobileNativeChatView' import type { MobileNativeChatController } from './use-mobile-native-chat-controller' +import type { MobileNativeChatPendingImage } from './use-mobile-native-chat-pending-images' type Props = { controller: MobileNativeChatController onAttachImage: () => void isAttaching: boolean + /** Overrides the controller's plain-text send (e.g. to deliver pending image + * attachments first). Falls back to controller.handleNativeChatSend. */ + onSend?: (text: string) => Promise + pendingImages?: readonly MobileNativeChatPendingImage[] + onRemovePendingImage?: (id: string) => void onMicPress: () => void micActive: boolean dictationMode: 'toggle' | 'hold' @@ -21,6 +27,9 @@ export function MobileNativeChatOverlay({ controller, onAttachImage, isAttaching, + onSend, + pendingImages, + onRemovePendingImage, onMicPress, micActive, dictationMode, @@ -54,12 +63,14 @@ export function MobileNativeChatOverlay({ hasMore={session.hasMore} loadingEarlier={session.loadingEarlier} onLoadEarlier={session.loadEarlier} - onSend={controller.handleNativeChatSend} + onSend={onSend ?? controller.handleNativeChatSend} pending={controller.chatPending} composerText={controller.chatComposerText} onComposerTextChange={controller.setChatComposerText} onAttachImage={onAttachImage} isAttaching={isAttaching} + pendingImages={pendingImages} + onRemovePendingImage={onRemovePendingImage} onMicPress={onMicPress} micActive={micActive} dictationMode={dictationMode} diff --git a/mobile/src/session/MobileNativeChatView.tsx b/mobile/src/session/MobileNativeChatView.tsx index 1cfe37e5d4d..ebbe4770623 100644 --- a/mobile/src/session/MobileNativeChatView.tsx +++ b/mobile/src/session/MobileNativeChatView.tsx @@ -23,6 +23,7 @@ import { useMobileNativeChatAskDismiss } from './use-mobile-native-chat-ask-dism import { useMobileNativeChatPinchGesture } from './use-mobile-native-chat-pinch-gesture' import { MobileAgentWorkingIndicator } from './MobileAgentWorkingIndicator' import { MobileNativeChatComposer } from './MobileNativeChatComposer' +import type { MobileNativeChatPendingImage } from './use-mobile-native-chat-pending-images' import { MobileNativeChatMessage } from './MobileNativeChatMessage' import { MobileNativeChatAsk } from './MobileNativeChatAsk' import type { AskAnswerSelection, AskPrompt } from './mobile-native-chat-ask' @@ -59,6 +60,8 @@ type Props = { onComposerTextChange: (text: string) => void onAttachImage?: () => void isAttaching?: boolean + pendingImages?: readonly MobileNativeChatPendingImage[] + onRemovePendingImage?: (id: string) => void onMicPress?: () => void micActive?: boolean dictationMode?: 'toggle' | 'hold' @@ -104,6 +107,8 @@ export function MobileNativeChatView({ onComposerTextChange, onAttachImage, isAttaching, + pendingImages, + onRemovePendingImage, onMicPress, micActive, dictationMode, @@ -404,6 +409,8 @@ export function MobileNativeChatView({ onSend={handleSend} onAttachImage={onAttachImage} isAttaching={isAttaching} + pendingImages={pendingImages} + onRemovePendingImage={onRemovePendingImage} onMicPress={onMicPress} micActive={micActive} dictationMode={dictationMode} diff --git a/mobile/src/session/mobile-image-source-picker.test.ts b/mobile/src/session/mobile-image-source-picker.test.ts index 3e35a7bed61..a165e963329 100644 --- a/mobile/src/session/mobile-image-source-picker.test.ts +++ b/mobile/src/session/mobile-image-source-picker.test.ts @@ -25,7 +25,7 @@ describe('pickMobileImage', () => { }) }) - expect(result).toEqual({ base64: 'AAAA' }) + expect(result).toEqual({ base64: 'AAAA', uri: 'file:///x.jpg' }) }) it('throws when photo library permission is denied', async () => { @@ -59,7 +59,10 @@ describe('pickMobileImage', () => { }) }) - expect(result).toEqual({ base64: Buffer.from(bytes).toString('base64') }) + expect(result).toEqual({ + base64: Buffer.from(bytes).toString('base64'), + uri: 'file:///doc.png' + }) fetchSpy.mockRestore() }) diff --git a/mobile/src/session/mobile-image-source-picker.ts b/mobile/src/session/mobile-image-source-picker.ts index 8bc824965fc..43c10eb6f1c 100644 --- a/mobile/src/session/mobile-image-source-picker.ts +++ b/mobile/src/session/mobile-image-source-picker.ts @@ -9,6 +9,9 @@ export type MobileImageSource = 'library' | 'files' export type PickedMobileImage = { // Raw base64 (no data: prefix); fed straight into the existing upload pipeline. readonly base64: string + // Local asset URI (file://…) when the picker provides one — lets the composer + // render a thumbnail without holding a second base64 copy in memory. + readonly uri?: string } export class ImageLibraryPermissionError extends Error { @@ -50,7 +53,7 @@ async function pickFromLibrary( if (!base64) { return null } - return { base64 } + return { base64, ...(asset?.uri ? { uri: asset.uri } : {}) } } async function pickFromFiles( @@ -68,7 +71,7 @@ async function pickFromFiles( if (!asset?.uri) { return null } - return { base64: await readUriAsBase64(asset.uri) } + return { base64: await readUriAsBase64(asset.uri), uri: asset.uri } } export async function pickMobileImage( diff --git a/mobile/src/session/use-mobile-image-attachment.ts b/mobile/src/session/use-mobile-image-attachment.ts index 417dd8d699b..8bd6ac176db 100644 --- a/mobile/src/session/use-mobile-image-attachment.ts +++ b/mobile/src/session/use-mobile-image-attachment.ts @@ -38,6 +38,21 @@ function getErrorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error) } +/** Shared toast copy for a failed image attach — used by the immediate + * terminal-paste flow here and the native-chat pending-attachment flow. */ +export function mobileImageAttachToastMessage(error: unknown, connected: boolean): string { + if (!connected) { + return 'Attach failed (disconnected)' + } + if (error instanceof ImageLibraryPermissionError) { + return 'Photo permission denied' + } + if (getErrorMessage(error) === 'Clipboard image is too large') { + return 'Image too large to attach' + } + return 'Attach failed' +} + export function useMobileImageAttachment({ client, activeHandle, @@ -72,19 +87,7 @@ export function useMobileImageAttachment({ } } catch (error) { onError() - if (connState !== 'connected') { - showToast('Attach failed (disconnected)', 1500) - return - } - if (error instanceof ImageLibraryPermissionError) { - showToast('Photo permission denied', 1500) - return - } - if (getErrorMessage(error) === 'Clipboard image is too large') { - showToast('Image too large to attach', 1500) - return - } - showToast('Attach failed', 1500) + showToast(mobileImageAttachToastMessage(error, connState === 'connected'), 1500) } finally { setIsAttaching(false) } diff --git a/mobile/src/session/use-mobile-native-chat-image-send.test.ts b/mobile/src/session/use-mobile-native-chat-image-send.test.ts new file mode 100644 index 00000000000..6c83df21fd8 --- /dev/null +++ b/mobile/src/session/use-mobile-native-chat-image-send.test.ts @@ -0,0 +1,133 @@ +import { createElement } from 'react' +import { act, create, type ReactTestRenderer } from 'react-test-renderer' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { RpcClient } from '../transport/rpc-client' +import { useMobileNativeChatImageSend } from './use-mobile-native-chat-image-send' +import type { MobileNativeChatReadyImage } from './use-mobile-native-chat-pending-images' + +const sendRequest = vi.fn() +const client = { sendRequest } as unknown as RpcClient +const sendText = vi.fn() +const consumeImages = vi.fn() +const onSendError = vi.fn() + +let readyImages: readonly MobileNativeChatReadyImage[] = [] +let leaseReady = true + +describe('useMobileNativeChatImageSend', () => { + let renderer: ReactTestRenderer | null = null + let send: ((text: string) => Promise) | null = null + + beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true + vi.clearAllMocks() + readyImages = [] + leaseReady = true + sendRequest.mockResolvedValue({ ok: true, result: { send: { accepted: true } } }) + sendText.mockResolvedValue(true) + }) + + afterEach(() => { + act(() => renderer?.unmount()) + renderer = null + send = null + }) + + function Harness(): null { + send = useMobileNativeChatImageSend({ + clientRef: { current: client }, + activeHandleRef: { current: 'term-1' }, + deviceTokenRef: { current: 'device-9' }, + inputLeaseReadyRef: { + get current() { + return leaseReady + } + }, + sendText, + getReadyImages: () => readyImages, + consumeImages, + onSendError + }) + return null + } + + function render(): void { + act(() => { + renderer = create(createElement(Harness)) + }) + } + + it('passes straight through to the text send when no images are pending', async () => { + render() + await expect(send!('hello')).resolves.toBe(true) + expect(sendText).toHaveBeenCalledWith('hello') + expect(sendRequest).not.toHaveBeenCalled() + }) + + it('pastes each image bracketed without enter, consumes it, then sends the text', async () => { + readyImages = [ + { id: 'a', hostPath: '/tmp/a.png' }, + { id: 'b', hostPath: '/tmp/b.png' } + ] + render() + + await expect(send!('what is this?')).resolves.toBe(true) + expect(sendRequest).toHaveBeenCalledTimes(2) + expect(sendRequest).toHaveBeenNthCalledWith(1, 'terminal.send', { + terminal: 'term-1', + text: '\x1b[200~/tmp/a.png\x1b[201~', + enter: false, + client: { id: 'device-9', type: 'mobile' } + }) + expect(sendRequest).toHaveBeenNthCalledWith(2, 'terminal.send', { + terminal: 'term-1', + text: '\x1b[200~/tmp/b.png\x1b[201~', + enter: false, + client: { id: 'device-9', type: 'mobile' } + }) + expect(consumeImages.mock.calls).toEqual([[['a']], [['b']]]) + expect(sendText).toHaveBeenCalledWith('what is this?') + }) + + it('submits an image-only send with a bare Enter', async () => { + readyImages = [{ id: 'a', hostPath: '/tmp/a.png' }] + render() + + await expect(send!('')).resolves.toBe(true) + expect(sendText).not.toHaveBeenCalled() + expect(sendRequest).toHaveBeenLastCalledWith('terminal.send', { + terminal: 'term-1', + text: '', + enter: true, + client: { id: 'device-9', type: 'mobile' } + }) + }) + + it('stops and keeps undelivered images when a paste is rejected', async () => { + readyImages = [ + { id: 'a', hostPath: '/tmp/a.png' }, + { id: 'b', hostPath: '/tmp/b.png' } + ] + sendRequest + .mockResolvedValueOnce({ ok: true, result: { send: { accepted: true } } }) + .mockResolvedValueOnce({ ok: true, result: { send: { accepted: false } } }) + render() + + await expect(send!('caption')).resolves.toBe(false) + // Only the delivered image is consumed; 'b' stays attached for retry. + expect(consumeImages.mock.calls).toEqual([[['a']]]) + expect(sendText).not.toHaveBeenCalled() + expect(onSendError).toHaveBeenCalledWith('Message not sent') + }) + + it('fails before pasting anything when the input lease is not ready', async () => { + readyImages = [{ id: 'a', hostPath: '/tmp/a.png' }] + leaseReady = false + render() + + await expect(send!('caption')).resolves.toBe(false) + expect(sendRequest).not.toHaveBeenCalled() + expect(consumeImages).not.toHaveBeenCalled() + expect(onSendError).toHaveBeenCalledWith('Message not sent (disconnected)') + }) +}) diff --git a/mobile/src/session/use-mobile-native-chat-image-send.ts b/mobile/src/session/use-mobile-native-chat-image-send.ts new file mode 100644 index 00000000000..f69d61f6cd7 --- /dev/null +++ b/mobile/src/session/use-mobile-native-chat-image-send.ts @@ -0,0 +1,95 @@ +import { useCallback } from 'react' +import type { RpcClient } from '../transport/rpc-client' +import { buildMobileImagePastePayload } from './mobile-clipboard-image' +import { sendMobileNativeChatMessage } from './mobile-native-chat-send' +import type { MobileNativeChatReadyImage } from './use-mobile-native-chat-pending-images' + +type CurrentRef = { + readonly current: T +} + +type UseMobileNativeChatImageSendArgs = { + readonly clientRef: CurrentRef + readonly activeHandleRef: CurrentRef + readonly deviceTokenRef: CurrentRef + readonly inputLeaseReadyRef: CurrentRef + /** The plain text send (controller-owned: optimistic bubble + draft clear). */ + readonly sendText: (text: string) => Promise + readonly getReadyImages: () => readonly MobileNativeChatReadyImage[] + readonly consumeImages: (ids: readonly string[]) => void + readonly onSendError: (message: string) => void +} + +/** Wraps the native-chat send so pending image attachments are pasted into the + * agent TUI (bracketed, like a desktop image paste) right before the message + * text — deferring delivery to send time is what lets the composer thumbnails + * be removable. Image-only sends submit with a bare Enter. */ +export function useMobileNativeChatImageSend({ + clientRef, + activeHandleRef, + deviceTokenRef, + inputLeaseReadyRef, + sendText, + getReadyImages, + consumeImages, + onSendError +}: UseMobileNativeChatImageSendArgs): (text: string) => Promise { + return useCallback( + async (text: string): Promise => { + const images = getReadyImages() + if (images.length === 0) { + return sendText(text) + } + const client = clientRef.current + const terminal = activeHandleRef.current + if (!client || !terminal || !inputLeaseReadyRef.current) { + onSendError('Message not sent (disconnected)') + return false + } + const mobileClient = deviceTokenRef.current + ? { id: deviceTokenRef.current, type: 'mobile' as const } + : undefined + for (const image of images) { + const accepted = await sendMobileNativeChatMessage({ + client, + terminal, + text: buildMobileImagePastePayload(image.hostPath), + enter: false, + ...(mobileClient ? { mobileClient } : {}) + }) + if (!accepted) { + onSendError('Message not sent') + return false + } + // Why: consume per image so a mid-sequence failure can't re-paste + // already-delivered paths on retry (they sit in the TUI input now). + consumeImages([image.id]) + } + if (text.trim().length > 0) { + return sendText(text) + } + // Image-only send: the paths are in the TUI input; submit with Enter. + const submitted = await sendMobileNativeChatMessage({ + client, + terminal, + text: '', + enter: true, + ...(mobileClient ? { mobileClient } : {}) + }) + if (!submitted) { + onSendError('Message not sent') + } + return submitted + }, + [ + activeHandleRef, + clientRef, + consumeImages, + deviceTokenRef, + getReadyImages, + inputLeaseReadyRef, + onSendError, + sendText + ] + ) +} diff --git a/mobile/src/session/use-mobile-native-chat-pending-images.test.ts b/mobile/src/session/use-mobile-native-chat-pending-images.test.ts new file mode 100644 index 00000000000..ebd13d6b204 --- /dev/null +++ b/mobile/src/session/use-mobile-native-chat-pending-images.test.ts @@ -0,0 +1,171 @@ +import { createElement } from 'react' +import { act, create, type ReactTestRenderer } from 'react-test-renderer' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const pickMobileImage = vi.fn() +// Fully mocked (no importOriginal): the real module imports expo pickers, whose +// react-native dependency chain doesn't parse under vitest. +vi.mock('./mobile-image-source-picker', () => ({ + pickMobileImage: (...args: unknown[]) => pickMobileImage(...args), + ImageLibraryPermissionError: class ImageLibraryPermissionError extends Error {} +})) + +const saveMobileClipboardImageAsTempFile = vi.fn() +vi.mock('./mobile-clipboard-image', () => ({ + saveMobileClipboardImageAsTempFile: (...args: unknown[]) => + saveMobileClipboardImageAsTempFile(...args), + buildMobileImagePastePayload: (filePath: string) => filePath +})) + +import type { RpcClient } from '../transport/rpc-client' +import { useMobileNativeChatPendingImages } from './use-mobile-native-chat-pending-images' + +type PendingImagesState = ReturnType + +const client = { sendRequest: vi.fn() } as unknown as RpcClient +const showToast = vi.fn() +const onSuccess = vi.fn() +const onError = vi.fn() + +describe('useMobileNativeChatPendingImages', () => { + let renderer: ReactTestRenderer | null = null + let state: PendingImagesState | null = null + + beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true + vi.clearAllMocks() + }) + + afterEach(() => { + act(() => renderer?.unmount()) + renderer = null + state = null + }) + + function Harness({ activeHandle = 't1' }: { activeHandle?: string | null }): null { + state = useMobileNativeChatPendingImages({ + client, + activeHandle, + canAttach: true, + connState: 'connected', + getConnectionId: async () => null, + showToast, + onSuccess, + onError + }) + return null + } + + function render(activeHandle?: string | null): void { + act(() => { + renderer = create(createElement(Harness, { activeHandle })) + }) + } + + it('adds an uploading entry on pick and flips it ready after the host upload', async () => { + pickMobileImage.mockResolvedValue({ base64: 'AAAA', uri: 'file:///pick.jpg' }) + let resolveUpload: (path: string) => void = () => {} + saveMobileClipboardImageAsTempFile.mockReturnValue( + new Promise((resolve) => { + resolveUpload = resolve + }) + ) + render() + + let attachDone: Promise = Promise.resolve() + await act(async () => { + attachDone = state!.attachPendingChatImage('library') + // Let the pick resolve and the uploading entry land. + await Promise.resolve() + await Promise.resolve() + }) + expect(state!.pendingChatImages).toEqual([ + { id: 'chat-image-1', thumbnailUri: 'file:///pick.jpg', status: 'uploading', hostPath: null } + ]) + expect(state!.getReadyChatImages()).toEqual([]) + + await act(async () => { + resolveUpload('/tmp/orca-img.png') + await attachDone + }) + expect(state!.pendingChatImages[0]).toMatchObject({ id: 'chat-image-1', status: 'ready' }) + expect(state!.getReadyChatImages()).toEqual([ + { id: 'chat-image-1', hostPath: '/tmp/orca-img.png' } + ]) + expect(onSuccess).toHaveBeenCalledTimes(1) + }) + + it('falls back to a data URI thumbnail when the picker has no file URI', async () => { + pickMobileImage.mockResolvedValue({ base64: 'AAAA' }) + saveMobileClipboardImageAsTempFile.mockResolvedValue('/tmp/a.png') + render() + + await act(async () => { + await state!.attachPendingChatImage('library') + }) + expect(state!.pendingChatImages[0]?.thumbnailUri).toBe('data:image/png;base64,AAAA') + }) + + it('adds nothing when the picker is cancelled', async () => { + pickMobileImage.mockResolvedValue(null) + render() + + await act(async () => { + await state!.attachPendingChatImage('library') + }) + expect(state!.pendingChatImages).toEqual([]) + expect(showToast).not.toHaveBeenCalled() + }) + + it('removes the entry and toasts when the upload fails', async () => { + pickMobileImage.mockResolvedValue({ base64: 'AAAA', uri: 'file:///pick.jpg' }) + saveMobileClipboardImageAsTempFile.mockRejectedValue(new Error('upload broke')) + render() + + await act(async () => { + await state!.attachPendingChatImage('library') + }) + expect(state!.pendingChatImages).toEqual([]) + expect(onError).toHaveBeenCalledTimes(1) + expect(showToast).toHaveBeenCalledWith('Attach failed', 1500) + }) + + it('removes a pending image and consumes delivered ones', async () => { + pickMobileImage + .mockResolvedValueOnce({ base64: 'AAAA', uri: 'file:///a.jpg' }) + .mockResolvedValueOnce({ base64: 'BBBB', uri: 'file:///b.jpg' }) + saveMobileClipboardImageAsTempFile + .mockResolvedValueOnce('/tmp/a.png') + .mockResolvedValueOnce('/tmp/b.png') + render() + + await act(async () => { + await state!.attachPendingChatImage('library') + await state!.attachPendingChatImage('library') + }) + expect(state!.getReadyChatImages()).toHaveLength(2) + + act(() => state!.removePendingChatImage('chat-image-1')) + expect(state!.getReadyChatImages()).toEqual([{ id: 'chat-image-2', hostPath: '/tmp/b.png' }]) + + act(() => state!.consumePendingChatImages(['chat-image-2'])) + expect(state!.pendingChatImages).toEqual([]) + }) + + it('drops pending images when the active terminal handle changes', async () => { + pickMobileImage.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + saveMobileClipboardImageAsTempFile.mockResolvedValue('/tmp/a.png') + render('t1') + + await act(async () => { + await state!.attachPendingChatImage('library') + }) + expect(state!.pendingChatImages).toHaveLength(1) + + act(() => { + renderer!.update(createElement(Harness, { activeHandle: 't2' })) + }) + expect(state!.pendingChatImages).toEqual([]) + expect(state!.getReadyChatImages()).toEqual([]) + }) +}) diff --git a/mobile/src/session/use-mobile-native-chat-pending-images.ts b/mobile/src/session/use-mobile-native-chat-pending-images.ts new file mode 100644 index 00000000000..b862057338f --- /dev/null +++ b/mobile/src/session/use-mobile-native-chat-pending-images.ts @@ -0,0 +1,178 @@ +import { useCallback, useRef, useState } from 'react' +import type { RpcClient } from '../transport/rpc-client' +import type { ConnectionState } from '../transport/types' +import { saveMobileClipboardImageAsTempFile } from './mobile-clipboard-image' +import { pickMobileImage, type MobileImageSource } from './mobile-image-source-picker' +import { mobileImageAttachToastMessage } from './use-mobile-image-attachment' + +/** A picked-but-unsent image shown as a removable thumbnail in the chat + * composer. Uploads to the host eagerly on pick; the resulting host path is + * pasted into the agent TUI only at send time so removing the thumbnail + * needs no TUI un-paste. */ +export type MobileNativeChatPendingImage = { + readonly id: string + readonly thumbnailUri: string + readonly status: 'uploading' | 'ready' +} + +export type MobileNativeChatReadyImage = { + readonly id: string + readonly hostPath: string +} + +type PendingImageEntry = MobileNativeChatPendingImage & { + readonly hostPath: string | null +} + +type UseMobileNativeChatPendingImagesArgs = { + readonly client: RpcClient | null + readonly activeHandle: string | null + readonly canAttach: boolean + readonly connState: ConnectionState + readonly getConnectionId: () => Promise + readonly showToast: (message: string, durationMs?: number) => void + readonly onSuccess: () => void + readonly onError: () => void +} + +type MobileNativeChatPendingImages = { + readonly pendingChatImages: readonly MobileNativeChatPendingImage[] + readonly attachPendingChatImage: (source: MobileImageSource) => Promise + readonly removePendingChatImage: (id: string) => void + /** Snapshot of fully-uploaded images, in attach order. Read at send time. */ + readonly getReadyChatImages: () => readonly MobileNativeChatReadyImage[] + /** Drop entries whose host paths were delivered to the TUI. */ + readonly consumePendingChatImages: (ids: readonly string[]) => void +} + +export function useMobileNativeChatPendingImages({ + client, + activeHandle, + canAttach, + connState, + getConnectionId, + showToast, + onSuccess, + onError +}: UseMobileNativeChatPendingImagesArgs): MobileNativeChatPendingImages { + const [entries, setEntries] = useState([]) + const entriesRef = useRef(entries) + const idCounter = useRef(0) + + const updateEntries = useCallback( + (updater: (previous: readonly PendingImageEntry[]) => readonly PendingImageEntry[]) => { + setEntries((previous) => { + const next = updater(previous) + entriesRef.current = next + return next + }) + }, + [] + ) + + // Why: pending images target the active terminal's agent; a tab/worktree + // switch would silently re-aim them, so drop instead (adjust during render, + // not in an effect, so a send in the same frame can't read stale entries). + const lastHandle = useRef(activeHandle) + if (lastHandle.current !== activeHandle) { + lastHandle.current = activeHandle + if (entriesRef.current.length > 0) { + entriesRef.current = [] + setEntries([]) + } + } + + const attachPendingChatImage = useCallback( + async (source: MobileImageSource): Promise => { + if (!client || !activeHandle || !canAttach) { + return + } + let entryId: string | null = null + try { + const picked = await pickMobileImage(source) + // Cancelled picker: no entry, no toast. + if (!picked) { + return + } + idCounter.current += 1 + const id = `chat-image-${idCounter.current}` + entryId = id + updateEntries((previous) => [ + ...previous, + { + id, + // Why: prefer the picker's file URI so the thumbnail doesn't hold a + // second base64 copy of the image in JS memory. + thumbnailUri: picked.uri ?? `data:image/png;base64,${picked.base64}`, + status: 'uploading', + hostPath: null + } + ]) + const connectionId = await getConnectionId() + const hostPath = await saveMobileClipboardImageAsTempFile(client, picked.base64, { + connectionId + }) + // A handle switch mid-upload cleared the entry; the orphaned host temp + // file is covered by the upload TTL. + updateEntries((previous) => + previous.map((entry) => + entry.id === id ? { ...entry, status: 'ready' as const, hostPath } : entry + ) + ) + onSuccess() + } catch (error) { + if (entryId !== null) { + const failedId = entryId + updateEntries((previous) => previous.filter((entry) => entry.id !== failedId)) + } + onError() + showToast(mobileImageAttachToastMessage(error, connState === 'connected'), 1500) + } + }, + [ + activeHandle, + canAttach, + client, + connState, + getConnectionId, + onError, + onSuccess, + showToast, + updateEntries + ] + ) + + const removePendingChatImage = useCallback( + (id: string) => { + updateEntries((previous) => previous.filter((entry) => entry.id !== id)) + }, + [updateEntries] + ) + + const getReadyChatImages = useCallback( + (): readonly MobileNativeChatReadyImage[] => + entriesRef.current + .filter((entry) => entry.status === 'ready' && entry.hostPath !== null) + .map((entry) => ({ id: entry.id, hostPath: entry.hostPath! })), + [] + ) + + const consumePendingChatImages = useCallback( + (ids: readonly string[]) => { + if (ids.length === 0) { + return + } + const consumed = new Set(ids) + updateEntries((previous) => previous.filter((entry) => !consumed.has(entry.id))) + }, + [updateEntries] + ) + + return { + pendingChatImages: entries, + attachPendingChatImage, + removePendingChatImage, + getReadyChatImages, + consumePendingChatImages + } +}