diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index 2dc0e163346..cd2253dcde7 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -83,7 +83,6 @@ mobile-config app/h/*/tasks.tsx mobile-config app/index.tsx mobile-config app/troubleshoot.tsx mobile-config scripts/mock-server.ts -mobile-config src/browser/MobileBrowserPane.tsx mobile-config src/components/NewWorktreeModal.tsx mobile-config src/terminal/terminal-webview-html.ts mobile-config src/transport/rpc-client.ts diff --git a/mobile/.oxlintrc.json b/mobile/.oxlintrc.json index f3338fc3616..7192820eb16 100644 --- a/mobile/.oxlintrc.json +++ b/mobile/.oxlintrc.json @@ -51,12 +51,6 @@ "max-lines": ["error", { "max": 1603, "skipBlankLines": true, "skipComments": true }] } }, - { - "files": ["src/browser/MobileBrowserPane.tsx"], - "rules": { - "max-lines": ["error", { "max": 1594, "skipBlankLines": true, "skipComments": true }] - } - }, { "files": ["app/index.tsx"], "rules": { diff --git a/mobile/src/browser/MobileBrowserPane.tsx b/mobile/src/browser/MobileBrowserPane.tsx index f823f309630..2922d2005a4 100644 --- a/mobile/src/browser/MobileBrowserPane.tsx +++ b/mobile/src/browser/MobileBrowserPane.tsx @@ -1,77 +1,31 @@ /* oxlint-disable react-doctor/no-adjust-state-on-prop-change -- Why: mobile browser state mirrors a remote desktop screencast session and CDP dialogs, which are external systems that cannot be derived during render. */ -import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react' -import { - ActivityIndicator, - AppState, - Image, - PanResponder, - PixelRatio, - Pressable, - Text, - TextInput, - View, - type GestureResponderEvent, - type PanResponderGestureState -} from 'react-native' -import { ArrowUp, ChevronLeft, ChevronRight, RefreshCw } from 'lucide-react-native' +import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react' +import { AppState, type Image, type View } from 'react-native' import type { RpcClient } from '../transport/rpc-client' -import type { RpcFailure, RpcSuccess } from '../transport/types' import type { BrowserScreencastFrame, BrowserScreencastFrameMetadata } from '../transport/browser-screencast-protocol' -import { colors } from '../theme/mobile-theme' -import { - MOBILE_BROWSER_FRAME_MIN_INTERVAL_MS, - buildMobileBrowserScreencastRequest, - type MobileBrowserViewMode -} from './browser-screencast-request' -import { - MobileBrowserPointerModifiers, - type BrowserPointerModifier -} from './MobileBrowserPointerModifiers' -import { MobileBrowserKeyRow } from './MobileBrowserKeyRow' -import { MobileBrowserToolbarIconButton } from './MobileBrowserToolbarIconButton' -import { MobileBrowserViewModeSwitch } from './MobileBrowserViewModeSwitch' +import type { MobileBrowserViewMode } from './browser-screencast-request' +import type { BrowserPointerModifier } from './MobileBrowserPointerModifiers' import { getInitialMobileBrowserViewMode, saveMobileBrowserViewMode } from './mobile-browser-view-mode-state' +import type { BrowserTouchLayout, BrowserZoomState } from './browser-touch-geometry' import { - clampBrowserZoomState, - computeBrowserFrameGeometry, - computeBrowserTouchClickRadiusCss, - mapScreenToBrowserPoint, - readLocalTouchPoint, - type BrowserPoint, - type BrowserTouchLayout, - type BrowserZoomState -} from './browser-touch-geometry' -import { - MAX_ZOOM, - MIN_ZOOM, - assertRpcOk, - browserErrorMessage, - browserFrameMetadataEqual, - buttonColor, - cacheBrowserFrame, clearCachedBrowserFramesForWorktree, - createBrowserFrameDataUri, - createPinchGesture, - getCachedBrowserFrame, makeBrowserFrameCacheKey, peekCachedBrowserFrame, - shouldSurfaceBrowserError, - updateBrowserImageSource, - updateBrowserLayerVisibility, - updatePinchZoom, type FrameLayer, type PinchGesture } from './mobile-browser-frame-state' -import { mobileBrowserPaneStyles as styles } from './mobile-browser-pane-styles' import { displayBrowserUrl, normalizeBrowserUrl } from './browser-url' -import { MobileBrowserAddressField } from './MobileBrowserAddressField' import { resolveMobileBrowserAddressSync } from './mobile-browser-address-sync' +import { MobileBrowserPaneView } from './MobileBrowserPaneView' +import { useMobileBrowserInteractions } from './use-mobile-browser-interactions' +import { useMobileBrowserPaneLayers } from './use-mobile-browser-pane-layers' +import { useMobileBrowserStream } from './use-mobile-browser-stream' export type MobileBrowserTab = { type: 'browser' @@ -108,26 +62,8 @@ type BrowserDialogState = { message: string } -const TAP_SLOP = 16 -const SCROLL_START_SLOP = 22 -const LONG_PRESS_MS = 550 -const WHEEL_INTERVAL_MS = 70 -const TOUCH_CLICK_RADIUS_DIP = 14 const DEFAULT_ZOOM: BrowserZoomState = { scale: 1, offsetX: 0, offsetY: 0 } -type BrowserPageParams = { - worktree: string - page: string -} - -type PendingWheelCommand = { - base: BrowserPageParams - point: BrowserPoint - gestureId: number - dx: number - dy: number -} - export function MobileBrowserPane({ client, worktreeId, @@ -180,17 +116,12 @@ export function MobileBrowserPane({ const frameThrottleTimerRef = useRef | null>(null) const dialogRef = useRef(null) const lastStreamCacheKeyRef = useRef(cacheKey) - const startPointRef = useRef<{ x: number; y: number; t: number } | null>(null) const longPressTimerRef = useRef | null>(null) - const rightClickSentRef = useRef(false) - const lastWheelRef = useRef<{ dx: number; dy: number; at: number }>({ dx: 0, dy: 0, at: 0 }) - const wheelGestureIdRef = useRef(0) - const pendingWheelCommandRef = useRef(null) - const wheelCommandInFlightRef = useRef(false) + const startPointRef = useRef<{ x: number; y: number; t: number } | null>(null) + const scrollingRef = useRef(false) const zoomRef = useRef(DEFAULT_ZOOM) const pinchRef = useRef(null) const panRef = useRef(null) - const scrollingRef = useRef(false) const lastZoomResetUrlRef = useRef(tab.url || 'about:blank') const clearLongPressTimer = useCallback(() => { @@ -265,313 +196,40 @@ export function MobileBrowserPane({ setBrowserViewMode(getInitialMobileBrowserViewMode(worktreeId, tab.browserPageId, tab.url)) }, [tab.browserPageId, tab.url, worktreeId]) - const pageParams = useCallback(() => { - if (!tab.browserPageId) { - return null - } - return { - worktree: `id:${worktreeId}`, - page: tab.browserPageId - } - }, [tab.browserPageId, worktreeId]) - - const applyFrame = useCallback((frame: BrowserScreencastFrame, frameCacheKey: string): void => { - if (!browserFrameMetadataEqual(frameMetadataRef.current, frame.metadata)) { - frameMetadataRef.current = frame.metadata - setFrameMetadata(frame.metadata) - } - const nextFrameUri = createBrowserFrameDataUri(frame) - cacheBrowserFrame(frameCacheKey, { uri: nextFrameUri, metadata: frame.metadata }) - if (!frameMountedRef.current) { - frameUriRef.current = nextFrameUri - frameMountedRef.current = true - setFrameUri(nextFrameUri) - updateBrowserImageSource(browserImageRefs.current[0], nextFrameUri) - } else if (pendingFrameLayerRef.current === null) { - // Why: decode the next frame offscreen and keep the previous layer visible - // until onLoad; replacing the visible Image directly flashes black. - const nextLayer: FrameLayer = visibleFrameLayerRef.current === 0 ? 1 : 0 - frameUriRef.current = nextFrameUri - pendingFrameLayerRef.current = nextLayer - updateBrowserImageSource(browserImageRefs.current[nextLayer], nextFrameUri) - } else { - // Why: popovers/menus can settle in one final frame while the previous - // offscreen frame is still decoding. Keep the hidden layer pointed at - // the newest frame instead of dropping the final static state. - frameUriRef.current = nextFrameUri - updateBrowserImageSource(browserImageRefs.current[pendingFrameLayerRef.current], nextFrameUri) - } - if (busyRef.current) { - busyRef.current = false - setBusy(false) - } - }, []) - - const clearFrameThrottle = useCallback(() => { - pendingThrottledFrameRef.current = null - if (frameThrottleTimerRef.current) { - clearTimeout(frameThrottleTimerRef.current) - frameThrottleTimerRef.current = null - } - }, []) - - const applyFrameThrottled = useCallback( - (frame: BrowserScreencastFrame, frameCacheKey: string): void => { - const now = Date.now() - const elapsed = now - lastAppliedFrameAtRef.current - if (lastAppliedFrameAtRef.current === 0 || elapsed >= MOBILE_BROWSER_FRAME_MIN_INTERVAL_MS) { - clearFrameThrottle() - lastAppliedFrameAtRef.current = now - applyFrame(frame, frameCacheKey) - return - } - - // Why: static UI changes can be the last frame Chromium emits. Coalesce - // throttled frames so the final visible state is applied after the delay. - pendingThrottledFrameRef.current = { frame, cacheKey: frameCacheKey } - if (frameThrottleTimerRef.current) { - return - } - frameThrottleTimerRef.current = setTimeout( - () => { - frameThrottleTimerRef.current = null - const pending = pendingThrottledFrameRef.current - pendingThrottledFrameRef.current = null - if (!pending) { - return - } - lastAppliedFrameAtRef.current = Date.now() - applyFrame(pending.frame, pending.cacheKey) - }, - Math.max(0, MOBILE_BROWSER_FRAME_MIN_INTERVAL_MS - elapsed) - ) - }, - [applyFrame, clearFrameThrottle] - ) - - const streamRequest = useMemo( - () => buildMobileBrowserScreencastRequest(layout, PixelRatio.get(), browserViewMode), - [browserViewMode, layout] - ) - - const frameGeometry = useMemo( - () => computeBrowserFrameGeometry(layout, frameMetadata), - [frameMetadata, layout] - ) - - useEffect(() => { - if (!frameGeometry) { - return - } - setZoom((current) => { - const next = clampBrowserZoomState(current, frameGeometry, MIN_ZOOM, MAX_ZOOM) - if ( - next.scale === current.scale && - next.offsetX === current.offsetX && - next.offsetY === current.offsetY - ) { - return current - } - // Why: rotation/layout changes can shrink the legal pan range while the - // current zoom state still points at the previous viewport geometry. - zoomRef.current = next - return next - }) - }, [frameGeometry]) - - useEffect(() => { - streamGenerationRef.current += 1 - const generation = streamGenerationRef.current - const sameStream = Boolean(cacheKey) && lastStreamCacheKeyRef.current === cacheKey - lastStreamCacheKeyRef.current = cacheKey - if (!sameStream || !frameUriRef.current) { - const cachedFrame = getCachedBrowserFrame(cacheKey) - if (cachedFrame) { - frameUriRef.current = cachedFrame.uri - frameMountedRef.current = true - frameMetadataRef.current = cachedFrame.metadata - setFrameUri(cachedFrame.uri) - setFrameMetadata(cachedFrame.metadata) - } else { - frameUriRef.current = null - frameMountedRef.current = false - setFrameUri(null) - setFrameMetadata(null) - frameMetadataRef.current = null - } - } else { - frameMountedRef.current = true - } - pendingFrameLayerRef.current = null - if (!sameStream || !frameUriRef.current) { - visibleFrameLayerRef.current = 0 - } - updateBrowserLayerVisibility(browserLayerRefs.current, visibleFrameLayerRef.current) - lastAppliedFrameAtRef.current = 0 - clearFrameThrottle() - busyRef.current = false - setDialog(null) - setError(null) - if ( - !client || - screencastSupported !== true || - !tab.browserPageId || - !appActive || - !streamRequest - ) { - busyRef.current = false - setBusy(false) - if (screencastSupported === false) { - setError('Update desktop Orca to stream browser tabs on mobile.') - } else if (screencastSupported === null) { - setError('Checking desktop browser streaming support.') - } else if (!tab.browserPageId) { - setError('Browser page is not available yet.') - } - return - } - busyRef.current = true - setBusy(true) - let startupTimer: ReturnType | null = setTimeout(() => { - if (streamGenerationRef.current !== generation) { - return - } - busyRef.current = false - setBusy(false) - setError('Browser stream timed out.') - }, 15_000) - const clearStartupTimer = (): void => { - if (startupTimer) { - clearTimeout(startupTimer) - startupTimer = null - } - } - const unsubscribe = client.subscribe( - 'browser.screencast', - { - worktree: `id:${worktreeId}`, - page: tab.browserPageId, - ...streamRequest - }, - (payload) => { - if (streamGenerationRef.current !== generation) { - return - } - const event = payload as { - type?: string - message?: string - error?: { message?: string } - dialogType?: string - tab?: { url?: string; title?: string; canGoBack?: boolean; canGoForward?: boolean } - } - if (event.type === 'ready') { - clearStartupTimer() - if (busyRef.current) { - busyRef.current = false - setBusy(false) - } - if (typeof event.tab?.url === 'string') { - setAddressValue(displayBrowserUrl(event.tab.url)) - if (event.tab.url !== lastZoomResetUrlRef.current) { - lastZoomResetUrlRef.current = event.tab.url - resetBrowserZoomState() - } - } - } else if (event.type === 'end') { - clearStartupTimer() - if (busyRef.current) { - busyRef.current = false - setBusy(false) - } - } else if (event.type === 'dialog') { - setDialog({ - dialogType: event.dialogType ?? 'alert', - message: event.message ?? 'Browser dialog' - }) - } else if (event.type === 'dialogClosed') { - setDialog(null) - } else if (event.type === 'error') { - clearStartupTimer() - if (busyRef.current) { - busyRef.current = false - setBusy(false) - } - const message = event.message ?? event.error?.message ?? 'Browser stream failed.' - if (shouldSurfaceBrowserError(message)) { - setError(message) - } - } - }, - { - onBinaryFrame: (frame) => { - if (streamGenerationRef.current !== generation) { - return - } - clearStartupTimer() - if (cacheKey) { - applyFrameThrottled(frame, cacheKey) - } - } - } - ) - return () => { - clearStartupTimer() - clearFrameThrottle() - unsubscribe() - } - }, [ + const { frameGeometry, pageParams, sendBrowserRequest } = useMobileBrowserStream({ appActive, - applyFrameThrottled, - clearFrameThrottle, + browserImageRefs, + browserLayerRefs, + browserViewMode, + busyRef, + cacheKey, client, + frameMetadata, + frameMetadataRef, + frameMountedRef, + frameThrottleTimerRef, + frameUriRef, + lastAppliedFrameAtRef, + lastStreamCacheKeyRef, + lastZoomResetUrlRef, + layout, + pendingFrameLayerRef, + pendingThrottledFrameRef, resetBrowserZoomState, screencastSupported, - streamRequest, - cacheKey, - tab.browserPageId, - worktreeId - ]) - - const sendBrowserRequest = useCallback( - async ( - method: string, - params: Record = {}, - opts: { showBusy?: boolean; suppressError?: boolean; timeoutMs?: number } = {} - ): Promise => { - const base = pageParams() - if (!client || !base) { - return null - } - if (opts.showBusy) { - busyRef.current = true - setBusy(true) - } - try { - const response = await client.sendRequest( - method, - { ...base, ...params }, - { timeoutMs: opts.timeoutMs ?? 15_000 } - ) - if (!response.ok) { - throw new Error((response as RpcFailure).error.message) - } - setError(null) - return (response as RpcSuccess).result - } catch (err) { - const message = browserErrorMessage(err, 'Browser command failed') - if (!opts.suppressError && shouldSurfaceBrowserError(message)) { - setError(message) - } - return null - } finally { - if (opts.showBusy) { - busyRef.current = false - setBusy(false) - } - } - }, - [client, pageParams] - ) + setAddressValue, + setBusy, + setDialog, + setError, + setFrameMetadata, + setFrameUri, + setZoom, + streamGenerationRef, + tab, + visibleFrameLayerRef, + worktreeId, + zoomRef + }) const navigateToAddress = useCallback(async () => { const url = normalizeBrowserUrl(addressValue) @@ -591,411 +249,45 @@ export function MobileBrowserPane({ } }, [addressValue, resetBrowserZoomState, sendBrowserRequest]) - const flushPendingWheelCommand = useCallback(() => { - if (wheelCommandInFlightRef.current) { - return - } - const pending = pendingWheelCommandRef.current - if (!pending || !client) { - return - } - pendingWheelCommandRef.current = null - wheelCommandInFlightRef.current = true - void (async () => { - try { - assertRpcOk( - await client.sendRequest('browser.mouseMove', { - ...pending.base, - x: pending.point.x, - y: pending.point.y - }), - 'Browser pointer move failed' - ) - assertRpcOk( - await client.sendRequest('browser.mouseWheel', { - ...pending.base, - dx: pending.dx, - dy: pending.dy - }), - 'Browser scroll failed' - ) - setError(null) - } catch { - // Scroll bursts commonly race page reload/navigation. Avoid replacing - // the live browser with transient command errors like selector_not_found. - } finally { - wheelCommandInFlightRef.current = false - flushPendingWheelCommand() - } - })() - }, [client]) + const { panResponder, sendDialogCommand, sendKeyboardText, sendKeypress, togglePointerModifier } = + useMobileBrowserInteractions({ + clearLongPressTimer, + client, + dialogRef, + frameGeometry, + frameMetadataRef, + keyboardValue, + layoutRef, + longPressTimerRef, + onToast, + pageParams, + panRef, + pinchRef, + pointerModifiers, + sendBrowserRequest, + scrollingRef, + startPointRef, + setDialog, + setError, + setKeyboardValue, + setPointerModifiers, + setZoom, + zoomRef + }) - const sendPointerClick = useCallback( - async (point: BrowserPoint, button: 'left' | 'right') => { - const base = pageParams() - if (!client || !base) { - return - } - const clickResult = await sendBrowserRequest( - 'browser.mouseClick', - { - x: point.x, - y: point.y, - button, - modifiers: pointerModifiers, - ...(button === 'left' - ? { - radius: computeBrowserTouchClickRadiusCss( - layoutRef.current, - frameMetadataRef.current, - zoomRef.current, - TOUCH_CLICK_RADIUS_DIP - ) - } - : {}) - }, - { suppressError: true, timeoutMs: 5_000 } - ) - if (clickResult !== null || pointerModifiers.length > 0) { - return - } - try { - assertRpcOk( - await client.sendRequest('browser.mouseMove', { ...base, x: point.x, y: point.y }), - 'Browser pointer move failed' - ) - assertRpcOk( - await client.sendRequest('browser.mouseDown', { ...base, button }), - 'Browser pointer down failed' - ) - assertRpcOk( - await client.sendRequest('browser.mouseUp', { ...base, button }), - 'Browser pointer up failed' - ) - setError(null) - } catch { - // Pointer commands can race page navigation. Keep the stream visible; - // actionable failures still surface through navigation/stream errors. - } - }, - [client, pageParams, pointerModifiers, sendBrowserRequest] - ) - - const togglePointerModifier = useCallback((modifier: BrowserPointerModifier) => { - setPointerModifiers((current) => - current.includes(modifier) - ? current.filter((candidate) => candidate !== modifier) - : [...current, modifier] - ) - }, []) - - const sendWheel = useCallback( - (point: BrowserPoint, screenDx: number, screenDy: number) => { - const base = pageParams() - if (!client || !base) { - return - } - const currentLayout = layoutRef.current - const geometry = computeBrowserFrameGeometry(currentLayout, frameMetadataRef.current) - const localZoom = zoomRef.current.scale - const scale = (geometry?.scale ?? 1) * localZoom - const cssDx = screenDx / scale - const cssDy = screenDy / scale - const delta = { dx: Math.round(-cssDx), dy: Math.round(-cssDy) } - if (Math.abs(delta.dx) < 1 && Math.abs(delta.dy) < 1) { - return - } - const pending = pendingWheelCommandRef.current - pendingWheelCommandRef.current = - pending && - pending.base.page === base.page && - pending.gestureId === wheelGestureIdRef.current - ? { - base, - point, - gestureId: wheelGestureIdRef.current, - dx: pending.dx + delta.dx, - dy: pending.dy + delta.dy - } - : { base, point, gestureId: wheelGestureIdRef.current, ...delta } - flushPendingWheelCommand() - }, - [client, flushPendingWheelCommand, pageParams] - ) - - const mapTouchPoint = useCallback((locationX: number, locationY: number): BrowserPoint | null => { - return mapScreenToBrowserPoint( - locationX, - locationY, - layoutRef.current, - frameMetadataRef.current, - zoomRef.current - ) - }, []) - - const handleResponderGrant = useCallback( - (event: GestureResponderEvent) => { - const pinch = createPinchGesture(event, frameGeometry, zoomRef.current) - if (pinch) { - clearLongPressTimer() - pinchRef.current = pinch - panRef.current = null - startPointRef.current = null - return - } - const startPoint = readLocalTouchPoint(event.nativeEvent) - if (!startPoint) { - return - } - startPointRef.current = { x: startPoint.x, y: startPoint.y, t: Date.now() } - rightClickSentRef.current = false - scrollingRef.current = false - wheelGestureIdRef.current += 1 - lastWheelRef.current = { dx: 0, dy: 0, at: 0 } - panRef.current = - zoomRef.current.scale > MIN_ZOOM - ? { - x: startPoint.x, - y: startPoint.y, - offsetX: zoomRef.current.offsetX, - offsetY: zoomRef.current.offsetY - } - : null - clearLongPressTimer() - longPressTimerRef.current = setTimeout(() => { - const start = startPointRef.current - if (!start) { - return - } - const point = mapTouchPoint(start.x, start.y) - if (!point) { - return - } - rightClickSentRef.current = true - void sendPointerClick(point, 'right') - onToast('Right click') - }, LONG_PRESS_MS) - }, - [clearLongPressTimer, frameGeometry, mapTouchPoint, onToast, sendPointerClick] - ) - - const handleResponderMove = useCallback( - (event: GestureResponderEvent, gesture: PanResponderGestureState) => { - const startedPinch = pinchRef.current - ? null - : createPinchGesture(event, frameGeometry, zoomRef.current) - if (startedPinch) { - clearLongPressTimer() - pinchRef.current = startedPinch - panRef.current = null - startPointRef.current = null - } - const activePinch = pinchRef.current - const nextPinch = activePinch ? updatePinchZoom(event, frameGeometry, activePinch) : null - if (nextPinch) { - clearLongPressTimer() - zoomRef.current = nextPinch - setZoom(nextPinch) - return - } - if (activePinch) { - pinchRef.current = null - } - const moved = Math.hypot(gesture.dx, gesture.dy) - if (moved > TAP_SLOP) { - clearLongPressTimer() - } - const activePan = panRef.current - if (activePan && frameGeometry) { - const currentPoint = readLocalTouchPoint(event.nativeEvent) - if (!currentPoint) { - return - } - if (!scrollingRef.current && moved <= TAP_SLOP) { - return - } - scrollingRef.current = true - startPointRef.current = null - const nextZoom = clampBrowserZoomState( - { - scale: zoomRef.current.scale, - offsetX: activePan.offsetX + currentPoint.x - activePan.x, - offsetY: activePan.offsetY + currentPoint.y - activePan.y - }, - frameGeometry, - MIN_ZOOM, - MAX_ZOOM - ) - zoomRef.current = nextZoom - setZoom(nextZoom) - return - } - if (!scrollingRef.current) { - if (moved <= SCROLL_START_SLOP) { - return - } - scrollingRef.current = true - startPointRef.current = null - } - const now = Date.now() - if (now - lastWheelRef.current.at < WHEEL_INTERVAL_MS) { - return - } - const deltaX = gesture.dx - lastWheelRef.current.dx - const deltaY = gesture.dy - lastWheelRef.current.dy - if (Math.abs(deltaX) + Math.abs(deltaY) < 8) { - return - } - const currentPoint = readLocalTouchPoint(event.nativeEvent) - if (!currentPoint) { - return - } - const point = mapTouchPoint(currentPoint.x, currentPoint.y) - if (!point) { - return - } - lastWheelRef.current = { dx: gesture.dx, dy: gesture.dy, at: now } - sendWheel(point, deltaX, deltaY) - }, - [clearLongPressTimer, frameGeometry, mapTouchPoint, sendWheel] - ) - - const handleResponderRelease = useCallback( - (event: GestureResponderEvent, gesture: PanResponderGestureState) => { - clearLongPressTimer() - pinchRef.current = null - panRef.current = null - const start = startPointRef.current - startPointRef.current = null - const wasScrolling = scrollingRef.current - scrollingRef.current = false - if (!start || rightClickSentRef.current || wasScrolling) { - return - } - const moved = Math.hypot(gesture.dx, gesture.dy) - if (moved <= TAP_SLOP && Date.now() - start.t < LONG_PRESS_MS) { - // Why: native browser taps resolve at touch-up. Using touch-down makes - // tiny finger drift feel like the click lands left/up of the finger. - const release = readLocalTouchPoint(event.nativeEvent) ?? start - const point = mapTouchPoint(release.x, release.y) - if (point) { - void sendPointerClick(point, 'left') - } - } - }, - [clearLongPressTimer, mapTouchPoint, sendPointerClick] - ) - - const panResponder = useMemo( - () => - PanResponder.create({ - onStartShouldSetPanResponder: () => dialogRef.current === null, - onMoveShouldSetPanResponder: () => dialogRef.current === null, - onPanResponderGrant: handleResponderGrant, - onPanResponderMove: handleResponderMove, - onPanResponderRelease: handleResponderRelease, - onPanResponderTerminate: () => { - clearLongPressTimer() - pinchRef.current = null - panRef.current = null - scrollingRef.current = false - startPointRef.current = null - }, - onPanResponderTerminationRequest: () => true - }), - [clearLongPressTimer, handleResponderGrant, handleResponderMove, handleResponderRelease] - ) - - const sendKeyboardText = useCallback(async () => { - const text = keyboardValue - if (!text) { - return - } - setKeyboardValue('') - const result = await sendBrowserRequest( - 'browser.keyboardInsertText', - { text }, - { suppressError: true } - ) - if (result !== null) { - onToast('Sent') - } else { - setKeyboardValue(text) - } - }, [keyboardValue, onToast, sendBrowserRequest]) - - const sendKeypress = useCallback( - async (key: string) => { - await sendBrowserRequest('browser.keypress', { key }, { suppressError: true }) - }, - [sendBrowserRequest] - ) - - const sendDialogCommand = useCallback( - async (method: 'browser.dialogAccept' | 'browser.dialogDismiss') => { - setDialog(null) - await sendBrowserRequest(method, {}, { suppressError: true, timeoutMs: 5_000 }) - }, - [sendBrowserRequest] - ) - - const setBrowserImageRef = useCallback((layer: FrameLayer, image: Image | null) => { - browserImageRefs.current[layer] = image - const currentFrameUri = frameUriRef.current - if (image && currentFrameUri) { - updateBrowserImageSource(image, currentFrameUri) - } - }, []) - const setBrowserLayerRef = useCallback((layer: FrameLayer, view: View | null) => { - browserLayerRefs.current[layer] = view - updateBrowserLayerVisibility(browserLayerRefs.current, visibleFrameLayerRef.current) - }, []) - const setBrowserLayer0Ref = useCallback( - (view: View | null) => setBrowserLayerRef(0, view), - [setBrowserLayerRef] - ) - const setBrowserLayer1Ref = useCallback( - (view: View | null) => setBrowserLayerRef(1, view), - [setBrowserLayerRef] - ) - const setBrowserImageLayer0Ref = useCallback( - (image: Image | null) => setBrowserImageRef(0, image), - [setBrowserImageRef] - ) - const setBrowserImageLayer1Ref = useCallback( - (image: Image | null) => setBrowserImageRef(1, image), - [setBrowserImageRef] - ) - - const handleBrowserImageLoad = useCallback((layer: FrameLayer) => { - if (pendingFrameLayerRef.current !== layer) { - return - } - pendingFrameLayerRef.current = null - visibleFrameLayerRef.current = layer - updateBrowserLayerVisibility(browserLayerRefs.current, layer) - }, []) - const handleBrowserImageLayer0Load = useCallback( - () => handleBrowserImageLoad(0), - [handleBrowserImageLoad] - ) - const handleBrowserImageLayer1Load = useCallback( - () => handleBrowserImageLoad(1), - [handleBrowserImageLoad] - ) - const handleBrowserImageError = useCallback((layer: FrameLayer) => { - if (pendingFrameLayerRef.current === layer) { - pendingFrameLayerRef.current = null - } - }, []) - const handleBrowserImageLayer0Error = useCallback( - () => handleBrowserImageError(0), - [handleBrowserImageError] - ) - const handleBrowserImageLayer1Error = useCallback( - () => handleBrowserImageError(1), - [handleBrowserImageError] - ) + const { + browserLayerRef, + frameLayerErrorHandler, + frameLayerLoadHandler, + frameLayerRef, + frameLayerStyle + } = useMobileBrowserPaneLayers({ + browserImageRefs, + browserLayerRefs, + frameUriRef, + pendingFrameLayerRef, + visibleFrameLayerRef + }) const controlsDisabled = !client || !tab.browserPageId || screencastSupported !== true const goBack = useCallback(() => { @@ -1016,6 +308,7 @@ export function MobileBrowserPane({ } void sendBrowserRequest('browser.reload', {}, { suppressError: true }) }, [controlsDisabled, sendBrowserRequest]) + const selectBrowserViewMode = useCallback( (mode: MobileBrowserViewMode) => { if (browserViewMode === mode) { @@ -1028,241 +321,48 @@ export function MobileBrowserPane({ }, [browserViewMode, resetBrowserZoomState, tab.browserPageId, worktreeId] ) + const renderedFrameSource = frameUriRef.current || frameUri ? { uri: frameUriRef.current ?? frameUri! } : null - const frameLayerStyle = useCallback((layer: FrameLayer) => { - return [ - styles.browserImageLayer, - visibleFrameLayerRef.current !== layer && styles.browserImageLayerHidden - ] - }, []) - const browserLayerRef = useCallback( - (layer: FrameLayer) => (layer === 0 ? setBrowserLayer0Ref : setBrowserLayer1Ref), - [setBrowserLayer0Ref, setBrowserLayer1Ref] - ) - const frameLayerRef = useCallback( - (layer: FrameLayer) => (layer === 0 ? setBrowserImageLayer0Ref : setBrowserImageLayer1Ref), - [setBrowserImageLayer0Ref, setBrowserImageLayer1Ref] - ) - const frameLayerLoadHandler = useCallback( - (layer: FrameLayer) => - layer === 0 ? handleBrowserImageLayer0Load : handleBrowserImageLayer1Load, - [handleBrowserImageLayer0Load, handleBrowserImageLayer1Load] - ) - const frameLayerErrorHandler = useCallback( - (layer: FrameLayer) => - layer === 0 ? handleBrowserImageLayer0Error : handleBrowserImageLayer1Error, - [handleBrowserImageLayer0Error, handleBrowserImageLayer1Error] - ) return ( - - - - - - - - - - - - setAddressFocused(true)} - onBlur={() => setAddressFocused(false)} - onSubmit={() => void navigateToAddress()} - focused={addressFocused} - disabled={controlsDisabled} - /> - - - - { - const next = { - width: event.nativeEvent.layout.width, - height: event.nativeEvent.layout.height - } - const current = layoutRef.current - if (current && current.width === next.width && current.height === next.height) { - return - } - layoutRef.current = next - setLayout(next) - }} - {...panResponder.panHandlers} - > - {renderedFrameSource ? ( - - {frameGeometry ? ( - - - {([0, 1] as const).map((layer) => ( - - - - ))} - - - ) : ( - ([0, 1] as const).map((layer) => ( - - - - )) - )} - - ) : null} - {!renderedFrameSource || busy || error ? ( - - {/* Why: a stream can report ready and then deliver no frames, so key the - indicator off actually having pixels or it clears into a blank pane. */} - {busy || (!renderedFrameSource && !error) ? ( - - ) : null} - {error ? {error} : null} - - ) : null} - {dialog ? ( - - - Browser Dialog - {dialog.message} - - {dialog.dialogType !== 'alert' ? ( - [ - styles.dialogButton, - pressed && styles.dialogButtonPressed - ]} - onPress={() => void sendDialogCommand('browser.dialogDismiss')} - > - Cancel - - ) : null} - [ - styles.dialogButton, - styles.dialogButtonPrimary, - pressed && styles.dialogButtonPressed - ]} - onPress={() => void sendDialogCommand('browser.dialogAccept')} - > - OK - - - - - ) : null} - - - - - void sendKeypress(key)} - /> - - void sendKeyboardText()} - /> - void sendKeyboardText()} - accessibilityLabel="Send text to browser" - > - - - - - + ) } diff --git a/mobile/src/browser/MobileBrowserPaneView.tsx b/mobile/src/browser/MobileBrowserPaneView.tsx new file mode 100644 index 00000000000..e36e7044837 --- /dev/null +++ b/mobile/src/browser/MobileBrowserPaneView.tsx @@ -0,0 +1,321 @@ +import type { Dispatch, RefObject, SetStateAction } from 'react' +import { + ActivityIndicator, + Image, + Pressable, + Text, + TextInput, + View, + type PanResponderInstance, + type StyleProp, + type ViewStyle +} from 'react-native' +import { ArrowUp, ChevronLeft, ChevronRight, RefreshCw } from 'lucide-react-native' +import { colors } from '../theme/mobile-theme' +import { MobileBrowserAddressField } from './MobileBrowserAddressField' +import { MobileBrowserKeyRow } from './MobileBrowserKeyRow' +import { + MobileBrowserPointerModifiers, + type BrowserPointerModifier +} from './MobileBrowserPointerModifiers' +import { MobileBrowserToolbarIconButton } from './MobileBrowserToolbarIconButton' +import { MobileBrowserViewModeSwitch } from './MobileBrowserViewModeSwitch' +import { buttonColor, type FrameLayer } from './mobile-browser-frame-state' +import { mobileBrowserPaneStyles as styles } from './mobile-browser-pane-styles' +import type { + BrowserFrameGeometry, + BrowserTouchLayout, + BrowserZoomState +} from './browser-touch-geometry' +import type { MobileBrowserViewMode } from './browser-screencast-request' +import type { MobileBrowserTab } from './MobileBrowserPane' + +type MobileBrowserPaneViewProps = { + addressFocused: boolean + addressValue: string + bottomInset: number + browserLayerRef: (layer: FrameLayer) => (view: View | null) => void + browserViewMode: MobileBrowserViewMode + busy: boolean + controlsDisabled: boolean + dialog: { dialogType: string; message: string } | null + error: string | null + frameGeometry: BrowserFrameGeometry | null + frameLayerErrorHandler: (layer: FrameLayer) => () => void + frameLayerLoadHandler: (layer: FrameLayer) => () => void + frameLayerRef: (layer: FrameLayer) => (image: Image | null) => void + frameLayerStyle: (layer: FrameLayer) => StyleProp + goBack: () => void + goForward: () => void + keyboardLift: number + keyboardValue: string + layoutRef: RefObject + navigateToAddress: () => Promise + panResponder: PanResponderInstance + pointerModifiers: BrowserPointerModifier[] + reloadPage: () => void + renderedFrameSource: { uri: string } | null + selectBrowserViewMode: (mode: MobileBrowserViewMode) => void + sendDialogCommand: (method: 'browser.dialogDismiss' | 'browser.dialogAccept') => Promise + sendKeyboardText: () => Promise + sendKeypress: (key: string) => Promise + setAddressFocused: Dispatch> + setAddressValue: Dispatch> + setKeyboardValue: Dispatch> + setLayout: Dispatch> + setRootViewRef: (view: View | null) => void + tab: MobileBrowserTab + togglePointerModifier: (modifier: BrowserPointerModifier) => void + zoom: BrowserZoomState +} + +export function MobileBrowserPaneView(props: MobileBrowserPaneViewProps) { + const { + addressFocused, + addressValue, + bottomInset, + browserLayerRef, + browserViewMode, + busy, + controlsDisabled, + dialog, + error, + frameGeometry, + frameLayerErrorHandler, + frameLayerLoadHandler, + frameLayerRef, + frameLayerStyle, + goBack, + goForward, + keyboardLift, + keyboardValue, + layoutRef, + navigateToAddress, + panResponder, + pointerModifiers, + reloadPage, + renderedFrameSource, + selectBrowserViewMode, + sendDialogCommand, + sendKeyboardText, + sendKeypress, + setAddressFocused, + setAddressValue, + setKeyboardValue, + setLayout, + setRootViewRef, + tab, + togglePointerModifier, + zoom + } = props + return ( + + + + + + + + + + + + setAddressFocused(true)} + onBlur={() => setAddressFocused(false)} + onSubmit={() => void navigateToAddress()} + focused={addressFocused} + disabled={controlsDisabled} + /> + + + + { + const next = { + width: event.nativeEvent.layout.width, + height: event.nativeEvent.layout.height + } + const current = layoutRef.current + if (current && current.width === next.width && current.height === next.height) { + return + } + layoutRef.current = next + setLayout(next) + }} + {...panResponder.panHandlers} + > + {renderedFrameSource ? ( + + {frameGeometry ? ( + + + {([0, 1] as const).map((layer) => ( + + + + ))} + + + ) : ( + ([0, 1] as const).map((layer) => ( + + + + )) + )} + + ) : null} + {!renderedFrameSource || busy || error ? ( + + {/* Why: a stream can report ready and then deliver no frames, so key the + indicator off actually having pixels or it clears into a blank pane. */} + {busy || (!renderedFrameSource && !error) ? ( + + ) : null} + {error ? {error} : null} + + ) : null} + {dialog ? ( + + + Browser Dialog + {dialog.message} + + {dialog.dialogType !== 'alert' ? ( + [ + styles.dialogButton, + pressed && styles.dialogButtonPressed + ]} + onPress={() => void sendDialogCommand('browser.dialogDismiss')} + > + Cancel + + ) : null} + [ + styles.dialogButton, + styles.dialogButtonPrimary, + pressed && styles.dialogButtonPressed + ]} + onPress={() => void sendDialogCommand('browser.dialogAccept')} + > + OK + + + + + ) : null} + + + + + void sendKeypress(key)} + /> + + void sendKeyboardText()} + /> + void sendKeyboardText()} + accessibilityLabel="Send text to browser" + > + + + + + + ) +} diff --git a/mobile/src/browser/mobile-browser-stream-events.ts b/mobile/src/browser/mobile-browser-stream-events.ts new file mode 100644 index 00000000000..5b65b2987f6 --- /dev/null +++ b/mobile/src/browser/mobile-browser-stream-events.ts @@ -0,0 +1,77 @@ +import type { Dispatch, SetStateAction } from 'react' +import { displayBrowserUrl } from './browser-url' +import { shouldSurfaceBrowserError } from './mobile-browser-frame-state' + +export type BrowserDialogState = { dialogType: string; message: string } + +export type ScreencastEvent = { + type?: string + message?: string + error?: { message?: string } + dialogType?: string + tab?: { url?: string; title?: string; canGoBack?: boolean; canGoForward?: boolean } +} + +type HandleScreencastEventArgs = { + busyRef: { current: boolean } + clearStartupTimer: () => void + event: ScreencastEvent + lastZoomResetUrlRef: { current: string } + resetBrowserZoomState: () => void + setAddressValue: Dispatch> + setBusy: Dispatch> + setDialog: Dispatch> + setError: Dispatch> +} + +export function handleBrowserScreencastEvent(args: HandleScreencastEventArgs): void { + const { + busyRef, + clearStartupTimer, + event, + lastZoomResetUrlRef, + resetBrowserZoomState, + setAddressValue, + setBusy, + setDialog, + setError + } = args + + if (event.type === 'ready') { + clearStartupTimer() + if (busyRef.current) { + busyRef.current = false + setBusy(false) + } + if (typeof event.tab?.url === 'string') { + setAddressValue(displayBrowserUrl(event.tab.url)) + if (event.tab.url !== lastZoomResetUrlRef.current) { + lastZoomResetUrlRef.current = event.tab.url + resetBrowserZoomState() + } + } + } else if (event.type === 'end') { + clearStartupTimer() + if (busyRef.current) { + busyRef.current = false + setBusy(false) + } + } else if (event.type === 'dialog') { + setDialog({ + dialogType: event.dialogType ?? 'alert', + message: event.message ?? 'Browser dialog' + }) + } else if (event.type === 'dialogClosed') { + setDialog(null) + } else if (event.type === 'error') { + clearStartupTimer() + if (busyRef.current) { + busyRef.current = false + setBusy(false) + } + const message = event.message ?? event.error?.message ?? 'Browser stream failed.' + if (shouldSurfaceBrowserError(message)) { + setError(message) + } + } +} diff --git a/mobile/src/browser/use-mobile-browser-commands.ts b/mobile/src/browser/use-mobile-browser-commands.ts new file mode 100644 index 00000000000..30554d4db5f --- /dev/null +++ b/mobile/src/browser/use-mobile-browser-commands.ts @@ -0,0 +1,248 @@ +import { useCallback, useRef, type Dispatch, type SetStateAction } from 'react' +import type { RpcClient } from '../transport/rpc-client' +import type { BrowserScreencastFrameMetadata } from '../transport/browser-screencast-protocol' +import { assertRpcOk } from './mobile-browser-frame-state' +import { + computeBrowserFrameGeometry, + computeBrowserTouchClickRadiusCss, + mapScreenToBrowserPoint, + type BrowserPoint, + type BrowserTouchLayout, + type BrowserZoomState +} from './browser-touch-geometry' +import type { BrowserPointerModifier } from './MobileBrowserPointerModifiers' + +const TOUCH_CLICK_RADIUS_DIP = 14 +type BrowserPageParams = { worktree: string; page: string } +type PendingWheelCommand = { + base: BrowserPageParams + point: BrowserPoint + gestureId: number + dx: number + dy: number +} +type SendBrowserRequest = ( + method: string, + params?: Record, + options?: { showBusy?: boolean; suppressError?: boolean; timeoutMs?: number } +) => Promise + +type MobileBrowserCommandArgs = { + client: RpcClient | null + frameMetadataRef: { current: BrowserScreencastFrameMetadata | null } + keyboardValue: string + layoutRef: { current: BrowserTouchLayout | null } + onToast: (message: string, durationMs?: number) => void + pageParams: () => BrowserPageParams | null + pointerModifiers: BrowserPointerModifier[] + sendBrowserRequest: SendBrowserRequest + setDialog: Dispatch> + setError: Dispatch> + setKeyboardValue: Dispatch> + setPointerModifiers: Dispatch> + zoomRef: { current: BrowserZoomState } +} + +export function useMobileBrowserCommands(args: MobileBrowserCommandArgs) { + const { + client, + frameMetadataRef, + keyboardValue, + layoutRef, + onToast, + pageParams, + pointerModifiers, + sendBrowserRequest, + setDialog, + setError, + setKeyboardValue, + setPointerModifiers, + zoomRef + } = args + + const pendingWheelCommandRef = useRef(null) + + const wheelCommandInFlightRef = useRef(false) + + const flushPendingWheelCommand = useCallback(() => { + if (wheelCommandInFlightRef.current) { + return + } + const pending = pendingWheelCommandRef.current + if (!pending || !client) { + return + } + pendingWheelCommandRef.current = null + wheelCommandInFlightRef.current = true + void (async () => { + try { + assertRpcOk( + await client.sendRequest('browser.mouseMove', { + ...pending.base, + x: pending.point.x, + y: pending.point.y + }), + 'Browser pointer move failed' + ) + assertRpcOk( + await client.sendRequest('browser.mouseWheel', { + ...pending.base, + dx: pending.dx, + dy: pending.dy + }), + 'Browser scroll failed' + ) + setError(null) + } catch { + // Scroll bursts commonly race page reload/navigation. Avoid replacing + // the live browser with transient command errors like selector_not_found. + } finally { + wheelCommandInFlightRef.current = false + flushPendingWheelCommand() + } + })() + }, [client]) + + const sendPointerClick = useCallback( + async (point: BrowserPoint, button: 'left' | 'right') => { + const base = pageParams() + if (!client || !base) { + return + } + const clickResult = await sendBrowserRequest( + 'browser.mouseClick', + { + x: point.x, + y: point.y, + button, + modifiers: pointerModifiers, + ...(button === 'left' + ? { + radius: computeBrowserTouchClickRadiusCss( + layoutRef.current, + frameMetadataRef.current, + zoomRef.current, + TOUCH_CLICK_RADIUS_DIP + ) + } + : {}) + }, + { suppressError: true, timeoutMs: 5_000 } + ) + if (clickResult !== null || pointerModifiers.length > 0) { + return + } + try { + assertRpcOk( + await client.sendRequest('browser.mouseMove', { ...base, x: point.x, y: point.y }), + 'Browser pointer move failed' + ) + assertRpcOk( + await client.sendRequest('browser.mouseDown', { ...base, button }), + 'Browser pointer down failed' + ) + assertRpcOk( + await client.sendRequest('browser.mouseUp', { ...base, button }), + 'Browser pointer up failed' + ) + setError(null) + } catch { + // Pointer commands can race page navigation. Keep the stream visible; + // actionable failures still surface through navigation/stream errors. + } + }, + [client, pageParams, pointerModifiers, sendBrowserRequest] + ) + + const togglePointerModifier = useCallback((modifier: BrowserPointerModifier) => { + setPointerModifiers((current) => + current.includes(modifier) + ? current.filter((candidate) => candidate !== modifier) + : [...current, modifier] + ) + }, []) + + const sendWheel = useCallback( + (point: BrowserPoint, screenDx: number, screenDy: number, gestureId: number) => { + const base = pageParams() + if (!client || !base) { + return + } + const currentLayout = layoutRef.current + const geometry = computeBrowserFrameGeometry(currentLayout, frameMetadataRef.current) + const localZoom = zoomRef.current.scale + const scale = (geometry?.scale ?? 1) * localZoom + const cssDx = screenDx / scale + const cssDy = screenDy / scale + const delta = { dx: Math.round(-cssDx), dy: Math.round(-cssDy) } + if (Math.abs(delta.dx) < 1 && Math.abs(delta.dy) < 1) { + return + } + const pending = pendingWheelCommandRef.current + pendingWheelCommandRef.current = + pending && pending.base.page === base.page && pending.gestureId === gestureId + ? { + base, + point, + gestureId, + dx: pending.dx + delta.dx, + dy: pending.dy + delta.dy + } + : { base, point, gestureId, ...delta } + flushPendingWheelCommand() + }, + [client, flushPendingWheelCommand, pageParams] + ) + + const mapTouchPoint = useCallback((locationX: number, locationY: number): BrowserPoint | null => { + return mapScreenToBrowserPoint( + locationX, + locationY, + layoutRef.current, + frameMetadataRef.current, + zoomRef.current + ) + }, []) + + const sendKeyboardText = useCallback(async () => { + const text = keyboardValue + if (!text) { + return + } + setKeyboardValue('') + const result = await sendBrowserRequest( + 'browser.keyboardInsertText', + { text }, + { suppressError: true } + ) + if (result !== null) { + onToast('Sent') + } else { + setKeyboardValue(text) + } + }, [keyboardValue, onToast, sendBrowserRequest]) + + const sendKeypress = useCallback( + async (key: string) => { + await sendBrowserRequest('browser.keypress', { key }, { suppressError: true }) + }, + [sendBrowserRequest] + ) + + const sendDialogCommand = useCallback( + async (method: 'browser.dialogAccept' | 'browser.dialogDismiss') => { + setDialog(null) + await sendBrowserRequest(method, {}, { suppressError: true, timeoutMs: 5_000 }) + }, + [sendBrowserRequest] + ) + return { + mapTouchPoint, + sendDialogCommand, + sendKeyboardText, + sendKeypress, + sendPointerClick, + sendWheel, + togglePointerModifier + } +} diff --git a/mobile/src/browser/use-mobile-browser-frame-apply.ts b/mobile/src/browser/use-mobile-browser-frame-apply.ts new file mode 100644 index 00000000000..8634edbc924 --- /dev/null +++ b/mobile/src/browser/use-mobile-browser-frame-apply.ts @@ -0,0 +1,122 @@ +import { useCallback, type Dispatch, type SetStateAction } from 'react' +import type { Image } from 'react-native' +import type { + BrowserScreencastFrame, + BrowserScreencastFrameMetadata +} from '../transport/browser-screencast-protocol' +import { MOBILE_BROWSER_FRAME_MIN_INTERVAL_MS } from './browser-screencast-request' +import { + browserFrameMetadataEqual, + cacheBrowserFrame, + createBrowserFrameDataUri, + updateBrowserImageSource, + type FrameLayer +} from './mobile-browser-frame-state' + +type PendingFrame = { frame: BrowserScreencastFrame; cacheKey: string } +type BrowserFrameApplyArgs = { + browserImageRefs: { current: [Image | null, Image | null] } + busyRef: { current: boolean } + frameMetadataRef: { current: BrowserScreencastFrameMetadata | null } + frameMountedRef: { current: boolean } + frameThrottleTimerRef: { current: ReturnType | null } + frameUriRef: { current: string | null } + lastAppliedFrameAtRef: { current: number } + pendingFrameLayerRef: { current: FrameLayer | null } + pendingThrottledFrameRef: { current: PendingFrame | null } + setBusy: Dispatch> + setFrameMetadata: Dispatch> + setFrameUri: Dispatch> + visibleFrameLayerRef: { current: FrameLayer } +} +export function useMobileBrowserFrameApply(args: BrowserFrameApplyArgs) { + const { + browserImageRefs, + busyRef, + frameMetadataRef, + frameMountedRef, + frameThrottleTimerRef, + frameUriRef, + lastAppliedFrameAtRef, + pendingFrameLayerRef, + pendingThrottledFrameRef, + setBusy, + setFrameMetadata, + setFrameUri, + visibleFrameLayerRef + } = args + const applyFrame = useCallback((frame: BrowserScreencastFrame, frameCacheKey: string): void => { + if (!browserFrameMetadataEqual(frameMetadataRef.current, frame.metadata)) { + frameMetadataRef.current = frame.metadata + setFrameMetadata(frame.metadata) + } + const nextFrameUri = createBrowserFrameDataUri(frame) + cacheBrowserFrame(frameCacheKey, { uri: nextFrameUri, metadata: frame.metadata }) + if (!frameMountedRef.current) { + frameUriRef.current = nextFrameUri + frameMountedRef.current = true + setFrameUri(nextFrameUri) + updateBrowserImageSource(browserImageRefs.current[0], nextFrameUri) + } else if (pendingFrameLayerRef.current === null) { + // Why: decode the next frame offscreen and keep the previous layer visible + // until onLoad; replacing the visible Image directly flashes black. + const nextLayer: FrameLayer = visibleFrameLayerRef.current === 0 ? 1 : 0 + frameUriRef.current = nextFrameUri + pendingFrameLayerRef.current = nextLayer + updateBrowserImageSource(browserImageRefs.current[nextLayer], nextFrameUri) + } else { + // Why: popovers/menus can settle in one final frame while the previous + // offscreen frame is still decoding. Keep the hidden layer pointed at + // the newest frame instead of dropping the final static state. + frameUriRef.current = nextFrameUri + updateBrowserImageSource(browserImageRefs.current[pendingFrameLayerRef.current], nextFrameUri) + } + if (busyRef.current) { + busyRef.current = false + setBusy(false) + } + }, []) + + const clearFrameThrottle = useCallback(() => { + pendingThrottledFrameRef.current = null + if (frameThrottleTimerRef.current) { + clearTimeout(frameThrottleTimerRef.current) + frameThrottleTimerRef.current = null + } + }, []) + + const applyFrameThrottled = useCallback( + (frame: BrowserScreencastFrame, frameCacheKey: string): void => { + const now = Date.now() + const elapsed = now - lastAppliedFrameAtRef.current + if (lastAppliedFrameAtRef.current === 0 || elapsed >= MOBILE_BROWSER_FRAME_MIN_INTERVAL_MS) { + clearFrameThrottle() + lastAppliedFrameAtRef.current = now + applyFrame(frame, frameCacheKey) + return + } + + // Why: static UI changes can be the last frame Chromium emits. Coalesce + // throttled frames so the final visible state is applied after the delay. + pendingThrottledFrameRef.current = { frame, cacheKey: frameCacheKey } + if (frameThrottleTimerRef.current) { + return + } + frameThrottleTimerRef.current = setTimeout( + () => { + frameThrottleTimerRef.current = null + const pending = pendingThrottledFrameRef.current + pendingThrottledFrameRef.current = null + if (!pending) { + return + } + lastAppliedFrameAtRef.current = Date.now() + applyFrame(pending.frame, pending.cacheKey) + }, + Math.max(0, MOBILE_BROWSER_FRAME_MIN_INTERVAL_MS - elapsed) + ) + }, + [applyFrame, clearFrameThrottle] + ) + return { applyFrameThrottled, clearFrameThrottle } +} diff --git a/mobile/src/browser/use-mobile-browser-interactions.ts b/mobile/src/browser/use-mobile-browser-interactions.ts new file mode 100644 index 00000000000..519ad45ab3b --- /dev/null +++ b/mobile/src/browser/use-mobile-browser-interactions.ts @@ -0,0 +1,290 @@ +import { useCallback, useMemo, useRef, type Dispatch, type SetStateAction } from 'react' +import { + PanResponder, + type GestureResponderEvent, + type PanResponderGestureState +} from 'react-native' +import type { RpcClient } from '../transport/rpc-client' +import { + createPinchGesture, + MAX_ZOOM, + MIN_ZOOM, + updatePinchZoom, + type PinchGesture +} from './mobile-browser-frame-state' +import { + clampBrowserZoomState, + readLocalTouchPoint, + type BrowserFrameGeometry, + type BrowserTouchLayout, + type BrowserZoomState +} from './browser-touch-geometry' +import type { BrowserPointerModifier } from './MobileBrowserPointerModifiers' +import type { BrowserScreencastFrameMetadata } from '../transport/browser-screencast-protocol' + +import { useMobileBrowserCommands } from './use-mobile-browser-commands' +const TAP_SLOP = 16 +const SCROLL_START_SLOP = 22 +const LONG_PRESS_MS = 550 +const WHEEL_INTERVAL_MS = 70 + +type BrowserPageParams = { worktree: string; page: string } +type PanGesture = { x: number; y: number; offsetX: number; offsetY: number } +type SendBrowserRequest = ( + method: string, + params?: Record, + options?: { showBusy?: boolean; suppressError?: boolean; timeoutMs?: number } +) => Promise + +type MobileBrowserInteractionArgs = { + clearLongPressTimer: () => void + client: RpcClient | null + dialogRef: { current: { dialogType: string; message: string } | null } + frameGeometry: BrowserFrameGeometry | null + frameMetadataRef: { current: BrowserScreencastFrameMetadata | null } + keyboardValue: string + layoutRef: { current: BrowserTouchLayout | null } + longPressTimerRef: { current: ReturnType | null } + onToast: (message: string, durationMs?: number) => void + pageParams: () => BrowserPageParams | null + panRef: { current: PanGesture | null } + pinchRef: { current: PinchGesture | null } + pointerModifiers: BrowserPointerModifier[] + sendBrowserRequest: SendBrowserRequest + setDialog: Dispatch> + setError: Dispatch> + setKeyboardValue: Dispatch> + scrollingRef: { current: boolean } + startPointRef: { current: { x: number; y: number; t: number } | null } + setPointerModifiers: Dispatch> + setZoom: Dispatch> + zoomRef: { current: BrowserZoomState } +} + +export function useMobileBrowserInteractions(args: MobileBrowserInteractionArgs) { + const { + clearLongPressTimer, + client, + dialogRef, + frameGeometry, + frameMetadataRef, + keyboardValue, + layoutRef, + longPressTimerRef, + onToast, + pageParams, + panRef, + pinchRef, + pointerModifiers, + scrollingRef, + sendBrowserRequest, + setDialog, + setError, + setKeyboardValue, + setPointerModifiers, + setZoom, + startPointRef, + zoomRef + } = args + const rightClickSentRef = useRef(false) + const lastWheelRef = useRef({ dx: 0, dy: 0, at: 0 }) + const wheelGestureIdRef = useRef(0) + const { + mapTouchPoint, + sendDialogCommand, + sendKeyboardText, + sendKeypress, + sendPointerClick, + sendWheel, + togglePointerModifier + } = useMobileBrowserCommands({ + client, + frameMetadataRef, + keyboardValue, + layoutRef, + onToast, + pageParams, + pointerModifiers, + sendBrowserRequest, + setDialog, + setError, + setKeyboardValue, + setPointerModifiers, + zoomRef + }) + + const handleResponderGrant = useCallback( + (event: GestureResponderEvent) => { + const pinch = createPinchGesture(event, frameGeometry, zoomRef.current) + if (pinch) { + clearLongPressTimer() + pinchRef.current = pinch + panRef.current = null + startPointRef.current = null + return + } + const startPoint = readLocalTouchPoint(event.nativeEvent) + if (!startPoint) { + return + } + startPointRef.current = { x: startPoint.x, y: startPoint.y, t: Date.now() } + rightClickSentRef.current = false + scrollingRef.current = false + wheelGestureIdRef.current += 1 + lastWheelRef.current = { dx: 0, dy: 0, at: 0 } + panRef.current = + zoomRef.current.scale > MIN_ZOOM + ? { + x: startPoint.x, + y: startPoint.y, + offsetX: zoomRef.current.offsetX, + offsetY: zoomRef.current.offsetY + } + : null + clearLongPressTimer() + longPressTimerRef.current = setTimeout(() => { + const start = startPointRef.current + if (!start) { + return + } + const point = mapTouchPoint(start.x, start.y) + if (!point) { + return + } + rightClickSentRef.current = true + void sendPointerClick(point, 'right') + onToast('Right click') + }, LONG_PRESS_MS) + }, + [clearLongPressTimer, frameGeometry, mapTouchPoint, onToast, sendPointerClick] + ) + + const handleResponderMove = useCallback( + (event: GestureResponderEvent, gesture: PanResponderGestureState) => { + const startedPinch = pinchRef.current + ? null + : createPinchGesture(event, frameGeometry, zoomRef.current) + if (startedPinch) { + clearLongPressTimer() + pinchRef.current = startedPinch + panRef.current = null + startPointRef.current = null + } + const activePinch = pinchRef.current + const nextPinch = activePinch ? updatePinchZoom(event, frameGeometry, activePinch) : null + if (nextPinch) { + clearLongPressTimer() + zoomRef.current = nextPinch + setZoom(nextPinch) + return + } + if (activePinch) { + pinchRef.current = null + } + const moved = Math.hypot(gesture.dx, gesture.dy) + if (moved > TAP_SLOP) { + clearLongPressTimer() + } + const activePan = panRef.current + if (activePan && frameGeometry) { + const currentPoint = readLocalTouchPoint(event.nativeEvent) + if (!currentPoint) { + return + } + if (!scrollingRef.current && moved <= TAP_SLOP) { + return + } + scrollingRef.current = true + startPointRef.current = null + const nextZoom = clampBrowserZoomState( + { + scale: zoomRef.current.scale, + offsetX: activePan.offsetX + currentPoint.x - activePan.x, + offsetY: activePan.offsetY + currentPoint.y - activePan.y + }, + frameGeometry, + MIN_ZOOM, + MAX_ZOOM + ) + zoomRef.current = nextZoom + setZoom(nextZoom) + return + } + if (!scrollingRef.current) { + if (moved <= SCROLL_START_SLOP) { + return + } + scrollingRef.current = true + startPointRef.current = null + } + const now = Date.now() + if (now - lastWheelRef.current.at < WHEEL_INTERVAL_MS) { + return + } + const deltaX = gesture.dx - lastWheelRef.current.dx + const deltaY = gesture.dy - lastWheelRef.current.dy + if (Math.abs(deltaX) + Math.abs(deltaY) < 8) { + return + } + const currentPoint = readLocalTouchPoint(event.nativeEvent) + if (!currentPoint) { + return + } + const point = mapTouchPoint(currentPoint.x, currentPoint.y) + if (!point) { + return + } + lastWheelRef.current = { dx: gesture.dx, dy: gesture.dy, at: now } + sendWheel(point, deltaX, deltaY, wheelGestureIdRef.current) + }, + [clearLongPressTimer, frameGeometry, mapTouchPoint, sendWheel] + ) + + const handleResponderRelease = useCallback( + (event: GestureResponderEvent, gesture: PanResponderGestureState) => { + clearLongPressTimer() + pinchRef.current = null + panRef.current = null + const start = startPointRef.current + startPointRef.current = null + const wasScrolling = scrollingRef.current + scrollingRef.current = false + if (!start || rightClickSentRef.current || wasScrolling) { + return + } + const moved = Math.hypot(gesture.dx, gesture.dy) + if (moved <= TAP_SLOP && Date.now() - start.t < LONG_PRESS_MS) { + // Why: native browser taps resolve at touch-up. Using touch-down makes + // tiny finger drift feel like the click lands left/up of the finger. + const release = readLocalTouchPoint(event.nativeEvent) ?? start + const point = mapTouchPoint(release.x, release.y) + if (point) { + void sendPointerClick(point, 'left') + } + } + }, + [clearLongPressTimer, mapTouchPoint, sendPointerClick] + ) + + const panResponder = useMemo( + () => + PanResponder.create({ + onStartShouldSetPanResponder: () => dialogRef.current === null, + onMoveShouldSetPanResponder: () => dialogRef.current === null, + onPanResponderGrant: handleResponderGrant, + onPanResponderMove: handleResponderMove, + onPanResponderRelease: handleResponderRelease, + onPanResponderTerminate: () => { + clearLongPressTimer() + pinchRef.current = null + panRef.current = null + scrollingRef.current = false + startPointRef.current = null + }, + onPanResponderTerminationRequest: () => true + }), + [clearLongPressTimer, handleResponderGrant, handleResponderMove, handleResponderRelease] + ) + + return { panResponder, sendDialogCommand, sendKeyboardText, sendKeypress, togglePointerModifier } +} diff --git a/mobile/src/browser/use-mobile-browser-pane-layers.ts b/mobile/src/browser/use-mobile-browser-pane-layers.ts new file mode 100644 index 00000000000..a91ca22e71f --- /dev/null +++ b/mobile/src/browser/use-mobile-browser-pane-layers.ts @@ -0,0 +1,141 @@ +import { useCallback } from 'react' +import type { Image, View } from 'react-native' +import { + updateBrowserImageSource, + updateBrowserLayerVisibility, + type FrameLayer +} from './mobile-browser-frame-state' +import { mobileBrowserPaneStyles as styles } from './mobile-browser-pane-styles' + +type BrowserLayerHandlersArgs = { + browserImageRefs: { current: [Image | null, Image | null] } + browserLayerRefs: { current: [View | null, View | null] } + frameUriRef: { current: string | null } + pendingFrameLayerRef: { current: FrameLayer | null } + visibleFrameLayerRef: { current: FrameLayer } +} + +export function useMobileBrowserPaneLayers(args: BrowserLayerHandlersArgs) { + const { + browserImageRefs, + browserLayerRefs, + frameUriRef, + pendingFrameLayerRef, + visibleFrameLayerRef + } = args + + const setBrowserImageRef = useCallback( + (layer: FrameLayer, image: Image | null) => { + browserImageRefs.current[layer] = image + const currentFrameUri = frameUriRef.current + if (image && currentFrameUri) { + updateBrowserImageSource(image, currentFrameUri) + } + }, + [browserImageRefs, frameUriRef] + ) + + const setBrowserLayerRef = useCallback( + (layer: FrameLayer, view: View | null) => { + browserLayerRefs.current[layer] = view + updateBrowserLayerVisibility(browserLayerRefs.current, visibleFrameLayerRef.current) + }, + [browserLayerRefs, visibleFrameLayerRef] + ) + + const setBrowserLayer0Ref = useCallback( + (view: View | null) => setBrowserLayerRef(0, view), + [setBrowserLayerRef] + ) + const setBrowserLayer1Ref = useCallback( + (view: View | null) => setBrowserLayerRef(1, view), + [setBrowserLayerRef] + ) + const setBrowserImageLayer0Ref = useCallback( + (image: Image | null) => setBrowserImageRef(0, image), + [setBrowserImageRef] + ) + const setBrowserImageLayer1Ref = useCallback( + (image: Image | null) => setBrowserImageRef(1, image), + [setBrowserImageRef] + ) + + const handleBrowserImageLoad = useCallback( + (layer: FrameLayer) => { + if (pendingFrameLayerRef.current !== layer) { + return + } + pendingFrameLayerRef.current = null + visibleFrameLayerRef.current = layer + updateBrowserLayerVisibility(browserLayerRefs.current, layer) + }, + [browserLayerRefs, pendingFrameLayerRef, visibleFrameLayerRef] + ) + + const handleBrowserImageLayer0Load = useCallback( + () => handleBrowserImageLoad(0), + [handleBrowserImageLoad] + ) + const handleBrowserImageLayer1Load = useCallback( + () => handleBrowserImageLoad(1), + [handleBrowserImageLoad] + ) + + const handleBrowserImageError = useCallback( + (layer: FrameLayer) => { + if (pendingFrameLayerRef.current === layer) { + pendingFrameLayerRef.current = null + } + }, + [pendingFrameLayerRef] + ) + + const handleBrowserImageLayer0Error = useCallback( + () => handleBrowserImageError(0), + [handleBrowserImageError] + ) + const handleBrowserImageLayer1Error = useCallback( + () => handleBrowserImageError(1), + [handleBrowserImageError] + ) + + const frameLayerStyle = useCallback( + (layer: FrameLayer) => { + return [ + styles.browserImageLayer, + visibleFrameLayerRef.current !== layer && styles.browserImageLayerHidden + ] + }, + [visibleFrameLayerRef] + ) + + const browserLayerRef = useCallback( + (layer: FrameLayer) => (layer === 0 ? setBrowserLayer0Ref : setBrowserLayer1Ref), + [setBrowserLayer0Ref, setBrowserLayer1Ref] + ) + + const frameLayerRef = useCallback( + (layer: FrameLayer) => (layer === 0 ? setBrowserImageLayer0Ref : setBrowserImageLayer1Ref), + [setBrowserImageLayer0Ref, setBrowserImageLayer1Ref] + ) + + const frameLayerLoadHandler = useCallback( + (layer: FrameLayer) => + layer === 0 ? handleBrowserImageLayer0Load : handleBrowserImageLayer1Load, + [handleBrowserImageLayer0Load, handleBrowserImageLayer1Load] + ) + + const frameLayerErrorHandler = useCallback( + (layer: FrameLayer) => + layer === 0 ? handleBrowserImageLayer0Error : handleBrowserImageLayer1Error, + [handleBrowserImageLayer0Error, handleBrowserImageLayer1Error] + ) + + return { + browserLayerRef, + frameLayerErrorHandler, + frameLayerLoadHandler, + frameLayerRef, + frameLayerStyle + } +} diff --git a/mobile/src/browser/use-mobile-browser-request.ts b/mobile/src/browser/use-mobile-browser-request.ts new file mode 100644 index 00000000000..311b9250ff0 --- /dev/null +++ b/mobile/src/browser/use-mobile-browser-request.ts @@ -0,0 +1,67 @@ +import { useCallback, type Dispatch, type SetStateAction } from 'react' +import type { RpcClient } from '../transport/rpc-client' +import type { RpcFailure, RpcSuccess } from '../transport/types' +import { browserErrorMessage, shouldSurfaceBrowserError } from './mobile-browser-frame-state' + +type BrowserRequestArgs = { + busyRef: { current: boolean } + client: RpcClient | null + pageId: string | null + setBusy: Dispatch> + setError: Dispatch> + worktreeId: string +} +export function useMobileBrowserRequest(args: BrowserRequestArgs) { + const { busyRef, client, pageId, setBusy, setError, worktreeId } = args + const pageParams = useCallback(() => { + if (!pageId) { + return null + } + return { + worktree: `id:${worktreeId}`, + page: pageId + } + }, [pageId, worktreeId]) + + const sendBrowserRequest = useCallback( + async ( + method: string, + params: Record = {}, + opts: { showBusy?: boolean; suppressError?: boolean; timeoutMs?: number } = {} + ): Promise => { + const base = pageParams() + if (!client || !base) { + return null + } + if (opts.showBusy) { + busyRef.current = true + setBusy(true) + } + try { + const response = await client.sendRequest( + method, + { ...base, ...params }, + { timeoutMs: opts.timeoutMs ?? 15_000 } + ) + if (!response.ok) { + throw new Error((response as RpcFailure).error.message) + } + setError(null) + return (response as RpcSuccess).result + } catch (err) { + const message = browserErrorMessage(err, 'Browser command failed') + if (!opts.suppressError && shouldSurfaceBrowserError(message)) { + setError(message) + } + return null + } finally { + if (opts.showBusy) { + busyRef.current = false + setBusy(false) + } + } + }, + [client, pageParams] + ) + return { pageParams, sendBrowserRequest } +} diff --git a/mobile/src/browser/use-mobile-browser-stream.ts b/mobile/src/browser/use-mobile-browser-stream.ts new file mode 100644 index 00000000000..5b55dd03ff5 --- /dev/null +++ b/mobile/src/browser/use-mobile-browser-stream.ts @@ -0,0 +1,283 @@ +import { useEffect, useMemo, type Dispatch, type SetStateAction } from 'react' +import { PixelRatio, type Image, type View } from 'react-native' +import type { RpcClient } from '../transport/rpc-client' +import type { + BrowserScreencastFrame, + BrowserScreencastFrameMetadata +} from '../transport/browser-screencast-protocol' +import { + buildMobileBrowserScreencastRequest, + type MobileBrowserViewMode +} from './browser-screencast-request' +import { + MAX_ZOOM, + MIN_ZOOM, + getCachedBrowserFrame, + updateBrowserLayerVisibility, + type FrameLayer +} from './mobile-browser-frame-state' +import { + clampBrowserZoomState, + computeBrowserFrameGeometry, + type BrowserTouchLayout, + type BrowserZoomState +} from './browser-touch-geometry' +import type { MobileBrowserTab } from './MobileBrowserPane' +import { + handleBrowserScreencastEvent, + type BrowserDialogState, + type ScreencastEvent +} from './mobile-browser-stream-events' +import { useMobileBrowserFrameApply } from './use-mobile-browser-frame-apply' +import { useMobileBrowserRequest } from './use-mobile-browser-request' + +type PendingFrame = { frame: BrowserScreencastFrame; cacheKey: string } + +type MobileBrowserStreamArgs = { + appActive: boolean + browserImageRefs: { current: [Image | null, Image | null] } + browserLayerRefs: { current: [View | null, View | null] } + browserViewMode: MobileBrowserViewMode + busyRef: { current: boolean } + cacheKey: string | null + client: RpcClient | null + frameMetadata: BrowserScreencastFrameMetadata | null + frameMetadataRef: { current: BrowserScreencastFrameMetadata | null } + frameMountedRef: { current: boolean } + frameThrottleTimerRef: { current: ReturnType | null } + frameUriRef: { current: string | null } + lastAppliedFrameAtRef: { current: number } + lastStreamCacheKeyRef: { current: string | null } + lastZoomResetUrlRef: { current: string } + layout: BrowserTouchLayout | null + pendingFrameLayerRef: { current: FrameLayer | null } + pendingThrottledFrameRef: { current: PendingFrame | null } + resetBrowserZoomState: () => void + screencastSupported: boolean | null + setAddressValue: Dispatch> + setBusy: Dispatch> + setDialog: Dispatch> + setError: Dispatch> + setFrameMetadata: Dispatch> + setFrameUri: Dispatch> + setZoom: Dispatch> + streamGenerationRef: { current: number } + tab: MobileBrowserTab + visibleFrameLayerRef: { current: FrameLayer } + worktreeId: string + zoomRef: { current: BrowserZoomState } +} + +export function useMobileBrowserStream(args: MobileBrowserStreamArgs) { + const { + appActive, + browserImageRefs, + browserLayerRefs, + browserViewMode, + busyRef, + cacheKey, + client, + frameMetadata, + frameMetadataRef, + frameMountedRef, + frameThrottleTimerRef, + frameUriRef, + lastAppliedFrameAtRef, + lastStreamCacheKeyRef, + lastZoomResetUrlRef, + layout, + pendingFrameLayerRef, + pendingThrottledFrameRef, + resetBrowserZoomState, + screencastSupported, + setAddressValue, + setBusy, + setDialog, + setError, + setFrameMetadata, + setFrameUri, + setZoom, + streamGenerationRef, + tab, + visibleFrameLayerRef, + worktreeId, + zoomRef + } = args + + const { pageParams, sendBrowserRequest } = useMobileBrowserRequest({ + busyRef, + client, + pageId: tab.browserPageId, + setBusy, + setError, + worktreeId + }) + + const { applyFrameThrottled, clearFrameThrottle } = useMobileBrowserFrameApply({ + browserImageRefs, + busyRef, + frameMetadataRef, + frameMountedRef, + frameThrottleTimerRef, + frameUriRef, + lastAppliedFrameAtRef, + pendingFrameLayerRef, + pendingThrottledFrameRef, + setBusy, + setFrameMetadata, + setFrameUri, + visibleFrameLayerRef + }) + + const streamRequest = useMemo( + () => buildMobileBrowserScreencastRequest(layout, PixelRatio.get(), browserViewMode), + [browserViewMode, layout] + ) + + const frameGeometry = useMemo( + () => computeBrowserFrameGeometry(layout, frameMetadata), + [frameMetadata, layout] + ) + + useEffect(() => { + if (!frameGeometry) { + return + } + setZoom((current) => { + const next = clampBrowserZoomState(current, frameGeometry, MIN_ZOOM, MAX_ZOOM) + if ( + next.scale === current.scale && + next.offsetX === current.offsetX && + next.offsetY === current.offsetY + ) { + return current + } + // Why: rotation/layout changes can shrink the legal pan range while the + // current zoom state still points at the previous viewport geometry. + zoomRef.current = next + return next + }) + }, [frameGeometry]) + + useEffect(() => { + streamGenerationRef.current += 1 + const generation = streamGenerationRef.current + const sameStream = Boolean(cacheKey) && lastStreamCacheKeyRef.current === cacheKey + lastStreamCacheKeyRef.current = cacheKey + if (!sameStream || !frameUriRef.current) { + const cachedFrame = getCachedBrowserFrame(cacheKey) + if (cachedFrame) { + frameUriRef.current = cachedFrame.uri + frameMountedRef.current = true + frameMetadataRef.current = cachedFrame.metadata + setFrameUri(cachedFrame.uri) + setFrameMetadata(cachedFrame.metadata) + } else { + frameUriRef.current = null + frameMountedRef.current = false + setFrameUri(null) + setFrameMetadata(null) + frameMetadataRef.current = null + } + } else { + frameMountedRef.current = true + } + pendingFrameLayerRef.current = null + if (!sameStream || !frameUriRef.current) { + visibleFrameLayerRef.current = 0 + } + updateBrowserLayerVisibility(browserLayerRefs.current, visibleFrameLayerRef.current) + lastAppliedFrameAtRef.current = 0 + clearFrameThrottle() + busyRef.current = false + setDialog(null) + setError(null) + if ( + !client || + screencastSupported !== true || + !tab.browserPageId || + !appActive || + !streamRequest + ) { + busyRef.current = false + setBusy(false) + if (screencastSupported === false) { + setError('Update desktop Orca to stream browser tabs on mobile.') + } else if (screencastSupported === null) { + setError('Checking desktop browser streaming support.') + } else if (!tab.browserPageId) { + setError('Browser page is not available yet.') + } + return + } + busyRef.current = true + setBusy(true) + let startupTimer: ReturnType | null = setTimeout(() => { + if (streamGenerationRef.current !== generation) { + return + } + busyRef.current = false + setBusy(false) + setError('Browser stream timed out.') + }, 15_000) + const clearStartupTimer = (): void => { + if (startupTimer) { + clearTimeout(startupTimer) + startupTimer = null + } + } + const unsubscribe = client.subscribe( + 'browser.screencast', + { + worktree: `id:${worktreeId}`, + page: tab.browserPageId, + ...streamRequest + }, + (payload) => { + if (streamGenerationRef.current !== generation) { + return + } + handleBrowserScreencastEvent({ + busyRef, + clearStartupTimer, + event: payload as ScreencastEvent, + lastZoomResetUrlRef, + resetBrowserZoomState, + setAddressValue, + setBusy, + setDialog, + setError + }) + }, + { + onBinaryFrame: (frame) => { + if (streamGenerationRef.current !== generation) { + return + } + clearStartupTimer() + if (cacheKey) { + applyFrameThrottled(frame, cacheKey) + } + } + } + ) + return () => { + clearStartupTimer() + clearFrameThrottle() + unsubscribe() + } + }, [ + appActive, + applyFrameThrottled, + clearFrameThrottle, + client, + resetBrowserZoomState, + screencastSupported, + streamRequest, + cacheKey, + tab.browserPageId, + worktreeId + ]) + + return { frameGeometry, pageParams, sendBrowserRequest } +}