mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 00:02:31 +00:00
Split mobile browser pane modules (#17141)
* Split speech session lifecycle * Split terminal output scheduler pipeline * Split mobile browser pane modules * Fix F3-speech for #17123 * Fix F1-cycle for #17131
This commit is contained in:
Neil
@@ -83,7 +83,6 @@ mobile-config app/h/*/tasks.tsx
|
||||
mobile-config app/index.tsx
|
||||
mobile-config app/troubleshoot.tsx
|
||||
mobile-config scripts/mock-server.ts
|
||||
mobile-config src/browser/MobileBrowserPane.tsx
|
||||
mobile-config src/components/NewWorktreeModal.tsx
|
||||
mobile-config src/terminal/terminal-webview-html.ts
|
||||
mobile-config src/transport/rpc-client.ts
|
||||
|
||||
@@ -51,12 +51,6 @@
|
||||
"max-lines": ["error", { "max": 1603, "skipBlankLines": true, "skipComments": true }]
|
||||
}
|
||||
},
|
||||
{
|
||||
"files": ["src/browser/MobileBrowserPane.tsx"],
|
||||
"rules": {
|
||||
"max-lines": ["error", { "max": 1594, "skipBlankLines": true, "skipComments": true }]
|
||||
}
|
||||
},
|
||||
{
|
||||
"files": ["app/index.tsx"],
|
||||
"rules": {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,321 @@
|
||||
import type { Dispatch, RefObject, SetStateAction } from 'react'
|
||||
import {
|
||||
ActivityIndicator,
|
||||
Image,
|
||||
Pressable,
|
||||
Text,
|
||||
TextInput,
|
||||
View,
|
||||
type PanResponderInstance,
|
||||
type StyleProp,
|
||||
type ViewStyle
|
||||
} from 'react-native'
|
||||
import { ArrowUp, ChevronLeft, ChevronRight, RefreshCw } from 'lucide-react-native'
|
||||
import { colors } from '../theme/mobile-theme'
|
||||
import { MobileBrowserAddressField } from './MobileBrowserAddressField'
|
||||
import { MobileBrowserKeyRow } from './MobileBrowserKeyRow'
|
||||
import {
|
||||
MobileBrowserPointerModifiers,
|
||||
type BrowserPointerModifier
|
||||
} from './MobileBrowserPointerModifiers'
|
||||
import { MobileBrowserToolbarIconButton } from './MobileBrowserToolbarIconButton'
|
||||
import { MobileBrowserViewModeSwitch } from './MobileBrowserViewModeSwitch'
|
||||
import { buttonColor, type FrameLayer } from './mobile-browser-frame-state'
|
||||
import { mobileBrowserPaneStyles as styles } from './mobile-browser-pane-styles'
|
||||
import type {
|
||||
BrowserFrameGeometry,
|
||||
BrowserTouchLayout,
|
||||
BrowserZoomState
|
||||
} from './browser-touch-geometry'
|
||||
import type { MobileBrowserViewMode } from './browser-screencast-request'
|
||||
import type { MobileBrowserTab } from './MobileBrowserPane'
|
||||
|
||||
type MobileBrowserPaneViewProps = {
|
||||
addressFocused: boolean
|
||||
addressValue: string
|
||||
bottomInset: number
|
||||
browserLayerRef: (layer: FrameLayer) => (view: View | null) => void
|
||||
browserViewMode: MobileBrowserViewMode
|
||||
busy: boolean
|
||||
controlsDisabled: boolean
|
||||
dialog: { dialogType: string; message: string } | null
|
||||
error: string | null
|
||||
frameGeometry: BrowserFrameGeometry | null
|
||||
frameLayerErrorHandler: (layer: FrameLayer) => () => void
|
||||
frameLayerLoadHandler: (layer: FrameLayer) => () => void
|
||||
frameLayerRef: (layer: FrameLayer) => (image: Image | null) => void
|
||||
frameLayerStyle: (layer: FrameLayer) => StyleProp<ViewStyle>
|
||||
goBack: () => void
|
||||
goForward: () => void
|
||||
keyboardLift: number
|
||||
keyboardValue: string
|
||||
layoutRef: RefObject<BrowserTouchLayout | null>
|
||||
navigateToAddress: () => Promise<void>
|
||||
panResponder: PanResponderInstance
|
||||
pointerModifiers: BrowserPointerModifier[]
|
||||
reloadPage: () => void
|
||||
renderedFrameSource: { uri: string } | null
|
||||
selectBrowserViewMode: (mode: MobileBrowserViewMode) => void
|
||||
sendDialogCommand: (method: 'browser.dialogDismiss' | 'browser.dialogAccept') => Promise<void>
|
||||
sendKeyboardText: () => Promise<void>
|
||||
sendKeypress: (key: string) => Promise<void>
|
||||
setAddressFocused: Dispatch<SetStateAction<boolean>>
|
||||
setAddressValue: Dispatch<SetStateAction<string>>
|
||||
setKeyboardValue: Dispatch<SetStateAction<string>>
|
||||
setLayout: Dispatch<SetStateAction<BrowserTouchLayout | null>>
|
||||
setRootViewRef: (view: View | null) => void
|
||||
tab: MobileBrowserTab
|
||||
togglePointerModifier: (modifier: BrowserPointerModifier) => void
|
||||
zoom: BrowserZoomState
|
||||
}
|
||||
|
||||
export function MobileBrowserPaneView(props: MobileBrowserPaneViewProps) {
|
||||
const {
|
||||
addressFocused,
|
||||
addressValue,
|
||||
bottomInset,
|
||||
browserLayerRef,
|
||||
browserViewMode,
|
||||
busy,
|
||||
controlsDisabled,
|
||||
dialog,
|
||||
error,
|
||||
frameGeometry,
|
||||
frameLayerErrorHandler,
|
||||
frameLayerLoadHandler,
|
||||
frameLayerRef,
|
||||
frameLayerStyle,
|
||||
goBack,
|
||||
goForward,
|
||||
keyboardLift,
|
||||
keyboardValue,
|
||||
layoutRef,
|
||||
navigateToAddress,
|
||||
panResponder,
|
||||
pointerModifiers,
|
||||
reloadPage,
|
||||
renderedFrameSource,
|
||||
selectBrowserViewMode,
|
||||
sendDialogCommand,
|
||||
sendKeyboardText,
|
||||
sendKeypress,
|
||||
setAddressFocused,
|
||||
setAddressValue,
|
||||
setKeyboardValue,
|
||||
setLayout,
|
||||
setRootViewRef,
|
||||
tab,
|
||||
togglePointerModifier,
|
||||
zoom
|
||||
} = props
|
||||
return (
|
||||
<View ref={setRootViewRef} style={styles.root}>
|
||||
<View style={styles.toolbar}>
|
||||
<MobileBrowserToolbarIconButton
|
||||
disabled={controlsDisabled || !tab.canGoBack}
|
||||
label="Back"
|
||||
onPress={goBack}
|
||||
>
|
||||
<ChevronLeft size={15} color={buttonColor(!controlsDisabled && tab.canGoBack)} />
|
||||
</MobileBrowserToolbarIconButton>
|
||||
<MobileBrowserToolbarIconButton
|
||||
disabled={controlsDisabled || !tab.canGoForward}
|
||||
label="Forward"
|
||||
onPress={goForward}
|
||||
>
|
||||
<ChevronRight size={15} color={buttonColor(!controlsDisabled && tab.canGoForward)} />
|
||||
</MobileBrowserToolbarIconButton>
|
||||
<MobileBrowserToolbarIconButton
|
||||
disabled={controlsDisabled}
|
||||
label="Reload"
|
||||
onPress={reloadPage}
|
||||
>
|
||||
<RefreshCw size={15} color={buttonColor(!controlsDisabled)} />
|
||||
</MobileBrowserToolbarIconButton>
|
||||
<MobileBrowserAddressField
|
||||
value={addressValue}
|
||||
onChangeText={setAddressValue}
|
||||
onFocus={() => setAddressFocused(true)}
|
||||
onBlur={() => setAddressFocused(false)}
|
||||
onSubmit={() => void navigateToAddress()}
|
||||
focused={addressFocused}
|
||||
disabled={controlsDisabled}
|
||||
/>
|
||||
<MobileBrowserViewModeSwitch
|
||||
disabled={controlsDisabled}
|
||||
value={browserViewMode}
|
||||
onChange={selectBrowserViewMode}
|
||||
/>
|
||||
</View>
|
||||
|
||||
<View
|
||||
style={styles.viewport}
|
||||
onLayout={(event) => {
|
||||
const next = {
|
||||
width: event.nativeEvent.layout.width,
|
||||
height: event.nativeEvent.layout.height
|
||||
}
|
||||
const current = layoutRef.current
|
||||
if (current && current.width === next.width && current.height === next.height) {
|
||||
return
|
||||
}
|
||||
layoutRef.current = next
|
||||
setLayout(next)
|
||||
}}
|
||||
{...panResponder.panHandlers}
|
||||
>
|
||||
{renderedFrameSource ? (
|
||||
<View style={styles.browserImageHost}>
|
||||
{frameGeometry ? (
|
||||
<View
|
||||
pointerEvents="none"
|
||||
style={[
|
||||
styles.browserZoomOffset,
|
||||
{
|
||||
width: frameGeometry.renderedWidth,
|
||||
height: frameGeometry.renderedHeight,
|
||||
transform: [{ translateX: zoom.offsetX }, { translateY: zoom.offsetY }]
|
||||
}
|
||||
]}
|
||||
>
|
||||
<View
|
||||
style={[
|
||||
styles.browserFrameBox,
|
||||
{
|
||||
width: frameGeometry.renderedWidth,
|
||||
height: frameGeometry.renderedHeight,
|
||||
transform: [{ scale: zoom.scale }]
|
||||
}
|
||||
]}
|
||||
>
|
||||
{([0, 1] as const).map((layer) => (
|
||||
<View
|
||||
key={layer}
|
||||
ref={browserLayerRef(layer)}
|
||||
pointerEvents="none"
|
||||
style={frameLayerStyle(layer)}
|
||||
>
|
||||
<Image
|
||||
ref={frameLayerRef(layer)}
|
||||
source={renderedFrameSource}
|
||||
resizeMode="stretch"
|
||||
fadeDuration={0}
|
||||
onLoad={frameLayerLoadHandler(layer)}
|
||||
onError={frameLayerErrorHandler(layer)}
|
||||
style={[
|
||||
styles.browserImage,
|
||||
{
|
||||
width: frameGeometry.renderedWidth,
|
||||
height: frameGeometry.renderedHeight
|
||||
}
|
||||
]}
|
||||
/>
|
||||
</View>
|
||||
))}
|
||||
</View>
|
||||
</View>
|
||||
) : (
|
||||
([0, 1] as const).map((layer) => (
|
||||
<View
|
||||
key={layer}
|
||||
ref={browserLayerRef(layer)}
|
||||
pointerEvents="none"
|
||||
style={frameLayerStyle(layer)}
|
||||
>
|
||||
<Image
|
||||
ref={frameLayerRef(layer)}
|
||||
source={renderedFrameSource}
|
||||
resizeMode="contain"
|
||||
fadeDuration={0}
|
||||
onLoad={frameLayerLoadHandler(layer)}
|
||||
onError={frameLayerErrorHandler(layer)}
|
||||
style={styles.browserImageFill}
|
||||
/>
|
||||
</View>
|
||||
))
|
||||
)}
|
||||
</View>
|
||||
) : null}
|
||||
{!renderedFrameSource || busy || error ? (
|
||||
<View pointerEvents="none" style={styles.overlay}>
|
||||
{/* Why: a stream can report ready and then deliver no frames, so key the
|
||||
indicator off actually having pixels or it clears into a blank pane. */}
|
||||
{busy || (!renderedFrameSource && !error) ? (
|
||||
<ActivityIndicator size="small" color={colors.textSecondary} />
|
||||
) : null}
|
||||
{error ? <Text style={styles.errorText}>{error}</Text> : null}
|
||||
</View>
|
||||
) : null}
|
||||
{dialog ? (
|
||||
<View style={styles.dialogOverlay}>
|
||||
<View style={styles.dialogCard}>
|
||||
<Text style={styles.dialogTitle}>Browser Dialog</Text>
|
||||
<Text style={styles.dialogMessage}>{dialog.message}</Text>
|
||||
<View style={styles.dialogActions}>
|
||||
{dialog.dialogType !== 'alert' ? (
|
||||
<Pressable
|
||||
style={({ pressed }) => [
|
||||
styles.dialogButton,
|
||||
pressed && styles.dialogButtonPressed
|
||||
]}
|
||||
onPress={() => void sendDialogCommand('browser.dialogDismiss')}
|
||||
>
|
||||
<Text style={styles.dialogButtonText}>Cancel</Text>
|
||||
</Pressable>
|
||||
) : null}
|
||||
<Pressable
|
||||
style={({ pressed }) => [
|
||||
styles.dialogButton,
|
||||
styles.dialogButtonPrimary,
|
||||
pressed && styles.dialogButtonPressed
|
||||
]}
|
||||
onPress={() => void sendDialogCommand('browser.dialogAccept')}
|
||||
>
|
||||
<Text style={[styles.dialogButtonText, styles.dialogButtonPrimaryText]}>OK</Text>
|
||||
</Pressable>
|
||||
</View>
|
||||
</View>
|
||||
</View>
|
||||
) : null}
|
||||
</View>
|
||||
|
||||
<View
|
||||
style={[
|
||||
styles.keyboardDock,
|
||||
{ paddingBottom: bottomInset, transform: [{ translateY: -keyboardLift }] }
|
||||
]}
|
||||
>
|
||||
<MobileBrowserPointerModifiers
|
||||
disabled={controlsDisabled}
|
||||
selectedModifiers={pointerModifiers}
|
||||
onToggle={togglePointerModifier}
|
||||
/>
|
||||
<MobileBrowserKeyRow
|
||||
disabled={controlsDisabled}
|
||||
onKeypress={(key) => void sendKeypress(key)}
|
||||
/>
|
||||
<View style={styles.inputRow}>
|
||||
<TextInput
|
||||
style={styles.keyboardInput}
|
||||
value={keyboardValue}
|
||||
onChangeText={setKeyboardValue}
|
||||
placeholder="Type on page…"
|
||||
placeholderTextColor={colors.textMuted}
|
||||
autoCapitalize="none"
|
||||
autoCorrect={false}
|
||||
editable={!controlsDisabled}
|
||||
onSubmitEditing={() => void sendKeyboardText()}
|
||||
/>
|
||||
<Pressable
|
||||
style={[styles.sendButton, (controlsDisabled || !keyboardValue) && styles.disabled]}
|
||||
disabled={controlsDisabled || !keyboardValue}
|
||||
onPress={() => void sendKeyboardText()}
|
||||
accessibilityLabel="Send text to browser"
|
||||
>
|
||||
<ArrowUp size={18} color={buttonColor(!controlsDisabled && !!keyboardValue)} />
|
||||
</Pressable>
|
||||
</View>
|
||||
</View>
|
||||
</View>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { Dispatch, SetStateAction } from 'react'
|
||||
import { displayBrowserUrl } from './browser-url'
|
||||
import { shouldSurfaceBrowserError } from './mobile-browser-frame-state'
|
||||
|
||||
export type BrowserDialogState = { dialogType: string; message: string }
|
||||
|
||||
export type ScreencastEvent = {
|
||||
type?: string
|
||||
message?: string
|
||||
error?: { message?: string }
|
||||
dialogType?: string
|
||||
tab?: { url?: string; title?: string; canGoBack?: boolean; canGoForward?: boolean }
|
||||
}
|
||||
|
||||
type HandleScreencastEventArgs = {
|
||||
busyRef: { current: boolean }
|
||||
clearStartupTimer: () => void
|
||||
event: ScreencastEvent
|
||||
lastZoomResetUrlRef: { current: string }
|
||||
resetBrowserZoomState: () => void
|
||||
setAddressValue: Dispatch<SetStateAction<string>>
|
||||
setBusy: Dispatch<SetStateAction<boolean>>
|
||||
setDialog: Dispatch<SetStateAction<BrowserDialogState | null>>
|
||||
setError: Dispatch<SetStateAction<string | null>>
|
||||
}
|
||||
|
||||
export function handleBrowserScreencastEvent(args: HandleScreencastEventArgs): void {
|
||||
const {
|
||||
busyRef,
|
||||
clearStartupTimer,
|
||||
event,
|
||||
lastZoomResetUrlRef,
|
||||
resetBrowserZoomState,
|
||||
setAddressValue,
|
||||
setBusy,
|
||||
setDialog,
|
||||
setError
|
||||
} = args
|
||||
|
||||
if (event.type === 'ready') {
|
||||
clearStartupTimer()
|
||||
if (busyRef.current) {
|
||||
busyRef.current = false
|
||||
setBusy(false)
|
||||
}
|
||||
if (typeof event.tab?.url === 'string') {
|
||||
setAddressValue(displayBrowserUrl(event.tab.url))
|
||||
if (event.tab.url !== lastZoomResetUrlRef.current) {
|
||||
lastZoomResetUrlRef.current = event.tab.url
|
||||
resetBrowserZoomState()
|
||||
}
|
||||
}
|
||||
} else if (event.type === 'end') {
|
||||
clearStartupTimer()
|
||||
if (busyRef.current) {
|
||||
busyRef.current = false
|
||||
setBusy(false)
|
||||
}
|
||||
} else if (event.type === 'dialog') {
|
||||
setDialog({
|
||||
dialogType: event.dialogType ?? 'alert',
|
||||
message: event.message ?? 'Browser dialog'
|
||||
})
|
||||
} else if (event.type === 'dialogClosed') {
|
||||
setDialog(null)
|
||||
} else if (event.type === 'error') {
|
||||
clearStartupTimer()
|
||||
if (busyRef.current) {
|
||||
busyRef.current = false
|
||||
setBusy(false)
|
||||
}
|
||||
const message = event.message ?? event.error?.message ?? 'Browser stream failed.'
|
||||
if (shouldSurfaceBrowserError(message)) {
|
||||
setError(message)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
import { useCallback, useRef, type Dispatch, type SetStateAction } from 'react'
|
||||
import type { RpcClient } from '../transport/rpc-client'
|
||||
import type { BrowserScreencastFrameMetadata } from '../transport/browser-screencast-protocol'
|
||||
import { assertRpcOk } from './mobile-browser-frame-state'
|
||||
import {
|
||||
computeBrowserFrameGeometry,
|
||||
computeBrowserTouchClickRadiusCss,
|
||||
mapScreenToBrowserPoint,
|
||||
type BrowserPoint,
|
||||
type BrowserTouchLayout,
|
||||
type BrowserZoomState
|
||||
} from './browser-touch-geometry'
|
||||
import type { BrowserPointerModifier } from './MobileBrowserPointerModifiers'
|
||||
|
||||
const TOUCH_CLICK_RADIUS_DIP = 14
|
||||
type BrowserPageParams = { worktree: string; page: string }
|
||||
type PendingWheelCommand = {
|
||||
base: BrowserPageParams
|
||||
point: BrowserPoint
|
||||
gestureId: number
|
||||
dx: number
|
||||
dy: number
|
||||
}
|
||||
type SendBrowserRequest = (
|
||||
method: string,
|
||||
params?: Record<string, unknown>,
|
||||
options?: { showBusy?: boolean; suppressError?: boolean; timeoutMs?: number }
|
||||
) => Promise<unknown | null>
|
||||
|
||||
type MobileBrowserCommandArgs = {
|
||||
client: RpcClient | null
|
||||
frameMetadataRef: { current: BrowserScreencastFrameMetadata | null }
|
||||
keyboardValue: string
|
||||
layoutRef: { current: BrowserTouchLayout | null }
|
||||
onToast: (message: string, durationMs?: number) => void
|
||||
pageParams: () => BrowserPageParams | null
|
||||
pointerModifiers: BrowserPointerModifier[]
|
||||
sendBrowserRequest: SendBrowserRequest
|
||||
setDialog: Dispatch<SetStateAction<{ dialogType: string; message: string } | null>>
|
||||
setError: Dispatch<SetStateAction<string | null>>
|
||||
setKeyboardValue: Dispatch<SetStateAction<string>>
|
||||
setPointerModifiers: Dispatch<SetStateAction<BrowserPointerModifier[]>>
|
||||
zoomRef: { current: BrowserZoomState }
|
||||
}
|
||||
|
||||
export function useMobileBrowserCommands(args: MobileBrowserCommandArgs) {
|
||||
const {
|
||||
client,
|
||||
frameMetadataRef,
|
||||
keyboardValue,
|
||||
layoutRef,
|
||||
onToast,
|
||||
pageParams,
|
||||
pointerModifiers,
|
||||
sendBrowserRequest,
|
||||
setDialog,
|
||||
setError,
|
||||
setKeyboardValue,
|
||||
setPointerModifiers,
|
||||
zoomRef
|
||||
} = args
|
||||
|
||||
const pendingWheelCommandRef = useRef<PendingWheelCommand | null>(null)
|
||||
|
||||
const wheelCommandInFlightRef = useRef(false)
|
||||
|
||||
const flushPendingWheelCommand = useCallback(() => {
|
||||
if (wheelCommandInFlightRef.current) {
|
||||
return
|
||||
}
|
||||
const pending = pendingWheelCommandRef.current
|
||||
if (!pending || !client) {
|
||||
return
|
||||
}
|
||||
pendingWheelCommandRef.current = null
|
||||
wheelCommandInFlightRef.current = true
|
||||
void (async () => {
|
||||
try {
|
||||
assertRpcOk(
|
||||
await client.sendRequest('browser.mouseMove', {
|
||||
...pending.base,
|
||||
x: pending.point.x,
|
||||
y: pending.point.y
|
||||
}),
|
||||
'Browser pointer move failed'
|
||||
)
|
||||
assertRpcOk(
|
||||
await client.sendRequest('browser.mouseWheel', {
|
||||
...pending.base,
|
||||
dx: pending.dx,
|
||||
dy: pending.dy
|
||||
}),
|
||||
'Browser scroll failed'
|
||||
)
|
||||
setError(null)
|
||||
} catch {
|
||||
// Scroll bursts commonly race page reload/navigation. Avoid replacing
|
||||
// the live browser with transient command errors like selector_not_found.
|
||||
} finally {
|
||||
wheelCommandInFlightRef.current = false
|
||||
flushPendingWheelCommand()
|
||||
}
|
||||
})()
|
||||
}, [client])
|
||||
|
||||
const sendPointerClick = useCallback(
|
||||
async (point: BrowserPoint, button: 'left' | 'right') => {
|
||||
const base = pageParams()
|
||||
if (!client || !base) {
|
||||
return
|
||||
}
|
||||
const clickResult = await sendBrowserRequest(
|
||||
'browser.mouseClick',
|
||||
{
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button,
|
||||
modifiers: pointerModifiers,
|
||||
...(button === 'left'
|
||||
? {
|
||||
radius: computeBrowserTouchClickRadiusCss(
|
||||
layoutRef.current,
|
||||
frameMetadataRef.current,
|
||||
zoomRef.current,
|
||||
TOUCH_CLICK_RADIUS_DIP
|
||||
)
|
||||
}
|
||||
: {})
|
||||
},
|
||||
{ suppressError: true, timeoutMs: 5_000 }
|
||||
)
|
||||
if (clickResult !== null || pointerModifiers.length > 0) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
assertRpcOk(
|
||||
await client.sendRequest('browser.mouseMove', { ...base, x: point.x, y: point.y }),
|
||||
'Browser pointer move failed'
|
||||
)
|
||||
assertRpcOk(
|
||||
await client.sendRequest('browser.mouseDown', { ...base, button }),
|
||||
'Browser pointer down failed'
|
||||
)
|
||||
assertRpcOk(
|
||||
await client.sendRequest('browser.mouseUp', { ...base, button }),
|
||||
'Browser pointer up failed'
|
||||
)
|
||||
setError(null)
|
||||
} catch {
|
||||
// Pointer commands can race page navigation. Keep the stream visible;
|
||||
// actionable failures still surface through navigation/stream errors.
|
||||
}
|
||||
},
|
||||
[client, pageParams, pointerModifiers, sendBrowserRequest]
|
||||
)
|
||||
|
||||
const togglePointerModifier = useCallback((modifier: BrowserPointerModifier) => {
|
||||
setPointerModifiers((current) =>
|
||||
current.includes(modifier)
|
||||
? current.filter((candidate) => candidate !== modifier)
|
||||
: [...current, modifier]
|
||||
)
|
||||
}, [])
|
||||
|
||||
const sendWheel = useCallback(
|
||||
(point: BrowserPoint, screenDx: number, screenDy: number, gestureId: number) => {
|
||||
const base = pageParams()
|
||||
if (!client || !base) {
|
||||
return
|
||||
}
|
||||
const currentLayout = layoutRef.current
|
||||
const geometry = computeBrowserFrameGeometry(currentLayout, frameMetadataRef.current)
|
||||
const localZoom = zoomRef.current.scale
|
||||
const scale = (geometry?.scale ?? 1) * localZoom
|
||||
const cssDx = screenDx / scale
|
||||
const cssDy = screenDy / scale
|
||||
const delta = { dx: Math.round(-cssDx), dy: Math.round(-cssDy) }
|
||||
if (Math.abs(delta.dx) < 1 && Math.abs(delta.dy) < 1) {
|
||||
return
|
||||
}
|
||||
const pending = pendingWheelCommandRef.current
|
||||
pendingWheelCommandRef.current =
|
||||
pending && pending.base.page === base.page && pending.gestureId === gestureId
|
||||
? {
|
||||
base,
|
||||
point,
|
||||
gestureId,
|
||||
dx: pending.dx + delta.dx,
|
||||
dy: pending.dy + delta.dy
|
||||
}
|
||||
: { base, point, gestureId, ...delta }
|
||||
flushPendingWheelCommand()
|
||||
},
|
||||
[client, flushPendingWheelCommand, pageParams]
|
||||
)
|
||||
|
||||
const mapTouchPoint = useCallback((locationX: number, locationY: number): BrowserPoint | null => {
|
||||
return mapScreenToBrowserPoint(
|
||||
locationX,
|
||||
locationY,
|
||||
layoutRef.current,
|
||||
frameMetadataRef.current,
|
||||
zoomRef.current
|
||||
)
|
||||
}, [])
|
||||
|
||||
const sendKeyboardText = useCallback(async () => {
|
||||
const text = keyboardValue
|
||||
if (!text) {
|
||||
return
|
||||
}
|
||||
setKeyboardValue('')
|
||||
const result = await sendBrowserRequest(
|
||||
'browser.keyboardInsertText',
|
||||
{ text },
|
||||
{ suppressError: true }
|
||||
)
|
||||
if (result !== null) {
|
||||
onToast('Sent')
|
||||
} else {
|
||||
setKeyboardValue(text)
|
||||
}
|
||||
}, [keyboardValue, onToast, sendBrowserRequest])
|
||||
|
||||
const sendKeypress = useCallback(
|
||||
async (key: string) => {
|
||||
await sendBrowserRequest('browser.keypress', { key }, { suppressError: true })
|
||||
},
|
||||
[sendBrowserRequest]
|
||||
)
|
||||
|
||||
const sendDialogCommand = useCallback(
|
||||
async (method: 'browser.dialogAccept' | 'browser.dialogDismiss') => {
|
||||
setDialog(null)
|
||||
await sendBrowserRequest(method, {}, { suppressError: true, timeoutMs: 5_000 })
|
||||
},
|
||||
[sendBrowserRequest]
|
||||
)
|
||||
return {
|
||||
mapTouchPoint,
|
||||
sendDialogCommand,
|
||||
sendKeyboardText,
|
||||
sendKeypress,
|
||||
sendPointerClick,
|
||||
sendWheel,
|
||||
togglePointerModifier
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
import { useCallback, type Dispatch, type SetStateAction } from 'react'
|
||||
import type { Image } from 'react-native'
|
||||
import type {
|
||||
BrowserScreencastFrame,
|
||||
BrowserScreencastFrameMetadata
|
||||
} from '../transport/browser-screencast-protocol'
|
||||
import { MOBILE_BROWSER_FRAME_MIN_INTERVAL_MS } from './browser-screencast-request'
|
||||
import {
|
||||
browserFrameMetadataEqual,
|
||||
cacheBrowserFrame,
|
||||
createBrowserFrameDataUri,
|
||||
updateBrowserImageSource,
|
||||
type FrameLayer
|
||||
} from './mobile-browser-frame-state'
|
||||
|
||||
type PendingFrame = { frame: BrowserScreencastFrame; cacheKey: string }
|
||||
type BrowserFrameApplyArgs = {
|
||||
browserImageRefs: { current: [Image | null, Image | null] }
|
||||
busyRef: { current: boolean }
|
||||
frameMetadataRef: { current: BrowserScreencastFrameMetadata | null }
|
||||
frameMountedRef: { current: boolean }
|
||||
frameThrottleTimerRef: { current: ReturnType<typeof setTimeout> | null }
|
||||
frameUriRef: { current: string | null }
|
||||
lastAppliedFrameAtRef: { current: number }
|
||||
pendingFrameLayerRef: { current: FrameLayer | null }
|
||||
pendingThrottledFrameRef: { current: PendingFrame | null }
|
||||
setBusy: Dispatch<SetStateAction<boolean>>
|
||||
setFrameMetadata: Dispatch<SetStateAction<BrowserScreencastFrameMetadata | null>>
|
||||
setFrameUri: Dispatch<SetStateAction<string | null>>
|
||||
visibleFrameLayerRef: { current: FrameLayer }
|
||||
}
|
||||
export function useMobileBrowserFrameApply(args: BrowserFrameApplyArgs) {
|
||||
const {
|
||||
browserImageRefs,
|
||||
busyRef,
|
||||
frameMetadataRef,
|
||||
frameMountedRef,
|
||||
frameThrottleTimerRef,
|
||||
frameUriRef,
|
||||
lastAppliedFrameAtRef,
|
||||
pendingFrameLayerRef,
|
||||
pendingThrottledFrameRef,
|
||||
setBusy,
|
||||
setFrameMetadata,
|
||||
setFrameUri,
|
||||
visibleFrameLayerRef
|
||||
} = args
|
||||
const applyFrame = useCallback((frame: BrowserScreencastFrame, frameCacheKey: string): void => {
|
||||
if (!browserFrameMetadataEqual(frameMetadataRef.current, frame.metadata)) {
|
||||
frameMetadataRef.current = frame.metadata
|
||||
setFrameMetadata(frame.metadata)
|
||||
}
|
||||
const nextFrameUri = createBrowserFrameDataUri(frame)
|
||||
cacheBrowserFrame(frameCacheKey, { uri: nextFrameUri, metadata: frame.metadata })
|
||||
if (!frameMountedRef.current) {
|
||||
frameUriRef.current = nextFrameUri
|
||||
frameMountedRef.current = true
|
||||
setFrameUri(nextFrameUri)
|
||||
updateBrowserImageSource(browserImageRefs.current[0], nextFrameUri)
|
||||
} else if (pendingFrameLayerRef.current === null) {
|
||||
// Why: decode the next frame offscreen and keep the previous layer visible
|
||||
// until onLoad; replacing the visible Image directly flashes black.
|
||||
const nextLayer: FrameLayer = visibleFrameLayerRef.current === 0 ? 1 : 0
|
||||
frameUriRef.current = nextFrameUri
|
||||
pendingFrameLayerRef.current = nextLayer
|
||||
updateBrowserImageSource(browserImageRefs.current[nextLayer], nextFrameUri)
|
||||
} else {
|
||||
// Why: popovers/menus can settle in one final frame while the previous
|
||||
// offscreen frame is still decoding. Keep the hidden layer pointed at
|
||||
// the newest frame instead of dropping the final static state.
|
||||
frameUriRef.current = nextFrameUri
|
||||
updateBrowserImageSource(browserImageRefs.current[pendingFrameLayerRef.current], nextFrameUri)
|
||||
}
|
||||
if (busyRef.current) {
|
||||
busyRef.current = false
|
||||
setBusy(false)
|
||||
}
|
||||
}, [])
|
||||
|
||||
const clearFrameThrottle = useCallback(() => {
|
||||
pendingThrottledFrameRef.current = null
|
||||
if (frameThrottleTimerRef.current) {
|
||||
clearTimeout(frameThrottleTimerRef.current)
|
||||
frameThrottleTimerRef.current = null
|
||||
}
|
||||
}, [])
|
||||
|
||||
const applyFrameThrottled = useCallback(
|
||||
(frame: BrowserScreencastFrame, frameCacheKey: string): void => {
|
||||
const now = Date.now()
|
||||
const elapsed = now - lastAppliedFrameAtRef.current
|
||||
if (lastAppliedFrameAtRef.current === 0 || elapsed >= MOBILE_BROWSER_FRAME_MIN_INTERVAL_MS) {
|
||||
clearFrameThrottle()
|
||||
lastAppliedFrameAtRef.current = now
|
||||
applyFrame(frame, frameCacheKey)
|
||||
return
|
||||
}
|
||||
|
||||
// Why: static UI changes can be the last frame Chromium emits. Coalesce
|
||||
// throttled frames so the final visible state is applied after the delay.
|
||||
pendingThrottledFrameRef.current = { frame, cacheKey: frameCacheKey }
|
||||
if (frameThrottleTimerRef.current) {
|
||||
return
|
||||
}
|
||||
frameThrottleTimerRef.current = setTimeout(
|
||||
() => {
|
||||
frameThrottleTimerRef.current = null
|
||||
const pending = pendingThrottledFrameRef.current
|
||||
pendingThrottledFrameRef.current = null
|
||||
if (!pending) {
|
||||
return
|
||||
}
|
||||
lastAppliedFrameAtRef.current = Date.now()
|
||||
applyFrame(pending.frame, pending.cacheKey)
|
||||
},
|
||||
Math.max(0, MOBILE_BROWSER_FRAME_MIN_INTERVAL_MS - elapsed)
|
||||
)
|
||||
},
|
||||
[applyFrame, clearFrameThrottle]
|
||||
)
|
||||
return { applyFrameThrottled, clearFrameThrottle }
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
import { useCallback, useMemo, useRef, type Dispatch, type SetStateAction } from 'react'
|
||||
import {
|
||||
PanResponder,
|
||||
type GestureResponderEvent,
|
||||
type PanResponderGestureState
|
||||
} from 'react-native'
|
||||
import type { RpcClient } from '../transport/rpc-client'
|
||||
import {
|
||||
createPinchGesture,
|
||||
MAX_ZOOM,
|
||||
MIN_ZOOM,
|
||||
updatePinchZoom,
|
||||
type PinchGesture
|
||||
} from './mobile-browser-frame-state'
|
||||
import {
|
||||
clampBrowserZoomState,
|
||||
readLocalTouchPoint,
|
||||
type BrowserFrameGeometry,
|
||||
type BrowserTouchLayout,
|
||||
type BrowserZoomState
|
||||
} from './browser-touch-geometry'
|
||||
import type { BrowserPointerModifier } from './MobileBrowserPointerModifiers'
|
||||
import type { BrowserScreencastFrameMetadata } from '../transport/browser-screencast-protocol'
|
||||
|
||||
import { useMobileBrowserCommands } from './use-mobile-browser-commands'
|
||||
const TAP_SLOP = 16
|
||||
const SCROLL_START_SLOP = 22
|
||||
const LONG_PRESS_MS = 550
|
||||
const WHEEL_INTERVAL_MS = 70
|
||||
|
||||
type BrowserPageParams = { worktree: string; page: string }
|
||||
type PanGesture = { x: number; y: number; offsetX: number; offsetY: number }
|
||||
type SendBrowserRequest = (
|
||||
method: string,
|
||||
params?: Record<string, unknown>,
|
||||
options?: { showBusy?: boolean; suppressError?: boolean; timeoutMs?: number }
|
||||
) => Promise<unknown | null>
|
||||
|
||||
type MobileBrowserInteractionArgs = {
|
||||
clearLongPressTimer: () => void
|
||||
client: RpcClient | null
|
||||
dialogRef: { current: { dialogType: string; message: string } | null }
|
||||
frameGeometry: BrowserFrameGeometry | null
|
||||
frameMetadataRef: { current: BrowserScreencastFrameMetadata | null }
|
||||
keyboardValue: string
|
||||
layoutRef: { current: BrowserTouchLayout | null }
|
||||
longPressTimerRef: { current: ReturnType<typeof setTimeout> | null }
|
||||
onToast: (message: string, durationMs?: number) => void
|
||||
pageParams: () => BrowserPageParams | null
|
||||
panRef: { current: PanGesture | null }
|
||||
pinchRef: { current: PinchGesture | null }
|
||||
pointerModifiers: BrowserPointerModifier[]
|
||||
sendBrowserRequest: SendBrowserRequest
|
||||
setDialog: Dispatch<SetStateAction<{ dialogType: string; message: string } | null>>
|
||||
setError: Dispatch<SetStateAction<string | null>>
|
||||
setKeyboardValue: Dispatch<SetStateAction<string>>
|
||||
scrollingRef: { current: boolean }
|
||||
startPointRef: { current: { x: number; y: number; t: number } | null }
|
||||
setPointerModifiers: Dispatch<SetStateAction<BrowserPointerModifier[]>>
|
||||
setZoom: Dispatch<SetStateAction<BrowserZoomState>>
|
||||
zoomRef: { current: BrowserZoomState }
|
||||
}
|
||||
|
||||
export function useMobileBrowserInteractions(args: MobileBrowserInteractionArgs) {
|
||||
const {
|
||||
clearLongPressTimer,
|
||||
client,
|
||||
dialogRef,
|
||||
frameGeometry,
|
||||
frameMetadataRef,
|
||||
keyboardValue,
|
||||
layoutRef,
|
||||
longPressTimerRef,
|
||||
onToast,
|
||||
pageParams,
|
||||
panRef,
|
||||
pinchRef,
|
||||
pointerModifiers,
|
||||
scrollingRef,
|
||||
sendBrowserRequest,
|
||||
setDialog,
|
||||
setError,
|
||||
setKeyboardValue,
|
||||
setPointerModifiers,
|
||||
setZoom,
|
||||
startPointRef,
|
||||
zoomRef
|
||||
} = args
|
||||
const rightClickSentRef = useRef(false)
|
||||
const lastWheelRef = useRef({ dx: 0, dy: 0, at: 0 })
|
||||
const wheelGestureIdRef = useRef(0)
|
||||
const {
|
||||
mapTouchPoint,
|
||||
sendDialogCommand,
|
||||
sendKeyboardText,
|
||||
sendKeypress,
|
||||
sendPointerClick,
|
||||
sendWheel,
|
||||
togglePointerModifier
|
||||
} = useMobileBrowserCommands({
|
||||
client,
|
||||
frameMetadataRef,
|
||||
keyboardValue,
|
||||
layoutRef,
|
||||
onToast,
|
||||
pageParams,
|
||||
pointerModifiers,
|
||||
sendBrowserRequest,
|
||||
setDialog,
|
||||
setError,
|
||||
setKeyboardValue,
|
||||
setPointerModifiers,
|
||||
zoomRef
|
||||
})
|
||||
|
||||
const handleResponderGrant = useCallback(
|
||||
(event: GestureResponderEvent) => {
|
||||
const pinch = createPinchGesture(event, frameGeometry, zoomRef.current)
|
||||
if (pinch) {
|
||||
clearLongPressTimer()
|
||||
pinchRef.current = pinch
|
||||
panRef.current = null
|
||||
startPointRef.current = null
|
||||
return
|
||||
}
|
||||
const startPoint = readLocalTouchPoint(event.nativeEvent)
|
||||
if (!startPoint) {
|
||||
return
|
||||
}
|
||||
startPointRef.current = { x: startPoint.x, y: startPoint.y, t: Date.now() }
|
||||
rightClickSentRef.current = false
|
||||
scrollingRef.current = false
|
||||
wheelGestureIdRef.current += 1
|
||||
lastWheelRef.current = { dx: 0, dy: 0, at: 0 }
|
||||
panRef.current =
|
||||
zoomRef.current.scale > MIN_ZOOM
|
||||
? {
|
||||
x: startPoint.x,
|
||||
y: startPoint.y,
|
||||
offsetX: zoomRef.current.offsetX,
|
||||
offsetY: zoomRef.current.offsetY
|
||||
}
|
||||
: null
|
||||
clearLongPressTimer()
|
||||
longPressTimerRef.current = setTimeout(() => {
|
||||
const start = startPointRef.current
|
||||
if (!start) {
|
||||
return
|
||||
}
|
||||
const point = mapTouchPoint(start.x, start.y)
|
||||
if (!point) {
|
||||
return
|
||||
}
|
||||
rightClickSentRef.current = true
|
||||
void sendPointerClick(point, 'right')
|
||||
onToast('Right click')
|
||||
}, LONG_PRESS_MS)
|
||||
},
|
||||
[clearLongPressTimer, frameGeometry, mapTouchPoint, onToast, sendPointerClick]
|
||||
)
|
||||
|
||||
const handleResponderMove = useCallback(
|
||||
(event: GestureResponderEvent, gesture: PanResponderGestureState) => {
|
||||
const startedPinch = pinchRef.current
|
||||
? null
|
||||
: createPinchGesture(event, frameGeometry, zoomRef.current)
|
||||
if (startedPinch) {
|
||||
clearLongPressTimer()
|
||||
pinchRef.current = startedPinch
|
||||
panRef.current = null
|
||||
startPointRef.current = null
|
||||
}
|
||||
const activePinch = pinchRef.current
|
||||
const nextPinch = activePinch ? updatePinchZoom(event, frameGeometry, activePinch) : null
|
||||
if (nextPinch) {
|
||||
clearLongPressTimer()
|
||||
zoomRef.current = nextPinch
|
||||
setZoom(nextPinch)
|
||||
return
|
||||
}
|
||||
if (activePinch) {
|
||||
pinchRef.current = null
|
||||
}
|
||||
const moved = Math.hypot(gesture.dx, gesture.dy)
|
||||
if (moved > TAP_SLOP) {
|
||||
clearLongPressTimer()
|
||||
}
|
||||
const activePan = panRef.current
|
||||
if (activePan && frameGeometry) {
|
||||
const currentPoint = readLocalTouchPoint(event.nativeEvent)
|
||||
if (!currentPoint) {
|
||||
return
|
||||
}
|
||||
if (!scrollingRef.current && moved <= TAP_SLOP) {
|
||||
return
|
||||
}
|
||||
scrollingRef.current = true
|
||||
startPointRef.current = null
|
||||
const nextZoom = clampBrowserZoomState(
|
||||
{
|
||||
scale: zoomRef.current.scale,
|
||||
offsetX: activePan.offsetX + currentPoint.x - activePan.x,
|
||||
offsetY: activePan.offsetY + currentPoint.y - activePan.y
|
||||
},
|
||||
frameGeometry,
|
||||
MIN_ZOOM,
|
||||
MAX_ZOOM
|
||||
)
|
||||
zoomRef.current = nextZoom
|
||||
setZoom(nextZoom)
|
||||
return
|
||||
}
|
||||
if (!scrollingRef.current) {
|
||||
if (moved <= SCROLL_START_SLOP) {
|
||||
return
|
||||
}
|
||||
scrollingRef.current = true
|
||||
startPointRef.current = null
|
||||
}
|
||||
const now = Date.now()
|
||||
if (now - lastWheelRef.current.at < WHEEL_INTERVAL_MS) {
|
||||
return
|
||||
}
|
||||
const deltaX = gesture.dx - lastWheelRef.current.dx
|
||||
const deltaY = gesture.dy - lastWheelRef.current.dy
|
||||
if (Math.abs(deltaX) + Math.abs(deltaY) < 8) {
|
||||
return
|
||||
}
|
||||
const currentPoint = readLocalTouchPoint(event.nativeEvent)
|
||||
if (!currentPoint) {
|
||||
return
|
||||
}
|
||||
const point = mapTouchPoint(currentPoint.x, currentPoint.y)
|
||||
if (!point) {
|
||||
return
|
||||
}
|
||||
lastWheelRef.current = { dx: gesture.dx, dy: gesture.dy, at: now }
|
||||
sendWheel(point, deltaX, deltaY, wheelGestureIdRef.current)
|
||||
},
|
||||
[clearLongPressTimer, frameGeometry, mapTouchPoint, sendWheel]
|
||||
)
|
||||
|
||||
const handleResponderRelease = useCallback(
|
||||
(event: GestureResponderEvent, gesture: PanResponderGestureState) => {
|
||||
clearLongPressTimer()
|
||||
pinchRef.current = null
|
||||
panRef.current = null
|
||||
const start = startPointRef.current
|
||||
startPointRef.current = null
|
||||
const wasScrolling = scrollingRef.current
|
||||
scrollingRef.current = false
|
||||
if (!start || rightClickSentRef.current || wasScrolling) {
|
||||
return
|
||||
}
|
||||
const moved = Math.hypot(gesture.dx, gesture.dy)
|
||||
if (moved <= TAP_SLOP && Date.now() - start.t < LONG_PRESS_MS) {
|
||||
// Why: native browser taps resolve at touch-up. Using touch-down makes
|
||||
// tiny finger drift feel like the click lands left/up of the finger.
|
||||
const release = readLocalTouchPoint(event.nativeEvent) ?? start
|
||||
const point = mapTouchPoint(release.x, release.y)
|
||||
if (point) {
|
||||
void sendPointerClick(point, 'left')
|
||||
}
|
||||
}
|
||||
},
|
||||
[clearLongPressTimer, mapTouchPoint, sendPointerClick]
|
||||
)
|
||||
|
||||
const panResponder = useMemo(
|
||||
() =>
|
||||
PanResponder.create({
|
||||
onStartShouldSetPanResponder: () => dialogRef.current === null,
|
||||
onMoveShouldSetPanResponder: () => dialogRef.current === null,
|
||||
onPanResponderGrant: handleResponderGrant,
|
||||
onPanResponderMove: handleResponderMove,
|
||||
onPanResponderRelease: handleResponderRelease,
|
||||
onPanResponderTerminate: () => {
|
||||
clearLongPressTimer()
|
||||
pinchRef.current = null
|
||||
panRef.current = null
|
||||
scrollingRef.current = false
|
||||
startPointRef.current = null
|
||||
},
|
||||
onPanResponderTerminationRequest: () => true
|
||||
}),
|
||||
[clearLongPressTimer, handleResponderGrant, handleResponderMove, handleResponderRelease]
|
||||
)
|
||||
|
||||
return { panResponder, sendDialogCommand, sendKeyboardText, sendKeypress, togglePointerModifier }
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
import { useCallback } from 'react'
|
||||
import type { Image, View } from 'react-native'
|
||||
import {
|
||||
updateBrowserImageSource,
|
||||
updateBrowserLayerVisibility,
|
||||
type FrameLayer
|
||||
} from './mobile-browser-frame-state'
|
||||
import { mobileBrowserPaneStyles as styles } from './mobile-browser-pane-styles'
|
||||
|
||||
type BrowserLayerHandlersArgs = {
|
||||
browserImageRefs: { current: [Image | null, Image | null] }
|
||||
browserLayerRefs: { current: [View | null, View | null] }
|
||||
frameUriRef: { current: string | null }
|
||||
pendingFrameLayerRef: { current: FrameLayer | null }
|
||||
visibleFrameLayerRef: { current: FrameLayer }
|
||||
}
|
||||
|
||||
export function useMobileBrowserPaneLayers(args: BrowserLayerHandlersArgs) {
|
||||
const {
|
||||
browserImageRefs,
|
||||
browserLayerRefs,
|
||||
frameUriRef,
|
||||
pendingFrameLayerRef,
|
||||
visibleFrameLayerRef
|
||||
} = args
|
||||
|
||||
const setBrowserImageRef = useCallback(
|
||||
(layer: FrameLayer, image: Image | null) => {
|
||||
browserImageRefs.current[layer] = image
|
||||
const currentFrameUri = frameUriRef.current
|
||||
if (image && currentFrameUri) {
|
||||
updateBrowserImageSource(image, currentFrameUri)
|
||||
}
|
||||
},
|
||||
[browserImageRefs, frameUriRef]
|
||||
)
|
||||
|
||||
const setBrowserLayerRef = useCallback(
|
||||
(layer: FrameLayer, view: View | null) => {
|
||||
browserLayerRefs.current[layer] = view
|
||||
updateBrowserLayerVisibility(browserLayerRefs.current, visibleFrameLayerRef.current)
|
||||
},
|
||||
[browserLayerRefs, visibleFrameLayerRef]
|
||||
)
|
||||
|
||||
const setBrowserLayer0Ref = useCallback(
|
||||
(view: View | null) => setBrowserLayerRef(0, view),
|
||||
[setBrowserLayerRef]
|
||||
)
|
||||
const setBrowserLayer1Ref = useCallback(
|
||||
(view: View | null) => setBrowserLayerRef(1, view),
|
||||
[setBrowserLayerRef]
|
||||
)
|
||||
const setBrowserImageLayer0Ref = useCallback(
|
||||
(image: Image | null) => setBrowserImageRef(0, image),
|
||||
[setBrowserImageRef]
|
||||
)
|
||||
const setBrowserImageLayer1Ref = useCallback(
|
||||
(image: Image | null) => setBrowserImageRef(1, image),
|
||||
[setBrowserImageRef]
|
||||
)
|
||||
|
||||
const handleBrowserImageLoad = useCallback(
|
||||
(layer: FrameLayer) => {
|
||||
if (pendingFrameLayerRef.current !== layer) {
|
||||
return
|
||||
}
|
||||
pendingFrameLayerRef.current = null
|
||||
visibleFrameLayerRef.current = layer
|
||||
updateBrowserLayerVisibility(browserLayerRefs.current, layer)
|
||||
},
|
||||
[browserLayerRefs, pendingFrameLayerRef, visibleFrameLayerRef]
|
||||
)
|
||||
|
||||
const handleBrowserImageLayer0Load = useCallback(
|
||||
() => handleBrowserImageLoad(0),
|
||||
[handleBrowserImageLoad]
|
||||
)
|
||||
const handleBrowserImageLayer1Load = useCallback(
|
||||
() => handleBrowserImageLoad(1),
|
||||
[handleBrowserImageLoad]
|
||||
)
|
||||
|
||||
const handleBrowserImageError = useCallback(
|
||||
(layer: FrameLayer) => {
|
||||
if (pendingFrameLayerRef.current === layer) {
|
||||
pendingFrameLayerRef.current = null
|
||||
}
|
||||
},
|
||||
[pendingFrameLayerRef]
|
||||
)
|
||||
|
||||
const handleBrowserImageLayer0Error = useCallback(
|
||||
() => handleBrowserImageError(0),
|
||||
[handleBrowserImageError]
|
||||
)
|
||||
const handleBrowserImageLayer1Error = useCallback(
|
||||
() => handleBrowserImageError(1),
|
||||
[handleBrowserImageError]
|
||||
)
|
||||
|
||||
const frameLayerStyle = useCallback(
|
||||
(layer: FrameLayer) => {
|
||||
return [
|
||||
styles.browserImageLayer,
|
||||
visibleFrameLayerRef.current !== layer && styles.browserImageLayerHidden
|
||||
]
|
||||
},
|
||||
[visibleFrameLayerRef]
|
||||
)
|
||||
|
||||
const browserLayerRef = useCallback(
|
||||
(layer: FrameLayer) => (layer === 0 ? setBrowserLayer0Ref : setBrowserLayer1Ref),
|
||||
[setBrowserLayer0Ref, setBrowserLayer1Ref]
|
||||
)
|
||||
|
||||
const frameLayerRef = useCallback(
|
||||
(layer: FrameLayer) => (layer === 0 ? setBrowserImageLayer0Ref : setBrowserImageLayer1Ref),
|
||||
[setBrowserImageLayer0Ref, setBrowserImageLayer1Ref]
|
||||
)
|
||||
|
||||
const frameLayerLoadHandler = useCallback(
|
||||
(layer: FrameLayer) =>
|
||||
layer === 0 ? handleBrowserImageLayer0Load : handleBrowserImageLayer1Load,
|
||||
[handleBrowserImageLayer0Load, handleBrowserImageLayer1Load]
|
||||
)
|
||||
|
||||
const frameLayerErrorHandler = useCallback(
|
||||
(layer: FrameLayer) =>
|
||||
layer === 0 ? handleBrowserImageLayer0Error : handleBrowserImageLayer1Error,
|
||||
[handleBrowserImageLayer0Error, handleBrowserImageLayer1Error]
|
||||
)
|
||||
|
||||
return {
|
||||
browserLayerRef,
|
||||
frameLayerErrorHandler,
|
||||
frameLayerLoadHandler,
|
||||
frameLayerRef,
|
||||
frameLayerStyle
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import { useCallback, type Dispatch, type SetStateAction } from 'react'
|
||||
import type { RpcClient } from '../transport/rpc-client'
|
||||
import type { RpcFailure, RpcSuccess } from '../transport/types'
|
||||
import { browserErrorMessage, shouldSurfaceBrowserError } from './mobile-browser-frame-state'
|
||||
|
||||
type BrowserRequestArgs = {
|
||||
busyRef: { current: boolean }
|
||||
client: RpcClient | null
|
||||
pageId: string | null
|
||||
setBusy: Dispatch<SetStateAction<boolean>>
|
||||
setError: Dispatch<SetStateAction<string | null>>
|
||||
worktreeId: string
|
||||
}
|
||||
export function useMobileBrowserRequest(args: BrowserRequestArgs) {
|
||||
const { busyRef, client, pageId, setBusy, setError, worktreeId } = args
|
||||
const pageParams = useCallback(() => {
|
||||
if (!pageId) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
worktree: `id:${worktreeId}`,
|
||||
page: pageId
|
||||
}
|
||||
}, [pageId, worktreeId])
|
||||
|
||||
const sendBrowserRequest = useCallback(
|
||||
async (
|
||||
method: string,
|
||||
params: Record<string, unknown> = {},
|
||||
opts: { showBusy?: boolean; suppressError?: boolean; timeoutMs?: number } = {}
|
||||
): Promise<unknown | null> => {
|
||||
const base = pageParams()
|
||||
if (!client || !base) {
|
||||
return null
|
||||
}
|
||||
if (opts.showBusy) {
|
||||
busyRef.current = true
|
||||
setBusy(true)
|
||||
}
|
||||
try {
|
||||
const response = await client.sendRequest(
|
||||
method,
|
||||
{ ...base, ...params },
|
||||
{ timeoutMs: opts.timeoutMs ?? 15_000 }
|
||||
)
|
||||
if (!response.ok) {
|
||||
throw new Error((response as RpcFailure).error.message)
|
||||
}
|
||||
setError(null)
|
||||
return (response as RpcSuccess).result
|
||||
} catch (err) {
|
||||
const message = browserErrorMessage(err, 'Browser command failed')
|
||||
if (!opts.suppressError && shouldSurfaceBrowserError(message)) {
|
||||
setError(message)
|
||||
}
|
||||
return null
|
||||
} finally {
|
||||
if (opts.showBusy) {
|
||||
busyRef.current = false
|
||||
setBusy(false)
|
||||
}
|
||||
}
|
||||
},
|
||||
[client, pageParams]
|
||||
)
|
||||
return { pageParams, sendBrowserRequest }
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
import { useEffect, useMemo, type Dispatch, type SetStateAction } from 'react'
|
||||
import { PixelRatio, type Image, type View } from 'react-native'
|
||||
import type { RpcClient } from '../transport/rpc-client'
|
||||
import type {
|
||||
BrowserScreencastFrame,
|
||||
BrowserScreencastFrameMetadata
|
||||
} from '../transport/browser-screencast-protocol'
|
||||
import {
|
||||
buildMobileBrowserScreencastRequest,
|
||||
type MobileBrowserViewMode
|
||||
} from './browser-screencast-request'
|
||||
import {
|
||||
MAX_ZOOM,
|
||||
MIN_ZOOM,
|
||||
getCachedBrowserFrame,
|
||||
updateBrowserLayerVisibility,
|
||||
type FrameLayer
|
||||
} from './mobile-browser-frame-state'
|
||||
import {
|
||||
clampBrowserZoomState,
|
||||
computeBrowserFrameGeometry,
|
||||
type BrowserTouchLayout,
|
||||
type BrowserZoomState
|
||||
} from './browser-touch-geometry'
|
||||
import type { MobileBrowserTab } from './MobileBrowserPane'
|
||||
import {
|
||||
handleBrowserScreencastEvent,
|
||||
type BrowserDialogState,
|
||||
type ScreencastEvent
|
||||
} from './mobile-browser-stream-events'
|
||||
import { useMobileBrowserFrameApply } from './use-mobile-browser-frame-apply'
|
||||
import { useMobileBrowserRequest } from './use-mobile-browser-request'
|
||||
|
||||
type PendingFrame = { frame: BrowserScreencastFrame; cacheKey: string }
|
||||
|
||||
type MobileBrowserStreamArgs = {
|
||||
appActive: boolean
|
||||
browserImageRefs: { current: [Image | null, Image | null] }
|
||||
browserLayerRefs: { current: [View | null, View | null] }
|
||||
browserViewMode: MobileBrowserViewMode
|
||||
busyRef: { current: boolean }
|
||||
cacheKey: string | null
|
||||
client: RpcClient | null
|
||||
frameMetadata: BrowserScreencastFrameMetadata | null
|
||||
frameMetadataRef: { current: BrowserScreencastFrameMetadata | null }
|
||||
frameMountedRef: { current: boolean }
|
||||
frameThrottleTimerRef: { current: ReturnType<typeof setTimeout> | null }
|
||||
frameUriRef: { current: string | null }
|
||||
lastAppliedFrameAtRef: { current: number }
|
||||
lastStreamCacheKeyRef: { current: string | null }
|
||||
lastZoomResetUrlRef: { current: string }
|
||||
layout: BrowserTouchLayout | null
|
||||
pendingFrameLayerRef: { current: FrameLayer | null }
|
||||
pendingThrottledFrameRef: { current: PendingFrame | null }
|
||||
resetBrowserZoomState: () => void
|
||||
screencastSupported: boolean | null
|
||||
setAddressValue: Dispatch<SetStateAction<string>>
|
||||
setBusy: Dispatch<SetStateAction<boolean>>
|
||||
setDialog: Dispatch<SetStateAction<BrowserDialogState | null>>
|
||||
setError: Dispatch<SetStateAction<string | null>>
|
||||
setFrameMetadata: Dispatch<SetStateAction<BrowserScreencastFrameMetadata | null>>
|
||||
setFrameUri: Dispatch<SetStateAction<string | null>>
|
||||
setZoom: Dispatch<SetStateAction<BrowserZoomState>>
|
||||
streamGenerationRef: { current: number }
|
||||
tab: MobileBrowserTab
|
||||
visibleFrameLayerRef: { current: FrameLayer }
|
||||
worktreeId: string
|
||||
zoomRef: { current: BrowserZoomState }
|
||||
}
|
||||
|
||||
export function useMobileBrowserStream(args: MobileBrowserStreamArgs) {
|
||||
const {
|
||||
appActive,
|
||||
browserImageRefs,
|
||||
browserLayerRefs,
|
||||
browserViewMode,
|
||||
busyRef,
|
||||
cacheKey,
|
||||
client,
|
||||
frameMetadata,
|
||||
frameMetadataRef,
|
||||
frameMountedRef,
|
||||
frameThrottleTimerRef,
|
||||
frameUriRef,
|
||||
lastAppliedFrameAtRef,
|
||||
lastStreamCacheKeyRef,
|
||||
lastZoomResetUrlRef,
|
||||
layout,
|
||||
pendingFrameLayerRef,
|
||||
pendingThrottledFrameRef,
|
||||
resetBrowserZoomState,
|
||||
screencastSupported,
|
||||
setAddressValue,
|
||||
setBusy,
|
||||
setDialog,
|
||||
setError,
|
||||
setFrameMetadata,
|
||||
setFrameUri,
|
||||
setZoom,
|
||||
streamGenerationRef,
|
||||
tab,
|
||||
visibleFrameLayerRef,
|
||||
worktreeId,
|
||||
zoomRef
|
||||
} = args
|
||||
|
||||
const { pageParams, sendBrowserRequest } = useMobileBrowserRequest({
|
||||
busyRef,
|
||||
client,
|
||||
pageId: tab.browserPageId,
|
||||
setBusy,
|
||||
setError,
|
||||
worktreeId
|
||||
})
|
||||
|
||||
const { applyFrameThrottled, clearFrameThrottle } = useMobileBrowserFrameApply({
|
||||
browserImageRefs,
|
||||
busyRef,
|
||||
frameMetadataRef,
|
||||
frameMountedRef,
|
||||
frameThrottleTimerRef,
|
||||
frameUriRef,
|
||||
lastAppliedFrameAtRef,
|
||||
pendingFrameLayerRef,
|
||||
pendingThrottledFrameRef,
|
||||
setBusy,
|
||||
setFrameMetadata,
|
||||
setFrameUri,
|
||||
visibleFrameLayerRef
|
||||
})
|
||||
|
||||
const streamRequest = useMemo(
|
||||
() => buildMobileBrowserScreencastRequest(layout, PixelRatio.get(), browserViewMode),
|
||||
[browserViewMode, layout]
|
||||
)
|
||||
|
||||
const frameGeometry = useMemo(
|
||||
() => computeBrowserFrameGeometry(layout, frameMetadata),
|
||||
[frameMetadata, layout]
|
||||
)
|
||||
|
||||
useEffect(() => {
|
||||
if (!frameGeometry) {
|
||||
return
|
||||
}
|
||||
setZoom((current) => {
|
||||
const next = clampBrowserZoomState(current, frameGeometry, MIN_ZOOM, MAX_ZOOM)
|
||||
if (
|
||||
next.scale === current.scale &&
|
||||
next.offsetX === current.offsetX &&
|
||||
next.offsetY === current.offsetY
|
||||
) {
|
||||
return current
|
||||
}
|
||||
// Why: rotation/layout changes can shrink the legal pan range while the
|
||||
// current zoom state still points at the previous viewport geometry.
|
||||
zoomRef.current = next
|
||||
return next
|
||||
})
|
||||
}, [frameGeometry])
|
||||
|
||||
useEffect(() => {
|
||||
streamGenerationRef.current += 1
|
||||
const generation = streamGenerationRef.current
|
||||
const sameStream = Boolean(cacheKey) && lastStreamCacheKeyRef.current === cacheKey
|
||||
lastStreamCacheKeyRef.current = cacheKey
|
||||
if (!sameStream || !frameUriRef.current) {
|
||||
const cachedFrame = getCachedBrowserFrame(cacheKey)
|
||||
if (cachedFrame) {
|
||||
frameUriRef.current = cachedFrame.uri
|
||||
frameMountedRef.current = true
|
||||
frameMetadataRef.current = cachedFrame.metadata
|
||||
setFrameUri(cachedFrame.uri)
|
||||
setFrameMetadata(cachedFrame.metadata)
|
||||
} else {
|
||||
frameUriRef.current = null
|
||||
frameMountedRef.current = false
|
||||
setFrameUri(null)
|
||||
setFrameMetadata(null)
|
||||
frameMetadataRef.current = null
|
||||
}
|
||||
} else {
|
||||
frameMountedRef.current = true
|
||||
}
|
||||
pendingFrameLayerRef.current = null
|
||||
if (!sameStream || !frameUriRef.current) {
|
||||
visibleFrameLayerRef.current = 0
|
||||
}
|
||||
updateBrowserLayerVisibility(browserLayerRefs.current, visibleFrameLayerRef.current)
|
||||
lastAppliedFrameAtRef.current = 0
|
||||
clearFrameThrottle()
|
||||
busyRef.current = false
|
||||
setDialog(null)
|
||||
setError(null)
|
||||
if (
|
||||
!client ||
|
||||
screencastSupported !== true ||
|
||||
!tab.browserPageId ||
|
||||
!appActive ||
|
||||
!streamRequest
|
||||
) {
|
||||
busyRef.current = false
|
||||
setBusy(false)
|
||||
if (screencastSupported === false) {
|
||||
setError('Update desktop Orca to stream browser tabs on mobile.')
|
||||
} else if (screencastSupported === null) {
|
||||
setError('Checking desktop browser streaming support.')
|
||||
} else if (!tab.browserPageId) {
|
||||
setError('Browser page is not available yet.')
|
||||
}
|
||||
return
|
||||
}
|
||||
busyRef.current = true
|
||||
setBusy(true)
|
||||
let startupTimer: ReturnType<typeof setTimeout> | null = setTimeout(() => {
|
||||
if (streamGenerationRef.current !== generation) {
|
||||
return
|
||||
}
|
||||
busyRef.current = false
|
||||
setBusy(false)
|
||||
setError('Browser stream timed out.')
|
||||
}, 15_000)
|
||||
const clearStartupTimer = (): void => {
|
||||
if (startupTimer) {
|
||||
clearTimeout(startupTimer)
|
||||
startupTimer = null
|
||||
}
|
||||
}
|
||||
const unsubscribe = client.subscribe(
|
||||
'browser.screencast',
|
||||
{
|
||||
worktree: `id:${worktreeId}`,
|
||||
page: tab.browserPageId,
|
||||
...streamRequest
|
||||
},
|
||||
(payload) => {
|
||||
if (streamGenerationRef.current !== generation) {
|
||||
return
|
||||
}
|
||||
handleBrowserScreencastEvent({
|
||||
busyRef,
|
||||
clearStartupTimer,
|
||||
event: payload as ScreencastEvent,
|
||||
lastZoomResetUrlRef,
|
||||
resetBrowserZoomState,
|
||||
setAddressValue,
|
||||
setBusy,
|
||||
setDialog,
|
||||
setError
|
||||
})
|
||||
},
|
||||
{
|
||||
onBinaryFrame: (frame) => {
|
||||
if (streamGenerationRef.current !== generation) {
|
||||
return
|
||||
}
|
||||
clearStartupTimer()
|
||||
if (cacheKey) {
|
||||
applyFrameThrottled(frame, cacheKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
return () => {
|
||||
clearStartupTimer()
|
||||
clearFrameThrottle()
|
||||
unsubscribe()
|
||||
}
|
||||
}, [
|
||||
appActive,
|
||||
applyFrameThrottled,
|
||||
clearFrameThrottle,
|
||||
client,
|
||||
resetBrowserZoomState,
|
||||
screencastSupported,
|
||||
streamRequest,
|
||||
cacheKey,
|
||||
tab.browserPageId,
|
||||
worktreeId
|
||||
])
|
||||
|
||||
return { frameGeometry, pageParams, sendBrowserRequest }
|
||||
}
|
||||
Reference in New Issue
Block a user