mirror of
https://github.com/stablyai/orca.git
synced 2026-10-02 16:02:15 +00:00
fix(mobile): hold-to-dictate, repeat keys and the browser long-press survive the page's long-press (#24277)
On the OTA page a held press died ~500 ms in: the WebView's long-press selected nearby text and that selection's selectionchange/touchcancel ended the press. Page text is now unselectable unless it opts in (as native), hold surfaces declare onLongPress, the browser pane refuses contextmenu termination, and the chat mic's swapped icons no longer steal the touch target.
This commit is contained in:
Jinwoo Hong
@@ -139,9 +139,17 @@ export const MOBILE_WEB_APP_ROOT_RESET =
|
||||
* Inputs and textareas: Chromium rings a focused one (`:focus-visible` matches every focused
|
||||
* text field); no native TextInput paints one, and the caret and the IME already mark focus.
|
||||
* Those only: a button reached by a hardware keyboard keeps the browser's ring.
|
||||
*
|
||||
* Text: native `Text` is unselectable unless `selectable`; react-native-web's is selectable. On
|
||||
* iOS a hold on it raises WebKit's selection over the page's own long press; on Android the
|
||||
* selection's `selectionchange` and `touchcancel` end a held press. `body`, not `#root`: modals
|
||||
* portal to `document.body`. Fields opt back in, callout too, since WebKit inherits it.
|
||||
*/
|
||||
export const MOBILE_WEB_APP_NATIVE_PARITY_STYLE =
|
||||
'<style id="orca-native-parity">:where(input:focus,textarea:focus){outline:none}</style>'
|
||||
'<style id="orca-native-parity">:where(input:focus,textarea:focus){outline:none}' +
|
||||
':where(body){-webkit-user-select:none;user-select:none;-webkit-touch-callout:none}' +
|
||||
':where(input,textarea,[contenteditable]){-webkit-user-select:text;user-select:text;' +
|
||||
'-webkit-touch-callout:default}</style>'
|
||||
|
||||
const PAGE_ASYNC_STORAGE_MODULE = join(
|
||||
mobileDir,
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
/**
|
||||
* Scratch routes for the held-press check: the surfaces whose press must outlive Android WebView's
|
||||
* long-press. Each mounts the shipped component with stubbed effects; what is under test is the
|
||||
* press responder, so handlers only count.
|
||||
*/
|
||||
|
||||
/** RN Web renders `nativeID` as the DOM `id`; the check touches each mic through it. */
|
||||
export const TERMINAL_MIC_ID = 'hold-probe-terminal-mic'
|
||||
export const CHAT_MIC_ID = 'hold-probe-chat-mic'
|
||||
|
||||
export function holdDictationProbeRouteSource({ terminalActionsModule, chatComposerModule }) {
|
||||
return `import { useRef, useState } from 'react'
|
||||
import { Text, View } from 'react-native'
|
||||
import { MobileTerminalInputActions } from ${JSON.stringify(terminalActionsModule)}
|
||||
import { MobileNativeChatComposer } from ${JSON.stringify(chatComposerModule)}
|
||||
|
||||
const noop = () => {}
|
||||
|
||||
export default function HoldDictationProbeRoute() {
|
||||
const [terminalHeld, setTerminalHeld] = useState(false)
|
||||
const [chatHeld, setChatHeld] = useState(false)
|
||||
const pressOutsRef = useRef({ terminal: 0, chat: 0 })
|
||||
globalThis.__orcaHeldPressProbe = { pressOuts: () => ({ ...pressOutsRef.current }) }
|
||||
return (
|
||||
<View style={{ padding: 24, gap: 24 }}>
|
||||
<Text>Model</Text>
|
||||
<View nativeID=${JSON.stringify(TERMINAL_MIC_ID)} style={{ flexDirection: 'row' }}>
|
||||
<MobileTerminalInputActions
|
||||
canSend
|
||||
isAttaching={false}
|
||||
dictation={{ isStarting: false, isRecording: terminalHeld, isProcessing: false }}
|
||||
dictationMode="hold"
|
||||
buttonStyle={{ width: 48, height: 48 }}
|
||||
activeButtonStyle={null}
|
||||
disabledButtonStyle={null}
|
||||
onAttachImage={noop}
|
||||
onAttachFile={noop}
|
||||
onDictationToggle={noop}
|
||||
onDictationPressIn={() => setTerminalHeld(true)}
|
||||
onDictationPressOut={() => {
|
||||
pressOutsRef.current.terminal += 1
|
||||
setTerminalHeld(false)
|
||||
}}
|
||||
onDictationCancel={noop}
|
||||
/>
|
||||
</View>
|
||||
<View nativeID=${JSON.stringify(CHAT_MIC_ID)}>
|
||||
<MobileNativeChatComposer
|
||||
value=""
|
||||
onChangeText={noop}
|
||||
onSend={() => Promise.resolve(false)}
|
||||
sendSurfaceId="hold-probe"
|
||||
getSendCompletionGeneration={() => 0}
|
||||
getComposerEditGeneration={() => 0}
|
||||
onMicPress={noop}
|
||||
micActive={chatHeld}
|
||||
dictationMode="hold"
|
||||
onMicPressIn={() => setChatHeld(true)}
|
||||
onMicPressOut={() => {
|
||||
pressOutsRef.current.chat += 1
|
||||
setChatHeld(false)
|
||||
}}
|
||||
/>
|
||||
</View>
|
||||
</View>
|
||||
)
|
||||
}
|
||||
`
|
||||
}
|
||||
|
||||
export const REPEAT_KEY_LABEL = 'Arrow Up'
|
||||
export const TAP_KEY_LABEL = 'Escape'
|
||||
|
||||
/**
|
||||
* The session's command dock with a stub controller, in buffered mode. The repeat cadence is
|
||||
* `use-mobile-session-accessory-selection.ts`'s (send on press-in, repeat after 400 ms, then every
|
||||
* 45 ms, stop on press-out), restated here because that hook is bound to the whole session.
|
||||
*/
|
||||
export function commandDockProbeRouteSource({ commandDockModule, keyDefinitionsModule }) {
|
||||
return `import { useRef } from 'react'
|
||||
import { MobileSessionCommandDock } from ${JSON.stringify(commandDockModule)}
|
||||
import { TERMINAL_ACCESSORY_KEY_DEFINITIONS } from ${JSON.stringify(keyDefinitionsModule)}
|
||||
|
||||
const noop = () => {}
|
||||
const KEYS = TERMINAL_ACCESSORY_KEY_DEFINITIONS.filter((key) => ['arrowUp', 'escape'].includes(key.id))
|
||||
const IDLE = { isStarting: false, isRecording: false, isProcessing: false }
|
||||
|
||||
export default function CommandDockProbeRoute() {
|
||||
const sentRef = useRef([])
|
||||
const timersRef = useRef({ timeout: null, interval: null })
|
||||
const stop = () => {
|
||||
clearTimeout(timersRef.current.timeout)
|
||||
clearInterval(timersRef.current.interval)
|
||||
}
|
||||
const send = (input) => {
|
||||
sentRef.current.push(input.bytes)
|
||||
return Promise.resolve()
|
||||
}
|
||||
const start = (input) => {
|
||||
stop()
|
||||
timersRef.current.timeout = setTimeout(() => {
|
||||
timersRef.current.interval = setInterval(() => void send(input), 45)
|
||||
}, 400)
|
||||
}
|
||||
globalThis.__orcaHeldPressProbe = { sent: () => [...sentRef.current] }
|
||||
const controller = {
|
||||
insets: { top: 0, bottom: 0, left: 0, right: 0 },
|
||||
bufferedTerminalDraftState: { input: '', setInput: noop },
|
||||
autocompleteEnabled: false,
|
||||
liveInputCapture: '',
|
||||
activeHandle: 'held-press-terminal',
|
||||
customKeys: [],
|
||||
setShowCustomKeyModal: noop,
|
||||
setDeleteKeyTarget: noop,
|
||||
visibleBuiltInAccessoryKeys: KEYS,
|
||||
terminalModes: new Map(),
|
||||
canPaste: false,
|
||||
dictationMode: 'toggle',
|
||||
bindCommandField: noop,
|
||||
handleLiveInputChange: noop,
|
||||
handleLiveInputKeyPress: noop,
|
||||
bindLiveInputField: noop,
|
||||
submitLiveInput: noop,
|
||||
canSend: true,
|
||||
canCompose: true,
|
||||
liveInputEnabled: false,
|
||||
focusLiveInput: noop,
|
||||
showNativeChat: false,
|
||||
dictation: IDLE,
|
||||
cancelDictation: noop,
|
||||
handleDictationToggle: noop,
|
||||
handleDictationPressIn: noop,
|
||||
handleDictationPressOut: noop,
|
||||
toggleDisplayMode: noop,
|
||||
handleSend: () => Promise.resolve(),
|
||||
handleAccessoryKey: send,
|
||||
dismissSoftwareKeyboard: noop,
|
||||
toggleLiveInput: noop,
|
||||
stopAccessoryRepeat: stop,
|
||||
startAccessoryRepeat: start,
|
||||
handlePaste: () => Promise.resolve(),
|
||||
isAttaching: false,
|
||||
attachImage: () => Promise.resolve(),
|
||||
activeMarkdownTab: null,
|
||||
activeFileTab: null,
|
||||
activeBrowserTab: null,
|
||||
keyboardLift: 0
|
||||
}
|
||||
return <MobileSessionCommandDock controller={controller} />
|
||||
}
|
||||
`
|
||||
}
|
||||
|
||||
export const BROWSER_VIEWPORT_ID = 'held-press-browser-viewport'
|
||||
|
||||
/**
|
||||
* The browser pane's view with its real interactions hook, which owns the 550 ms long-press
|
||||
* right-click. The stub client records each request, so a right-click is read where the host would
|
||||
* see it, not only from the hook's "Right click" toast.
|
||||
*/
|
||||
export function browserPaneProbeRouteSource({
|
||||
paneViewModule,
|
||||
interactionsModule,
|
||||
geometryModule
|
||||
}) {
|
||||
return `import { useRef, useState } from 'react'
|
||||
import { View } from 'react-native'
|
||||
import { MobileBrowserPaneView } from ${JSON.stringify(paneViewModule)}
|
||||
import { useMobileBrowserInteractions } from ${JSON.stringify(interactionsModule)}
|
||||
import { computeBrowserFrameGeometry } from ${JSON.stringify(geometryModule)}
|
||||
|
||||
const noop = () => {}
|
||||
const LAYOUT = { width: 390, height: 500 }
|
||||
const LAYER = { attachView: noop, attachImage: noop, onLoad: noop, onError: noop }
|
||||
const TAB = {
|
||||
type: 'browser',
|
||||
id: 'held-press-tab',
|
||||
title: 'Example',
|
||||
browserWorkspaceId: 'held-press-workspace',
|
||||
browserPageId: 'held-press-page',
|
||||
url: 'https://example.com/',
|
||||
loading: false,
|
||||
canGoBack: false,
|
||||
canGoForward: false
|
||||
}
|
||||
|
||||
export default function BrowserPaneProbeRoute() {
|
||||
const toastsRef = useRef([])
|
||||
const requestsRef = useRef([])
|
||||
const clientRef = useRef({
|
||||
sendRequest: (method, params) => {
|
||||
requestsRef.current.push({ method, params })
|
||||
return Promise.resolve({ ok: true, id: String(requestsRef.current.length), result: {} })
|
||||
}
|
||||
})
|
||||
const layoutRef = useRef(LAYOUT)
|
||||
const longPressTimerRef = useRef(null)
|
||||
const zoom = { scale: 1, offsetX: 0, offsetY: 0 }
|
||||
const [dialog, setDialog] = useState(null)
|
||||
const dialogRef = useRef(null)
|
||||
dialogRef.current = dialog
|
||||
const frameGeometry = computeBrowserFrameGeometry(LAYOUT, null)
|
||||
const interactions = useMobileBrowserInteractions({
|
||||
clearLongPressTimer: () => clearTimeout(longPressTimerRef.current),
|
||||
client: clientRef.current,
|
||||
dialogRef,
|
||||
frameGeometry,
|
||||
frameMetadataRef: { current: null },
|
||||
keyboardValue: '',
|
||||
layoutRef,
|
||||
longPressTimerRef,
|
||||
onToast: (message) => toastsRef.current.push(message),
|
||||
pageParams: () => ({ worktree: 'held-press-worktree', page: TAB.browserPageId }),
|
||||
panRef: useRef(null),
|
||||
pinchRef: useRef(null),
|
||||
pointerModifiers: [],
|
||||
sendBrowserRequest: () => Promise.resolve(null),
|
||||
scrollingRef: useRef(false),
|
||||
startPointRef: useRef(null),
|
||||
setDialog,
|
||||
setError: noop,
|
||||
setKeyboardValue: noop,
|
||||
setPointerModifiers: noop,
|
||||
setZoom: noop,
|
||||
zoomRef: { current: zoom }
|
||||
})
|
||||
globalThis.__orcaHeldPressProbe = {
|
||||
toasts: () => [...toastsRef.current],
|
||||
requests: () => [...requestsRef.current]
|
||||
}
|
||||
return (
|
||||
<View nativeID=${JSON.stringify(BROWSER_VIEWPORT_ID)} style={{ flex: 1 }}>
|
||||
<MobileBrowserPaneView
|
||||
addressFocused={false}
|
||||
addressValue={TAB.url}
|
||||
bottomInset={0}
|
||||
browserViewMode="mobile"
|
||||
busy={false}
|
||||
controlsDisabled={false}
|
||||
dialog={dialog}
|
||||
error={null}
|
||||
frameGeometry={frameGeometry}
|
||||
frameLayers={[LAYER, LAYER]}
|
||||
goBack={noop}
|
||||
goForward={noop}
|
||||
keyboardLift={0}
|
||||
keyboardValue=""
|
||||
layoutRef={layoutRef}
|
||||
navigateToAddress={() => Promise.resolve()}
|
||||
panResponder={interactions.panResponder}
|
||||
pointerModifiers={[]}
|
||||
reloadPage={noop}
|
||||
renderedFrameSource={null}
|
||||
selectBrowserViewMode={noop}
|
||||
sendDialogCommand={interactions.sendDialogCommand}
|
||||
sendKeyboardText={() => Promise.resolve()}
|
||||
sendKeypress={() => Promise.resolve()}
|
||||
setAddressFocused={noop}
|
||||
setAddressValue={noop}
|
||||
setKeyboardValue={noop}
|
||||
setLayout={noop}
|
||||
setRootViewRef={noop}
|
||||
tab={TAB}
|
||||
togglePointerModifier={noop}
|
||||
zoom={zoom}
|
||||
/>
|
||||
</View>
|
||||
)
|
||||
}
|
||||
`
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { chromium } from 'playwright-core'
|
||||
import { buildMobileWebAppBundle } from './build-mobile-web-app-bundle.mjs'
|
||||
import { mobileWebAppDependenciesPresent } from './mobile-web-app-bundle-dependencies.mjs'
|
||||
import {
|
||||
BROWSER_VIEWPORT_ID,
|
||||
CHAT_MIC_ID,
|
||||
REPEAT_KEY_LABEL,
|
||||
TAP_KEY_LABEL,
|
||||
TERMINAL_MIC_ID,
|
||||
browserPaneProbeRouteSource,
|
||||
commandDockProbeRouteSource,
|
||||
holdDictationProbeRouteSource
|
||||
} from './mobile-web-app-held-press-probe-routes.mjs'
|
||||
import { MOBILE_WEB_APP_ROUTE_ROOT } from './mobile-web-app-route-manifest.mjs'
|
||||
import {
|
||||
createBundleServer,
|
||||
installShellDouble,
|
||||
projectDir,
|
||||
readBridgeFaultGrant,
|
||||
readBridgeProtocolVersion,
|
||||
readShellCsp
|
||||
} from './mobile-web-app-render-harness.mjs'
|
||||
import { LAYOUT_SOURCE } from './mobile-web-app-terminal-probe-route.mjs'
|
||||
|
||||
/**
|
||||
* Held presses on the page. Held ~500 ms, Android WebView turns a touch into a long-press: it fires
|
||||
* `contextmenu`, and where text can be selected it starts a selection and then cancels the touch.
|
||||
* react-native-web ends a press on all three unless the press refuses `contextmenu` (a Pressable
|
||||
* with `onLongPress`), and the page's native-parity style leaves no text to select. Traced on an
|
||||
* emulator; headless Chromium generates no long-press from CDP touches, so `holdLikeAndroidWebView`
|
||||
* plays it.
|
||||
*/
|
||||
|
||||
const ROUTES = {
|
||||
mic: `/${MOBILE_WEB_APP_ROUTE_ROOT}/hold-dictation-probe`,
|
||||
keys: `/${MOBILE_WEB_APP_ROUTE_ROOT}/command-dock-probe`,
|
||||
browser: `/${MOBILE_WEB_APP_ROUTE_ROOT}/browser-pane-probe`
|
||||
}
|
||||
/** When Android WebView's long-press lands; a selection's cancel follows ~20-50 ms later. */
|
||||
const LONG_PRESS_MS = 500
|
||||
const bundles = mobileWebAppDependenciesPresent()
|
||||
const describeRender = bundles ? describe : describe.skip
|
||||
|
||||
let browser = null
|
||||
let origin = null
|
||||
let scratch = null
|
||||
let server = null
|
||||
let bridgeVersion = null
|
||||
let faultGrant = null
|
||||
|
||||
beforeAll(async () => {
|
||||
if (!bundles) {
|
||||
return
|
||||
}
|
||||
const sessionDir = join(projectDir, 'mobile', 'src', 'session')
|
||||
const browserDir = join(projectDir, 'mobile', 'src', 'browser')
|
||||
const terminalDir = join(projectDir, 'mobile', 'src', 'terminal')
|
||||
const cspHeader = await readShellCsp()
|
||||
bridgeVersion = await readBridgeProtocolVersion()
|
||||
faultGrant = await readBridgeFaultGrant()
|
||||
scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-held-press-'))
|
||||
const appDir = join(scratch, 'app')
|
||||
const routeDir = join(appDir, MOBILE_WEB_APP_ROUTE_ROOT)
|
||||
await mkdir(routeDir, { recursive: true })
|
||||
await writeFile(join(routeDir, '_layout.tsx'), LAYOUT_SOURCE)
|
||||
await writeFile(
|
||||
join(routeDir, 'hold-dictation-probe.tsx'),
|
||||
holdDictationProbeRouteSource({
|
||||
terminalActionsModule: join(sessionDir, 'MobileTerminalInputActions'),
|
||||
chatComposerModule: join(sessionDir, 'MobileNativeChatComposer')
|
||||
})
|
||||
)
|
||||
await writeFile(
|
||||
join(routeDir, 'command-dock-probe.tsx'),
|
||||
commandDockProbeRouteSource({
|
||||
commandDockModule: join(sessionDir, 'MobileSessionCommandDock'),
|
||||
keyDefinitionsModule: join(terminalDir, 'terminal-key-definitions')
|
||||
})
|
||||
)
|
||||
await writeFile(
|
||||
join(routeDir, 'browser-pane-probe.tsx'),
|
||||
browserPaneProbeRouteSource({
|
||||
paneViewModule: join(browserDir, 'MobileBrowserPaneView'),
|
||||
interactionsModule: join(browserDir, 'use-mobile-browser-interactions'),
|
||||
geometryModule: join(browserDir, 'browser-touch-geometry')
|
||||
})
|
||||
)
|
||||
const built = await buildMobileWebAppBundle({
|
||||
appDir,
|
||||
outDir: join(scratch, 'bundle'),
|
||||
pageRoutes: Object.values(ROUTES).map((pathname) => ({ pathname, grants: [] }))
|
||||
})
|
||||
const served = await createBundleServer({ outDir: built.outDir, cspHeader })
|
||||
server = served.server
|
||||
origin = served.origin
|
||||
const executablePath = process.env.ORCA_MOBILE_WEB_RENDER_BROWSER
|
||||
browser = await chromium.launch({ headless: true, ...(executablePath ? { executablePath } : {}) })
|
||||
}, 600_000)
|
||||
|
||||
afterAll(async () => {
|
||||
await browser?.close()
|
||||
server?.close()
|
||||
if (scratch) {
|
||||
await rm(scratch, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
async function openProbe(pathname) {
|
||||
const page = await browser.newPage({ viewport: { width: 390, height: 844 }, hasTouch: true })
|
||||
await page.addInitScript(installShellDouble, {
|
||||
version: bridgeVersion,
|
||||
sessionId: 'held-press-session',
|
||||
buildId: 'held-press-build',
|
||||
route: { pathname, params: {} },
|
||||
host: { id: 'held-host', name: 'Held Host', endpoint: 'ws://held', lastConnected: 1 },
|
||||
storage: {},
|
||||
faultGrant,
|
||||
grants: [faultGrant],
|
||||
pageRoutes: Object.values(ROUTES),
|
||||
replies: {}
|
||||
})
|
||||
const errors = []
|
||||
page.on('pageerror', (error) => errors.push(`${error.name}: ${error.message}`))
|
||||
await page.goto(`${origin}/`, { waitUntil: 'load' })
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
globalThis.__orcaHeldPressProbe !== undefined ||
|
||||
(globalThis.__orcaRenderCheckFaults ?? []).length > 0,
|
||||
{ timeout: 60_000, polling: 100 }
|
||||
)
|
||||
expect(await page.evaluate(() => globalThis.__orcaRenderCheckFaults ?? [])).toEqual([])
|
||||
return { errors, page }
|
||||
}
|
||||
|
||||
/**
|
||||
* Touch-holds `selector` for `holdMs`, reads `read()` mid-hold, then lifts. At 500 ms it plays
|
||||
* Android WebView's long-press: `contextmenu`, then — only where the touch could select text —
|
||||
* `selectionchange` and `touchcancel`, which is what the device did while page text was selectable.
|
||||
*/
|
||||
async function holdLikeAndroidWebView(page, selector, { holdMs, read }) {
|
||||
// A handle, not a locator: a label the selector matches can change as soon as the press lands.
|
||||
const target = await page.waitForSelector(selector)
|
||||
const box = await target.boundingBox()
|
||||
const point = { x: box.x + box.width / 2, y: box.y + box.height / 2 }
|
||||
const input = await page.context().newCDPSession(page)
|
||||
await input.send('Input.dispatchTouchEvent', { type: 'touchStart', touchPoints: [point] })
|
||||
await page.waitForTimeout(LONG_PRESS_MS)
|
||||
const selectable = await target.evaluate((node, at) => {
|
||||
node.dispatchEvent(
|
||||
new MouseEvent('contextmenu', {
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
clientX: at.x,
|
||||
clientY: at.y
|
||||
})
|
||||
)
|
||||
// The nearest explicit user-select up the tree decides, as Blink resolves `auto`.
|
||||
for (let element = node; element; element = element.parentElement) {
|
||||
const value = getComputedStyle(element).userSelect
|
||||
if (value && value !== 'auto') {
|
||||
return value !== 'none'
|
||||
}
|
||||
}
|
||||
return true
|
||||
}, point)
|
||||
if (selectable) {
|
||||
await page.evaluate(() => document.dispatchEvent(new Event('selectionchange')))
|
||||
await input.send('Input.dispatchTouchEvent', { type: 'touchCancel', touchPoints: [] })
|
||||
}
|
||||
await page.waitForTimeout(holdMs - LONG_PRESS_MS)
|
||||
const midHold = { selectable, ...(await read(target)) }
|
||||
if (!selectable) {
|
||||
await input.send('Input.dispatchTouchEvent', { type: 'touchEnd', touchPoints: [] })
|
||||
}
|
||||
await page.waitForTimeout(300)
|
||||
return { midHold, afterRelease: await read(target) }
|
||||
}
|
||||
|
||||
const pressOuts = (page) => page.evaluate(() => globalThis.__orcaHeldPressProbe.pressOuts())
|
||||
|
||||
describeRender(
|
||||
'held presses on the page',
|
||||
() => {
|
||||
describe('hold-to-dictate', () => {
|
||||
it('keeps the terminal mic held until the finger lifts', async () => {
|
||||
const { errors, page } = await openProbe(ROUTES.mic)
|
||||
const { midHold, afterRelease } = await holdLikeAndroidWebView(
|
||||
page,
|
||||
`#${TERMINAL_MIC_ID} [aria-label="Start voice dictation"]`,
|
||||
{
|
||||
holdMs: 1500,
|
||||
read: async (mic) => ({
|
||||
label: await mic.getAttribute('aria-label'),
|
||||
pressOuts: await pressOuts(page)
|
||||
})
|
||||
}
|
||||
)
|
||||
expect(midHold).toEqual({
|
||||
selectable: false,
|
||||
label: 'Stop voice dictation',
|
||||
pressOuts: { terminal: 0, chat: 0 }
|
||||
})
|
||||
expect(afterRelease.pressOuts).toEqual({ terminal: 1, chat: 0 })
|
||||
expect(errors).toEqual([])
|
||||
await page.close()
|
||||
}, 300_000)
|
||||
|
||||
it('keeps the chat mic held until the finger lifts', async () => {
|
||||
const { errors, page } = await openProbe(ROUTES.mic)
|
||||
const { midHold, afterRelease } = await holdLikeAndroidWebView(
|
||||
page,
|
||||
`#${CHAT_MIC_ID} [aria-label="Dictate"]`,
|
||||
{
|
||||
holdMs: 1500,
|
||||
read: async (mic) => ({
|
||||
label: await mic.getAttribute('aria-label'),
|
||||
pressOuts: await pressOuts(page)
|
||||
})
|
||||
}
|
||||
)
|
||||
expect(midHold).toEqual({
|
||||
selectable: false,
|
||||
label: 'Stop dictation',
|
||||
pressOuts: { terminal: 0, chat: 0 }
|
||||
})
|
||||
// The icon under the finger swaps on press, so the release has to reach the Pressable.
|
||||
expect(afterRelease.pressOuts).toEqual({ terminal: 0, chat: 1 })
|
||||
expect(errors).toEqual([])
|
||||
await page.close()
|
||||
}, 300_000)
|
||||
})
|
||||
|
||||
describe('the key bar', () => {
|
||||
const sentCount = async (page) =>
|
||||
(await page.evaluate(() => globalThis.__orcaHeldPressProbe.sent())).length
|
||||
|
||||
it('keeps repeating a held arrow key until the finger lifts', async () => {
|
||||
const { errors, page } = await openProbe(ROUTES.keys)
|
||||
const { midHold, afterRelease } = await holdLikeAndroidWebView(
|
||||
page,
|
||||
`[aria-label="${REPEAT_KEY_LABEL}"]`,
|
||||
{ holdMs: 1500, read: async () => ({ sent: await sentCount(page) }) }
|
||||
)
|
||||
// One send on press-in, then every 45 ms from 400 ms: ~25 by 1.5 s, against 1 when the
|
||||
// long-press ends the press before the first repeat.
|
||||
expect(midHold.sent).toBeGreaterThan(20)
|
||||
expect(midHold.selectable).toBe(false)
|
||||
// Nothing more once the finger is up.
|
||||
expect(afterRelease.sent - midHold.sent).toBeLessThan(3)
|
||||
expect(errors).toEqual([])
|
||||
await page.close()
|
||||
}, 300_000)
|
||||
|
||||
it('selects nothing on a long-press of a tap key', async () => {
|
||||
const { errors, page } = await openProbe(ROUTES.keys)
|
||||
const { midHold } = await holdLikeAndroidWebView(page, `[aria-label="${TAP_KEY_LABEL}"]`, {
|
||||
holdMs: 1000,
|
||||
read: () => page.evaluate(() => ({ selection: window.getSelection()?.toString() ?? '' }))
|
||||
})
|
||||
expect(midHold).toEqual({ selectable: false, selection: '' })
|
||||
expect(errors).toEqual([])
|
||||
await page.close()
|
||||
}, 300_000)
|
||||
|
||||
it('sends a tapped key once', async () => {
|
||||
const { errors, page } = await openProbe(ROUTES.keys)
|
||||
const key = await page.waitForSelector(`[aria-label="${TAP_KEY_LABEL}"]`)
|
||||
await key.tap()
|
||||
await page.waitForFunction(() => globalThis.__orcaHeldPressProbe.sent().length === 1)
|
||||
expect(await page.evaluate(() => globalThis.__orcaHeldPressProbe.sent())).toEqual(['\x1b'])
|
||||
expect(errors).toEqual([])
|
||||
await page.close()
|
||||
}, 300_000)
|
||||
})
|
||||
|
||||
describe('the browser pane', () => {
|
||||
const toasts = (page) => page.evaluate(() => globalThis.__orcaHeldPressProbe.toasts())
|
||||
const clicks = (page) =>
|
||||
page.evaluate(() =>
|
||||
globalThis.__orcaHeldPressProbe
|
||||
.requests()
|
||||
.filter((request) => request.method === 'browser.mouseClick')
|
||||
.map((request) => ({ button: request.params.button, page: request.params.page }))
|
||||
)
|
||||
|
||||
it('right-clicks on a 1 s hold', async () => {
|
||||
const { errors, page } = await openProbe(ROUTES.browser)
|
||||
const { midHold, afterRelease } = await holdLikeAndroidWebView(
|
||||
page,
|
||||
`#${BROWSER_VIEWPORT_ID}`,
|
||||
{
|
||||
holdMs: 1000,
|
||||
read: async () => ({ toasts: await toasts(page), clicks: await clicks(page) })
|
||||
}
|
||||
)
|
||||
// The hook's timer is 550 ms, after the WebView's long-press would have ended the press.
|
||||
const rightClick = { button: 'right', page: 'held-press-page' }
|
||||
expect(midHold).toEqual({
|
||||
selectable: false,
|
||||
toasts: ['Right click'],
|
||||
clicks: [rightClick]
|
||||
})
|
||||
// The release after a right-click sends no left click.
|
||||
expect(afterRelease.clicks).toEqual([rightClick])
|
||||
expect(errors).toEqual([])
|
||||
await page.close()
|
||||
}, 300_000)
|
||||
|
||||
it('left-clicks once on a tap', async () => {
|
||||
const { errors, page } = await openProbe(ROUTES.browser)
|
||||
const viewport = await page.waitForSelector(`#${BROWSER_VIEWPORT_ID}`)
|
||||
await viewport.tap()
|
||||
await page.waitForFunction(() => globalThis.__orcaHeldPressProbe.requests().length > 0)
|
||||
expect(await clicks(page)).toEqual([{ button: 'left', page: 'held-press-page' }])
|
||||
expect(await toasts(page)).toEqual([])
|
||||
expect(errors).toEqual([])
|
||||
await page.close()
|
||||
}, 300_000)
|
||||
})
|
||||
},
|
||||
900_000
|
||||
)
|
||||
@@ -47,7 +47,7 @@ const ENGINES = [
|
||||
const PAGE_ENTRY = `
|
||||
import { createElement as h } from 'react'
|
||||
import { createRoot } from 'react-dom/client'
|
||||
const { StyleSheet, TextInput, View } = require('react-native')
|
||||
const { Modal, StyleSheet, Text, TextInput, View } = require('react-native')
|
||||
const styles = StyleSheet.create({
|
||||
separatorBox: { paddingTop: 10.1, width: 12, backgroundColor: '#ffffff' },
|
||||
separator: { height: StyleSheet.hairlineWidth, backgroundColor: '#000000' },
|
||||
@@ -55,7 +55,7 @@ const styles = StyleSheet.create({
|
||||
input: { height: 40 }
|
||||
})
|
||||
createRoot(document.getElementById('root')).render(
|
||||
h(View, null, h(View, { testID: 'separator-box', style: styles.separatorBox }, h(View, { style: styles.separator })), h(View, { testID: 'hairline', style: styles.hairline }), h(TextInput, { testID: 'input', style: styles.input }))
|
||||
h(View, null, h(View, { testID: 'separator-box', style: styles.separatorBox }, h(View, { style: styles.separator })), h(View, { testID: 'hairline', style: styles.hairline }), h(TextInput, { testID: 'input', style: styles.input }), h(Text, { testID: 'plain-text' }, 'row'), h(Text, { testID: 'selectable-text', selectable: true }, 'message'), location.hash === '#modal' ? h(Modal, { visible: true, transparent: true }, h(View, null, h(Text, { testID: 'modal-text' }, 'sheet row'), h(TextInput, { testID: 'modal-input', style: styles.input }))) : null)
|
||||
)
|
||||
`
|
||||
|
||||
@@ -111,9 +111,9 @@ afterAll(async () => {
|
||||
}
|
||||
})
|
||||
|
||||
async function openPage(engine) {
|
||||
async function openPage(engine, hash = '') {
|
||||
const page = await browsers.get(engine.name).newPage(engine.pageOptions)
|
||||
await page.goto(`${origin}/`, { waitUntil: 'domcontentloaded' })
|
||||
await page.goto(`${origin}/${hash}`, { waitUntil: 'domcontentloaded' })
|
||||
await page.waitForSelector('[data-testid="input"]')
|
||||
return page
|
||||
}
|
||||
@@ -168,4 +168,81 @@ describeParity.each(ENGINES)('the page against native, at a phone density, in $n
|
||||
await page.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('leaves only selectable text and editable fields selectable, as native does', async () => {
|
||||
// The iOS shell keeps WebKit text interaction on so fields take text, and a hold on plain text
|
||||
// would otherwise raise WebKit's selection over the page's own long press.
|
||||
const page = await openPage(engine)
|
||||
try {
|
||||
await page.type('[data-testid="input"]', 'typed')
|
||||
const measured = await page.evaluate(() => {
|
||||
const select = (id) => getComputedStyle(document.querySelector(`[data-testid="${id}"]`))
|
||||
const read = (id) => select(id).webkitUserSelect || select(id).userSelect
|
||||
return {
|
||||
plain: read('plain-text'),
|
||||
selectable: read('selectable-text'),
|
||||
input: read('input'),
|
||||
value: document.querySelector('[data-testid="input"]').value
|
||||
}
|
||||
})
|
||||
expect(measured).toEqual({ plain: 'none', selectable: 'text', input: 'text', value: 'typed' })
|
||||
} finally {
|
||||
await page.close()
|
||||
}
|
||||
})
|
||||
|
||||
it("selects through the browser's own gesture only what native lets the user select", async () => {
|
||||
// A double-click, not a Range: a script can select text a user cannot.
|
||||
const page = await openPage(engine)
|
||||
try {
|
||||
const selectedBy = async (testId) => {
|
||||
await page.evaluate(() => window.getSelection()?.removeAllRanges())
|
||||
await page.dblclick(`[data-testid="${testId}"]`)
|
||||
return page.evaluate(() => window.getSelection()?.toString() ?? '')
|
||||
}
|
||||
expect(await selectedBy('plain-text')).toBe('')
|
||||
// react-native-web's Text/index.js:115 adds `styles.selectable` (`userSelect: 'text'`).
|
||||
expect(await selectedBy('selectable-text')).not.toBe('')
|
||||
|
||||
await page.type('[data-testid="input"]', 'hello world')
|
||||
await page.dblclick('[data-testid="input"]')
|
||||
const selectedInField = await page.evaluate(() => {
|
||||
const input = document.querySelector('[data-testid="input"]')
|
||||
return input.selectionEnd - input.selectionStart
|
||||
})
|
||||
expect(selectedInField).toBeGreaterThan(0)
|
||||
} finally {
|
||||
await page.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('applies the same to text a modal portals outside #root, and keeps its fields selectable', async () => {
|
||||
// react-native-web's Modal appends to document.body, so a rule on #root would miss sheets.
|
||||
// The fields' callout exemption goes unmeasured: desktop WebKit lacks the property.
|
||||
const page = await openPage(engine, '#modal')
|
||||
try {
|
||||
await page.waitForSelector('[data-testid="modal-text"]')
|
||||
const measured = await page.evaluate(() => {
|
||||
const text = document.querySelector('[data-testid="modal-text"]')
|
||||
const input = document.querySelector('[data-testid="modal-input"]')
|
||||
const read = (node) =>
|
||||
getComputedStyle(node).webkitUserSelect || getComputedStyle(node).userSelect
|
||||
return {
|
||||
outsideRoot: !document.getElementById('root').contains(text),
|
||||
text: read(text),
|
||||
input: read(input)
|
||||
}
|
||||
})
|
||||
expect(measured).toEqual({
|
||||
outsideRoot: true,
|
||||
text: 'none',
|
||||
input: 'text'
|
||||
})
|
||||
await page.evaluate(() => window.getSelection()?.removeAllRanges())
|
||||
await page.dblclick('[data-testid="modal-text"]')
|
||||
expect(await page.evaluate(() => window.getSelection()?.toString() ?? '')).toBe('')
|
||||
} finally {
|
||||
await page.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -20,7 +20,11 @@ import { fileURLToPath } from 'node:url'
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import * as esbuild from 'esbuild'
|
||||
import { chromium, webkit } from 'playwright-core'
|
||||
import { MOBILE_WEB_APP_ROOT_RESET, lucideBarrelPlugin } from './build-mobile-web-app-bundle.mjs'
|
||||
import {
|
||||
MOBILE_WEB_APP_NATIVE_PARITY_STYLE,
|
||||
MOBILE_WEB_APP_ROOT_RESET,
|
||||
lucideBarrelPlugin
|
||||
} from './build-mobile-web-app-bundle.mjs'
|
||||
import { mobileWebAppDependenciesPresent } from './mobile-web-app-bundle-dependencies.mjs'
|
||||
import { textInputFontSizeFloor } from './mobile-web-app-text-input-font-size-seam.mjs'
|
||||
import {
|
||||
@@ -270,7 +274,8 @@ beforeAll(async () => {
|
||||
join(outDir, 'index.html'),
|
||||
// The root reset the shipped document carries: every box below the mount is `flex: 1`, so
|
||||
// without a definite height on all three the editor measures 0 and paints nothing.
|
||||
`<!doctype html><html><head><meta charset="utf-8">${MOBILE_WEB_APP_ROOT_RESET}</head>` +
|
||||
`<!doctype html><html><head><meta charset="utf-8">${MOBILE_WEB_APP_ROOT_RESET}` +
|
||||
`${MOBILE_WEB_APP_NATIVE_PARITY_STYLE}</head>` +
|
||||
'<body><div id="root"></div>' +
|
||||
'<script type="module" src="/rich-markdown-check.js"></script></body></html>'
|
||||
)
|
||||
@@ -512,6 +517,28 @@ describeEditor(
|
||||
* a paragraph inline reported its own text with no marker, so the bullet the user pressed
|
||||
* was gone the moment the host saved what the document reported.
|
||||
*/
|
||||
it('edits and selects under the shipped document style, which makes other text unselectable', async () => {
|
||||
const { page, consoleErrors } = await openPage(browser)
|
||||
try {
|
||||
await setContent(page, 'alpha', '<p>alpha</p>')
|
||||
const word = page.locator('#first-surface #editor p')
|
||||
await word.click()
|
||||
await page.keyboard.press('End')
|
||||
await page.keyboard.type(' bravo')
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
document.querySelector('#first-surface #editor p')?.textContent === 'alpha bravo'
|
||||
)
|
||||
// The browser's own gesture, not a Range: `body` is `user-select: none` on the page.
|
||||
await page.evaluate(() => window.getSelection()?.removeAllRanges())
|
||||
await word.dblclick()
|
||||
expect(await page.evaluate(() => window.getSelection()?.toString().trim())).not.toBe('')
|
||||
expect(consoleErrors).toEqual([])
|
||||
} finally {
|
||||
await page.close()
|
||||
}
|
||||
}, 600_000)
|
||||
|
||||
it('reports a typed bullet list as a list, and renders that markdown back as one', async () => {
|
||||
const { page, consoleErrors } = await openPage(browser)
|
||||
try {
|
||||
|
||||
@@ -281,7 +281,9 @@ export function useMobileBrowserInteractions(args: MobileBrowserInteractionArgs)
|
||||
scrollingRef.current = false
|
||||
startPointRef.current = null
|
||||
},
|
||||
onPanResponderTerminationRequest: () => true
|
||||
// The long-press right-click outlives the WebView's own long-press contextmenu at ~500 ms.
|
||||
onPanResponderTerminationRequest: (event) =>
|
||||
!('type' in event.nativeEvent) || event.nativeEvent.type !== 'contextmenu'
|
||||
}),
|
||||
[clearLongPressTimer, handleResponderGrant, handleResponderMove, handleResponderRelease]
|
||||
)
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
} from './MobileNativeChatSessionOptionPickers'
|
||||
import type { PendingNativeChatImage } from './mobile-native-chat-image-attachment'
|
||||
import { mobileNativeChatInputStyles } from './mobile-native-chat-input-styles'
|
||||
import { keepHeldPressThroughLongPress } from './held-press-long-press'
|
||||
|
||||
const NO_FILE_PATHS: string[] = []
|
||||
const NO_ATTACHMENTS: PendingNativeChatImage[] = []
|
||||
@@ -301,17 +302,26 @@ export function MobileNativeChatComposer({
|
||||
onPress={dictationMode === 'hold' ? undefined : onMicPress}
|
||||
onPressIn={dictationMode === 'hold' ? onMicPressIn : undefined}
|
||||
onPressOut={dictationMode === 'hold' ? onMicPressOut : undefined}
|
||||
onLongPress={dictationMode === 'hold' ? keepHeldPressThroughLongPress : undefined}
|
||||
disabled={disabled}
|
||||
>
|
||||
{/* The icon swaps on press; as the page's touch target, its removal would send
|
||||
touchend to a detached node and lose the release. */}
|
||||
{micActive ? (
|
||||
<Square
|
||||
pointerEvents="none"
|
||||
size={18}
|
||||
color={colors.statusRed}
|
||||
strokeWidth={2.4}
|
||||
fill={colors.statusRed}
|
||||
/>
|
||||
) : (
|
||||
<Mic size={20} color={colors.textSecondary} strokeWidth={2} />
|
||||
<Mic
|
||||
pointerEvents="none"
|
||||
size={20}
|
||||
color={colors.textSecondary}
|
||||
strokeWidth={2}
|
||||
/>
|
||||
)}
|
||||
</Pressable>
|
||||
) : null}
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
} from '../terminal/terminal-keyboard-type'
|
||||
import { MobileTerminalLiveInputStatus } from './MobileTerminalLiveInputStatus'
|
||||
import { MobileTerminalInputActions } from './MobileTerminalInputActions'
|
||||
import { keepHeldPressThroughLongPress } from './held-press-long-press'
|
||||
import { isTerminalPhoneDisplayMode } from './mobile-session-route-helpers'
|
||||
import { colors } from '../theme/mobile-theme'
|
||||
import { styles } from './mobile-session-styles'
|
||||
@@ -210,6 +211,7 @@ export function MobileSessionCommandDock({ controller }: { controller: MobileSes
|
||||
}
|
||||
void handleAccessoryKey(createTerminalLiveAccessoryInput(key))
|
||||
}}
|
||||
onLongPress={key.repeatable ? keepHeldPressThroughLongPress : undefined}
|
||||
accessibilityLabel={key.accessibilityLabel ?? `Send ${key.label}`}
|
||||
>
|
||||
<Text
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { ActivityIndicator, Pressable, type StyleProp, type ViewStyle } from 'react-native'
|
||||
import { ImagePlus, Mic } from 'lucide-react-native'
|
||||
import { colors } from '../theme/mobile-theme'
|
||||
import { keepHeldPressThroughLongPress } from './held-press-long-press'
|
||||
|
||||
type DictationState = {
|
||||
readonly isStarting: boolean
|
||||
@@ -74,7 +75,7 @@ export function MobileTerminalInputActions({
|
||||
onDictationCancel()
|
||||
}
|
||||
}
|
||||
: undefined
|
||||
: keepHeldPressThroughLongPress
|
||||
}
|
||||
accessibilityLabel={
|
||||
dictation.isRecording
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
// RNW holds a press past contextmenu only with onLongPress; body user-select:none stops touchcancel
|
||||
export function keepHeldPressThroughLongPress(): void {}
|
||||
Reference in New Issue
Block a user