mirror of
https://github.com/stablyai/orca.git
synced 2026-11-18 20:06:10 +00:00
* fix(emulator): restore scrcpy control encoders and track the video size Bring back the scrcpy v2.4 control-message encoders removed as unwired in #13528 (touch, keycode, text), re-checked against control_msg.c and the server's ControlMessageReader: generic-finger pointer id, the official u16 pressure fixed point, and text split into <=300-byte messages on code point boundaries. The server drops touch events whose screen size differs from the current video size. v2.4 reports the size once in its codec meta; after a rotation the new size is only in the next SPS, so the session reads it from every config packet. ScrcpyStreamSession.sendControl() returns, the control socket disables Nagle, and the registry keeps the live session's control target next to its video. * fix(emulator): send Android input through the scrcpy control socket Android pane input ran one `adb shell input` process per action (~150 ms each on a USB phone) and replayed drags only on pointer-up as a fixed 300 ms straight swipe between the endpoints. Pane pointer phases now go to main as they happen over a new fire-and-forget `emulator:scrcpyTouch` IPC and are written to the live session's scrcpy control socket, so drags and scrolls track the pointer. CLI and agent tap, gesture, type and button use the same socket whenever a stream is live; `adb shell input` remains the fallback without one. Input routing and the `wm size` cache the fallback needs move into AndroidDeviceInput. iOS is unchanged. * test(emulator): keep the adb fallback test from leaking a failing sendControl * docs(emulator): explain why pane touches have no per-phase adb fallback * fix(emulator): keep iOS touch callback stable across interact changes sendTouch depended on canInteract only for the scrcpy branch, but the new dep also rebuilt it for iOS, so a canInteract flip mid-drag ran the frame's cleanup, ended the live touch early and replayed the drag over RPC. The frame already gates touch begins on canInteract. * fix(emulator): narrow scrcpy touch payloads without a type assertion The changed-code casting gate forbids `as`; use `in` narrowing like the other IPC payload guards. --------- Co-authored-by: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com>
84 lines
2.3 KiB
TypeScript
84 lines
2.3 KiB
TypeScript
import type { EmulatorSessionInfo } from '../emulator-types'
|
|
import { scrcpyVideoRegistry } from '../scrcpy-video-registry'
|
|
import type { AndroidCommandRunner } from './android-command-runner'
|
|
import type { AndroidSdkPaths } from './android-sdk-discovery'
|
|
import { ScrcpyStreamSession } from './scrcpy-stream-session'
|
|
|
|
export type AndroidStreamHandle = { close: () => void }
|
|
|
|
export type StartAndroidStreamParams = {
|
|
runner: AndroidCommandRunner
|
|
sdk: AndroidSdkPaths
|
|
serial: string
|
|
jarPath: string
|
|
maxSize?: number
|
|
}
|
|
|
|
export type StartAndroidStream = (
|
|
params: StartAndroidStreamParams
|
|
) => Promise<{ info: EmulatorSessionInfo; handle: AndroidStreamHandle }>
|
|
|
|
function toArrayBuffer(buffer: Buffer): ArrayBuffer {
|
|
const arrayBuffer = new ArrayBuffer(buffer.byteLength)
|
|
new Uint8Array(arrayBuffer).set(buffer)
|
|
return arrayBuffer
|
|
}
|
|
|
|
export const startAndroidStreamSession: StartAndroidStream = async ({
|
|
runner,
|
|
sdk,
|
|
serial,
|
|
jarPath,
|
|
maxSize
|
|
}) => {
|
|
let session: ScrcpyStreamSession | null = null
|
|
// Register before startup so early config/keyframes enter the replay cache.
|
|
scrcpyVideoRegistry.register(serial, () => session?.close())
|
|
try {
|
|
session = await ScrcpyStreamSession.start(
|
|
{ runner, sdk, serial, localJarPath: jarPath, maxSize },
|
|
{
|
|
onMeta: (meta) => scrcpyVideoRegistry.pushMeta(serial, meta),
|
|
onFrame: (frame) =>
|
|
scrcpyVideoRegistry.pushFrame(serial, {
|
|
config: frame.config,
|
|
keyFrame: frame.keyFrame,
|
|
pts: frame.pts.toString(),
|
|
bytes: toArrayBuffer(frame.data)
|
|
}),
|
|
onError: () => scrcpyVideoRegistry.stop(serial),
|
|
onClose: () => scrcpyVideoRegistry.stop(serial)
|
|
}
|
|
)
|
|
} catch (error) {
|
|
scrcpyVideoRegistry.stop(serial)
|
|
throw error
|
|
}
|
|
scrcpyVideoRegistry.attachControl(serial, session)
|
|
|
|
let closed = false
|
|
return {
|
|
info: androidStreamSessionInfo(serial),
|
|
handle: {
|
|
close: () => {
|
|
if (closed) {
|
|
return
|
|
}
|
|
closed = true
|
|
session?.close()
|
|
scrcpyVideoRegistry.stop(serial)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
export function androidStreamSessionInfo(serial: string): EmulatorSessionInfo {
|
|
return {
|
|
deviceUdid: serial,
|
|
streamUrl: `scrcpy://${serial}`,
|
|
wsUrl: '',
|
|
streamCodec: 'h264',
|
|
backend: 'android'
|
|
}
|
|
}
|