mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 10:58:10 +00:00
Relay remote accepted chat prompts to desktop telemetry
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
# Remote accepted prompt observations
|
||||
|
||||
The committed journal acceptance observer in `native-chat-prompt-submissions.ts` owns the evidence. A fresh personal send from a transport-proven desktop or phone retains its device class and consent-owning desktop in memory until provider acceptance. Queue conversion keeps the original draft's provenance. Reservation, handover, rejected sends, withdrawn drafts, conversation commands, unattended sends and operation replay do not emit an accepted prompt observation.
|
||||
|
||||
An initialized desktop records `native_chat_prompt_submitted` through its existing telemetry client, including when consent is disabled (the client drops the observation). A headless host instead uses the existing live structured-chat relay on `runtime.clientEvents.subscribe`. The additive `structured-chat-prompt-telemetry-v1` capability isolates the new event from older readers; older hosts simply omit it. Prompt delivery and journal storage are unchanged.
|
||||
|
||||
The event contains only `type`, a fresh observation UUID, `agent_kind` and `client` (`desktop` or `mobile`). No prompt text, session/message/request identifiers, paths, fingerprints, timestamps or device identifiers cross this observation channel. The owner routing key stays on the host: a paired desktop uses its authenticated device ID, and a delegated phone uses its authenticated parent desktop ID. Only an authenticated runtime desktop stream with that owner and the new capability receives the event. Another desktop, a phone stream, the CLI or a declared client without proven identity cannot receive it. A phone paired directly to the local desktop follows the existing local telemetry path.
|
||||
|
||||
Both shared-control and one-shot subscription transports use the same desktop-main subscription decoder. The decoder checks the current transport generation, strictly validates the event and deduplicates its UUID before calling the consent-owning telemetry client with `runtime: remote`. It never forwards accepted observations to the renderer. The UUID is not an analytics property. The existing protected milestone admission preserves first-interaction/day ordering, installation attribution, consent and sampling behavior. UUID deduplication retains the newest 1,024 observations across desktop subscriptions; the existing per-install milestone cursor provides additional milestone deduplication.
|
||||
|
||||
## Deliberate undercount limits
|
||||
|
||||
- The relay is live and best effort, matching the existing agent-start observation channel. Provider acceptance while the desktop is disconnected or before its event stream opens is missed. Reconnect registers a new listener and does not backfill historical acceptance, retry prompt delivery or infer success from send receipts. Desktop restart also clears received UUIDs, but the host does not replay observations.
|
||||
- Pending prompt provenance is keyed by the live journal object and capped at 256 entries per journal. Host restart or journal replacement loses it. A queued draft or an unaccepted submission persisted before restart may later be accepted without earning a milestone; operation replay and history reads cannot restore its provenance. Older pending entries evicted at the cap also undercount.
|
||||
- Persisting provenance or recovering disconnected acceptance would require a separate design for durable owner identity, acceptance ordering, retention, consent changes and reader compatibility. This change deliberately adds no outbox, journal annotation or history-derived inference. A later fresh, proven send can still earn the missing milestone.
|
||||
|
||||
These limits prefer missing an observation over attributing automation, replay or an unproven user action to an installation.
|
||||
@@ -0,0 +1,106 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
|
||||
import { decodeRuntimeEnvironmentSubscriptionEvents } from './runtime-environment-subscription-events'
|
||||
import {
|
||||
cleanupTelemetryClientTest,
|
||||
setupTelemetryClientTest,
|
||||
type TelemetryClientTestState
|
||||
} from '../../telemetry/client-test-harness'
|
||||
|
||||
let state: TelemetryClientTestState
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(new Date('2026-11-08T23:58:00Z'))
|
||||
state = setupTelemetryClientTest({
|
||||
optedIn: true,
|
||||
installId: '00000000-0000-4000-8000-000000000000',
|
||||
existedBeforeTelemetryRelease: false,
|
||||
protectedMilestones: { installCohort: 'fresh' }
|
||||
})
|
||||
})
|
||||
afterEach(() => {
|
||||
cleanupTelemetryClientTest(state.envStash)
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
const prompt = (client: 'desktop' | 'mobile' = 'desktop') => ({
|
||||
type: 'structuredChatPromptAccepted',
|
||||
id: randomUUID(),
|
||||
agent_kind: 'codex',
|
||||
client
|
||||
})
|
||||
const frame = (result: unknown) => ({
|
||||
type: 'response' as const,
|
||||
response: { id: 'events-1', ok: true as const, result, _meta: { runtimeId: 'remote-runtime' } }
|
||||
})
|
||||
const cursor = () => state.settings.telemetry?.protectedMilestones
|
||||
const captured = (name: string) =>
|
||||
state.mock.capture.mock.calls.map(([event]) => event).filter((event) => event.event === name)
|
||||
|
||||
it('consumes live accepted observations in desktop main once, without exposing their IDs to analytics', () => {
|
||||
const delivered: unknown[] = []
|
||||
let current = false
|
||||
const observe = decodeRuntimeEnvironmentSubscriptionEvents(
|
||||
'runtime.clientEvents.subscribe',
|
||||
(event) => delivered.push(event),
|
||||
() => current
|
||||
)
|
||||
const event = prompt()
|
||||
observe(frame(event))
|
||||
expect(cursor()).toEqual({ installCohort: 'fresh' })
|
||||
current = true
|
||||
observe(frame(event))
|
||||
expect(cursor()).toEqual({
|
||||
installCohort: 'fresh',
|
||||
firstAgentInteractionDay: 20_765,
|
||||
meaningfulActivityDay: 20_765
|
||||
})
|
||||
expect(captured('first_agent_interaction')[0].properties).toMatchObject({
|
||||
source: 'native_chat',
|
||||
agent_kind: 'codex'
|
||||
})
|
||||
expect(captured('meaningful_activity_day')[0].properties.trigger).toBe('native_chat')
|
||||
expect(delivered).toEqual([frame(event)])
|
||||
expect(JSON.stringify(state.mock.capture.mock.calls)).not.toContain(event.id)
|
||||
|
||||
vi.setSystemTime(new Date('2026-11-09T00:01:00Z'))
|
||||
observe(frame(event))
|
||||
expect(cursor()?.meaningfulActivityDay).toBe(20_765)
|
||||
observe(frame(prompt('mobile')))
|
||||
expect(cursor()?.meaningfulActivityDay).toBe(20_766)
|
||||
expect(captured('meaningful_activity_day').map((entry) => entry.properties.trigger)).toEqual([
|
||||
'native_chat',
|
||||
'mobile_chat'
|
||||
])
|
||||
})
|
||||
|
||||
it('drops opted-out observations permanently and rejects unbounded or content-bearing facts', () => {
|
||||
const observe = decodeRuntimeEnvironmentSubscriptionEvents(
|
||||
'runtime.clientEvents.subscribe',
|
||||
() => {},
|
||||
() => true
|
||||
)
|
||||
const event = prompt('mobile')
|
||||
state.settings.telemetry!.optedIn = false
|
||||
observe(frame(event))
|
||||
expect(cursor()).toEqual({ installCohort: 'fresh' })
|
||||
state.settings.telemetry!.optedIn = true
|
||||
observe(frame(event))
|
||||
observe(frame({ ...prompt(), body: 'private prompt must never reach analytics' }))
|
||||
observe(frame({ ...prompt(), agent_kind: 'arbitrary-agent-name' }))
|
||||
const wrongMethod = decodeRuntimeEnvironmentSubscriptionEvents(
|
||||
'agentSession.subscribe',
|
||||
() => {},
|
||||
() => true
|
||||
)
|
||||
wrongMethod(frame(prompt()))
|
||||
expect(cursor()).toEqual({ installCohort: 'fresh' })
|
||||
|
||||
observe(frame(prompt('mobile')))
|
||||
expect(cursor()).toEqual({
|
||||
installCohort: 'fresh',
|
||||
firstAgentInteractionDay: 20_765,
|
||||
meaningfulActivityDay: 20_765
|
||||
})
|
||||
expect(captured('first_agent_interaction')[0].properties.source).toBe('mobile_chat')
|
||||
})
|
||||
@@ -19,6 +19,7 @@ export function decodeRuntimeEnvironmentSubscriptionEvents(
|
||||
event !== null &&
|
||||
'type' in event &&
|
||||
(event.type === 'structuredChatMessageDispatch' ||
|
||||
event.type === 'structuredChatPromptAccepted' ||
|
||||
event.type === 'structuredChatAgentStart') &&
|
||||
recordRemoteStructuredChatTelemetry(event)
|
||||
) {
|
||||
|
||||
+2
-1
@@ -233,7 +233,8 @@ async function foundConversation(
|
||||
? openingMessageOptions(candidate.record.options)
|
||||
: {})
|
||||
},
|
||||
caller.userClient
|
||||
caller.userClient,
|
||||
caller.telemetryOwnerKey
|
||||
).catch((error: unknown) => {
|
||||
if (!(error instanceof CommandReceiptExistsError)) {
|
||||
throw error
|
||||
|
||||
@@ -65,6 +65,8 @@ export type StructuredAgentSessionCaller = {
|
||||
clientKind?: 'mobile' | 'runtime'
|
||||
/** Proven by the caller's transport, never inferred from an unnamed/runtime client. */
|
||||
userClient?: 'desktop' | 'mobile'
|
||||
/** Consent-owning paired desktop, including the authenticated parent of a delegated phone. */
|
||||
telemetryOwnerKey?: string
|
||||
readsOptionsWithoutModel?: boolean
|
||||
}
|
||||
|
||||
|
||||
+83
-4
@@ -3,12 +3,15 @@ import type { RpcContext } from '../../../../runtime/rpc/core'
|
||||
import type { RpcCallerIdentity } from '../../../../runtime/rpc/rpc-caller-identity'
|
||||
import { structuredCallerFor } from '../../../../runtime/rpc/methods/structured-sessions/creation/structured-agent-session-gate'
|
||||
import { createTestParams } from './structured-agent-session-create-test-fixture'
|
||||
import { track } from '../../../../telemetry/client'
|
||||
import { isTelemetryInitialized, track } from '../../../../telemetry/client'
|
||||
import { subscribeStructuredChatTelemetry } from '../../../../telemetry/structured-chat-dispatch'
|
||||
import type { StructuredChatTelemetryEvent } from '../../../../../shared/structured-chat-dispatch-telemetry'
|
||||
import {
|
||||
createQueuedMessageTestRig,
|
||||
type QueuedMessageTestRig
|
||||
} from './structured-agent-session-queued-message-rig.test-fixture'
|
||||
import { eventually } from './structured-agent-session-queued-message-rig-provider.test-fixture'
|
||||
import { openRigTurnFor } from './structured-agent-session-queued-rig-turn.test-fixture'
|
||||
import {
|
||||
HOST_TEST_SESSION,
|
||||
HOST_TEST_THREAD,
|
||||
@@ -17,14 +20,21 @@ import {
|
||||
hostTestSendParams
|
||||
} from './structured-agent-session-host-test-data'
|
||||
|
||||
vi.mock('../../../../telemetry/client', () => ({ track: vi.fn() }))
|
||||
vi.mock('../../../../telemetry/client', () => ({
|
||||
track: vi.fn(),
|
||||
isTelemetryInitialized: vi.fn(() => true)
|
||||
}))
|
||||
|
||||
let rig: QueuedMessageTestRig
|
||||
const prompts = () =>
|
||||
vi.mocked(track).mock.calls.filter(([name]) => name === 'native_chat_prompt_submitted')
|
||||
|
||||
function transportCaller(caller?: RpcCallerIdentity, clientKind?: 'mobile' | 'runtime') {
|
||||
const context: Partial<RpcContext> = { caller, clientKind }
|
||||
function transportCaller(
|
||||
caller?: RpcCallerIdentity,
|
||||
clientKind?: 'mobile' | 'runtime',
|
||||
pairedDeviceParentId?: string
|
||||
) {
|
||||
const context: Partial<RpcContext> = { caller, clientKind, pairedDeviceParentId }
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: caller derivation reads only transport identity fields supplied here.
|
||||
return structuredCallerFor(context as RpcContext)
|
||||
}
|
||||
@@ -32,6 +42,7 @@ const desktop = () => transportCaller({ kind: 'desktop' })
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks()
|
||||
vi.mocked(isTelemetryInitialized).mockReturnValue(true)
|
||||
rig = await createQueuedMessageTestRig()
|
||||
})
|
||||
afterEach(async () => rig.dispose())
|
||||
@@ -169,3 +180,71 @@ it('counts the trusted desktop opening message only after its provider accepts i
|
||||
['native_chat_prompt_submitted', { agent_kind: 'codex', runtime: 'unknown', client: 'desktop' }]
|
||||
])
|
||||
})
|
||||
|
||||
it('relays only provider acceptance with the authenticated desktop or delegated phone owner', async () => {
|
||||
vi.mocked(isTelemetryInitialized).mockReturnValue(false)
|
||||
const events: { event: StructuredChatTelemetryEvent; owner: string }[] = []
|
||||
const unsubscribe = subscribeStructuredChatTelemetry((event, owner) =>
|
||||
events.push({ event, owner })
|
||||
)
|
||||
try {
|
||||
const caller = transportCaller({ kind: 'paired-device', deviceId: 'desktop-a' }, 'runtime')
|
||||
const desktopSend = { ...hostTestSendParams('private desktop prompt'), userSend: true as const }
|
||||
await rig.host.send(caller, desktopSend)
|
||||
await eventually(async () =>
|
||||
expect(
|
||||
(await rig.submission(desktopSend.envelope.clientOperationId))?.handedOverAt
|
||||
).toBeDefined()
|
||||
)
|
||||
expect(events).toEqual([])
|
||||
|
||||
const queuedId = hostTestOperationId()
|
||||
const fields = {
|
||||
body: hostTestMessage('private phone prompt'),
|
||||
delivery: 'queue-if-active' as const
|
||||
}
|
||||
const phone = transportCaller(
|
||||
{ kind: 'paired-device', deviceId: 'delegated-phone' },
|
||||
'mobile',
|
||||
'desktop-a'
|
||||
)
|
||||
await rig.host.send(phone, {
|
||||
envelope: rig.envelope(fields, 'agentSession.send', queuedId),
|
||||
...fields,
|
||||
userSend: true
|
||||
})
|
||||
expect(events).toEqual([])
|
||||
await rig.settleAccepted(
|
||||
desktopSend.envelope.clientOperationId,
|
||||
`of-${desktopSend.envelope.clientOperationId}`
|
||||
)
|
||||
await rig.host.send(caller, desktopSend)
|
||||
await openRigTurnFor(rig, desktopSend.envelope.clientOperationId, 'completed', 'success')
|
||||
await eventually(async () => expect((await rig.handoff(queuedId))?.handedOverAt).toBeDefined())
|
||||
await rig.settleAccepted(await rig.handoffId(queuedId), 'remote-phone-turn')
|
||||
expect(events).toEqual([
|
||||
{
|
||||
owner: 'desktop-a',
|
||||
event: {
|
||||
type: 'structuredChatPromptAccepted',
|
||||
id: expect.any(String),
|
||||
agent_kind: 'codex',
|
||||
client: 'desktop'
|
||||
}
|
||||
},
|
||||
{
|
||||
owner: 'desktop-a',
|
||||
event: {
|
||||
type: 'structuredChatPromptAccepted',
|
||||
id: expect.any(String),
|
||||
agent_kind: 'codex',
|
||||
client: 'mobile'
|
||||
}
|
||||
}
|
||||
])
|
||||
expect(events[0].event.id).not.toBe(events[1].event.id)
|
||||
expect(prompts()).toEqual([])
|
||||
} finally {
|
||||
unsubscribe()
|
||||
}
|
||||
})
|
||||
|
||||
+2
-1
@@ -119,7 +119,8 @@ export function sendStructuredAgentSessionTurn(
|
||||
ctx.journal,
|
||||
params.envelope.clientOperationId,
|
||||
ctx.agent,
|
||||
caller.userClient
|
||||
caller.userClient,
|
||||
caller.telemetryOwnerKey
|
||||
)
|
||||
}
|
||||
try {
|
||||
|
||||
+9
-2
@@ -133,7 +133,8 @@ export async function performSend(
|
||||
options?: Readonly<Record<string, string>>
|
||||
selection?: Readonly<Record<string, string>>
|
||||
},
|
||||
userClient?: 'desktop' | 'mobile'
|
||||
userClient?: 'desktop' | 'mobile',
|
||||
telemetryOwnerKey?: string
|
||||
): Promise<TurnOutcome<AgentSessionSendResult>> {
|
||||
const existing = ctx.journal.submission(input.clientMessageId)
|
||||
if (existing && existing.payloadFingerprint !== input.payloadFingerprint) {
|
||||
@@ -154,7 +155,13 @@ export async function performSend(
|
||||
}
|
||||
try {
|
||||
if (!input.body.command) {
|
||||
rememberNativeChatPrompt(ctx.journal, input.clientMessageId, ctx.agent, userClient)
|
||||
rememberNativeChatPrompt(
|
||||
ctx.journal,
|
||||
input.clientMessageId,
|
||||
ctx.agent,
|
||||
userClient,
|
||||
telemetryOwnerKey
|
||||
)
|
||||
}
|
||||
await ctx.journal.appendSubmission(
|
||||
{ ...input, options: captured.options, fence: ctx.fence, handoverRecorded: true },
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
import { expect, it } from 'vitest'
|
||||
import {
|
||||
STRUCTURED_CHAT_PROMPT_TELEMETRY_CLIENT_CAPABILITY,
|
||||
STRUCTURED_CHAT_START_TELEMETRY_CLIENT_CAPABILITY
|
||||
} from '../../../../../shared/structured-chat-dispatch-telemetry'
|
||||
import { recordStructuredChatPromptAccepted } from '../../../../telemetry/structured-chat-dispatch'
|
||||
import type { OrcaRuntimeService } from '../../../orca-runtime'
|
||||
import { eraseRpcMethods, isStreamingMethod, type RpcContext } from '../../core'
|
||||
// Import the registry to avoid client-events' module-init cycle through ipc/ssh.
|
||||
import { ALL_RPC_METHODS } from '../rpc-method-registry'
|
||||
|
||||
const subscribe = eraseRpcMethods(ALL_RPC_METHODS)
|
||||
.filter(isStreamingMethod)
|
||||
.find((method) => method.name === 'runtime.clientEvents.subscribe')!
|
||||
|
||||
function open(overrides: Partial<RpcContext> = {}) {
|
||||
const events: unknown[] = []
|
||||
let cleanup = () => {}
|
||||
const fake = {
|
||||
onClientEvent: () => () => {},
|
||||
getClientSettings: () => ({}),
|
||||
registerSubscriptionCleanup: (_id: string, close: () => void) => {
|
||||
cleanup = close
|
||||
}
|
||||
}
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the stream reads only these runtime members.
|
||||
const runtime = fake as unknown as OrcaRuntimeService
|
||||
const done = subscribe.handler(
|
||||
{},
|
||||
{
|
||||
runtime,
|
||||
caller: { kind: 'paired-device', deviceId: 'desktop-a' },
|
||||
pairedDeviceId: 'desktop-a',
|
||||
clientKind: 'runtime',
|
||||
clientCapabilities: [STRUCTURED_CHAT_PROMPT_TELEMETRY_CLIENT_CAPABILITY],
|
||||
...overrides
|
||||
},
|
||||
(event) => {
|
||||
if (
|
||||
typeof event === 'object' &&
|
||||
event !== null &&
|
||||
'type' in event &&
|
||||
event.type === 'structuredChatPromptAccepted'
|
||||
) {
|
||||
events.push(event)
|
||||
}
|
||||
}
|
||||
)
|
||||
return { events, done, close: () => cleanup() }
|
||||
}
|
||||
|
||||
it('addresses live observations only to capable authenticated owning desktops, with no reconnect backfill', async () => {
|
||||
const owner = open()
|
||||
const other = open({
|
||||
caller: { kind: 'paired-device', deviceId: 'desktop-b' },
|
||||
pairedDeviceId: 'desktop-b'
|
||||
})
|
||||
const old = open({ clientCapabilities: [STRUCTURED_CHAT_START_TELEMETRY_CLIENT_CAPABILITY] })
|
||||
const phone = open({ clientKind: 'mobile' })
|
||||
const unproven = open({ caller: undefined, clientId: 'desktop-a' })
|
||||
const cli = open({ caller: { kind: 'local-cli' }, clientId: 'desktop-a' })
|
||||
const streams = [owner, other, old, phone, unproven, cli]
|
||||
let reconnected: ReturnType<typeof open> | undefined
|
||||
try {
|
||||
recordStructuredChatPromptAccepted(
|
||||
{ agent_kind: 'codex', client: 'mobile', runtime: 'unknown' },
|
||||
'desktop-a'
|
||||
)
|
||||
expect(owner.events).toEqual([
|
||||
{
|
||||
type: 'structuredChatPromptAccepted',
|
||||
id: expect.any(String),
|
||||
agent_kind: 'codex',
|
||||
client: 'mobile'
|
||||
}
|
||||
])
|
||||
expect([other.events, old.events, phone.events, unproven.events, cli.events]).toEqual([
|
||||
[],
|
||||
[],
|
||||
[],
|
||||
[],
|
||||
[]
|
||||
])
|
||||
|
||||
owner.close()
|
||||
await owner.done
|
||||
recordStructuredChatPromptAccepted(
|
||||
{ agent_kind: 'codex', client: 'desktop', runtime: 'unknown' },
|
||||
'desktop-a'
|
||||
)
|
||||
reconnected = open()
|
||||
expect(reconnected.events).toEqual([])
|
||||
recordStructuredChatPromptAccepted(
|
||||
{ agent_kind: 'claude-code', client: 'desktop', runtime: 'unknown' },
|
||||
'desktop-a'
|
||||
)
|
||||
expect(reconnected.events).toEqual([
|
||||
{
|
||||
type: 'structuredChatPromptAccepted',
|
||||
id: expect.any(String),
|
||||
agent_kind: 'claude-code',
|
||||
client: 'desktop'
|
||||
}
|
||||
])
|
||||
expect(owner.events).toHaveLength(1)
|
||||
} finally {
|
||||
for (const stream of [...streams, ...(reconnected ? [reconnected] : [])]) {
|
||||
stream.close()
|
||||
await stream.done
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -10,7 +10,10 @@ import {
|
||||
} from '../../../../../shared/rpc-contract/client-events-params'
|
||||
import { BROWSER_CLIENT_HOST_REQUEST_CAPABILITY } from '../../../../../shared/browser/client-hosting/browser-client-host-request-protocol'
|
||||
import { getBrowserHostRequestRegistry } from '../../../browser-host/leases/browser-host-request-registry'
|
||||
import { STRUCTURED_CHAT_START_TELEMETRY_CLIENT_CAPABILITY } from '../../../../../shared/structured-chat-dispatch-telemetry'
|
||||
import {
|
||||
STRUCTURED_CHAT_START_TELEMETRY_CLIENT_CAPABILITY,
|
||||
STRUCTURED_CHAT_PROMPT_TELEMETRY_CLIENT_CAPABILITY
|
||||
} from '../../../../../shared/structured-chat-dispatch-telemetry'
|
||||
import { subscribeStructuredChatTelemetry } from '../../../../telemetry/structured-chat-dispatch'
|
||||
import { structuredCallerFor } from '../structured-sessions/creation/structured-agent-session-gate'
|
||||
|
||||
@@ -38,18 +41,24 @@ export const CLIENT_EVENT_METHODS = [
|
||||
clientCapabilities?.includes(BROWSER_CLIENT_HOST_REQUEST_CAPABILITY)
|
||||
? getBrowserHostRequestRegistry(runtime).register({ pairedDeviceId, emit })
|
||||
: undefined
|
||||
const unsubscribeTelemetry = clientCapabilities?.includes(
|
||||
const readsStarts = clientCapabilities?.includes(
|
||||
STRUCTURED_CHAT_START_TELEMETRY_CLIENT_CAPABILITY
|
||||
)
|
||||
? subscribeStructuredChatTelemetry((event, callerKey) => {
|
||||
if (
|
||||
callerKey === structuredCallerFor(ctx).callerKey &&
|
||||
clientCapabilities?.includes(STRUCTURED_CHAT_START_TELEMETRY_CLIENT_CAPABILITY)
|
||||
) {
|
||||
emit(event)
|
||||
}
|
||||
})
|
||||
: undefined
|
||||
const readsPrompts =
|
||||
ctx.caller?.kind === 'paired-device' &&
|
||||
clientKind === 'runtime' &&
|
||||
clientCapabilities?.includes(STRUCTURED_CHAT_PROMPT_TELEMETRY_CLIENT_CAPABILITY)
|
||||
const unsubscribeTelemetry =
|
||||
readsStarts || readsPrompts
|
||||
? subscribeStructuredChatTelemetry((event, callerKey) => {
|
||||
if (
|
||||
callerKey === structuredCallerFor(ctx).callerKey &&
|
||||
(event.type === 'structuredChatPromptAccepted' ? readsPrompts : readsStarts)
|
||||
) {
|
||||
emit(event)
|
||||
}
|
||||
})
|
||||
: undefined
|
||||
|
||||
const seq = ++clientEventSubscriptionSeq
|
||||
const subscriptionId = `runtime-client-events-${connectionId ?? 'inproc'}-${seq}`
|
||||
|
||||
+8
@@ -132,6 +132,14 @@ export async function requireRenameStructuredHost(
|
||||
export function structuredCallerFor(ctx: RpcContext): StructuredAgentSessionCaller {
|
||||
return {
|
||||
clientKind: ctx.clientKind,
|
||||
telemetryOwnerKey:
|
||||
ctx.caller?.kind === 'paired-device'
|
||||
? ctx.clientKind === 'runtime'
|
||||
? ctx.caller.deviceId
|
||||
: ctx.clientKind === 'mobile'
|
||||
? ctx.pairedDeviceParentId
|
||||
: undefined
|
||||
: undefined,
|
||||
userClient:
|
||||
ctx.caller?.kind === 'desktop'
|
||||
? 'desktop'
|
||||
|
||||
@@ -3,11 +3,14 @@ import { isTuiAgent } from '../../shared/agent-launch/catalog/tui-agent-config'
|
||||
import type { EventProps } from '../../shared/telemetry-events'
|
||||
import type { AgentSessionJournal } from '../native-chat/agent-session-journal/journal-store'
|
||||
import type { JournalRow } from '../native-chat/agent-session-journal/durable-rows/journal-row-schema'
|
||||
import { track } from './client'
|
||||
import { recordStructuredChatPromptAccepted } from './structured-chat-dispatch'
|
||||
|
||||
type PromptJournal = Pick<AgentSessionJournal, 'submission'>
|
||||
type PromptSource = EventProps<'native_chat_prompt_submitted'>
|
||||
const pendingPrompts = new WeakMap<PromptJournal, Map<string, PromptSource>>()
|
||||
const pendingPrompts = new WeakMap<
|
||||
PromptJournal,
|
||||
Map<string, { source: PromptSource; ownerKey?: string }>
|
||||
>()
|
||||
const MAX_PENDING_PROMPTS_PER_JOURNAL = 256
|
||||
|
||||
/** Fresh user sends retain device provenance through queued-message conversion, without persisting telemetry state. */
|
||||
@@ -15,7 +18,8 @@ export function rememberNativeChatPrompt(
|
||||
journal: PromptJournal,
|
||||
clientMessageId: string,
|
||||
agent: string,
|
||||
userClient: 'desktop' | 'mobile' | undefined
|
||||
userClient: 'desktop' | 'mobile' | undefined,
|
||||
ownerKey?: string
|
||||
): void {
|
||||
if (!userClient) {
|
||||
return
|
||||
@@ -35,10 +39,13 @@ export function rememberNativeChatPrompt(
|
||||
}
|
||||
}
|
||||
pending.set(clientMessageId, {
|
||||
agent_kind: isTuiAgent(agent) ? tuiAgentToAgentKind(agent) : 'other',
|
||||
// The host cannot establish the desktop renderer's execution-target classification.
|
||||
runtime: 'unknown',
|
||||
client: userClient
|
||||
ownerKey,
|
||||
source: {
|
||||
agent_kind: isTuiAgent(agent) ? tuiAgentToAgentKind(agent) : 'other',
|
||||
// The host cannot establish the desktop renderer's execution-target classification.
|
||||
runtime: 'unknown',
|
||||
client: userClient
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -67,7 +74,7 @@ export function observeNativeChatPromptSubmissions(
|
||||
if (row.state === 'accepted' && source) {
|
||||
pending.delete(sourceId)
|
||||
try {
|
||||
track('native_chat_prompt_submitted', source)
|
||||
recordStructuredChatPromptAccepted(source.source, source.ownerKey)
|
||||
} catch (error) {
|
||||
console.warn('[telemetry] native chat prompt observation failed', error)
|
||||
}
|
||||
|
||||
@@ -3,21 +3,31 @@ import { randomUUID } from 'node:crypto'
|
||||
import { z } from 'zod'
|
||||
import type {
|
||||
StructuredChatStartTelemetryEvent,
|
||||
StructuredChatPromptTelemetryEvent,
|
||||
StructuredChatTelemetryEvent
|
||||
} from '../../shared/structured-chat-dispatch-telemetry'
|
||||
import { agentKindSchema } from '../../shared/telemetry-property-schemas'
|
||||
import { isTelemetryInitialized } from './client'
|
||||
import { isTelemetryInitialized, track } from './client'
|
||||
import type { EventProps } from '../../shared/telemetry-events'
|
||||
|
||||
type TelemetryListener = (event: StructuredChatTelemetryEvent, callerKey: string) => void
|
||||
const remoteListeners = new Set<TelemetryListener>()
|
||||
const receivedEvents = new Set<string>()
|
||||
const telemetryEventSchema = z.strictObject({
|
||||
type: z.literal('structuredChatAgentStart'),
|
||||
id: z.uuid(),
|
||||
agent_kind: agentKindSchema,
|
||||
launch_source: z.literal('unknown'),
|
||||
request_kind: z.enum(['new', 'resume'])
|
||||
})
|
||||
const telemetryEventSchema = z.discriminatedUnion('type', [
|
||||
z.strictObject({
|
||||
type: z.literal('structuredChatAgentStart'),
|
||||
id: z.uuid(),
|
||||
agent_kind: agentKindSchema,
|
||||
launch_source: z.literal('unknown'),
|
||||
request_kind: z.enum(['new', 'resume'])
|
||||
}),
|
||||
z.strictObject({
|
||||
type: z.literal('structuredChatPromptAccepted'),
|
||||
id: z.uuid(),
|
||||
agent_kind: agentKindSchema,
|
||||
client: z.enum(['desktop', 'mobile'])
|
||||
})
|
||||
])
|
||||
|
||||
/** Live, best-effort delivery; outcomes settled without the caller connected are not replayed. */
|
||||
export function subscribeStructuredChatTelemetry(listener: TelemetryListener): () => void {
|
||||
@@ -40,13 +50,39 @@ export function recordRemoteStructuredChatTelemetry(value: unknown): boolean {
|
||||
receivedEvents.delete(receivedEvents.values().next().value!)
|
||||
}
|
||||
try {
|
||||
recordAgentStartTelemetry(event, 'structured')
|
||||
if (event.type === 'structuredChatPromptAccepted') {
|
||||
track('native_chat_prompt_submitted', {
|
||||
agent_kind: event.agent_kind,
|
||||
runtime: 'remote',
|
||||
client: event.client
|
||||
})
|
||||
} else {
|
||||
recordAgentStartTelemetry(event, 'structured')
|
||||
}
|
||||
} catch {
|
||||
// A telemetry failure must not interrupt runtime event delivery.
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
/** Only the committed acceptance observer supplies these facts; remote hosts have no desktop consent. */
|
||||
export function recordStructuredChatPromptAccepted(
|
||||
source: EventProps<'native_chat_prompt_submitted'>,
|
||||
ownerKey: string | undefined
|
||||
): void {
|
||||
if (isTelemetryInitialized()) {
|
||||
track('native_chat_prompt_submitted', source)
|
||||
} else if (ownerKey) {
|
||||
const event: StructuredChatPromptTelemetryEvent = {
|
||||
type: 'structuredChatPromptAccepted',
|
||||
id: randomUUID(),
|
||||
agent_kind: source.agent_kind,
|
||||
client: source.client
|
||||
}
|
||||
relayStructuredChatTelemetry(event, ownerKey)
|
||||
}
|
||||
}
|
||||
|
||||
export function recordStructuredChatAgentStart(
|
||||
telemetry: Pick<
|
||||
StructuredChatStartTelemetryEvent,
|
||||
|
||||
@@ -3,7 +3,10 @@ import { GIT_GRAPH_TAB_CAPABILITY } from './source-control/runtime/git-graph-cap
|
||||
import { STRUCTURED_AGENT_SESSION_RENDERER_CLIENT_CAPABILITIES } from './agent-session/structured-agent-session-renderer-client-capabilities'
|
||||
import { TERMINAL_IMAGE_VIEWER_CAPABILITY } from './terminal/images/terminal-image-viewer-capability'
|
||||
import { BROWSER_CLIENT_HOST_REQUEST_CAPABILITY } from './browser/client-hosting/browser-client-host-request-protocol'
|
||||
import { STRUCTURED_CHAT_START_TELEMETRY_CLIENT_CAPABILITY } from './structured-chat-dispatch-telemetry'
|
||||
import {
|
||||
STRUCTURED_CHAT_START_TELEMETRY_CLIENT_CAPABILITY,
|
||||
STRUCTURED_CHAT_PROMPT_TELEMETRY_CLIENT_CAPABILITY
|
||||
} from './structured-chat-dispatch-telemetry'
|
||||
import {
|
||||
AGENT_STATUS_RESTORED_UNCONFIRMED_RUNTIME_CAPABILITY,
|
||||
BROWSER_CLIENT_HOST_RUNTIME_CAPABILITY,
|
||||
@@ -20,6 +23,7 @@ export const ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES = [
|
||||
GIT_GRAPH_TAB_CAPABILITY,
|
||||
GIT_REVERT_CONFLICT_CLIENT_CAPABILITY,
|
||||
STRUCTURED_CHAT_START_TELEMETRY_CLIENT_CAPABILITY,
|
||||
STRUCTURED_CHAT_PROMPT_TELEMETRY_CLIENT_CAPABILITY,
|
||||
...NATIVE_REMOTE_RUNTIME_CLIENT_CAPABILITIES,
|
||||
TERMINAL_IMAGE_VIEWER_CAPABILITY,
|
||||
// The renderer that reads a paired host's structured chats is the one that reads its own.
|
||||
|
||||
@@ -3,6 +3,9 @@ import type { AgentKind } from './telemetry-events'
|
||||
export const STRUCTURED_CHAT_START_TELEMETRY_CLIENT_CAPABILITY =
|
||||
'structured-chat-start-telemetry-v1'
|
||||
|
||||
export const STRUCTURED_CHAT_PROMPT_TELEMETRY_CLIENT_CAPABILITY =
|
||||
'structured-chat-prompt-telemetry-v1'
|
||||
|
||||
export type StructuredChatStartTelemetryEvent = {
|
||||
type: 'structuredChatAgentStart'
|
||||
id: string
|
||||
@@ -11,4 +14,13 @@ export type StructuredChatStartTelemetryEvent = {
|
||||
request_kind: 'new' | 'resume'
|
||||
}
|
||||
|
||||
export type StructuredChatTelemetryEvent = StructuredChatStartTelemetryEvent
|
||||
export type StructuredChatPromptTelemetryEvent = {
|
||||
type: 'structuredChatPromptAccepted'
|
||||
id: string
|
||||
agent_kind: AgentKind
|
||||
client: 'desktop' | 'mobile'
|
||||
}
|
||||
|
||||
export type StructuredChatTelemetryEvent =
|
||||
| StructuredChatStartTelemetryEvent
|
||||
| StructuredChatPromptTelemetryEvent
|
||||
|
||||
Reference in New Issue
Block a user