test: require native IME engagement proof for the digit case

This commit is contained in:
Neil
2026-09-05 22:02:39 -07:00
parent 8a9c6e133c
commit 397f29554e
4 changed files with 32 additions and 14 deletions
+6 -1
View File
@@ -677,8 +677,13 @@ describe('PR E2E gate contract', () => {
// Why pin the titles: the runner requires one receipt per name, so a rename that nobody
// mirrored here would fail the lane loudly instead of quietly halving it.
const nativeDigitSpec = readFileSync(
join(projectDir, 'tests/e2e/terminal-hangul-terminating-digit-native.spec.ts'),
'utf8'
)
expect(nativeDigitSpec).toContain('appendImeEngagementReceipt(testInfo.title, trace)')
for (const title of EXPECTED_NATIVE_IME_TESTS) {
expect(nativeImeSpec, title).toContain(title)
expect(nativeImeSpec + nativeDigitSpec, title).toContain(title)
}
})
@@ -13,7 +13,8 @@ export const IME_ENGAGEMENT_RECEIPT_ENV = 'ORCA_E2E_IME_ENGAGEMENT_RECEIPT'
/** The tests that must each leave a receipt. Pinned so deleting one cannot quietly shrink the lane. */
export const EXPECTED_NATIVE_IME_TESTS = [
'forwards the issue exact-byte sequence without loss or duplication',
'forwards the issue sentence stress sequence without leaked ASCII'
'forwards the issue sentence stress sequence without leaked ASCII',
'a digit typed right after a Hangul syllable reaches the pty'
]
function parseReceipts(text) {
@@ -4,7 +4,7 @@ import {
verifyImeEngagementReceipts
} from './terminal-ime-engagement-receipt.mjs'
const [firstTest, secondTest] = EXPECTED_NATIVE_IME_TESTS
const [firstTest, secondTest, thirdTest] = EXPECTED_NATIVE_IME_TESTS
function receipt(test, overrides = {}) {
return JSON.stringify({
@@ -18,9 +18,11 @@ function receipt(test, overrides = {}) {
describe('verifyImeEngagementReceipts', () => {
it('accepts a run where every expected test observed real composition', () => {
expect(verifyImeEngagementReceipts(`${receipt(firstTest)}\n${receipt(secondTest)}\n`)).toEqual(
[]
)
expect(
verifyImeEngagementReceipts(
`${receipt(firstTest)}\n${receipt(secondTest)}\n${receipt(thirdTest)}\n`
)
).toEqual([])
})
// The failure this whole mechanism exists for: Playwright reports a skipped test as a pass, so
@@ -35,13 +37,20 @@ describe('verifyImeEngagementReceipts', () => {
it('rejects a partial run where only one test reached the engine', () => {
expect(verifyImeEngagementReceipts(`${receipt(firstTest)}\n`)).toEqual([
`no engagement receipt for "${secondTest}" — it was skipped, filtered out, or renamed`
`no engagement receipt for "${secondTest}" — it was skipped, filtered out, or renamed`,
`no engagement receipt for "${thirdTest}" — it was skipped, filtered out, or renamed`
])
})
it('requires the digit receipt even when both original native tests passed', () => {
expect(verifyImeEngagementReceipts(`${receipt(firstTest)}\n${receipt(secondTest)}\n`)).toEqual([
`no engagement receipt for "${thirdTest}" — it was skipped, filtered out, or renamed`
])
})
it('rejects a run that typed keys but never opened a composition', () => {
const problems = verifyImeEngagementReceipts(
`${receipt(firstTest, { compositionStart: 0 })}\n${receipt(secondTest)}\n`
`${receipt(firstTest, { compositionStart: 0 })}\n${receipt(secondTest)}\n${receipt(thirdTest)}\n`
)
expect(problems).toEqual([
`"${firstTest}" recorded no compositionstart — the IME never engaged`
@@ -50,7 +59,7 @@ describe('verifyImeEngagementReceipts', () => {
it('rejects a composition that produced no Hangul, which a latin passthrough would satisfy', () => {
const problems = verifyImeEngagementReceipts(
`${receipt(firstTest, { hangulComposition: 0 })}\n${receipt(secondTest)}\n`
`${receipt(firstTest, { hangulComposition: 0 })}\n${receipt(secondTest)}\n${receipt(thirdTest)}\n`
)
expect(problems).toEqual([
`"${firstTest}" recorded no Hangul composition data — the engine produced no syllables`
@@ -59,7 +68,7 @@ describe('verifyImeEngagementReceipts', () => {
it('rejects a renamed test rather than counting it toward coverage', () => {
const problems = verifyImeEngagementReceipts(
`${receipt(firstTest)}\n${receipt(secondTest)}\n${receipt('some new scenario')}\n`
`${receipt(firstTest)}\n${receipt(secondTest)}\n${receipt(thirdTest)}\n${receipt('some new scenario')}\n`
)
expect(problems).toEqual([
'unexpected engagement receipt for "some new scenario" — update EXPECTED_NATIVE_IME_TESTS'
@@ -68,7 +77,7 @@ describe('verifyImeEngagementReceipts', () => {
it('reports a truncated receipt rather than parsing around it', () => {
const problems = verifyImeEngagementReceipts(
`${receipt(firstTest)}\n{"test":"trunc\n${receipt(secondTest)}\n`
`${receipt(firstTest)}\n{"test":"trunc\n${receipt(secondTest)}\n${receipt(thirdTest)}\n`
)
expect(problems).toEqual(['malformed receipt line: {"test":"trunc'])
})
@@ -43,6 +43,7 @@ import { mkdirSync, writeFileSync } from 'node:fs'
import path from 'node:path'
import type { Page, TestInfo } from '@stablyai/playwright-test'
import { test, expect } from './helpers/orca-app'
import { appendImeEngagementReceipt } from './terminal-ime-engagement-receipt'
import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store'
import {
focusActiveTerminalInput,
@@ -230,6 +231,11 @@ test.describe('Hangul terminating digit @headful', () => {
}
receivedBytes = await waitForTerminalImeBytes(page, reader, 20_000)
expect(receivedBytes.map((hex) => Buffer.from(hex, 'hex').toString('utf8'))).toEqual(
Array.from({ length: REPETITIONS }, () => `${EXPECTED_LINE}\n`)
)
const trace = await readTerminalImeBoundaryTrace(page)
appendImeEngagementReceipt(testInfo.title, trace)
} finally {
await writeEvidence(page, testInfo, 'hangul-terminating-digit', {
expectedHex,
@@ -241,8 +247,5 @@ test.describe('Hangul terminating digit @headful', () => {
await sendToTerminal(page, ptyId, '\x03').catch(() => undefined)
removeTerminalImeByteReader(reader)
}
expect(receivedBytes.map((hex) => Buffer.from(hex, 'hex').toString('utf8'))).toEqual(
Array.from({ length: REPETITIONS }, () => `${EXPECTED_LINE}\n`)
)
})
})