mirror of
https://github.com/stablyai/orca.git
synced 2026-09-21 16:02:20 +00:00
test: cover native X11 Hangul-plus-digit PTY bytes in CI (#19013)
* test: run the native Hangul terminating-digit regression in CI * test: distinguish X11 byte coverage from the manual Wayland repro * test: require native IME engagement proof for the digit case
This commit is contained in:
@@ -636,13 +636,8 @@ describe('PR E2E gate contract', () => {
|
||||
.filter((spec) => nativeGateExpression.test(readFileSync(join(projectDir, spec), 'utf8')))
|
||||
expect(nativeGatedSpecs.length).toBeGreaterThan(0)
|
||||
|
||||
// Why exempt: the digit repro needs a nested gnome-shell, which no hosted runner provides
|
||||
// (headless mutter never answers RemoteDesktop.CreateSession); the macOS spec needs a real
|
||||
// macOS input source, and no macOS runner exists on any PR or scheduled lane.
|
||||
const unreachableSpecs = new Set([
|
||||
'tests/e2e/terminal-hangul-terminating-digit-native.spec.ts',
|
||||
'tests/e2e/terminal-macos-2set-korean-native.spec.ts'
|
||||
])
|
||||
// The macOS spec needs a native input source; PR and scheduled IME lanes use Linux.
|
||||
const unreachableSpecs = new Set(['tests/e2e/terminal-macos-2set-korean-native.spec.ts'])
|
||||
const unclaimed = nativeGatedSpecs.filter(
|
||||
(spec) => !unreachableSpecs.has(spec) && !nativeImeRunner.includes(spec)
|
||||
)
|
||||
@@ -682,8 +677,13 @@ describe('PR E2E gate contract', () => {
|
||||
|
||||
// Why pin the titles: the runner requires one receipt per name, so a rename that nobody
|
||||
// mirrored here would fail the lane loudly instead of quietly halving it.
|
||||
const nativeDigitSpec = readFileSync(
|
||||
join(projectDir, 'tests/e2e/terminal-hangul-terminating-digit-native.spec.ts'),
|
||||
'utf8'
|
||||
)
|
||||
expect(nativeDigitSpec).toContain('appendImeEngagementReceipt(testInfo.title, trace)')
|
||||
for (const title of EXPECTED_NATIVE_IME_TESTS) {
|
||||
expect(nativeImeSpec, title).toContain(title)
|
||||
expect(nativeImeSpec + nativeDigitSpec, title).toContain(title)
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@@ -199,7 +199,8 @@ async function runInsideSession(evidenceDir) {
|
||||
'test:e2e:headful',
|
||||
'--workers=1',
|
||||
'--',
|
||||
'tests/e2e/terminal-ibus-hangul-native.spec.ts'
|
||||
'tests/e2e/terminal-ibus-hangul-native.spec.ts',
|
||||
'tests/e2e/terminal-hangul-terminating-digit-native.spec.ts'
|
||||
],
|
||||
{
|
||||
cwd: projectDir,
|
||||
|
||||
@@ -13,7 +13,8 @@ export const IME_ENGAGEMENT_RECEIPT_ENV = 'ORCA_E2E_IME_ENGAGEMENT_RECEIPT'
|
||||
/** The tests that must each leave a receipt. Pinned so deleting one cannot quietly shrink the lane. */
|
||||
export const EXPECTED_NATIVE_IME_TESTS = [
|
||||
'forwards the issue exact-byte sequence without loss or duplication',
|
||||
'forwards the issue sentence stress sequence without leaked ASCII'
|
||||
'forwards the issue sentence stress sequence without leaked ASCII',
|
||||
'a digit typed right after a Hangul syllable reaches the pty'
|
||||
]
|
||||
|
||||
function parseReceipts(text) {
|
||||
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
verifyImeEngagementReceipts
|
||||
} from './terminal-ime-engagement-receipt.mjs'
|
||||
|
||||
const [firstTest, secondTest] = EXPECTED_NATIVE_IME_TESTS
|
||||
const [firstTest, secondTest, thirdTest] = EXPECTED_NATIVE_IME_TESTS
|
||||
|
||||
function receipt(test, overrides = {}) {
|
||||
return JSON.stringify({
|
||||
@@ -18,9 +18,11 @@ function receipt(test, overrides = {}) {
|
||||
|
||||
describe('verifyImeEngagementReceipts', () => {
|
||||
it('accepts a run where every expected test observed real composition', () => {
|
||||
expect(verifyImeEngagementReceipts(`${receipt(firstTest)}\n${receipt(secondTest)}\n`)).toEqual(
|
||||
[]
|
||||
)
|
||||
expect(
|
||||
verifyImeEngagementReceipts(
|
||||
`${receipt(firstTest)}\n${receipt(secondTest)}\n${receipt(thirdTest)}\n`
|
||||
)
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
// The failure this whole mechanism exists for: Playwright reports a skipped test as a pass, so
|
||||
@@ -35,13 +37,20 @@ describe('verifyImeEngagementReceipts', () => {
|
||||
|
||||
it('rejects a partial run where only one test reached the engine', () => {
|
||||
expect(verifyImeEngagementReceipts(`${receipt(firstTest)}\n`)).toEqual([
|
||||
`no engagement receipt for "${secondTest}" — it was skipped, filtered out, or renamed`
|
||||
`no engagement receipt for "${secondTest}" — it was skipped, filtered out, or renamed`,
|
||||
`no engagement receipt for "${thirdTest}" — it was skipped, filtered out, or renamed`
|
||||
])
|
||||
})
|
||||
|
||||
it('requires the digit receipt even when both original native tests passed', () => {
|
||||
expect(verifyImeEngagementReceipts(`${receipt(firstTest)}\n${receipt(secondTest)}\n`)).toEqual([
|
||||
`no engagement receipt for "${thirdTest}" — it was skipped, filtered out, or renamed`
|
||||
])
|
||||
})
|
||||
|
||||
it('rejects a run that typed keys but never opened a composition', () => {
|
||||
const problems = verifyImeEngagementReceipts(
|
||||
`${receipt(firstTest, { compositionStart: 0 })}\n${receipt(secondTest)}\n`
|
||||
`${receipt(firstTest, { compositionStart: 0 })}\n${receipt(secondTest)}\n${receipt(thirdTest)}\n`
|
||||
)
|
||||
expect(problems).toEqual([
|
||||
`"${firstTest}" recorded no compositionstart — the IME never engaged`
|
||||
@@ -50,7 +59,7 @@ describe('verifyImeEngagementReceipts', () => {
|
||||
|
||||
it('rejects a composition that produced no Hangul, which a latin passthrough would satisfy', () => {
|
||||
const problems = verifyImeEngagementReceipts(
|
||||
`${receipt(firstTest, { hangulComposition: 0 })}\n${receipt(secondTest)}\n`
|
||||
`${receipt(firstTest, { hangulComposition: 0 })}\n${receipt(secondTest)}\n${receipt(thirdTest)}\n`
|
||||
)
|
||||
expect(problems).toEqual([
|
||||
`"${firstTest}" recorded no Hangul composition data — the engine produced no syllables`
|
||||
@@ -59,7 +68,7 @@ describe('verifyImeEngagementReceipts', () => {
|
||||
|
||||
it('rejects a renamed test rather than counting it toward coverage', () => {
|
||||
const problems = verifyImeEngagementReceipts(
|
||||
`${receipt(firstTest)}\n${receipt(secondTest)}\n${receipt('some new scenario')}\n`
|
||||
`${receipt(firstTest)}\n${receipt(secondTest)}\n${receipt(thirdTest)}\n${receipt('some new scenario')}\n`
|
||||
)
|
||||
expect(problems).toEqual([
|
||||
'unexpected engagement receipt for "some new scenario" — update EXPECTED_NATIVE_IME_TESTS'
|
||||
@@ -68,7 +77,7 @@ describe('verifyImeEngagementReceipts', () => {
|
||||
|
||||
it('reports a truncated receipt rather than parsing around it', () => {
|
||||
const problems = verifyImeEngagementReceipts(
|
||||
`${receipt(firstTest)}\n{"test":"trunc\n${receipt(secondTest)}\n`
|
||||
`${receipt(firstTest)}\n{"test":"trunc\n${receipt(secondTest)}\n${receipt(thirdTest)}\n`
|
||||
)
|
||||
expect(problems).toEqual(['malformed receipt line: {"test":"trunc'])
|
||||
})
|
||||
|
||||
@@ -3,20 +3,18 @@
|
||||
* the pty. Written to reproduce #15299, where a digit typed straight after a Hangul syllable was
|
||||
* dropped under Wayland but not under X11.
|
||||
*
|
||||
* THIS DOES NOT RUN IN CI. It is gated on ORCA_E2E_NATIVE_IBUS_HANGUL=1 and needs a compositor
|
||||
* session that CI does not have, so it is a manual reproduction harness rather than coverage.
|
||||
* That is stated plainly because this repo already carries native IME specs that are skipped
|
||||
* everywhere and were mistaken for coverage they never provided.
|
||||
* CI runs the default xdotool injector under X11, checking exact Hangul-plus-digit PTY bytes.
|
||||
* That path passed even before the Wayland fix; it does not prove #15299 is fixed.
|
||||
* Reproducing #15299 still requires the nested Wayland session below.
|
||||
*
|
||||
* To run it, on a machine with gnome-shell and ibus-hangul:
|
||||
* To run the Wayland reproduction on a machine with gnome-shell and ibus-hangul:
|
||||
*
|
||||
* Xvfb :65 -extension GLX &
|
||||
* DISPLAY=:65 gnome-shell --nested --wayland # nested, NOT --headless
|
||||
* ORCA_E2E_NATIVE_IBUS_HANGUL=1 ORCA_E2E_IME_INJECTOR=nested npx playwright test \
|
||||
* tests/e2e/terminal-hangul-terminating-digit-native.spec.ts
|
||||
*
|
||||
* Eight things that decide whether a run is real or a silent false negative, each of which cost a
|
||||
* failed attempt:
|
||||
* Nested Wayland prerequisites:
|
||||
*
|
||||
* - Nested, not headless. A headless mutter never answers RemoteDesktop.CreateSession, so there
|
||||
* is no way to inject input; nested makes the whole compositor an X window that xdotool can
|
||||
@@ -45,6 +43,7 @@ import { mkdirSync, writeFileSync } from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import type { Page, TestInfo } from '@stablyai/playwright-test'
|
||||
import { test, expect } from './helpers/orca-app'
|
||||
import { appendImeEngagementReceipt } from './terminal-ime-engagement-receipt'
|
||||
import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store'
|
||||
import {
|
||||
focusActiveTerminalInput,
|
||||
@@ -232,6 +231,11 @@ test.describe('Hangul terminating digit @headful', () => {
|
||||
}
|
||||
|
||||
receivedBytes = await waitForTerminalImeBytes(page, reader, 20_000)
|
||||
expect(receivedBytes.map((hex) => Buffer.from(hex, 'hex').toString('utf8'))).toEqual(
|
||||
Array.from({ length: REPETITIONS }, () => `${EXPECTED_LINE}\n`)
|
||||
)
|
||||
const trace = await readTerminalImeBoundaryTrace(page)
|
||||
appendImeEngagementReceipt(testInfo.title, trace)
|
||||
} finally {
|
||||
await writeEvidence(page, testInfo, 'hangul-terminating-digit', {
|
||||
expectedHex,
|
||||
@@ -243,8 +247,5 @@ test.describe('Hangul terminating digit @headful', () => {
|
||||
await sendToTerminal(page, ptyId, '\x03').catch(() => undefined)
|
||||
removeTerminalImeByteReader(reader)
|
||||
}
|
||||
expect(receivedBytes.map((hex) => Buffer.from(hex, 'hex').toString('utf8'))).toEqual(
|
||||
Array.from({ length: REPETITIONS }, () => `${EXPECTED_LINE}\n`)
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user