mirror of
https://github.com/stablyai/orca.git
synced 2026-09-30 00:03:15 +00:00
fix(native-chat): stop the name normalizer stripping legitimate joiners
`\p{Cf}` swept up U+200C and U+200D, which are not hostile formatting: they
hold multi-part emoji together and are orthographic in Persian and Hindi.
"Fix 👨👩👧 layout" came out as three separate people, and "میخواهم" lost its
ZWNJ — while the naming prompt asks the model to write in the user's own
language. Name the bidi controls and zero-width marks the comment already
claimed to target instead, so the hardening stays and the joiners survive.
The test that looked like it covered this asserted on "Résumé du fil ☕",
characters in categories the regex never touched.
This commit is contained in:
@@ -75,3 +75,38 @@ describe('normalizeAgentSessionConversationName hostile text', () => {
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('normalizeAgentSessionConversationName joiners', () => {
|
||||
// U+200C/U+200D carry meaning: stripping them as "format characters" splits a
|
||||
// family emoji into three people and breaks Persian and Hindi orthography.
|
||||
// The naming prompt asks for the user's own language, so this is normal input.
|
||||
const ZWJ = '\u200D'
|
||||
const ZWNJ = '\u200C'
|
||||
|
||||
it.each([
|
||||
['family emoji', `Fix \u{1F468}${ZWJ}\u{1F469}${ZWJ}\u{1F467} layout`],
|
||||
['flag emoji', `Ship \u{1F3F3}\uFE0F${ZWJ}\u{1F308} theme`],
|
||||
['profession emoji', `Add \u{1F469}${ZWJ}\u{1F4BB} avatar`],
|
||||
['Persian ZWNJ', `می${ZWNJ}خواهم تست`],
|
||||
['Hindi ZWNJ conjunct', `क्${ZWNJ}ष ठीक`]
|
||||
])('keeps the joiners in a %s name', (_label, name) => {
|
||||
expect(normalizeAgentSessionConversationName(name)).toBe(name)
|
||||
})
|
||||
|
||||
// Kept from the hardening: allowing the joiners must not readmit these.
|
||||
it.each([
|
||||
['bidi override', 'Fix\u202Egnp.exe probe', 'Fix gnp.exe probe'],
|
||||
['isolate pair', 'Fix\u2066the\u2069 probe', 'Fix the probe'],
|
||||
['Arabic letter mark', 'Fix\u061Cthe probe', 'Fix the probe'],
|
||||
['soft hyphen', 'Fix\u00ADthe probe', 'Fix the probe'],
|
||||
['word joiner', 'Fix\u2060the probe', 'Fix the probe'],
|
||||
['zero-width space', 'Fix\u200Bthe probe', 'Fix the probe'],
|
||||
['byte order mark', 'Fix\uFEFFthe probe', 'Fix the probe']
|
||||
])('still strips a %s', (_label, name, expected) => {
|
||||
expect(normalizeAgentSessionConversationName(name)).toBe(expected)
|
||||
})
|
||||
|
||||
it('rejects a name that is only invisible controls', () => {
|
||||
expect(normalizeAgentSessionConversationName('\u202E\u200B\u2060')).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -10,10 +10,13 @@ import { sliceAtCodeUnitLimit } from './surrogate-safe-text-slice'
|
||||
/** Well past any provider's own cap, short enough that a pasted essay cannot become a tab label. */
|
||||
export const AGENT_SESSION_CONVERSATION_NAME_MAX_LENGTH = 200
|
||||
|
||||
/** Whitespace, plus the C0/C1 controls and format characters `\s` misses. A
|
||||
* bidi override renders a label that reads as text the name does not contain,
|
||||
* and a zero-width run renders as nothing at all. */
|
||||
const UNRENDERABLE_RUN = /[\s\p{Cc}\p{Cf}\p{Zl}\p{Zp}]+/gu
|
||||
/** Whitespace, plus the C0/C1 controls, bidi controls and zero-width marks `\s`
|
||||
* misses. A bidi override renders a label that reads as text the name does not
|
||||
* contain, and a zero-width run renders as nothing at all. Named rather than
|
||||
* taken as all of `\p{Cf}`, which would also strip U+200C/U+200D — joiners that
|
||||
* are load-bearing in Persian, Hindi and every multi-part emoji. */
|
||||
const UNRENDERABLE_RUN =
|
||||
/[\s\p{Cc}\p{Zl}\p{Zp}\u00AD\u061C\u200B\u200E\u200F\u202A-\u202E\u2060\u2066-\u2069\uFEFF]+/gu
|
||||
|
||||
export function normalizeAgentSessionConversationName(value: unknown): string | null {
|
||||
if (typeof value !== 'string') {
|
||||
|
||||
Reference in New Issue
Block a user