mirror of
https://github.com/stablyai/orca.git
synced 2026-10-18 08:03:22 +00:00
fix(editor): preserve Unicode reconciliation seeds
This commit is contained in:
@@ -348,7 +348,7 @@ describe('reconcileSerializedMarkdown', () => {
|
||||
// previously threw; every such edit must reconcile and stay render-equal to `edited`.
|
||||
const lines: string[] = []
|
||||
for (let i = 0; i < 20; i++) {
|
||||
lines.push(`_强调${i}_ 这是一段包含很多中文字符的文本内容用来测试多字节偏移问题`)
|
||||
lines.push(`_强调${i}_ 😀 café 这是一段包含很多中文字符的文本内容用来测试多字节偏移问题`)
|
||||
}
|
||||
const originalSource = `# 标题\n\n${lines.join('\n')}\n`
|
||||
const baseCanonical = fakeCanonicalize(originalSource)
|
||||
@@ -356,11 +356,13 @@ describe('reconcileSerializedMarkdown', () => {
|
||||
// Sweep insert positions so we exercise the offsets that used to overshoot the byte target.
|
||||
const chars = [...baseCanonical]
|
||||
for (let pos = 5; pos < chars.length - 5; pos += 3) {
|
||||
const edited = `${chars.slice(0, pos).join('')}插${chars.slice(pos).join('')}`
|
||||
const inserted = ['插', '😀', 'é'][Math.floor(pos / 3) % 3]
|
||||
const edited = `${chars.slice(0, pos).join('')}${inserted}${chars.slice(pos).join('')}`
|
||||
|
||||
const reconciled = reconcileWithFake(originalSource, edited)
|
||||
|
||||
// Never throws, and the reconciled bytes render exactly to `edited`.
|
||||
// Never throws or falls back to a whole-document canonical rewrite.
|
||||
expect(reconciled).not.toBe(edited)
|
||||
expect(fakeCanonicalize(reconciled).trimEnd()).toBe(edited.trimEnd())
|
||||
}
|
||||
})
|
||||
|
||||
@@ -80,13 +80,16 @@ export function reconcileSerializedMarkdown({
|
||||
diffs = cleanupEfficiency(diffs)
|
||||
}
|
||||
const patches = makePatches(baseLf, diffs)
|
||||
// Why: dmp's adjustIndiciesToUcs2 treats patch.start1 (a char index) as a UTF-8 byte offset, so on multi-byte
|
||||
// text (Chinese/emoji/accented) its byte counter overshoots the target and throws "Failed to determine byte
|
||||
// offset" (#9492). allowExceedingIndices returns the nearest index instead; the fuzzy match relocates the hunk
|
||||
// and branch 6's round-trip proof rejects any misplacement, so the seed offset need not be exact.
|
||||
const [reconciledLf, results] = applyPatches(patches, originalSourceLf, {
|
||||
allowExceedingIndices: true
|
||||
})
|
||||
// Why: applyPatches decodes starts as UTF-8 offsets even though makePatches returns UTF-16 indices; encode against the divergent text being patched so decoding preserves the fuzzy-match seed.
|
||||
const utf8Offsets = getUtf8OffsetsAtCodeUnitIndices(
|
||||
originalSourceLf,
|
||||
patches.flatMap((patch) => [patch.start1, patch.start2])
|
||||
)
|
||||
for (const patch of patches) {
|
||||
patch.start1 = utf8Offsets.get(patch.start1) ?? 0
|
||||
patch.start2 = utf8Offsets.get(patch.start2) ?? 0
|
||||
}
|
||||
const [reconciledLf, results] = applyPatches(patches, originalSourceLf)
|
||||
|
||||
// Branch 5: a hunk failed to locate in the non-canonical source → unreliable fuzzy match, fall back to canonical.
|
||||
if (results.some((applied) => !applied)) {
|
||||
@@ -128,6 +131,42 @@ function normalizeForSafety(text: string): string {
|
||||
return text.replace(/\r\n/g, '\n')
|
||||
}
|
||||
|
||||
function getUtf8OffsetsAtCodeUnitIndices(
|
||||
text: string,
|
||||
codeUnitIndices: number[]
|
||||
): Map<number, number> {
|
||||
const targets = [...new Set(codeUnitIndices)].sort((a, b) => a - b)
|
||||
const offsets = new Map<number, number>()
|
||||
let codeUnitIndex = 0
|
||||
let byteOffset = 0
|
||||
for (const target of targets) {
|
||||
const boundedTarget = Math.max(0, Math.min(target, text.length))
|
||||
while (codeUnitIndex < boundedTarget) {
|
||||
const codePoint = text.codePointAt(codeUnitIndex)
|
||||
if (codePoint === undefined) {
|
||||
break
|
||||
}
|
||||
byteOffset += utf8CodePointLength(codePoint)
|
||||
codeUnitIndex += codePoint > 0xffff ? 2 : 1
|
||||
}
|
||||
offsets.set(target, byteOffset)
|
||||
}
|
||||
return offsets
|
||||
}
|
||||
|
||||
function utf8CodePointLength(codePoint: number): number {
|
||||
if (codePoint <= 0x7f) {
|
||||
return 1
|
||||
}
|
||||
if (codePoint <= 0x7ff) {
|
||||
return 2
|
||||
}
|
||||
if (codePoint <= 0xffff) {
|
||||
return 3
|
||||
}
|
||||
return 4
|
||||
}
|
||||
|
||||
function hasRepeatedHalfMatchSeed(textA: string, textB: string): boolean {
|
||||
// Match diff-match-patch's prefix/suffix trimming so a small edit isn't mistaken for a repetitive replacement.
|
||||
const minimumLength = Math.min(textA.length, textB.length)
|
||||
|
||||
Reference in New Issue
Block a user