fix(editor): preserve Unicode reconciliation seeds

This commit is contained in:
Brennan Benson
2026-07-20 19:03:09 -07:00
parent 905e8c1903
commit 4cb093c4f7
2 changed files with 51 additions and 10 deletions
@@ -348,7 +348,7 @@ describe('reconcileSerializedMarkdown', () => {
// previously threw; every such edit must reconcile and stay render-equal to `edited`.
const lines: string[] = []
for (let i = 0; i < 20; i++) {
lines.push(`_强调${i}_ 这是一段包含很多中文字符的文本内容用来测试多字节偏移问题`)
lines.push(`_强调${i}_ 😀 café 这是一段包含很多中文字符的文本内容用来测试多字节偏移问题`)
}
const originalSource = `# 标题\n\n${lines.join('\n')}\n`
const baseCanonical = fakeCanonicalize(originalSource)
@@ -356,11 +356,13 @@ describe('reconcileSerializedMarkdown', () => {
// Sweep insert positions so we exercise the offsets that used to overshoot the byte target.
const chars = [...baseCanonical]
for (let pos = 5; pos < chars.length - 5; pos += 3) {
const edited = `${chars.slice(0, pos).join('')}插${chars.slice(pos).join('')}`
const inserted = ['插', '😀', 'é'][Math.floor(pos / 3) % 3]
const edited = `${chars.slice(0, pos).join('')}${inserted}${chars.slice(pos).join('')}`
const reconciled = reconcileWithFake(originalSource, edited)
// Never throws, and the reconciled bytes render exactly to `edited`.
// Never throws or falls back to a whole-document canonical rewrite.
expect(reconciled).not.toBe(edited)
expect(fakeCanonicalize(reconciled).trimEnd()).toBe(edited.trimEnd())
}
})
@@ -80,13 +80,16 @@ export function reconcileSerializedMarkdown({
diffs = cleanupEfficiency(diffs)
}
const patches = makePatches(baseLf, diffs)
// Why: dmp's adjustIndiciesToUcs2 treats patch.start1 (a char index) as a UTF-8 byte offset, so on multi-byte
// text (Chinese/emoji/accented) its byte counter overshoots the target and throws "Failed to determine byte
// offset" (#9492). allowExceedingIndices returns the nearest index instead; the fuzzy match relocates the hunk
// and branch 6's round-trip proof rejects any misplacement, so the seed offset need not be exact.
const [reconciledLf, results] = applyPatches(patches, originalSourceLf, {
allowExceedingIndices: true
})
// Why: applyPatches decodes starts as UTF-8 offsets even though makePatches returns UTF-16 indices; encode against the divergent text being patched so decoding preserves the fuzzy-match seed.
const utf8Offsets = getUtf8OffsetsAtCodeUnitIndices(
originalSourceLf,
patches.flatMap((patch) => [patch.start1, patch.start2])
)
for (const patch of patches) {
patch.start1 = utf8Offsets.get(patch.start1) ?? 0
patch.start2 = utf8Offsets.get(patch.start2) ?? 0
}
const [reconciledLf, results] = applyPatches(patches, originalSourceLf)
// Branch 5: a hunk failed to locate in the non-canonical source → unreliable fuzzy match, fall back to canonical.
if (results.some((applied) => !applied)) {
@@ -128,6 +131,42 @@ function normalizeForSafety(text: string): string {
return text.replace(/\r\n/g, '\n')
}
function getUtf8OffsetsAtCodeUnitIndices(
text: string,
codeUnitIndices: number[]
): Map<number, number> {
const targets = [...new Set(codeUnitIndices)].sort((a, b) => a - b)
const offsets = new Map<number, number>()
let codeUnitIndex = 0
let byteOffset = 0
for (const target of targets) {
const boundedTarget = Math.max(0, Math.min(target, text.length))
while (codeUnitIndex < boundedTarget) {
const codePoint = text.codePointAt(codeUnitIndex)
if (codePoint === undefined) {
break
}
byteOffset += utf8CodePointLength(codePoint)
codeUnitIndex += codePoint > 0xffff ? 2 : 1
}
offsets.set(target, byteOffset)
}
return offsets
}
function utf8CodePointLength(codePoint: number): number {
if (codePoint <= 0x7f) {
return 1
}
if (codePoint <= 0x7ff) {
return 2
}
if (codePoint <= 0xffff) {
return 3
}
return 4
}
function hasRepeatedHalfMatchSeed(textA: string, textB: string): boolean {
// Match diff-match-patch's prefix/suffix trimming so a small edit isn't mistaken for a repetitive replacement.
const minimumLength = Math.min(textA.length, textB.length)