fix(editor): stop the rich markdown editor from mangling dollar-heavy files

Opening a markdown file full of dollar amounts in rich mode reformatted it
on load, and any save then rewrote the whole file:

- marked emits an `escape` inline token for `\$`, `\*`, `\_`, `\[`; Tiptap's
  markdown parser has no case for it, so the escaped character was deleted
  from the document as soon as the file opened. The marked facade now
  rewrites escape tokens to text tokens, in place, so table cells keep them
  too.
- the inline-math tokenizer treated any same-line `$...$` pair as LaTeX, so
  "from $10 to $20" or "entered 2021 at $0" became a KaTeX atom whose text
  was trimmed. It now requires Pandoc-style boundaries: both `$` must touch
  the formula and the closing `$` must not be followed by a digit.
- the source-preserving reconcile mis-applied any edit at the end of a file
  whose source ends with a newline: getMarkdown never emits one, so the
  end-of-document hunk landed one character late, failed the round-trip
  proof, and fell back to canonical output for the entire file, rewriting
  every `\$`, `&` and table in it. It now patches the newline-stripped
  bodies and re-attaches the source's trailing newline run.
This commit is contained in:
averydev
2026-09-18 04:43:48 -07:00
committed by Neil
parent 58890c59fb
commit 01d6567c02
6 changed files with 157 additions and 10 deletions
@@ -189,6 +189,27 @@ function roundTripMarkdown(content: string): string {
}
}
function countInlineMathNodes(content: string): number {
const codec = createRichMarkdownEditorCodec()
const editor = new Editor({
element: null,
extensions: createRichMarkdownExtensions({ codec }),
content: encodeRawMarkdownHtmlForRichEditor(content, codec),
contentType: 'markdown'
})
try {
let count = 0
editor.state.doc.descendants((node) => {
if (node.type.name === 'inlineMath') {
count += 1
}
})
return count
} finally {
editor.destroy()
}
}
function markdownAfterTextReplace(content: string, search: string, replacement: string): string {
const codec = createRichMarkdownEditorCodec()
const editor = new Editor({
@@ -603,6 +624,28 @@ describe('rich markdown round trip', () => {
expect(slashCommandSelectionParent(commandId)).toBe('detailsSummary')
})
it('keeps backslash-escaped characters instead of dropping them on load', () => {
// Why: marked emits `escape` tokens that Tiptap's parser otherwise discards, deleting the character.
expect(roundTripMarkdown('cost was \\$1,200, rate 5\\*, file\\_name, see \\[note\\].\n')).toBe(
'cost was $1,200, rate 5*, file_name, see [note].'
)
})
it('keeps escaped dollars inside table cells', () => {
expect(roundTripMarkdown('| Item | Amount |\n|---|---|\n| Fee | \\$500 |\n')).toContain('$500')
})
it('keeps dollar amounts as text instead of inline math', () => {
const content = 'from $10 to $20, then (deficit $509,542 by end-2020) entered 2021 at $0'
expect(countInlineMathNodes(content)).toBe(0)
expect(roundTripMarkdown(`${content}\n`)).toBe(content)
})
it('still parses inline math that touches its dollar signs', () => {
expect(countInlineMathNodes('Energy is $E = mc^2$ here, and $x_1$ too.')).toBe(2)
expect(roundTripMarkdown('Energy is $E = mc^2$ here.\n')).toBe('Energy is $E = mc^2$ here.')
})
it('preserves markdown tables', () => {
expect(roundTripMarkdown('| a | b |\n| - | - |\n| 1 | 2 |\n')).toContain('| a')
})
@@ -49,6 +49,11 @@ import { renderRichMarkdownCodeBlock } from './rich-markdown-code-block-markdown
const lowlight = createCachedLowlight(createLowlight(common))
const RichMarkdownLink = Link.extend({
// Keep link priority below code so linked code labels serialize correctly.
priority: 90
})
const RichMarkdownCode = Code.extend({
// Why: Markdown supports linked code labels, so code cannot exclude the link
// mark even though it should still stay exclusive with emphasis marks.
@@ -96,7 +101,7 @@ export function createRichMarkdownExtensions({
lowlight,
defaultLanguage: null
}),
Link.configure({
RichMarkdownLink.configure({
openOnClick: false,
autolink: true,
linkOnPaste: true
@@ -2,7 +2,7 @@ import { InlineMath } from '@tiptap/extension-mathematics'
// Why: the common dialect requires a non-space next to each delimiter and forbids a
// newline inside, which is what keeps `US$ 5,000 and R$ 40,000` out of a math span.
const INLINE_MATH = /^\$(?![\s$])((?:[^$\n]*[^\s$])?)\$(?!\$)/
const INLINE_MATH = /^\$(?![\s$])((?:[^$\n]*[^\s$])?)\$(?![\d$])/
const baseTokenizer = InlineMath.config.markdownTokenizer
if (!baseTokenizer) {
@@ -400,6 +400,27 @@ describe('serializeRichMarkdownForReconcile (real editor pipeline)', () => {
const serialize = (md: string): string | null =>
serializeRichMarkdownForReconcile(md, serializerContext)
it('patches an edit at the end of a dollar-and-ampersand doc with the real serializer', () => {
// Why: the parser used to drop `\\$` and the serializer still writes `&` and re-pads tables;
// an end-of-file edit must keep the untouched source bytes instead of canonicalizing the file.
const originalSource =
'Cost was \\$1,200 for Nell & Mary.\n\n| Item | Amount |\n|--------------|-------:|\n| Fee | \\$500 |\n\nTrailing paragraph.\n'
const baseCanonical = serialize(originalSource)!
expect(baseCanonical.endsWith('\n')).toBe(false) // getMarkdown never emits a trailing newline
const edited = `${baseCanonical} Added word.`
const reconciled = reconcileSerializedMarkdown({
originalSource,
baseCanonical,
edited,
roundTrip: (md) => serialize(md)
})
expect(reconciled).toBe(
'Cost was \\$1,200 for Nell & Mary.\n\n| Item | Amount |\n|--------------|-------:|\n| Fee | \\$500 |\n\nTrailing paragraph. Added word.\n'
)
})
it('applies normalizeEmptyListItems so empty list items round-trip stably', () => {
// `3. ` immediately before a heading parses as an empty list item; without the
// normalize step the safety re-parse would spuriously mismatch and no-op.
@@ -684,3 +705,56 @@ describe('serializeRichMarkdownForReconcile (real editor pipeline)', () => {
expect(reconciled.replace(/\r\n/g, '')).not.toContain('\n')
})
})
describe('reconcileSerializedMarkdown end-of-document edits', () => {
// Why: real getMarkdown never emits a trailing newline; mimic that on both the base and the safety re-parse.
const canonicalWithoutTrailingNewline = (md: string): string =>
fakeCanonicalize(md).replace(/\n+$/, '')
it('patches an appended edit into a source that ends with a newline', () => {
const originalSource = '# Title\n\n_emphasis_ and __strong__\n\nTrailing paragraph.\n'
const baseCanonical = canonicalWithoutTrailingNewline(originalSource)
const edited = `${baseCanonical} Added word.`
const reconciled = reconcileSerializedMarkdown({
originalSource,
baseCanonical,
edited,
roundTrip: canonicalWithoutTrailingNewline
})
expect(reconciled).toBe(
'# Title\n\n_emphasis_ and __strong__\n\nTrailing paragraph. Added word.\n'
)
})
it('patches an appended edit into a CRLF source that ends with a newline', () => {
const originalSource = '# Title\r\n\r\n_emphasis_\r\n\r\nTrailing paragraph.\r\n'
const baseCanonical = canonicalWithoutTrailingNewline(originalSource.replace(/\r\n/g, '\n'))
const edited = `${baseCanonical} Added word.`
const reconciled = reconcileSerializedMarkdown({
originalSource,
baseCanonical,
edited,
roundTrip: canonicalWithoutTrailingNewline
})
expect(reconciled).toBe('# Title\r\n\r\n_emphasis_\r\n\r\nTrailing paragraph. Added word.\r\n')
})
it('still patches an edit at the end of a source without a trailing newline', () => {
const originalSource = '# Title\n\n_emphasis_\n\nTrailing paragraph.'
const baseCanonical = canonicalWithoutTrailingNewline(originalSource)
const edited = `${baseCanonical} Added word.`
const reconciled = reconcileSerializedMarkdown({
originalSource,
baseCanonical,
edited,
roundTrip: canonicalWithoutTrailingNewline
})
expect(reconciled).toBe('# Title\n\n_emphasis_\n\nTrailing paragraph. Added word.')
})
})
@@ -83,11 +83,18 @@ export function reconcileSerializedMarkdown({
return restoreMarkdownSourceLineEndings(edited, originalSource)
}
// Why: getMarkdown omits one non-semantic final newline; making it shared diff context keeps EOF edits before that newline.
const omittedSourceEof = !baseLf.endsWith('\n') && originalTrailingNewlines === '\n' ? '\n' : ''
const patchBase = baseLf + omittedSourceEof
const patchEdited = editedLf.endsWith('\n') ? editedLf : editedLf + omittedSourceEof
let diffs = makeDiff(patchBase, patchEdited, {
// Patch newline-stripped bodies so an EOF edit cannot land after the source's
// non-semantic trailing newline and force a whole-file canonical rewrite.
const sourceBody = stripTrailingNewlines(originalSourceLf)
const baseBody = stripTrailingNewlines(baseLf)
const editedBody = stripTrailingNewlines(editedLf)
const sourceTrailingNewlines = originalSourceLf.match(/\n+$/)?.[0] ?? ''
const baseTrailingNewlines = baseLf.match(/\n+$/)?.[0] ?? ''
const editedTrailingNewlines = editedLf.match(/\n+$/)?.[0] ?? ''
const reconciledTrailingNewlines =
(editedTrailingNewlines === baseTrailingNewlines ? '' : editedTrailingNewlines) +
sourceTrailingNewlines
let diffs = makeDiff(baseBody, editedBody, {
checkLines: true,
timeout: RECONCILE_DIFF_TIMEOUT_SECONDS
})
@@ -95,17 +102,18 @@ export function reconcileSerializedMarkdown({
diffs = cleanupSemantic(diffs)
diffs = cleanupEfficiency(diffs)
}
const patches = makePatches(patchBase, diffs)
const patches = makePatches(baseBody, diffs)
// Why: applyPatches decodes starts as UTF-8 offsets even though makePatches returns UTF-16 indices; encode against the divergent text being patched so decoding preserves the fuzzy-match seed.
const utf8Offsets = getUtf8OffsetsAtCodeUnitIndices(
originalSourceLf,
sourceBody,
patches.flatMap((patch) => [patch.start1, patch.start2])
)
for (const patch of patches) {
patch.start1 = utf8Offsets.get(patch.start1) ?? 0
patch.start2 = utf8Offsets.get(patch.start2) ?? 0
}
const [reconciledLf, results] = applyPatches(patches, originalSourceLf)
const [reconciledBody, results] = applyPatches(patches, sourceBody)
const reconciledLf = reconciledBody + reconciledTrailingNewlines
// Branch 5: a hunk failed to locate in the non-canonical source → unreliable fuzzy match, fall back to canonical.
if (results.some((applied) => !applied)) {
@@ -13,6 +13,17 @@ import {
marked
} from 'marked'
/** Rewrites escape tokens in place (marked fills caller-owned arrays for table cells). */
function preserveEscapedCharacters(tokens: Token[]): Token[] {
for (let index = 0; index < tokens.length; index += 1) {
const token = tokens[index]
if (token.type === 'escape') {
tokens[index] = { type: 'text', raw: token.raw, text: token.text, escaped: false }
}
}
return tokens
}
export function createTiptapMarkedFacade(): typeof marked {
const registry = new Marked()
@@ -26,6 +37,12 @@ export function createTiptapMarkedFacade(): typeof marked {
extensions: registry.defaults.extensions
})
}
// Why: Tiptap's markdown parser has no case for marked's `escape` token, so
// `\$`, `\*`, `\_`, `\[` would be deleted from the document on load.
inlineTokens(src: string, tokens: Token[] = []): Token[] {
return preserveEscapedCharacters(super.inlineTokens(src, tokens))
}
}
const parser = (tokens: Token[], options?: MarkedOptions) => registry.parser(tokens, options)