mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 16:02:32 +00:00
fix(editor): require non-space delimiters for inline math
The upstream tokenizer matched `$([^$]+)$` with no delimiter rule, so a second dollar sign anywhere in the paragraph turned prose into a math span: `Costs are US$ 5,000 and R$ 40,000` lost the space after `US$`. Require a non-space next to each delimiter and forbid a newline inside, matching the common dialect. Real math still parses.
This commit is contained in:
committed by
Neil
parent
fd1c8847bb
commit
b22fc02610
@@ -9,7 +9,7 @@ import { createRichMarkdownTable } from './rich-markdown-table'
|
||||
import { TableCell } from '@tiptap/extension-table-cell'
|
||||
import { TableHeader } from '@tiptap/extension-table-header'
|
||||
import { TableRow } from '@tiptap/extension-table-row'
|
||||
import { BlockMath, InlineMath } from '@tiptap/extension-mathematics'
|
||||
import { BlockMath } from '@tiptap/extension-mathematics'
|
||||
import { createRichMarkdownExtension } from './rich-markdown-extension'
|
||||
import { createLowlight, common } from 'lowlight'
|
||||
import {
|
||||
@@ -38,6 +38,7 @@ import { createRichMarkdownHtmlSuperscriptLink } from './rich-markdown-html-supe
|
||||
import type { RichMarkdownHtmlSuperscriptLinkContext } from './rich-markdown-html-superscript-link-context'
|
||||
import { RichMarkdownOrderedList } from './rich-markdown-ordered-list'
|
||||
import { RichMarkdownParagraph } from './rich-markdown-paragraph'
|
||||
import { RichMarkdownInlineMath } from './rich-markdown-inline-math'
|
||||
import { RichMarkdownCodeBlockLowlight } from './rich-markdown-lowlight'
|
||||
import { RichMarkdownTaskList } from './rich-markdown-task-list'
|
||||
import { createCachedLowlight } from './rich-markdown-lowlight-cache'
|
||||
@@ -224,7 +225,7 @@ export function createRichMarkdownExtensions({
|
||||
TableRow,
|
||||
TableHeader,
|
||||
TableCell,
|
||||
InlineMath.configure({
|
||||
RichMarkdownInlineMath.configure({
|
||||
katexOptions: {
|
||||
throwOnError: false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { Editor } from '@tiptap/core'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { encodeRawMarkdownHtmlForRichEditor } from './raw-markdown-html'
|
||||
import { createRichMarkdownExtensions } from './rich-markdown-extensions'
|
||||
import { createRichMarkdownEditorCodec } from './rich-markdown-source-transport'
|
||||
|
||||
function withEditor<T>(source: string, read: (editor: Editor) => T): T {
|
||||
const codec = createRichMarkdownEditorCodec()
|
||||
const editor = new Editor({
|
||||
element: null,
|
||||
extensions: createRichMarkdownExtensions({ codec }),
|
||||
content: encodeRawMarkdownHtmlForRichEditor(source, codec),
|
||||
contentType: 'markdown'
|
||||
})
|
||||
try {
|
||||
return read(editor)
|
||||
} finally {
|
||||
editor.destroy()
|
||||
}
|
||||
}
|
||||
|
||||
function roundTrip(source: string): string {
|
||||
return withEditor(source, (editor) => editor.getMarkdown().trimEnd())
|
||||
}
|
||||
|
||||
function countInlineMath(source: string): number {
|
||||
return withEditor(source, (editor) => {
|
||||
let total = 0
|
||||
editor.state.doc.descendants((node) => {
|
||||
if (node.type.name === 'inlineMath') {
|
||||
total += 1
|
||||
}
|
||||
})
|
||||
return total
|
||||
})
|
||||
}
|
||||
|
||||
describe('inline math delimiters', () => {
|
||||
it.each([
|
||||
['Costs are US$ 5,000 and R$ 40,000 total.'],
|
||||
['R$ 40,000 and R$ 50,000'],
|
||||
['The fee is $ 100 and the tax is $ 20.'],
|
||||
['one $ two $ three'],
|
||||
['$ not math $'],
|
||||
['R$ 40,000 and US$5,000 in one paragraph.'],
|
||||
['A line with US$ 5,000 here\nand another with R$ 40,000 there.']
|
||||
])('leaves %j untouched', (source) => {
|
||||
expect(roundTrip(source)).toBe(source)
|
||||
expect(countInlineMath(source)).toBe(0)
|
||||
})
|
||||
|
||||
it.each([
|
||||
['$x$', 1],
|
||||
['Real math: $E = mc^2$ here.', 1],
|
||||
['$a+b$ and $c+d$ two real', 2]
|
||||
])('still parses math in %j', (source, expected) => {
|
||||
expect(roundTrip(source)).toBe(source)
|
||||
expect(countInlineMath(source)).toBe(expected)
|
||||
})
|
||||
|
||||
it('does not span a soft line break', () => {
|
||||
expect(countInlineMath('A line with US$ 5,000 here\nand another with R$ 40,000 there.')).toBe(0)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,27 @@
|
||||
import { InlineMath } from '@tiptap/extension-mathematics'
|
||||
|
||||
// Why: the common dialect requires a non-space next to each delimiter and forbids a
|
||||
// newline inside, which is what keeps `US$ 5,000 and R$ 40,000` out of a math span.
|
||||
const INLINE_MATH = /^\$(?![\s$])((?:[^$\n]*[^\s$])?)\$(?!\$)/
|
||||
|
||||
const baseTokenizer = InlineMath.config.markdownTokenizer
|
||||
if (!baseTokenizer) {
|
||||
throw new Error('InlineMath must provide a Markdown tokenizer')
|
||||
}
|
||||
|
||||
export const RichMarkdownInlineMath = InlineMath.extend({
|
||||
markdownTokenizer: {
|
||||
...baseTokenizer,
|
||||
tokenize(src) {
|
||||
const match = src.match(INLINE_MATH)
|
||||
if (!match) {
|
||||
return undefined
|
||||
}
|
||||
return {
|
||||
type: 'inlineMath',
|
||||
raw: match[0],
|
||||
latex: match[1]
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
Reference in New Issue
Block a user