From a83cb6ee3308a380fa4b8bb11bb5d0d2310a2f58 Mon Sep 17 00:00:00 2001 From: OrcaWin Date: Thu, 29 Oct 2026 15:50:57 -0700 Subject: [PATCH] Reuse Markdown tokenizer and details start searches across paragraph suffixes (#35374) * Reuse Markdown tokenizer start searches across paragraph suffixes * Reuse details start searches across Markdown paragraph suffixes * test(editor): drop prototype spy-count tokenizer cursor tests --------- Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com> --- .../editor/markdown-tokenizer-start.test.ts | 39 +++++++++++++ .../editor/markdown-tokenizer-start.ts | 56 ++++++++++++++++++- .../editor/rich-markdown-details-extension.ts | 8 ++- .../components/editor/tiptap-marked-facade.ts | 15 +++-- 4 files changed, 109 insertions(+), 9 deletions(-) diff --git a/src/renderer/src/components/editor/markdown-tokenizer-start.test.ts b/src/renderer/src/components/editor/markdown-tokenizer-start.test.ts index c190b3b7e5a..f6f4dc3f5a8 100644 --- a/src/renderer/src/components/editor/markdown-tokenizer-start.test.ts +++ b/src/renderer/src/components/editor/markdown-tokenizer-start.test.ts @@ -1,7 +1,9 @@ import { expect, it, vi } from 'vitest' import { createTiptapMarkedFacade } from './tiptap-marked-facade' +import { createOrcaDetailsExtensions } from './rich-markdown-details-extension' import { createMarkdownTokenizerStart, + createMarkdownTokenizerStartCursor, registerMarkdownTokenizerStart } from './markdown-tokenizer-start' @@ -94,3 +96,40 @@ it('skips custom searches when their required marker is absent', () => { marked.lexer('Prose\n
') expect(search).toHaveReturnedWith(5) }) + +it('recognizes a suffix-start directive skipped by an earlier line-start search', () => { + const marked = createTiptapMarkedFacade() + marked.use({ + extensions: [ + { + name: 'directive', + level: 'block', + start: createMarkdownTokenizerStart(':::x', true), + tokenizer: (source) => + source.startsWith(':::x') ? { type: 'directive', raw: ':::x', text: 'found' } : undefined + } + ] + }) + expect(marked.lexer('Prose\n\nx:::x\n\n:::x').map(({ type, raw }) => ({ type, raw }))).toEqual([ + { type: 'paragraph', raw: 'Prose' }, + { type: 'space', raw: '\n\n' }, + { type: 'paragraph', raw: 'x' }, + { type: 'directive', raw: ':::x' }, + { type: 'space', raw: '\n\n' }, + { type: 'directive', raw: ':::x' } + ]) +}) + +function getDetailsTokenizerStart() { + const tokenizer = createOrcaDetailsExtensions()[0].config.markdownTokenizer + if (!tokenizer || typeof tokenizer === 'function' || typeof tokenizer.start !== 'function') { + throw new Error('Details tokenizer must expose its start hint') + } + return tokenizer.start +} + +it('rechecks details start hints when a suffix discards their lookbehind', () => { + const start = createMarkdownTokenizerStartCursor(getDetailsTokenizerStart()) + const source = 'before\n
\n\nnext\n
' + expect([0, 7, 10, 26].map((offset) => start(source.slice(offset)))).toEqual([10, 19, 16, -1]) +}) diff --git a/src/renderer/src/components/editor/markdown-tokenizer-start.ts b/src/renderer/src/components/editor/markdown-tokenizer-start.ts index b32a4539fac..77b7c227578 100644 --- a/src/renderer/src/components/editor/markdown-tokenizer-start.ts +++ b/src/renderer/src/components/editor/markdown-tokenizer-start.ts @@ -1,4 +1,7 @@ -const startMarkers = new WeakMap<(source: string) => number | void, string>() +type MarkdownTokenizerStart = (source: string) => number | void + +const startMarkers = new WeakMap() +const startCursorFactories = new WeakMap MarkdownTokenizerStart>() export function registerMarkdownTokenizerStart(marker: string, start: (source: string) => number) { startMarkers.set(start, marker) @@ -14,9 +17,60 @@ export function createMarkdownTokenizerStart(marker: string, lineStart = false) } return position } + if (lineStart && /[\n\r\u2028\u2029]/.test(marker)) { + return registerMarkdownTokenizerStart(marker, start) + } + startCursorFactories.set(start, () => + createMarkdownTokenizerStartSearchCursor(start, 0, lineStart ? marker : undefined) + ) return registerMarkdownTokenizerStart(marker, start) } export function getMarkdownTokenizerStartMarker(start: (source: string) => number | void) { return startMarkers.get(start) } + +/** Bind to one blockTokens call, where Marked passes monotonically shorter suffixes. */ +export function createMarkdownTokenizerStartCursor( + start: MarkdownTokenizerStart +): MarkdownTokenizerStart { + return startCursorFactories.get(start)?.() ?? start +} + +/** Removing a prefix must only remove matches; each match needs at most lookbehindLength preceding units. */ +export function registerMarkdownLookbehindTokenizerStart( + marker: string, + start: (source: string) => number, + lookbehindLength: number +) { + startCursorFactories.set(start, () => + createMarkdownTokenizerStartSearchCursor(start, lookbehindLength) + ) + return registerMarkdownTokenizerStart(marker, start) +} + +function createMarkdownTokenizerStartSearchCursor( + start: (source: string) => number, + lookbehindLength: number, + suffixStartMarker?: string +): MarkdownTokenizerStart { + let firstLength: number | undefined + let nextPosition: number | undefined + return (source) => { + firstLength ??= source.length + const position = firstLength - source.length + // A suffix can begin at a marker an earlier line-start search skipped. + if (suffixStartMarker !== undefined && source.startsWith(suffixStartMarker)) { + return 0 + } + if (nextPosition === -1) { + return -1 + } + if (nextPosition !== undefined && nextPosition >= position + lookbehindLength) { + return nextPosition - position + } + const match = start(source) + nextPosition = match === -1 ? -1 : position + match + return match + } +} diff --git a/src/renderer/src/components/editor/rich-markdown-details-extension.ts b/src/renderer/src/components/editor/rich-markdown-details-extension.ts index 5162c72cd6f..58e6359d3ab 100644 --- a/src/renderer/src/components/editor/rich-markdown-details-extension.ts +++ b/src/renderer/src/components/editor/rich-markdown-details-extension.ts @@ -1,4 +1,4 @@ -import { registerMarkdownTokenizerStart } from './markdown-tokenizer-start' +import { registerMarkdownLookbehindTokenizerStart } from './markdown-tokenizer-start' import { guardMarkdownDirectiveTokenizer } from './markdown-directive-tokenizer' import { decodeHtmlEntities, type AnyExtension, type Editor } from '@tiptap/core' import { Details, DetailsContent, DetailsSummary } from '@tiptap/extension-details' @@ -233,8 +233,10 @@ const OrcaDetails = Details.extend({ name: 'details', level: 'block', // HTML blocks open only at a line start; marked passes source.slice(1), so offset 0 never qualifies. - start: registerMarkdownTokenizerStart('<', (source) => - source.search(/(?<=[\n\r] {0,3}) source.search(/(?<=[\n\r] {0,3}) { - const marker = getMarkdownTokenizerStartMarker(start) - return !marker || source.includes(marker) - }) + const startBlock = extensions.startBlock + .filter((start) => { + const marker = getMarkdownTokenizerStartMarker(start) + return !marker || source.includes(marker) + }) + .map(createMarkdownTokenizerStartCursor) this.options.extensions = { ...extensions, startBlock } try { return super.blockTokens(source, tokens, lastParagraphClipped)