Reuse Markdown tokenizer and details start searches across paragraph suffixes (#35374)

* Reuse Markdown tokenizer start searches across paragraph suffixes

* Reuse details start searches across Markdown paragraph suffixes

* test(editor): drop prototype spy-count tokenizer cursor tests

---------

Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>

src/renderer/src/components/editor/markdown-tokenizer-start.test.ts

@@ -1,7 +1,9 @@
 import { expect, it, vi } from 'vitest'
 import { createTiptapMarkedFacade } from './tiptap-marked-facade'
+import { createOrcaDetailsExtensions } from './rich-markdown-details-extension'
 import {
   createMarkdownTokenizerStart,
+  createMarkdownTokenizerStartCursor,
   registerMarkdownTokenizerStart
 } from './markdown-tokenizer-start'
 
@@ -94,3 +96,40 @@ it('skips custom searches when their required marker is absent', () => {
   marked.lexer('Prose\n<DETAILS>')
   expect(search).toHaveReturnedWith(5)
 })
+
+it('recognizes a suffix-start directive skipped by an earlier line-start search', () => {
+  const marked = createTiptapMarkedFacade()
+  marked.use({
+    extensions: [
+      {
+        name: 'directive',
+        level: 'block',
+        start: createMarkdownTokenizerStart(':::x', true),
+        tokenizer: (source) =>
+          source.startsWith(':::x') ? { type: 'directive', raw: ':::x', text: 'found' } : undefined
+      }
+    ]
+  })
+  expect(marked.lexer('Prose\n\nx:::x\n\n:::x').map(({ type, raw }) => ({ type, raw }))).toEqual([
+    { type: 'paragraph', raw: 'Prose' },
+    { type: 'space', raw: '\n\n' },
+    { type: 'paragraph', raw: 'x' },
+    { type: 'directive', raw: ':::x' },
+    { type: 'space', raw: '\n\n' },
+    { type: 'directive', raw: ':::x' }
+  ])
+})
+
+function getDetailsTokenizerStart() {
+  const tokenizer = createOrcaDetailsExtensions()[0].config.markdownTokenizer
+  if (!tokenizer || typeof tokenizer === 'function' || typeof tokenizer.start !== 'function') {
+    throw new Error('Details tokenizer must expose its start hint')
+  }
+  return tokenizer.start
+}
+
+it('rechecks details start hints when a suffix discards their lookbehind', () => {
+  const start = createMarkdownTokenizerStartCursor(getDetailsTokenizerStart())
+  const source = 'before\n   <details>\n\nnext\n<details>'
+  expect([0, 7, 10, 26].map((offset) => start(source.slice(offset)))).toEqual([10, 19, 16, -1])
+})

src/renderer/src/components/editor/markdown-tokenizer-start.ts

@@ -1,4 +1,7 @@
-const startMarkers = new WeakMap<(source: string) => number | void, string>()
+type MarkdownTokenizerStart = (source: string) => number | void
+
+const startMarkers = new WeakMap<MarkdownTokenizerStart, string>()
+const startCursorFactories = new WeakMap<MarkdownTokenizerStart, () => MarkdownTokenizerStart>()
 
 export function registerMarkdownTokenizerStart(marker: string, start: (source: string) => number) {
   startMarkers.set(start, marker)
@@ -14,9 +17,60 @@ export function createMarkdownTokenizerStart(marker: string, lineStart = false)
     }
     return position
   }
+  if (lineStart && /[\n\r\u2028\u2029]/.test(marker)) {
+    return registerMarkdownTokenizerStart(marker, start)
+  }
+  startCursorFactories.set(start, () =>
+    createMarkdownTokenizerStartSearchCursor(start, 0, lineStart ? marker : undefined)
+  )
   return registerMarkdownTokenizerStart(marker, start)
 }
 
 export function getMarkdownTokenizerStartMarker(start: (source: string) => number | void) {
   return startMarkers.get(start)
 }
+
+/** Bind to one blockTokens call, where Marked passes monotonically shorter suffixes. */
+export function createMarkdownTokenizerStartCursor(
+  start: MarkdownTokenizerStart
+): MarkdownTokenizerStart {
+  return startCursorFactories.get(start)?.() ?? start
+}
+
+/** Removing a prefix must only remove matches; each match needs at most lookbehindLength preceding units. */
+export function registerMarkdownLookbehindTokenizerStart(
+  marker: string,
+  start: (source: string) => number,
+  lookbehindLength: number
+) {
+  startCursorFactories.set(start, () =>
+    createMarkdownTokenizerStartSearchCursor(start, lookbehindLength)
+  )
+  return registerMarkdownTokenizerStart(marker, start)
+}
+
+function createMarkdownTokenizerStartSearchCursor(
+  start: (source: string) => number,
+  lookbehindLength: number,
+  suffixStartMarker?: string
+): MarkdownTokenizerStart {
+  let firstLength: number | undefined
+  let nextPosition: number | undefined
+  return (source) => {
+    firstLength ??= source.length
+    const position = firstLength - source.length
+    // A suffix can begin at a marker an earlier line-start search skipped.
+    if (suffixStartMarker !== undefined && source.startsWith(suffixStartMarker)) {
+      return 0
+    }
+    if (nextPosition === -1) {
+      return -1
+    }
+    if (nextPosition !== undefined && nextPosition >= position + lookbehindLength) {
+      return nextPosition - position
+    }
+    const match = start(source)
+    nextPosition = match === -1 ? -1 : position + match
+    return match
+  }
+}

src/renderer/src/components/editor/rich-markdown-details-extension.ts

@@ -1,4 +1,4 @@
-import { registerMarkdownTokenizerStart } from './markdown-tokenizer-start'
+import { registerMarkdownLookbehindTokenizerStart } from './markdown-tokenizer-start'
 import { guardMarkdownDirectiveTokenizer } from './markdown-directive-tokenizer'
 import { decodeHtmlEntities, type AnyExtension, type Editor } from '@tiptap/core'
 import { Details, DetailsContent, DetailsSummary } from '@tiptap/extension-details'
@@ -233,8 +233,10 @@ const OrcaDetails = Details.extend({
     name: 'details',
     level: 'block',
     // HTML blocks open only at a line start; marked passes source.slice(1), so offset 0 never qualifies.
-    start: registerMarkdownTokenizerStart('<', (source) =>
-      source.search(/(?<=[\n\r] {0,3})<details\b/i)
+    start: registerMarkdownLookbehindTokenizerStart(
+      '<',
+      (source) => source.search(/(?<=[\n\r] {0,3})<details\b/i),
+      4
     ),
     tokenize(src, _tokens, lexer) {
       const detailsBlock = matchDetailsHtmlBlock(src, 0)

src/renderer/src/components/editor/tiptap-marked-facade.ts

@@ -1,4 +1,7 @@
-import { getMarkdownTokenizerStartMarker } from './markdown-tokenizer-start'
+import {
+  createMarkdownTokenizerStartCursor,
+  getMarkdownTokenizerStartMarker
+} from './markdown-tokenizer-start'
 import {
   Hooks,
   Lexer,
@@ -88,10 +91,12 @@ export function createTiptapMarkedFacade(): typeof marked {
       if (!extensions?.startBlock) {
         return super.blockTokens(source, tokens, lastParagraphClipped)
       }
-      const startBlock = extensions.startBlock.filter((start) => {
-        const marker = getMarkdownTokenizerStartMarker(start)
-        return !marker || source.includes(marker)
-      })
+      const startBlock = extensions.startBlock
+        .filter((start) => {
+          const marker = getMarkdownTokenizerStartMarker(start)
+          return !marker || source.includes(marker)
+        })
+        .map(createMarkdownTokenizerStartCursor)
       this.options.extensions = { ...extensions, startBlock }
       try {
         return super.blockTokens(source, tokens, lastParagraphClipped)