feat(editor): VS Code markdown highlighting and sticky headings in source mode (#34537)

* feat(editor): VS Code markdown highlighting and sticky headings in source mode

* fix(editor): address review on markdown TextMate highlighting

- resolve fence grammars from the first word of the info string
- give fence grammars the cold-compile retry budget
- fall back to Monarch markdown when TextMate fails to load
- bound heading-scan line length

src/renderer/src/lib/monaco-languages/markdown-heading-outline.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from 'vitest'
+import {
+  buildMarkdownHeadingSections,
+  type MarkdownHeadingSection
+} from './markdown-heading-outline'
+
+type Outline = { title: string; lines: [number, number]; children?: Outline[] }
+
+function outline(sections: MarkdownHeadingSection[]): Outline[] {
+  return sections.map((section) => ({
+    title: section.title,
+    lines: [section.startLine, section.endLine],
+    ...(section.children.length > 0 ? { children: outline(section.children) } : {})
+  }))
+}
+
+describe('buildMarkdownHeadingSections', () => {
+  it('nests sections by level and ends each before the next same-or-higher heading', () => {
+    const lines = ['# A', 'text', '## B', '### C', 'x', '## D', '# E', 'tail']
+    expect(outline(buildMarkdownHeadingSections(lines))).toEqual([
+      {
+        title: 'A',
+        lines: [1, 6],
+        children: [
+          { title: 'B', lines: [3, 5], children: [{ title: 'C', lines: [4, 5] }] },
+          { title: 'D', lines: [6, 6] }
+        ]
+      },
+      { title: 'E', lines: [7, 8] }
+    ])
+  })
+
+  it('ignores fenced code, front matter, and thematic breaks but reads setext headings', () => {
+    const lines = [
+      '---',
+      'title: # not a heading',
+      '---',
+      '````md',
+      '# fenced',
+      '```',
+      '````',
+      'Setext Title',
+      '===',
+      '',
+      '---',
+      '- item',
+      '---',
+      '## Closed ##'
+    ]
+    expect(outline(buildMarkdownHeadingSections(lines))).toEqual([
+      { title: 'Setext Title', lines: [8, 14], children: [{ title: 'Closed', lines: [14, 14] }] }
+    ])
+  })
+})

src/renderer/src/lib/monaco-languages/markdown-heading-outline.ts

@@ -0,0 +1,112 @@
+export type MarkdownHeadingSection = {
+  level: number
+  title: string
+  /** 1-based line of the heading text; a setext heading starts on its text line. */
+  startLine: number
+  /** 1-based last line of the section's own heading line(s). */
+  headingEndLine: number
+  /** 1-based last line before the next heading of the same or a higher level. */
+  endLine: number
+  children: MarkdownHeadingSection[]
+}
+
+// Why: bounds the heading regexes' backtracking on pathological lines; real headings are far shorter.
+const MAX_HEADING_LINE_LENGTH = 1000
+const ATX_HEADING = /^ {0,3}(#{1,6})(?:[ \t]+(.*?))?(?:[ \t]+#+)?[ \t]*$/
+const SETEXT_UNDERLINE = /^ {0,3}(=+|-+)[ \t]*$/
+const FENCE_OPEN = /^ {0,3}(`{3,}|~{3,})/
+const FRONT_MATTER_FENCE = /^(---|\+\+\+)[ \t]*$/
+// Lines that cannot be a setext heading's text: they open their own block.
+const NON_PARAGRAPH_START = /^ {0,3}(?:[-+*>|]|\d{1,9}[.)]|#{1,6}(?:[ \t]|$)|<)|^(?: {4}|\t)/
+
+function isFenceClose(line: string, fence: string): boolean {
+  const trimmed = line.trimStart()
+  if (line.length - trimmed.length > 3 || !trimmed.startsWith(fence)) {
+    return false
+  }
+  const run = trimmed.match(fence[0] === '`' ? /^`+/ : /^~+/)?.[0] ?? ''
+  return run.length >= fence.length && trimmed.slice(run.length).trim() === ''
+}
+
+function frontMatterEnd(lines: readonly string[]): number {
+  const open = lines[0]?.match(FRONT_MATTER_FENCE)?.[1]
+  if (!open) {
+    return 0
+  }
+  for (let index = 1; index < lines.length; index += 1) {
+    const line = lines[index].trimEnd()
+    if (line === open || (open === '---' && line === '...')) {
+      return index + 1
+    }
+  }
+  return 0
+}
+
+type FlatHeading = Omit<MarkdownHeadingSection, 'endLine' | 'children'>
+
+// Why: a line scan keeps sticky scroll cheap on every edit; a full remark parse would block typing in large files.
+function scanHeadings(lines: readonly string[]): FlatHeading[] {
+  const headings: FlatHeading[] = []
+  let fence: string | null = null
+  let previousIsParagraph = false
+  for (let index = frontMatterEnd(lines); index < lines.length; index += 1) {
+    const line = lines[index]
+    if (fence) {
+      if (isFenceClose(line, fence)) {
+        fence = null
+      }
+      previousIsParagraph = false
+      continue
+    }
+    if (line.length > MAX_HEADING_LINE_LENGTH) {
+      previousIsParagraph = true
+      continue
+    }
+    const fenceOpen = line.match(FENCE_OPEN)?.[1]
+    if (fenceOpen) {
+      fence = fenceOpen
+      previousIsParagraph = false
+      continue
+    }
+    const atx = line.match(ATX_HEADING)
+    if (atx) {
+      headings.push({
+        level: atx[1].length,
+        title: (atx[2] ?? '').trim(),
+        startLine: index + 1,
+        headingEndLine: index + 1
+      })
+      previousIsParagraph = false
+      continue
+    }
+    const underline = line.match(SETEXT_UNDERLINE)?.[1]
+    if (underline && previousIsParagraph) {
+      headings.push({
+        level: underline[0] === '=' ? 1 : 2,
+        title: lines[index - 1].trim(),
+        startLine: index,
+        headingEndLine: index + 1
+      })
+      previousIsParagraph = false
+      continue
+    }
+    previousIsParagraph = line.trim() !== '' && !NON_PARAGRAPH_START.test(line)
+  }
+  return headings
+}
+
+/** Heading sections nested by level, each spanning until the next heading of the same or a higher level. */
+export function buildMarkdownHeadingSections(lines: readonly string[]): MarkdownHeadingSection[] {
+  const roots: MarkdownHeadingSection[] = []
+  const open: MarkdownHeadingSection[] = []
+  for (const heading of scanHeadings(lines)) {
+    for (let last = open.at(-1); last && last.level >= heading.level; last = open.at(-1)) {
+      last.endLine = heading.startLine - 1
+      open.pop()
+    }
+    const section: MarkdownHeadingSection = { ...heading, endLine: lines.length, children: [] }
+    ;(open.at(-1)?.children ?? roots).push(section)
+    open.push(section)
+  }
+  return roots
+}

src/renderer/src/lib/monaco-languages/markdown-textmate-tokens.test.ts

@@ -0,0 +1,64 @@
+import type * as Monaco from 'monaco-editor'
+import { describe, expect, it, vi } from 'vitest'
+import { loadNodeOniguruma } from '../syntax-highlighting/oniguruma-test-harness'
+import { createMarkdownTokensProvider } from './markdown-textmate-tokens'
+
+vi.mock('../syntax-highlighting/oniguruma', () => ({ loadOniguruma: loadNodeOniguruma }))
+
+function tokenizeLines(provider: Monaco.languages.TokensProvider, lines: string[]): string[][] {
+  let state = provider.getInitialState()
+  return lines.map((line) => {
+    const result = provider.tokenize(line, state)
+    state = result.endState
+    return result.tokens.map((token) => token.scopes)
+  })
+}
+
+describe('createMarkdownTokensProvider', () => {
+  it('styles markdown with Dark+ rules and highlights a fence once its grammar loads', async () => {
+    const defineTheme = vi.fn()
+    let replacement: Monaco.languages.TokensProvider | undefined
+    const provider = await createMarkdownTokensProvider({
+      editor: { defineTheme },
+      languages: {
+        setTokensProvider: (_language, next) => {
+          if ('tokenize' in next && 'getInitialState' in next && !('tokenizeEncoded' in next)) {
+            replacement = next
+          }
+          return { dispose: () => {} }
+        }
+      }
+    })
+
+    expect(defineTheme).toHaveBeenCalledWith(
+      'vs-dark',
+      expect.objectContaining({
+        rules: expect.arrayContaining([
+          { token: 'md.markup.heading', foreground: '#569cd6', fontStyle: 'bold' }
+        ])
+      })
+    )
+    const [heading, list] = tokenizeLines(provider, ['## Quick', '- **State:** open'])
+    expect(new Set(heading)).toEqual(new Set(['md.markup.heading.markdown']))
+    expect(list).toEqual([
+      'md.punctuation.definition.list.begin.markdown',
+      'md.text.html.markdown',
+      'md.markup.bold.markdown',
+      'md.markup.bold.markdown',
+      'md.markup.bold.markdown',
+      'md.text.html.markdown'
+    ])
+
+    const fence = ['```ts', 'const x = 1', '```']
+    tokenizeLines(provider, fence)
+    const refreshed = await vi.waitFor(() => {
+      if (!replacement) {
+        throw new Error('fence grammar has not loaded')
+      }
+      return replacement
+    })
+    expect(tokenizeLines(refreshed, ['```ts title="a.ts"', 'const x = 1', '```'])[1]).toContain(
+      'md.storage.type.ts'
+    )
+  })
+})

src/renderer/src/lib/monaco-languages/markdown-textmate-tokens.ts

@@ -0,0 +1,167 @@
+import type * as Monaco from 'monaco-editor'
+import type { ThemeRegistrationRaw } from 'shiki/core'
+import type { IToken } from 'shiki/textmate'
+import type { BundledLanguageInfo } from 'shiki/types'
+import { COLD_GRAMMAR_LINE_ATTEMPTS } from '../syntax-highlighting/textmate-core'
+import { createTokensProvider, loadEditorCore } from './textmate-token-provider'
+
+type MonacoModule = {
+  editor: Pick<typeof Monaco.editor, 'defineTheme'>
+  languages: Pick<typeof Monaco.languages, 'setTokensProvider'>
+}
+
+const MARKDOWN_LANGUAGE_ID = 'markdown'
+const MARKDOWN_SCOPE_NAME = 'text.html.markdown'
+// Why: namespaced rules color only markdown tokens, leaving every Monarch language's vs/vs-dark colors untouched.
+const MARKDOWN_TOKEN_PREFIX = 'md.'
+// The grammar's catch-all fence rule; a specific fence rule takes over once its grammar loads.
+const UNKNOWN_FENCE_LANGUAGE_SCOPE = 'fenced_code.block.language'
+// The catch-all rule captures the whole info string, e.g. `ts title="a.ts"`.
+const FENCE_INFO_SEPARATOR = /[\s{,:?]/
+
+type ThemeRule = Monaco.editor.ITokenThemeRule
+
+function simpleScopeSelectors(scope: string | string[] | undefined): string[] {
+  const selectors = typeof scope === 'string' ? scope.split(',') : (scope ?? [])
+  // Monaco matches one scope by dotted prefix, so descendant and exclusion selectors cannot apply.
+  return selectors
+    .map((selector) => selector.trim())
+    .filter((selector) => /^[^\s]+$/.test(selector))
+}
+
+function markdownThemeRules(theme: ThemeRegistrationRaw): ThemeRule[] {
+  return (theme.tokenColors ?? []).flatMap(({ scope, settings }) =>
+    simpleScopeSelectors(scope).map((selector) => ({
+      token: `${MARKDOWN_TOKEN_PREFIX}${selector}`,
+      ...(settings.foreground ? { foreground: settings.foreground } : {}),
+      ...(settings.fontStyle !== undefined ? { fontStyle: settings.fontStyle } : {})
+    }))
+  )
+}
+
+function hasMatchingSelector(selectors: ReadonlySet<string>, scope: string): boolean {
+  for (let end = scope.length; end > 0; end = scope.lastIndexOf('.', end - 1)) {
+    if (selectors.has(scope.slice(0, end))) {
+      return true
+    }
+  }
+  return false
+}
+
+/** Monaco takes one scope per token, so pick the innermost one a theme rule styles, as VS Code's cascade would. */
+function createScopeMapper(selectors: ReadonlySet<string>): (scopes: readonly string[]) => string {
+  return (scopes) => {
+    for (let index = scopes.length - 1; index >= 0; index -= 1) {
+      if (hasMatchingSelector(selectors, scopes[index])) {
+        return `${MARKDOWN_TOKEN_PREFIX}${scopes[index]}`
+      }
+    }
+    return MARKDOWN_TOKEN_PREFIX + MARKDOWN_SCOPE_NAME
+  }
+}
+
+function fenceLanguagesByLabel(
+  bundledLanguagesInfo: readonly BundledLanguageInfo[],
+  embeddedLanguageIds: readonly string[]
+): Map<string, BundledLanguageInfo> {
+  const embedded = new Set(embeddedLanguageIds)
+  const byLabel = new Map<string, BundledLanguageInfo>()
+  for (const info of bundledLanguagesInfo) {
+    if (embedded.has(info.id)) {
+      for (const label of [info.id, ...(info.aliases ?? [])]) {
+        byLabel.set(label, info)
+      }
+    }
+  }
+  return byLabel
+}
+
+/** VS Code's markdown grammar with Dark+/Light+ colors; fence grammars load on first sight. */
+export async function createMarkdownTokensProvider(
+  monaco: MonacoModule
+): Promise<Monaco.languages.TokensProvider> {
+  const [editorCore, { default: markdownGrammars }, { bundledLanguagesInfo }, dark, light] =
+    await Promise.all([
+      loadEditorCore(),
+      import('shiki/langs/markdown.mjs'),
+      import('shiki/langs'),
+      import('shiki/themes/dark-plus.mjs'),
+      import('shiki/themes/light-plus.mjs')
+    ])
+  await editorCore.core.loadLanguage(...markdownGrammars)
+
+  const darkRules = markdownThemeRules(dark.default)
+  const lightRules = markdownThemeRules(light.default)
+  // Why: Orca's editors all use the built-in theme names; redefining them keeps every setTheme caller unchanged.
+  monaco.editor.defineTheme('vs-dark', {
+    base: 'vs-dark',
+    inherit: true,
+    rules: darkRules,
+    colors: {}
+  })
+  monaco.editor.defineTheme('vs', { base: 'vs', inherit: true, rules: lightRules, colors: {} })
+  const mapScopes = createScopeMapper(
+    new Set(
+      [...darkRules, ...lightRules].map((rule) => rule.token.slice(MARKDOWN_TOKEN_PREFIX.length))
+    )
+  )
+
+  const fenceLanguages = fenceLanguagesByLabel(
+    bundledLanguagesInfo,
+    markdownGrammars[0]?.embeddedLangsLazy ?? []
+  )
+  const requestedFenceLanguages = new Set<string>()
+  let refreshQueued = false
+
+  function refreshProvider(): void {
+    if (refreshQueued) {
+      return
+    }
+    refreshQueued = true
+    queueMicrotask(() => {
+      refreshQueued = false
+      // Why: loading a grammar rebuilds markdown's grammar; replacing the provider retokenizes from its fresh start state.
+      monaco.languages.setTokensProvider(MARKDOWN_LANGUAGE_ID, buildProvider())
+    })
+  }
+
+  function loadFenceLanguage(label: string): void {
+    const info = fenceLanguages.get(label.trim().split(FENCE_INFO_SEPARATOR)[0].toLowerCase())
+    if (!info || requestedFenceLanguages.has(info.id)) {
+      return
+    }
+    requestedFenceLanguages.add(info.id)
+    void info
+      .import()
+      .then(({ default: grammars }) => editorCore.core.loadLanguage(...grammars))
+      .then(refreshProvider)
+      .catch((error: unknown) => {
+        console.warn(
+          `[markdown] failed to load the ${info.id} fence grammar; it stays plain`,
+          error
+        )
+      })
+  }
+
+  function observeFenceLabels(line: string, tokens: readonly IToken[]): void {
+    for (const token of tokens) {
+      if (token.scopes.at(-1) === UNKNOWN_FENCE_LANGUAGE_SCOPE) {
+        loadFenceLanguage(line.slice(token.startIndex, token.endIndex))
+      }
+    }
+  }
+
+  function buildProvider(): Monaco.languages.TokensProvider {
+    return createTokensProvider(
+      editorCore,
+      editorCore.core.getLanguage(MARKDOWN_LANGUAGE_ID),
+      MARKDOWN_SCOPE_NAME,
+      mapScopes,
+      observeFenceLabels,
+      // Why: each fence grammar compiles on its first block, long after the markdown root has warmed.
+      COLD_GRAMMAR_LINE_ATTEMPTS
+    )
+  }
+
+  return buildProvider()
+}

src/renderer/src/lib/monaco-languages/register-markdown.ts

@@ -0,0 +1,62 @@
+import type * as Monaco from 'monaco-editor'
+import {
+  buildMarkdownHeadingSections,
+  type MarkdownHeadingSection
+} from './markdown-heading-outline'
+
+type MonacoModule = Pick<typeof Monaco, 'languages' | 'editor'>
+
+const MARKDOWN_LANGUAGE_ID = 'markdown'
+
+function toDocumentSymbol(
+  monaco: MonacoModule,
+  model: Monaco.editor.ITextModel,
+  section: MarkdownHeadingSection
+): Monaco.languages.DocumentSymbol {
+  return {
+    name: section.title || '#'.repeat(section.level),
+    detail: '',
+    kind: monaco.languages.SymbolKind.String,
+    tags: [],
+    range: {
+      startLineNumber: section.startLine,
+      startColumn: 1,
+      endLineNumber: section.endLine,
+      endColumn: model.getLineMaxColumn(section.endLine)
+    },
+    selectionRange: {
+      startLineNumber: section.startLine,
+      startColumn: 1,
+      endLineNumber: section.headingEndLine,
+      endColumn: model.getLineMaxColumn(section.headingEndLine)
+    },
+    children: section.children.map((child) => toDocumentSymbol(monaco, model, child))
+  }
+}
+
+/** VS Code's markdown editing: TextMate colors, no bracket-pair colors, and a heading outline for sticky scroll. */
+export function registerMarkdownLanguage(monaco: MonacoModule): void {
+  // Why: Monaco merges configurations per field, so this keeps the built-in markdown config and drops only bracket colors.
+  monaco.languages.setLanguageConfiguration(MARKDOWN_LANGUAGE_ID, { colorizedBracketPairs: [] })
+
+  monaco.languages.registerTokensProviderFactory(MARKDOWN_LANGUAGE_ID, {
+    // Monaco caches this result, so a TextMate load failure falls back to the built-in Monarch tokenizer.
+    create: () =>
+      import('./markdown-textmate-tokens')
+        .then(({ createMarkdownTokensProvider }) => createMarkdownTokensProvider(monaco))
+        .catch(async (error: unknown) => {
+          console.warn('[markdown] TextMate highlighting failed to load; using Monarch', error)
+          const { language } =
+            await import('monaco-editor/esm/vs/basic-languages/markdown/markdown.js')
+          return language
+        })
+  })
+
+  monaco.languages.registerDocumentSymbolProvider(MARKDOWN_LANGUAGE_ID, {
+    displayName: 'Markdown headings',
+    provideDocumentSymbols: (model) =>
+      buildMarkdownHeadingSections(model.getLinesContent()).map((section) =>
+        toDocumentSymbol(monaco, model, section)
+      )
+  })
+}

src/renderer/src/lib/monaco-languages/textmate-token-provider.ts

@@ -1,7 +1,8 @@
 import type * as Monaco from 'monaco-editor'
 import type { Grammar, LanguageRegistration } from 'shiki/core'
-import type { StateStack } from 'shiki/textmate'
+import type { IToken, StateStack } from 'shiki/textmate'
 import {
+  COLD_GRAMMAR_LINE_ATTEMPTS,
   createTextMateCore,
   tokenizeLineWithinLimits,
   type TextMateCore
@@ -20,7 +21,7 @@ export type TextMateTokensProviderOptions = {
 let editorCorePromise: Promise<TextMateCore> | undefined
 
 // Why: chat's registry holds the catalogue's Nim and Typst under the same scope names as these vendored grammars.
-function loadEditorCore(): Promise<TextMateCore> {
+export function loadEditorCore(): Promise<TextMateCore> {
   editorCorePromise ??= createTextMateCore([]).catch((error: unknown) => {
     editorCorePromise = undefined
     throw error
@@ -47,17 +48,20 @@ class TextMateTokenizerState implements Monaco.languages.IState {
   }
 }
 
-function createTokensProvider(
+export function createTokensProvider(
   { textmate }: TextMateCore,
   grammar: Grammar,
   fallbackScopeName: string,
-  mapTokenScopes?: (scopes: readonly string[]) => string
+  mapTokenScopes?: (scopes: readonly string[]) => string,
+  onLineTokens?: (line: string, tokens: readonly IToken[]) => void,
+  lineAttempts?: number
 ): TextMateTokensProvider {
   const plain = (state: TextMateTokenizerState): Monaco.languages.ILineTokens => ({
     endState: state,
     tokens: [{ startIndex: 0, scopes: fallbackScopeName }]
   })
   const skipped = new TextMateTokenizerState(null)
+  let warm = false
   return {
     getInitialState() {
       return new TextMateTokenizerState(textmate.INITIAL)
@@ -67,13 +71,17 @@ function createTokensProvider(
       if (!ruleStack) {
         return plain(skipped)
       }
-      const result = tokenizeLineWithinLimits(line, (text, timeLimitMs) =>
-        grammar.tokenizeLine(text, ruleStack, timeLimitMs)
+      const result = tokenizeLineWithinLimits(
+        line,
+        (text, timeLimitMs) => grammar.tokenizeLine(text, ruleStack, timeLimitMs),
+        lineAttempts ?? (warm ? undefined : COLD_GRAMMAR_LINE_ATTEMPTS)
       )
       if (!result) {
         // Why: the line's end state is unknown, so continuing would color later lines from a guess.
         return plain(skipped)
       }
+      warm = true
+      onLineTokens?.(line, result.tokens)
       return {
         endState: new TextMateTokenizerState(result.ruleStack),
         tokens: result.tokens.map((token) => ({

src/renderer/src/lib/monaco-setup.ts

@@ -11,6 +11,7 @@ import tsWorker from 'monaco-editor/esm/vs/language/typescript/ts.worker?worker'
 import { registerAstroLanguage } from './monaco-languages/register-astro'
 import { registerJsonlLanguage } from './monaco-languages/register-jsonl'
 import { registerLeanLanguage } from './monaco-languages/register-lean'
+import { registerMarkdownLanguage } from './monaco-languages/register-markdown'
 import { registerNimLanguage } from './monaco-languages/register-nim'
 import { registerQuartoLanguage } from './monaco-languages/register-quarto'
 import { registerShellMarkdownAliases } from './monaco-languages/register-shell-markdown-aliases'
@@ -76,6 +77,7 @@ runMonacoSetupSteps([
   ['Zig language registration', () => registerZigLanguage(monaco)],
   ['JSONL language registration', () => registerJsonlLanguage(monaco)],
   ['Quarto language registration', () => registerQuartoLanguage(monaco)],
+  ['Markdown language registration', () => registerMarkdownLanguage(monaco)],
   ['shell Markdown alias registration', () => registerShellMarkdownAliases(monaco)],
   ['delayer cancellation guard', installMonacoDelayerCancellationGuard],
   ['diff editor disposal guard', () => installMonacoDiffEditorDisposalGuard(monaco)],