From 84decfd2a2a7f4b9f604fe15343a65447e0e22ce Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:46:30 -0700 Subject: [PATCH] feat(editor): VS Code markdown highlighting and sticky headings in source mode (#34537) * feat(editor): VS Code markdown highlighting and sticky headings in source mode * fix(editor): address review on markdown TextMate highlighting - resolve fence grammars from the first word of the info string - give fence grammars the cold-compile retry budget - fall back to Monarch markdown when TextMate fails to load - bound heading-scan line length --- .../markdown-heading-outline.test.ts | 54 ++++++ .../markdown-heading-outline.ts | 112 ++++++++++++ .../markdown-textmate-tokens.test.ts | 64 +++++++ .../markdown-textmate-tokens.ts | 167 ++++++++++++++++++ .../lib/monaco-languages/register-markdown.ts | 62 +++++++ .../textmate-token-provider.ts | 20 ++- src/renderer/src/lib/monaco-setup.ts | 2 + 7 files changed, 475 insertions(+), 6 deletions(-) create mode 100644 src/renderer/src/lib/monaco-languages/markdown-heading-outline.test.ts create mode 100644 src/renderer/src/lib/monaco-languages/markdown-heading-outline.ts create mode 100644 src/renderer/src/lib/monaco-languages/markdown-textmate-tokens.test.ts create mode 100644 src/renderer/src/lib/monaco-languages/markdown-textmate-tokens.ts create mode 100644 src/renderer/src/lib/monaco-languages/register-markdown.ts diff --git a/src/renderer/src/lib/monaco-languages/markdown-heading-outline.test.ts b/src/renderer/src/lib/monaco-languages/markdown-heading-outline.test.ts new file mode 100644 index 00000000000..1e09be4d866 --- /dev/null +++ b/src/renderer/src/lib/monaco-languages/markdown-heading-outline.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest' +import { + buildMarkdownHeadingSections, + type MarkdownHeadingSection +} from './markdown-heading-outline' + +type Outline = { title: string; lines: [number, number]; children?: Outline[] } + +function outline(sections: MarkdownHeadingSection[]): Outline[] { + return sections.map((section) => ({ + title: section.title, + lines: [section.startLine, section.endLine], + ...(section.children.length > 0 ? { children: outline(section.children) } : {}) + })) +} + +describe('buildMarkdownHeadingSections', () => { + it('nests sections by level and ends each before the next same-or-higher heading', () => { + const lines = ['# A', 'text', '## B', '### C', 'x', '## D', '# E', 'tail'] + expect(outline(buildMarkdownHeadingSections(lines))).toEqual([ + { + title: 'A', + lines: [1, 6], + children: [ + { title: 'B', lines: [3, 5], children: [{ title: 'C', lines: [4, 5] }] }, + { title: 'D', lines: [6, 6] } + ] + }, + { title: 'E', lines: [7, 8] } + ]) + }) + + it('ignores fenced code, front matter, and thematic breaks but reads setext headings', () => { + const lines = [ + '---', + 'title: # not a heading', + '---', + '````md', + '# fenced', + '```', + '````', + 'Setext Title', + '===', + '', + '---', + '- item', + '---', + '## Closed ##' + ] + expect(outline(buildMarkdownHeadingSections(lines))).toEqual([ + { title: 'Setext Title', lines: [8, 14], children: [{ title: 'Closed', lines: [14, 14] }] } + ]) + }) +}) diff --git a/src/renderer/src/lib/monaco-languages/markdown-heading-outline.ts b/src/renderer/src/lib/monaco-languages/markdown-heading-outline.ts new file mode 100644 index 00000000000..3f27b3d8b5e --- /dev/null +++ b/src/renderer/src/lib/monaco-languages/markdown-heading-outline.ts @@ -0,0 +1,112 @@ +export type MarkdownHeadingSection = { + level: number + title: string + /** 1-based line of the heading text; a setext heading starts on its text line. */ + startLine: number + /** 1-based last line of the section's own heading line(s). */ + headingEndLine: number + /** 1-based last line before the next heading of the same or a higher level. */ + endLine: number + children: MarkdownHeadingSection[] +} + +// Why: bounds the heading regexes' backtracking on pathological lines; real headings are far shorter. +const MAX_HEADING_LINE_LENGTH = 1000 +const ATX_HEADING = /^ {0,3}(#{1,6})(?:[ \t]+(.*?))?(?:[ \t]+#+)?[ \t]*$/ +const SETEXT_UNDERLINE = /^ {0,3}(=+|-+)[ \t]*$/ +const FENCE_OPEN = /^ {0,3}(`{3,}|~{3,})/ +const FRONT_MATTER_FENCE = /^(---|\+\+\+)[ \t]*$/ +// Lines that cannot be a setext heading's text: they open their own block. +const NON_PARAGRAPH_START = /^ {0,3}(?:[-+*>|]|\d{1,9}[.)]|#{1,6}(?:[ \t]|$)|<)|^(?: {4}|\t)/ + +function isFenceClose(line: string, fence: string): boolean { + const trimmed = line.trimStart() + if (line.length - trimmed.length > 3 || !trimmed.startsWith(fence)) { + return false + } + const run = trimmed.match(fence[0] === '`' ? /^`+/ : /^~+/)?.[0] ?? '' + return run.length >= fence.length && trimmed.slice(run.length).trim() === '' +} + +function frontMatterEnd(lines: readonly string[]): number { + const open = lines[0]?.match(FRONT_MATTER_FENCE)?.[1] + if (!open) { + return 0 + } + for (let index = 1; index < lines.length; index += 1) { + const line = lines[index].trimEnd() + if (line === open || (open === '---' && line === '...')) { + return index + 1 + } + } + return 0 +} + +type FlatHeading = Omit + +// Why: a line scan keeps sticky scroll cheap on every edit; a full remark parse would block typing in large files. +function scanHeadings(lines: readonly string[]): FlatHeading[] { + const headings: FlatHeading[] = [] + let fence: string | null = null + let previousIsParagraph = false + for (let index = frontMatterEnd(lines); index < lines.length; index += 1) { + const line = lines[index] + if (fence) { + if (isFenceClose(line, fence)) { + fence = null + } + previousIsParagraph = false + continue + } + if (line.length > MAX_HEADING_LINE_LENGTH) { + previousIsParagraph = true + continue + } + const fenceOpen = line.match(FENCE_OPEN)?.[1] + if (fenceOpen) { + fence = fenceOpen + previousIsParagraph = false + continue + } + const atx = line.match(ATX_HEADING) + if (atx) { + headings.push({ + level: atx[1].length, + title: (atx[2] ?? '').trim(), + startLine: index + 1, + headingEndLine: index + 1 + }) + previousIsParagraph = false + continue + } + const underline = line.match(SETEXT_UNDERLINE)?.[1] + if (underline && previousIsParagraph) { + headings.push({ + level: underline[0] === '=' ? 1 : 2, + title: lines[index - 1].trim(), + startLine: index, + headingEndLine: index + 1 + }) + previousIsParagraph = false + continue + } + previousIsParagraph = line.trim() !== '' && !NON_PARAGRAPH_START.test(line) + } + return headings +} + +/** Heading sections nested by level, each spanning until the next heading of the same or a higher level. */ +export function buildMarkdownHeadingSections(lines: readonly string[]): MarkdownHeadingSection[] { + const roots: MarkdownHeadingSection[] = [] + const open: MarkdownHeadingSection[] = [] + for (const heading of scanHeadings(lines)) { + for (let last = open.at(-1); last && last.level >= heading.level; last = open.at(-1)) { + last.endLine = heading.startLine - 1 + open.pop() + } + const section: MarkdownHeadingSection = { ...heading, endLine: lines.length, children: [] } + ;(open.at(-1)?.children ?? roots).push(section) + open.push(section) + } + return roots +} diff --git a/src/renderer/src/lib/monaco-languages/markdown-textmate-tokens.test.ts b/src/renderer/src/lib/monaco-languages/markdown-textmate-tokens.test.ts new file mode 100644 index 00000000000..b9bf5a082b0 --- /dev/null +++ b/src/renderer/src/lib/monaco-languages/markdown-textmate-tokens.test.ts @@ -0,0 +1,64 @@ +import type * as Monaco from 'monaco-editor' +import { describe, expect, it, vi } from 'vitest' +import { loadNodeOniguruma } from '../syntax-highlighting/oniguruma-test-harness' +import { createMarkdownTokensProvider } from './markdown-textmate-tokens' + +vi.mock('../syntax-highlighting/oniguruma', () => ({ loadOniguruma: loadNodeOniguruma })) + +function tokenizeLines(provider: Monaco.languages.TokensProvider, lines: string[]): string[][] { + let state = provider.getInitialState() + return lines.map((line) => { + const result = provider.tokenize(line, state) + state = result.endState + return result.tokens.map((token) => token.scopes) + }) +} + +describe('createMarkdownTokensProvider', () => { + it('styles markdown with Dark+ rules and highlights a fence once its grammar loads', async () => { + const defineTheme = vi.fn() + let replacement: Monaco.languages.TokensProvider | undefined + const provider = await createMarkdownTokensProvider({ + editor: { defineTheme }, + languages: { + setTokensProvider: (_language, next) => { + if ('tokenize' in next && 'getInitialState' in next && !('tokenizeEncoded' in next)) { + replacement = next + } + return { dispose: () => {} } + } + } + }) + + expect(defineTheme).toHaveBeenCalledWith( + 'vs-dark', + expect.objectContaining({ + rules: expect.arrayContaining([ + { token: 'md.markup.heading', foreground: '#569cd6', fontStyle: 'bold' } + ]) + }) + ) + const [heading, list] = tokenizeLines(provider, ['## Quick', '- **State:** open']) + expect(new Set(heading)).toEqual(new Set(['md.markup.heading.markdown'])) + expect(list).toEqual([ + 'md.punctuation.definition.list.begin.markdown', + 'md.text.html.markdown', + 'md.markup.bold.markdown', + 'md.markup.bold.markdown', + 'md.markup.bold.markdown', + 'md.text.html.markdown' + ]) + + const fence = ['```ts', 'const x = 1', '```'] + tokenizeLines(provider, fence) + const refreshed = await vi.waitFor(() => { + if (!replacement) { + throw new Error('fence grammar has not loaded') + } + return replacement + }) + expect(tokenizeLines(refreshed, ['```ts title="a.ts"', 'const x = 1', '```'])[1]).toContain( + 'md.storage.type.ts' + ) + }) +}) diff --git a/src/renderer/src/lib/monaco-languages/markdown-textmate-tokens.ts b/src/renderer/src/lib/monaco-languages/markdown-textmate-tokens.ts new file mode 100644 index 00000000000..0f37936e4a1 --- /dev/null +++ b/src/renderer/src/lib/monaco-languages/markdown-textmate-tokens.ts @@ -0,0 +1,167 @@ +import type * as Monaco from 'monaco-editor' +import type { ThemeRegistrationRaw } from 'shiki/core' +import type { IToken } from 'shiki/textmate' +import type { BundledLanguageInfo } from 'shiki/types' +import { COLD_GRAMMAR_LINE_ATTEMPTS } from '../syntax-highlighting/textmate-core' +import { createTokensProvider, loadEditorCore } from './textmate-token-provider' + +type MonacoModule = { + editor: Pick + languages: Pick +} + +const MARKDOWN_LANGUAGE_ID = 'markdown' +const MARKDOWN_SCOPE_NAME = 'text.html.markdown' +// Why: namespaced rules color only markdown tokens, leaving every Monarch language's vs/vs-dark colors untouched. +const MARKDOWN_TOKEN_PREFIX = 'md.' +// The grammar's catch-all fence rule; a specific fence rule takes over once its grammar loads. +const UNKNOWN_FENCE_LANGUAGE_SCOPE = 'fenced_code.block.language' +// The catch-all rule captures the whole info string, e.g. `ts title="a.ts"`. +const FENCE_INFO_SEPARATOR = /[\s{,:?]/ + +type ThemeRule = Monaco.editor.ITokenThemeRule + +function simpleScopeSelectors(scope: string | string[] | undefined): string[] { + const selectors = typeof scope === 'string' ? scope.split(',') : (scope ?? []) + // Monaco matches one scope by dotted prefix, so descendant and exclusion selectors cannot apply. + return selectors + .map((selector) => selector.trim()) + .filter((selector) => /^[^\s]+$/.test(selector)) +} + +function markdownThemeRules(theme: ThemeRegistrationRaw): ThemeRule[] { + return (theme.tokenColors ?? []).flatMap(({ scope, settings }) => + simpleScopeSelectors(scope).map((selector) => ({ + token: `${MARKDOWN_TOKEN_PREFIX}${selector}`, + ...(settings.foreground ? { foreground: settings.foreground } : {}), + ...(settings.fontStyle !== undefined ? { fontStyle: settings.fontStyle } : {}) + })) + ) +} + +function hasMatchingSelector(selectors: ReadonlySet, scope: string): boolean { + for (let end = scope.length; end > 0; end = scope.lastIndexOf('.', end - 1)) { + if (selectors.has(scope.slice(0, end))) { + return true + } + } + return false +} + +/** Monaco takes one scope per token, so pick the innermost one a theme rule styles, as VS Code's cascade would. */ +function createScopeMapper(selectors: ReadonlySet): (scopes: readonly string[]) => string { + return (scopes) => { + for (let index = scopes.length - 1; index >= 0; index -= 1) { + if (hasMatchingSelector(selectors, scopes[index])) { + return `${MARKDOWN_TOKEN_PREFIX}${scopes[index]}` + } + } + return MARKDOWN_TOKEN_PREFIX + MARKDOWN_SCOPE_NAME + } +} + +function fenceLanguagesByLabel( + bundledLanguagesInfo: readonly BundledLanguageInfo[], + embeddedLanguageIds: readonly string[] +): Map { + const embedded = new Set(embeddedLanguageIds) + const byLabel = new Map() + for (const info of bundledLanguagesInfo) { + if (embedded.has(info.id)) { + for (const label of [info.id, ...(info.aliases ?? [])]) { + byLabel.set(label, info) + } + } + } + return byLabel +} + +/** VS Code's markdown grammar with Dark+/Light+ colors; fence grammars load on first sight. */ +export async function createMarkdownTokensProvider( + monaco: MonacoModule +): Promise { + const [editorCore, { default: markdownGrammars }, { bundledLanguagesInfo }, dark, light] = + await Promise.all([ + loadEditorCore(), + import('shiki/langs/markdown.mjs'), + import('shiki/langs'), + import('shiki/themes/dark-plus.mjs'), + import('shiki/themes/light-plus.mjs') + ]) + await editorCore.core.loadLanguage(...markdownGrammars) + + const darkRules = markdownThemeRules(dark.default) + const lightRules = markdownThemeRules(light.default) + // Why: Orca's editors all use the built-in theme names; redefining them keeps every setTheme caller unchanged. + monaco.editor.defineTheme('vs-dark', { + base: 'vs-dark', + inherit: true, + rules: darkRules, + colors: {} + }) + monaco.editor.defineTheme('vs', { base: 'vs', inherit: true, rules: lightRules, colors: {} }) + const mapScopes = createScopeMapper( + new Set( + [...darkRules, ...lightRules].map((rule) => rule.token.slice(MARKDOWN_TOKEN_PREFIX.length)) + ) + ) + + const fenceLanguages = fenceLanguagesByLabel( + bundledLanguagesInfo, + markdownGrammars[0]?.embeddedLangsLazy ?? [] + ) + const requestedFenceLanguages = new Set() + let refreshQueued = false + + function refreshProvider(): void { + if (refreshQueued) { + return + } + refreshQueued = true + queueMicrotask(() => { + refreshQueued = false + // Why: loading a grammar rebuilds markdown's grammar; replacing the provider retokenizes from its fresh start state. + monaco.languages.setTokensProvider(MARKDOWN_LANGUAGE_ID, buildProvider()) + }) + } + + function loadFenceLanguage(label: string): void { + const info = fenceLanguages.get(label.trim().split(FENCE_INFO_SEPARATOR)[0].toLowerCase()) + if (!info || requestedFenceLanguages.has(info.id)) { + return + } + requestedFenceLanguages.add(info.id) + void info + .import() + .then(({ default: grammars }) => editorCore.core.loadLanguage(...grammars)) + .then(refreshProvider) + .catch((error: unknown) => { + console.warn( + `[markdown] failed to load the ${info.id} fence grammar; it stays plain`, + error + ) + }) + } + + function observeFenceLabels(line: string, tokens: readonly IToken[]): void { + for (const token of tokens) { + if (token.scopes.at(-1) === UNKNOWN_FENCE_LANGUAGE_SCOPE) { + loadFenceLanguage(line.slice(token.startIndex, token.endIndex)) + } + } + } + + function buildProvider(): Monaco.languages.TokensProvider { + return createTokensProvider( + editorCore, + editorCore.core.getLanguage(MARKDOWN_LANGUAGE_ID), + MARKDOWN_SCOPE_NAME, + mapScopes, + observeFenceLabels, + // Why: each fence grammar compiles on its first block, long after the markdown root has warmed. + COLD_GRAMMAR_LINE_ATTEMPTS + ) + } + + return buildProvider() +} diff --git a/src/renderer/src/lib/monaco-languages/register-markdown.ts b/src/renderer/src/lib/monaco-languages/register-markdown.ts new file mode 100644 index 00000000000..a1d5f833da5 --- /dev/null +++ b/src/renderer/src/lib/monaco-languages/register-markdown.ts @@ -0,0 +1,62 @@ +import type * as Monaco from 'monaco-editor' +import { + buildMarkdownHeadingSections, + type MarkdownHeadingSection +} from './markdown-heading-outline' + +type MonacoModule = Pick + +const MARKDOWN_LANGUAGE_ID = 'markdown' + +function toDocumentSymbol( + monaco: MonacoModule, + model: Monaco.editor.ITextModel, + section: MarkdownHeadingSection +): Monaco.languages.DocumentSymbol { + return { + name: section.title || '#'.repeat(section.level), + detail: '', + kind: monaco.languages.SymbolKind.String, + tags: [], + range: { + startLineNumber: section.startLine, + startColumn: 1, + endLineNumber: section.endLine, + endColumn: model.getLineMaxColumn(section.endLine) + }, + selectionRange: { + startLineNumber: section.startLine, + startColumn: 1, + endLineNumber: section.headingEndLine, + endColumn: model.getLineMaxColumn(section.headingEndLine) + }, + children: section.children.map((child) => toDocumentSymbol(monaco, model, child)) + } +} + +/** VS Code's markdown editing: TextMate colors, no bracket-pair colors, and a heading outline for sticky scroll. */ +export function registerMarkdownLanguage(monaco: MonacoModule): void { + // Why: Monaco merges configurations per field, so this keeps the built-in markdown config and drops only bracket colors. + monaco.languages.setLanguageConfiguration(MARKDOWN_LANGUAGE_ID, { colorizedBracketPairs: [] }) + + monaco.languages.registerTokensProviderFactory(MARKDOWN_LANGUAGE_ID, { + // Monaco caches this result, so a TextMate load failure falls back to the built-in Monarch tokenizer. + create: () => + import('./markdown-textmate-tokens') + .then(({ createMarkdownTokensProvider }) => createMarkdownTokensProvider(monaco)) + .catch(async (error: unknown) => { + console.warn('[markdown] TextMate highlighting failed to load; using Monarch', error) + const { language } = + await import('monaco-editor/esm/vs/basic-languages/markdown/markdown.js') + return language + }) + }) + + monaco.languages.registerDocumentSymbolProvider(MARKDOWN_LANGUAGE_ID, { + displayName: 'Markdown headings', + provideDocumentSymbols: (model) => + buildMarkdownHeadingSections(model.getLinesContent()).map((section) => + toDocumentSymbol(monaco, model, section) + ) + }) +} diff --git a/src/renderer/src/lib/monaco-languages/textmate-token-provider.ts b/src/renderer/src/lib/monaco-languages/textmate-token-provider.ts index a37a60aff26..f99b8be3bcc 100644 --- a/src/renderer/src/lib/monaco-languages/textmate-token-provider.ts +++ b/src/renderer/src/lib/monaco-languages/textmate-token-provider.ts @@ -1,7 +1,8 @@ import type * as Monaco from 'monaco-editor' import type { Grammar, LanguageRegistration } from 'shiki/core' -import type { StateStack } from 'shiki/textmate' +import type { IToken, StateStack } from 'shiki/textmate' import { + COLD_GRAMMAR_LINE_ATTEMPTS, createTextMateCore, tokenizeLineWithinLimits, type TextMateCore @@ -20,7 +21,7 @@ export type TextMateTokensProviderOptions = { let editorCorePromise: Promise | undefined // Why: chat's registry holds the catalogue's Nim and Typst under the same scope names as these vendored grammars. -function loadEditorCore(): Promise { +export function loadEditorCore(): Promise { editorCorePromise ??= createTextMateCore([]).catch((error: unknown) => { editorCorePromise = undefined throw error @@ -47,17 +48,20 @@ class TextMateTokenizerState implements Monaco.languages.IState { } } -function createTokensProvider( +export function createTokensProvider( { textmate }: TextMateCore, grammar: Grammar, fallbackScopeName: string, - mapTokenScopes?: (scopes: readonly string[]) => string + mapTokenScopes?: (scopes: readonly string[]) => string, + onLineTokens?: (line: string, tokens: readonly IToken[]) => void, + lineAttempts?: number ): TextMateTokensProvider { const plain = (state: TextMateTokenizerState): Monaco.languages.ILineTokens => ({ endState: state, tokens: [{ startIndex: 0, scopes: fallbackScopeName }] }) const skipped = new TextMateTokenizerState(null) + let warm = false return { getInitialState() { return new TextMateTokenizerState(textmate.INITIAL) @@ -67,13 +71,17 @@ function createTokensProvider( if (!ruleStack) { return plain(skipped) } - const result = tokenizeLineWithinLimits(line, (text, timeLimitMs) => - grammar.tokenizeLine(text, ruleStack, timeLimitMs) + const result = tokenizeLineWithinLimits( + line, + (text, timeLimitMs) => grammar.tokenizeLine(text, ruleStack, timeLimitMs), + lineAttempts ?? (warm ? undefined : COLD_GRAMMAR_LINE_ATTEMPTS) ) if (!result) { // Why: the line's end state is unknown, so continuing would color later lines from a guess. return plain(skipped) } + warm = true + onLineTokens?.(line, result.tokens) return { endState: new TextMateTokenizerState(result.ruleStack), tokens: result.tokens.map((token) => ({ diff --git a/src/renderer/src/lib/monaco-setup.ts b/src/renderer/src/lib/monaco-setup.ts index 402d4964ebf..d332395b1c9 100644 --- a/src/renderer/src/lib/monaco-setup.ts +++ b/src/renderer/src/lib/monaco-setup.ts @@ -11,6 +11,7 @@ import tsWorker from 'monaco-editor/esm/vs/language/typescript/ts.worker?worker' import { registerAstroLanguage } from './monaco-languages/register-astro' import { registerJsonlLanguage } from './monaco-languages/register-jsonl' import { registerLeanLanguage } from './monaco-languages/register-lean' +import { registerMarkdownLanguage } from './monaco-languages/register-markdown' import { registerNimLanguage } from './monaco-languages/register-nim' import { registerQuartoLanguage } from './monaco-languages/register-quarto' import { registerShellMarkdownAliases } from './monaco-languages/register-shell-markdown-aliases' @@ -76,6 +77,7 @@ runMonacoSetupSteps([ ['Zig language registration', () => registerZigLanguage(monaco)], ['JSONL language registration', () => registerJsonlLanguage(monaco)], ['Quarto language registration', () => registerQuartoLanguage(monaco)], + ['Markdown language registration', () => registerMarkdownLanguage(monaco)], ['shell Markdown alias registration', () => registerShellMarkdownAliases(monaco)], ['delayer cancellation guard', installMonacoDelayerCancellationGuard], ['diff editor disposal guard', () => installMonacoDiffEditorDisposalGuard(monaco)],