From d3943c6d81e98e62060f52853302eea9bbc56b9b Mon Sep 17 00:00:00 2001 From: Yoshihiko Kunisato Date: Tue, 6 Oct 2026 09:18:05 +0900 Subject: [PATCH] feat(editor): highlight Quarto and R Markdown files (#17772) Highlight Quarto and R Markdown documents using the bundled Markdown, YAML, R, Python, and JavaScript tokenizers. Preserve guarded startup registration and bound inherited inline embed recursion. Thanks to Yoshihiko Kunisato (@ykunisato) for the implementation and regression tests. Fixes #17771 Co-authored-by: Yoshihiko Kunisato <12838333+ykunisato@users.noreply.github.com> Co-authored-by: Claude Opus 5 --- .../journal-submission-positions.test.ts | 1 - src/renderer/src/env.d.ts | 5 + .../src/lib/file-type-icon-extension-table.ts | 3 + src/renderer/src/lib/language-detect.test.ts | 8 + src/renderer/src/lib/language-detect.ts | 5 + .../register-quarto-adversarial.test.ts | 78 +++++++++ .../monaco-languages/register-quarto.test.ts | 163 ++++++++++++++++++ .../lib/monaco-languages/register-quarto.ts | 104 +++++++++++ src/renderer/src/lib/monaco-setup.ts | 2 + 9 files changed, 368 insertions(+), 1 deletion(-) create mode 100644 src/renderer/src/lib/monaco-languages/register-quarto-adversarial.test.ts create mode 100644 src/renderer/src/lib/monaco-languages/register-quarto.test.ts create mode 100644 src/renderer/src/lib/monaco-languages/register-quarto.ts diff --git a/src/main/native-chat/agent-session-journal/journal-submission-positions.test.ts b/src/main/native-chat/agent-session-journal/journal-submission-positions.test.ts index 2ae99a42b45..3e05bbdbbcc 100644 --- a/src/main/native-chat/agent-session-journal/journal-submission-positions.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-submission-positions.test.ts @@ -7,7 +7,6 @@ import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' -import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import { agentSessionFailureFact } from '../../../shared/agent-session-failure' import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' import { diff --git a/src/renderer/src/env.d.ts b/src/renderer/src/env.d.ts index 63a0b063ddd..a4359ea87d2 100644 --- a/src/renderer/src/env.d.ts +++ b/src/renderer/src/env.d.ts @@ -12,6 +12,11 @@ declare module 'monaco-editor/esm/vs/basic-languages/python/python.js' { export const language: languages.IMonarchLanguage } +declare module 'monaco-editor/esm/vs/basic-languages/markdown/markdown.js' { + export const conf: languages.LanguageConfiguration + export const language: languages.IMonarchLanguage +} + // Monaco ships these contributions without public type declarations. We only // touch the paste-override surface, so declare the minimal shape we use. declare module 'monaco-editor/esm/vs/editor/contrib/clipboard/browser/clipboard.js' { diff --git a/src/renderer/src/lib/file-type-icon-extension-table.ts b/src/renderer/src/lib/file-type-icon-extension-table.ts index 8ecd418ddb2..94774618a84 100644 --- a/src/renderer/src/lib/file-type-icon-extension-table.ts +++ b/src/renderer/src/lib/file-type-icon-extension-table.ts @@ -142,9 +142,12 @@ export const FILE_ICON_BY_EXTENSION: Record = { psd: FileImage, pub: FileKey, py: FileCode, + qmd: FileText, r: FileCode, rar: FileArchive, rb: FileCode, + rmarkdown: FileText, + rmd: FileText, rst: FileText, rs: FileCode, rtf: FileText, diff --git a/src/renderer/src/lib/language-detect.test.ts b/src/renderer/src/lib/language-detect.test.ts index b8cd6e45353..ec8a79ce80f 100644 --- a/src/renderer/src/lib/language-detect.test.ts +++ b/src/renderer/src/lib/language-detect.test.ts @@ -77,6 +77,14 @@ describe('detectLanguage', () => { expect(detectLanguage('api/v1/service.proto')).toBe('proto') }) + it('maps Quarto and R Markdown files to the quarto language id', () => { + expect(detectLanguage('slides/talk.qmd')).toBe('quarto') + expect(detectLanguage('analysis/report.Rmd')).toBe('quarto') + expect(detectLanguage('/home/remote/report.rmarkdown')).toBe('quarto') + expect(detectLanguage('C:\\repo\\NOTES.QMD')).toBe('quarto') + expect(detectLanguage('README.md')).toBe('markdown') + }) + it('maps .jsonl files to the dedicated jsonl language id (case-insensitive)', () => { expect(detectLanguage('/home/user/.claude/sessions/transcript.jsonl')).toBe('jsonl') expect(detectLanguage('C:\\Users\\alice\\.codex\\LOG.JSONL')).toBe('jsonl') diff --git a/src/renderer/src/lib/language-detect.ts b/src/renderer/src/lib/language-detect.ts index b90b5a36ae2..a58c0e8e5c7 100644 --- a/src/renderer/src/lib/language-detect.ts +++ b/src/renderer/src/lib/language-detect.ts @@ -31,6 +31,11 @@ const EXT_TO_LANGUAGE: Record = { '.ipynb': 'notebook', '.md': 'markdown', '.mdx': 'markdown', + // Why: Quarto/R Markdown are Markdown plus a YAML header and executable + // cells; the 'markdown' grammar leaves ```{r}/```{python} cells uncolored. + '.qmd': 'quarto', + '.rmd': 'quarto', + '.rmarkdown': 'quarto', '.mmd': 'mermaid', '.mermaid': 'mermaid', '.css': 'css', diff --git a/src/renderer/src/lib/monaco-languages/register-quarto-adversarial.test.ts b/src/renderer/src/lib/monaco-languages/register-quarto-adversarial.test.ts new file mode 100644 index 00000000000..c70f5bbe943 --- /dev/null +++ b/src/renderer/src/lib/monaco-languages/register-quarto-adversarial.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from 'vitest' +import { + createMonarchTokenizer, + endEmbeddedLanguages, + measureNestedDepth, + tokenizeMonarchDocument, + tokenTypeAt +} from './monarch-tokenizer-test-harness' +import { EMBED_ENTRY_REST_OF_LINE_BUDGET } from './monarch-embed-entry-budget' +import { QUARTO_LANGUAGE_ID, quartoMonarchLanguage } from './register-quarto' + +const FENCE = '```' + +function tokenize(source: string[]) { + return tokenizeMonarchDocument(QUARTO_LANGUAGE_ID, quartoMonarchLanguage, source.join('\n')) +} + +describe('Quarto adversarial documents', () => { + it('bounds inherited Markdown script and style embed recursion', () => { + for (const tag of ['script', 'style']) { + const tokenizer = createMonarchTokenizer(QUARTO_LANGUAGE_ID, quartoMonarchLanguage) + const { error, maxNestedDepth } = measureNestedDepth(tokenizer, [ + `<${tag}>x`.repeat(800) + ]) + expect(error).toBeUndefined() + expect(maxNestedDepth).toBeLessThanOrEqual(EMBED_ENTRY_REST_OF_LINE_BUDGET) + } + }) + + it('preserves ordinary inline script and style highlighting', () => { + for (const tag of ['script', 'style']) { + const lines = tokenize([`<${tag}>x`, '# After']) + expect(lines[0].tokens.some((token) => token.language !== QUARTO_LANGUAGE_ID)).toBe(true) + expect(tokenTypeAt(lines[1], 0)).toBe('keyword') + } + }) + + it('does not treat front matter after a blank first line as YAML', () => { + expect(endEmbeddedLanguages(tokenize(['', '---', 'title: demo', '---']))).toEqual([ + null, + null, + null, + null + ]) + }) + + it('keeps malformed cell headers out of embedded languages', () => { + for (const header of ['{python', '{=}', '{}', '{{python}']) { + expect(endEmbeddedLanguages(tokenize([`${FENCE}${header}`, 'x', FENCE]))).not.toContain( + 'python' + ) + } + }) + + it('does not close cells on fence-like code or trailing text', () => { + expect( + endEmbeddedLanguages( + tokenize([`${FENCE}{python}`, `${FENCE} python`, `${FENCE} # comment`, FENCE, '# After']) + ) + ).toEqual(['python', 'python', 'python', null, null]) + }) + + it('closes unknown engines and escaped long fences without swallowing headings', () => { + for (const header of ['{unknown-engine}', '{{python}}']) { + const lines = tokenize([`\`\`\`\`${header}`, FENCE, 'inside', '`````', '# After']) + expect(lines[3].endEmbeddedLanguageId).toBeNull() + expect(tokenTypeAt(lines[4], 0)).toBe('keyword') + } + }) + + it('accepts YAML end markers and whitespace after closing fences', () => { + expect( + endEmbeddedLanguages( + tokenize(['---', 'title: demo', '...', `${FENCE}{r}`, 'x <- 1', `${FENCE} \t`, '# After']) + ) + ).toEqual(['yaml', 'yaml', null, 'r', 'r', null, null]) + }) +}) diff --git a/src/renderer/src/lib/monaco-languages/register-quarto.test.ts b/src/renderer/src/lib/monaco-languages/register-quarto.test.ts new file mode 100644 index 00000000000..63f4c0fe15a --- /dev/null +++ b/src/renderer/src/lib/monaco-languages/register-quarto.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, it, vi } from 'vitest' +import { + endEmbeddedLanguages, + tokenizeMonarchDocument, + tokenTypeAt +} from './monarch-tokenizer-test-harness' +import { + QUARTO_LANGUAGE_ID, + quartoLanguageConfiguration, + quartoMonarchLanguage, + registerQuartoLanguage +} from './register-quarto' + +const FENCE = '```' +const LONG_FENCE = '````' + +function tokenizeQuarto(source: string) { + return tokenizeMonarchDocument(QUARTO_LANGUAGE_ID, quartoMonarchLanguage, source) +} + +function embedsFor(source: string): (string | null)[] { + return endEmbeddedLanguages(tokenizeQuarto(source)) +} + +function createMonacoMock(existingLanguageIds: string[] = ['markdown']) { + const languages = existingLanguageIds.map((id) => ({ id })) + return { + languages: { + getLanguages: vi.fn(() => languages), + register: vi.fn((entry: { id: string }) => { + languages.push({ id: entry.id }) + }), + setLanguageConfiguration: vi.fn(), + setMonarchTokensProvider: vi.fn() + } + } +} + +describe('registerQuartoLanguage', () => { + it('registers the quarto language, tokenizer, and configuration once', () => { + const monaco = createMonacoMock() + + registerQuartoLanguage(monaco) + registerQuartoLanguage(monaco) + + expect(monaco.languages.register).toHaveBeenCalledTimes(1) + expect(monaco.languages.register).toHaveBeenCalledWith( + expect.objectContaining({ + id: QUARTO_LANGUAGE_ID, + extensions: ['.qmd', '.rmd', '.rmarkdown'] + }) + ) + expect(monaco.languages.setMonarchTokensProvider).toHaveBeenCalledWith( + QUARTO_LANGUAGE_ID, + quartoMonarchLanguage + ) + expect(monaco.languages.setLanguageConfiguration).toHaveBeenCalledWith( + QUARTO_LANGUAGE_ID, + quartoLanguageConfiguration + ) + }) + + it('hands a leading --- block to yaml and returns to markdown', () => { + expect(embedsFor(['---', 'format: revealjs', '---', '# Title'].join('\n'))).toEqual([ + 'yaml', + 'yaml', + null, + null + ]) + }) + + it('treats a mid-document --- as markdown, not front matter', () => { + // Why: revealjs decks use `---` as a slide separator, so only line 1 may open YAML. + expect(embedsFor(['# Title', '', '---', '', '## Next slide'].join('\n'))).toEqual([ + null, + null, + null, + null, + null + ]) + }) + + it('colors executable cells with the engine language', () => { + expect( + embedsFor([`${FENCE}{r setup, include=FALSE}`, 'x <- 1', FENCE, 'after'].join('\n')) + ).toEqual(['r', 'r', null, null]) + expect(embedsFor([`${FENCE}{python}`, 'print(1)', FENCE].join('\n'))).toEqual([ + 'python', + 'python', + null + ]) + expect(embedsFor([`${FENCE}{=html}`, 'x', FENCE].join('\n'))[0]).toBe('html') + // {ojs} and {d3} are JavaScript dialects Monaco has no language id for. + expect(embedsFor([`${FENCE}{ojs}`, 'x = 1', FENCE].join('\n'))[0]).toBe('javascript') + }) + + it('leaves an engine Monaco does not know uncolored instead of failing', () => { + expect(embedsFor([`${FENCE}{tikz}`, '\\draw;', FENCE, 'after'].join('\n'))).toEqual([ + 'tikz', + 'tikz', + null, + null + ]) + }) + + it('keeps an escaped ```{{python}} cell out of the engine tokenizer', () => { + // Why: Quarto's double-brace form shows a cell without running it. + const lines = tokenizeQuarto([`${FENCE}{{python}}`, 'print(1)', FENCE, '## After'].join('\n')) + expect(endEmbeddedLanguages(lines)).toEqual([null, null, null, null]) + expect(tokenTypeAt(lines[3], 0)).toBe('keyword') + }) + + it('closes a cell only on a fence at least as long as the one that opened it', () => { + // Why: markdown's own code-block states close on exactly three backticks, so a + // ````-fenced cell used to run to the end of the file. + expect( + embedsFor( + [ + `${LONG_FENCE}{python}`, + `print("${FENCE}")`, + FENCE, + 'still inside', + LONG_FENCE, + '# After' + ].join('\n') + ) + ).toEqual(['python', 'python', 'python', 'python', null, null]) + // A closing fence longer than the opener still closes, as CommonMark requires. + expect(embedsFor([`${FENCE}{r}`, 'x', LONG_FENCE, '# After'].join('\n'))).toEqual([ + 'r', + 'r', + null, + null + ]) + }) + + it('routes long plain fences through the same fence-aware states', () => { + // Why: ````-fenced blocks are how a Quarto document shows a ``` fence, and + // markdown's own fence rules stop at three backticks. + const lines = tokenizeQuarto( + [LONG_FENCE, `${FENCE}r`, 'x', FENCE, LONG_FENCE, '## After'].join('\n') + ) + expect(endEmbeddedLanguages(lines)).toEqual([null, null, null, null, null, null]) + expect(tokenTypeAt(lines[5], 0)).toBe('keyword') + }) + + it('keeps plain markdown fences and headings working', () => { + expect(embedsFor([`${FENCE}python`, 'print(1)', FENCE, 'text'].join('\n'))).toEqual([ + 'python', + 'python', + null, + null + ]) + const heading = tokenizeQuarto('## Slide title')[0] + expect(tokenTypeAt(heading, 0)).toBe('keyword') + }) + + it('marks pandoc fenced divs', () => { + const lines = tokenizeQuarto(['::: {.callout-note}', 'text', ':::'].join('\n')) + expect(tokenTypeAt(lines[0], 0)).toBe('meta.separator') + expect(tokenTypeAt(lines[2], 0)).toBe('meta.separator') + }) +}) diff --git a/src/renderer/src/lib/monaco-languages/register-quarto.ts b/src/renderer/src/lib/monaco-languages/register-quarto.ts new file mode 100644 index 00000000000..bc0c7ea6122 --- /dev/null +++ b/src/renderer/src/lib/monaco-languages/register-quarto.ts @@ -0,0 +1,104 @@ +import type * as Monaco from 'monaco-editor' +import { + conf as markdownConf, + language as markdownLanguage +} from 'monaco-editor/esm/vs/basic-languages/markdown/markdown.js' + +import { tagCloseWithinEmbedBudget } from './monarch-embed-entry-budget' + +type QuartoMonaco = { + languages: Pick< + typeof Monaco.languages, + 'getLanguages' | 'register' | 'setLanguageConfiguration' | 'setMonarchTokensProvider' + > +} + +export const QUARTO_LANGUAGE_ID = 'quarto' + +// Quarto files are edited as Markdown: same comment syntax and bracket pairs. +export const quartoLanguageConfiguration: Monaco.languages.LanguageConfiguration = markdownConf + +const markdownTokenizer = markdownLanguage.tokenizer + +// `$S2` substitutes the opening fence; a regex literal would interpret `$` as an anchor. +const CLOSING_FENCE = '^\\s*$S2`*\\s*$' + +// Extend Markdown so bundled rules and embedded tokenizers stay authoritative. +export const quartoMonarchLanguage: Monaco.languages.IMonarchLanguage = { + ...markdownLanguage, + tokenPostfix: '.qmd', + start: 'quartoStart', + tokenizer: { + ...markdownTokenizer, + // Inline script/style embeds inherit Markdown's recursion risk. + tag: [ + ...markdownTokenizer.tag.map((rule): Monaco.languages.IMonarchLanguageRule => + Array.isArray(rule) && rule[0] instanceof RegExp && rule[0].source === '>' + ? [tagCloseWithinEmbedBudget, rule[1]] + : rule + ), + [/>/, 'tag', '@pop'] + ], + // Only the initial state may interpret `---` as YAML front matter. + quartoStart: [ + [ + /^---\s*$/, + { token: 'meta.separator', switchTo: '@quartoFrontMatter', nextEmbedded: 'yaml' } + ], + [/.*/, { token: '@rematch', switchTo: '@root' }] + ], + quartoFrontMatter: [ + [/^(?:---|\.\.\.)\s*$/, { token: 'meta.separator', switchTo: '@root', nextEmbedded: '@pop' }], + [/.*$/, 'variable.source'] + ], + root: [ + // Escaped cells must consume their closing fence without starting an embed. + [/^\s*(`{3,})\s*\{\{[^}]*\}\}.*$/, { token: 'string', next: '@quartoRawCell.$1' }], + // ```{ojs} / ```{d3} are JavaScript dialects Monaco has no language id for. + [ + /^\s*(`{3,})\s*\{\s*(?:ojs|d3)\b[^}]*\}.*$/, + { token: 'string', next: '@quartoCell.$1', nextEmbedded: 'javascript' } + ], + // Unknown engine ids fall back to uncolored content in Monaco. + [ + /^\s*(`{3,})\s*\{=?\s*([A-Za-z][\w.+-]*)[^}]*\}.*$/, + { token: 'string', next: '@quartoCell.$1', nextEmbedded: '$2' } + ], + // Long plain fences must keep shorter inner fences inside the block. + [ + /^\s*(`{4,})\s*((?:\w|[/\-#])+).*$/, + { token: 'string', next: '@quartoCell.$1', nextEmbedded: '$2' } + ], + [/^\s*(`{4,})\s*$/, { token: 'string', next: '@quartoRawCell.$1' }], + // Pandoc fenced divs: ::: {.callout-note} + [/^\s*:{3,}.*$/, 'meta.separator'], + ...markdownTokenizer.root + ], + // Embedded-language exits match the pattern alone; guards cannot enforce fence length. + quartoCell: [ + [CLOSING_FENCE, { token: 'string', next: '@pop', nextEmbedded: '@pop' }], + [/.*$/, 'variable.source'] + ], + quartoRawCell: [ + [CLOSING_FENCE, { token: 'string', next: '@pop' }], + [/.*$/, 'variable.source'] + ] + } +} + +export function registerQuartoLanguage(monaco: QuartoMonaco): void { + const languageAlreadyRegistered = monaco.languages + .getLanguages() + .some((language) => language.id === QUARTO_LANGUAGE_ID) + if (languageAlreadyRegistered) { + return + } + + monaco.languages.register({ + id: QUARTO_LANGUAGE_ID, + extensions: ['.qmd', '.rmd', '.rmarkdown'], + aliases: ['Quarto', 'quarto', 'R Markdown'] + }) + monaco.languages.setLanguageConfiguration(QUARTO_LANGUAGE_ID, quartoLanguageConfiguration) + monaco.languages.setMonarchTokensProvider(QUARTO_LANGUAGE_ID, quartoMonarchLanguage) +} diff --git a/src/renderer/src/lib/monaco-setup.ts b/src/renderer/src/lib/monaco-setup.ts index 5c2c69d24cb..be9b001adf4 100644 --- a/src/renderer/src/lib/monaco-setup.ts +++ b/src/renderer/src/lib/monaco-setup.ts @@ -11,6 +11,7 @@ import tsWorker from 'monaco-editor/esm/vs/language/typescript/ts.worker?worker' import { registerAstroLanguage } from './monaco-languages/register-astro' import { registerJsonlLanguage } from './monaco-languages/register-jsonl' import { registerNimLanguage } from './monaco-languages/register-nim' +import { registerQuartoLanguage } from './monaco-languages/register-quarto' import { registerShellMarkdownAliases } from './monaco-languages/register-shell-markdown-aliases' import { registerSvelteLanguage } from './monaco-languages/register-svelte' import { registerTypstLanguage } from './monaco-languages/register-typst' @@ -85,6 +86,7 @@ runMonacoSetupSteps([ ['Nim language registration', () => registerNimLanguage(monaco)], ['Typst language registration', () => registerTypstLanguage(monaco)], ['JSONL language registration', () => registerJsonlLanguage(monaco)], + ['Quarto language registration', () => registerQuartoLanguage(monaco)], ['shell Markdown alias registration', () => registerShellMarkdownAliases(monaco)], ['delayer cancellation guard', installMonacoDelayerCancellationGuard], ['diff editor disposal guard', () => installMonacoDiffEditorDisposalGuard(monaco)],