From 2cb2bef67e3a8c9435c8ebd30ce6f2f0b4dadf08 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:08:42 +0000 Subject: [PATCH] feat(native-chat): highlight code blocks and shell commands (#26343) * feat(native-chat): highlight code blocks and shell commands Uses Shiki rather than the highlight.js already in the preview: it can resume mid-block while a reply streams. * fix(native-chat): leave the tool row command unhighlighted * refactor(native-chat): highlight code through a shared TextMate interface Chat code now goes through one renderer highlighting interface built on Shiki core and the Oniguruma WASM the editor already loads for Nim and Typst, with VS Code Light+/Dark+. Lines that are too long or too slow leave the rest of the block plain, failed grammar loads retry from a mounted block, every block shares one time slice, finished-block caching counts retained text, each theme keeps its own formatting, Vue template expressions are colored, and tool-row commands take the grammar of the shell they ran in. * chore(deps): drop pnpm self-install entries from the lockfile Keeps the lockfile change to Shiki alone; the earlier commit picked up the @pnpm/exe entries pnpm 12 writes when it runs in a fresh checkout. * test(native-chat): give cold grammar loads time in highlighting tests The first colored block in a test file loads its grammar and the regex engine, which took over a second on a busy CI runner. --------- Co-authored-by: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> --- package.json | 1 + pnpm-lock.yaml | 146 +++++++ .../NativeChatAssistantMessageRow.tsx | 4 +- .../NativeChatCodeBlock.load-failure.test.tsx | 58 +++ .../native-chat/NativeChatCodeBlock.test.tsx | 81 +++- .../native-chat/NativeChatCodeBlock.tsx | 21 +- .../native-chat/NativeChatHighlightedCode.tsx | 83 ++++ .../native-chat/NativeChatMessageRow.test.tsx | 4 + .../NativeChatReasoningDisclosure.tsx | 4 +- .../native-chat/NativeChatToolLine.test.tsx | 36 +- .../native-chat/NativeChatToolLine.tsx | 10 +- .../native-chat-command-language.test.ts | 39 ++ .../native-chat-command-language.ts | 46 +++ .../native-chat-tool-line-label.ts | 7 +- .../src/hooks/use-highlighted-syntax.ts | 135 +++++++ .../textmate-token-provider.ts | 24 +- .../incremental-syntax-tokenizer.test.ts | 151 ++++++++ .../incremental-syntax-tokenizer.ts | 99 +++++ .../oniguruma-test-harness.ts | 18 + .../src/lib/syntax-highlighting/oniguruma.ts | 24 ++ .../syntax-highlight-scheduler.test.ts | 49 +++ .../syntax-highlight-scheduler.ts | 54 +++ .../syntax-highlighter.test.ts | 196 ++++++++++ .../syntax-highlighting/syntax-highlighter.ts | 358 ++++++++++++++++++ .../vue-template-injections.ts | 25 ++ 25 files changed, 1644 insertions(+), 29 deletions(-) create mode 100644 src/renderer/src/components/native-chat/NativeChatCodeBlock.load-failure.test.tsx create mode 100644 src/renderer/src/components/native-chat/NativeChatHighlightedCode.tsx create mode 100644 src/renderer/src/components/native-chat/native-chat-command-language.test.ts create mode 100644 src/renderer/src/components/native-chat/native-chat-command-language.ts create mode 100644 src/renderer/src/hooks/use-highlighted-syntax.ts create mode 100644 src/renderer/src/lib/syntax-highlighting/incremental-syntax-tokenizer.test.ts create mode 100644 src/renderer/src/lib/syntax-highlighting/incremental-syntax-tokenizer.ts create mode 100644 src/renderer/src/lib/syntax-highlighting/oniguruma-test-harness.ts create mode 100644 src/renderer/src/lib/syntax-highlighting/oniguruma.ts create mode 100644 src/renderer/src/lib/syntax-highlighting/syntax-highlight-scheduler.test.ts create mode 100644 src/renderer/src/lib/syntax-highlighting/syntax-highlight-scheduler.ts create mode 100644 src/renderer/src/lib/syntax-highlighting/syntax-highlighter.test.ts create mode 100644 src/renderer/src/lib/syntax-highlighting/syntax-highlighter.ts create mode 100644 src/renderer/src/lib/syntax-highlighting/vue-template-injections.ts diff --git a/package.json b/package.json index 18ff04a8082..a7c729b6f09 100644 --- a/package.json +++ b/package.json @@ -318,6 +318,7 @@ "remark-rehype": "11.1.2", "remend": "1.4.0", "shadcn": "^4.13.1", + "shiki": "4.5.0", "sonner": "2.0.7", "tailwind-merge": "^3.7.0", "tailwindcss": "^4.2.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 544ad1f899e..ef6f038cb1b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -582,6 +582,9 @@ importers: shadcn: specifier: ^4.13.1 version: 4.13.1(supports-color@7.2.0)(typescript@7.0.2) + shiki: + specifier: 4.5.0 + version: 4.5.0 sonner: specifier: 2.0.7 version: 2.0.7(react-dom@19.2.8(react@19.2.8))(react@19.2.8) @@ -3164,6 +3167,37 @@ packages: resolution: {integrity: sha512-8mpfsoPeRhesY3pOrzNZBL8uG6N5GVX1EHLBYbd4gzKs+c7vaEIqpTNK5VrffU33qQN4cwpP2v3u4aPPBU32sw==} engines: {node: '>=16'} + '@shikijs/core@4.5.0': + resolution: {integrity: sha512-6CIgwq+bt2JZ3OSjb/nBkkwEf/CfpHcbam+Mg5Pllco7HAGRQ2/EFhZrrJtbyNTgKwknfVS+llsXGygiDGwJWw==} + engines: {node: '>=20'} + + '@shikijs/engine-javascript@4.5.0': + resolution: {integrity: sha512-Ed6UdEj84LyYkwnxM5CvSMWAD+id9ExTYri0/+x41U9zrgNifPSS9lZm0JqBP0VfYI8yr8tY5nbNq9+p9R0kFA==} + engines: {node: '>=20'} + + '@shikijs/engine-oniguruma@4.5.0': + resolution: {integrity: sha512-dKVmbJecVdB9recl8a/CU4jdpNXyjJQrIcesIkpS70s5HS02ohpnCSpMUbotPnEUzbAcLQ+S4hwZYbkBBiTcJw==} + engines: {node: '>=20'} + + '@shikijs/langs@4.5.0': + resolution: {integrity: sha512-wwBTqwduXSz0X4v9Qa2EWeFaYJEw1clCncWjoRNQRbVDu3SPRTRWucnouYapKj3b8Q+sk/41Vbc4B5c8nuYzLA==} + engines: {node: '>=20'} + + '@shikijs/primitive@4.5.0': + resolution: {integrity: sha512-UM7ShlA8oz0OWxLLO8BmO9Ip6fVqdkt8yF0EH7nOCqHfrx1eEAzF0ZUVK7L+I8NEgeu1PEWSRlxaiPPeCb4iDg==} + engines: {node: '>=20'} + + '@shikijs/themes@4.5.0': + resolution: {integrity: sha512-zGb87H3UBbpuI4yoh5UguB1XbU+2BDU3kflM7UfZ+Cy2hWo9lBtqjSVK20gmLY6ukYl040hAypGXMgud9vul8Q==} + engines: {node: '>=20'} + + '@shikijs/types@4.5.0': + resolution: {integrity: sha512-drtPrtD8N0DENyD2cyt6Ajy+IWsiCLgMxODwLTpvH1UsN6a9jf90tNTPjqwHvA+0Lzu9e0txk8DjfqH1p9UJRw==} + engines: {node: '>=20'} + + '@shikijs/vscode-textmate@10.0.2': + resolution: {integrity: sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==} + '@sindresorhus/is@4.6.0': resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==} engines: {node: '>=10'} @@ -3883,6 +3917,9 @@ packages: '@types/hast@3.0.4': resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==} + '@types/hast@3.0.5': + resolution: {integrity: sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==} + '@types/http-cache-semantics@4.2.0': resolution: {integrity: sha512-L3LgimLHXtGkWikKnsPg0/VFx9OGZaC+eN1u4r+OB1XRqH3meBIAVC2zr1WdMH+RHmnRkqliQAOHNJ/E0j/e0Q==} @@ -5450,6 +5487,9 @@ packages: hast-util-sanitize@5.0.2: resolution: {integrity: sha512-3yTWghByc50aGS7JlGhk61SPenfE/p1oaFeNwkOOyrscaOkMGrcW9+Cy/QAIOBpZxP1yqDIzFMR0+Np0i0+usg==} + hast-util-to-html@9.0.5: + resolution: {integrity: sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==} + hast-util-to-jsx-runtime@2.3.6: resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==} @@ -6481,6 +6521,12 @@ packages: resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==} engines: {node: '>=18'} + oniguruma-parser@0.12.2: + resolution: {integrity: sha512-6HVa5oIrgMC6aA6WF6XyyqbhRPJrKR02L20+2+zpDtO5QAzGHAUGw5TKQvwi5vctNnRHkJYmjAhRVQF2EKdTQw==} + + oniguruma-to-es@4.3.6: + resolution: {integrity: sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA==} + open@11.0.0: resolution: {integrity: sha512-smsWv2LzFjP03xmvFoJ331ss6h+jixfA4UUV/Bsiyuu4YJPfN+FIQGOIiv4w9/+MoHkfkJ22UIaQWRVFRfH6Vw==} engines: {node: '>=20'} @@ -6966,6 +7012,15 @@ packages: resolution: {integrity: sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==} engines: {node: '>=8'} + regex-recursion@6.0.2: + resolution: {integrity: sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==} + + regex-utilities@2.3.0: + resolution: {integrity: sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==} + + regex@6.1.0: + resolution: {integrity: sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg==} + rehype-highlight@7.0.2: resolution: {integrity: sha512-k158pK7wdC2qL3M5NcZROZ2tR/l7zOzjxXd5VGdcfIyoijjQqpHd3JKtYSBDpDZ38UI2WJWuFAtkMDxmx5kstA==} @@ -7248,6 +7303,10 @@ packages: sherpa-onnx@1.12.37: resolution: {integrity: sha512-3luwSdHwR8BtJiiFwqHfb15FE2FX0KsN4aOBbfq9Ma23r3w9C3bprFc/WBusXk56nUbzcEN5YczN7t9w1JwdtQ==} + shiki@4.5.0: + resolution: {integrity: sha512-OcRW72ngQvc+MSVWBRIovC/BTW589iUE/Vxav7VeM4RcY9MUjqB+IR6ILG1k9nuW6LiG/nct9uma4/MBtnZbmQ==} + engines: {node: '>=20'} + shipstyles@1.0.0: resolution: {integrity: sha512-1AcihfXo/mSxqeOdoKqq/5TNv+vSH0dBYPFq88PUX5FBkK5pRmY5TP9SuMNio85PQ+euUI67+KpSVk1cUVDCgA==} @@ -10193,6 +10252,46 @@ snapshots: '@shaderfrog/glsl-parser@7.0.1': {} + '@shikijs/core@4.5.0': + dependencies: + '@shikijs/primitive': 4.5.0 + '@shikijs/types': 4.5.0 + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.5 + hast-util-to-html: 9.0.5 + + '@shikijs/engine-javascript@4.5.0': + dependencies: + '@shikijs/types': 4.5.0 + '@shikijs/vscode-textmate': 10.0.2 + oniguruma-to-es: 4.3.6 + + '@shikijs/engine-oniguruma@4.5.0': + dependencies: + '@shikijs/types': 4.5.0 + '@shikijs/vscode-textmate': 10.0.2 + + '@shikijs/langs@4.5.0': + dependencies: + '@shikijs/types': 4.5.0 + + '@shikijs/primitive@4.5.0': + dependencies: + '@shikijs/types': 4.5.0 + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.5 + + '@shikijs/themes@4.5.0': + dependencies: + '@shikijs/types': 4.5.0 + + '@shikijs/types@4.5.0': + dependencies: + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.5 + + '@shikijs/vscode-textmate@10.0.2': {} + '@sindresorhus/is@4.6.0': {} '@sindresorhus/merge-streams@4.0.0': {} @@ -10877,6 +10976,10 @@ snapshots: dependencies: '@types/unist': 3.0.3 + '@types/hast@3.0.5': + dependencies: + '@types/unist': 3.0.3 + '@types/http-cache-semantics@4.2.0': {} '@types/json-schema@7.0.15': {} @@ -12633,6 +12736,20 @@ snapshots: '@ungap/structured-clone': 1.3.1 unist-util-position: 5.0.0 + hast-util-to-html@9.0.5: + dependencies: + '@types/hast': 3.0.4 + '@types/unist': 3.0.3 + ccount: 2.0.1 + comma-separated-tokens: 2.0.3 + hast-util-whitespace: 3.0.0 + html-void-elements: 3.0.0 + mdast-util-to-hast: 13.2.1 + property-information: 7.1.0 + space-separated-tokens: 2.0.2 + stringify-entities: 4.0.4 + zwitch: 2.0.4 + hast-util-to-jsx-runtime@2.3.6(supports-color@7.2.0): dependencies: '@types/estree': 1.0.8 @@ -13847,6 +13964,14 @@ snapshots: dependencies: mimic-function: 5.0.1 + oniguruma-parser@0.12.2: {} + + oniguruma-to-es@4.3.6: + dependencies: + oniguruma-parser: 0.12.2 + regex: 6.1.0 + regex-recursion: 6.0.2 + open@11.0.0: dependencies: default-browser: 5.5.0 @@ -14489,6 +14614,16 @@ snapshots: indent-string: 4.0.0 strip-indent: 3.0.0 + regex-recursion@6.0.2: + dependencies: + regex-utilities: 2.3.0 + + regex-utilities@2.3.0: {} + + regex@6.1.0: + dependencies: + regex-utilities: 2.3.0 + rehype-highlight@7.0.2: dependencies: '@types/hast': 3.0.4 @@ -14854,6 +14989,17 @@ snapshots: sherpa-onnx@1.12.37: {} + shiki@4.5.0: + dependencies: + '@shikijs/core': 4.5.0 + '@shikijs/engine-javascript': 4.5.0 + '@shikijs/engine-oniguruma': 4.5.0 + '@shikijs/langs': 4.5.0 + '@shikijs/themes': 4.5.0 + '@shikijs/types': 4.5.0 + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.5 + shipstyles@1.0.0: {} side-channel-list@1.0.1: diff --git a/src/renderer/src/components/native-chat/NativeChatAssistantMessageRow.tsx b/src/renderer/src/components/native-chat/NativeChatAssistantMessageRow.tsx index ca3a2e8550b..4eb95baa651 100644 --- a/src/renderer/src/components/native-chat/NativeChatAssistantMessageRow.tsx +++ b/src/renderer/src/components/native-chat/NativeChatAssistantMessageRow.tsx @@ -5,7 +5,7 @@ import type { MessageRowProps } from './NativeChatMessageRow' import { NativeChatPacedMarkdown } from './NativeChatPacedMarkdown' import { NATIVE_CHAT_QUOTE_SOURCE_PROPS } from './native-chat-quote-selection' import { NativeChatToolRun } from './NativeChatToolRun' -import { NativeChatCodeBlock } from './NativeChatCodeBlock' +import { NativeChatCodeBlock, NativeChatPlainCodeBlock } from './NativeChatCodeBlock' import { NativeChatAgentControls, NativeChatImageAttachments } from './NativeChatTranscriptChrome' import type { useNativeChatWorkRun } from './use-native-chat-work-run' @@ -74,7 +74,7 @@ export function NativeChatAssistantMessageRow({ content={words} variant="document" className="text-sm native-chat-message-text" - renderCodeBlock={NativeChatCodeBlock} + renderCodeBlock={isSystem ? NativeChatPlainCodeBlock : NativeChatCodeBlock} onLinkClick={onLinkClick} allowFileUriLinks={allowFileUriLinks} linkifyFilePaths={onLinkClick !== undefined} diff --git a/src/renderer/src/components/native-chat/NativeChatCodeBlock.load-failure.test.tsx b/src/renderer/src/components/native-chat/NativeChatCodeBlock.load-failure.test.tsx new file mode 100644 index 00000000000..628757f2505 --- /dev/null +++ b/src/renderer/src/components/native-chat/NativeChatCodeBlock.load-failure.test.tsx @@ -0,0 +1,58 @@ +// @vitest-environment happy-dom + +import '@testing-library/jest-dom/vitest' +import { cleanup, render, waitFor } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { loadSyntaxLanguage } from '@/lib/syntax-highlighting/syntax-highlighter' +import { NativeChatCodeBlock } from './NativeChatCodeBlock' + +// Its own file, so the regex engine has not loaded before the failure. +const oniguruma = vi.hoisted(() => ({ fails: false })) + +vi.mock('@/lib/syntax-highlighting/oniguruma', async () => { + const { loadNodeOniguruma } = await import('@/lib/syntax-highlighting/oniguruma-test-harness') + return { + loadOniguruma: () => + oniguruma.fails ? Promise.reject(new Error('regex engine unavailable')) : loadNodeOniguruma() + } +}) + +afterEach(() => { + cleanup() + vi.restoreAllMocks() +}) + +describe('NativeChatCodeBlock grammar load failure', () => { + it('colors a block still streaming once a failed grammar load succeeds on retry', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}) + let now = 0 + vi.spyOn(performance, 'now').mockImplementation(() => now) + oniguruma.fails = true + const block = (code: string) => ( + + {code} + + ) + try { + const { container, rerender } = render(block('fn main() {\n')) + await expect(loadSyntaxLanguage('rust')).resolves.toBe('failed') + expect(container.querySelector('pre span')).toBeNull() + + oniguruma.fails = false + now += 5000 + rerender(block('fn main() {\n let answer = 42;\n')) + + await waitFor( + () => { + expect(container.querySelector('pre span[style*="--syntax-dark"]')).not.toBeNull() + }, + { timeout: 10_000 } + ) + expect(container.querySelector('pre')?.textContent).toBe( + 'fn main() {\n let answer = 42;\n' + ) + } finally { + oniguruma.fails = false + } + }) +}) diff --git a/src/renderer/src/components/native-chat/NativeChatCodeBlock.test.tsx b/src/renderer/src/components/native-chat/NativeChatCodeBlock.test.tsx index c7cc84e838c..210332b5215 100644 --- a/src/renderer/src/components/native-chat/NativeChatCodeBlock.test.tsx +++ b/src/renderer/src/components/native-chat/NativeChatCodeBlock.test.tsx @@ -3,7 +3,13 @@ import '@testing-library/jest-dom/vitest' import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' import { afterEach, describe, expect, it, vi } from 'vitest' -import { NativeChatCodeBlock } from './NativeChatCodeBlock' +import { loadSyntaxLanguage } from '@/lib/syntax-highlighting/syntax-highlighter' +import { NativeChatCodeBlock, NativeChatPlainCodeBlock } from './NativeChatCodeBlock' + +vi.mock('@/lib/syntax-highlighting/oniguruma', async () => ({ + loadOniguruma: (await import('@/lib/syntax-highlighting/oniguruma-test-harness')) + .loadNodeOniguruma +})) afterEach(() => { cleanup() @@ -41,4 +47,77 @@ describe('NativeChatCodeBlock', () => { }) expect(await screen.findByRole('button', { name: 'Copied' })).toBeInTheDocument() }) + + it('colors a fenced block without changing its text', async () => { + const code = 'const answer = 42\nconsole.log(answer)\n' + const { container } = render( + + {code} + + ) + const pre = container.querySelector('pre') + + await waitFor( + () => { + expect(pre?.querySelector('span[style*="--syntax-dark"]')).not.toBeNull() + }, + { timeout: 10_000 } + ) + expect(pre?.textContent).toBe(code) + }) + + it('keeps Windows line endings and finishes a block longer than one slice', async () => { + const code = `${'const value = compute(1, "two", [3])\r\n'.repeat(400)}const lastLine = 1\r\n` + const { container } = render( + + {code} + + ) + + expect(await screen.findByText('lastLine', {}, { timeout: 10_000 })).toBeInTheDocument() + expect(container.querySelector('pre')?.textContent).toBe(code) + }) + + it('leaves the code of a dimmed row uncolored', async () => { + const code = {'const dimmed = 1\n'} + await loadSyntaxLanguage('typescript') + const colored = render({code}) + expect(colored.container.querySelector('pre span')).not.toBeNull() + + const plain = render( + {code} + ) + + expect(plain.container.querySelector('pre span')).toBeNull() + expect(plain.container.querySelector('pre')?.textContent).toBe('const dimmed = 1\n') + }) + + it('paints a finished block in color at once when its row mounts again', async () => { + // Long enough that tokenizing it again could not finish within one render. + const block = ( + + {`${'const a=[1,2,3].map((x)=>x+1);f(a,{b:1,c:"s"});\n'.repeat(300)}const lastLine = 1\n`} + + ) + const first = render(block) + expect(await screen.findByText('lastLine', {}, { timeout: 10_000 })).toBeInTheDocument() + const colored = first.container.querySelectorAll('span[style*="--syntax-dark"]').length + first.unmount() + + const second = render(block) + + expect(second.container.querySelectorAll('span[style*="--syntax-dark"]')).toHaveLength(colored) + }) + + it('leaves a fence in an unknown language as plain text', async () => { + const { container } = render( + + {'plain words\n'} + + ) + const pre = container.querySelector('pre') + + await waitFor(() => expect(pre?.textContent).toBe('plain words\n')) + expect(pre?.querySelector('span')).toBeNull() + }) }) diff --git a/src/renderer/src/components/native-chat/NativeChatCodeBlock.tsx b/src/renderer/src/components/native-chat/NativeChatCodeBlock.tsx index 208802c8538..f6c266feed3 100644 --- a/src/renderer/src/components/native-chat/NativeChatCodeBlock.tsx +++ b/src/renderer/src/components/native-chat/NativeChatCodeBlock.tsx @@ -3,14 +3,17 @@ import { translate } from '@/i18n/i18n' import { cn } from '@/lib/utils' import { getCodeBlockLanguageLabel } from '@/components/editor/rich-markdown-code-block-languages' import { NativeChatCopyButton } from './NativeChatCopyButton' +import { NativeChatHighlightedCode } from './NativeChatHighlightedCode' /** Code fences need their own copy target rather than the whole chat message. */ export function NativeChatCodeBlock({ children, - language + language, + plain = false }: { children?: React.ReactNode language?: string + plain?: boolean }): React.JSX.Element { const code = extractCodeText(children) @@ -40,7 +43,13 @@ export function NativeChatCodeBlock({ language ? 'px-3.5 pt-0.5 pb-3' : 'p-3 pr-10' )} > - {children} + {language && code && !plain ? ( + + + + ) : ( + children + )} {code && !language ? ( +} + function extractCodeText(node: React.ReactNode): string { if (typeof node === 'string' || typeof node === 'number') { return String(node) diff --git a/src/renderer/src/components/native-chat/NativeChatHighlightedCode.tsx b/src/renderer/src/components/native-chat/NativeChatHighlightedCode.tsx new file mode 100644 index 00000000000..84e2757345e --- /dev/null +++ b/src/renderer/src/components/native-chat/NativeChatHighlightedCode.tsx @@ -0,0 +1,83 @@ +import { memo } from 'react' +import { useHighlightedSyntax } from '@/hooks/use-highlighted-syntax' +import type { SyntaxLine } from '@/lib/syntax-highlighting/incremental-syntax-tokenizer' +import { SYNTAX_FONT_STYLE, type SyntaxToken } from '@/lib/syntax-highlighting/syntax-highlighter' + +type TokenStyle = React.CSSProperties & Record<`--syntax-${string}`, string | undefined> + +// Both themes ride on the token so a scheme change needs no re-tokenizing. +const COLORED_TOKEN_CLASS = 'text-(color:--syntax-light) dark:text-(color:--syntax-dark)' +const FORMATTED_TOKEN_CLASS = + 'text-(color:--syntax-light) dark:text-(color:--syntax-dark) [font-style:var(--syntax-light-font-style)] dark:[font-style:var(--syntax-dark-font-style)] [font-weight:var(--syntax-light-font-weight)] dark:[font-weight:var(--syntax-dark-font-weight)] [text-decoration-line:var(--syntax-light-decoration)] dark:[text-decoration-line:var(--syntax-dark-decoration)]' + +function formatting(theme: 'light' | 'dark', fontStyle: number): TokenStyle { + const decorations = [ + fontStyle & SYNTAX_FONT_STYLE.underline ? 'underline' : '', + fontStyle & SYNTAX_FONT_STYLE.strikethrough ? 'line-through' : '' + ].filter(Boolean) + return { + [`--syntax-${theme}-font-style`]: fontStyle & SYNTAX_FONT_STYLE.italic ? 'italic' : 'normal', + [`--syntax-${theme}-font-weight`]: fontStyle & SYNTAX_FONT_STYLE.bold ? 'bold' : 'normal', + [`--syntax-${theme}-decoration`]: decorations.join(' ') || 'none' + } +} + +function tokenStyle(token: SyntaxToken): TokenStyle { + const colors: TokenStyle = { '--syntax-light': token.light, '--syntax-dark': token.dark } + return token.lightFontStyle || token.darkFontStyle + ? { + ...colors, + ...formatting('light', token.lightFontStyle), + ...formatting('dark', token.darkFontStyle) + } + : colors +} + +const HighlightedLine = memo(function HighlightedLine({ line }: { line: SyntaxLine }) { + return ( + <> + {line.tokens.map((token) => + // Uncolored text needs no element of its own. + token.content.trim() && + (token.light || token.dark || token.lightFontStyle || token.darkFontStyle) ? ( + + {token.content} + + ) : ( + token.content + ) + )} + {line.ending} + + ) +}) + +/** Colors `code` in place; its text is unchanged, so layout and copy match the plain fallback. */ +export const NativeChatHighlightedCode = memo(function NativeChatHighlightedCode({ + code, + language +}: { + code: string + language: string +}): React.ReactNode { + const progress = useHighlightedSyntax(code, language) + if (!progress) { + return code + } + return ( + <> + {progress.lines.map((line) => ( + + ))} + {code.slice(progress.highlightedLength)} + + ) +}) diff --git a/src/renderer/src/components/native-chat/NativeChatMessageRow.test.tsx b/src/renderer/src/components/native-chat/NativeChatMessageRow.test.tsx index 44ad235f4aa..4c0e470aa16 100644 --- a/src/renderer/src/components/native-chat/NativeChatMessageRow.test.tsx +++ b/src/renderer/src/components/native-chat/NativeChatMessageRow.test.tsx @@ -9,6 +9,10 @@ import type { NativeChatRewindSurface } from './use-native-chat-rewind' import { readNativeChatQuotableSelection } from './native-chat-quote-selection' const confirm = vi.hoisted(() => vi.fn()) +vi.mock('@/lib/syntax-highlighting/oniguruma', async () => ({ + loadOniguruma: (await import('@/lib/syntax-highlighting/oniguruma-test-harness')) + .loadNodeOniguruma +})) vi.mock('@/components/confirmation-dialog-context', () => ({ useConfirmationDialog: () => confirm })) diff --git a/src/renderer/src/components/native-chat/NativeChatReasoningDisclosure.tsx b/src/renderer/src/components/native-chat/NativeChatReasoningDisclosure.tsx index 5d8ded6e4df..62d7584d2cc 100644 --- a/src/renderer/src/components/native-chat/NativeChatReasoningDisclosure.tsx +++ b/src/renderer/src/components/native-chat/NativeChatReasoningDisclosure.tsx @@ -1,6 +1,6 @@ import { ChevronRight } from 'lucide-react' import type { CommentMarkdownLinkClickHandler } from '@/components/sidebar/CommentMarkdown' -import { NativeChatCodeBlock } from './NativeChatCodeBlock' +import { NativeChatPlainCodeBlock } from './NativeChatCodeBlock' import { NativeChatMarkdown } from './NativeChatMarkdown' /** A reasoning block's text, under the live activity line or its finished row: quieter prose in the @@ -27,7 +27,7 @@ export function NativeChatReasoningBody({ content={markdown} variant="document" className="text-sm native-chat-message-text" - renderCodeBlock={NativeChatCodeBlock} + renderCodeBlock={NativeChatPlainCodeBlock} onLinkClick={onLinkClick} allowFileUriLinks={allowFileUriLinks} linkifyFilePaths={onLinkClick !== undefined} diff --git a/src/renderer/src/components/native-chat/NativeChatToolLine.test.tsx b/src/renderer/src/components/native-chat/NativeChatToolLine.test.tsx index 1a1195c9cb5..854d5a7813d 100644 --- a/src/renderer/src/components/native-chat/NativeChatToolLine.test.tsx +++ b/src/renderer/src/components/native-chat/NativeChatToolLine.test.tsx @@ -1,11 +1,16 @@ // @vitest-environment happy-dom import '@testing-library/jest-dom/vitest' -import { cleanup, fireEvent, render, screen } from '@testing-library/react' -import { afterEach, describe, expect, it } from 'vitest' +import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' import { NativeChatToolLine } from './NativeChatToolLine' import { NativeChatToolRun } from './NativeChatToolRun' import type { NativeChatToolCallBlock } from '../../../../shared/native-chat-types' +vi.mock('@/lib/syntax-highlighting/oniguruma', async () => ({ + loadOniguruma: (await import('@/lib/syntax-highlighting/oniguruma-test-harness')) + .loadNodeOniguruma +})) + afterEach(cleanup) describe('tool sentence rows', () => { @@ -320,6 +325,33 @@ describe('tool sentence rows', () => { } ) + it.each([ + ['PowerShell', 'Get-ChildItem', true], + ['execute', 'Get-ChildItem', false] + ])('colors the full %s command only when its shell is known', async (name, verb, colored) => { + const command = `${verb} -Path ${'C:\\repo\\deep\\'.repeat(40)} | Where-Object { $_.Length -gt 10 }` + const { container } = render( + + ) + fireEvent.click(screen.getByRole('button')) + const detail = container.querySelector('pre') + + if (colored) { + // The first PowerShell block in this file loads the grammar and its regex engine. + await waitFor( + () => { + expect(detail?.querySelector('span[style*="--syntax-dark"]')).not.toBeNull() + }, + { timeout: 10_000 } + ) + } else { + expect(detail?.querySelector('span')).toBeNull() + } + expect(detail?.textContent).toBe(command) + }) + it('keeps a long plain command with surrounding whitespace complete', () => { const command = ` pnpm test ${'src/renderer/tests/long-path/'.repeat(170)}end.test.ts ` const { container } = render( diff --git a/src/renderer/src/components/native-chat/NativeChatToolLine.tsx b/src/renderer/src/components/native-chat/NativeChatToolLine.tsx index fbef701dc53..049f9106e53 100644 --- a/src/renderer/src/components/native-chat/NativeChatToolLine.tsx +++ b/src/renderer/src/components/native-chat/NativeChatToolLine.tsx @@ -18,6 +18,7 @@ import { import { NativeChatToolIcon } from './NativeChatToolIcon' import { NativeChatToolFileTarget } from './NativeChatToolFileTarget' import { NativeChatDiffView } from './NativeChatDiffView' +import { NativeChatHighlightedCode } from './NativeChatHighlightedCode' import { nativeChatToolLineLabel } from './native-chat-tool-line-label' import { diffFromText, diffFromToolCall } from './native-chat-diff' import { NativeChatExpandable } from './NativeChatExpandable' @@ -33,12 +34,14 @@ function ToolLineDetail({ block, body, fullCommand, + commandLanguage, results, onLinkClick }: { block: NativeChatBlock body: NativeChatToolResultBlock | undefined fullCommand: string | null + commandLanguage: string | null results: NonNullable onLinkClick?: CommentMarkdownLinkClickHandler }): React.JSX.Element { @@ -58,7 +61,11 @@ function ToolLineDetail({ data-native-chat-code-content className="max-h-64 overflow-auto whitespace-pre-wrap break-words rounded-lg border border-chat-code-border bg-chat-code-surface p-2 font-mono text-xs text-chat-foreground scrollbar-sleek" > - {fullCommand} + {commandLanguage ? ( + + ) : ( + fullCommand + )} ) : null} {body ? ( @@ -190,6 +197,7 @@ export function NativeChatToolLine({ block={block} body={body} fullCommand={fullCommand} + commandLanguage={label?.commandLanguage ?? null} results={results} onLinkClick={onLinkClick} /> diff --git a/src/renderer/src/components/native-chat/native-chat-command-language.test.ts b/src/renderer/src/components/native-chat/native-chat-command-language.test.ts new file mode 100644 index 00000000000..cb542834254 --- /dev/null +++ b/src/renderer/src/components/native-chat/native-chat-command-language.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest' +import { nativeChatCommandLanguage } from './native-chat-command-language' + +describe('nativeChatCommandLanguage', () => { + it.each([ + ['Bash', { command: 'git status' }, 'shellscript'], + ['PowerShell', { command: 'Get-ChildItem' }, 'powershell'], + // A bash tool still ran bash when its command calls another shell. + ['Bash', { command: 'cmd.exe /c dir' }, 'shellscript'] + ])('trusts a %s tool to name its shell', (name, input, language) => { + expect(nativeChatCommandLanguage(name, input)).toBe(language) + }) + + it.each([ + [{ command: "/bin/zsh -lc 'git status'" }, 'shellscript'], + [{ command: ['bash', '-lc', 'ls -la'] }, 'shellscript'], + [{ command: 'wsl.exe -e bash -lc "ls"' }, 'shellscript'], + [ + { + command: + '"C:\\\\WINDOWS\\\\System32\\\\WindowsPowerShell\\\\v1.0\\\\powershell.exe" -Command "Get-ChildItem"' + }, + 'powershell' + ], + [{ command: 'pwsh -NoProfile -Command Get-Date' }, 'powershell'], + [{ cmd: 'C:\\Windows\\System32\\cmd.exe /c dir' }, 'bat'] + ])('reads the shell a generic tool wrapped %j in', (input, language) => { + expect(nativeChatCommandLanguage('shell', input)).toBe(language) + }) + + it.each([ + { command: 'git status' }, + { command: 'bash' }, + { command: 'python -c "print(1)"' }, + {} + ])('leaves %j plain when nothing says which shell ran it', (input) => { + expect(nativeChatCommandLanguage('execute', input)).toBeNull() + }) +}) diff --git a/src/renderer/src/components/native-chat/native-chat-command-language.ts b/src/renderer/src/components/native-chat/native-chat-command-language.ts new file mode 100644 index 00000000000..5577b1ec2f1 --- /dev/null +++ b/src/renderer/src/components/native-chat/native-chat-command-language.ts @@ -0,0 +1,46 @@ +import { nativeChatToolInputText } from './native-chat-tool-input-text' + +// A tool named after its shell ran there, whatever the host's own shell is. +const LANGUAGE_BY_TOOL_NAME: ReadonlyMap = new Map([ + ['bash', 'shellscript'], + ['powershell', 'powershell'] +]) + +// The program a generic command tool spawned to run the command. +const LANGUAGE_BY_SHELL_PROGRAM: ReadonlyMap = new Map([ + ['bash', 'shellscript'], + ['sh', 'shellscript'], + ['zsh', 'shellscript'], + ['dash', 'shellscript'], + ['ksh', 'shellscript'], + ['wsl', 'shellscript'], + ['fish', 'fish'], + ['pwsh', 'powershell'], + ['powershell', 'powershell'], + ['cmd', 'bat'] +]) + +// The first word, quoted or bare, when arguments follow it. +const PROGRAM_WITH_ARGUMENTS = /^\s*(?:"([^"]+)"|'([^']+)'|(\S+))\s+\S/ + +function shellProgram(command: string): string | null { + const match = PROGRAM_WITH_ARGUMENTS.exec(command) + const path = match?.[1] ?? match?.[2] ?? match?.[3] + const name = path?.split(/[\\/]/).at(-1)?.toLowerCase() + return name?.replace(/\.exe$/, '') ?? null +} + +/** + * The grammar for a command row, from the shell the command actually ran in: a + * tool named after its shell, or the shell program the agent wrapped the + * command in. Null when neither says, so the command stays plain. + */ +export function nativeChatCommandLanguage(toolName: string, input: unknown): string | null { + const byName = LANGUAGE_BY_TOOL_NAME.get(toolName.trim().toLowerCase()) + if (byName) { + return byName + } + const command = nativeChatToolInputText(input, 'command') || nativeChatToolInputText(input, 'cmd') + const program = command ? shellProgram(command) : null + return (program && LANGUAGE_BY_SHELL_PROGRAM.get(program)) ?? null +} diff --git a/src/renderer/src/components/native-chat/native-chat-tool-line-label.ts b/src/renderer/src/components/native-chat/native-chat-tool-line-label.ts index c2d2179092c..72419af7f07 100644 --- a/src/renderer/src/components/native-chat/native-chat-tool-line-label.ts +++ b/src/renderer/src/components/native-chat/native-chat-tool-line-label.ts @@ -1,4 +1,5 @@ import { translate } from '@/i18n/i18n' +import { nativeChatCommandLanguage } from './native-chat-command-language' import { nativeChatToolCategory } from './native-chat-tool-category' import { nativeChatFullCommand, @@ -25,6 +26,8 @@ export function nativeChatToolLineLabel( filePath: string | null /** The whole command, when the row had to shorten it. */ commandDetail: string | null + /** Grammar of `commandDetail`; null when the row holds no command text. */ + commandLanguage: string | null } { const display = createToolInputDisplay(call.input) const category = nativeChatToolCategory(call.name, call.mcpIdentity) @@ -36,6 +39,7 @@ export function nativeChatToolLineLabel( (call.state === 'completed' || result !== undefined) const failed = call.state === 'failed' || result?.isError === true let commandDetail: string | null = null + let commandLanguage: string | null = null let verb: string | null = null let target = display.label let title = display.filePath ?? target @@ -46,6 +50,7 @@ export function nativeChatToolLineLabel( target = fullCommand === null ? target : summarizeToolInput(fullCommand) title = fullCommand ?? target if (fullCommand !== null) { + commandLanguage = nativeChatCommandLanguage(call.name, call.input) const plainCommand = nativeChatPlainCommandInput(call.input) if (plainCommand?.trim() === fullCommand) { commandDetail = plainCommand === target ? null : plainCommand @@ -115,5 +120,5 @@ export function nativeChatToolLineLabel( verb = translate('components.native-chat.tool.row.triedSearchWeb', 'Tried to search the web') } } - return { verb, target, title, command, filePath, commandDetail } + return { verb, target, title, command, filePath, commandDetail, commandLanguage } } diff --git a/src/renderer/src/hooks/use-highlighted-syntax.ts b/src/renderer/src/hooks/use-highlighted-syntax.ts new file mode 100644 index 00000000000..5d2c9b884dc --- /dev/null +++ b/src/renderer/src/hooks/use-highlighted-syntax.ts @@ -0,0 +1,135 @@ +import { useEffect, useMemo, useReducer, useRef, useState } from 'react' +import { + createIncrementalSyntaxTokenizer, + type SyntaxProgress +} from '@/lib/syntax-highlighting/incremental-syntax-tokenizer' +import { scheduleSyntaxHighlighting } from '@/lib/syntax-highlighting/syntax-highlight-scheduler' +import { + loadedSyntaxHighlighter, + loadSyntaxLanguage +} from '@/lib/syntax-highlighting/syntax-highlighter' +import { BoundedMap } from '../../../shared/bounded-map' + +// Each token is a DOM node, so only the start of a very long block is colored. +const MAX_HIGHLIGHTED_CODE_LENGTH = 50_000 +// Tokenizing runs on the renderer thread, so it is spent in slices shorter than a frame. +const RENDER_BUDGET_MS = 4 +const MAX_FINISHED_BYTES = 8 * 1024 * 1024 +const MAX_FINISHED_ENTRIES = 256 +// Rough retained size of one token object beyond its text. +const TOKEN_OVERHEAD_BYTES = 120 +const LINE_OVERHEAD_BYTES = 80 + +let renderDeadline: number | null = null + +/** One budget for every block that renders in the same task, however many mount at once. */ +function remainingRenderBudgetMs(): number { + if (renderDeadline === null) { + renderDeadline = performance.now() + RENDER_BUDGET_MS + setTimeout(() => { + renderDeadline = null + }, 0) + } + return Math.max(0, renderDeadline - performance.now()) +} + +/** UTF-16 text in the key and the tokens' copies of it, plus the objects holding them. */ +function finishedBytes(progress: SyntaxProgress, key: string): number { + let bytes = key.length * 2 + for (const line of progress.lines) { + bytes += LINE_OVERHEAD_BYTES + for (const token of line.tokens) { + bytes += TOKEN_OVERHEAD_BYTES + token.content.length * 2 + } + } + return bytes +} + +// Finished blocks, so a row scrolled back into view paints in color without tokenizing again. +const finished = new BoundedMap({ + maxEntries: MAX_FINISHED_ENTRIES, + maxBytes: MAX_FINISHED_BYTES, + sizeOf: finishedBytes +}) + +function finishedKey(language: string, source: string): string { + return `${language}\n${source}` +} + +/** + * Tokens for as much of `code` as has been colored so far, or null while there + * is nothing to color it with. The rest, `code.slice(highlightedLength)`, is + * still plain and arrives over later renders. + */ +export function useHighlightedSyntax(code: string, language: string): SyntaxProgress | null { + const source = + code.length > MAX_HIGHLIGHTED_CODE_LENGTH ? code.slice(0, MAX_HIGHLIGHTED_CODE_LENGTH) : code + const highlighter = loadedSyntaxHighlighter(language) + const [, rerender] = useReducer((count: number) => count + 1, 0) + + // Why: `source` is a dependency so a block still streaming asks again after a failed load. + useEffect(() => { + if (highlighter) { + return + } + let cancelled = false + void loadSyntaxLanguage(language).then((status) => { + if (!cancelled && status === 'ready') { + rerender() + } + }) + return () => { + cancelled = true + } + }, [language, highlighter, source]) + + const tokenize = useMemo( + () => (highlighter ? createIncrementalSyntaxTokenizer(highlighter) : null), + [highlighter] + ) + // Short blocks finish here and paint in color at once; longer ones continue below. + const started = useMemo( + () => + finished.get(finishedKey(language, source)) ?? + (tokenize ? tokenize(source, remainingRenderBudgetMs()) : null), + [language, source, tokenize] + ) + const [continued, setContinued] = useState<{ + after: SyntaxProgress + progress: SyntaxProgress + } | null>(null) + const progress = continued?.after === started ? continued.progress : started + + useEffect(() => { + if (!tokenize || !started || started.highlightedLength >= source.length) { + return + } + return scheduleSyntaxHighlighting((budgetMs) => { + const next = tokenize(source, budgetMs) + setContinued({ after: started, progress: next }) + return next.highlightedLength >= source.length + }) + }, [source, started, tokenize]) + + const latest = useRef({ language, source, progress }) + useEffect(() => { + latest.current = { language, source, progress } + }) + // Why: remembered on unmount, not on every streamed chunk, which would fill the cache with prefixes. + useEffect( + () => () => { + const last = latest.current + // A degraded block may only have lost its grammar mid-stream, so it is colored afresh next time. + if ( + last.progress && + !last.progress.degraded && + last.progress.highlightedLength >= last.source.length + ) { + finished.set(finishedKey(last.language, last.source), last.progress) + } + }, + [] + ) + + return progress +} diff --git a/src/renderer/src/lib/monaco-languages/textmate-token-provider.ts b/src/renderer/src/lib/monaco-languages/textmate-token-provider.ts index ace5ca9966e..b61e5c78e93 100644 --- a/src/renderer/src/lib/monaco-languages/textmate-token-provider.ts +++ b/src/renderer/src/lib/monaco-languages/textmate-token-provider.ts @@ -1,7 +1,7 @@ import type * as Monaco from 'monaco-editor' import { INITIAL, Registry } from 'vscode-textmate' import type { IGrammar, IOnigLib, IRawGrammar, StateStack } from 'vscode-textmate' -import onigurumaWasmUrl from 'vscode-oniguruma/release/onig.wasm?url' +import { loadOniguruma } from '../syntax-highlighting/oniguruma' type TextMateTokensProvider = Monaco.languages.TokensProvider @@ -13,24 +13,12 @@ export type TextMateTokensProviderOptions = { loadOniguruma?: () => Promise } -let browserOnigurumaPromise: Promise | undefined - async function loadBrowserOniguruma(): Promise { - browserOnigurumaPromise ??= (async () => { - const oniguruma = await import('vscode-oniguruma') - const response = await fetch(onigurumaWasmUrl) - if (!response.ok) { - throw new Error(`Failed to load TextMate regex engine from ${onigurumaWasmUrl}`) - } - - await oniguruma.loadWASM(response) - return { - createOnigScanner: oniguruma.createOnigScanner, - createOnigString: oniguruma.createOnigString - } - })() - - return browserOnigurumaPromise + const oniguruma = await loadOniguruma() + return { + createOnigScanner: oniguruma.createOnigScanner, + createOnigString: oniguruma.createOnigString + } } class TextMateTokenizerState implements Monaco.languages.IState { diff --git a/src/renderer/src/lib/syntax-highlighting/incremental-syntax-tokenizer.test.ts b/src/renderer/src/lib/syntax-highlighting/incremental-syntax-tokenizer.test.ts new file mode 100644 index 00000000000..aa8f73630a2 --- /dev/null +++ b/src/renderer/src/lib/syntax-highlighting/incremental-syntax-tokenizer.test.ts @@ -0,0 +1,151 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + createIncrementalSyntaxTokenizer, + type SyntaxProgress +} from './incremental-syntax-tokenizer' +import { + loadedSyntaxHighlighter, + loadSyntaxLanguage, + type SyntaxHighlighter, + type SyntaxToken +} from './syntax-highlighter' + +vi.mock('./oniguruma', async () => ({ + loadOniguruma: (await import('./oniguruma-test-harness')).loadNodeOniguruma +})) + +const DOCUMENT = [ + 'const greeting = `hello', + '// still inside the template string', + '${name}` /* a comment', + 'that spans lines */ + 1', + 'export default greeting', + '' +].join('\n') + +async function typescriptTokenizer(): Promise { + await loadSyntaxLanguage('typescript') + const highlighter = loadedSyntaxHighlighter('typescript') + if (!highlighter) { + throw new Error('typescript grammar did not load') + } + return highlighter +} + +function colorsOf(lines: readonly (readonly SyntaxToken[])[]): unknown { + return lines.map((line) => line.map((token) => [token.content, token.light, token.dark])) +} + +function colors(progress: SyntaxProgress): unknown { + return colorsOf(progress.lines.map((line) => line.tokens)) +} + +function text(progress: SyntaxProgress, code: string): string { + return ( + progress.lines + .map((line) => line.tokens.map((token) => token.content).join('') + line.ending) + .join('') + code.slice(progress.highlightedLength) + ) +} + +afterEach(() => { + vi.restoreAllMocks() +}) + +describe('createIncrementalSyntaxTokenizer', () => { + it('colors a streamed document exactly like a single full pass', async () => { + const tokenizer = await typescriptTokenizer() + const tokenize = createIncrementalSyntaxTokenizer(tokenizer) + + for (let length = 1; length <= DOCUMENT.length; length += 1) { + const streamed = DOCUMENT.slice(0, length) + const fullPass = tokenizer(streamed).lines + // A full pass reports the empty line after a final newline; nothing renders for it. + if (streamed.endsWith('\n')) { + fullPass.pop() + } + expect(colors(tokenize(streamed, Infinity)), streamed).toEqual(colorsOf(fullPass)) + } + }) + + // Markdown ends a fence that is still being written with a newline; a raw stream does not. + it.each(['', '\n'])( + 'keeps completed lines by identity while a document ending in %j grows', + async (ending) => { + const tokenize = createIncrementalSyntaxTokenizer(await typescriptTokenizer()) + const grown = `const a = 1\nconst b = 2${ending}` + + const before = tokenize(`const a = 1\nconst b${ending}`, Infinity) + const after = tokenize(grown, Infinity) + + expect(after.lines[0]).toBe(before.lines[0]) + expect(after.lines[1]).not.toBe(before.lines[1]) + expect(text(after, grown)).toBe(grown) + } + ) + + it('starts over when the document is replaced rather than extended', async () => { + const tokenizer = await typescriptTokenizer() + const tokenize = createIncrementalSyntaxTokenizer(tokenizer) + + tokenize('const a = `open\nstill open\n', Infinity) + const replaced = 'let b = 2\nlet c = 3' + + expect(colors(tokenize(replaced, Infinity))).toEqual( + colors(createIncrementalSyntaxTokenizer(tokenizer)(replaced, Infinity)) + ) + }) + + it('stops when its time is spent and finishes over later calls', async () => { + const tokenizer = await typescriptTokenizer() + // Every batch takes one tick of a clock that otherwise stands still. + let now = 0 + vi.spyOn(performance, 'now').mockImplementation(() => now) + const tokenize = createIncrementalSyntaxTokenizer((code, state) => { + now += 1 + return tokenizer(code, state) + }) + const code = 'const value = compute(1, "two", [3])\n'.repeat(200) + + let progress = tokenize(code, 1) + expect(progress.highlightedLength).toBeGreaterThan(0) + expect(progress.highlightedLength).toBeLessThan(code.length) + expect(text(progress, code)).toBe(code) + + while (progress.highlightedLength < code.length) { + progress = tokenize(code, 1) + } + expect(colors(progress)).toEqual( + colors(createIncrementalSyntaxTokenizer(tokenizer)(code, Infinity)) + ) + }) + + it('keeps every line after a skipped one plain, however the document streams', async () => { + const tokenize = createIncrementalSyntaxTokenizer(await typescriptTokenizer()) + const code = `const a = 1\nconst long = "${'x'.repeat(2000)}"\nconst b = 2\nconst c = 3\n` + + let progress = tokenize(code.slice(0, code.indexOf('const b')), Infinity) + progress = tokenize(code, Infinity) + + expect(progress.degraded).toBe(true) + expect(colors(progress)).toEqual([ + expect.arrayContaining([['const', '#0000FF', '#569CD6']]), + [[`const long = "${'x'.repeat(2000)}"`, undefined, undefined]], + [['const b = 2', undefined, undefined]], + [['const c = 3', undefined, undefined]] + ]) + expect(text(progress, code)).toBe(code) + }) + + it.each(['a = 1\r\nb = 2\r\n', 'a = 1\r\nb = 2\nc = 3\r', 'a = 1\rb = 2\r\n\r\nc'])( + 'keeps the line endings of %j', + async (code) => { + const tokenize = createIncrementalSyntaxTokenizer(await typescriptTokenizer()) + + for (let length = 1; length <= code.length; length += 1) { + const streamed = code.slice(0, length) + expect(text(tokenize(streamed, Infinity), streamed)).toBe(streamed) + } + } + ) +}) diff --git a/src/renderer/src/lib/syntax-highlighting/incremental-syntax-tokenizer.ts b/src/renderer/src/lib/syntax-highlighting/incremental-syntax-tokenizer.ts new file mode 100644 index 00000000000..656b46f02fd --- /dev/null +++ b/src/renderer/src/lib/syntax-highlighting/incremental-syntax-tokenizer.ts @@ -0,0 +1,99 @@ +import type { SyntaxHighlighter, SyntaxHighlightState, SyntaxToken } from './syntax-highlighter' + +export type SyntaxLine = { + /** Offset of the line in the document; unique, so it keys the line. */ + start: number + tokens: readonly SyntaxToken[] + /** The separator the source has after this line, CRLF included. */ + ending: string +} + +export type SyntaxProgress = { + lines: SyntaxLine[] + /** `code.slice(highlightedLength)` has not been tokenized yet. */ + highlightedLength: number + /** A line was skipped, so the lines after it stay plain. */ + degraded: boolean +} + +// A batch cannot be interrupted, so it stays near a millisecond of dense code. +const BATCH_LENGTH = 250 +const BATCH_LINES = 10 + +/** + * The text of a run of lines without its final separator, copied out of the + * document: a substring keeps its parent alive, so tokens cut straight from a + * streaming block would pin every copy of it that streamed past. + */ +function detachedLines(text: string): string { + return ` ${text.replace(/\r?\n$/, '')}`.slice(1) +} + +/** + * Tokenizes a document a few lines at a time, resuming after the last completed + * line, and stops once `budgetMs` is spent. Completed lines keep their identity. + */ +export function createIncrementalSyntaxTokenizer( + highlight: SyntaxHighlighter +): (code: string, budgetMs: number) => SyntaxProgress { + let completed = '' + let state: SyntaxHighlightState | undefined + let lines: SyntaxLine[] = [] + let degraded = false + + return (code, budgetMs) => { + if (!code.startsWith(completed)) { + completed = '' + state = undefined + lines = [] + degraded = false + } + const deadline = performance.now() + budgetMs + // Why: markdown ends a fence still being written with a newline, so the last line can grow. + const lastBreak = code.lastIndexOf('\n', code.length - 2) + 1 + while (completed.length < lastBreak) { + if (performance.now() >= deadline) { + return { lines, highlightedLength: completed.length, degraded } + } + const from = completed.length + let end = from + for (let count = 0; count < BATCH_LINES && end < lastBreak; count += 1) { + const next = code.indexOf('\n', end) + 1 + if (count > 0 && next - from > BATCH_LENGTH) { + break + } + end = next + } + const tokenized = highlight(detachedLines(code.slice(from, end)), state) + state = tokenized.state + degraded ||= tokenized.outcome !== 'ok' + let start = from + const batch = tokenized.lines.map((tokens) => { + const lineEnd = tokens.reduce((offset, token) => offset + token.content.length, start) + const ending = code.startsWith('\r\n', lineEnd) ? '\r\n' : '\n' + const line = { start, tokens, ending } + start = lineEnd + ending.length + return line + }) + lines = [...lines, ...batch] + completed = code.slice(0, end) + } + const last = code.slice(completed.length) + if (!last) { + return { lines, highlightedLength: code.length, degraded } + } + if (performance.now() >= deadline) { + return { lines, highlightedLength: completed.length, degraded } + } + const text = detachedLines(last) + const partial = highlight(text, state) + return { + lines: [ + ...lines, + { start: completed.length, tokens: partial.lines[0] ?? [], ending: last.slice(text.length) } + ], + highlightedLength: code.length, + degraded: degraded || partial.outcome !== 'ok' + } + } +} diff --git a/src/renderer/src/lib/syntax-highlighting/oniguruma-test-harness.ts b/src/renderer/src/lib/syntax-highlighting/oniguruma-test-harness.ts new file mode 100644 index 00000000000..487e50fd47f --- /dev/null +++ b/src/renderer/src/lib/syntax-highlighting/oniguruma-test-harness.ts @@ -0,0 +1,18 @@ +import { readFile } from 'node:fs/promises' +import { createRequire } from 'node:module' +import type { Oniguruma } from './oniguruma' + +let nodeOnigurumaPromise: Promise | undefined + +/** The bundled Oniguruma WASM read from disk, for tests running outside a browser. */ +export function loadNodeOniguruma(): Promise { + nodeOnigurumaPromise ??= (async () => { + const oniguruma = await import('vscode-oniguruma') + const wasm = await readFile( + createRequire(import.meta.url).resolve('vscode-oniguruma/release/onig.wasm') + ) + await oniguruma.loadWASM(wasm.buffer.slice(wasm.byteOffset, wasm.byteOffset + wasm.byteLength)) + return oniguruma + })() + return nodeOnigurumaPromise +} diff --git a/src/renderer/src/lib/syntax-highlighting/oniguruma.ts b/src/renderer/src/lib/syntax-highlighting/oniguruma.ts new file mode 100644 index 00000000000..bd2243f1a6e --- /dev/null +++ b/src/renderer/src/lib/syntax-highlighting/oniguruma.ts @@ -0,0 +1,24 @@ +import type * as OnigurumaModule from 'vscode-oniguruma' +import onigurumaWasmUrl from 'vscode-oniguruma/release/onig.wasm?url' + +export type Oniguruma = typeof OnigurumaModule + +let onigurumaPromise: Promise | undefined + +/** The renderer's one Oniguruma WASM instance, shared by every TextMate tokenizer. */ +export function loadOniguruma(): Promise { + onigurumaPromise ??= (async () => { + const oniguruma = await import('vscode-oniguruma') + const response = await fetch(onigurumaWasmUrl) + if (!response.ok) { + throw new Error(`Failed to load TextMate regex engine from ${onigurumaWasmUrl}`) + } + await oniguruma.loadWASM(response) + return oniguruma + })().catch((error: unknown) => { + // Why: a failed fetch would otherwise leave every grammar plain until restart. + onigurumaPromise = undefined + throw error + }) + return onigurumaPromise +} diff --git a/src/renderer/src/lib/syntax-highlighting/syntax-highlight-scheduler.test.ts b/src/renderer/src/lib/syntax-highlighting/syntax-highlight-scheduler.test.ts new file mode 100644 index 00000000000..3ca8ab469bb --- /dev/null +++ b/src/renderer/src/lib/syntax-highlighting/syntax-highlight-scheduler.test.ts @@ -0,0 +1,49 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { scheduleSyntaxHighlighting } from './syntax-highlight-scheduler' + +afterEach(() => { + vi.restoreAllMocks() +}) + +describe('scheduleSyntaxHighlighting', () => { + it('shares one slice between blocks and takes turns', async () => { + let now = 0 + vi.spyOn(performance, 'now').mockImplementation(() => now) + const runs: string[] = [] + const budgets: number[] = [] + const job = (name: string, steps: number) => { + let left = steps + return (budgetMs: number) => { + runs.push(name) + budgets.push(budgetMs) + // Each step spends the whole slice. + now += budgetMs + left -= 1 + return left === 0 + } + } + + scheduleSyntaxHighlighting(job('a', 2)) + scheduleSyntaxHighlighting(job('b', 2)) + await vi.waitFor(() => expect(runs).toHaveLength(4)) + + expect(runs).toEqual(['a', 'b', 'a', 'b']) + // One block per slice: the first spends it all, so the other waits for the next. + expect(budgets).toEqual([6, 6, 6, 6]) + }) + + it('stops a cancelled block', async () => { + const steps: string[] = [] + const cancel = scheduleSyntaxHighlighting(() => { + steps.push('cancelled') + return false + }) + cancel() + scheduleSyntaxHighlighting(() => { + steps.push('kept') + return true + }) + + await vi.waitFor(() => expect(steps).toEqual(['kept'])) + }) +}) diff --git a/src/renderer/src/lib/syntax-highlighting/syntax-highlight-scheduler.ts b/src/renderer/src/lib/syntax-highlighting/syntax-highlight-scheduler.ts new file mode 100644 index 00000000000..2453015b2e9 --- /dev/null +++ b/src/renderer/src/lib/syntax-highlighting/syntax-highlight-scheduler.ts @@ -0,0 +1,54 @@ +import { yieldToEventLoop } from '../../../../shared/event-loop-yield' + +/** Runs for at most `budgetMs`; true once its work is finished. */ +export type SyntaxHighlightJob = (budgetMs: number) => boolean + +// Tokenizing runs on the renderer thread, so every block together gets one slice shorter than a frame. +const SLICE_BUDGET_MS = 6 + +const jobs = new Set() +let running = false + +async function runJobs(): Promise { + running = true + try { + while (jobs.size > 0) { + await yieldToEventLoop() + const deadline = performance.now() + SLICE_BUDGET_MS + // A snapshot, so a job sent to the back waits for the next slice. + for (const job of Array.from(jobs)) { + const remaining = deadline - performance.now() + if (remaining <= 0) { + break + } + if (!jobs.has(job)) { + continue + } + let finished = true + try { + finished = job(remaining) + } catch { + // The lines colored so far stay; the rest keeps its plain text. + } + jobs.delete(job) + if (!finished) { + // Why: to the back, so a long block cannot starve the others. + jobs.add(job) + } + } + } + } finally { + running = false + } +} + +/** Shares one time slice per task across every block still highlighting; returns a cancel. */ +export function scheduleSyntaxHighlighting(job: SyntaxHighlightJob): () => void { + jobs.add(job) + if (!running) { + void runJobs() + } + return () => { + jobs.delete(job) + } +} diff --git a/src/renderer/src/lib/syntax-highlighting/syntax-highlighter.test.ts b/src/renderer/src/lib/syntax-highlighting/syntax-highlighter.test.ts new file mode 100644 index 00000000000..c905b92cc4b --- /dev/null +++ b/src/renderer/src/lib/syntax-highlighting/syntax-highlighter.test.ts @@ -0,0 +1,196 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type * as SyntaxHighlighterModule from './syntax-highlighter' +import { loadNodeOniguruma } from './oniguruma-test-harness' +import { + loadedSyntaxHighlighter, + loadSyntaxLanguage, + type SyntaxHighlighter, + type SyntaxHighlightResult, + type SyntaxToken +} from './syntax-highlighter' + +const oniguruma = vi.hoisted(() => ({ fails: false, loads: 0 })) + +vi.mock('./oniguruma', () => ({ + loadOniguruma: () => { + oniguruma.loads += 1 + return oniguruma.fails + ? Promise.reject(new Error('regex engine unavailable')) + : loadNodeOniguruma() + } +})) + +afterEach(() => { + vi.restoreAllMocks() +}) + +async function highlighterFor(language: string): Promise { + await expect(loadSyntaxLanguage(language)).resolves.toBe('ready') + const highlighter = loadedSyntaxHighlighter(language) + if (!highlighter) { + throw new Error(`${language} did not load`) + } + return highlighter +} + +/** A module whose regex engine has not loaded yet. */ +async function freshHighlighterModule(): Promise { + vi.resetModules() + vi.spyOn(console, 'warn').mockImplementation(() => {}) + return import('./syntax-highlighter') +} + +function find(result: SyntaxHighlightResult, content: string): SyntaxToken | undefined { + return result.lines.flat().find((token) => token.content.trim() === content) +} + +function colors(lines: readonly (readonly SyntaxToken[])[]): unknown { + return lines.map((line) => line.map((token) => [token.content, token.light, token.dark])) +} + +describe('highlightSyntax', () => { + it('colors each theme on its own and leaves default text to the surface', async () => { + const result = (await highlighterFor('ts'))('const answer = 42') + + expect(result.outcome).toBe('ok') + expect(find(result, 'const')).toMatchObject({ light: '#0000FF', dark: '#569CD6' }) + expect(find(result, '=')).toMatchObject({ light: undefined, dark: undefined }) + expect(result.lines[0].map((token) => token.content).join('')).toBe('const answer = 42') + }) + + it('continues from a returned state exactly as one pass over the whole document', async () => { + const python = await highlighterFor('python') + const lines = [ + 'def run(x):', + ' s = f"""multi', + ' line {x}', + ' """', + ' with open(s) as f:', + ' return f.read()' + ] + const whole = python(lines.join('\n')) + const head = python(lines.slice(0, 2).join('\n')) + const tail = python(lines.slice(2).join('\n'), head.state) + + expect(colors([...head.lines, ...tail.lines])).toEqual(colors(whole.lines)) + // After the multi-line f-string, `with` is a keyword again. + expect(find(whole, 'with')).toMatchObject({ dark: '#C586C0' }) + }) + + it('reports formatting per theme', async () => { + const result = (await highlighterFor('markdown'))('*emphasis* and **strong**') + + expect(find(result, '*emphasis*')).toMatchObject({ lightFontStyle: 1, darkFontStyle: 1 }) + expect(find(result, '**strong**')).toMatchObject({ lightFontStyle: 2, darkFontStyle: 2 }) + }) + + it.each([ + ['one line', ''], + ['several lines', ''] + ])('colors Vue template expressions over %s', async (_, code) => { + // Markdown names Vue's scope first, which is what kept these plain. + await highlighterFor('markdown') + const result = (await highlighterFor('vue'))(code) + + expect(find(result, 'format')).toMatchObject({ dark: '#DCDCAA' }) + expect(find(result, 'total')).toMatchObject({ dark: '#9CDCFE' }) + }) + + it('leaves a line too long to color plain, along with every line after it', async () => { + const typescript = await highlighterFor('typescript') + const code = ['const a = 1', `const long = "${'x'.repeat(2000)}"`, 'const b = 2'].join('\n') + + const result = typescript(code) + + expect(result.outcome).toBe('degraded') + expect(find(result, 'const')).toMatchObject({ dark: '#569CD6' }) + expect(result.lines[1]).toEqual([expect.objectContaining({ dark: undefined })]) + expect(result.lines[2]).toEqual([ + { + content: 'const b = 2', + offset: code.lastIndexOf('const b'), + light: undefined, + dark: undefined, + lightFontStyle: 0, + darkFontStyle: 0 + } + ]) + expect(typescript('const c = 3', result.state)).toMatchObject({ + outcome: 'degraded', + lines: [[expect.objectContaining({ content: 'const c = 3', dark: undefined })]] + }) + }) + + it('leaves a line that runs out of time plain instead of guessing the rest', async () => { + const typescript = await highlighterFor('typescript') + let now = 0 + vi.spyOn(Date, 'now').mockImplementation(() => (now += 1000)) + + const result = typescript('const a = `template ${value} string`') + + expect(result.outcome).toBe('degraded') + expect(result.lines[0]).toEqual([expect.objectContaining({ dark: undefined })]) + }) + + it('does not resume from a state taken before its grammar reloaded', async () => { + const vue = await highlighterFor('vue') + const head = vue('