diff --git a/package.json b/package.json index 18ff04a8082..a7c729b6f09 100644 --- a/package.json +++ b/package.json @@ -318,6 +318,7 @@ "remark-rehype": "11.1.2", "remend": "1.4.0", "shadcn": "^4.13.1", + "shiki": "4.5.0", "sonner": "2.0.7", "tailwind-merge": "^3.7.0", "tailwindcss": "^4.2.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 544ad1f899e..ef6f038cb1b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -582,6 +582,9 @@ importers: shadcn: specifier: ^4.13.1 version: 4.13.1(supports-color@7.2.0)(typescript@7.0.2) + shiki: + specifier: 4.5.0 + version: 4.5.0 sonner: specifier: 2.0.7 version: 2.0.7(react-dom@19.2.8(react@19.2.8))(react@19.2.8) @@ -3164,6 +3167,37 @@ packages: resolution: {integrity: sha512-8mpfsoPeRhesY3pOrzNZBL8uG6N5GVX1EHLBYbd4gzKs+c7vaEIqpTNK5VrffU33qQN4cwpP2v3u4aPPBU32sw==} engines: {node: '>=16'} + '@shikijs/core@4.5.0': + resolution: {integrity: sha512-6CIgwq+bt2JZ3OSjb/nBkkwEf/CfpHcbam+Mg5Pllco7HAGRQ2/EFhZrrJtbyNTgKwknfVS+llsXGygiDGwJWw==} + engines: {node: '>=20'} + + '@shikijs/engine-javascript@4.5.0': + resolution: {integrity: sha512-Ed6UdEj84LyYkwnxM5CvSMWAD+id9ExTYri0/+x41U9zrgNifPSS9lZm0JqBP0VfYI8yr8tY5nbNq9+p9R0kFA==} + engines: {node: '>=20'} + + '@shikijs/engine-oniguruma@4.5.0': + resolution: {integrity: sha512-dKVmbJecVdB9recl8a/CU4jdpNXyjJQrIcesIkpS70s5HS02ohpnCSpMUbotPnEUzbAcLQ+S4hwZYbkBBiTcJw==} + engines: {node: '>=20'} + + '@shikijs/langs@4.5.0': + resolution: {integrity: sha512-wwBTqwduXSz0X4v9Qa2EWeFaYJEw1clCncWjoRNQRbVDu3SPRTRWucnouYapKj3b8Q+sk/41Vbc4B5c8nuYzLA==} + engines: {node: '>=20'} + + '@shikijs/primitive@4.5.0': + resolution: {integrity: sha512-UM7ShlA8oz0OWxLLO8BmO9Ip6fVqdkt8yF0EH7nOCqHfrx1eEAzF0ZUVK7L+I8NEgeu1PEWSRlxaiPPeCb4iDg==} + engines: {node: '>=20'} + + '@shikijs/themes@4.5.0': + resolution: {integrity: sha512-zGb87H3UBbpuI4yoh5UguB1XbU+2BDU3kflM7UfZ+Cy2hWo9lBtqjSVK20gmLY6ukYl040hAypGXMgud9vul8Q==} + engines: {node: '>=20'} + + '@shikijs/types@4.5.0': + resolution: {integrity: sha512-drtPrtD8N0DENyD2cyt6Ajy+IWsiCLgMxODwLTpvH1UsN6a9jf90tNTPjqwHvA+0Lzu9e0txk8DjfqH1p9UJRw==} + engines: {node: '>=20'} + + '@shikijs/vscode-textmate@10.0.2': + resolution: {integrity: sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==} + '@sindresorhus/is@4.6.0': resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==} engines: {node: '>=10'} @@ -3883,6 +3917,9 @@ packages: '@types/hast@3.0.4': resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==} + '@types/hast@3.0.5': + resolution: {integrity: sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==} + '@types/http-cache-semantics@4.2.0': resolution: {integrity: sha512-L3LgimLHXtGkWikKnsPg0/VFx9OGZaC+eN1u4r+OB1XRqH3meBIAVC2zr1WdMH+RHmnRkqliQAOHNJ/E0j/e0Q==} @@ -5450,6 +5487,9 @@ packages: hast-util-sanitize@5.0.2: resolution: {integrity: sha512-3yTWghByc50aGS7JlGhk61SPenfE/p1oaFeNwkOOyrscaOkMGrcW9+Cy/QAIOBpZxP1yqDIzFMR0+Np0i0+usg==} + hast-util-to-html@9.0.5: + resolution: {integrity: sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==} + hast-util-to-jsx-runtime@2.3.6: resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==} @@ -6481,6 +6521,12 @@ packages: resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==} engines: {node: '>=18'} + oniguruma-parser@0.12.2: + resolution: {integrity: sha512-6HVa5oIrgMC6aA6WF6XyyqbhRPJrKR02L20+2+zpDtO5QAzGHAUGw5TKQvwi5vctNnRHkJYmjAhRVQF2EKdTQw==} + + oniguruma-to-es@4.3.6: + resolution: {integrity: sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA==} + open@11.0.0: resolution: {integrity: sha512-smsWv2LzFjP03xmvFoJ331ss6h+jixfA4UUV/Bsiyuu4YJPfN+FIQGOIiv4w9/+MoHkfkJ22UIaQWRVFRfH6Vw==} engines: {node: '>=20'} @@ -6966,6 +7012,15 @@ packages: resolution: {integrity: sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==} engines: {node: '>=8'} + regex-recursion@6.0.2: + resolution: {integrity: sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==} + + regex-utilities@2.3.0: + resolution: {integrity: sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==} + + regex@6.1.0: + resolution: {integrity: sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg==} + rehype-highlight@7.0.2: resolution: {integrity: sha512-k158pK7wdC2qL3M5NcZROZ2tR/l7zOzjxXd5VGdcfIyoijjQqpHd3JKtYSBDpDZ38UI2WJWuFAtkMDxmx5kstA==} @@ -7248,6 +7303,10 @@ packages: sherpa-onnx@1.12.37: resolution: {integrity: sha512-3luwSdHwR8BtJiiFwqHfb15FE2FX0KsN4aOBbfq9Ma23r3w9C3bprFc/WBusXk56nUbzcEN5YczN7t9w1JwdtQ==} + shiki@4.5.0: + resolution: {integrity: sha512-OcRW72ngQvc+MSVWBRIovC/BTW589iUE/Vxav7VeM4RcY9MUjqB+IR6ILG1k9nuW6LiG/nct9uma4/MBtnZbmQ==} + engines: {node: '>=20'} + shipstyles@1.0.0: resolution: {integrity: sha512-1AcihfXo/mSxqeOdoKqq/5TNv+vSH0dBYPFq88PUX5FBkK5pRmY5TP9SuMNio85PQ+euUI67+KpSVk1cUVDCgA==} @@ -10193,6 +10252,46 @@ snapshots: '@shaderfrog/glsl-parser@7.0.1': {} + '@shikijs/core@4.5.0': + dependencies: + '@shikijs/primitive': 4.5.0 + '@shikijs/types': 4.5.0 + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.5 + hast-util-to-html: 9.0.5 + + '@shikijs/engine-javascript@4.5.0': + dependencies: + '@shikijs/types': 4.5.0 + '@shikijs/vscode-textmate': 10.0.2 + oniguruma-to-es: 4.3.6 + + '@shikijs/engine-oniguruma@4.5.0': + dependencies: + '@shikijs/types': 4.5.0 + '@shikijs/vscode-textmate': 10.0.2 + + '@shikijs/langs@4.5.0': + dependencies: + '@shikijs/types': 4.5.0 + + '@shikijs/primitive@4.5.0': + dependencies: + '@shikijs/types': 4.5.0 + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.5 + + '@shikijs/themes@4.5.0': + dependencies: + '@shikijs/types': 4.5.0 + + '@shikijs/types@4.5.0': + dependencies: + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.5 + + '@shikijs/vscode-textmate@10.0.2': {} + '@sindresorhus/is@4.6.0': {} '@sindresorhus/merge-streams@4.0.0': {} @@ -10877,6 +10976,10 @@ snapshots: dependencies: '@types/unist': 3.0.3 + '@types/hast@3.0.5': + dependencies: + '@types/unist': 3.0.3 + '@types/http-cache-semantics@4.2.0': {} '@types/json-schema@7.0.15': {} @@ -12633,6 +12736,20 @@ snapshots: '@ungap/structured-clone': 1.3.1 unist-util-position: 5.0.0 + hast-util-to-html@9.0.5: + dependencies: + '@types/hast': 3.0.4 + '@types/unist': 3.0.3 + ccount: 2.0.1 + comma-separated-tokens: 2.0.3 + hast-util-whitespace: 3.0.0 + html-void-elements: 3.0.0 + mdast-util-to-hast: 13.2.1 + property-information: 7.1.0 + space-separated-tokens: 2.0.2 + stringify-entities: 4.0.4 + zwitch: 2.0.4 + hast-util-to-jsx-runtime@2.3.6(supports-color@7.2.0): dependencies: '@types/estree': 1.0.8 @@ -13847,6 +13964,14 @@ snapshots: dependencies: mimic-function: 5.0.1 + oniguruma-parser@0.12.2: {} + + oniguruma-to-es@4.3.6: + dependencies: + oniguruma-parser: 0.12.2 + regex: 6.1.0 + regex-recursion: 6.0.2 + open@11.0.0: dependencies: default-browser: 5.5.0 @@ -14489,6 +14614,16 @@ snapshots: indent-string: 4.0.0 strip-indent: 3.0.0 + regex-recursion@6.0.2: + dependencies: + regex-utilities: 2.3.0 + + regex-utilities@2.3.0: {} + + regex@6.1.0: + dependencies: + regex-utilities: 2.3.0 + rehype-highlight@7.0.2: dependencies: '@types/hast': 3.0.4 @@ -14854,6 +14989,17 @@ snapshots: sherpa-onnx@1.12.37: {} + shiki@4.5.0: + dependencies: + '@shikijs/core': 4.5.0 + '@shikijs/engine-javascript': 4.5.0 + '@shikijs/engine-oniguruma': 4.5.0 + '@shikijs/langs': 4.5.0 + '@shikijs/themes': 4.5.0 + '@shikijs/types': 4.5.0 + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.5 + shipstyles@1.0.0: {} side-channel-list@1.0.1: diff --git a/src/renderer/src/components/native-chat/NativeChatAssistantMessageRow.tsx b/src/renderer/src/components/native-chat/NativeChatAssistantMessageRow.tsx index ca3a2e8550b..4eb95baa651 100644 --- a/src/renderer/src/components/native-chat/NativeChatAssistantMessageRow.tsx +++ b/src/renderer/src/components/native-chat/NativeChatAssistantMessageRow.tsx @@ -5,7 +5,7 @@ import type { MessageRowProps } from './NativeChatMessageRow' import { NativeChatPacedMarkdown } from './NativeChatPacedMarkdown' import { NATIVE_CHAT_QUOTE_SOURCE_PROPS } from './native-chat-quote-selection' import { NativeChatToolRun } from './NativeChatToolRun' -import { NativeChatCodeBlock } from './NativeChatCodeBlock' +import { NativeChatCodeBlock, NativeChatPlainCodeBlock } from './NativeChatCodeBlock' import { NativeChatAgentControls, NativeChatImageAttachments } from './NativeChatTranscriptChrome' import type { useNativeChatWorkRun } from './use-native-chat-work-run' @@ -74,7 +74,7 @@ export function NativeChatAssistantMessageRow({ content={words} variant="document" className="text-sm native-chat-message-text" - renderCodeBlock={NativeChatCodeBlock} + renderCodeBlock={isSystem ? NativeChatPlainCodeBlock : NativeChatCodeBlock} onLinkClick={onLinkClick} allowFileUriLinks={allowFileUriLinks} linkifyFilePaths={onLinkClick !== undefined} diff --git a/src/renderer/src/components/native-chat/NativeChatCodeBlock.load-failure.test.tsx b/src/renderer/src/components/native-chat/NativeChatCodeBlock.load-failure.test.tsx new file mode 100644 index 00000000000..628757f2505 --- /dev/null +++ b/src/renderer/src/components/native-chat/NativeChatCodeBlock.load-failure.test.tsx @@ -0,0 +1,58 @@ +// @vitest-environment happy-dom + +import '@testing-library/jest-dom/vitest' +import { cleanup, render, waitFor } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { loadSyntaxLanguage } from '@/lib/syntax-highlighting/syntax-highlighter' +import { NativeChatCodeBlock } from './NativeChatCodeBlock' + +// Its own file, so the regex engine has not loaded before the failure. +const oniguruma = vi.hoisted(() => ({ fails: false })) + +vi.mock('@/lib/syntax-highlighting/oniguruma', async () => { + const { loadNodeOniguruma } = await import('@/lib/syntax-highlighting/oniguruma-test-harness') + return { + loadOniguruma: () => + oniguruma.fails ? Promise.reject(new Error('regex engine unavailable')) : loadNodeOniguruma() + } +}) + +afterEach(() => { + cleanup() + vi.restoreAllMocks() +}) + +describe('NativeChatCodeBlock grammar load failure', () => { + it('colors a block still streaming once a failed grammar load succeeds on retry', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}) + let now = 0 + vi.spyOn(performance, 'now').mockImplementation(() => now) + oniguruma.fails = true + const block = (code: string) => ( + + {code} + + ) + try { + const { container, rerender } = render(block('fn main() {\n')) + await expect(loadSyntaxLanguage('rust')).resolves.toBe('failed') + expect(container.querySelector('pre span')).toBeNull() + + oniguruma.fails = false + now += 5000 + rerender(block('fn main() {\n let answer = 42;\n')) + + await waitFor( + () => { + expect(container.querySelector('pre span[style*="--syntax-dark"]')).not.toBeNull() + }, + { timeout: 10_000 } + ) + expect(container.querySelector('pre')?.textContent).toBe( + 'fn main() {\n let answer = 42;\n' + ) + } finally { + oniguruma.fails = false + } + }) +}) diff --git a/src/renderer/src/components/native-chat/NativeChatCodeBlock.test.tsx b/src/renderer/src/components/native-chat/NativeChatCodeBlock.test.tsx index c7cc84e838c..210332b5215 100644 --- a/src/renderer/src/components/native-chat/NativeChatCodeBlock.test.tsx +++ b/src/renderer/src/components/native-chat/NativeChatCodeBlock.test.tsx @@ -3,7 +3,13 @@ import '@testing-library/jest-dom/vitest' import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' import { afterEach, describe, expect, it, vi } from 'vitest' -import { NativeChatCodeBlock } from './NativeChatCodeBlock' +import { loadSyntaxLanguage } from '@/lib/syntax-highlighting/syntax-highlighter' +import { NativeChatCodeBlock, NativeChatPlainCodeBlock } from './NativeChatCodeBlock' + +vi.mock('@/lib/syntax-highlighting/oniguruma', async () => ({ + loadOniguruma: (await import('@/lib/syntax-highlighting/oniguruma-test-harness')) + .loadNodeOniguruma +})) afterEach(() => { cleanup() @@ -41,4 +47,77 @@ describe('NativeChatCodeBlock', () => { }) expect(await screen.findByRole('button', { name: 'Copied' })).toBeInTheDocument() }) + + it('colors a fenced block without changing its text', async () => { + const code = 'const answer = 42\nconsole.log(answer)\n' + const { container } = render( + + {code} + + ) + const pre = container.querySelector('pre') + + await waitFor( + () => { + expect(pre?.querySelector('span[style*="--syntax-dark"]')).not.toBeNull() + }, + { timeout: 10_000 } + ) + expect(pre?.textContent).toBe(code) + }) + + it('keeps Windows line endings and finishes a block longer than one slice', async () => { + const code = `${'const value = compute(1, "two", [3])\r\n'.repeat(400)}const lastLine = 1\r\n` + const { container } = render( + + {code} + + ) + + expect(await screen.findByText('lastLine', {}, { timeout: 10_000 })).toBeInTheDocument() + expect(container.querySelector('pre')?.textContent).toBe(code) + }) + + it('leaves the code of a dimmed row uncolored', async () => { + const code = {'const dimmed = 1\n'} + await loadSyntaxLanguage('typescript') + const colored = render({code}) + expect(colored.container.querySelector('pre span')).not.toBeNull() + + const plain = render( + {code} + ) + + expect(plain.container.querySelector('pre span')).toBeNull() + expect(plain.container.querySelector('pre')?.textContent).toBe('const dimmed = 1\n') + }) + + it('paints a finished block in color at once when its row mounts again', async () => { + // Long enough that tokenizing it again could not finish within one render. + const block = ( + + {`${'const a=[1,2,3].map((x)=>x+1);f(a,{b:1,c:"s"});\n'.repeat(300)}const lastLine = 1\n`} + + ) + const first = render(block) + expect(await screen.findByText('lastLine', {}, { timeout: 10_000 })).toBeInTheDocument() + const colored = first.container.querySelectorAll('span[style*="--syntax-dark"]').length + first.unmount() + + const second = render(block) + + expect(second.container.querySelectorAll('span[style*="--syntax-dark"]')).toHaveLength(colored) + }) + + it('leaves a fence in an unknown language as plain text', async () => { + const { container } = render( + + {'plain words\n'} + + ) + const pre = container.querySelector('pre') + + await waitFor(() => expect(pre?.textContent).toBe('plain words\n')) + expect(pre?.querySelector('span')).toBeNull() + }) }) diff --git a/src/renderer/src/components/native-chat/NativeChatCodeBlock.tsx b/src/renderer/src/components/native-chat/NativeChatCodeBlock.tsx index 208802c8538..f6c266feed3 100644 --- a/src/renderer/src/components/native-chat/NativeChatCodeBlock.tsx +++ b/src/renderer/src/components/native-chat/NativeChatCodeBlock.tsx @@ -3,14 +3,17 @@ import { translate } from '@/i18n/i18n' import { cn } from '@/lib/utils' import { getCodeBlockLanguageLabel } from '@/components/editor/rich-markdown-code-block-languages' import { NativeChatCopyButton } from './NativeChatCopyButton' +import { NativeChatHighlightedCode } from './NativeChatHighlightedCode' /** Code fences need their own copy target rather than the whole chat message. */ export function NativeChatCodeBlock({ children, - language + language, + plain = false }: { children?: React.ReactNode language?: string + plain?: boolean }): React.JSX.Element { const code = extractCodeText(children) @@ -40,7 +43,13 @@ export function NativeChatCodeBlock({ language ? 'px-3.5 pt-0.5 pb-3' : 'p-3 pr-10' )} > - {children} + {language && code && !plain ? ( + + + + ) : ( + children + )} {code && !language ? ( +} + function extractCodeText(node: React.ReactNode): string { if (typeof node === 'string' || typeof node === 'number') { return String(node) diff --git a/src/renderer/src/components/native-chat/NativeChatHighlightedCode.tsx b/src/renderer/src/components/native-chat/NativeChatHighlightedCode.tsx new file mode 100644 index 00000000000..84e2757345e --- /dev/null +++ b/src/renderer/src/components/native-chat/NativeChatHighlightedCode.tsx @@ -0,0 +1,83 @@ +import { memo } from 'react' +import { useHighlightedSyntax } from '@/hooks/use-highlighted-syntax' +import type { SyntaxLine } from '@/lib/syntax-highlighting/incremental-syntax-tokenizer' +import { SYNTAX_FONT_STYLE, type SyntaxToken } from '@/lib/syntax-highlighting/syntax-highlighter' + +type TokenStyle = React.CSSProperties & Record<`--syntax-${string}`, string | undefined> + +// Both themes ride on the token so a scheme change needs no re-tokenizing. +const COLORED_TOKEN_CLASS = 'text-(color:--syntax-light) dark:text-(color:--syntax-dark)' +const FORMATTED_TOKEN_CLASS = + 'text-(color:--syntax-light) dark:text-(color:--syntax-dark) [font-style:var(--syntax-light-font-style)] dark:[font-style:var(--syntax-dark-font-style)] [font-weight:var(--syntax-light-font-weight)] dark:[font-weight:var(--syntax-dark-font-weight)] [text-decoration-line:var(--syntax-light-decoration)] dark:[text-decoration-line:var(--syntax-dark-decoration)]' + +function formatting(theme: 'light' | 'dark', fontStyle: number): TokenStyle { + const decorations = [ + fontStyle & SYNTAX_FONT_STYLE.underline ? 'underline' : '', + fontStyle & SYNTAX_FONT_STYLE.strikethrough ? 'line-through' : '' + ].filter(Boolean) + return { + [`--syntax-${theme}-font-style`]: fontStyle & SYNTAX_FONT_STYLE.italic ? 'italic' : 'normal', + [`--syntax-${theme}-font-weight`]: fontStyle & SYNTAX_FONT_STYLE.bold ? 'bold' : 'normal', + [`--syntax-${theme}-decoration`]: decorations.join(' ') || 'none' + } +} + +function tokenStyle(token: SyntaxToken): TokenStyle { + const colors: TokenStyle = { '--syntax-light': token.light, '--syntax-dark': token.dark } + return token.lightFontStyle || token.darkFontStyle + ? { + ...colors, + ...formatting('light', token.lightFontStyle), + ...formatting('dark', token.darkFontStyle) + } + : colors +} + +const HighlightedLine = memo(function HighlightedLine({ line }: { line: SyntaxLine }) { + return ( + <> + {line.tokens.map((token) => + // Uncolored text needs no element of its own. + token.content.trim() && + (token.light || token.dark || token.lightFontStyle || token.darkFontStyle) ? ( + + {token.content} + + ) : ( + token.content + ) + )} + {line.ending} + + ) +}) + +/** Colors `code` in place; its text is unchanged, so layout and copy match the plain fallback. */ +export const NativeChatHighlightedCode = memo(function NativeChatHighlightedCode({ + code, + language +}: { + code: string + language: string +}): React.ReactNode { + const progress = useHighlightedSyntax(code, language) + if (!progress) { + return code + } + return ( + <> + {progress.lines.map((line) => ( + + ))} + {code.slice(progress.highlightedLength)} + + ) +}) diff --git a/src/renderer/src/components/native-chat/NativeChatMessageRow.test.tsx b/src/renderer/src/components/native-chat/NativeChatMessageRow.test.tsx index 44ad235f4aa..4c0e470aa16 100644 --- a/src/renderer/src/components/native-chat/NativeChatMessageRow.test.tsx +++ b/src/renderer/src/components/native-chat/NativeChatMessageRow.test.tsx @@ -9,6 +9,10 @@ import type { NativeChatRewindSurface } from './use-native-chat-rewind' import { readNativeChatQuotableSelection } from './native-chat-quote-selection' const confirm = vi.hoisted(() => vi.fn()) +vi.mock('@/lib/syntax-highlighting/oniguruma', async () => ({ + loadOniguruma: (await import('@/lib/syntax-highlighting/oniguruma-test-harness')) + .loadNodeOniguruma +})) vi.mock('@/components/confirmation-dialog-context', () => ({ useConfirmationDialog: () => confirm })) diff --git a/src/renderer/src/components/native-chat/NativeChatReasoningDisclosure.tsx b/src/renderer/src/components/native-chat/NativeChatReasoningDisclosure.tsx index 5d8ded6e4df..62d7584d2cc 100644 --- a/src/renderer/src/components/native-chat/NativeChatReasoningDisclosure.tsx +++ b/src/renderer/src/components/native-chat/NativeChatReasoningDisclosure.tsx @@ -1,6 +1,6 @@ import { ChevronRight } from 'lucide-react' import type { CommentMarkdownLinkClickHandler } from '@/components/sidebar/CommentMarkdown' -import { NativeChatCodeBlock } from './NativeChatCodeBlock' +import { NativeChatPlainCodeBlock } from './NativeChatCodeBlock' import { NativeChatMarkdown } from './NativeChatMarkdown' /** A reasoning block's text, under the live activity line or its finished row: quieter prose in the @@ -27,7 +27,7 @@ export function NativeChatReasoningBody({ content={markdown} variant="document" className="text-sm native-chat-message-text" - renderCodeBlock={NativeChatCodeBlock} + renderCodeBlock={NativeChatPlainCodeBlock} onLinkClick={onLinkClick} allowFileUriLinks={allowFileUriLinks} linkifyFilePaths={onLinkClick !== undefined} diff --git a/src/renderer/src/components/native-chat/NativeChatToolLine.test.tsx b/src/renderer/src/components/native-chat/NativeChatToolLine.test.tsx index 1a1195c9cb5..854d5a7813d 100644 --- a/src/renderer/src/components/native-chat/NativeChatToolLine.test.tsx +++ b/src/renderer/src/components/native-chat/NativeChatToolLine.test.tsx @@ -1,11 +1,16 @@ // @vitest-environment happy-dom import '@testing-library/jest-dom/vitest' -import { cleanup, fireEvent, render, screen } from '@testing-library/react' -import { afterEach, describe, expect, it } from 'vitest' +import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' import { NativeChatToolLine } from './NativeChatToolLine' import { NativeChatToolRun } from './NativeChatToolRun' import type { NativeChatToolCallBlock } from '../../../../shared/native-chat-types' +vi.mock('@/lib/syntax-highlighting/oniguruma', async () => ({ + loadOniguruma: (await import('@/lib/syntax-highlighting/oniguruma-test-harness')) + .loadNodeOniguruma +})) + afterEach(cleanup) describe('tool sentence rows', () => { @@ -320,6 +325,33 @@ describe('tool sentence rows', () => { } ) + it.each([ + ['PowerShell', 'Get-ChildItem', true], + ['execute', 'Get-ChildItem', false] + ])('colors the full %s command only when its shell is known', async (name, verb, colored) => { + const command = `${verb} -Path ${'C:\\repo\\deep\\'.repeat(40)} | Where-Object { $_.Length -gt 10 }` + const { container } = render( + + ) + fireEvent.click(screen.getByRole('button')) + const detail = container.querySelector('pre') + + if (colored) { + // The first PowerShell block in this file loads the grammar and its regex engine. + await waitFor( + () => { + expect(detail?.querySelector('span[style*="--syntax-dark"]')).not.toBeNull() + }, + { timeout: 10_000 } + ) + } else { + expect(detail?.querySelector('span')).toBeNull() + } + expect(detail?.textContent).toBe(command) + }) + it('keeps a long plain command with surrounding whitespace complete', () => { const command = ` pnpm test ${'src/renderer/tests/long-path/'.repeat(170)}end.test.ts ` const { container } = render( diff --git a/src/renderer/src/components/native-chat/NativeChatToolLine.tsx b/src/renderer/src/components/native-chat/NativeChatToolLine.tsx index fbef701dc53..049f9106e53 100644 --- a/src/renderer/src/components/native-chat/NativeChatToolLine.tsx +++ b/src/renderer/src/components/native-chat/NativeChatToolLine.tsx @@ -18,6 +18,7 @@ import { import { NativeChatToolIcon } from './NativeChatToolIcon' import { NativeChatToolFileTarget } from './NativeChatToolFileTarget' import { NativeChatDiffView } from './NativeChatDiffView' +import { NativeChatHighlightedCode } from './NativeChatHighlightedCode' import { nativeChatToolLineLabel } from './native-chat-tool-line-label' import { diffFromText, diffFromToolCall } from './native-chat-diff' import { NativeChatExpandable } from './NativeChatExpandable' @@ -33,12 +34,14 @@ function ToolLineDetail({ block, body, fullCommand, + commandLanguage, results, onLinkClick }: { block: NativeChatBlock body: NativeChatToolResultBlock | undefined fullCommand: string | null + commandLanguage: string | null results: NonNullable onLinkClick?: CommentMarkdownLinkClickHandler }): React.JSX.Element { @@ -58,7 +61,11 @@ function ToolLineDetail({ data-native-chat-code-content className="max-h-64 overflow-auto whitespace-pre-wrap break-words rounded-lg border border-chat-code-border bg-chat-code-surface p-2 font-mono text-xs text-chat-foreground scrollbar-sleek" > - {fullCommand} + {commandLanguage ? ( + + ) : ( + fullCommand + )} ) : null} {body ? ( @@ -190,6 +197,7 @@ export function NativeChatToolLine({ block={block} body={body} fullCommand={fullCommand} + commandLanguage={label?.commandLanguage ?? null} results={results} onLinkClick={onLinkClick} /> diff --git a/src/renderer/src/components/native-chat/native-chat-command-language.test.ts b/src/renderer/src/components/native-chat/native-chat-command-language.test.ts new file mode 100644 index 00000000000..cb542834254 --- /dev/null +++ b/src/renderer/src/components/native-chat/native-chat-command-language.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest' +import { nativeChatCommandLanguage } from './native-chat-command-language' + +describe('nativeChatCommandLanguage', () => { + it.each([ + ['Bash', { command: 'git status' }, 'shellscript'], + ['PowerShell', { command: 'Get-ChildItem' }, 'powershell'], + // A bash tool still ran bash when its command calls another shell. + ['Bash', { command: 'cmd.exe /c dir' }, 'shellscript'] + ])('trusts a %s tool to name its shell', (name, input, language) => { + expect(nativeChatCommandLanguage(name, input)).toBe(language) + }) + + it.each([ + [{ command: "/bin/zsh -lc 'git status'" }, 'shellscript'], + [{ command: ['bash', '-lc', 'ls -la'] }, 'shellscript'], + [{ command: 'wsl.exe -e bash -lc "ls"' }, 'shellscript'], + [ + { + command: + '"C:\\\\WINDOWS\\\\System32\\\\WindowsPowerShell\\\\v1.0\\\\powershell.exe" -Command "Get-ChildItem"' + }, + 'powershell' + ], + [{ command: 'pwsh -NoProfile -Command Get-Date' }, 'powershell'], + [{ cmd: 'C:\\Windows\\System32\\cmd.exe /c dir' }, 'bat'] + ])('reads the shell a generic tool wrapped %j in', (input, language) => { + expect(nativeChatCommandLanguage('shell', input)).toBe(language) + }) + + it.each([ + { command: 'git status' }, + { command: 'bash' }, + { command: 'python -c "print(1)"' }, + {} + ])('leaves %j plain when nothing says which shell ran it', (input) => { + expect(nativeChatCommandLanguage('execute', input)).toBeNull() + }) +}) diff --git a/src/renderer/src/components/native-chat/native-chat-command-language.ts b/src/renderer/src/components/native-chat/native-chat-command-language.ts new file mode 100644 index 00000000000..5577b1ec2f1 --- /dev/null +++ b/src/renderer/src/components/native-chat/native-chat-command-language.ts @@ -0,0 +1,46 @@ +import { nativeChatToolInputText } from './native-chat-tool-input-text' + +// A tool named after its shell ran there, whatever the host's own shell is. +const LANGUAGE_BY_TOOL_NAME: ReadonlyMap = new Map([ + ['bash', 'shellscript'], + ['powershell', 'powershell'] +]) + +// The program a generic command tool spawned to run the command. +const LANGUAGE_BY_SHELL_PROGRAM: ReadonlyMap = new Map([ + ['bash', 'shellscript'], + ['sh', 'shellscript'], + ['zsh', 'shellscript'], + ['dash', 'shellscript'], + ['ksh', 'shellscript'], + ['wsl', 'shellscript'], + ['fish', 'fish'], + ['pwsh', 'powershell'], + ['powershell', 'powershell'], + ['cmd', 'bat'] +]) + +// The first word, quoted or bare, when arguments follow it. +const PROGRAM_WITH_ARGUMENTS = /^\s*(?:"([^"]+)"|'([^']+)'|(\S+))\s+\S/ + +function shellProgram(command: string): string | null { + const match = PROGRAM_WITH_ARGUMENTS.exec(command) + const path = match?.[1] ?? match?.[2] ?? match?.[3] + const name = path?.split(/[\\/]/).at(-1)?.toLowerCase() + return name?.replace(/\.exe$/, '') ?? null +} + +/** + * The grammar for a command row, from the shell the command actually ran in: a + * tool named after its shell, or the shell program the agent wrapped the + * command in. Null when neither says, so the command stays plain. + */ +export function nativeChatCommandLanguage(toolName: string, input: unknown): string | null { + const byName = LANGUAGE_BY_TOOL_NAME.get(toolName.trim().toLowerCase()) + if (byName) { + return byName + } + const command = nativeChatToolInputText(input, 'command') || nativeChatToolInputText(input, 'cmd') + const program = command ? shellProgram(command) : null + return (program && LANGUAGE_BY_SHELL_PROGRAM.get(program)) ?? null +} diff --git a/src/renderer/src/components/native-chat/native-chat-tool-line-label.ts b/src/renderer/src/components/native-chat/native-chat-tool-line-label.ts index c2d2179092c..72419af7f07 100644 --- a/src/renderer/src/components/native-chat/native-chat-tool-line-label.ts +++ b/src/renderer/src/components/native-chat/native-chat-tool-line-label.ts @@ -1,4 +1,5 @@ import { translate } from '@/i18n/i18n' +import { nativeChatCommandLanguage } from './native-chat-command-language' import { nativeChatToolCategory } from './native-chat-tool-category' import { nativeChatFullCommand, @@ -25,6 +26,8 @@ export function nativeChatToolLineLabel( filePath: string | null /** The whole command, when the row had to shorten it. */ commandDetail: string | null + /** Grammar of `commandDetail`; null when the row holds no command text. */ + commandLanguage: string | null } { const display = createToolInputDisplay(call.input) const category = nativeChatToolCategory(call.name, call.mcpIdentity) @@ -36,6 +39,7 @@ export function nativeChatToolLineLabel( (call.state === 'completed' || result !== undefined) const failed = call.state === 'failed' || result?.isError === true let commandDetail: string | null = null + let commandLanguage: string | null = null let verb: string | null = null let target = display.label let title = display.filePath ?? target @@ -46,6 +50,7 @@ export function nativeChatToolLineLabel( target = fullCommand === null ? target : summarizeToolInput(fullCommand) title = fullCommand ?? target if (fullCommand !== null) { + commandLanguage = nativeChatCommandLanguage(call.name, call.input) const plainCommand = nativeChatPlainCommandInput(call.input) if (plainCommand?.trim() === fullCommand) { commandDetail = plainCommand === target ? null : plainCommand @@ -115,5 +120,5 @@ export function nativeChatToolLineLabel( verb = translate('components.native-chat.tool.row.triedSearchWeb', 'Tried to search the web') } } - return { verb, target, title, command, filePath, commandDetail } + return { verb, target, title, command, filePath, commandDetail, commandLanguage } } diff --git a/src/renderer/src/hooks/use-highlighted-syntax.ts b/src/renderer/src/hooks/use-highlighted-syntax.ts new file mode 100644 index 00000000000..5d2c9b884dc --- /dev/null +++ b/src/renderer/src/hooks/use-highlighted-syntax.ts @@ -0,0 +1,135 @@ +import { useEffect, useMemo, useReducer, useRef, useState } from 'react' +import { + createIncrementalSyntaxTokenizer, + type SyntaxProgress +} from '@/lib/syntax-highlighting/incremental-syntax-tokenizer' +import { scheduleSyntaxHighlighting } from '@/lib/syntax-highlighting/syntax-highlight-scheduler' +import { + loadedSyntaxHighlighter, + loadSyntaxLanguage +} from '@/lib/syntax-highlighting/syntax-highlighter' +import { BoundedMap } from '../../../shared/bounded-map' + +// Each token is a DOM node, so only the start of a very long block is colored. +const MAX_HIGHLIGHTED_CODE_LENGTH = 50_000 +// Tokenizing runs on the renderer thread, so it is spent in slices shorter than a frame. +const RENDER_BUDGET_MS = 4 +const MAX_FINISHED_BYTES = 8 * 1024 * 1024 +const MAX_FINISHED_ENTRIES = 256 +// Rough retained size of one token object beyond its text. +const TOKEN_OVERHEAD_BYTES = 120 +const LINE_OVERHEAD_BYTES = 80 + +let renderDeadline: number | null = null + +/** One budget for every block that renders in the same task, however many mount at once. */ +function remainingRenderBudgetMs(): number { + if (renderDeadline === null) { + renderDeadline = performance.now() + RENDER_BUDGET_MS + setTimeout(() => { + renderDeadline = null + }, 0) + } + return Math.max(0, renderDeadline - performance.now()) +} + +/** UTF-16 text in the key and the tokens' copies of it, plus the objects holding them. */ +function finishedBytes(progress: SyntaxProgress, key: string): number { + let bytes = key.length * 2 + for (const line of progress.lines) { + bytes += LINE_OVERHEAD_BYTES + for (const token of line.tokens) { + bytes += TOKEN_OVERHEAD_BYTES + token.content.length * 2 + } + } + return bytes +} + +// Finished blocks, so a row scrolled back into view paints in color without tokenizing again. +const finished = new BoundedMap({ + maxEntries: MAX_FINISHED_ENTRIES, + maxBytes: MAX_FINISHED_BYTES, + sizeOf: finishedBytes +}) + +function finishedKey(language: string, source: string): string { + return `${language}\n${source}` +} + +/** + * Tokens for as much of `code` as has been colored so far, or null while there + * is nothing to color it with. The rest, `code.slice(highlightedLength)`, is + * still plain and arrives over later renders. + */ +export function useHighlightedSyntax(code: string, language: string): SyntaxProgress | null { + const source = + code.length > MAX_HIGHLIGHTED_CODE_LENGTH ? code.slice(0, MAX_HIGHLIGHTED_CODE_LENGTH) : code + const highlighter = loadedSyntaxHighlighter(language) + const [, rerender] = useReducer((count: number) => count + 1, 0) + + // Why: `source` is a dependency so a block still streaming asks again after a failed load. + useEffect(() => { + if (highlighter) { + return + } + let cancelled = false + void loadSyntaxLanguage(language).then((status) => { + if (!cancelled && status === 'ready') { + rerender() + } + }) + return () => { + cancelled = true + } + }, [language, highlighter, source]) + + const tokenize = useMemo( + () => (highlighter ? createIncrementalSyntaxTokenizer(highlighter) : null), + [highlighter] + ) + // Short blocks finish here and paint in color at once; longer ones continue below. + const started = useMemo( + () => + finished.get(finishedKey(language, source)) ?? + (tokenize ? tokenize(source, remainingRenderBudgetMs()) : null), + [language, source, tokenize] + ) + const [continued, setContinued] = useState<{ + after: SyntaxProgress + progress: SyntaxProgress + } | null>(null) + const progress = continued?.after === started ? continued.progress : started + + useEffect(() => { + if (!tokenize || !started || started.highlightedLength >= source.length) { + return + } + return scheduleSyntaxHighlighting((budgetMs) => { + const next = tokenize(source, budgetMs) + setContinued({ after: started, progress: next }) + return next.highlightedLength >= source.length + }) + }, [source, started, tokenize]) + + const latest = useRef({ language, source, progress }) + useEffect(() => { + latest.current = { language, source, progress } + }) + // Why: remembered on unmount, not on every streamed chunk, which would fill the cache with prefixes. + useEffect( + () => () => { + const last = latest.current + // A degraded block may only have lost its grammar mid-stream, so it is colored afresh next time. + if ( + last.progress && + !last.progress.degraded && + last.progress.highlightedLength >= last.source.length + ) { + finished.set(finishedKey(last.language, last.source), last.progress) + } + }, + [] + ) + + return progress +} diff --git a/src/renderer/src/lib/monaco-languages/textmate-token-provider.ts b/src/renderer/src/lib/monaco-languages/textmate-token-provider.ts index ace5ca9966e..b61e5c78e93 100644 --- a/src/renderer/src/lib/monaco-languages/textmate-token-provider.ts +++ b/src/renderer/src/lib/monaco-languages/textmate-token-provider.ts @@ -1,7 +1,7 @@ import type * as Monaco from 'monaco-editor' import { INITIAL, Registry } from 'vscode-textmate' import type { IGrammar, IOnigLib, IRawGrammar, StateStack } from 'vscode-textmate' -import onigurumaWasmUrl from 'vscode-oniguruma/release/onig.wasm?url' +import { loadOniguruma } from '../syntax-highlighting/oniguruma' type TextMateTokensProvider = Monaco.languages.TokensProvider @@ -13,24 +13,12 @@ export type TextMateTokensProviderOptions = { loadOniguruma?: () => Promise } -let browserOnigurumaPromise: Promise | undefined - async function loadBrowserOniguruma(): Promise { - browserOnigurumaPromise ??= (async () => { - const oniguruma = await import('vscode-oniguruma') - const response = await fetch(onigurumaWasmUrl) - if (!response.ok) { - throw new Error(`Failed to load TextMate regex engine from ${onigurumaWasmUrl}`) - } - - await oniguruma.loadWASM(response) - return { - createOnigScanner: oniguruma.createOnigScanner, - createOnigString: oniguruma.createOnigString - } - })() - - return browserOnigurumaPromise + const oniguruma = await loadOniguruma() + return { + createOnigScanner: oniguruma.createOnigScanner, + createOnigString: oniguruma.createOnigString + } } class TextMateTokenizerState implements Monaco.languages.IState { diff --git a/src/renderer/src/lib/syntax-highlighting/incremental-syntax-tokenizer.test.ts b/src/renderer/src/lib/syntax-highlighting/incremental-syntax-tokenizer.test.ts new file mode 100644 index 00000000000..aa8f73630a2 --- /dev/null +++ b/src/renderer/src/lib/syntax-highlighting/incremental-syntax-tokenizer.test.ts @@ -0,0 +1,151 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + createIncrementalSyntaxTokenizer, + type SyntaxProgress +} from './incremental-syntax-tokenizer' +import { + loadedSyntaxHighlighter, + loadSyntaxLanguage, + type SyntaxHighlighter, + type SyntaxToken +} from './syntax-highlighter' + +vi.mock('./oniguruma', async () => ({ + loadOniguruma: (await import('./oniguruma-test-harness')).loadNodeOniguruma +})) + +const DOCUMENT = [ + 'const greeting = `hello', + '// still inside the template string', + '${name}` /* a comment', + 'that spans lines */ + 1', + 'export default greeting', + '' +].join('\n') + +async function typescriptTokenizer(): Promise { + await loadSyntaxLanguage('typescript') + const highlighter = loadedSyntaxHighlighter('typescript') + if (!highlighter) { + throw new Error('typescript grammar did not load') + } + return highlighter +} + +function colorsOf(lines: readonly (readonly SyntaxToken[])[]): unknown { + return lines.map((line) => line.map((token) => [token.content, token.light, token.dark])) +} + +function colors(progress: SyntaxProgress): unknown { + return colorsOf(progress.lines.map((line) => line.tokens)) +} + +function text(progress: SyntaxProgress, code: string): string { + return ( + progress.lines + .map((line) => line.tokens.map((token) => token.content).join('') + line.ending) + .join('') + code.slice(progress.highlightedLength) + ) +} + +afterEach(() => { + vi.restoreAllMocks() +}) + +describe('createIncrementalSyntaxTokenizer', () => { + it('colors a streamed document exactly like a single full pass', async () => { + const tokenizer = await typescriptTokenizer() + const tokenize = createIncrementalSyntaxTokenizer(tokenizer) + + for (let length = 1; length <= DOCUMENT.length; length += 1) { + const streamed = DOCUMENT.slice(0, length) + const fullPass = tokenizer(streamed).lines + // A full pass reports the empty line after a final newline; nothing renders for it. + if (streamed.endsWith('\n')) { + fullPass.pop() + } + expect(colors(tokenize(streamed, Infinity)), streamed).toEqual(colorsOf(fullPass)) + } + }) + + // Markdown ends a fence that is still being written with a newline; a raw stream does not. + it.each(['', '\n'])( + 'keeps completed lines by identity while a document ending in %j grows', + async (ending) => { + const tokenize = createIncrementalSyntaxTokenizer(await typescriptTokenizer()) + const grown = `const a = 1\nconst b = 2${ending}` + + const before = tokenize(`const a = 1\nconst b${ending}`, Infinity) + const after = tokenize(grown, Infinity) + + expect(after.lines[0]).toBe(before.lines[0]) + expect(after.lines[1]).not.toBe(before.lines[1]) + expect(text(after, grown)).toBe(grown) + } + ) + + it('starts over when the document is replaced rather than extended', async () => { + const tokenizer = await typescriptTokenizer() + const tokenize = createIncrementalSyntaxTokenizer(tokenizer) + + tokenize('const a = `open\nstill open\n', Infinity) + const replaced = 'let b = 2\nlet c = 3' + + expect(colors(tokenize(replaced, Infinity))).toEqual( + colors(createIncrementalSyntaxTokenizer(tokenizer)(replaced, Infinity)) + ) + }) + + it('stops when its time is spent and finishes over later calls', async () => { + const tokenizer = await typescriptTokenizer() + // Every batch takes one tick of a clock that otherwise stands still. + let now = 0 + vi.spyOn(performance, 'now').mockImplementation(() => now) + const tokenize = createIncrementalSyntaxTokenizer((code, state) => { + now += 1 + return tokenizer(code, state) + }) + const code = 'const value = compute(1, "two", [3])\n'.repeat(200) + + let progress = tokenize(code, 1) + expect(progress.highlightedLength).toBeGreaterThan(0) + expect(progress.highlightedLength).toBeLessThan(code.length) + expect(text(progress, code)).toBe(code) + + while (progress.highlightedLength < code.length) { + progress = tokenize(code, 1) + } + expect(colors(progress)).toEqual( + colors(createIncrementalSyntaxTokenizer(tokenizer)(code, Infinity)) + ) + }) + + it('keeps every line after a skipped one plain, however the document streams', async () => { + const tokenize = createIncrementalSyntaxTokenizer(await typescriptTokenizer()) + const code = `const a = 1\nconst long = "${'x'.repeat(2000)}"\nconst b = 2\nconst c = 3\n` + + let progress = tokenize(code.slice(0, code.indexOf('const b')), Infinity) + progress = tokenize(code, Infinity) + + expect(progress.degraded).toBe(true) + expect(colors(progress)).toEqual([ + expect.arrayContaining([['const', '#0000FF', '#569CD6']]), + [[`const long = "${'x'.repeat(2000)}"`, undefined, undefined]], + [['const b = 2', undefined, undefined]], + [['const c = 3', undefined, undefined]] + ]) + expect(text(progress, code)).toBe(code) + }) + + it.each(['a = 1\r\nb = 2\r\n', 'a = 1\r\nb = 2\nc = 3\r', 'a = 1\rb = 2\r\n\r\nc'])( + 'keeps the line endings of %j', + async (code) => { + const tokenize = createIncrementalSyntaxTokenizer(await typescriptTokenizer()) + + for (let length = 1; length <= code.length; length += 1) { + const streamed = code.slice(0, length) + expect(text(tokenize(streamed, Infinity), streamed)).toBe(streamed) + } + } + ) +}) diff --git a/src/renderer/src/lib/syntax-highlighting/incremental-syntax-tokenizer.ts b/src/renderer/src/lib/syntax-highlighting/incremental-syntax-tokenizer.ts new file mode 100644 index 00000000000..656b46f02fd --- /dev/null +++ b/src/renderer/src/lib/syntax-highlighting/incremental-syntax-tokenizer.ts @@ -0,0 +1,99 @@ +import type { SyntaxHighlighter, SyntaxHighlightState, SyntaxToken } from './syntax-highlighter' + +export type SyntaxLine = { + /** Offset of the line in the document; unique, so it keys the line. */ + start: number + tokens: readonly SyntaxToken[] + /** The separator the source has after this line, CRLF included. */ + ending: string +} + +export type SyntaxProgress = { + lines: SyntaxLine[] + /** `code.slice(highlightedLength)` has not been tokenized yet. */ + highlightedLength: number + /** A line was skipped, so the lines after it stay plain. */ + degraded: boolean +} + +// A batch cannot be interrupted, so it stays near a millisecond of dense code. +const BATCH_LENGTH = 250 +const BATCH_LINES = 10 + +/** + * The text of a run of lines without its final separator, copied out of the + * document: a substring keeps its parent alive, so tokens cut straight from a + * streaming block would pin every copy of it that streamed past. + */ +function detachedLines(text: string): string { + return ` ${text.replace(/\r?\n$/, '')}`.slice(1) +} + +/** + * Tokenizes a document a few lines at a time, resuming after the last completed + * line, and stops once `budgetMs` is spent. Completed lines keep their identity. + */ +export function createIncrementalSyntaxTokenizer( + highlight: SyntaxHighlighter +): (code: string, budgetMs: number) => SyntaxProgress { + let completed = '' + let state: SyntaxHighlightState | undefined + let lines: SyntaxLine[] = [] + let degraded = false + + return (code, budgetMs) => { + if (!code.startsWith(completed)) { + completed = '' + state = undefined + lines = [] + degraded = false + } + const deadline = performance.now() + budgetMs + // Why: markdown ends a fence still being written with a newline, so the last line can grow. + const lastBreak = code.lastIndexOf('\n', code.length - 2) + 1 + while (completed.length < lastBreak) { + if (performance.now() >= deadline) { + return { lines, highlightedLength: completed.length, degraded } + } + const from = completed.length + let end = from + for (let count = 0; count < BATCH_LINES && end < lastBreak; count += 1) { + const next = code.indexOf('\n', end) + 1 + if (count > 0 && next - from > BATCH_LENGTH) { + break + } + end = next + } + const tokenized = highlight(detachedLines(code.slice(from, end)), state) + state = tokenized.state + degraded ||= tokenized.outcome !== 'ok' + let start = from + const batch = tokenized.lines.map((tokens) => { + const lineEnd = tokens.reduce((offset, token) => offset + token.content.length, start) + const ending = code.startsWith('\r\n', lineEnd) ? '\r\n' : '\n' + const line = { start, tokens, ending } + start = lineEnd + ending.length + return line + }) + lines = [...lines, ...batch] + completed = code.slice(0, end) + } + const last = code.slice(completed.length) + if (!last) { + return { lines, highlightedLength: code.length, degraded } + } + if (performance.now() >= deadline) { + return { lines, highlightedLength: completed.length, degraded } + } + const text = detachedLines(last) + const partial = highlight(text, state) + return { + lines: [ + ...lines, + { start: completed.length, tokens: partial.lines[0] ?? [], ending: last.slice(text.length) } + ], + highlightedLength: code.length, + degraded: degraded || partial.outcome !== 'ok' + } + } +} diff --git a/src/renderer/src/lib/syntax-highlighting/oniguruma-test-harness.ts b/src/renderer/src/lib/syntax-highlighting/oniguruma-test-harness.ts new file mode 100644 index 00000000000..487e50fd47f --- /dev/null +++ b/src/renderer/src/lib/syntax-highlighting/oniguruma-test-harness.ts @@ -0,0 +1,18 @@ +import { readFile } from 'node:fs/promises' +import { createRequire } from 'node:module' +import type { Oniguruma } from './oniguruma' + +let nodeOnigurumaPromise: Promise | undefined + +/** The bundled Oniguruma WASM read from disk, for tests running outside a browser. */ +export function loadNodeOniguruma(): Promise { + nodeOnigurumaPromise ??= (async () => { + const oniguruma = await import('vscode-oniguruma') + const wasm = await readFile( + createRequire(import.meta.url).resolve('vscode-oniguruma/release/onig.wasm') + ) + await oniguruma.loadWASM(wasm.buffer.slice(wasm.byteOffset, wasm.byteOffset + wasm.byteLength)) + return oniguruma + })() + return nodeOnigurumaPromise +} diff --git a/src/renderer/src/lib/syntax-highlighting/oniguruma.ts b/src/renderer/src/lib/syntax-highlighting/oniguruma.ts new file mode 100644 index 00000000000..bd2243f1a6e --- /dev/null +++ b/src/renderer/src/lib/syntax-highlighting/oniguruma.ts @@ -0,0 +1,24 @@ +import type * as OnigurumaModule from 'vscode-oniguruma' +import onigurumaWasmUrl from 'vscode-oniguruma/release/onig.wasm?url' + +export type Oniguruma = typeof OnigurumaModule + +let onigurumaPromise: Promise | undefined + +/** The renderer's one Oniguruma WASM instance, shared by every TextMate tokenizer. */ +export function loadOniguruma(): Promise { + onigurumaPromise ??= (async () => { + const oniguruma = await import('vscode-oniguruma') + const response = await fetch(onigurumaWasmUrl) + if (!response.ok) { + throw new Error(`Failed to load TextMate regex engine from ${onigurumaWasmUrl}`) + } + await oniguruma.loadWASM(response) + return oniguruma + })().catch((error: unknown) => { + // Why: a failed fetch would otherwise leave every grammar plain until restart. + onigurumaPromise = undefined + throw error + }) + return onigurumaPromise +} diff --git a/src/renderer/src/lib/syntax-highlighting/syntax-highlight-scheduler.test.ts b/src/renderer/src/lib/syntax-highlighting/syntax-highlight-scheduler.test.ts new file mode 100644 index 00000000000..3ca8ab469bb --- /dev/null +++ b/src/renderer/src/lib/syntax-highlighting/syntax-highlight-scheduler.test.ts @@ -0,0 +1,49 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { scheduleSyntaxHighlighting } from './syntax-highlight-scheduler' + +afterEach(() => { + vi.restoreAllMocks() +}) + +describe('scheduleSyntaxHighlighting', () => { + it('shares one slice between blocks and takes turns', async () => { + let now = 0 + vi.spyOn(performance, 'now').mockImplementation(() => now) + const runs: string[] = [] + const budgets: number[] = [] + const job = (name: string, steps: number) => { + let left = steps + return (budgetMs: number) => { + runs.push(name) + budgets.push(budgetMs) + // Each step spends the whole slice. + now += budgetMs + left -= 1 + return left === 0 + } + } + + scheduleSyntaxHighlighting(job('a', 2)) + scheduleSyntaxHighlighting(job('b', 2)) + await vi.waitFor(() => expect(runs).toHaveLength(4)) + + expect(runs).toEqual(['a', 'b', 'a', 'b']) + // One block per slice: the first spends it all, so the other waits for the next. + expect(budgets).toEqual([6, 6, 6, 6]) + }) + + it('stops a cancelled block', async () => { + const steps: string[] = [] + const cancel = scheduleSyntaxHighlighting(() => { + steps.push('cancelled') + return false + }) + cancel() + scheduleSyntaxHighlighting(() => { + steps.push('kept') + return true + }) + + await vi.waitFor(() => expect(steps).toEqual(['kept'])) + }) +}) diff --git a/src/renderer/src/lib/syntax-highlighting/syntax-highlight-scheduler.ts b/src/renderer/src/lib/syntax-highlighting/syntax-highlight-scheduler.ts new file mode 100644 index 00000000000..2453015b2e9 --- /dev/null +++ b/src/renderer/src/lib/syntax-highlighting/syntax-highlight-scheduler.ts @@ -0,0 +1,54 @@ +import { yieldToEventLoop } from '../../../../shared/event-loop-yield' + +/** Runs for at most `budgetMs`; true once its work is finished. */ +export type SyntaxHighlightJob = (budgetMs: number) => boolean + +// Tokenizing runs on the renderer thread, so every block together gets one slice shorter than a frame. +const SLICE_BUDGET_MS = 6 + +const jobs = new Set() +let running = false + +async function runJobs(): Promise { + running = true + try { + while (jobs.size > 0) { + await yieldToEventLoop() + const deadline = performance.now() + SLICE_BUDGET_MS + // A snapshot, so a job sent to the back waits for the next slice. + for (const job of Array.from(jobs)) { + const remaining = deadline - performance.now() + if (remaining <= 0) { + break + } + if (!jobs.has(job)) { + continue + } + let finished = true + try { + finished = job(remaining) + } catch { + // The lines colored so far stay; the rest keeps its plain text. + } + jobs.delete(job) + if (!finished) { + // Why: to the back, so a long block cannot starve the others. + jobs.add(job) + } + } + } + } finally { + running = false + } +} + +/** Shares one time slice per task across every block still highlighting; returns a cancel. */ +export function scheduleSyntaxHighlighting(job: SyntaxHighlightJob): () => void { + jobs.add(job) + if (!running) { + void runJobs() + } + return () => { + jobs.delete(job) + } +} diff --git a/src/renderer/src/lib/syntax-highlighting/syntax-highlighter.test.ts b/src/renderer/src/lib/syntax-highlighting/syntax-highlighter.test.ts new file mode 100644 index 00000000000..c905b92cc4b --- /dev/null +++ b/src/renderer/src/lib/syntax-highlighting/syntax-highlighter.test.ts @@ -0,0 +1,196 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type * as SyntaxHighlighterModule from './syntax-highlighter' +import { loadNodeOniguruma } from './oniguruma-test-harness' +import { + loadedSyntaxHighlighter, + loadSyntaxLanguage, + type SyntaxHighlighter, + type SyntaxHighlightResult, + type SyntaxToken +} from './syntax-highlighter' + +const oniguruma = vi.hoisted(() => ({ fails: false, loads: 0 })) + +vi.mock('./oniguruma', () => ({ + loadOniguruma: () => { + oniguruma.loads += 1 + return oniguruma.fails + ? Promise.reject(new Error('regex engine unavailable')) + : loadNodeOniguruma() + } +})) + +afterEach(() => { + vi.restoreAllMocks() +}) + +async function highlighterFor(language: string): Promise { + await expect(loadSyntaxLanguage(language)).resolves.toBe('ready') + const highlighter = loadedSyntaxHighlighter(language) + if (!highlighter) { + throw new Error(`${language} did not load`) + } + return highlighter +} + +/** A module whose regex engine has not loaded yet. */ +async function freshHighlighterModule(): Promise { + vi.resetModules() + vi.spyOn(console, 'warn').mockImplementation(() => {}) + return import('./syntax-highlighter') +} + +function find(result: SyntaxHighlightResult, content: string): SyntaxToken | undefined { + return result.lines.flat().find((token) => token.content.trim() === content) +} + +function colors(lines: readonly (readonly SyntaxToken[])[]): unknown { + return lines.map((line) => line.map((token) => [token.content, token.light, token.dark])) +} + +describe('highlightSyntax', () => { + it('colors each theme on its own and leaves default text to the surface', async () => { + const result = (await highlighterFor('ts'))('const answer = 42') + + expect(result.outcome).toBe('ok') + expect(find(result, 'const')).toMatchObject({ light: '#0000FF', dark: '#569CD6' }) + expect(find(result, '=')).toMatchObject({ light: undefined, dark: undefined }) + expect(result.lines[0].map((token) => token.content).join('')).toBe('const answer = 42') + }) + + it('continues from a returned state exactly as one pass over the whole document', async () => { + const python = await highlighterFor('python') + const lines = [ + 'def run(x):', + ' s = f"""multi', + ' line {x}', + ' """', + ' with open(s) as f:', + ' return f.read()' + ] + const whole = python(lines.join('\n')) + const head = python(lines.slice(0, 2).join('\n')) + const tail = python(lines.slice(2).join('\n'), head.state) + + expect(colors([...head.lines, ...tail.lines])).toEqual(colors(whole.lines)) + // After the multi-line f-string, `with` is a keyword again. + expect(find(whole, 'with')).toMatchObject({ dark: '#C586C0' }) + }) + + it('reports formatting per theme', async () => { + const result = (await highlighterFor('markdown'))('*emphasis* and **strong**') + + expect(find(result, '*emphasis*')).toMatchObject({ lightFontStyle: 1, darkFontStyle: 1 }) + expect(find(result, '**strong**')).toMatchObject({ lightFontStyle: 2, darkFontStyle: 2 }) + }) + + it.each([ + ['one line', ''], + ['several lines', ''] + ])('colors Vue template expressions over %s', async (_, code) => { + // Markdown names Vue's scope first, which is what kept these plain. + await highlighterFor('markdown') + const result = (await highlighterFor('vue'))(code) + + expect(find(result, 'format')).toMatchObject({ dark: '#DCDCAA' }) + expect(find(result, 'total')).toMatchObject({ dark: '#9CDCFE' }) + }) + + it('leaves a line too long to color plain, along with every line after it', async () => { + const typescript = await highlighterFor('typescript') + const code = ['const a = 1', `const long = "${'x'.repeat(2000)}"`, 'const b = 2'].join('\n') + + const result = typescript(code) + + expect(result.outcome).toBe('degraded') + expect(find(result, 'const')).toMatchObject({ dark: '#569CD6' }) + expect(result.lines[1]).toEqual([expect.objectContaining({ dark: undefined })]) + expect(result.lines[2]).toEqual([ + { + content: 'const b = 2', + offset: code.lastIndexOf('const b'), + light: undefined, + dark: undefined, + lightFontStyle: 0, + darkFontStyle: 0 + } + ]) + expect(typescript('const c = 3', result.state)).toMatchObject({ + outcome: 'degraded', + lines: [[expect.objectContaining({ content: 'const c = 3', dark: undefined })]] + }) + }) + + it('leaves a line that runs out of time plain instead of guessing the rest', async () => { + const typescript = await highlighterFor('typescript') + let now = 0 + vi.spyOn(Date, 'now').mockImplementation(() => (now += 1000)) + + const result = typescript('const a = `template ${value} string`') + + expect(result.outcome).toBe('degraded') + expect(result.lines[0]).toEqual([expect.objectContaining({ dark: undefined })]) + }) + + it('does not resume from a state taken before its grammar reloaded', async () => { + const vue = await highlighterFor('vue') + const head = vue('