mirror of
https://github.com/stablyai/orca.git
synced 2026-11-01 01:27:35 +00:00
feat(native-chat): highlight code blocks and shell commands (#26343)
* feat(native-chat): highlight code blocks and shell commands Uses Shiki rather than the highlight.js already in the preview: it can resume mid-block while a reply streams. * fix(native-chat): leave the tool row command unhighlighted * refactor(native-chat): highlight code through a shared TextMate interface Chat code now goes through one renderer highlighting interface built on Shiki core and the Oniguruma WASM the editor already loads for Nim and Typst, with VS Code Light+/Dark+. Lines that are too long or too slow leave the rest of the block plain, failed grammar loads retry from a mounted block, every block shares one time slice, finished-block caching counts retained text, each theme keeps its own formatting, Vue template expressions are colored, and tool-row commands take the grammar of the shell they ran in. * chore(deps): drop pnpm self-install entries from the lockfile Keeps the lockfile change to Shiki alone; the earlier commit picked up the @pnpm/exe entries pnpm 12 writes when it runs in a fresh checkout. * test(native-chat): give cold grammar loads time in highlighting tests The first colored block in a test file loads its grammar and the regex engine, which took over a second on a busy CI runner. --------- Co-authored-by: Brennan Benson <79079362+brennanb2025@users.noreply.github.com>
This commit is contained in:
Kelvin Amoaba
co-authored by
Brennan Benson
Brennan Benson
parent
470f685b80
commit
2cb2bef67e
@@ -318,6 +318,7 @@
|
||||
"remark-rehype": "11.1.2",
|
||||
"remend": "1.4.0",
|
||||
"shadcn": "^4.13.1",
|
||||
"shiki": "4.5.0",
|
||||
"sonner": "2.0.7",
|
||||
"tailwind-merge": "^3.7.0",
|
||||
"tailwindcss": "^4.2.4",
|
||||
|
||||
Generated
+146
@@ -582,6 +582,9 @@ importers:
|
||||
shadcn:
|
||||
specifier: ^4.13.1
|
||||
version: 4.13.1(supports-color@7.2.0)(typescript@7.0.2)
|
||||
shiki:
|
||||
specifier: 4.5.0
|
||||
version: 4.5.0
|
||||
sonner:
|
||||
specifier: 2.0.7
|
||||
version: 2.0.7(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
|
||||
@@ -3164,6 +3167,37 @@ packages:
|
||||
resolution: {integrity: sha512-8mpfsoPeRhesY3pOrzNZBL8uG6N5GVX1EHLBYbd4gzKs+c7vaEIqpTNK5VrffU33qQN4cwpP2v3u4aPPBU32sw==}
|
||||
engines: {node: '>=16'}
|
||||
|
||||
'@shikijs/core@4.5.0':
|
||||
resolution: {integrity: sha512-6CIgwq+bt2JZ3OSjb/nBkkwEf/CfpHcbam+Mg5Pllco7HAGRQ2/EFhZrrJtbyNTgKwknfVS+llsXGygiDGwJWw==}
|
||||
engines: {node: '>=20'}
|
||||
|
||||
'@shikijs/engine-javascript@4.5.0':
|
||||
resolution: {integrity: sha512-Ed6UdEj84LyYkwnxM5CvSMWAD+id9ExTYri0/+x41U9zrgNifPSS9lZm0JqBP0VfYI8yr8tY5nbNq9+p9R0kFA==}
|
||||
engines: {node: '>=20'}
|
||||
|
||||
'@shikijs/engine-oniguruma@4.5.0':
|
||||
resolution: {integrity: sha512-dKVmbJecVdB9recl8a/CU4jdpNXyjJQrIcesIkpS70s5HS02ohpnCSpMUbotPnEUzbAcLQ+S4hwZYbkBBiTcJw==}
|
||||
engines: {node: '>=20'}
|
||||
|
||||
'@shikijs/langs@4.5.0':
|
||||
resolution: {integrity: sha512-wwBTqwduXSz0X4v9Qa2EWeFaYJEw1clCncWjoRNQRbVDu3SPRTRWucnouYapKj3b8Q+sk/41Vbc4B5c8nuYzLA==}
|
||||
engines: {node: '>=20'}
|
||||
|
||||
'@shikijs/primitive@4.5.0':
|
||||
resolution: {integrity: sha512-UM7ShlA8oz0OWxLLO8BmO9Ip6fVqdkt8yF0EH7nOCqHfrx1eEAzF0ZUVK7L+I8NEgeu1PEWSRlxaiPPeCb4iDg==}
|
||||
engines: {node: '>=20'}
|
||||
|
||||
'@shikijs/themes@4.5.0':
|
||||
resolution: {integrity: sha512-zGb87H3UBbpuI4yoh5UguB1XbU+2BDU3kflM7UfZ+Cy2hWo9lBtqjSVK20gmLY6ukYl040hAypGXMgud9vul8Q==}
|
||||
engines: {node: '>=20'}
|
||||
|
||||
'@shikijs/types@4.5.0':
|
||||
resolution: {integrity: sha512-drtPrtD8N0DENyD2cyt6Ajy+IWsiCLgMxODwLTpvH1UsN6a9jf90tNTPjqwHvA+0Lzu9e0txk8DjfqH1p9UJRw==}
|
||||
engines: {node: '>=20'}
|
||||
|
||||
'@shikijs/vscode-textmate@10.0.2':
|
||||
resolution: {integrity: sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==}
|
||||
|
||||
'@sindresorhus/is@4.6.0':
|
||||
resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==}
|
||||
engines: {node: '>=10'}
|
||||
@@ -3883,6 +3917,9 @@ packages:
|
||||
'@types/hast@3.0.4':
|
||||
resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==}
|
||||
|
||||
'@types/hast@3.0.5':
|
||||
resolution: {integrity: sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==}
|
||||
|
||||
'@types/http-cache-semantics@4.2.0':
|
||||
resolution: {integrity: sha512-L3LgimLHXtGkWikKnsPg0/VFx9OGZaC+eN1u4r+OB1XRqH3meBIAVC2zr1WdMH+RHmnRkqliQAOHNJ/E0j/e0Q==}
|
||||
|
||||
@@ -5450,6 +5487,9 @@ packages:
|
||||
hast-util-sanitize@5.0.2:
|
||||
resolution: {integrity: sha512-3yTWghByc50aGS7JlGhk61SPenfE/p1oaFeNwkOOyrscaOkMGrcW9+Cy/QAIOBpZxP1yqDIzFMR0+Np0i0+usg==}
|
||||
|
||||
hast-util-to-html@9.0.5:
|
||||
resolution: {integrity: sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==}
|
||||
|
||||
hast-util-to-jsx-runtime@2.3.6:
|
||||
resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==}
|
||||
|
||||
@@ -6481,6 +6521,12 @@ packages:
|
||||
resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
oniguruma-parser@0.12.2:
|
||||
resolution: {integrity: sha512-6HVa5oIrgMC6aA6WF6XyyqbhRPJrKR02L20+2+zpDtO5QAzGHAUGw5TKQvwi5vctNnRHkJYmjAhRVQF2EKdTQw==}
|
||||
|
||||
oniguruma-to-es@4.3.6:
|
||||
resolution: {integrity: sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA==}
|
||||
|
||||
open@11.0.0:
|
||||
resolution: {integrity: sha512-smsWv2LzFjP03xmvFoJ331ss6h+jixfA4UUV/Bsiyuu4YJPfN+FIQGOIiv4w9/+MoHkfkJ22UIaQWRVFRfH6Vw==}
|
||||
engines: {node: '>=20'}
|
||||
@@ -6966,6 +7012,15 @@ packages:
|
||||
resolution: {integrity: sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
regex-recursion@6.0.2:
|
||||
resolution: {integrity: sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==}
|
||||
|
||||
regex-utilities@2.3.0:
|
||||
resolution: {integrity: sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==}
|
||||
|
||||
regex@6.1.0:
|
||||
resolution: {integrity: sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg==}
|
||||
|
||||
rehype-highlight@7.0.2:
|
||||
resolution: {integrity: sha512-k158pK7wdC2qL3M5NcZROZ2tR/l7zOzjxXd5VGdcfIyoijjQqpHd3JKtYSBDpDZ38UI2WJWuFAtkMDxmx5kstA==}
|
||||
|
||||
@@ -7248,6 +7303,10 @@ packages:
|
||||
sherpa-onnx@1.12.37:
|
||||
resolution: {integrity: sha512-3luwSdHwR8BtJiiFwqHfb15FE2FX0KsN4aOBbfq9Ma23r3w9C3bprFc/WBusXk56nUbzcEN5YczN7t9w1JwdtQ==}
|
||||
|
||||
shiki@4.5.0:
|
||||
resolution: {integrity: sha512-OcRW72ngQvc+MSVWBRIovC/BTW589iUE/Vxav7VeM4RcY9MUjqB+IR6ILG1k9nuW6LiG/nct9uma4/MBtnZbmQ==}
|
||||
engines: {node: '>=20'}
|
||||
|
||||
shipstyles@1.0.0:
|
||||
resolution: {integrity: sha512-1AcihfXo/mSxqeOdoKqq/5TNv+vSH0dBYPFq88PUX5FBkK5pRmY5TP9SuMNio85PQ+euUI67+KpSVk1cUVDCgA==}
|
||||
|
||||
@@ -10193,6 +10252,46 @@ snapshots:
|
||||
|
||||
'@shaderfrog/glsl-parser@7.0.1': {}
|
||||
|
||||
'@shikijs/core@4.5.0':
|
||||
dependencies:
|
||||
'@shikijs/primitive': 4.5.0
|
||||
'@shikijs/types': 4.5.0
|
||||
'@shikijs/vscode-textmate': 10.0.2
|
||||
'@types/hast': 3.0.5
|
||||
hast-util-to-html: 9.0.5
|
||||
|
||||
'@shikijs/engine-javascript@4.5.0':
|
||||
dependencies:
|
||||
'@shikijs/types': 4.5.0
|
||||
'@shikijs/vscode-textmate': 10.0.2
|
||||
oniguruma-to-es: 4.3.6
|
||||
|
||||
'@shikijs/engine-oniguruma@4.5.0':
|
||||
dependencies:
|
||||
'@shikijs/types': 4.5.0
|
||||
'@shikijs/vscode-textmate': 10.0.2
|
||||
|
||||
'@shikijs/langs@4.5.0':
|
||||
dependencies:
|
||||
'@shikijs/types': 4.5.0
|
||||
|
||||
'@shikijs/primitive@4.5.0':
|
||||
dependencies:
|
||||
'@shikijs/types': 4.5.0
|
||||
'@shikijs/vscode-textmate': 10.0.2
|
||||
'@types/hast': 3.0.5
|
||||
|
||||
'@shikijs/themes@4.5.0':
|
||||
dependencies:
|
||||
'@shikijs/types': 4.5.0
|
||||
|
||||
'@shikijs/types@4.5.0':
|
||||
dependencies:
|
||||
'@shikijs/vscode-textmate': 10.0.2
|
||||
'@types/hast': 3.0.5
|
||||
|
||||
'@shikijs/vscode-textmate@10.0.2': {}
|
||||
|
||||
'@sindresorhus/is@4.6.0': {}
|
||||
|
||||
'@sindresorhus/merge-streams@4.0.0': {}
|
||||
@@ -10877,6 +10976,10 @@ snapshots:
|
||||
dependencies:
|
||||
'@types/unist': 3.0.3
|
||||
|
||||
'@types/hast@3.0.5':
|
||||
dependencies:
|
||||
'@types/unist': 3.0.3
|
||||
|
||||
'@types/http-cache-semantics@4.2.0': {}
|
||||
|
||||
'@types/json-schema@7.0.15': {}
|
||||
@@ -12633,6 +12736,20 @@ snapshots:
|
||||
'@ungap/structured-clone': 1.3.1
|
||||
unist-util-position: 5.0.0
|
||||
|
||||
hast-util-to-html@9.0.5:
|
||||
dependencies:
|
||||
'@types/hast': 3.0.4
|
||||
'@types/unist': 3.0.3
|
||||
ccount: 2.0.1
|
||||
comma-separated-tokens: 2.0.3
|
||||
hast-util-whitespace: 3.0.0
|
||||
html-void-elements: 3.0.0
|
||||
mdast-util-to-hast: 13.2.1
|
||||
property-information: 7.1.0
|
||||
space-separated-tokens: 2.0.2
|
||||
stringify-entities: 4.0.4
|
||||
zwitch: 2.0.4
|
||||
|
||||
hast-util-to-jsx-runtime@2.3.6(supports-color@7.2.0):
|
||||
dependencies:
|
||||
'@types/estree': 1.0.8
|
||||
@@ -13847,6 +13964,14 @@ snapshots:
|
||||
dependencies:
|
||||
mimic-function: 5.0.1
|
||||
|
||||
oniguruma-parser@0.12.2: {}
|
||||
|
||||
oniguruma-to-es@4.3.6:
|
||||
dependencies:
|
||||
oniguruma-parser: 0.12.2
|
||||
regex: 6.1.0
|
||||
regex-recursion: 6.0.2
|
||||
|
||||
open@11.0.0:
|
||||
dependencies:
|
||||
default-browser: 5.5.0
|
||||
@@ -14489,6 +14614,16 @@ snapshots:
|
||||
indent-string: 4.0.0
|
||||
strip-indent: 3.0.0
|
||||
|
||||
regex-recursion@6.0.2:
|
||||
dependencies:
|
||||
regex-utilities: 2.3.0
|
||||
|
||||
regex-utilities@2.3.0: {}
|
||||
|
||||
regex@6.1.0:
|
||||
dependencies:
|
||||
regex-utilities: 2.3.0
|
||||
|
||||
rehype-highlight@7.0.2:
|
||||
dependencies:
|
||||
'@types/hast': 3.0.4
|
||||
@@ -14854,6 +14989,17 @@ snapshots:
|
||||
|
||||
sherpa-onnx@1.12.37: {}
|
||||
|
||||
shiki@4.5.0:
|
||||
dependencies:
|
||||
'@shikijs/core': 4.5.0
|
||||
'@shikijs/engine-javascript': 4.5.0
|
||||
'@shikijs/engine-oniguruma': 4.5.0
|
||||
'@shikijs/langs': 4.5.0
|
||||
'@shikijs/themes': 4.5.0
|
||||
'@shikijs/types': 4.5.0
|
||||
'@shikijs/vscode-textmate': 10.0.2
|
||||
'@types/hast': 3.0.5
|
||||
|
||||
shipstyles@1.0.0: {}
|
||||
|
||||
side-channel-list@1.0.1:
|
||||
|
||||
@@ -5,7 +5,7 @@ import type { MessageRowProps } from './NativeChatMessageRow'
|
||||
import { NativeChatPacedMarkdown } from './NativeChatPacedMarkdown'
|
||||
import { NATIVE_CHAT_QUOTE_SOURCE_PROPS } from './native-chat-quote-selection'
|
||||
import { NativeChatToolRun } from './NativeChatToolRun'
|
||||
import { NativeChatCodeBlock } from './NativeChatCodeBlock'
|
||||
import { NativeChatCodeBlock, NativeChatPlainCodeBlock } from './NativeChatCodeBlock'
|
||||
import { NativeChatAgentControls, NativeChatImageAttachments } from './NativeChatTranscriptChrome'
|
||||
import type { useNativeChatWorkRun } from './use-native-chat-work-run'
|
||||
|
||||
@@ -74,7 +74,7 @@ export function NativeChatAssistantMessageRow({
|
||||
content={words}
|
||||
variant="document"
|
||||
className="text-sm native-chat-message-text"
|
||||
renderCodeBlock={NativeChatCodeBlock}
|
||||
renderCodeBlock={isSystem ? NativeChatPlainCodeBlock : NativeChatCodeBlock}
|
||||
onLinkClick={onLinkClick}
|
||||
allowFileUriLinks={allowFileUriLinks}
|
||||
linkifyFilePaths={onLinkClick !== undefined}
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
// @vitest-environment happy-dom
|
||||
|
||||
import '@testing-library/jest-dom/vitest'
|
||||
import { cleanup, render, waitFor } from '@testing-library/react'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { loadSyntaxLanguage } from '@/lib/syntax-highlighting/syntax-highlighter'
|
||||
import { NativeChatCodeBlock } from './NativeChatCodeBlock'
|
||||
|
||||
// Its own file, so the regex engine has not loaded before the failure.
|
||||
const oniguruma = vi.hoisted(() => ({ fails: false }))
|
||||
|
||||
vi.mock('@/lib/syntax-highlighting/oniguruma', async () => {
|
||||
const { loadNodeOniguruma } = await import('@/lib/syntax-highlighting/oniguruma-test-harness')
|
||||
return {
|
||||
loadOniguruma: () =>
|
||||
oniguruma.fails ? Promise.reject(new Error('regex engine unavailable')) : loadNodeOniguruma()
|
||||
}
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
cleanup()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
describe('NativeChatCodeBlock grammar load failure', () => {
|
||||
it('colors a block still streaming once a failed grammar load succeeds on retry', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
let now = 0
|
||||
vi.spyOn(performance, 'now').mockImplementation(() => now)
|
||||
oniguruma.fails = true
|
||||
const block = (code: string) => (
|
||||
<NativeChatCodeBlock language="rust">
|
||||
<code>{code}</code>
|
||||
</NativeChatCodeBlock>
|
||||
)
|
||||
try {
|
||||
const { container, rerender } = render(block('fn main() {\n'))
|
||||
await expect(loadSyntaxLanguage('rust')).resolves.toBe('failed')
|
||||
expect(container.querySelector('pre span')).toBeNull()
|
||||
|
||||
oniguruma.fails = false
|
||||
now += 5000
|
||||
rerender(block('fn main() {\n let answer = 42;\n'))
|
||||
|
||||
await waitFor(
|
||||
() => {
|
||||
expect(container.querySelector('pre span[style*="--syntax-dark"]')).not.toBeNull()
|
||||
},
|
||||
{ timeout: 10_000 }
|
||||
)
|
||||
expect(container.querySelector('pre')?.textContent).toBe(
|
||||
'fn main() {\n let answer = 42;\n'
|
||||
)
|
||||
} finally {
|
||||
oniguruma.fails = false
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -3,7 +3,13 @@
|
||||
import '@testing-library/jest-dom/vitest'
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { NativeChatCodeBlock } from './NativeChatCodeBlock'
|
||||
import { loadSyntaxLanguage } from '@/lib/syntax-highlighting/syntax-highlighter'
|
||||
import { NativeChatCodeBlock, NativeChatPlainCodeBlock } from './NativeChatCodeBlock'
|
||||
|
||||
vi.mock('@/lib/syntax-highlighting/oniguruma', async () => ({
|
||||
loadOniguruma: (await import('@/lib/syntax-highlighting/oniguruma-test-harness'))
|
||||
.loadNodeOniguruma
|
||||
}))
|
||||
|
||||
afterEach(() => {
|
||||
cleanup()
|
||||
@@ -41,4 +47,77 @@ describe('NativeChatCodeBlock', () => {
|
||||
})
|
||||
expect(await screen.findByRole('button', { name: 'Copied' })).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('colors a fenced block without changing its text', async () => {
|
||||
const code = 'const answer = 42\nconsole.log(answer)\n'
|
||||
const { container } = render(
|
||||
<NativeChatCodeBlock language="typescript">
|
||||
<code>{code}</code>
|
||||
</NativeChatCodeBlock>
|
||||
)
|
||||
const pre = container.querySelector('pre')
|
||||
|
||||
await waitFor(
|
||||
() => {
|
||||
expect(pre?.querySelector('span[style*="--syntax-dark"]')).not.toBeNull()
|
||||
},
|
||||
{ timeout: 10_000 }
|
||||
)
|
||||
expect(pre?.textContent).toBe(code)
|
||||
})
|
||||
|
||||
it('keeps Windows line endings and finishes a block longer than one slice', async () => {
|
||||
const code = `${'const value = compute(1, "two", [3])\r\n'.repeat(400)}const lastLine = 1\r\n`
|
||||
const { container } = render(
|
||||
<NativeChatCodeBlock language="typescript">
|
||||
<code>{code}</code>
|
||||
</NativeChatCodeBlock>
|
||||
)
|
||||
|
||||
expect(await screen.findByText('lastLine', {}, { timeout: 10_000 })).toBeInTheDocument()
|
||||
expect(container.querySelector('pre')?.textContent).toBe(code)
|
||||
})
|
||||
|
||||
it('leaves the code of a dimmed row uncolored', async () => {
|
||||
const code = <code>{'const dimmed = 1\n'}</code>
|
||||
await loadSyntaxLanguage('typescript')
|
||||
const colored = render(<NativeChatCodeBlock language="typescript">{code}</NativeChatCodeBlock>)
|
||||
expect(colored.container.querySelector('pre span')).not.toBeNull()
|
||||
|
||||
const plain = render(
|
||||
<NativeChatPlainCodeBlock language="typescript">{code}</NativeChatPlainCodeBlock>
|
||||
)
|
||||
|
||||
expect(plain.container.querySelector('pre span')).toBeNull()
|
||||
expect(plain.container.querySelector('pre')?.textContent).toBe('const dimmed = 1\n')
|
||||
})
|
||||
|
||||
it('paints a finished block in color at once when its row mounts again', async () => {
|
||||
// Long enough that tokenizing it again could not finish within one render.
|
||||
const block = (
|
||||
<NativeChatCodeBlock language="typescript">
|
||||
<code>{`${'const a=[1,2,3].map((x)=>x+1);f(a,{b:1,c:"s"});\n'.repeat(300)}const lastLine = 1\n`}</code>
|
||||
</NativeChatCodeBlock>
|
||||
)
|
||||
const first = render(block)
|
||||
expect(await screen.findByText('lastLine', {}, { timeout: 10_000 })).toBeInTheDocument()
|
||||
const colored = first.container.querySelectorAll('span[style*="--syntax-dark"]').length
|
||||
first.unmount()
|
||||
|
||||
const second = render(block)
|
||||
|
||||
expect(second.container.querySelectorAll('span[style*="--syntax-dark"]')).toHaveLength(colored)
|
||||
})
|
||||
|
||||
it('leaves a fence in an unknown language as plain text', async () => {
|
||||
const { container } = render(
|
||||
<NativeChatCodeBlock language="not-a-real-language">
|
||||
<code>{'plain words\n'}</code>
|
||||
</NativeChatCodeBlock>
|
||||
)
|
||||
const pre = container.querySelector('pre')
|
||||
|
||||
await waitFor(() => expect(pre?.textContent).toBe('plain words\n'))
|
||||
expect(pre?.querySelector('span')).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -3,14 +3,17 @@ import { translate } from '@/i18n/i18n'
|
||||
import { cn } from '@/lib/utils'
|
||||
import { getCodeBlockLanguageLabel } from '@/components/editor/rich-markdown-code-block-languages'
|
||||
import { NativeChatCopyButton } from './NativeChatCopyButton'
|
||||
import { NativeChatHighlightedCode } from './NativeChatHighlightedCode'
|
||||
|
||||
/** Code fences need their own copy target rather than the whole chat message. */
|
||||
export function NativeChatCodeBlock({
|
||||
children,
|
||||
language
|
||||
language,
|
||||
plain = false
|
||||
}: {
|
||||
children?: React.ReactNode
|
||||
language?: string
|
||||
plain?: boolean
|
||||
}): React.JSX.Element {
|
||||
const code = extractCodeText(children)
|
||||
|
||||
@@ -40,7 +43,13 @@ export function NativeChatCodeBlock({
|
||||
language ? 'px-3.5 pt-0.5 pb-3' : 'p-3 pr-10'
|
||||
)}
|
||||
>
|
||||
{children}
|
||||
{language && code && !plain ? (
|
||||
<code>
|
||||
<NativeChatHighlightedCode code={code} language={language} />
|
||||
</code>
|
||||
) : (
|
||||
children
|
||||
)}
|
||||
</pre>
|
||||
{code && !language ? (
|
||||
<NativeChatCopyButton
|
||||
@@ -53,6 +62,14 @@ export function NativeChatCodeBlock({
|
||||
)
|
||||
}
|
||||
|
||||
/** For dimmed rows: their code keeps the row's faint color, so it is not tokenized at all. */
|
||||
export function NativeChatPlainCodeBlock(props: {
|
||||
children?: React.ReactNode
|
||||
language?: string
|
||||
}): React.JSX.Element {
|
||||
return <NativeChatCodeBlock {...props} plain />
|
||||
}
|
||||
|
||||
function extractCodeText(node: React.ReactNode): string {
|
||||
if (typeof node === 'string' || typeof node === 'number') {
|
||||
return String(node)
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
import { memo } from 'react'
|
||||
import { useHighlightedSyntax } from '@/hooks/use-highlighted-syntax'
|
||||
import type { SyntaxLine } from '@/lib/syntax-highlighting/incremental-syntax-tokenizer'
|
||||
import { SYNTAX_FONT_STYLE, type SyntaxToken } from '@/lib/syntax-highlighting/syntax-highlighter'
|
||||
|
||||
type TokenStyle = React.CSSProperties & Record<`--syntax-${string}`, string | undefined>
|
||||
|
||||
// Both themes ride on the token so a scheme change needs no re-tokenizing.
|
||||
const COLORED_TOKEN_CLASS = 'text-(color:--syntax-light) dark:text-(color:--syntax-dark)'
|
||||
const FORMATTED_TOKEN_CLASS =
|
||||
'text-(color:--syntax-light) dark:text-(color:--syntax-dark) [font-style:var(--syntax-light-font-style)] dark:[font-style:var(--syntax-dark-font-style)] [font-weight:var(--syntax-light-font-weight)] dark:[font-weight:var(--syntax-dark-font-weight)] [text-decoration-line:var(--syntax-light-decoration)] dark:[text-decoration-line:var(--syntax-dark-decoration)]'
|
||||
|
||||
function formatting(theme: 'light' | 'dark', fontStyle: number): TokenStyle {
|
||||
const decorations = [
|
||||
fontStyle & SYNTAX_FONT_STYLE.underline ? 'underline' : '',
|
||||
fontStyle & SYNTAX_FONT_STYLE.strikethrough ? 'line-through' : ''
|
||||
].filter(Boolean)
|
||||
return {
|
||||
[`--syntax-${theme}-font-style`]: fontStyle & SYNTAX_FONT_STYLE.italic ? 'italic' : 'normal',
|
||||
[`--syntax-${theme}-font-weight`]: fontStyle & SYNTAX_FONT_STYLE.bold ? 'bold' : 'normal',
|
||||
[`--syntax-${theme}-decoration`]: decorations.join(' ') || 'none'
|
||||
}
|
||||
}
|
||||
|
||||
function tokenStyle(token: SyntaxToken): TokenStyle {
|
||||
const colors: TokenStyle = { '--syntax-light': token.light, '--syntax-dark': token.dark }
|
||||
return token.lightFontStyle || token.darkFontStyle
|
||||
? {
|
||||
...colors,
|
||||
...formatting('light', token.lightFontStyle),
|
||||
...formatting('dark', token.darkFontStyle)
|
||||
}
|
||||
: colors
|
||||
}
|
||||
|
||||
const HighlightedLine = memo(function HighlightedLine({ line }: { line: SyntaxLine }) {
|
||||
return (
|
||||
<>
|
||||
{line.tokens.map((token) =>
|
||||
// Uncolored text needs no element of its own.
|
||||
token.content.trim() &&
|
||||
(token.light || token.dark || token.lightFontStyle || token.darkFontStyle) ? (
|
||||
<span
|
||||
key={token.offset}
|
||||
className={
|
||||
token.lightFontStyle || token.darkFontStyle
|
||||
? FORMATTED_TOKEN_CLASS
|
||||
: COLORED_TOKEN_CLASS
|
||||
}
|
||||
style={tokenStyle(token)}
|
||||
>
|
||||
{token.content}
|
||||
</span>
|
||||
) : (
|
||||
token.content
|
||||
)
|
||||
)}
|
||||
{line.ending}
|
||||
</>
|
||||
)
|
||||
})
|
||||
|
||||
/** Colors `code` in place; its text is unchanged, so layout and copy match the plain fallback. */
|
||||
export const NativeChatHighlightedCode = memo(function NativeChatHighlightedCode({
|
||||
code,
|
||||
language
|
||||
}: {
|
||||
code: string
|
||||
language: string
|
||||
}): React.ReactNode {
|
||||
const progress = useHighlightedSyntax(code, language)
|
||||
if (!progress) {
|
||||
return code
|
||||
}
|
||||
return (
|
||||
<>
|
||||
{progress.lines.map((line) => (
|
||||
<HighlightedLine key={line.start} line={line} />
|
||||
))}
|
||||
{code.slice(progress.highlightedLength)}
|
||||
</>
|
||||
)
|
||||
})
|
||||
@@ -9,6 +9,10 @@ import type { NativeChatRewindSurface } from './use-native-chat-rewind'
|
||||
import { readNativeChatQuotableSelection } from './native-chat-quote-selection'
|
||||
|
||||
const confirm = vi.hoisted(() => vi.fn())
|
||||
vi.mock('@/lib/syntax-highlighting/oniguruma', async () => ({
|
||||
loadOniguruma: (await import('@/lib/syntax-highlighting/oniguruma-test-harness'))
|
||||
.loadNodeOniguruma
|
||||
}))
|
||||
vi.mock('@/components/confirmation-dialog-context', () => ({
|
||||
useConfirmationDialog: () => confirm
|
||||
}))
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { ChevronRight } from 'lucide-react'
|
||||
import type { CommentMarkdownLinkClickHandler } from '@/components/sidebar/CommentMarkdown'
|
||||
import { NativeChatCodeBlock } from './NativeChatCodeBlock'
|
||||
import { NativeChatPlainCodeBlock } from './NativeChatCodeBlock'
|
||||
import { NativeChatMarkdown } from './NativeChatMarkdown'
|
||||
|
||||
/** A reasoning block's text, under the live activity line or its finished row: quieter prose in the
|
||||
@@ -27,7 +27,7 @@ export function NativeChatReasoningBody({
|
||||
content={markdown}
|
||||
variant="document"
|
||||
className="text-sm native-chat-message-text"
|
||||
renderCodeBlock={NativeChatCodeBlock}
|
||||
renderCodeBlock={NativeChatPlainCodeBlock}
|
||||
onLinkClick={onLinkClick}
|
||||
allowFileUriLinks={allowFileUriLinks}
|
||||
linkifyFilePaths={onLinkClick !== undefined}
|
||||
|
||||
@@ -1,11 +1,16 @@
|
||||
// @vitest-environment happy-dom
|
||||
import '@testing-library/jest-dom/vitest'
|
||||
import { cleanup, fireEvent, render, screen } from '@testing-library/react'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { NativeChatToolLine } from './NativeChatToolLine'
|
||||
import { NativeChatToolRun } from './NativeChatToolRun'
|
||||
import type { NativeChatToolCallBlock } from '../../../../shared/native-chat-types'
|
||||
|
||||
vi.mock('@/lib/syntax-highlighting/oniguruma', async () => ({
|
||||
loadOniguruma: (await import('@/lib/syntax-highlighting/oniguruma-test-harness'))
|
||||
.loadNodeOniguruma
|
||||
}))
|
||||
|
||||
afterEach(cleanup)
|
||||
|
||||
describe('tool sentence rows', () => {
|
||||
@@ -320,6 +325,33 @@ describe('tool sentence rows', () => {
|
||||
}
|
||||
)
|
||||
|
||||
it.each([
|
||||
['PowerShell', 'Get-ChildItem', true],
|
||||
['execute', 'Get-ChildItem', false]
|
||||
])('colors the full %s command only when its shell is known', async (name, verb, colored) => {
|
||||
const command = `${verb} -Path ${'C:\\repo\\deep\\'.repeat(40)} | Where-Object { $_.Length -gt 10 }`
|
||||
const { container } = render(
|
||||
<NativeChatToolLine
|
||||
block={{ type: 'tool-call', name, input: { command }, state: 'completed' }}
|
||||
/>
|
||||
)
|
||||
fireEvent.click(screen.getByRole('button'))
|
||||
const detail = container.querySelector('pre')
|
||||
|
||||
if (colored) {
|
||||
// The first PowerShell block in this file loads the grammar and its regex engine.
|
||||
await waitFor(
|
||||
() => {
|
||||
expect(detail?.querySelector('span[style*="--syntax-dark"]')).not.toBeNull()
|
||||
},
|
||||
{ timeout: 10_000 }
|
||||
)
|
||||
} else {
|
||||
expect(detail?.querySelector('span')).toBeNull()
|
||||
}
|
||||
expect(detail?.textContent).toBe(command)
|
||||
})
|
||||
|
||||
it('keeps a long plain command with surrounding whitespace complete', () => {
|
||||
const command = ` pnpm test ${'src/renderer/tests/long-path/'.repeat(170)}end.test.ts `
|
||||
const { container } = render(
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
import { NativeChatToolIcon } from './NativeChatToolIcon'
|
||||
import { NativeChatToolFileTarget } from './NativeChatToolFileTarget'
|
||||
import { NativeChatDiffView } from './NativeChatDiffView'
|
||||
import { NativeChatHighlightedCode } from './NativeChatHighlightedCode'
|
||||
import { nativeChatToolLineLabel } from './native-chat-tool-line-label'
|
||||
import { diffFromText, diffFromToolCall } from './native-chat-diff'
|
||||
import { NativeChatExpandable } from './NativeChatExpandable'
|
||||
@@ -33,12 +34,14 @@ function ToolLineDetail({
|
||||
block,
|
||||
body,
|
||||
fullCommand,
|
||||
commandLanguage,
|
||||
results,
|
||||
onLinkClick
|
||||
}: {
|
||||
block: NativeChatBlock
|
||||
body: NativeChatToolResultBlock | undefined
|
||||
fullCommand: string | null
|
||||
commandLanguage: string | null
|
||||
results: NonNullable<NativeChatToolCallBlock['webSearchResults']>
|
||||
onLinkClick?: CommentMarkdownLinkClickHandler
|
||||
}): React.JSX.Element {
|
||||
@@ -58,7 +61,11 @@ function ToolLineDetail({
|
||||
data-native-chat-code-content
|
||||
className="max-h-64 overflow-auto whitespace-pre-wrap break-words rounded-lg border border-chat-code-border bg-chat-code-surface p-2 font-mono text-xs text-chat-foreground scrollbar-sleek"
|
||||
>
|
||||
{fullCommand}
|
||||
{commandLanguage ? (
|
||||
<NativeChatHighlightedCode code={fullCommand} language={commandLanguage} />
|
||||
) : (
|
||||
fullCommand
|
||||
)}
|
||||
</pre>
|
||||
) : null}
|
||||
{body ? (
|
||||
@@ -190,6 +197,7 @@ export function NativeChatToolLine({
|
||||
block={block}
|
||||
body={body}
|
||||
fullCommand={fullCommand}
|
||||
commandLanguage={label?.commandLanguage ?? null}
|
||||
results={results}
|
||||
onLinkClick={onLinkClick}
|
||||
/>
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { nativeChatCommandLanguage } from './native-chat-command-language'
|
||||
|
||||
describe('nativeChatCommandLanguage', () => {
|
||||
it.each([
|
||||
['Bash', { command: 'git status' }, 'shellscript'],
|
||||
['PowerShell', { command: 'Get-ChildItem' }, 'powershell'],
|
||||
// A bash tool still ran bash when its command calls another shell.
|
||||
['Bash', { command: 'cmd.exe /c dir' }, 'shellscript']
|
||||
])('trusts a %s tool to name its shell', (name, input, language) => {
|
||||
expect(nativeChatCommandLanguage(name, input)).toBe(language)
|
||||
})
|
||||
|
||||
it.each([
|
||||
[{ command: "/bin/zsh -lc 'git status'" }, 'shellscript'],
|
||||
[{ command: ['bash', '-lc', 'ls -la'] }, 'shellscript'],
|
||||
[{ command: 'wsl.exe -e bash -lc "ls"' }, 'shellscript'],
|
||||
[
|
||||
{
|
||||
command:
|
||||
'"C:\\\\WINDOWS\\\\System32\\\\WindowsPowerShell\\\\v1.0\\\\powershell.exe" -Command "Get-ChildItem"'
|
||||
},
|
||||
'powershell'
|
||||
],
|
||||
[{ command: 'pwsh -NoProfile -Command Get-Date' }, 'powershell'],
|
||||
[{ cmd: 'C:\\Windows\\System32\\cmd.exe /c dir' }, 'bat']
|
||||
])('reads the shell a generic tool wrapped %j in', (input, language) => {
|
||||
expect(nativeChatCommandLanguage('shell', input)).toBe(language)
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ command: 'git status' },
|
||||
{ command: 'bash' },
|
||||
{ command: 'python -c "print(1)"' },
|
||||
{}
|
||||
])('leaves %j plain when nothing says which shell ran it', (input) => {
|
||||
expect(nativeChatCommandLanguage('execute', input)).toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,46 @@
|
||||
import { nativeChatToolInputText } from './native-chat-tool-input-text'
|
||||
|
||||
// A tool named after its shell ran there, whatever the host's own shell is.
|
||||
const LANGUAGE_BY_TOOL_NAME: ReadonlyMap<string, string> = new Map([
|
||||
['bash', 'shellscript'],
|
||||
['powershell', 'powershell']
|
||||
])
|
||||
|
||||
// The program a generic command tool spawned to run the command.
|
||||
const LANGUAGE_BY_SHELL_PROGRAM: ReadonlyMap<string, string> = new Map([
|
||||
['bash', 'shellscript'],
|
||||
['sh', 'shellscript'],
|
||||
['zsh', 'shellscript'],
|
||||
['dash', 'shellscript'],
|
||||
['ksh', 'shellscript'],
|
||||
['wsl', 'shellscript'],
|
||||
['fish', 'fish'],
|
||||
['pwsh', 'powershell'],
|
||||
['powershell', 'powershell'],
|
||||
['cmd', 'bat']
|
||||
])
|
||||
|
||||
// The first word, quoted or bare, when arguments follow it.
|
||||
const PROGRAM_WITH_ARGUMENTS = /^\s*(?:"([^"]+)"|'([^']+)'|(\S+))\s+\S/
|
||||
|
||||
function shellProgram(command: string): string | null {
|
||||
const match = PROGRAM_WITH_ARGUMENTS.exec(command)
|
||||
const path = match?.[1] ?? match?.[2] ?? match?.[3]
|
||||
const name = path?.split(/[\\/]/).at(-1)?.toLowerCase()
|
||||
return name?.replace(/\.exe$/, '') ?? null
|
||||
}
|
||||
|
||||
/**
|
||||
* The grammar for a command row, from the shell the command actually ran in: a
|
||||
* tool named after its shell, or the shell program the agent wrapped the
|
||||
* command in. Null when neither says, so the command stays plain.
|
||||
*/
|
||||
export function nativeChatCommandLanguage(toolName: string, input: unknown): string | null {
|
||||
const byName = LANGUAGE_BY_TOOL_NAME.get(toolName.trim().toLowerCase())
|
||||
if (byName) {
|
||||
return byName
|
||||
}
|
||||
const command = nativeChatToolInputText(input, 'command') || nativeChatToolInputText(input, 'cmd')
|
||||
const program = command ? shellProgram(command) : null
|
||||
return (program && LANGUAGE_BY_SHELL_PROGRAM.get(program)) ?? null
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { translate } from '@/i18n/i18n'
|
||||
import { nativeChatCommandLanguage } from './native-chat-command-language'
|
||||
import { nativeChatToolCategory } from './native-chat-tool-category'
|
||||
import {
|
||||
nativeChatFullCommand,
|
||||
@@ -25,6 +26,8 @@ export function nativeChatToolLineLabel(
|
||||
filePath: string | null
|
||||
/** The whole command, when the row had to shorten it. */
|
||||
commandDetail: string | null
|
||||
/** Grammar of `commandDetail`; null when the row holds no command text. */
|
||||
commandLanguage: string | null
|
||||
} {
|
||||
const display = createToolInputDisplay(call.input)
|
||||
const category = nativeChatToolCategory(call.name, call.mcpIdentity)
|
||||
@@ -36,6 +39,7 @@ export function nativeChatToolLineLabel(
|
||||
(call.state === 'completed' || result !== undefined)
|
||||
const failed = call.state === 'failed' || result?.isError === true
|
||||
let commandDetail: string | null = null
|
||||
let commandLanguage: string | null = null
|
||||
let verb: string | null = null
|
||||
let target = display.label
|
||||
let title = display.filePath ?? target
|
||||
@@ -46,6 +50,7 @@ export function nativeChatToolLineLabel(
|
||||
target = fullCommand === null ? target : summarizeToolInput(fullCommand)
|
||||
title = fullCommand ?? target
|
||||
if (fullCommand !== null) {
|
||||
commandLanguage = nativeChatCommandLanguage(call.name, call.input)
|
||||
const plainCommand = nativeChatPlainCommandInput(call.input)
|
||||
if (plainCommand?.trim() === fullCommand) {
|
||||
commandDetail = plainCommand === target ? null : plainCommand
|
||||
@@ -115,5 +120,5 @@ export function nativeChatToolLineLabel(
|
||||
verb = translate('components.native-chat.tool.row.triedSearchWeb', 'Tried to search the web')
|
||||
}
|
||||
}
|
||||
return { verb, target, title, command, filePath, commandDetail }
|
||||
return { verb, target, title, command, filePath, commandDetail, commandLanguage }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
import { useEffect, useMemo, useReducer, useRef, useState } from 'react'
|
||||
import {
|
||||
createIncrementalSyntaxTokenizer,
|
||||
type SyntaxProgress
|
||||
} from '@/lib/syntax-highlighting/incremental-syntax-tokenizer'
|
||||
import { scheduleSyntaxHighlighting } from '@/lib/syntax-highlighting/syntax-highlight-scheduler'
|
||||
import {
|
||||
loadedSyntaxHighlighter,
|
||||
loadSyntaxLanguage
|
||||
} from '@/lib/syntax-highlighting/syntax-highlighter'
|
||||
import { BoundedMap } from '../../../shared/bounded-map'
|
||||
|
||||
// Each token is a DOM node, so only the start of a very long block is colored.
|
||||
const MAX_HIGHLIGHTED_CODE_LENGTH = 50_000
|
||||
// Tokenizing runs on the renderer thread, so it is spent in slices shorter than a frame.
|
||||
const RENDER_BUDGET_MS = 4
|
||||
const MAX_FINISHED_BYTES = 8 * 1024 * 1024
|
||||
const MAX_FINISHED_ENTRIES = 256
|
||||
// Rough retained size of one token object beyond its text.
|
||||
const TOKEN_OVERHEAD_BYTES = 120
|
||||
const LINE_OVERHEAD_BYTES = 80
|
||||
|
||||
let renderDeadline: number | null = null
|
||||
|
||||
/** One budget for every block that renders in the same task, however many mount at once. */
|
||||
function remainingRenderBudgetMs(): number {
|
||||
if (renderDeadline === null) {
|
||||
renderDeadline = performance.now() + RENDER_BUDGET_MS
|
||||
setTimeout(() => {
|
||||
renderDeadline = null
|
||||
}, 0)
|
||||
}
|
||||
return Math.max(0, renderDeadline - performance.now())
|
||||
}
|
||||
|
||||
/** UTF-16 text in the key and the tokens' copies of it, plus the objects holding them. */
|
||||
function finishedBytes(progress: SyntaxProgress, key: string): number {
|
||||
let bytes = key.length * 2
|
||||
for (const line of progress.lines) {
|
||||
bytes += LINE_OVERHEAD_BYTES
|
||||
for (const token of line.tokens) {
|
||||
bytes += TOKEN_OVERHEAD_BYTES + token.content.length * 2
|
||||
}
|
||||
}
|
||||
return bytes
|
||||
}
|
||||
|
||||
// Finished blocks, so a row scrolled back into view paints in color without tokenizing again.
|
||||
const finished = new BoundedMap<string, SyntaxProgress>({
|
||||
maxEntries: MAX_FINISHED_ENTRIES,
|
||||
maxBytes: MAX_FINISHED_BYTES,
|
||||
sizeOf: finishedBytes
|
||||
})
|
||||
|
||||
function finishedKey(language: string, source: string): string {
|
||||
return `${language}\n${source}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokens for as much of `code` as has been colored so far, or null while there
|
||||
* is nothing to color it with. The rest, `code.slice(highlightedLength)`, is
|
||||
* still plain and arrives over later renders.
|
||||
*/
|
||||
export function useHighlightedSyntax(code: string, language: string): SyntaxProgress | null {
|
||||
const source =
|
||||
code.length > MAX_HIGHLIGHTED_CODE_LENGTH ? code.slice(0, MAX_HIGHLIGHTED_CODE_LENGTH) : code
|
||||
const highlighter = loadedSyntaxHighlighter(language)
|
||||
const [, rerender] = useReducer((count: number) => count + 1, 0)
|
||||
|
||||
// Why: `source` is a dependency so a block still streaming asks again after a failed load.
|
||||
useEffect(() => {
|
||||
if (highlighter) {
|
||||
return
|
||||
}
|
||||
let cancelled = false
|
||||
void loadSyntaxLanguage(language).then((status) => {
|
||||
if (!cancelled && status === 'ready') {
|
||||
rerender()
|
||||
}
|
||||
})
|
||||
return () => {
|
||||
cancelled = true
|
||||
}
|
||||
}, [language, highlighter, source])
|
||||
|
||||
const tokenize = useMemo(
|
||||
() => (highlighter ? createIncrementalSyntaxTokenizer(highlighter) : null),
|
||||
[highlighter]
|
||||
)
|
||||
// Short blocks finish here and paint in color at once; longer ones continue below.
|
||||
const started = useMemo(
|
||||
() =>
|
||||
finished.get(finishedKey(language, source)) ??
|
||||
(tokenize ? tokenize(source, remainingRenderBudgetMs()) : null),
|
||||
[language, source, tokenize]
|
||||
)
|
||||
const [continued, setContinued] = useState<{
|
||||
after: SyntaxProgress
|
||||
progress: SyntaxProgress
|
||||
} | null>(null)
|
||||
const progress = continued?.after === started ? continued.progress : started
|
||||
|
||||
useEffect(() => {
|
||||
if (!tokenize || !started || started.highlightedLength >= source.length) {
|
||||
return
|
||||
}
|
||||
return scheduleSyntaxHighlighting((budgetMs) => {
|
||||
const next = tokenize(source, budgetMs)
|
||||
setContinued({ after: started, progress: next })
|
||||
return next.highlightedLength >= source.length
|
||||
})
|
||||
}, [source, started, tokenize])
|
||||
|
||||
const latest = useRef({ language, source, progress })
|
||||
useEffect(() => {
|
||||
latest.current = { language, source, progress }
|
||||
})
|
||||
// Why: remembered on unmount, not on every streamed chunk, which would fill the cache with prefixes.
|
||||
useEffect(
|
||||
() => () => {
|
||||
const last = latest.current
|
||||
// A degraded block may only have lost its grammar mid-stream, so it is colored afresh next time.
|
||||
if (
|
||||
last.progress &&
|
||||
!last.progress.degraded &&
|
||||
last.progress.highlightedLength >= last.source.length
|
||||
) {
|
||||
finished.set(finishedKey(last.language, last.source), last.progress)
|
||||
}
|
||||
},
|
||||
[]
|
||||
)
|
||||
|
||||
return progress
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import type * as Monaco from 'monaco-editor'
|
||||
import { INITIAL, Registry } from 'vscode-textmate'
|
||||
import type { IGrammar, IOnigLib, IRawGrammar, StateStack } from 'vscode-textmate'
|
||||
import onigurumaWasmUrl from 'vscode-oniguruma/release/onig.wasm?url'
|
||||
import { loadOniguruma } from '../syntax-highlighting/oniguruma'
|
||||
|
||||
type TextMateTokensProvider = Monaco.languages.TokensProvider
|
||||
|
||||
@@ -13,24 +13,12 @@ export type TextMateTokensProviderOptions = {
|
||||
loadOniguruma?: () => Promise<IOnigLib>
|
||||
}
|
||||
|
||||
let browserOnigurumaPromise: Promise<IOnigLib> | undefined
|
||||
|
||||
async function loadBrowserOniguruma(): Promise<IOnigLib> {
|
||||
browserOnigurumaPromise ??= (async () => {
|
||||
const oniguruma = await import('vscode-oniguruma')
|
||||
const response = await fetch(onigurumaWasmUrl)
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to load TextMate regex engine from ${onigurumaWasmUrl}`)
|
||||
}
|
||||
|
||||
await oniguruma.loadWASM(response)
|
||||
return {
|
||||
createOnigScanner: oniguruma.createOnigScanner,
|
||||
createOnigString: oniguruma.createOnigString
|
||||
}
|
||||
})()
|
||||
|
||||
return browserOnigurumaPromise
|
||||
const oniguruma = await loadOniguruma()
|
||||
return {
|
||||
createOnigScanner: oniguruma.createOnigScanner,
|
||||
createOnigString: oniguruma.createOnigString
|
||||
}
|
||||
}
|
||||
|
||||
class TextMateTokenizerState implements Monaco.languages.IState {
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
createIncrementalSyntaxTokenizer,
|
||||
type SyntaxProgress
|
||||
} from './incremental-syntax-tokenizer'
|
||||
import {
|
||||
loadedSyntaxHighlighter,
|
||||
loadSyntaxLanguage,
|
||||
type SyntaxHighlighter,
|
||||
type SyntaxToken
|
||||
} from './syntax-highlighter'
|
||||
|
||||
vi.mock('./oniguruma', async () => ({
|
||||
loadOniguruma: (await import('./oniguruma-test-harness')).loadNodeOniguruma
|
||||
}))
|
||||
|
||||
const DOCUMENT = [
|
||||
'const greeting = `hello',
|
||||
'// still inside the template string',
|
||||
'${name}` /* a comment',
|
||||
'that spans lines */ + 1',
|
||||
'export default greeting',
|
||||
''
|
||||
].join('\n')
|
||||
|
||||
async function typescriptTokenizer(): Promise<SyntaxHighlighter> {
|
||||
await loadSyntaxLanguage('typescript')
|
||||
const highlighter = loadedSyntaxHighlighter('typescript')
|
||||
if (!highlighter) {
|
||||
throw new Error('typescript grammar did not load')
|
||||
}
|
||||
return highlighter
|
||||
}
|
||||
|
||||
function colorsOf(lines: readonly (readonly SyntaxToken[])[]): unknown {
|
||||
return lines.map((line) => line.map((token) => [token.content, token.light, token.dark]))
|
||||
}
|
||||
|
||||
function colors(progress: SyntaxProgress): unknown {
|
||||
return colorsOf(progress.lines.map((line) => line.tokens))
|
||||
}
|
||||
|
||||
function text(progress: SyntaxProgress, code: string): string {
|
||||
return (
|
||||
progress.lines
|
||||
.map((line) => line.tokens.map((token) => token.content).join('') + line.ending)
|
||||
.join('') + code.slice(progress.highlightedLength)
|
||||
)
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
describe('createIncrementalSyntaxTokenizer', () => {
|
||||
it('colors a streamed document exactly like a single full pass', async () => {
|
||||
const tokenizer = await typescriptTokenizer()
|
||||
const tokenize = createIncrementalSyntaxTokenizer(tokenizer)
|
||||
|
||||
for (let length = 1; length <= DOCUMENT.length; length += 1) {
|
||||
const streamed = DOCUMENT.slice(0, length)
|
||||
const fullPass = tokenizer(streamed).lines
|
||||
// A full pass reports the empty line after a final newline; nothing renders for it.
|
||||
if (streamed.endsWith('\n')) {
|
||||
fullPass.pop()
|
||||
}
|
||||
expect(colors(tokenize(streamed, Infinity)), streamed).toEqual(colorsOf(fullPass))
|
||||
}
|
||||
})
|
||||
|
||||
// Markdown ends a fence that is still being written with a newline; a raw stream does not.
|
||||
it.each(['', '\n'])(
|
||||
'keeps completed lines by identity while a document ending in %j grows',
|
||||
async (ending) => {
|
||||
const tokenize = createIncrementalSyntaxTokenizer(await typescriptTokenizer())
|
||||
const grown = `const a = 1\nconst b = 2${ending}`
|
||||
|
||||
const before = tokenize(`const a = 1\nconst b${ending}`, Infinity)
|
||||
const after = tokenize(grown, Infinity)
|
||||
|
||||
expect(after.lines[0]).toBe(before.lines[0])
|
||||
expect(after.lines[1]).not.toBe(before.lines[1])
|
||||
expect(text(after, grown)).toBe(grown)
|
||||
}
|
||||
)
|
||||
|
||||
it('starts over when the document is replaced rather than extended', async () => {
|
||||
const tokenizer = await typescriptTokenizer()
|
||||
const tokenize = createIncrementalSyntaxTokenizer(tokenizer)
|
||||
|
||||
tokenize('const a = `open\nstill open\n', Infinity)
|
||||
const replaced = 'let b = 2\nlet c = 3'
|
||||
|
||||
expect(colors(tokenize(replaced, Infinity))).toEqual(
|
||||
colors(createIncrementalSyntaxTokenizer(tokenizer)(replaced, Infinity))
|
||||
)
|
||||
})
|
||||
|
||||
it('stops when its time is spent and finishes over later calls', async () => {
|
||||
const tokenizer = await typescriptTokenizer()
|
||||
// Every batch takes one tick of a clock that otherwise stands still.
|
||||
let now = 0
|
||||
vi.spyOn(performance, 'now').mockImplementation(() => now)
|
||||
const tokenize = createIncrementalSyntaxTokenizer((code, state) => {
|
||||
now += 1
|
||||
return tokenizer(code, state)
|
||||
})
|
||||
const code = 'const value = compute(1, "two", [3])\n'.repeat(200)
|
||||
|
||||
let progress = tokenize(code, 1)
|
||||
expect(progress.highlightedLength).toBeGreaterThan(0)
|
||||
expect(progress.highlightedLength).toBeLessThan(code.length)
|
||||
expect(text(progress, code)).toBe(code)
|
||||
|
||||
while (progress.highlightedLength < code.length) {
|
||||
progress = tokenize(code, 1)
|
||||
}
|
||||
expect(colors(progress)).toEqual(
|
||||
colors(createIncrementalSyntaxTokenizer(tokenizer)(code, Infinity))
|
||||
)
|
||||
})
|
||||
|
||||
it('keeps every line after a skipped one plain, however the document streams', async () => {
|
||||
const tokenize = createIncrementalSyntaxTokenizer(await typescriptTokenizer())
|
||||
const code = `const a = 1\nconst long = "${'x'.repeat(2000)}"\nconst b = 2\nconst c = 3\n`
|
||||
|
||||
let progress = tokenize(code.slice(0, code.indexOf('const b')), Infinity)
|
||||
progress = tokenize(code, Infinity)
|
||||
|
||||
expect(progress.degraded).toBe(true)
|
||||
expect(colors(progress)).toEqual([
|
||||
expect.arrayContaining([['const', '#0000FF', '#569CD6']]),
|
||||
[[`const long = "${'x'.repeat(2000)}"`, undefined, undefined]],
|
||||
[['const b = 2', undefined, undefined]],
|
||||
[['const c = 3', undefined, undefined]]
|
||||
])
|
||||
expect(text(progress, code)).toBe(code)
|
||||
})
|
||||
|
||||
it.each(['a = 1\r\nb = 2\r\n', 'a = 1\r\nb = 2\nc = 3\r', 'a = 1\rb = 2\r\n\r\nc'])(
|
||||
'keeps the line endings of %j',
|
||||
async (code) => {
|
||||
const tokenize = createIncrementalSyntaxTokenizer(await typescriptTokenizer())
|
||||
|
||||
for (let length = 1; length <= code.length; length += 1) {
|
||||
const streamed = code.slice(0, length)
|
||||
expect(text(tokenize(streamed, Infinity), streamed)).toBe(streamed)
|
||||
}
|
||||
}
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,99 @@
|
||||
import type { SyntaxHighlighter, SyntaxHighlightState, SyntaxToken } from './syntax-highlighter'
|
||||
|
||||
export type SyntaxLine = {
|
||||
/** Offset of the line in the document; unique, so it keys the line. */
|
||||
start: number
|
||||
tokens: readonly SyntaxToken[]
|
||||
/** The separator the source has after this line, CRLF included. */
|
||||
ending: string
|
||||
}
|
||||
|
||||
export type SyntaxProgress = {
|
||||
lines: SyntaxLine[]
|
||||
/** `code.slice(highlightedLength)` has not been tokenized yet. */
|
||||
highlightedLength: number
|
||||
/** A line was skipped, so the lines after it stay plain. */
|
||||
degraded: boolean
|
||||
}
|
||||
|
||||
// A batch cannot be interrupted, so it stays near a millisecond of dense code.
|
||||
const BATCH_LENGTH = 250
|
||||
const BATCH_LINES = 10
|
||||
|
||||
/**
|
||||
* The text of a run of lines without its final separator, copied out of the
|
||||
* document: a substring keeps its parent alive, so tokens cut straight from a
|
||||
* streaming block would pin every copy of it that streamed past.
|
||||
*/
|
||||
function detachedLines(text: string): string {
|
||||
return ` ${text.replace(/\r?\n$/, '')}`.slice(1)
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokenizes a document a few lines at a time, resuming after the last completed
|
||||
* line, and stops once `budgetMs` is spent. Completed lines keep their identity.
|
||||
*/
|
||||
export function createIncrementalSyntaxTokenizer(
|
||||
highlight: SyntaxHighlighter
|
||||
): (code: string, budgetMs: number) => SyntaxProgress {
|
||||
let completed = ''
|
||||
let state: SyntaxHighlightState | undefined
|
||||
let lines: SyntaxLine[] = []
|
||||
let degraded = false
|
||||
|
||||
return (code, budgetMs) => {
|
||||
if (!code.startsWith(completed)) {
|
||||
completed = ''
|
||||
state = undefined
|
||||
lines = []
|
||||
degraded = false
|
||||
}
|
||||
const deadline = performance.now() + budgetMs
|
||||
// Why: markdown ends a fence still being written with a newline, so the last line can grow.
|
||||
const lastBreak = code.lastIndexOf('\n', code.length - 2) + 1
|
||||
while (completed.length < lastBreak) {
|
||||
if (performance.now() >= deadline) {
|
||||
return { lines, highlightedLength: completed.length, degraded }
|
||||
}
|
||||
const from = completed.length
|
||||
let end = from
|
||||
for (let count = 0; count < BATCH_LINES && end < lastBreak; count += 1) {
|
||||
const next = code.indexOf('\n', end) + 1
|
||||
if (count > 0 && next - from > BATCH_LENGTH) {
|
||||
break
|
||||
}
|
||||
end = next
|
||||
}
|
||||
const tokenized = highlight(detachedLines(code.slice(from, end)), state)
|
||||
state = tokenized.state
|
||||
degraded ||= tokenized.outcome !== 'ok'
|
||||
let start = from
|
||||
const batch = tokenized.lines.map((tokens) => {
|
||||
const lineEnd = tokens.reduce((offset, token) => offset + token.content.length, start)
|
||||
const ending = code.startsWith('\r\n', lineEnd) ? '\r\n' : '\n'
|
||||
const line = { start, tokens, ending }
|
||||
start = lineEnd + ending.length
|
||||
return line
|
||||
})
|
||||
lines = [...lines, ...batch]
|
||||
completed = code.slice(0, end)
|
||||
}
|
||||
const last = code.slice(completed.length)
|
||||
if (!last) {
|
||||
return { lines, highlightedLength: code.length, degraded }
|
||||
}
|
||||
if (performance.now() >= deadline) {
|
||||
return { lines, highlightedLength: completed.length, degraded }
|
||||
}
|
||||
const text = detachedLines(last)
|
||||
const partial = highlight(text, state)
|
||||
return {
|
||||
lines: [
|
||||
...lines,
|
||||
{ start: completed.length, tokens: partial.lines[0] ?? [], ending: last.slice(text.length) }
|
||||
],
|
||||
highlightedLength: code.length,
|
||||
degraded: degraded || partial.outcome !== 'ok'
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { createRequire } from 'node:module'
|
||||
import type { Oniguruma } from './oniguruma'
|
||||
|
||||
let nodeOnigurumaPromise: Promise<Oniguruma> | undefined
|
||||
|
||||
/** The bundled Oniguruma WASM read from disk, for tests running outside a browser. */
|
||||
export function loadNodeOniguruma(): Promise<Oniguruma> {
|
||||
nodeOnigurumaPromise ??= (async () => {
|
||||
const oniguruma = await import('vscode-oniguruma')
|
||||
const wasm = await readFile(
|
||||
createRequire(import.meta.url).resolve('vscode-oniguruma/release/onig.wasm')
|
||||
)
|
||||
await oniguruma.loadWASM(wasm.buffer.slice(wasm.byteOffset, wasm.byteOffset + wasm.byteLength))
|
||||
return oniguruma
|
||||
})()
|
||||
return nodeOnigurumaPromise
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import type * as OnigurumaModule from 'vscode-oniguruma'
|
||||
import onigurumaWasmUrl from 'vscode-oniguruma/release/onig.wasm?url'
|
||||
|
||||
export type Oniguruma = typeof OnigurumaModule
|
||||
|
||||
let onigurumaPromise: Promise<Oniguruma> | undefined
|
||||
|
||||
/** The renderer's one Oniguruma WASM instance, shared by every TextMate tokenizer. */
|
||||
export function loadOniguruma(): Promise<Oniguruma> {
|
||||
onigurumaPromise ??= (async () => {
|
||||
const oniguruma = await import('vscode-oniguruma')
|
||||
const response = await fetch(onigurumaWasmUrl)
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to load TextMate regex engine from ${onigurumaWasmUrl}`)
|
||||
}
|
||||
await oniguruma.loadWASM(response)
|
||||
return oniguruma
|
||||
})().catch((error: unknown) => {
|
||||
// Why: a failed fetch would otherwise leave every grammar plain until restart.
|
||||
onigurumaPromise = undefined
|
||||
throw error
|
||||
})
|
||||
return onigurumaPromise
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { scheduleSyntaxHighlighting } from './syntax-highlight-scheduler'
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
describe('scheduleSyntaxHighlighting', () => {
|
||||
it('shares one slice between blocks and takes turns', async () => {
|
||||
let now = 0
|
||||
vi.spyOn(performance, 'now').mockImplementation(() => now)
|
||||
const runs: string[] = []
|
||||
const budgets: number[] = []
|
||||
const job = (name: string, steps: number) => {
|
||||
let left = steps
|
||||
return (budgetMs: number) => {
|
||||
runs.push(name)
|
||||
budgets.push(budgetMs)
|
||||
// Each step spends the whole slice.
|
||||
now += budgetMs
|
||||
left -= 1
|
||||
return left === 0
|
||||
}
|
||||
}
|
||||
|
||||
scheduleSyntaxHighlighting(job('a', 2))
|
||||
scheduleSyntaxHighlighting(job('b', 2))
|
||||
await vi.waitFor(() => expect(runs).toHaveLength(4))
|
||||
|
||||
expect(runs).toEqual(['a', 'b', 'a', 'b'])
|
||||
// One block per slice: the first spends it all, so the other waits for the next.
|
||||
expect(budgets).toEqual([6, 6, 6, 6])
|
||||
})
|
||||
|
||||
it('stops a cancelled block', async () => {
|
||||
const steps: string[] = []
|
||||
const cancel = scheduleSyntaxHighlighting(() => {
|
||||
steps.push('cancelled')
|
||||
return false
|
||||
})
|
||||
cancel()
|
||||
scheduleSyntaxHighlighting(() => {
|
||||
steps.push('kept')
|
||||
return true
|
||||
})
|
||||
|
||||
await vi.waitFor(() => expect(steps).toEqual(['kept']))
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,54 @@
|
||||
import { yieldToEventLoop } from '../../../../shared/event-loop-yield'
|
||||
|
||||
/** Runs for at most `budgetMs`; true once its work is finished. */
|
||||
export type SyntaxHighlightJob = (budgetMs: number) => boolean
|
||||
|
||||
// Tokenizing runs on the renderer thread, so every block together gets one slice shorter than a frame.
|
||||
const SLICE_BUDGET_MS = 6
|
||||
|
||||
const jobs = new Set<SyntaxHighlightJob>()
|
||||
let running = false
|
||||
|
||||
async function runJobs(): Promise<void> {
|
||||
running = true
|
||||
try {
|
||||
while (jobs.size > 0) {
|
||||
await yieldToEventLoop()
|
||||
const deadline = performance.now() + SLICE_BUDGET_MS
|
||||
// A snapshot, so a job sent to the back waits for the next slice.
|
||||
for (const job of Array.from(jobs)) {
|
||||
const remaining = deadline - performance.now()
|
||||
if (remaining <= 0) {
|
||||
break
|
||||
}
|
||||
if (!jobs.has(job)) {
|
||||
continue
|
||||
}
|
||||
let finished = true
|
||||
try {
|
||||
finished = job(remaining)
|
||||
} catch {
|
||||
// The lines colored so far stay; the rest keeps its plain text.
|
||||
}
|
||||
jobs.delete(job)
|
||||
if (!finished) {
|
||||
// Why: to the back, so a long block cannot starve the others.
|
||||
jobs.add(job)
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
running = false
|
||||
}
|
||||
}
|
||||
|
||||
/** Shares one time slice per task across every block still highlighting; returns a cancel. */
|
||||
export function scheduleSyntaxHighlighting(job: SyntaxHighlightJob): () => void {
|
||||
jobs.add(job)
|
||||
if (!running) {
|
||||
void runJobs()
|
||||
}
|
||||
return () => {
|
||||
jobs.delete(job)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type * as SyntaxHighlighterModule from './syntax-highlighter'
|
||||
import { loadNodeOniguruma } from './oniguruma-test-harness'
|
||||
import {
|
||||
loadedSyntaxHighlighter,
|
||||
loadSyntaxLanguage,
|
||||
type SyntaxHighlighter,
|
||||
type SyntaxHighlightResult,
|
||||
type SyntaxToken
|
||||
} from './syntax-highlighter'
|
||||
|
||||
const oniguruma = vi.hoisted(() => ({ fails: false, loads: 0 }))
|
||||
|
||||
vi.mock('./oniguruma', () => ({
|
||||
loadOniguruma: () => {
|
||||
oniguruma.loads += 1
|
||||
return oniguruma.fails
|
||||
? Promise.reject(new Error('regex engine unavailable'))
|
||||
: loadNodeOniguruma()
|
||||
}
|
||||
}))
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
async function highlighterFor(language: string): Promise<SyntaxHighlighter> {
|
||||
await expect(loadSyntaxLanguage(language)).resolves.toBe('ready')
|
||||
const highlighter = loadedSyntaxHighlighter(language)
|
||||
if (!highlighter) {
|
||||
throw new Error(`${language} did not load`)
|
||||
}
|
||||
return highlighter
|
||||
}
|
||||
|
||||
/** A module whose regex engine has not loaded yet. */
|
||||
async function freshHighlighterModule(): Promise<typeof SyntaxHighlighterModule> {
|
||||
vi.resetModules()
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
return import('./syntax-highlighter')
|
||||
}
|
||||
|
||||
function find(result: SyntaxHighlightResult, content: string): SyntaxToken | undefined {
|
||||
return result.lines.flat().find((token) => token.content.trim() === content)
|
||||
}
|
||||
|
||||
function colors(lines: readonly (readonly SyntaxToken[])[]): unknown {
|
||||
return lines.map((line) => line.map((token) => [token.content, token.light, token.dark]))
|
||||
}
|
||||
|
||||
describe('highlightSyntax', () => {
|
||||
it('colors each theme on its own and leaves default text to the surface', async () => {
|
||||
const result = (await highlighterFor('ts'))('const answer = 42')
|
||||
|
||||
expect(result.outcome).toBe('ok')
|
||||
expect(find(result, 'const')).toMatchObject({ light: '#0000FF', dark: '#569CD6' })
|
||||
expect(find(result, '=')).toMatchObject({ light: undefined, dark: undefined })
|
||||
expect(result.lines[0].map((token) => token.content).join('')).toBe('const answer = 42')
|
||||
})
|
||||
|
||||
it('continues from a returned state exactly as one pass over the whole document', async () => {
|
||||
const python = await highlighterFor('python')
|
||||
const lines = [
|
||||
'def run(x):',
|
||||
' s = f"""multi',
|
||||
' line {x}',
|
||||
' """',
|
||||
' with open(s) as f:',
|
||||
' return f.read()'
|
||||
]
|
||||
const whole = python(lines.join('\n'))
|
||||
const head = python(lines.slice(0, 2).join('\n'))
|
||||
const tail = python(lines.slice(2).join('\n'), head.state)
|
||||
|
||||
expect(colors([...head.lines, ...tail.lines])).toEqual(colors(whole.lines))
|
||||
// After the multi-line f-string, `with` is a keyword again.
|
||||
expect(find(whole, 'with')).toMatchObject({ dark: '#C586C0' })
|
||||
})
|
||||
|
||||
it('reports formatting per theme', async () => {
|
||||
const result = (await highlighterFor('markdown'))('*emphasis* and **strong**')
|
||||
|
||||
expect(find(result, '*emphasis*')).toMatchObject({ lightFontStyle: 1, darkFontStyle: 1 })
|
||||
expect(find(result, '**strong**')).toMatchObject({ lightFontStyle: 2, darkFontStyle: 2 })
|
||||
})
|
||||
|
||||
it.each([
|
||||
['one line', '<template>\n <p :title="label">{{ format(total) }}</p>\n</template>'],
|
||||
['several lines', '<template>\n <p>{{\n format(total)\n }}</p>\n</template>']
|
||||
])('colors Vue template expressions over %s', async (_, code) => {
|
||||
// Markdown names Vue's scope first, which is what kept these plain.
|
||||
await highlighterFor('markdown')
|
||||
const result = (await highlighterFor('vue'))(code)
|
||||
|
||||
expect(find(result, 'format')).toMatchObject({ dark: '#DCDCAA' })
|
||||
expect(find(result, 'total')).toMatchObject({ dark: '#9CDCFE' })
|
||||
})
|
||||
|
||||
it('leaves a line too long to color plain, along with every line after it', async () => {
|
||||
const typescript = await highlighterFor('typescript')
|
||||
const code = ['const a = 1', `const long = "${'x'.repeat(2000)}"`, 'const b = 2'].join('\n')
|
||||
|
||||
const result = typescript(code)
|
||||
|
||||
expect(result.outcome).toBe('degraded')
|
||||
expect(find(result, 'const')).toMatchObject({ dark: '#569CD6' })
|
||||
expect(result.lines[1]).toEqual([expect.objectContaining({ dark: undefined })])
|
||||
expect(result.lines[2]).toEqual([
|
||||
{
|
||||
content: 'const b = 2',
|
||||
offset: code.lastIndexOf('const b'),
|
||||
light: undefined,
|
||||
dark: undefined,
|
||||
lightFontStyle: 0,
|
||||
darkFontStyle: 0
|
||||
}
|
||||
])
|
||||
expect(typescript('const c = 3', result.state)).toMatchObject({
|
||||
outcome: 'degraded',
|
||||
lines: [[expect.objectContaining({ content: 'const c = 3', dark: undefined })]]
|
||||
})
|
||||
})
|
||||
|
||||
it('leaves a line that runs out of time plain instead of guessing the rest', async () => {
|
||||
const typescript = await highlighterFor('typescript')
|
||||
let now = 0
|
||||
vi.spyOn(Date, 'now').mockImplementation(() => (now += 1000))
|
||||
|
||||
const result = typescript('const a = `template ${value} string`')
|
||||
|
||||
expect(result.outcome).toBe('degraded')
|
||||
expect(result.lines[0]).toEqual([expect.objectContaining({ dark: undefined })])
|
||||
})
|
||||
|
||||
it('does not resume from a state taken before its grammar reloaded', async () => {
|
||||
const vue = await highlighterFor('vue')
|
||||
const head = vue('<template>')
|
||||
// Vue embeds YAML lazily, so loading YAML rebuilds the Vue grammar.
|
||||
await highlighterFor('yaml')
|
||||
|
||||
expect(vue(' <p>hi</p>', head.state).outcome).toBe('degraded')
|
||||
expect(vue('<template>\n <p>hi</p>').outcome).toBe('ok')
|
||||
})
|
||||
})
|
||||
|
||||
describe('loadSyntaxLanguage', () => {
|
||||
it('reports a language without a grammar as unsupported', async () => {
|
||||
await expect(loadSyntaxLanguage('not-a-real-language')).resolves.toBe('unsupported')
|
||||
expect(loadedSyntaxHighlighter('not-a-real-language')).toBeNull()
|
||||
})
|
||||
|
||||
it('accepts aliases and any letter case', async () => {
|
||||
await expect(loadSyntaxLanguage('PS1')).resolves.toBe('ready')
|
||||
expect(loadedSyntaxHighlighter('ps1')).not.toBeNull()
|
||||
})
|
||||
|
||||
it('retries a failed load after a pause', async () => {
|
||||
const { loadSyntaxLanguage, loadedSyntaxHighlighter } = await freshHighlighterModule()
|
||||
let now = 0
|
||||
vi.spyOn(performance, 'now').mockImplementation(() => now)
|
||||
oniguruma.fails = true
|
||||
try {
|
||||
await expect(loadSyntaxLanguage('rust')).resolves.toBe('failed')
|
||||
const loads = oniguruma.loads
|
||||
now += 10
|
||||
await expect(loadSyntaxLanguage('rust')).resolves.toBe('failed')
|
||||
expect(oniguruma.loads).toBe(loads)
|
||||
|
||||
oniguruma.fails = false
|
||||
now += 5000
|
||||
await expect(loadSyntaxLanguage('rust')).resolves.toBe('ready')
|
||||
expect(loadedSyntaxHighlighter('rust')).not.toBeNull()
|
||||
} finally {
|
||||
oniguruma.fails = false
|
||||
}
|
||||
})
|
||||
|
||||
it('stops retrying after a few failed loads', async () => {
|
||||
const { loadSyntaxLanguage } = await freshHighlighterModule()
|
||||
let now = 0
|
||||
vi.spyOn(performance, 'now').mockImplementation(() => now)
|
||||
oniguruma.fails = true
|
||||
try {
|
||||
for (let attempt = 0; attempt < 3; attempt += 1) {
|
||||
await expect(loadSyntaxLanguage('go')).resolves.toBe('failed')
|
||||
now += 5000
|
||||
}
|
||||
oniguruma.fails = false
|
||||
const loads = oniguruma.loads
|
||||
await expect(loadSyntaxLanguage('go')).resolves.toBe('failed')
|
||||
expect(oniguruma.loads).toBe(loads)
|
||||
} finally {
|
||||
oniguruma.fails = false
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,358 @@
|
||||
import type { Grammar, HighlighterCore, RegexEngine } from 'shiki/core'
|
||||
import type * as ShikiTextmate from 'shiki/textmate'
|
||||
import type { StateStack } from 'shiki/textmate'
|
||||
import { BoundedMap } from '../../../../shared/bounded-map'
|
||||
import { loadOniguruma, type Oniguruma } from './oniguruma'
|
||||
import { withVueTemplateInjections } from './vue-template-injections'
|
||||
|
||||
export type SyntaxToken = {
|
||||
content: string
|
||||
/** Position in the highlighted text; unique within a line. */
|
||||
offset: number
|
||||
/** Undefined where the theme uses its default text color, so the surface's own color shows. */
|
||||
light: string | undefined
|
||||
dark: string | undefined
|
||||
/** `SYNTAX_FONT_STYLE` bits; each theme formats on its own. */
|
||||
lightFontStyle: number
|
||||
darkFontStyle: number
|
||||
}
|
||||
|
||||
/** TextMate's font style bits. */
|
||||
export const SYNTAX_FONT_STYLE = { italic: 1, bold: 2, underline: 4, strikethrough: 8 } as const
|
||||
|
||||
/**
|
||||
* `ok`: every line is colored. `degraded`: a line was too long or too slow, so
|
||||
* it and every later line of the document stay plain. `failed`: the grammar
|
||||
* threw, so the code stays plain.
|
||||
*/
|
||||
export type SyntaxHighlightOutcome = 'ok' | 'degraded' | 'failed'
|
||||
|
||||
/** Opaque: where a document's highlighting stopped, to continue it after its last line. */
|
||||
export type SyntaxHighlightState = ResumeState
|
||||
|
||||
export type SyntaxHighlightResult = {
|
||||
lines: SyntaxToken[][]
|
||||
state: SyntaxHighlightState
|
||||
outcome: SyntaxHighlightOutcome
|
||||
}
|
||||
|
||||
/**
|
||||
* Colors `code` (one or more lines) in VS Code's Light+ and Dark+ themes,
|
||||
* continuing from `state` when that is what the call for the text just before
|
||||
* `code` returned.
|
||||
*/
|
||||
export type SyntaxHighlighter = (
|
||||
code: string,
|
||||
state?: SyntaxHighlightState
|
||||
) => SyntaxHighlightResult
|
||||
|
||||
/** `unsupported`: no grammar for the language. `failed`: the grammar could not load. */
|
||||
export type SyntaxLanguageStatus = 'ready' | 'unsupported' | 'failed'
|
||||
|
||||
const THEMES = { light: 'light-plus', dark: 'dark-plus' } as const
|
||||
// Why: one dense line cannot be interrupted, and its cost grows faster than its length.
|
||||
const MAX_HIGHLIGHTED_LINE_LENGTH = 1000
|
||||
// Why: TextMate checks this between regex scans, so a pathological line ends instead of freezing chat.
|
||||
const LINE_TIME_LIMIT_MS = 50
|
||||
const LINE_ATTEMPTS = 3
|
||||
const MAX_LOAD_ATTEMPTS = 3
|
||||
const LOAD_RETRY_DELAY_MS = 2000
|
||||
// Arbitrary fence labels must not grow the table without limit.
|
||||
const MAX_PENDING_LANGUAGES = 256
|
||||
|
||||
type ThemeStacks = { light: StateStack; dark: StateStack }
|
||||
|
||||
class ResumeState {
|
||||
/** `stacks` is null once the document degraded; it never resumes coloring after that. */
|
||||
constructor(
|
||||
readonly grammar: Grammar | null,
|
||||
readonly stacks: ThemeStacks | null
|
||||
) {}
|
||||
}
|
||||
|
||||
const DEGRADED = new ResumeState(null, null)
|
||||
|
||||
type Highlighter = { core: HighlighterCore; textmate: typeof ShikiTextmate }
|
||||
|
||||
type PendingLanguage =
|
||||
| { status: 'loading'; promise: Promise<SyntaxLanguageStatus>; attempts: number }
|
||||
| { status: 'unsupported' }
|
||||
| { status: 'failed'; attempts: number; retryAt: number }
|
||||
|
||||
// Only catalogue names reach this map, so it is bounded by the catalogue.
|
||||
const readyLanguages = new Map<string, SyntaxHighlighter>()
|
||||
const pendingLanguages = new BoundedMap<string, PendingLanguage>({
|
||||
maxEntries: MAX_PENDING_LANGUAGES
|
||||
})
|
||||
let highlighterPromise: Promise<Highlighter> | undefined
|
||||
let loadFailureLogged = false
|
||||
|
||||
function onigurumaEngine(oniguruma: Oniguruma): RegexEngine {
|
||||
return {
|
||||
createScanner: (patterns) => {
|
||||
const scanner = oniguruma.createOnigScanner(
|
||||
patterns.map((pattern) => (typeof pattern === 'string' ? pattern : pattern.source))
|
||||
)
|
||||
return {
|
||||
// Why: Shiki's TextMate always passes no find options, so none are forwarded.
|
||||
findNextMatchSync: (text, startPosition) =>
|
||||
scanner.findNextMatchSync(
|
||||
typeof text === 'string' || text instanceof oniguruma.OnigString ? text : text.content,
|
||||
startPosition
|
||||
),
|
||||
dispose: () => scanner.dispose()
|
||||
}
|
||||
},
|
||||
createString: (text) => oniguruma.createOnigString(text)
|
||||
}
|
||||
}
|
||||
|
||||
function loadHighlighter(): Promise<Highlighter> {
|
||||
highlighterPromise ??= (async () => {
|
||||
const [{ createHighlighterCore }, textmate, oniguruma, light, dark] = await Promise.all([
|
||||
import('shiki/core'),
|
||||
import('shiki/textmate'),
|
||||
loadOniguruma(),
|
||||
import('shiki/themes/light-plus.mjs'),
|
||||
import('shiki/themes/dark-plus.mjs')
|
||||
])
|
||||
const core = await createHighlighterCore({
|
||||
themes: [light.default, dark.default],
|
||||
langs: [],
|
||||
// Why: Orca already ships this regex engine for the editor; a second WASM copy would add ~150 KB.
|
||||
engine: onigurumaEngine(oniguruma)
|
||||
})
|
||||
return { core, textmate }
|
||||
})().catch((error: unknown) => {
|
||||
highlighterPromise = undefined
|
||||
throw error
|
||||
})
|
||||
return highlighterPromise
|
||||
}
|
||||
|
||||
function plainLine(line: string, offset: number): SyntaxToken[] {
|
||||
return line
|
||||
? [
|
||||
{
|
||||
content: line,
|
||||
offset,
|
||||
light: undefined,
|
||||
dark: undefined,
|
||||
lightFontStyle: 0,
|
||||
darkFontStyle: 0
|
||||
}
|
||||
]
|
||||
: []
|
||||
}
|
||||
|
||||
/** Lines split like Shiki splits them, each with its offset in `code`. */
|
||||
function splitLines(code: string): { line: string; offset: number }[] {
|
||||
let offset = 0
|
||||
return code.split(/\r?\n/).map((line) => {
|
||||
const start = offset
|
||||
offset = code.indexOf('\n', start + line.length) + 1
|
||||
return { line, offset: start }
|
||||
})
|
||||
}
|
||||
|
||||
function plainResult(code: string, outcome: SyntaxHighlightOutcome): SyntaxHighlightResult {
|
||||
return {
|
||||
lines: splitLines(code).map(({ line, offset }) => plainLine(line, offset)),
|
||||
state: DEGRADED,
|
||||
outcome
|
||||
}
|
||||
}
|
||||
|
||||
type ThemeTokens = { starts: number[]; colors: (string | undefined)[]; styles: number[] }
|
||||
|
||||
/** Tokenizes in one theme up to the first line that is too long or too slow. */
|
||||
function tokenizeWithTheme(
|
||||
{ core, textmate }: Highlighter,
|
||||
grammar: Grammar,
|
||||
theme: string,
|
||||
lines: string[],
|
||||
initial: StateStack
|
||||
): { tokens: ThemeTokens[]; stack: StateStack } {
|
||||
const { colorMap, theme: resolved } = core.setTheme(theme)
|
||||
const defaultColor = resolved.fg.toUpperCase()
|
||||
const { getForeground, getFontStyle } = textmate.EncodedTokenMetadata
|
||||
const tokens: ThemeTokens[] = []
|
||||
let stack = initial
|
||||
for (const line of lines) {
|
||||
if (line.length > MAX_HIGHLIGHTED_LINE_LENGTH) {
|
||||
break
|
||||
}
|
||||
let result = grammar.tokenizeLine2(line, stack, LINE_TIME_LIMIT_MS)
|
||||
// Why: the first lines through a grammar also compile its regexes, which happens only once.
|
||||
for (let attempt = 1; result.stoppedEarly && attempt < LINE_ATTEMPTS; attempt += 1) {
|
||||
result = grammar.tokenizeLine2(line, stack, LINE_TIME_LIMIT_MS)
|
||||
}
|
||||
if (result.stoppedEarly) {
|
||||
break
|
||||
}
|
||||
const lineTokens: ThemeTokens = { starts: [], colors: [], styles: [] }
|
||||
for (let index = 0; index < result.tokens.length; index += 2) {
|
||||
const metadata = result.tokens[index + 1]
|
||||
const color = colorMap[getForeground(metadata)]
|
||||
lineTokens.starts.push(result.tokens[index])
|
||||
lineTokens.colors.push(color?.toUpperCase() === defaultColor ? undefined : color)
|
||||
lineTokens.styles.push(Math.max(0, getFontStyle(metadata)))
|
||||
}
|
||||
tokens.push(lineTokens)
|
||||
stack = result.ruleStack
|
||||
}
|
||||
return { tokens, stack }
|
||||
}
|
||||
|
||||
/** Splits the line wherever either theme starts a token, so each piece carries both themes. */
|
||||
function mergeThemeTokens(
|
||||
line: string,
|
||||
offset: number,
|
||||
light: ThemeTokens,
|
||||
dark: ThemeTokens
|
||||
): SyntaxToken[] {
|
||||
const tokens: SyntaxToken[] = []
|
||||
let lightIndex = 0
|
||||
let darkIndex = 0
|
||||
let start = 0
|
||||
while (start < line.length) {
|
||||
while (lightIndex + 1 < light.starts.length && light.starts[lightIndex + 1] <= start) {
|
||||
lightIndex += 1
|
||||
}
|
||||
while (darkIndex + 1 < dark.starts.length && dark.starts[darkIndex + 1] <= start) {
|
||||
darkIndex += 1
|
||||
}
|
||||
const end = Math.min(
|
||||
light.starts[lightIndex + 1] ?? line.length,
|
||||
dark.starts[darkIndex + 1] ?? line.length,
|
||||
line.length
|
||||
)
|
||||
tokens.push({
|
||||
content: line.slice(start, end),
|
||||
offset: offset + start,
|
||||
light: light.colors[lightIndex],
|
||||
dark: dark.colors[darkIndex],
|
||||
lightFontStyle: light.styles[lightIndex] ?? 0,
|
||||
darkFontStyle: dark.styles[darkIndex] ?? 0
|
||||
})
|
||||
start = end
|
||||
}
|
||||
return tokens
|
||||
}
|
||||
|
||||
function createSyntaxHighlighter(highlighter: Highlighter, grammarName: string): SyntaxHighlighter {
|
||||
return (code, state) => {
|
||||
if (state && !state.stacks) {
|
||||
return plainResult(code, 'degraded')
|
||||
}
|
||||
try {
|
||||
const grammar = highlighter.core.getLanguage(grammarName)
|
||||
// Why: loading a language another grammar embeds reloads that grammar, which cannot read old states.
|
||||
if (state && state.grammar !== grammar) {
|
||||
return plainResult(code, 'degraded')
|
||||
}
|
||||
const lines = splitLines(code)
|
||||
const text = lines.map(({ line }) => line)
|
||||
const { INITIAL } = highlighter.textmate
|
||||
const light = tokenizeWithTheme(
|
||||
highlighter,
|
||||
grammar,
|
||||
THEMES.light,
|
||||
text,
|
||||
state?.stacks?.light ?? INITIAL
|
||||
)
|
||||
// A line Light+ gave up on stays plain, so Dark+ need not try it.
|
||||
const dark = tokenizeWithTheme(
|
||||
highlighter,
|
||||
grammar,
|
||||
THEMES.dark,
|
||||
text.slice(0, light.tokens.length),
|
||||
state?.stacks?.dark ?? INITIAL
|
||||
)
|
||||
const complete = Math.min(light.tokens.length, dark.tokens.length)
|
||||
const tokens = lines.map(({ line, offset }, index) =>
|
||||
index < complete
|
||||
? mergeThemeTokens(line, offset, light.tokens[index], dark.tokens[index])
|
||||
: plainLine(line, offset)
|
||||
)
|
||||
return complete < lines.length
|
||||
? { lines: tokens, state: DEGRADED, outcome: 'degraded' }
|
||||
: {
|
||||
lines: tokens,
|
||||
state: new ResumeState(grammar, { light: light.stack, dark: dark.stack }),
|
||||
outcome: 'ok'
|
||||
}
|
||||
} catch {
|
||||
return plainResult(code, 'failed')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function loadLanguage(key: string, attempts: number): Promise<SyntaxLanguageStatus> {
|
||||
try {
|
||||
const { bundledLanguagesInfo } = await import('shiki/langs')
|
||||
const info = bundledLanguagesInfo.find(
|
||||
(language) => language.id === key || language.aliases?.includes(key)
|
||||
)
|
||||
if (!info) {
|
||||
pendingLanguages.set(key, { status: 'unsupported' })
|
||||
return 'unsupported'
|
||||
}
|
||||
const [highlighter, { default: grammars }] = await Promise.all([
|
||||
loadHighlighter(),
|
||||
info.import()
|
||||
])
|
||||
await highlighter.core.loadLanguage(...withVueTemplateInjections(grammars))
|
||||
readyLanguages.set(key, createSyntaxHighlighter(highlighter, info.id))
|
||||
pendingLanguages.delete(key)
|
||||
return 'ready'
|
||||
} catch (error) {
|
||||
pendingLanguages.set(key, {
|
||||
status: 'failed',
|
||||
attempts: attempts + 1,
|
||||
retryAt: performance.now() + LOAD_RETRY_DELAY_MS
|
||||
})
|
||||
if (!loadFailureLogged) {
|
||||
loadFailureLogged = true
|
||||
console.warn('[syntax-highlight] failed to load a grammar; code stays plain', error)
|
||||
}
|
||||
return 'failed'
|
||||
}
|
||||
}
|
||||
|
||||
function languageKey(language: string): string {
|
||||
return language.trim().toLowerCase()
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads `language`'s grammar for `loadedSyntaxHighlighter`. A failed load is
|
||||
* retried by a later call, a few times at most.
|
||||
*/
|
||||
export function loadSyntaxLanguage(language: string): Promise<SyntaxLanguageStatus> {
|
||||
const key = languageKey(language)
|
||||
if (readyLanguages.has(key)) {
|
||||
return Promise.resolve('ready')
|
||||
}
|
||||
const pending = pendingLanguages.get(key)
|
||||
if (pending?.status === 'loading') {
|
||||
return pending.promise
|
||||
}
|
||||
if (pending?.status === 'unsupported') {
|
||||
return Promise.resolve('unsupported')
|
||||
}
|
||||
const attempts = pending?.attempts ?? 0
|
||||
if (
|
||||
pending?.status === 'failed' &&
|
||||
(attempts >= MAX_LOAD_ATTEMPTS || performance.now() < pending.retryAt)
|
||||
) {
|
||||
return Promise.resolve('failed')
|
||||
}
|
||||
const promise = loadLanguage(key, attempts)
|
||||
pendingLanguages.set(key, { status: 'loading', promise, attempts })
|
||||
return promise
|
||||
}
|
||||
|
||||
/** The highlighter for a language that has loaded, so a block can color in the same render. */
|
||||
export function loadedSyntaxHighlighter(language: string): SyntaxHighlighter | null {
|
||||
return readyLanguages.get(languageKey(language)) ?? null
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import type { LanguageRegistration } from 'shiki/core'
|
||||
|
||||
const CATALOGUE_VUE_SCOPE = 'source.vue'
|
||||
const VUE_ROOT_SCOPE = 'text.html.vue'
|
||||
|
||||
/**
|
||||
* Vue's template grammars inject into `source.vue`, but Vue's root scope is
|
||||
* `text.html.vue`, so `{{ }}` and directive values would stay plain. They go in
|
||||
* the root grammar's own injections, which apply whatever loaded first.
|
||||
*/
|
||||
export function withVueTemplateInjections(
|
||||
grammars: readonly LanguageRegistration[]
|
||||
): LanguageRegistration[] {
|
||||
const injections: NonNullable<LanguageRegistration['injections']> = {}
|
||||
for (const grammar of grammars) {
|
||||
if (grammar.injectTo?.includes(CATALOGUE_VUE_SCOPE) && grammar.injectionSelector) {
|
||||
injections[grammar.injectionSelector] = { patterns: [{ include: grammar.scopeName }] }
|
||||
}
|
||||
}
|
||||
return grammars.map((grammar) =>
|
||||
grammar.scopeName === VUE_ROOT_SCOPE && Object.keys(injections).length > 0
|
||||
? { ...grammar, injections: { ...grammar.injections, ...injections } }
|
||||
: grammar
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user