starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-09-28 16:02:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(monaco): re-embed script and style bodies after an over-budget opening tag

Browse Source 
`scriptBodyPlain` / `styleBodyPlain` were the only over-budget mirror states
without a re-entry rule, and they dropped `$S2` as well. A `<script>` or
`<style>` opening tag carrying more than 512 trailing characters therefore left
the whole block unhighlighted until its closing tag, however short the following
lines were. Carry the language through and re-enter the embed as soon as the
rest of the line fits, matching the markup and expression mirror states.

Also extends the recursion ramp so the densest embed shape (`{a}` / `{{a}}`) is
driven at Monaco's line cap (19_800 / 19_528 chars) instead of stopping at
7_500, and renames the inverted private `restOfLineTooLong` constant.

The budget stays at 512: an A/B of the real tokenizer at 512 vs 256 over
realistic SFCs differs on 8 lines, all of them 256 losing the html or
typescript embed on ordinary shapes such as a ~430-character Tailwind class
attribute.
This commit is contained in:
Neil
2026-09-10 17:09:49 -07:00
parent 91634fb2c2
commit 3096ef4c4b
5 changed files with 96 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/renderer/src/lib/monaco-languages/monarch-embed-entry-budget.ts
+7 -3
View File
@@ -12,12 +12,16 @@
// colours instead of a dead renderer. Worst case measured at this budget is 341
// levels (a whole line of `{a}`), against a ~1000-level ceiling in the same
// runtime.
//
// Not safe to halve: at 256 a realistic ~430-character Tailwind class attribute
// stops entering the html embed at every re-entry point, so ordinary markup
// loses attribute-level highlighting. Measured A/B on real-shaped SFCs.
export const EMBED_ENTRY_REST_OF_LINE_BUDGET = 512
const restOfLineTooLong = `(?!.{${EMBED_ENTRY_REST_OF_LINE_BUDGET + 1}})`
const restOfLineWithinBudget = `(?!.{${EMBED_ENTRY_REST_OF_LINE_BUDGET + 1}})`
/** Zero-width: matches only while the rest of the line is within budget. */
export const restOfLineWithinEmbedBudget = new RegExp(restOfLineTooLong)
export const restOfLineWithinEmbedBudget = new RegExp(restOfLineWithinBudget)
/** `>` (script/style tag close) followed by a within-budget rest of line. */
export const tagCloseWithinEmbedBudget = new RegExp(`>${restOfLineTooLong}`)
export const tagCloseWithinEmbedBudget = new RegExp(`>${restOfLineWithinBudget}`)
src/renderer/src/lib/monaco-languages/monarch-embedded-recursion-depth.test.ts
+53 -3
View File
@@ -15,9 +15,11 @@ import { vueMonarchLanguage } from './register-vue'
// levels and died with `RangeError: Maximum call stack size exceeded` — the
// renderer-side STATUS_STACK_OVERFLOW this suite guards.
type MonarchEndState = { embeddedLanguageData?: { languageId: string } | null }
type MonarchTokenizerInstance = {
getInitialState: () => unknown
tokenize: (line: string, hasEOL: boolean, state: unknown) => { endState: unknown }
tokenize: (line: string, hasEOL: boolean, state: unknown) => { endState: MonarchEndState }
_nestedTokenize: (...args: unknown[]) => unknown
}
@@ -86,7 +88,24 @@ function measureNestedDepth(
return { maxNestedDepth, error }
}
const RAMP = [50, 200, 500, 1000, 2500]
// The embedded language each line *ends* in — `null` means the line left the
// tokenizer with no embed, i.e. that region renders unhighlighted.
function embeddedLanguagePerLine(
tokenizer: MonarchTokenizerInstance,
lines: string[]
): (string | null)[] {
let state: unknown = tokenizer.getInitialState()
return lines.map((line) => {
const endState = tokenizer.tokenize(line, true, state).endState
state = endState
return endState.embeddedLanguageData?.languageId ?? null
})
}
// 6600 is the largest `{a}` count under Monaco's line cap (19_800 chars); the
// filter below drops it for the longer chunk shapes, so the densest embed
// shape is the one that gets driven at maximum length.
const RAMP = [50, 200, 500, 1000, 2500, 6600]
function interpolationLine(count: number): string {
return `<p>${Array.from({ length: count }, (_, index) => `{a${index}}`).join('')}</p>`
@@ -151,6 +170,24 @@ describe.each([
expect(measurement.maxNestedDepth).toBeGreaterThan(0)
})
it.each([
['script', 'ts', 'typescript'],
['style', 'scss', 'scss']
])('re-embeds a %s body after an over-budget opening line', (tag, lang, embeddedLanguageId) => {
// The opening tag plus code on the same line pushes the tag close past the
// budget, so the body starts unembedded. Every following short line must
// recover the embed (and the `lang=` language) instead of leaving the whole
// block unhighlighted until the closing tag.
const embeds = embeddedLanguagePerLine(createMonarchTokenizer(languageId, language), [
`<${tag} lang="${lang}">a = "${'x'.repeat(EMBED_ENTRY_REST_OF_LINE_BUDGET)}"`,
' b',
' c',
`</${tag}>`
])
expect(embeds).toEqual([null, embeddedLanguageId, embeddedLanguageId, null])
})
it('keeps tokenizing after an over-budget line and re-embeds on the next one', () => {
const overBudget = `<div class="${'x'.repeat(EMBED_ENTRY_REST_OF_LINE_BUDGET)}">{value}</div>`
const measurement = measureNestedDepth(createMonarchTokenizer(languageId, language), [
@@ -196,7 +233,7 @@ describe('vue embedded-tokenizer recursion depth', () => {
`<template><p>${'{{a}}'.repeat(count)}</p></template>`
it('stays within the embed budget for a line of interpolations', () => {
const ramp = [50, 200, 1000, 2500].filter(
const ramp = [50, 200, 1000, 2500, 3900].filter(
(count) => templateLine(count).length < DEFAULT_MAX_TOKENIZATION_LINE_LENGTH
)
expect(ramp.length).toBeGreaterThanOrEqual(3)
@@ -214,6 +251,19 @@ describe('vue embedded-tokenizer recursion depth', () => {
)
})
it.each([
['script', 'ts', 'typescript'],
['style', 'scss', 'scss']
])('re-embeds a %s body after an over-budget opening line', (tag, lang, embeddedLanguageId) => {
const embeds = embeddedLanguagePerLine(createMonarchTokenizer('vue', vueMonarchLanguage), [
`<${tag} lang="${lang}">a = "${'x'.repeat(EMBED_ENTRY_REST_OF_LINE_BUDGET)}"`,
' b',
`</${tag}>`
])
expect(embeds).toEqual([null, embeddedLanguageId, null])
})
it('tokenizes a template interpolation without dropping the embed', () => {
const measurement = measureNestedDepth(createMonarchTokenizer('vue', vueMonarchLanguage), [
'<template>',
src/renderer/src/lib/monaco-languages/register-astro.ts
+12 -3
View File
@@ -113,7 +113,7 @@ export const astroMonarchLanguage: Monaco.languages.IMonarchLanguage = {
tagCloseWithinEmbedBudget,
{ token: 'tag', switchTo: '@scriptBody.$S2', nextEmbedded: '$S2' }
],
[/>/, { token: 'tag', switchTo: '@scriptBodyPlain' }],
[/>/, { token: 'tag', switchTo: '@scriptBodyPlain.$S2' }],
[/lang(?=\s*=)/, { token: 'attribute.name', switchTo: '@scriptLangBeforeEquals.$S2' }],
{ include: '@tagAttributes' }
],
@@ -143,9 +143,14 @@ export const astroMonarchLanguage: Monaco.languages.IMonarchLanguage = {
scriptBody: [
[/<\/script\s*>/, { token: 'tag', switchTo: '@markupReenter', nextEmbedded: '@pop' }]
],
// Same body, no embed: reached only past the budget.
// Over-budget mirror of the body: re-enters `$S2` as soon as the rest of
// the line fits, so a long opening line does not grey out the whole block.
scriptBodyPlain: [
[/<\/script\s*>/, { token: 'tag', switchTo: '@markupReenter' }],
[
restOfLineWithinEmbedBudget,
{ token: '@rematch', switchTo: '@scriptBody.$S2', nextEmbedded: '$S2' }
],
[/[^<]+/, ''],
[/./, '']
],
@@ -155,7 +160,7 @@ export const astroMonarchLanguage: Monaco.languages.IMonarchLanguage = {
tagCloseWithinEmbedBudget,
{ token: 'tag', switchTo: '@styleBody.$S2', nextEmbedded: '$S2' }
],
[/>/, { token: 'tag', switchTo: '@styleBodyPlain' }],
[/>/, { token: 'tag', switchTo: '@styleBodyPlain.$S2' }],
[/lang(?=\s*=)/, { token: 'attribute.name', switchTo: '@styleLangBeforeEquals.$S2' }],
{ include: '@tagAttributes' }
],
@@ -187,6 +192,10 @@ export const astroMonarchLanguage: Monaco.languages.IMonarchLanguage = {
],
styleBodyPlain: [
[/<\/style\s*>/, { token: 'tag', switchTo: '@markupReenter' }],
[
restOfLineWithinEmbedBudget,
{ token: '@rematch', switchTo: '@styleBody.$S2', nextEmbedded: '$S2' }
],
[/[^<]+/, ''],
[/./, '']
],
src/renderer/src/lib/monaco-languages/register-svelte.ts
+12 -3
View File
@@ -164,7 +164,7 @@ export const svelteMonarchLanguage: Monaco.languages.IMonarchLanguage = {
tagCloseWithinEmbedBudget,
{ token: 'tag', switchTo: '@scriptBody.$S2', nextEmbedded: '$S2' }
],
[/>/, { token: 'tag', switchTo: '@scriptBodyPlain' }],
[/>/, { token: 'tag', switchTo: '@scriptBodyPlain.$S2' }],
[/lang(?=\s*=)/, { token: 'attribute.name', switchTo: '@scriptLangBeforeEquals.$S2' }],
{ include: '@tagAttributes' }
],
@@ -194,9 +194,14 @@ export const svelteMonarchLanguage: Monaco.languages.IMonarchLanguage = {
scriptBody: [
[/<\/script\s*>/, { token: 'tag', switchTo: '@markupReenter', nextEmbedded: '@pop' }]
],
// Same body, no embed: reached only past the budget.
// Over-budget mirror of the body: re-enters `$S2` as soon as the rest of
// the line fits, so a long opening line does not grey out the whole block.
scriptBodyPlain: [
[/<\/script\s*>/, { token: 'tag', switchTo: '@markupReenter' }],
[
restOfLineWithinEmbedBudget,
{ token: '@rematch', switchTo: '@scriptBody.$S2', nextEmbedded: '$S2' }
],
[/[^<]+/, ''],
[/./, '']
],
@@ -206,7 +211,7 @@ export const svelteMonarchLanguage: Monaco.languages.IMonarchLanguage = {
tagCloseWithinEmbedBudget,
{ token: 'tag', switchTo: '@styleBody.$S2', nextEmbedded: '$S2' }
],
[/>/, { token: 'tag', switchTo: '@styleBodyPlain' }],
[/>/, { token: 'tag', switchTo: '@styleBodyPlain.$S2' }],
[/lang(?=\s*=)/, { token: 'attribute.name', switchTo: '@styleLangBeforeEquals.$S2' }],
{ include: '@tagAttributes' }
],
@@ -238,6 +243,10 @@ export const svelteMonarchLanguage: Monaco.languages.IMonarchLanguage = {
],
styleBodyPlain: [
[/<\/style\s*>/, { token: 'tag', switchTo: '@markupReenter' }],
[
restOfLineWithinEmbedBudget,
{ token: '@rematch', switchTo: '@styleBody.$S2', nextEmbedded: '$S2' }
],
[/[^<]+/, ''],
[/./, '']
],
src/renderer/src/lib/monaco-languages/register-vue.ts
+12 -3
View File
@@ -97,7 +97,7 @@ export const vueMonarchLanguage: Monaco.languages.IMonarchLanguage = {
tagCloseWithinEmbedBudget,
{ token: 'tag', switchTo: '@scriptBody.$S2', nextEmbedded: '$S2' }
],
[/>/, { token: 'tag', switchTo: '@scriptBodyPlain' }],
[/>/, { token: 'tag', switchTo: '@scriptBodyPlain.$S2' }],
[/lang(?=\s*=)/, { token: 'attribute.name', switchTo: '@scriptLangBeforeEquals.$S2' }],
{ include: '@tagAttributes' }
],
@@ -125,9 +125,14 @@ export const vueMonarchLanguage: Monaco.languages.IMonarchLanguage = {
[/\s+/, 'white']
],
scriptBody: [[/<\/script\s*>/, { token: 'tag', next: '@pop', nextEmbedded: '@pop' }]],
// Same body, no embed: reached only past the budget.
// Over-budget mirror of the body: re-enters `$S2` as soon as the rest of
// the line fits, so a long opening line does not grey out the whole block.
scriptBodyPlain: [
[/<\/script\s*>/, { token: 'tag', next: '@pop' }],
[
restOfLineWithinEmbedBudget,
{ token: '@rematch', switchTo: '@scriptBody.$S2', nextEmbedded: '$S2' }
],
[/[^<]+/, ''],
[/./, '']
],
@@ -137,7 +142,7 @@ export const vueMonarchLanguage: Monaco.languages.IMonarchLanguage = {
tagCloseWithinEmbedBudget,
{ token: 'tag', switchTo: '@styleBody.$S2', nextEmbedded: '$S2' }
],
[/>/, { token: 'tag', switchTo: '@styleBodyPlain' }],
[/>/, { token: 'tag', switchTo: '@styleBodyPlain.$S2' }],
[/lang(?=\s*=)/, { token: 'attribute.name', switchTo: '@styleLangBeforeEquals.$S2' }],
{ include: '@tagAttributes' }
],
@@ -167,6 +172,10 @@ export const vueMonarchLanguage: Monaco.languages.IMonarchLanguage = {
styleBody: [[/<\/style\s*>/, { token: 'tag', next: '@pop', nextEmbedded: '@pop' }]],
styleBodyPlain: [
[/<\/style\s*>/, { token: 'tag', next: '@pop' }],
[
restOfLineWithinEmbedBudget,
{ token: '@rematch', switchTo: '@styleBody.$S2', nextEmbedded: '$S2' }
],
[/[^<]+/, ''],
[/./, '']
],