Use released table parser fix and remove redundant preview work (#24919)

This commit is contained in:
Neil
2026-10-02 18:09:30 -07:00
committed by GitHub
parent 328caa2160
commit c99b20203f
14 changed files with 115 additions and 243 deletions
@@ -1,119 +0,0 @@
diff --git a/dev/lib/edit-map.js b/dev/lib/edit-map.js
index 983d0556e2713dc92982faa6a09b1093ef4d02db..818e702f5de8a1a8b1d35b25b5f6a0568989b152 100644
--- a/dev/lib/edit-map.js
+++ b/dev/lib/edit-map.js
@@ -34,6 +34,7 @@ export class EditMap {
* @type {Array<Change>}
*/
this.map = []
+ this.indexByOffset = new Map()
}
/**
@@ -124,6 +125,7 @@ export class EditMap {
// Truncate everything.
this.map.length = 0
+ this.indexByOffset.clear()
}
}
@@ -137,32 +139,16 @@ export class EditMap {
* @returns {undefined}
*/
function addImplementation(editMap, at, remove, add) {
- let index = 0
-
- /* c8 ignore next 3 -- `resolve` is never called without tables, so without edits. */
- if (remove === 0 && add.length === 0) {
+ if (remove === 0 && add.length === 0) return
+ const existing = editMap.indexByOffset.get(at)
+ if (existing) {
+ existing[1] += remove
+ existing[2].push(...add)
return
}
-
- while (index < editMap.map.length) {
- if (editMap.map[index][0] === at) {
- editMap.map[index][1] += remove
-
- // To do: before not used by tables, use when moving to micromark.
- // if (before) {
- // add.push(...editMap.map[index][2])
- // editMap.map[index][2] = add
- // } else {
- editMap.map[index][2].push(...add)
- // }
-
- return
- }
-
- index += 1
- }
-
- editMap.map.push([at, remove, add])
+ const change = [at, remove, add]
+ editMap.map.push(change)
+ editMap.indexByOffset.set(at, change)
}
// /**
diff --git a/lib/edit-map.js b/lib/edit-map.js
index ecc8bce784d29a48e0869be1c4a2dd5ccf1d3d01..8cb2282f2669f978f1ae60ab23f65f1e6523864a 100644
--- a/lib/edit-map.js
+++ b/lib/edit-map.js
@@ -34,6 +34,7 @@ export class EditMap {
* @type {Array<Change>}
*/
this.map = [];
+ this.indexByOffset = new Map();
}
/**
@@ -117,6 +118,7 @@ export class EditMap {
// Truncate everything.
this.map.length = 0;
+ this.indexByOffset.clear();
}
}
@@ -130,29 +132,16 @@ export class EditMap {
* @returns {undefined}
*/
function addImplementation(editMap, at, remove, add) {
- let index = 0;
-
- /* c8 ignore next 3 -- `resolve` is never called without tables, so without edits. */
- if (remove === 0 && add.length === 0) {
+ if (remove === 0 && add.length === 0) return;
+ const existing = editMap.indexByOffset.get(at);
+ if (existing) {
+ existing[1] += remove;
+ existing[2].push(...add);
return;
}
- while (index < editMap.map.length) {
- if (editMap.map[index][0] === at) {
- editMap.map[index][1] += remove;
-
- // To do: before not used by tables, use when moving to micromark.
- // if (before) {
- // add.push(...editMap.map[index][2])
- // editMap.map[index][2] = add
- // } else {
- editMap.map[index][2].push(...add);
- // }
-
- return;
- }
- index += 1;
- }
- editMap.map.push([at, remove, add]);
+ const change = [at, remove, add];
+ editMap.map.push(change);
+ editMap.indexByOffset.set(at, change);
}
// /**
@@ -1,83 +0,0 @@
import { createRequire } from 'node:module'
import path from 'node:path'
import { pathToFileURL } from 'node:url'
import { describe, expect, it } from 'vitest'
import { unified } from 'unified'
import remarkParse from 'remark-parse'
import remarkGfm from 'remark-gfm'
const require = createRequire(import.meta.url)
const resolvedRoot = path.dirname(require.resolve('micromark-extension-gfm-table'))
const root = path.basename(resolvedRoot) === 'dev' ? path.dirname(resolvedRoot) : resolvedRoot
for (const directory of ['lib', 'dev/lib']) {
const { EditMap } = await import(pathToFileURL(path.join(root, directory, 'edit-map.js')).href)
describe(`table edit map ${directory}`, () => {
it('merges repeated offsets, sorts edits, and resets for reuse', () => {
const edits = new EditMap()
edits.add(3, 1, ['c'])
edits.add(1, 1, ['a'])
edits.add(3, 1, ['d'])
edits.add(0, 0, [])
const events = [0, 1, 2, 3, 4, 5]
edits.consume(events)
expect(events).toEqual([0, 'a', 2, 'c', 'd', 5])
edits.add(1, 1, ['new'])
edits.consume(events)
expect(events).toEqual([0, 'new', 2, 'c', 'd', 5])
})
it('does not scan prior offsets when adding thousands of distinct edits', () => {
const edits = new EditMap()
edits.add(0, 1, ['first'])
let reads = 0
const first = edits.map[0]
Object.defineProperty(edits.map, '0', {
configurable: true,
get() {
reads += 1
return first
}
})
for (let offset = 1; offset < 5000; offset += 1) {
edits.add(offset, 1, [offset])
}
expect(reads).toBe(0)
expect(edits.map).toHaveLength(5000)
})
it('preserves complete parsed trees and source positions against the previous merge algorithm', () => {
const parser = unified().use(remarkParse).use(remarkGfm)
const sources = [
'| a | b |\n| :- | -: |\n| x | y |\n',
'> | a | b |\n> | --- | --- |\n> | **bold** | [link][r] |\n\n[r]: https://example.com',
'- item\n\n | a | b |\n | --- | --- |\n | x | y |',
'| escaped \\| pipe | `code` |\n| --- | --- |\n| ~~del~~ | 😀 |\n',
`Before\n\n${'| a | b |\n| --- | --- |\n| x | y |\n\n'.repeat(200)}`
]
const patched = sources.map((source) => parser.parse(source))
const originalAdd = EditMap.prototype.add
let stockCalls = 0
try {
EditMap.prototype.add = function (at, remove, add) {
stockCalls += 1
if (remove === 0 && add.length === 0) {
return
}
const change = this.map.find((entry) => entry[0] === at)
if (change) {
change[1] += remove
change[2].push(...add)
} else {
this.map.push([at, remove, add])
}
}
expect(sources.map((source) => parser.parse(source))).toEqual(patched)
if (directory === 'dev/lib') {
expect(stockCalls).toBeGreaterThan(0)
}
} finally {
EditMap.prototype.add = originalAdd
}
})
})
}
+4 -5
View File
@@ -175,7 +175,6 @@ patchedDependencies:
'@xterm/xterm@6.1.0-beta.303': dd0ccc59cd1ccf99f4d76e5aa2456da165fa0804dce19a833d7638bd07ffa393
i18next-cli@1.74.2: 7955b89d3aa229f477408608d331f78c85148a85a85913729f89fac65ad8b207
lint-staged@16.4.0: 7333b3837f80a7fbd045964db6d76ba4fc118e49134bdbabb00585b6b7b60673
micromark-extension-gfm-table@2.1.1: 97cbc5201c0dcf03d72f5d4e3617e28723268fd058ab05fd48fb11b46aa6e5ed
node-pty@1.1.0: 92c95cffab383d86b3b13a460c75a08074468ebf1f3911db8e874e083201f192
importers:
@@ -6203,8 +6202,8 @@ packages:
micromark-extension-gfm-strikethrough@2.1.0:
resolution: {integrity: sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==}
micromark-extension-gfm-table@2.1.1:
resolution: {integrity: sha512-t2OU/dXXioARrC6yWfJ4hqB7rct14e8f7m0cbI5hUmDyyIlwv5vEtooptH8INkbLzOatzKuVbQmAYcbWoyz6Dg==}
micromark-extension-gfm-table@2.1.2:
resolution: {integrity: sha512-pRzm4kDTu0MjlmBkxmS9yYhw60nncfcEwu9NNdPFSQEFXS95ZKyIIyTSHu/o3ReBUrLKYEq+7YaXCRn/bPB4MA==}
micromark-extension-gfm-tagfilter@2.0.0:
resolution: {integrity: sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==}
@@ -13508,7 +13507,7 @@ snapshots:
micromark-util-symbol: 2.0.1
micromark-util-types: 2.0.2
micromark-extension-gfm-table@2.1.1(patch_hash=97cbc5201c0dcf03d72f5d4e3617e28723268fd058ab05fd48fb11b46aa6e5ed):
micromark-extension-gfm-table@2.1.2:
dependencies:
devlop: 1.1.0
micromark-factory-space: 2.0.1
@@ -13533,7 +13532,7 @@ snapshots:
micromark-extension-gfm-autolink-literal: 2.1.0
micromark-extension-gfm-footnote: 2.1.0
micromark-extension-gfm-strikethrough: 2.1.0
micromark-extension-gfm-table: 2.1.1(patch_hash=97cbc5201c0dcf03d72f5d4e3617e28723268fd058ab05fd48fb11b46aa6e5ed)
micromark-extension-gfm-table: 2.1.2
micromark-extension-gfm-tagfilter: 2.0.0
micromark-extension-gfm-task-list-item: 2.1.0
micromark-util-combine-extensions: 2.0.1
-1
View File
@@ -63,4 +63,3 @@ patchedDependencies:
lint-staged@16.4.0: config/patches/lint-staged@16.4.0.patch
'@vscode/windows-process-tree@0.8.0': config/patches/@vscode__windows-process-tree@0.8.0.patch
i18next-cli@1.74.2: config/patches/i18next-cli@1.74.2.patch
micromark-extension-gfm-table@2.1.1: config/patches/micromark-extension-gfm-table@2.1.1.patch
@@ -94,7 +94,6 @@ export function VirtualMarkdownPreviewBody({
} | null>(null)
const [anchor, setAnchor] = useState<PreviewReveal | null>(null)
const virtualBodyRef = useRef<HTMLDivElement>(null)
const pendingHighlight = useRef<Range[]>([])
const completedAnchor = useRef<PreviewReveal | null>(null)
const completedSearch = useRef<{
client: MarkdownPreviewDocumentClient
@@ -239,16 +238,13 @@ export function VirtualMarkdownPreviewBody({
? body.querySelector<HTMLElement>(`[data-preview-block-index="${activeMatch.block}"]`)
: null
clearMarkdownPreviewSearchHighlights(searchInstance)
pendingHighlight.current = block
const ranges = block
? applyMarkdownPreviewSearchHighlights(searchInstance, block, query, { documentOnly: true })
: []
setActiveMarkdownPreviewSearchMatch(
searchInstance,
pendingHighlight.current,
activeMatch?.occurrence ?? -1,
{ scrollIntoView: false }
)
const range = pendingHighlight.current[activeMatch?.occurrence ?? -1]
setActiveMarkdownPreviewSearchMatch(searchInstance, ranges, activeMatch?.occurrence ?? -1, {
scrollIntoView: false
})
const range = ranges[activeMatch?.occurrence ?? -1]
if (
activeMatch &&
range &&
@@ -260,7 +256,6 @@ export function VirtualMarkdownPreviewBody({
completedSearch.current = { client, match: activeMatch, query }
}
return () => {
pendingHighlight.current = []
clearMarkdownPreviewSearchHighlights(searchInstance)
}
}, [activeMatch, bodyRef, client, query, rendered, searchInstance])
@@ -64,7 +64,6 @@ export class MarkdownPreviewDocumentEngine {
}
const { tree, document } = parseMarkdownPreviewDocument(content)
this.tree = tree
this.cache.clear()
return document
}
@@ -2,8 +2,13 @@ import type { Root, RootContent, Nodes } from 'hast'
import type { Root as MarkdownRoot, Nodes as MarkdownNodes } from 'mdast'
import remarkParse from 'remark-parse'
import remarkRehype from 'remark-rehype'
import rehypeRaw from 'rehype-raw'
import { unified } from 'unified'
import { MARKDOWN_REMARK_PLUGINS, MARKDOWN_REHYPE_PLUGINS } from './markdown-preview-plugins'
import {
MARKDOWN_REMARK_PLUGINS,
MARKDOWN_REHYPE_NORMALIZATION_PLUGINS,
MARKDOWN_REHYPE_EXPANSION_PLUGINS
} from './markdown-preview-plugins'
import {
MARKDOWN_PREVIEW_DOCUMENT_MAX_NODES,
MARKDOWN_PREVIEW_BLOCK_MAX_NODES,
@@ -128,7 +133,8 @@ export function parseMarkdownPreviewDocument(content: string): {
const containsHtml = hasHtml(parsed)
const processor = parser()
.use(remarkRehype, { allowDangerousHtml: true })
.use(containsHtml ? MARKDOWN_REHYPE_PLUGINS.slice(0, -2) : MARKDOWN_REHYPE_PLUGINS.slice(1, -2))
.use(containsHtml ? [rehypeRaw] : [])
.use(MARKDOWN_REHYPE_NORMALIZATION_PLUGINS)
const tree = processor.runSync(parsed)
assertDocumentBudget(tree)
tree.children = tree.children.filter(
@@ -156,19 +162,20 @@ function hasHtml(node: MarkdownNodes): boolean {
return 'children' in node && node.children.some((child) => hasHtml(child))
}
const expansion = unified().use(MARKDOWN_REHYPE_PLUGINS.slice(-2))
const expansion = unified().use(MARKDOWN_REHYPE_EXPANSION_PLUGINS)
export function renderMarkdownPreviewBlock(
node: RootContent,
index: number
): MarkdownPreviewRenderedBlock {
const tree: Root = { type: 'root', children: [structuredClone(node)] }
const sourceTree: Root = { type: 'root', children: [node] }
const tooLarge =
countMarkdownPreviewNodes(tree, MARKDOWN_PREVIEW_BLOCK_MAX_NODES) >
countMarkdownPreviewNodes(sourceTree, MARKDOWN_PREVIEW_BLOCK_MAX_NODES) >
MARKDOWN_PREVIEW_BLOCK_MAX_NODES || getMarkdownPreviewTreeText(node).length > 32_768
if (tooLarge) {
return { index, tree: { type: 'root', children: [] }, oversized: true }
}
const tree = structuredClone(sourceTree)
const expanded = expansion.runSync(tree)
if (!isHastRoot(expanded)) {
throw new Error('Invalid rendered preview block.')
@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest'
import type { Nodes } from 'hast'
import { describe, expect, it, vi } from 'vitest'
import type { Nodes, RootContent } from 'hast'
import { createElement } from 'react'
import { renderToStaticMarkup } from 'react-dom/server'
import Markdown from 'react-markdown'
@@ -24,6 +24,27 @@ function elements(node: Nodes): Nodes[] {
}
describe('large Markdown preview documents', () => {
it('rejects atomic blocks before allocating a copy of their trees', () => {
const { tree } = parseMarkdownPreviewDocument(`\`\`\`text\n${'x'.repeat(32_769)}\n\`\`\``)
const broadBlock: RootContent = {
type: 'element',
tagName: 'div',
properties: {},
children: Array.from({ length: MARKDOWN_PREVIEW_BLOCK_MAX_NODES }, () => ({
type: 'text',
value: 'x'
}))
}
const clone = vi.spyOn(globalThis, 'structuredClone')
try {
expect(renderMarkdownPreviewBlock(tree.children[0], 0).oversized).toBe(true)
expect(renderMarkdownPreviewBlock(broadBlock, 1).oversized).toBe(true)
expect(clone).not.toHaveBeenCalled()
} finally {
clone.mockRestore()
}
})
it('renders the same safe HTML as the ordinary preview pipeline', () => {
const content =
'# Repeat\n\n[Global][end] **bold** ~~deleted~~\n\n# Repeat\n\n' +
@@ -59,11 +59,15 @@ export const MARKDOWN_REMARK_PLUGINS: MarkdownPluginList = [
remarkMath,
remarkMarkdownDocLinks
]
// Why: sanitize raw HTML before KaTeX/highlight expand it.
export const MARKDOWN_REHYPE_NORMALIZATION_PLUGINS: MarkdownPluginList = [
[rehypeSanitize, markdownPreviewSanitizeSchema],
rehypeSlug
]
export const MARKDOWN_REHYPE_EXPANSION_PLUGINS: MarkdownPluginList = [rehypeHighlight, rehypeKatex]
// Sanitize raw HTML before math and syntax expansion.
export const MARKDOWN_REHYPE_PLUGINS: MarkdownPluginList = [
rehypeRaw,
[rehypeSanitize, markdownPreviewSanitizeSchema],
rehypeSlug,
rehypeHighlight,
rehypeKatex
...MARKDOWN_REHYPE_NORMALIZATION_PLUGINS,
...MARKDOWN_REHYPE_EXPANSION_PLUGINS
]
@@ -0,0 +1,61 @@
import { describe, expect, it } from 'vitest'
import { unified } from 'unified'
import remarkParse from 'remark-parse'
import remarkGfm from 'remark-gfm'
import { parseMarkdownPreviewDocument } from './markdown-preview-document-tree'
const parser = unified().use(remarkParse).use(remarkGfm)
describe('Markdown table parsing', () => {
it('preserves alignment, escaped pipes, inline syntax, and source positions', () => {
const source =
'| left | right |\n| :- | -: |\n| escaped \\| pipe | `code` |\n| ~~deleted~~ | [link][r] |\n\n[r]: https://example.com\n'
const table = parser.parse(source).children[0]
expect(table.type).toBe('table')
if (table.type !== 'table') {
throw new Error('Expected a table')
}
expect(table.align).toEqual(['left', 'right'])
expect(table.position).toMatchObject({
start: { line: 1, column: 1, offset: 0 },
end: { line: 4, column: 28 }
})
expect(table.children[1].children.map((cell) => cell.children)).toMatchObject([
[{ type: 'text', value: 'escaped | pipe' }],
[{ type: 'inlineCode', value: 'code' }]
])
expect(table.children[2].children.map((cell) => cell.children)).toMatchObject([
[{ type: 'delete', children: [{ type: 'text', value: 'deleted' }] }],
[{ type: 'linkReference', identifier: 'r' }]
])
})
it('keeps tables inside block quotes and list items', () => {
const quote = parser.parse('> | a | b |\n> | --- | --- |\n> | x | y |\n').children[0]
expect(quote).toMatchObject({ type: 'blockquote', children: [{ type: 'table' }] })
const list = parser.parse('- item\n\n | a | b |\n | --- | --- |\n | x | y |\n').children[0]
expect(list).toMatchObject({
type: 'list',
children: [{ type: 'listItem', children: [{ type: 'paragraph' }, { type: 'table' }] }]
})
})
it('parses thousands of tables with global references and original line ranges', () => {
const section = '## Table\n\n[Reference][later]\n\n| a | b |\n| --- | --- |\n| x | y |\n\n'
const { tree, document } = parseMarkdownPreviewDocument(
`${section.repeat(2000)}[later]: https://example.com\n`
)
expect(document.blocks).toHaveLength(6000)
expect(document.toc).toHaveLength(2000)
expect(document.blocks.at(-1)).toMatchObject({
sourceLine: 1999 * 8 + 5,
sourceEndLine: 1999 * 8 + 7
})
expect(tree.children.at(-1)).toMatchObject({ type: 'element', tagName: 'table' })
expect(tree.children[5998]).toMatchObject({
type: 'element',
tagName: 'p',
children: [{ type: 'element', tagName: 'a', properties: { href: 'https://example.com' } }]
})
})
})
@@ -8,7 +8,6 @@ import {
import type { MarkdownDocument } from '../../../../shared/filesystem-entry-types'
import { createMarkdownDocumentIndex } from './markdown-doc-links'
import { exceedsMarkdownRichModeSizeLimit } from './markdown-rich-size-limit'
import { extractFrontMatter, markdownFrontMatterInner } from './markdown-frontmatter'
import { previewHasAnnotationBlockKey } from './markdown-preview-annotation-shortcut'
import { selectMarkdownTableOfContents } from './markdown-toc-visibility-gate'
import type { NotesSendMenuScope } from './NotesSendMenu'
@@ -52,7 +51,6 @@ export function useMarkdownPreviewFoundation({
markdownComments
} = source
const frontMatter = useMemo(() => extractFrontMatter(renderedContent), [renderedContent])
const tableOfContentsItems = useMemo(
() =>
selectMarkdownTableOfContents(
@@ -65,7 +63,6 @@ export function useMarkdownPreviewFoundation({
() => createMarkdownDocumentIndex(markdownDocuments),
[markdownDocuments]
)
const frontMatterInner = useMemo(() => markdownFrontMatterInner(frontMatter), [frontMatter])
const toggleableSourceFileId: string | null = sourceFileId ?? null
const frontmatterVisible = toggleableSourceFileId
? (frontmatterVisibleByFile[toggleableSourceFileId] ?? true)
@@ -122,10 +119,8 @@ export function useMarkdownPreviewFoundation({
return {
...source,
frontMatter,
tableOfContentsItems,
markdownDocumentIndex,
frontMatterInner,
frontmatterVisible,
activeAnnotationBlockKey,
setActiveAnnotationBlockKey,
@@ -5,12 +5,10 @@ import type { MarkdownPreviewFoundation } from './use-markdown-preview-foundatio
export function useMarkdownPreviewScrollViewport({
foundation,
scrollCacheKey,
readyRevision,
restorePixels = true
}: {
foundation: MarkdownPreviewFoundation
restorePixels?: boolean
readyRevision?: unknown
scrollCacheKey: string
}): void {
const { rootRef, renderedContent } = foundation
@@ -76,5 +74,5 @@ export function useMarkdownPreviewScrollViewport({
tryRestore()
return () => window.cancelAnimationFrame(frameId)
}, [rootRef, scrollCacheKey, renderedContent, readyRevision, restorePixels])
}, [rootRef, scrollCacheKey, renderedContent, restorePixels])
}
@@ -64,7 +64,6 @@ export function useMarkdownPreviewViewport({
useMarkdownPreviewScrollViewport({
foundation,
scrollCacheKey,
readyRevision: largeDocument,
restorePixels: !largePreview
})
-3
View File
@@ -2942,9 +2942,6 @@
"title": "Agent dashboard"
}
},
"editor": {
"markdownPreview": {}
},
"fileExplorer": {
"root": {
"omittedFiles": "Files omitted by sparse checkout are not searched.",