diff --git a/src/editor/lineTokens.test.ts b/src/editor/lineTokens.test.ts new file mode 100644 index 00000000..fe6514f3 --- /dev/null +++ b/src/editor/lineTokens.test.ts @@ -0,0 +1,82 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { EditorState } from "@codemirror/state"; + +const disk = new Map(); +const editors = new Map(); + +vi.mock(import("@/platform/ipc"), async (importOriginal) => ({ + ...(await importOriginal()), + commands: { fsReadText: (path: string) => path } as never, + call: (async (path: string) => disk.get(path) ?? Promise.reject(new Error("missing"))) as never, +})); +vi.mock("@/editor/api", () => ({ + editorsFor: (path: string) => { + const state = editors.get(path); + return state ? [{ view: { state } }] : []; + }, +})); + +const { clearLineTokens, lineTokens, peekLineTokens } = await import("./lineTokens"); + +const SOURCE = ["/**", " * A read-only virtual document (library", " */", 'const lib = "x";'].join("\n"); +const classesOf = (text: string, tokens: Array<{ from: number; to: number; cls: string }>) => + tokens.map((t) => `${text.slice(t.from, t.to)}:${t.cls}`); + +describe("lineTokens", () => { + beforeEach(() => { + clearLineTokens(); + disk.clear(); + editors.clear(); + }); + + it("colours a line inside a block comment with the whole file as context", async () => { + disk.set("/p/a.ts", SOURCE); + const text = "* A read-only virtual document (library"; + const tokens = await lineTokens({ path: "/p/a.ts", line: 1, col: 1 }, text); + expect(classesOf(text, tokens)).toEqual([`${text}:tok-comment`]); + expect(peekLineTokens({ path: "/p/a.ts", line: 1, col: 1 }, text)).toBe(tokens); + }); + + it("keeps positions relative to the shown text", async () => { + disk.set("/p/a.ts", SOURCE); + const text = 'lib = "x";'; + const tokens = await lineTokens({ path: "/p/a.ts", line: 3, col: 6 }, text); + expect(classesOf(text, tokens)).toContain('"x":tok-string'); + }); + + it("parses the line on its own when the file no longer has it there", async () => { + disk.set("/p/a.ts", SOURCE); + const text = "* A read-only virtual document (library"; + const tokens = await lineTokens({ path: "/p/a.ts", line: 0, col: 0 }, text); + expect(classesOf(text, tokens)).not.toContain(`${text}:tok-comment`); + }); + + it("parses again when the open editor's document changes", async () => { + editors.set("/p/a.ts", EditorState.create({ doc: SOURCE.replace("/**", "//") })); + const text = "* A read-only virtual document (library"; + expect(classesOf(text, await lineTokens({ path: "/p/a.ts", line: 1, col: 1 }, text))).not.toContain( + `${text}:tok-comment`, + ); + editors.set("/p/a.ts", EditorState.create({ doc: SOURCE })); + expect(classesOf(text, await lineTokens({ path: "/p/a.ts", line: 1, col: 1 }, text))).toEqual([ + `${text}:tok-comment`, + ]); + }); + + it("parses a big file in slices, letting other tasks run in between", async () => { + const body = "const a = 1; // filler line for the parser\n".repeat(8000); + disk.set("/p/big.ts", `${body}const lib = "x";`); + let ticks = 0; + const timer = setInterval(() => ticks++, 0); + const text = 'lib = "x";'; + const tokens = await lineTokens({ path: "/p/big.ts", line: 8000, col: 6 }, text); + clearInterval(timer); + expect(classesOf(text, tokens)).toContain('"x":tok-string'); + expect(ticks).toBeGreaterThan(0); + }); + + it("leaves files of unknown languages plain", async () => { + disk.set("/p/a.unknownext", SOURCE); + expect(await lineTokens({ path: "/p/a.unknownext", line: 1, col: 1 }, "* A read-only")).toEqual([]); + }); +}); diff --git a/src/editor/lineTokens.tsx b/src/editor/lineTokens.tsx new file mode 100644 index 00000000..7fa21dff --- /dev/null +++ b/src/editor/lineTokens.tsx @@ -0,0 +1,167 @@ +/** + * Syntax colours for result lines (Find in Files, Show Usages), like IDEA's + * `ChunkExtractor`: each file is parsed whole once, so a line inside a block + * comment or a multi-line string is coloured as such. + */ +import { useEffect, useState, type ReactNode } from "react"; +import { Text } from "@codemirror/state"; +import { editorsFor } from "@/editor/api"; +import { codeHighlighter, highlightTokens, languageForPath, type CodeToken } from "@/editor/lsp/markdown"; +import { call, commands } from "@/platform/ipc"; +import { isVirtualUri, virtualDocument } from "@/platform/virtualDocs"; + +export type { CodeToken }; + +/** Where a result line's `text` sits in its file: 0-based `line`, column `col`. */ +export interface LineSource { + path: string; + line: number; + col: number; +} + +const texts = new Map>(); + +/** + * Text of a file that isn't open (a library source or a read-only backend + * read that doesn't open a document), cached until `clearLineTokens`. + */ +export function readFileText(path: string): Promise { + let p = texts.get(path); + if (!p) { + p = isVirtualUri(path) ? virtualDocument(path).then((d) => d.text) : call(commands.fsReadText(path)); + p.catch(() => texts.delete(path)); + texts.set(path, p); + if (texts.size > 40) texts.delete(texts.keys().next().value!); + } + return p; +} + +/** Drops cached file texts and parses (a new search reads the files again). */ +export function clearLineTokens() { + texts.clear(); + highlighted.clear(); +} + +/** Bigger files aren't parsed whole: their result lines are coloured on their own. */ +const MAX_PARSE = 500_000; + +interface Parsed { + doc: Text; + highlight: (from: number, to: number) => CodeToken[]; +} + +/** + * One file parsed whole: `source` is the editor document or the text it was + * parsed from, so an edit (a new `Text`) or a re-read (a new string) parses + * again. + */ +interface Highlighted { + source: Text | string; + parsed: Promise; + /** Tokens per result line (`lineKey`). */ + lines: Map; +} +const highlighted = new Map(); + +const lineKey = (line: number, col: number, text: string) => `${line}\u0000${col}\u0000${text}`; + +/** `lineTokens` already computed for the file as last parsed (for a first render without flicker). */ +export function peekLineTokens({ path, line, col }: LineSource, text: string): CodeToken[] | null { + return highlighted.get(path)?.lines.get(lineKey(line, col, text)) ?? null; +} + +/** + * Syntax tokens of a result line (`text` at `source`), positions within + * `text`, coloured with the whole file as context. When the file no longer + * has `text` there (changed since the search) or is too big, the line is + * parsed on its own. + */ +export async function lineTokens({ path, line, col }: LineSource, text: string): Promise { + const language = languageForPath(path); + if (!language || !text) return []; + const source = editorsFor(path)[0]?.view.state.doc ?? (await readFileText(path).catch(() => null)); + if (source == null) return highlightTokens(text, language); + let h = highlighted.get(path); + if (h?.source !== source) { + h = { source, parsed: parseWhole(source, language), lines: new Map() }; + highlighted.delete(path); + highlighted.set(path, h); + if (highlighted.size > 40) highlighted.delete(highlighted.keys().next().value!); + } + const key = lineKey(line, col, text); + const known = h.lines.get(key); + if (known) return known; + const parsed = await h.parsed; + let tokens: CodeToken[] | null = null; + if (parsed && line < parsed.doc.lines) { + const l = parsed.doc.line(line + 1); + if (l.text.slice(col, col + text.length) === text) { + const from = l.from + col; + tokens = parsed.highlight(from, from + text.length).map((t) => ({ ...t, from: t.from - from, to: t.to - from })); + } + } + tokens ??= await highlightTokens(text, language); + h.lines.set(key, tokens); + return tokens; +} + +async function parseWhole(source: Text | string, language: string): Promise { + if (source.length > MAX_PARSE) return null; + const text = typeof source === "string" ? source : source.toString(); + const highlight = await codeHighlighter(text, language); + if (!highlight) return null; + // Splitting on "\n" alone keeps a "\r" in its line, so offsets match `text`. + return { doc: typeof source === "string" ? Text.of(text.split("\n")) : source, highlight }; +} + +/** `lineTokens` for a rendered line; null until known (and without a `source`). */ +export function useLineTokens(source: LineSource | undefined, text: string): CodeToken[] | null { + const path = source?.path ?? null; + const line = source?.line ?? 0; + const col = source?.col ?? 0; + const key = `${path}\u0000${line}\u0000${col}\u0000${text}`; + const [loaded, setLoaded] = useState<{ key: string; tokens: CodeToken[] } | null>(null); + useEffect(() => { + if (path == null) return; + let alive = true; + void lineTokens({ path, line, col }, text).then((tokens) => { + if (alive) setLoaded({ key, tokens }); + }); + return () => { + alive = false; + }; + }, [key, path, line, col, text]); + if (path == null) return null; + return loaded?.key === key ? loaded.tokens : peekLineTokens({ path, line, col }, text); +} + +/** + * `text` between `from` and `to` as spans with the tokens' `tok-*` classes + * (styled by the caller); `key` prefixes the React keys. + */ +export function tokenSpans( + text: string, + from: number, + to: number, + tokens: readonly CodeToken[] | null, + key = "", +): ReactNode[] { + if (!tokens?.length) return [text.slice(from, to)]; + const out: ReactNode[] = []; + let pos = from; + for (const t of tokens) { + if (t.to <= pos) continue; + if (t.from >= to) break; + const s = Math.max(t.from, pos); + const e = Math.min(t.to, to); + if (s > pos) out.push(text.slice(pos, s)); + out.push( + + {text.slice(s, e)} + , + ); + pos = e; + } + if (pos < to) out.push(text.slice(pos, to)); + return out; +} diff --git a/src/editor/lsp/markdown.ts b/src/editor/lsp/markdown.ts index f280a27b..a1e85ee4 100644 --- a/src/editor/lsp/markdown.ts +++ b/src/editor/lsp/markdown.ts @@ -7,9 +7,9 @@ * is prefetched when the app goes idle; until then markup shows as plain text. */ import { useMemo, useSyncExternalStore } from "react"; -import { LanguageDescription } from "@codemirror/language"; +import { LanguageDescription, type Language } from "@codemirror/language"; import { languages } from "@codemirror/language-data"; -import { classHighlighter, highlightCode } from "@lezer/highlight"; +import { classHighlighter, highlightTree } from "@lezer/highlight"; import type { Markup } from "@/bindings"; import { maybeEditorService } from "@/workbench/contributions"; import { isTauri } from "@/platform/ipc"; @@ -114,26 +114,55 @@ export interface CodeToken { /** Highlighted ranges of `text`; empty when the language is unknown or its grammar fails to load. */ export async function highlightTokens(text: string, language: string | null): Promise { + return (await codeHighlighter(text, language))?.(0, text.length) ?? []; +} + +/** + * Parses `text` once; the result gives the highlighted ranges (positions in + * `text`) between `from` and `to`, so a line is coloured with the whole file + * as context. Null when the language is unknown or its grammar fails to load. + */ +export async function codeHighlighter( + text: string, + language: string | null, +): Promise<((from: number, to: number) => CodeToken[]) | null> { const desc = language ? findLanguage(language) : null; - if (!desc) return []; + if (!desc) return null; try { const support = desc.support ?? (await desc.load()); - const tree = support.language.parser.parse(text); - const out: CodeToken[] = []; - let pos = 0; - highlightCode( - text, - tree, - classHighlighter, - (t, cls) => { - if (cls) out.push({ from: pos, to: pos + t.length, cls }); - pos += t.length; - }, - () => pos++, - ); - return out; + const tree = await parseSliced(support.language.parser, text); + return (from, to) => { + const out: CodeToken[] = []; + highlightTree( + tree, + classHighlighter, + (a, b, cls) => { + const s = Math.max(a, from); + const e = Math.min(b, to); + if (e > s) out.push({ from: s, to: e, cls }); + }, + from, + to, + ); + return out; + }; } catch { - return []; + return null; + } +} + +/** Parse budget per task: a whole file (up to ~1s) yields to input and rendering in between. */ +const PARSE_SLICE_MS = 8; + +async function parseSliced(parser: Language["parser"], text: string): Promise> { + const parse = parser.startParse(text); + for (;;) { + const start = performance.now(); + while (performance.now() - start < PARSE_SLICE_MS) { + const tree = parse.advance(); + if (tree) return tree; + } + await new Promise((r) => setTimeout(r, 0)); } } diff --git a/src/features/code-insight/UsagesView.module.css b/src/features/code-insight/UsagesView.module.css index 6398fb41..2bd6b942 100644 --- a/src/features/code-insight/UsagesView.module.css +++ b/src/features/code-insight/UsagesView.module.css @@ -92,6 +92,40 @@ color: var(--fg-search-match, inherit); border-radius: 2px; } +/* Lezer classHighlighter tokens of usage lines (same colours as Find in Files results). */ +.preview :global(.tok-keyword), +.preview :global(.tok-modifier), +.preview :global(.tok-controlKeyword), +.preview :global(.tok-operatorKeyword) { + color: var(--syn-keyword); +} +.preview :global(.tok-string), +.preview :global(.tok-string2) { + color: var(--syn-string); +} +.preview :global(.tok-number), +.preview :global(.tok-bool), +.preview :global(.tok-atom) { + color: var(--syn-number); +} +.preview :global(.tok-typeName), +.preview :global(.tok-className), +.preview :global(.tok-namespace) { + color: var(--syn-type); +} +.preview :global(.tok-comment) { + color: var(--syn-comment); +} +.preview :global(.tok-meta), +.preview :global(.tok-macroName) { + color: var(--syn-annotation); +} +.preview :global(.tok-propertyName) { + color: var(--syn-property); +} +.preview :global(.tok-variableName.tok-definition) { + color: var(--syn-function); +} .note { min-width: 0; diff --git a/src/features/code-insight/UsagesView.tsx b/src/features/code-insight/UsagesView.tsx index 26de1de2..c59e5049 100644 --- a/src/features/code-insight/UsagesView.tsx +++ b/src/features/code-insight/UsagesView.tsx @@ -5,6 +5,7 @@ import { useCallback, useMemo, useRef, useState, type KeyboardEvent, type ReactNode } from "react"; import clsx from "clsx"; import { create } from "zustand"; +import { tokenSpans, useLineTokens } from "@/editor/lineTokens"; import { formatMessage, useTx } from "@/platform/i18n"; import { shortcutLabel } from "@/platform/keymap"; import { basename, relativeDir, useProject } from "@/platform/project"; @@ -46,12 +47,13 @@ type Node = | { kind: "usage"; key: string; usage: Usage }; export function UsagePreview({ usage }: { usage: Usage }) { - const { text, from, to } = trimPreview(usage); + const { text, textCol, from, to } = trimPreview(usage); + const tokens = useLineTokens({ path: usage.path, line: usage.line, col: textCol }, text); return ( - {text.slice(0, from)} - {text.slice(from, to)} - {text.slice(to)} + {tokenSpans(text, 0, from, tokens, "b:")} + {tokenSpans(text, from, to, tokens, "m:")} + {tokenSpans(text, to, text.length, tokens, "a:")} ); } diff --git a/src/features/code-insight/usagePreview.test.tsx b/src/features/code-insight/usagePreview.test.tsx new file mode 100644 index 00000000..51b8c205 --- /dev/null +++ b/src/features/code-insight/usagePreview.test.tsx @@ -0,0 +1,29 @@ +import { describe, expect, it, vi } from "vitest"; +import { render, waitFor } from "@testing-library/react"; + +vi.mock(import("@/platform/ipc"), async (importOriginal) => ({ + ...(await importOriginal()), + commands: { fsReadText: (path: string) => path } as never, + call: (async () => "/**\n * Uses foo()\n */\n const x = foo();\n") as never, +})); +vi.mock(import("@/editor/api"), async (importOriginal) => ({ ...(await importOriginal()), editorsFor: () => [] })); + +const { UsagePreview } = await import("./UsagesView"); + +describe("UsagePreview", () => { + it("colours the usage line with its file as context", async () => { + const { container } = render( + , + ); + await waitFor(() => expect(container.querySelector(".tok-keyword")?.textContent).toBe("const")); + expect(container.querySelector("mark")?.textContent).toBe("foo"); + expect(container.textContent).toBe("const x = foo();"); + }); + + it("colours a usage inside a doc comment as a comment", async () => { + const { container } = render( + , + ); + await waitFor(() => expect(container.querySelector("mark .tok-comment")?.textContent).toBe("foo")); + }); +}); diff --git a/src/features/code-insight/usages.ts b/src/features/code-insight/usages.ts index 39998570..a0c9ded3 100644 --- a/src/features/code-insight/usages.ts +++ b/src/features/code-insight/usages.ts @@ -1,6 +1,7 @@ /** Find Usages (⌥F7 → "usages" tool window) and Show Usages (⌥⌘F7 → popup). */ import { create } from "zustand"; import type { Location, ReferencesIncomplete, RequestId } from "@/bindings"; +import { clearLineTokens } from "@/editor/lineTokens"; import { commands, isCancelled, nextRequestId } from "@/platform/ipc"; import { toolWindowService } from "@/workbench/contributions"; import { @@ -57,12 +58,12 @@ export function usageCount(groups: readonly UsageGroup[]): number { } /** Trims leading whitespace of a preview line, shifting the match range accordingly. */ -export function trimPreview(u: Usage): { text: string; from: number; to: number } { +export function trimPreview(u: Usage): { text: string; textCol: number; from: number; to: number } { const lead = u.text.length - u.text.trimStart().length; const text = u.text.trim(); const from = Math.max(0, u.col - lead); const to = Math.max(from, Math.min(text.length, u.endCol - lead)); - return { text, from, to }; + return { text, textCol: lead, from, to }; } export interface UsageSearch { @@ -170,6 +171,8 @@ export function incompleteNote(reason: ReferencesIncomplete, word: string | null * Cancel with `commands.cancelRequest(request)`. */ export async function fetchUsages(c: CaretContext, request: RequestId = nextRequestId()): Promise { + // Usage lines are read afresh: so are the files their syntax colours come from. + clearLineTokens(); let locs: Location[]; let note: string | null = null; try { @@ -259,6 +262,7 @@ export async function showUsageResults(arg: unknown) { const base = { id, symbol, rerun: null }; openUsageTab({ ...base, loading: true, groups: [], error: null, note: null }); toolWindowService().show("usages"); + clearLineTokens(); const files = await readManyFileLines(locs.map((l) => l.path)); replaceUsageTab(id, { ...base, loading: false, groups: groupUsages(locs, files), error: null, note: null }); } diff --git a/src/features/code-insight/views.test.ts b/src/features/code-insight/views.test.ts index 15887ab1..ac6d7dd9 100644 --- a/src/features/code-insight/views.test.ts +++ b/src/features/code-insight/views.test.ts @@ -27,7 +27,7 @@ describe("usages", () => { ]); expect(usageCount(groups)).toBe(3); expect(groups[0]!.usages[0]!.text).toBe(""); - expect(trimPreview(groups[1]!.usages[1]!)).toEqual({ text: "const x = foo();", from: 10, to: 13 }); + expect(trimPreview(groups[1]!.usages[1]!)).toEqual({ text: "const x = foo();", textCol: 2, from: 10, to: 13 }); }); }); diff --git a/src/features/search/__tests__/matchText.test.tsx b/src/features/search/__tests__/matchText.test.tsx new file mode 100644 index 00000000..22af9390 --- /dev/null +++ b/src/features/search/__tests__/matchText.test.tsx @@ -0,0 +1,36 @@ +import { describe, expect, it, vi } from "vitest"; +import { render, waitFor } from "@testing-library/react"; + +vi.mock(import("@/platform/ipc"), async (importOriginal) => ({ + ...(await importOriginal()), + commands: { fsReadText: (path: string) => path } as never, + call: (async () => '/*\n * lib\n */\nconst lib = "lib";\n') as never, +})); +vi.mock(import("@/editor/api"), async (importOriginal) => ({ ...(await importOriginal()), editorsFor: () => [] })); + +const { MatchText } = await import("../find/parts"); + +describe("MatchText", () => { + it("colours the line by its file's tokens under the match highlight", async () => { + const { container } = render( + , + ); + await waitFor(() => expect(container.querySelector(".tok-keyword")?.textContent).toBe("const")); + expect(container.querySelector("mark")?.textContent).toBe("lib"); + expect(container.querySelector(".tok-string")?.textContent).toBe('"lib"'); + expect(container.textContent).toBe('const lib = "lib";'); + }); + + it("colours a line inside a block comment as a comment", async () => { + const { container } = render( + , + ); + await waitFor(() => expect(container.querySelector("mark .tok-comment")?.textContent).toBe("lib")); + }); + + it("stays plain without a source", () => { + const { container } = render(); + expect(container.querySelector("[class^=tok-]")).toBeNull(); + expect(container.querySelector("mark")?.textContent).toBe("lib"); + }); +}); diff --git a/src/features/search/find/FindInFiles.tsx b/src/features/search/find/FindInFiles.tsx index 96eb6421..63f740d5 100644 --- a/src/features/search/find/FindInFiles.tsx +++ b/src/features/search/find/FindInFiles.tsx @@ -685,6 +685,7 @@ function FindInFilesDialog() { ; starts: Array<{ line: number; col: number }>; }; @@ -95,6 +97,7 @@ export function buildFindTree(files: readonly FileMatches[], groupByDirectory = col: first[0] + m.textOffset, endCol: first[1] + m.textOffset, text, + textCol: lead + m.textOffset, ranges: m.ranges .map(([s, e]) => [Math.max(0, s - lead), Math.min(text.length, e - lead)] as [number, number]) .filter(([s, e]) => e > s), @@ -531,6 +534,7 @@ function FindNodeContent({ node, state, tab }: { node: Node; state: TreeRowState ; @@ -26,12 +31,14 @@ export function MatchText({ replacement?: string | null; query?: Pick | null; preserveCase?: boolean; + source?: LineSource; }) { + const tokens = useLineTokens(source, text); const parts: ReactNode[] = []; let pos = 0; ranges.forEach(([s, e], i) => { if (s < pos) return; - if (s > pos) parts.push(text.slice(pos, s)); + if (s > pos) parts.push(...tokenSpans(text, pos, s, tokens, `g${i}:`)); const matched = text.slice(s, e); if (replacement != null && query) { parts.push( @@ -45,14 +52,14 @@ export function MatchText({ } else { parts.push( - {matched} + {tokenSpans(text, s, e, tokens, "")} , ); } pos = e; }); - if (pos < text.length) parts.push(text.slice(pos)); - return {parts}; + if (pos < text.length) parts.push(...tokenSpans(text, pos, text.length, tokens, "end:")); + return {parts}; } const WINDOW = 300; diff --git a/src/features/search/find/preview.ts b/src/features/search/find/preview.ts index d12a7fc5..7d56128f 100644 --- a/src/features/search/find/preview.ts +++ b/src/features/search/find/preview.ts @@ -1,27 +1,17 @@ import { editorsFor } from "@/editor/api"; -import { call, commands } from "@/platform/ipc"; - -const cache = new Map>(); +import { clearLineTokens, readFileText } from "@/editor/lineTokens"; /** * File text for the preview pane: the live buffer of an open editor, else a * read-only backend read (doesn't open a document or touch recent files). + * Shared with the result lines' syntax colours. */ export function loadPreviewText(path: string): Promise { const view = editorsFor(path)[0]?.view; if (view) return Promise.resolve(view.state.doc.toString()); - let p = cache.get(path); - if (!p) { - p = (async () => { - return call(commands.fsReadText(path)); - })(); - p.catch(() => cache.delete(path)); - cache.set(path, p); - if (cache.size > 40) cache.delete(cache.keys().next().value!); - } - return p; + return readFileText(path); } export function clearPreviewCache() { - cache.clear(); + clearLineTokens(); } diff --git a/src/features/search/find/session.ts b/src/features/search/find/session.ts index 794fd748..51b40e2d 100644 --- a/src/features/search/find/session.ts +++ b/src/features/search/find/session.ts @@ -145,6 +145,8 @@ export interface MatchRow { line: number; /** Preview text (leading whitespace trimmed). */ text: string; + /** Column of `text` in the real line. */ + textCol: number; /** Highlight ranges within `text`. */ ranges: Array<[number, number]>; /** First match column in the real line. */ @@ -172,6 +174,7 @@ export function matchRows(files: readonly FileMatches[]): MatchRow[] { relPath: f.relPath, line: m.line, text, + textCol: lead + m.textOffset, ranges, col: first[0] + m.textOffset, endCol: first[1] + m.textOffset,