diff --git a/docs/plugins/themes.md b/docs/plugins/themes.md
index 96b639c2..760a7c3d 100644
--- a/docs/plugins/themes.md
+++ b/docs/plugins/themes.md
@@ -324,7 +324,7 @@ a token.
-228 themeable tokens: 121 in the UI theme file, 107 in the editor colour scheme file.
+235 themeable tokens: 121 in the UI theme file, 114 in the editor colour scheme file.
### Surfaces (UI theme)
@@ -434,6 +434,13 @@ a token.
| `--syn-param` | `#bcbec4` | `#080808` |
| `--syn-property` | `#c77dbb` | `#871094` |
| `--syn-constant` | `#c77dbb` | `#871094` |
+| `--syn-python-local-variable` | `var(--syn-variable)` | `var(--syn-variable)` |
+| `--syn-python-global-variable` | `var(--syn-global)` | `var(--syn-global)` |
+| `--syn-python-instance-attribute` | `var(--syn-property)` | `var(--syn-property)` |
+| `--syn-python-class-attribute` | `var(--syn-property)` | `var(--syn-property)` |
+| `--syn-python-constant` | `var(--syn-constant)` | `var(--syn-constant)` |
+| `--syn-python-property` | `var(--syn-property)` | `var(--syn-property)` |
+| `--syn-python-type-parameter` | `var(--syn-type)` | `var(--syn-type)` |
| `--syn-builtin` | `#8888c6` | `#000080` |
| `--syn-self` | `#94558d` | `#94558d` |
| `--syn-keyword-arg` | `#aa4926` | `#660099` |
diff --git a/e2e/pythonHighlight.fixture.ts b/e2e/pythonHighlight.fixture.ts
new file mode 100644
index 00000000..e42a08e4
--- /dev/null
+++ b/e2e/pythonHighlight.fixture.ts
@@ -0,0 +1,84 @@
+import { EditorState } from "@codemirror/state";
+import { EditorView } from "@codemirror/view";
+import { syntaxHighlighting } from "@codemirror/language";
+import { openideaHighlightStyle } from "../src/editor/core/cm/highlight";
+import { findLanguageDescription, loadSupport } from "../src/editor/core/cm/languages";
+import { semanticExtension } from "../src/editor/lsp/semantic";
+import { lspDoc } from "../src/editor/lsp/lsp";
+import { lspTheme } from "../src/editor/lsp/theme";
+import { parseIcls } from "../src/platform/icls";
+import { schemeCss } from "../src/platform/colorScheme";
+import type { EditorDocContext } from "../src/editor/api";
+
+const attributes = {
+ TEXT: { FOREGROUND: "ddeeff", BACKGROUND: "101820" },
+ "PY.KEYWORD": { FOREGROUND: "112233" },
+ "PY.FUNC_DEFINITION": { FOREGROUND: "224466" },
+ "PY.NESTED_FUNC_DEFINITION": { FOREGROUND: "335577" },
+ DEFAULT_FUNCTION_CALL: { FOREGROUND: "446688" },
+ DEFAULT_LINE_COMMENT: { FOREGROUND: "667788" },
+ DEFAULT_BLOCK_COMMENT: { FOREGROUND: "aabbcc" },
+ "PY.METHOD_CALL": { FOREGROUND: "557799" },
+ "PY.DOC_COMMENT": { FOREGROUND: "6688aa", EFFECT_TYPE: "1" },
+ "PY.STRING.B": { FOREGROUND: "7799bb", FONT_TYPE: "1" },
+ "PY.VALID_STRING_ESCAPE": { FOREGROUND: "88aacc" },
+ "PY.ANNOTATION": { FOREGROUND: "99bbdd", BACKGROUND: "182838" },
+ "PY.SELF_PARAMETER": { FOREGROUND: "aa7788" },
+ "PY.BUILTIN_NAME": { FOREGROUND: "bb8899" },
+ "PY.KEYWORD_ARGUMENT": { FOREGROUND: "cc99aa" },
+ OPENIDEA_pyLocalVariable: { FOREGROUND: "aa1101" },
+ OPENIDEA_pyGlobalVariable: { FOREGROUND: "aa2202" },
+ OPENIDEA_pyInstanceAttribute: { FOREGROUND: "aa3303" },
+ OPENIDEA_pyClassAttribute: { FOREGROUND: "aa4404" },
+ OPENIDEA_pyConstant: { FOREGROUND: "aa5505" },
+ OPENIDEA_pyProperty: { FOREGROUND: "aa6606" },
+ OPENIDEA_pyTypeParameter: { FOREGROUND: "aa7707" },
+};
+const xml = `${Object.entries(attributes)
+ .map(
+ ([key, values]) =>
+ ``,
+ )
+ .join("")}`;
+const style = document.createElement("style");
+style.textContent =
+ schemeCss("custom-python", {}, null, { "custom-python": parseIcls(xml).scheme }) +
+ "body { margin: 24px; background: #101820; color: #ddeeff; } .cm-editor { font: 14px/1.6 monospace; }";
+document.head.append(style);
+const support = await loadSupport(findLanguageDescription("Python", "demo.py")!);
+new EditorView({
+ parent: document.body,
+ state: EditorState.create({
+ doc: `MAX_SIZE: Final = 10
+name = "global"
+T = TypeVar("T")
+
+def outer(x: T) -> T:
+ """Syntax Highlighting Demo"""
+ def nested(y):
+ print(y + 1)
+ nested(x)
+
+class Foo[T]:
+ count = 0
+ def __init__(self: Foo, value: T):
+ value = b"newline:\\n"
+ local = 1
+ self.instance_attr = value
+ print(local, MAX_SIZE, name, T)
+ self.run(whatever=1) # Comment
+ @property
+ def size(self) -> int:
+ return self.instance_attr
+`,
+ extensions: [
+ support!,
+ syntaxHighlighting(openideaHighlightStyle),
+ lspTheme,
+ lspDoc.of({ language: "python" } as EditorDocContext),
+ ...(new URLSearchParams(location.search).has("baseline") ? [] : [semanticExtension()]),
+ ],
+ }),
+});
diff --git a/e2e/pythonHighlight.html b/e2e/pythonHighlight.html
new file mode 100644
index 00000000..f3414d19
--- /dev/null
+++ b/e2e/pythonHighlight.html
@@ -0,0 +1,9 @@
+
+
+
+
+
+
+
+
+
diff --git a/e2e/pythonHighlight.spec.ts b/e2e/pythonHighlight.spec.ts
new file mode 100644
index 00000000..f3e5b084
--- /dev/null
+++ b/e2e/pythonHighlight.spec.ts
@@ -0,0 +1,51 @@
+import { expect, test } from "@playwright/test";
+
+test("Python syntax supplements retain PyCharm colours without a language server", async ({ page }, info) => {
+ const errors: string[] = [];
+ page.on("pageerror", (e) => errors.push(e.message));
+ await page.goto("/e2e/pythonHighlight.html?baseline");
+ await expect(page.locator(".cm-content")).toBeVisible();
+ await page.screenshot({ path: info.outputPath("before.png") });
+ await page.goto("/e2e/pythonHighlight.html");
+ await expect(page.locator(".cm-sem-python-definition")).toHaveText("outer");
+ await expect(page.locator(".cm-sem-python-definition")).toHaveCSS("color", "rgb(34, 68, 102)");
+ await expect(page.locator(".cm-sem-python-nested")).toHaveCSS("color", "rgb(51, 85, 119)");
+ await expect(page.locator(".cm-sem-python-docstring")).toHaveCSS("color", "rgb(102, 136, 170)");
+ await expect(page.locator(".cm-sem-python-docstring")).toHaveCSS("text-decoration-line", "none");
+ await expect(page.locator(".cm-sem-python-bytes").first()).toHaveCSS("color", "rgb(119, 153, 187)");
+ await expect(page.locator(".cm-sem-python-bytes").first()).toHaveCSS("font-weight", "700");
+ await expect(page.locator(".cm-sem-python-escape")).toHaveCSS("color", "rgb(136, 170, 204)");
+ await expect(page.locator(".cm-sem-python-annotation").first()).toHaveCSS("color", "rgb(153, 187, 221)");
+ await expect(page.locator(".cm-sem-self").first()).toHaveCSS("color", "rgb(170, 119, 136)");
+ await expect(page.locator(".cm-sem-python-method-call")).toHaveCSS("color", "rgb(85, 119, 153)");
+ await expect(
+ page
+ .locator(".cm-line")
+ .filter({ hasText: "nested(x)" })
+ .locator(".cm-sem-function")
+ .filter({ hasText: /^nested$/ }),
+ ).toHaveCSS("color", "rgb(68, 102, 136)");
+ await expect(
+ page
+ .locator(".cm-line")
+ .filter({ hasText: "# Comment" })
+ .locator("span")
+ .filter({ hasText: /^# Comment$/ }),
+ ).toHaveCSS("color", "rgb(102, 119, 136)");
+ const print = page
+ .locator(".cm-sem-python-builtin")
+ .filter({ hasText: /^print$/ })
+ .first();
+ await expect(print).toHaveText("print");
+ await expect(print).toHaveCSS("color", "rgb(187, 136, 153)");
+ await expect(page.locator(".cm-sem-python-keyword-arg")).toHaveCSS("color", "rgb(204, 153, 170)");
+ await expect(page.locator(".cm-sem-python-local-variable").first()).toHaveCSS("color", "rgb(170, 17, 1)");
+ await expect(page.locator(".cm-sem-python-global-variable").first()).toHaveCSS("color", "rgb(170, 34, 2)");
+ await expect(page.locator(".cm-sem-python-instance-attribute").first()).toHaveCSS("color", "rgb(170, 51, 3)");
+ await expect(page.locator(".cm-sem-python-class-attribute").first()).toHaveCSS("color", "rgb(170, 68, 4)");
+ await expect(page.locator(".cm-sem-python-constant").first()).toHaveCSS("color", "rgb(170, 85, 5)");
+ await expect(page.locator(".cm-sem-python-property").first()).toHaveCSS("color", "rgb(170, 102, 6)");
+ await expect(page.locator(".cm-sem-python-type-parameter").first()).toHaveCSS("color", "rgb(170, 119, 7)");
+ expect(errors).toEqual([]);
+ await page.screenshot({ path: info.outputPath("python-highlighting.png") });
+});
diff --git a/knip.ts b/knip.ts
index 572c234a..9fab7673 100644
--- a/knip.ts
+++ b/knip.ts
@@ -2,6 +2,7 @@ import type { KnipConfig } from "knip";
const config: KnipConfig = {
// Entries come from the Vite (index.html), Vitest, Playwright and package.json script plugins.
+ entry: ["e2e/pythonHighlight.fixture.ts"],
project: [
"src/**/*.{ts,tsx,css}",
"!src/editor/core/cm/angular/parser.terms.ts",
diff --git a/plugins/official/harbor-night/themes/harbor-night.scheme.json b/plugins/official/harbor-night/themes/harbor-night.scheme.json
index 41bd2966..a6f7dc8e 100644
--- a/plugins/official/harbor-night/themes/harbor-night.scheme.json
+++ b/plugins/official/harbor-night/themes/harbor-night.scheme.json
@@ -22,6 +22,13 @@
"--syn-param": "#d9dfe8",
"--syn-property": "#b9a3f0",
"--syn-constant": "#e58fa3",
+ "--syn-python-local-variable": "#d9dfe8",
+ "--syn-python-global-variable": "#d9dfe8",
+ "--syn-python-instance-attribute": "#b9a3f0",
+ "--syn-python-class-attribute": "#b9a3f0",
+ "--syn-python-constant": "#e58fa3",
+ "--syn-python-property": "#b9a3f0",
+ "--syn-python-type-parameter": "#8fd4c9",
"--syn-builtin": "#6fc3df",
"--syn-self": "#d6849c",
"--syn-keyword-arg": "#c4a0f2",
diff --git a/src/editor/lsp/features.test.ts b/src/editor/lsp/features.test.ts
index 0d23cf9b..4f221b34 100644
--- a/src/editor/lsp/features.test.ts
+++ b/src/editor/lsp/features.test.ts
@@ -35,6 +35,14 @@ describe("semanticRanges", () => {
expect(semanticRanges(doc, input, false, "java")[0]?.className).toBe("cm-sem-property cm-sem-static");
});
+ it("routes Python library symbols through its built-in name colour", () => {
+ const input = { tokenTypes: ["function"], tokenModifiers: ["defaultLibrary"], data: [0, 0, 3, 0, 1] };
+ expect(semanticRanges(doc, input, false, "python")[0]?.className).toBe(
+ "cm-sem-function cm-sem-builtin cm-sem-python-builtin",
+ );
+ expect(semanticRanges(doc, input, false, "java")[0]?.className).toBe("cm-sem-function cm-sem-builtin");
+ });
+
it("ignores type and modifier names that only exist on Object.prototype", () => {
const input = {
tokenTypes: ["method", "toString"],
diff --git a/src/editor/lsp/pythonHighlight.test.ts b/src/editor/lsp/pythonHighlight.test.ts
new file mode 100644
index 00000000..81ba2d62
--- /dev/null
+++ b/src/editor/lsp/pythonHighlight.test.ts
@@ -0,0 +1,179 @@
+import { describe, expect, it } from "vitest";
+import { EditorState } from "@codemirror/state";
+import { ensureSyntaxTree, syntaxTree } from "@codemirror/language";
+import { findLanguageDescription, loadSupport } from "../core/cm/languages";
+import { pythonSyntaxRanges, pythonFallbackRanges, pythonSymbolKinds } from "./pythonHighlight";
+import { semanticRanges } from "./semantic";
+
+async function parse(source: string) {
+ const support = await loadSupport(findLanguageDescription("Python", "demo.py")!);
+ const state = EditorState.create({ doc: source, extensions: [support!] });
+ ensureSyntaxTree(state, state.doc.length, 1000);
+ return state;
+}
+
+describe("Python PyCharm categories", () => {
+ it("distinguishes definitions, nested definitions, docstrings, self, bytes and keyword arguments", async () => {
+ const state = await parse(`"""module doc"""
+def outer(x):
+ """function doc"""
+ def nested(y):
+ return y
+ nested(x)
+class Foo:
+ """class doc"""
+ def __init__(self):
+ value = b"bytes\\n"
+ self.run(whatever=1)
+ print(self.__doc__)
+`);
+ const ranges = pythonSyntaxRanges(syntaxTree(state), state.doc);
+ const styles = ranges.map((r) => [state.doc.sliceString(r.from, r.to), r.className]);
+ expect(styles).toContainEqual(["outer", "cm-sem-python-definition"]);
+ expect(styles).toContainEqual(["nested", "cm-sem-python-nested"]);
+ expect(styles).toContainEqual(["__init__", "cm-sem-python-predefined-definition"]);
+ expect(styles).toContainEqual(["whatever", "cm-sem-python-keyword-arg"]);
+ expect(styles).toContainEqual(["__doc__", "cm-sem-python-predefined-usage"]);
+ expect(styles.filter(([, c]) => c === "cm-sem-python-docstring")).toHaveLength(3);
+ expect(styles.filter(([, c]) => c === "cm-sem-self")).toHaveLength(3);
+ expect(styles).toContainEqual(['b"bytes\\n"', "cm-sem-python-bytes"]);
+ expect(styles).toContainEqual(["\\n", "cm-sem-python-escape"]);
+ });
+
+ it("does not treat a locally shadowed builtin as a library symbol", async () => {
+ const state = await parse("def f(print):\n print(1)\nlen([1])");
+ const ranges = pythonFallbackRanges(syntaxTree(state), state.doc);
+ const print = ranges.filter((r) => state.doc.sliceString(r.from, r.to) === "print");
+ expect(print.map((r) => r.className)).toEqual(["cm-sem-parameter", "cm-sem-parameter"]);
+ expect(ranges.find((r) => state.doc.sliceString(r.from, r.to) === "len")?.className).toContain(
+ "cm-sem-python-builtin",
+ );
+ });
+
+ it("lets server classification override the local builtin guess", async () => {
+ const state = await parse("print(1)");
+ const ranges = semanticRanges(
+ state.doc,
+ { tokenTypes: ["variable"], tokenModifiers: [], data: [0, 0, 5, 0, 0] },
+ false,
+ "python",
+ syntaxTree(state),
+ );
+ expect(ranges.some((r) => r.className.includes("python-builtin"))).toBe(false);
+ });
+
+ it("keeps calls yellow when Pyrefly omits declaration modifiers", async () => {
+ const state = await parse("def outer(x):\n outer(x)");
+ const result = semanticRanges(
+ state.doc,
+ {
+ tokenTypes: ["function"],
+ tokenModifiers: [],
+ data: [0, 4, 5, 0, 0, 1, 4, 5, 0, 0],
+ },
+ false,
+ "python",
+ syntaxTree(state),
+ );
+ expect(result.filter((r) => state.doc.sliceString(r.from, r.to) === "outer").map((r) => r.className)).toEqual([
+ "cm-sem-python-definition",
+ "cm-sem-function",
+ ]);
+ });
+
+ it("does not misclassify ordinary strings, later expressions, bytes or f strings as docstrings", async () => {
+ const state = await parse(
+ 'x = "ordinary"\n"later"\ndef f():\n b"bytes"\n "later"\ndef g():\n f"format {x}"',
+ );
+ expect(
+ pythonSyntaxRanges(syntaxTree(state), state.doc).some((r) => r.className === "cm-sem-python-docstring"),
+ ).toBe(false);
+ });
+
+ it("honours Pyrefly selfParameter modifiers without applying rainbow colours", async () => {
+ const state = await parse("self arg");
+ const result = semanticRanges(
+ state.doc,
+ {
+ tokenTypes: ["parameter"],
+ tokenModifiers: ["selfParameter"],
+ data: [0, 0, 4, 0, 1, 0, 5, 3, 0, 0],
+ },
+ true,
+ "python",
+ );
+ expect(result[0]?.className).toBe("cm-sem-parameter cm-sem-self");
+ expect(result[1]?.className).toContain("cm-sem-rainbow-");
+ });
+
+ it("classifies Python variables, attributes, properties, constants and type parameters", async () => {
+ const state = await parse(`MAX_SIZE: Final = 10
+name = "x"
+T = TypeVar("T")
+type Alias[U] = list[U]
+
+class Box[T]:
+ count = 0
+ ICON: Final = "box"
+
+ def __init__(self, value: T):
+ self.value = value
+ local = 1
+ self.count
+ self.value
+ self.size
+
+ @property
+ def size(self) -> T:
+ return self.value
+
+ def read(self) -> T:
+ return Box.count + Box.ICON
+`);
+ const tree = syntaxTree(state);
+ const kinds = pythonSymbolKinds(tree, state.doc);
+ const kindOf = (name: string) =>
+ [...kinds]
+ .filter(([range]) => {
+ const [from, to] = range.split(":").map(Number);
+ return state.doc.sliceString(from!, to!) === name;
+ })
+ .map(([, kind]) => kind);
+ expect(kindOf("local")).toEqual(["localVariable"]);
+ expect(kindOf("name")).toEqual(["globalVariable"]);
+ expect(kindOf("MAX_SIZE")).toEqual(["constant"]);
+ expect(kindOf("value")).toContain("instanceAttribute");
+ expect(kindOf("count")).toContain("classAttribute");
+ expect(kindOf("ICON")).toContain("constant");
+ expect(kindOf("size")).toContain("property");
+ expect(kindOf("T")).toEqual(expect.arrayContaining(["typeParameter"]));
+ expect(kindOf("U")).toEqual(["typeParameter", "typeParameter"]);
+
+ const syntax = pythonSyntaxRanges(tree, state.doc, kinds);
+ const syntaxClass = (name: string) =>
+ syntax.filter((range) => state.doc.sliceString(range.from, range.to) === name).map((range) => range.className);
+ expect(syntaxClass("size")).toContain("cm-sem-property cm-sem-python-property");
+ expect(syntaxClass("T")).toContain("cm-sem-typeParameter cm-sem-python-type-parameter");
+ });
+
+ it("keeps Pyrefly tokens but applies the finer Python category", async () => {
+ const state = await parse("MAX_SIZE: Final = 10\ndef f(x):\n local = x\n return local");
+ const result = semanticRanges(
+ state.doc,
+ {
+ tokenTypes: ["variable"],
+ tokenModifiers: [],
+ data: [0, 0, 8, 0, 0, 2, 4, 5, 0, 1, 3, 11, 5, 0, 2],
+ },
+ true,
+ "python",
+ syntaxTree(state),
+ );
+ const classOf = (name: string) =>
+ result.filter((range) => state.doc.sliceString(range.from, range.to) === name).map((range) => range.className);
+ expect(classOf("MAX_SIZE")).toEqual(["cm-sem-constant cm-sem-python-constant"]);
+ expect(classOf("local")[0]).toContain("cm-sem-variable cm-sem-python-local-variable cm-sem-rainbow-");
+ expect(classOf("local")[1]).toContain("cm-sem-python-local-variable");
+ expect(classOf("MAX_SIZE")[0]).not.toContain("cm-sem-rainbow-");
+ });
+});
diff --git a/src/editor/lsp/pythonHighlight.ts b/src/editor/lsp/pythonHighlight.ts
new file mode 100644
index 00000000..a57c0e58
--- /dev/null
+++ b/src/editor/lsp/pythonHighlight.ts
@@ -0,0 +1,513 @@
+import type { SyntaxNode, Tree } from "@lezer/common";
+import type { Text } from "@codemirror/state";
+
+interface PythonRange {
+ from: number;
+ to: number;
+ className: string;
+ source: "pythonSyntax" | "pythonFallback";
+}
+
+export type PythonSymbolKind =
+ | "function"
+ | "type"
+ | "parameter"
+ | "localVariable"
+ | "globalVariable"
+ | "instanceAttribute"
+ | "classAttribute"
+ | "constant"
+ | "property"
+ | "typeParameter"
+ | "builtin";
+
+const PYTHON_CLASSES: Record = {
+ function: "cm-sem-function",
+ type: "cm-sem-type",
+ parameter: "cm-sem-parameter",
+ localVariable: "cm-sem-variable cm-sem-python-local-variable",
+ globalVariable: "cm-sem-global cm-sem-python-global-variable",
+ instanceAttribute: "cm-sem-property cm-sem-python-instance-attribute",
+ classAttribute: "cm-sem-property cm-sem-python-class-attribute",
+ constant: "cm-sem-constant cm-sem-python-constant",
+ property: "cm-sem-property cm-sem-python-property",
+ typeParameter: "cm-sem-typeParameter cm-sem-python-type-parameter",
+ builtin: "cm-sem-builtin cm-sem-python-builtin",
+};
+
+export const pythonSymbolClasses = (kind: PythonSymbolKind) => PYTHON_CLASSES[kind];
+
+const BUILTIN_TYPES = new Set([
+ "int",
+ "str",
+ "bool",
+ "bytes",
+ "float",
+ "complex",
+ "list",
+ "dict",
+ "tuple",
+ "set",
+ "frozenset",
+ "object",
+ "type",
+ "None",
+]);
+
+function hasAncestor(node: SyntaxNode, name: string): boolean {
+ for (let parent = node.parent; parent; parent = parent.parent) if (parent.name === name) return true;
+ return false;
+}
+
+function enclosingFunction(node: SyntaxNode | null): SyntaxNode | null {
+ for (; node; node = node.parent) if (node.name === "FunctionDefinition") return node;
+ return null;
+}
+
+const FUNCTION_SCOPES = new Set(["FunctionDefinition", "LambdaExpression"]);
+const COMPREHENSION_SCOPES = new Set([
+ "ArrayComprehensionExpression",
+ "DictionaryComprehensionExpression",
+ "SetComprehensionExpression",
+]);
+const SCOPE_NODES = new Set([
+ "FunctionDefinition",
+ "LambdaExpression",
+ "ClassDefinition",
+ "TypeDefinition",
+ "Script",
+ ...COMPREHENSION_SCOPES,
+]);
+const PATTERN_CONTAINERS = new Set([
+ "TupleExpression",
+ "ArrayExpression",
+ "ParenthesizedExpression",
+ "SequencePattern",
+ "PatternArgList",
+ "StarPattern",
+]);
+const TYPE_VAR_CALLS = new Set(["TypeVar", "ParamSpec", "TypeVarTuple"]);
+const KIND_PRIORITY: Record = {
+ globalVariable: 70,
+ typeParameter: 90,
+ parameter: 80,
+ property: 80,
+ constant: 70,
+ classAttribute: 60,
+ localVariable: 50,
+ function: 40,
+ type: 40,
+ instanceAttribute: 40,
+ builtin: 30,
+};
+
+function scope(node: SyntaxNode | null): SyntaxNode | null {
+ for (; node; node = node.parent) if (SCOPE_NODES.has(node.name)) return node;
+ return null;
+}
+
+function enclosingClass(node: SyntaxNode | null): SyntaxNode | null {
+ for (; node; node = node.parent) if (node.name === "ClassDefinition") return node;
+ return null;
+}
+
+function directName(node: SyntaxNode | null): SyntaxNode | null {
+ for (let child = node?.firstChild; child; child = child.nextSibling) if (child.name === "VariableName") return child;
+ return null;
+}
+
+function isConstantName(name: string): boolean {
+ return /^_*[A-Z][A-Z0-9_]*$/.test(name);
+}
+
+function hasFinalAnnotation(node: SyntaxNode, doc: Text): boolean {
+ for (let child = node.firstChild; child; child = child.nextSibling) {
+ if (
+ (child.name === "VariableName" || child.name === "PropertyName") &&
+ doc.sliceString(child.from, child.to) === "Final"
+ )
+ return true;
+ if (hasFinalAnnotation(child, doc)) return true;
+ }
+ return false;
+}
+
+function callName(node: SyntaxNode | null, doc: Text): string | undefined {
+ if (node?.name !== "CallExpression") return undefined;
+ const callee = node.firstChild;
+ if (callee?.name === "VariableName") return doc.sliceString(callee.from, callee.to);
+ if (callee?.name === "MemberExpression") {
+ const property = callee.getChild("PropertyName");
+ if (property) return doc.sliceString(property.from, property.to);
+ }
+ return undefined;
+}
+
+function assignmentRight(node: SyntaxNode): SyntaxNode | null {
+ const assign = node.getChild("AssignOp");
+ return assign?.nextSibling ?? null;
+}
+
+function assignmentAnnotation(node: SyntaxNode): SyntaxNode | null {
+ return node.getChild("TypeDef");
+}
+
+function collectPattern(node: SyntaxNode, out: SyntaxNode[]) {
+ if (node.name === "VariableName" || node.name === "MemberExpression") {
+ out.push(node);
+ return;
+ }
+ if (!PATTERN_CONTAINERS.has(node.name)) return;
+ for (let child = node.firstChild; child; child = child.nextSibling) collectPattern(child, out);
+}
+
+function assignmentTargets(node: SyntaxNode, stop: string): SyntaxNode[] {
+ const out: SyntaxNode[] = [];
+ for (let child = node.firstChild; child; child = child.nextSibling) {
+ if (child.name === stop) break;
+ if (child.name === "TypeDef") continue;
+ collectPattern(child, out);
+ }
+ return out;
+}
+
+function forTargets(node: SyntaxNode): SyntaxNode[] {
+ const out: SyntaxNode[] = [];
+ let collecting = false;
+ for (let child = node.firstChild; child; child = child.nextSibling) {
+ if (child.name === "for") {
+ collecting = true;
+ continue;
+ }
+ if (!collecting) continue;
+ if (child.name === "in") break;
+ collectPattern(child, out);
+ }
+ return out;
+}
+
+function aliasTargets(node: SyntaxNode): SyntaxNode[] {
+ const out: SyntaxNode[] = [];
+ for (let child = node.firstChild; child; child = child.nextSibling) {
+ if (child.name === "as") {
+ const target = child.nextSibling;
+ if (target) collectPattern(target, out);
+ }
+ }
+ return out;
+}
+
+function isPropertyFunction(node: SyntaxNode, name: string, doc: Text): boolean {
+ const decorated = node.parent;
+ if (decorated?.name !== "DecoratedStatement") return false;
+ for (let child = decorated.firstChild; child; child = child.nextSibling) {
+ if (child.name !== "Decorator") continue;
+ const decorator = doc.sliceString(child.from + 1, child.to).trim();
+ if (decorator === "property" || decorator === "cached_property" || decorator.endsWith(".cached_property"))
+ return true;
+ const dot = decorator.lastIndexOf(".");
+ if (
+ dot > 0 &&
+ decorator.slice(0, dot) === name &&
+ ["setter", "getter", "deleter"].includes(decorator.slice(dot + 1))
+ )
+ return true;
+ }
+ return false;
+}
+
+/** Classifies every Python identifier from scope bindings; the server only distinguishes some of them. */
+export function pythonSymbolKinds(tree: Tree, doc: Text): Map {
+ const bindings = new Map>();
+ const classByName = new Map();
+ const classData = new Map>();
+ const propertyNames = new Set();
+ const key = (node: SyntaxNode) => `${node.name}:${node.from}`;
+ const text = (node: SyntaxNode) => doc.sliceString(node.from, node.to);
+ const dataFor = (node: SyntaxNode) => {
+ const scopeKey = key(node);
+ let data = classData.get(scopeKey);
+ if (!data) classData.set(scopeKey, (data = new Set()));
+ return data;
+ };
+ const bind = (owner: SyntaxNode | null, name: string, kind: PythonSymbolKind) => {
+ if (!owner) return;
+ let names = bindings.get(key(owner));
+ if (!names) bindings.set(key(owner), (names = new Map()));
+ const previous = names.get(name);
+ if (previous && KIND_PRIORITY[previous] >= KIND_PRIORITY[kind]) return;
+ names.set(name, kind);
+ };
+
+ tree.iterate({
+ enter(cursor) {
+ const node = cursor.node;
+ if (node.name === "ClassDefinition") {
+ const classNode = directName(node);
+ if (classNode) {
+ bind(scope(node.parent), text(classNode), "type");
+ classByName.set(text(classNode), node);
+ dataFor(node);
+ }
+ } else if (node.name === "TypeDefinition") {
+ const typeNode = directName(node);
+ if (typeNode) bind(scope(node.parent), text(typeNode), "type");
+ } else if (node.name === "FunctionDefinition") {
+ const functionNode = directName(node);
+ if (!functionNode) return;
+ const functionName = text(functionNode);
+ const property = isPropertyFunction(node, functionName, doc);
+ bind(scope(node.parent), functionName, property ? "property" : "function");
+ if (property) {
+ propertyNames.add(functionName);
+ }
+ } else if (node.name === "ParamList") {
+ const owner = scope(node);
+ for (let child = node.firstChild; child; child = child.nextSibling)
+ if (child.name === "VariableName") bind(owner, text(child), "parameter");
+ } else if (node.name === "TypeParam") {
+ const param = directName(node);
+ if (param) bind(scope(node), text(param), "typeParameter");
+ } else if (node.name === "ScopeStatement" && node.firstChild?.name === "global") {
+ const owner = scope(node);
+ for (let child: SyntaxNode | null = node.firstChild; child; child = child.nextSibling)
+ if (child.name === "VariableName") bind(owner, text(child), "globalVariable");
+ }
+ if (node.name === "CapturePattern") {
+ const capture = directName(node);
+ if (capture) {
+ const owner = scope(node);
+ const kind =
+ owner?.name === "ClassDefinition"
+ ? "classAttribute"
+ : owner?.name === "Script"
+ ? "globalVariable"
+ : "localVariable";
+ bind(owner, text(capture), kind);
+ }
+ }
+ },
+ });
+
+ const assignmentKind = (owner: SyntaxNode | null, name: string, annotation: SyntaxNode | null): PythonSymbolKind => {
+ if (annotation && hasFinalAnnotation(annotation, doc)) return "constant";
+ if (isConstantName(name)) return "constant";
+ if (owner?.name === "Script") return "globalVariable";
+ if (owner?.name === "ClassDefinition") return "classAttribute";
+ return "localVariable";
+ };
+
+ tree.iterate({
+ enter(cursor) {
+ const node = cursor.node;
+ if (node.name === "AssignStatement" || node.name === "UpdateStatement") {
+ const stop = node.name === "AssignStatement" ? "AssignOp" : "UpdateOp";
+ const owner = scope(node);
+ const targets = assignmentTargets(node, stop);
+ const annotation = node.name === "AssignStatement" ? assignmentAnnotation(node) : null;
+ const called = node.name === "AssignStatement" ? callName(assignmentRight(node), doc) : undefined;
+ const typeParameter = called !== undefined && TYPE_VAR_CALLS.has(called);
+ for (const target of targets) {
+ if (target.name === "MemberExpression") {
+ const object = target.firstChild;
+ const property = target.getChild("PropertyName");
+ if (object?.name === "VariableName" && property && text(object) === "self") {
+ const ownerClass = enclosingClass(node);
+ if (ownerClass) dataFor(ownerClass).add(text(property));
+ }
+ continue;
+ }
+ const name = text(target);
+ bind(owner, name, typeParameter ? "typeParameter" : assignmentKind(owner, name, annotation));
+ }
+ } else if (node.name === "NamedExpression") {
+ const target = node.firstChild;
+ if (target?.name === "VariableName")
+ bind(scope(node), text(target), assignmentKind(scope(node), text(target), null));
+ } else if (node.name === "ForStatement") {
+ const owner = scope(node);
+ for (const target of forTargets(node))
+ if (target.name === "VariableName") bind(owner, text(target), assignmentKind(owner, text(target), null));
+ } else if (COMPREHENSION_SCOPES.has(node.name)) {
+ const owner = scope(node);
+ for (const target of forTargets(node))
+ if (target.name === "VariableName") bind(owner, text(target), "localVariable");
+ } else if (node.name === "WithStatement" || node.name === "TryStatement") {
+ const owner = scope(node);
+ for (const target of aliasTargets(node))
+ if (target.name === "VariableName") bind(owner, text(target), assignmentKind(owner, text(target), null));
+ }
+ },
+ });
+
+ const resolve = (
+ node: SyntaxNode,
+ name: string,
+ start: SyntaxNode | null = scope(node),
+ ): PythonSymbolKind | undefined => {
+ let crossedFunction = false;
+ for (let current = start; current; current = scope(current.parent)) {
+ if (FUNCTION_SCOPES.has(current.name)) crossedFunction = true;
+ const kind = bindings.get(key(current))?.get(name);
+ if (!kind) continue;
+ if (crossedFunction && current.name === "ClassDefinition" && kind !== "typeParameter") continue;
+ return kind;
+ }
+ return undefined;
+ };
+ const classMember = (node: SyntaxNode | null, name: string) =>
+ node ? bindings.get(key(node))?.get(name) : undefined;
+ const memberKind = (node: SyntaxNode, name: string): PythonSymbolKind | undefined => {
+ const member = node.parent;
+ if (member?.name !== "MemberExpression") return undefined;
+ const object = member.firstChild;
+ if (!object) return undefined;
+ const objectName = object.name === "VariableName" ? text(object) : undefined;
+ const currentClass = enclosingClass(node);
+ const namedClass = objectName ? classByName.get(objectName) : undefined;
+ const owner = objectName === "self" || objectName === "cls" ? currentClass : namedClass;
+ const bound = classMember(owner ?? null, name);
+ const call = member.parent?.name === "CallExpression" && member.parent.firstChild?.from === member.from;
+ if (call) return bound === "property" || bound === "constant" ? bound : undefined;
+ if (objectName === "self" && currentClass) {
+ if (bound === "property") return "property";
+ if (owner && dataFor(owner).has(name)) return "instanceAttribute";
+ if (bound) return bound;
+ if (propertyNames.has(name)) return "property";
+ if (isConstantName(name)) return "constant";
+ return "instanceAttribute";
+ }
+ if (objectName === "cls" && currentClass) {
+ if (bound) return bound;
+ if (isConstantName(name)) return "constant";
+ return "classAttribute";
+ }
+ if (namedClass) {
+ if (bound) return bound;
+ if (isConstantName(name)) return "constant";
+ return "classAttribute";
+ }
+ if (propertyNames.has(name)) return "property";
+ if (isConstantName(name)) return "constant";
+ return "instanceAttribute";
+ };
+ const kindFor = (node: SyntaxNode): PythonSymbolKind | undefined => {
+ const name = text(node);
+ if (node.name === "PropertyName") return memberKind(node, name);
+ const start = node.parent?.name === "FunctionDefinition" ? scope(node.parent.parent) : scope(node);
+ const resolved = resolve(node, name, start);
+ if (resolved) return resolved;
+ if (isConstantName(name) && !BUILTIN_NAMES.has(name)) return "constant";
+ return undefined;
+ };
+
+ const out = new Map();
+ tree.iterate({
+ enter(cursor) {
+ const node = cursor.node;
+ if (node.name !== "VariableName" && node.name !== "PropertyName") return;
+ const kind = kindFor(node);
+ if (kind) out.set(`${node.from}:${node.to}`, kind);
+ },
+ });
+ return out;
+}
+
+/** Python categories absent from LSP tokens, derived from the editor's existing parse tree. */
+export function pythonSyntaxRanges(
+ tree: Tree,
+ doc: Text,
+ symbols: Map = pythonSymbolKinds(tree, doc),
+): PythonRange[] {
+ const ranges: PythonRange[] = [];
+ tree.iterate({
+ enter(cursor) {
+ const node = cursor.node;
+ if (!["VariableName", "PropertyName", "String", "Escape", "(", ")", "[", "]", "{", "}"].includes(node.name))
+ return;
+ const text = doc.sliceString(node.from, node.to);
+ const symbol = symbols.get(`${node.from}:${node.to}`);
+ let style: string | undefined;
+ let className: string | undefined;
+ if (["(", ")"].includes(node.name)) style = "python-parentheses";
+ else if (["[", "]"].includes(node.name)) style = "python-brackets";
+ else if (["{", "}"].includes(node.name)) style = "python-braces";
+ else if (node.name === "VariableName") {
+ if (symbol === "typeParameter") className = pythonSymbolClasses(symbol);
+ else if (node.parent?.name === "FunctionDefinition") {
+ if (symbol === "property") className = pythonSymbolClasses(symbol);
+ else
+ style = /^__.*__$/.test(text)
+ ? "python-predefined-definition"
+ : enclosingFunction(node.parent.parent)
+ ? "python-nested"
+ : "python-definition";
+ } else if (hasAncestor(node, "TypeDef") && !BUILTIN_TYPES.has(text)) style = "python-annotation";
+ else if ((text === "self" || text === "cls") && enclosingFunction(node.parent)) style = "self";
+ else if (node.parent?.name === "ArgList" && node.nextSibling?.name === "AssignOp") style = "python-keyword-arg";
+ } else if (node.name === "PropertyName" && /^__.*__$/.test(text)) style = "python-predefined-usage";
+ else if (
+ node.name === "PropertyName" &&
+ node.parent?.name === "MemberExpression" &&
+ node.parent.parent?.name === "CallExpression"
+ )
+ style = "python-method-call";
+ else if (node.name === "String") {
+ const statement = node.parent;
+ const body = statement?.parent;
+ let first = body?.firstChild;
+ while (first && (first.name === ":" || first.name === "Comment")) first = first.nextSibling;
+ const docstring =
+ statement?.name === "ExpressionStatement" &&
+ statement.firstChild?.from === node.from &&
+ statement.firstChild?.to === node.to &&
+ first?.from === statement.from &&
+ (body?.name === "Script" ||
+ (body?.name === "Body" && ["FunctionDefinition", "ClassDefinition"].includes(body.parent?.name ?? "")));
+ if (docstring && !/^[bf]/i.test(text)) style = "python-docstring";
+ else if (/^[rub]*b[rub]*['"]/i.test(text)) style = "python-bytes";
+ } else if (node.name === "Escape") style = "python-escape";
+ if (style) className = `cm-sem-${style}`;
+ if (className) ranges.push({ from: node.from, to: node.to, className, source: "pythonSyntax" });
+ if (style === "python-docstring") return false;
+ },
+ });
+ return ranges;
+}
+
+const BUILTIN_NAMES = new Set(
+ "abs aiter all anext any ascii bin bool breakpoint bytearray bytes callable chr classmethod compile complex delattr dict dir divmod enumerate eval exec filter float format frozenset getattr globals hasattr hash help hex id input int isinstance issubclass iter len list locals map max memoryview min next object oct open ord pow print property range repr reversed round set setattr slice sorted staticmethod str sum super tuple type vars zip __import__ BaseException Exception ArithmeticError AssertionError AttributeError EOFError ImportError IndexError KeyError LookupError NameError NotImplementedError OSError OverflowError RuntimeError StopIteration SyntaxError TypeError UnboundLocalError ValueError ZeroDivisionError".split(
+ " ",
+ ),
+);
+
+/** Fast local guesses while LSP is pending. Server tokens take precedence over these. */
+export function pythonFallbackRanges(
+ tree: Tree,
+ doc: Text,
+ rainbow?: (name: string) => string,
+ symbols: Map = pythonSymbolKinds(tree, doc),
+): PythonRange[] {
+ const ranges: PythonRange[] = [];
+ tree.iterate({
+ enter(cursor) {
+ const node = cursor.node;
+ if (node.name !== "VariableName" && node.name !== "PropertyName") return;
+ const name = doc.sliceString(node.from, node.to);
+ const kind =
+ symbols.get(`${node.from}:${node.to}`) ??
+ (node.name === "VariableName" && BUILTIN_NAMES.has(name) ? "builtin" : undefined);
+ if (!kind) return;
+ const local = kind === "parameter" || kind === "localVariable";
+ ranges.push({
+ from: node.from,
+ to: node.to,
+ className:
+ pythonSymbolClasses(kind) +
+ (local && rainbow && name !== "self" && name !== "cls" ? ` ${rainbow(name)}` : ""),
+ source: "pythonFallback",
+ });
+ },
+ });
+ return ranges;
+}
diff --git a/src/editor/lsp/semantic.ts b/src/editor/lsp/semantic.ts
index 9b3d3e95..3d847381 100644
--- a/src/editor/lsp/semantic.ts
+++ b/src/editor/lsp/semantic.ts
@@ -9,6 +9,15 @@
* `cm-sem-rainbow-`, `n` derived from the name, so one name keeps one
* colour (`--syn-rainbow-`) throughout the file.
*/
+import { syntaxTree } from "@codemirror/language";
+import type { Tree } from "@lezer/common";
+import {
+ pythonSyntaxRanges,
+ pythonFallbackRanges,
+ pythonSymbolClasses,
+ pythonSymbolKinds,
+ type PythonSymbolKind,
+} from "./pythonHighlight";
import { Prec, StateEffect, StateField, type Extension, type Text } from "@codemirror/state";
import { Decoration, EditorView, ViewPlugin, type DecorationSet, type ViewUpdate } from "@codemirror/view";
import { commands, type SemanticTokens } from "@/platform/ipc";
@@ -57,6 +66,11 @@ const MODIFIER_CLASS: Record = {
definition: "declaration",
defaultLibrary: "builtin",
global: "global",
+ selfParameter: "self",
+ byteString: "python-bytes",
+ escapeSequence: "python-escape",
+ formatSpecifier: "python-format",
+ formatPlaceholder: "python-format",
};
/** Number of `--syn-rainbow-` colours (IDEA's five semantic highlighting colours). */
@@ -75,15 +89,60 @@ export function rainbowIndex(name: string): number {
/** Locals and parameters only: no fields, statics, constants, library or `self` names. */
const RAINBOW_TYPES = new Set(["variable", "parameter"]);
const NOT_LOCAL = new Set(["static", "constant", "defaultLibrary", "global", "classScope", "fileScope"]);
+const PYTHON_VARIABLE_KINDS = new Set([
+ "localVariable",
+ "globalVariable",
+ "instanceAttribute",
+ "classAttribute",
+ "constant",
+ "property",
+ "typeParameter",
+]);
+const PYTHON_SYMBOL_BASES = new Set([
+ "cm-sem-variable",
+ "cm-sem-global",
+ "cm-sem-property",
+ "cm-sem-constant",
+ "cm-sem-type",
+ "cm-sem-typeParameter",
+ "cm-sem-builtin",
+]);
export interface SemanticRange {
from: number;
to: number;
className: string;
+ source?: "pythonSyntax" | "pythonFallback";
+}
+
+function rangeCoverage(ranges: Array<{ from: number; to: number }>) {
+ ranges.sort((a, b) => a.from - b.from || a.to - b.to);
+ const ends: number[] = [];
+ for (const r of ranges) ends.push(Math.max(ends.at(-1) ?? 0, r.to));
+ return (from: number, to: number) => {
+ let low = 0,
+ high = ranges.length;
+ while (low < high) {
+ const mid = (low + high) >>> 1;
+ if (ranges[mid]!.from <= from) low = mid + 1;
+ else high = mid;
+ }
+ return low > 0 && ends[low - 1]! >= to;
+ };
}
/** Decodes absolute `[line, col, len, type, modifiers]` quintuples into styled ranges. */
-export function semanticRanges(doc: Text, tokens: SemanticTokens, rainbow = false, language?: string): SemanticRange[] {
+export function semanticRanges(
+ doc: Text,
+ tokens: SemanticTokens,
+ rainbow = false,
+ language?: string,
+ tree?: Tree,
+): SemanticRange[] {
+ const symbols = language === "python" && tree ? pythonSymbolKinds(tree, doc) : undefined;
+ const syntax = language === "python" && tree ? pythonSyntaxRanges(tree, doc, symbols) : [];
+ const covered = rangeCoverage(syntax);
+ const serverRanges: Array<{ from: number; to: number }> = [];
const out: SemanticRange[] = [];
const { data, tokenTypes, tokenModifiers } = tokens;
// Servers with a `local` modifier (TypeScript) tell module-level variables apart.
@@ -95,7 +154,12 @@ export function semanticRanges(doc: Text, tokens: SemanticTokens, rainbow = fals
typeIdx = data[i + 3]!,
mods = data[i + 4]!;
if (line >= doc.lines || len <= 0) continue;
+ const l = doc.line(line + 1);
+ const from = l.from + Math.min(col, l.length);
+ const to = Math.min(from + len, l.to);
+ if (from >= to) continue;
const type = tokenTypes[typeIdx] ?? "";
+ const pythonKind = symbols?.get(`${from}:${to}`);
const names: string[] = [];
for (let b = 0; b < tokenModifiers.length; b++) if (mods & (1 << b)) names.push(tokenModifiers[b]!);
const classes: string[] = [];
@@ -109,27 +173,49 @@ export function semanticRanges(doc: Text, tokens: SemanticTokens, rainbow = fals
// `hasOwn`: servers send names like `constructor` (jdtls) that `Object.prototype` also has.
for (const n of names) if (Object.hasOwn(MODIFIER_CLASS, n)) classes.push(`cm-sem-${MODIFIER_CLASS[n]}`);
if (language === "python" && names.includes("static")) classes.push("cm-sem-python-static");
+ if (language === "python" && type === "method") classes.push("cm-sem-python-method-call");
+ if (language === "python" && names.includes("defaultLibrary")) classes.push("cm-sem-python-builtin");
+ if (pythonKind && PYTHON_VARIABLE_KINDS.has(pythonKind)) {
+ for (let j = classes.length - 1; j >= 0; j--) if (PYTHON_SYMBOL_BASES.has(classes[j]!)) classes.splice(j, 1);
+ classes.push(...pythonSymbolClasses(pythonKind).split(" "));
+ }
const local =
rainbow &&
!constant &&
- RAINBOW_TYPES.has(type) &&
+ !names.includes("selfParameter") &&
!names.some((n) => NOT_LOCAL.has(n)) &&
- (type === "parameter" || !marksLocals || names.includes("local"));
+ (language === "python"
+ ? pythonKind
+ ? pythonKind === "localVariable" || pythonKind === "parameter"
+ : type === "parameter"
+ : RAINBOW_TYPES.has(type) && (type === "parameter" || !marksLocals || names.includes("local")));
+ serverRanges.push({ from, to });
if (!classes.length && !local) continue;
- const l = doc.line(line + 1);
- const from = l.from + Math.min(col, l.length);
- const to = Math.min(from + len, l.to);
- if (from >= to) continue;
+ if (covered(from, to)) continue;
if (local) classes.push(`cm-sem-rainbow-${rainbowIndex(doc.sliceString(from, to))}`);
out.push({ from, to, className: classes.join(" ") });
}
+ if (language === "python" && tree) {
+ const serverCovered = rangeCoverage(serverRanges);
+ out.push(
+ ...pythonFallbackRanges(
+ tree,
+ doc,
+ rainbow ? (name) => `cm-sem-rainbow-${rainbowIndex(name)}` : undefined,
+ symbols,
+ ).filter((r) => !covered(r.from, r.to) && !serverCovered(r.from, r.to)),
+ );
+ }
+ out.push(...syntax);
return out.sort((a, b) => a.from - b.from || a.to - b.to);
}
const markCache = new Map();
-const markFor = (cls: string) => {
- let m = markCache.get(cls);
- if (!m) markCache.set(cls, (m = Decoration.mark({ class: cls })));
+const markFor = (cls: string, source?: "pythonSyntax" | "pythonFallback", symbol?: string) => {
+ if (symbol !== undefined) return Decoration.mark({ class: cls, symbol });
+ const key = `${source ?? "semantic"}:${cls}`;
+ let m = markCache.get(key);
+ if (!m) markCache.set(key, (m = Decoration.mark({ class: cls, pythonSyntax: source !== undefined })));
return m;
};
@@ -151,12 +237,24 @@ const semanticPlugin = ViewPlugin.fromClass(
/** Last answer, re-decorated without a request when Semantic highlighting is toggled. */
private last: { doc: Text; tokens: SemanticTokens | null } | null = null;
private destroyed = false;
+ private tree: Tree | null = null;
constructor(private view: EditorView) {
+ queueMicrotask(() => {
+ if (!this.destroyed) this.decorate(this.view.state.doc, null);
+ });
this.refresh();
this.offReady = onLspRefresh("semanticTokens", () => this.refresh());
}
update(u: ViewUpdate) {
- if (u.docChanged) this.refresh();
+ const tree = syntaxTree(u.state);
+ if (u.docChanged || tree !== this.tree) {
+ this.tree = tree;
+ this.refresh();
+ const doc = u.state.doc;
+ queueMicrotask(() => {
+ if (!this.destroyed && this.view.state.doc === doc) this.refreshSyntax(doc);
+ });
+ }
const rainbow = editorPrefs().semanticHighlighting;
if (rainbow === this.rainbow) return;
this.rainbow = rainbow;
@@ -168,13 +266,52 @@ const semanticPlugin = ViewPlugin.fromClass(
});
else this.refresh();
}
+ refreshSyntax(doc: Text) {
+ const tree = syntaxTree(this.view.state);
+ const python = docOf(this.view)?.language === "python";
+ const symbols = python ? pythonSymbolKinds(tree, doc) : undefined;
+ const syntax = python ? pythonSyntaxRanges(tree, doc, symbols) : [];
+ const covered = rangeCoverage(syntax);
+ const retained: Array<{ from: number; to: number }> = [];
+ const marks = syntax.map((r) => markFor(r.className, r.source).range(r.from, r.to));
+ this.view.state.field(semanticField).between(0, doc.length, (from, to, decoration) => {
+ if (
+ !decoration.spec.pythonSyntax &&
+ !covered(from, to) &&
+ decoration.spec.symbol === doc.sliceString(from, to)
+ ) {
+ marks.push(decoration.range(from, to));
+ retained.push({ from, to });
+ }
+ });
+ if (docOf(this.view)?.language === "python") {
+ const retainedCovered = rangeCoverage(retained);
+ for (const r of pythonFallbackRanges(
+ tree,
+ doc,
+ this.rainbow ? (name) => `cm-sem-rainbow-${rainbowIndex(name)}` : undefined,
+ symbols,
+ ))
+ if (!covered(r.from, r.to) && !retainedCovered(r.from, r.to))
+ marks.push(markFor(r.className, r.source).range(r.from, r.to));
+ }
+ this.view.dispatch({ effects: setSemantic.of(Decoration.set(marks, true)) });
+ }
decorate(doc: Text, tokens: SemanticTokens | null) {
this.last = { doc, tokens };
- const ranges = tokens ? semanticRanges(doc, tokens, this.rainbow, docOf(this.view)?.language) : [];
+ const ranges = semanticRanges(
+ doc,
+ tokens ?? { data: [], tokenTypes: [], tokenModifiers: [] },
+ this.rainbow,
+ docOf(this.view)?.language,
+ syntaxTree(this.view.state),
+ );
this.view.dispatch({
effects: setSemantic.of(
Decoration.set(
- ranges.map((r) => markFor(r.className).range(r.from, r.to)),
+ ranges.map((r) =>
+ markFor(r.className, r.source, r.source ? undefined : doc.sliceString(r.from, r.to)).range(r.from, r.to),
+ ),
true,
),
),
diff --git a/src/editor/lsp/semanticTyping.test.ts b/src/editor/lsp/semanticTyping.test.ts
new file mode 100644
index 00000000..9d4fc85d
--- /dev/null
+++ b/src/editor/lsp/semanticTyping.test.ts
@@ -0,0 +1,90 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { EditorState } from "@codemirror/state";
+import { EditorView } from "@codemirror/view";
+import { findLanguageDescription, loadSupport } from "../core/cm/languages";
+import type { SemanticTokens } from "@/platform/ipc";
+import type { EditorDocContext } from "@/editor/api";
+import { lspDoc } from "./lsp";
+import { semanticExtension } from "./semantic";
+
+const requests = vi.hoisted(() => [] as Array<(tokens: SemanticTokens) => void>);
+vi.mock("./lsp", async (importOriginal) => ({
+ ...(await importOriginal()),
+ lspRequest: () => new Promise((resolve) => requests.push(resolve)),
+}));
+vi.mock("./availability", () => ({
+ featureBackoff: () => ({ active: () => false, mark: () => {} }),
+ onLspRefresh: () => () => {},
+}));
+
+afterEach(() => {
+ vi.useRealTimers();
+ requests.length = 0;
+});
+
+describe("semantic colours while typing", () => {
+ it("colours a newly completed builtin and parameters before any LSP response", async () => {
+ const support = await loadSupport(findLanguageDescription("Python", "demo.py")!);
+ vi.useFakeTimers();
+ const view = new EditorView({
+ parent: document.body,
+ state: EditorState.create({
+ doc: "def f(x):\n pri",
+ extensions: [support!, lspDoc.of({ doc: 92, language: "python" } as EditorDocContext), semanticExtension()],
+ }),
+ });
+ try {
+ await vi.advanceTimersByTimeAsync(0);
+ expect(view.dom.querySelector(".cm-sem-python-builtin")).toBeNull();
+ view.dispatch({ changes: { from: view.state.doc.length, insert: "nt(x)" } });
+ await vi.advanceTimersByTimeAsync(0);
+ expect(requests).toHaveLength(0);
+ expect(view.dom.querySelector(".cm-sem-python-builtin")?.textContent).toBe("print");
+ expect(view.dom.querySelectorAll(".cm-sem-parameter")).toHaveLength(2);
+ } finally {
+ view.destroy();
+ }
+ });
+
+ it("retains mapped server colours until the delayed response arrives, while updating syntax colours", async () => {
+ const support = await loadSupport(findLanguageDescription("Python", "demo.py")!);
+ vi.useFakeTimers();
+ const view = new EditorView({
+ parent: document.body,
+ state: EditorState.create({
+ doc: "def f(x):\n print(x)",
+ extensions: [support!, lspDoc.of({ doc: 91, language: "python" } as EditorDocContext), semanticExtension()],
+ }),
+ });
+ try {
+ await vi.advanceTimersByTimeAsync(600);
+ requests.shift()!({
+ tokenTypes: ["function", "parameter"],
+ tokenModifiers: ["defaultLibrary"],
+ data: [1, 4, 5, 0, 1, 1, 10, 1, 1, 0],
+ });
+ await vi.advanceTimersByTimeAsync(0);
+ expect(view.dom.querySelector(".cm-sem-python-builtin")?.textContent).toBe("print");
+ expect(view.dom.querySelector(".cm-sem-parameter")?.textContent).toBe("x");
+
+ view.dispatch({ changes: { from: 0, insert: "# comment\n" } });
+ await vi.advanceTimersByTimeAsync(0);
+ expect(view.dom.querySelector(".cm-sem-python-builtin")?.textContent).toBe("print");
+ expect(view.dom.querySelector(".cm-sem-parameter")?.textContent).toBe("x");
+ expect(view.dom.querySelector(".cm-sem-python-definition")?.textContent).toBe("f");
+
+ await vi.advanceTimersByTimeAsync(600);
+ expect(requests).toHaveLength(1);
+ expect(view.dom.querySelector(".cm-sem-python-builtin")?.textContent).toBe("print");
+ requests.shift()!({
+ tokenTypes: ["function", "parameter"],
+ tokenModifiers: ["defaultLibrary"],
+ data: [2, 4, 5, 0, 1, 2, 10, 1, 1, 0],
+ });
+ await vi.advanceTimersByTimeAsync(0);
+ expect(view.dom.querySelector(".cm-sem-python-builtin")?.textContent).toBe("print");
+ } finally {
+ view.destroy();
+ }
+ });
+});
diff --git a/src/editor/lsp/theme.ts b/src/editor/lsp/theme.ts
index 8ea732ac..79350190 100644
--- a/src/editor/lsp/theme.ts
+++ b/src/editor/lsp/theme.ts
@@ -69,6 +69,16 @@ const markdown = {
".tok-invalid": { color: "var(--syn-invalid)" },
};
+function pythonStyle(token: string, fallback: string) {
+ return {
+ color: `var(--syn-${token}, var(--syn-${fallback}))`,
+ fontWeight: `var(--syn-${token}-bold, inherit)`,
+ fontStyle: `var(--syn-${token}-italic, inherit)`,
+ backgroundColor: `var(--syn-${token}-bg, transparent)`,
+ textDecoration: `var(--syn-${token}-effect, none)`,
+ };
+}
+
export const lspTheme = EditorView.theme({
// --- completion (IDEA lookup: icon column, bold blue match, grey tail/type, advertiser) ---
".cm-tooltip.cm-tooltip-autocomplete.cm-lsp-completion": {
@@ -348,15 +358,31 @@ export const lspTheme = EditorView.theme({
".cm-lsp-inlay-pl": { marginLeft: "4px" },
".cm-lsp-inlay-pr": { marginRight: "4px" },
+ ".cm-sem-python-method-call": pythonStyle("python-method-call", "function-call"),
+ ".cm-sem-python-annotation": pythonStyle("python-annotation", "type"),
+ ".cm-sem-python-parentheses": pythonStyle("python-parentheses", "punct"),
+ ".cm-sem-python-brackets": pythonStyle("python-brackets", "punct"),
+ ".cm-sem-python-braces": pythonStyle("python-braces", "punct"),
+ ".cm-sem-python-definition": pythonStyle("function", "function"),
+ ".cm-sem-python-nested": pythonStyle("python-nested-function", "function"),
+ ".cm-sem-python-predefined-definition": pythonStyle("python-predefined-definition", "function"),
+ ".cm-sem-python-predefined-usage": pythonStyle("python-predefined-usage", "self"),
+ ".cm-sem-python-docstring, .cm-sem-python-docstring *": pythonStyle("docstring", "string"),
+ ".cm-sem-python-bytes": pythonStyle("python-bytes", "string"),
+ ".cm-sem-python-escape": pythonStyle("escape", "escape"),
+ ".cm-sem-python-format": pythonStyle("python-format", "escape"),
+ ".cm-sem-python-keyword-arg": pythonStyle("keyword-arg", "param"),
// --- semantic tokens -----------------------------------------------------
".cm-sem-type, .cm-sem-typeParameter": { color: "var(--syn-type)" },
// IDEA colours declarations, calls stay (nearly) plain text.
".cm-sem-function": { color: "var(--syn-function-call, var(--syn-function))" },
".cm-sem-function.cm-sem-declaration": { color: "var(--syn-function)" },
".cm-sem-function.cm-sem-builtin": { color: "var(--syn-builtin, var(--syn-function))" },
+ ".cm-sem-function.cm-sem-python-method-call": pythonStyle("python-method-call", "function-call"),
".cm-sem-macro": { color: "var(--syn-annotation)" },
".cm-sem-annotation": { color: "var(--syn-decorator, var(--syn-annotation))" },
".cm-sem-constant": { color: "var(--syn-constant)" },
+ ".cm-sem-variable": { color: "var(--syn-variable)" },
".cm-sem-property": { color: "var(--syn-property)" },
".cm-sem-parameter": { color: "var(--syn-param, var(--syn-variable))" },
".cm-sem-self": { color: "var(--syn-self, var(--syn-keyword))" },
@@ -364,6 +390,7 @@ export const lspTheme = EditorView.theme({
// Before the builtin rule: library globals (`print`, `string`) keep the predefined colour.
".cm-sem-global:not(.cm-sem-function)": { color: "var(--syn-global, inherit)" },
".cm-sem-builtin:not(.cm-sem-function)": { color: "var(--syn-builtin, inherit)" },
+ ".cm-sem-builtin.cm-sem-python-builtin": pythonStyle("predefined", "builtin"),
".cm-sem-static": { fontStyle: "var(--syn-static-italic, italic)" },
".cm-sem-property.cm-sem-static, .cm-sem-property.cm-sem-static *": {
color: "var(--syn-static-property, var(--syn-property))",
@@ -380,6 +407,13 @@ export const lspTheme = EditorView.theme({
textUnderlineOffset: "3px",
},
".cm-sem-deprecated": { textDecoration: "line-through" },
+ ".cm-sem-python-local-variable": pythonStyle("python-local-variable", "variable"),
+ ".cm-sem-global.cm-sem-python-global-variable": pythonStyle("python-global-variable", "global"),
+ ".cm-sem-python-instance-attribute": pythonStyle("python-instance-attribute", "property"),
+ ".cm-sem-python-class-attribute": pythonStyle("python-class-attribute", "property"),
+ ".cm-sem-python-constant": pythonStyle("python-constant", "constant"),
+ ".cm-sem-python-property": pythonStyle("python-property", "property"),
+ ".cm-sem-python-type-parameter": pythonStyle("python-type-parameter", "type"),
// Semantic highlighting: after the base classes so a parameter's colour gives way, and over the
// syntax highlighter's span inside.
...Object.fromEntries(
diff --git a/src/platform/__fixtures__/colorSchemes/forest.icls b/src/platform/__fixtures__/colorSchemes/forest.icls
index b38a1aae..b1a61ccf 100644
--- a/src/platform/__fixtures__/colorSchemes/forest.icls
+++ b/src/platform/__fixtures__/colorSchemes/forest.icls
@@ -15,7 +15,7 @@
-
+
diff --git a/src/platform/colorScheme.ts b/src/platform/colorScheme.ts
index 4e0dbc5e..ef2e9534 100644
--- a/src/platform/colorScheme.ts
+++ b/src/platform/colorScheme.ts
@@ -135,6 +135,16 @@ const BASE_ATTRIBUTES: readonly AttributeDef[] = [
...[1, 2, 3, 4, 5].map((n) => syn(`semantic${n}`, `Semantic highlighting › Color ${n}`, [LD], `rainbow-${n}`)),
...[1, 2, 3].map((n) => syn(`bracket${n}`, `Rainbow brackets › Level ${n}`, [LD], `bracket-${n}`)),
+ syn("pyMethodCall", "Method call", ["Python"], "python-method-call"),
+ syn("pyAnnotation", "Type annotation", ["Python"], "python-annotation"),
+ syn("pyParentheses", "Parentheses", ["Python"], "python-parentheses"),
+ syn("pyBrackets", "Brackets", ["Python"], "python-brackets"),
+ syn("pyBraces", "Braces", ["Python"], "python-braces"),
+ syn("pyNestedFunction", "Nested function definition", ["Python"], "python-nested-function"),
+ syn("pyBytes", "Byte string", ["Python"], "python-bytes"),
+ syn("pyPredefinedDefinition", "Predefined function definition", ["Python"], "python-predefined-definition"),
+ syn("pyPredefinedUsage", "Predefined name usage", ["Python"], "python-predefined-usage"),
+ syn("pyFormat", "Format specifier", ["Python"], "python-format"),
syn("pySelf", "Self parameter", ["Python"], "self"),
{
id: "pyStaticSymbol",
@@ -148,6 +158,13 @@ const BASE_ATTRIBUTES: readonly AttributeDef[] = [
syn("pyDocstring", "Docstring", ["Python"], "docstring"),
syn("pyBuiltin", "Built-in name", ["Python"], "predefined"),
syn("pyDecorator", "Decorator", ["Python"], "annotation"),
+ syn("pyLocalVariable", "Local variable", ["Python"], "python-local-variable"),
+ syn("pyGlobalVariable", "Global variable", ["Python"], "python-global-variable"),
+ syn("pyInstanceAttribute", "Instance attribute", ["Python"], "python-instance-attribute"),
+ syn("pyClassAttribute", "Class attribute", ["Python"], "python-class-attribute"),
+ syn("pyConstant", "Constant", ["Python"], "python-constant"),
+ syn("pyProperty", "Property", ["Python"], "python-property"),
+ syn("pyTypeParameter", "Type parameter", ["Python"], "python-type-parameter"),
syn("mdHeading", "Heading", ["Markdown"], "markup-heading"),
syn("mdCode", "Code span", ["Markdown"], "markup-code"),
syn("mdLink", "Link", ["Markdown"], "link"),
diff --git a/src/platform/icls.test.ts b/src/platform/icls.test.ts
index 4f72ec1d..018a3003 100644
--- a/src/platform/icls.test.ts
+++ b/src/platform/icls.test.ts
@@ -77,6 +77,51 @@ describe(".icls schemes", () => {
expect(customs).toEqual(before);
});
+ it("imports PyCharm dot keys after shared defaults without inherited font effects", () => {
+ const xml = `
+
+
+
+
+
+
+
+ `;
+ const attrs = parseIcls(xml).scheme.attributes;
+ expect(attrs.keyword).toEqual({ foreground: "#112233", bold: false, italic: false, effect: "none" });
+ expect(attrs.pyNestedFunction?.foreground).toBe("#223344");
+ expect(attrs.pyDocstring?.foreground).toBe("#334455");
+ expect(attrs.pyDocstring?.effect).toBe("none");
+ expect(attrs.pyBytes?.foreground).toBe("#445566");
+ expect(attrs.pyBytes?.bold).toBe(true);
+ expect(attrs.pyBuiltin?.foreground).toBe("#00abcd");
+ const imported = importScheme({}, parseIcls(xml).scheme);
+ const roundTrip = parseIcls(exportIcls(imported.id, imported.customs)).scheme.attributes;
+ expect(roundTrip.pyNestedFunction?.foreground).toBe("#223344");
+ expect(roundTrip.pyBytes?.foreground).toBe("#445566");
+ });
+
+ it("round trips the Python identifier attributes independently", () => {
+ const names = [
+ ["pyLocalVariable", "101010"],
+ ["pyGlobalVariable", "202020"],
+ ["pyInstanceAttribute", "303030"],
+ ["pyClassAttribute", "404040"],
+ ["pyConstant", "505050"],
+ ["pyProperty", "606060"],
+ ["pyTypeParameter", "707070"],
+ ] as const;
+ const xml = `${names
+ .map(
+ ([id, value]) =>
+ ``,
+ )
+ .join("")}`;
+ const { id, customs } = importScheme({}, parseIcls(xml).scheme);
+ const attrs = parseIcls(exportIcls(id, customs)).scheme.attributes;
+ for (const [name, value] of names) expect(attrs[name]?.foreground).toBe(`#${value}`);
+ });
+
it("imports the global variable colour", () => {
const xml =
'