From 1572fc36ea00cf030b30a54e80920cbdd9799aa1 Mon Sep 17 00:00:00 2001 From: l0ng-ai <24760907+l0ng-ai@users.noreply.github.com> Date: Sat, 24 Oct 2026 09:53:40 +0800 Subject: [PATCH] feat(editor): Highlight languages without a CodeMirror mode from tree-sitter Elixir files opened as plain text: highlighting only came from @codemirror/language-data, which has no Elixir mode. Grammars now carry their own highlights.scm; `doc_highlights` folds its captures into a small kind vocabulary (inner nodes and later patterns win, interpolation clears the string colour), memoised per document version on the incremental tree. The editor maps kinds onto Lezer tags, so colours follow the active scheme, and requests them only when no CodeMirror mode exists. Enabled for Elixir; other grammar-only languages need their query wired. Claude-Session: https://claude.ai/code/session_01B65qLEKnkeYTGdbwhLpjCu --- crates/ide-language/src/lib.rs | 5 +- crates/ide-language/src/syntax/cache.rs | 21 +- crates/ide-language/src/syntax/grammars.rs | 38 +++ crates/ide-language/src/syntax/highlight.rs | 296 ++++++++++++++++++++ crates/ide-language/src/syntax/lines.rs | 2 +- crates/ide-language/src/syntax/mod.rs | 24 +- src-tauri/src/commands/doc.rs | 18 +- src-tauri/src/commands/mod.rs | 1 + src/bindings.ts | 16 ++ src/editor/core/backend.ts | 7 + src/editor/core/cm/setup.ts | 2 + src/editor/core/cm/syntaxHighlight.test.ts | 37 +++ src/editor/core/cm/syntaxHighlight.ts | 66 +++++ src/editor/core/model/document.ts | 17 +- 14 files changed, 543 insertions(+), 7 deletions(-) create mode 100644 crates/ide-language/src/syntax/highlight.rs create mode 100644 src/editor/core/cm/syntaxHighlight.test.ts create mode 100644 src/editor/core/cm/syntaxHighlight.ts diff --git a/crates/ide-language/src/lib.rs b/crates/ide-language/src/lib.rs index 67f736cd..97b1b352 100644 --- a/crates/ide-language/src/lib.rs +++ b/crates/ide-language/src/lib.rs @@ -12,7 +12,10 @@ use std::collections::BTreeMap; mod registry; mod syntax; pub use registry::LanguageRegistry; -pub use syntax::{MAX_CACHED_TREES, MAX_PARSE_BYTES, ParsedTree, Syntax, SyntaxCache, SyntaxCacheStats, TreeError}; +pub use syntax::{ + MAX_CACHED_TREES, MAX_PARSE_BYTES, ParsedTree, Syntax, SyntaxCache, SyntaxCacheStats, SyntaxHighlightKind, + SyntaxHighlights, TreeError, +}; /// Parsed form of `languages/.toml`. Backend-only; the UI gets [`LanguageInfo`]. #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/crates/ide-language/src/syntax/cache.rs b/crates/ide-language/src/syntax/cache.rs index fef3cb9b..0969fe7d 100644 --- a/crates/ide-language/src/syntax/cache.rs +++ b/crates/ide-language/src/syntax/cache.rs @@ -5,7 +5,10 @@ //! Structure / folding / Extend Selection on a large file cost an incremental //! parse instead of a full one. -use super::{Grammar, MAX_PARSE_BYTES, ParsedTree, Syntax, TreeError, folding, outline, parse_cancellable, selection}; +use super::{ + Grammar, MAX_PARSE_BYTES, ParsedTree, Syntax, SyntaxHighlights, TreeError, folding, highlight, outline, + parse_cancellable, selection, +}; use crate::FoldRange; use ide_core::{DocId, LanguageId, OutlineItem, Range}; use ide_text::ByteEdit; @@ -59,6 +62,7 @@ struct Memo { key: Option<(u32, LanguageId)>, outline: Option>>, folding: Option>>, + highlights: Option>>, } impl Memo { @@ -211,6 +215,21 @@ impl SyntaxCache { ) } + /// Highlighting from the grammar's own query; empty without one. + pub fn highlights(&self, doc: DocId, lang: &LanguageId, version: u32, text: &str) -> SyntaxHighlights { + SyntaxHighlights::new(self.memoized( + doc, + lang, + version, + |m| &mut m.highlights, + || { + self.with_tree(doc, lang, version, text, |g, tree, text| { + Some(highlight::highlights(g.highlight_query()?, tree, text)) + }) + }, + )) + } + /// `compute()` once per document version; `None` (no grammar / query) is not memoized. fn memoized( &self, diff --git a/crates/ide-language/src/syntax/grammars.rs b/crates/ide-language/src/syntax/grammars.rs index beb969ea..fad2d5c5 100644 --- a/crates/ide-language/src/syntax/grammars.rs +++ b/crates/ide-language/src/syntax/grammars.rs @@ -9,6 +9,8 @@ pub(crate) struct Spec { pub language: LanguageFn, /// `queries//outline.scm`; see `queries/README.md` for the capture vocabulary. pub outline: &'static str, + /// The grammar's `highlights.scm`, for languages without a CodeMirror mode. + pub highlights: Option<&'static str>, /// Node kinds folded when they span several lines (in addition to every /// bracket-delimited node, which is folded generically). pub fold_nodes: &'static [&'static str], @@ -33,6 +35,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "rust", language: tree_sitter_rust::LANGUAGE, outline: outline!("rust"), + highlights: None, fold_nodes: NONE, fold_bodies: NONE, imports: &["use_declaration", "extern_crate_declaration"], @@ -42,6 +45,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "typescript", language: tree_sitter_typescript::LANGUAGE_TYPESCRIPT, outline: outline!("typescript"), + highlights: None, fold_nodes: &["template_string"], fold_bodies: NONE, imports: &["import_statement"], @@ -51,6 +55,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "tsx", language: tree_sitter_typescript::LANGUAGE_TSX, outline: outline!("typescript"), + highlights: None, fold_nodes: &["template_string", "jsx_element"], fold_bodies: NONE, imports: &["import_statement"], @@ -60,6 +65,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "javascript", language: tree_sitter_javascript::LANGUAGE, outline: outline!("javascript"), + highlights: None, fold_nodes: &["template_string", "jsx_element"], fold_bodies: NONE, imports: &["import_statement"], @@ -69,6 +75,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "python", language: tree_sitter_python::LANGUAGE, outline: outline!("python"), + highlights: None, fold_nodes: &["string"], fold_bodies: &["block"], imports: &["import_statement", "import_from_statement", "future_import_statement"], @@ -78,6 +85,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "go", language: tree_sitter_go::LANGUAGE, outline: outline!("go"), + highlights: None, fold_nodes: &["raw_string_literal"], fold_bodies: NONE, imports: &["import_declaration"], @@ -87,6 +95,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "java", language: tree_sitter_java::LANGUAGE, outline: outline!("java"), + highlights: None, fold_nodes: &["text_block"], fold_bodies: NONE, imports: &["import_declaration"], @@ -96,6 +105,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "kotlin", language: tree_sitter_kotlin_ng::LANGUAGE, outline: outline!("kotlin"), + highlights: None, fold_nodes: &["string_literal"], fold_bodies: NONE, imports: &["import"], @@ -105,6 +115,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "scala", language: tree_sitter_scala::LANGUAGE, outline: outline!("scala"), + highlights: None, fold_nodes: &["string"], fold_bodies: &["indented_block", "template_body"], imports: &["import_declaration"], @@ -114,6 +125,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "c", language: tree_sitter_c::LANGUAGE, outline: outline!("c"), + highlights: None, fold_nodes: &["preproc_if", "preproc_ifdef", "preproc_function_def"], fold_bodies: NONE, imports: &["preproc_include"], @@ -123,6 +135,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "cpp", language: tree_sitter_cpp::LANGUAGE, outline: outline!("cpp"), + highlights: None, fold_nodes: &["preproc_if", "preproc_ifdef", "preproc_function_def", "raw_string_literal"], fold_bodies: NONE, imports: &["preproc_include", "using_declaration"], @@ -132,6 +145,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "csharp", language: tree_sitter_c_sharp::LANGUAGE, outline: outline!("csharp"), + highlights: None, fold_nodes: &["preproc_region", "raw_string_literal", "verbatim_string_literal"], fold_bodies: NONE, imports: &["using_directive"], @@ -141,6 +155,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "swift", language: tree_sitter_swift::LANGUAGE, outline: outline!("swift"), + highlights: None, // Control-flow statements hold `{ statements }` without a block node of their own. fold_nodes: &[ "multi_line_string_literal", @@ -160,6 +175,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "ruby", language: tree_sitter_ruby::LANGUAGE, outline: outline!("ruby"), + highlights: None, fold_nodes: &[ "method", "singleton_method", @@ -185,6 +201,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "php", language: tree_sitter_php::LANGUAGE_PHP, outline: outline!("php"), + highlights: None, fold_nodes: &["heredoc", "nowdoc"], fold_bodies: NONE, imports: &["namespace_use_declaration"], @@ -194,6 +211,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "lua", language: tree_sitter_lua::LANGUAGE, outline: outline!("lua"), + highlights: None, fold_nodes: &[ "function_declaration", "function_definition", @@ -212,6 +230,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "dart", language: tree_sitter_dart::LANGUAGE, outline: outline!("dart"), + highlights: None, fold_nodes: NONE, fold_bodies: NONE, imports: &["import_or_export", "library_name", "part_directive", "part_of_directive"], @@ -221,6 +240,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "zig", language: tree_sitter_zig::LANGUAGE, outline: outline!("zig"), + highlights: None, fold_nodes: NONE, fold_bodies: NONE, imports: NONE, @@ -230,6 +250,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "elixir", language: tree_sitter_elixir::LANGUAGE, outline: outline!("elixir"), + highlights: Some(tree_sitter_elixir::HIGHLIGHTS_QUERY), fold_nodes: &["do_block", "anonymous_function", "string", "sigil"], fold_bodies: NONE, imports: NONE, @@ -239,6 +260,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "haskell", language: tree_sitter_haskell::LANGUAGE, outline: outline!("haskell"), + highlights: None, // `bind`: equations without arguments (`main = do …`). fold_nodes: &["function", "bind", "data_type", "newtype", "class", "instance", "signature"], fold_bodies: NONE, @@ -249,6 +271,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "ocaml", language: tree_sitter_ocaml::LANGUAGE_OCAML, outline: outline!("ocaml"), + highlights: None, fold_nodes: &[ "value_definition", "type_definition", @@ -266,6 +289,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "ocaml_interface", language: tree_sitter_ocaml::LANGUAGE_OCAML_INTERFACE, outline: outline!("ocaml_interface"), + highlights: None, fold_nodes: &["module_definition", "module_type_definition", "signature", "type_definition"], fold_bodies: NONE, imports: &["open_module"], @@ -275,6 +299,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "bash", language: tree_sitter_bash::LANGUAGE, outline: outline!("bash"), + highlights: None, fold_nodes: &["do_group", "if_statement", "case_statement", "heredoc_body"], fold_bodies: NONE, imports: NONE, @@ -284,6 +309,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "html", language: tree_sitter_html::LANGUAGE, outline: outline!("html"), + highlights: None, fold_nodes: &["element", "script_element", "style_element"], fold_bodies: NONE, imports: NONE, @@ -293,6 +319,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "xml", language: tree_sitter_xml::LANGUAGE_XML, outline: outline!("xml"), + highlights: None, fold_nodes: &["element", "CDSect"], fold_bodies: NONE, imports: NONE, @@ -302,6 +329,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "css", language: tree_sitter_css::LANGUAGE, outline: outline!("css"), + highlights: None, fold_nodes: NONE, fold_bodies: NONE, imports: &["import_statement"], @@ -311,6 +339,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "json", language: tree_sitter_json::LANGUAGE, outline: outline!("json"), + highlights: None, fold_nodes: NONE, fold_bodies: NONE, imports: NONE, @@ -320,6 +349,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "yaml", language: tree_sitter_yaml::LANGUAGE, outline: outline!("yaml"), + highlights: None, fold_nodes: &["block_mapping_pair", "block_sequence_item", "block_scalar"], fold_bodies: NONE, imports: NONE, @@ -329,6 +359,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "toml", language: tree_sitter_toml_ng::LANGUAGE, outline: outline!("toml"), + highlights: None, fold_nodes: &["table", "table_array_element", "string"], fold_bodies: NONE, imports: NONE, @@ -338,6 +369,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "markdown", language: tree_sitter_md::LANGUAGE, outline: outline!("markdown"), + highlights: None, fold_nodes: &["section", "fenced_code_block", "list", "block_quote", "html_block"], fold_bodies: NONE, imports: NONE, @@ -347,6 +379,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "hcl", language: tree_sitter_hcl::LANGUAGE, outline: outline!("hcl"), + highlights: None, fold_nodes: &["block", "heredoc_template"], fold_bodies: NONE, imports: NONE, @@ -356,6 +389,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "nix", language: tree_sitter_nix::LANGUAGE, outline: outline!("nix"), + highlights: None, fold_nodes: &["let_expression", "indented_string_expression"], fold_bodies: NONE, imports: NONE, @@ -365,6 +399,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "proto", language: tree_sitter_proto::LANGUAGE, outline: outline!("proto"), + highlights: None, fold_nodes: NONE, fold_bodies: NONE, imports: &["import"], @@ -374,6 +409,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "graphql", language: tree_sitter_graphql::LANGUAGE, outline: outline!("graphql"), + highlights: None, fold_nodes: NONE, fold_bodies: NONE, imports: NONE, @@ -383,6 +419,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "make", language: tree_sitter_make::LANGUAGE, outline: outline!("make"), + highlights: None, fold_nodes: &["rule", "conditional", "define_directive"], fold_bodies: NONE, imports: &["include_directive"], @@ -392,6 +429,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "cmake", language: tree_sitter_cmake::LANGUAGE, outline: outline!("cmake"), + highlights: None, fold_nodes: &["function_def", "macro_def", "if_condition", "foreach_loop", "while_loop", "block_def"], fold_bodies: NONE, imports: NONE, diff --git a/crates/ide-language/src/syntax/highlight.rs b/crates/ide-language/src/syntax/highlight.rs new file mode 100644 index 00000000..629c081c --- /dev/null +++ b/crates/ide-language/src/syntax/highlight.rs @@ -0,0 +1,296 @@ +//! Syntax highlighting from a grammar's own `highlights.scm`, for languages the +//! UI has no CodeMirror mode for. +//! +//! Capture names follow the nvim-treesitter / Zed conventions (`@keyword`, +//! `@string.special.symbol`, ...) and are folded into [`SyntaxHighlightKind`] by +//! their longest known dotted prefix. Overlaps resolve the way those editors +//! do: a smaller node wins over the node containing it, and for the same node +//! the later pattern wins. `@embedded` (string interpolation) clears the +//! surrounding colour. + +use super::lines::LineIndex; +use serde::{Deserialize, Serialize}; +use specta::Type; +use tree_sitter::{Language, Query, QueryCursor, StreamingIterator, Tree}; + +/// Highlight classes sent to the UI, which maps each onto a Lezer tag. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub enum SyntaxHighlightKind { + Keyword, + String, + StringSpecial, + Escape, + Regexp, + Atom, + Number, + Constant, + Comment, + DocComment, + Function, + FunctionCall, + Builtin, + Type, + Namespace, + Variable, + SelfVariable, + Property, + Attribute, + Macro, + Label, + Tag, + Operator, + Punctuation, +} + +impl SyntaxHighlightKind { + /// Every kind, in `as u32` order: the legend for [`SyntaxHighlights::data`]. + pub const ALL: [Self; 24] = [ + Self::Keyword, + Self::String, + Self::StringSpecial, + Self::Escape, + Self::Regexp, + Self::Atom, + Self::Number, + Self::Constant, + Self::Comment, + Self::DocComment, + Self::Function, + Self::FunctionCall, + Self::Builtin, + Self::Type, + Self::Namespace, + Self::Variable, + Self::SelfVariable, + Self::Property, + Self::Attribute, + Self::Macro, + Self::Label, + Self::Tag, + Self::Operator, + Self::Punctuation, + ]; +} + +/// Highlighted ranges of a whole document, packed like LSP semantic tokens. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct SyntaxHighlights { + /// Legend: `data` kind `i` is `kinds[i]`. + pub kinds: Vec, + /// `[from, to, kind]` triples: sorted, non-overlapping UTF-16 offsets. + pub data: Vec, +} + +impl SyntaxHighlights { + pub(crate) fn new(data: Vec) -> Self { + Self { kinds: SyntaxHighlightKind::ALL.to_vec(), data } + } +} + +/// What a capture paints over its node. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Paint { + Kind(SyntaxHighlightKind), + /// `@embedded`: code inside a string, uncoloured unless captured itself. + Clear, +} + +/// Capture name prefixes, most specific first within each family. +const CAPTURES: &[(&str, Paint)] = { + use Paint::{Clear, Kind}; + use SyntaxHighlightKind as K; + &[ + ("embedded", Clear), + ("keyword", Kind(K::Keyword)), + ("include", Kind(K::Keyword)), + ("import", Kind(K::Keyword)), + ("conditional", Kind(K::Keyword)), + ("repeat", Kind(K::Keyword)), + ("exception", Kind(K::Keyword)), + ("boolean", Kind(K::Keyword)), + ("string.escape", Kind(K::Escape)), + ("escape", Kind(K::Escape)), + ("string.regex", Kind(K::Regexp)), + ("string.regexp", Kind(K::Regexp)), + ("string.special.symbol", Kind(K::Atom)), + ("symbol", Kind(K::Atom)), + ("string.special", Kind(K::StringSpecial)), + ("string", Kind(K::String)), + ("character", Kind(K::String)), + ("number", Kind(K::Number)), + ("float", Kind(K::Number)), + ("constant.builtin", Kind(K::Constant)), + ("constant.macro", Kind(K::Macro)), + ("constant", Kind(K::Constant)), + // nvim greys out unused bindings (`_acc`); IDEA shows them as variables. + ("comment.unused", Kind(K::Variable)), + ("comment.doc", Kind(K::DocComment)), + ("comment.documentation", Kind(K::DocComment)), + ("comment", Kind(K::Comment)), + ("function.builtin", Kind(K::Builtin)), + ("function.macro", Kind(K::Macro)), + ("function.definition", Kind(K::Function)), + ("function", Kind(K::FunctionCall)), + ("method", Kind(K::FunctionCall)), + ("constructor", Kind(K::Type)), + ("type.builtin", Kind(K::Builtin)), + ("type", Kind(K::Type)), + ("module", Kind(K::Namespace)), + ("namespace", Kind(K::Namespace)), + ("variable.builtin", Kind(K::SelfVariable)), + ("variable.member", Kind(K::Property)), + ("variable.field", Kind(K::Property)), + ("variable", Kind(K::Variable)), + ("parameter", Kind(K::Variable)), + ("property", Kind(K::Property)), + ("field", Kind(K::Property)), + ("attribute", Kind(K::Attribute)), + ("label", Kind(K::Label)), + ("tag", Kind(K::Tag)), + ("operator", Kind(K::Operator)), + ("punctuation", Kind(K::Punctuation)), + ] +}; + +/// The paint for a capture name: its longest prefix (on `.` boundaries) in +/// [`CAPTURES`]. Private captures (`@__name__`) and unknown names paint nothing. +fn paint_for(name: &str) -> Option { + if name.starts_with('_') { + return None; + } + let mut prefix = name; + loop { + if let Some((_, p)) = CAPTURES.iter().find(|(n, _)| *n == prefix) { + return Some(*p); + } + prefix = &prefix[..prefix.rfind('.')?]; + } +} + +pub(crate) struct HighlightQuery { + query: Query, + /// By capture index. + paints: Vec>, +} + +impl HighlightQuery { + pub(crate) fn new(language: &Language, source: &str) -> Result { + let query = Query::new(language, source)?; + let paints = query.capture_names().iter().map(|n| paint_for(n)).collect(); + Ok(Self { query, paints }) + } +} + +/// Byte value for "no colour" in the paint buffer; kinds are `1 + index`. +const NONE: u8 = 0; + +/// `[from, to, kind]` UTF-16 triples for `text`; see [`SyntaxHighlights::data`]. +pub(crate) fn highlights(q: &HighlightQuery, tree: &Tree, text: &str) -> Vec { + // (byte len, pattern, start, end, paint): painted largest first, then in + // pattern order, so inner nodes and later patterns overwrite. + let mut captures: Vec<(usize, usize, usize, usize, Paint)> = Vec::new(); + let mut cursor = QueryCursor::new(); + let mut matches = cursor.matches(&q.query, tree.root_node(), text.as_bytes()); + while let Some(m) = matches.next() { + for c in m.captures { + if let Some(paint) = q.paints[c.index as usize] { + let r = c.node.byte_range(); + if r.start < r.end { + captures.push((r.end - r.start, m.pattern_index, r.start, r.end, paint)); + } + } + } + } + captures.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1))); + + let mut painted = vec![NONE; text.len()]; + for &(_, _, start, end, paint) in &captures { + let v = match paint { + Paint::Kind(k) => 1 + k as u8, + Paint::Clear => NONE, + }; + painted[start..end].fill(v); + } + + let lines = LineIndex::new(text); + let mut data = Vec::new(); + let mut i = 0; + while i < painted.len() { + let v = painted[i]; + let start = i; + while i < painted.len() && painted[i] == v { + i += 1; + } + // Node boundaries are char boundaries; whitespace-only runs stay uncoloured. + if v != NONE && text.is_char_boundary(start) && text.is_char_boundary(i) && !text[start..i].trim().is_empty() { + data.extend([lines.utf16_at(start) as u32, lines.utf16_at(i) as u32, (v - 1) as u32]); + } + } + data +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn capture_names_fold_to_the_longest_known_prefix() { + use SyntaxHighlightKind as K; + assert_eq!(paint_for("keyword.function"), Some(Paint::Kind(K::Keyword))); + assert_eq!(paint_for("string.special.symbol"), Some(Paint::Kind(K::Atom))); + assert_eq!(paint_for("string.special.path"), Some(Paint::Kind(K::StringSpecial))); + assert_eq!(paint_for("comment.doc.__attribute__"), Some(Paint::Kind(K::DocComment))); + assert_eq!(paint_for("punctuation.bracket"), Some(Paint::Kind(K::Punctuation))); + assert_eq!(paint_for("embedded"), Some(Paint::Clear)); + assert_eq!(paint_for("__name__"), None); + assert_eq!(paint_for("spell"), None); + } + + /// `(text, kind)` of every highlighted range. + fn spans(lang: &str, text: &str) -> Vec<(String, SyntaxHighlightKind)> { + let h = crate::Syntax::new().highlights(&ide_core::LanguageId::new(lang), text).unwrap(); + let units: Vec = text.encode_utf16().collect(); + h.data + .chunks(3) + .map(|t| (String::from_utf16(&units[t[0] as usize..t[1] as usize]).unwrap(), h.kinds[t[2] as usize])) + .collect() + } + + #[test] + fn highlights_elixir() { + use SyntaxHighlightKind as K; + let src = "defmodule Greeter do\n @moduledoc \"Hi\"\n # 😀 note\n def hello(name, _opts) do\n IO.puts(\"Hello #{name}\")\n :ok\n end\nend\n"; + let got = spans("elixir", src); + let kind = |t: &str| got.iter().find(|(s, _)| s == t).map(|(_, k)| *k); + assert_eq!(kind("defmodule"), Some(K::Keyword)); + assert_eq!(kind("Greeter"), Some(K::Namespace)); + assert_eq!(kind("\"Hi\""), Some(K::DocComment)); + assert_eq!(kind("# 😀 note"), Some(K::Comment)); + assert_eq!(kind("def"), Some(K::Keyword)); + assert_eq!(kind("hello"), Some(K::FunctionCall)); + assert_eq!(kind("_opts"), Some(K::Variable)); + assert_eq!(kind("puts"), Some(K::FunctionCall)); + // Interpolation clears the string colour; its contents are coloured on their own. + assert_eq!(kind("\"Hello "), Some(K::String)); + assert_eq!(kind("#{"), Some(K::Punctuation)); + assert_eq!(kind(":ok"), Some(K::Atom)); + assert_eq!(kind("end"), Some(K::Keyword)); + // Sorted and non-overlapping. + let h = crate::Syntax::new().highlights(&ide_core::LanguageId::new("elixir"), src).unwrap(); + assert!(h.data.chunks(3).zip(h.data.chunks(3).skip(1)).all(|(a, b)| a[0] < a[1] && a[1] <= b[0])); + } + + #[test] + fn languages_without_a_query_have_none() { + assert_eq!(crate::Syntax::new().highlights(&ide_core::LanguageId::new("rust"), "fn a() {}"), None); + } + + #[test] + fn legend_matches_discriminants() { + for (i, k) in SyntaxHighlightKind::ALL.iter().enumerate() { + assert_eq!(*k as usize, i); + } + } +} diff --git a/crates/ide-language/src/syntax/lines.rs b/crates/ide-language/src/syntax/lines.rs index 7b3291d2..5b8de206 100644 --- a/crates/ide-language/src/syntax/lines.rs +++ b/crates/ide-language/src/syntax/lines.rs @@ -40,7 +40,7 @@ impl<'a> LineIndex<'a> { } /// UTF-16 offset of a char-boundary byte offset. - fn utf16_at(&self, byte: usize) -> usize { + pub(crate) fn utf16_at(&self, byte: usize) -> usize { if self.checkpoints.is_empty() { return byte; } diff --git a/crates/ide-language/src/syntax/mod.rs b/crates/ide-language/src/syntax/mod.rs index e6633bb2..783dfe9c 100644 --- a/crates/ide-language/src/syntax/mod.rs +++ b/crates/ide-language/src/syntax/mod.rs @@ -1,8 +1,9 @@ -//! Tree-sitter driven structure: outline, folding, symbol tags, selection ranges. +//! Tree-sitter driven structure: outline, folding, symbol tags, selection ranges, highlighting. mod cache; mod folding; mod grammars; +mod highlight; pub(crate) mod lines; mod outline; mod selection; @@ -10,6 +11,7 @@ mod selection; use crate::{FoldRange, LanguageConfig, LanguageRegistry, SymbolTag}; pub use cache::{MAX_CACHED_TREES, SyntaxCache, SyntaxCacheStats}; use grammars::{SPECS, Spec}; +pub use highlight::{SyntaxHighlightKind, SyntaxHighlights}; use ide_core::{LanguageId, OutlineItem, Range}; use std::cell::RefCell; use std::collections::HashMap; @@ -31,6 +33,7 @@ pub(crate) struct Grammar { pub(crate) spec: &'static Spec, pub(crate) language: Language, outline: OnceLock>, + highlights: OnceLock>, kinds: OnceLock, } @@ -50,6 +53,17 @@ impl Grammar { }) .as_ref() } + + fn highlight_query(&self) -> Option<&highlight::HighlightQuery> { + self.highlights + .get_or_init(|| { + let source = self.spec.highlights?; + highlight::HighlightQuery::new(&self.language, source) + .inspect_err(|e| tracing::error!("highlights query for {} is invalid: {e}", self.spec.name)) + .ok() + }) + .as_ref() + } } thread_local! { @@ -72,6 +86,7 @@ impl Syntax { spec, language: Language::new(spec.language), outline: OnceLock::new(), + highlights: OnceLock::new(), kinds: OnceLock::new(), }) .collect(); @@ -150,6 +165,13 @@ impl Syntax { .unwrap_or_default() } + /// Highlighting from the grammar's own query; `None` without one. + pub fn highlights(&self, lang: &LanguageId, text: &str) -> Option { + self.analyze(lang, text, |g, tree, text| { + Some(SyntaxHighlights::new(highlight::highlights(g.highlight_query()?, tree, text))) + }) + } + /// Syntax-aware selection ranges for IDEA "Extend Selection" (⌥↑), innermost first. pub fn selection_ranges(&self, lang: &LanguageId, text: &str, offset_utf16: u32) -> Vec { match self.analyze(lang, text, |_, tree, text| Some(selection::selection_ranges(tree, text, offset_utf16))) { diff --git a/src-tauri/src/commands/doc.rs b/src-tauri/src/commands/doc.rs index 59c4f708..719dcbca 100644 --- a/src-tauri/src/commands/doc.rs +++ b/src-tauri/src/commands/doc.rs @@ -1,6 +1,6 @@ use crate::state::AppState; use ide_core::{ChangeSet, DocId, IdeError, IdeResult, OutlineItem, Range}; -use ide_language::FoldRange; +use ide_language::{FoldRange, SyntaxHighlights}; use ide_text::LineEnding; use ide_workspace::{DocumentInfo, HistoryEntry}; use std::path::PathBuf; @@ -136,6 +136,22 @@ pub async fn doc_folding(window: Window, state: State<'_, AppState>, doc: DocId) .await } +/// Tree-sitter highlighting for languages the UI has no CodeMirror mode for; +/// empty when the grammar has no highlights query or the document is large. +#[tauri::command] +#[specta::specta] +pub async fn doc_highlights(window: Window, state: State<'_, AppState>, doc: DocId) -> IdeResult { + let project = state.project(window.label())?; + blocking(move || { + let snap = project.documents.text_snapshot(doc)?; + if snap.large { + return Ok(SyntaxHighlights::default()); + } + Ok(project.syntax.highlights(doc, &snap.language, snap.version, &snap.text)) + }) + .await +} + /// Extend Selection (⌥↑): syntax ranges around `offset`, innermost first. /// Large documents fall back to word / line / document ranges. #[tauri::command] diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 4c4f60fa..dc0d79f6 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -74,6 +74,7 @@ pub fn builder() -> tauri_specta::Builder { doc::doc_set_line_ending, doc::doc_outline, doc::doc_folding, + doc::doc_highlights, doc::doc_selection_ranges, syntax::syntax_support, syntax::syntax_node_at, diff --git a/src/bindings.ts b/src/bindings.ts index 2afc2305..22f7dc89 100644 --- a/src/bindings.ts +++ b/src/bindings.ts @@ -92,6 +92,11 @@ export const commands = { */ docOutline: (doc: DocId) => typedError(__TAURI_INVOKE("doc_outline", { doc })), docFolding: (doc: DocId) => typedError(__TAURI_INVOKE("doc_folding", { doc })), + /** + * Tree-sitter highlighting for languages the UI has no CodeMirror mode for; + * empty when the grammar has no highlights query or the document is large. + */ + docHighlights: (doc: DocId) => typedError(__TAURI_INVOKE("doc_highlights", { doc })), /** * Extend Selection (⌥↑): syntax ranges around `offset`, innermost first. * Large documents fall back to word / line / document ranges. @@ -3683,6 +3688,17 @@ export type SyntaxErrors = { truncated: boolean, }; +/** Highlight classes sent to the UI, which maps each onto a Lezer tag. */ +export type SyntaxHighlightKind = "keyword" | "string" | "stringSpecial" | "escape" | "regexp" | "atom" | "number" | "constant" | "comment" | "docComment" | "function" | "functionCall" | "builtin" | "type" | "namespace" | "variable" | "selfVariable" | "property" | "attribute" | "macro" | "label" | "tag" | "operator" | "punctuation"; + +/** Highlighted ranges of a whole document, packed like LSP semantic tokens. */ +export type SyntaxHighlights = { + /** Legend: `data` kind `i` is `kinds[i]`. */ + kinds: SyntaxHighlightKind[], + /** `[from, to, kind]` triples: sorted, non-overlapping UTF-16 offsets. */ + data: number[], +}; + /** One tree-sitter node. */ export type SyntaxNode = { /** Grammar node kind, e.g. `method_invocation`, `identifier`, `;`. */ diff --git a/src/editor/core/backend.ts b/src/editor/core/backend.ts index 75de63a2..e075ec6e 100644 --- a/src/editor/core/backend.ts +++ b/src/editor/core/backend.ts @@ -18,6 +18,7 @@ import { type LineEnding, type OutlineItem, type Range, + type SyntaxHighlights, type TextChange, } from "@/platform/ipc"; @@ -43,6 +44,8 @@ export interface DocBackend { setEncoding(doc: DocId, encoding: string, reload: boolean): Promise; outline(doc: DocId): Promise; folding(doc: DocId): Promise; + /** Tree-sitter highlighting, for languages without a CodeMirror mode. */ + highlights(doc: DocId): Promise; selectionRanges(doc: DocId, offset: number): Promise; languages(): Promise; onPatched(handler: (e: DocPatchedEvent) => void): () => void; @@ -62,6 +65,7 @@ export const ipcBackend: DocBackend = { setEncoding: (doc, encoding, reload) => call(commands.docSetEncoding(doc, encoding, reload)), outline: (doc) => call(commands.docOutline(doc)), folding: (doc) => call(commands.docFolding(doc)), + highlights: (doc) => call(commands.docHighlights(doc)), selectionRanges: (doc, offset) => call(commands.docSelectionRanges(doc, offset)), languages: () => commands.languages(), onPatched: (h) => onIdeEvent("docPatched", (e) => h(e)), @@ -343,6 +347,9 @@ export function createMemoryBackend(files: Record = {}): MemoryB } return items; }, + async highlights() { + return { kinds: [], data: [] }; + }, async folding(doc) { const d = get(doc); const out: FoldRange[] = []; diff --git a/src/editor/core/cm/setup.ts b/src/editor/core/cm/setup.ts index 62cb91df..e5618dde 100644 --- a/src/editor/core/cm/setup.ts +++ b/src/editor/core/cm/setup.ts @@ -25,6 +25,7 @@ import { openideaTheme } from "./theme"; import { caretRowHighlight } from "./caretRow"; import { changeLane, overviewLane } from "./lanes"; import { foldingExtension } from "./folding"; +import { syntaxHighlightField } from "./syntaxHighlight"; import { indentGuides } from "./indentGuides"; import { stickyMaxLines, stickyScroll } from "./stickyScroll"; import { findExtension } from "./find"; @@ -267,6 +268,7 @@ export function baseExtensions(o: SetupOptions): Extension { clipboardHistoryExtension, indentOnInput(), syntaxHighlighting(openideaHighlightStyle), + syntaxHighlightField, closeBrackets(), caretRowHighlight, fontZoom, diff --git a/src/editor/core/cm/syntaxHighlight.test.ts b/src/editor/core/cm/syntaxHighlight.test.ts new file mode 100644 index 00000000..a3a1ecbb --- /dev/null +++ b/src/editor/core/cm/syntaxHighlight.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { EditorState } from "@codemirror/state"; +import { highlightingFor, syntaxHighlighting } from "@codemirror/language"; +import { tags as t } from "@lezer/highlight"; +import { openideaHighlightStyle } from "./highlight"; +import { setSyntaxHighlights, syntaxHighlightField } from "./syntaxHighlight"; + +function marks(state: EditorState) { + const out: { text: string; cls: string }[] = []; + state.field(syntaxHighlightField).between(0, state.doc.length, (from, to, d) => { + out.push({ text: state.doc.sliceString(from, to), cls: d.spec.class as string }); + }); + return out; +} + +describe("syntaxHighlightField", () => { + const create = () => + EditorState.create({ doc: "def hi, do: :ok", extensions: [syntaxHighlighting(openideaHighlightStyle), syntaxHighlightField] }); + + it("styles backend ranges with the active highlight style and maps them through edits", () => { + let state = create(); + state = state.update({ + effects: setSyntaxHighlights.of({ kinds: ["keyword", "atom", "functionCall"], data: [0, 3, 0, 4, 6, 2, 12, 15, 1, 99, 120, 0] }), + }).state; + const keyword = highlightingFor(state, [t.keyword]); + const atom = highlightingFor(state, [t.atom]); + expect(keyword).toBeTruthy(); + expect(marks(state)).toEqual([ + { text: "def", cls: keyword }, + { text: "hi", cls: highlightingFor(state, [t.function(t.variableName)]) }, + { text: ":ok", cls: atom }, + ]); + + state = state.update({ changes: { from: 0, insert: " " } }).state; + expect(marks(state).map((m) => m.text)).toEqual(["def", "hi", ":ok"]); + }); +}); diff --git a/src/editor/core/cm/syntaxHighlight.ts b/src/editor/core/cm/syntaxHighlight.ts new file mode 100644 index 00000000..8ced4899 --- /dev/null +++ b/src/editor/core/cm/syntaxHighlight.ts @@ -0,0 +1,66 @@ +/** + * Tree-sitter highlighting (`doc_highlights`) for languages without a + * CodeMirror mode (Elixir, ...): backend kinds → Lezer tags, so the colours + * come from the same highlight style and Color Scheme settings as Lezer + * highlighting. Refreshed with folding; in between, marks are mapped through + * the changes. + */ +import { StateEffect, StateField, type EditorState } from "@codemirror/state"; +import { Decoration, EditorView, type DecorationSet } from "@codemirror/view"; +import { highlightingFor } from "@codemirror/language"; +import { tags as t, type Tag } from "@lezer/highlight"; +import type { SyntaxHighlightKind, SyntaxHighlights } from "@/platform/ipc"; + +const TAGS: Record = { + keyword: t.keyword, + string: t.string, + stringSpecial: t.special(t.string), + escape: t.escape, + regexp: t.regexp, + atom: t.atom, + number: t.number, + constant: t.constant(t.variableName), + comment: t.comment, + docComment: t.docComment, + function: t.function(t.definition(t.variableName)), + functionCall: t.function(t.variableName), + builtin: t.standard(t.variableName), + type: t.typeName, + namespace: t.namespace, + variable: t.variableName, + selfVariable: t.self, + property: t.propertyName, + attribute: t.annotation, + macro: t.macroName, + label: t.labelName, + tag: t.tagName, + operator: t.operator, + punctuation: t.punctuation, +}; + +export const setSyntaxHighlights = StateEffect.define(); + +function decorations(state: EditorState, h: SyntaxHighlights): DecorationSet { + const marks = h.kinds.map((k) => { + const cls = highlightingFor(state, [TAGS[k]]); + return cls ? Decoration.mark({ class: cls }) : null; + }); + const len = state.doc.length; + const ranges = []; + for (let i = 0; i + 2 < h.data.length; i += 3) { + const from = h.data[i]!; + const to = Math.min(h.data[i + 1]!, len); + const mark = marks[h.data[i + 2]!]; + if (mark && from < to) ranges.push(mark.range(from, to)); + } + return Decoration.set(ranges, true); +} + +export const syntaxHighlightField = StateField.define({ + create: () => Decoration.none, + update(deco, tr) { + for (const e of tr.effects) if (e.is(setSyntaxHighlights)) return decorations(tr.state, e.value); + return tr.docChanged ? deco.map(tr.changes) : deco; + }, + provide: (f) => EditorView.decorations.from(f), +}); diff --git a/src/editor/core/model/document.ts b/src/editor/core/model/document.ts index 799e6e6c..981289c9 100644 --- a/src/editor/core/model/document.ts +++ b/src/editor/core/model/document.ts @@ -7,7 +7,7 @@ import { EditorView } from "@codemirror/view"; import type { LanguageDescription, LanguageSupport } from "@codemirror/language"; import { create } from "zustand"; import type { EditorConfigProps } from "@/bindings"; -import type { DocId, DocumentInfo, EditorSettings, FileSettings, LineEnding, OutlineItem, Settings } from "@/platform/ipc"; +import type { DocId, DocumentInfo, EditorSettings, FileSettings, LineEnding, OutlineItem, Settings, SyntaxHighlights } from "@/platform/ipc"; import { _emitEditorEvent, _saveParticipants, @@ -22,6 +22,7 @@ import type { DocBackend, LanguageInfo } from "../backend"; import { findLanguageDescription, languageInfo, loadLanguageInfos, loadSupport, loadedSupport, commentTokens } from "../cm/languages"; import { foldEffect } from "@codemirror/language"; import { backendFolds, setBackendFolds } from "../cm/folding"; +import { setSyntaxHighlights } from "../cm/syntaxHighlight"; import { caretFor, compartments, @@ -265,6 +266,7 @@ export class DocModel { addSlot(slot: EditorSlot) { this.slots.add(slot); + if (this.highlights?.counter === this.changeCounter) slot.reconfigure([setSyntaxHighlights.of(this.highlights.value)]); } removeSlot(slot: EditorSlot) { @@ -332,6 +334,8 @@ export class DocModel { indentOverride: { tabSize: number; insertSpaces: boolean } | null = null; /** Indentation detected in the file's text at open time. */ detectedIndent: { insertSpaces: boolean; size?: number } | null = null; + /** Last tree-sitter highlighting, for slots opened before the next refresh. */ + private highlights: { counter: number; value: SyntaxHighlights } | null = null; /** Fold-by-default has been applied (first folding ranges of the document). */ private initialFolds = false; /** Signature of settings that require contributions to be rebuilt. */ @@ -423,7 +427,12 @@ export class DocModel { await this.sync.whenIdle(); const counter = this.changeCounter; const backend = this.manager.backend; - const [folds, outline] = await Promise.allSettled([backend.folding(this.doc), backend.outline(this.doc)]); + const [folds, outline, highlights] = await Promise.allSettled([ + backend.folding(this.doc), + backend.outline(this.doc), + // Without a CodeMirror mode, colours come from the grammar's tree-sitter query. + this.desc ? null : backend.highlights(this.doc), + ]); if (this.closed) return; if (counter !== this.changeCounter) { this.scheduleRefresh(); @@ -436,6 +445,10 @@ export class DocModel { this.foldByDefault(); } } + if (highlights.status === "fulfilled" && highlights.value) { + this.highlights = { counter, value: highlights.value }; + for (const s of this.slots) s.reconfigure([setSyntaxHighlights.of(highlights.value)]); + } if (outline.status === "fulfilled") { useOutline.setState({ [this.path]: outline.value }); const lines = methodLines(outline.value);