diff --git a/crates/ide-language/src/lib.rs b/crates/ide-language/src/lib.rs index 67f736cd..97b1b352 100644 --- a/crates/ide-language/src/lib.rs +++ b/crates/ide-language/src/lib.rs @@ -12,7 +12,10 @@ use std::collections::BTreeMap; mod registry; mod syntax; pub use registry::LanguageRegistry; -pub use syntax::{MAX_CACHED_TREES, MAX_PARSE_BYTES, ParsedTree, Syntax, SyntaxCache, SyntaxCacheStats, TreeError}; +pub use syntax::{ + MAX_CACHED_TREES, MAX_PARSE_BYTES, ParsedTree, Syntax, SyntaxCache, SyntaxCacheStats, SyntaxHighlightKind, + SyntaxHighlights, TreeError, +}; /// Parsed form of `languages/.toml`. Backend-only; the UI gets [`LanguageInfo`]. #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/crates/ide-language/src/syntax/cache.rs b/crates/ide-language/src/syntax/cache.rs index fef3cb9b..0969fe7d 100644 --- a/crates/ide-language/src/syntax/cache.rs +++ b/crates/ide-language/src/syntax/cache.rs @@ -5,7 +5,10 @@ //! Structure / folding / Extend Selection on a large file cost an incremental //! parse instead of a full one. -use super::{Grammar, MAX_PARSE_BYTES, ParsedTree, Syntax, TreeError, folding, outline, parse_cancellable, selection}; +use super::{ + Grammar, MAX_PARSE_BYTES, ParsedTree, Syntax, SyntaxHighlights, TreeError, folding, highlight, outline, + parse_cancellable, selection, +}; use crate::FoldRange; use ide_core::{DocId, LanguageId, OutlineItem, Range}; use ide_text::ByteEdit; @@ -59,6 +62,7 @@ struct Memo { key: Option<(u32, LanguageId)>, outline: Option>>, folding: Option>>, + highlights: Option>>, } impl Memo { @@ -211,6 +215,21 @@ impl SyntaxCache { ) } + /// Highlighting from the grammar's own query; empty without one. + pub fn highlights(&self, doc: DocId, lang: &LanguageId, version: u32, text: &str) -> SyntaxHighlights { + SyntaxHighlights::new(self.memoized( + doc, + lang, + version, + |m| &mut m.highlights, + || { + self.with_tree(doc, lang, version, text, |g, tree, text| { + Some(highlight::highlights(g.highlight_query()?, tree, text)) + }) + }, + )) + } + /// `compute()` once per document version; `None` (no grammar / query) is not memoized. fn memoized( &self, diff --git a/crates/ide-language/src/syntax/grammars.rs b/crates/ide-language/src/syntax/grammars.rs index beb969ea..fad2d5c5 100644 --- a/crates/ide-language/src/syntax/grammars.rs +++ b/crates/ide-language/src/syntax/grammars.rs @@ -9,6 +9,8 @@ pub(crate) struct Spec { pub language: LanguageFn, /// `queries//outline.scm`; see `queries/README.md` for the capture vocabulary. pub outline: &'static str, + /// The grammar's `highlights.scm`, for languages without a CodeMirror mode. + pub highlights: Option<&'static str>, /// Node kinds folded when they span several lines (in addition to every /// bracket-delimited node, which is folded generically). pub fold_nodes: &'static [&'static str], @@ -33,6 +35,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "rust", language: tree_sitter_rust::LANGUAGE, outline: outline!("rust"), + highlights: None, fold_nodes: NONE, fold_bodies: NONE, imports: &["use_declaration", "extern_crate_declaration"], @@ -42,6 +45,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "typescript", language: tree_sitter_typescript::LANGUAGE_TYPESCRIPT, outline: outline!("typescript"), + highlights: None, fold_nodes: &["template_string"], fold_bodies: NONE, imports: &["import_statement"], @@ -51,6 +55,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "tsx", language: tree_sitter_typescript::LANGUAGE_TSX, outline: outline!("typescript"), + highlights: None, fold_nodes: &["template_string", "jsx_element"], fold_bodies: NONE, imports: &["import_statement"], @@ -60,6 +65,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "javascript", language: tree_sitter_javascript::LANGUAGE, outline: outline!("javascript"), + highlights: None, fold_nodes: &["template_string", "jsx_element"], fold_bodies: NONE, imports: &["import_statement"], @@ -69,6 +75,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "python", language: tree_sitter_python::LANGUAGE, outline: outline!("python"), + highlights: None, fold_nodes: &["string"], fold_bodies: &["block"], imports: &["import_statement", "import_from_statement", "future_import_statement"], @@ -78,6 +85,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "go", language: tree_sitter_go::LANGUAGE, outline: outline!("go"), + highlights: None, fold_nodes: &["raw_string_literal"], fold_bodies: NONE, imports: &["import_declaration"], @@ -87,6 +95,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "java", language: tree_sitter_java::LANGUAGE, outline: outline!("java"), + highlights: None, fold_nodes: &["text_block"], fold_bodies: NONE, imports: &["import_declaration"], @@ -96,6 +105,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "kotlin", language: tree_sitter_kotlin_ng::LANGUAGE, outline: outline!("kotlin"), + highlights: None, fold_nodes: &["string_literal"], fold_bodies: NONE, imports: &["import"], @@ -105,6 +115,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "scala", language: tree_sitter_scala::LANGUAGE, outline: outline!("scala"), + highlights: None, fold_nodes: &["string"], fold_bodies: &["indented_block", "template_body"], imports: &["import_declaration"], @@ -114,6 +125,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "c", language: tree_sitter_c::LANGUAGE, outline: outline!("c"), + highlights: None, fold_nodes: &["preproc_if", "preproc_ifdef", "preproc_function_def"], fold_bodies: NONE, imports: &["preproc_include"], @@ -123,6 +135,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "cpp", language: tree_sitter_cpp::LANGUAGE, outline: outline!("cpp"), + highlights: None, fold_nodes: &["preproc_if", "preproc_ifdef", "preproc_function_def", "raw_string_literal"], fold_bodies: NONE, imports: &["preproc_include", "using_declaration"], @@ -132,6 +145,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "csharp", language: tree_sitter_c_sharp::LANGUAGE, outline: outline!("csharp"), + highlights: None, fold_nodes: &["preproc_region", "raw_string_literal", "verbatim_string_literal"], fold_bodies: NONE, imports: &["using_directive"], @@ -141,6 +155,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "swift", language: tree_sitter_swift::LANGUAGE, outline: outline!("swift"), + highlights: None, // Control-flow statements hold `{ statements }` without a block node of their own. fold_nodes: &[ "multi_line_string_literal", @@ -160,6 +175,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "ruby", language: tree_sitter_ruby::LANGUAGE, outline: outline!("ruby"), + highlights: None, fold_nodes: &[ "method", "singleton_method", @@ -185,6 +201,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "php", language: tree_sitter_php::LANGUAGE_PHP, outline: outline!("php"), + highlights: None, fold_nodes: &["heredoc", "nowdoc"], fold_bodies: NONE, imports: &["namespace_use_declaration"], @@ -194,6 +211,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "lua", language: tree_sitter_lua::LANGUAGE, outline: outline!("lua"), + highlights: None, fold_nodes: &[ "function_declaration", "function_definition", @@ -212,6 +230,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "dart", language: tree_sitter_dart::LANGUAGE, outline: outline!("dart"), + highlights: None, fold_nodes: NONE, fold_bodies: NONE, imports: &["import_or_export", "library_name", "part_directive", "part_of_directive"], @@ -221,6 +240,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "zig", language: tree_sitter_zig::LANGUAGE, outline: outline!("zig"), + highlights: None, fold_nodes: NONE, fold_bodies: NONE, imports: NONE, @@ -230,6 +250,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "elixir", language: tree_sitter_elixir::LANGUAGE, outline: outline!("elixir"), + highlights: Some(tree_sitter_elixir::HIGHLIGHTS_QUERY), fold_nodes: &["do_block", "anonymous_function", "string", "sigil"], fold_bodies: NONE, imports: NONE, @@ -239,6 +260,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "haskell", language: tree_sitter_haskell::LANGUAGE, outline: outline!("haskell"), + highlights: None, // `bind`: equations without arguments (`main = do …`). fold_nodes: &["function", "bind", "data_type", "newtype", "class", "instance", "signature"], fold_bodies: NONE, @@ -249,6 +271,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "ocaml", language: tree_sitter_ocaml::LANGUAGE_OCAML, outline: outline!("ocaml"), + highlights: None, fold_nodes: &[ "value_definition", "type_definition", @@ -266,6 +289,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "ocaml_interface", language: tree_sitter_ocaml::LANGUAGE_OCAML_INTERFACE, outline: outline!("ocaml_interface"), + highlights: None, fold_nodes: &["module_definition", "module_type_definition", "signature", "type_definition"], fold_bodies: NONE, imports: &["open_module"], @@ -275,6 +299,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "bash", language: tree_sitter_bash::LANGUAGE, outline: outline!("bash"), + highlights: None, fold_nodes: &["do_group", "if_statement", "case_statement", "heredoc_body"], fold_bodies: NONE, imports: NONE, @@ -284,6 +309,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "html", language: tree_sitter_html::LANGUAGE, outline: outline!("html"), + highlights: None, fold_nodes: &["element", "script_element", "style_element"], fold_bodies: NONE, imports: NONE, @@ -293,6 +319,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "xml", language: tree_sitter_xml::LANGUAGE_XML, outline: outline!("xml"), + highlights: None, fold_nodes: &["element", "CDSect"], fold_bodies: NONE, imports: NONE, @@ -302,6 +329,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "css", language: tree_sitter_css::LANGUAGE, outline: outline!("css"), + highlights: None, fold_nodes: NONE, fold_bodies: NONE, imports: &["import_statement"], @@ -311,6 +339,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "json", language: tree_sitter_json::LANGUAGE, outline: outline!("json"), + highlights: None, fold_nodes: NONE, fold_bodies: NONE, imports: NONE, @@ -320,6 +349,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "yaml", language: tree_sitter_yaml::LANGUAGE, outline: outline!("yaml"), + highlights: None, fold_nodes: &["block_mapping_pair", "block_sequence_item", "block_scalar"], fold_bodies: NONE, imports: NONE, @@ -329,6 +359,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "toml", language: tree_sitter_toml_ng::LANGUAGE, outline: outline!("toml"), + highlights: None, fold_nodes: &["table", "table_array_element", "string"], fold_bodies: NONE, imports: NONE, @@ -338,6 +369,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "markdown", language: tree_sitter_md::LANGUAGE, outline: outline!("markdown"), + highlights: None, fold_nodes: &["section", "fenced_code_block", "list", "block_quote", "html_block"], fold_bodies: NONE, imports: NONE, @@ -347,6 +379,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "hcl", language: tree_sitter_hcl::LANGUAGE, outline: outline!("hcl"), + highlights: None, fold_nodes: &["block", "heredoc_template"], fold_bodies: NONE, imports: NONE, @@ -356,6 +389,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "nix", language: tree_sitter_nix::LANGUAGE, outline: outline!("nix"), + highlights: None, fold_nodes: &["let_expression", "indented_string_expression"], fold_bodies: NONE, imports: NONE, @@ -365,6 +399,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "proto", language: tree_sitter_proto::LANGUAGE, outline: outline!("proto"), + highlights: None, fold_nodes: NONE, fold_bodies: NONE, imports: &["import"], @@ -374,6 +409,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "graphql", language: tree_sitter_graphql::LANGUAGE, outline: outline!("graphql"), + highlights: None, fold_nodes: NONE, fold_bodies: NONE, imports: NONE, @@ -383,6 +419,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "make", language: tree_sitter_make::LANGUAGE, outline: outline!("make"), + highlights: None, fold_nodes: &["rule", "conditional", "define_directive"], fold_bodies: NONE, imports: &["include_directive"], @@ -392,6 +429,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "cmake", language: tree_sitter_cmake::LANGUAGE, outline: outline!("cmake"), + highlights: None, fold_nodes: &["function_def", "macro_def", "if_condition", "foreach_loop", "while_loop", "block_def"], fold_bodies: NONE, imports: NONE, diff --git a/crates/ide-language/src/syntax/highlight.rs b/crates/ide-language/src/syntax/highlight.rs new file mode 100644 index 00000000..629c081c --- /dev/null +++ b/crates/ide-language/src/syntax/highlight.rs @@ -0,0 +1,296 @@ +//! Syntax highlighting from a grammar's own `highlights.scm`, for languages the +//! UI has no CodeMirror mode for. +//! +//! Capture names follow the nvim-treesitter / Zed conventions (`@keyword`, +//! `@string.special.symbol`, ...) and are folded into [`SyntaxHighlightKind`] by +//! their longest known dotted prefix. Overlaps resolve the way those editors +//! do: a smaller node wins over the node containing it, and for the same node +//! the later pattern wins. `@embedded` (string interpolation) clears the +//! surrounding colour. + +use super::lines::LineIndex; +use serde::{Deserialize, Serialize}; +use specta::Type; +use tree_sitter::{Language, Query, QueryCursor, StreamingIterator, Tree}; + +/// Highlight classes sent to the UI, which maps each onto a Lezer tag. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub enum SyntaxHighlightKind { + Keyword, + String, + StringSpecial, + Escape, + Regexp, + Atom, + Number, + Constant, + Comment, + DocComment, + Function, + FunctionCall, + Builtin, + Type, + Namespace, + Variable, + SelfVariable, + Property, + Attribute, + Macro, + Label, + Tag, + Operator, + Punctuation, +} + +impl SyntaxHighlightKind { + /// Every kind, in `as u32` order: the legend for [`SyntaxHighlights::data`]. + pub const ALL: [Self; 24] = [ + Self::Keyword, + Self::String, + Self::StringSpecial, + Self::Escape, + Self::Regexp, + Self::Atom, + Self::Number, + Self::Constant, + Self::Comment, + Self::DocComment, + Self::Function, + Self::FunctionCall, + Self::Builtin, + Self::Type, + Self::Namespace, + Self::Variable, + Self::SelfVariable, + Self::Property, + Self::Attribute, + Self::Macro, + Self::Label, + Self::Tag, + Self::Operator, + Self::Punctuation, + ]; +} + +/// Highlighted ranges of a whole document, packed like LSP semantic tokens. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct SyntaxHighlights { + /// Legend: `data` kind `i` is `kinds[i]`. + pub kinds: Vec, + /// `[from, to, kind]` triples: sorted, non-overlapping UTF-16 offsets. + pub data: Vec, +} + +impl SyntaxHighlights { + pub(crate) fn new(data: Vec) -> Self { + Self { kinds: SyntaxHighlightKind::ALL.to_vec(), data } + } +} + +/// What a capture paints over its node. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Paint { + Kind(SyntaxHighlightKind), + /// `@embedded`: code inside a string, uncoloured unless captured itself. + Clear, +} + +/// Capture name prefixes, most specific first within each family. +const CAPTURES: &[(&str, Paint)] = { + use Paint::{Clear, Kind}; + use SyntaxHighlightKind as K; + &[ + ("embedded", Clear), + ("keyword", Kind(K::Keyword)), + ("include", Kind(K::Keyword)), + ("import", Kind(K::Keyword)), + ("conditional", Kind(K::Keyword)), + ("repeat", Kind(K::Keyword)), + ("exception", Kind(K::Keyword)), + ("boolean", Kind(K::Keyword)), + ("string.escape", Kind(K::Escape)), + ("escape", Kind(K::Escape)), + ("string.regex", Kind(K::Regexp)), + ("string.regexp", Kind(K::Regexp)), + ("string.special.symbol", Kind(K::Atom)), + ("symbol", Kind(K::Atom)), + ("string.special", Kind(K::StringSpecial)), + ("string", Kind(K::String)), + ("character", Kind(K::String)), + ("number", Kind(K::Number)), + ("float", Kind(K::Number)), + ("constant.builtin", Kind(K::Constant)), + ("constant.macro", Kind(K::Macro)), + ("constant", Kind(K::Constant)), + // nvim greys out unused bindings (`_acc`); IDEA shows them as variables. + ("comment.unused", Kind(K::Variable)), + ("comment.doc", Kind(K::DocComment)), + ("comment.documentation", Kind(K::DocComment)), + ("comment", Kind(K::Comment)), + ("function.builtin", Kind(K::Builtin)), + ("function.macro", Kind(K::Macro)), + ("function.definition", Kind(K::Function)), + ("function", Kind(K::FunctionCall)), + ("method", Kind(K::FunctionCall)), + ("constructor", Kind(K::Type)), + ("type.builtin", Kind(K::Builtin)), + ("type", Kind(K::Type)), + ("module", Kind(K::Namespace)), + ("namespace", Kind(K::Namespace)), + ("variable.builtin", Kind(K::SelfVariable)), + ("variable.member", Kind(K::Property)), + ("variable.field", Kind(K::Property)), + ("variable", Kind(K::Variable)), + ("parameter", Kind(K::Variable)), + ("property", Kind(K::Property)), + ("field", Kind(K::Property)), + ("attribute", Kind(K::Attribute)), + ("label", Kind(K::Label)), + ("tag", Kind(K::Tag)), + ("operator", Kind(K::Operator)), + ("punctuation", Kind(K::Punctuation)), + ] +}; + +/// The paint for a capture name: its longest prefix (on `.` boundaries) in +/// [`CAPTURES`]. Private captures (`@__name__`) and unknown names paint nothing. +fn paint_for(name: &str) -> Option { + if name.starts_with('_') { + return None; + } + let mut prefix = name; + loop { + if let Some((_, p)) = CAPTURES.iter().find(|(n, _)| *n == prefix) { + return Some(*p); + } + prefix = &prefix[..prefix.rfind('.')?]; + } +} + +pub(crate) struct HighlightQuery { + query: Query, + /// By capture index. + paints: Vec>, +} + +impl HighlightQuery { + pub(crate) fn new(language: &Language, source: &str) -> Result { + let query = Query::new(language, source)?; + let paints = query.capture_names().iter().map(|n| paint_for(n)).collect(); + Ok(Self { query, paints }) + } +} + +/// Byte value for "no colour" in the paint buffer; kinds are `1 + index`. +const NONE: u8 = 0; + +/// `[from, to, kind]` UTF-16 triples for `text`; see [`SyntaxHighlights::data`]. +pub(crate) fn highlights(q: &HighlightQuery, tree: &Tree, text: &str) -> Vec { + // (byte len, pattern, start, end, paint): painted largest first, then in + // pattern order, so inner nodes and later patterns overwrite. + let mut captures: Vec<(usize, usize, usize, usize, Paint)> = Vec::new(); + let mut cursor = QueryCursor::new(); + let mut matches = cursor.matches(&q.query, tree.root_node(), text.as_bytes()); + while let Some(m) = matches.next() { + for c in m.captures { + if let Some(paint) = q.paints[c.index as usize] { + let r = c.node.byte_range(); + if r.start < r.end { + captures.push((r.end - r.start, m.pattern_index, r.start, r.end, paint)); + } + } + } + } + captures.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1))); + + let mut painted = vec![NONE; text.len()]; + for &(_, _, start, end, paint) in &captures { + let v = match paint { + Paint::Kind(k) => 1 + k as u8, + Paint::Clear => NONE, + }; + painted[start..end].fill(v); + } + + let lines = LineIndex::new(text); + let mut data = Vec::new(); + let mut i = 0; + while i < painted.len() { + let v = painted[i]; + let start = i; + while i < painted.len() && painted[i] == v { + i += 1; + } + // Node boundaries are char boundaries; whitespace-only runs stay uncoloured. + if v != NONE && text.is_char_boundary(start) && text.is_char_boundary(i) && !text[start..i].trim().is_empty() { + data.extend([lines.utf16_at(start) as u32, lines.utf16_at(i) as u32, (v - 1) as u32]); + } + } + data +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn capture_names_fold_to_the_longest_known_prefix() { + use SyntaxHighlightKind as K; + assert_eq!(paint_for("keyword.function"), Some(Paint::Kind(K::Keyword))); + assert_eq!(paint_for("string.special.symbol"), Some(Paint::Kind(K::Atom))); + assert_eq!(paint_for("string.special.path"), Some(Paint::Kind(K::StringSpecial))); + assert_eq!(paint_for("comment.doc.__attribute__"), Some(Paint::Kind(K::DocComment))); + assert_eq!(paint_for("punctuation.bracket"), Some(Paint::Kind(K::Punctuation))); + assert_eq!(paint_for("embedded"), Some(Paint::Clear)); + assert_eq!(paint_for("__name__"), None); + assert_eq!(paint_for("spell"), None); + } + + /// `(text, kind)` of every highlighted range. + fn spans(lang: &str, text: &str) -> Vec<(String, SyntaxHighlightKind)> { + let h = crate::Syntax::new().highlights(&ide_core::LanguageId::new(lang), text).unwrap(); + let units: Vec = text.encode_utf16().collect(); + h.data + .chunks(3) + .map(|t| (String::from_utf16(&units[t[0] as usize..t[1] as usize]).unwrap(), h.kinds[t[2] as usize])) + .collect() + } + + #[test] + fn highlights_elixir() { + use SyntaxHighlightKind as K; + let src = "defmodule Greeter do\n @moduledoc \"Hi\"\n # 😀 note\n def hello(name, _opts) do\n IO.puts(\"Hello #{name}\")\n :ok\n end\nend\n"; + let got = spans("elixir", src); + let kind = |t: &str| got.iter().find(|(s, _)| s == t).map(|(_, k)| *k); + assert_eq!(kind("defmodule"), Some(K::Keyword)); + assert_eq!(kind("Greeter"), Some(K::Namespace)); + assert_eq!(kind("\"Hi\""), Some(K::DocComment)); + assert_eq!(kind("# 😀 note"), Some(K::Comment)); + assert_eq!(kind("def"), Some(K::Keyword)); + assert_eq!(kind("hello"), Some(K::FunctionCall)); + assert_eq!(kind("_opts"), Some(K::Variable)); + assert_eq!(kind("puts"), Some(K::FunctionCall)); + // Interpolation clears the string colour; its contents are coloured on their own. + assert_eq!(kind("\"Hello "), Some(K::String)); + assert_eq!(kind("#{"), Some(K::Punctuation)); + assert_eq!(kind(":ok"), Some(K::Atom)); + assert_eq!(kind("end"), Some(K::Keyword)); + // Sorted and non-overlapping. + let h = crate::Syntax::new().highlights(&ide_core::LanguageId::new("elixir"), src).unwrap(); + assert!(h.data.chunks(3).zip(h.data.chunks(3).skip(1)).all(|(a, b)| a[0] < a[1] && a[1] <= b[0])); + } + + #[test] + fn languages_without_a_query_have_none() { + assert_eq!(crate::Syntax::new().highlights(&ide_core::LanguageId::new("rust"), "fn a() {}"), None); + } + + #[test] + fn legend_matches_discriminants() { + for (i, k) in SyntaxHighlightKind::ALL.iter().enumerate() { + assert_eq!(*k as usize, i); + } + } +} diff --git a/crates/ide-language/src/syntax/lines.rs b/crates/ide-language/src/syntax/lines.rs index 7b3291d2..5b8de206 100644 --- a/crates/ide-language/src/syntax/lines.rs +++ b/crates/ide-language/src/syntax/lines.rs @@ -40,7 +40,7 @@ impl<'a> LineIndex<'a> { } /// UTF-16 offset of a char-boundary byte offset. - fn utf16_at(&self, byte: usize) -> usize { + pub(crate) fn utf16_at(&self, byte: usize) -> usize { if self.checkpoints.is_empty() { return byte; } diff --git a/crates/ide-language/src/syntax/mod.rs b/crates/ide-language/src/syntax/mod.rs index e6633bb2..783dfe9c 100644 --- a/crates/ide-language/src/syntax/mod.rs +++ b/crates/ide-language/src/syntax/mod.rs @@ -1,8 +1,9 @@ -//! Tree-sitter driven structure: outline, folding, symbol tags, selection ranges. +//! Tree-sitter driven structure: outline, folding, symbol tags, selection ranges, highlighting. mod cache; mod folding; mod grammars; +mod highlight; pub(crate) mod lines; mod outline; mod selection; @@ -10,6 +11,7 @@ mod selection; use crate::{FoldRange, LanguageConfig, LanguageRegistry, SymbolTag}; pub use cache::{MAX_CACHED_TREES, SyntaxCache, SyntaxCacheStats}; use grammars::{SPECS, Spec}; +pub use highlight::{SyntaxHighlightKind, SyntaxHighlights}; use ide_core::{LanguageId, OutlineItem, Range}; use std::cell::RefCell; use std::collections::HashMap; @@ -31,6 +33,7 @@ pub(crate) struct Grammar { pub(crate) spec: &'static Spec, pub(crate) language: Language, outline: OnceLock>, + highlights: OnceLock>, kinds: OnceLock, } @@ -50,6 +53,17 @@ impl Grammar { }) .as_ref() } + + fn highlight_query(&self) -> Option<&highlight::HighlightQuery> { + self.highlights + .get_or_init(|| { + let source = self.spec.highlights?; + highlight::HighlightQuery::new(&self.language, source) + .inspect_err(|e| tracing::error!("highlights query for {} is invalid: {e}", self.spec.name)) + .ok() + }) + .as_ref() + } } thread_local! { @@ -72,6 +86,7 @@ impl Syntax { spec, language: Language::new(spec.language), outline: OnceLock::new(), + highlights: OnceLock::new(), kinds: OnceLock::new(), }) .collect(); @@ -150,6 +165,13 @@ impl Syntax { .unwrap_or_default() } + /// Highlighting from the grammar's own query; `None` without one. + pub fn highlights(&self, lang: &LanguageId, text: &str) -> Option { + self.analyze(lang, text, |g, tree, text| { + Some(SyntaxHighlights::new(highlight::highlights(g.highlight_query()?, tree, text))) + }) + } + /// Syntax-aware selection ranges for IDEA "Extend Selection" (⌥↑), innermost first. pub fn selection_ranges(&self, lang: &LanguageId, text: &str, offset_utf16: u32) -> Vec { match self.analyze(lang, text, |_, tree, text| Some(selection::selection_ranges(tree, text, offset_utf16))) { diff --git a/src-tauri/src/commands/doc.rs b/src-tauri/src/commands/doc.rs index 59c4f708..719dcbca 100644 --- a/src-tauri/src/commands/doc.rs +++ b/src-tauri/src/commands/doc.rs @@ -1,6 +1,6 @@ use crate::state::AppState; use ide_core::{ChangeSet, DocId, IdeError, IdeResult, OutlineItem, Range}; -use ide_language::FoldRange; +use ide_language::{FoldRange, SyntaxHighlights}; use ide_text::LineEnding; use ide_workspace::{DocumentInfo, HistoryEntry}; use std::path::PathBuf; @@ -136,6 +136,22 @@ pub async fn doc_folding(window: Window, state: State<'_, AppState>, doc: DocId) .await } +/// Tree-sitter highlighting for languages the UI has no CodeMirror mode for; +/// empty when the grammar has no highlights query or the document is large. +#[tauri::command] +#[specta::specta] +pub async fn doc_highlights(window: Window, state: State<'_, AppState>, doc: DocId) -> IdeResult { + let project = state.project(window.label())?; + blocking(move || { + let snap = project.documents.text_snapshot(doc)?; + if snap.large { + return Ok(SyntaxHighlights::default()); + } + Ok(project.syntax.highlights(doc, &snap.language, snap.version, &snap.text)) + }) + .await +} + /// Extend Selection (⌥↑): syntax ranges around `offset`, innermost first. /// Large documents fall back to word / line / document ranges. #[tauri::command] diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 4c4f60fa..dc0d79f6 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -74,6 +74,7 @@ pub fn builder() -> tauri_specta::Builder { doc::doc_set_line_ending, doc::doc_outline, doc::doc_folding, + doc::doc_highlights, doc::doc_selection_ranges, syntax::syntax_support, syntax::syntax_node_at, diff --git a/src/bindings.ts b/src/bindings.ts index 2afc2305..22f7dc89 100644 --- a/src/bindings.ts +++ b/src/bindings.ts @@ -92,6 +92,11 @@ export const commands = { */ docOutline: (doc: DocId) => typedError(__TAURI_INVOKE("doc_outline", { doc })), docFolding: (doc: DocId) => typedError(__TAURI_INVOKE("doc_folding", { doc })), + /** + * Tree-sitter highlighting for languages the UI has no CodeMirror mode for; + * empty when the grammar has no highlights query or the document is large. + */ + docHighlights: (doc: DocId) => typedError(__TAURI_INVOKE("doc_highlights", { doc })), /** * Extend Selection (⌥↑): syntax ranges around `offset`, innermost first. * Large documents fall back to word / line / document ranges. @@ -3683,6 +3688,17 @@ export type SyntaxErrors = { truncated: boolean, }; +/** Highlight classes sent to the UI, which maps each onto a Lezer tag. */ +export type SyntaxHighlightKind = "keyword" | "string" | "stringSpecial" | "escape" | "regexp" | "atom" | "number" | "constant" | "comment" | "docComment" | "function" | "functionCall" | "builtin" | "type" | "namespace" | "variable" | "selfVariable" | "property" | "attribute" | "macro" | "label" | "tag" | "operator" | "punctuation"; + +/** Highlighted ranges of a whole document, packed like LSP semantic tokens. */ +export type SyntaxHighlights = { + /** Legend: `data` kind `i` is `kinds[i]`. */ + kinds: SyntaxHighlightKind[], + /** `[from, to, kind]` triples: sorted, non-overlapping UTF-16 offsets. */ + data: number[], +}; + /** One tree-sitter node. */ export type SyntaxNode = { /** Grammar node kind, e.g. `method_invocation`, `identifier`, `;`. */ diff --git a/src/editor/core/backend.ts b/src/editor/core/backend.ts index 75de63a2..e075ec6e 100644 --- a/src/editor/core/backend.ts +++ b/src/editor/core/backend.ts @@ -18,6 +18,7 @@ import { type LineEnding, type OutlineItem, type Range, + type SyntaxHighlights, type TextChange, } from "@/platform/ipc"; @@ -43,6 +44,8 @@ export interface DocBackend { setEncoding(doc: DocId, encoding: string, reload: boolean): Promise; outline(doc: DocId): Promise; folding(doc: DocId): Promise; + /** Tree-sitter highlighting, for languages without a CodeMirror mode. */ + highlights(doc: DocId): Promise; selectionRanges(doc: DocId, offset: number): Promise; languages(): Promise; onPatched(handler: (e: DocPatchedEvent) => void): () => void; @@ -62,6 +65,7 @@ export const ipcBackend: DocBackend = { setEncoding: (doc, encoding, reload) => call(commands.docSetEncoding(doc, encoding, reload)), outline: (doc) => call(commands.docOutline(doc)), folding: (doc) => call(commands.docFolding(doc)), + highlights: (doc) => call(commands.docHighlights(doc)), selectionRanges: (doc, offset) => call(commands.docSelectionRanges(doc, offset)), languages: () => commands.languages(), onPatched: (h) => onIdeEvent("docPatched", (e) => h(e)), @@ -343,6 +347,9 @@ export function createMemoryBackend(files: Record = {}): MemoryB } return items; }, + async highlights() { + return { kinds: [], data: [] }; + }, async folding(doc) { const d = get(doc); const out: FoldRange[] = []; diff --git a/src/editor/core/cm/setup.ts b/src/editor/core/cm/setup.ts index 62cb91df..e5618dde 100644 --- a/src/editor/core/cm/setup.ts +++ b/src/editor/core/cm/setup.ts @@ -25,6 +25,7 @@ import { openideaTheme } from "./theme"; import { caretRowHighlight } from "./caretRow"; import { changeLane, overviewLane } from "./lanes"; import { foldingExtension } from "./folding"; +import { syntaxHighlightField } from "./syntaxHighlight"; import { indentGuides } from "./indentGuides"; import { stickyMaxLines, stickyScroll } from "./stickyScroll"; import { findExtension } from "./find"; @@ -267,6 +268,7 @@ export function baseExtensions(o: SetupOptions): Extension { clipboardHistoryExtension, indentOnInput(), syntaxHighlighting(openideaHighlightStyle), + syntaxHighlightField, closeBrackets(), caretRowHighlight, fontZoom, diff --git a/src/editor/core/cm/syntaxHighlight.test.ts b/src/editor/core/cm/syntaxHighlight.test.ts new file mode 100644 index 00000000..a3a1ecbb --- /dev/null +++ b/src/editor/core/cm/syntaxHighlight.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { EditorState } from "@codemirror/state"; +import { highlightingFor, syntaxHighlighting } from "@codemirror/language"; +import { tags as t } from "@lezer/highlight"; +import { openideaHighlightStyle } from "./highlight"; +import { setSyntaxHighlights, syntaxHighlightField } from "./syntaxHighlight"; + +function marks(state: EditorState) { + const out: { text: string; cls: string }[] = []; + state.field(syntaxHighlightField).between(0, state.doc.length, (from, to, d) => { + out.push({ text: state.doc.sliceString(from, to), cls: d.spec.class as string }); + }); + return out; +} + +describe("syntaxHighlightField", () => { + const create = () => + EditorState.create({ doc: "def hi, do: :ok", extensions: [syntaxHighlighting(openideaHighlightStyle), syntaxHighlightField] }); + + it("styles backend ranges with the active highlight style and maps them through edits", () => { + let state = create(); + state = state.update({ + effects: setSyntaxHighlights.of({ kinds: ["keyword", "atom", "functionCall"], data: [0, 3, 0, 4, 6, 2, 12, 15, 1, 99, 120, 0] }), + }).state; + const keyword = highlightingFor(state, [t.keyword]); + const atom = highlightingFor(state, [t.atom]); + expect(keyword).toBeTruthy(); + expect(marks(state)).toEqual([ + { text: "def", cls: keyword }, + { text: "hi", cls: highlightingFor(state, [t.function(t.variableName)]) }, + { text: ":ok", cls: atom }, + ]); + + state = state.update({ changes: { from: 0, insert: " " } }).state; + expect(marks(state).map((m) => m.text)).toEqual(["def", "hi", ":ok"]); + }); +}); diff --git a/src/editor/core/cm/syntaxHighlight.ts b/src/editor/core/cm/syntaxHighlight.ts new file mode 100644 index 00000000..8ced4899 --- /dev/null +++ b/src/editor/core/cm/syntaxHighlight.ts @@ -0,0 +1,66 @@ +/** + * Tree-sitter highlighting (`doc_highlights`) for languages without a + * CodeMirror mode (Elixir, ...): backend kinds → Lezer tags, so the colours + * come from the same highlight style and Color Scheme settings as Lezer + * highlighting. Refreshed with folding; in between, marks are mapped through + * the changes. + */ +import { StateEffect, StateField, type EditorState } from "@codemirror/state"; +import { Decoration, EditorView, type DecorationSet } from "@codemirror/view"; +import { highlightingFor } from "@codemirror/language"; +import { tags as t, type Tag } from "@lezer/highlight"; +import type { SyntaxHighlightKind, SyntaxHighlights } from "@/platform/ipc"; + +const TAGS: Record = { + keyword: t.keyword, + string: t.string, + stringSpecial: t.special(t.string), + escape: t.escape, + regexp: t.regexp, + atom: t.atom, + number: t.number, + constant: t.constant(t.variableName), + comment: t.comment, + docComment: t.docComment, + function: t.function(t.definition(t.variableName)), + functionCall: t.function(t.variableName), + builtin: t.standard(t.variableName), + type: t.typeName, + namespace: t.namespace, + variable: t.variableName, + selfVariable: t.self, + property: t.propertyName, + attribute: t.annotation, + macro: t.macroName, + label: t.labelName, + tag: t.tagName, + operator: t.operator, + punctuation: t.punctuation, +}; + +export const setSyntaxHighlights = StateEffect.define(); + +function decorations(state: EditorState, h: SyntaxHighlights): DecorationSet { + const marks = h.kinds.map((k) => { + const cls = highlightingFor(state, [TAGS[k]]); + return cls ? Decoration.mark({ class: cls }) : null; + }); + const len = state.doc.length; + const ranges = []; + for (let i = 0; i + 2 < h.data.length; i += 3) { + const from = h.data[i]!; + const to = Math.min(h.data[i + 1]!, len); + const mark = marks[h.data[i + 2]!]; + if (mark && from < to) ranges.push(mark.range(from, to)); + } + return Decoration.set(ranges, true); +} + +export const syntaxHighlightField = StateField.define({ + create: () => Decoration.none, + update(deco, tr) { + for (const e of tr.effects) if (e.is(setSyntaxHighlights)) return decorations(tr.state, e.value); + return tr.docChanged ? deco.map(tr.changes) : deco; + }, + provide: (f) => EditorView.decorations.from(f), +}); diff --git a/src/editor/core/model/document.ts b/src/editor/core/model/document.ts index 799e6e6c..981289c9 100644 --- a/src/editor/core/model/document.ts +++ b/src/editor/core/model/document.ts @@ -7,7 +7,7 @@ import { EditorView } from "@codemirror/view"; import type { LanguageDescription, LanguageSupport } from "@codemirror/language"; import { create } from "zustand"; import type { EditorConfigProps } from "@/bindings"; -import type { DocId, DocumentInfo, EditorSettings, FileSettings, LineEnding, OutlineItem, Settings } from "@/platform/ipc"; +import type { DocId, DocumentInfo, EditorSettings, FileSettings, LineEnding, OutlineItem, Settings, SyntaxHighlights } from "@/platform/ipc"; import { _emitEditorEvent, _saveParticipants, @@ -22,6 +22,7 @@ import type { DocBackend, LanguageInfo } from "../backend"; import { findLanguageDescription, languageInfo, loadLanguageInfos, loadSupport, loadedSupport, commentTokens } from "../cm/languages"; import { foldEffect } from "@codemirror/language"; import { backendFolds, setBackendFolds } from "../cm/folding"; +import { setSyntaxHighlights } from "../cm/syntaxHighlight"; import { caretFor, compartments, @@ -265,6 +266,7 @@ export class DocModel { addSlot(slot: EditorSlot) { this.slots.add(slot); + if (this.highlights?.counter === this.changeCounter) slot.reconfigure([setSyntaxHighlights.of(this.highlights.value)]); } removeSlot(slot: EditorSlot) { @@ -332,6 +334,8 @@ export class DocModel { indentOverride: { tabSize: number; insertSpaces: boolean } | null = null; /** Indentation detected in the file's text at open time. */ detectedIndent: { insertSpaces: boolean; size?: number } | null = null; + /** Last tree-sitter highlighting, for slots opened before the next refresh. */ + private highlights: { counter: number; value: SyntaxHighlights } | null = null; /** Fold-by-default has been applied (first folding ranges of the document). */ private initialFolds = false; /** Signature of settings that require contributions to be rebuilt. */ @@ -423,7 +427,12 @@ export class DocModel { await this.sync.whenIdle(); const counter = this.changeCounter; const backend = this.manager.backend; - const [folds, outline] = await Promise.allSettled([backend.folding(this.doc), backend.outline(this.doc)]); + const [folds, outline, highlights] = await Promise.allSettled([ + backend.folding(this.doc), + backend.outline(this.doc), + // Without a CodeMirror mode, colours come from the grammar's tree-sitter query. + this.desc ? null : backend.highlights(this.doc), + ]); if (this.closed) return; if (counter !== this.changeCounter) { this.scheduleRefresh(); @@ -436,6 +445,10 @@ export class DocModel { this.foldByDefault(); } } + if (highlights.status === "fulfilled" && highlights.value) { + this.highlights = { counter, value: highlights.value }; + for (const s of this.slots) s.reconfigure([setSyntaxHighlights.of(highlights.value)]); + } if (outline.status === "fulfilled") { useOutline.setState({ [this.path]: outline.value }); const lines = methodLines(outline.value);