diff --git a/crates/ide-language/queries/graphql/highlights.scm b/crates/ide-language/queries/graphql/highlights.scm new file mode 100644 index 00000000..bcea7da9 --- /dev/null +++ b/crates/ide-language/queries/graphql/highlights.scm @@ -0,0 +1,49 @@ +; GraphQL highlighting (the grammar crate ships no query). Later patterns win. + +(comment) @comment +(description (string_value) @comment.doc) +(string_value) @string +(int_value) @number +(float_value) @number +[(boolean_value) (null_value)] @keyword +(enum_value (name) @constant) + +(variable) @variable +(argument (name) @variable.parameter) +(input_value_definition (name) @variable.parameter) +(object_field (name) @property) + +(field (name) @property) +(alias (name) @property) +(field_definition (name) @property) +(enum_value_definition (enum_value (name) @constant)) + +(named_type (name) @type) +(object_type_definition (name) @type) +(interface_type_definition (name) @type) +(union_type_definition (name) @type) +(enum_type_definition (name) @type) +(input_object_type_definition (name) @type) +(scalar_type_definition (name) @type) +(object_type_extension (name) @type) +(interface_type_extension (name) @type) +(union_type_extension (name) @type) +(enum_type_extension (name) @type) +(input_object_type_extension (name) @type) +(scalar_type_extension (name) @type) + +(operation_definition (name) @function) +(fragment_name (name) @function) +(directive_definition (name) @attribute) +(directive (name) @attribute) +"@" @attribute +(directive_location) @constant + +(operation_type) @keyword +[ + "query" "mutation" "subscription" "fragment" "on" "type" "interface" "union" + "enum" "input" "scalar" "schema" "extend" "implements" "directive" "repeatable" +] @keyword + +["(" ")" "[" "]" "{" "}"] @punctuation.bracket +[":" "=" "|" "&" "!" "..."] @punctuation.delimiter diff --git a/crates/ide-language/queries/hcl/highlights.scm b/crates/ide-language/queries/hcl/highlights.scm new file mode 100644 index 00000000..372f3b1d --- /dev/null +++ b/crates/ide-language/queries/hcl/highlights.scm @@ -0,0 +1,29 @@ +; HCL / Terraform highlighting (the grammar crate ships no query). Later patterns win. + +(comment) @comment +(string_lit) @string +(quoted_template) @string +(heredoc_template) @string +[(heredoc_start) (heredoc_identifier)] @string.special +(template_literal) @string +(numeric_lit) @number +[(bool_lit) (null_lit)] @keyword + +(identifier) @variable +(attribute (identifier) @property) +(object_elem key: (expression (variable_expr (identifier) @property))) +(get_attr (identifier) @property) +(function_call (identifier) @function) +; `resource "aws_instance" "web" {` +(block (identifier) @keyword) +(block (string_lit (template_literal) @type)) + +(template_interpolation) @embedded +[(template_interpolation_start) (template_interpolation_end)] @punctuation.special +[(template_directive_start) (template_directive_end) (strip_marker)] @punctuation.special +["for" "in" "if" "else" "endfor" "endif"] @keyword + +(operation) @operator +["!" "!=" "%" "&&" "*" "+" "-" "/" "<" "<=" "==" ">" ">=" "||" "?" "=>"] @operator +["=" ":" "," "."] @punctuation.delimiter +["(" ")" "[" "]" "{" "}" "[*]" ".*"] @punctuation.bracket diff --git a/crates/ide-language/src/syntax/grammars.rs b/crates/ide-language/src/syntax/grammars.rs index fad2d5c5..2045bde4 100644 --- a/crates/ide-language/src/syntax/grammars.rs +++ b/crates/ide-language/src/syntax/grammars.rs @@ -10,7 +10,7 @@ pub(crate) struct Spec { /// `queries//outline.scm`; see `queries/README.md` for the capture vocabulary. pub outline: &'static str, /// The grammar's `highlights.scm`, for languages without a CodeMirror mode. - pub highlights: Option<&'static str>, + pub highlights: Option, /// Node kinds folded when they span several lines (in addition to every /// bracket-delimited node, which is folded generically). pub fold_nodes: &'static [&'static str], @@ -22,6 +22,21 @@ pub(crate) struct Spec { pub constructors: &'static [&'static str], } +/// A highlights query and how its patterns rank against each other. +pub(crate) struct Highlights { + pub query: &'static str, + /// For the same node, the first matching pattern wins (tree-sitter CLI / + /// GitHub queries) instead of the last one (nvim / Zed queries). + pub first_wins: bool, +} + +/// Our own `queries//highlights.scm`, for grammars that ship none. +macro_rules! highlights { + ($name:literal) => { + include_str!(concat!("../../queries/", $name, "/highlights.scm")) + }; +} + macro_rules! outline { ($name:literal) => { include_str!(concat!("../../queries/", $name, "/outline.scm")) @@ -240,7 +255,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "zig", language: tree_sitter_zig::LANGUAGE, outline: outline!("zig"), - highlights: None, + highlights: Some(Highlights { query: tree_sitter_zig::HIGHLIGHTS_QUERY, first_wins: false }), fold_nodes: NONE, fold_bodies: NONE, imports: NONE, @@ -250,7 +265,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "elixir", language: tree_sitter_elixir::LANGUAGE, outline: outline!("elixir"), - highlights: Some(tree_sitter_elixir::HIGHLIGHTS_QUERY), + highlights: Some(Highlights { query: tree_sitter_elixir::HIGHLIGHTS_QUERY, first_wins: false }), fold_nodes: &["do_block", "anonymous_function", "string", "sigil"], fold_bodies: NONE, imports: NONE, @@ -379,7 +394,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "hcl", language: tree_sitter_hcl::LANGUAGE, outline: outline!("hcl"), - highlights: None, + highlights: Some(Highlights { query: highlights!("hcl"), first_wins: false }), fold_nodes: &["block", "heredoc_template"], fold_bodies: NONE, imports: NONE, @@ -389,7 +404,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "nix", language: tree_sitter_nix::LANGUAGE, outline: outline!("nix"), - highlights: None, + highlights: Some(Highlights { query: tree_sitter_nix::HIGHLIGHTS_QUERY, first_wins: true }), fold_nodes: &["let_expression", "indented_string_expression"], fold_bodies: NONE, imports: NONE, @@ -409,7 +424,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "graphql", language: tree_sitter_graphql::LANGUAGE, outline: outline!("graphql"), - highlights: None, + highlights: Some(Highlights { query: highlights!("graphql"), first_wins: false }), fold_nodes: NONE, fold_bodies: NONE, imports: NONE, @@ -419,7 +434,7 @@ pub(crate) static SPECS: &[Spec] = &[ name: "make", language: tree_sitter_make::LANGUAGE, outline: outline!("make"), - highlights: None, + highlights: Some(Highlights { query: tree_sitter_make::HIGHLIGHTS_QUERY, first_wins: false }), fold_nodes: &["rule", "conditional", "define_directive"], fold_bodies: NONE, imports: &["include_directive"], diff --git a/crates/ide-language/src/syntax/highlight.rs b/crates/ide-language/src/syntax/highlight.rs index 629c081c..2daab82f 100644 --- a/crates/ide-language/src/syntax/highlight.rs +++ b/crates/ide-language/src/syntax/highlight.rs @@ -5,13 +5,14 @@ //! `@string.special.symbol`, ...) and are folded into [`SyntaxHighlightKind`] by //! their longest known dotted prefix. Overlaps resolve the way those editors //! do: a smaller node wins over the node containing it, and for the same node -//! the later pattern wins. `@embedded` (string interpolation) clears the -//! surrounding colour. +//! the later pattern wins (the first one for tree-sitter CLI style queries). `@embedded` (string interpolation) clears the +//! surrounding colour. Besides tree-sitter's own text predicates, nvim's +//! `#lua-match?` is understood; patterns with other predicates never match. use super::lines::LineIndex; use serde::{Deserialize, Serialize}; use specta::Type; -use tree_sitter::{Language, Query, QueryCursor, StreamingIterator, Tree}; +use tree_sitter::{Language, Query, QueryCursor, QueryMatch, QueryPredicateArg, StreamingIterator, Tree}; /// Highlight classes sent to the UI, which maps each onto a Lezer tag. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Type)] @@ -169,18 +170,113 @@ fn paint_for(name: &str) -> Option { } } +/// A predicate tree-sitter leaves to the caller. +enum Check { + /// `#lua-match?` / `#not-lua-match?` on a capture's text. + Matches { + capture: u32, + regex: regex_lite::Regex, + negate: bool, + }, + Unsupported, +} + pub(crate) struct HighlightQuery { query: Query, /// By capture index. paints: Vec>, + /// By pattern index. + checks: Vec>, + first_wins: bool, } impl HighlightQuery { - pub(crate) fn new(language: &Language, source: &str) -> Result { + pub(crate) fn new(language: &Language, source: &str, first_wins: bool) -> Result { let query = Query::new(language, source)?; let paints = query.capture_names().iter().map(|n| paint_for(n)).collect(); - Ok(Self { query, paints }) + let checks = (0..query.pattern_count()) + .map(|i| { + query + .general_predicates(i) + .iter() + .filter_map(|p| match (&*p.operator, &*p.args) { + // Locals are not tracked: builtins are assumed not to be shadowed. + ("is-not?", [QueryPredicateArg::String(s)]) if &**s == "local" => None, + ( + op @ ("lua-match?" | "not-lua-match?"), + [QueryPredicateArg::Capture(c), QueryPredicateArg::String(pat)], + ) => Some(match regex_lite::Regex::new(&lua_pattern_to_regex(pat)) { + Ok(regex) => Check::Matches { capture: *c, regex, negate: op.starts_with("not-") }, + Err(_) => Check::Unsupported, + }), + _ => Some(Check::Unsupported), + }) + .collect() + }) + .collect(); + Ok(Self { query, paints, checks, first_wins }) } + + fn accepts(&self, m: &QueryMatch<'_, '_>, text: &str) -> bool { + self.checks[m.pattern_index].iter().all(|check| match check { + Check::Matches { capture, regex, negate } => m + .captures + .iter() + .filter(|c| c.index == *capture) + .all(|c| regex.is_match(&text[c.node.byte_range()]) != *negate), + Check::Unsupported => false, + }) + } +} + +/// Translates the Lua patterns used by nvim queries (`^[A-Z][A-Z_0-9]+$`, `^//!`) to regex syntax. +fn lua_pattern_to_regex(pat: &str) -> String { + let class = |c: char| match c { + 'a' => Some("A-Za-z"), + 'd' => Some("0-9"), + 'l' => Some("a-z"), + 'u' => Some("A-Z"), + 'w' => Some("A-Za-z0-9"), + 's' => Some("\\s"), + 'x' => Some("0-9A-Fa-f"), + 'p' => Some("[:punct:]"), + _ => None, + }; + let mut out = String::with_capacity(pat.len() + 8); + let mut in_set = false; + let mut chars = pat.chars(); + while let Some(c) = chars.next() { + match c { + '%' => { + let Some(next) = chars.next() else { break }; + match class(next) { + Some(set) if in_set => out.push_str(set), + Some(set) => { + out.push('['); + out.push_str(set); + out.push(']'); + } + None => out.push_str(®ex_lite::escape(&next.to_string())), + } + } + '[' if !in_set => { + in_set = true; + out.push(c); + } + ']' if in_set => { + in_set = false; + out.push(c); + } + // Lua's lazy `*` (inside a set, `-` is a range). + '-' if !in_set => out.push_str("*?"), + '\\' | '{' | '}' | '|' => { + out.push('\\'); + out.push(c); + } + _ => out.push(c), + } + } + out } /// Byte value for "no colour" in the paint buffer; kinds are `1 + index`. @@ -188,17 +284,21 @@ const NONE: u8 = 0; /// `[from, to, kind]` UTF-16 triples for `text`; see [`SyntaxHighlights::data`]. pub(crate) fn highlights(q: &HighlightQuery, tree: &Tree, text: &str) -> Vec { - // (byte len, pattern, start, end, paint): painted largest first, then in - // pattern order, so inner nodes and later patterns overwrite. + // (byte len, rank, start, end, paint): painted largest first, then by rank, + // so inner nodes and winning patterns overwrite. let mut captures: Vec<(usize, usize, usize, usize, Paint)> = Vec::new(); let mut cursor = QueryCursor::new(); let mut matches = cursor.matches(&q.query, tree.root_node(), text.as_bytes()); while let Some(m) = matches.next() { + if !q.accepts(m, text) { + continue; + } for c in m.captures { if let Some(paint) = q.paints[c.index as usize] { let r = c.node.byte_range(); if r.start < r.end { - captures.push((r.end - r.start, m.pattern_index, r.start, r.end, paint)); + let rank = if q.first_wins { usize::MAX - m.pattern_index } else { m.pattern_index }; + captures.push((r.end - r.start, rank, r.start, r.end, paint)); } } } @@ -282,11 +382,98 @@ mod tests { assert!(h.data.chunks(3).zip(h.data.chunks(3).skip(1)).all(|(a, b)| a[0] < a[1] && a[1] <= b[0])); } + #[test] + fn highlights_other_grammar_only_languages() { + use SyntaxHighlightKind as K; + type Expect<'a> = &'a [(&'a str, SyntaxHighlightKind)]; + let cases: &[(&str, &str, Expect)] = &[ + ( + "zig", + "const std = @import(\"std\");\n// hi\npub fn main() void {\n var x: u32 = 42;\n _ = x;\n}\n", + &[ + ("const", K::Keyword), + ("\"std\"", K::String), + ("// hi", K::Comment), + ("42", K::Number), + ("u32", K::Builtin), + ], + ), + ( + "nix", + "{ pkgs ? import {} }:\n# hi\nlet x = 1; in pkgs.mkShell { name = \"dev\"; }\n", + &[ + ("let", K::Keyword), + ("# hi", K::Comment), + ("1", K::Number), + ("\"dev\"", K::String), + ("import", K::Builtin), + ], + ), + ( + "graphql", + "# hi\nquery Hero($id: ID!) @cached {\n hero(id: $id) { name ...Friends }\n}\nenum Episode { NEWHOPE }\n", + &[ + ("# hi", K::Comment), + ("query", K::Keyword), + ("Hero", K::FunctionCall), + ("$id", K::Variable), + ("ID", K::Type), + ("@cached", K::Attribute), + ("name", K::Property), + ("Friends", K::FunctionCall), + ("NEWHOPE", K::Constant), + ], + ), + ( + "hcl", + "# hi\nresource \"aws_instance\" \"web\" {\n ami = var.ami\n count = 2\n tags = { Name = \"web-${var.env}\" }\n name = upper(\"x\")\n}\n", + &[ + ("# hi", K::Comment), + ("resource", K::Keyword), + ("aws_instance", K::Type), + ("ami", K::Property), + ("2", K::Number), + ("upper", K::FunctionCall), + ("\"web-", K::String), + ("env", K::Property), + ("var", K::Variable), + ], + ), + ("make", "# hi\nCC := gcc\nall: main.o\n\t$(CC) -o app main.o\n", &[("# hi", K::Comment)]), + ]; + for (lang, src, expect) in cases { + let got = spans(lang, src); + for (text, kind) in *expect { + assert!(got.iter().any(|(s, k)| s == text && k == kind), "{lang}: {text:?} as {kind:?} in {got:?}"); + } + } + } + #[test] fn languages_without_a_query_have_none() { assert_eq!(crate::Syntax::new().highlights(&ide_core::LanguageId::new("rust"), "fn a() {}"), None); } + #[test] + fn every_highlights_query_compiles() { + for spec in super::super::grammars::SPECS { + if let Some(h) = &spec.highlights { + let lang = Language::new(spec.language); + if let Err(e) = HighlightQuery::new(&lang, h.query, h.first_wins) { + panic!("{}: {e}", spec.name); + } + } + } + } + + #[test] + fn lua_patterns_translate() { + assert_eq!(lua_pattern_to_regex("^[A-Z][A-Z_0-9]+$"), "^[A-Z][A-Z_0-9]+$"); + assert_eq!(lua_pattern_to_regex("^//!"), "^//!"); + assert_eq!(lua_pattern_to_regex("^%u[%w_]*$"), "^[A-Z][A-Za-z0-9_]*$"); + assert_eq!(lua_pattern_to_regex("a.-b%."), "a.*?b\\."); + } + #[test] fn legend_matches_discriminants() { for (i, k) in SyntaxHighlightKind::ALL.iter().enumerate() { diff --git a/crates/ide-language/src/syntax/mod.rs b/crates/ide-language/src/syntax/mod.rs index 783dfe9c..fed5514f 100644 --- a/crates/ide-language/src/syntax/mod.rs +++ b/crates/ide-language/src/syntax/mod.rs @@ -57,8 +57,8 @@ impl Grammar { fn highlight_query(&self) -> Option<&highlight::HighlightQuery> { self.highlights .get_or_init(|| { - let source = self.spec.highlights?; - highlight::HighlightQuery::new(&self.language, source) + let h = self.spec.highlights.as_ref()?; + highlight::HighlightQuery::new(&self.language, h.query, h.first_wins) .inspect_err(|e| tracing::error!("highlights query for {} is invalid: {e}", self.spec.name)) .ok() })