mirror of
https://github.com/l0ng-ai/openidea.git
synced 2026-11-06 08:07:49 +00:00
feat(editor): Tree-sitter highlighting for Zig, Nix, Make, GraphQL and HCL
Every grammar-only language now gets colours: Zig, Nix and Make from the queries their crates ship, GraphQL and HCL (Terraform) from our own, as those crates ship none. Grammar queries follow two conventions, so the precedence is set per grammar: nvim-style (the later pattern wins) or tree-sitter CLI style (the first one wins, Nix). nvim's `#lua-match?` is evaluated; patterns with predicates we cannot evaluate no longer match unconditionally, which coloured every Zig identifier as a constant. Claude-Session: https://claude.ai/code/session_01B65qLEKnkeYTGdbwhLpjCu
This commit is contained in:
l0ng-ai
@@ -0,0 +1,49 @@
|
||||
; GraphQL highlighting (the grammar crate ships no query). Later patterns win.
|
||||
|
||||
(comment) @comment
|
||||
(description (string_value) @comment.doc)
|
||||
(string_value) @string
|
||||
(int_value) @number
|
||||
(float_value) @number
|
||||
[(boolean_value) (null_value)] @keyword
|
||||
(enum_value (name) @constant)
|
||||
|
||||
(variable) @variable
|
||||
(argument (name) @variable.parameter)
|
||||
(input_value_definition (name) @variable.parameter)
|
||||
(object_field (name) @property)
|
||||
|
||||
(field (name) @property)
|
||||
(alias (name) @property)
|
||||
(field_definition (name) @property)
|
||||
(enum_value_definition (enum_value (name) @constant))
|
||||
|
||||
(named_type (name) @type)
|
||||
(object_type_definition (name) @type)
|
||||
(interface_type_definition (name) @type)
|
||||
(union_type_definition (name) @type)
|
||||
(enum_type_definition (name) @type)
|
||||
(input_object_type_definition (name) @type)
|
||||
(scalar_type_definition (name) @type)
|
||||
(object_type_extension (name) @type)
|
||||
(interface_type_extension (name) @type)
|
||||
(union_type_extension (name) @type)
|
||||
(enum_type_extension (name) @type)
|
||||
(input_object_type_extension (name) @type)
|
||||
(scalar_type_extension (name) @type)
|
||||
|
||||
(operation_definition (name) @function)
|
||||
(fragment_name (name) @function)
|
||||
(directive_definition (name) @attribute)
|
||||
(directive (name) @attribute)
|
||||
"@" @attribute
|
||||
(directive_location) @constant
|
||||
|
||||
(operation_type) @keyword
|
||||
[
|
||||
"query" "mutation" "subscription" "fragment" "on" "type" "interface" "union"
|
||||
"enum" "input" "scalar" "schema" "extend" "implements" "directive" "repeatable"
|
||||
] @keyword
|
||||
|
||||
["(" ")" "[" "]" "{" "}"] @punctuation.bracket
|
||||
[":" "=" "|" "&" "!" "..."] @punctuation.delimiter
|
||||
@@ -0,0 +1,29 @@
|
||||
; HCL / Terraform highlighting (the grammar crate ships no query). Later patterns win.
|
||||
|
||||
(comment) @comment
|
||||
(string_lit) @string
|
||||
(quoted_template) @string
|
||||
(heredoc_template) @string
|
||||
[(heredoc_start) (heredoc_identifier)] @string.special
|
||||
(template_literal) @string
|
||||
(numeric_lit) @number
|
||||
[(bool_lit) (null_lit)] @keyword
|
||||
|
||||
(identifier) @variable
|
||||
(attribute (identifier) @property)
|
||||
(object_elem key: (expression (variable_expr (identifier) @property)))
|
||||
(get_attr (identifier) @property)
|
||||
(function_call (identifier) @function)
|
||||
; `resource "aws_instance" "web" {`
|
||||
(block (identifier) @keyword)
|
||||
(block (string_lit (template_literal) @type))
|
||||
|
||||
(template_interpolation) @embedded
|
||||
[(template_interpolation_start) (template_interpolation_end)] @punctuation.special
|
||||
[(template_directive_start) (template_directive_end) (strip_marker)] @punctuation.special
|
||||
["for" "in" "if" "else" "endfor" "endif"] @keyword
|
||||
|
||||
(operation) @operator
|
||||
["!" "!=" "%" "&&" "*" "+" "-" "/" "<" "<=" "==" ">" ">=" "||" "?" "=>"] @operator
|
||||
["=" ":" "," "."] @punctuation.delimiter
|
||||
["(" ")" "[" "]" "{" "}" "[*]" ".*"] @punctuation.bracket
|
||||
@@ -10,7 +10,7 @@ pub(crate) struct Spec {
|
||||
/// `queries/<name>/outline.scm`; see `queries/README.md` for the capture vocabulary.
|
||||
pub outline: &'static str,
|
||||
/// The grammar's `highlights.scm`, for languages without a CodeMirror mode.
|
||||
pub highlights: Option<&'static str>,
|
||||
pub highlights: Option<Highlights>,
|
||||
/// Node kinds folded when they span several lines (in addition to every
|
||||
/// bracket-delimited node, which is folded generically).
|
||||
pub fold_nodes: &'static [&'static str],
|
||||
@@ -22,6 +22,21 @@ pub(crate) struct Spec {
|
||||
pub constructors: &'static [&'static str],
|
||||
}
|
||||
|
||||
/// A highlights query and how its patterns rank against each other.
|
||||
pub(crate) struct Highlights {
|
||||
pub query: &'static str,
|
||||
/// For the same node, the first matching pattern wins (tree-sitter CLI /
|
||||
/// GitHub queries) instead of the last one (nvim / Zed queries).
|
||||
pub first_wins: bool,
|
||||
}
|
||||
|
||||
/// Our own `queries/<name>/highlights.scm`, for grammars that ship none.
|
||||
macro_rules! highlights {
|
||||
($name:literal) => {
|
||||
include_str!(concat!("../../queries/", $name, "/highlights.scm"))
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! outline {
|
||||
($name:literal) => {
|
||||
include_str!(concat!("../../queries/", $name, "/outline.scm"))
|
||||
@@ -240,7 +255,7 @@ pub(crate) static SPECS: &[Spec] = &[
|
||||
name: "zig",
|
||||
language: tree_sitter_zig::LANGUAGE,
|
||||
outline: outline!("zig"),
|
||||
highlights: None,
|
||||
highlights: Some(Highlights { query: tree_sitter_zig::HIGHLIGHTS_QUERY, first_wins: false }),
|
||||
fold_nodes: NONE,
|
||||
fold_bodies: NONE,
|
||||
imports: NONE,
|
||||
@@ -250,7 +265,7 @@ pub(crate) static SPECS: &[Spec] = &[
|
||||
name: "elixir",
|
||||
language: tree_sitter_elixir::LANGUAGE,
|
||||
outline: outline!("elixir"),
|
||||
highlights: Some(tree_sitter_elixir::HIGHLIGHTS_QUERY),
|
||||
highlights: Some(Highlights { query: tree_sitter_elixir::HIGHLIGHTS_QUERY, first_wins: false }),
|
||||
fold_nodes: &["do_block", "anonymous_function", "string", "sigil"],
|
||||
fold_bodies: NONE,
|
||||
imports: NONE,
|
||||
@@ -379,7 +394,7 @@ pub(crate) static SPECS: &[Spec] = &[
|
||||
name: "hcl",
|
||||
language: tree_sitter_hcl::LANGUAGE,
|
||||
outline: outline!("hcl"),
|
||||
highlights: None,
|
||||
highlights: Some(Highlights { query: highlights!("hcl"), first_wins: false }),
|
||||
fold_nodes: &["block", "heredoc_template"],
|
||||
fold_bodies: NONE,
|
||||
imports: NONE,
|
||||
@@ -389,7 +404,7 @@ pub(crate) static SPECS: &[Spec] = &[
|
||||
name: "nix",
|
||||
language: tree_sitter_nix::LANGUAGE,
|
||||
outline: outline!("nix"),
|
||||
highlights: None,
|
||||
highlights: Some(Highlights { query: tree_sitter_nix::HIGHLIGHTS_QUERY, first_wins: true }),
|
||||
fold_nodes: &["let_expression", "indented_string_expression"],
|
||||
fold_bodies: NONE,
|
||||
imports: NONE,
|
||||
@@ -409,7 +424,7 @@ pub(crate) static SPECS: &[Spec] = &[
|
||||
name: "graphql",
|
||||
language: tree_sitter_graphql::LANGUAGE,
|
||||
outline: outline!("graphql"),
|
||||
highlights: None,
|
||||
highlights: Some(Highlights { query: highlights!("graphql"), first_wins: false }),
|
||||
fold_nodes: NONE,
|
||||
fold_bodies: NONE,
|
||||
imports: NONE,
|
||||
@@ -419,7 +434,7 @@ pub(crate) static SPECS: &[Spec] = &[
|
||||
name: "make",
|
||||
language: tree_sitter_make::LANGUAGE,
|
||||
outline: outline!("make"),
|
||||
highlights: None,
|
||||
highlights: Some(Highlights { query: tree_sitter_make::HIGHLIGHTS_QUERY, first_wins: false }),
|
||||
fold_nodes: &["rule", "conditional", "define_directive"],
|
||||
fold_bodies: NONE,
|
||||
imports: &["include_directive"],
|
||||
|
||||
@@ -5,13 +5,14 @@
|
||||
//! `@string.special.symbol`, ...) and are folded into [`SyntaxHighlightKind`] by
|
||||
//! their longest known dotted prefix. Overlaps resolve the way those editors
|
||||
//! do: a smaller node wins over the node containing it, and for the same node
|
||||
//! the later pattern wins. `@embedded` (string interpolation) clears the
|
||||
//! surrounding colour.
|
||||
//! the later pattern wins (the first one for tree-sitter CLI style queries). `@embedded` (string interpolation) clears the
|
||||
//! surrounding colour. Besides tree-sitter's own text predicates, nvim's
|
||||
//! `#lua-match?` is understood; patterns with other predicates never match.
|
||||
|
||||
use super::lines::LineIndex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use specta::Type;
|
||||
use tree_sitter::{Language, Query, QueryCursor, StreamingIterator, Tree};
|
||||
use tree_sitter::{Language, Query, QueryCursor, QueryMatch, QueryPredicateArg, StreamingIterator, Tree};
|
||||
|
||||
/// Highlight classes sent to the UI, which maps each onto a Lezer tag.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Type)]
|
||||
@@ -169,18 +170,113 @@ fn paint_for(name: &str) -> Option<Paint> {
|
||||
}
|
||||
}
|
||||
|
||||
/// A predicate tree-sitter leaves to the caller.
|
||||
enum Check {
|
||||
/// `#lua-match?` / `#not-lua-match?` on a capture's text.
|
||||
Matches {
|
||||
capture: u32,
|
||||
regex: regex_lite::Regex,
|
||||
negate: bool,
|
||||
},
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
pub(crate) struct HighlightQuery {
|
||||
query: Query,
|
||||
/// By capture index.
|
||||
paints: Vec<Option<Paint>>,
|
||||
/// By pattern index.
|
||||
checks: Vec<Vec<Check>>,
|
||||
first_wins: bool,
|
||||
}
|
||||
|
||||
impl HighlightQuery {
|
||||
pub(crate) fn new(language: &Language, source: &str) -> Result<Self, tree_sitter::QueryError> {
|
||||
pub(crate) fn new(language: &Language, source: &str, first_wins: bool) -> Result<Self, tree_sitter::QueryError> {
|
||||
let query = Query::new(language, source)?;
|
||||
let paints = query.capture_names().iter().map(|n| paint_for(n)).collect();
|
||||
Ok(Self { query, paints })
|
||||
let checks = (0..query.pattern_count())
|
||||
.map(|i| {
|
||||
query
|
||||
.general_predicates(i)
|
||||
.iter()
|
||||
.filter_map(|p| match (&*p.operator, &*p.args) {
|
||||
// Locals are not tracked: builtins are assumed not to be shadowed.
|
||||
("is-not?", [QueryPredicateArg::String(s)]) if &**s == "local" => None,
|
||||
(
|
||||
op @ ("lua-match?" | "not-lua-match?"),
|
||||
[QueryPredicateArg::Capture(c), QueryPredicateArg::String(pat)],
|
||||
) => Some(match regex_lite::Regex::new(&lua_pattern_to_regex(pat)) {
|
||||
Ok(regex) => Check::Matches { capture: *c, regex, negate: op.starts_with("not-") },
|
||||
Err(_) => Check::Unsupported,
|
||||
}),
|
||||
_ => Some(Check::Unsupported),
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
Ok(Self { query, paints, checks, first_wins })
|
||||
}
|
||||
|
||||
fn accepts(&self, m: &QueryMatch<'_, '_>, text: &str) -> bool {
|
||||
self.checks[m.pattern_index].iter().all(|check| match check {
|
||||
Check::Matches { capture, regex, negate } => m
|
||||
.captures
|
||||
.iter()
|
||||
.filter(|c| c.index == *capture)
|
||||
.all(|c| regex.is_match(&text[c.node.byte_range()]) != *negate),
|
||||
Check::Unsupported => false,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Translates the Lua patterns used by nvim queries (`^[A-Z][A-Z_0-9]+$`, `^//!`) to regex syntax.
|
||||
fn lua_pattern_to_regex(pat: &str) -> String {
|
||||
let class = |c: char| match c {
|
||||
'a' => Some("A-Za-z"),
|
||||
'd' => Some("0-9"),
|
||||
'l' => Some("a-z"),
|
||||
'u' => Some("A-Z"),
|
||||
'w' => Some("A-Za-z0-9"),
|
||||
's' => Some("\\s"),
|
||||
'x' => Some("0-9A-Fa-f"),
|
||||
'p' => Some("[:punct:]"),
|
||||
_ => None,
|
||||
};
|
||||
let mut out = String::with_capacity(pat.len() + 8);
|
||||
let mut in_set = false;
|
||||
let mut chars = pat.chars();
|
||||
while let Some(c) = chars.next() {
|
||||
match c {
|
||||
'%' => {
|
||||
let Some(next) = chars.next() else { break };
|
||||
match class(next) {
|
||||
Some(set) if in_set => out.push_str(set),
|
||||
Some(set) => {
|
||||
out.push('[');
|
||||
out.push_str(set);
|
||||
out.push(']');
|
||||
}
|
||||
None => out.push_str(®ex_lite::escape(&next.to_string())),
|
||||
}
|
||||
}
|
||||
'[' if !in_set => {
|
||||
in_set = true;
|
||||
out.push(c);
|
||||
}
|
||||
']' if in_set => {
|
||||
in_set = false;
|
||||
out.push(c);
|
||||
}
|
||||
// Lua's lazy `*` (inside a set, `-` is a range).
|
||||
'-' if !in_set => out.push_str("*?"),
|
||||
'\\' | '{' | '}' | '|' => {
|
||||
out.push('\\');
|
||||
out.push(c);
|
||||
}
|
||||
_ => out.push(c),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Byte value for "no colour" in the paint buffer; kinds are `1 + index`.
|
||||
@@ -188,17 +284,21 @@ const NONE: u8 = 0;
|
||||
|
||||
/// `[from, to, kind]` UTF-16 triples for `text`; see [`SyntaxHighlights::data`].
|
||||
pub(crate) fn highlights(q: &HighlightQuery, tree: &Tree, text: &str) -> Vec<u32> {
|
||||
// (byte len, pattern, start, end, paint): painted largest first, then in
|
||||
// pattern order, so inner nodes and later patterns overwrite.
|
||||
// (byte len, rank, start, end, paint): painted largest first, then by rank,
|
||||
// so inner nodes and winning patterns overwrite.
|
||||
let mut captures: Vec<(usize, usize, usize, usize, Paint)> = Vec::new();
|
||||
let mut cursor = QueryCursor::new();
|
||||
let mut matches = cursor.matches(&q.query, tree.root_node(), text.as_bytes());
|
||||
while let Some(m) = matches.next() {
|
||||
if !q.accepts(m, text) {
|
||||
continue;
|
||||
}
|
||||
for c in m.captures {
|
||||
if let Some(paint) = q.paints[c.index as usize] {
|
||||
let r = c.node.byte_range();
|
||||
if r.start < r.end {
|
||||
captures.push((r.end - r.start, m.pattern_index, r.start, r.end, paint));
|
||||
let rank = if q.first_wins { usize::MAX - m.pattern_index } else { m.pattern_index };
|
||||
captures.push((r.end - r.start, rank, r.start, r.end, paint));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -282,11 +382,98 @@ mod tests {
|
||||
assert!(h.data.chunks(3).zip(h.data.chunks(3).skip(1)).all(|(a, b)| a[0] < a[1] && a[1] <= b[0]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn highlights_other_grammar_only_languages() {
|
||||
use SyntaxHighlightKind as K;
|
||||
type Expect<'a> = &'a [(&'a str, SyntaxHighlightKind)];
|
||||
let cases: &[(&str, &str, Expect)] = &[
|
||||
(
|
||||
"zig",
|
||||
"const std = @import(\"std\");\n// hi\npub fn main() void {\n var x: u32 = 42;\n _ = x;\n}\n",
|
||||
&[
|
||||
("const", K::Keyword),
|
||||
("\"std\"", K::String),
|
||||
("// hi", K::Comment),
|
||||
("42", K::Number),
|
||||
("u32", K::Builtin),
|
||||
],
|
||||
),
|
||||
(
|
||||
"nix",
|
||||
"{ pkgs ? import <nixpkgs> {} }:\n# hi\nlet x = 1; in pkgs.mkShell { name = \"dev\"; }\n",
|
||||
&[
|
||||
("let", K::Keyword),
|
||||
("# hi", K::Comment),
|
||||
("1", K::Number),
|
||||
("\"dev\"", K::String),
|
||||
("import", K::Builtin),
|
||||
],
|
||||
),
|
||||
(
|
||||
"graphql",
|
||||
"# hi\nquery Hero($id: ID!) @cached {\n hero(id: $id) { name ...Friends }\n}\nenum Episode { NEWHOPE }\n",
|
||||
&[
|
||||
("# hi", K::Comment),
|
||||
("query", K::Keyword),
|
||||
("Hero", K::FunctionCall),
|
||||
("$id", K::Variable),
|
||||
("ID", K::Type),
|
||||
("@cached", K::Attribute),
|
||||
("name", K::Property),
|
||||
("Friends", K::FunctionCall),
|
||||
("NEWHOPE", K::Constant),
|
||||
],
|
||||
),
|
||||
(
|
||||
"hcl",
|
||||
"# hi\nresource \"aws_instance\" \"web\" {\n ami = var.ami\n count = 2\n tags = { Name = \"web-${var.env}\" }\n name = upper(\"x\")\n}\n",
|
||||
&[
|
||||
("# hi", K::Comment),
|
||||
("resource", K::Keyword),
|
||||
("aws_instance", K::Type),
|
||||
("ami", K::Property),
|
||||
("2", K::Number),
|
||||
("upper", K::FunctionCall),
|
||||
("\"web-", K::String),
|
||||
("env", K::Property),
|
||||
("var", K::Variable),
|
||||
],
|
||||
),
|
||||
("make", "# hi\nCC := gcc\nall: main.o\n\t$(CC) -o app main.o\n", &[("# hi", K::Comment)]),
|
||||
];
|
||||
for (lang, src, expect) in cases {
|
||||
let got = spans(lang, src);
|
||||
for (text, kind) in *expect {
|
||||
assert!(got.iter().any(|(s, k)| s == text && k == kind), "{lang}: {text:?} as {kind:?} in {got:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn languages_without_a_query_have_none() {
|
||||
assert_eq!(crate::Syntax::new().highlights(&ide_core::LanguageId::new("rust"), "fn a() {}"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_highlights_query_compiles() {
|
||||
for spec in super::super::grammars::SPECS {
|
||||
if let Some(h) = &spec.highlights {
|
||||
let lang = Language::new(spec.language);
|
||||
if let Err(e) = HighlightQuery::new(&lang, h.query, h.first_wins) {
|
||||
panic!("{}: {e}", spec.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lua_patterns_translate() {
|
||||
assert_eq!(lua_pattern_to_regex("^[A-Z][A-Z_0-9]+$"), "^[A-Z][A-Z_0-9]+$");
|
||||
assert_eq!(lua_pattern_to_regex("^//!"), "^//!");
|
||||
assert_eq!(lua_pattern_to_regex("^%u[%w_]*$"), "^[A-Z][A-Za-z0-9_]*$");
|
||||
assert_eq!(lua_pattern_to_regex("a.-b%."), "a.*?b\\.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legend_matches_discriminants() {
|
||||
for (i, k) in SyntaxHighlightKind::ALL.iter().enumerate() {
|
||||
|
||||
@@ -57,8 +57,8 @@ impl Grammar {
|
||||
fn highlight_query(&self) -> Option<&highlight::HighlightQuery> {
|
||||
self.highlights
|
||||
.get_or_init(|| {
|
||||
let source = self.spec.highlights?;
|
||||
highlight::HighlightQuery::new(&self.language, source)
|
||||
let h = self.spec.highlights.as_ref()?;
|
||||
highlight::HighlightQuery::new(&self.language, h.query, h.first_wins)
|
||||
.inspect_err(|e| tracing::error!("highlights query for {} is invalid: {e}", self.spec.name))
|
||||
.ok()
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user