use std::fmt;
use lanekeep_core::Position;
use lanekeep_lang::{Language, LanguageId};
use streaming_iterator::StreamingIterator;
use thiserror::Error;
use tree_sitter::{Node, Query, QueryCursor, QueryErrorKind, Tree};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CompileErrorKind {
Syntax,
UnknownNodeKind,
UnknownField,
UnknownCapture,
ImpossiblePattern,
NoCaptures,
UnsupportedPredicate,
Other,
}
impl CompileErrorKind {
const fn describe(self) -> &'static str {
match self {
Self::Syntax => "the query is not valid s-expression syntax",
Self::UnknownNodeKind => "no such node kind in this grammar",
Self::UnknownField => "no such field in this grammar",
Self::UnknownCapture => "the query refers to a capture it never binds",
Self::ImpossiblePattern => "this pattern can never match",
Self::NoCaptures => "the query binds no captures",
Self::UnsupportedPredicate => "the query uses a predicate that is never applied",
Self::Other => "the grammar rejected this query",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Error)]
pub struct CompileError {
pub kind: CompileErrorKind,
pub language: LanguageId,
pub position: Position,
pub offset: usize,
pub detail: String,
pub line: String,
}
impl fmt::Display for CompileError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
writeln!(f, "query error: {}", self.kind.describe())?;
if !self.detail.is_empty() {
match self.kind {
CompileErrorKind::UnknownNodeKind => writeln!(
f,
" the {} grammar has no node kind `{}`",
self.language, self.detail
)?,
CompileErrorKind::UnknownField => writeln!(
f,
" the {} grammar has no field `{}`",
self.language, self.detail
)?,
CompileErrorKind::UnsupportedPredicate => writeln!(
f,
" the predicate `{}` is parsed but never applied; remove it",
self.detail
)?,
_ => writeln!(f, " {}", self.detail)?,
}
}
if !self.line.is_empty() {
let gutter = format!("{}", self.position.line);
let pad = " ".repeat(gutter.len());
writeln!(
f,
"{pad} --> query:{}:{}",
self.position.line, self.position.column
)?;
writeln!(f, "{pad} |")?;
writeln!(f, "{gutter} | {}", self.line)?;
let caret_pad = " ".repeat(self.position.column.saturating_sub(1) as usize);
writeln!(f, "{pad} | {caret_pad}^")?;
}
Ok(())
}
}
#[derive(Debug)]
pub struct CompiledQuery {
query: Query,
language: LanguageId,
capture_names: Vec<String>,
}
impl CompiledQuery {
pub fn compile(language: &dyn Language, source: &str) -> Result<Self, CompileError> {
let grammar = language.grammar();
let id = language.id();
let query = Query::new(&grammar, source).map_err(|err| {
let kind = match err.kind {
QueryErrorKind::Syntax => CompileErrorKind::Syntax,
QueryErrorKind::NodeType => CompileErrorKind::UnknownNodeKind,
QueryErrorKind::Field => CompileErrorKind::UnknownField,
QueryErrorKind::Capture => CompileErrorKind::UnknownCapture,
QueryErrorKind::Structure => CompileErrorKind::ImpossiblePattern,
_ => CompileErrorKind::Other,
};
let detail = match kind {
CompileErrorKind::Syntax => String::new(),
_ => err.message.trim_matches('"').to_owned(),
};
CompileError {
kind,
language: id,
position: Position::new(
u32::try_from(err.row).unwrap_or(u32::MAX).saturating_add(1),
u32::try_from(err.column)
.unwrap_or(u32::MAX)
.saturating_add(1),
),
offset: err.offset,
detail,
line: source.lines().nth(err.row).unwrap_or_default().to_owned(),
}
})?;
for pattern in 0..query.pattern_count() {
let operator = query
.general_predicates(pattern)
.first()
.map(|predicate| predicate.operator.to_string())
.or_else(|| {
query
.property_predicates(pattern)
.first()
.map(|(_, is_positive)| {
if *is_positive { "is?" } else { "is-not?" }.to_owned()
})
})
.or_else(|| {
query
.property_settings(pattern)
.first()
.map(|_| "set!".to_owned())
});
if let Some(operator) = operator {
let start = query.start_byte_for_pattern(pattern);
let end = query.end_byte_for_pattern(pattern);
let needle = format!("#{operator}");
let offset = source[start..end]
.find(&needle)
.map_or(start, |rel| start + rel);
let (position, line) = position_at(source, offset);
return Err(CompileError {
kind: CompileErrorKind::UnsupportedPredicate,
language: id,
position,
offset,
detail: needle,
line: line.to_owned(),
});
}
}
let capture_names: Vec<String> = query
.capture_names()
.iter()
.map(|name| (*name).to_owned())
.collect();
if capture_names.is_empty() {
return Err(CompileError {
kind: CompileErrorKind::NoCaptures,
language: id,
position: Position::START,
offset: 0,
detail: "add a capture such as `@match` so the handler can reference the node"
.to_owned(),
line: source.lines().next().unwrap_or_default().to_owned(),
});
}
Ok(Self {
query,
language: id,
capture_names,
})
}
#[must_use]
pub const fn language(&self) -> LanguageId {
self.language
}
#[must_use]
pub fn capture_names(&self) -> &[String] {
&self.capture_names
}
#[must_use]
pub fn pattern_count(&self) -> usize {
self.query.pattern_count()
}
pub fn for_each_match<'tree>(
&self,
tree: &'tree Tree,
source: &[u8],
visit: impl FnMut(QueryMatch<'_, 'tree>),
) {
self.for_each_match_in(tree.root_node(), source, visit);
}
pub fn for_each_match_in<'tree>(
&self,
node: Node<'tree>,
source: &[u8],
mut visit: impl FnMut(QueryMatch<'_, 'tree>),
) {
let mut cursor = QueryCursor::new();
let mut matches = cursor.matches(&self.query, node, source);
while let Some(m) = matches.next() {
let captures = m
.captures
.iter()
.map(|capture| {
let name = self
.capture_names
.get(capture.index as usize)
.map_or("", String::as_str);
(name, capture.node)
})
.collect();
visit(QueryMatch {
pattern_index: m.pattern_index,
captures,
});
}
}
}
#[derive(Debug, Clone)]
pub struct QueryMatch<'q, 'tree> {
pub pattern_index: usize,
pub captures: Vec<(&'q str, Node<'tree>)>,
}
impl<'tree> QueryMatch<'_, 'tree> {
#[must_use]
pub fn get(&self, name: &str) -> Option<Node<'tree>> {
self.captures
.iter()
.find(|(n, _)| *n == name)
.map(|(_, node)| *node)
}
pub fn get_all<'a>(&'a self, name: &'a str) -> impl Iterator<Item = Node<'tree>> + 'a {
self.captures
.iter()
.filter(move |(n, _)| *n == name)
.map(|(_, node)| *node)
}
}
fn position_at(source: &str, offset: usize) -> (Position, &str) {
let before = &source[..offset.min(source.len())];
let line = u32::try_from(before.bytes().filter(|&b| b == b'\n').count())
.unwrap_or(u32::MAX)
.saturating_add(1);
let last_nl = before.rfind('\n').map_or(0, |i| i + 1);
let column = u32::try_from(before[last_nl..].chars().count())
.unwrap_or(u32::MAX)
.saturating_add(1);
let line_text = source.lines().nth((line - 1) as usize).unwrap_or_default();
(Position::new(line, column), line_text)
}
#[cfg(test)]
mod tests {
use lanekeep_lang_js::{JavaScript, Tsx, TypeScript};
use super::*;
fn parse(language: &dyn Language, source: &str) -> Tree {
let mut parser = tree_sitter::Parser::new();
parser
.set_language(&language.grammar())
.expect("grammar loads");
parser.parse(source, None).expect("parser returns a tree")
}
fn compile(source: &str) -> CompiledQuery {
CompiledQuery::compile(&TypeScript, source).expect("query compiles")
}
fn compile_err(source: &str) -> CompileError {
CompiledQuery::compile(&TypeScript, source).expect_err("query should not compile")
}
fn run(query: &CompiledQuery, source: &str) -> Vec<Vec<String>> {
let tree = parse(&TypeScript, source);
let mut out = Vec::new();
query.for_each_match(&tree, source.as_bytes(), |m| {
out.push(
m.captures
.iter()
.map(|(name, node)| {
let text = node.utf8_text(source.as_bytes()).unwrap_or("<invalid>");
format!("{name}={text}")
})
.collect(),
);
});
out
}
#[test]
fn compiles_a_simple_query() {
let query = compile("(identifier) @id");
assert_eq!(query.capture_names(), ["id"]);
assert_eq!(query.pattern_count(), 1);
assert_eq!(query.language().as_str(), "typescript");
}
#[test]
fn reports_capture_names_in_index_order() {
let query =
compile("(pair key: (property_identifier) @prop value: (number) @value) @match");
assert_eq!(query.capture_names(), ["prop", "value", "match"]);
}
#[test]
fn matches_expose_captures_by_name() {
let query =
compile("(pair key: (property_identifier) @prop value: (number) @value) @match");
let matches = run(&query, "const s = { padding: 12, margin: 4 };");
assert_eq!(matches.len(), 2);
assert!(matches[0].contains(&"prop=padding".to_owned()));
assert!(matches[0].contains(&"value=12".to_owned()));
assert!(matches[1].contains(&"prop=margin".to_owned()));
assert!(matches[1].contains(&"value=4".to_owned()));
}
#[test]
fn get_returns_the_node_for_a_capture() {
let query = compile("(pair key: (property_identifier) @prop) @match");
let source = "const s = { padding: 12 };";
let tree = parse(&TypeScript, source);
let mut seen = Vec::new();
query.for_each_match(&tree, source.as_bytes(), |m| {
let prop = m.get("prop").expect("prop is bound");
seen.push(
prop.utf8_text(source.as_bytes())
.unwrap_or_default()
.to_owned(),
);
assert!(m.get("nope").is_none(), "unbound capture must be None");
});
assert_eq!(seen, ["padding"]);
}
#[test]
fn match_order_is_deterministic() {
let query = compile("(identifier) @id");
let source = "const alpha = 1; const beta = 2; function gamma() { return delta }";
let first = run(&query, source);
for _ in 0..25 {
assert_eq!(
run(&query, source),
first,
"match order varied between runs"
);
}
assert!(
first.len() >= 4,
"expected several matches, got {}",
first.len()
);
}
#[test]
fn a_query_matching_nothing_yields_no_matches() {
let query = compile("(class_declaration) @c");
assert!(run(&query, "const x = 1;").is_empty());
}
#[test]
fn handles_an_empty_source_file() {
let query = compile("(identifier) @id");
assert!(run(&query, "").is_empty());
}
#[test]
fn rejects_a_query_with_no_captures() {
let err = compile_err("(identifier)");
assert_eq!(err.kind, CompileErrorKind::NoCaptures);
assert!(
err.to_string().contains("@match"),
"should suggest adding a capture"
);
}
#[test]
fn rejects_an_unknown_node_kind_with_a_useful_message() {
let err = compile_err("(nonexistent_node) @x");
assert_eq!(err.kind, CompileErrorKind::UnknownNodeKind);
assert_eq!(err.detail, "nonexistent_node");
let rendered = err.to_string();
assert!(
rendered.contains("typescript"),
"should name the grammar: {rendered}"
);
assert!(
rendered.contains("nonexistent_node"),
"should name the node: {rendered}"
);
assert!(
rendered.contains("-->"),
"should point at a position: {rendered}"
);
assert!(rendered.contains('^'), "should carry a caret: {rendered}");
}
#[test]
fn rejects_an_unknown_field() {
let err = compile_err("(pair nonexistent_field: (number) @n) @m");
assert_eq!(err.kind, CompileErrorKind::UnknownField);
assert_eq!(err.detail, "nonexistent_field");
assert!(err.to_string().contains("no field"), "{err}");
}
#[test]
fn rejects_malformed_syntax() {
let err = compile_err("(pair key: (property_identifier) @a");
assert_eq!(err.kind, CompileErrorKind::Syntax);
}
#[test]
fn points_at_the_right_line_of_a_multiline_query() {
let err = CompiledQuery::compile(
&TypeScript,
"(pair\n key: (property_identifier) @prop\n value: (nonexistent_node) @v) @m",
)
.expect_err("should not compile");
assert_eq!(err.position.line, 3, "should point at the third line");
assert!(
err.line.contains("nonexistent_node"),
"excerpt should be that line: {err:?}"
);
let rendered = err.to_string();
assert!(rendered.contains("query:3:"), "{rendered}");
let caret_line = rendered.lines().last().unwrap_or_default();
let caret_col = caret_line.find('^').unwrap_or(0);
assert!(
caret_col > 4,
"caret should be indented to the token: {rendered}"
);
}
#[test]
fn compiles_against_each_language() {
let jsx = "(jsx_element) @el";
assert!(
CompiledQuery::compile(&Tsx, jsx).is_ok(),
"TSX should know jsx_element"
);
assert!(
CompiledQuery::compile(&JavaScript, jsx).is_ok(),
"JS should know jsx_element"
);
let err = CompiledQuery::compile(&TypeScript, jsx)
.expect_err("plain TypeScript has no JSX nodes");
assert_eq!(err.kind, CompileErrorKind::UnknownNodeKind);
assert_eq!(err.language.as_str(), "typescript");
let types = "(type_annotation) @t";
assert!(CompiledQuery::compile(&TypeScript, types).is_ok());
assert!(
CompiledQuery::compile(&JavaScript, types).is_err(),
"JavaScript has no type annotations"
);
}
#[test]
fn supports_alternations_and_multiple_patterns() {
let query = compile("[(number) (string)] @literal");
let matches = run(&query, "const a = 1; const b = 'two';");
assert_eq!(matches.len(), 2);
let two = compile("(number) @n\n(string) @s");
assert_eq!(two.pattern_count(), 2);
assert_eq!(two.capture_names(), ["n", "s"]);
}
#[test]
fn get_all_returns_every_binding_of_a_repeated_capture() {
let query = compile("(object (pair) @entry) @obj");
let source = "const s = { a: 1, b: 2, c: 3 };";
let tree = parse(&TypeScript, source);
let mut counts = Vec::new();
query.for_each_match(&tree, source.as_bytes(), |m| {
counts.push(m.get_all("entry").count());
assert!(m.get("entry").is_some());
});
assert!(!counts.is_empty(), "expected at least one match");
}
#[test]
fn nodes_carry_positions_usable_for_reporting() {
let query = compile("(number) @n");
let source = "const a = 1;\nconst b = 22;";
let tree = parse(&TypeScript, source);
let mut positions = Vec::new();
query.for_each_match(&tree, source.as_bytes(), |m| {
let node = m.get("n").expect("bound");
let start = node.start_position();
positions.push((start.row + 1, start.column + 1));
});
assert_eq!(positions, [(1, 11), (2, 11)]);
}
#[test]
fn text_predicates_filter_matches() {
let source = "const alpha = 1; const beta = 2;";
let query = compile("((identifier) @id (#eq? @id \"alpha\"))");
assert_eq!(run(&query, source), vec![vec!["id=alpha".to_owned()]]);
let query = compile("((identifier) @id (#not-eq? @id \"alpha\"))");
assert_eq!(run(&query, source), vec![vec!["id=beta".to_owned()]]);
let query = compile("((identifier) @id (#match? @id \"^a\"))");
assert_eq!(run(&query, source), vec![vec!["id=alpha".to_owned()]]);
let query = compile("((identifier) @id (#not-match? @id \"^a\"))");
assert_eq!(run(&query, source), vec![vec!["id=beta".to_owned()]]);
let source = "const alpha = 1; const beta = 2; const gamma = 3;";
let query = compile("((identifier) @id (#any-of? @id \"alpha\" \"gamma\"))");
assert_eq!(
run(&query, source),
vec![vec!["id=alpha".to_owned()], vec!["id=gamma".to_owned()]]
);
let query = compile("((identifier) @id (#not-any-of? @id \"alpha\" \"gamma\"))");
assert_eq!(run(&query, source), vec![vec!["id=beta".to_owned()]]);
}
#[test]
fn rejects_a_general_predicate_naming_the_operator() {
let err = compile_err("((identifier) @id (#is? @id \"x\"))");
assert_eq!(err.kind, CompileErrorKind::UnsupportedPredicate);
let rendered = err.to_string();
assert!(
rendered.contains("#is?"),
"should name the operator: {rendered}"
);
assert!(
rendered.contains("-->"),
"should point at a position: {rendered}"
);
assert!(rendered.contains('^'), "should carry a caret: {rendered}");
let err = compile_err("((identifier) @id (#set! @id \"x\"))");
assert_eq!(err.kind, CompileErrorKind::UnsupportedPredicate);
assert!(err.to_string().contains("#set!"), "{}", err);
let err = compile_err("((identifier) @id (#is-not? @id \"x\"))");
assert_eq!(err.kind, CompileErrorKind::UnsupportedPredicate);
assert!(err.to_string().contains("#is-not?"), "{}", err);
let err = compile_err("((identifier) @id (#foo? @id \"x\"))");
assert_eq!(err.kind, CompileErrorKind::UnsupportedPredicate);
assert!(err.to_string().contains("#foo?"), "{}", err);
}
#[test]
fn general_predicate_error_points_at_the_operator() {
let err = CompiledQuery::compile(
&TypeScript,
"((pair\n key: (property_identifier) @prop\n value: (number) @v) @m\n (#is? @prop \"x\"))",
)
.expect_err("should not compile");
assert_eq!(err.kind, CompileErrorKind::UnsupportedPredicate);
assert_eq!(err.position.line, 4, "should point at the fourth line");
assert!(
err.line.contains("#is?"),
"excerpt should be that line: {err:?}"
);
let rendered = err.to_string();
assert!(rendered.contains("query:4:"), "{rendered}");
let caret_line = rendered.lines().last().unwrap_or_default();
let caret_col = caret_line.find('^').unwrap_or(0);
assert!(
caret_col > 2,
"caret should be indented to the token: {rendered}"
);
}
}