use std::borrow::Cow;
use std::collections::HashMap;
use std::collections::hash_map::Entry;
use karet_core::BytePos;
use karet_core::Span;
mod detect;
mod injection;
mod registry;
mod walk;
#[cfg(test)]
mod tests;
pub use detect::language_id_from_injection_name;
pub use detect::language_id_from_path;
pub use detect::language_name_from_path;
pub use injection::InjectionRegion;
pub use walk::WalkControl;
pub use walk::WalkNode;
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum TsError {
#[error("unknown language")]
UnknownLanguage,
#[error("parse failed")]
ParseFailed,
#[error("invalid query: {0}")]
InvalidQuery(String),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct LanguageId(pub u16);
#[must_use]
pub fn highlights_query(lang: LanguageId) -> Option<&'static str> {
registry::grammar(lang).map(|g| g.highlights)
}
#[must_use]
pub fn injections_query(lang: LanguageId) -> Option<Cow<'static, str>> {
let g = registry::grammar(lang)?;
match (g.injections, g.injections_extra) {
(Some(base), Some(extra)) => Some(Cow::Owned(format!("{base}\n{extra}"))),
(Some(q), None) | (None, Some(q)) => Some(Cow::Borrowed(q)),
(None, None) => None,
}
}
#[must_use]
pub fn semantic_query(lang: LanguageId) -> Option<&'static str> {
registry::semantic_query(lang)
}
#[must_use]
pub fn outline_query(lang: LanguageId) -> Option<Cow<'static, str>> {
registry::outline_query(lang)
}
#[derive(Default)]
pub struct ParserPool {
parsers: HashMap<LanguageId, tree_sitter::Parser>,
}
impl ParserPool {
#[must_use]
pub fn new() -> Self {
Self::default()
}
fn parser_for(&mut self, lang: LanguageId) -> Result<&mut tree_sitter::Parser, TsError> {
match self.parsers.entry(lang) {
Entry::Occupied(e) => Ok(e.into_mut()),
Entry::Vacant(e) => {
let info = registry::grammar(lang).ok_or(TsError::UnknownLanguage)?;
let mut parser = tree_sitter::Parser::new();
parser
.set_language(&(info.language)())
.map_err(|_| TsError::UnknownLanguage)?;
Ok(e.insert(parser))
},
}
}
}
pub type Edit = karet_core::AppliedEdit;
pub struct SyntaxTree {
tree: tree_sitter::Tree,
lang: LanguageId,
}
impl SyntaxTree {
pub fn parse(pool: &mut ParserPool, lang: LanguageId, text: &str) -> Result<Self, TsError> {
let parser = pool.parser_for(lang)?;
let tree = parser
.parse(text.as_bytes(), None)
.ok_or(TsError::ParseFailed)?;
Ok(Self { tree, lang })
}
#[must_use]
pub fn language(&self) -> LanguageId {
self.lang
}
#[must_use]
pub fn span(&self) -> Span {
let root = self.tree.root_node();
Span {
start: BytePos(root.start_byte()),
end: BytePos(root.end_byte()),
}
}
#[must_use]
pub fn named_ancestors_at(&self, byte: BytePos) -> Vec<SyntaxNode> {
let root = self.tree.root_node();
let point = byte.0.min(root.end_byte());
let mut node = root.descendant_for_byte_range(point, point).unwrap_or(root);
let mut nodes = Vec::new();
loop {
if node.is_named() {
nodes.push(SyntaxNode {
kind: node.kind().to_owned(),
span: Span {
start: BytePos(node.start_byte()),
end: BytePos(node.end_byte()),
},
});
}
let Some(parent) = node.parent() else {
break;
};
node = parent;
}
nodes
}
#[must_use]
pub fn error_lines(&self) -> Vec<(u32, u32)> {
let root = self.tree.root_node();
if !root.has_error() {
return Vec::new();
}
let mut out = Vec::new();
let mut cursor = root.walk();
let mut stack = vec![root];
while let Some(node) = stack.pop() {
if node.is_error() || node.is_missing() {
let start = node.start_position();
let end = node.end_position();
let start_row = u32::try_from(start.row).unwrap_or(u32::MAX);
let mut end_row = u32::try_from(end.row).unwrap_or(u32::MAX);
if end.column == 0 && end_row > start_row {
end_row -= 1;
}
out.push((start_row, end_row.max(start_row)));
continue;
}
if !node.has_error() {
continue;
}
stack.extend(node.children(&mut cursor));
}
out.sort_unstable();
out
}
pub fn parse_ranges(
pool: &mut ParserPool,
lang: LanguageId,
text: &str,
ranges: &[Span],
) -> Result<Self, TsError> {
Self::parse_ranges_indexed(pool, lang, text, ranges, &karet_core::LineIndex::new(text))
}
fn parse_ranges_indexed(
pool: &mut ParserPool,
lang: LanguageId,
text: &str,
ranges: &[Span],
starts: &karet_core::LineIndex,
) -> Result<Self, TsError> {
if ranges.is_empty() {
return Err(TsError::ParseFailed);
}
let ts_ranges = to_ts_ranges(ranges, starts);
let parser = pool.parser_for(lang)?;
let result = parser
.set_included_ranges(&ts_ranges)
.map_err(|_| TsError::ParseFailed)
.and_then(|()| {
parser
.parse(text.as_bytes(), None)
.ok_or(TsError::ParseFailed)
});
parser.set_included_ranges(&[]).ok();
Ok(Self {
tree: result?,
lang,
})
}
#[must_use]
pub fn injections(&self, query: &Query, text: &str) -> Vec<InjectionRegion> {
injection::extract(&self.tree, query, text)
}
pub fn reparse(&mut self, pool: &mut ParserPool, text: &str) -> Result<(), TsError> {
let parser = pool.parser_for(self.lang)?;
let tree = parser
.parse(text.as_bytes(), Some(&self.tree))
.ok_or(TsError::ParseFailed)?;
self.tree = tree;
Ok(())
}
pub fn edit(&mut self, edit: &Edit) {
self.tree.edit(&tree_sitter::InputEdit {
start_byte: edit.start_byte,
old_end_byte: edit.old_end_byte,
new_end_byte: edit.new_end_byte,
start_position: to_point(edit.start_point),
old_end_position: to_point(edit.old_end_point),
new_end_position: to_point(edit.new_end_point),
});
}
pub fn reparse_with<T, F>(&mut self, pool: &mut ParserPool, mut read: F) -> Result<(), TsError>
where
T: AsRef<[u8]>,
F: FnMut(usize) -> T,
{
let parser = pool.parser_for(self.lang)?;
let mut callback = |byte: usize, _: tree_sitter::Point| read(byte);
let tree = parser
.parse_with_options(&mut callback, Some(&self.tree), None)
.ok_or(TsError::ParseFailed)?;
self.tree = tree;
Ok(())
}
#[must_use]
pub fn changed_ranges(&self, old: &SyntaxTree) -> Vec<Span> {
old.tree
.changed_ranges(&self.tree)
.map(|r| Span {
start: BytePos(r.start_byte),
end: BytePos(r.end_byte),
})
.collect()
}
#[must_use]
pub fn captures(&self, query: &Query, text: &str) -> Vec<RawCapture> {
use tree_sitter::StreamingIterator;
let mut cursor = tree_sitter::QueryCursor::new();
let mut it = cursor.captures(&query.inner, self.tree.root_node(), text.as_bytes());
let mut out = Vec::new();
while let Some((m, idx)) = it.next() {
if let Some(cap) = m.captures.get(*idx) {
out.push(RawCapture {
capture: cap.index,
span: Span {
start: BytePos(cap.node.start_byte()),
end: BytePos(cap.node.end_byte()),
},
});
}
}
out
}
#[must_use]
pub fn matches(&self, query: &Query, text: &str) -> Vec<QueryMatch> {
use tree_sitter::StreamingIterator;
let mut cursor = tree_sitter::QueryCursor::new();
let mut it = cursor.matches(&query.inner, self.tree.root_node(), text.as_bytes());
let mut out = Vec::new();
while let Some(m) = it.next() {
out.push(QueryMatch {
pattern: m.pattern_index,
captures: m
.captures
.iter()
.map(|cap| RawCapture {
capture: cap.index,
span: Span {
start: BytePos(cap.node.start_byte()),
end: BytePos(cap.node.end_byte()),
},
})
.collect(),
});
}
out
}
#[must_use]
pub fn multiline_named_spans(&self) -> Vec<MultilineSpan> {
let mut out = Vec::new();
let mut cursor = self.tree.walk();
'walk: loop {
let node = cursor.node();
if node.is_named() {
let start = node.start_position().row;
let end = node.end_position().row;
if end > start {
out.push(MultilineSpan {
span: Span {
start: BytePos(node.start_byte()),
end: BytePos(node.end_byte()),
},
start_row: start as u32,
end_row: end as u32,
});
}
}
if cursor.goto_first_child() {
continue;
}
loop {
if cursor.goto_next_sibling() {
continue 'walk;
}
if !cursor.goto_parent() {
break 'walk;
}
}
}
out
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SyntaxNode {
pub kind: String,
pub span: Span,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct MultilineSpan {
pub span: Span,
pub start_row: u32,
pub end_row: u32,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct RawCapture {
pub capture: u32,
pub span: Span,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct QueryMatch {
pub pattern: usize,
pub captures: Vec<RawCapture>,
}
pub struct Query {
pub(crate) inner: tree_sitter::Query,
}
impl Query {
pub fn compile(lang: LanguageId, source: &str) -> Result<Self, TsError> {
let info = registry::grammar(lang).ok_or(TsError::UnknownLanguage)?;
let language = (info.language)();
let inner = tree_sitter::Query::new(&language, source)
.map_err(|e| TsError::InvalidQuery(e.to_string()))?;
Ok(Self { inner })
}
#[must_use]
pub fn capture_names(&self) -> &[&str] {
self.inner.capture_names()
}
}
fn to_point(point: karet_core::BytePoint) -> tree_sitter::Point {
let (row, column) = (point.row, point.col);
tree_sitter::Point { row, column }
}
fn point_at(starts: &karet_core::LineIndex, byte: usize) -> tree_sitter::Point {
let (row, column) = starts.point_at(karet_core::BytePos(byte));
tree_sitter::Point {
row: row as usize,
column,
}
}
fn to_ts_ranges(ranges: &[Span], starts: &karet_core::LineIndex) -> Vec<tree_sitter::Range> {
ranges
.iter()
.map(|s| tree_sitter::Range {
start_byte: s.start.0,
end_byte: s.end.0,
start_point: point_at(starts, s.start.0),
end_point: point_at(starts, s.end.0),
})
.collect()
}
const MAX_INJECTION_DEPTH: usize = 4;
const MAX_INJECTION_LAYERS: usize = 4096;
#[derive(Default)]
pub struct LayeredParser {
pool: ParserPool,
injections: HashMap<LanguageId, Option<Query>>,
}
impl LayeredParser {
#[must_use]
pub fn new() -> Self {
Self::default()
}
pub fn pool(&mut self) -> &mut ParserPool {
&mut self.pool
}
pub fn parse(&mut self, lang: LanguageId, text: &str) -> Result<LayeredTree, TsError> {
let root = SyntaxTree::parse(&mut self.pool, lang, text)?;
let children = self.build_layers(&root, text);
Ok(LayeredTree { root, children })
}
pub fn reparse(
&mut self,
tree: &mut LayeredTree,
edits: &[Edit],
text: &str,
) -> Result<(), TsError> {
for edit in edits {
tree.root.edit(edit);
}
tree.root.reparse(&mut self.pool, text)?;
tree.children = self.build_layers(&tree.root, text);
Ok(())
}
fn build_layers(&mut self, root: &SyntaxTree, text: &str) -> Vec<SyntaxTree> {
let starts = karet_core::LineIndex::new(text);
let mut out: Vec<SyntaxTree> = Vec::new();
let mut frontier = self.expand(root, text, &starts);
for _ in 1..=MAX_INJECTION_DEPTH {
if frontier.is_empty() {
break;
}
let room = MAX_INJECTION_LAYERS.saturating_sub(out.len());
if frontier.len() >= room {
frontier.truncate(room);
out.append(&mut frontier);
break;
}
let mut next = Vec::new();
for layer in &frontier {
next.extend(self.expand(layer, text, &starts));
}
out.append(&mut frontier);
frontier = next;
}
out
}
fn expand(
&mut self,
parent: &SyntaxTree,
text: &str,
starts: &karet_core::LineIndex,
) -> Vec<SyntaxTree> {
let Some(regions) = self.regions_of(parent, text) else {
return Vec::new();
};
let mut combined: Vec<(LanguageId, Vec<Span>)> = Vec::new();
let mut separate: Vec<(LanguageId, Vec<Span>)> = Vec::new();
for region in regions {
if region.combined {
match combined.iter_mut().find(|(l, _)| *l == region.lang) {
Some((_, ranges)) => ranges.extend(region.ranges),
None => combined.push((region.lang, region.ranges)),
}
} else {
separate.push((region.lang, region.ranges));
}
}
let mut children = Vec::new();
for (lang, ranges) in combined.into_iter().chain(separate) {
let ranges = injection::normalize(ranges);
if ranges.is_empty() || covers_same_text(parent, &ranges, lang) {
continue;
}
if let Ok(child) =
SyntaxTree::parse_ranges_indexed(&mut self.pool, lang, text, &ranges, starts)
{
children.push(child);
}
}
children
}
fn regions_of(&mut self, tree: &SyntaxTree, text: &str) -> Option<Vec<InjectionRegion>> {
let lang = tree.language();
self.injections.entry(lang).or_insert_with(|| {
let source = injections_query(lang)?;
Query::compile(lang, &source).ok()
});
let query = self.injections.get(&lang)?.as_ref()?;
Some(tree.injections(query, text))
}
}
fn covers_same_text(parent: &SyntaxTree, ranges: &[Span], lang: LanguageId) -> bool {
lang == parent.language()
&& ranges.len() == 1
&& ranges[0].start.0 == parent.tree.root_node().start_byte()
&& ranges[0].end.0 == parent.tree.root_node().end_byte()
}
pub struct LayeredTree {
root: SyntaxTree,
children: Vec<SyntaxTree>,
}
impl LayeredTree {
#[must_use]
pub fn root(&self) -> &SyntaxTree {
&self.root
}
#[must_use]
pub fn children(&self) -> &[SyntaxTree] {
&self.children
}
pub fn layers(&self) -> impl Iterator<Item = &SyntaxTree> {
std::iter::once(&self.root).chain(&self.children)
}
#[must_use]
pub fn error_lines(&self) -> Vec<(u32, u32)> {
let mut out: Vec<(u32, u32)> = self.layers().flat_map(SyntaxTree::error_lines).collect();
out.sort_unstable();
out
}
}