1use std::collections::HashMap;
12
13use streaming_iterator::StreamingIterator;
14pub mod languages;
15
16use ropey::Rope;
17use strop_core::id::BufferRevision;
18use tree_sitter::{Parser, Query, QueryCursor, TextProvider};
19
20#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
23pub enum Class {
24 Keyword,
25 Function,
26 Type,
27 String,
28 Comment,
29 Number,
30 Operator,
31 Punctuation,
32 Constant,
33 Variable,
34 Attribute,
35}
36
37impl Class {
38 fn from_capture(name: &str) -> Self {
39 let head = name.split('.').next().unwrap_or(name);
40 match head {
41 "keyword" => Class::Keyword,
42 "function" | "constructor" => Class::Function,
43 "type" => Class::Type,
44 "string" | "character" => Class::String,
45 "comment" => Class::Comment,
46 "number" | "float" => Class::Number,
47 "operator" => Class::Operator,
48 "punctuation" => Class::Punctuation,
49 "constant" | "boolean" => Class::Constant,
50 "attribute" | "property" => Class::Attribute,
51 _ => Class::Variable,
52 }
53 }
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq)]
58pub struct Span {
59 pub start: usize,
60 pub end: usize,
61 pub class: Class,
62}
63
64#[derive(Debug, Clone, Copy, PartialEq, Eq)]
68pub enum HighlightError {
69 Parse,
73}
74
75impl std::fmt::Display for HighlightError {
76 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
77 match self {
78 HighlightError::Parse => f.write_str("tree-sitter produced no parse tree"),
79 }
80 }
81}
82
83impl std::error::Error for HighlightError {}
84
85struct RopeText<'a> {
90 rope: &'a Rope,
91}
92
93impl<'a> TextProvider<&'a [u8]> for RopeText<'a> {
94 type I = RopeSlices<'a>;
95
96 fn text(&mut self, node: tree_sitter::Node<'_>) -> Self::I {
97 RopeSlices {
98 rope: self.rope,
99 start: node.start_byte(),
100 end: node.end_byte(),
101 }
102 }
103}
104
105struct RopeSlices<'a> {
109 rope: &'a Rope,
110 start: usize,
111 end: usize,
112}
113
114impl<'a> Iterator for RopeSlices<'a> {
115 type Item = &'a [u8];
116
117 fn next(&mut self) -> Option<Self::Item> {
118 if self.start >= self.end {
119 return None;
120 }
121 let (chunk, chunk_start, ..) = self.rope.chunk_at_byte(self.start);
122 let head = &chunk[self.start - chunk_start..];
123 let take = head.len().min(self.end - self.start);
127 let slice = &head.as_bytes()[..take];
128 self.start += take;
129 Some(slice)
130 }
131}
132
133pub struct Highlighter {
138 parser: Parser,
139 query: Query,
140 classes: Vec<Class>,
142 source_hash: BufferRevision,
143 spans: Vec<Span>,
144 tree: Option<tree_sitter::Tree>,
146 tree_revision: BufferRevision,
147}
148
149impl Highlighter {
150 pub fn invalidate(&mut self) {
153 self.tree = None;
154 }
155
156 pub fn apply_edits(&mut self, edits: &[strop_core::InputEdit], revision: BufferRevision) {
162 if revision == self.tree_revision {
163 return;
164 }
165 if let Some(tree) = &mut self.tree {
166 for edit in edits {
167 tree.edit(&tree_sitter::InputEdit {
168 start_byte: edit.start_byte,
169 old_end_byte: edit.old_end_byte,
170 new_end_byte: edit.new_end_byte,
171 start_position: tree_sitter::Point {
172 row: edit.start_point.0,
173 column: edit.start_point.1,
174 },
175 old_end_position: tree_sitter::Point {
176 row: edit.old_end_point.0,
177 column: edit.old_end_point.1,
178 },
179 new_end_position: tree_sitter::Point {
180 row: edit.new_end_point.0,
181 column: edit.new_end_point.1,
182 },
183 });
184 }
185 }
186 self.tree_revision = revision;
189 }
190
191 pub fn for_path(path: &std::path::Path, rope: &Rope) -> Option<Self> {
197 let spec = languages::detect(path, Some(&first_line_bounded(rope)))?;
198 Self::from_spec(spec)
199 }
200
201 fn from_spec(spec: languages::LanguageSpec) -> Option<Self> {
202 let mut parser = Parser::new();
203 parser.set_language(&spec.language).ok()?;
204 let query = Query::new(&spec.language, spec.highlights).ok()?;
205 let classes = query
206 .capture_names()
207 .iter()
208 .map(|n| Class::from_capture(n))
209 .collect();
210 Some(Self {
211 parser,
212 query,
213 classes,
214 source_hash: BufferRevision::from(u64::MAX), spans: Vec::new(),
216 tree: None,
217 tree_revision: BufferRevision::new(0),
218 })
219 }
220
221 pub fn highlight(
228 &mut self,
229 rope: &Rope,
230 revision: BufferRevision,
231 first_byte: usize,
232 last_byte: usize,
233 ) -> Result<Vec<Span>, HighlightError> {
234 if revision != self.source_hash {
235 let tree = self
239 .parser
240 .parse_with_options(
241 &mut |byte: usize, _| {
242 if byte >= rope.len_bytes() {
247 return "";
248 }
249 let (chunk, start, _, _) = rope.chunk_at_byte(byte);
250 &chunk[byte - start..]
251 },
252 self.tree.as_ref(),
253 None,
254 )
255 .ok_or(HighlightError::Parse)?;
256 self.tree = Some(tree.clone());
257 self.tree_revision = revision;
258 let mut cursor = QueryCursor::new();
262 let mut by_byte: HashMap<usize, (usize, Class)> = HashMap::new();
263 let mut matches = cursor.matches(&self.query, tree.root_node(), RopeText { rope });
264 while let Some(m) = { StreamingIterator::next(&mut matches) } {
265 for cap in m.captures {
266 let node = cap.node;
267 let class = self.classes[cap.index as usize];
268 let entry = by_byte
270 .entry(node.start_byte())
271 .or_insert((node.end_byte(), class));
272 if node.end_byte() - node.start_byte() <= entry.0 - node.start_byte() {
273 *entry = (node.end_byte(), class);
274 }
275 }
276 }
277 let mut spans: Vec<Span> = by_byte
278 .into_iter()
279 .map(|(start, (end, class))| Span { start, end, class })
280 .collect();
281 spans.sort_by_key(|s| (s.start, s.end));
282 self.spans = spans;
283 self.source_hash = revision;
284 }
285 let lo = self.spans.partition_point(|s| s.end <= first_byte);
287 let hi = self.spans.partition_point(|s| s.start < last_byte);
288 Ok(self.spans[lo..hi.max(lo)].to_vec())
289 }
290}
291
292fn cut_at_boundary(head: &str, want: usize) -> usize {
296 let mut take = want.min(head.len());
297 while take > 0 && !head.is_char_boundary(take) {
298 take -= 1;
299 }
300 take
301}
302
303fn first_line_bounded(rope: &Rope) -> String {
308 const CAP: usize = 256;
309 if rope.len_bytes() == 0 || rope.byte(0) != b'#' {
311 return String::new();
312 }
313 let limit = rope.len_bytes().min(CAP);
314 let mut line = String::new();
315 let mut byte = 0;
316 while byte < limit {
317 let (chunk, start, ..) = rope.chunk_at_byte(byte);
318 let head = &chunk[byte - start..];
319 let stop = head.find('\n').unwrap_or(head.len());
320 let take = cut_at_boundary(head, stop.min(limit - byte));
321 if take == 0 {
322 break; }
324 line.push_str(&head[..take]);
325 if take == stop {
326 break; }
328 byte += take;
329 }
330 line
331}
332
333#[cfg(test)]
334mod tests {
335 use super::*;
336 use strop_core::{Buffer, Range};
337
338 fn clean_spans(path: &std::path::Path, rope: &Rope) -> Vec<Span> {
339 let mut hl = Highlighter::for_path(path, rope).expect("language");
340 hl.highlight(rope, BufferRevision::new(0), 0, rope.len_bytes())
341 .expect("parse")
342 }
343
344 fn classes_for(path: &std::path::Path, src: &str) -> Vec<Class> {
345 clean_spans(path, &Rope::from_str(src))
346 .iter()
347 .map(|s| s.class)
348 .collect()
349 }
350
351 fn apply_and_highlight(
356 buf: &mut Buffer,
357 hl: &mut Highlighter,
358 (start, end): (usize, usize),
359 text: &str,
360 ) -> Vec<Span> {
361 buf.edit()
362 .replace(Range::charwise(start, end), text)
363 .expect("edit");
364 for change in buf.changes() {
365 hl.apply_edits(std::slice::from_ref(&change.edit), change.revision);
366 }
367 buf.clear_changes();
368 hl.highlight(buf.text(), buf.revision(), 0, buf.len_bytes())
369 .expect("parse")
370 }
371
372 #[test]
373 fn rust_keywords_and_strings() {
374 let classes = classes_for(
375 std::path::Path::new("x.rs"),
376 "fn main() { let s = \"hi\"; }\n",
377 );
378 assert!(classes.contains(&Class::Keyword), "{classes:?}");
379 assert!(classes.contains(&Class::String), "{classes:?}");
380 }
381
382 #[test]
383 fn cpp_highlights_with_cxx_scanner() {
384 let classes = classes_for(std::path::Path::new("x.cpp"), "auto edge = hone(blade);\n");
387 assert!(!classes.is_empty(), "cpp grammar produced no spans");
388 assert!(classes.contains(&Class::Type), "{classes:?}"); }
390
391 #[test]
392 fn python_and_go_and_ts() {
393 assert!(
394 classes_for(std::path::Path::new("x.py"), "def f(x):\n return x\n")
395 .contains(&Class::Keyword)
396 );
397 assert!(classes_for(
398 std::path::Path::new("x.go"),
399 "package main\nfunc main() {}\n"
400 )
401 .contains(&Class::Keyword));
402 assert!(!classes_for(std::path::Path::new("x.ts"), "const x: number = 1;\n").is_empty());
403 assert!(!classes_for(std::path::Path::new("x.json"), "{\"a\": 1}\n").is_empty());
404 assert!(!classes_for(std::path::Path::new("x.sh"), "#!/bin/sh\necho hi\n").is_empty());
405 }
406
407 #[test]
408 fn fish_lua_and_sql() {
409 assert!(!classes_for(std::path::Path::new("x.fish"), "set -l name rust\n").is_empty());
412 assert!(
413 classes_for(std::path::Path::new("x.lua"), "local x = 1\n").contains(&Class::Keyword)
414 );
415 assert!(
416 classes_for(std::path::Path::new("x.sql"), "SELECT * FROM users;\n")
417 .contains(&Class::Keyword)
418 );
419 }
420
421 #[test]
422 fn for_path_is_pure_over_path_and_rope() {
423 let src = "#!/usr/bin/env bash\necho hi\n";
426 let rope = Rope::from_str(src);
427 let mut hl =
428 Highlighter::for_path(std::path::Path::new("strop-syntax-purity-probe"), &rope)
429 .expect("bash via rope shebang");
430 assert!(!hl
431 .highlight(&rope, BufferRevision::new(0), 0, rope.len_bytes())
432 .expect("parse")
433 .is_empty());
434 }
435
436 #[test]
437 fn shebang_detection_is_bounded_at_256_bytes() {
438 let over = format!("#!/bin/{}\nls\n", "b".repeat(300));
440 let rope = Rope::from_str(&over);
441 assert!(Highlighter::for_path(std::path::Path::new("probe"), &rope).is_none());
442 let under = Rope::from_str("#!/bin/bash\nls\n");
444 assert!(Highlighter::for_path(std::path::Path::new("probe"), &under).is_some());
445 let wide = format!("#!/bin/{}\nls\n", "é".repeat(200));
447 let rope = Rope::from_str(&wide);
448 assert!(Highlighter::for_path(std::path::Path::new("probe"), &rope).is_none());
449 }
450
451 #[test]
452 fn first_line_assembly_walks_rope_chunks() {
453 let mut line = String::from("#!/usr/bin/env bash");
455 line.push_str(&" # padding ".repeat(600));
456 let rope = Rope::from_str(&format!("{line}\nls\n"));
457 assert!(
458 rope.chunks().count() > 1,
459 "precondition: multi-chunk first line"
460 );
461 let got = first_line_bounded(&rope);
462 assert!(got.starts_with("#!/usr/bin/env bash"));
463 assert!(line.starts_with(got.as_str()), "bounded prefix of the line");
464 assert_eq!(got.len(), 256, "ASCII content fills the cap exactly");
465 }
466
467 #[test]
468 fn window_and_cache_semantics_hold() {
469 let rope = Rope::from_str("fn main() { let s = \"hi\"; let n = 7; }\n");
470 let path = std::path::Path::new("x.rs");
471 let mut hl = Highlighter::for_path(path, &rope).unwrap();
472 let full = hl
473 .highlight(&rope, BufferRevision::new(0), 0, rope.len_bytes())
474 .unwrap();
475 assert!(!full.is_empty());
476 let anchor = full
478 .iter()
479 .find(|s| s.class == Class::String)
480 .expect("a string span");
481 let (w0, w1) = (anchor.start - 1, anchor.end + 1);
482 let window = hl.highlight(&rope, BufferRevision::new(0), w0, w1).unwrap();
483 let expect: Vec<Span> = full
484 .iter()
485 .copied()
486 .filter(|s| s.end > w0 && s.start < w1)
487 .collect();
488 assert_eq!(window, expect);
489 let again = hl
491 .highlight(&rope, BufferRevision::new(0), 0, rope.len_bytes())
492 .unwrap();
493 assert_eq!(again, full);
494 }
495
496 #[test]
497 fn incremental_replacement_matches_clean_parse() {
498 let path = std::path::Path::new("x.rs");
502 let mut buf = Buffer::from_text("fn main() { let s = \"hi\"; let t = 2; }\n");
503 let mut hl = Highlighter::for_path(path, buf.text()).unwrap();
504 let warm = hl
505 .highlight(buf.text(), buf.revision(), 0, buf.len_bytes())
506 .unwrap();
507 assert!(warm.iter().any(|s| s.class == Class::String));
508
509 let start = buf.text().to_string().find("\"hi\"").unwrap();
510 let got = apply_and_highlight(&mut buf, &mut hl, (start, start + 4), "\"wörld → 🌍\"");
511 assert!(got.iter().any(|s| s.class == Class::String));
512 assert_eq!(got, clean_spans(path, buf.text()));
513 }
514
515 #[test]
516 fn incremental_multiline_edit_matches_clean_parse() {
517 let path = std::path::Path::new("x.rs");
520 let mut buf = Buffer::from_text("fn a() {}\nfn b() {}\n");
521 let mut hl = Highlighter::for_path(path, buf.text()).unwrap();
522 hl.highlight(buf.text(), buf.revision(), 0, buf.len_bytes())
523 .unwrap();
524
525 let got = apply_and_highlight(&mut buf, &mut hl, (7, 7), " let s = \"one\ntwo 🌍\";");
526 assert_eq!(got, clean_spans(path, buf.text()));
527
528 let text = buf.text().to_string();
529 let start = text.find("\"one\ntwo 🌍\"").unwrap();
530 let got = apply_and_highlight(
531 &mut buf,
532 &mut hl,
533 (start, start + "\"one\ntwo 🌍\"".len()),
534 "\"x\"",
535 );
536 assert_eq!(got, clean_spans(path, buf.text()));
537 }
538
539 #[test]
540 fn incremental_undo_roundtrip_matches_clean_parse() {
541 let path = std::path::Path::new("x.rs");
545 let original = "fn main() { let x = 1; }\n";
546 let mut buf = Buffer::from_text(original);
547 let mut hl = Highlighter::for_path(path, buf.text()).unwrap();
548 hl.highlight(buf.text(), buf.revision(), 0, buf.len_bytes())
549 .unwrap();
550
551 let start = original.find("1").unwrap();
552 let forward = apply_and_highlight(&mut buf, &mut hl, (start, start + 1), "0x1f 🌍");
553 assert_eq!(forward, clean_spans(path, buf.text()));
554
555 let back = apply_and_highlight(&mut buf, &mut hl, (start, start + "0x1f 🌍".len()), "1");
556 assert_eq!(back, clean_spans(path, buf.text()));
557 assert_eq!(buf.text().to_string(), original);
558 }
559
560 #[test]
561 fn predicates_evaluate_across_rope_chunk_boundaries() {
562 const CAPS: &str = "A_VERY_LONG_CAPS_IDENTIFIER_STRADDLING_ROPE_CHUNKS";
569 let tail = "fn tail() { let filler = \"padding to a multi-chunk rope\"; }\n".repeat(90);
570 let mut found = false;
571 for pad in (0..1050usize).step_by(11) {
572 let src = format!(
573 "fn main() {{\n //{}\n let {} = 1;\n let {}_use = {};\n}}\n{}",
574 "x".repeat(pad),
575 CAPS,
576 CAPS.to_lowercase(),
577 CAPS,
578 tail,
579 );
580 let rope = Rope::from_str(&src);
581 let start = src.find(CAPS).unwrap();
582 let end = start + CAPS.len();
583 if !spans_chunks(&rope, start, end) {
584 continue;
585 }
586 let spans = clean_spans(std::path::Path::new("x.rs"), &rope);
587 let class = spans.iter().find(|s| s.start == start).map(|s| s.class);
588 assert_eq!(
589 class,
590 Some(Class::Constant),
591 "pad {pad}: predicate must see the whole identifier across chunks"
592 );
593 found = true;
594 break;
595 }
596 assert!(
597 found,
598 "the sweep never produced a chunk-straddling identifier"
599 );
600 }
601
602 fn spans_chunks(rope: &Rope, start: usize, end: usize) -> bool {
605 let mut offset = 0;
606 for chunk in rope.chunks() {
607 if offset > start && offset < end {
608 return true;
609 }
610 offset += chunk.len();
611 }
612 false
613 }
614
615 #[test]
616 fn incremental_on_multichunk_rope_matches_clean_parse() {
617 let path = std::path::Path::new("x.rs");
620 let mut big = String::from("fn top() {}\n");
621 for i in 0..80 {
622 big.push_str(&format!("fn f{i}() {{ let s{i} = \"{i}\"; }}\n"));
623 }
624 big.push_str("fn bottom() {}\n");
625 let mut buf = Buffer::from_text(&big);
626 let mut hl = Highlighter::for_path(path, buf.text()).unwrap();
627 assert!(
628 buf.text().chunks().count() > 1,
629 "precondition: multi-chunk rope"
630 );
631 hl.highlight(buf.text(), buf.revision(), 0, buf.len_bytes())
632 .unwrap();
633
634 let mid = big.len() / 2;
635 let line_start = big[..mid].rfind('\n').map(|i| i + 1).unwrap_or(0);
636 let got = apply_and_highlight(
637 &mut buf,
638 &mut hl,
639 (line_start, line_start),
640 "let inserted_mid_rope = \"x\";\n",
641 );
642 assert_eq!(got, clean_spans(path, buf.text()));
643 }
644
645 #[test]
646 fn highlight_survives_backtracking_requests() {
647 let mut big = String::from("namespace std {\n");
651 for i in 0..400 {
652 big.push_str(&format!(
653 "template <typename T{i}> struct O{i} {{ T{i} v; O{i} f() {{ return O{i}{{}}; }} }};\n"
654 ));
655 }
656 big.push_str("}\n");
657 let rope = ropey::Rope::from_str(&big);
658 let mut hl = Highlighter::for_path(std::path::Path::new("x.hpp"), &rope).unwrap();
659 let spans = hl
660 .highlight(&rope, BufferRevision::new(0), 0, rope.len_bytes())
661 .unwrap();
662 assert!(!spans.is_empty(), "the big file highlights");
663 let edited = big.replacen("namespace", "namespace extra_long_name_here", 1);
666 let rope2 = ropey::Rope::from_str(&edited);
667 let spans2 = hl
668 .highlight(&rope2, BufferRevision::new(1), 0, rope2.len_bytes())
669 .unwrap();
670 assert!(!spans2.is_empty());
671 }
672}