1#![doc = include_str!("../README.md")]
2
3use std::num::NonZeroU8;
4use std::sync::Arc;
5use std::sync::atomic::AtomicUsize;
6
7use termosaic::{Doc, TokenId, tokens};
8use tree_sitter::{Language, Node, QueryError, Tree};
9use tree_sitter_highlight::{Highlight, HighlightConfiguration, HighlightEvent, Highlighter};
10use unicode_width::UnicodeWidthStr;
11
12pub use tree_sitter;
13pub use tree_sitter_highlight;
14
15#[derive(Clone, Debug, Eq, PartialEq)]
16struct CaptureMapping {
17 name: String,
18 tokens: Box<[TokenId]>,
19}
20
21#[derive(Debug, Eq, PartialEq)]
22struct CaptureMapData {
23 entries: Box<[CaptureMapping]>,
24}
25
26#[derive(Clone, Debug, Eq, PartialEq)]
28pub struct CaptureMap(Arc<CaptureMapData>);
29
30impl CaptureMap {
31 pub fn builder() -> CaptureMapBuilder {
32 CaptureMapBuilder::default()
33 }
34
35 pub fn standard() -> Self {
37 let mut builder = CaptureMapBuilder::default();
38 for (name, token) in standard_captures() {
39 builder.push_static(name, token);
40 }
41 builder.build()
42 }
43
44 pub fn entries(&self) -> impl ExactSizeIterator<Item = (&str, &[TokenId])> {
45 self.0
46 .entries
47 .iter()
48 .map(|entry| (entry.name.as_str(), entry.tokens.as_ref()))
49 }
50
51 pub fn customize(&self) -> CaptureMapBuilder {
52 CaptureMapBuilder {
53 entries: self.0.entries.to_vec(),
54 }
55 }
56
57 pub fn is_compatible_with(&self, other: &Self) -> bool {
63 self == other
64 }
65
66 fn configure(&self, configuration: &mut HighlightConfiguration) {
67 let names = self
68 .0
69 .entries
70 .iter()
71 .map(|entry| entry.name.as_str())
72 .collect::<Vec<_>>();
73 configuration.configure(&names);
74 }
75
76 fn tokens(&self, highlight: Highlight) -> Option<&[TokenId]> {
77 self.0
78 .entries
79 .get(highlight.0)
80 .map(|entry| entry.tokens.as_ref())
81 }
82}
83
84#[derive(Default)]
85pub struct CaptureMapBuilder {
86 entries: Vec<CaptureMapping>,
87}
88
89impl CaptureMapBuilder {
90 pub fn capture(
91 mut self,
92 name: impl Into<String>,
93 tokens: impl IntoIterator<Item = TokenId>,
94 ) -> Result<Self, CaptureMapError> {
95 self.insert(name, tokens)?;
96 Ok(self)
97 }
98
99 pub fn insert(
100 &mut self,
101 name: impl Into<String>,
102 tokens: impl IntoIterator<Item = TokenId>,
103 ) -> Result<&mut Self, CaptureMapError> {
104 let name = name.into();
105 if name.is_empty() {
106 return Err(CaptureMapError::EmptyName);
107 }
108 if self.entries.iter().any(|entry| entry.name == name) {
109 return Err(CaptureMapError::DuplicateName { name });
110 }
111 let tokens = tokens.into_iter().collect::<Vec<_>>().into_boxed_slice();
112 if tokens.is_empty() {
113 return Err(CaptureMapError::NoTokens { name });
114 }
115 self.entries.push(CaptureMapping { name, tokens });
116 Ok(self)
117 }
118
119 pub fn build(self) -> CaptureMap {
120 CaptureMap(Arc::new(CaptureMapData {
121 entries: self.entries.into_boxed_slice(),
122 }))
123 }
124
125 pub fn replace(
126 &mut self,
127 name: &str,
128 tokens: impl IntoIterator<Item = TokenId>,
129 ) -> Result<&mut Self, CaptureMapError> {
130 let Some(entry) = self.entries.iter_mut().find(|entry| entry.name == name) else {
131 return Err(CaptureMapError::UnknownName {
132 name: name.to_owned(),
133 });
134 };
135 let tokens = tokens.into_iter().collect::<Vec<_>>().into_boxed_slice();
136 if tokens.is_empty() {
137 return Err(CaptureMapError::NoTokens {
138 name: name.to_owned(),
139 });
140 }
141 entry.tokens = tokens;
142 Ok(self)
143 }
144
145 pub fn set(
150 &mut self,
151 name: impl Into<String>,
152 tokens: impl IntoIterator<Item = TokenId>,
153 ) -> Result<&mut Self, CaptureMapError> {
154 let name = name.into();
155 if name.is_empty() {
156 return Err(CaptureMapError::EmptyName);
157 }
158 let tokens = tokens.into_iter().collect::<Vec<_>>().into_boxed_slice();
159 if tokens.is_empty() {
160 return Err(CaptureMapError::NoTokens { name });
161 }
162 if let Some(entry) = self.entries.iter_mut().find(|entry| entry.name == name) {
163 entry.tokens = tokens;
164 } else {
165 self.entries.push(CaptureMapping { name, tokens });
166 }
167 Ok(self)
168 }
169
170 pub fn remove(&mut self, name: &str) -> Result<&mut Self, CaptureMapError> {
171 let Some(index) = self.entries.iter().position(|entry| entry.name == name) else {
172 return Err(CaptureMapError::UnknownName {
173 name: name.to_owned(),
174 });
175 };
176 self.entries.remove(index);
177 Ok(self)
178 }
179
180 fn push_static(&mut self, name: &'static str, token: TokenId) {
181 self.entries.push(CaptureMapping {
182 name: name.to_owned(),
183 tokens: Box::new([token]),
184 });
185 }
186}
187
188#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)]
189pub enum CaptureMapError {
190 #[error("Tree-sitter capture name is empty")]
191 EmptyName,
192 #[error("Tree-sitter capture {name:?} is already mapped")]
193 DuplicateName { name: String },
194 #[error("Tree-sitter capture {name:?} has no semantic tokens")]
195 NoTokens { name: String },
196 #[error("Tree-sitter capture {name:?} is not mapped")]
197 UnknownName { name: String },
198}
199
200pub struct LanguageConfiguration {
202 inner: HighlightConfiguration,
203 captures: CaptureMap,
204}
205
206impl LanguageConfiguration {
207 pub fn from_language(
209 language: Language,
210 name: impl Into<String>,
211 highlights_query: &str,
212 injection_query: &str,
213 locals_query: &str,
214 captures: CaptureMap,
215 ) -> Result<Self, QueryError> {
216 let configuration = HighlightConfiguration::new(
217 language,
218 name,
219 highlights_query,
220 injection_query,
221 locals_query,
222 )?;
223 Ok(Self::new(configuration, captures))
224 }
225
226 pub fn standard(
229 language: Language,
230 name: impl Into<String>,
231 highlights_query: &str,
232 ) -> Result<Self, QueryError> {
233 Self::standard_with_queries(language, name, highlights_query, "", "")
234 }
235
236 pub fn standard_with_queries(
239 language: Language,
240 name: impl Into<String>,
241 highlights_query: &str,
242 injection_query: &str,
243 locals_query: &str,
244 ) -> Result<Self, QueryError> {
245 Self::from_language(
246 language,
247 name,
248 highlights_query,
249 injection_query,
250 locals_query,
251 CaptureMap::standard(),
252 )
253 }
254
255 pub fn for_injection(
260 parent: &Self,
261 language: Language,
262 name: impl Into<String>,
263 highlights_query: &str,
264 injection_query: &str,
265 locals_query: &str,
266 ) -> Result<Self, QueryError> {
267 Self::from_language(
268 language,
269 name,
270 highlights_query,
271 injection_query,
272 locals_query,
273 parent.captures.clone(),
274 )
275 }
276
277 pub fn new(mut configuration: HighlightConfiguration, captures: CaptureMap) -> Self {
278 captures.configure(&mut configuration);
279 Self {
280 inner: configuration,
281 captures,
282 }
283 }
284
285 pub fn inner(&self) -> &HighlightConfiguration {
286 &self.inner
287 }
288
289 pub fn capture_map(&self) -> &CaptureMap {
290 &self.captures
291 }
292}
293
294#[derive(Clone, Debug)]
295pub struct CodeOptions {
296 tab_width: NonZeroU8,
297 root_tokens: Vec<TokenId>,
298}
299
300impl Default for CodeOptions {
301 fn default() -> Self {
302 Self {
303 tab_width: NonZeroU8::new(4).expect("default tab width is nonzero"),
304 root_tokens: vec![tokens::SYNTAX_CODE],
305 }
306 }
307}
308
309impl CodeOptions {
310 pub fn new() -> Self {
311 Self::default()
312 }
313
314 pub fn tab_width(mut self, tab_width: NonZeroU8) -> Self {
315 self.tab_width = tab_width;
316 self
317 }
318
319 pub fn root_token(mut self, token: TokenId) -> Self {
320 self.root_tokens.push(token);
321 self
322 }
323
324 pub fn root_tokens(mut self, tokens: impl IntoIterator<Item = TokenId>) -> Self {
325 self.root_tokens.extend(tokens);
326 self
327 }
328
329 pub const fn configured_tab_width(&self) -> NonZeroU8 {
330 self.tab_width
331 }
332
333 pub fn configured_root_tokens(&self) -> &[TokenId] {
334 &self.root_tokens
335 }
336}
337
338pub struct TreeLayoutContext<'source> {
340 source: &'source str,
341 tab_width: NonZeroU8,
342}
343
344impl<'source> TreeLayoutContext<'source> {
345 pub fn source(&self) -> &'source str {
346 self.source
347 }
348
349 pub const fn tab_width(&self) -> NonZeroU8 {
350 self.tab_width
351 }
352
353 pub fn node_text(&self, node: Node<'_>) -> Result<&'source str, TreeSourceError> {
354 let range = node.byte_range();
355 self.source
356 .get(range.clone())
357 .ok_or(TreeSourceError::InvalidNodeRange {
358 start: range.start,
359 end: range.end,
360 source_len: self.source.len(),
361 })
362 }
363
364 pub fn text(&self, node: Node<'_>) -> Result<Doc, TreeSourceError> {
365 Ok(Doc::from_text(
366 self.node_text(node)?,
367 termosaic::TextLayout::Preformatted {
368 tab_width: self.tab_width,
369 },
370 ))
371 }
372
373 pub fn token_text(&self, node: Node<'_>, token: TokenId) -> Result<Doc, TreeSourceError> {
374 Ok(Doc::token(token, self.text(node)?))
375 }
376
377 pub fn tokens_text(
378 &self,
379 node: Node<'_>,
380 tokens: impl IntoIterator<Item = TokenId>,
381 ) -> Result<Doc, TreeSourceError> {
382 Ok(Doc::tokens(tokens, self.text(node)?))
383 }
384}
385
386pub trait TreeFormatter {
388 type Error: From<TreeSourceError>;
389
390 fn format(&self, context: &TreeLayoutContext<'_>, node: Node<'_>) -> Result<Doc, Self::Error>;
391}
392
393pub fn document_from_tree<F: TreeFormatter>(
395 tree: &Tree,
396 source: &str,
397 formatter: &F,
398 options: &CodeOptions,
399) -> Result<Doc, F::Error> {
400 let context = TreeLayoutContext {
401 source,
402 tab_width: options.tab_width,
403 };
404 let document = formatter.format(&context, tree.root_node())?;
405 Ok(Doc::tokens(options.root_tokens.clone(), document))
406}
407
408#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)]
409pub enum TreeSourceError {
410 #[error("Tree-sitter node range {start}..{end} is invalid for {source_len} UTF-8 bytes")]
411 InvalidNodeRange {
412 start: usize,
413 end: usize,
414 source_len: usize,
415 },
416}
417
418pub struct SemanticHighlighter {
420 inner: Highlighter,
421}
422
423impl Default for SemanticHighlighter {
424 fn default() -> Self {
425 Self::new()
426 }
427}
428
429impl SemanticHighlighter {
430 pub fn new() -> Self {
431 Self {
432 inner: Highlighter::new(),
433 }
434 }
435
436 pub fn highlighter(&mut self) -> &mut Highlighter {
437 &mut self.inner
438 }
439
440 pub fn document(
441 &mut self,
442 configuration: &LanguageConfiguration,
443 source: &str,
444 options: &CodeOptions,
445 ) -> Result<Doc, HighlightError> {
446 self.document_with_injections(configuration, source, options, None, |_| None)
447 }
448
449 pub fn document_with_injections<'a>(
450 &'a mut self,
451 configuration: &'a LanguageConfiguration,
452 source: &'a str,
453 options: &CodeOptions,
454 cancellation_flag: Option<&'a AtomicUsize>,
455 mut injection_callback: impl FnMut(&str) -> Option<&'a LanguageConfiguration> + 'a,
456 ) -> Result<Doc, HighlightError> {
457 let captures = configuration.captures.clone();
458 let mut mismatched_capture_map = false;
459 let events = self.inner.highlight(
460 &configuration.inner,
461 source.as_bytes(),
462 cancellation_flag,
463 |name| match injection_callback(name) {
464 Some(injection) if captures.is_compatible_with(&injection.captures) => {
465 Some(&injection.inner)
466 }
467 Some(_) => {
468 mismatched_capture_map = true;
469 None
470 }
471 None => None,
472 },
473 )?;
474
475 let mut builder = SourceBuilder::new(source, options.tab_width, &captures);
476 for event in events {
477 builder.event(event?)?;
478 }
479 let document = builder.finish()?;
480 if mismatched_capture_map {
481 return Err(HighlightError::InjectionCaptureMapMismatch);
482 }
483 Ok(Doc::tokens(options.root_tokens.clone(), document))
484 }
485}
486
487struct SourceBuilder<'a> {
488 source: &'a str,
489 tab_width: usize,
490 captures: &'a CaptureMap,
491 active: Vec<Highlight>,
492 docs: Vec<Doc>,
493 cursor: usize,
494 column: usize,
495 pending_carriage_return: bool,
496}
497
498impl<'a> SourceBuilder<'a> {
499 fn new(source: &'a str, tab_width: NonZeroU8, captures: &'a CaptureMap) -> Self {
500 Self {
501 source,
502 tab_width: usize::from(tab_width.get()),
503 captures,
504 active: Vec::new(),
505 docs: Vec::new(),
506 cursor: 0,
507 column: 0,
508 pending_carriage_return: false,
509 }
510 }
511
512 fn event(&mut self, event: HighlightEvent) -> Result<(), HighlightError> {
513 match event {
514 HighlightEvent::Source { start, end } => {
515 if start < self.cursor || end < start || end > self.source.len() {
516 return Err(HighlightError::InvalidSourceRange {
517 start,
518 end,
519 source_len: self.source.len(),
520 });
521 }
522 if start > self.cursor {
523 self.push_range(self.cursor, start)?;
524 }
525 self.push_range(start, end)?;
526 self.cursor = end;
527 }
528 HighlightEvent::HighlightStart(highlight) => {
529 if self.captures.tokens(highlight).is_none() {
530 return Err(HighlightError::UnknownHighlight { index: highlight.0 });
531 }
532 self.active.push(highlight);
533 }
534 HighlightEvent::HighlightEnd => {
535 self.active
536 .pop()
537 .ok_or(HighlightError::UnbalancedHighlightEnd)?;
538 }
539 }
540 Ok(())
541 }
542
543 fn finish(mut self) -> Result<Doc, HighlightError> {
544 if self.cursor < self.source.len() {
545 self.push_range(self.cursor, self.source.len())?;
546 }
547 if !self.active.is_empty() {
548 return Err(HighlightError::UnclosedHighlights {
549 count: self.active.len(),
550 });
551 }
552 if self.pending_carriage_return {
553 self.docs.push(Doc::hard_line());
554 }
555 Ok(Doc::concat(self.docs))
556 }
557
558 fn push_range(&mut self, start: usize, end: usize) -> Result<(), HighlightError> {
559 let value = self
560 .source
561 .get(start..end)
562 .ok_or(HighlightError::InvalidUtf8Boundary { start, end })?;
563 let mut segment_start = 0usize;
564 for (offset, character) in value.char_indices() {
565 if self.pending_carriage_return {
566 self.docs.push(Doc::hard_line());
567 self.column = 0;
568 self.pending_carriage_return = false;
569 if character == '\n' {
570 segment_start = offset + character.len_utf8();
571 continue;
572 }
573 }
574 if character == '\r'
575 || character == '\n'
576 || character == '\t'
577 || unsafe_human(character)
578 {
579 self.push_segment(&value[segment_start..offset])?;
580 segment_start = offset + character.len_utf8();
581 match character {
582 '\r' => self.pending_carriage_return = true,
583 '\n' => {
584 self.docs.push(Doc::hard_line());
585 self.column = 0;
586 }
587 '\t' => {
588 let spaces = self.tab_width - (self.column % self.tab_width);
589 self.push_segment(&" ".repeat(spaces))?;
590 }
591 _ => self.push_segment("\u{fffd}")?,
592 }
593 }
594 }
595 self.push_segment(&value[segment_start..])
596 }
597
598 fn push_segment(&mut self, value: &str) -> Result<(), HighlightError> {
599 if value.is_empty() {
600 return Ok(());
601 }
602 self.column = self
603 .column
604 .checked_add(UnicodeWidthStr::width(value))
605 .ok_or(HighlightError::DisplayColumnOverflow)?;
606 let mut tokens = Vec::new();
607 for highlight in &self.active {
608 tokens.extend_from_slice(
609 self.captures
610 .tokens(*highlight)
611 .ok_or(HighlightError::UnknownHighlight { index: highlight.0 })?,
612 );
613 }
614 self.docs.push(Doc::tokens(tokens, Doc::text(value)));
615 Ok(())
616 }
617}
618
619fn unsafe_human(character: char) -> bool {
620 matches!(character as u32,
621 0x0000..=0x001f
622 | 0x007f..=0x009f
623 | 0x202a..=0x202e
624 | 0x2066..=0x2069)
625}
626
627#[derive(Debug, thiserror::Error)]
628pub enum HighlightError {
629 #[error(transparent)]
630 TreeSitter(#[from] tree_sitter_highlight::Error),
631 #[error("Tree-sitter source range {start}..{end} is invalid for {source_len} bytes")]
632 InvalidSourceRange {
633 start: usize,
634 end: usize,
635 source_len: usize,
636 },
637 #[error("Tree-sitter source range {start}..{end} is not on UTF-8 boundaries")]
638 InvalidUtf8Boundary { start: usize, end: usize },
639 #[error("Tree-sitter emitted unknown highlight index {index}")]
640 UnknownHighlight { index: usize },
641 #[error("Tree-sitter emitted a highlight end without a matching start")]
642 UnbalancedHighlightEnd,
643 #[error("Tree-sitter left {count} highlight scopes open")]
644 UnclosedHighlights { count: usize },
645 #[error("an injected language uses an incompatible capture map or capture order")]
646 InjectionCaptureMapMismatch,
647 #[error("highlighted source display column overflowed usize")]
648 DisplayColumnOverflow,
649}
650
651fn standard_captures() -> Vec<(&'static str, TokenId)> {
652 vec![
653 ("attribute", tokens::SYNTAX_ATTRIBUTE),
654 ("boolean", tokens::SYNTAX_BOOLEAN),
655 ("character", tokens::SYNTAX_CHARACTER),
656 ("comment", tokens::SYNTAX_COMMENT),
657 ("conditional", tokens::SYNTAX_CONDITIONAL),
658 ("constant", tokens::SYNTAX_CONSTANT),
659 ("constant.builtin", tokens::SYNTAX_CONSTANT_BUILTIN),
660 ("constructor", tokens::SYNTAX_CONSTRUCTOR),
661 ("embedded", tokens::SYNTAX_EMBEDDED),
662 ("exception", tokens::SYNTAX_EXCEPTION),
663 ("field", tokens::SYNTAX_FIELD),
664 ("float", tokens::SYNTAX_FLOAT),
665 ("function", tokens::SYNTAX_FUNCTION),
666 ("function.builtin", tokens::SYNTAX_FUNCTION_BUILTIN),
667 ("function.call", tokens::SYNTAX_FUNCTION_CALL),
668 ("function.macro", tokens::SYNTAX_FUNCTION_MACRO),
669 ("include", tokens::SYNTAX_INCLUDE),
670 ("keyword", tokens::SYNTAX_KEYWORD),
671 ("keyword.function", tokens::SYNTAX_KEYWORD_FUNCTION),
672 ("keyword.operator", tokens::SYNTAX_KEYWORD_OPERATOR),
673 ("label", tokens::SYNTAX_LABEL),
674 ("method", tokens::SYNTAX_METHOD),
675 ("method.call", tokens::SYNTAX_METHOD_CALL),
676 ("namespace", tokens::SYNTAX_NAMESPACE),
677 ("number", tokens::SYNTAX_NUMBER),
678 ("operator", tokens::SYNTAX_OPERATOR),
679 ("parameter", tokens::SYNTAX_PARAMETER),
680 ("property", tokens::SYNTAX_PROPERTY),
681 ("punctuation", tokens::SYNTAX_PUNCTUATION),
682 ("punctuation.bracket", tokens::SYNTAX_PUNCTUATION_BRACKET),
683 (
684 "punctuation.delimiter",
685 tokens::SYNTAX_PUNCTUATION_DELIMITER,
686 ),
687 ("punctuation.special", tokens::SYNTAX_PUNCTUATION_SPECIAL),
688 ("repeat", tokens::SYNTAX_REPEAT),
689 ("string", tokens::SYNTAX_STRING),
690 ("string.escape", tokens::SYNTAX_STRING_ESCAPE),
691 ("string.regexp", tokens::SYNTAX_STRING_REGEXP),
692 ("tag", tokens::SYNTAX_TAG),
693 ("type", tokens::SYNTAX_TYPE),
694 ("type.builtin", tokens::SYNTAX_TYPE_BUILTIN),
695 ("variable", tokens::SYNTAX_VARIABLE),
696 ("variable.builtin", tokens::SYNTAX_VARIABLE_BUILTIN),
697 ]
698}
699
700#[cfg(test)]
701mod tests {
702 use std::io::{self, Write};
703 use std::num::{NonZeroU8, NonZeroU16};
704 use std::sync::{Arc, Mutex};
705
706 use termosaic::{
707 AnsiColor, Color, ColorChoice, ColorDepth, Console, ConsoleOptions, DestinationOptions,
708 OutputClass, Stream, TerminalCapabilities, Theme, TokenStyle,
709 };
710 use tree_sitter::Parser;
711 use tree_sitter_highlight::{Highlight, HighlightEvent};
712
713 use super::*;
714
715 #[derive(Clone, Default)]
716 struct Capture(Arc<Mutex<Vec<u8>>>);
717
718 impl Capture {
719 fn text(&self) -> String {
720 String::from_utf8(
721 self.0
722 .lock()
723 .unwrap_or_else(|poisoned| poisoned.into_inner())
724 .clone(),
725 )
726 .expect("captured output is UTF-8")
727 }
728 }
729
730 impl Write for Capture {
731 fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
732 self.0
733 .lock()
734 .unwrap_or_else(|poisoned| poisoned.into_inner())
735 .extend_from_slice(bytes);
736 Ok(bytes.len())
737 }
738
739 fn flush(&mut self) -> io::Result<()> {
740 Ok(())
741 }
742 }
743
744 fn render(doc: &Doc, theme: Theme, color: ColorChoice) -> String {
745 let capture = Capture::default();
746 let destination = DestinationOptions::new(
747 OutputClass::Human,
748 color,
749 TerminalCapabilities {
750 is_terminal: color == ColorChoice::Always,
751 color_depth: ColorDepth::Ansi16,
752 width: NonZeroU16::new(80),
753 interactive_cursor: false,
754 },
755 );
756 let console = Console::with_writers(
757 capture.clone(),
758 io::sink(),
759 ConsoleOptions::new(destination, destination, theme),
760 )
761 .expect("Tree-sitter test console");
762 console
763 .human(Stream::Stdout)
764 .expect("human writer")
765 .doc(doc)
766 .expect("render Tree-sitter document");
767 capture.text()
768 }
769
770 fn capture_map() -> CaptureMap {
771 let mut builder = CaptureMap::standard().customize();
772 builder
773 .set("string.special.key", [tokens::SYNTAX_PROPERTY])
774 .expect("JSON property capture");
775 builder
776 .set("escape", [tokens::SYNTAX_STRING_ESCAPE])
777 .expect("JSON escape capture");
778 builder.build()
779 }
780
781 fn json_configuration(captures: CaptureMap) -> LanguageConfiguration {
782 LanguageConfiguration::from_language(
783 tree_sitter_json::LANGUAGE.into(),
784 "json",
785 tree_sitter_json::HIGHLIGHTS_QUERY,
786 "",
787 "",
788 captures,
789 )
790 .expect("JSON highlight query")
791 }
792
793 #[test]
794 fn real_json_highlighting_preserves_source_and_supports_reuse() {
795 let configuration = json_configuration(capture_map());
796 let options = CodeOptions::new().root_token(tokens::LANGUAGE_JSON);
797 let mut highlighter = SemanticHighlighter::new();
798
799 for source in [r#"{"name":"value","ready":true}"#, r#"[1,null,false]"#] {
800 let doc = highlighter
801 .document(&configuration, source, &options)
802 .expect("highlight JSON");
803 assert_eq!(
804 render(&doc, termosaic::themes::default(), ColorChoice::Never),
805 format!("{source}\n")
806 );
807 }
808 }
809
810 #[test]
811 fn standard_language_configuration_handles_common_captures() {
812 let configuration = LanguageConfiguration::standard(
813 tree_sitter_json::LANGUAGE.into(),
814 "json",
815 tree_sitter_json::HIGHLIGHTS_QUERY,
816 )
817 .expect("JSON highlight query");
818 let doc = SemanticHighlighter::new()
819 .document(&configuration, "true", &CodeOptions::new())
820 .expect("highlight JSON");
821 assert_eq!(
822 render(&doc, termosaic::themes::default(), ColorChoice::Never),
823 "true\n"
824 );
825 }
826
827 #[test]
828 fn capture_set_adds_and_overrides_without_branching() {
829 let mut captures = CaptureMap::standard().customize();
830 captures
831 .set("keyword", [tokens::STATUS_WARNING])
832 .expect("override existing capture")
833 .set("custom", [tokens::STATUS_SUCCESS])
834 .expect("add custom capture");
835 let captures = captures.build();
836
837 assert_eq!(
838 captures
839 .entries()
840 .find(|(name, _)| *name == "keyword")
841 .expect("keyword mapping")
842 .1,
843 &[tokens::STATUS_WARNING]
844 );
845 assert_eq!(
846 captures
847 .entries()
848 .find(|(name, _)| *name == "custom")
849 .expect("custom mapping")
850 .1,
851 &[tokens::STATUS_SUCCESS]
852 );
853 }
854
855 #[test]
856 fn independently_built_equivalent_capture_maps_are_injection_compatible() {
857 let first = CaptureMap::standard();
858 let second = CaptureMap::standard();
859 assert!(first.is_compatible_with(&second));
860
861 let mut changed = second.customize();
862 changed
863 .set("keyword", [tokens::STATUS_WARNING])
864 .expect("change keyword mapping");
865 assert!(!first.is_compatible_with(&changed.build()));
866
867 let parent = LanguageConfiguration::standard(
868 tree_sitter_json::LANGUAGE.into(),
869 "json",
870 tree_sitter_json::HIGHLIGHTS_QUERY,
871 )
872 .expect("parent language");
873 let injection = LanguageConfiguration::for_injection(
874 &parent,
875 tree_sitter_json::LANGUAGE.into(),
876 "json-injection",
877 tree_sitter_json::HIGHLIGHTS_QUERY,
878 "",
879 "",
880 )
881 .expect("injection language");
882 assert!(
883 parent
884 .capture_map()
885 .is_compatible_with(injection.capture_map())
886 );
887 }
888
889 #[test]
890 fn structural_formatter_receives_a_real_tree_and_source_context() {
891 struct RootFormatter;
892
893 impl TreeFormatter for RootFormatter {
894 type Error = TreeSourceError;
895
896 fn format(
897 &self,
898 context: &TreeLayoutContext<'_>,
899 node: Node<'_>,
900 ) -> Result<Doc, Self::Error> {
901 context.tokens_text(node, [tokens::LANGUAGE_JSON, tokens::SYNTAX_CODE])
902 }
903 }
904
905 let source = r#"{"value":1}"#;
906 let mut parser = Parser::new();
907 parser
908 .set_language(&tree_sitter_json::LANGUAGE.into())
909 .expect("load JSON grammar");
910 let tree = parser.parse(source, None).expect("parse JSON");
911 assert!(!tree.root_node().has_error());
912
913 let doc = document_from_tree(&tree, source, &RootFormatter, &CodeOptions::new())
914 .expect("format parsed tree");
915 assert_eq!(
916 render(&doc, termosaic::themes::default(), ColorChoice::Never),
917 format!("{source}\n")
918 );
919
920 let mismatched_context = TreeLayoutContext {
921 source: "",
922 tab_width: NonZeroU8::new(4).expect("tab width"),
923 };
924 assert!(matches!(
925 mismatched_context.node_text(tree.root_node()),
926 Err(TreeSourceError::InvalidNodeRange {
927 start: 0,
928 end,
929 source_len: 0,
930 }) if end == source.len()
931 ));
932 }
933
934 #[test]
935 fn source_events_preserve_semantics_and_normalize_terminal_text_safely() {
936 let captures = CaptureMap::builder()
937 .capture("property", [tokens::SYNTAX_PROPERTY])
938 .expect("property capture")
939 .build();
940 let source = "a\t界\r\nb\u{85}";
941 let split_after_carriage_return = source.find('\n').expect("line feed");
942 let mut builder =
943 SourceBuilder::new(source, NonZeroU8::new(4).expect("tab width"), &captures);
944 builder
945 .event(HighlightEvent::HighlightStart(Highlight(0)))
946 .expect("start property");
947 builder
948 .event(HighlightEvent::Source {
949 start: 0,
950 end: split_after_carriage_return,
951 })
952 .expect("first source range");
953 builder
954 .event(HighlightEvent::Source {
955 start: split_after_carriage_return,
956 end: source.len(),
957 })
958 .expect("second source range");
959 builder
960 .event(HighlightEvent::HighlightEnd)
961 .expect("end property");
962 let doc = builder.finish().expect("finish source events");
963
964 let theme = Theme::builder("tree-sitter-test")
965 .rule(
966 tokens::SYNTAX_PROPERTY,
967 TokenStyle::foreground(Color::Ansi(AnsiColor::Red)),
968 )
969 .expect("property rule")
970 .build()
971 .expect("test theme");
972 assert_eq!(
973 render(&doc, theme, ColorChoice::Always),
974 "\u{1b}[31ma 界\u{1b}[0m\n\u{1b}[31mb�\u{1b}[0m\n"
975 );
976 }
977
978 #[test]
979 fn capture_map_validation_and_customization_are_typed() {
980 assert!(matches!(
981 CaptureMap::builder().capture("", [tokens::SYNTAX_STRING]),
982 Err(CaptureMapError::EmptyName)
983 ));
984 assert!(matches!(
985 CaptureMap::builder().capture("string", []),
986 Err(CaptureMapError::NoTokens { name }) if name == "string"
987 ));
988 let mut builder = CaptureMap::builder()
989 .capture("string", [tokens::SYNTAX_STRING])
990 .expect("initial string capture");
991 assert!(matches!(
992 builder.insert("string", [tokens::DATA_STRING]),
993 Err(CaptureMapError::DuplicateName { name }) if name == "string"
994 ));
995 assert!(matches!(
996 builder.replace("missing", [tokens::SYNTAX_STRING]),
997 Err(CaptureMapError::UnknownName { name }) if name == "missing"
998 ));
999 assert!(matches!(
1000 builder.remove("missing"),
1001 Err(CaptureMapError::UnknownName { name }) if name == "missing"
1002 ));
1003 builder
1004 .replace("string", [tokens::DATA_STRING])
1005 .expect("replace capture");
1006 let map = builder.build();
1007 assert_eq!(
1008 map.entries().collect::<Vec<_>>(),
1009 vec![("string", &[tokens::DATA_STRING][..])]
1010 );
1011 }
1012
1013 #[test]
1014 fn malformed_highlight_event_streams_return_specific_errors() {
1015 let captures = CaptureMap::builder()
1016 .capture("string", [tokens::SYNTAX_STRING])
1017 .expect("string capture")
1018 .build();
1019 let tab_width = NonZeroU8::new(4).expect("tab width");
1020
1021 let mut invalid_range = SourceBuilder::new("x", tab_width, &captures);
1022 assert!(matches!(
1023 invalid_range.event(HighlightEvent::Source { start: 0, end: 2 }),
1024 Err(HighlightError::InvalidSourceRange {
1025 start: 0,
1026 end: 2,
1027 source_len: 1,
1028 })
1029 ));
1030
1031 let mut invalid_boundary = SourceBuilder::new("é", tab_width, &captures);
1032 assert!(matches!(
1033 invalid_boundary.event(HighlightEvent::Source { start: 0, end: 1 }),
1034 Err(HighlightError::InvalidUtf8Boundary { start: 0, end: 1 })
1035 ));
1036
1037 let mut unknown = SourceBuilder::new("x", tab_width, &captures);
1038 assert!(matches!(
1039 unknown.event(HighlightEvent::HighlightStart(Highlight(1))),
1040 Err(HighlightError::UnknownHighlight { index: 1 })
1041 ));
1042
1043 let mut unbalanced = SourceBuilder::new("x", tab_width, &captures);
1044 assert!(matches!(
1045 unbalanced.event(HighlightEvent::HighlightEnd),
1046 Err(HighlightError::UnbalancedHighlightEnd)
1047 ));
1048
1049 let mut unclosed = SourceBuilder::new("x", tab_width, &captures);
1050 unclosed
1051 .event(HighlightEvent::HighlightStart(Highlight(0)))
1052 .expect("start highlight");
1053 assert!(matches!(
1054 unclosed.finish(),
1055 Err(HighlightError::UnclosedHighlights { count: 1 })
1056 ));
1057
1058 let mut overflowing_column = SourceBuilder::new("x", tab_width, &captures);
1059 overflowing_column.column = usize::MAX;
1060 assert!(matches!(
1061 overflowing_column.event(HighlightEvent::Source { start: 0, end: 1 }),
1062 Err(HighlightError::DisplayColumnOverflow)
1063 ));
1064 }
1065}