carta_highlight/
highlighter.rs

1//! The tokenizer: walks a stack of contexts over source text, emitting classified [`Token`]s.
2//!
3//! For each line the current context's rules are tried in order; the first to match consumes text,
4//! emits a token, and may switch contexts. When nothing matches, the context either falls through to
5//! another context or consumes a run of ordinary text. Regular-expression rules match anchored at the
6//! current position; compiled patterns and resolved keyword sets are cached across lines.
7
8use std::cell::RefCell;
9use std::collections::{BTreeMap, BTreeSet};
10use std::rc::Rc;
11
12use fancy_regex::Regex;
13
14use crate::grammar::Grammar;
15use crate::registry::Registry;
16use crate::token::SourceLine;
17
18mod helpers;
19mod tokenizer;
20
21use helpers::{build_regex, split_lines};
22use tokenizer::Tokenizer;
23
24/// Tokenizes source code using a catalog of syntax definitions.
25#[derive(Debug, Default)]
26pub struct Highlighter {
27    registry: Registry,
28    regexes: RefCell<BTreeMap<RegexKey, Option<Rc<Regex>>>>,
29    keyword_sets: RefCell<BTreeMap<String, BTreeMap<String, Rc<KeywordSet>>>>,
30}
31
32#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
33struct RegexKey {
34    pattern: String,
35    insensitive: bool,
36    minimal: bool,
37}
38
39#[derive(Debug)]
40pub(crate) struct KeywordSet {
41    words: BTreeSet<String>,
42    case_sensitive: bool,
43}
44
45impl KeywordSet {
46    fn contains(&self, word: &str) -> bool {
47        if self.case_sensitive {
48            return self.words.contains(word);
49        }
50        // An ASCII word with no uppercase letters is already its own lowercase form.
51        if word
52            .bytes()
53            .all(|b| b.is_ascii() && !b.is_ascii_uppercase())
54        {
55            return self.words.contains(word);
56        }
57        self.words.contains(&word.to_lowercase())
58    }
59}
60
61/// One entry on the context stack: the definition and context it names, and any captures carried in
62/// from the rule that entered it.
63#[derive(Debug, Clone)]
64struct Frame {
65    grammar: Rc<Grammar>,
66    context: usize,
67    captures: Vec<String>,
68}
69
70impl Highlighter {
71    /// A highlighter over the bundled syntax definitions.
72    #[must_use]
73    pub fn new() -> Self {
74        Highlighter::default()
75    }
76
77    /// The catalog backing this highlighter.
78    pub fn registry(&self) -> &Registry {
79        &self.registry
80    }
81
82    /// The catalog backing this highlighter, mutably (to register user definitions).
83    pub fn registry_mut(&mut self) -> &mut Registry {
84        &mut self.registry
85    }
86
87    /// Tokenize `code` as the given language, returning one [`SourceLine`] per line, or `None` if the
88    /// language is unknown.
89    pub fn highlight(&self, language: &str, code: &str) -> Option<Vec<SourceLine>> {
90        let grammar = self.registry.resolve(language)?;
91        Some(self.tokenize(grammar, code))
92    }
93
94    fn tokenize(&self, start: Rc<Grammar>, code: &str) -> Vec<SourceLine> {
95        let mut state = Tokenizer::new(self, start);
96        split_lines(code)
97            .into_iter()
98            .map(|line| state.tokenize_line(line))
99            .collect()
100    }
101
102    fn compiled_regex(&self, key: &RegexKey) -> Option<Rc<Regex>> {
103        if let Some(entry) = self.regexes.borrow().get(key) {
104            return entry.clone();
105        }
106        let compiled = build_regex(key);
107        self.regexes
108            .borrow_mut()
109            .insert(key.clone(), compiled.clone());
110        compiled
111    }
112
113    fn keyword_set(&self, grammar: &Rc<Grammar>, list: &str) -> Rc<KeywordSet> {
114        if let Some(set) = self
115            .keyword_sets
116            .borrow()
117            .get(grammar.name.as_str())
118            .and_then(|lists| lists.get(list))
119        {
120            return Rc::clone(set);
121        }
122        let mut words = BTreeSet::new();
123        let case_sensitive = grammar.keywords.case_sensitive;
124        self.collect_words(
125            grammar,
126            list,
127            case_sensitive,
128            &mut words,
129            &mut BTreeSet::new(),
130        );
131        let set = Rc::new(KeywordSet {
132            words,
133            case_sensitive,
134        });
135        self.keyword_sets
136            .borrow_mut()
137            .entry(grammar.name.clone())
138            .or_default()
139            .insert(list.to_string(), Rc::clone(&set));
140        set
141    }
142
143    fn collect_words(
144        &self,
145        grammar: &Rc<Grammar>,
146        list: &str,
147        case_sensitive: bool,
148        out: &mut BTreeSet<String>,
149        visited: &mut BTreeSet<(String, String)>,
150    ) {
151        let key = (grammar.name.clone(), list.to_string());
152        if !visited.insert(key) {
153            return;
154        }
155        if let Some(words) = grammar.keyword_lists.get(list) {
156            for word in words {
157                if word.is_empty() {
158                    continue;
159                }
160                out.insert(if case_sensitive {
161                    word.clone()
162                } else {
163                    word.to_lowercase()
164                });
165            }
166        }
167        for include in &grammar.keyword_includes {
168            if include.target_list != list {
169                continue;
170            }
171            if include.source_language == grammar.name {
172                self.collect_words(grammar, &include.source_list, case_sensitive, out, visited);
173            } else if let Some(source) = self.registry.resolve_reference(&include.source_language) {
174                self.collect_words(&source, &include.source_list, case_sensitive, out, visited);
175            }
176        }
177    }
178}
179
180#[cfg(test)]
181mod tests;
carta_highlight/highlighter.rs

carta_highlight/
highlighter.rs
