syntaxmate 0.2.0

Rust-native TextMate syntax highlighting with bundled grammars and themes
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

//! Rust-native TextMate grammar engine.

pub mod cache;
pub mod checkpoint;
pub mod counters;
pub mod grammar;
pub(crate) mod grammar_closure;
pub(crate) mod grammar_ir;
pub(crate) mod hashing;
pub mod line;
pub mod regex;
pub mod scopes;
pub mod state;
pub mod tokenizer;

#[cfg(test)]
mod closure_parity_tests;

use crate::{Error, Result};
use tokenizer::{GrammarSet, LazyGrammar};

/// Build the bundled grammar set for `language` from its recorded closure.
///
/// Only the root is decoded here; other members decode on first access. The
/// recorded member traits let the repository-context walk skip members that
/// cannot bind a non-empty context, and recorded availability proofs answer
/// include-availability checks for members the tokenizer has not entered.
pub(crate) fn load_grammar_set(language: &str) -> Result<(GrammarSet, state::GrammarId)> {
    load_catalog_grammar_set(crate::grammars::embedded_bundle_shared(), language)
}

pub(crate) fn load_catalog_grammar_set(
    bundle: &std::sync::Arc<crate::grammars::bundle::Bundle>,
    language: &str,
) -> Result<(GrammarSet, state::GrammarId)> {
    let missing = || {
        Error::Bundle(crate::BundleError::new(
            crate::BundleErrorKind::MissingGrammar,
            Some(language.to_owned()),
            format!("bundled TextMate grammar `{language}` is missing"),
        ))
    };
    let root_index = bundle
        .grammar_blob_index_for_language(language)
        .ok_or_else(missing)?;
    let root_blob = &bundle.grammar_blobs[root_index];
    let mut grammars = GrammarSet::new();
    let mut root = None;
    let root_graph = bundle.grammar_graphs.get(root_index).ok_or_else(missing)?;
    for member in &root_graph.closure {
        let blob = &bundle.grammar_blobs[member.blob as usize];
        let grammar_id = grammars.add_lazy(LazyGrammar {
            bundle: std::sync::Arc::clone(bundle),
            index: member.blob as usize,
            traits: member.traits,
        });
        if blob.scope_name == root_blob.scope_name {
            root = Some(grammar_id);
        }
    }

    // Community grammars occasionally retain optional repository includes
    // supplied only by a host editor extension. The tokenizer skips those
    // references rather than disabling the complete bundled backend.
    let root = root.ok_or_else(missing)?;
    if grammars.grammar(root).is_none() {
        let error = root_blob.compiled_grammar(root).err();
        return Err(Error::Bundle(crate::BundleError::new(
            crate::BundleErrorKind::Decode,
            Some(root_blob.language.to_string()),
            format!(
                "failed to decode bundled TextMate grammar `{}`: {error:?}",
                root_blob.language
            ),
        )));
    }
    Ok((grammars, root))
}

/// Decode exactly the compiled external-include closure of one root.
#[cfg(test)]
fn compiled_grammar_closure(
    bundle: &crate::grammars::bundle::Bundle,
    root_scope: &str,
) -> Result<Vec<grammar::CompiledGrammar>> {
    let mut grammars = decode_bundle_grammars(bundle);
    let root = grammars
        .iter()
        .rposition(|grammar| grammar.scope_name == root_scope)
        .ok_or_else(|| {
            Error::Bundle(crate::BundleError::new(
                crate::BundleErrorKind::MissingGrammar,
                Some(root_scope.to_owned()),
                format!("missing root `{root_scope}`"),
            ))
        })?;
    let members = grammar_closure::dependency_closure(&grammars, root);
    let mut closure = Vec::with_capacity(members.len());
    for index in members.into_iter().rev() {
        closure.push(grammars.swap_remove(index));
    }
    closure.reverse();
    for (id, grammar) in closure.iter_mut().enumerate() {
        grammar.id = state::GrammarId(u16::try_from(id).expect("closure fits in GrammarId"));
    }
    Ok(closure)
}

#[cfg(test)]
fn decode_bundle_grammars(
    bundle: &crate::grammars::bundle::Bundle,
) -> Vec<grammar::CompiledGrammar> {
    bundle
        .grammar_blobs
        .iter()
        .map(|blob| {
            blob.compiled_grammar(state::GrammarId(0))
                .unwrap_or_else(|error| panic!("{}: {error:?}", blob.language))
        })
        .collect()
}