syntaxmate 0.1.2

Rust-native TextMate syntax highlighting with bundled grammars and themes
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

use syntaxmate::{GrammarRegistry, Tokenizer, TokenizerOptions};

fn line_scopes_complete(
    source: &str,
    grammar: &str,
) -> Vec<Vec<(std::ops::Range<usize>, Vec<String>)>> {
    let mut registry = GrammarRegistry::new();
    let root = registry.add_json(grammar).expect("grammar");
    let mut tokenizer = Tokenizer::new(
        &registry,
        root,
        TokenizerOptions {
            max_line_bytes: 16 * 1024,
            line_cache_entries: 64,
        },
    )
    .expect("tokenizer");
    tokenizer
        .tokenize(source)
        .lines()
        .iter()
        .map(|line| {
            line.spans()
                .iter()
                .map(|span| {
                    (
                        span.range(),
                        line.scope_names(span.scope_stack())
                            .map(str::to_owned)
                            .collect(),
                    )
                })
                .collect()
        })
        .collect()
}

fn line_scopes_incremental(
    source: &str,
    grammar: &str,
) -> Vec<Vec<(std::ops::Range<usize>, Vec<String>)>> {
    let mut registry = GrammarRegistry::new();
    let root = registry.add_json(grammar).expect("grammar");
    let mut tokenizer = Tokenizer::new(
        &registry,
        root,
        TokenizerOptions {
            max_line_bytes: 16 * 1024,
            line_cache_entries: 64,
        },
    )
    .expect("tokenizer");
    let mut state = tokenizer.initial_state();
    source
        .split('\n')
        .map(|line| {
            let tokenized = tokenizer.tokenize_line(line, &mut state).expect("line");
            tokenized
                .tokens()
                .iter()
                .map(|token| (token.range(), token.scopes().map(str::to_owned).collect()))
                .collect()
        })
        .collect()
}

#[test]
fn final_line_without_trailing_newline_can_differ_from_tokenize_line() {
    // Lookahead class includes `\s`, so the synthetic newline that
    // `tokenize_line` always appends changes the match at EOF.
    // The fuzz harness therefore skips complete↔incremental parity for the
    // final line when the source has no trailing newline.
    let grammar = r#"{
        "scopeName": "source.seed.eol",
        "patterns": [
            {
                "match": "(?<word>[[:alpha:]]+)(?=\\s)",
                "name": "meta.word"
            }
        ]
    }"#;
    let source = "nabarbaal";

    let complete = line_scopes_complete(source, grammar);
    let incremental = line_scopes_incremental(source, grammar);
    assert_ne!(
        complete, incremental,
        "complete tokenize and tokenize_line intentionally disagree without a trailing newline"
    );
}