ferroni 2.2.1

Oniguruma's regex engine in pure Rust: full Oniguruma syntax, a vscode-oniguruma-compatible TextMate scanner, no C toolchain
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

//! Tokenizing a line with the multi-pattern Scanner API.
//!
//! The Scanner is the operation behind TextMate-based syntax highlighting:
//! many patterns are matched against the same line, and the leftmost match
//! wins. It is API-compatible with vscode-oniguruma, so the same loop drives
//! vscode-textmate and Shiki grammars.
//!
//! Run with:
//! cargo run --example scanner_tokenize

use ferroni::prelude::*;

/// The scope name reported for each pattern, by pattern index.
const SCOPES: &[&str] = &["keyword", "string", "comment", "number", "identifier"];

fn main() -> Result<(), RegexError> {
    let mut scanner = Scanner::new(&[
        r"\b(?:const|let|var|function|return)\b",
        r#""[^"]*""#,
        r"//.*$",
        r"\b\d+(?:\.\d+)?\b",
        r"\b[A-Za-z_]\w*\b",
    ])?;

    let line = r#"const label = "answer" + 42 // "not" a string"#;

    // Walk the line the way a highlighter does: find the next match, emit the
    // token, then continue from the end of that match.
    let mut position = 0;
    while let Some(m) = scanner.find_next_match(line, position, ScannerFindOptions::NONE) {
        let whole = &m.captures()[0];
        println!(
            "{:>10}  {:>3}..{:<3} {}",
            SCOPES[m.index],
            whole.start,
            whole.end,
            &line[whole.start..whole.end],
        );

        // Guard against zero-width matches, which would loop forever. Step to
        // the next character boundary: stepping one byte could land inside a
        // multibyte character and panic on the next slice.
        let next_boundary =
            whole.start + line[whole.start..].chars().next().map_or(1, char::len_utf8);
        position = whole.end.max(next_boundary);
    }

    // vscode-textmate and Shiki address text in UTF-16 code units. Wrap the
    // line in an `OnigString` and the offsets come back in the same units.
    let mut scanner = Scanner::new(&["world"])?;
    let text = OnigString::new("hello 🌍 world");
    let m = scanner
        .find_next_match_utf16(&text, 0, ScannerFindOptions::NONE)
        .expect("`world` is in the text");
    println!(
        "\nUTF-16 offsets: {}..{} (the emoji counts as two units)",
        m.captures()[0].start,
        m.captures()[0].end,
    );

    Ok(())
}