rustine 0.1.1

High-performance Gel syntax parser transforming to JSON/XML (Rust + PyO3)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

use rustine::exec::streaming::StreamingEvent;
use rustine::parser::ast::GelDocument;
use rustine::StreamingRunner;

fn parse_doc(src: &str) -> GelDocument {
    let tokens = rustine::parser::lexer::lex(src).unwrap();
    rustine::parser::syntax::parse_gel_document(&tokens).unwrap()
}

#[test]
fn streaming_single_chunk_matches_batch() {
    let src = "\
grammar main:
    match /(\\w+)/:
        out.create(\"word\", $1)
";
    let input = "hello world";
    let mut doc = parse_doc(src);

    // Batch execution
    let batch = rustine::exec::execute(&mut doc.clone(), "main", input).unwrap();

    // Streaming: feed all at once
    let mut sr = StreamingRunner::new(&mut doc, "main").unwrap();
    sr.feed(input);
    sr.set_eof();
    let result = sr.finish().unwrap();

    assert_eq!(result.consumed, batch.consumed, "consumed bytes should match");
}

#[test]
fn streaming_chunked_input() {
    let src = "\
grammar main:
    match /(\\w+)/:
        out.create(\"word\", $1)
    skip /\\s+/
";
    let mut doc = parse_doc(src);
    let mut sr = StreamingRunner::new(&mut doc, "main").unwrap();

    // Feed in two chunks
    sr.feed("hello ");
    let events1 = sr.step().unwrap();
    // Should have matched "hello" and consumed the space
    let has_match = events1
        .iter()
        .any(|e| matches!(e, StreamingEvent::MatchConsumed { .. }));
    assert!(has_match, "first chunk should produce a match event");

    // Feed more data + EOF
    sr.feed("world");
    sr.set_eof();
    let events2 = sr.step().unwrap();
    let has_match2 = events2
        .iter()
        .any(|e| matches!(e, StreamingEvent::MatchConsumed { .. }));
    assert!(has_match2, "second chunk should produce a match event");

    let has_finished = events2.iter().any(|e| matches!(e, StreamingEvent::Finished { .. }));
    assert!(has_finished, "should finish after eof");
}

#[test]
fn streaming_events_include_create() {
    let src = "\
grammar main:
    match /(\\w+)/:
        out.create(\"item\", $1)
    skip /\\s+/
";
    let mut doc = parse_doc(src);
    let mut sr = StreamingRunner::new(&mut doc, "main").unwrap();
    sr.feed("alpha beta");
    sr.set_eof();
    let events = sr.step().unwrap();

    let creates: Vec<_> = events
        .iter()
        .filter(|e| matches!(e, StreamingEvent::Create { .. }))
        .collect();
    assert!(
        creates.len() >= 2,
        "should have at least 2 Create events, got {}",
        creates.len()
    );
}

#[test]
fn streaming_stalled_without_eof() {
    let src = "\
grammar main:
    match /(\\d+)/:
        out.create(\"num\", $1)
";
    let mut doc = parse_doc(src);
    let mut sr = StreamingRunner::new(&mut doc, "main").unwrap();
    sr.feed("abc");
    let events = sr.step().unwrap();
    // Input doesn't match grammar, no EOF → stalled
    let has_stalled = events.iter().any(|e| matches!(e, StreamingEvent::Stalled { .. }));
    assert!(has_stalled, "should stall when no match and no eof");
}

#[test]
fn streaming_finish_produces_full_result() {
    let src = "\
grammar main:
    match /([a-z]+)=([0-9]+)/:
        out.create(\"pair/$1\", $2)
    skip /\\s+/
";
    let mut doc = parse_doc(src);
    let mut sr = StreamingRunner::new(&mut doc, "main").unwrap();
    sr.feed("x=1 y=2 z=3");
    sr.set_eof();
    let result = sr.finish().unwrap();

    assert!(result.consumed > 0, "should consume some input");
    assert!(
        result.capture_history.len() >= 3,
        "should have 3 capture rounds, got {}",
        result.capture_history.len()
    );
}