udled 0.8.0

Tokenizer and parser
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

use udled::{prelude::*, Buffer, Char, Either, Input, Item, Span, EOF};

#[test]
fn char_tokenizer() {
    let mut input = Input::new("abc");
    let item = input.parse(Char).unwrap();
    assert_eq!(item.value, 'a');
    assert_eq!(item.span, Span::new(0, 1));

    let item = input.parse(Char).unwrap();
    assert_eq!(item.value, 'b');
    assert_eq!(item.span, Span::new(1, 2));
}

#[test]
fn char_tokenizer_fails_on_empty() {
    let mut input = Input::new("");
    assert!(input.parse(Char).is_err());
}

#[test]
fn literal_char() {
    let mut input = Input::new("xyz");
    let item = input.parse('x').unwrap();
    assert_eq!(item.value, 'x');
    assert_eq!(item.span, Span::new(0, 1));

    assert!(input.parse('x').is_err());
}

#[test]
fn literal_string() {
    let mut input = Input::new("hello world");
    let item = input.parse("hello").unwrap();
    assert_eq!(item.value, "hello");
    assert_eq!(item.span, Span::new(0, 5));

    assert!(input.parse("hello").is_err());
}

#[test]
fn literal_string_empty_fails() {
    let mut input = Input::new("abc");
    assert!(input.parse("").is_err());
}

#[test]
fn char_range() {
    let mut input = Input::new("37a");
    let item = input.parse('0'..='9').unwrap();
    assert_eq!(item.value, '3');

    let item = input.parse('0'..='9').unwrap();
    assert_eq!(item.value, '7');

    assert!(input.parse('0'..='9').is_err());
}

#[test]
fn eof_tokenizer() {
    let mut input = Input::new("");
    assert_eq!(input.parse(EOF).unwrap(), 0);

    let mut input = Input::new("a");
    assert!(input.parse(EOF).is_err());
}

#[test]
fn next_tokenizer() {
    use udled::tokenizers::Next;

    let mut input = Input::new("ab");
    assert_eq!(input.parse(Next).unwrap(), 'a');
    assert_eq!(input.parse(Next).unwrap(), 'b');
    assert!(input.parse(Next).is_err());
}

#[test]
fn either_tokenizer() {
    let mut input = Input::new("42");
    let tok = Either::<char, char>::Left('4').or(Either::<char, char>::Right('5'));
    let _ = input.parse(tok).unwrap();

    let mut input = Input::new("5");
    let tok = Either::<char, char>::Left('4').or(Either::<char, char>::Right('5'));
    let result = input.parse(tok).unwrap();
    assert!(matches!(result, Either::Right(_)));
}

#[test]
fn tuple_tokenizer() {
    let mut input = Input::new("ab");
    let (a, b) = input.parse(('a', 'b')).unwrap();
    assert_eq!(a.value, 'a');
    assert_eq!(b.value, 'b');
}

#[test]
fn tokenizer_eat_and_peek() {
    let mut input = Input::new("abc");
    assert!(input.is('a'));

    input.eat('a').unwrap();
    let remaining = input.parse("bc").unwrap();
    assert_eq!(remaining.value, "bc");
}

#[test]
fn input_buffer() {
    let input = Input::new("abc");
    assert_eq!(input.buffer().source(), "abc");
}

#[test]
fn item_map() {
    let item = Item::new(Span::new(0, 1), 'a');
    let mapped = item.map(|c| c.to_string());
    assert_eq!(mapped.value, "a");
    assert_eq!(mapped.span, Span::new(0, 1));
}