udled 0.8.0

Tokenizer and parser
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

use udled::tokenizers::{
    AlphaNumeric, Alphabetic, AsciiWhiteSpace, LineBreak, Numeric, Punct, WhiteSpace,
};
use udled::Input;

#[test]
fn alphabetic() {
    let mut input = Input::new("abc123");
    assert_eq!(input.parse(Alphabetic).unwrap().value, 'a');
    assert_eq!(input.parse(Alphabetic).unwrap().value, 'b');
    assert_eq!(input.parse(Alphabetic).unwrap().value, 'c');
    assert!(input.parse(Alphabetic).is_err());
}

#[test]
fn alphanumeric() {
    let mut input = Input::new("a1!");
    assert_eq!(input.parse(AlphaNumeric).unwrap().value, 'a');
    assert_eq!(input.parse(AlphaNumeric).unwrap().value, '1');
    assert!(input.parse(AlphaNumeric).is_err());
}

#[test]
fn numeric() {
    let mut input = Input::new("123a");
    assert_eq!(input.parse(Numeric).unwrap().value, '1');
    assert_eq!(input.parse(Numeric).unwrap().value, '2');
    assert_eq!(input.parse(Numeric).unwrap().value, '3');
    assert!(input.parse(Numeric).is_err());
}

#[test]
fn punct_tokenizer() {
    let mut input = Input::new("!@#");
    assert_eq!(input.parse(Punct).unwrap().value, '!');
    assert_eq!(input.parse(Punct).unwrap().value, '@');
    assert_eq!(input.parse(Punct).unwrap().value, '#');
    assert!(input.parse(Punct).is_err());
}

#[test]
fn ascii_whitespace() {
    let mut input = Input::new(" \t\n");
    assert_eq!(input.parse(AsciiWhiteSpace).unwrap().value, ' ');
    assert_eq!(input.parse(AsciiWhiteSpace).unwrap().value, '\t');
    assert_eq!(input.parse(AsciiWhiteSpace).unwrap().value, '\n');
    assert!(input.parse(AsciiWhiteSpace).is_err());
}

#[test]
fn whitespace() {
    let mut input = Input::new(" \u{00A0}");
    assert_eq!(input.parse(WhiteSpace).unwrap().value, ' ');
    assert_eq!(input.parse(WhiteSpace).unwrap().value, '\u{00A0}');
}

#[test]
fn linebreak() {
    let mut input = Input::new("\n\r\u{2028}");
    assert_eq!(input.parse(LineBreak).unwrap().value, '\n');
    assert_eq!(input.parse(LineBreak).unwrap().value, '\r');
    assert_eq!(input.parse(LineBreak).unwrap().value, '\u{2028}');
}

#[test]
fn digit_base() {
    use udled::tokenizers::Digit;

    let mut input = Input::new("9");
    assert_eq!(input.parse(Digit::default()).unwrap().value, 9);

    let mut input = Input::new("F");
    assert_eq!(input.parse(Digit(16)).unwrap().value, 15);

    let mut input = Input::new("G");
    assert!(input.parse(Digit(16)).is_err());

    let mut input = Input::new("7");
    assert!(input.parse(Digit(4)).is_err());
}