ll-sparql-parser 0.6.0

A resilient LL parser for SPARQL
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

use logos::Logos;

use crate::syntax_kind::SyntaxKind;

fn tokenize(input: &str) -> Vec<SyntaxKind> {
    let mut token_kinds = Vec::new();
    let lexer = SyntaxKind::lexer(input);
    for result in lexer {
        match result {
            Ok(kind) if !kind.is_trivia() => token_kinds.push(kind),
            Err(_) => token_kinds.push(SyntaxKind::Error),
            _ => {}
        }
    }
    return token_kinds;
}

#[test]
fn tokenize_langtag() {
    let tokens = tokenize(r#""dings"@de "foo"@a-109283"#);
    assert_eq!(
        tokens,
        vec![
            SyntaxKind::STRING_LITERAL2,
            SyntaxKind::LANGTAG,
            SyntaxKind::STRING_LITERAL2,
            SyntaxKind::LANGTAG
        ]
    )
}

#[test]
fn tokenize_brack() {
    let tokens = tokenize("[] [ ] [ ?var ] ");
    assert_eq!(
        tokens,
        vec![
            SyntaxKind::ANON,
            SyntaxKind::ANON,
            SyntaxKind::LBrack,
            SyntaxKind::VAR1,
            SyntaxKind::RBrack
        ]
    )
}

#[test]
fn tokenize_a() {
    let tokens = tokenize("abc a affiliation");
    assert_eq!(
        tokens,
        vec![SyntaxKind::Error, SyntaxKind::a, SyntaxKind::Error,]
    )
}

#[test]
fn tokenize_variables() {
    let tokens = tokenize("?var $x ?x2 ?münchen ?42 ?2· ?x ?a_b");
    assert_eq!(
        tokens,
        vec![
            SyntaxKind::VAR1,
            SyntaxKind::VAR2,
            SyntaxKind::VAR1,
            SyntaxKind::VAR1,
            SyntaxKind::VAR1,
            SyntaxKind::VAR1,
            SyntaxKind::VAR1,
            SyntaxKind::VAR1,
        ]
    )
}

#[test]
fn tokenize_numbers() {
    let tokens = tokenize("42 4.2 .42 +1 -1 +1.2 -1.3 -.2 1.2e+9");
    assert_eq!(
        tokens,
        vec![
            SyntaxKind::INTEGER,
            SyntaxKind::DECIMAL,
            SyntaxKind::DECIMAL,
            SyntaxKind::INTEGER_POSITIVE,
            SyntaxKind::INTEGER_NEGATIVE,
            SyntaxKind::DECIMAL_POSITIVE,
            SyntaxKind::DECIMAL_NEGATIVE,
            SyntaxKind::DECIMAL_NEGATIVE,
            SyntaxKind::DOUBLE,
        ]
    )
}

#[test]
fn tokenize_iris() {
    let tokens = tokenize("<simple> prefix: ns:local2 ns:123 ns:%32 x....42: äöü:öäü");
    assert_eq!(
        tokens,
        vec![
            SyntaxKind::IRIREF,
            SyntaxKind::PNAME_NS,
            SyntaxKind::PNAME_LN,
            SyntaxKind::PNAME_LN,
            SyntaxKind::PNAME_LN,
            SyntaxKind::PNAME_NS,
            SyntaxKind::PNAME_LN,
        ]
    )
}