udled 0.6.2

Tokenizer and parser
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

use alloc::string::ToString;

use crate::{AsBytes, AsChar, Buffer, Item, Reader, Result, Span, Tokenizer};

pub struct IgnoreCase<T>(pub T);

impl<'lit, T, B> Tokenizer<'lit, B> for IgnoreCase<T>
where
    T: AsRef<str>,
    B: Buffer<'lit>,
    B::Item: AsChar,
    B::Source: AsBytes<'lit>,
{
    type Token = Item<&'lit str>;
    fn to_token(&self, reader: &mut Reader<'_, 'lit, B>) -> Result<Self::Token> {
        let tokens = self.0.as_ref().chars();

        let start = reader.position();

        for token in tokens {
            let Some(next) = reader.read()?.as_char() else {
                return Err(reader.error(self.0.as_ref().to_string()));
            };

            if token.is_ascii() && next.is_ascii() {
                if !token.eq_ignore_ascii_case(&next) {
                    return Err(reader.error(self.0.as_ref().to_string()));
                }
            } else {
                let t = token.to_lowercase();
                let n = next.to_lowercase();

                if t.len() != n.len() {
                    return Err(reader.error(self.0.as_ref().to_string()));
                }

                for (t, n) in t.zip(n) {
                    if t != n {
                        return Err(reader.error(self.0.as_ref().to_string()));
                    }
                }
            }
        }

        if start == reader.position() {
            return Err(reader.error(self.0.as_ref().to_string()));
        }

        let span = Span {
            start,
            end: reader.position(),
        };

        let string = reader.buffer().source().as_bytes();
        let string = unsafe { core::str::from_utf8_unchecked(string) };

        Ok(Item {
            value: span.slice(string).unwrap(),
            span,
        })
    }

    fn peek(&self, reader: &mut Reader<'_, 'lit, B>) -> bool {
        let tokens = self.0.as_ref().chars();
        for (idx, next) in tokens.enumerate() {
            if Some(next) == reader.peek_chn(idx).and_then(|m| m.as_char()) {
                continue;
            }
            return false;
        }

        true
    }
}

#[cfg(test)]
mod test {
    use crate::Input;

    use super::IgnoreCase;

    macro_rules! parse {
        ($parser: literal, $($input:literal),+) => {
          $(
            let mut input = Input::new($input);
            let ret = input.parse(IgnoreCase($parser)).expect("parse");

            assert_eq!($input,ret.value);
          )+
        };
    }

    #[test]

    fn ignore_case() {
        parse!("DOCTYPE", "docType", "DOCTYPE", "DocType");
        parse!("ÆæpÅLLÆ", "ææpållæ");
    }
}