udled_tokenizers/
ws.rs

1use udled::{AsChar, AsSlice, AsStr, Buffer, Char, Item, Tokenizer, TokenizerExt};
2
3#[derive(Debug, Clone, Copy)]
4pub struct Whitespace;
5
6impl<'input, T> Tokenizer<'input, T> for Whitespace
7where
8    T: Buffer<'input>,
9    T::Item: AsChar,
10{
11    type Token = Item<char>;
12
13    fn to_token(
14        &self,
15        reader: &mut udled::Reader<'_, 'input, T>,
16    ) -> Result<Self::Token, udled::Error> {
17        let char = reader.parse(Char)?;
18        if char.value.is_whitespace() {
19            Ok(char)
20        } else {
21            Err(reader.error("Whitespace"))
22        }
23    }
24
25    fn eat(&self, reader: &mut udled::Reader<'_, 'input, T>) -> Result<(), udled::Error> {
26        let _ = self.to_token(reader)?;
27        Ok(())
28    }
29
30    fn peek(&self, reader: &mut udled::Reader<'_, 'input, T>) -> bool {
31        let Ok(ret) = reader.parse(Char) else {
32            return false;
33        };
34
35        ret.value.is_whitespace()
36    }
37}
38
39#[derive(Debug, Clone, Copy)]
40pub struct AsciiWhitespace;
41
42impl<'input, T> Tokenizer<'input, T> for AsciiWhitespace
43where
44    T: Buffer<'input>,
45    T::Item: AsChar,
46{
47    type Token = Item<char>;
48
49    fn to_token(
50        &self,
51        reader: &mut udled::Reader<'_, 'input, T>,
52    ) -> Result<Self::Token, udled::Error> {
53        let char = reader.parse(Char)?;
54        if char.value.is_ascii_whitespace() {
55            Ok(char)
56        } else {
57            Err(reader.error("Whitespace"))
58        }
59    }
60
61    fn eat(&self, reader: &mut udled::Reader<'_, 'input, T>) -> Result<(), udled::Error> {
62        let _ = self.to_token(reader)?;
63        Ok(())
64    }
65
66    fn peek(&self, reader: &mut udled::Reader<'_, 'input, T>) -> bool {
67        let Ok(ret) = reader.parse(Char) else {
68            return false;
69        };
70
71        ret.value.is_ascii_whitespace()
72    }
73}
74
75#[derive(Debug, Clone, Copy)]
76pub struct LineFeed;
77
78impl<'input, T> Tokenizer<'input, T> for LineFeed
79where
80    T: Buffer<'input>,
81    T::Item: AsChar,
82    T::Source: AsStr<'input> + AsSlice<'input>,
83{
84    type Token = Item<<T::Source as AsSlice<'input>>::Slice>;
85
86    fn to_token(
87        &self,
88        reader: &mut udled::Reader<'_, 'input, T>,
89    ) -> Result<Self::Token, udled::Error> {
90        let Ok(char) = reader.parse('\n'.or('\r').or('\u{2028}').or('\u{2029}').slice()) else {
91            return Err(reader.error("Line Feed"));
92        };
93        Ok(char)
94    }
95
96    fn eat(&self, reader: &mut udled::Reader<'_, 'input, T>) -> Result<(), udled::Error> {
97        let _ = self.to_token(reader)?;
98        Ok(())
99    }
100
101    fn peek(&self, reader: &mut udled::Reader<'_, 'input, T>) -> bool {
102        reader.is('\n'.or('\r').or('\u{2028}').or('\u{2029}'))
103    }
104}
105
106#[derive(Debug, Clone, Copy)]
107pub struct Space;
108
109impl<'input, T> Tokenizer<'input, T> for Space
110where
111    T: Buffer<'input>,
112    T::Item: AsChar,
113{
114    type Token = Item<char>;
115
116    fn to_token(
117        &self,
118        reader: &mut udled::Reader<'_, 'input, T>,
119    ) -> Result<Self::Token, udled::Error> {
120        let char = reader.parse(' '.or('\t'))?.unify();
121        Ok(char)
122    }
123
124    fn eat(&self, reader: &mut udled::Reader<'_, 'input, T>) -> Result<(), udled::Error> {
125        let _ = self.to_token(reader)?;
126        Ok(())
127    }
128
129    fn peek(&self, reader: &mut udled::Reader<'_, 'input, T>) -> bool {
130        reader.is(' '.or('\t'))
131    }
132}
udled_tokenizers/ws.rs

udled_tokenizers/
ws.rs
