udled_tokenizers/
numeric.rs

1use alloc::string::ToString;
2use udled::{
3    any,
4    token::{Digit, Opt, Spanned},
5    Error, Item, Reader, Span, StringExt, Tokenizer,
6};
7/// Match a rust style integer
8#[derive(Debug, Clone, Copy, Default)]
9pub struct Int;
10
11impl Tokenizer for Int {
12    type Token<'a> = Item<i128>;
13    fn to_token<'a>(&self, reader: &mut Reader<'_, 'a>) -> Result<Self::Token<'a>, Error> {
14        let mut val: i128 = 0;
15
16        let start = reader.position();
17
18        let sign = if reader.parse("-").is_ok() { -1 } else { 1 };
19
20        let mut base = 10;
21        if reader.parse("0x").is_ok() {
22            base = 16
23        };
24        if reader.parse("0b").is_ok() {
25            base = 2
26        };
27
28        loop {
29            let ch = reader.parse(Digit(base))?;
30
31            val = (base as i128) * val + (ch as i128);
32
33            let Some(ch) = reader.peek_ch() else {
34                break;
35            };
36
37            // Allow underscores as separators
38            if ch == "_" {
39                reader.eat_ch()?;
40                continue;
41            }
42
43            if ch == "\0" {
44                break;
45            }
46
47            if !ch.is_digit(base) {
48                break;
49            }
50        }
51
52        return Ok(Item::new(sign * val, Span::new(start, reader.position())));
53    }
54
55    fn peek(&self, reader: &mut Reader<'_, '_>) -> Result<bool, Error> {
56        let Some(mut ch) = reader.peek_ch() else {
57            return Ok(false);
58        };
59
60        if ch == "-" {
61            let Some(next) = reader.peek_chn(1) else {
62                return Ok(false);
63            };
64
65            ch = next;
66        }
67
68        Ok(ch.is_digit(10))
69    }
70}
71
72/// Match a integer
73pub struct Integer;
74
75impl Tokenizer for Integer {
76    type Token<'a> = Item<i128>;
77
78    fn to_token<'a>(&self, reader: &mut Reader<'_, 'a>) -> Result<Self::Token<'a>, Error> {
79        let start = reader.position();
80        let mut val: i128 = 0;
81        let base = 10;
82
83        loop {
84            let ch = reader.parse(Digit(base))?;
85
86            val = (base as i128) * val + (ch as i128);
87
88            let Some(ch) = reader.peek_ch() else {
89                break;
90            };
91
92            // Allow underscores as separators
93            if ch == "_" {
94                reader.eat_ch()?;
95                continue;
96            }
97
98            if ch == "\0" {
99                break;
100            }
101
102            if !ch.is_digit(base) {
103                break;
104            }
105        }
106
107        return Ok(Item::new(val, Span::new(start, reader.position())));
108    }
109
110    fn peek(&self, reader: &mut Reader<'_, '_>) -> Result<bool, Error> {
111        reader.peek(Digit(10))
112    }
113}
114
115/// Match a rust style float
116#[derive(Debug, Clone, Copy, Default)]
117pub struct Float;
118
119impl Tokenizer for Float {
120    type Token<'a> = Item<f64>;
121
122    fn to_token<'a>(&self, reader: &mut Reader<'_, 'a>) -> Result<Self::Token<'a>, Error> {
123        let start = reader.parse(Spanned(Integer))?;
124        reader.eat('.')?;
125        let mut end = reader.parse(Spanned(Integer))?;
126
127        if reader.peek(any!('E', 'e'))? {
128            end = reader.parse(Spanned((any!('E', 'e'), Opt('-'), Integer)))?;
129        }
130
131        let input = (start + end)
132            .slice(reader.source())
133            .ok_or_else(|| reader.error("Invalid range"))?;
134
135        let float: f64 = input
136            .parse()
137            .map_err(|err: core::num::ParseFloatError| reader.error(err.to_string()))?;
138
139        Ok(Item::new(float, start + end))
140    }
141}
142
143#[cfg(test)]
144mod test {
145    use udled::{token::Ws, Input};
146
147    use super::{Float, Int, Integer};
148
149    #[test]
150    fn integer() {
151        let mut input = Input::new("10203 0 42");
152
153        let (a, _, b, _, c) = input.parse((Integer, Ws, Integer, Ws, Integer)).unwrap();
154
155        assert_eq!(a.value, 10203);
156        assert_eq!(b.value, 0);
157        assert_eq!(c.value, 42);
158    }
159
160    #[test]
161    fn int() {
162        let mut input = Input::new("0x202 0b11 42");
163
164        let (a, _, b, _, c) = input.parse((Int, Ws, Int, Ws, Int)).unwrap();
165
166        assert_eq!(a.value, 0x202);
167        assert_eq!(b.value, 0b11);
168        assert_eq!(c.value, 42);
169    }
170
171    #[test]
172    fn float() {
173        let mut input = Input::new("1.0 2003.303 12.03e-20");
174
175        let (a, _, b, _, c) = input.parse((Float, Ws, Float, Ws, Float)).unwrap();
176
177        assert_eq!(a.value, 1.0);
178        assert_eq!(b.value, 2003.303);
179        assert_eq!(c.value, 12.03e-20);
180    }
181}
udled_tokenizers/numeric.rs

udled_tokenizers/
numeric.rs
