udled_tokenizers/
string.rs

1use udled::{token::Char, Error, Lex, Reader, Span, Tokenizer};
2
3#[derive(Debug, Clone, Copy, Default)]
4pub struct Str;
5
6impl Tokenizer for Str {
7    type Token<'a> = Lex<'a>;
8    fn to_token<'a>(&self, reader: &mut Reader<'_, 'a>) -> Result<Self::Token<'a>, Error> {
9        let start = reader.parse('"')?;
10
11        let end = loop {
12            if reader.eof() {
13                return Err(reader.error("Unexpected end of input while parsing string literal"));
14            }
15
16            let ch = reader.parse(Char)?;
17
18            if ch == r#"""# {
19                break ch.span;
20            }
21
22            if ch == "\\" {
23                match reader.eat_ch()? {
24                    "\\" | "\'" | "\"" | "t" | "r" | "n" | "0" => {
25                        continue;
26                    }
27                    _ => return Err(reader.error("Unknown escape sequence")),
28                }
29            }
30        };
31
32        let span = start + end;
33
34        let str = if span.len() == 2 {
35            Some("")
36        } else {
37            Span::new(span.start + 1, span.end - 1).slice(reader.source())
38        };
39
40        Ok(Lex::new(str.unwrap(), span))
41    }
42
43    fn peek<'a>(&self, reader: &mut Reader<'_, '_>) -> Result<bool, Error> {
44        reader.peek('"')
45    }
46}
47
48#[cfg(test)]
49mod test {
50
51    use udled::Input;
52
53    use super::*;
54
55    #[test]
56    fn empty_string() {
57        let mut input = Input::new(r#""""#);
58        let str = input.parse(Str).unwrap();
59        assert_eq!(str.value, "");
60        assert_eq!(str.span, Span::new(0, 2));
61    }
62
63    #[test]
64    fn string() {
65        let mut input = Input::new(r#""Hello, World!""#);
66        let str = input.parse(Str).unwrap();
67        assert_eq!(str.value, "Hello, World!");
68        assert_eq!(str.span, Span::new(0, 15));
69    }
70}
udled_tokenizers/string.rs

udled_tokenizers/
string.rs
