any_lexer/lexers/
json.rs

1use text_scanner::{ext::JsonScannerExt, Scanner};
2
3use crate::{impl_lexer_from_scanner, ScanToken, ScannerExt, TokenSpan};
4
5#[derive(PartialEq, Eq, Clone, Copy, Debug)]
6pub enum JsonToken {
7    Space,
8    String,
9    Number,
10    Null,
11    True,
12    False,
13    /// Punctuation e.g. `:`, `,`.
14    Punct,
15    /// Delimiter e.g. `{`, `}`, `[`, and `]`.
16    Delim,
17    /// Given valid JSON, then this variant should never be encountered. If
18    /// is is encountered, then check if an issue has already been submitted,
19    /// otherwise please [submit an issue].
20    ///
21    /// [submit an issue]: https://github.com/vallentin/colorblast/issues
22    Unknown,
23}
24
25impl ScanToken for JsonToken {
26    fn scan_token<'text>(scanner: &mut Scanner<'text>) -> Option<(Self, TokenSpan<'text>)> {
27        let (r, _s) = scanner.skip_whitespace();
28        if !r.is_empty() {
29            return Some((Self::Space, scanner.span(r)));
30        }
31
32        if let Ok((r, _c)) = scanner.accept_char_any(&['{', '}', '[', ']']) {
33            return Some((Self::Delim, scanner.span(r)));
34        } else if let Ok((r, _c)) = scanner.accept_char_any(&[':', ',']) {
35            return Some((Self::Punct, scanner.span(r)));
36        } else if let Ok((r, _c)) = scanner.scan_json_string() {
37            return Some((Self::String, scanner.span(r)));
38        } else if let Ok((r, _c)) = scanner.scan_json_number() {
39            return Some((Self::Number, scanner.span(r)));
40        }
41
42        let backtrack = scanner.cursor_pos();
43        let res = scanner.scan_with(|scanner| {
44            scanner.accept_if(|c| c.is_ascii_alphabetic())?;
45            scanner.skip_while(|c| c.is_ascii_alphabetic());
46
47            Ok(())
48        });
49        if let Ok((r, s)) = res {
50            match s {
51                "null" => return Some((Self::Null, scanner.span(r))),
52                "true" => return Some((Self::True, scanner.span(r))),
53                "false" => return Some((Self::False, scanner.span(r))),
54                _ => {
55                    scanner.set_cursor_pos(backtrack);
56                }
57            }
58        }
59
60        let (r, _c) = scanner.next().ok()?;
61        Some((Self::Unknown, scanner.span(r)))
62    }
63}
64
65/// JSON lexer producing [`JsonToken`]s.
66///
67/// **Note:** Cloning `JsonLexer` is essentially a copy, as it just contains
68/// a `&str` and a `usize` for its `cursor`. However, `Copy` is not
69/// implemented, to avoid accidentally copying immutable `JsonLexer`s.
70#[derive(Clone, Debug)]
71pub struct JsonLexer<'text> {
72    scanner: Scanner<'text>,
73}
74
75impl<'text> JsonLexer<'text> {
76    #[inline]
77    pub fn new(text: &'text str) -> Self {
78        Self {
79            scanner: Scanner::new(text),
80        }
81    }
82}
83
84impl_lexer_from_scanner!('text, JsonLexer<'text>, JsonToken, scanner);
85
86#[cfg(test)]
87mod tests {
88    use super::*;
89
90    #[test]
91    fn test_json_lexer_spans() {
92        // This intentionally uses Rust code as input, as it is
93        // only testing that JsonLexer returns all characters
94        let input = include_str!("../../../text-scanner/src/ext/rust.rs");
95        let mut output = String::new();
96
97        let lexer = JsonLexer::new(input);
98        for (_tok, span) in lexer {
99            output.push_str(span.as_str());
100        }
101
102        assert_eq!(input, output);
103    }
104}
any_lexer/lexers/json.rs

any_lexer/lexers/
json.rs
