any_lexer/lexers/
jsonc.rs

1use text_scanner::{ext::JsonCScannerExt, Scanner};
2
3use crate::{impl_lexer_from_scanner, JsonToken, ScanToken, ScannerExt, TokenSpan};
4
5#[derive(PartialEq, Eq, Clone, Copy, Debug)]
6pub enum JsonCToken {
7    Space,
8    LineComment,
9    BlockComment,
10    String,
11    Number,
12    Null,
13    True,
14    False,
15    /// Punctuation e.g. `:`, `,`.
16    Punct,
17    /// Delimiter e.g. `{`, `}`, `[`, and `]`.
18    Delim,
19    /// Given valid JSON with Comments, then this variant should never be
20    /// encountered. If is is encountered, then check if an issue has already
21    /// been submitted, otherwise please [submit an issue].
22    ///
23    /// [submit an issue]: https://github.com/vallentin/colorblast/issues
24    Unknown,
25}
26
27impl ScanToken for JsonCToken {
28    fn scan_token<'text>(scanner: &mut Scanner<'text>) -> Option<(Self, TokenSpan<'text>)> {
29        if let Ok((r, _s)) = scanner.scan_jsonc_line_comment() {
30            return Some((Self::LineComment, scanner.span(r)));
31        } else if let Ok((r, _s)) = scanner.scan_jsonc_block_comment() {
32            return Some((Self::BlockComment, scanner.span(r)));
33        }
34
35        let (tok, span) = JsonToken::scan_token(scanner)?;
36        match tok {
37            JsonToken::Space => Some((JsonCToken::Space, span)),
38            JsonToken::String => Some((JsonCToken::String, span)),
39            JsonToken::Number => Some((JsonCToken::Number, span)),
40            JsonToken::Null => Some((JsonCToken::Null, span)),
41            JsonToken::True => Some((JsonCToken::True, span)),
42            JsonToken::False => Some((JsonCToken::False, span)),
43            JsonToken::Punct => Some((JsonCToken::Punct, span)),
44            JsonToken::Delim => Some((JsonCToken::Delim, span)),
45            JsonToken::Unknown => Some((JsonCToken::Unknown, span)),
46        }
47    }
48}
49
50/// [JSON with Comments] lexer producing [`JsonCToken`]s.
51///
52/// **Note:** Cloning `JsonCLexer` is essentially a copy, as it just contains
53/// a `&str` and a `usize` for its `cursor`. However, `Copy` is not
54/// implemented, to avoid accidentally copying immutable `JsonCLexer`s.
55///
56/// [JSON with Comments]: https://code.visualstudio.com/docs/languages/json#_json-with-comments
57#[derive(Clone, Debug)]
58pub struct JsonCLexer<'text> {
59    scanner: Scanner<'text>,
60}
61
62impl<'text> JsonCLexer<'text> {
63    #[inline]
64    pub fn new(text: &'text str) -> Self {
65        Self {
66            scanner: Scanner::new(text),
67        }
68    }
69}
70
71impl_lexer_from_scanner!('text, JsonCLexer<'text>, JsonCToken, scanner);
72
73#[cfg(test)]
74mod tests {
75    use super::*;
76
77    #[test]
78    fn test_jsonc_lexer_spans() {
79        // This intentionally uses Rust code as input, as it is
80        // only testing that JsonCLexer returns all characters
81        let input = include_str!("../../../text-scanner/src/ext/rust.rs");
82        let mut output = String::new();
83
84        let lexer = JsonCLexer::new(input);
85        for (_tok, span) in lexer {
86            output.push_str(span.as_str());
87        }
88
89        assert_eq!(input, output);
90    }
91}
any_lexer/lexers/jsonc.rs

any_lexer/lexers/
jsonc.rs
