jjpwrgem_parse/tokens/
lexical.rs

1use core::{fmt::Display, range::RangeInclusive};
2
3use crate::tokens::{CharWithContext, Token};
4
5/// see [`JsonChar::is_whitespace`]
6pub(crate) fn trim_end_whitespace(s: &str) -> &str {
7    let end = s
8        .char_indices()
9        .map(Into::<CharWithContext>::into)
10        .rev()
11        .find_map(|CharWithContext(r, c)| (!c.is_whitespace()).then_some(r.end))
12        .unwrap_or_default();
13
14    &s[..end]
15}
16
17#[derive(Debug, PartialEq, Eq, Clone, Copy, PartialOrd, Ord)]
18pub struct JsonChar(pub char);
19
20#[derive(Debug, PartialEq, Eq, Clone, Copy, PartialOrd, Ord)]
21pub struct JsonByte(pub u8);
22
23impl JsonByte {
24    pub(crate) fn is_whitespace(self) -> bool {
25        matches!(self.0, b' ' | b'\t' | b'\n' | b'\r')
26    }
27
28    pub(crate) fn as_token(self) -> Option<Token> {
29        let token = match self.0 {
30            b'{' => Token::OpenCurlyBrace,
31            b'}' => Token::ClosedCurlyBrace,
32            b':' => Token::Colon,
33            b',' => Token::Comma,
34            b'[' => Token::OpenSquareBracket,
35            b']' => Token::ClosedSquareBracket,
36            _ => return None,
37        };
38        Some(token)
39    }
40}
41
42impl JsonChar {
43    ///```abnf
44    /// char = unescaped /
45    ///       escape (
46    ///           %x22 /          ; "    quotation mark  U+0022
47    ///           %x5C /          ; \    reverse solidus U+005C
48    ///           %x2F /          ; /    solidus         U+002F
49    ///           %x62 /          ; b    backspace       U+0008
50    ///           %x66 /          ; f    form feed       U+000C
51    ///           %x6E /          ; n    line feed       U+000A
52    ///           %x72 /          ; r    carriage return U+000D
53    ///           %x74 /          ; t    tab             U+0009
54    ///           %x75 4HEXDIG )  ; uXXXX                U+XXXX
55    ///       escape = %x5C              ; \
56    ///
57    /// quotation-mark = %x22      ; "
58    ///
59    /// unescaped = %x20-21 / %x23-5B / %x5D-10FFFF
60    /// ```
61    /// see [RFC 8249 section 7](https://datatracker.ietf.org/doc/html/rfc8259#section-7)
62    pub(crate) fn direct_escape(self) -> Option<&'static str> {
63        match self.0 {
64            '"' => Some(r#"\""#),
65            '\\' => Some(r"\\"),
66            '/' => Some(r"\/"),
67            '\u{0008}' => Some(r"\b"),
68            '\u{000C}' => Some(r"\f"),
69            '\n' => Some(r"\n"),
70            '\r' => Some(r"\r"),
71            '\t' => Some(r"\t"),
72            _ => None,
73        }
74    }
75
76    #[cfg(any(test, feature = "serde"))]
77    pub(crate) fn minimal_escape(self) -> Option<&'static str> {
78        self.direct_escape().filter(|escaped| *escaped != r"\/")
79    }
80
81    pub(crate) fn escape(self) -> String {
82        match self.direct_escape() {
83            Some(escaped) => escaped.into(),
84            None => format!("\\u{:04X}", u32::from(self.0)),
85        }
86    }
87
88    pub(crate) fn is_hexdigit(self) -> bool {
89        self.0.is_ascii_hexdigit()
90    }
91
92    pub(crate) fn can_be_escaped_directly(self) -> bool {
93        matches!(self.0, '"' | '\\' | '/' | 'b' | 'f' | 'n' | 'r' | 't')
94    }
95
96    /// See [RFC 8259, Section 7](https://datatracker.ietf.org/doc/html/rfc8259#section-7)
97    pub(crate) const CONTROL_RANGE: RangeInclusive<char> = '\u{0000}'..='\u{001F}';
98
99    /// see [`Self::CONTROL_RANGE`]
100    pub(crate) fn is_control(self) -> bool {
101        Self::CONTROL_RANGE.contains(&self.0)
102    }
103
104    /// See [RFC 8259, Section 2](https://datatracker.ietf.org/doc/html/rfc8259#section-2):
105    ///
106    ///```abnf
107    /// ws = *(
108    ///         %x20 /              ; Space
109    ///         %x09 /              ; Horizontal tab
110    ///         %x0A /              ; Line feed or New line
111    ///         %x0D )              ; Carriage return
112    /// ```
113    pub(crate) fn is_whitespace(self) -> bool {
114        matches!(self.0, ' ' | '\t' | '\n' | '\r')
115    }
116
117    /// See [RFC 8259, Section 2](https://datatracker.ietf.org/doc/html/rfc8259#section-2):
118    ///
119    /// ```abnf
120    /// structural-character = begin-array / begin-object / end-array /
121    ///                        end-object / name-separator / value-separator
122    ///
123    /// begin-array     = ws %x5B ws  ; [ left square bracket
124    /// begin-object    = ws %x7B ws  ; { left curly bracket
125    /// end-array       = ws %x5D ws  ; ] right square bracket
126    /// end-object      = ws %x7D ws  ; } right curly bracket
127    /// name-separator  = ws %x3A ws  ; : colon
128    /// value-separator = ws %x2C ws  ; , comma
129    /// ```
130    pub(crate) fn is_structural(self) -> bool {
131        matches!(self.0, '{' | '}' | '[' | ']' | ':' | ',')
132    }
133}
134
135impl Display for JsonChar {
136    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
137        let rep = if self.is_control() {
138            self.escape()
139        } else {
140            self.0.to_string()
141        };
142        write!(f, "{rep}")
143    }
144}
145
146impl From<char> for JsonChar {
147    fn from(value: char) -> Self {
148        Self(value)
149    }
150}
151
152impl From<u8> for JsonByte {
153    fn from(value: u8) -> Self {
154        Self(value)
155    }
156}
157
158#[cfg(test)]
159mod tests {
160    use rstest::rstest;
161
162    use super::*;
163
164    #[rstest]
165    #[case('"', Some(r#"\""#))]
166    #[case('\\', Some(r"\\"))]
167    #[case('/', Some(r"\/"))]
168    #[case('\n', Some(r"\n"))]
169    #[case('a', None)]
170    fn direct_escape_cases(#[case] input: char, #[case] expected: Option<&str>) {
171        assert_eq!(JsonChar(input).direct_escape(), expected);
172    }
173
174    #[rstest]
175    #[case('"', Some(r#"\""#))]
176    #[case('\\', Some(r"\\"))]
177    #[case('/', None)]
178    #[case('\n', Some(r"\n"))]
179    #[case('a', None)]
180    fn minimal_escape_cases(#[case] input: char, #[case] expected: Option<&str>) {
181        assert_eq!(JsonChar(input).minimal_escape(), expected);
182    }
183
184    #[rstest]
185    #[case('\u{0008}', r"\b")]
186    #[case('\u{000C}', r"\f")]
187    #[case('\n', r"\n")]
188    #[case('\r', r"\r")]
189    #[case('\t', r"\t")]
190    fn escape_char_for_json_string_short_forms(#[case] input: char, #[case] expected: &str) {
191        assert_eq!(JsonChar(input).to_string(), expected);
192    }
193
194    #[rstest]
195    #[case('\u{0000}', "\\u0000")]
196    #[case('\u{001F}', "\\u001F")]
197    #[case('\u{0011}', "\\u0011")]
198    fn escape_char_for_json_string_control_chars(#[case] input: char, #[case] expected: &str) {
199        assert_eq!(JsonChar(input).to_string(), expected);
200    }
201
202    #[rstest]
203    #[case('a')]
204    #[case('Z')]
205    #[case('0')]
206    #[case(' ')]
207    #[case('{')]
208    #[case('"')]
209    #[case('\\')]
210    #[case('/')]
211    fn escape_char_for_json_string_leaves_non_specials(#[case] input: char) {
212        assert_eq!(JsonChar(input).to_string(), input.to_string());
213    }
214
215    #[rstest]
216    #[case("h \t\n\r", "h")]
217    #[case("\u{000B} h ", "\u{000B} h")]
218    #[case("rust", "rust")]
219    fn trim_end_whitespace_cases(#[case] input: &str, #[case] output: &str) {
220        assert_eq!(trim_end_whitespace(input), output);
221    }
222}
jjpwrgem_parse/tokens/lexical.rs

jjpwrgem_parse/tokens/
lexical.rs
