ferroui_base/utilities/
span_string_tokenizer.rs

1//! Allocation-free tokenizer used by the `parse` implementations of the value types.
2
3use std::borrow::Cow;
4use std::fmt;
5
6use crate::utilities::span_helpers::{try_parse_double, try_parse_int, NumberStyles};
7
8const DEFAULT_SEPARATOR_CHAR: char = ',';
9
10const DEFAULT_FORMAT_MESSAGE: &str = "One of the identified items was in an invalid format.";
11
12/// Error returned when a string is not in the expected format.
13#[derive(Clone, Debug, PartialEq, Eq)]
14pub struct FormatError {
15    message: Cow<'static, str>,
16}
17
18impl FormatError {
19    /// Creates an error with a static message.
20    #[inline]
21    pub const fn new(message: &'static str) -> Self {
22        Self {
23            message: Cow::Borrowed(message),
24        }
25    }
26
27    /// Creates an error with a computed message.
28    #[inline]
29    pub fn from_string(message: String) -> Self {
30        Self {
31            message: Cow::Owned(message),
32        }
33    }
34
35    /// Creates the error reported when a number cannot be parsed from `input`.
36    pub fn invalid_input(input: &str) -> Self {
37        Self::from_string(format!(
38            "The input string '{input}' was not in a correct format."
39        ))
40    }
41
42    /// Gets the error message.
43    #[inline]
44    pub fn message(&self) -> &str {
45        &self.message
46    }
47}
48
49impl Default for FormatError {
50    fn default() -> Self {
51        Self::new(DEFAULT_FORMAT_MESSAGE)
52    }
53}
54
55impl fmt::Display for FormatError {
56    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
57        f.write_str(&self.message)
58    }
59}
60
61impl std::error::Error for FormatError {}
62
63/// Splits a string into tokens separated by white space and/or a single separator
64/// character, reading them as integers, doubles or raw slices.
65///
66/// Reading can fail in two distinct ways, mirrored by the `try_read_*` signatures:
67/// `Ok(None)` means "no (valid) value here", `Err` means the input is malformed
68/// (empty token, doubled separator, trailing separator, ...).
69#[derive(Clone, Debug)]
70pub struct SpanStringTokenizer<'a> {
71    s: &'a str,
72    separator: char,
73    exception_message: Option<&'static str>,
74    /// Byte offset of the read cursor.
75    index: usize,
76    /// Byte offset and byte length of the current token.
77    token: Option<(usize, usize)>,
78}
79
80impl<'a> SpanStringTokenizer<'a> {
81    /// Creates a tokenizer using `,` as separator and the default error message.
82    #[inline]
83    pub fn new(s: &'a str) -> Self {
84        Self::with_separator(s, DEFAULT_SEPARATOR_CHAR, None)
85    }
86
87    /// Creates a tokenizer using `,` as separator (the invariant-culture separator)
88    /// and the given error message.
89    #[inline]
90    pub fn with_message(s: &'a str, exception_message: &'static str) -> Self {
91        Self::with_separator(s, DEFAULT_SEPARATOR_CHAR, Some(exception_message))
92    }
93
94    /// Creates a tokenizer with an explicit separator and optional error message.
95    pub fn with_separator(
96        s: &'a str,
97        separator: char,
98        exception_message: Option<&'static str>,
99    ) -> Self {
100        let trimmed = s.trim_start_matches(char::is_whitespace);
101        Self {
102            s,
103            separator,
104            exception_message,
105            index: s.len() - trimmed.len(),
106            token: None,
107        }
108    }
109
110    /// Byte offset of the current token, or `None` when there is no current token.
111    #[inline]
112    pub fn current_token_index(&self) -> Option<usize> {
113        self.token.map(|(index, _)| index)
114    }
115
116    /// The current token, or `None` when there is no current token.
117    #[inline]
118    pub fn current_token(&self) -> Option<&'a str> {
119        self.token.map(|(index, len)| &self.s[index..index + len])
120    }
121
122    /// The current token, or an empty slice when there is no current token.
123    #[inline]
124    pub fn current_token_span(&self) -> &'a str {
125        self.current_token().unwrap_or("")
126    }
127
128    /// Verifies that the whole input was consumed.
129    ///
130    /// This is the counterpart of disposing the tokenizer in the reference
131    /// implementation and must be called once reading is done.
132    #[inline]
133    pub fn finish(&self) -> Result<(), FormatError> {
134        if self.index != self.s.len() {
135            return Err(self.format_error());
136        }
137        Ok(())
138    }
139
140    /// Runs `f` and then [`finish`](Self::finish). An error from `finish` takes
141    /// precedence over an error from `f`, exactly like an exception thrown while
142    /// leaving a scoped-disposal block replaces the one in flight.
143    #[inline]
144    pub fn scope<T>(
145        mut self,
146        f: impl FnOnce(&mut Self) -> Result<T, FormatError>,
147    ) -> Result<T, FormatError> {
148        let result = f(&mut self);
149        self.finish()?;
150        result
151    }
152
153    #[inline]
154    pub fn try_read_int32(&mut self) -> Result<Option<i32>, FormatError> {
155        self.try_read_int32_with(None)
156    }
157
158    pub fn try_read_int32_with(
159        &mut self,
160        separator: Option<char>,
161    ) -> Result<Option<i32>, FormatError> {
162        Ok(match self.try_read_span_with(separator)? {
163            Some(token) => try_parse_int(token, NumberStyles::INTEGER),
164            None => None,
165        })
166    }
167
168    #[inline]
169    pub fn read_int32(&mut self) -> Result<i32, FormatError> {
170        self.read_int32_with(None)
171    }
172
173    pub fn read_int32_with(&mut self, separator: Option<char>) -> Result<i32, FormatError> {
174        match self.try_read_int32_with(separator)? {
175            Some(value) => Ok(value),
176            None => Err(self.format_error()),
177        }
178    }
179
180    #[inline]
181    pub fn try_read_double(&mut self) -> Result<Option<f64>, FormatError> {
182        self.try_read_double_with(None)
183    }
184
185    pub fn try_read_double_with(
186        &mut self,
187        separator: Option<char>,
188    ) -> Result<Option<f64>, FormatError> {
189        Ok(match self.try_read_span_with(separator)? {
190            Some(token) => try_parse_double(token, NumberStyles::FLOAT),
191            None => None,
192        })
193    }
194
195    #[inline]
196    pub fn read_double(&mut self) -> Result<f64, FormatError> {
197        self.read_double_with(None)
198    }
199
200    pub fn read_double_with(&mut self, separator: Option<char>) -> Result<f64, FormatError> {
201        match self.try_read_double_with(separator)? {
202            Some(value) => Ok(value),
203            None => Err(self.format_error()),
204        }
205    }
206
207    /// Same as [`try_read_span`](Self::try_read_span); tokens are borrowed slices.
208    #[inline]
209    pub fn try_read_string(&mut self) -> Result<Option<&'a str>, FormatError> {
210        self.try_read_span_with(None)
211    }
212
213    #[inline]
214    pub fn try_read_string_with(
215        &mut self,
216        separator: Option<char>,
217    ) -> Result<Option<&'a str>, FormatError> {
218        self.try_read_span_with(separator)
219    }
220
221    /// Same as [`read_span`](Self::read_span); tokens are borrowed slices.
222    #[inline]
223    pub fn read_string(&mut self) -> Result<&'a str, FormatError> {
224        self.read_span_with(None)
225    }
226
227    #[inline]
228    pub fn read_string_with(&mut self, separator: Option<char>) -> Result<&'a str, FormatError> {
229        self.read_span_with(separator)
230    }
231
232    #[inline]
233    pub fn try_read_span(&mut self) -> Result<Option<&'a str>, FormatError> {
234        self.try_read_span_with(None)
235    }
236
237    pub fn try_read_span_with(
238        &mut self,
239        separator: Option<char>,
240    ) -> Result<Option<&'a str>, FormatError> {
241        let success = self.try_read_token(separator.unwrap_or(self.separator))?;
242        Ok(if success { self.current_token() } else { None })
243    }
244
245    #[inline]
246    pub fn read_span(&mut self) -> Result<&'a str, FormatError> {
247        self.read_span_with(None)
248    }
249
250    pub fn read_span_with(&mut self, separator: Option<char>) -> Result<&'a str, FormatError> {
251        match self.try_read_span_with(separator)? {
252            Some(token) => Ok(token),
253            None => Err(self.format_error()),
254        }
255    }
256
257    #[inline]
258    fn peek(&self) -> Option<char> {
259        self.s[self.index..].chars().next()
260    }
261
262    fn try_read_token(&mut self, separator: char) -> Result<bool, FormatError> {
263        self.token = None;
264
265        if self.index >= self.s.len() {
266            return Ok(false);
267        }
268
269        let index = self.index;
270
271        while let Some(c) = self.peek() {
272            if c.is_whitespace() || c == separator {
273                break;
274            }
275            self.index += c.len_utf8();
276        }
277
278        let length = self.index - index;
279
280        self.skip_to_next_token(separator)?;
281
282        self.token = Some((index, length));
283
284        if length < 1 {
285            return Err(self.format_error());
286        }
287
288        Ok(true)
289    }
290
291    fn skip_to_next_token(&mut self, separator: char) -> Result<(), FormatError> {
292        if let Some(c) = self.peek() {
293            if c != separator && !c.is_whitespace() {
294                return Err(self.format_error());
295            }
296
297            let mut length = 0;
298
299            while let Some(c) = self.peek() {
300                if c == separator {
301                    length += 1;
302                    self.index += c.len_utf8();
303
304                    if length > 1 {
305                        return Err(self.format_error());
306                    }
307                } else {
308                    if !c.is_whitespace() {
309                        break;
310                    }
311
312                    self.index += c.len_utf8();
313                }
314            }
315
316            if length > 0 && self.index >= self.s.len() {
317                return Err(self.format_error());
318            }
319        }
320
321        Ok(())
322    }
323
324    fn format_error(&self) -> FormatError {
325        match self.exception_message {
326            Some(message) => FormatError::new(message),
327            None => FormatError::default(),
328        }
329    }
330}
331
332#[cfg(test)]
333mod tests {
334    use super::*;
335
336    #[test]
337    fn read_int32_reads_values() {
338        let mut target = SpanStringTokenizer::new("123,456");
339
340        assert_eq!(123, target.read_int32().unwrap());
341        assert_eq!(456, target.read_int32().unwrap());
342        assert!(target.read_int32().is_err());
343    }
344
345    #[test]
346    fn read_double_reads_values() {
347        let mut target = SpanStringTokenizer::new("12.3,45.6");
348
349        assert_eq!(12.3, target.read_double().unwrap());
350        assert_eq!(45.6, target.read_double().unwrap());
351        assert!(target.read_double().is_err());
352    }
353
354    #[test]
355    fn try_read_int32_reads_values() {
356        let mut target = SpanStringTokenizer::new("123,456");
357
358        assert_eq!(Ok(Some(123)), target.try_read_int32());
359        assert_eq!(Ok(Some(456)), target.try_read_int32());
360        assert_eq!(Ok(None), target.try_read_int32());
361    }
362
363    #[test]
364    fn try_read_int32_doesnt_throw() {
365        let mut target = SpanStringTokenizer::new("abc");
366
367        assert_eq!(Ok(None), target.try_read_int32());
368    }
369
370    #[test]
371    fn try_read_double_reads_values() {
372        let mut target = SpanStringTokenizer::new("12.3,45.6");
373
374        assert_eq!(Ok(Some(12.3)), target.try_read_double());
375        assert_eq!(Ok(Some(45.6)), target.try_read_double());
376        assert_eq!(Ok(None), target.try_read_double());
377    }
378
379    #[test]
380    fn try_read_double_doesnt_throw() {
381        let mut target = SpanStringTokenizer::new("abc");
382
383        assert_eq!(Ok(None), target.try_read_double());
384    }
385
386    #[test]
387    fn read_span_and_read_string_reads_same() {
388        let mut target1 = SpanStringTokenizer::new("abc,def");
389        let mut target2 = SpanStringTokenizer::new("abc,def");
390
391        assert_eq!(target1.read_string().unwrap(), target2.read_span().unwrap());
392        assert_eq!(target1.read_span().unwrap(), target2.read_string().unwrap());
393    }
394
395    // Additional coverage of the malformed-input rules.
396
397    #[test]
398    fn rejects_empty_tokens_and_dangling_separators() {
399        assert!(SpanStringTokenizer::new(",1").read_int32().is_err());
400        assert!(SpanStringTokenizer::new("1,,2").read_int32().is_err());
401        assert!(SpanStringTokenizer::new("1,").read_int32().is_err());
402
403        let mut t = SpanStringTokenizer::with_message("1 , 2 3", "Invalid.");
404        assert_eq!(Ok(1), t.read_int32());
405        assert_eq!(Ok(2), t.read_int32());
406        assert_eq!(Err(FormatError::new("Invalid.")), t.finish());
407        assert_eq!(Ok(3), t.read_int32());
408        assert_eq!(Ok(()), t.finish());
409    }
410
411    #[test]
412    fn tracks_current_token() {
413        let mut t = SpanStringTokenizer::new("  ab  cd");
414        assert_eq!(None, t.current_token());
415        assert_eq!("", t.current_token_span());
416        assert_eq!(Ok("ab"), t.read_span());
417        assert_eq!(Some(2), t.current_token_index());
418        assert_eq!(Ok("cd"), t.read_span());
419        assert_eq!(Some("cd"), t.current_token());
420        assert_eq!(Ok(None), t.try_read_span());
421        assert_eq!(None, t.current_token_index());
422    }
423}
ferroui_base/utilities/span_string_tokenizer.rs

ferroui_base/utilities/
span_string_tokenizer.rs
