token_parser/
lib.rs

1#![deny(missing_docs)]
2
3/*!
4Some utilities for parsing some format based on nested lists into arbitrary data structures.
5It's also meant to be used as a backend for parsers.
6**/
7
8use std::{path::PathBuf, rc::Rc, sync::Arc};
9
10use thiserror::Error;
11
12#[cfg(feature = "derive")]
13pub use token_parser_derive::{Parsable, SymbolParsable};
14
15/// A trait required for all contexts being used for token parsing.
16///
17/// By default, only the empty tuple implements it.
18/// It currently does not contain anything by default. It's just there to achieve compatibility with features and to allow more changes without breaking anything.
19pub trait Context {
20    #[cfg(feature = "radix-parsing")]
21    #[inline]
22    /// Specifies the radix if the feature radix parsing is enabled.
23    fn radix(&self) -> u32 {
24        10
25    }
26}
27
28impl Context for () {}
29
30/// A source range with line, column, and byte offsets.
31#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
32pub struct Span {
33    /// The line number of the start position (0-based).
34    pub line: usize,
35    /// The column number of the start position (0-based).
36    pub column: usize,
37    /// The byte offset of the start of the span within the source.
38    pub start: usize,
39    /// The byte offset one past the end of the span within the source.
40    pub end: usize,
41}
42
43impl std::fmt::Display for Span {
44    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
45        write!(f, "{}:{}", self.line + 1, self.column + 1)
46    }
47}
48
49/// The kind of error that occurred during token parsing.
50#[derive(Debug, Error)]
51#[non_exhaustive]
52pub enum ErrorKind {
53    /// The sublist contains fewer elements than the given minimum.
54    #[error("Not enough elements: at least {0} expected")]
55    NotEnoughElements(usize),
56
57    /// The sublist contains more elements than expected by a specified amount.
58    #[error("Too many elements: {0} unexpected")]
59    TooManyElements(usize),
60
61    /// No list is allowed in this context.
62    #[error("List not allowed")]
63    ListNotAllowed,
64
65    /// No symbol is allowed in this context.
66    #[error("Symbol not allowed")]
67    SymbolNotAllowed,
68
69    /// String parsing failed for the named type.
70    #[error("Expected {type_name}: {source}")]
71    StringParsing {
72        /// The name of the type that failed to parse.
73        type_name: &'static str,
74        /// The underlying parse error from `FromStr`.
75        #[source]
76        source: Box<dyn std::error::Error + Send + Sync>,
77    },
78
79    /// The named field does not exist.
80    #[error("Unknown field {0}")]
81    UnknownField(Box<str>),
82
83    /// Some specific element is invalid.
84    #[error("Invalid element")]
85    InvalidElement,
86}
87
88/// The error type for token parsing, containing a kind and an optional source position.
89#[derive(Debug)]
90pub struct Error {
91    /// The kind of error.
92    pub kind: ErrorKind,
93    /// The source position where the error occurred, if known.
94    pub span: Option<Span>,
95    /// Optional context describing what was being parsed.
96    pub context: Option<Box<str>>,
97}
98
99impl std::fmt::Display for Error {
100    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
101        if let Some(span) = self.span {
102            write!(f, "{span}: ")?;
103        }
104        if let Some(ctx) = &self.context {
105            write!(f, "{ctx}: ")?;
106        }
107        write!(f, "{}", self.kind)
108    }
109}
110
111impl std::error::Error for Error {}
112
113impl From<ErrorKind> for Error {
114    fn from(kind: ErrorKind) -> Self {
115        Self {
116            kind,
117            span: None,
118            context: None,
119        }
120    }
121}
122
123impl Error {
124    /// Attaches a source position to the error, keeping an already attached position.
125    #[must_use]
126    pub const fn at(mut self, span: Span) -> Self {
127        if self.span.is_none() {
128            self.span = Some(span);
129        }
130        self
131    }
132
133    /// Adds descriptive context to the error (e.g., which field was being parsed), keeping already attached context.
134    pub fn context(mut self, msg: impl Into<Box<str>>) -> Self {
135        if self.context.is_none() {
136            self.context = Some(msg.into());
137        }
138        self
139    }
140}
141
142/// The result type for token parsing.
143pub type Result<T> = std::result::Result<T, Error>;
144
145/// Some unit, which represents an intermediate state.
146#[derive(Clone)]
147pub enum Unit {
148    /// The current unit is a single symbol.
149    Symbol(Box<str>, Span),
150    /// The current unit is a parser, which can yield multiple units.
151    Parser(Parser),
152}
153
154impl Unit {
155    /// Returns the source span of this unit.
156    #[must_use]
157    pub const fn span(&self) -> Span {
158        match self {
159            Self::Symbol(_, span) => *span,
160            Self::Parser(parser) => parser.span,
161        }
162    }
163
164    /// Returns the symbol, if applicable, as a result type.
165    ///
166    /// # Errors
167    ///
168    /// Returns an error if this unit is a parser (a sublist), not a symbol.
169    pub fn symbol(self) -> Result<Box<str>> {
170        match self {
171            Self::Symbol(name, _) => Ok(name),
172            Self::Parser(parser) => Err(Error::from(ErrorKind::ListNotAllowed).at(parser.span)),
173        }
174    }
175
176    /// Returns the parser, if applicable, as a result type.
177    ///
178    /// # Errors
179    ///
180    /// Returns an error if this unit is a symbol, not a parser.
181    pub fn parser(self) -> Result<Parser> {
182        match self {
183            Self::Parser(parser) => Ok(parser),
184            Self::Symbol(_, span) => Err(Error::from(ErrorKind::SymbolNotAllowed).at(span)),
185        }
186    }
187
188    /// Replaces all occurrences of a symbol with another symbol, recursively.
189    pub fn substitute(&mut self, variable: &str, value: &str) {
190        match self {
191            Self::Symbol(name, _) => {
192                if name.as_ref() == variable {
193                    *name = value.into();
194                }
195            }
196            Self::Parser(parser) => parser.substitute(variable, value),
197        }
198    }
199}
200
201impl<C: Context> Parsable<C> for Unit {
202    fn parse_symbol(name: Box<str>, span: Span, _context: &C) -> Result<Self> {
203        Ok(Self::Symbol(name, span))
204    }
205
206    fn parse_list(parser: &mut Parser, _context: &C) -> Result<Self> {
207        let form = std::mem::take(&mut parser.form);
208        let span = parser.span;
209        Ok(Self::Parser(Parser {
210            form,
211            count: 0,
212            span,
213        }))
214    }
215}
216
217/// This trait needs to be implemented for every struct which can be parsed using the token parser.
218#[expect(clippy::boxed_local)]
219pub trait Parsable<C: Context>: Sized {
220    /// When a symbol is found by the parser, this will be called.
221    ///
222    /// # Errors
223    ///
224    /// The default implementation always returns `ErrorKind::SymbolNotAllowed`;
225    /// implementors override this for types parsable from a symbol.
226    fn parse_symbol(_name: Box<str>, _span: Span, _context: &C) -> Result<Self> {
227        Err(ErrorKind::SymbolNotAllowed.into())
228    }
229
230    /// When a subparser is found by the parser, this will be called.
231    ///
232    /// # Errors
233    ///
234    /// The default implementation always returns `ErrorKind::ListNotAllowed`;
235    /// implementors override this for types parsable from a sublist.
236    fn parse_list(_parser: &mut Parser, _context: &C) -> Result<Self> {
237        Err(ErrorKind::ListNotAllowed.into())
238    }
239}
240
241/// The reverse of `Parsable`: emits a token `Unit` from a value.
242///
243/// Round-trip target: for a value `v` of type `T: Parsable<C> + Unparsable<C>`,
244/// parsing the unit returned by `v.to_unit(context)` yields a value equal to `v`.
245pub trait Unparsable<C: Context> {
246    /// Produces the unit representing `self` in the given context.
247    fn to_unit(&self, context: &C) -> Unit;
248}
249
250fn parse<C: Context, P: Parsable<C>>(unit: Unit, context: &C) -> Result<P> {
251    match unit {
252        Unit::Symbol(name, span) => {
253            Parsable::parse_symbol(name, span, context).map_err(|e| e.at(span))
254        }
255        Unit::Parser(mut parser) => {
256            let span = parser.span;
257            Parsable::parse_list(&mut parser, context).map_err(|e| e.at(span))
258        }
259    }
260}
261
262impl<C: Context, T: Parsable<C>> Parsable<C> for Box<T> {
263    fn parse_symbol(name: Box<str>, span: Span, context: &C) -> Result<Self> {
264        Ok(Self::new(Parsable::parse_symbol(name, span, context)?))
265    }
266
267    fn parse_list(parser: &mut Parser, context: &C) -> Result<Self> {
268        Ok(Self::new(parser.parse_list(context)?))
269    }
270}
271
272impl<C: Context, T: Parsable<C>> Parsable<C> for Rc<T> {
273    fn parse_symbol(name: Box<str>, span: Span, context: &C) -> Result<Self> {
274        Ok(Self::new(Parsable::parse_symbol(name, span, context)?))
275    }
276
277    fn parse_list(parser: &mut Parser, context: &C) -> Result<Self> {
278        Ok(Self::new(parser.parse_list(context)?))
279    }
280}
281
282impl<C: Context, T: Parsable<C>> Parsable<C> for Arc<T> {
283    fn parse_symbol(name: Box<str>, span: Span, context: &C) -> Result<Self> {
284        Ok(Self::new(Parsable::parse_symbol(name, span, context)?))
285    }
286
287    fn parse_list(parser: &mut Parser, context: &C) -> Result<Self> {
288        Ok(Self::new(parser.parse_list(context)?))
289    }
290}
291
292impl<C: Context, T: Parsable<C>> Parsable<C> for Vec<T> {
293    fn parse_list(parser: &mut Parser, context: &C) -> Result<Self> {
294        let Parser { form, count, .. } = parser;
295        form.drain(..)
296            .rev()
297            .map(|unit| {
298                *count += 1;
299                parse(unit, context)
300            })
301            .collect()
302    }
303}
304
305impl<C: Context> Parsable<C> for String {
306    fn parse_symbol(name: Box<str>, _span: Span, _context: &C) -> Result<Self> {
307        Ok(name.into())
308    }
309}
310
311impl<C: Context> Parsable<C> for Box<str> {
312    fn parse_symbol(name: Box<str>, _span: Span, _context: &C) -> Result<Self> {
313        Ok(name)
314    }
315}
316
317impl<C: Context> Parsable<C> for PathBuf {
318    fn parse_symbol(name: Box<str>, _span: Span, _context: &C) -> Result<Self> {
319        Ok(name.as_ref().into())
320    }
321}
322
323impl<C: Context, T: Unparsable<C> + ?Sized> Unparsable<C> for Box<T> {
324    fn to_unit(&self, context: &C) -> Unit {
325        (**self).to_unit(context)
326    }
327}
328
329impl<C: Context, T: Unparsable<C> + ?Sized> Unparsable<C> for Rc<T> {
330    fn to_unit(&self, context: &C) -> Unit {
331        (**self).to_unit(context)
332    }
333}
334
335impl<C: Context, T: Unparsable<C> + ?Sized> Unparsable<C> for Arc<T> {
336    fn to_unit(&self, context: &C) -> Unit {
337        (**self).to_unit(context)
338    }
339}
340
341impl<C: Context, T: Unparsable<C>> Unparsable<C> for Vec<T> {
342    fn to_unit(&self, context: &C) -> Unit {
343        let items: Vec<Unit> = self.iter().map(|item| item.to_unit(context)).collect();
344        Unit::Parser(Parser::new(items))
345    }
346}
347
348impl<C: Context> Unparsable<C> for String {
349    fn to_unit(&self, _context: &C) -> Unit {
350        Unit::Symbol(self.clone().into_boxed_str(), Span::default())
351    }
352}
353
354impl<C: Context> Unparsable<C> for str {
355    fn to_unit(&self, _context: &C) -> Unit {
356        Unit::Symbol(self.into(), Span::default())
357    }
358}
359
360impl<C: Context> Unparsable<C> for PathBuf {
361    fn to_unit(&self, _context: &C) -> Unit {
362        Unit::Symbol(self.to_string_lossy().into(), Span::default())
363    }
364}
365
366/// Derives `Parsable` from symbol for types which implement `FromStr`.
367#[macro_export]
368macro_rules! derive_symbol_parsable {
369    ($t:ty) => {
370        impl<C: $crate::Context> $crate::Parsable<C> for $t {
371            fn parse_symbol(name: Box<str>, _span: $crate::Span, _context: &C) -> $crate::Result<Self> {
372                name.parse().map_err(|error| $crate::ErrorKind::StringParsing {
373                    type_name: stringify!($t),
374                    source: Box::new(error),
375                }.into())
376            }
377        }
378    };
379    ($t:ty, $($rest:ty),+) => {
380        derive_symbol_parsable!($t);
381        derive_symbol_parsable!($($rest),+);
382    };
383}
384
385/// Derives `Unparsable` from symbol for types which implement `Display`.
386#[macro_export]
387macro_rules! derive_symbol_unparsable {
388    ($t:ty) => {
389        impl<C: $crate::Context> $crate::Unparsable<C> for $t {
390            fn to_unit(&self, _context: &C) -> $crate::Unit {
391                $crate::Unit::Symbol(
392                    ::std::string::ToString::to_string(self).into(),
393                    $crate::Span::default(),
394                )
395            }
396        }
397    };
398    ($t:ty, $($rest:ty),+) => {
399        derive_symbol_unparsable!($t);
400        derive_symbol_unparsable!($($rest),+);
401    };
402}
403
404#[cfg(not(feature = "radix-parsing"))]
405mod numbers;
406derive_symbol_parsable!(bool);
407derive_symbol_unparsable!(bool);
408derive_symbol_unparsable!(i8, i16, i32, i64, i128);
409derive_symbol_unparsable!(u8, u16, u32, u64, u128);
410derive_symbol_unparsable!(f32, f64);
411derive_symbol_unparsable!(usize);
412
413/// The token parser to parse the units into wanted types.
414#[derive(Clone)]
415pub struct Parser {
416    form: Vec<Unit>,
417    count: usize,
418    span: Span,
419}
420
421impl Parser {
422    /// Creates a new parser from a list of objects.
423    pub fn new<I: IntoIterator>(form: I) -> Self
424    where
425        I::Item: Into<Unit>,
426    {
427        let mut form: Vec<_> = form.into_iter().map(I::Item::into).collect();
428        form.reverse();
429        Self {
430            form,
431            count: 0,
432            span: Span::default(),
433        }
434    }
435
436    /// Sets the span for this parser (builder pattern).
437    #[must_use]
438    pub const fn with_span(mut self, span: Span) -> Self {
439        self.span = span;
440        self
441    }
442
443    /// Returns the source span of this parser.
444    #[must_use]
445    pub const fn span(&self) -> Span {
446        self.span
447    }
448
449    /// Returns whether the parser has no remaining elements.
450    #[must_use]
451    pub const fn is_empty(&self) -> bool {
452        self.form.is_empty()
453    }
454
455    /// Returns the number of remaining elements.
456    #[must_use]
457    pub const fn len(&self) -> usize {
458        self.form.len()
459    }
460
461    /// Replaces all occurrences of a symbol with another symbol, recursively.
462    pub fn substitute(&mut self, variable: &str, value: &str) {
463        for unit in &mut self.form {
464            unit.substitute(variable, value);
465        }
466    }
467
468    /// Returns the next unit without parsing it, or `None` if empty.
469    pub fn next_unit(&mut self) -> Option<Unit> {
470        self.count += 1;
471        self.form.pop()
472    }
473
474    /// Tries to parse the next unit as the required type.
475    ///
476    /// # Errors
477    ///
478    /// Returns `ErrorKind::NotEnoughElements` if no elements remain, or
479    /// whatever error `T`'s `Parsable` implementation returns.
480    pub fn parse_next<C: Context, T: Parsable<C>>(&mut self, context: &C) -> Result<T> {
481        self.count += 1;
482        if let Some(token) = self.form.pop() {
483            parse(token, context)
484        } else {
485            Result::Err(Error {
486                kind: ErrorKind::NotEnoughElements(self.count),
487                span: Some(self.span),
488                context: None,
489            })
490        }
491    }
492
493    /// Tries to parse the next unit as the required type, returning `None` if no elements remain.
494    ///
495    /// # Errors
496    ///
497    /// Returns whatever error `T`'s `Parsable` implementation returns.
498    pub fn parse_next_optional<C: Context, T: Parsable<C>>(
499        &mut self,
500        context: &C,
501    ) -> Result<Option<T>> {
502        if Self::is_empty(self) {
503            Ok(None)
504        } else {
505            self.parse_next(context).map(Some)
506        }
507    }
508
509    /// Tries to parse the rest of the current list into the required type.
510    /// If not every available token is used, this will be an error.
511    ///
512    /// # Errors
513    ///
514    /// Returns `ErrorKind::TooManyElements` if tokens remain unconsumed
515    /// after parsing, or whatever error `T`'s `Parsable` implementation
516    /// returns.
517    pub fn parse_rest<C: Context, T: Parsable<C>>(&mut self, context: &C) -> Result<T> {
518        let result = self.parse_list(context);
519        let count = self.form.len();
520        if count > 0 {
521            self.form.clear();
522            result?;
523            Err(Error {
524                kind: ErrorKind::TooManyElements(count),
525                span: Some(self.span),
526                context: None,
527            })
528        } else {
529            result
530        }
531    }
532
533    /// Tries to parse as many tokens of the current list as needed into the required type.
534    ///
535    /// # Errors
536    ///
537    /// Returns whatever error `T`'s `Parsable` implementation returns.
538    pub fn parse_list<C: Context, T: Parsable<C>>(&mut self, context: &C) -> Result<T> {
539        Parsable::parse_list(self, context)
540    }
541}
542
543impl Iterator for Parser {
544    type Item = Result<Self>;
545
546    fn next(&mut self) -> Option<Result<Self>> {
547        self.count += 1;
548        Some(self.form.pop()?.parser())
549    }
550
551    fn size_hint(&self) -> (usize, Option<usize>) {
552        let remaining = self.form.len();
553        (remaining, Some(remaining))
554    }
555}
556
557#[cfg(feature = "radix-parsing")]
558/// Contains utilities for radix parsing.
559pub mod radix;
token_parser/lib.rs

token_parser/
lib.rs
