lexerus/
token.rs

1use super::*;
2
3/// Token indicator
4///
5/// Trait indicating that the struct is ready for [Lexer]. This is not included as part of [Lexer]
6/// as there are instances where the user may wish to handroll the [Token] representation. In any
7/// case they refer to distincting things: [Token] represents the outward represntation of the
8/// struct after parsing.
9pub trait Token<'code>
10where
11    Self: Sized,
12{
13    /// Name of the token
14    const NAME: &'static str;
15
16    /// Underlying [Buffer] on the [Token]. This is an [Option] because certain types of
17    /// structures, e.g. [std::marker::PhantomData] do not actually store a [Buffer] and it is
18    /// therefore theoretically possible that a [Token] has no inherent [Buffer]. Examples include
19    /// [Not] and [Peek]
20    fn buffer(&self) -> Option<Buffer<'code>>;
21}
lexerus/token.rs

lexerus/
token.rs
