alkale 2.0.0

A simple LL(1) lexer library for Rust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

//! Module for [`Token`]s and related types.

use core::fmt::{self, Debug};

use crate::span::{Span, Spanned};

/// Container struct for bundling a token's raw data with its [`Span`] information.
///
/// This should generally be used as part of the output of lexers, and should be used instead of
/// [`Spanned`] in that context.
#[derive(PartialEq, Eq, Clone, Copy)]
#[expect(clippy::exhaustive_structs)]
pub struct Token<T>(
    /// The primary data held by this token.
    pub T,
    /// The span information of this token.
    pub Span,
);

impl<T> Token<T> {
    /// Create a new [`Token`] from data and [`Span`].
    #[inline(always)]
    pub const fn new(data: T, span: Span) -> Self {
        Self(data, span)
    }

    /// Get a reference to this token's data / variant.
    #[inline(always)]
    pub const fn data(&self) -> &T {
        &self.0
    }

    /// Get a reference to this token's [`Span`].
    #[inline(always)]
    pub const fn span(&self) -> &Span {
        &self.1
    }

    /// Get a mutable reference to this token's data / variant.
    #[inline(always)]
    pub fn data_mut(&mut self) -> &mut T {
        &mut self.0
    }

    /// Get a mutable reference to this token's [`Span`].
    #[inline(always)]
    pub fn span_mut(&mut self) -> &mut Span {
        &mut self.1
    }

    /// Get this token's data / variant.
    #[inline(always)]
    pub fn into_data(self) -> T {
        self.0
    }

    /// Get this token's [`Span`].
    #[inline(always)]
    pub fn into_span(self) -> Span {
        self.1
    }

    /// Convert this token into a [`Spanned`] token-data.
    #[inline(always)]
    pub fn into_spanned(self) -> Spanned<T> {
        self.1.wrap(self.0)
    }

    /// Convert this token into a [`Spanned`] token-data.
    #[inline(always)]
    pub fn from_spanned(span: Spanned<T>) -> Self {
        Self(span.data, span.span)
    }
}

// Erase the existence of the underlying Spanned and unwrap the span
// into a range for readability.
impl<T: Debug> Debug for Token<T> {
    #[inline]
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_tuple("Token")
            .field(&(self.1.index()..self.1.end_index()))
            .field(&self.0)
            .finish()
    }
}

impl<T> From<Spanned<T>> for Token<T> {
    #[inline]
    fn from(value: Spanned<T>) -> Self {
        Self::from_spanned(value)
    }
}