sipha_core/
token.rs

1//! Token representations shared by the parser infrastructure.
2
3use crate::{span::Span, traits::TokenKind};
4use builder_pattern::Builder;
5
6/// Piece of trivia (comment/whitespace) attached to a token.
7#[derive(Builder, Clone, Debug)]
8pub struct TokenTrivia<K: TokenKind> {
9    /// Kind of the trivia token.
10    pub kind: K,
11    /// Span covered by the trivia.
12    pub span: Span,
13    /// Raw textual representation.
14    #[into]
15    pub text: String,
16}
17
18impl<K: TokenKind> TokenTrivia<K> {
19    /// Construct a new trivia record.
20    ///
21    /// For builder pattern usage, use `TokenTriviaBuilder::new()` instead.
22    pub fn create(kind: K, span: Span, text: impl Into<String>) -> Self {
23        Self {
24            kind,
25            span,
26            text: text.into(),
27        }
28    }
29}
30
31/// Token emitted by the lexer.
32#[derive(Builder, Clone, Debug)]
33pub struct Token<K: TokenKind> {
34    /// Token kind enum.
35    pub kind: K,
36    /// Source span covered by the token.
37    pub span: Span,
38    /// Original textual lexeme.
39    #[into]
40    pub lexeme: String,
41    /// Trivia that appears before the token.
42    #[default(Vec::new())]
43    pub leading_trivia: Vec<TokenTrivia<K>>,
44    /// Trivia that appears immediately after the token.
45    #[default(Vec::new())]
46    pub trailing_trivia: Vec<TokenTrivia<K>>,
47}
48
49impl<K: TokenKind> Token<K> {
50    /// Construct a new token with the provided trivia.
51    ///
52    /// For builder pattern usage, use `TokenBuilder::new()` instead.
53    pub fn create(
54        kind: K,
55        span: Span,
56        lexeme: impl Into<String>,
57        leading_trivia: Vec<TokenTrivia<K>>,
58        trailing_trivia: Vec<TokenTrivia<K>>,
59    ) -> Self {
60        Self {
61            kind,
62            span,
63            lexeme: lexeme.into(),
64            leading_trivia,
65            trailing_trivia,
66        }
67    }
68
69    /// Create a token with attached trivia.
70    pub fn with_trivia(
71        mut self,
72        leading: Vec<TokenTrivia<K>>,
73        trailing: Vec<TokenTrivia<K>>,
74    ) -> Self {
75        self.leading_trivia = leading;
76        self.trailing_trivia = trailing;
77        self
78    }
79}
sipha_core/token.rs

sipha_core/
token.rs
