perl_lexer/
token.rs

1//! Token types and structures for the Perl lexer.
2//!
3//! [`TokenType`] classifies every token the lexer can emit, and [`Token`]
4//! bundles a type with its source text and byte span.
5
6use std::sync::Arc;
7
8/// Parts of an interpolated string
9#[derive(Debug, Clone, PartialEq)]
10pub enum StringPart {
11    /// Literal text
12    Literal(Arc<str>),
13    /// Variable interpolation: $var, @array, %hash
14    Variable(Arc<str>),
15    /// Expression interpolation: ${expr}, @{expr}
16    Expression(Arc<str>),
17    /// Method call: `->method()`
18    MethodCall(Arc<str>),
19    /// Array slice: [1..3]
20    ArraySlice(Arc<str>),
21}
22
23/// Token types for Perl
24#[derive(Debug, Clone, PartialEq)]
25pub enum TokenType {
26    // Slash-derived tokens
27    /// Division operator: /
28    Division,
29    /// Regex match: m// or //
30    RegexMatch,
31    /// Substitution: s///
32    Substitution,
33    /// Transliteration: tr/// or y///
34    Transliteration,
35    /// Quote regex: qr//
36    QuoteRegex,
37
38    // String and quote tokens
39    /// String literal: "string" or 'string'
40    StringLiteral,
41    /// Single quote: q//
42    QuoteSingle,
43    /// Double quote: qq//
44    QuoteDouble,
45    /// Quote words: qw//
46    QuoteWords,
47    /// Quote command: qx// or `backticks`
48    QuoteCommand,
49
50    // String interpolation tokens
51    /// String with interpolated parts
52    InterpolatedString(Vec<StringPart>),
53
54    // Heredoc tokens
55    /// Heredoc start: <<EOF or <<'EOF'
56    HeredocStart,
57    /// Heredoc body content
58    HeredocBody(Arc<str>),
59
60    // Format declarations
61    /// Format body content
62    FormatBody(Arc<str>),
63
64    // Version strings
65    /// Version string: v5.32.0
66    Version(Arc<str>),
67
68    // POD documentation
69    /// POD documentation block
70    Pod,
71
72    // Data sections
73    /// Data section marker: __DATA__ or __END__
74    DataMarker(Arc<str>),
75    /// Data section body content
76    DataBody(Arc<str>),
77
78    // Error recovery
79    /// Unknown rest of input (used when budget exceeded)
80    UnknownRest,
81
82    // Identifiers and literals
83    /// Identifier or variable name
84    Identifier(Arc<str>),
85    /// Numeric literal
86    Number(Arc<str>),
87    /// Operator
88    Operator(Arc<str>),
89    /// Keyword
90    Keyword(Arc<str>),
91
92    // Delimiters
93    /// Left parenthesis: (
94    LeftParen,
95    /// Right parenthesis: )
96    RightParen,
97    /// Left bracket: [
98    LeftBracket,
99    /// Right bracket: ]
100    RightBracket,
101    /// Left brace: {
102    LeftBrace,
103    /// Right brace: }
104    RightBrace,
105
106    // Punctuation
107    /// Semicolon: ;
108    Semicolon,
109    /// Comma: ,
110    Comma,
111    /// Colon: :
112    Colon,
113    /// Arrow: ->
114    Arrow,
115    /// Fat comma: =>
116    FatComma,
117
118    // Whitespace and comments
119    /// Whitespace (usually not returned)
120    Whitespace,
121    /// Newline character
122    Newline,
123    /// Comment text
124    Comment(Arc<str>),
125
126    // Special tokens
127    /// End of file
128    EOF,
129    /// Error token for invalid input
130    Error(Arc<str>),
131}
132
133impl TokenType {
134    /// Return `true` when this token is ignorable trivia.
135    pub fn is_trivia(&self) -> bool {
136        matches!(self, Self::Whitespace | Self::Newline | Self::Comment(_))
137    }
138
139    /// Return `true` when lexing could not continue safely.
140    pub fn is_recovery_token(&self) -> bool {
141        matches!(self, Self::UnknownRest | Self::Error(_))
142    }
143}
144
145/// A single token produced by [`PerlLexer`](crate::PerlLexer).
146///
147/// Carries its [`TokenType`], the original source text (as a cheap-to-clone
148/// `Arc<str>`), and the byte span within the input.
149#[derive(Debug, Clone)]
150pub struct Token {
151    /// Classification of this token (keyword, operator, literal, etc.).
152    pub token_type: TokenType,
153    /// Original source text that this token spans.
154    pub text: Arc<str>,
155    /// Starting byte offset (inclusive) in the source input.
156    pub start: usize,
157    /// Ending byte offset (exclusive) in the source input.
158    pub end: usize,
159}
160
161impl Token {
162    /// Create a new token with the given type, source text, and byte span.
163    pub fn new(token_type: TokenType, text: impl Into<Arc<str>>, start: usize, end: usize) -> Self {
164        Self { token_type, text: text.into(), start, end }
165    }
166
167    /// Return the byte length of this token's span (`end - start`).
168    pub fn len(&self) -> usize {
169        self.end - self.start
170    }
171
172    /// Return `true` if the token has a zero-length span.
173    pub fn is_empty(&self) -> bool {
174        self.start == self.end
175    }
176}
177
178#[cfg(test)]
179mod tests {
180    use super::{Token, TokenType};
181    use std::sync::Arc;
182
183    #[test]
184    fn token_type_trivia_classifier_matches_expected_variants()
185    -> Result<(), Box<dyn std::error::Error>> {
186        assert!(TokenType::Whitespace.is_trivia());
187        assert!(TokenType::Newline.is_trivia());
188        assert!(TokenType::Comment(Arc::from("# note")).is_trivia());
189        assert!(!TokenType::Identifier(Arc::from("foo")).is_trivia());
190        Ok(())
191    }
192
193    #[test]
194    fn token_type_recovery_classifier_matches_expected_variants()
195    -> Result<(), Box<dyn std::error::Error>> {
196        assert!(TokenType::UnknownRest.is_recovery_token());
197        assert!(TokenType::Error(Arc::from("oops")).is_recovery_token());
198        assert!(!TokenType::EOF.is_recovery_token());
199        Ok(())
200    }
201
202    // --- Token::new ---
203
204    #[test]
205    fn token_new_stores_type_text_and_span() -> Result<(), Box<dyn std::error::Error>> {
206        let tok = Token::new(TokenType::Semicolon, ";", 3, 4);
207        assert_eq!(tok.token_type, TokenType::Semicolon);
208        assert_eq!(tok.text.as_ref(), ";");
209        assert_eq!(tok.start, 3);
210        assert_eq!(tok.end, 4);
211        Ok(())
212    }
213
214    #[test]
215    fn token_new_accepts_arc_str_directly() -> Result<(), Box<dyn std::error::Error>> {
216        let text: Arc<str> = Arc::from("hello");
217        let tok = Token::new(TokenType::Identifier(Arc::from("hello")), text, 0, 5);
218        assert_eq!(tok.start, 0);
219        assert_eq!(tok.end, 5);
220        Ok(())
221    }
222
223    // --- Token::len ---
224
225    #[test]
226    fn token_len_returns_end_minus_start() -> Result<(), Box<dyn std::error::Error>> {
227        let tok = Token::new(TokenType::Semicolon, ";", 10, 17);
228        assert_eq!(tok.len(), 7);
229        Ok(())
230    }
231
232    #[test]
233    fn token_len_zero_when_span_is_empty() -> Result<(), Box<dyn std::error::Error>> {
234        let tok = Token::new(TokenType::EOF, "", 5, 5);
235        assert_eq!(tok.len(), 0);
236        Ok(())
237    }
238
239    #[test]
240    fn token_len_single_byte_span() -> Result<(), Box<dyn std::error::Error>> {
241        let tok = Token::new(TokenType::Comma, ",", 0, 1);
242        assert_eq!(tok.len(), 1);
243        Ok(())
244    }
245
246    // --- Token::is_empty ---
247
248    #[test]
249    fn token_is_empty_true_for_zero_length_span() -> Result<(), Box<dyn std::error::Error>> {
250        let tok = Token::new(TokenType::EOF, "", 0, 0);
251        assert!(tok.is_empty());
252        Ok(())
253    }
254
255    #[test]
256    fn token_is_empty_true_for_non_zero_start_matching_end()
257    -> Result<(), Box<dyn std::error::Error>> {
258        let tok = Token::new(TokenType::EOF, "", 42, 42);
259        assert!(tok.is_empty());
260        Ok(())
261    }
262
263    #[test]
264    fn token_is_empty_false_for_non_zero_length_span() -> Result<(), Box<dyn std::error::Error>> {
265        let tok = Token::new(TokenType::Semicolon, ";", 0, 1);
266        assert!(!tok.is_empty());
267        Ok(())
268    }
269}
perl_lexer/token.rs

perl_lexer/
token.rs
