ass_core/tokenizer/tokens/
token.rs

1//! Zero-copy token produced by the ASS tokenizer.
2//!
3//! Defines the [`Token`] struct pairing a [`TokenType`] discriminant with a
4//! `&'a str` span and source location for error reporting and editor integration.
5
6use core::fmt;
7
8use super::TokenType;
9
10/// Token produced by ASS tokenizer with zero-copy span
11///
12/// Represents a lexical unit in ASS script with location information.
13/// The span references the original source text to avoid allocations.
14#[derive(Debug, Clone, PartialEq, Eq)]
15pub struct Token<'a> {
16    /// Token type discriminant
17    pub token_type: TokenType,
18
19    /// Zero-copy span referencing source text
20    pub span: &'a str,
21
22    /// Line number where token starts (1-based)
23    pub line: usize,
24
25    /// Column number where token starts (1-based)
26    pub column: usize,
27}
28
29impl<'a> Token<'a> {
30    /// Create new token with full location information
31    #[must_use]
32    pub const fn new(token_type: TokenType, span: &'a str, line: usize, column: usize) -> Self {
33        Self {
34            token_type,
35            span,
36            line,
37            column,
38        }
39    }
40
41    /// Get token length in characters
42    #[must_use]
43    pub fn len(&self) -> usize {
44        self.span.chars().count()
45    }
46
47    /// Check if token is empty (should not happen in normal tokenization)
48    #[must_use]
49    pub const fn is_empty(&self) -> bool {
50        self.span.is_empty()
51    }
52
53    /// Get end column position
54    #[must_use]
55    pub fn end_column(&self) -> usize {
56        self.column + self.len()
57    }
58
59    /// Check if this token represents whitespace
60    #[must_use]
61    pub const fn is_whitespace(&self) -> bool {
62        matches!(self.token_type, TokenType::Whitespace)
63    }
64
65    /// Check if this token represents a delimiter
66    #[must_use]
67    pub const fn is_delimiter(&self) -> bool {
68        matches!(
69            self.token_type,
70            TokenType::Comma
71                | TokenType::Colon
72                | TokenType::SectionOpen
73                | TokenType::SectionClose
74                | TokenType::OverrideOpen
75                | TokenType::OverrideClose
76        )
77    }
78
79    /// Check if this token represents content (text, numbers, etc.)
80    #[must_use]
81    pub const fn is_content(&self) -> bool {
82        matches!(
83            self.token_type,
84            TokenType::Text
85                | TokenType::Number
86                | TokenType::HexValue
87                | TokenType::SectionName
88                | TokenType::OverrideBlock
89        )
90    }
91
92    /// Validate that span references valid UTF-8
93    #[must_use]
94    pub const fn validate_utf8(&self) -> bool {
95        true
96    }
97}
98
99impl fmt::Display for Token<'_> {
100    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
101        write!(
102            f,
103            "{:?}@{}:{} '{}'",
104            self.token_type, self.line, self.column, self.span
105        )
106    }
107}
ass_core/tokenizer/tokens/token.rs

ass_core/tokenizer/tokens/
token.rs
