tcss_core/
token.rs

1/// Represents the type of a token in TCSS
2#[derive(Debug, Clone, PartialEq)]
3pub enum TokenType {
4    // Keywords
5    Fn,           // @fn
6    Var,          // @var
7    Import,       // @import
8    Return,       // return
9    
10    // Identifiers and Literals
11    Identifier(String),
12    Number(f64),
13    String(String),
14    Color(String),  // #hex or named colors
15    Unit(String),   // px, em, rem, %, etc.
16    
17    // Operators
18    Plus,         // +
19    Minus,        // -
20    Star,         // *
21    Slash,        // /
22    Percent,      // %
23    
24    // Comparison (for future use)
25    Equal,        // ==
26    NotEqual,     // !=
27    Less,         // <
28    Greater,      // >
29    LessEqual,    // <=
30    GreaterEqual, // >=
31    
32    // Delimiters
33    Colon,        // :
34    Semicolon,    // ;
35    Comma,        // ,
36    Dot,          // .
37    At,           // @
38    
39    // Brackets
40    LeftParen,    // (
41    RightParen,   // )
42    LeftBrace,    // {
43    RightBrace,   // }
44    LeftBracket,  // [
45    RightBracket, // ]
46    
47    // Special
48    Arrow,        // ->
49    Assign,       // =
50    
51    // Whitespace and Structure
52    Newline,
53    Indent,
54    Dedent,
55    
56    // Comments
57    Comment(String),
58    
59    // End of file
60    Eof,
61}
62
63/// Represents a token with its type, value, and position in source
64#[derive(Debug, Clone, PartialEq)]
65pub struct Token {
66    pub token_type: TokenType,
67    pub lexeme: String,
68    pub line: usize,
69    pub column: usize,
70}
71
72impl Token {
73    /// Create a new token
74    pub fn new(token_type: TokenType, lexeme: String, line: usize, column: usize) -> Self {
75        Self {
76            token_type,
77            lexeme,
78            line,
79            column,
80        }
81    }
82
83    /// Get the source location of this token
84    pub fn location(&self) -> crate::error::SourceLocation {
85        crate::error::SourceLocation::new(self.line, self.column)
86    }
87
88    /// Check if token is a keyword
89    pub fn is_keyword(&self) -> bool {
90        matches!(
91            self.token_type,
92            TokenType::Fn | TokenType::Var | TokenType::Import | TokenType::Return
93        )
94    }
95
96    /// Check if token is an operator
97    pub fn is_operator(&self) -> bool {
98        matches!(
99            self.token_type,
100            TokenType::Plus
101                | TokenType::Minus
102                | TokenType::Star
103                | TokenType::Slash
104                | TokenType::Percent
105        )
106    }
107
108    /// Check if token is a delimiter
109    pub fn is_delimiter(&self) -> bool {
110        matches!(
111            self.token_type,
112            TokenType::Colon
113                | TokenType::Semicolon
114                | TokenType::Comma
115                | TokenType::Dot
116        )
117    }
118}
119
120impl std::fmt::Display for Token {
121    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
122        write!(
123            f,
124            "Token({:?}, '{}', {}:{})",
125            self.token_type, self.lexeme, self.line, self.column
126        )
127    }
128}
129
130#[cfg(test)]
131mod tests {
132    use super::*;
133
134    #[test]
135    fn test_token_creation() {
136        let token = Token::new(TokenType::Identifier("test".to_string()), "test".to_string(), 1, 1);
137        assert_eq!(token.line, 1);
138        assert_eq!(token.column, 1);
139        assert_eq!(token.lexeme, "test");
140    }
141
142    #[test]
143    fn test_is_keyword() {
144        let fn_token = Token::new(TokenType::Fn, "@fn".to_string(), 1, 1);
145        assert!(fn_token.is_keyword());
146        
147        let id_token = Token::new(TokenType::Identifier("test".to_string()), "test".to_string(), 1, 1);
148        assert!(!id_token.is_keyword());
149    }
150
151    #[test]
152    fn test_is_operator() {
153        let plus_token = Token::new(TokenType::Plus, "+".to_string(), 1, 1);
154        assert!(plus_token.is_operator());
155    }
156}
157
tcss_core/token.rs

tcss_core/
token.rs
