ocelot_parser/lexer/
token.rs

1use crate::lexer::token_type::TokenType;
2use ocelot_base::span::Span;
3
4/// Lexical token with its source span.
5#[derive(Debug, Clone, PartialEq, Eq)]
6pub struct Token {
7    pub token_type: TokenType,
8    pub span: Span,
9}
10
11impl Token {
12    /// Creates a token from its kind and byte offsets.
13    pub const fn new(token_type: TokenType, start: usize, end: usize) -> Self {
14        Self {
15            token_type,
16            span: Span::new(start, end),
17        }
18    }
19}
20
21#[cfg(test)]
22mod tests {
23    use super::Token;
24    use crate::lexer::token_type::TokenType;
25
26    #[test]
27    fn token_captures_kind_and_span() {
28        let token = Token::new(TokenType::Identifier, 0, 7);
29
30        assert_eq!(token.token_type, TokenType::Identifier);
31        assert_eq!(token.span.start(), 0);
32        assert_eq!(token.span.end(), 7);
33    }
34
35    #[test]
36    fn println_script_tokens_are_representable() {
37        let tokens = [
38            Token::new(TokenType::Identifier, 0, 7),
39            Token::new(TokenType::LeftParen, 7, 8),
40            Token::new(TokenType::String, 8, 15),
41            Token::new(TokenType::RightParen, 15, 16),
42            Token::new(TokenType::Semicolon, 16, 17),
43            Token::new(TokenType::EndOfFile, 17, 17),
44        ];
45
46        assert_eq!(tokens[0].token_type, TokenType::Identifier);
47        assert_eq!(tokens[1].token_type, TokenType::LeftParen);
48        assert_eq!(tokens[2].token_type, TokenType::String);
49        assert_eq!(tokens[3].token_type, TokenType::RightParen);
50        assert_eq!(tokens[4].token_type, TokenType::Semicolon);
51        assert_eq!(tokens[5].token_type, TokenType::EndOfFile);
52    }
53}
ocelot_parser/lexer/token.rs

ocelot_parser/lexer/
token.rs
