styx_parse/
token.rs

1//! Token types for the Styx lexer.
2
3use crate::Span;
4
5/// The kind of a token.
6#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
7pub enum TokenKind {
8    // Structural tokens
9    /// `{`
10    LBrace,
11    /// `}`
12    RBrace,
13    /// `(`
14    LParen,
15    /// `)`
16    RParen,
17    /// `,`
18    Comma,
19    /// `>`
20    Gt,
21    /// `@`
22    At,
23
24    // Scalar tokens
25    /// Bare (unquoted) scalar: `hello`, `42`, `true`
26    BareScalar,
27    /// Quoted scalar: `"hello world"`
28    QuotedScalar,
29    /// Raw scalar: `r#"..."#`
30    RawScalar,
31    /// Heredoc start marker: `<<DELIM` (includes the newline)
32    HeredocStart,
33    /// Heredoc content (the actual text)
34    HeredocContent,
35    /// Heredoc end marker: the closing delimiter
36    HeredocEnd,
37
38    // Comment tokens
39    /// Line comment: `// ...`
40    LineComment,
41    /// Doc comment line: `/// ...`
42    DocComment,
43
44    // Whitespace tokens (significant for separator detection)
45    /// Horizontal whitespace: spaces and tabs
46    Whitespace,
47    /// Newline: `\n` or `\r\n`
48    Newline,
49
50    // Special tokens
51    /// End of file
52    Eof,
53    /// Lexer error (unrecognized input)
54    Error,
55}
56
57impl TokenKind {
58    /// Whether this token is trivia (whitespace or comments).
59    pub fn is_trivia(&self) -> bool {
60        matches!(
61            self,
62            TokenKind::Whitespace | TokenKind::Newline | TokenKind::LineComment
63        )
64    }
65
66    /// Whether this token starts a scalar value.
67    pub fn is_scalar_start(&self) -> bool {
68        matches!(
69            self,
70            TokenKind::BareScalar
71                | TokenKind::QuotedScalar
72                | TokenKind::RawScalar
73                | TokenKind::HeredocStart
74        )
75    }
76}
77
78/// A token with its kind, span, and source text slice.
79#[derive(Debug, Clone, PartialEq, Eq)]
80pub struct Token<'src> {
81    /// The kind of token.
82    pub kind: TokenKind,
83    /// The span in the source text.
84    pub span: Span,
85    /// The source text of this token.
86    pub text: &'src str,
87}
88
89impl<'src> Token<'src> {
90    /// Create a new token.
91    pub fn new(kind: TokenKind, span: Span, text: &'src str) -> Self {
92        Self { kind, span, text }
93    }
94}
styx_parse/token.rs

styx_parse/
token.rs
