styx_tokenizer/
token.rs

1//! Token types for the Styx lexer.
2
3use crate::Span;
4
5/// The kind of a token.
6#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
7pub enum TokenKind {
8    // Structural tokens
9    /// `{`
10    LBrace,
11    /// `}`
12    RBrace,
13    /// `(`
14    LParen,
15    /// `)`
16    RParen,
17    /// `,`
18    Comma,
19    /// `>`
20    Gt,
21    /// `@` (standalone unit)
22    At,
23    /// `@name` (tag with identifier)
24    Tag,
25
26    // Scalar tokens
27    /// Bare (unquoted) scalar: `hello`, `42`, `true`
28    BareScalar,
29    /// Quoted scalar: `"hello world"`
30    QuotedScalar,
31    /// Raw scalar: `r#"..."#`
32    RawScalar,
33    /// Heredoc start marker: `<<DELIM` (includes the newline)
34    HeredocStart,
35    /// Heredoc content (the actual text)
36    HeredocContent,
37    /// Heredoc end marker: the closing delimiter
38    HeredocEnd,
39
40    // Comment tokens
41    /// Line comment: `// ...`
42    LineComment,
43    /// Doc comment line: `/// ...`
44    DocComment,
45
46    // Whitespace tokens (significant for separator detection)
47    /// Horizontal whitespace: spaces and tabs
48    Whitespace,
49    /// Newline: `\n` or `\r\n`
50    Newline,
51
52    // Special tokens
53    /// End of file
54    Eof,
55    /// Lexer error (unrecognized input)
56    Error,
57}
58
59impl TokenKind {
60    /// Whether this token is trivia (whitespace or comments).
61    pub fn is_trivia(&self) -> bool {
62        matches!(
63            self,
64            TokenKind::Whitespace | TokenKind::Newline | TokenKind::LineComment
65        )
66    }
67
68    /// Whether this token starts a scalar value.
69    pub fn is_scalar_start(&self) -> bool {
70        matches!(
71            self,
72            TokenKind::BareScalar
73                | TokenKind::QuotedScalar
74                | TokenKind::RawScalar
75                | TokenKind::HeredocStart
76        )
77    }
78}
79
80/// A token with its kind, span, and source text slice.
81#[derive(Debug, Clone, PartialEq, Eq)]
82pub struct Token<'src> {
83    /// The kind of token.
84    pub kind: TokenKind,
85    /// The span in the source text.
86    pub span: Span,
87    /// The source text of this token.
88    pub text: &'src str,
89}
90
91impl<'src> Token<'src> {
92    /// Create a new token.
93    pub fn new(kind: TokenKind, span: Span, text: &'src str) -> Self {
94        Self { kind, span, text }
95    }
96}
styx_tokenizer/token.rs

styx_tokenizer/
token.rs
