alba_syntax/
token.rs

1//! Token types produced by the lexer.
2
3/// A byte-offset span into the original source text.
4///
5/// `start` is inclusive, `end` is exclusive. Spans are used to report
6/// diagnostics (see the `alba-diagnostics` crate) with a caret under the
7/// exact offending text, so accuracy here matters.
8#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9pub struct Span {
10    pub start: usize,
11    pub end: usize,
12}
13
14impl Span {
15    /// Creates a new span from a byte-offset range `[start, end)`.
16    pub fn new(start: usize, end: usize) -> Self {
17        Self { start, end }
18    }
19}
20
21/// The kind of a lexical token in the Beamfile DSL.
22#[derive(Debug, Clone, PartialEq, Eq)]
23pub enum TokenKind {
24    /// An identifier, e.g. a beam name or a variable name.
25    Ident(String),
26    /// The decoded content of a string literal, after processing escapes.
27    /// Interpolation braces (`{`, `}`) are left as raw text; a later parsing
28    /// stage re-scans this content to resolve interpolation.
29    Str(String),
30
31    // Keywords.
32    KwVersion,
33    KwImport,
34    KwAs,
35    KwLet,
36    KwDefault,
37    KwBeam,
38    KwIf,
39    KwThen,
40    KwElse,
41    KwTrue,
42    KwFalse,
43    KwHook,
44
45    // Punctuation and operators.
46    LBrace,
47    RBrace,
48    LBracket,
49    RBracket,
50    LParen,
51    RParen,
52    Comma,
53    Colon,
54    /// `=`
55    Eq,
56    /// `==`
57    EqEq,
58    /// `!=`
59    NotEq,
60    /// `&&`
61    AndAnd,
62    /// `||`
63    OrOr,
64    Plus,
65    /// `.`, member access (`git.branch`).
66    Dot,
67
68    /// A statement separator. Emitted only strictly between two other
69    /// tokens: leading and trailing newline trivia (including blank lines
70    /// and comments) are dropped, and a run of consecutive newlines
71    /// collapses into a single `Newline` token.
72    Newline,
73    /// End of input. Yielded exactly once, as the last token.
74    Eof,
75}
76
77/// A single lexical token together with its span in the source text.
78#[derive(Debug, Clone, PartialEq, Eq)]
79pub struct Token {
80    pub kind: TokenKind,
81    pub span: Span,
82}
83
84impl Token {
85    /// Creates a new token from its kind and span.
86    pub fn new(kind: TokenKind, span: Span) -> Self {
87        Self { kind, span }
88    }
89}
alba_syntax/token.rs

alba_syntax/
token.rs
