o7/parser/
tokens.rs

1use std::fmt;
2
3/// A token with its type, value, and source location.
4#[derive(Debug, Clone, PartialEq)]
5pub struct Token {
6    pub ty: TokenType,
7    pub value: String,
8    pub line: usize,
9    pub column: usize,
10}
11
12impl Token {
13    /// Create a new token with the given type, value, and source location.
14    pub fn new(ty: TokenType, value: impl Into<String>, line: usize, column: usize) -> Self {
15        Token {
16            ty,
17            value: value.into(),
18            line,
19            column,
20        }
21    }
22
23    /// Create a synthetic token with an empty value (used for Indent, Dedent, Newline, Eof).
24    pub fn synthetic(ty: TokenType, line: usize, column: usize) -> Self {
25        Token {
26            ty,
27            value: String::new(),
28            line,
29            column,
30        }
31    }
32}
33
34/// All token types for the O7 DSL.
35///
36/// INDENT, DEDENT, NEWLINE, and EOF are synthetic tokens injected
37/// by the indent post-processor, not by the hand-written lexer.
38#[derive(Debug, Clone, PartialEq, Eq, Hash)]
39pub enum TokenType {
40    // Keywords
41    Version,
42    Workflow,
43    Run,
44    If,
45    Not,
46    While,
47    ParAnd,
48    Exec,
49    Harness,
50    Prompt,
51    PromptFile,
52    Args,
53    FailPolicy,
54    Match,
55    Else,
56
57    // Literals and identifiers
58    Name,
59    Number,
60    Str,
61    BareValue,
62
63    // Punctuation
64    Colon,
65    Arrow,
66
67    // Synthetic tokens (injected by indent processor)
68    Indent,
69    Dedent,
70    Newline,
71    Eof,
72}
73
74impl fmt::Display for TokenType {
75    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
76        match self {
77            TokenType::Version => write!(f, "version"),
78            TokenType::Workflow => write!(f, "workflow"),
79            TokenType::Run => write!(f, "run"),
80            TokenType::If => write!(f, "if"),
81            TokenType::Not => write!(f, "not"),
82            TokenType::While => write!(f, "while"),
83            TokenType::ParAnd => write!(f, "par-and"),
84            TokenType::Exec => write!(f, "exec"),
85            TokenType::Harness => write!(f, "harness"),
86            TokenType::Prompt => write!(f, "prompt"),
87            TokenType::PromptFile => write!(f, "prompt_file"),
88            TokenType::Args => write!(f, "args"),
89            TokenType::FailPolicy => write!(f, "fail-policy"),
90            TokenType::Match => write!(f, "match"),
91            TokenType::Else => write!(f, "else"),
92            TokenType::Name => write!(f, "name"),
93            TokenType::Number => write!(f, "number"),
94            TokenType::Str => write!(f, "string"),
95            TokenType::BareValue => write!(f, "bare value"),
96            TokenType::Colon => write!(f, ":"),
97            TokenType::Arrow => write!(f, "->"),
98            TokenType::Indent => write!(f, "INDENT"),
99            TokenType::Dedent => write!(f, "DEDENT"),
100            TokenType::Newline => write!(f, "NEWLINE"),
101            TokenType::Eof => write!(f, "EOF"),
102        }
103    }
104}
105
106#[cfg(test)]
107mod tests {
108    use super::*;
109
110    #[test]
111    fn test_token_type_display() {
112        assert_eq!(format!("{}", TokenType::Version), "version");
113        assert_eq!(format!("{}", TokenType::ParAnd), "par-and");
114        assert_eq!(format!("{}", TokenType::Eof), "EOF");
115    }
116
117    #[test]
118    fn test_token_creation() {
119        let tok = Token {
120            ty: TokenType::Workflow,
121            value: "workflow".to_string(),
122            line: 1,
123            column: 1,
124        };
125        assert_eq!(tok.ty, TokenType::Workflow);
126        assert_eq!(tok.line, 1);
127    }
128}
o7/parser/tokens.rs

o7/parser/
tokens.rs
