harn_lexer/
token.rs

1use std::fmt;
2
3/// A segment of an interpolated string.
4#[derive(Debug, Clone, PartialEq)]
5pub enum StringSegment {
6    Literal(String),
7    /// An interpolated expression with its source position (line, column).
8    Expression(String, usize, usize),
9}
10
11impl fmt::Display for StringSegment {
12    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
13        match self {
14            StringSegment::Literal(s) => write!(f, "{s}"),
15            StringSegment::Expression(e, _, _) => write!(f, "${{{e}}}"),
16        }
17    }
18}
19
20/// Source location for error reporting.
21#[derive(Debug, Clone, Copy, PartialEq, Eq)]
22pub struct Span {
23    /// Byte offset from start of source (inclusive).
24    pub start: usize,
25    /// Byte offset from start of source (exclusive).
26    pub end: usize,
27    /// 1-based line number of start position.
28    pub line: usize,
29    /// 1-based column number of start position.
30    pub column: usize,
31    /// 1-based line number of end position (for multiline span detection).
32    pub end_line: usize,
33}
34
35impl Span {
36    pub fn with_offsets(start: usize, end: usize, line: usize, column: usize) -> Self {
37        Self {
38            start,
39            end,
40            line,
41            column,
42            end_line: line,
43        }
44    }
45
46    /// Create a span covering two spans (from start of `a` to end of `b`).
47    pub fn merge(a: Span, b: Span) -> Span {
48        Span {
49            start: a.start,
50            end: b.end,
51            line: a.line,
52            column: a.column,
53            end_line: b.end_line,
54        }
55    }
56
57    /// A dummy span for synthetic/generated nodes.
58    pub fn dummy() -> Self {
59        Self {
60            start: 0,
61            end: 0,
62            line: 0,
63            column: 0,
64            end_line: 0,
65        }
66    }
67}
68
69impl fmt::Display for Span {
70    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
71        write!(f, "{}:{}", self.line, self.column)
72    }
73}
74
75/// Canonical list of Harn language keywords.
76///
77/// This is the single source of truth for keyword tokens. The lexer's
78/// identifier-to-keyword match in `lexer.rs` must stay in sync; the unit test
79/// `test_keywords_const_covers_lexer` verifies parity between the two.
80///
81/// Tooling that needs the keyword set (syntax highlighters, the LSP, etc.)
82/// should read `KEYWORDS` rather than hard-coding a duplicate list.
83pub const KEYWORDS: &[&str] = &[
84    "ask",
85    "break",
86    "catch",
87    "continue",
88    "deadline",
89    "else",
90    "enum",
91    "extends",
92    "false",
93    "finally",
94    "fn",
95    "for",
96    "from",
97    "guard",
98    "if",
99    "impl",
100    "import",
101    "in",
102    "interface",
103    "let",
104    "match",
105    "mutex",
106    "nil",
107    "override",
108    "parallel",
109    "parallel_map",
110    "parallel_settle",
111    "pipeline",
112    "pub",
113    "require",
114    "retry",
115    "return",
116    "select",
117    "spawn",
118    "struct",
119    "throw",
120    "thru",
121    "tool",
122    "true",
123    "try",
124    "type",
125    "upto",
126    "var",
127    "while",
128    "yield",
129];
130
131/// Token kinds produced by the lexer.
132#[derive(Debug, Clone, PartialEq)]
133pub enum TokenKind {
134    // Keywords
135    Pipeline,
136    Extends,
137    Override,
138    Let,
139    Var,
140    If,
141    Else,
142    For,
143    In,
144    Match,
145    Retry,
146    Parallel,
147    ParallelMap,
148    ParallelSettle,
149    Return,
150    Import,
151    True,
152    False,
153    Nil,
154    Try,
155    Catch,
156    Throw,
157    Finally,
158    Fn,
159    Spawn,
160    While,
161    TypeKw,
162    Enum,
163    Struct,
164    Interface,
165    Pub,
166    From,
167    Thru,
168    Tool,
169    Upto,
170    Guard,
171    Require,
172    Ask,
173    Deadline,
174    Yield,
175    Mutex,
176    Break,
177    Continue,
178    Select,
179    Impl,
180
181    // Literals
182    Identifier(String),
183    StringLiteral(String),
184    InterpolatedString(Vec<StringSegment>),
185    /// Raw string literal `r"..."` — no escape processing, no interpolation.
186    RawStringLiteral(String),
187    IntLiteral(i64),
188    FloatLiteral(f64),
189    /// Duration literal in milliseconds: 500ms, 5s, 30m, 2h
190    DurationLiteral(u64),
191
192    // Two-character operators
193    Eq,            // ==
194    Neq,           // !=
195    And,           // &&
196    Or,            // ||
197    Pipe,          // |>
198    NilCoal,       // ??
199    QuestionDot,   // ?.
200    Arrow,         // ->
201    Lte,           // <=
202    Gte,           // >=
203    PlusAssign,    // +=
204    MinusAssign,   // -=
205    StarAssign,    // *=
206    SlashAssign,   // /=
207    PercentAssign, // %=
208
209    // Single-character operators
210    Assign,   // =
211    Not,      // !
212    Dot,      // .
213    Plus,     // +
214    Minus,    // -
215    Star,     // *
216    Slash,    // /
217    Percent,  // %
218    Lt,       // <
219    Gt,       // >
220    Question, // ?
221    Bar,      // |  (for union types)
222
223    // Delimiters
224    LBrace,    // {
225    RBrace,    // }
226    LParen,    // (
227    RParen,    // )
228    LBracket,  // [
229    RBracket,  // ]
230    Comma,     // ,
231    Colon,     // :
232    Semicolon, // ;
233
234    // Comments
235    LineComment(String),  // // text
236    BlockComment(String), // /* text */
237
238    // Special
239    Newline,
240    Eof,
241}
242
243impl fmt::Display for TokenKind {
244    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
245        match self {
246            TokenKind::Pipeline => write!(f, "pipeline"),
247            TokenKind::Extends => write!(f, "extends"),
248            TokenKind::Override => write!(f, "override"),
249            TokenKind::Let => write!(f, "let"),
250            TokenKind::Var => write!(f, "var"),
251            TokenKind::If => write!(f, "if"),
252            TokenKind::Else => write!(f, "else"),
253            TokenKind::For => write!(f, "for"),
254            TokenKind::In => write!(f, "in"),
255            TokenKind::Match => write!(f, "match"),
256            TokenKind::Retry => write!(f, "retry"),
257            TokenKind::Parallel => write!(f, "parallel"),
258            TokenKind::ParallelMap => write!(f, "parallel_map"),
259            TokenKind::ParallelSettle => write!(f, "parallel_settle"),
260            TokenKind::Return => write!(f, "return"),
261            TokenKind::Import => write!(f, "import"),
262            TokenKind::True => write!(f, "true"),
263            TokenKind::False => write!(f, "false"),
264            TokenKind::Nil => write!(f, "nil"),
265            TokenKind::Try => write!(f, "try"),
266            TokenKind::Catch => write!(f, "catch"),
267            TokenKind::Throw => write!(f, "throw"),
268            TokenKind::Finally => write!(f, "finally"),
269            TokenKind::Fn => write!(f, "fn"),
270            TokenKind::Spawn => write!(f, "spawn"),
271            TokenKind::While => write!(f, "while"),
272            TokenKind::TypeKw => write!(f, "type"),
273            TokenKind::Enum => write!(f, "enum"),
274            TokenKind::Struct => write!(f, "struct"),
275            TokenKind::Interface => write!(f, "interface"),
276            TokenKind::Pub => write!(f, "pub"),
277            TokenKind::From => write!(f, "from"),
278            TokenKind::Thru => write!(f, "thru"),
279            TokenKind::Tool => write!(f, "tool"),
280            TokenKind::Upto => write!(f, "upto"),
281            TokenKind::Guard => write!(f, "guard"),
282            TokenKind::Require => write!(f, "require"),
283            TokenKind::Ask => write!(f, "ask"),
284            TokenKind::Deadline => write!(f, "deadline"),
285            TokenKind::Yield => write!(f, "yield"),
286            TokenKind::Mutex => write!(f, "mutex"),
287            TokenKind::Break => write!(f, "break"),
288            TokenKind::Continue => write!(f, "continue"),
289            TokenKind::Select => write!(f, "select"),
290            TokenKind::Impl => write!(f, "impl"),
291            TokenKind::Identifier(s) => write!(f, "id({s})"),
292            TokenKind::StringLiteral(s) => write!(f, "str({s})"),
293            TokenKind::InterpolatedString(_) => write!(f, "istr(...)"),
294            TokenKind::RawStringLiteral(s) => write!(f, "rstr({s})"),
295            TokenKind::IntLiteral(n) => write!(f, "int({n})"),
296            TokenKind::FloatLiteral(n) => write!(f, "float({n})"),
297            TokenKind::DurationLiteral(ms) => write!(f, "duration({ms}ms)"),
298            TokenKind::Eq => write!(f, "=="),
299            TokenKind::Neq => write!(f, "!="),
300            TokenKind::And => write!(f, "&&"),
301            TokenKind::Or => write!(f, "||"),
302            TokenKind::Pipe => write!(f, "|>"),
303            TokenKind::NilCoal => write!(f, "??"),
304            TokenKind::QuestionDot => write!(f, "?."),
305            TokenKind::Arrow => write!(f, "->"),
306            TokenKind::Lte => write!(f, "<="),
307            TokenKind::Gte => write!(f, ">="),
308            TokenKind::PlusAssign => write!(f, "+="),
309            TokenKind::MinusAssign => write!(f, "-="),
310            TokenKind::StarAssign => write!(f, "*="),
311            TokenKind::SlashAssign => write!(f, "/="),
312            TokenKind::PercentAssign => write!(f, "%="),
313            TokenKind::Assign => write!(f, "="),
314            TokenKind::Not => write!(f, "!"),
315            TokenKind::Dot => write!(f, "."),
316            TokenKind::Plus => write!(f, "+"),
317            TokenKind::Minus => write!(f, "-"),
318            TokenKind::Star => write!(f, "*"),
319            TokenKind::Slash => write!(f, "/"),
320            TokenKind::Percent => write!(f, "%"),
321            TokenKind::Lt => write!(f, "<"),
322            TokenKind::Gt => write!(f, ">"),
323            TokenKind::Question => write!(f, "?"),
324            TokenKind::Bar => write!(f, "|"),
325            TokenKind::LBrace => write!(f, "{{"),
326            TokenKind::RBrace => write!(f, "}}"),
327            TokenKind::LParen => write!(f, "("),
328            TokenKind::RParen => write!(f, ")"),
329            TokenKind::LBracket => write!(f, "["),
330            TokenKind::RBracket => write!(f, "]"),
331            TokenKind::Comma => write!(f, ","),
332            TokenKind::Colon => write!(f, ":"),
333            TokenKind::Semicolon => write!(f, ";"),
334            TokenKind::LineComment(s) => write!(f, "// {s}"),
335            TokenKind::BlockComment(s) => write!(f, "/* {s} */"),
336            TokenKind::Newline => write!(f, "\\n"),
337            TokenKind::Eof => write!(f, "EOF"),
338        }
339    }
340}
341
342/// A token with its kind and source location.
343#[derive(Debug, Clone, PartialEq)]
344pub struct Token {
345    pub kind: TokenKind,
346    pub span: Span,
347}
348
349impl Token {
350    pub fn with_span(kind: TokenKind, span: Span) -> Self {
351        Self { kind, span }
352    }
353}
harn_lexer/token.rs

harn_lexer/
token.rs
