harn_lexer/
token.rs

1use std::fmt;
2
3/// A segment of an interpolated string.
4#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
5pub enum StringSegment {
6    Literal(String),
7    /// An interpolated expression with its source position (line, column).
8    Expression(String, usize, usize),
9}
10
11impl fmt::Display for StringSegment {
12    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
13        match self {
14            StringSegment::Literal(s) => write!(f, "{s}"),
15            StringSegment::Expression(e, _, _) => write!(f, "${{{e}}}"),
16        }
17    }
18}
19
20/// Source location for error reporting.
21#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
22pub struct Span {
23    /// Byte offset from start of source (inclusive).
24    pub start: usize,
25    /// Byte offset from start of source (exclusive).
26    pub end: usize,
27    /// 1-based line number of start position.
28    pub line: usize,
29    /// 1-based column number of start position.
30    pub column: usize,
31    /// 1-based line number of end position (for multiline span detection).
32    pub end_line: usize,
33}
34
35impl Span {
36    pub fn with_offsets(start: usize, end: usize, line: usize, column: usize) -> Self {
37        Self {
38            start,
39            end,
40            line,
41            column,
42            end_line: line,
43        }
44    }
45
46    /// Create a span covering two spans (from start of `a` to end of `b`).
47    pub fn merge(a: Span, b: Span) -> Span {
48        Span {
49            start: a.start,
50            end: b.end,
51            line: a.line,
52            column: a.column,
53            end_line: b.end_line,
54        }
55    }
56
57    /// A dummy span for synthetic/generated nodes.
58    pub fn dummy() -> Self {
59        Self {
60            start: 0,
61            end: 0,
62            line: 0,
63            column: 0,
64            end_line: 0,
65        }
66    }
67}
68
69impl fmt::Display for Span {
70    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
71        write!(f, "{}:{}", self.line, self.column)
72    }
73}
74
75/// A machine-applicable text replacement for autofixing diagnostics.
76#[derive(Debug, Clone)]
77pub struct FixEdit {
78    /// The source span to replace.
79    pub span: Span,
80    /// The replacement text (empty string = deletion).
81    pub replacement: String,
82}
83
84impl FixEdit {
85    /// Sort edits right-to-left by start offset and drop any that overlap an
86    /// already-accepted edit, returning the survivors in descending-start
87    /// order — ready to splice right-to-left without invalidating earlier
88    /// offsets. This is the single source of truth for the "apply all fixes,
89    /// drop conflicts" policy that `harn fmt`, `harn lint --fix`, and the LSP
90    /// on-save fixer must agree on byte-for-byte.
91    pub fn dedupe_overlapping(edits: &[FixEdit]) -> Vec<FixEdit> {
92        let mut sorted = edits.to_vec();
93        sorted.sort_by_key(|edit| std::cmp::Reverse(edit.span.start));
94        let mut accepted: Vec<FixEdit> = Vec::new();
95        for edit in sorted {
96            let overlaps = accepted
97                .iter()
98                .any(|prev| edit.span.start < prev.span.end && edit.span.end > prev.span.start);
99            if !overlaps {
100                accepted.push(edit);
101            }
102        }
103        accepted
104    }
105
106    /// Apply `edits` to `source`, dropping overlaps via
107    /// [`Self::dedupe_overlapping`] and splicing right-to-left. Callers that
108    /// also need the accepted-edit list (e.g. to build LSP `TextEdit`s) should
109    /// call `dedupe_overlapping` directly.
110    pub fn apply_all(source: &str, edits: &[FixEdit]) -> String {
111        let mut out = source.to_string();
112        for edit in Self::dedupe_overlapping(edits) {
113            let before = &out[..edit.span.start];
114            let after = &out[edit.span.end..];
115            out = format!("{before}{}{after}", edit.replacement);
116        }
117        out
118    }
119}
120
121#[cfg(test)]
122mod fix_edit_tests {
123    use super::*;
124
125    fn edit(start: usize, end: usize, replacement: &str) -> FixEdit {
126        FixEdit {
127            span: Span::with_offsets(start, end, 1, start + 1),
128            replacement: replacement.to_string(),
129        }
130    }
131
132    #[test]
133    fn apply_all_splices_right_to_left() {
134        // Order-independent input; non-overlapping edits both apply.
135        let out = FixEdit::apply_all("0123456789", &[edit(2, 4, "AB"), edit(6, 8, "CD")]);
136        assert_eq!(out, "01AB45CD89");
137    }
138
139    #[test]
140    fn apply_all_drops_overlapping_edits_descending_start_wins() {
141        // Sorted descending by start, edit(4,8) is accepted and edit(2,6)
142        // overlaps it, so it's dropped — matching fmt/lsp/cli semantics.
143        let out = FixEdit::apply_all("0123456789", &[edit(2, 6, "XXXX"), edit(4, 8, "YYYY")]);
144        assert_eq!(out, "0123YYYY89");
145        assert_eq!(
146            FixEdit::dedupe_overlapping(&[edit(2, 6, "x"), edit(4, 8, "y")]).len(),
147            1
148        );
149    }
150}
151
152/// Canonical list of Harn language keywords. Single source of truth; the lexer's
153/// identifier-to-keyword match must stay in sync (enforced by
154/// `test_keywords_const_covers_lexer`). External tooling should consume this
155/// rather than duplicate it.
156pub const KEYWORDS: &[&str] = &[
157    "ask_user",
158    "break",
159    "catch",
160    "const",
161    "continue",
162    "deadline",
163    "defer",
164    "dual_control",
165    "else",
166    "emit",
167    "enum",
168    "escalate_to",
169    "eval_pack",
170    "exclusive",
171    "extends",
172    "false",
173    "finally",
174    "fn",
175    "for",
176    "from",
177    "guard",
178    "if",
179    "impl",
180    "import",
181    "in",
182    "interface",
183    "let",
184    "match",
185    "mutex",
186    "nil",
187    "override",
188    "parallel",
189    "pipeline",
190    "pub",
191    "request_approval",
192    "require",
193    "retry",
194    "return",
195    "select",
196    "skill",
197    "spawn",
198    "struct",
199    "throw",
200    "to",
201    "tool",
202    "true",
203    "try",
204    "type",
205    "var",
206    "while",
207    "yield",
208];
209
210/// Token kinds produced by the lexer.
211#[derive(Debug, Clone, PartialEq)]
212pub enum TokenKind {
213    Pipeline,
214    Extends,
215    Override,
216    Let,
217    Const,
218    Var,
219    If,
220    Else,
221    For,
222    In,
223    Match,
224    Retry,
225    Parallel,
226    Return,
227    Import,
228    True,
229    False,
230    Nil,
231    Try,
232    Catch,
233    Throw,
234    Finally,
235    Fn,
236    Spawn,
237    While,
238    TypeKw,
239    Enum,
240    EvalPack,
241    Struct,
242    Interface,
243    Emit,
244    Pub,
245    From,
246    To,
247    Tool,
248    Exclusive,
249    Guard,
250    Require,
251    Deadline,
252    Defer,
253    Yield,
254    Mutex,
255    Break,
256    Continue,
257    Select,
258    Impl,
259    Skill,
260    /// First-class HITL primitive: `request_approval(...)`.
261    RequestApproval,
262    /// First-class HITL primitive: `dual_control(...)`.
263    DualControl,
264    /// First-class HITL primitive: `ask_user(...)`.
265    AskUser,
266    /// First-class HITL primitive: `escalate_to(...)`.
267    EscalateTo,
268
269    Identifier(String),
270    StringLiteral(String),
271    InterpolatedString(Vec<StringSegment>),
272    /// Raw string literal `r"..."` — no escape processing, no interpolation.
273    RawStringLiteral(String),
274    IntLiteral(i64),
275    FloatLiteral(f64),
276    /// Duration literal in milliseconds: 500ms, 5s, 30m, 2h, 1d, 1w
277    DurationLiteral(u64),
278
279    Eq,            // ==
280    Neq,           // !=
281    And,           // &&
282    Or,            // ||
283    Pipe,          // |>
284    NilCoal,       // ??
285    Pow,           // **
286    QuestionDot,   // ?.
287    Arrow,         // ->
288    Lte,           // <=
289    Gte,           // >=
290    PlusAssign,    // +=
291    MinusAssign,   // -=
292    StarAssign,    // *=
293    SlashAssign,   // /=
294    PercentAssign, // %=
295
296    Assign,   // =
297    Not,      // !
298    Dot,      // .
299    Plus,     // +
300    Minus,    // -
301    Star,     // *
302    Slash,    // /
303    Percent,  // %
304    Lt,       // <
305    Gt,       // >
306    Question, // ?
307    Bar,      // |  (for union types)
308    Amp,      // &  (for intersection types)
309
310    LBrace,    // {
311    RBrace,    // }
312    LParen,    // (
313    RParen,    // )
314    LBracket,  // [
315    RBracket,  // ]
316    Comma,     // ,
317    Colon,     // :
318    Semicolon, // ;
319    At,        // @ (attribute prefix)
320
321    LineComment {
322        text: String,
323        is_doc: bool,
324    }, // // text or /// text
325    BlockComment {
326        text: String,
327        is_doc: bool,
328    }, // /* text */ or /** text */
329
330    Newline,
331    Eof,
332}
333
334impl fmt::Display for TokenKind {
335    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
336        match self {
337            TokenKind::Pipeline => write!(f, "pipeline"),
338            TokenKind::Extends => write!(f, "extends"),
339            TokenKind::Override => write!(f, "override"),
340            TokenKind::Let => write!(f, "let"),
341            TokenKind::Const => write!(f, "const"),
342            TokenKind::Var => write!(f, "var"),
343            TokenKind::If => write!(f, "if"),
344            TokenKind::Else => write!(f, "else"),
345            TokenKind::For => write!(f, "for"),
346            TokenKind::In => write!(f, "in"),
347            TokenKind::Match => write!(f, "match"),
348            TokenKind::Retry => write!(f, "retry"),
349            TokenKind::Parallel => write!(f, "parallel"),
350            TokenKind::Return => write!(f, "return"),
351            TokenKind::Import => write!(f, "import"),
352            TokenKind::True => write!(f, "true"),
353            TokenKind::False => write!(f, "false"),
354            TokenKind::Nil => write!(f, "nil"),
355            TokenKind::Try => write!(f, "try"),
356            TokenKind::Catch => write!(f, "catch"),
357            TokenKind::Throw => write!(f, "throw"),
358            TokenKind::Finally => write!(f, "finally"),
359            TokenKind::Fn => write!(f, "fn"),
360            TokenKind::Spawn => write!(f, "spawn"),
361            TokenKind::While => write!(f, "while"),
362            TokenKind::TypeKw => write!(f, "type"),
363            TokenKind::Enum => write!(f, "enum"),
364            TokenKind::EvalPack => write!(f, "eval_pack"),
365            TokenKind::Struct => write!(f, "struct"),
366            TokenKind::Interface => write!(f, "interface"),
367            TokenKind::Emit => write!(f, "emit"),
368            TokenKind::Pub => write!(f, "pub"),
369            TokenKind::From => write!(f, "from"),
370            TokenKind::To => write!(f, "to"),
371            TokenKind::Tool => write!(f, "tool"),
372            TokenKind::Exclusive => write!(f, "exclusive"),
373            TokenKind::Guard => write!(f, "guard"),
374            TokenKind::Require => write!(f, "require"),
375            TokenKind::Deadline => write!(f, "deadline"),
376            TokenKind::Defer => write!(f, "defer"),
377            TokenKind::Yield => write!(f, "yield"),
378            TokenKind::Mutex => write!(f, "mutex"),
379            TokenKind::Break => write!(f, "break"),
380            TokenKind::Continue => write!(f, "continue"),
381            TokenKind::Select => write!(f, "select"),
382            TokenKind::Impl => write!(f, "impl"),
383            TokenKind::Skill => write!(f, "skill"),
384            TokenKind::RequestApproval => write!(f, "request_approval"),
385            TokenKind::DualControl => write!(f, "dual_control"),
386            TokenKind::AskUser => write!(f, "ask_user"),
387            TokenKind::EscalateTo => write!(f, "escalate_to"),
388            TokenKind::Identifier(s) => write!(f, "id({s})"),
389            TokenKind::StringLiteral(s) => write!(f, "str({s})"),
390            TokenKind::InterpolatedString(_) => write!(f, "istr(...)"),
391            TokenKind::RawStringLiteral(s) => write!(f, "rstr({s})"),
392            TokenKind::IntLiteral(n) => write!(f, "int({n})"),
393            TokenKind::FloatLiteral(n) => write!(f, "float({n})"),
394            TokenKind::DurationLiteral(ms) => write!(f, "duration({ms}ms)"),
395            TokenKind::Eq => write!(f, "=="),
396            TokenKind::Neq => write!(f, "!="),
397            TokenKind::And => write!(f, "&&"),
398            TokenKind::Or => write!(f, "||"),
399            TokenKind::Pipe => write!(f, "|>"),
400            TokenKind::NilCoal => write!(f, "??"),
401            TokenKind::Pow => write!(f, "**"),
402            TokenKind::QuestionDot => write!(f, "?."),
403            TokenKind::Arrow => write!(f, "->"),
404            TokenKind::Lte => write!(f, "<="),
405            TokenKind::Gte => write!(f, ">="),
406            TokenKind::PlusAssign => write!(f, "+="),
407            TokenKind::MinusAssign => write!(f, "-="),
408            TokenKind::StarAssign => write!(f, "*="),
409            TokenKind::SlashAssign => write!(f, "/="),
410            TokenKind::PercentAssign => write!(f, "%="),
411            TokenKind::Assign => write!(f, "="),
412            TokenKind::Not => write!(f, "!"),
413            TokenKind::Dot => write!(f, "."),
414            TokenKind::Plus => write!(f, "+"),
415            TokenKind::Minus => write!(f, "-"),
416            TokenKind::Star => write!(f, "*"),
417            TokenKind::Slash => write!(f, "/"),
418            TokenKind::Percent => write!(f, "%"),
419            TokenKind::Lt => write!(f, "<"),
420            TokenKind::Gt => write!(f, ">"),
421            TokenKind::Question => write!(f, "?"),
422            TokenKind::Bar => write!(f, "|"),
423            TokenKind::Amp => write!(f, "&"),
424            TokenKind::LBrace => write!(f, "{{"),
425            TokenKind::RBrace => write!(f, "}}"),
426            TokenKind::LParen => write!(f, "("),
427            TokenKind::RParen => write!(f, ")"),
428            TokenKind::LBracket => write!(f, "["),
429            TokenKind::RBracket => write!(f, "]"),
430            TokenKind::Comma => write!(f, ","),
431            TokenKind::Colon => write!(f, ":"),
432            TokenKind::Semicolon => write!(f, ";"),
433            TokenKind::At => write!(f, "@"),
434            TokenKind::LineComment { text, is_doc } => {
435                let prefix = if *is_doc { "///" } else { "//" };
436                write!(f, "{prefix} {text}")
437            }
438            TokenKind::BlockComment { text, is_doc } => {
439                let prefix = if *is_doc { "/**" } else { "/*" };
440                write!(f, "{prefix} {text} */")
441            }
442            TokenKind::Newline => write!(f, "\\n"),
443            TokenKind::Eof => write!(f, "EOF"),
444        }
445    }
446}
447
448/// A token with its kind and source location.
449#[derive(Debug, Clone, PartialEq)]
450pub struct Token {
451    pub kind: TokenKind,
452    pub span: Span,
453}
454
455impl Token {
456    pub fn with_span(kind: TokenKind, span: Span) -> Self {
457        Self { kind, span }
458    }
459}
harn_lexer/token.rs

harn_lexer/
token.rs
