harn_lexer/
token.rs

1use std::fmt;
2
3/// A segment of an interpolated string.
4#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
5pub enum StringSegment {
6    Literal(String),
7    /// An interpolated expression with its source position (line, column).
8    Expression(String, usize, usize),
9}
10
11impl fmt::Display for StringSegment {
12    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
13        match self {
14            StringSegment::Literal(s) => write!(f, "{s}"),
15            StringSegment::Expression(e, _, _) => write!(f, "${{{e}}}"),
16        }
17    }
18}
19
20/// Source location for error reporting.
21#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
22pub struct Span {
23    /// Byte offset from start of source (inclusive).
24    pub start: usize,
25    /// Byte offset from start of source (exclusive).
26    pub end: usize,
27    /// 1-based line number of start position.
28    pub line: usize,
29    /// 1-based column number of start position.
30    pub column: usize,
31    /// 1-based line number of end position (for multiline span detection).
32    pub end_line: usize,
33}
34
35/// Resolve a one-based lexer line/column to an absolute UTF-8 byte offset.
36///
37/// Harn columns count Unicode scalar values, while edit spans use byte offsets.
38/// Keeping this conversion in the lexer prevents diagnostic and repair clients
39/// from maintaining subtly different source-coordinate projections.
40#[must_use]
41pub fn byte_offset_for_position(source: &str, line: usize, column: usize) -> Option<usize> {
42    if line == 0 || column == 0 {
43        return None;
44    }
45
46    let mut current_line = 1usize;
47    let mut current_column = 1usize;
48    for (offset, character) in source.char_indices() {
49        if current_line == line && current_column == column {
50            return Some(offset);
51        }
52        if character == '\n' {
53            current_line += 1;
54            current_column = 1;
55        } else {
56            current_column += 1;
57        }
58    }
59
60    (current_line == line && current_column == column).then_some(source.len())
61}
62
63impl Span {
64    pub fn with_offsets(start: usize, end: usize, line: usize, column: usize) -> Self {
65        Self {
66            start,
67            end,
68            line,
69            column,
70            end_line: line,
71        }
72    }
73
74    /// Create a span covering two spans (from start of `a` to end of `b`).
75    pub fn merge(a: Span, b: Span) -> Span {
76        Span {
77            start: a.start,
78            end: b.end,
79            line: a.line,
80            column: a.column,
81            end_line: b.end_line,
82        }
83    }
84
85    /// A dummy span for synthetic/generated nodes.
86    pub fn dummy() -> Self {
87        Self {
88            start: 0,
89            end: 0,
90            line: 0,
91            column: 0,
92            end_line: 0,
93        }
94    }
95}
96
97impl fmt::Display for Span {
98    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
99        write!(f, "{}:{}", self.line, self.column)
100    }
101}
102
103/// A machine-applicable text replacement for autofixing diagnostics.
104#[derive(Debug, Clone)]
105pub struct FixEdit {
106    /// The source span to replace.
107    pub span: Span,
108    /// The replacement text (empty string = deletion).
109    pub replacement: String,
110}
111
112impl FixEdit {
113    /// Sort edits right-to-left by start offset and drop any that overlap an
114    /// already-accepted edit, returning the survivors in descending-start
115    /// order — ready to splice right-to-left without invalidating earlier
116    /// offsets. This is the single source of truth for the "apply all fixes,
117    /// drop conflicts" policy that `harn fmt`, `harn lint --fix`, and the LSP
118    /// on-save fixer must agree on byte-for-byte.
119    pub fn dedupe_overlapping(edits: &[FixEdit]) -> Vec<FixEdit> {
120        let mut sorted = edits.to_vec();
121        // At the same offset, apply a replacement before an insertion. This
122        // makes independently synthesized "project this argument" and
123        // "prepend a new argument" edits compose as
124        // `root, root.capability` instead of overwriting the insertion.
125        sorted.sort_by_key(|edit| {
126            (
127                std::cmp::Reverse(edit.span.start),
128                std::cmp::Reverse(edit.span.end),
129            )
130        });
131        let mut accepted: Vec<FixEdit> = Vec::new();
132        for edit in sorted {
133            let overlaps = accepted
134                .iter()
135                .any(|prev| edit.span.start < prev.span.end && edit.span.end > prev.span.start);
136            if !overlaps {
137                accepted.push(edit);
138            }
139        }
140        accepted
141    }
142
143    /// Apply `edits` to `source`, dropping overlaps via
144    /// [`Self::dedupe_overlapping`] and splicing right-to-left. Callers that
145    /// also need the accepted-edit list (e.g. to build LSP `TextEdit`s) should
146    /// call `dedupe_overlapping` directly.
147    #[expect(
148        clippy::string_slice,
149        reason = "FixEdit spans are lexed token byte offsets, which lie on char boundaries"
150    )]
151    pub fn apply_all(source: &str, edits: &[FixEdit]) -> String {
152        let mut out = source.to_string();
153        for edit in Self::dedupe_overlapping(edits) {
154            let before = &out[..edit.span.start];
155            let after = &out[edit.span.end..];
156            out = format!("{before}{}{after}", edit.replacement);
157        }
158        out
159    }
160}
161
162#[cfg(test)]
163mod fix_edit_tests {
164    use super::*;
165
166    fn edit(start: usize, end: usize, replacement: &str) -> FixEdit {
167        FixEdit {
168            span: Span::with_offsets(start, end, 1, start + 1),
169            replacement: replacement.to_string(),
170        }
171    }
172
173    #[test]
174    fn apply_all_splices_right_to_left() {
175        // Order-independent input; non-overlapping edits both apply.
176        let out = FixEdit::apply_all("0123456789", &[edit(2, 4, "AB"), edit(6, 8, "CD")]);
177        assert_eq!(out, "01AB45CD89");
178    }
179
180    #[test]
181    fn apply_all_drops_overlapping_edits_descending_start_wins() {
182        // Sorted descending by start, edit(4,8) is accepted and edit(2,6)
183        // overlaps it, so it's dropped — matching fmt/lsp/cli semantics.
184        let out = FixEdit::apply_all("0123456789", &[edit(2, 6, "XXXX"), edit(4, 8, "YYYY")]);
185        assert_eq!(out, "0123YYYY89");
186        assert_eq!(
187            FixEdit::dedupe_overlapping(&[edit(2, 6, "x"), edit(4, 8, "y")]).len(),
188            1
189        );
190    }
191
192    #[test]
193    fn apply_all_composes_replacement_then_insertion_at_same_offset() {
194        let source = "call(harness, value)";
195        let start = source.find("harness").unwrap();
196        let out = FixEdit::apply_all(
197            source,
198            &[
199                edit(start, start, "harness, "),
200                edit(start, start + "harness".len(), "harness.fs"),
201            ],
202        );
203        assert_eq!(out, "call(harness, harness.fs, value)");
204    }
205}
206
207macro_rules! define_keyword_vocabulary {
208    (
209        keywords { $( $keyword:literal => $token:ident ),* $(,)? }
210        literals { $( $literal:literal => $literal_token:ident ),* $(,)? }
211    ) => {
212        /// Canonical Harn keyword vocabulary consumed by parser and tooling.
213        pub const KEYWORDS: &[&str] = &[$($keyword,)* $($literal,)*];
214
215        /// Keyword-shaped literal values for syntax-highlighting projections.
216        pub const LITERAL_KEYWORDS: &[&str] = &[$($literal,)*];
217
218        /// Tokenize a canonical keyword with a compile-time match table.
219        ///
220        /// This function and both public vocabulary projections are generated
221        /// by the same declaration, so adding syntax cannot update one surface
222        /// while leaving another stale.
223        pub fn keyword_token_kind(value: &str) -> Option<TokenKind> {
224            match value {
225                $($keyword => Some(TokenKind::$token),)*
226                $($literal => Some(TokenKind::$literal_token),)*
227                _ => None,
228            }
229        }
230    };
231}
232
233define_keyword_vocabulary! {
234    keywords {
235        "break" => Break,
236        "catch" => Catch,
237        "const" => Const,
238        "continue" => Continue,
239        "deadline" => Deadline,
240        "defer" => Defer,
241        "else" => Else,
242        "emit" => Emit,
243        "enum" => Enum,
244        "eval_pack" => EvalPack,
245        "exclusive" => Exclusive,
246        "extends" => Extends,
247        "finally" => Finally,
248        "fn" => Fn,
249        "for" => For,
250        "from" => From,
251        "guard" => Guard,
252        "if" => If,
253        "impl" => Impl,
254        "import" => Import,
255        "in" => In,
256        "interface" => Interface,
257        "let" => Let,
258        "match" => Match,
259        "mutex" => Mutex,
260        "override" => Override,
261        "parallel" => Parallel,
262        "pipeline" => Pipeline,
263        "pub" => Pub,
264        "require" => Require,
265        "retry" => Retry,
266        "return" => Return,
267        "select" => Select,
268        "skill" => Skill,
269        "spawn" => Spawn,
270        "struct" => Struct,
271        "throw" => Throw,
272        "throws" => Throws,
273        "to" => To,
274        "tool" => Tool,
275        "try" => Try,
276        "type" => TypeKw,
277        "var" => Var,
278        "while" => While,
279        "yield" => Yield,
280    }
281    literals {
282        "false" => False,
283        "nil" => Nil,
284        "true" => True,
285    }
286}
287
288/// Token kinds produced by the lexer.
289#[derive(Debug, Clone, PartialEq)]
290pub enum TokenKind {
291    Pipeline,
292    Extends,
293    Override,
294    Let,
295    Const,
296    Var,
297    If,
298    Else,
299    For,
300    In,
301    Match,
302    Retry,
303    Parallel,
304    Return,
305    Import,
306    True,
307    False,
308    Nil,
309    Try,
310    Catch,
311    Throw,
312    /// `throws` — declares a callable's exception channel in its signature
313    /// (`fn f() -> R throws E`). Distinct from `Throw` (the statement).
314    Throws,
315    Finally,
316    Fn,
317    Spawn,
318    While,
319    TypeKw,
320    Enum,
321    EvalPack,
322    Struct,
323    Interface,
324    Emit,
325    Pub,
326    From,
327    To,
328    Tool,
329    Exclusive,
330    Guard,
331    Require,
332    Deadline,
333    Defer,
334    Yield,
335    Mutex,
336    Break,
337    Continue,
338    Select,
339    Impl,
340    Skill,
341    /// First-class HITL primitive: `request_approval(...)`.
342    RequestApproval,
343    /// First-class HITL primitive: `dual_control(...)`.
344    DualControl,
345    /// First-class HITL primitive: `ask_user(...)`.
346    AskUser,
347    /// First-class HITL primitive: `escalate_to(...)`.
348    EscalateTo,
349
350    Identifier(String),
351    StringLiteral(String),
352    InterpolatedString(Vec<StringSegment>),
353    /// Raw string literal `r"..."` — no escape processing, no interpolation.
354    RawStringLiteral(String),
355    IntLiteral(i64),
356    FloatLiteral(f64),
357    /// Duration literal in milliseconds: 500ms, 5s, 30m, 2h, 1d, 1w
358    DurationLiteral(u64),
359
360    Eq,            // ==
361    Neq,           // !=
362    And,           // &&
363    Or,            // ||
364    Pipe,          // |>
365    NilCoal,       // ??
366    Pow,           // **
367    QuestionDot,   // ?.
368    Arrow,         // ->
369    Lte,           // <=
370    Gte,           // >=
371    PlusAssign,    // +=
372    MinusAssign,   // -=
373    StarAssign,    // *=
374    SlashAssign,   // /=
375    PercentAssign, // %=
376
377    Assign,   // =
378    Not,      // !
379    Dot,      // .
380    Plus,     // +
381    Minus,    // -
382    Star,     // *
383    Slash,    // /
384    Percent,  // %
385    Lt,       // <
386    Gt,       // >
387    Question, // ?
388    Bar,      // |  (for union types)
389    Amp,      // &  (for intersection types)
390
391    LBrace,    // {
392    RBrace,    // }
393    LParen,    // (
394    RParen,    // )
395    LBracket,  // [
396    RBracket,  // ]
397    Comma,     // ,
398    Colon,     // :
399    Semicolon, // ;
400    At,        // @ (attribute prefix)
401
402    LineComment {
403        text: String,
404        is_doc: bool,
405    }, // // text or /// text
406    BlockComment {
407        text: String,
408        is_doc: bool,
409    }, // /* text */ or /** text */
410
411    Newline,
412    Eof,
413}
414
415impl fmt::Display for TokenKind {
416    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
417        match self {
418            TokenKind::Pipeline => write!(f, "pipeline"),
419            TokenKind::Extends => write!(f, "extends"),
420            TokenKind::Override => write!(f, "override"),
421            TokenKind::Let => write!(f, "let"),
422            TokenKind::Const => write!(f, "const"),
423            TokenKind::Var => write!(f, "var"),
424            TokenKind::If => write!(f, "if"),
425            TokenKind::Else => write!(f, "else"),
426            TokenKind::For => write!(f, "for"),
427            TokenKind::In => write!(f, "in"),
428            TokenKind::Match => write!(f, "match"),
429            TokenKind::Retry => write!(f, "retry"),
430            TokenKind::Parallel => write!(f, "parallel"),
431            TokenKind::Return => write!(f, "return"),
432            TokenKind::Import => write!(f, "import"),
433            TokenKind::True => write!(f, "true"),
434            TokenKind::False => write!(f, "false"),
435            TokenKind::Nil => write!(f, "nil"),
436            TokenKind::Try => write!(f, "try"),
437            TokenKind::Catch => write!(f, "catch"),
438            TokenKind::Throw => write!(f, "throw"),
439            TokenKind::Throws => write!(f, "throws"),
440            TokenKind::Finally => write!(f, "finally"),
441            TokenKind::Fn => write!(f, "fn"),
442            TokenKind::Spawn => write!(f, "spawn"),
443            TokenKind::While => write!(f, "while"),
444            TokenKind::TypeKw => write!(f, "type"),
445            TokenKind::Enum => write!(f, "enum"),
446            TokenKind::EvalPack => write!(f, "eval_pack"),
447            TokenKind::Struct => write!(f, "struct"),
448            TokenKind::Interface => write!(f, "interface"),
449            TokenKind::Emit => write!(f, "emit"),
450            TokenKind::Pub => write!(f, "pub"),
451            TokenKind::From => write!(f, "from"),
452            TokenKind::To => write!(f, "to"),
453            TokenKind::Tool => write!(f, "tool"),
454            TokenKind::Exclusive => write!(f, "exclusive"),
455            TokenKind::Guard => write!(f, "guard"),
456            TokenKind::Require => write!(f, "require"),
457            TokenKind::Deadline => write!(f, "deadline"),
458            TokenKind::Defer => write!(f, "defer"),
459            TokenKind::Yield => write!(f, "yield"),
460            TokenKind::Mutex => write!(f, "mutex"),
461            TokenKind::Break => write!(f, "break"),
462            TokenKind::Continue => write!(f, "continue"),
463            TokenKind::Select => write!(f, "select"),
464            TokenKind::Impl => write!(f, "impl"),
465            TokenKind::Skill => write!(f, "skill"),
466            TokenKind::RequestApproval => write!(f, "request_approval"),
467            TokenKind::DualControl => write!(f, "dual_control"),
468            TokenKind::AskUser => write!(f, "ask_user"),
469            TokenKind::EscalateTo => write!(f, "escalate_to"),
470            TokenKind::Identifier(s) => write!(f, "id({s})"),
471            TokenKind::StringLiteral(s) => write!(f, "str({s})"),
472            TokenKind::InterpolatedString(_) => write!(f, "istr(...)"),
473            TokenKind::RawStringLiteral(s) => write!(f, "rstr({s})"),
474            TokenKind::IntLiteral(n) => write!(f, "int({n})"),
475            TokenKind::FloatLiteral(n) => write!(f, "float({n})"),
476            TokenKind::DurationLiteral(ms) => write!(f, "duration({ms}ms)"),
477            TokenKind::Eq => write!(f, "=="),
478            TokenKind::Neq => write!(f, "!="),
479            TokenKind::And => write!(f, "&&"),
480            TokenKind::Or => write!(f, "||"),
481            TokenKind::Pipe => write!(f, "|>"),
482            TokenKind::NilCoal => write!(f, "??"),
483            TokenKind::Pow => write!(f, "**"),
484            TokenKind::QuestionDot => write!(f, "?."),
485            TokenKind::Arrow => write!(f, "->"),
486            TokenKind::Lte => write!(f, "<="),
487            TokenKind::Gte => write!(f, ">="),
488            TokenKind::PlusAssign => write!(f, "+="),
489            TokenKind::MinusAssign => write!(f, "-="),
490            TokenKind::StarAssign => write!(f, "*="),
491            TokenKind::SlashAssign => write!(f, "/="),
492            TokenKind::PercentAssign => write!(f, "%="),
493            TokenKind::Assign => write!(f, "="),
494            TokenKind::Not => write!(f, "!"),
495            TokenKind::Dot => write!(f, "."),
496            TokenKind::Plus => write!(f, "+"),
497            TokenKind::Minus => write!(f, "-"),
498            TokenKind::Star => write!(f, "*"),
499            TokenKind::Slash => write!(f, "/"),
500            TokenKind::Percent => write!(f, "%"),
501            TokenKind::Lt => write!(f, "<"),
502            TokenKind::Gt => write!(f, ">"),
503            TokenKind::Question => write!(f, "?"),
504            TokenKind::Bar => write!(f, "|"),
505            TokenKind::Amp => write!(f, "&"),
506            TokenKind::LBrace => write!(f, "{{"),
507            TokenKind::RBrace => write!(f, "}}"),
508            TokenKind::LParen => write!(f, "("),
509            TokenKind::RParen => write!(f, ")"),
510            TokenKind::LBracket => write!(f, "["),
511            TokenKind::RBracket => write!(f, "]"),
512            TokenKind::Comma => write!(f, ","),
513            TokenKind::Colon => write!(f, ":"),
514            TokenKind::Semicolon => write!(f, ";"),
515            TokenKind::At => write!(f, "@"),
516            TokenKind::LineComment { text, is_doc } => {
517                let prefix = if *is_doc { "///" } else { "//" };
518                write!(f, "{prefix} {text}")
519            }
520            TokenKind::BlockComment { text, is_doc } => {
521                let prefix = if *is_doc { "/**" } else { "/*" };
522                write!(f, "{prefix} {text} */")
523            }
524            TokenKind::Newline => write!(f, "\\n"),
525            TokenKind::Eof => write!(f, "EOF"),
526        }
527    }
528}
529
530/// A token with its kind and source location.
531#[derive(Debug, Clone, PartialEq)]
532pub struct Token {
533    pub kind: TokenKind,
534    pub span: Span,
535}
536
537impl Token {
538    pub fn with_span(kind: TokenKind, span: Span) -> Self {
539        Self { kind, span }
540    }
541}
542
543#[cfg(test)]
544mod tests {
545    use super::byte_offset_for_position;
546
547    #[test]
548    fn source_positions_map_unicode_columns_to_byte_offsets() {
549        let source = "αβ\n  call(\"value\")\n";
550        let expected = source.find("\"value\"").expect("first argument");
551
552        assert_eq!(byte_offset_for_position(source, 2, 8), Some(expected));
553        assert_eq!(byte_offset_for_position(source, 3, 1), Some(source.len()));
554        assert_eq!(byte_offset_for_position(source, 0, 1), None);
555        assert_eq!(byte_offset_for_position(source, 2, 99), None);
556    }
557}
harn_lexer/token.rs

harn_lexer/
token.rs
