active_core/parser/lexer.rs
1//! Complete lexer for the Active alpha.
2//!
3//! Handles:
4//! - Identifiers, keywords (let, true, false, null, env, secret)
5//! - Strings (double & single quoted) with escape sequences
6//! - Integers (with underscores) and floats
7//! - Unit suffixes immediately after numbers (5s, 512MiB, 100/req/s …)
8//! - Operators: = . , ?: + - * / == != < <= > >= && || !
9//! - Delimiters: [ ] { } ( )
10//! - Newlines and full-line comments (# …)
11//! - Exact source spans on every token
12//!
13//! Healing: the lexer detects unterminated strings and records a repair
14//! diagnostic (H001) rather than rejecting the input. All other healing
15//! phases live in the parser (comma and bracket repairs) and in
16//! `crate::heal` (semantic and type healing).
17
18use crate::diag::{thresholds, Diagnostic, Repair, RepairKind, Severity, Span};
19
20/// Internal sentinel for an escaped dollar (`\$`) in a double-quoted string.
21///
22/// The lexer decodes escapes, which would otherwise erase the difference
23/// between `\$` (a literal dollar) and `\\` followed by `$` (a backslash and
24/// an interpolation opener). The sentinel preserves that distinction to the
25/// parser. It is rendered back as `$` in every string value, and the formatter
26/// re-emits it as `\$`, so it never reaches user data.
27pub const ESCAPED_DOLLAR: char = '\u{E000}';
28
29/// Result of tokenizing: the token stream, hard lexical errors, healing
30/// diagnostics, and the comments found along the way (trivia for the
31/// formatter).
32pub struct Lexed {
33 pub tokens: Vec<Token>,
34 pub repairs: Vec<Diagnostic>,
35 pub errors: Vec<Diagnostic>,
36 pub comments: Vec<(String, Span)>,
37}
38
39/// Build the H001 repair for a string that reached EOF without its closing
40/// delimiter: insert the missing delimiter. Safe repair (confidence ≥ 0.95).
41fn unclosed_string_repair(line: u32, col: u32, delimiter: &str) -> Diagnostic {
42 Diagnostic {
43 severity: Severity::Info, // safe repair: ≥ AUTO_APPLY
44 code: "H001".into(),
45 message: format!(
46 "string literal reached end of input without its closing {}; inserted",
47 delimiter
48 ),
49 path: None,
50 span: Some(Span {
51 line,
52 column: col,
53 end_line: line,
54 end_column: col + 1,
55 }),
56 repair: Some(Repair {
57 original: String::new(),
58 suggested: delimiter.to_string(),
59 kind: RepairKind::InsertQuote,
60 safe: true,
61 }),
62 confidence: thresholds::AUTO_APPLY + 0.03,
63 // The parser stamps the session file (includes get their own).
64 file: None,
65 }
66}
67
68#[derive(Debug, Clone, PartialEq, Eq)]
69pub enum TokenKind {
70 Eof,
71 Newline,
72 /// Opened an indented block (spec/dual-syntax.md §4).
73 Indent,
74 /// Closed an indented block.
75 Dedent,
76
77 // Literals & identifiers
78 Ident,
79 /// Double-quoted string: escapes processed, `${…}` interpolated.
80 /// Carries the source position (line offset, column) of every decoded
81 /// character, exactly like MlString: nested interpolation inside
82 /// sub-lexed expression text needs the same position fidelity.
83 String(Vec<(u32, u32)>),
84 /// Single-quoted string: escapes processed, `${…}` literal (spec §2.5).
85 SingleString,
86 /// Triple-double-quoted multi-line string: escapes processed, `${…}`
87 /// interpolated, common-indent stripping at parse time (spec §1.4.1).
88 /// Carries the source position (line offset, column) of every decoded
89 /// character: escapes decode to characters of different source width
90 /// (`\n` is two source columns becoming one decoded newline on the
91 /// same physical line), so only a per-character map keeps diagnostics
92 /// on their true positions.
93 MlString(Vec<(u32, u32)>),
94 /// Triple-single-quoted multi-line string: fully literal — backslashes
95 /// kept verbatim, `${…}` never interpolated (spec §1.4.1).
96 MlSingleString,
97 Integer,
98 Float,
99 True,
100 False,
101 Null,
102
103 // Keywords
104 Let,
105 Env,
106 Secret,
107
108 // Operators
109 Assign, // =
110 Dot, // .
111 Comma, // ,
112 QuestionColon, // ?:
113 Plus, // +
114 Minus, // -
115 Star, // *
116 Slash, // /
117 EqEq, // ==
118 NotEq, // !=
119 Lt, // <
120 LtEq, // <=
121 Gt, // >
122 GtEq, // >=
123 AndAnd, // &&
124 OrOr, // ||
125 Bang, // !
126
127 // Delimiters
128 LBracket, // [
129 RBracket, // ]
130 LBrace, // {
131 RBrace, // }
132 LParen, // (
133 RParen, // )
134 Colon, // :
135 At, // @ (semantic-block tags, spec §1.4.1)
136
137 // Unit suffix emitted right after a number when a known unit is present
138 Unit(String),
139}
140
141#[derive(Debug, Clone)]
142pub struct Token {
143 pub kind: TokenKind,
144 pub text: String,
145 pub line: u32,
146 pub col: u32,
147 pub end_line: u32,
148 pub end_col: u32,
149}
150
151#[derive(Debug, Clone, Copy, PartialEq, Eq)]
152pub enum LineClass {
153 /// Blank line, or a line whose first content is a comment: no tokens,
154 /// no indentation-stack change.
155 Blank,
156 /// A line carrying real content.
157 Content,
158}
159
160pub struct Lexer {
161 chars: Vec<char>,
162 pos: usize,
163 line: u32,
164 col: u32,
165 /// Stack of open `(`/`[`/`{` delimiters. Indentation is measured only
166 /// when this is empty (spec/dual-syntax.md §3): inside brackets, layout
167 /// is cosmetic. Only a *matching* closer pops, so malformed input cannot
168 /// change layout state.
169 open_delims: Vec<char>,
170 /// Indentation columns of the open blocks, initialised to `[0]`.
171 indent_stack: Vec<usize>,
172 /// True at the start of a physical line that still needs measurement.
173 at_line_start: bool,
174}
175
176impl Lexer {
177 pub fn new(src: &str) -> Self {
178 // A leading byte-order mark is not content (research HIST-YAML-021;
179 // JSON RFC policy: parsers may ignore it). Strip one U+FEFF so it
180 // never becomes an E005 token or shifts the first column.
181 let src = src.strip_prefix('\u{FEFF}').unwrap_or(src);
182 Self {
183 chars: src.chars().collect(),
184 pos: 0,
185 line: 1,
186 col: 1,
187 open_delims: Vec::new(),
188 indent_stack: vec![0],
189 at_line_start: true,
190 }
191 }
192
193 fn peek(&self) -> Option<char> {
194 self.chars.get(self.pos).copied()
195 }
196
197 fn peek_ahead(&self, n: usize) -> Option<char> {
198 self.chars.get(self.pos + n).copied()
199 }
200
201 fn bump(&mut self) -> Option<char> {
202 let c = self.peek()?;
203 self.pos += 1;
204 if c == '\n' {
205 self.line += 1;
206 self.col = 1;
207 } else {
208 self.col += 1;
209 }
210 Some(c)
211 }
212
213 fn make_token(&self, kind: TokenKind, text: String, start_line: u32, start_col: u32) -> Token {
214 Token {
215 kind,
216 text,
217 line: start_line,
218 col: start_col,
219 end_line: self.line,
220 end_col: self.col,
221 }
222 }
223
224 /// Lex a triple-double-quoted body after the opening `"""`. Processes
225 /// escapes exactly like `"…"` (including the `\$` sentinel) and leaves
226 /// `${…}` intact for the parser. Returns the decoded text, whether the
227 /// closing `"""` was found, and the source position (physical line
228 /// offset, column) of every decoded character for diagnostic mapping.
229 /// Newlines, brackets, braces, and `#` inside
230 /// the body are content — they never affect layout, delimiter, or
231 /// comment state, because the loop below consumes them directly.
232 fn lex_ml_string(&mut self, quote: char) -> (String, bool, Vec<(u32, u32)>) {
233 let mut s = String::new();
234 // Source position (physical line offset, column) of every decoded
235 // character, recorded *before* consuming: self.col is the column
236 // of the current peek character and advances with each bump().
237 let mut pos: Vec<(u32, u32)> = Vec::new();
238 let mut real_off: u32 = 0;
239 let mut escaped = false;
240 while let Some(ch) = self.peek() {
241 if escaped {
242 // The decoded character takes the backslash's position;
243 // bump() then advances past both source characters.
244 let at = (real_off, self.col);
245 match ch {
246 'n' => {
247 s.push('\n');
248 pos.push(at);
249 }
250 '\n' => {
251 // Backslash followed by a physical newline: an
252 // unknown escape, so both characters are kept — and
253 // the physical line still advances for mapping.
254 s.push('\\');
255 pos.push(at);
256 s.push('\n');
257 pos.push((real_off, self.col + 1));
258 real_off += 1;
259 }
260 't' => {
261 s.push('\t');
262 pos.push(at);
263 }
264 'r' => {
265 s.push('\r');
266 pos.push(at);
267 }
268 '\\' => {
269 s.push('\\');
270 pos.push(at);
271 }
272 '"' => {
273 s.push('"');
274 pos.push(at);
275 }
276 '0' => {
277 s.push('\0');
278 pos.push(at);
279 }
280 '$' => {
281 s.push(ESCAPED_DOLLAR);
282 pos.push(at);
283 }
284 other => {
285 s.push('\\');
286 pos.push(at);
287 s.push(other);
288 pos.push((real_off, self.col + 1));
289 }
290 }
291 escaped = false;
292 self.bump();
293 } else if ch == '\\' {
294 escaped = true;
295 self.bump();
296 } else if ch == quote
297 && self.peek_ahead(1) == Some(quote)
298 && self.peek_ahead(2) == Some(quote)
299 {
300 self.bump();
301 self.bump();
302 self.bump();
303 return (s, true, pos);
304 } else if ch == '\r' {
305 // Same CRLF normalization as single-line strings.
306 if self.peek_ahead(1) == Some('\n') {
307 self.bump();
308 continue;
309 }
310 pos.push((real_off, self.col));
311 s.push('\n');
312 self.bump();
313 } else {
314 if ch == '\n' {
315 pos.push((real_off, self.col));
316 real_off += 1;
317 } else {
318 pos.push((real_off, self.col));
319 }
320 s.push(ch);
321 self.bump();
322 }
323 }
324 if escaped {
325 s.push('\\');
326 pos.push((real_off, self.col));
327 }
328 (s, false, pos)
329 }
330
331 /// Lex a triple-single-quoted body after the opening `'''`. Fully
332 /// literal: no escapes, no interpolation. Returns the raw text and
333 /// whether the closing `'''` was found.
334 fn lex_ml_literal(&mut self, quote: char) -> (String, bool) {
335 let mut s = String::new();
336 while let Some(ch) = self.peek() {
337 if ch == quote && self.peek_ahead(1) == Some(quote) && self.peek_ahead(2) == Some(quote)
338 {
339 self.bump();
340 self.bump();
341 self.bump();
342 return (s, true);
343 }
344 // Same CRLF normalization; fully literal otherwise.
345 if ch == '\r' {
346 if self.peek_ahead(1) == Some('\n') {
347 self.bump();
348 continue;
349 }
350 s.push('\n');
351 self.bump();
352 continue;
353 }
354 s.push(ch);
355 self.bump();
356 }
357 (s, false)
358 }
359
360 /// Measure one line's indentation and update the stack, emitting Indent
361 /// and Dedent tokens (spec/dual-syntax.md §4). Returns whether the line
362 /// carries content.
363 fn measure_indentation(
364 &mut self,
365 tokens: &mut Vec<Token>,
366 errors: &mut Vec<Diagnostic>,
367 repairs: &mut Vec<Diagnostic>,
368 ) -> LineClass {
369 let line = self.line;
370 let start_col = self.col;
371 let mut column: usize = 0;
372 let mut had_tab = false;
373 let mut tab_span: Option<(u32, u32)> = None;
374
375 while let Some(ch) = self.peek() {
376 match ch {
377 ' ' => {
378 column += 1;
379 self.bump();
380 }
381 '\t' => {
382 // Each tab counts as one column; the first one is E006.
383 if !had_tab {
384 had_tab = true;
385 tab_span = Some((self.line, self.col));
386 }
387 column += 1;
388 self.bump();
389 }
390 _ => break,
391 }
392 }
393
394 if let Some((l, col)) = tab_span {
395 errors.push(Diagnostic {
396 severity: Severity::Error,
397 code: "E006".into(),
398 message: "tab character in leading whitespace; use spaces".into(),
399 path: None,
400 span: Some(Span {
401 line: l,
402 column: col,
403 end_line: l,
404 end_column: col + 1,
405 }),
406 repair: None,
407 confidence: 0.0,
408 // The parser stamps the session file (includes get theirs).
409 file: None,
410 });
411 }
412
413 // Blank line, or a line whose first content is a comment: no tokens
414 // and no stack change. (A comment-only line's comment is skipped by
415 // the main loop, which then sees the newline.)
416 match self.peek() {
417 None | Some('\n') | Some('#') => return LineClass::Blank,
418 _ => {}
419 }
420
421 let top = *self.indent_stack.last().unwrap_or(&0);
422 if column > top {
423 self.indent_stack.push(column);
424 tokens.push(self.make_token(TokenKind::Indent, column.to_string(), line, start_col));
425 } else if column < top {
426 while let Some(&level) = self.indent_stack.last() {
427 if level <= column {
428 break;
429 }
430 self.indent_stack.pop();
431 let target = self.indent_stack.last().copied().unwrap_or(0);
432 tokens.push(self.make_token(
433 TokenKind::Dedent,
434 target.to_string(),
435 line,
436 start_col,
437 ));
438 }
439 if self.indent_stack.last().copied() != Some(column) {
440 // Unmatched dedentation: the stack has already popped to the
441 // nearest level below, and the column is not pushed.
442 errors.push(Diagnostic {
443 severity: Severity::Error,
444 code: "E007".into(),
445 message: format!("inconsistent dedentation to column {column}"),
446 path: None,
447 span: Some(Span {
448 line,
449 column: start_col,
450 end_line: line,
451 end_column: start_col + column as u32,
452 }),
453 repair: None,
454 confidence: 0.0,
455 // The parser stamps the session file.
456 file: None,
457 });
458 }
459 }
460
461 let _ = repairs; // lexical repairs are recorded elsewhere in the loop
462 LineClass::Content
463 }
464
465 pub fn tokenize(&mut self) -> Lexed {
466 let mut tokens = Vec::new();
467 let mut repairs: Vec<Diagnostic> = Vec::new();
468 let mut errors: Vec<Diagnostic> = Vec::new();
469 let mut comments: Vec<(String, Span)> = Vec::new();
470
471 loop {
472 // Indentation is measured at the start of each physical line, and
473 // only at bracket depth 0 (spec/dual-syntax.md §3).
474 if self.at_line_start {
475 self.at_line_start = false;
476 if self.open_delims.is_empty() {
477 let class = self.measure_indentation(&mut tokens, &mut errors, &mut repairs);
478 if class == LineClass::Blank {
479 // Blank or comment-only line: the stack must not
480 // change, and the next physical line is measured.
481 self.at_line_start = true;
482 }
483 }
484 }
485
486 let c = match self.peek() {
487 Some(c) => c,
488 None => break,
489 };
490
491 match c {
492 // Horizontal whitespace
493 ' ' | '\t' | '\r' => {
494 self.bump();
495 }
496
497 // Newline
498 '\n' => {
499 let (l, col) = (self.line, self.col);
500 self.bump();
501 tokens.push(self.make_token(TokenKind::Newline, "\n".into(), l, col));
502 self.at_line_start = true;
503 }
504
505 // Full-line or trailing comment: retained as trivia with its
506 // span so the formatter can re-emit it (§7-§8). A
507 // trailing carriage return (CRLF documents) is not
508 // content: keeping it would poison annotation payloads
509 // (`true\r` is not `true`) and formatter output.
510 '#' => {
511 let (l, col) = (self.line, self.col);
512 let mut text = String::new();
513 while let Some(ch) = self.peek() {
514 if ch == '\n' {
515 break;
516 }
517 text.push(ch);
518 self.bump();
519 }
520 // Span end tracks the text: a stripped terminator
521 // is one character column, ASCII by definition.
522 let stripped = text.ends_with('\r');
523 if stripped {
524 text.pop();
525 }
526 comments.push((
527 text,
528 Span {
529 line: l,
530 column: col,
531 end_line: self.line,
532 end_column: if stripped {
533 self.col.saturating_sub(1)
534 } else {
535 self.col
536 },
537 },
538 ));
539 }
540
541 // Multi-character operators
542 '?' if self.peek_ahead(1) == Some(':') => {
543 let (l, col) = (self.line, self.col);
544 self.bump();
545 self.bump();
546 tokens.push(self.make_token(TokenKind::QuestionColon, "?:".into(), l, col));
547 }
548 '=' if self.peek_ahead(1) == Some('=') => {
549 let (l, col) = (self.line, self.col);
550 self.bump();
551 self.bump();
552 tokens.push(self.make_token(TokenKind::EqEq, "==".into(), l, col));
553 }
554 '!' if self.peek_ahead(1) == Some('=') => {
555 let (l, col) = (self.line, self.col);
556 self.bump();
557 self.bump();
558 tokens.push(self.make_token(TokenKind::NotEq, "!=".into(), l, col));
559 }
560 '<' if self.peek_ahead(1) == Some('=') => {
561 let (l, col) = (self.line, self.col);
562 self.bump();
563 self.bump();
564 tokens.push(self.make_token(TokenKind::LtEq, "<=".into(), l, col));
565 }
566 '>' if self.peek_ahead(1) == Some('=') => {
567 let (l, col) = (self.line, self.col);
568 self.bump();
569 self.bump();
570 tokens.push(self.make_token(TokenKind::GtEq, ">=".into(), l, col));
571 }
572 '&' if self.peek_ahead(1) == Some('&') => {
573 let (l, col) = (self.line, self.col);
574 self.bump();
575 self.bump();
576 tokens.push(self.make_token(TokenKind::AndAnd, "&&".into(), l, col));
577 }
578 '|' if self.peek_ahead(1) == Some('|') => {
579 let (l, col) = (self.line, self.col);
580 self.bump();
581 self.bump();
582 tokens.push(self.make_token(TokenKind::OrOr, "||".into(), l, col));
583 }
584
585 // Single-character operators & delimiters
586 '=' => {
587 let (l, col) = (self.line, self.col);
588 self.bump();
589 tokens.push(self.make_token(TokenKind::Assign, "=".into(), l, col));
590 }
591 '.' => {
592 let (l, col) = (self.line, self.col);
593 self.bump();
594 tokens.push(self.make_token(TokenKind::Dot, ".".into(), l, col));
595 }
596 ',' => {
597 let (l, col) = (self.line, self.col);
598 self.bump();
599 tokens.push(self.make_token(TokenKind::Comma, ",".into(), l, col));
600 }
601 '+' => {
602 let (l, col) = (self.line, self.col);
603 self.bump();
604 tokens.push(self.make_token(TokenKind::Plus, "+".into(), l, col));
605 }
606 '-' => {
607 let (l, col) = (self.line, self.col);
608 self.bump();
609 tokens.push(self.make_token(TokenKind::Minus, "-".into(), l, col));
610 }
611 '*' => {
612 let (l, col) = (self.line, self.col);
613 self.bump();
614 tokens.push(self.make_token(TokenKind::Star, "*".into(), l, col));
615 }
616 '/' => {
617 let (l, col) = (self.line, self.col);
618 self.bump();
619 tokens.push(self.make_token(TokenKind::Slash, "/".into(), l, col));
620 }
621 '<' => {
622 let (l, col) = (self.line, self.col);
623 self.bump();
624 tokens.push(self.make_token(TokenKind::Lt, "<".into(), l, col));
625 }
626 '>' => {
627 let (l, col) = (self.line, self.col);
628 self.bump();
629 tokens.push(self.make_token(TokenKind::Gt, ">".into(), l, col));
630 }
631 '!' => {
632 let (l, col) = (self.line, self.col);
633 self.bump();
634 tokens.push(self.make_token(TokenKind::Bang, "!".into(), l, col));
635 }
636 '[' => {
637 let (l, col) = (self.line, self.col);
638 self.bump();
639 self.open_delims.push('[');
640 tokens.push(self.make_token(TokenKind::LBracket, "[".into(), l, col));
641 }
642 ']' => {
643 let (l, col) = (self.line, self.col);
644 self.bump();
645 if self.open_delims.last() == Some(&'[') {
646 self.open_delims.pop();
647 }
648 tokens.push(self.make_token(TokenKind::RBracket, "]".into(), l, col));
649 }
650 '{' => {
651 let (l, col) = (self.line, self.col);
652 self.bump();
653 self.open_delims.push('{');
654 tokens.push(self.make_token(TokenKind::LBrace, "{".into(), l, col));
655 }
656 '}' => {
657 let (l, col) = (self.line, self.col);
658 self.bump();
659 if self.open_delims.last() == Some(&'{') {
660 self.open_delims.pop();
661 }
662 tokens.push(self.make_token(TokenKind::RBrace, "}".into(), l, col));
663 }
664 '(' => {
665 let (l, col) = (self.line, self.col);
666 self.bump();
667 self.open_delims.push('(');
668 tokens.push(self.make_token(TokenKind::LParen, "(".into(), l, col));
669 }
670 ')' => {
671 let (l, col) = (self.line, self.col);
672 self.bump();
673 if self.open_delims.last() == Some(&'(') {
674 self.open_delims.pop();
675 }
676 tokens.push(self.make_token(TokenKind::RParen, ")".into(), l, col));
677 }
678 ':' => {
679 // A bare colon (block headers, `key: value`). The `?:`
680 // combination is matched earlier as QuestionColon.
681 let (l, col) = (self.line, self.col);
682 self.bump();
683 tokens.push(self.make_token(TokenKind::Colon, ":".into(), l, col));
684 }
685 '@' => {
686 let (l, col) = (self.line, self.col);
687 self.bump();
688 tokens.push(self.make_token(TokenKind::At, "@".into(), l, col));
689 }
690
691 // Triple-quoted multi-line strings (spec §1.4.1). The
692 // triple-quote check must come before the single-quote
693 // arms: `"""` and `'''` open a multi-line string.
694 '"' if self.peek_ahead(1) == Some('"') && self.peek_ahead(2) == Some('"') => {
695 let (l, col) = (self.line, self.col);
696 self.bump();
697 self.bump();
698 self.bump(); // opening """
699 let (raw, terminated, src_pos) = self.lex_ml_string('"');
700 if !terminated {
701 repairs.push(unclosed_string_repair(l, col, "\"\"\""));
702 }
703 tokens.push(self.make_token(TokenKind::MlString(src_pos), raw, l, col));
704 }
705 '\'' if self.peek_ahead(1) == Some('\'') && self.peek_ahead(2) == Some('\'') => {
706 let (l, col) = (self.line, self.col);
707 self.bump();
708 self.bump();
709 self.bump(); // opening '''
710 let (raw, terminated) = self.lex_ml_literal('\'');
711 if !terminated {
712 repairs.push(unclosed_string_repair(l, col, "'''"));
713 }
714 tokens.push(self.make_token(TokenKind::MlSingleString, raw, l, col));
715 }
716
717 // Double-quoted strings (escapes + ${…} left intact for the parser)
718 '"' => {
719 let (l, col) = (self.line, self.col);
720 self.bump(); // opening "
721 let mut s = String::new();
722 // Per-character source positions (physical line offset,
723 // column), recorded before consuming: self.col is the
724 // column of the current peek character.
725 let mut pos: Vec<(u32, u32)> = Vec::new();
726 let mut real_off: u32 = 0;
727 let mut escaped = false;
728 let mut terminated = false;
729 while let Some(ch) = self.peek() {
730 if escaped {
731 let at = (real_off, self.col);
732 match ch {
733 'n' => {
734 s.push('\n');
735 pos.push(at);
736 }
737 '\n' => {
738 // Backslash followed by a physical
739 // newline: an unknown escape, so both
740 // characters are kept — and the
741 // physical line still advances.
742 s.push('\\');
743 pos.push(at);
744 s.push('\n');
745 pos.push((real_off, self.col + 1));
746 real_off += 1;
747 }
748 't' => {
749 s.push('\t');
750 pos.push(at);
751 }
752 'r' => {
753 s.push('\r');
754 pos.push(at);
755 }
756 '\\' => {
757 s.push('\\');
758 pos.push(at);
759 }
760 '"' => {
761 s.push('"');
762 pos.push(at);
763 }
764 '0' => {
765 s.push('\0');
766 pos.push(at);
767 }
768 // `\$` keeps its own identity through the
769 // escape table so the parser can tell it
770 // apart from a real backslash before `${`.
771 '$' => {
772 s.push(ESCAPED_DOLLAR);
773 pos.push(at);
774 }
775 other => {
776 s.push('\\');
777 pos.push(at);
778 s.push(other);
779 pos.push((real_off, self.col + 1));
780 }
781 }
782 escaped = false;
783 self.bump();
784 } else if ch == '\\' {
785 escaped = true;
786 self.bump();
787 } else if ch == '"' {
788 self.bump();
789 terminated = true;
790 break;
791 } else if ch == '\r' {
792 // CRLF normalization (research HIST-YAML-014):
793 // `\r\n` contributes one newline; a lone CR
794 // becomes LF. Explicit `\r` escapes decode
795 // elsewhere and never reach this branch.
796 if self.peek_ahead(1) == Some('\n') {
797 self.bump();
798 continue;
799 }
800 pos.push((real_off, self.col));
801 s.push('\n');
802 self.bump();
803 } else {
804 if ch == '\n' {
805 pos.push((real_off, self.col));
806 real_off += 1;
807 } else {
808 pos.push((real_off, self.col));
809 }
810 s.push(ch);
811 self.bump();
812 }
813 }
814 if !terminated {
815 // A dangling escape (string ended inside `\`) has
816 // nothing to escape: preserve the backslash so the
817 // content is not silently altered.
818 if escaped {
819 s.push('\\');
820 pos.push((real_off, self.col));
821 }
822 // Healing phase 1: insert the missing closing quote.
823 repairs.push(unclosed_string_repair(l, col, "\""));
824 }
825 tokens.push(self.make_token(TokenKind::String(pos), s, l, col));
826 }
827
828 // Single-quoted strings (no interpolation semantics)
829 '\'' => {
830 let (l, col) = (self.line, self.col);
831 self.bump();
832 let mut s = String::new();
833 let mut escaped = false;
834 let mut terminated = false;
835 while let Some(ch) = self.peek() {
836 if escaped {
837 match ch {
838 '\\' => s.push('\\'),
839 '\'' => s.push('\''),
840 other => {
841 s.push('\\');
842 s.push(other);
843 }
844 }
845 escaped = false;
846 self.bump();
847 } else if ch == '\\' {
848 escaped = true;
849 self.bump();
850 } else if ch == '\'' {
851 self.bump();
852 terminated = true;
853 break;
854 } else if ch == '\r' {
855 // Same CRLF normalization as double-quoted.
856 if self.peek_ahead(1) == Some('\n') {
857 self.bump();
858 continue;
859 }
860 s.push('\n');
861 self.bump();
862 } else {
863 s.push(ch);
864 self.bump();
865 }
866 }
867 if !terminated {
868 // A dangling escape has nothing to escape; preserve
869 // the backslash so the content is not altered.
870 if escaped {
871 s.push('\\');
872 }
873 // Healing phase 1: insert the missing closing quote.
874 repairs.push(unclosed_string_repair(l, col, "'"));
875 }
876 tokens.push(self.make_token(TokenKind::SingleString, s, l, col));
877 }
878
879 // Numbers + optional unit
880 c if c.is_ascii_digit() => {
881 let (l, col) = (self.line, self.col);
882 let mut s = String::new();
883 let mut is_float = false;
884
885 while let Some(ch) = self.peek() {
886 if ch.is_ascii_digit() || ch == '_' {
887 s.push(ch);
888 self.bump();
889 } else if ch == '.' && !is_float {
890 if let Some(next) = self.peek_ahead(1) {
891 if next.is_ascii_digit() {
892 is_float = true;
893 s.push(ch);
894 self.bump();
895 continue;
896 }
897 }
898 break;
899 } else {
900 break;
901 }
902 }
903
904 let cleaned: String = s.chars().filter(|c| *c != '_').collect();
905 let kind = if is_float {
906 TokenKind::Float
907 } else {
908 TokenKind::Integer
909 };
910 tokens.push(self.make_token(kind, cleaned, l, col));
911
912 // Unit must be attached with no intervening space
913 if let Some(u) = self.try_consume_unit() {
914 let (ul, uc) = (self.line, self.col);
915 tokens.push(self.make_token(TokenKind::Unit(u.clone()), u, ul, uc));
916 }
917 }
918
919 // Identifiers & keywords
920 c if c.is_ascii_alphabetic() || c == '_' => {
921 let (l, col) = (self.line, self.col);
922 let mut s = String::new();
923 while let Some(ch) = self.peek() {
924 if ch.is_ascii_alphanumeric() || ch == '_' {
925 s.push(ch);
926 self.bump();
927 } else {
928 break;
929 }
930 }
931 let kind = match s.as_str() {
932 "let" => TokenKind::Let,
933 "true" => TokenKind::True,
934 "false" => TokenKind::False,
935 "null" => TokenKind::Null,
936 "env" => TokenKind::Env,
937 "secret" => TokenKind::Secret,
938 _ => TokenKind::Ident,
939 };
940 tokens.push(self.make_token(kind, s, l, col));
941 }
942
943 // Unknown character — a hard error in every mode: no rule
944 // accepts it, and skipping it would silently change the
945 // configuration. Reported with a precise span.
946 _ => {
947 let (l, col) = (self.line, self.col);
948 self.bump();
949 errors.push(Diagnostic {
950 severity: Severity::Error,
951 code: "E005".into(),
952 message: format!("unexpected character {:?}", c),
953 path: None,
954 span: Some(Span {
955 line: l,
956 column: col,
957 end_line: l,
958 end_column: col + 1,
959 }),
960 repair: None,
961 confidence: 0.0,
962 // The parser stamps the session file.
963 file: None,
964 });
965 }
966 }
967 }
968
969 // End of input closes every open indentation block
970 // (spec/dual-syntax.md §4.2).
971 while self.indent_stack.len() > 1 {
972 self.indent_stack.pop();
973 let target = self.indent_stack.last().copied().unwrap_or(0);
974 tokens.push(self.make_token(
975 TokenKind::Dedent,
976 target.to_string(),
977 self.line,
978 self.col,
979 ));
980 }
981
982 tokens.push(self.make_token(TokenKind::Eof, String::new(), self.line, self.col));
983 Lexed {
984 tokens,
985 repairs,
986 errors,
987 comments,
988 }
989 }
990
991 /// Consume a known unit suffix if present.
992 fn try_consume_unit(&mut self) -> Option<String> {
993 let start_pos = self.pos;
994 let start_line = self.line;
995 let start_col = self.col;
996
997 let mut s = String::new();
998 while let Some(ch) = self.peek() {
999 // Unicode-aware so the documented `µs` microsecond suffix is
1000 // collectable (the whitelist carries it).
1001 if ch.is_alphanumeric() || ch == '/' || ch == '_' || ch == '%' {
1002 s.push(ch);
1003 self.bump();
1004 } else {
1005 break;
1006 }
1007 }
1008
1009 if s.is_empty() {
1010 return None;
1011 }
1012
1013 const KNOWN: &[&str] = &[
1014 "ns", "us", "µs", "ms", "s", "m", "h", "d", "B", "KiB", "MiB", "GiB", "TiB", "KB",
1015 "MB", "GB", "TB", "/s", "/m", "/h", "per_s", "per_m", "per_h", "req/s", "req_s",
1016 "/req/s", "/req/m", "/req/h", "req/m", "req_m", "req/h", "req_h", "%", "Hz", "kHz",
1017 "MHz", "GHz", "THz", "b", "Kb", "Mb", "Gb", "bps", "Kbps", "Mbps", "Gbps", "MB/s",
1018 "week", "sec", "min",
1019 ];
1020
1021 if KNOWN.contains(&s.as_str()) {
1022 Some(s)
1023 } else {
1024 // rollback
1025 self.pos = start_pos;
1026 self.line = start_line;
1027 self.col = start_col;
1028 None
1029 }
1030 }
1031}