aam_core/pipeline/
lexer.rs

1//! Lexer stage: tokenizes raw AAML text into a stream of tokens.
2//!
3//! The Lexer scans through raw text and produces a `Vec<Token>` with positional
4//! information preserved for error diagnostics.
5
6use crate::error::{AamlError, ErrorDiagnostics};
7
8/// A single token produced by the Lexer.
9///
10/// Each token carries its line and column number for error reporting and its
11/// byte span inside the original source.
12#[derive(Debug, Clone, PartialEq, Eq)]
13pub struct Token<'a> {
14    pub kind: TokenKind,
15    pub line: usize,
16    pub column: usize,
17    pub text: std::borrow::Cow<'a, str>,
18    pub start: usize,
19    pub end: usize,
20    pub source: &'a str,
21}
22
23impl<'a> Token<'a> {
24    pub fn new(
25        kind: TokenKind,
26        line: usize,
27        column: usize,
28        text: impl Into<std::borrow::Cow<'a, str>>,
29        start: usize,
30        end: usize,
31        source: &'a str,
32    ) -> Self {
33        Self {
34            kind,
35            line,
36            column,
37            text: text.into(),
38            start,
39            end,
40            source,
41        }
42    }
43}
44
45/// The type of token.
46#[derive(Debug, Clone, PartialEq, Eq)]
47pub enum TokenKind {
48    /// Identifier or unquoted value (e.g., `host`, `localhost`)
49    Identifier,
50    /// The `=` operator in assignments
51    Assign,
52    /// String literal (quoted with `"` or `'`)
53    String,
54    /// Number literal (integer or float)
55    Number,
56    /// Boolean literal (`true` or `false`)
57    Boolean,
58    /// Opening brace `{`
59    LeftBrace,
60    /// Closing brace `}`
61    RightBrace,
62    /// Opening bracket `[`
63    LeftBracket,
64    /// Closing bracket `]`
65    RightBracket,
66    /// Comma separator `,`
67    Comma,
68    /// The `@` directive prefix
69    At,
70    /// End of line / newline
71    Newline,
72    /// Comment (including the `#`)
73    Comment,
74}
75
76/// Trait for lexical analysis stage.
77pub trait Lexer: Send + Sync {
78    /// Tokenizes raw AAML content and returns a stream of tokens with line/column info.
79    ///
80    /// # Errors
81    /// Returns `AamlError::LexError` if the input contains invalid tokens or
82    /// unclosed delimiters.
83    fn tokenize<'a>(&self, content: &'a str) -> Result<Vec<Token<'a>>, AamlError>;
84}
85
86/// Default implementation of the Lexer stage.
87pub struct DefaultLexer;
88
89impl DefaultLexer {
90    #[must_use]
91    pub const fn new() -> Self {
92        Self
93    }
94
95    /// Checks if a character is whitespace (excluding newlines)
96    const fn is_whitespace(c: char) -> bool {
97        c == ' ' || c == '\t' || c == '\r'
98    }
99
100    /// Checks if a character can start an identifier
101    fn is_id_start(c: char) -> bool {
102        c.is_alphabetic() || c == '_' || c == '@' || c == '#' || c == '/'
103    }
104
105    /// Checks if a character can continue an identifier
106    fn is_id_cont(c: char) -> bool {
107        c.is_alphanumeric()
108            || c == '_'
109            || c == ':'
110            || c == '.'
111            || c == '*'
112            || c == '#'
113            || c == '-'
114            || c == '/'
115            || c == '<'
116            || c == '>'
117    }
118
119    // Keep legacy quirk: `#` starts a comment only when followed by whitespace.
120    fn is_comment_start(chars: &std::iter::Peekable<std::str::Chars>) -> bool {
121        chars.clone().nth(1).is_some_and(char::is_whitespace)
122    }
123
124    /// Checks if a character is a digit
125    const fn is_digit(c: char) -> bool {
126        c.is_ascii_digit()
127    }
128
129    /// Checks if a character can be part of a number
130    const fn is_number_part(c: char) -> bool {
131        c.is_ascii_digit() || c == '.' || c == '-' || c == 'e' || c == 'E'
132    }
133}
134
135impl Default for DefaultLexer {
136    fn default() -> Self {
137        Self::new()
138    }
139}
140
141impl Lexer for DefaultLexer {
142    #[allow(clippy::too_many_lines)]
143    fn tokenize<'a>(&self, content: &'a str) -> Result<Vec<Token<'a>>, AamlError> {
144        let mut tokens = Vec::new();
145        let mut line = 1;
146        let mut column = 1;
147        let mut byte_offset = 0usize;
148        let mut chars = content.chars().peekable();
149
150        while let Some(&ch) = chars.peek() {
151            match ch {
152                '\n' => {
153                    Self::handle_newline(
154                        &mut tokens,
155                        &mut chars,
156                        &mut line,
157                        &mut column,
158                        &mut byte_offset,
159                        content,
160                    );
161                }
162                c if Self::is_whitespace(c) => {
163                    chars.next();
164                    column += 1;
165                    byte_offset += c.len_utf8();
166                }
167                '#' => {
168                    if Self::is_comment_start(&chars) {
169                        Self::handle_comment(
170                            &mut tokens,
171                            &mut chars,
172                            line,
173                            &mut column,
174                            &mut byte_offset,
175                            content,
176                        );
177                    } else {
178                        Self::handle_identifier(
179                            &mut tokens,
180                            &mut chars,
181                            line,
182                            &mut column,
183                            &mut byte_offset,
184                            content,
185                        );
186                    }
187                }
188                '=' => Self::push_single_token(
189                    &mut tokens,
190                    TokenKind::Assign,
191                    line,
192                    column,
193                    "=",
194                    &mut chars,
195                    &mut column,
196                    &mut byte_offset,
197                    content,
198                ),
199                '{' => Self::push_single_token(
200                    &mut tokens,
201                    TokenKind::LeftBrace,
202                    line,
203                    column,
204                    "{",
205                    &mut chars,
206                    &mut column,
207                    &mut byte_offset,
208                    content,
209                ),
210                '}' => Self::push_single_token(
211                    &mut tokens,
212                    TokenKind::RightBrace,
213                    line,
214                    column,
215                    "}",
216                    &mut chars,
217                    &mut column,
218                    &mut byte_offset,
219                    content,
220                ),
221                '[' => Self::push_single_token(
222                    &mut tokens,
223                    TokenKind::LeftBracket,
224                    line,
225                    column,
226                    "[",
227                    &mut chars,
228                    &mut column,
229                    &mut byte_offset,
230                    content,
231                ),
232                ']' => Self::push_single_token(
233                    &mut tokens,
234                    TokenKind::RightBracket,
235                    line,
236                    column,
237                    "]",
238                    &mut chars,
239                    &mut column,
240                    &mut byte_offset,
241                    content,
242                ),
243                ',' => Self::push_single_token(
244                    &mut tokens,
245                    TokenKind::Comma,
246                    line,
247                    column,
248                    ",",
249                    &mut chars,
250                    &mut column,
251                    &mut byte_offset,
252                    content,
253                ),
254                '@' => Self::push_single_token(
255                    &mut tokens,
256                    TokenKind::At,
257                    line,
258                    column,
259                    "@",
260                    &mut chars,
261                    &mut column,
262                    &mut byte_offset,
263                    content,
264                ),
265                '"' | '\'' => {
266                    Self::handle_string(
267                        &mut tokens,
268                        &mut chars,
269                        ch,
270                        line,
271                        &mut column,
272                        &mut line,
273                        &mut byte_offset,
274                        content,
275                    );
276                }
277                _ if Self::is_digit(ch)
278                    || (ch == '-' && chars.clone().nth(1).is_some_and(Self::is_digit)) =>
279                {
280                    Self::handle_number(
281                        &mut tokens,
282                        &mut chars,
283                        ch,
284                        line,
285                        &mut column,
286                        &mut byte_offset,
287                        content,
288                    );
289                }
290                _ if Self::is_id_start(ch) => {
291                    Self::handle_identifier(
292                        &mut tokens,
293                        &mut chars,
294                        line,
295                        &mut column,
296                        &mut byte_offset,
297                        content,
298                    );
299                }
300                _ => {
301                    return Err(AamlError::LexError {
302                        line,
303                        column,
304                        character: ch.to_string(),
305                        diagnostics: Some(Box::new(ErrorDiagnostics::new(
306                            "Invalid character in input",
307                            format!("Unexpected character '{ch}' at {line}:{column}"),
308                            "Check for typos or unsupported characters",
309                        ))),
310                    });
311                }
312            }
313        }
314
315        // Add final newline if not present
316        if tokens.is_empty() || tokens.last().is_none_or(|t| t.kind != TokenKind::Newline) {
317            tokens.push(Token::new(
318                TokenKind::Newline,
319                line,
320                column,
321                "\n".to_string(),
322                byte_offset,
323                byte_offset + 1,
324                content,
325            ));
326        }
327
328        Ok(tokens)
329    }
330}
331
332impl DefaultLexer {
333    fn handle_newline<'a>(
334        tokens: &mut Vec<Token<'a>>,
335        chars: &mut std::iter::Peekable<std::str::Chars>,
336        line: &mut usize,
337        column: &mut usize,
338        byte_offset: &mut usize,
339        source: &'a str,
340    ) {
341        let start = *byte_offset;
342        tokens.push(Token::new(
343            TokenKind::Newline,
344            *line,
345            *column,
346            "\n".to_string(),
347            start,
348            start + 1,
349            source,
350        ));
351        chars.next();
352        *line += 1;
353        *column = 1;
354        *byte_offset += 1;
355    }
356
357    fn handle_comment<'a>(
358        tokens: &mut Vec<Token<'a>>,
359        chars: &mut std::iter::Peekable<std::str::Chars>,
360        line: usize,
361        column: &mut usize,
362        byte_offset: &mut usize,
363        source: &'a str,
364    ) {
365        let col = *column;
366        let start = *byte_offset;
367        let mut text = String::new();
368        while let Some(&c) = chars.peek() {
369            if c == '\n' {
370                break;
371            }
372            text.push(c);
373            chars.next();
374            *column += 1;
375            *byte_offset += c.len_utf8();
376        }
377        tokens.push(Token::new(
378            TokenKind::Comment,
379            line,
380            col,
381            text,
382            start,
383            *byte_offset,
384            source,
385        ));
386    }
387
388    #[allow(clippy::too_many_arguments)]
389    fn push_single_token<'a>(
390        tokens: &mut Vec<Token<'a>>,
391        kind: TokenKind,
392        line: usize,
393        column: usize,
394        text: &str,
395        chars: &mut std::iter::Peekable<std::str::Chars>,
396        col_ref: &mut usize,
397        byte_offset: &mut usize,
398        source: &'a str,
399    ) {
400        let start = *byte_offset;
401        let end = start + text.len();
402        tokens.push(Token::new(
403            kind,
404            line,
405            column,
406            text.to_string(),
407            start,
408            end,
409            source,
410        ));
411        chars.next();
412        *col_ref += 1;
413        *byte_offset = end;
414    }
415
416    const fn update_string_scan_state(
417        c: char,
418        quote: char,
419        escaped: &mut bool,
420        line: &mut usize,
421        column: &mut usize,
422    ) -> bool {
423        if *escaped {
424            *escaped = false;
425            return false;
426        }
427
428        if c == '\\' {
429            *escaped = true;
430            return false;
431        }
432
433        if c == quote {
434            return true;
435        }
436
437        if c == '\n' {
438            *line += 1;
439            *column = 1;
440        }
441
442        false
443    }
444
445    #[allow(clippy::too_many_arguments)]
446    fn handle_string<'a>(
447        tokens: &mut Vec<Token<'a>>,
448        chars: &mut std::iter::Peekable<std::str::Chars>,
449        quote: char,
450        mut line: usize,
451        column: &mut usize,
452        line_ref: &mut usize,
453        byte_offset: &mut usize,
454        source: &'a str,
455    ) {
456        let col = *column;
457        let start = *byte_offset;
458        chars.next();
459        *column += 1;
460        *byte_offset += quote.len_utf8();
461        let mut text = String::from(quote);
462        let mut escaped = false;
463
464        while let Some(&c) = chars.peek() {
465            text.push(c);
466            chars.next();
467            *column += 1;
468            *byte_offset += c.len_utf8();
469
470            if Self::update_string_scan_state(c, quote, &mut escaped, &mut line, column) {
471                break;
472            }
473        }
474
475        tokens.push(Token::new(
476            TokenKind::String,
477            line,
478            col,
479            text,
480            start,
481            *byte_offset,
482            source,
483        ));
484        *line_ref = line;
485    }
486
487    fn handle_number<'a>(
488        tokens: &mut Vec<Token<'a>>,
489        chars: &mut std::iter::Peekable<std::str::Chars>,
490        first_ch: char,
491        line: usize,
492        column: &mut usize,
493        byte_offset: &mut usize,
494        source: &'a str,
495    ) {
496        let col = *column;
497        let start = *byte_offset;
498        let mut text = String::new();
499
500        if first_ch == '-' {
501            text.push('-');
502            chars.next();
503            *column += 1;
504            *byte_offset += '-'.len_utf8();
505        }
506
507        while let Some(&c) = chars.peek() {
508            if Self::is_number_part(c) {
509                text.push(c);
510                chars.next();
511                *column += 1;
512                *byte_offset += c.len_utf8();
513            } else {
514                break;
515            }
516        }
517
518        let kind = if text == "true" || text == "false" {
519            TokenKind::Boolean
520        } else {
521            TokenKind::Number
522        };
523
524        tokens.push(Token::new(
525            kind,
526            line,
527            col,
528            text,
529            start,
530            *byte_offset,
531            source,
532        ));
533    }
534
535    fn handle_identifier<'a>(
536        tokens: &mut Vec<Token<'a>>,
537        chars: &mut std::iter::Peekable<std::str::Chars>,
538        line: usize,
539        column: &mut usize,
540        byte_offset: &mut usize,
541        source: &'a str,
542    ) {
543        let col = *column;
544        let start = *byte_offset;
545        let mut text = String::new();
546
547        while let Some(&c) = chars.peek() {
548            if Self::is_id_cont(c) {
549                text.push(c);
550                chars.next();
551                *column += 1;
552                *byte_offset += c.len_utf8();
553            } else {
554                break;
555            }
556        }
557
558        let kind = match text.as_str() {
559            "true" | "false" => TokenKind::Boolean,
560            _ => TokenKind::Identifier,
561        };
562
563        tokens.push(Token::new(
564            kind,
565            line,
566            col,
567            text,
568            start,
569            *byte_offset,
570            source,
571        ));
572    }
573}
574
575#[cfg(test)]
576mod tests {
577    use super::*;
578
579    #[test]
580    fn test_simple_assignment() {
581        let lexer = DefaultLexer::new();
582        let tokens = lexer.tokenize("host = localhost").unwrap();
583
584        assert_eq!(tokens.len(), 4); // host, =, localhost, newline
585        assert_eq!(tokens[0].kind, TokenKind::Identifier);
586        assert_eq!(tokens[1].kind, TokenKind::Assign);
587        assert_eq!(tokens[2].kind, TokenKind::Identifier);
588        assert_eq!(tokens[3].kind, TokenKind::Newline);
589    }
590
591    #[test]
592    fn test_quoted_string() {
593        let lexer = DefaultLexer::new();
594        let tokens = lexer.tokenize("name = \"John Doe\"").unwrap();
595
596        assert!(tokens.iter().any(|t| t.kind == TokenKind::String));
597    }
598
599    #[test]
600    fn test_number_literal() {
601        let lexer = DefaultLexer::new();
602        let tokens = lexer.tokenize("port = 8080").unwrap();
603
604        assert!(tokens.iter().any(|t| t.kind == TokenKind::Number));
605    }
606
607    #[test]
608    fn test_boolean_literal() {
609        let lexer = DefaultLexer::new();
610        let tokens = lexer.tokenize("enabled = true").unwrap();
611
612        assert!(tokens.iter().any(|t| t.kind == TokenKind::Boolean));
613    }
614
615    #[test]
616    fn test_braces_and_brackets() {
617        let lexer = DefaultLexer::new();
618        let tokens = lexer.tokenize("obj = { key = val }").unwrap();
619
620        assert!(tokens.iter().any(|t| t.kind == TokenKind::LeftBrace));
621        assert!(tokens.iter().any(|t| t.kind == TokenKind::RightBrace));
622    }
623
624    #[test]
625    fn test_directive() {
626        let lexer = DefaultLexer::new();
627        let tokens = lexer.tokenize("@import base.aam").unwrap();
628
629        assert_eq!(tokens[0].kind, TokenKind::At);
630        assert_eq!(tokens[1].kind, TokenKind::Identifier);
631    }
632}
aam_core/pipeline/lexer.rs

aam_core/pipeline/
lexer.rs
