o7 0.1.0

//! O7 DSL Lexer
//!
//! Produces a flat token array from source text, including synthetic
//! INDENT/DEDENT tokens based on indentation changes.

use crate::parser::errors::ParseError;
use crate::parser::tokens::{Token, TokenType};

/// Result of lexing a source file.
pub struct LexResult {
    pub tokens: Vec<Token>,
    pub errors: Vec<ParseError>,
}

/// Scanner state: wraps source string and provides peek/advance/is_at_end helpers.
struct Scanner {
    source: Vec<char>,
    pos: usize,
    line: usize,
    column: usize,
}

impl Scanner {
    fn new(source: &str) -> Self {
        Scanner {
            source: source.chars().collect(),
            pos: 0,
            line: 1,
            column: 1,
        }
    }

    /// Look at the current character without consuming it.
    fn peek(&self) -> char {
        self.source.get(self.pos).copied().unwrap_or('\0')
    }

    /// Look ahead by n characters without consuming.
    fn peek_at(&self, n: usize) -> char {
        self.source.get(self.pos + n).copied().unwrap_or('\0')
    }

    /// Consume and return the current character, advancing position.
    fn advance(&mut self) -> char {
        let ch = self.peek();
        self.pos += 1;
        if ch == '\n' {
            self.line += 1;
            self.column = 1;
        } else {
            self.column += 1;
        }
        ch
    }

    /// Whether we have reached the end of the source.
    fn is_at_end(&self) -> bool {
        self.pos >= self.source.len()
    }
}

/// Manages the indentation stack and emits INDENT/DEDENT tokens.
struct IndentTracker {
    /// Stack of indent widths; starts with 0 (column 0).
    stack: Vec<usize>,
    /// The indent width unit once established (0 = not yet established).
    unit_width: usize,
}

impl IndentTracker {
    fn new() -> Self {
        IndentTracker {
            stack: vec![0],
            unit_width: 0,
        }
    }

    /// Get the current indent level (top of stack).
    fn current_indent(&self) -> usize {
        *self.stack.last().unwrap()
    }

    /// Process a new line's indentation width. Returns INDENT/DEDENT tokens
    /// to emit, or an error if the indentation is invalid.
    fn process(
        &mut self,
        width: usize,
        line: usize,
        filename: &str,
    ) -> (Vec<Token>, Option<ParseError>) {
        let current = self.current_indent();
        let mut tokens = Vec::new();

        if width > current {
            // New indent level
            if self.unit_width == 0 {
                // First indent: establish the unit width
                self.unit_width = width - current;
            } else {
                // Subsequent indent: must be exactly one unit deeper
                let expected = current + self.unit_width;
                if width != expected {
                    return (
                        vec![],
                        Some(ParseError::new(
                            filename, line, width + 1,
                            format!(
                                "Inconsistent indentation: expected {} spaces but got {}",
                                expected, width
                            ),
                        )),
                    );
                }
            }
            self.stack.push(width);
            tokens.push(Token::synthetic(TokenType::Indent, line, 1));
        } else if width < current {
            // Dedent: pop until we find a matching level
            while self.stack.len() > 1 && self.current_indent() > width {
                self.stack.pop();
                tokens.push(Token::synthetic(TokenType::Dedent, line, 1));
            }
            if self.current_indent() != width {
                return (
                    tokens,
                    Some(ParseError::new(
                        filename, line, width + 1,
                        format!(
                            "Indentation does not match any outer level (got {} spaces)",
                            width
                        ),
                    )),
                );
            }
        }
        // If width == current, no indent/dedent tokens needed.

        (tokens, None)
    }

    /// At end of file, emit DEDENT tokens for any remaining open indentation levels.
    fn flush(&mut self, line: usize) -> Vec<Token> {
        let mut tokens = Vec::new();
        while self.stack.len() > 1 {
            self.stack.pop();
            tokens.push(Token::synthetic(TokenType::Dedent, line, 1));
        }
        tokens
    }
}

/// Whether a character can start an identifier.
fn is_ident_start(ch: char) -> bool {
    ch.is_ascii_alphabetic()
}

/// Whether a character can continue an identifier.
fn is_ident_part(ch: char) -> bool {
    ch.is_ascii_alphanumeric() || ch == '_' || ch == '-'
}

/// Lex a word (keyword or name). Handles:
/// - Simple identifiers: `main`, `run-impl`
/// - Namespaced names: `reviews::security`
/// - Keywords: `version`, `par-and`, `fail-policy`, `prompt_file`
fn lex_word(scanner: &mut Scanner) -> Token {
    let start_line = scanner.line;
    let start_col = scanner.column;
    let mut word = String::new();

    // Read first identifier segment
    while !scanner.is_at_end() && is_ident_part(scanner.peek()) {
        word.push(scanner.advance());
    }

    // Check for `::` namespace separator -- keep consuming segments
    while !scanner.is_at_end() && scanner.peek() == ':' && scanner.peek_at(1) == ':' {
        word.push(scanner.advance()); // first ':'
        word.push(scanner.advance()); // second ':'
        // Read next segment
        while !scanner.is_at_end() && is_ident_part(scanner.peek()) {
            word.push(scanner.advance());
        }
    }

    // Check if the word is a keyword
    let ty = match word.as_str() {
        "version" => TokenType::Version,
        "workflow" => TokenType::Workflow,
        "run" => TokenType::Run,
        "if" => TokenType::If,
        "not" => TokenType::Not,
        "while" => TokenType::While,
        "par-and" => TokenType::ParAnd,
        "exec" => TokenType::Exec,
        "harness" => TokenType::Harness,
        "prompt_file" => TokenType::PromptFile,
        "prompt" => TokenType::Prompt,
        "args" => TokenType::Args,
        "fail-policy" => TokenType::FailPolicy,
        "match" => TokenType::Match,
        "else" => TokenType::Else,
        _ => TokenType::Name,
    };

    Token::new(ty, word, start_line, start_col)
}

/// Lex a number literal.
fn lex_number(scanner: &mut Scanner) -> Token {
    let start_line = scanner.line;
    let start_col = scanner.column;
    let mut num = String::new();

    while !scanner.is_at_end() && scanner.peek().is_ascii_digit() {
        num.push(scanner.advance());
    }

    Token::new(TokenType::Number, num, start_line, start_col)
}

/// Lex a double-quoted string literal. Returns a token and/or an error.
fn lex_string(scanner: &mut Scanner, filename: &str) -> (Option<Token>, Option<ParseError>) {
    let start_line = scanner.line;
    let start_col = scanner.column;
    scanner.advance(); // consume opening '"'

    let mut value = String::new();
    while !scanner.is_at_end() {
        let ch = scanner.peek();

        if ch == '\n' {
            // Unterminated string at end of line
            return (
                None,
                Some(ParseError::new(
                    filename, start_line, start_col,
                    "Unterminated string literal",
                )),
            );
        }

        if ch == '\\' {
            scanner.advance(); // consume backslash
            if scanner.is_at_end() {
                return (
                    None,
                    Some(ParseError::new(
                        filename, start_line, start_col,
                        "Unterminated string literal",
                    )),
                );
            }
            let escaped = scanner.advance();
            match escaped {
                'n' => value.push('\n'),
                't' => value.push('\t'),
                '\\' => value.push('\\'),
                '"' => value.push('"'),
                other => {
                    value.push('\\');
                    value.push(other);
                }
            }
            continue;
        }

        if ch == '"' {
            scanner.advance(); // consume closing '"'
            return (
                Some(Token::new(TokenType::Str, value, start_line, start_col)),
                None,
            );
        }

        value.push(scanner.advance());
    }

    // Reached end of file without closing quote
    (
        None,
        Some(ParseError::new(
            filename, start_line, start_col,
            "Unterminated string literal",
        )),
    )
}

/// Skip to end of line (consume everything up to and including the newline).
fn skip_to_end_of_line(scanner: &mut Scanner) {
    while !scanner.is_at_end() && scanner.peek() != '\n' {
        scanner.advance();
    }
    if !scanner.is_at_end() {
        scanner.advance(); // consume the newline
    }
}

/// Lex a source string into tokens.
///
/// Processes input character by character, tracking indentation and emitting
/// synthetic INDENT/DEDENT/NEWLINE/EOF tokens.
pub fn lex(source: &str, filename: &str) -> LexResult {
    let mut scanner = Scanner::new(source);
    let mut indent = IndentTracker::new();
    let mut tokens: Vec<Token> = Vec::new();
    let mut errors: Vec<ParseError> = Vec::new();

    // Whether we are at the start of a line (need to check indentation).
    let mut at_line_start = true;
    // Whether we have emitted any content token yet.
    let mut has_emitted_content = false;

    while !scanner.is_at_end() {
        if at_line_start {
            // Measure indentation at the start of a line
            let line_start_line = scanner.line;
            let mut indent_width: usize = 0;
            let mut has_tabs = false;
            let mut has_mixed_tabs_and_spaces = false;

            while !scanner.is_at_end()
                && (scanner.peek() == ' ' || scanner.peek() == '\t')
            {
                let ch = scanner.advance();
                if ch == '\t' {
                    if indent_width > 0 && !has_tabs {
                        has_mixed_tabs_and_spaces = true;
                    }
                    has_tabs = true;
                } else if has_tabs {
                    has_mixed_tabs_and_spaces = true;
                }
                indent_width += 1;
            }

            // Check for tab errors
            if has_mixed_tabs_and_spaces {
                errors.push(ParseError::new(
                    filename, line_start_line, 1,
                    "Mixed tabs and spaces in indentation",
                ));
                skip_to_end_of_line(&mut scanner);
                at_line_start = true;
                continue;
            }
            if has_tabs {
                errors.push(ParseError::new(
                    filename, line_start_line, 1,
                    "Tabs are not allowed for indentation; use spaces",
                ));
                skip_to_end_of_line(&mut scanner);
                at_line_start = true;
                continue;
            }

            // Skip \r at line start (after indentation)
            if !scanner.is_at_end() && scanner.peek() == '\r' {
                scanner.advance();
            }

            // If the rest of the line is blank or a comment, skip it entirely
            // (blank lines and comment-only lines do not affect indentation)
            if scanner.is_at_end() || scanner.peek() == '\n' {
                if !scanner.is_at_end() {
                    scanner.advance(); // consume the newline
                }
                at_line_start = true;
                continue;
            }
            if scanner.peek() == '#' {
                // Comment-only line: skip to end of line
                skip_to_end_of_line(&mut scanner);
                at_line_start = true;
                continue;
            }

            // Non-blank, non-comment line: process indentation
            if has_emitted_content || indent_width > 0 {
                let (indent_tokens, error) =
                    indent.process(indent_width, line_start_line, filename);
                tokens.extend(indent_tokens);
                if let Some(err) = error {
                    errors.push(err);
                    skip_to_end_of_line(&mut scanner);
                    at_line_start = true;
                    continue;
                }
            }

            at_line_start = false;
        }

        let ch = scanner.peek();

        // Skip spaces within a line (not at line start - that's handled above)
        if ch == ' ' {
            scanner.advance();
            continue;
        }

        // Carriage return: skip it (handle CRLF by treating \r as whitespace)
        if ch == '\r' {
            scanner.advance();
            continue;
        }

        // Newline
        if ch == '\n' {
            if has_emitted_content {
                tokens.push(Token::new(TokenType::Newline, "\n", scanner.line, scanner.column));
            }
            scanner.advance();
            at_line_start = true;
            continue;
        }

        // Comment: skip to end of line (but don't consume the newline)
        if ch == '#' {
            while !scanner.is_at_end() && scanner.peek() != '\n' {
                scanner.advance();
            }
            continue;
        }

        // Colon
        if ch == ':' {
            tokens.push(Token::new(TokenType::Colon, ":", scanner.line, scanner.column));
            scanner.advance();
            has_emitted_content = true;

            // After a colon, check if the rest of the line is a bare value
            // (contains characters that aren't valid identifiers, like / or .)
            // Skip leading spaces first, using lookahead without consuming
            let mut temp_pos = scanner.pos;
            while temp_pos < scanner.source.len() && scanner.source[temp_pos] == ' ' {
                temp_pos += 1;
            }
            // Check if the next non-space char starts a bare value
            let next_ch = scanner
                .source
                .get(temp_pos)
                .copied()
                .unwrap_or('\0');
            if next_ch != '\0'
                && next_ch != '\n'
                && next_ch != '\r'
                && next_ch != '"'
                && next_ch != '#'
            {
                // Scan ahead to see if this line contains non-identifier characters
                let mut has_bare_chars = false;
                let mut scan_pos = temp_pos;
                while scan_pos < scanner.source.len()
                    && scanner.source[scan_pos] != '\n'
                    && scanner.source[scan_pos] != '\r'
                    && scanner.source[scan_pos] != '#'
                {
                    let c = scanner.source[scan_pos];
                    if c != ' '
                        && c != '\t'
                        && !is_ident_part(c)
                        && !c.is_ascii_digit()
                        && c != ':'
                    {
                        has_bare_chars = true;
                        break;
                    }
                    scan_pos += 1;
                }
                if has_bare_chars {
                    // Consume spaces
                    while !scanner.is_at_end() && scanner.peek() == ' ' {
                        scanner.advance();
                    }
                    // Lex a bare value: everything until newline, \r, or # (trimmed)
                    let value_line = scanner.line;
                    let value_col = scanner.column;
                    let mut bare_value = String::new();
                    while !scanner.is_at_end()
                        && scanner.peek() != '\n'
                        && scanner.peek() != '\r'
                        && scanner.peek() != '#'
                    {
                        bare_value.push(scanner.advance());
                    }
                    let trimmed = bare_value.trim_end().to_string();
                    if !trimmed.is_empty() {
                        tokens.push(Token::new(TokenType::BareValue, trimmed, value_line, value_col));
                    }
                    continue;
                }
            }
            continue;
        }

        // String literal
        if ch == '"' {
            let (token, error) = lex_string(&mut scanner, filename);
            if let Some(err) = error {
                errors.push(err);
            }
            if let Some(tok) = token {
                tokens.push(tok);
                has_emitted_content = true;
            }
            continue;
        }

        // Number
        if ch.is_ascii_digit() {
            let num_token = lex_number(&mut scanner);
            tokens.push(num_token);
            has_emitted_content = true;
            continue;
        }

        // Arrow token: `->`
        if ch == '-' {
            if scanner.peek_at(1) == '>' {
                let line = scanner.line;
                let col = scanner.column;
                scanner.advance(); // consume '-'
                scanner.advance(); // consume '>'
                tokens.push(Token::new(TokenType::Arrow, "->", line, col));
                has_emitted_content = true;
                continue;
            } else {
                errors.push(ParseError::new(
                    filename, scanner.line, scanner.column,
                    format!("Unexpected character: '{}'", ch),
                ));
                scanner.advance();
                continue;
            }
        }

        // Bare `>` — give a helpful diagnostic
        if ch == '>' {
            errors.push(ParseError::new(
                filename, scanner.line, scanner.column,
                "Unexpected '>' — did you mean '->'? Whitespace is required around '->'",
            ));
            scanner.advance();
            continue;
        }

        // Keywords and names
        if is_ident_start(ch) {
            let word_result = lex_word(&mut scanner);
            tokens.push(word_result);
            has_emitted_content = true;
            continue;
        }

        // Unknown character: report error and skip
        errors.push(ParseError::new(
            filename, scanner.line, scanner.column,
            format!("Unexpected character: '{}'", ch),
        ));
        scanner.advance();
    }

    // At end of file: flush remaining dedents
    let dedents = indent.flush(scanner.line);
    tokens.extend(dedents);

    // Emit EOF
    tokens.push(Token::synthetic(TokenType::Eof, scanner.line, scanner.column));

    LexResult { tokens, errors }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Lex source and assert no errors, returning the token list.
    fn lex_ok(source: &str) -> Vec<Token> {
        let result = lex(source, "test.7");
        assert!(result.errors.is_empty(), "errors: {:?}", result.errors);
        result.tokens
    }

    /// Find the first token of the given type.
    fn find_token<'a>(tokens: &'a [Token], ty: TokenType) -> &'a Token {
        tokens.iter().find(|t| t.ty == ty)
            .unwrap_or_else(|| panic!("token {:?} not found", ty))
    }

    #[test]
    fn test_lex_minimal_valid() {
        let result = lex("version 1\n\nworkflow main\n  run greet\n", "test.7");
        assert!(result.errors.is_empty(), "errors: {:?}", result.errors);
        let types: Vec<_> = result.tokens.iter().map(|t| &t.ty).collect();
        assert_eq!(
            types,
            &[
                &TokenType::Version,
                &TokenType::Number, // version 1
                &TokenType::Newline,
                &TokenType::Workflow,
                &TokenType::Name, // workflow main
                &TokenType::Newline,
                &TokenType::Indent,
                &TokenType::Run,
                &TokenType::Name, // run greet
                &TokenType::Newline,
                &TokenType::Dedent,
                &TokenType::Eof,
            ]
        );
    }

    #[test]
    fn test_lex_exec_block_with_bare_value() {
        let tokens = lex_ok(
            "version 1\nworkflow w\n  exec\n    harness: echo\n    prompt_file: prompts/foo.md\n",
        );
        assert_eq!(find_token(&tokens, TokenType::BareValue).value, "prompts/foo.md");
    }

    #[test]
    fn test_lex_tabs_rejected() {
        let result = lex("version 1\nworkflow main\n\trun greet\n", "test.7");
        assert!(!result.errors.is_empty());
        assert!(result.errors[0].message.contains("Tabs"));
    }

    #[test]
    fn test_lex_missing_version() {
        let tokens = lex_ok("workflow main\n");
        assert_eq!(tokens[0].ty, TokenType::Workflow);
    }

    #[test]
    fn test_lex_comment_lines_skipped() {
        let tokens = lex_ok("version 1\n# comment\nworkflow main\n");
        assert!(!tokens.iter().any(|t| t.value.contains('#')));
    }

    #[test]
    fn test_lex_string_with_escapes() {
        let tokens = lex_ok(
            "version 1\nworkflow w\n  exec\n    harness: h\n    prompt: \"hello \\\"world\\\"\\n\"\n",
        );
        assert_eq!(find_token(&tokens, TokenType::Str).value, "hello \"world\"\n");
    }

    #[test]
    fn test_lex_namespace_name() {
        let tokens = lex_ok("version 1\nworkflow reviews::security\n");
        assert_eq!(find_token(&tokens, TokenType::Name).value, "reviews::security");
    }

    #[test]
    fn test_lex_dedent_multiple_levels() {
        let tokens = lex_ok("version 1\nworkflow main\n  if check\n    run a\n  run b\n");
        let dedent_count = tokens.iter().filter(|t| t.ty == TokenType::Dedent).count();
        assert!(dedent_count >= 2, "expected >= 2 dedents, got {}", dedent_count);
    }

    #[test]
    fn test_lex_par_and_keyword() {
        let tokens = lex_ok("version 1\nworkflow w\n  par-and\n    run a\n    run b\n");
        assert_eq!(find_token(&tokens, TokenType::ParAnd).value, "par-and");
    }

    #[test]
    fn test_lex_fail_policy_keyword() {
        let tokens = lex_ok(
            "version 1\nworkflow w\n  exec\n    harness: h\n    fail-policy: abort\n",
        );
        assert_eq!(find_token(&tokens, TokenType::FailPolicy).value, "fail-policy");
    }

    #[test]
    fn test_lex_prompt_file_before_prompt() {
        let tokens = lex_ok(
            "version 1\nworkflow w\n  exec\n    harness: h\n    prompt_file: test.md\n",
        );
        assert_eq!(find_token(&tokens, TokenType::PromptFile).value, "prompt_file");
    }

    #[test]
    fn test_lex_inline_comment() {
        let tokens = lex_ok("version 1 # inline comment\nworkflow main\n");
        assert!(!tokens.iter().any(|t| t.value.contains('#')));
        assert_eq!(tokens[0].ty, TokenType::Version);
        assert_eq!(tokens[1].ty, TokenType::Number);
    }

    #[test]
    fn test_lex_empty_source() {
        let tokens = lex_ok("");
        assert_eq!(tokens.len(), 1);
        assert_eq!(tokens[0].ty, TokenType::Eof);
    }

    #[test]
    fn test_lex_blank_lines_only() {
        let tokens = lex_ok("\n\n\n");
        assert_eq!(tokens.len(), 1);
        assert_eq!(tokens[0].ty, TokenType::Eof);
    }

    #[test]
    fn test_lex_unterminated_string() {
        let result = lex(
            "version 1\nworkflow w\n  exec\n    harness: h\n    prompt: \"hello\n",
            "test.7",
        );
        assert!(!result.errors.is_empty());
        assert!(result.errors[0].message.contains("Unterminated"));
    }

    #[test]
    fn test_lex_colon_token() {
        let tokens = lex_ok("version 1\nworkflow w\n  exec\n    harness: echo\n");
        assert!(tokens.iter().any(|t| t.ty == TokenType::Colon));
    }

    #[test]
    fn test_lex_mixed_tabs_and_spaces() {
        let result = lex("version 1\nworkflow main\n \trun greet\n", "test.7");
        assert!(!result.errors.is_empty());
        assert!(result.errors[0].message.contains("Mixed tabs"));
    }

    #[test]
    fn test_lex_match_keyword() {
        let tokens = lex_ok("version 1\nworkflow w\n  match check\n    small -> run a\n");
        assert_eq!(find_token(&tokens, TokenType::Match).value, "match");
    }

    #[test]
    fn test_lex_else_keyword() {
        let tokens = lex_ok("version 1\nworkflow w\n  match check\n    else -> run a\n");
        assert_eq!(find_token(&tokens, TokenType::Else).value, "else");
    }

    #[test]
    fn test_lex_arrow_token() {
        let tokens = lex_ok("version 1\nworkflow w\n  match check\n    small -> run a\n");
        assert_eq!(find_token(&tokens, TokenType::Arrow).value, "->");
    }

    #[test]
    fn test_lex_bare_gt_error() {
        let result = lex("version 1\nworkflow w\n  match check\n    small> run a\n", "test.7");
        assert!(!result.errors.is_empty());
        assert!(result.errors[0].message.contains("->"));
    }
}