kaish-kernel 0.16.0

//! Lexer tests using rstest for parameterization.
//!
//! These tests replace the custom tokens.txt test file format with native Rust tests.

// Test-fixture code: unwrap/expect on known-good setup is the idiom here.
#![allow(clippy::unwrap_used, clippy::expect_used)]

use kaish_kernel::lexer::{tokenize, LexerError, Token};
use rstest::rstest;

/// Format a Token into the test format string.
fn format_token(token: &Token) -> String {
    fn escape_for_display(s: &str) -> String {
        s.replace('\n', "\\n")
            .replace('\t', "\\t")
            .replace('\r', "\\r")
    }

    match token {
        // Keywords
        Token::Set => "SET".to_string(),
        Token::Local => "LOCAL".to_string(),
        Token::If => "IF".to_string(),
        Token::Then => "THEN".to_string(),
        Token::Else => "ELSE".to_string(),
        Token::Elif => "ELIF".to_string(),
        Token::Fi => "FI".to_string(),
        Token::For => "FOR".to_string(),
        Token::While => "WHILE".to_string(),
        Token::In => "IN".to_string(),
        Token::Do => "DO".to_string(),
        Token::Done => "DONE".to_string(),
        Token::Case => "CASE".to_string(),
        Token::Esac => "ESAC".to_string(),
        Token::Function => "FUNCTION".to_string(),
        Token::Break => "BREAK".to_string(),
        Token::Continue => "CONTINUE".to_string(),
        Token::Return => "RETURN".to_string(),
        Token::Exit => "EXIT".to_string(),
        Token::True => "BOOL(true)".to_string(),
        Token::False => "BOOL(false)".to_string(),

        // Type keywords
        Token::TypeString => "TYPESTRING".to_string(),
        Token::TypeInt => "TYPEINT".to_string(),
        Token::TypeFloat => "TYPEFLOAT".to_string(),
        Token::TypeBool => "TYPEBOOL".to_string(),

        // Operators
        Token::And => "AMPAMP".to_string(),
        Token::Or => "PIPEPIPE".to_string(),
        Token::EqEq => "EQEQ".to_string(),
        Token::NotEq => "NEQ".to_string(),
        Token::Match => "MATCH".to_string(),
        Token::NotMatch => "NOTMATCH".to_string(),
        Token::GtEq => "GEQ".to_string(),
        Token::LtEq => "LEQ".to_string(),
        Token::GtGt => "REDIR_APPEND".to_string(),
        Token::StderrToStdout => "REDIR_MERGE".to_string(),
        Token::StdoutToStderr => "REDIR_STDOUT_TO_STDERR".to_string(),
        Token::StdoutToStderr2 => "REDIR_STDOUT_TO_STDERR".to_string(),
        Token::Stderr => "REDIR_ERR".to_string(),
        Token::Both => "REDIR_BOTH".to_string(),
        Token::HereDocStart => "HEREDOC_START".to_string(),
        Token::HereString => "HERESTRING".to_string(),
        Token::DoubleSemi => "DOUBLESEMI".to_string(),

        // Single-char operators
        Token::Eq => "EQ".to_string(),
        Token::Pipe => "PIPE".to_string(),
        Token::Amp => "AMP".to_string(),
        Token::Gt => "GT".to_string(),
        Token::Lt => "LT".to_string(),
        Token::Semi => "SEMI".to_string(),
        Token::Colon => "COLON".to_string(),
        Token::Comma => "COMMA".to_string(),
        Token::Dot => "DOT".to_string(),
        Token::DotDot => "DOTDOT".to_string(),
        Token::DotDotDot => "DOTDOTDOT".to_string(),
        Token::Tilde => "TILDE".to_string(),
        Token::TildePath(s) => format!("TILDEPATH({})", s),
        Token::RelativePath(s) => format!("RELPATH({})", s),
        Token::DotSlashPath(s) => format!("DOTSLASH({})", s),

        // Brackets
        Token::LBrace => "LBRACE".to_string(),
        Token::RBrace => "RBRACE".to_string(),
        Token::LBracket => "LBRACK".to_string(),
        Token::RBracket => "RBRACK".to_string(),
        Token::LParen => "LPAREN".to_string(),
        Token::RParen => "RPAREN".to_string(),
        Token::Star => "STAR".to_string(),
        Token::Bang => "BANG".to_string(),
        Token::Question => "QUESTION".to_string(),
        Token::GlobWord(s) => format!("GLOB({})", s),

        // Arithmetic and command substitution
        Token::Arithmetic(s) => format!("ARITH({})", s),
        Token::CmdSubstStart => "CMDSUBST".to_string(),

        // Flags
        Token::LongFlag(s) => format!("LONGFLAG({})", s),
        Token::ShortFlag(s) => format!("SHORTFLAG({})", s),
        Token::PlusFlag(s) => format!("PLUSFLAG({})", s),
        Token::DoubleDash => "DOUBLEDASH".to_string(),
        // Bare words starting with + or -
        Token::PlusBare(s) => format!("PLUSBARE({})", s),
        Token::MinusBare(s) => format!("MINUSBARE({})", s),
        Token::DoubleDashBare(s) => format!("DOUBLEDASHBARE({})", s),
        Token::JobSpec(s) => format!("JOBSPEC({})", s),
        Token::MinusAlone => "MINUSALONE".to_string(),

        // Literals
        Token::String(s) => format!("STRING({})", escape_for_display(s)),
        Token::SingleString(s) => format!("SINGLESTRING({})", s),
        Token::HereDoc(d) => format!("HEREDOC({}, literal={})", escape_for_display(&d.content), d.literal),
        Token::VarRef(s) => format!("VARREF({})", s),
        Token::SimpleVarRef(s) => format!("SIMPLEVARREF({})", s),
        Token::Positional(n) => format!("POSITIONAL({})", n),
        Token::AllArgs => "ALLARGS".to_string(),
        Token::ArgCount => "ARGCOUNT".to_string(),
        Token::LastExitCode => "LASTEXITCODE".to_string(),
        Token::CurrentPid => "CURRENTPID".to_string(),
        Token::VarLength(s) => format!("VARLENGTH({})", s),
        Token::Int(n) => format!("INT({})", n),
        Token::Float(n) => {
            let s = n.to_string();
            if s.contains('.') {
                format!("FLOAT({})", s)
            } else {
                format!("FLOAT({}.0)", s)
            }
        }

        // Identifiers and paths
        Token::Ident(s) => format!("IDENT({})", s),
        Token::NumberIdent(s) => format!("NUMIDENT({})", s),
        Token::DashNumWord(s) => format!("DASHNUM({})", s),
        Token::AtWord(s) => format!("ATWORD({})", s),
        Token::DottedIdent(s) => format!("DOTIDENT({})", s),
        Token::Path(s) => format!("PATH({})", s),

        // Structural
        Token::Comment => "COMMENT".to_string(),
        Token::Newline => "NEWLINE".to_string(),
        Token::LineContinuation => "LINECONT".to_string(),

        // Invalid variants (should never be produced)
        Token::InvalidFloatNoLeading => "INVALID_FLOAT_NO_LEADING".to_string(),
        Token::InvalidFloatNoTrailing => "INVALID_FLOAT_NO_TRAILING".to_string(),
        Token::BacktickRejected => "BACKTICK_REJECTED".to_string(),

        // `Token` is `#[non_exhaustive]` (kaish-kernel enums grow as the
        // language does). This formatter is a full inventory of every
        // variant on purpose — a new one must be added above, loudly, not
        // silently rendered as a mislabeled existing variant.
        _ => panic!("format_token: unhandled Token variant {token:?} — add a case above"),
    }
}

/// Run a lexer test that expects successful tokenization.
fn run_lexer_test(input: &str, expected: &[&str]) {
    let tokens = tokenize(input).expect("lexing should succeed");
    let actual: Vec<String> = tokens
        .iter()
        .filter(|s| !matches!(s.token, Token::Newline | Token::Comment))
        .map(|s| format_token(&s.token))
        .collect();
    let expected: Vec<String> = expected.iter().map(|s| s.to_string()).collect();
    assert_eq!(actual, expected, "input: {:?}", input);
}

/// Run a lexer test that expects an error.
fn run_lexer_error_test(input: &str) {
    let result = tokenize(input);
    assert!(result.is_err(), "expected error for input: {:?}", input);
}

/// Run a lexer test that expects a *specific* error variant. Asserting only
/// `is_err()` would still pass if a curated diagnostic regressed to the generic
/// `UnexpectedCharacter`, so the negative suites pin the variant they document.
fn run_lexer_error_variant(input: &str, expected: LexerError) {
    let errors = tokenize(input).expect_err(&format!("expected error for input: {input:?}"));
    assert!(
        errors.iter().any(|e| e.token == expected),
        "input {input:?}: expected {expected:?}, got {:?}",
        errors.iter().map(|e| &e.token).collect::<Vec<_>>(),
    );
}

/// Like [`run_lexer_error_variant`] but matches against a predicate, for error
/// variants that carry a payload — `LexerError::NonAsciiName` names the
/// offending word, so an equality assertion would have to spell it twice.
fn run_lexer_error_matching(input: &str, pred: impl Fn(&LexerError) -> bool, what: &str) {
    let errors = tokenize(input).expect_err(&format!("expected error for input: {input:?}"));
    assert!(
        errors.iter().any(|e| pred(&e.token)),
        "input {input:?}: expected {what}, got {:?}",
        errors.iter().map(|e| &e.token).collect::<Vec<_>>(),
    );
}

// =============================================================================
// Keywords
// =============================================================================

#[rstest]
#[case::keyword_set("set", &["SET"])]
#[case::keyword_local("local", &["LOCAL"])]
#[case::keyword_if("if", &["IF"])]
#[case::keyword_then("then", &["THEN"])]
#[case::keyword_else("else", &["ELSE"])]
#[case::keyword_elif("elif", &["ELIF"])]
#[case::keyword_fi("fi", &["FI"])]
#[case::keyword_for("for", &["FOR"])]
#[case::keyword_in("in", &["IN"])]
#[case::keyword_do("do", &["DO"])]
#[case::keyword_done("done", &["DONE"])]
fn lexer_keywords(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Identifiers
// =============================================================================

#[rstest]
#[case::ident_simple("foo", &["IDENT(foo)"])]
#[case::ident_underscore("foo_bar", &["IDENT(foo_bar)"])]
#[case::ident_hyphen("foo-bar", &["IDENT(foo-bar)"])]
#[case::ident_private("_private", &["IDENT(_private)"])]
#[case::ident_with_number("x1", &["IDENT(x1)"])]
fn lexer_identifiers(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// Digit-leading bare words are valid argv tokens: SHA prefixes (019dda1c),
// UUIDs, version-ish identifiers. Lex them as NumberIdent so the parser
// can treat them as bareword strings. (Pure digit sequences still lex as
// Int — at least one alpha character is required to land here.)
#[rstest]
#[case::numident_hex("019dda1c", &["NUMIDENT(019dda1c)"])]
#[case::numident_alpha_after_digit("123abc", &["NUMIDENT(123abc)"])]
#[case::numident_with_dash("019dda1c-5b3f-7000", &["NUMIDENT(019dda1c-5b3f-7000)"])]
#[case::numident_with_dot("019dda1c.commit", &["NUMIDENT(019dda1c.commit)"])]
fn lexer_number_idents(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// Dot-prefixed bare words: `.gitignore`, `.parent`, `.parent.parent`. Must
// lex as a single token, not Dot + Ident, so they are not misparsed as the
// POSIX `.` (source) command followed by an argument.
#[rstest]
#[case::dotident_simple(".parent", &["DOTIDENT(.parent)"])]
#[case::dotident_chained(".parent.parent", &["DOTIDENT(.parent.parent)"])]
#[case::dotident_hidden_file(".gitignore", &["DOTIDENT(.gitignore)"])]
#[case::dotident_with_dash(".foo-bar", &["DOTIDENT(.foo-bar)"])]
fn lexer_dot_idents(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// `. file` (with whitespace) must remain Dot + Ident so the source alias works.
#[rstest]
#[case::source_alias_with_space(". script", &["DOT", "IDENT(script)"])]
#[case::source_alias_with_dotted_file(". script.kai", &["DOT", "IDENT(script.kai)"])]
fn lexer_source_alias_preserved(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Integers
// =============================================================================

#[rstest]
#[case::int_zero("0", &["INT(0)"])]
#[case::int_positive("123", &["INT(123)"])]
#[case::int_negative("-456", &["INT(-456)"])]
#[case::int_large("999999999", &["INT(999999999)"])]
fn lexer_integers(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Floats
// =============================================================================

#[rstest]
#[case::float_zero("0.0", &["FLOAT(0.0)"])]
#[case::float_pi("3.14", &["FLOAT(3.14)"])]
#[case::float_negative("-2.5", &["FLOAT(-2.5)"])]
fn lexer_floats(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

#[rstest]
#[case::float_no_leading(".5", LexerError::InvalidFloatNoLeading)]
#[case::float_no_trailing("5.", LexerError::InvalidFloatNoTrailing)]
fn lexer_float_errors(#[case] input: &str, #[case] expected: LexerError) {
    run_lexer_error_variant(input, expected);
}

// =============================================================================
// Booleans
// =============================================================================

#[rstest]
#[case::bool_true("true", &["BOOL(true)"])]
#[case::bool_false("false", &["BOOL(false)"])]
fn lexer_booleans(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

/// Only lowercase `true` and `false` are boolean literals. Everything that
/// merely *looks* like one is an ordinary word.
///
/// The lexer used to reject these as ambiguous, which cost more than it
/// bought: `yes | head -3` could not run the POSIX utility, `cat no` could
/// not read a file named `no`, and `grep TRUE data.csv` could not search for
/// a common CSV value. A lexer cannot see whether a boolean was wanted, so it
/// rejected the word in every position to catch the one where it might have
/// mattered — and it did not even do that consistently, since `1`, `0`, `on`,
/// `off`, `y`, and `n` were always accepted.
#[rstest]
#[case::bool_upper_true("TRUE")]
#[case::bool_upper_false("FALSE")]
#[case::bool_mixed_true("True")]
#[case::bool_like_yes("yes")]
#[case::bool_like_no("no")]
#[case::bool_like_yes_upper("YES")]
#[case::bool_like_no_upper("NO")]
fn boolean_lookalikes_are_ordinary_identifiers(#[case] input: &str) {
    run_lexer_test(input, &[&format!("IDENT({input})")]);
}

// =============================================================================
// Double-Quoted Strings
// =============================================================================

#[rstest]
#[case::string_simple(r#""hello""#, &["STRING(hello)"])]
#[case::string_with_spaces(r#""hello world""#, &["STRING(hello world)"])]
#[case::string_empty(r#""""#, &["STRING()"])]
#[case::string_newline(r#""line\nbreak""#, &["STRING(line\\nbreak)"])]
#[case::string_tab(r#""tab\there""#, &["STRING(tab\\there)"])]
#[case::string_quote(r#""quote\"here""#, &["STRING(quote\"here)"])]
#[case::string_backslash(r#""slash\\here""#, &["STRING(slash\\here)"])]
#[case::string_unicode(r#""unicode\u0041""#, &["STRING(unicodeA)"])]
fn lexer_double_quoted_strings(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

#[rstest]
#[case::string_unterminated(r#""unterminated"#)]
fn lexer_string_errors(#[case] input: &str) {
    // The gap this used to pin is closed. It read: an unterminated string
    // never matched the flat `String` regex, so logos surfaced the generic
    // `UnexpectedCharacter` rather than the curated `UnterminatedString`, and
    // "improving the diagnostic to `UnterminatedString` should update this
    // assertion." `lex_string` scans for the closing quote itself now, so it
    // is the one that reports the failure and it names it. The span is
    // unchanged; only the wording improved.
    run_lexer_error_variant(input, LexerError::UnterminatedString);
}

/// A `$(` left open inside the string names the missing `)`, not the string.
/// Both are unterminated; only the paren is the mistake, and the parser's
/// sibling scanner has always said so for the forms that reached it.
#[rstest]
#[case::in_substitution(r#""$(echo hi"#)]
#[case::after_inner_quote(r#""$(echo "hi""#)]
#[case::nested_substitution(r#""$(echo "$(echo hi)""#)]
fn lexer_unterminated_cmdsubst_in_string(#[case] input: &str) {
    run_lexer_error_variant(input, LexerError::UnterminatedCommandSubst);
}

// =============================================================================
// Single-Quoted Strings
// =============================================================================

#[rstest]
#[case::singlestring_simple("'hello'", &["SINGLESTRING(hello)"])]
#[case::singlestring_with_spaces("'hello world'", &["SINGLESTRING(hello world)"])]
#[case::singlestring_empty("''", &["SINGLESTRING()"])]
#[case::singlestring_no_var("'no $VAR here'", &["SINGLESTRING(no $VAR here)"])]
#[case::singlestring_no_escapes(r"'no escapes: \n'", &[r"SINGLESTRING(no escapes: \n)"])]
fn lexer_single_quoted_strings(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

#[rstest]
#[case::singlestring_unterminated("'unterminated")]
fn lexer_singlestring_errors(#[case] input: &str) {
    run_lexer_error_test(input);
}

// =============================================================================
// Variable References
// =============================================================================

#[rstest]
#[case::varref_braced("${X}", &["VARREF(${X})"])]
#[case::varref_braced_lower("${foo}", &["VARREF(${foo})"])]
#[case::varref_braced_underscore("${foo_bar}", &["VARREF(${foo_bar})"])]
#[case::varref_field("${X.field}", &["VARREF(${X.field})"])]
#[case::varref_index("${X[0]}", &["VARREF(${X[0]})"])]
#[case::varref_path("${X.a.b[0].c}", &["VARREF(${X.a.b[0].c})"])]
fn lexer_braced_varrefs(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

#[rstest]
#[case::varref_unterminated_brace("${")]
#[case::varref_unterminated_name("${X")]
fn lexer_varref_errors(#[case] input: &str) {
    run_lexer_error_test(input);
}

#[rstest]
#[case::simple_varref("$X", &["SIMPLEVARREF(X)"])]
#[case::simple_varref_lower("$foo", &["SIMPLEVARREF(foo)"])]
#[case::simple_varref_underscore("$foo_bar", &["SIMPLEVARREF(foo_bar)"])]
#[case::simple_varref_private("$_private", &["SIMPLEVARREF(_private)"])]
fn lexer_simple_varrefs(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Operators
// =============================================================================

#[rstest]
#[case::op_eq("=", &["EQ"])]
#[case::op_eqeq("==", &["EQEQ"])]
#[case::op_neq("!=", &["NEQ"])]
#[case::op_lt("<", &["LT"])]
#[case::op_gt(">", &["GT"])]
#[case::op_leq("<=", &["LEQ"])]
#[case::op_geq(">=", &["GEQ"])]
#[case::op_pipe("|", &["PIPE"])]
#[case::op_amp("&", &["AMP"])]
#[case::op_ampamp("&&", &["AMPAMP"])]
#[case::op_pipepipe("||", &["PIPEPIPE"])]
fn lexer_operators(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Redirects
// =============================================================================

#[rstest]
#[case::redir_append(">>", &["REDIR_APPEND"])]
#[case::redir_err("2>", &["REDIR_ERR"])]
#[case::redir_both("&>", &["REDIR_BOTH"])]
#[case::redir_merge("2>&1", &["REDIR_MERGE"])]
#[case::redir_merge_in_cmd("cmd 2>&1", &["IDENT(cmd)", "REDIR_MERGE"])]
#[case::redir_merge_pipe("cmd 2>&1 | cat", &["IDENT(cmd)", "REDIR_MERGE", "PIPE", "IDENT(cat)"])]
#[case::herestring_alone("<<<", &["HERESTRING"])]
#[case::herestring_with_word("<<< hi", &["HERESTRING", "IDENT(hi)"])]
#[case::herestring_with_var("<<< \"$R\"", &["HERESTRING", "STRING($R)"])]
#[case::herestring_no_space("<<<hi", &["HERESTRING", "IDENT(hi)"])]
#[case::herestring_in_cmd("cat <<< hi", &["IDENT(cat)", "HERESTRING", "IDENT(hi)"])]
#[case::heredoc_start_still_preprocesses("<< EOF\nEOF", &["HEREDOC_START", "HEREDOC(, literal=false)"])]
#[case::four_lt_greedy_match("<<<<", &["HERESTRING", "LT"])]
fn lexer_redirects(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Brackets
// =============================================================================

#[rstest]
#[case::bracket_lbrace("{", &["LBRACE"])]
#[case::bracket_rbrace("}", &["RBRACE"])]
#[case::bracket_lbrack("[", &["LBRACK"])]
#[case::bracket_rbrack("]", &["RBRACK"])]
#[case::bracket_lparen("(", &["LPAREN"])]
#[case::bracket_rparen(")", &["RPAREN"])]
#[case::bracket_double_lbrack("[[", &["LBRACK", "LBRACK"])]
#[case::bracket_double_rbrack("]]", &["RBRACK", "RBRACK"])]
fn lexer_brackets(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Punctuation
// =============================================================================

#[rstest]
#[case::punct_comma(",", &["COMMA"])]
#[case::punct_colon(":", &["COLON"])]
#[case::punct_semi(";", &["SEMI"])]
#[case::punct_dot(".", &["DOT"])]
#[case::punct_star("*", &["STAR"])]
#[case::punct_question("?", &["QUESTION"])]
#[case::punct_bang("!", &["BANG"])]
fn lexer_punctuation(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Bang operator priority (multi-char operators should take precedence)
// =============================================================================

#[rstest]
#[case::bang_alone("!", &["BANG"])]
#[case::bang_neq_takes_priority("!=", &["NEQ"])]
#[case::bang_notmatch_takes_priority("!~", &["NOTMATCH"])]
#[case::bang_then_eq("! =", &["BANG", "EQ"])]
#[case::bang_with_command("! true", &["BANG", "BOOL(true)"])]
#[case::negated_char_class_pattern("[!a-z]", &["GLOB([!a-z])"])]
fn lexer_bang_operator(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Glob word merging
// =============================================================================

#[rstest]
#[case::star_dot_txt("*.txt", &["GLOB(*.txt)"])]
#[case::double_star_slash_rs("**/*.rs", &["GLOB(**/*.rs)"])]
#[case::path_star_go("src/*.go", &["GLOB(src/*.go)"])]
#[case::question_mark_glob("file?.log", &["GLOB(file?.log)"])]
#[case::bracket_class("[a-z].txt", &["GLOB([a-z].txt)"])]
#[case::star_alone("*", &["STAR"])]
#[case::question_alone("?", &["QUESTION"])]
#[case::quoted_not_glob("\"*.txt\"", &["STRING(*.txt)"])]
#[case::no_glob_chars("foo bar", &["IDENT(foo)", "IDENT(bar)"])]
#[case::star_dot_tar_gz("*.tar.gz", &["GLOB(*.tar.gz)"])]
#[case::dot_star(".*", &["GLOB(.*)"])]
#[case::star_dot_brace_rs_go("*.{rs,go}", &["GLOB(*.{rs,go})"])]
#[case::glob_colon_merge("foo::bar*.txt", &["GLOB(foo::bar*.txt)"])]
#[case::glob_tilde_path("~/src/*.rs", &["GLOB(~/src/*.rs)"])]
#[case::glob_relative_path("../src/*.rs", &["GLOB(../src/*.rs)"])]
#[case::glob_dot_slash_path("./*.rs", &["GLOB(./*.rs)"])]
#[case::glob_adjacent_to_semi("*.txt;", &["GLOB(*.txt)", "SEMI"])]
fn lexer_glob_word(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Whitespace & Comments
// =============================================================================

#[rstest]
#[case::string_with_spaces_preserved(r#""  spaces  ""#, &["STRING(  spaces  )"])]
fn lexer_whitespace_in_strings(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Flags
// =============================================================================

#[rstest]
#[case::shortflag_l("-l", &["SHORTFLAG(l)"])]
#[case::shortflag_a("-a", &["SHORTFLAG(a)"])]
#[case::shortflag_combined("-la", &["SHORTFLAG(la)"])]
#[case::shortflag_triple("-vvv", &["SHORTFLAG(vvv)"])]
fn lexer_short_flags(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

#[rstest]
#[case::longflag_force("--force", &["LONGFLAG(force)"])]
#[case::longflag_verbose("--verbose", &["LONGFLAG(verbose)"])]
#[case::longflag_hyphen("--foo-bar", &["LONGFLAG(foo-bar)"])]
#[case::longflag_message("--message", &["LONGFLAG(message)"])]
fn lexer_long_flags(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

#[rstest]
#[case::plusflag_e("+e", &["PLUSFLAG(e)"])]
#[case::plusflag_x("+x", &["PLUSFLAG(x)"])]
#[case::plusflag_combined("+ex", &["PLUSFLAG(ex)"])]
fn lexer_plus_flags(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

#[rstest]
#[case::doubledash("--", &["DOUBLEDASH"])]
fn lexer_double_dash(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// Note: Single dash "-" is now valid as MinusAlone (for cat - stdin indicator).
// A `--`-prefixed word whose 3rd char isn't a letter is ONE `DoubleDashBare`
// token — it used to fragment into `DoubleDash` + a leftover token, silently
// truncating a dash-only operand (`echo ---` printed `-` instead of `---`,
// GH #137). A lone `--` (nothing following) is unaffected: it still lexes as
// plain `DOUBLEDASH`.
#[rstest]
#[case::single_dash("-", &["MINUSALONE"])]
#[case::triple_dash("---", &["DOUBLEDASHBARE(---)"])]
#[case::quad_dash("----", &["DOUBLEDASHBARE(----)"])]
#[case::double_dash_equals("--=x", &["DOUBLEDASHBARE(--=x)"])]
#[case::double_dash_digit("--1", &["DOUBLEDASHBARE(--1)"])]
#[case::double_dash_then_word("---foo", &["DOUBLEDASHBARE(---foo)"])]
fn lexer_dash_variants(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

#[rstest]
#[case::date_format("+%s", &["PLUSBARE(+%s)"])]
#[case::date_format_complex("+%Y-%m-%d", &["PLUSBARE(+%Y-%m-%d)"])]
fn lexer_plus_bare(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// GH #144: `DoubleDashBare`/`PlusBare`/`MinusBare` used to match a trailing
// `[^\s]*`, which happily swallowed an immediately-adjacent shell operator
// with no whitespace in between — `---)` lexed as the single token
// `DoubleDashBare("---)")`, leaving no separate `RParen` for the case-branch
// parser to find. This is the same silent-truncation failure family as #137
// (`---` losing its dashes), just the operator disappearing into the bare
// word instead of the word losing characters to the operator. The fix
// excludes `()|&;<>` from both the differentiator char and the trailing run,
// so these tokens stop exactly where an unquoted operator starts, same as
// `Ident`/`ShortFlag`/`LongFlag` already do.
#[rstest]
#[case::triple_dash_before_rparen("---)", &["DOUBLEDASHBARE(---)", "RPAREN"])]
#[case::triple_dash_before_semi("---;", &["DOUBLEDASHBARE(---)", "SEMI"])]
#[case::triple_dash_before_pipe("---|", &["DOUBLEDASHBARE(---)", "PIPE"])]
#[case::triple_dash_before_amp("---&", &["DOUBLEDASHBARE(---)", "AMP"])]
#[case::minus_bare_before_rparen("-%)", &["MINUSBARE(-%)", "RPAREN"])]
#[case::minus_bare_before_semi("-%;", &["MINUSBARE(-%)", "SEMI"])]
#[case::plus_bare_before_rparen("+%s)", &["PLUSBARE(+%s)", "RPAREN"])]
#[case::plus_bare_before_semi("+%s;", &["PLUSBARE(+%s)", "SEMI"])]
// A lone `-` immediately before an operator (no differentiator char left
// over) falls back all the way to `MinusAlone`, same as a lone `-` before
// whitespace/EOF.
#[case::lone_dash_before_rparen("-)", &["MINUSALONE", "RPAREN"])]
// `--` immediately before an operator stays plain `DoubleDash`: the bare-word
// regex's differentiator char would have to BE the operator, which is now
// excluded, so the two-char exact-match token wins instead.
#[case::double_dash_before_rparen("--)", &["DOUBLEDASH", "RPAREN"])]
fn lexer_dash_plus_bare_stops_at_operators(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Flags vs Negative Numbers
// =============================================================================

#[rstest]
#[case::negative_123("-123", &["INT(-123)"])]
#[case::negative_1("-1", &["INT(-1)"])]
#[case::flag_l("-l", &["SHORTFLAG(l)"])]
// A minus-led numeric word with a non-numeric suffix (`-1a`, `-1k`, `-30d`)
// is one contiguous word (the `find -size -1k` class), not Int(-1)+Ident(a).
#[case::negative_1_then_ident("-1a", &["DASHNUM(-1a)"])]
fn lexer_flag_vs_number(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Combined Sequences
// =============================================================================

#[rstest]
#[case::bash_assign("X=5", &["IDENT(X)", "EQ", "INT(5)"])]
#[case::echo_string(r#"echo "hi""#, &["IDENT(echo)", "STRING(hi)"])]
#[case::cmd_named_args("cmd a=1 b=2", &["IDENT(cmd)", "IDENT(a)", "EQ", "INT(1)", "IDENT(b)", "EQ", "INT(2)"])]
#[case::pipe_chain("a | b | c", &["IDENT(a)", "PIPE", "IDENT(b)", "PIPE", "IDENT(c)"])]
#[case::redirect("x > file", &["IDENT(x)", "GT", "IDENT(file)"])]
fn lexer_combined_sequences(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Flag Sequences
// =============================================================================

#[rstest]
#[case::ls_l("ls -l", &["IDENT(ls)", "SHORTFLAG(l)"])]
#[case::ls_la("ls -la", &["IDENT(ls)", "SHORTFLAG(la)"])]
#[case::git_force("git --force", &["IDENT(git)", "LONGFLAG(force)"])]
#[case::git_push_force("git push --force", &["IDENT(git)", "IDENT(push)", "LONGFLAG(force)"])]
#[case::set_e("set -e", &["SET", "SHORTFLAG(e)"])]
#[case::set_plus_e("set +e", &["SET", "PLUSFLAG(e)"])]
#[case::set_multi_flags("set -e -u", &["SET", "SHORTFLAG(e)", "SHORTFLAG(u)"])]
#[case::cmd_doubledash_flag("cmd -- -flag", &["IDENT(cmd)", "DOUBLEDASH", "SHORTFLAG(flag)"])]
fn lexer_flag_sequences(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Flag-metachar merging (awk -F: idiom)
// =============================================================================
//
// Only `:` (Colon) fuses onto a short flag when span-adjacent (no whitespace).
// `;` (Semi) and `|` (Pipe) are shell operators and must NEVER fuse — even when
// typed immediately after a flag with no space — because they are statement
// terminators and pipe operators respectively.  bash also requires quoting for
// those forms: `awk -F';'`, not `awk -F;`.
// Guarded by span-adjacency: a *space*-separated flag and colon must stay
// as separate tokens.

#[rstest]
// Colon glued onto short flag: `awk -F:` idiom — should fuse
#[case::shortflag_colon("-F:", &["SHORTFLAG(F:)"])]
// Double-colon run: `-F::` fuses completely into one token
#[case::shortflag_double_colon("-F::", &["SHORTFLAG(F::)"])]
// Semicolon glued onto short flag: must NOT fuse — `;` is a shell operator
#[case::shortflag_semi("-F;", &["SHORTFLAG(F)", "SEMI"])]
// Pipe glued onto short flag: must NOT fuse — `|` is a shell operator
#[case::shortflag_pipe("-F|", &["SHORTFLAG(F)", "PIPE"])]
// Colon in full command context: awk -F: is one token; the program is another
#[case::awk_colon_full(r"awk -F: '{print $1}'", &["IDENT(awk)", "SHORTFLAG(F:)", "SINGLESTRING({print $1})"])]
// Regression: `ls -l|cat` — the Pipe must survive as a real pipeline operator
#[case::ls_pipe_cat("ls -l|cat", &["IDENT(ls)", "SHORTFLAG(l)", "PIPE", "IDENT(cat)"])]
// Regression: `cmd -x;cmd2` — the Semi must survive as a real statement separator
#[case::cmd_semi_cmd2("cmd -x;cmd2", &["IDENT(cmd)", "SHORTFLAG(x)", "SEMI", "IDENT(cmd2)"])]
// Space-separated: flag and operator MUST stay separate (no merge)
#[case::shortflag_space_colon("-F :", &["SHORTFLAG(F)", "COLON"])]
#[case::shortflag_space_semi("-F ;", &["SHORTFLAG(F)", "SEMI"])]
#[case::shortflag_space_pipe("-F |", &["SHORTFLAG(F)", "PIPE"])]
fn lexer_flag_metachar_merge(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Special Variables
// =============================================================================

#[rstest]
#[case::positional_0("$0", &["POSITIONAL(0)"])]
#[case::positional_1("$1", &["POSITIONAL(1)"])]
#[case::positional_9("$9", &["POSITIONAL(9)"])]
#[case::all_args("$@", &["ALLARGS"])]
#[case::arg_count("$#", &["ARGCOUNT"])]
#[case::last_exit_code("$?", &["LASTEXITCODE"])]
#[case::current_pid("$$", &["CURRENTPID"])]
#[case::var_length("${#NAME}", &["VARLENGTH(NAME)"])]
fn lexer_special_variables(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

#[rstest]
#[case::echo_exit_code("echo $?", &["IDENT(echo)", "LASTEXITCODE"])]
#[case::echo_pid("echo $$", &["IDENT(echo)", "CURRENTPID"])]
#[case::echo_all_args("echo $@", &["IDENT(echo)", "ALLARGS"])]
#[case::echo_arg_count("echo $#", &["IDENT(echo)", "ARGCOUNT"])]
fn lexer_special_variables_in_context(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

/// Arithmetic inside command substitutions must not be preprocessed by the outer lexer.
/// The inner command sub is re-lexed when evaluated, so $((expr)) inside $(...) is
/// handled by the inner lexer, not the outer preprocessing pass.
#[rstest]
#[case::arith_inside_cmd_sub_single_quoted(
    "$(kaish -c 'echo $((2 + 2))')",
    &["CMDSUBST", "IDENT(kaish)", "SHORTFLAG(c)", "SINGLESTRING(echo $((2 + 2)))", "RPAREN"]
)]
#[case::arith_outside_cmd_sub(
    "X=$((1 + 2)); $(echo hello)",
    &["IDENT(X)", "EQ", "ARITH(1 + 2)", "SEMI", "CMDSUBST", "IDENT(echo)", "IDENT(hello)", "RPAREN"]
)]
// Bug A: single quote inside double quotes must not start quote mode
#[case::single_quote_in_double_quotes(
    r#"echo "It's $((1+1))""#,
    &["IDENT(echo)", "STRING(It's ${__ARITH:1+1__})"]
)]
// Arithmetic in simple double-quoted string
#[case::arith_in_double_quote_string(
    r#"echo "$((2+3))""#,
    &["IDENT(echo)", "STRING(${__ARITH:2+3__})"]
)]
// Bug B: paren in string inside command sub shouldn't break skipper
#[case::paren_in_string_inside_cmd_sub(
    r#"$(echo "foo ) bar")"#,
    &["CMDSUBST", "IDENT(echo)", "STRING(foo ) bar)", "RPAREN"]
)]
fn lexer_arithmetic_in_command_substitution(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Navigation tokens: .., ~, ~/path, ../path
// =============================================================================

#[rstest]
#[case::dotdot("..", &["DOTDOT"])]
#[case::tilde("~", &["TILDE"])]
#[case::tilde_path("~/foo", &["TILDEPATH(~/foo)"])]
#[case::tilde_path_nested("~/src/kaish", &["TILDEPATH(~/src/kaish)"])]
#[case::relative_path("../foo", &["RELPATH(../foo)"])]
#[case::relative_path_nested("../foo/bar", &["RELPATH(../foo/bar)"])]
#[case::cd_dotdot("cd ..", &["IDENT(cd)", "DOTDOT"])]
#[case::cd_tilde("cd ~", &["IDENT(cd)", "TILDE"])]
#[case::cd_tilde_path("cd ~/foo", &["IDENT(cd)", "TILDEPATH(~/foo)"])]
#[case::cd_relative("cd ../bar", &["IDENT(cd)", "RELPATH(../bar)"])]
#[case::dot_slash("./foo", &["DOTSLASH(./foo)"])]
#[case::dot_slash_nested("./src/main.rs", &["DOTSLASH(./src/main.rs)"])]
#[case::cd_dot_slash("cd ./crates", &["IDENT(cd)", "DOTSLASH(./crates)"])]
// Bare relative paths (no ./ or ../ prefix) — regression test for cd only
// traversing the first path component
#[case::cd_bare_relative("cd src/kaish", &["IDENT(cd)", "RELPATH(src/kaish)"])]
#[case::cd_bare_relative_nested("cd src/kaish/crates", &["IDENT(cd)", "RELPATH(src/kaish/crates)"])]
// Trailing slash must stay attached to the path word. Splitting `dest/` into
// `Ident(dest)` + `Path(/)` silently turned `cp a b dest/` into 4 operands.
#[case::bare_relative_trailing_slash("cp a.txt dest/", &["IDENT(cp)", "IDENT(a.txt)", "RELPATH(dest/)"])]
#[case::bare_relative_nested_trailing_slash("cd src/kaish/", &["IDENT(cd)", "RELPATH(src/kaish/)"])]
fn lexer_navigation_tokens(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Assignment lvalue subscripts: `fruits[0]=kiwi` must NOT fuse into a
// `GlobWord` — the glob-merge pass suppresses fusion for a bracket run (no
// `*`/`?`) immediately followed by `=`, distinct from the value-position
// (RHS) suppression used by list/record literals. See `docs/LANGUAGE.md`,
// "Assignment", and `lexer::flush_glob_run`'s `followed_by_eq` parameter.
// =============================================================================

#[rstest]
#[case::single_index("fruits[0]=kiwi", &["IDENT(fruits)", "LBRACK", "INT(0)", "RBRACK", "EQ", "IDENT(kiwi)"])]
#[case::negative_index("xs[-1]=7", &["IDENT(xs)", "LBRACK", "INT(-1)", "RBRACK", "EQ", "INT(7)"])]
#[case::bareword_key("user[email]=x", &["IDENT(user)", "LBRACK", "IDENT(email)", "RBRACK", "EQ", "IDENT(x)"])]
#[case::chained_keys(
    "s[web][port]=9000",
    &["IDENT(s)", "LBRACK", "IDENT(web)", "RBRACK", "LBRACK", "IDENT(port)", "RBRACK", "EQ", "INT(9000)"]
)]
#[case::local_spaced(
    "local xs[0] = 9",
    &["LOCAL", "IDENT(xs)", "LBRACK", "INT(0)", "RBRACK", "EQ", "INT(9)"]
)]
fn lexer_assignment_lvalue_subscript_not_fused(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

/// A real glob pattern (has `*`) followed by `=` is NOT an lvalue and keeps
/// fusing into a `GlobWord` — the `has_star_or_question` guard in
/// `flush_glob_run` protects this.
#[test]
fn lexer_glob_pattern_before_eq_still_fuses() {
    run_lexer_test("[[ $x = [0-9]*.txt ]]", &[
        "LBRACK", "LBRACK", "SIMPLEVARREF(x)", "EQ", "GLOB([0-9]*.txt)", "RBRACK", "RBRACK",
    ]);
}

/// A bracket run NOT followed by `=` (ordinary glob usage) is unaffected by
/// the lvalue suppression and still fuses normally.
#[test]
fn lexer_bracket_char_class_without_eq_still_fuses() {
    run_lexer_test("ls [dog]", &["IDENT(ls)", "GLOB([dog])"]);
}

/// A BARE char-class operand of a `[[ ]]` string comparison (`[[ [a] = b ]]`)
/// starts with `[`, not an `Ident`, so the lvalue suppression must NOT fire —
/// `[a]` keeps fusing to a `GlobWord` and `=` stays string equality against
/// the literal "[a]". The lvalue trigger only fires on an `Ident`-led run
/// (`arr[0]=`), where the root identifier is the first token of the run.
#[test]
fn lexer_bare_char_class_operand_before_eq_still_fuses() {
    run_lexer_test("[[ [a] = b ]]", &[
        "LBRACK", "LBRACK", "GLOB([a])", "EQ", "IDENT(b)", "RBRACK", "RBRACK",
    ]);
}

// =============================================================================
// `push`'s bracket-path TARGET (`push services[web][tags] item`): fused
// verbatim into a single `Ident` — never a `GlobWord` to glob-expand — by a
// THIRD, independent trigger (`PushTarget`, `flush_glob_run`'s `push_target`
// parameter). The target has no trailing `=` to key off the way an
// assignment lvalue does, so it needs its own recognition. See GH #183 and
// `docs/LANGUAGE.md`, "Assignment".
// =============================================================================

#[rstest]
#[case::nested_keys(
    "push services[web][tags] item",
    &["IDENT(push)", "IDENT(services[web][tags])", "IDENT(item)"]
)]
#[case::single_index("push xs[0]", &["IDENT(push)", "IDENT(xs[0])"])]
#[case::bareword_target_unaffected("push xs c", &["IDENT(push)", "IDENT(xs)", "IDENT(c)"])]
fn lexer_push_bracket_target_fused_verbatim(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

/// `push` used as a plain variable name (`push=5`) is unaffected — the
/// tracker is independent of the assignment DFA and never steals its
/// lvalue-root slot.
#[test]
fn lexer_push_as_variable_name_unaffected() {
    run_lexer_test("push=5", &["IDENT(push)", "EQ", "INT(5)"]);
}

/// `push` as a plain bareword ARGUMENT to another command (not the command
/// word itself, so not at statement head) must not trigger the tracker.
#[test]
fn lexer_push_as_bareword_argument_unaffected() {
    run_lexer_test("echo push xs", &["IDENT(echo)", "IDENT(push)", "IDENT(xs)"]);
}

/// A pushed VALUE that itself looks like a bracket path keeps globbing as
/// before — only the first word right after `push` is the target.
#[test]
fn lexer_push_value_after_target_still_globs() {
    run_lexer_test(
        "push xs values[0]",
        &["IDENT(push)", "IDENT(xs)", "GLOB(values[0])"],
    );
}

/// A variable literally named `push`, GLUED to a bracket subscript and
/// assigned (`push[0]=x`), is unaffected by the target tracker: seeing
/// `push` at statement-head sets `PushTarget::AwaitingRoot`, but the very
/// next token here is `LBracket`, not `Ident` — `AwaitingRoot`'s only
/// transition arm requires an `Ident` — so the tracker falls through to its
/// catch-all reset (`PushTarget::None`) and never suppresses this run.
/// The ordinary `=`-followed lvalue trigger (`followed_by_eq`, entirely
/// independent of `PushTarget`) still recognizes `push[0]` as an assignment
/// lvalue on its own, exactly like any other identifier.
#[test]
fn lexer_push_named_variable_bracket_assignment_unaffected() {
    run_lexer_test(
        "push[0]=x",
        &["IDENT(push)", "LBRACK", "INT(0)", "RBRACK", "EQ", "IDENT(x)"],
    );
}

/// `push` immediately after a pipe is still the command word — a pipe is a
/// statement boundary, so both `StmtHead` and the independent `PushTarget`
/// tracker reset to `Start`/`None` there, and `push`'s target-fusion trigger
/// re-arms cleanly on the far side.
#[test]
fn lexer_push_bracket_target_after_pipe() {
    run_lexer_test(
        "echo x | push xs[0] item",
        &[
            "IDENT(echo)", "IDENT(x)", "PIPE",
            "IDENT(push)", "IDENT(xs[0])", "IDENT(item)",
        ],
    );
}

/// Same reset, via `;` instead of a pipe.
#[test]
fn lexer_push_bracket_target_after_semicolon() {
    run_lexer_test(
        "echo x; push xs[0] item",
        &[
            "IDENT(echo)", "IDENT(x)", "SEMI",
            "IDENT(push)", "IDENT(xs[0])", "IDENT(item)",
        ],
    );
}

/// Same reset, via `&&` — a bareword `push` right after a chain operator is
/// still recognized as the command word, not swallowed by whatever DFA state
/// the left-hand command left behind.
#[test]
fn lexer_push_bracket_target_after_and_chain() {
    run_lexer_test(
        "true && push xs[0] item",
        &[
            "BOOL(true)", "AMPAMP",
            "IDENT(push)", "IDENT(xs[0])", "IDENT(item)",
        ],
    );
}

// =============================================================================
// Non-ASCII words
//
// Every bareword/path rule used an ASCII-only character class, so
// `echo café`, `ls /tmp/日本語`, and `cd ~/文書` were all lexer errors —
// quoting was the only way through. bash's rule is that a word is anything
// that is not an operator or whitespace; it never inspects bytes for
// alphabetic-ness. These rules now match that: any non-ASCII scalar value
// (`\u{80}` and up) is an ordinary word character, same as an ASCII letter.
//
// Flag names and `$name` variable references deliberately did NOT widen —
// see the errors section below.
// =============================================================================

#[rstest]
#[case::ident_cafe("café", &["IDENT(café)"])]
#[case::ident_cjk("日本語", &["IDENT(日本語)"])]
#[case::ident_cyrillic("привет", &["IDENT(привет)"])]
#[case::ident_internal_dash("a-café", &["IDENT(a-café)"])]
#[case::ident_trailing_dash("café-a", &["IDENT(café-a)"])]
fn lexer_non_ascii_idents(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

#[rstest]
#[case::absolute_path("/tmp/日本語", &["PATH(/tmp/日本語)"])]
#[case::tilde_path("~/文書", &["TILDEPATH(~/文書)"])]
#[case::dot_slash_path("./café.txt", &["DOTSLASH(./café.txt)"])]
#[case::dot_dot_path("../café/x", &["RELPATH(../café/x)"])]
#[case::bare_relative_path("café/foo", &["RELPATH(café/foo)"])]
#[case::bare_relative_leading_non_ascii("日本語/foo", &["RELPATH(日本語/foo)"])]
#[case::dotted_ident(".日本語", &["DOTIDENT(.日本語)"])]
#[case::number_ident("019café", &["NUMIDENT(019café)"])]
#[case::at_word("@café/pkg", &["ATWORD(@café/pkg)"])]
#[case::dash_num_word("2024-café", &["DASHNUM(2024-café)"])]
#[case::glob_merge_still_fuses("café*.txt", &["GLOB(café*.txt)"])]
fn lexer_non_ascii_paths(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

/// The reported failing commands, minus the ones that stay
/// errors (covered separately below).
#[rstest]
#[case::echo_cafe("echo café", &["IDENT(echo)", "IDENT(café)"])]
#[case::ls_cjk_path("ls /tmp/日本語", &["IDENT(ls)", "PATH(/tmp/日本語)"])]
#[case::cd_tilde_cjk("cd ~/文書", &["IDENT(cd)", "TILDEPATH(~/文書)"])]
#[case::echo_dot_slash_cafe("echo ./café.txt", &["IDENT(echo)", "DOTSLASH(./café.txt)"])]
#[case::echo_dot_dot_cafe("echo ../café/x", &["IDENT(echo)", "RELPATH(../café/x)"])]
#[case::assignment_value("X=café", &["IDENT(X)", "EQ", "IDENT(café)"])]
#[case::word_with_internal_dash("echo a-café", &["IDENT(echo)", "IDENT(a-café)"])]
fn lexer_non_ascii_words_in_context(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

/// Variable names take the same characters as barewords, in both positions,
/// so a reference is spelled in whatever script the name is.
#[rstest]
#[case::varref_cafe("$café", &["SIMPLEVARREF(café)"])]
#[case::varref_japanese("$名前", &["SIMPLEVARREF(名前)"])]
#[case::varref_emoji("$😁", &["SIMPLEVARREF(😁)"])]
#[case::varref_mixed("$x😁", &["SIMPLEVARREF(x😁)"])]
fn lexer_non_ascii_variable_names(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

/// Flag names stay ASCII. A flag spelled in another script is ambiguous — a
/// flag no tool defines, or a word the caller meant literally — so kaish
/// refuses rather than guessing, and the error says to quote it.
///
/// Without special handling, a non-ASCII tail glued onto an otherwise-valid
/// flag prefix would silently lex as TWO tokens, since `Ident`'s leading
/// character class admits non-ASCII — `--café` as `LongFlag(caf)` followed by
/// a stray `Ident(é)` argument, turning a typo into a silently-wrong argument
/// count. These regexes claim the full non-ASCII tail and the callback rejects
/// it, so the diagnostic is one `NonAsciiName` error, never a split.
#[rstest]
#[case::long_flag_cafe("--café")]
#[case::long_flag_cafe_in_context("grep --café x")]
#[case::short_flag_cafe("-café")]
// A real single-letter short flag with a glued non-ASCII tail — the
// sharpest case, since `ShortFlag` (unlike `LongFlag`) is excluded from
// the parser's no-token-pasting glue guard (it carries the `cut -d,`
// glued-value idiom), so this rule's own ASCII check is the only thing
// that keeps it loud.
#[case::short_flag_single_letter_glued_cafe("-lé")]
#[case::plus_flag_cafe("+café")]
fn lexer_non_ascii_names_stay_ascii_errors(#[case] input: &str) {
    run_lexer_error_matching(
        input,
        |e| matches!(e, LexerError::NonAsciiName { .. }),
        "LexerError::NonAsciiName",
    );
}

/// Quoting remains the escape hatch out of ASCII-only flag/variable rules —
/// unaffected by this change, pinned here so a future regression in the
/// quoted-string path still agrees with the unquoted one.
#[rstest]
#[case::quoted_cafe(r#""café""#, &["STRING(café)"])]
fn lexer_non_ascii_quoting_still_escapes(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

// =============================================================================
// Differential corpus — priority-interaction regression guard.
//
// Widening 11 bareword/path character classes to admit non-ASCII risked
// shifting which rule wins the longest-match race for existing ASCII input
// (NumberIdent vs. Int, DashNumWord vs. Int/flags, RelativePath vs.
// Ident+Path, glob-merge boundaries, flag-metachar colon fusion). This
// corpus pins one representative case per widened rule family — every case
// below passed before this change and must keep producing the exact same
// token stream after it.
// =============================================================================

#[rstest]
#[case::ident_boundary("foo-bar", &["IDENT(foo-bar)"])]
#[case::numident_boundary("123abc", &["NUMIDENT(123abc)"])]
#[case::numident_pure_int_unaffected("123", &["INT(123)"])]
#[case::dashnum_boundary("2024-01-02", &["DASHNUM(2024-01-02)"])]
#[case::dashnum_minus_led_boundary("-1a", &["DASHNUM(-1a)"])]
#[case::dashnum_pure_int_unaffected("-123", &["INT(-123)"])]
#[case::atword_boundary("@scope/pkg", &["ATWORD(@scope/pkg)"])]
#[case::dotident_boundary(".gitignore", &["DOTIDENT(.gitignore)"])]
#[case::tildepath_boundary("~/src/kaish", &["TILDEPATH(~/src/kaish)"])]
#[case::relpath_dotdot_boundary("../foo/bar", &["RELPATH(../foo/bar)"])]
#[case::relpath_bareword_boundary("src/kaish", &["RELPATH(src/kaish)"])]
#[case::dotslash_boundary("./src/main.rs", &["DOTSLASH(./src/main.rs)"])]
#[case::path_boundary("/etc/hosts", &["PATH(/etc/hosts)"])]
#[case::longflag_boundary("--force", &["LONGFLAG(force)"])]
#[case::shortflag_boundary("-la", &["SHORTFLAG(la)"])]
#[case::plusflag_boundary("+ex", &["PLUSFLAG(ex)"])]
#[case::simplevarref_boundary("$foo_bar", &["SIMPLEVARREF(foo_bar)"])]
#[case::glob_still_wins_over_ident("*.txt", &["GLOB(*.txt)"])]
#[case::glob_colon_merge_unaffected("foo::bar*.txt", &["GLOB(foo::bar*.txt)"])]
#[case::shortflag_colon_fuse_unaffected("-F:", &["SHORTFLAG(F:)"])]
#[case::trailing_slash_boundary("dest/", &["RELPATH(dest/)"])]
#[case::triple_dash_bare_unaffected("---foo", &["DOUBLEDASHBARE(---foo)"])]
#[case::plus_bare_unaffected("+%Y-%m-%d", &["PLUSBARE(+%Y-%m-%d)"])]
// `--foo=bar`'s equals form (`docs/LANGUAGE.md`'s `curl --header="..."`)
// must still split at `=` — the widened LongFlag continuation class added
// only the `\u{80}-\u{10FFFF}` range, never `=`, so this is unaffected,
// but it is exactly the shape a too-greedy widening (e.g. `\S*`) would
// have broken by swallowing the value into the flag name.
#[case::longflag_equals_form_unaffected("--foo=bar", &["LONGFLAG(foo)", "EQ", "IDENT(bar)"])]
// The leading character after the sigil stays ASCII-only (unchanged) for
// all three flag rules, so a non-ASCII-FIRST word after `--`/`+`/`-` is
// never claimed by the widened flag rules — it falls through to the
// existing bareword-fallback rules exactly as it did before this change.
#[case::doubledashbare_leading_non_ascii_unaffected("--é", &["DOUBLEDASHBARE(--é)"])]
#[case::plusbare_leading_non_ascii_unaffected("+é", &["PLUSBARE(+é)"])]
#[case::minusbare_leading_non_ascii_unaffected("-é", &["MINUSBARE(-é)"])]
fn lexer_widen_does_not_disturb_ascii_priority(#[case] input: &str, #[case] expected: &[&str]) {
    run_lexer_test(input, expected);
}

/// A file full of unterminated openers must not cost O(N²).
///
/// A token that scans for its own terminator (`"`, `${`) reads to end-of-input
/// when the terminator is missing, and logos then retries from the next
/// character — so N openers cost N scans of the remainder: 20000 `"$(echo "`
/// openers took 4.6s uncapped and 0.02s capped. Both renderers join every
/// collected error, so the cap bounds the printed diagnostic too. Pinning the
/// count catches a regression here as a slow test rather than as a wrong
/// answer.
#[test]
fn lexer_errors_are_capped() {
    // One opening quote, then openers that never pair off. `"$(echo "` repeated
    // on its own does NOT reproduce: its quotes pair into empty strings and
    // only two ever reach end-of-input. The shape matters more than the size.
    let source = format!("echo \"{}", r#"$(echo ""#.repeat(5000));
    let errors = tokenize(&source).expect_err("unterminated openers must be an error");
    // Exactly the cap, not merely "bounded": 5000 openers produce ~5001 errors
    // uncapped, so `<= 64` alone would also pass with a cap of 1, which would
    // swallow diagnostics a caller needs.
    assert_eq!(
        errors.len(),
        64,
        "expected exactly the cap when the input overruns it"
    );
}