lumesh 0.18.2

use crate::tokens::{Input, Token, TokenKind};
use crate::utils::is_cfm_mode;
use detached_str::StrSlice;
use nom::{IResult, branch::alt, error::ParseError};

#[derive(Debug, Copy, Clone, PartialEq, Eq)]
struct NotFoundError;

const NOT_FOUND: nom::Err<NotFoundError> = nom::Err::Error(NotFoundError);

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Diagnostic {
    Valid,
    InvalidNumber(StrSlice),
    IllegalChar(StrSlice),
    NotTokenized(StrSlice),
    UnterminatedString(StrSlice),
}

impl<I> ParseError<I> for NotFoundError {
    fn from_error_kind(_: I, _: nom::error::ErrorKind) -> Self {
        NotFoundError
    }

    fn append(_: I, _: nom::error::ErrorKind, other: Self) -> Self {
        other
    }
}

type TokenizationResult<'a, T = StrSlice> = IResult<Input<'a>, T, NotFoundError>;

/// Context based on the previous token's last character.
/// Determines how the next token is classified.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Ctx {
    Number, // Digits sequence, used to detect postfix units like K,M,G,T
    Letter, // Alphabetic sequence
    Start,  // line start / initial state
    Space,  // prev token ended with whitespace (Whitespace/LineBreak/Comment)
    Word,   // prev token ended with  `_`, or closing bracket/quote
    Open,   // other (non-space, non-word symbol)
}

impl Ctx {
    /// Determine the next context based on the current token's ending character.
    /// Whitespace/LineBreak/Comment → Space
    /// Alphanumeric/`_`/closing bracket/quote → Word
    /// Other symbols → Open
    #[inline]
    fn after_token(token: &Token, original: &str) -> Self {
        let last_char = token.range.to_str(original).chars().next_back();
        match token.kind {
            TokenKind::Whitespace | TokenKind::Comment => Ctx::Space,
            TokenKind::LineBreak => Ctx::Start,
            TokenKind::IntegerLiteral | TokenKind::FloatLiteral => Ctx::Number,
            // TokenKind::Symbol => Ctx::Letter,
            _ => match last_char {
                // Some(c) if c.is_ascii_whitespace() => Ctx::Space,
                // Some(c) if c.is_ascii_digit() => Ctx::Number,
                // Some(c) if c.is_ascii_alphabetic() => Ctx::Letter,
                Some(c) if c.is_ascii_alphanumeric() => Ctx::Letter,
                Some(')' | ']' | '}' | '\'' | '"' | '`' | '_') => Ctx::Word,
                Some('(' | '[' | '{' | '|') => Ctx::Start,
                _ => Ctx::Open,
            },
        }
    }
}

/// Dispatch tokenization based on the first character and context.
/// Each character triggers a specific parsing path.
fn parse_token_dispatch(
    input: Input<'_>,
    ctx: Ctx,
    last_ctx: Ctx,
    is_cfm: bool,
) -> TokenizationResult<'_, (Token, Diagnostic)> {
    let first = match input.chars().next() {
        Some(c) => c,
        None => return Err(NOT_FOUND),
    };

    macro_rules! m {
        ($p:expr, $k:expr) => {
            map_valid_token($p, $k)(input)
        };
    }

    match first {
        ' ' | '\t' | '\0' => m!(whitespace, TokenKind::Whitespace),

        ';' => m!(punctuation_tag(";"), TokenKind::LineBreak),
        // #[cfg(windows)]
        '\r' => alt((
            map_valid_token(punctuation_tag("\r\n"), TokenKind::LineBreak), // $var
            map_valid_token(punctuation_tag("\r"), TokenKind::Whitespace),  // $ as symbol
        ))(input),
        '\n' => m!(punctuation_tag("\n"), TokenKind::LineBreak),

        '\\' => {
            if let Ok(r) = m!(line_continuation, TokenKind::Whitespace) {
                return Ok(r);
            }
            m!(punctuation_tag("\\"), TokenKind::Symbol)
        }

        '#' => m!(comment, TokenKind::Comment),

        '"' | '\'' | '`' => string_literal(input),

        '.' => dot_dispatch(input, ctx), // context-aware: method call/range/path/customop

        '-' => minus_dispatch(input, ctx, is_cfm), // context-aware: negative vs flag vs operator

        '(' | ')' | '[' | ']' | '{' | '}' | ',' => paren_dispatch(input, ctx, first),

        '%' => percent_dispatch(input, ctx),

        '!' => bang_dispatch(input, ctx), // context-aware: prefix negate vs postfix call

        '?' => question_dispatch(input, ctx),

        '$' => alt((
            map_valid_token(prefix_tag("$"), TokenKind::OperatorPrefix), // $var
            map_valid_token(punctuation_tag("$"), TokenKind::Symbol),    // $ as symbol
        ))(input),

        '^' => circum_dispatch(input, ctx),
        '&' => and_dispatch(input, ctx),
        '|' => m!(pipe_parser, TokenKind::Operator),
        '=' => m!(equal_parser, TokenKind::Operator),
        '<' => m!(less_parser, TokenKind::Operator),
        '>' => m!(greater_parser, TokenKind::Operator),

        '+' => plus_dispatch(input, ctx, is_cfm),

        '*' => star_dispatch(input, ctx),

        '/' => slash_dispatch(input, ctx),

        '~' => tiled_dispatch(input, ctx),

        ':' => colon_dispatch(input, ctx),

        '@' => at_dispatch(input, ctx),

        '_' => underscore_dispatch(input, ctx), // standalone _ vs _ in symbol

        '0' => radix_literal(input),
        '1'..='9' => number_literal(input),

        'a'..='z' | 'A'..='Z' => alpha_dispatch(input, ctx, last_ctx, first, is_cfm), // keyword / value_symbol / string / symbol

        c if !c.is_ascii() => m!(non_ascii, TokenKind::StringRaw),

        _ => {
            let (rest, range) = input.split_at(first.len_utf8());
            Ok((
                rest,
                (
                    Token::new(TokenKind::Symbol, range),
                    Diagnostic::IllegalChar(range),
                ),
            ))
        }
    }
}

#[inline]
fn map_valid_token(
    mut parser: impl FnMut(Input<'_>) -> TokenizationResult<'_>,
    kind: TokenKind,
) -> impl FnMut(Input<'_>) -> TokenizationResult<'_, (Token, Diagnostic)> {
    move |input| {
        let (input, s) = parser(input)?;
        Ok((input, (Token::new(kind, s), Diagnostic::Valid)))
    }
}

// ========== Context-aware dispatch helpers ==========

/// `(` or `[` after Word context → OperatorPostfix (function call/index)
/// Otherwise → Punctuation (standalone bracket/comma)
fn paren_dispatch(
    input: Input<'_>,
    ctx: Ctx,
    first: char,
) -> TokenizationResult<'_, (Token, Diagnostic)> {
    if matches!(ctx, Ctx::Letter | Ctx::Word) && matches!(first, '(' | '[') {
        map_valid_token(
            punctuation_tag(&first.to_string()),
            TokenKind::OperatorPostfix,
        )(input)
    } else {
        map_valid_token(punctuation_tag(&first.to_string()), TokenKind::Punctuation)(input)
    }
}

fn equal_parser(input: Input<'_>) -> TokenizationResult<'_> {
    alt((
        space_brace_followed_tag("=>"),
        // punctuation_tag("!=="),
        punctuation_tag("==="),
        // punctuation_tag("!="),
        punctuation_tag("=="),
        punctuation_tag("="),
        // punctuation_tag(">="),
        // punctuation_tag("<="),\
    ))(input)
}
fn less_parser(input: Input<'_>) -> TokenizationResult<'_> {
    alt((
        punctuation_tag("<="),
        punctuation_tag("<<"),
        punctuation_tag("<"),
    ))(input)
}
fn greater_parser(input: Input<'_>) -> TokenizationResult<'_> {
    alt((
        punctuation_tag(">="),
        punctuation_tag(">>"),
        punctuation_tag(">!"),
        punctuation_tag(">"),
    ))(input)
}
fn plus_dispatch(
    input: Input<'_>,
    ctx: Ctx,
    is_cfm: bool,
) -> TokenizationResult<'_, (Token, Diagnostic)> {
    match ctx {
        Ctx::Space if is_cfm => alt((
            map_valid_token(punctuation_tag("+="), TokenKind::Operator),
            // map_valid_token(space_followed_tag("+"), TokenKind::Operator), //already contained in postfix_break_tag
            map_valid_token(postfix_break_tag("+"), TokenKind::Operator), // for not fail: `1 +`
            map_valid_token(whole_word("+"), TokenKind::Symbol),          //important for `chmod +x`
        ))(input),
        Ctx::Space => alt((
            map_valid_token(punctuation_tag("+="), TokenKind::Operator),
            // map_valid_token(space_followed_tag("+"), TokenKind::Operator),
            map_valid_token(postfix_break_tag("+"), TokenKind::Operator), // for not fail: `1 +`
            map_valid_token(operator_tag("+"), TokenKind::OperatorPrefix), //important for `chmod +x`
        ))(input),
        _ => alt((
            map_valid_token(punctuation_tag("+="), TokenKind::Operator),
            map_valid_token(punctuation_tag("+"), TokenKind::Operator),
        ))(input),
    }
}
fn pipe_parser(input: Input<'_>) -> TokenizationResult<'_> {
    alt((
        operator_tag("||"),
        punctuation_tag("|>"),
        punctuation_tag("|^"),
        punctuation_tag("|"),
    ))(input)
}

fn and_dispatch(input: Input<'_>, ctx: Ctx) -> TokenizationResult<'_, (Token, Diagnostic)> {
    match ctx {
        Ctx::Word | Ctx::Number | Ctx::Letter => {
            alt((map_valid_token(punctuation_tag("&"), TokenKind::Symbol),))(input)
        } //NEVER USE
        Ctx::Space | Ctx::Open => alt((
            map_valid_token(space_brace_followed_tag("&:"), TokenKind::Operator), //onsuccess catch
            map_valid_token(operator_tag("&&"), TokenKind::Operator),
            map_valid_token(postfix_break_tag("&+"), TokenKind::StringRaw),
            map_valid_token(postfix_break_tag("&-"), TokenKind::StringRaw),
            map_valid_token(postfix_break_tag("&?"), TokenKind::StringRaw),
            map_valid_token(postfix_break_tag("&."), TokenKind::StringRaw),
            map_valid_token(postfix_break_tag("&"), TokenKind::StringRaw),
        ))(input),
        Ctx::Start => alt((map_valid_token(punctuation_tag("&"), TokenKind::Symbol),))(input),
    }
}

fn star_dispatch(input: Input<'_>, ctx: Ctx) -> TokenizationResult<'_, (Token, Diagnostic)> {
    match ctx {
        Ctx::Letter | Ctx::Word | Ctx::Number => {
            map_valid_token(operator_tag("*"), TokenKind::Operator)(input)
        }
        Ctx::Start | Ctx::Space | Ctx::Open => alt((
            map_valid_token(operator_tag("*="), TokenKind::Operator),
            map_valid_token(path_tag("**/", true), TokenKind::Symbol), // **/ path as symbo, to match wildcard
            map_valid_token(path_tag("*/", true), TokenKind::Symbol),  // */ path
            map_valid_token(path_tag("*.", true), TokenKind::Symbol),  // *. path
            map_valid_token(operator_tag("*"), TokenKind::Operator),
            map_valid_token(last_path_tag("*"), TokenKind::Symbol), // `ls *` path at end
        ))(input),
    }
}

fn tiled_dispatch(input: Input<'_>, ctx: Ctx) -> TokenizationResult<'_, (Token, Diagnostic)> {
    match ctx {
        Ctx::Letter | Ctx::Word | Ctx::Number => {
            map_valid_token(punctuation_tag("~"), TokenKind::Symbol)(input) //NEVER
        } //NEVER USE
        Ctx::Start | Ctx::Space | Ctx::Open => alt((
            map_valid_token(operator_tag("~:"), TokenKind::Operator),
            // map_valid_token(operator_tag("~="), TokenKind::Operator),
            map_valid_token(path_tag("~/", true), TokenKind::Symbol), // ~/ path as symbo, to match wildcard
            map_valid_token(last_path_tag("~"), TokenKind::StringRaw), // `ls ~` path at end
                                                                      // map_valid_token(postfix_break_tag("~"), TokenKind::Operator), // reverse?
        ))(input),
    }
}

fn slash_dispatch(input: Input<'_>, ctx: Ctx) -> TokenizationResult<'_, (Token, Diagnostic)> {
    match ctx {
        Ctx::Letter | Ctx::Word | Ctx::Number => {
            alt((map_valid_token(punctuation_tag("/"), TokenKind::Operator),))(input)
        } //divide
        Ctx::Start | Ctx::Space | Ctx::Open => alt((
            map_valid_token(punctuation_tag("/="), TokenKind::Operator),
            map_valid_token(path_tag("/", false), TokenKind::Symbol), // /x path as symbo, to match wildcard
            map_valid_token(last_path_tag("/"), TokenKind::StringRaw), // `ls /` path at end
            map_valid_token(postfix_break_tag("/"), TokenKind::Operator), // divide
        ))(input),
    }
}

/// `Word` context → `...=`/`...`/`..=`/`..` infix range, or `.` postfix (method call)
/// `Start/Space/Open` context → `..` operator, `.` prefix (pipemethod), argument path, number literal, or standalone `.`/`..`
fn dot_dispatch(input: Input<'_>, ctx: Ctx) -> TokenizationResult<'_, (Token, Diagnostic)> {
    match ctx {
        Ctx::Letter | Ctx::Number | Ctx::Word => alt((
            map_valid_token(prefix_range_tag("...="), TokenKind::OperatorInfix),
            map_valid_token(prefix_range_tag("..."), TokenKind::OperatorInfix),
            map_valid_token(prefix_range_tag("..="), TokenKind::OperatorInfix),
            map_valid_token(prefix_range_tag(".."), TokenKind::OperatorInfix), //a..b range
            map_valid_token(postfix_range_tag(".."), TokenKind::OperatorPostfix), //a.. range
            map_valid_token(alpha_followed_tag("."), TokenKind::OperatorPostfix), //call
        ))(input),
        Ctx::Start | Ctx::Space | Ctx::Open => alt((
            map_valid_token(prefix_range_tag("..="), TokenKind::OperatorPrefix), // ..=b range
            map_valid_token(prefix_range_tag(".."), TokenKind::OperatorPrefix),  // ..b range
            number_literal,                                                      //.5
            map_valid_token(alpha_followed_tag("."), TokenKind::OperatorPrefix), //.pipemethod
            map_valid_token(path_tag("../", true), TokenKind::Symbol), //path as symbo, to match wildcard
            map_valid_token(path_tag("./", true), TokenKind::Symbol),
            // eager eat, must bellow others
            map_valid_token(punct_seq_tag(".."), TokenKind::Operator), // ..+ customOp
            map_valid_token(postfix_break_tag(".."), TokenKind::StringRaw), // parent path
            map_valid_token(postfix_break_tag("."), TokenKind::StringRaw), // current path                                                                     // TODO ..3
        ))(input),
    }
}

/// Context-aware `-` dispatch:
/// - Word context: operator only (`-=`, `->`, `-` or symbol) — no prefix/argument
/// - Start/Space: prefix `-` when followed by literal/number/paren/letter (parser distinguishes negation vs flag);
///   `argument_symbol` only for `--` style flags; `number_literal` for bare digits
/// - Open: prefix `-` when followed by literal/number/paren/letter
fn minus_dispatch(
    input: Input<'_>,
    ctx: Ctx,
    is_cfm: bool,
) -> TokenizationResult<'_, (Token, Diagnostic)> {
    match ctx {
        Ctx::Letter | Ctx::Word | Ctx::Number => alt((
            map_valid_token(punctuation_tag("-="), TokenKind::Operator),
            map_valid_token(punctuation_tag("->"), TokenKind::Operator),
            map_valid_token(punctuation_tag("-"), TokenKind::Operator), //a-b as operator
                                                                        // map_valid_token(symbol, TokenKind::Symbol),
        ))(input),
        Ctx::Start => alt((
            map_valid_token(prefix_tag("-"), TokenKind::OperatorPrefix),
            map_valid_token(punctuation_tag("-"), TokenKind::Symbol), //never useful
        ))(input),
        Ctx::Space if is_cfm => alt((
            map_valid_token(punctuation_tag("-="), TokenKind::Operator),
            map_valid_token(punctuation_tag("->"), TokenKind::Operator),
            // bare `-` as operator (e.g. `- ` followed by space)
            map_valid_token(space_followed_tag("-"), TokenKind::Operator),
            // must after '- ' to exclude it
            map_valid_token(postfix_break_tag("-"), TokenKind::StringRaw), //ls | cat -
            // `--flag`/`-s`  → argument symbol
            map_valid_token(whole_word("-"), TokenKind::StringRaw),
            map_valid_token(punctuation_tag("-"), TokenKind::Symbol), //never useful
        ))(input),
        Ctx::Space => alt((
            map_valid_token(punctuation_tag("-="), TokenKind::Operator),
            map_valid_token(punctuation_tag("->"), TokenKind::Operator),
            // `--flag` style: two dashes → argument symbol
            map_valid_token(whole_word("--"), TokenKind::StringRaw),
            // single `-` followed by literal/number/paren/letter → OperatorPrefix
            map_valid_token(prefix_tag("-"), TokenKind::OperatorPrefix),
            // bare `-` as operator (e.g. `- ` followed by space)
            map_valid_token(space_followed_tag("-"), TokenKind::Operator),
            // must after '- ' to exclude it
            map_valid_token(postfix_break_tag("-"), TokenKind::StringRaw), //ls | cat -
            map_valid_token(punctuation_tag("-"), TokenKind::Symbol),      //never useful
        ))(input),
        Ctx::Open => alt((
            map_valid_token(prefix_tag("-"), TokenKind::OperatorPrefix),
            map_valid_token(space_followed_tag("-"), TokenKind::Operator), //not useful
            map_valid_token(punctuation_tag("-"), TokenKind::Symbol),      //never useful
        ))(input),
    }
}

/// Maches range prefix, allow followed by literal/number/(_:]
/// prefix: ..b  .._ ..-2 ..(c)
/// infix : a..-2  a.._  a..(c)
#[inline]
fn prefix_range_tag(prefix: &str) -> impl '_ + Fn(Input<'_>) -> TokenizationResult<'_> {
    move |input: Input<'_>| {
        input
            .strip_prefix(prefix)
            .filter(|(rest, _)| {
                rest.starts_with(|c: char| {
                    c.is_ascii_alphanumeric() || matches!(c, '(' | '-' | '_')
                })
            })
            .ok_or(NOT_FOUND)
    }
}

/// a..:2  [a..]
/// a..  _.. followed by space/:/] or end delimeter
#[inline]
fn postfix_range_tag(prefix: &str) -> impl '_ + Fn(Input<'_>) -> TokenizationResult<'_> {
    move |input: Input<'_>| {
        input
            .strip_prefix(prefix)
            .filter(|(rest, _)| {
                rest.is_empty()
                    || rest.starts_with(|c: char| is_path_delimiter(c) || matches!(c, ':'))
            })
            .ok_or(NOT_FOUND)
    }
}

fn circum_dispatch(input: Input<'_>, ctx: Ctx) -> TokenizationResult<'_, (Token, Diagnostic)> {
    match ctx {
        Ctx::Letter | Ctx::Word => alt((
            // (map_valid_token(space_followed_tag("^"), TokenKind::OperatorPostfix)),
            (map_valid_token(postfix_break_tag("^"), TokenKind::OperatorPostfix)), //for SymbolRaw
            (map_valid_token(punctuation_tag("^"), TokenKind::Operator)),
        ))(input),
        Ctx::Start | Ctx::Space | Ctx::Open | Ctx::Number => {
            map_valid_token(punctuation_tag("^"), TokenKind::Operator)(input)
        }
    }
}

fn percent_dispatch(input: Input<'_>, ctx: Ctx) -> TokenizationResult<'_, (Token, Diagnostic)> {
    match ctx {
        Ctx::Number => alt((map_valid_token(
            punctuation_tag("%"),
            TokenKind::OperatorPostfix,
        ),))(input), //5%
        Ctx::Letter | Ctx::Word => alt((
            map_valid_token(punctuation_tag("%{"), TokenKind::Punctuation),
            map_valid_token(punctuation_tag("%"), TokenKind::Operator),
        ))(input), //a%b

        Ctx::Start | Ctx::Space | Ctx::Open => alt((
            map_valid_token(punctuation_tag("%{"), TokenKind::Punctuation),
            map_valid_token(punctuation_tag("%"), TokenKind::Operator),
        ))(input),
    }
}

/// Context-aware `!` dispatch:
/// - Word context: `!=`/`!==` comparison, `!~:` pattern match, or `!` postfix (flat call)
/// - Start/Space/Open: same operators but `!` as prefix (negation) instead of postfix
fn bang_dispatch(input: Input<'_>, ctx: Ctx) -> TokenizationResult<'_, (Token, Diagnostic)> {
    match ctx {
        Ctx::Letter | Ctx::Word | Ctx::Number => alt((
            map_valid_token(punctuation_tag("!=="), TokenKind::Operator),
            map_valid_token(punctuation_tag("!="), TokenKind::Operator),
            map_valid_token(operator_tag("!~:"), TokenKind::Operator),
            map_valid_token(postfix_break_tag("!"), TokenKind::OperatorPostfix),
            map_valid_token(punctuation_tag("!"), TokenKind::Punctuation),
        ))(input),
        Ctx::Start | Ctx::Space | Ctx::Open => alt((
            map_valid_token(punctuation_tag("!=="), TokenKind::Operator),
            map_valid_token(punctuation_tag("!="), TokenKind::Operator),
            map_valid_token(operator_tag("!~:"), TokenKind::Operator),
            map_valid_token(prefix_tag("!"), TokenKind::OperatorPrefix),
            map_valid_token(punctuation_tag("!"), TokenKind::Punctuation),
        ))(input),
    }
}

fn question_dispatch(input: Input<'_>, _ctx: Ctx) -> TokenizationResult<'_, (Token, Diagnostic)> {
    alt((
        map_valid_token(question_operator, TokenKind::Operator),
        map_valid_token(operator_tag("?"), TokenKind::Operator),
        map_valid_token(punctuation_tag("?"), TokenKind::Symbol),
    ))(input)
}

fn underscore_dispatch(input: Input<'_>, ctx: Ctx) -> TokenizationResult<'_, (Token, Diagnostic)> {
    match ctx {
        Ctx::Letter | Ctx::Word | Ctx::Number => alt((
            // custom unary op __+ as OperatorPostfix: 'xx'__+
            // map_valid_token(punct_seq_tag("__"), TokenKind::OperatorPostfix), //custom op use
            map_valid_token(punctuation_tag("_"), TokenKind::Symbol),
        ))(input),
        _ => alt((
            map_valid_token(space_brace_followed_tag("_:"), TokenKind::Operator), //onEmpty catch
            map_valid_token(space_brace_followed_tag("_!"), TokenKind::Operator), //TerminateOnEmpty catch
            // custom unary op __+ as Operator: a __+
            map_valid_token(punct_seq_tag("__"), TokenKind::Operator), //custom op define
            //`ls _` `[0.._]` `[_..9]` `a[.._:2]
            map_valid_token(
                |input| {
                    input
                        .strip_prefix("_")
                        .filter(|(rest, _)| {
                            rest.is_empty()
                                || rest.starts_with(&[' ', '\n', ')', ']', '}', ':', ';'])
                                || rest.starts_with("..")
                        })
                        .ok_or(NOT_FOUND)
                },
                TokenKind::ValueSymbol,
            ),
            // cfm never affect _
            map_valid_token(|input| symbol(input, false, ctx, ctx), TokenKind::Symbol), // failback to symbol: _foo
        ))(input),
    }
}

fn alpha_dispatch(
    input: Input<'_>,
    ctx: Ctx,
    last_ctx: Ctx,
    first: char,
    is_cfm: bool,
) -> TokenizationResult<'_, (Token, Diagnostic)> {
    // filesize
    if ctx == Ctx::Number && matches!(&first, 'B' | 'K' | 'M' | 'G' | 'T' | 'P') {
        return map_valid_token(
            postfix_break_tag(&first.to_string()),
            TokenKind::OperatorPostfix,
        )(input);
    }
    match input.chars().nth(1) {
        Some(second) => {
            // HMap/Map/Set
            if second == '{'
                && matches!(ctx, Ctx::Space | Ctx::Start)
                && matches!(&first, 'H' | 'M' | 'S')
            {
                return map_valid_token(
                    punctuation_tag(&format!("{first}{{")),
                    TokenKind::Punctuation,
                )(input);
            }

            if matches!(second, '\'' | '"') {
                return match first {
                    'r' => hashed_literal(&'r')(input),
                    'g' => parse_prefixed_string(input, first, second, TokenKind::Regex),
                    't' => parse_prefixed_string(input, first, second, TokenKind::Time),
                    's' => parse_prefixed_string(input, first, second, TokenKind::StringSafe),
                    'b' => parse_prefixed_string(input, first, second, TokenKind::Bytes),
                    _ => Err(NOT_FOUND),
                };
            }

            if second == '#' {
                if matches!(first, 'r' | 'g' | 't' | 's' | 'b') {
                    let result = hashed_literal(&first)(input);
                    if let Ok(hs) = result {
                        return Ok(hs);
                    }
                }
            }

            #[cfg(windows)]
            if let Ok(r) = map_valid_token(win_abpath_tag, TokenKind::StringRaw)(input) {
                return Ok(r);
            }

            // keyword should only in ctx::Start/Space, not in ctx::Open, like `regex.match`
            if ctx == Ctx::Start || ctx == Ctx::Space {
                if let Ok(r) = map_valid_token(any_keyword, TokenKind::Keyword)(input) {
                    return Ok(r);
                }
            }

            // try others
            alt((
                map_valid_token(value_symbol, TokenKind::ValueSymbol),
                map_valid_token(protocols, TokenKind::StringRaw),
                symbol_literal(ctx, last_ctx, is_cfm),
            ))(input)
        }
        None => {
            // symbol with one char
            return symbol_literal(ctx, last_ctx, is_cfm)(input);
        }
    }
}

/// 扫描 `#`*N + quote 的开定界符，返回 (hash 数量, 引号字符, 消耗的字节数)
fn hash_quote_prefix(input: Input<'_>) -> Option<(usize, char)> {
    let hashes = input.chars().take_while(|&c| c == '#').count();
    let quote = input.chars().nth(hashes)?;
    matches!(quote, '\'' | '"' | '`').then_some((hashes, quote))
}

fn hashed_literal(
    prefix: &char,
) -> impl FnMut(Input<'_>) -> TokenizationResult<'_, (Token, Diagnostic)> {
    move |input: Input<'_>| {
        if let Some((after_r, _)) = input.strip_prefix(&prefix.to_string()) {
            if let Some((hashes, quote)) = hash_quote_prefix(after_r) {
                let kind = match (prefix, quote) {
                    ('r', '\'') => TokenKind::StringRaw,
                    ('r', '"') => TokenKind::StringLiteral,
                    ('r', '`') => TokenKind::StringTemplate,
                    ('g', _) => TokenKind::Regex,
                    ('t', _) => TokenKind::Time,
                    ('s', _) => TokenKind::StringSafe,
                    ('b', _) => TokenKind::Bytes,
                    _ => return Err(NOT_FOUND),
                };
                return parse_hashed_string(input, hashes, quote, kind);
            }
        }
        Err(NOT_FOUND)
    }
}

fn parse_hashed_string<'a>(
    input: Input<'a>,
    // open_len: usize, // "r" + hashes + quote 的总长度
    hashes: usize,
    quote: char,
    kind: TokenKind,
) -> TokenizationResult<'a, (Token, Diagnostic)> {
    let open_len = 1 + hashes + 1;
    let close_delim: String = std::iter::once(quote)
        .chain(std::iter::repeat('#').take(hashes))
        .collect();

    let src = input.as_ref();
    match src[open_len..].find(close_delim.as_str()) {
        Some(pos) => {
            let total_len = open_len + pos + close_delim.len();
            let (input, full_range) = input.split_at(total_len);
            Ok((
                input,
                (
                    Token::new_quoted(kind, full_range, open_len as u8, close_delim.len() as u8),
                    Diagnostic::Valid,
                ),
            ))
        }
        None => {
            // 找不到收定界符 -> UnterminatedString，range 取到输入末尾
            let (input, full_range) = input.split_at(input.len());

            Ok((
                input,
                (
                    Token::new_quoted(kind, full_range, open_len as u8, 0),
                    Diagnostic::UnterminatedString(full_range),
                ),
            ))
        }
    }
}

fn symbol_literal(
    ctx: Ctx,
    last_ctx: Ctx,
    is_cfm: bool,
) -> impl FnMut(Input<'_>) -> TokenizationResult<'_, (Token, Diagnostic)> {
    move |input: Input<'_>| {
        map_valid_token(
            |input| symbol(input, is_cfm, ctx, last_ctx),
            TokenKind::Symbol,
        )(input)
    }
}

// fn try_map_or_symbol(
//     input: Input<'_>,
//     ctx: Ctx,
//     last_ctx: Ctx,
//     first: char,
//     is_cfm: bool,
// ) -> TokenizationResult<'_, (Token, Diagnostic)> {
//     // H{, M{, S{ — check if followed by {
//     let bytes = input.as_ref().as_bytes();
//     if bytes.len() > 1 && bytes[1] == b'{' {
//         map_valid_token(
//             punctuation_tag(&format!("{first}{{")),
//             TokenKind::Punctuation,
//         )(input)
//     } else {
//         map_valid_token(
//             |input| symbol(input, is_cfm, ctx, last_ctx),
//             TokenKind::Symbol,
//         )(input)
//     }
// }

/// `::` module call infix operator (e.g. `mod::func`).
/// Requires Word context (preceded by identifier) and followed by identifier.
fn colon_dispatch(input: Input<'_>, ctx: Ctx) -> TokenizationResult<'_, (Token, Diagnostic)> {
    match ctx {
        Ctx::Letter => alt((
            map_valid_token(alpha_followed_tag("::"), TokenKind::OperatorInfix),
            map_valid_token(punctuation_tag(":="), TokenKind::Operator),
            map_valid_token(punctuation_tag(":"), TokenKind::Operator), //{k:v} a?b:c
        ))(input),
        Ctx::Start => alt((
            map_valid_token(punctuation_tag(":="), TokenKind::Operator),
            map_valid_token(operator_tag(":"), TokenKind::OperatorPrefix), // a[:2]
        ))(input),
        _ => alt((
            map_valid_token(punctuation_tag(":="), TokenKind::Operator),
            map_valid_token(punctuation_tag(":"), TokenKind::Operator),
        ))(input),
    }
}

/// `@` as prefix operator for decorators (e.g. `@deco`).
/// Requires non-Word context (standalone `@` before identifier).
fn at_dispatch(input: Input<'_>, ctx: Ctx) -> TokenizationResult<'_, (Token, Diagnostic)> {
    match ctx {
        Ctx::Start | Ctx::Space => alt((
            map_valid_token(prefix_tag("@"), TokenKind::OperatorPrefix), // @deco, @(expr)
        ))(input),
        _ => {
            // Inside an identifier, `@` is not valid — fall through to symbol
            map_valid_token(punctuation_tag("@"), TokenKind::Symbol)(input)
        }
    }
}

/// Matches `?`-prefixed multi-char operators (`?+`, `?.`, `??`, `?>`, `?!`, `?:`, `?~`).
fn question_operator(input: Input<'_>) -> TokenizationResult<'_> {
    alt((
        postfix_break_tag("?+"),
        postfix_break_tag("?."),
        postfix_break_tag("??"),
        postfix_break_tag("?>"),
        postfix_break_tag("?!"),
        space_brace_followed_tag("?:"),
        // space_brace_followed_tag("&:"),
        postfix_break_tag("?~"),
    ))(input)
}

fn any_keyword(input: Input<'_>) -> TokenizationResult<'_> {
    alt((
        space_followed_tag("let"),
        space_followed_tag("set"),
        space_followed_tag("alias"),
        space_followed_tag("export"),
        space_brace_followed_tag("if"),
        space_brace_followed_tag("else"),
        space_followed_tag("fn"),
        space_brace_followed_tag("match"),
        space_followed_tag("for"),
        space_followed_tag("in"),
        space_brace_followed_tag("while"),
        space_brace_followed_tag("loop"),
        postfix_break_tag("break"),
        postfix_break_tag("continue"),
        postfix_break_tag("return"),
        postfix_break_tag("shift"),
        space_followed_tag("del"),
        space_followed_tag("use"),
    ))(input)
}

/// Matches a sequence of ASCII punctuation starting with `punct` (length ≥ 2).
///
/// Used for `..` custom operator at expression-start/non-word positions.
#[inline]
fn punct_seq_tag(punct: &str) -> impl '_ + Fn(Input<'_>) -> TokenizationResult<'_> {
    move |input: Input<'_>| {
        if input.starts_with(punct) {
            let places = input.chars().take_while(char::is_ascii_punctuation).count();
            if places > punct.len() {
                return Ok(input.split_at(places));
            }
        }
        Err(NOT_FOUND)
    }
}

/// Path/argument prefix matcher: consumes a `punct` prefix then scans forward
/// until a delimiter or end of input.
///
/// Delimiters: whitespace, `;`, `` ` ``, `)`, `]`, `}`, `|`, `>`.
/// Escape: `\X` skips the next byte (on Unix, `\ ` / `\`"` / `\''` are escaped pairs).
fn path_tag(punct: &str, alone_ok: bool) -> impl '_ + Fn(Input<'_>) -> TokenizationResult<'_> {
    move |input: Input<'_>| {
        if !input.starts_with(punct) {
            return Err(NOT_FOUND);
        }

        let bytes = input.as_ref().as_bytes();
        let prefix_len = punct.len();
        let mut i = prefix_len;

        while i < bytes.len() {
            let b = bytes[i];

            // escape sequence: skip backslash + next byte
            if b == b'\\' {
                i += 1; // skip backslash
                if i < bytes.len() {
                    let char_len = match bytes[i] {
                        0x00..=0x7F => 1,
                        0xC0..=0xDF => 2,
                        0xE0..=0xEF => 3, // 中文字符落在此区间
                        _ => 4,
                    };
                    i += char_len; // skip escaped byte
                    continue;
                }
                // backslash at end of input — treat as literal, stop
                break;
            }

            // delimiter — stop scanning
            if b < 0x80 && is_path_delimiter(b as char) {
                break;
            }

            // skip multi-byte UTF-8: advance by char length
            let char_len = match b {
                0x00..=0x7F => 1,
                0xC0..=0xDF => 2,
                0xE0..=0xEF => 3, // 中文字符落在此区间
                _ => 4,
            };
            i += char_len;
        }

        // need at least 1 byte of content beyond the prefix
        if alone_ok || i > prefix_len {
            Ok(input.split_at(i))
        } else {
            Err(NOT_FOUND)
        }
    }
}

fn last_path_tag(punct: &str) -> impl '_ + Fn(Input<'_>) -> TokenizationResult<'_> {
    move |input: Input<'_>| {
        if !input.starts_with(punct) {
            return Err(NOT_FOUND);
        }
        let bytes = input.as_ref().as_bytes();
        let prefix_len = punct.len();
        let mut i = prefix_len;

        while i < bytes.len() {
            let b = bytes[i];
            if !matches!(b, b' ' | b'\t') {
                if is_path_delimiter(b as char) {
                    return Ok(input.split_at(i));
                }
                return Err(NOT_FOUND);
            }
            i += (b as char).len_utf8();
        }
        return Ok(input.split_at(i));
    }
}

/// Returns true if byte `b` is a path-scanning delimiter.
#[inline]
fn is_path_delimiter(c: char) -> bool {
    c.is_ascii_whitespace() || matches!(c, ';' | '`' | ')' | ']' | '}' | '|' | '>' | ',')
}

#[cfg(windows)]
fn win_abpath_tag(input: Input<'_>) -> TokenizationResult<'_> {
    let mut it = input.chars();
    if input.len() > 1
        && it.next().map_or(false, |c| c.is_ascii_uppercase())
        && it.next().map_or(false, |c| c == ':')
    {
        let byte_len = input
            .chars()
            .take_while(|&c| !is_path_delimiter(c))
            .map(char::len_utf8)
            .sum::<usize>();
        Ok(input.split_at(byte_len))
    } else {
        Err(NOT_FOUND)
    }
}

fn protocols(input: Input<'_>) -> TokenizationResult<'_> {
    alt((
        whole_word("https://"),
        whole_word("http://"),
        whole_word("ftps://"),
        whole_word("ftp://"),
        whole_word("file://"),
    ))(input)
}

fn string_literal(input: Input<'_>) -> TokenizationResult<'_, (Token, Diagnostic)> {
    match input.chars().next() {
        Some('"') => parse_string(input, '"', TokenKind::StringLiteral),
        Some('\'') => parse_string(input, '\'', TokenKind::StringRaw),
        Some('`') => parse_string(input, '`', TokenKind::StringTemplate),
        _ => Err(NOT_FOUND),
    }
}

/// Core string parser: scan for matching close quote, return (rest, (token, diagnostic)).
fn parse_string(
    input: Input<'_>,
    quote: char,
    kind: TokenKind,
) -> TokenizationResult<'_, (Token, Diagnostic)> {
    let quote_str = quote.to_string();
    let (inner, _) = input.strip_prefix(&quote_str).ok_or(NOT_FOUND)?;

    let (rest_after_content, diagnostic) = parse_string_inner(inner, quote)?;

    let (rest, content) = finish_string(input, rest_after_content, quote);

    let token = Token::new_quoted(kind, content, 1, 1);
    Ok((rest, (token, diagnostic)))
}

fn parse_prefixed_string<'a>(
    input: Input<'a>,
    leading: char,
    quote: char,
    kind: TokenKind,
) -> TokenizationResult<'a, (Token, Diagnostic)> {
    let mut prefix = String::with_capacity(2);
    prefix.push(leading);
    prefix.push(quote);

    let (inner, _prefix) = input.strip_prefix(&prefix).ok_or(NOT_FOUND)?;
    let (rest_after_content, diagnostic) = parse_string_inner(inner, quote)?;

    let (rest, content) = finish_string(input, rest_after_content, quote);

    let token = Token::new_quoted(kind, content, 2, 1);
    Ok((rest, (token, diagnostic)))
}

/// Given the full input and the position after the string content,
/// consume the closing quote (if present) and return (rest, content_range).
fn finish_string<'a>(
    input: Input<'a>,
    rest_after_content: Input<'a>,
    quote: char,
) -> (Input<'a>, StrSlice) {
    let quote_str = quote.to_string();
    match rest_after_content.strip_prefix(&quote_str) {
        Some((after_close, _)) => {
            let (_, content) = input.split_until(after_close);
            (after_close, content)
        }
        None => (rest_after_content, input.split_until(rest_after_content).1),
    }
}

fn ip_literal(input: Input<'_>) -> TokenizationResult<'_, (Token, Diagnostic)> {
    // Simple IP detection: four groups of digits separated by dots
    let mut parts = 0;
    let mut i = 0;
    let bytes = input.as_ref().as_bytes();
    while i < bytes.len() {
        // consume digits
        let start = i;
        while i < bytes.len() && bytes[i].is_ascii_digit() {
            i += 1;
        }
        if i == start {
            break;
        }
        parts += 1;
        if parts == 4 {
            break;
        }
        // expect a dot
        if i < bytes.len() && bytes[i] == b'.' {
            i += 1;
        } else {
            break;
        }
    }
    if parts == 4 {
        let (rest, range) = input.split_at(i);
        Ok((
            rest,
            (Token::new(TokenKind::StringRaw, range), Diagnostic::Valid),
        ))
    } else {
        Err(NOT_FOUND)
    }
}

fn radix_literal(input: Input<'_>) -> TokenizationResult<'_, (Token, Diagnostic)> {
    let radix_specs: [(&str, TokenKind, fn(char) -> bool); 3] = [
        ("0b", TokenKind::Radix2, |c: char| c == '0' || c == '1'),
        ("0o", TokenKind::Radix8, |c: char| c.is_digit(8)),
        ("0x", TokenKind::Radix16, |c: char| c.is_ascii_hexdigit()),
    ];

    for (prefix, kind, is_digit) in radix_specs {
        if let Some((after_prefix, _)) = input.strip_prefix(prefix) {
            let places = after_prefix
                .chars()
                .take_while(|&c| is_digit(c) || c == '_')
                .count();

            let (remain, _) = after_prefix.split_at(places);
            let (remain, number) = input.split_until(remain);

            if places == 0 {
                // 前缀后没有合法数字
                return Ok((
                    remain,
                    (
                        Token::new_quoted(kind, number, 2, 0),
                        Diagnostic::InvalidNumber(number),
                    ),
                ));
            }

            return Ok((
                remain,
                (
                    Token::new_quoted(kind, number, 2, 0), // 整体 token 含 "0x..","0b..","0o.."
                    Diagnostic::Valid,
                ),
            ));
        }
    }
    number_literal(input)
}

fn number_literal(input: Input<'_>) -> TokenizationResult<'_, (Token, Diagnostic)> {
    // First, try to parse as an IP address (e.g., 192.168.0.1)
    if let Ok(res) = ip_literal(input) {
        return Ok(res);
    }
    // Original numeric literal parsing
    let bytes = input.as_ref().as_bytes();
    let mut i = 0;
    if bytes.get(0) == Some(&b'.') {
        i += 1;
    }
    // skip leading digits
    let digit_start = i;
    while i < bytes.len() && (bytes[i].is_ascii_digit() || bytes[i] == b'_') {
        i += 1;
    }
    if i == digit_start {
        return Err(NOT_FOUND);
    }

    let has_tailing_dot = bytes.get(i) == Some(&b'.');
    // `N..` → integer literal, leave `..` for range operator
    if has_tailing_dot && bytes.get(i + 1) == Some(&b'.') {
        let (rest, range) = input.split_at(i);
        return Ok((
            rest,
            (
                Token::new(TokenKind::IntegerLiteral, range),
                Diagnostic::Valid,
            ),
        ));
    }

    // optional fractional part
    if has_tailing_dot {
        i += 1;
        let frac_start = i;
        while i < bytes.len() && (bytes[i].is_ascii_digit() || bytes[i] == b'_') {
            i += 1;
        }
        if i == frac_start {
            // `3.` — invalid float (no fractional digits)
            let (rest, range) = input.split_at(i);
            return Ok((
                rest,
                (
                    Token::new(TokenKind::FloatLiteral, range),
                    Diagnostic::InvalidNumber(range),
                ),
            ));
        }
    }

    let (rest, range) = input.split_at(i);

    let kind = if digit_start > 0 || has_tailing_dot {
        TokenKind::FloatLiteral
    } else {
        TokenKind::IntegerLiteral
    };
    Ok((rest, (Token::new(kind, range), Diagnostic::Valid)))
}

fn value_symbol(input: Input<'_>) -> TokenizationResult<'_> {
    alt((
        postfix_break_tag("true"),
        postfix_break_tag("false"),
        postfix_break_tag("none"),
        space_punc_followed_tag("_"),
    ))(input)
}

fn symbol(input: Input<'_>, is_cfm: bool, ctx: Ctx, last_ctx: Ctx) -> TokenizationResult<'_> {
    let len = if is_cfm {
        let is_param_ctx = ctx == Ctx::Space && last_ctx == Ctx::Letter;
        let is_cmd_ctx = ctx == Ctx::Start || last_ctx == Ctx::Start && ctx == Ctx::Space;

        input
            .chars()
            .take_while(|&c| is_symbol_char_cfm(c, is_param_ctx, is_cmd_ctx))
            .count()
    } else {
        input.chars().take_while(|&c| is_symbol_char(c)).count()
    };

    if len == 0 {
        return Err(NOT_FOUND);
    }

    Ok(input.split_at(len))
}

fn whitespace(input: Input<'_>) -> TokenizationResult<'_> {
    let ws_chars = input
        .chars()
        .take_while(|c| matches!(c, ' ' | '\t' | '\0'))
        .count();

    if ws_chars == 0 {
        return Err(NOT_FOUND);
    }
    Ok(input.split_at(ws_chars))
}

fn non_ascii(input: Input<'_>) -> TokenizationResult<'_> {
    // 处理UTF-8连续非ASCII字符
    let len = input
        .chars()
        .take_while(|&c| !c.is_ascii())
        .map(char::len_utf8)
        .sum();

    if len == 0 {
        return Err(NOT_FOUND);
    }
    Ok(input.split_at(len))
}

// fn linebreak(input: Input<'_>) -> TokenizationResult<'_> {
//     let (input, _) = {
//         let ws_chars = input.chars().take_while(|c| *c == ' ').count();
//         if ws_chars > 0 {
//             input.split_at(ws_chars)
//         } else {
//             (input, input.as_str_slice())
//         }
//     };

//     #[cfg(windows)]
//     if let Some((rest, nl_slice)) = input.strip_prefix("\r\n") {
//         return Ok((rest, nl_slice));
//     }

//     if let Some((rest, nl_slice)) = input.strip_prefix("\n") {
//         Ok((rest, nl_slice))
//     } else if let Some((rest, matched)) = input.strip_prefix(";") {
//         Ok((rest, matched))
//     } else {
//         Err(NOT_FOUND)
//     }
// }
// 新增续行符解析函数
fn line_continuation(input: Input<'_>) -> TokenizationResult<'_> {
    if let Some((rest, matched)) = input.strip_prefix("\\\n") {
        Ok((rest, matched))
    } else {
        #[cfg(windows)]
        if let Some((rest, matched)) = input.strip_prefix("\\\r\n") {
            return Ok((rest, matched));
        }
        Err(NOT_FOUND)
    }
}

fn comment(input: Input<'_>) -> TokenizationResult<'_> {
    let len = input
        .chars()
        .take_while(|&c| !matches!(c, '\n' | '\r'))
        .map(char::len_utf8)
        .sum();
    Ok(input.split_at(len))
}

fn parse_string_inner(input: Input<'_>, quote_char: char) -> TokenizationResult<'_, Diagnostic> {
    let start_range = input.as_str_slice();
    let quote_byte = quote_char as u8;

    // 使用字节扫描，极速跳过非转义字符
    let bytes = input.as_ref().as_bytes();
    let mut pos = 0;

    while pos < bytes.len() {
        let b = bytes[pos];
        if b == quote_byte {
            // 匹配到结束引号
            return Ok((input.split_at(pos).0, Diagnostic::Valid));
        } else {
            pos += 1;
            if b == b'\\' && pos + 1 < bytes.len() {
                // 处理转义：跳过反斜杠及其后一个字节
                pos += 1;
            }
        }
    }

    // 未能找到结束引号
    Ok((
        input.split_at(pos).0,
        Diagnostic::UnterminatedString(start_range),
    ))
}

/// Matches a literal string prefix without any continuation restrictions.
#[inline]
fn punctuation_tag(punct: &str) -> impl '_ + Fn(Input<'_>) -> TokenizationResult<'_> {
    move |input: Input<'_>| input.strip_prefix(punct).ok_or(NOT_FOUND)
}

/// Matches a keyword/operator that must NOT be followed by symbol characters.
/// Prevents operators from merging into longer symbols (e.g. `&&` vs `&&&`).
// fn keyword_tag(keyword: &str) -> impl '_ + Fn(Input<'_>) -> TokenizationResult<'_> {
//     move |input: Input<'_>| {
//         input
//             .strip_prefix(keyword)
//             .filter(|(rest, _)| !rest.starts_with(is_symbol_char))
//             .ok_or(NOT_FOUND)
//     }
// }

/// Matches a keyword that must be followed by whitespace (not end-of-input).
/// Used for standalone keywords like `let`, `set`, `if`, `fn`, `match`.
#[inline]
fn space_followed_tag(keyword: &str) -> impl '_ + Fn(Input<'_>) -> TokenizationResult<'_> {
    move |input: Input<'_>| {
        input
            .strip_prefix(keyword)
            .filter(|(rest, _)| rest.starts_with(char::is_whitespace))
            .ok_or(NOT_FOUND)
    }
}
#[inline]
fn alpha_followed_tag(keyword: &str) -> impl '_ + Fn(Input<'_>) -> TokenizationResult<'_> {
    move |input: Input<'_>| {
        input
            .strip_prefix(keyword)
            .filter(|(rest, _)| rest.starts_with(|x| char::is_ascii_alphabetic(&x)))
            .ok_or(NOT_FOUND)
    }
}
#[inline]
fn space_brace_followed_tag(keyword: &str) -> impl '_ + Fn(Input<'_>) -> TokenizationResult<'_> {
    move |input: Input<'_>| {
        input
            .strip_prefix(keyword)
            .filter(|(rest, _)| {
                rest.starts_with(char::is_whitespace)
                    || rest.starts_with('{')
                    || rest.starts_with('(')
            })
            .ok_or(NOT_FOUND)
    }
}

/// Matches an operator that must NOT be followed by ASCII punctuation.
/// Prevents single-char operators from merging into longer sequences (e.g. `+` vs `+=`).
#[inline]
fn operator_tag(keyword: &str) -> impl '_ + Fn(Input<'_>) -> TokenizationResult<'_> {
    move |input: Input<'_>| {
        input
            .strip_prefix(keyword)
            .filter(|(rest, _)| {
                rest.starts_with(|c: char| c.is_whitespace() || !c.is_ascii_punctuation())
            })
            .ok_or(NOT_FOUND)
    }
}

/// Mathes whole word with a prefix
/// similar with path_tag,but don't skip anything. eg `\ `
#[inline]
fn whole_word(prefix: &str) -> impl '_ + Fn(Input<'_>) -> TokenizationResult<'_> {
    move |input: Input<'_>| {
        if input.starts_with(prefix) {
            let len = input
                .chars()
                .take_while(|c| !matches!(c, ' ' | ';' | '\n' | '\t' | '\r' | ')' | ']' | '}'))
                .map(char::len_utf8)
                .sum();
            Ok(input.split_at(len))
        } else {
            Err(NOT_FOUND)
        }
    }
}

/// Matches a token that must be followed by whitespace or punctuation (not letters).
/// Used for `_` to distinguish standalone `_` value from `_` within a symbol like `foo_bar`.
#[inline]
fn space_punc_followed_tag(keyword: &str) -> impl '_ + Fn(Input<'_>) -> TokenizationResult<'_> {
    move |input: Input<'_>| {
        input
            .strip_prefix(keyword)
            .filter(|(rest, _)| {
                rest.is_empty()
                    || rest.starts_with(|c: char| c.is_whitespace() || c.is_ascii_punctuation())
            })
            .ok_or(NOT_FOUND)
    }
}

/// Matches a prefix operator that must be followed by a value-start character.
/// After stripping the prefix, checks the rest starts with alphanumeric, `(`, `[`, `{`, or `$` `.`.
/// This lets the parser decide: `-42` → negation, `-arg` → flag, `-(expr)` → grouped negation.
#[inline]
fn prefix_tag(keyword: &str) -> impl '_ + Fn(Input<'_>) -> TokenizationResult<'_> {
    move |input: Input<'_>| {
        input
            .strip_prefix(keyword)
            .filter(|(rest, _)| {
                rest.starts_with(|c: char| {
                    c.is_ascii_alphanumeric() || matches!(&c, '(' | '[' | '{' | '$' | '.')
                })
            })
            .ok_or(NOT_FOUND)
    }
}

/// Matches a postfix operator that must be followed by whitespace or dilimeter or end-of-input.
/// Used for postfix `!` and `^` to prevent merging with following characters.
#[inline]
fn postfix_break_tag(keyword: &str) -> impl '_ + Fn(Input<'_>) -> TokenizationResult<'_> {
    move |input: Input<'_>| {
        input
            .strip_prefix(keyword)
            .filter(|(rest, _)| rest.is_empty() || rest.starts_with(|c: char| is_path_delimiter(c)))
            .ok_or(NOT_FOUND)
    }
}
/// Checks whether the character is allowed in a symbol.
/// Symbol chars: alphanumeric, `_`, `~`, `?`, `&`, `#`, `$`, `-`, `/`, `\`
/// Excluded (cause operator/punctuation parsing instead): `+`, `=`, `<`, `>`, `*`, `%`, `^`, `|`, `:`, `@`, `!`, `.`, `,`, `;`, `(`, `)`, `[`, `]`, `{`, `}`, `'`, `"`, backtick, whitespace
#[inline]
fn is_symbol_char(c: char) -> bool {
    if c.is_ascii_whitespace() {
        return false;
    }
    // allow `a-b` as symbo
    matches!(
        c,
        'a'..='z' | 'A'..='Z' | '0'..='9' | '_' | '~' | '?' | '&' | '#' | '$' | '@' | '-' | '/' | '\\'
    )
}
#[inline]
fn is_symbol_char_cfm(c: char, is_param_ctx: bool, is_cmd_ctx: bool) -> bool {
    if c.is_ascii_whitespace() {
        return false;
    }

    if is_cmd_ctx {
        // never eat `=`, leave it to env sign
        // never eat `+`, for binary plus
        // never eat `.` for bultin lib call
        return matches!(
            c,
            'a'..='z' | 'A'..='Z' | '0'..='9' | '_' | '~' | '?' | '&' | '#' | '$' | '@' | '-' | '/' | '\\'
        );
    }
    if is_param_ctx {
        // eat `.` only on param_ctx, for 'git tag v0.0.1'
        // eat `:` only on param_ctx, for 'cut -d:'
        // eat `+` for `cmd arg+`
        // eat `=` for `dd if=/dev`
        return matches!(
            c,
            'a'..='z' | 'A'..='Z' | '0'..='9' | '_' | '~' | '?' | '&' | '#' | '$' | '@' | '-' | '/' | '\\' | '=' | '+' | '.' | ':'
        );
    }
    // other ? same to cfm off
    return matches!(
        c,
        'a'..='z' | 'A'..='Z' | '0'..='9' | '_' | '~' | '?' | '&' | '#' | '$' | '@' | '-' | '/' | '\\'
    );
}

/// Main tokenization entry point.
/// - Single-line input (no `:`) → CFM (Command First Mode) for shell-like parsing
/// - Multi-line or `:`-prefixed → Expression mode with context-aware dispatch
pub(crate) fn parse_tokens(input: Input<'_>) -> (Vec<Token>, Vec<Diagnostic>) {
    // if is_cfm_mode(input) {
    //     return parse_command_tokens(input);
    // }
    let is_cfm = is_cfm_mode(input.as_original_str());
    let leading_char = if is_cfm { ">" } else { ":" };

    let mut tokens = Vec::new();
    let mut diagnostics = Vec::new();
    let mut ctx = Ctx::Start;
    let mut last_ctx = Ctx::Start;
    let mut input = input;

    // skip multiline mode prefix `:`
    if let Ok((new_input, (token, diagnostic))) =
        map_valid_token(punctuation_tag(leading_char), TokenKind::ModeTip)(input)
    {
        input = new_input;
        tokens.push(token);
        diagnostics.push(diagnostic);
    }

    // tokenize one by one with context tracking
    loop {
        match parse_token_dispatch(input, ctx, last_ctx, is_cfm) {
            Err(_) => break,
            Ok((new_input, (token, diagnostic))) => {
                last_ctx = ctx;
                ctx = Ctx::after_token(&token, input.as_original_str());
                input = new_input;
                tokens.push(token);
                diagnostics.push(diagnostic);
            }
        }
    }
    if !input.is_empty() {
        diagnostics.push(Diagnostic::NotTokenized(input.as_str_slice()))
    }
    // dbg!(&tokens);
    (tokens, diagnostics)
}

pub fn tokenize(input: &str) -> (Vec<Token>, Vec<Diagnostic>) {
    let str = input.into();
    let input = Input::new(&str);
    parse_tokens(input)
}