kcl-lib 0.2.192

//! Adapts the `kcl-syntax` lexer to the parser's `TokenType` stream.
//!
//! - [`syntax_kind_to_token_type`] maps lexer kinds to parser token types.
//!   Recovery kinds map to `TokenType::Unknown` so invalid input is not accepted as trivia.
//! - [`lex_with_diagnostics`] records lexical errors before recovery kinds are mapped to `Unknown`.
//! - [`keyword_before_paren_as_word`] classifies adjacent `import(` and `use(` tokens as function names.

use kcl_error::KclErrorDetails;
use kcl_syntax::syntax_kind::SyntaxKind;

use super::Token;
use super::TokenStream;
use super::TokenType;
use crate::ModuleId;
use crate::SourceRange;
use crate::errors::KclError;

/// Classification of a recoverable lexical problem, retained from the rich
/// `SyntaxKind` stream so a good message can be produced before the token stream
/// collapses recovery kinds to `TokenType::Unknown`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LexErrorKind {
    UnknownToken,
    UnterminatedString,
    UnterminatedBlockComment,
}

/// A single lexical diagnostic over a byte range of the source.
///
/// Deliberately NOT a [`crate::errors::CompilationIssue`]: that type carries a
/// single range and is recategorised to a *syntax* error by `parse_errs_as_err`.
/// Lexical problems stay lexical by converting to `KclError::new_lexical` (see
/// [`LexResult::to_lexical_error`]).
#[derive(Debug, Clone)]
pub struct LexDiagnostic {
    kind: LexErrorKind,
    /// Offending source text, used to render `found unknown token '<x>'`.
    text: String,
    range: SourceRange,
}

impl LexDiagnostic {
    fn message(&self) -> String {
        match self.kind {
            LexErrorKind::UnknownToken => format!("found unknown token '{}'", self.text),
            LexErrorKind::UnterminatedString => "unterminated string literal".to_owned(),
            LexErrorKind::UnterminatedBlockComment => "unterminated block comment".to_owned(),
        }
    }
}

/// Result of lexing with the new lexer: the (recovery-collapsed) token stream plus
/// every lexical diagnostic found. Callers choose how to surface `issues`: the
/// public parse path folds them into one `KclError::new_lexical` via
/// [`LexResult::to_lexical_error`]; the LSP renders each as a diagnostic while
/// keeping `tokens` for highlighting.
#[derive(Debug)]
pub struct LexResult {
    pub tokens: TokenStream,
    pub issues: Vec<LexDiagnostic>,
}

impl LexResult {
    /// Fold the lexical issues into a single lexical `KclError`, or `None` when
    /// there are none. Matches the legacy public contract: a run of unknown
    /// tokens aggregates into one message with all ranges (mirroring
    /// `parse_tokens`); any other case reports the first issue in source order.
    pub fn to_lexical_error(&self) -> Option<KclError> {
        let issues = &self.issues;
        let first = issues.first()?;

        if issues.iter().all(|issue| issue.kind == LexErrorKind::UnknownToken) {
            let ranges: Vec<SourceRange> = issues.iter().map(|issue| issue.range).collect();
            let message = if issues.len() == 1 {
                format!("found unknown token '{}'", first.text)
            } else {
                let list = issues
                    .iter()
                    .map(|issue| issue.text.as_str())
                    .collect::<Vec<_>>()
                    .join(", ");
                format!("found unknown tokens [{list}]")
            };
            return Some(KclError::new_lexical(KclErrorDetails::new(message, ranges)));
        }

        Some(KclError::new_lexical(KclErrorDetails::new(
            first.message(),
            vec![first.range],
        )))
    }
}

/// Lex `source` with the `kcl-syntax` lexer, mapping to the legacy token stream
/// and collecting lexical diagnostics. Never fails: recoverable problems appear
/// in `tokens` (as `Unknown`) and in `issues`.
pub fn lex_with_diagnostics(source: &str, module_id: ModuleId) -> LexResult {
    let lexed = kcl_syntax::lexer::lex(source);

    let mut tokens: Vec<Token> = Vec::with_capacity(lexed.len());
    let mut issues: Vec<LexDiagnostic> = Vec::new();

    for token in lexed.tokens() {
        let kind = token.kind();
        let range = token.range();
        let text = token.text();

        if let Some(error_kind) = recovery_kind(kind) {
            issues.push(LexDiagnostic {
                kind: error_kind,
                text: text.to_owned(),
                range: SourceRange::new(range.start, range.end, module_id),
            });
        }

        tokens.push(Token::from_range(
            range,
            module_id,
            syntax_kind_to_token_type(kind),
            text.to_owned(),
        ));
    }

    keyword_before_paren_as_word(&mut tokens);

    LexResult {
        tokens: TokenStream::new(tokens),
        issues,
    }
}

/// The recovery classification of a kind, if it is one; `None` for normal kinds.
fn recovery_kind(kind: SyntaxKind) -> Option<LexErrorKind> {
    match kind {
        SyntaxKind::Unknown => Some(LexErrorKind::UnknownToken),
        SyntaxKind::UnterminatedString => Some(LexErrorKind::UnterminatedString),
        SyntaxKind::UnterminatedBlockComment => Some(LexErrorKind::UnterminatedBlockComment),
        _ => None,
    }
}

/// Preserve the function-name exception for keywords immediately followed by
/// `(`. The parser requires a `Word` token for these function names.
fn keyword_before_paren_as_word(tokens: &mut [Token]) {
    for i in 0..tokens.len() {
        if tokens[i].token_type != TokenType::Keyword || !matches!(tokens[i].value.as_str(), "import" | "use") {
            continue;
        }
        let keyword_end = tokens[i].end;
        let followed_by_open_paren = tokens
            .get(i + 1)
            .is_some_and(|next| next.token_type == TokenType::Brace && next.value == "(" && next.start == keyword_end);
        if followed_by_open_paren {
            tokens[i].token_type = TokenType::Word;
        }
    }
}

/// Pure, total mapping from a new-lexer `SyntaxKind` to the legacy `TokenType`.
///
/// `TokenType::Type` and `TokenType::Function` are intentionally never produced
/// here -- the parser assigns those later. Recovery kinds map to `Unknown`.
pub(crate) fn syntax_kind_to_token_type(kind: SyntaxKind) -> TokenType {
    match kind {
        SyntaxKind::Number => TokenType::Number,
        SyntaxKind::Word => TokenType::Word,
        SyntaxKind::GtEq
        | SyntaxKind::LtEq
        | SyntaxKind::EqEq
        | SyntaxKind::FatArrow
        | SyntaxKind::BangEq
        | SyntaxKind::PipeGt
        | SyntaxKind::Star
        | SyntaxKind::Plus
        | SyntaxKind::Minus
        | SyntaxKind::Slash
        | SyntaxKind::Percent
        | SyntaxKind::Eq
        | SyntaxKind::Lt
        | SyntaxKind::Gt
        | SyntaxKind::Backslash
        | SyntaxKind::Caret
        | SyntaxKind::PipePipe
        | SyntaxKind::AmpAmp
        | SyntaxKind::Pipe
        | SyntaxKind::Amp => TokenType::Operator,
        SyntaxKind::String => TokenType::String,
        SyntaxKind::IfKw
        | SyntaxKind::ElseKw
        | SyntaxKind::ForKw
        | SyntaxKind::WhileKw
        | SyntaxKind::ReturnKw
        | SyntaxKind::BreakKw
        | SyntaxKind::ContinueKw
        | SyntaxKind::FnKw
        | SyntaxKind::LetKw
        | SyntaxKind::MutKw
        | SyntaxKind::AsKw
        | SyntaxKind::LoopKw
        | SyntaxKind::TrueKw
        | SyntaxKind::FalseKw
        | SyntaxKind::NilKw
        | SyntaxKind::AndKw
        | SyntaxKind::OrKw
        | SyntaxKind::NotKw
        | SyntaxKind::VarKw
        | SyntaxKind::ConstKw
        | SyntaxKind::ImportKw
        | SyntaxKind::UseKw
        | SyntaxKind::ExportKw
        | SyntaxKind::TypeKw
        | SyntaxKind::InterfaceKw
        | SyntaxKind::NewKw
        | SyntaxKind::SelfKw
        | SyntaxKind::RecordKw
        | SyntaxKind::StructKw
        | SyntaxKind::ObjectKw => TokenType::Keyword,
        SyntaxKind::OpenParen
        | SyntaxKind::CloseParen
        | SyntaxKind::OpenBrace
        | SyntaxKind::CloseBrace
        | SyntaxKind::OpenBracket
        | SyntaxKind::CloseBracket => TokenType::Brace,
        SyntaxKind::Hash => TokenType::Hash,
        SyntaxKind::Bang => TokenType::Bang,
        SyntaxKind::Dollar => TokenType::Dollar,
        SyntaxKind::Whitespace => TokenType::Whitespace,
        SyntaxKind::Comma => TokenType::Comma,
        SyntaxKind::Colon => TokenType::Colon,
        SyntaxKind::DoubleColon => TokenType::DoubleColon,
        SyntaxKind::Period => TokenType::Period,
        SyntaxKind::DoublePeriod => TokenType::DoublePeriod,
        SyntaxKind::DoublePeriodLessThan => TokenType::DoublePeriodLessThan,
        SyntaxKind::LineComment => TokenType::LineComment,
        SyntaxKind::BlockComment => TokenType::BlockComment,
        // Recovery kinds collapse to Unknown so the parser can never treat
        // invalid input as trivia; the rich pass has already recorded a
        // diagnostic with the precise kind.
        SyntaxKind::UnterminatedString | SyntaxKind::UnterminatedBlockComment | SyntaxKind::Unknown => {
            TokenType::Unknown
        }
        SyntaxKind::QuestionMark => TokenType::QuestionMark,
        SyntaxKind::At => TokenType::At,
        SyntaxKind::SemiColon => TokenType::SemiColon,
    }
}

#[cfg(test)]
mod tests {
    use std::ops::Range;

    use super::*;

    #[test]
    fn adapter_preserves_source_text_and_ranges() {
        let source = "use(3)\n亞當 = 🙂\n\"unterminated";
        let lexed = kcl_syntax::lexer::lex(source);
        let result = lex_with_diagnostics(source, ModuleId::default());
        let expected: Vec<_> = lexed
            .tokens()
            .iter()
            .map(|token| (token.text(), token.range()))
            .collect();
        let actual: Vec<_> = result
            .tokens
            .iter()
            .map(|token| (token.value.as_str(), token.start..token.end))
            .collect();
        assert_eq!(actual, expected);
    }

    #[test]
    fn unknown_token_errors_report_source_byte_ranges() {
        for (source, expected_message, expected_range) in [
            ("~", "found unknown token '~'", 0..1),
            ("12 ~ 8", "found unknown token '~'", 3..4),
            ("a\u{00A0}b", "found unknown token '\u{00A0}'", 1..3),
            ("a\u{1F642}b", "found unknown token '\u{1F642}'", 1..5),
            ("~ \u{4E9E}", "found unknown token '~'", 0..1),
        ] {
            assert_lexical_error(source, expected_message, expected_range);
        }
    }

    #[test]
    fn unterminated_string_errors_report_source_byte_ranges() {
        for (source, expected_range) in [
            ("\"abc", 0..4),
            ("\"", 0..1),
            ("{\"\u{78E}\u{78E}\0\0\0\"\".", 10..12),
            ("(/=e\"\u{616}\u{75D}\"\"", 10..11),
        ] {
            assert_lexical_error(source, "unterminated string literal", expected_range);
        }
    }

    #[test]
    fn unterminated_block_comment_error_reports_source_byte_range() {
        assert_lexical_error("/* abc", "unterminated block comment", 0..6);
    }

    #[test]
    fn valid_strings_are_single_tokens_without_lexical_issues() {
        for source in ["\"line one\nline two\"", "\"// a comment\""] {
            let result = lex_with_diagnostics(source, ModuleId::default());
            assert!(result.issues.is_empty(), "{source:?}");
            let tokens: Vec<_> = result
                .tokens
                .iter()
                .map(|token| (token.token_type, token.start, token.end))
                .collect();
            assert_eq!(tokens, vec![(TokenType::String, 0, source.len())], "{source:?}");
        }
    }

    fn assert_lexical_error(source: &str, expected_message: &str, expected_range: Range<usize>) {
        let module_id = ModuleId::default();
        let result = lex_with_diagnostics(source, module_id);
        let error = result
            .to_lexical_error()
            .expect("invalid input should produce a lexical error");
        assert_eq!(error.error_type(), "lexical", "{source:?}");
        assert_eq!(error.message(), expected_message, "{source:?}");
        assert_eq!(
            error.source_ranges(),
            vec![SourceRange::new(expected_range.start, expected_range.end, module_id)],
            "{source:?}"
        );
    }
}