tessellate-core 0.9.0

use crate::diagnostic::Diagnostic;
use crate::source::{SourceFile, Span};
use unicode_normalization::UnicodeNormalization;

/// A reserved word in the Tess surface language.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Keyword {
    Mod,
    Use,
    Enum,
    Record,
    Fn,
    Rule,
    Fixture,
    Test,
    Assert,
    Override,
    Let,
    From,
    Expect,
    Some,
    O,
    X,
    Unknown,
}

impl Keyword {
    #[must_use]
    pub const fn as_str(self) -> &'static str {
        match self {
            Self::Mod => "mod",
            Self::Use => "use",
            Self::Enum => "enum",
            Self::Record => "record",
            Self::Fn => "fn",
            Self::Rule => "rule",
            Self::Fixture => "fixture",
            Self::Test => "test",
            Self::Assert => "assert",
            Self::Override => "override",
            Self::Let => "let",
            Self::From => "from",
            Self::Expect => "expect",
            Self::Some => "some",
            Self::O => "O",
            Self::X => "X",
            Self::Unknown => "unknown",
        }
    }

    fn from_identifier(identifier: &str) -> Option<Self> {
        Some(match identifier {
            "mod" => Self::Mod,
            "use" => Self::Use,
            "enum" => Self::Enum,
            "record" => Self::Record,
            "fn" => Self::Fn,
            "rule" => Self::Rule,
            "fixture" => Self::Fixture,
            "test" => Self::Test,
            "assert" => Self::Assert,
            "override" => Self::Override,
            "let" => Self::Let,
            "from" => Self::From,
            "expect" => Self::Expect,
            "some" => Self::Some,
            "O" => Self::O,
            "X" => Self::X,
            "unknown" => Self::Unknown,
            _ => return None,
        })
    }

    /// Whether this word may be used as an identifier outside its grammar context.
    #[must_use]
    pub const fn is_contextual(self) -> bool {
        matches!(
            self,
            Self::Rule
                | Self::Fixture
                | Self::Test
                | Self::Assert
                | Self::Override
                | Self::Let
                | Self::From
                | Self::Expect
                | Self::Some
        )
    }
}

/// Return the NFC spelling of a valid Tess identifier.
///
/// Contextual keywords are accepted because the parser accepts them anywhere
/// an identifier is unambiguous. Hard keywords such as `record`, `O`, and `X`
/// are rejected.
#[must_use]
pub fn canonical_identifier(identifier: &str) -> Option<String> {
    let normalized = identifier.nfc().collect::<String>();
    let mut characters = normalized.chars();
    let first = characters.next()?;
    if first != '_' && !unicode_ident::is_xid_start(first) {
        return None;
    }
    if !characters.all(|character| character == '_' || unicode_ident::is_xid_continue(character)) {
        return None;
    }
    if Keyword::from_identifier(&normalized).is_some_and(|keyword| !keyword.is_contextual()) {
        return None;
    }
    Some(normalized)
}

/// The semantic kind of a lexical token.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum TokenKind {
    Identifier(String),
    Number(String),
    String(String),
    RawText(String),
    Keyword(Keyword),
    Indent,
    Dedent,
    LeftBrace,
    RightBrace,
    LeftParen,
    RightParen,
    LeftBracket,
    RightBracket,
    Colon,
    Comma,
    Dot,
    Question,
    Semicolon,
    Equal,
    NotEqual,
    Greater,
    GreaterEqual,
    Less,
    LessEqual,
    Plus,
    Minus,
    Star,
    Slash,
    Ampersand,
    Pipe,
    Newline,
    Eof,
}

impl TokenKind {
    #[must_use]
    pub fn description(&self) -> &'static str {
        match self {
            Self::Identifier(_) => "identifier",
            Self::Number(_) => "number",
            Self::String(_) => "string",
            Self::RawText(_) => "raw text",
            Self::Keyword(_) => "keyword",
            Self::Indent => "indent",
            Self::Dedent => "dedent",
            Self::LeftBrace => "`{`",
            Self::RightBrace => "`}`",
            Self::LeftParen => "`(`",
            Self::RightParen => "`)`",
            Self::LeftBracket => "`[`",
            Self::RightBracket => "`]`",
            Self::Colon => "`:`",
            Self::Comma => "`,`",
            Self::Dot => "`.`",
            Self::Question => "`?`",
            Self::Semicolon => "`;`",
            Self::Equal => "`=`",
            Self::NotEqual => "`!=`",
            Self::Greater => "`>`",
            Self::GreaterEqual => "`>=`",
            Self::Less => "`<`",
            Self::LessEqual => "`<=`",
            Self::Plus => "`+`",
            Self::Minus => "`-`",
            Self::Star => "`*`",
            Self::Slash => "`/`",
            Self::Ampersand => "`&`",
            Self::Pipe => "`|`",
            Self::Newline => "newline",
            Self::Eof => "end of file",
        }
    }
}

/// A token paired with its half-open byte range in the original source.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Token {
    pub kind: TokenKind,
    pub span: Span,
}

/// A comment retained as source trivia.
///
/// Comments are kept out of the parser token stream so they cannot affect the
/// grammar, while formatters and other source-aware tools can still preserve
/// their original spelling and placement.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Comment {
    /// The complete comment text, including its `#`, `#<`, or `#>` marker.
    ///
    /// A block comment includes the physical newlines between its marker lines,
    /// but not the newline following its closing `#>` marker.
    pub text: String,
    pub span: Span,
}

#[derive(Clone, Debug, Default)]
pub struct LexOutput {
    pub tokens: Vec<Token>,
    pub comments: Vec<Comment>,
    pub diagnostics: Vec<Diagnostic>,
}

/// Tokenize a UTF-8 source file. Newlines are significant and are retained.
#[must_use]
pub fn lex(source: &SourceFile) -> LexOutput {
    Lexer::new(&source.text).run()
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum SuiteKind {
    Code,
    Fragment,
}

#[derive(Clone, Copy, Debug)]
struct PendingSuite {
    kind: SuiteKind,
    base_indent: usize,
    colon_span: Span,
}

#[derive(Clone, Copy, Debug)]
struct LineInfo {
    start: usize,
    indent_end: usize,
    indent_width: usize,
    first_tab: Option<usize>,
    end: usize,
    newline_end: usize,
    blank: bool,
}

struct Lexer<'a> {
    text: &'a str,
    cursor: usize,
    output: LexOutput,
    indent_stack: Vec<usize>,
    at_line_start: bool,
    logical_line_start: usize,
    delimiter_depth: usize,
    pending_suite: Option<PendingSuite>,
}

impl<'a> Lexer<'a> {
    fn new(text: &'a str) -> Self {
        Self {
            text,
            cursor: 0,
            output: LexOutput::default(),
            indent_stack: vec![0],
            at_line_start: true,
            logical_line_start: 0,
            delimiter_depth: 0,
            pending_suite: None,
        }
    }

    fn run(mut self) -> LexOutput {
        while self.peek().is_some() {
            if self.at_line_start {
                if self.try_lex_fragment_suite() {
                    continue;
                }
                self.prepare_line_start();
                if self.peek().is_none() {
                    break;
                }
            }

            let character = self.peek().expect("the cursor was checked above");
            let start = self.cursor;
            match character {
                ' ' | '\t' | '\u{000C}' => {
                    self.bump();
                }
                '\n' | '\r' => self.lex_newline(start),
                '#' if self.is_first_non_whitespace_on_line(start)
                    && self.peek_second() == Some('<') =>
                {
                    self.lex_block_comment(start);
                }
                '#' if self.is_first_non_whitespace_on_line(start)
                    && self.peek_second() == Some('>') =>
                {
                    self.report_marker_trailing_text(start);
                    self.output.diagnostics.push(
                        Diagnostic::error(
                            "L0008",
                            "unmatched block comment closing marker",
                            Span::new(start, start + 2),
                        )
                        .with_help("remove `#>` or add a matching `#<` marker before it"),
                    );
                    self.lex_line_comment(start);
                }
                '#' => self.lex_line_comment(start),
                '"' => self.lex_string(start),
                '0'..='9' => self.lex_number(start),
                '_' => self.lex_identifier(start),
                value if unicode_ident::is_xid_start(value) => self.lex_identifier(start),
                '{' => self.single(TokenKind::LeftBrace),
                '}' => self.single(TokenKind::RightBrace),
                '(' => self.single(TokenKind::LeftParen),
                ')' => self.single(TokenKind::RightParen),
                '[' => self.single(TokenKind::LeftBracket),
                ']' => self.single(TokenKind::RightBracket),
                ':' => self.single(TokenKind::Colon),
                ',' => self.single(TokenKind::Comma),
                '?' => self.single(TokenKind::Question),
                ';' => self.single(TokenKind::Semicolon),
                '+' => self.single(TokenKind::Plus),
                '*' => self.single(TokenKind::Star),
                '/' => self.single(TokenKind::Slash),
                '&' => self.single(TokenKind::Ampersand),
                '|' => self.single(TokenKind::Pipe),
                '.' => self.single(TokenKind::Dot),
                '-' => self.single(TokenKind::Minus),
                '=' => self.single(TokenKind::Equal),
                '!' if self.peek_second() == Some('=') => self.double(TokenKind::NotEqual),
                '>' if self.peek_second() == Some('=') => self.double(TokenKind::GreaterEqual),
                '>' => self.single(TokenKind::Greater),
                '<' if self.peek_second() == Some('=') => self.double(TokenKind::LessEqual),
                '<' => self.single(TokenKind::Less),
                _ => {
                    self.bump();
                    self.output.diagnostics.push(
                        Diagnostic::error(
                            "L0001",
                            format!("unexpected character `{character}`"),
                            Span::new(start, self.cursor),
                        )
                        .with_help("remove the character or replace it with a Tess token"),
                    );
                }
            }
        }

        if self.delimiter_depth == 0 {
            self.finish_logical_line();
        }
        if let Some(pending) = self.pending_suite.take() {
            self.report_expected_suite(pending);
        }
        while self.indent_stack.len() > 1 {
            self.indent_stack.pop();
            self.emit_layout(TokenKind::Dedent, Span::new(self.cursor, self.cursor));
        }
        self.output.tokens.push(Token {
            kind: TokenKind::Eof,
            span: Span::new(self.cursor, self.cursor),
        });
        self.output
    }

    fn prepare_line_start(&mut self) {
        let line = self.line_info(self.cursor);
        self.report_leading_tab(line);
        self.cursor = line.indent_end;
        self.at_line_start = false;

        let comment_only = self.text[line.indent_end..line.end].starts_with('#');
        if line.blank || comment_only || self.delimiter_depth > 0 {
            return;
        }

        let valid_multiple = line.indent_width % 4 == 0;
        if !valid_multiple {
            self.output.diagnostics.push(
                Diagnostic::error(
                    "L0010",
                    "indentation must be a multiple of 4 spaces",
                    Span::new(line.start, line.indent_end),
                )
                .with_help("use 4 spaces for each structural depth"),
            );
        }

        self.apply_layout(line, valid_multiple);
    }

    fn try_lex_fragment_suite(&mut self) -> bool {
        let Some(pending) = self
            .pending_suite
            .filter(|pending| pending.kind == SuiteKind::Fragment)
        else {
            return false;
        };
        if self.delimiter_depth > 0 {
            return false;
        }

        let baseline = pending.base_indent + 4;
        let block_start = self.cursor;
        let mut scan = self.cursor;
        let mut lines = Vec::new();
        let mut last_raw_content = None;
        let mut blank_after_content = false;
        let mut code_start = None;

        while scan < self.text.len() {
            let line = self.line_info(scan);
            if !line.blank && line.indent_width < baseline {
                break;
            }
            if !line.blank
                && line.indent_width == baseline
                && blank_after_content
                && (self.is_fragment_code_start(line)
                    || (self.is_code_comment_line(line)
                        && self.fragment_code_follows_trivia(line.start, baseline)))
            {
                code_start = Some(line.start);
                break;
            }
            lines.push(line);
            if line.blank {
                blank_after_content = last_raw_content.is_some();
            } else {
                last_raw_content = Some(lines.len() - 1);
                blank_after_content = false;
            }
            if line.newline_end == scan {
                break;
            }
            scan = line.newline_end;
        }

        let Some(last_raw_content) = last_raw_content else {
            self.pending_suite = None;
            self.report_expected_suite(pending);
            return false;
        };

        self.pending_suite = None;
        let first_content_index = lines
            .iter()
            .position(|line| !line.blank)
            .expect("a raw suite with content has a first content line");
        let first_content = lines[first_content_index];
        let (first_content_start, _) = self.raw_content_start(first_content, baseline);

        self.indent_stack.push(baseline);
        self.emit_layout(
            TokenKind::Indent,
            Span::new(first_content.start, first_content_start),
        );

        let mut value = String::new();
        for line in lines.iter().take(last_raw_content + 1) {
            let (content_start, tab) = self.raw_content_start(*line, baseline);
            if let Some(tab) = tab {
                self.report_raw_indent_tab(tab);
            }
            value.push_str(&self.text[content_start..line.end]);
            if line.newline_end > line.end {
                value.push('\n');
            }
        }

        let raw_end = lines[last_raw_content].newline_end;
        self.emit_layout(TokenKind::RawText(value), Span::new(block_start, raw_end));

        self.cursor = code_start.unwrap_or(scan);
        self.at_line_start = true;
        self.logical_line_start = self.output.tokens.len();
        true
    }

    fn is_fragment_code_start(&self, line: LineInfo) -> bool {
        let content = self.text[line.indent_end..line.end]
            .split_once('#')
            .map_or_else(
                || self.text[line.indent_end..line.end].trim_end(),
                |(code, _)| code.trim_end(),
            );
        if let Some(header) = content
            .strip_prefix("rule ")
            .and_then(|rest| rest.strip_suffix(':'))
        {
            if let Some((name, parameters)) = header.split_once('(') {
                return canonical_identifier(name).as_deref() == Some(name)
                    && parameters.strip_suffix(')').is_some();
            }
            return canonical_identifier(header).as_deref() == Some(header);
        }
        let Some(header) = content
            .strip_prefix("fn ")
            .and_then(|rest| rest.strip_suffix(':'))
        else {
            return false;
        };
        let Some(open) = header.find('(') else {
            let mut parts = header.split_whitespace();
            let Some(name) = parts.next() else {
                return false;
            };
            let return_type = parts.next();
            return parts.next().is_none()
                && canonical_identifier(name).as_deref() == Some(name)
                && return_type.is_none_or(Self::is_qualified_identifier_text);
        };
        let Some(close_offset) = header[open + 1..].find(')') else {
            return false;
        };
        let close = open + 1 + close_offset;
        let name = &header[..open];
        let parameters = &header[open + 1..close];
        let return_type = header[close + 1..].trim();
        let valid_return_type =
            return_type.is_empty() || Self::is_qualified_identifier_text(return_type);
        canonical_identifier(name).as_deref() == Some(name)
            && !parameters.contains('(')
            && !parameters.contains(')')
            && valid_return_type
    }

    fn is_code_comment_line(&self, line: LineInfo) -> bool {
        self.text[line.indent_end..line.end].starts_with('#')
    }

    fn fragment_code_follows_trivia(&self, start: usize, baseline: usize) -> bool {
        let mut scan = start;
        let mut in_block = false;
        while scan < self.text.len() {
            let line = self.line_info(scan);
            if in_block {
                if self.text[line.indent_end..line.end].trim_end() == "#>" {
                    in_block = false;
                }
            } else if line.blank {
                // Blank lines are comment trivia once a code boundary is possible.
            } else if line.indent_width != baseline {
                return false;
            } else {
                let content = self.text[line.indent_end..line.end].trim_end();
                if content == "#<" {
                    in_block = true;
                } else if content.starts_with('#') {
                    // A same-depth line comment is code trivia.
                } else {
                    return self.is_fragment_code_start(line);
                }
            }
            if line.newline_end == scan {
                return false;
            }
            scan = line.newline_end;
        }
        false
    }

    fn is_qualified_identifier_text(text: &str) -> bool {
        !text.is_empty()
            && text
                .split('.')
                .all(|segment| canonical_identifier(segment).as_deref() == Some(segment))
    }

    fn raw_content_start(&self, line: LineInfo, baseline: usize) -> (usize, Option<usize>) {
        let mut width = 0;
        let mut content_start = line.start;
        let mut first_tab = None;
        for (offset, character) in self.text[line.start..line.indent_end].char_indices() {
            if width >= baseline {
                break;
            }
            match character {
                ' ' => width += 1,
                '\t' => {
                    first_tab.get_or_insert(line.start + offset);
                    width += 4 - width % 4;
                }
                _ => break,
            }
            content_start = line.start + offset + character.len_utf8();
        }
        (content_start, first_tab)
    }

    fn report_raw_indent_tab(&mut self, tab: usize) {
        self.output.diagnostics.push(
            Diagnostic::error(
                "L0009",
                "tabs are not allowed in leading indentation",
                Span::new(tab, tab + 1),
            )
            .with_help("replace the tab with spaces"),
        );
    }

    fn apply_layout(&mut self, line: LineInfo, valid_multiple: bool) {
        if let Some(pending) = self.pending_suite.take() {
            let expected = pending.base_indent + 4;
            if line.indent_width <= pending.base_indent {
                self.report_expected_suite(pending);
            } else {
                let effective = if valid_multiple {
                    line.indent_width
                } else {
                    expected
                };
                if valid_multiple && line.indent_width != expected {
                    self.output.diagnostics.push(
                        Diagnostic::error(
                            "L0011",
                            "a suite must increase indentation by exactly 4 spaces",
                            Span::new(line.start, line.indent_end),
                        )
                        .with_label(pending.colon_span, "this suite starts here")
                        .with_help(format!("indent this line by {expected} spaces")),
                    );
                }
                self.indent_stack.push(effective);
                self.emit_layout(TokenKind::Indent, Span::new(line.start, line.indent_end));
                return;
            }
        }

        let current = *self
            .indent_stack
            .last()
            .expect("the indentation stack always contains zero");
        let effective = if valid_multiple {
            line.indent_width
        } else if line.indent_width > current {
            current
        } else {
            self.indent_stack
                .iter()
                .copied()
                .rev()
                .find(|indent| *indent <= line.indent_width)
                .unwrap_or(0)
        };

        if effective > current {
            self.output.diagnostics.push(
                Diagnostic::error(
                    "L0011",
                    "unexpected indentation",
                    Span::new(line.start, line.indent_end),
                )
                .with_help("only a header ending in `:` may introduce an indented suite"),
            );
            self.indent_stack.push(effective);
            self.emit_layout(TokenKind::Indent, Span::new(line.start, line.indent_end));
            return;
        }

        while self
            .indent_stack
            .last()
            .is_some_and(|indent| *indent > effective)
        {
            self.indent_stack.pop();
            self.emit_layout(
                TokenKind::Dedent,
                Span::new(line.indent_end, line.indent_end),
            );
        }

        if valid_multiple
            && self
                .indent_stack
                .last()
                .is_some_and(|indent| *indent != effective)
        {
            self.output.diagnostics.push(
                Diagnostic::error(
                    "L0012",
                    "dedent does not match an enclosing indentation level",
                    Span::new(line.start, line.indent_end),
                )
                .with_help("align this line with an existing 4-space indentation level"),
            );
        }
    }

    fn finish_logical_line(&mut self) {
        let semantic: Vec<&Token> = self.output.tokens[self.logical_line_start..]
            .iter()
            .filter(|token| {
                !matches!(
                    token.kind,
                    TokenKind::Indent | TokenKind::Dedent | TokenKind::Newline
                )
            })
            .collect();
        let Some(last) = semantic.last() else {
            return;
        };
        if last.kind != TokenKind::Colon {
            return;
        }

        let kind = if self.indent_stack.last() == Some(&0) && Self::is_fragment_header(&semantic) {
            SuiteKind::Fragment
        } else {
            SuiteKind::Code
        };
        self.pending_suite = Some(PendingSuite {
            kind,
            base_indent: *self
                .indent_stack
                .last()
                .expect("the indentation stack always contains zero"),
            colon_span: last.span,
        });
    }

    fn is_fragment_header(tokens: &[&Token]) -> bool {
        let Some((last, body)) = tokens.split_last() else {
            return false;
        };
        if last.kind != TokenKind::Colon {
            return false;
        }
        Self::is_qualified_identifier_tokens(body)
    }

    fn is_qualified_identifier_tokens(tokens: &[&Token]) -> bool {
        !tokens.is_empty()
            && tokens.iter().enumerate().all(|(index, token)| {
                if index % 2 == 0 {
                    matches!(token.kind, TokenKind::Identifier(_))
                        || (index > 0
                            && matches!(
                                token.kind,
                                TokenKind::Keyword(keyword) if keyword.is_contextual()
                            ))
                } else {
                    token.kind == TokenKind::Dot
                }
            })
            && tokens.len() % 2 == 1
    }

    fn report_expected_suite(&mut self, pending: PendingSuite) {
        let expected = pending.base_indent + 4;
        let description = match pending.kind {
            SuiteKind::Code => "expected an indented suite after `:`",
            SuiteKind::Fragment => "expected indented raw text after the fragment header",
        };
        self.output.diagnostics.push(
            Diagnostic::error("L0013", description, pending.colon_span)
                .with_help(format!("indent the next content line by {expected} spaces")),
        );
    }

    fn emit_layout(&mut self, kind: TokenKind, span: Span) {
        self.output.tokens.push(Token { kind, span });
    }

    fn report_leading_tab(&mut self, line: LineInfo) {
        let Some(tab) = line.first_tab else {
            return;
        };
        self.output.diagnostics.push(
            Diagnostic::error(
                "L0009",
                "tabs are not allowed in leading indentation",
                Span::new(tab, tab + 1),
            )
            .with_help("replace the tab with spaces"),
        );
    }

    fn line_info(&self, start: usize) -> LineInfo {
        let end = self.line_end(start);
        let mut indent_end = start;
        let mut indent_width = 0;
        let mut first_tab = None;
        for (offset, character) in self.text[start..end].char_indices() {
            match character {
                ' ' => {
                    indent_width += 1;
                    indent_end = start + offset + 1;
                }
                '\t' => {
                    first_tab.get_or_insert(start + offset);
                    indent_width += 4 - indent_width % 4;
                    indent_end = start + offset + 1;
                }
                _ => break,
            }
        }
        let newline_end = if end == self.text.len() {
            end
        } else if self.text.as_bytes().get(end) == Some(&b'\r')
            && self.text.as_bytes().get(end + 1) == Some(&b'\n')
        {
            end + 2
        } else {
            end + 1
        };
        LineInfo {
            start,
            indent_end,
            indent_width,
            first_tab,
            end,
            newline_end,
            blank: indent_end == end,
        }
    }

    fn peek(&self) -> Option<char> {
        self.text[self.cursor..].chars().next()
    }

    fn peek_second(&self) -> Option<char> {
        let mut characters = self.text[self.cursor..].chars();
        characters.next()?;
        characters.next()
    }

    fn bump(&mut self) -> Option<char> {
        let character = self.peek()?;
        self.cursor += character.len_utf8();
        Some(character)
    }

    fn push(&mut self, kind: TokenKind, start: usize) {
        self.output.tokens.push(Token {
            kind,
            span: Span::new(start, self.cursor),
        });
    }

    fn single(&mut self, kind: TokenKind) {
        let start = self.cursor;
        self.bump();
        match kind {
            TokenKind::LeftBrace | TokenKind::LeftParen | TokenKind::LeftBracket => {
                self.delimiter_depth += 1;
            }
            TokenKind::RightBrace | TokenKind::RightParen | TokenKind::RightBracket => {
                self.delimiter_depth = self.delimiter_depth.saturating_sub(1);
            }
            _ => {}
        }
        self.push(kind, start);
    }

    fn double(&mut self, kind: TokenKind) {
        let start = self.cursor;
        self.bump();
        self.bump();
        self.push(kind, start);
    }

    fn lex_newline(&mut self, start: usize) {
        let structural = self.delimiter_depth == 0;
        if structural {
            self.finish_logical_line();
        }
        if self.bump() == Some('\r') && self.peek() == Some('\n') {
            self.bump();
        }
        if structural {
            self.push(TokenKind::Newline, start);
            self.logical_line_start = self.output.tokens.len();
        }
        self.at_line_start = true;
    }

    fn lex_line_comment(&mut self, start: usize) {
        while !matches!(self.peek(), None | Some('\n' | '\r')) {
            self.bump();
        }
        self.output.comments.push(Comment {
            text: self.text[start..self.cursor].to_owned(),
            span: Span::new(start, self.cursor),
        });
    }

    fn lex_block_comment(&mut self, start: usize) {
        self.report_marker_trailing_text(start);
        let mut terminated = false;

        while self.peek().is_some() {
            if self.cursor != start
                && self.peek() == Some('#')
                && self.peek_second() == Some('>')
                && self.is_first_non_whitespace_on_line(self.cursor)
            {
                self.report_marker_trailing_text(self.cursor);
                self.consume_to_line_end();
                terminated = true;
                break;
            }

            if matches!(self.peek(), Some('\n' | '\r')) {
                self.lex_newline(self.cursor);
            } else {
                self.bump();
            }
        }

        self.output.comments.push(Comment {
            text: self.text[start..self.cursor].to_owned(),
            span: Span::new(start, self.cursor),
        });
        self.at_line_start = self.cursor == 0
            || self.text[..self.cursor].ends_with('\n')
            || self.text[..self.cursor].ends_with('\r');

        if !terminated {
            self.output.diagnostics.push(
                Diagnostic::error(
                    "L0007",
                    "unterminated block comment",
                    Span::new(start, self.cursor),
                )
                .with_help("add a `#>` marker on its own line"),
            );
        }
    }

    fn is_first_non_whitespace_on_line(&self, byte: usize) -> bool {
        let line_start = self.text[..byte]
            .rfind(['\n', '\r'])
            .map_or(0, |newline| newline + 1);
        self.text[line_start..byte]
            .chars()
            .all(Self::is_horizontal_whitespace)
    }

    fn report_marker_trailing_text(&mut self, marker_start: usize) {
        let line_end = self.line_end(marker_start + 2);
        let trailing = &self.text[marker_start + 2..line_end];
        let Some(relative_start) = trailing.char_indices().find_map(|(index, character)| {
            (!Self::is_horizontal_whitespace(character)).then_some(index)
        }) else {
            return;
        };
        let trailing_start = marker_start + 2 + relative_start;
        self.output.diagnostics.push(
            Diagnostic::error(
                "L0006",
                "block comment marker must be alone on its line",
                Span::new(trailing_start, line_end),
            )
            .with_help("move comment text between the `#<` and `#>` marker lines"),
        );
    }

    fn consume_to_line_end(&mut self) {
        while !matches!(self.peek(), None | Some('\n' | '\r')) {
            self.bump();
        }
    }

    fn line_end(&self, start: usize) -> usize {
        self.text[start..]
            .find(['\n', '\r'])
            .map_or(self.text.len(), |offset| start + offset)
    }

    const fn is_horizontal_whitespace(character: char) -> bool {
        matches!(character, ' ' | '\t')
    }

    fn lex_identifier(&mut self, start: usize) {
        self.bump();
        while self
            .peek()
            .is_some_and(|value| value == '_' || unicode_ident::is_xid_continue(value))
        {
            self.bump();
        }
        let raw = &self.text[start..self.cursor];
        let normalized: String = raw.nfc().collect();
        if let Some(keyword) = Keyword::from_identifier(&normalized) {
            self.push(TokenKind::Keyword(keyword), start);
        } else {
            self.push(TokenKind::Identifier(normalized), start);
        }
    }

    fn lex_number(&mut self, start: usize) {
        while self
            .peek()
            .is_some_and(|value| value.is_ascii_digit() || value == '_')
        {
            self.bump();
        }
        if self.peek() == Some('.')
            && self
                .peek_second()
                .is_some_and(|value| value.is_ascii_digit())
        {
            self.bump();
            while self
                .peek()
                .is_some_and(|value| value.is_ascii_digit() || value == '_')
            {
                self.bump();
            }
        }
        let text = &self.text[start..self.cursor];
        if text.as_bytes().iter().enumerate().any(|(index, value)| {
            *value == b'_'
                && (!index
                    .checked_sub(1)
                    .and_then(|previous| text.as_bytes().get(previous))
                    .is_some_and(u8::is_ascii_digit)
                    || !text
                        .as_bytes()
                        .get(index + 1)
                        .is_some_and(u8::is_ascii_digit))
        }) {
            self.output.diagnostics.push(
                Diagnostic::error(
                    "L0005",
                    "invalid `_` placement in numeric literal",
                    Span::new(start, self.cursor),
                )
                .with_help("place `_` only between digits, for example `100_000`"),
            );
        }
        self.push(TokenKind::Number(text.to_owned()), start);
    }

    fn lex_string(&mut self, start: usize) {
        self.bump();
        let mut value = String::new();
        let mut terminated = false;
        while let Some(character) = self.peek() {
            match character {
                '"' => {
                    self.bump();
                    terminated = true;
                    break;
                }
                '\n' | '\r' => break,
                '\\' => {
                    let escape_start = self.cursor;
                    self.bump();
                    self.lex_escape(&mut value, escape_start);
                }
                other => {
                    self.bump();
                    value.push(other);
                }
            }
        }
        if !terminated {
            self.output.diagnostics.push(
                Diagnostic::error(
                    "L0002",
                    "unterminated string literal",
                    Span::new(start, self.cursor),
                )
                .with_help("add a closing double quote before the end of the line"),
            );
        }
        self.push(TokenKind::String(value), start);
    }

    fn lex_escape(&mut self, value: &mut String, start: usize) {
        let Some(escaped) = self.bump() else {
            return;
        };
        match escaped {
            'n' => value.push('\n'),
            'r' => value.push('\r'),
            't' => value.push('\t'),
            '0' => value.push('\0'),
            '\\' => value.push('\\'),
            '"' => value.push('"'),
            'u' => self.lex_unicode_escape(value, start),
            other => {
                value.push(other);
                self.output.diagnostics.push(
                    Diagnostic::error(
                        "L0003",
                        format!("unknown string escape `\\{other}`"),
                        Span::new(start, self.cursor),
                    )
                    .with_help(r#"supported escapes are \n, \r, \t, \0, \\, \", and \u{...}"#),
                );
            }
        }
    }

    fn lex_unicode_escape(&mut self, value: &mut String, start: usize) {
        if self.peek() != Some('{') {
            self.output.diagnostics.push(Diagnostic::error(
                "L0004",
                "Unicode escape must use `\\u{...}` syntax",
                Span::new(start, self.cursor),
            ));
            return;
        }
        self.bump();
        let digits_start = self.cursor;
        while self
            .peek()
            .is_some_and(|character| character.is_ascii_hexdigit())
        {
            self.bump();
        }
        let digits_end = self.cursor;
        if self.peek() == Some('}') {
            self.bump();
        } else {
            self.output.diagnostics.push(Diagnostic::error(
                "L0004",
                "unterminated Unicode escape",
                Span::new(start, self.cursor),
            ));
            return;
        }
        let scalar = u32::from_str_radix(&self.text[digits_start..digits_end], 16)
            .ok()
            .and_then(char::from_u32);
        if let Some(character) = scalar {
            value.push(character);
        } else {
            self.output.diagnostics.push(Diagnostic::error(
                "L0004",
                "invalid Unicode scalar value",
                Span::new(start, self.cursor),
            ));
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn output(text: &str) -> LexOutput {
        lex(&SourceFile::new("test.tes", text))
    }

    fn kinds(text: &str) -> Vec<TokenKind> {
        output(text)
            .tokens
            .into_iter()
            .map(|token| token.kind)
            .collect()
    }

    fn codes(text: &str) -> Vec<String> {
        output(text)
            .diagnostics
            .into_iter()
            .map(|diagnostic| diagnostic.code)
            .collect()
    }

    #[test]
    fn recognizes_the_surface_keyword_set() {
        let keywords = [
            ("mod", Keyword::Mod),
            ("use", Keyword::Use),
            ("enum", Keyword::Enum),
            ("record", Keyword::Record),
            ("fn", Keyword::Fn),
            ("rule", Keyword::Rule),
            ("fixture", Keyword::Fixture),
            ("test", Keyword::Test),
            ("assert", Keyword::Assert),
            ("override", Keyword::Override),
            ("let", Keyword::Let),
            ("from", Keyword::From),
            ("expect", Keyword::Expect),
            ("some", Keyword::Some),
            ("O", Keyword::O),
            ("X", Keyword::X),
            ("unknown", Keyword::Unknown),
        ];
        for (spelling, keyword) in keywords {
            assert_eq!(
                kinds(spelling),
                vec![TokenKind::Keyword(keyword), TokenKind::Eof]
            );
            assert_eq!(keyword.as_str(), spelling);
        }
    }

    #[test]
    fn non_keywords_are_plain_identifiers() {
        for word in [
            "module",
            "import",
            "entity",
            "derive",
            "decision",
            "case",
            "invariant",
            "source",
            "then",
            "exactly",
            "zero",
            "cardinality",
            "range",
            "domain",
            "frag",
            "text",
            "also",
            "for",
            "all",
            "if",
            "one",
            "many",
            "optional",
            "continue",
            "state",
            "action",
            "transition",
            "trace",
            "on",
            "set",
            "when",
            "initially",
            "always",
            "terminates",
            "within",
            "steps",
            "no",
            "dead",
            "ends",
            "and",
            "or",
            "not",
            "true",
            "false",
        ] {
            assert_eq!(
                kinds(word),
                vec![TokenKind::Identifier(word.into()), TokenKind::Eof]
            );
        }
    }

    #[test]
    fn recognizes_korean_and_normalizes_identifiers_to_nfc() {
        assert_eq!(
            kinds("강의평가 e\u{301}"),
            vec![
                TokenKind::Identifier("강의평가".into()),
                TokenKind::Identifier("é".into()),
                TokenKind::Eof,
            ]
        );
        assert_eq!(canonical_identifier("e\u{301}").as_deref(), Some("é"));
        assert_eq!(canonical_identifier("state").as_deref(), Some("state"));
        assert!(canonical_identifier("record").is_none());
        assert!(canonical_identifier("O").is_none());
        assert!(canonical_identifier("X").is_none());
        assert!(canonical_identifier("1st").is_none());
    }

    #[test]
    fn recognizes_colons_decimal_dots_intervals_and_boolean_operators() {
        assert_eq!(
            kinds("a: b: c [0, 100) 0.70 s.점수 & O | X"),
            vec![
                TokenKind::Identifier("a".into()),
                TokenKind::Colon,
                TokenKind::Identifier("b".into()),
                TokenKind::Colon,
                TokenKind::Identifier("c".into()),
                TokenKind::LeftBracket,
                TokenKind::Number("0".into()),
                TokenKind::Comma,
                TokenKind::Number("100".into()),
                TokenKind::RightParen,
                TokenKind::Number("0.70".into()),
                TokenKind::Identifier("s".into()),
                TokenKind::Dot,
                TokenKind::Identifier("점수".into()),
                TokenKind::Ampersand,
                TokenKind::Keyword(Keyword::O),
                TokenKind::Pipe,
                TokenKind::Keyword(Keyword::X),
                TokenKind::Eof,
            ]
        );
    }

    #[test]
    fn handles_numeric_literals_and_reports_bad_separators() {
        let valid = output("100_000 1_234.50_01 [0,9_999]");
        assert!(valid.diagnostics.is_empty(), "{:?}", valid.diagnostics);
        assert_eq!(
            valid
                .tokens
                .into_iter()
                .map(|token| token.kind)
                .collect::<Vec<_>>(),
            vec![
                TokenKind::Number("100_000".into()),
                TokenKind::Number("1_234.50_01".into()),
                TokenKind::LeftBracket,
                TokenKind::Number("0".into()),
                TokenKind::Comma,
                TokenKind::Number("9_999".into()),
                TokenKind::RightBracket,
                TokenKind::Eof,
            ]
        );
        for literal in ["1__000", "1_", "1_.0", "1.0_", "1.0__1"] {
            assert_eq!(codes(literal), vec!["L0005"], "{literal}");
        }
    }

    #[test]
    fn emits_indent_and_dedent_around_four_space_suites() {
        let source = concat!(
            "mod policy\n",
            "record Person:\n",
            "    age: Int\n",
            "    active: Bool\n",
            "\n",
            "rule eligible:\n",
            "    O: active = O\n",
        );
        let result = output(source);
        assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics);
        assert_eq!(
            result
                .tokens
                .iter()
                .filter(|token| token.kind == TokenKind::Indent)
                .count(),
            2
        );
        assert_eq!(
            result
                .tokens
                .iter()
                .filter(|token| token.kind == TokenKind::Dedent)
                .count(),
            2
        );
        let rule = result
            .tokens
            .iter()
            .position(|token| token.kind == TokenKind::Keyword(Keyword::Rule))
            .unwrap();
        assert_eq!(result.tokens[rule - 1].kind, TokenKind::Dedent);
        assert_eq!(
            result.tokens[result.tokens.len() - 2].kind,
            TokenKind::Dedent
        );
    }

    #[test]
    fn blank_and_comment_only_lines_do_not_change_layout_or_consume_a_suite() {
        let source = concat!(
            "record R:\n",
            "\n",
            "        # indentation here is trivia\n",
            "#<\n",
            "opaque comment body\n",
            "#>\n",
            "    value: Int\n",
            "next\n",
        );
        let result = output(source);
        assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics);
        assert_eq!(
            result
                .tokens
                .iter()
                .filter(|token| token.kind == TokenKind::Indent)
                .count(),
            1
        );
        assert_eq!(
            result
                .tokens
                .iter()
                .filter(|token| token.kind == TokenKind::Dedent)
                .count(),
            1
        );
    }

    #[test]
    fn supports_unbounded_structural_depth() {
        let result = output("record R:\n    rule r:\n        if O:\n            continue\n");
        assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics);
        assert_eq!(
            result
                .tokens
                .iter()
                .filter(|token| token.kind == TokenKind::Indent)
                .count(),
            3
        );
        assert_eq!(
            result
                .tokens
                .iter()
                .filter(|token| token.kind == TokenKind::Dedent)
                .count(),
            3
        );
    }

    #[test]
    fn reports_layout_errors_with_stable_codes() {
        assert_eq!(codes("record R:\n\tvalue\n"), vec!["L0009"]);
        assert_eq!(codes("record R:\n  value\n"), vec!["L0010"]);
        assert_eq!(codes("record R:\n        value\n"), vec!["L0011"]);
        assert_eq!(codes("record R:\nvalue\n"), vec!["L0013"]);
        assert_eq!(
            codes("record R:\n        value\n    other\n"),
            vec!["L0011", "L0012"]
        );
        assert_eq!(codes("mod m\n    stray\n"), vec!["L0011"]);
    }

    #[test]
    fn suppresses_layout_inside_parentheses_and_braces() {
        let source = concat!(
            "fn combine(\n",
            "  left,\n",
            "       right\n",
            "): Bool\n",
            "let values = {\n",
            "  enabled: O\n",
            "}\n",
        );
        let result = output(source);
        assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics);
        assert!(
            !result
                .tokens
                .iter()
                .any(|token| matches!(token.kind, TokenKind::Indent | TokenKind::Dedent))
        );
        assert_eq!(
            kinds("[a\n    & b]"),
            vec![
                TokenKind::LeftBracket,
                TokenKind::Identifier("a".into()),
                TokenKind::Ampersand,
                TokenKind::Identifier("b".into()),
                TokenKind::RightBracket,
                TokenKind::Eof,
            ]
        );
    }

    #[test]
    fn lexes_fragment_source_as_one_raw_token() {
        let source = concat!(
            "policy.article:\n",
            "    First # is literal\n",
            "        rule and keywords stay raw\r\n",
            "    \n",
            "    Last\r\n",
            "    \n",
            "    rule applies: # source ends above\n",
            "        O: result = O\n",
        );
        let result = output(source);
        assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics);
        assert_eq!(result.comments.len(), 1);
        let raw = result
            .tokens
            .iter()
            .find(|token| matches!(token.kind, TokenKind::RawText(_)))
            .unwrap();
        assert_eq!(
            raw.kind,
            TokenKind::RawText(
                "First # is literal\n    rule and keywords stay raw\n\nLast\n".into()
            )
        );
        assert_eq!(
            &source[raw.span.start..raw.span.end],
            concat!(
                "    First # is literal\n",
                "        rule and keywords stay raw\r\n",
                "    \n",
                "    Last\r\n",
            )
        );
        let raw_index = result
            .tokens
            .iter()
            .position(|token| matches!(token.kind, TokenKind::RawText(_)))
            .unwrap();
        assert_eq!(result.tokens[raw_index - 1].kind, TokenKind::Indent);
        assert_eq!(
            result.tokens[raw_index + 1].kind,
            TokenKind::Keyword(Keyword::Rule)
        );
    }

    #[test]
    fn raw_text_preserves_extra_indent_and_omits_separator_blank_lines() {
        let result = output("policy.article:\n        deep\n\n\nnext\n");
        assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics);
        assert!(
            result
                .tokens
                .iter()
                .any(|token| { token.kind == TokenKind::RawText("    deep\n".into()) })
        );
    }

    #[test]
    fn fragment_function_headers_allow_inferred_and_explicit_return_types() {
        for header in [
            "fn amount:",
            "fn amount: # inferred inputs and result",
            "fn amount Decimal:",
            "fn amount domain.types.Decimal:",
            "fn amount(request Request):",
            "fn amount(request Request): # inferred",
            "fn amount(request Request)Decimal:",
            "fn amount(request Request) Decimal:",
            "fn amount(request Request) domain.types.Decimal:",
        ] {
            let source = format!(
                "law.formula:\n    Formula source.\n\n    {header}\n        request.value * 0.7\n"
            );
            let result = output(&source);
            assert!(
                result.diagnostics.is_empty(),
                "{header}: {:?}",
                result.diagnostics
            );
            assert!(
                result
                    .tokens
                    .iter()
                    .any(|token| { token.kind == TokenKind::Keyword(Keyword::Fn) }),
                "{header} remained raw text"
            );
        }
    }

    #[test]
    fn text_is_not_raw_when_it_is_not_the_only_top_level_header() {
        let source = "fn f(text:\n    value\n: Bool\nfoo text:\n    value\n";
        let result = output(source);
        assert!(
            !result
                .tokens
                .iter()
                .any(|token| matches!(token.kind, TokenKind::RawText(_)))
        );
    }

    #[test]
    fn reports_a_raw_block_that_does_not_reach_its_baseline() {
        assert_eq!(
            codes("policy.article:\n  too shallow\n"),
            vec!["L0013", "L0010"]
        );
    }

    #[test]
    fn comments_use_hash_only_and_retain_physical_newlines() {
        let source = "a # first\n# second\n#<\nopaque / #<\r\n#>\nb // c";
        let result = output(source);
        assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics);
        assert_eq!(result.comments.len(), 3);
        assert_eq!(result.comments[0].text, "# first");
        assert_eq!(result.comments[1].text, "# second");
        assert_eq!(result.comments[2].text, "#<\nopaque / #<\r\n#>");
        assert_eq!(
            result
                .tokens
                .iter()
                .filter(|token| token.kind == TokenKind::Newline)
                .count(),
            5
        );
        assert_eq!(
            result
                .tokens
                .iter()
                .filter(|token| token.kind == TokenKind::Slash)
                .count(),
            2
        );
    }

    #[test]
    fn block_markers_are_line_leading_marker_only_and_not_nested() {
        let inline = output("a #< one line\nb\n");
        assert_eq!(inline.comments[0].text, "#< one line");

        let nested = output("#<\n#<\n#>\n#>");
        assert_eq!(nested.comments[0].text, "#<\n#<\n#>");
        assert_eq!(nested.comments[1].text, "#>");
        assert_eq!(nested.diagnostics.len(), 1);
        assert_eq!(nested.diagnostics[0].code, "L0008");

        assert_eq!(
            codes("#< heading\nopaque\n#> trailing\n"),
            vec!["L0006", "L0006"]
        );
    }

    #[test]
    fn reports_unterminated_and_unmatched_block_markers() {
        let unterminated = output("#<\nnever closed\n");
        assert_eq!(unterminated.comments[0].text, "#<\nnever closed\n");
        assert_eq!(unterminated.diagnostics[0].code, "L0007");
        let unmatched = output("  #>\nmod m");
        assert_eq!(unmatched.comments[0].text, "#>");
        assert_eq!(unmatched.diagnostics[0].code, "L0008");
    }

    #[test]
    fn comment_markers_inside_strings_are_not_trivia() {
        let result = output("\"# text // text\" # actual");
        assert_eq!(result.comments.len(), 1);
        assert_eq!(result.comments[0].text, "# actual");
        assert_eq!(
            result.tokens[0].kind,
            TokenKind::String("# text // text".into())
        );
    }

    #[test]
    fn keeps_one_newline_token_for_crlf() {
        let result = output("a\r\nb\n");
        assert_eq!(
            result
                .tokens
                .iter()
                .filter(|token| token.kind == TokenKind::Newline)
                .count(),
            2
        );
        assert_eq!(result.tokens[1].span, Span::new(1, 3));
    }

    #[test]
    fn decodes_string_escapes_and_recovers_from_an_unterminated_string() {
        assert_eq!(
            kinds(r#""line\n\u{D55C}\"""#),
            vec![TokenKind::String("line\n한\"".into()), TokenKind::Eof]
        );
        let unterminated = output("\"oops\nmod ok");
        assert_eq!(unterminated.diagnostics[0].code, "L0002");
        assert!(
            unterminated
                .tokens
                .iter()
                .any(|token| { token.kind == TokenKind::Keyword(Keyword::Mod) })
        );
    }

    #[test]
    fn raw_fragment_text_stops_before_code_comment_trivia() {
        let result = output(
            "policy.source:\n    Original # text.\n\n    # code note\n    #<\n    longer note\n    #>\n    rule apply(Request): # rule note\n        : result(Request) = O\n",
        );
        assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics);
        let raw = result.tokens.iter().find_map(|token| match &token.kind {
            TokenKind::RawText(value) => Some(value.as_str()),
            _ => None,
        });
        assert_eq!(raw, Some("Original # text.\n"));
        assert_eq!(result.comments.len(), 3);
        assert_eq!(result.comments[0].text, "# code note");
        assert!(result.comments[1].text.starts_with("#<"));
        assert_eq!(result.comments[2].text, "# rule note");
    }

    #[test]
    fn raw_rule_like_prose_is_not_a_code_boundary() {
        let result = output(
            "policy.source:\n    A phrase about the\n\n    rule of law:\n    remains source text.\n",
        );
        assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics);
        let raw = result.tokens.iter().find_map(|token| match &token.kind {
            TokenKind::RawText(value) => Some(value.as_str()),
            _ => None,
        });
        assert_eq!(
            raw,
            Some("A phrase about the\n\nrule of law:\nremains source text.\n")
        );
    }

    #[test]
    fn comment_like_lines_without_a_code_sentinel_remain_raw_text() {
        let result = output("policy.source:\n    Original.\n\n    # source note\n");
        assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics);
        let raw = result.tokens.iter().find_map(|token| match &token.kind {
            TokenKind::RawText(value) => Some(value.as_str()),
            _ => None,
        });
        assert_eq!(raw, Some("Original.\n\n# source note\n"));
        assert!(result.comments.is_empty());
    }

    #[test]
    fn reports_invalid_char_and_continues() {
        let result = output("a @ b");
        assert_eq!(result.diagnostics.len(), 1);
        assert_eq!(result.tokens.len(), 3);
    }

    #[test]
    fn package_private_fragment_ids_are_not_source_identifiers() {
        let result = output("rule leak:\n    basis @pkg_common.policy.private\n    : result = O\n");
        assert!(
            result
                .diagnostics
                .iter()
                .any(|diagnostic| diagnostic.code == "L0001")
        );
    }
}