ridl_syntax/

lexer.rs

//! The family lexer (typl reference §1.4 and §2).
//!
//! One lexer serves the whole RIDL family. It recognises the shared token set —
//! the reserved-word registry (§1.4), identifiers, numeric, string, regex and
//! duration literals, punctuation, and the comment forms — and it is total:
//! concatenating the text of every returned token reproduces the input byte for
//! byte, so a rowan tree built from these tokens round-trips (ADR-0004 §2).
//!
//! `logos` lexes the raw stream; two source-driven refinements sit on top,
//! because they depend on more than a single token's bytes:
//!
//! - **Regex literals** (§2.7, §5.3). A `/` whose previous significant token is
//!   `=` (an init or const value) or the `match` keyword (a string constraint)
//!   begins a regex literal. It is scanned from the source directly and the raw
//!   stream inside it is discarded by re-lexing from the literal's end, so the
//!   way `logos` happened to tokenise the pattern bytes does not matter.
//! - **Duration literals** (§2.8). A number immediately followed by a UCUM time
//!   atom (`us`, `ms`, `s`, `min`, `h`) is merged into one `Duration` token. The
//!   typl profile rejects durations later (TYPL-302); the family lexer still
//!   recognises them, because timing belongs to the other profiles.
//!
//! Identifiers are mapped against the family registry ([`crate::keywords`])
//! after lexing: a used keyword becomes its keyword kind, any other registry
//! word becomes [`SyntaxKind::ReservedWord`], everything else stays
//! [`SyntaxKind::Ident`].

use logos::Logos;

use crate::keywords::{self, Profile};
use crate::syntax_kind::SyntaxKind;

/// A lexed token: its [`SyntaxKind`] and the exact source slice it covers.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Token<'a> {
    pub kind: SyntaxKind,
    pub text: &'a str,
}

/// The `logos`-derived token set, private to this module. [`lex`] maps each
/// variant into [`SyntaxKind`] — the family lexer's stable public interface —
/// and applies the regex, duration, and keyword refinements on top.
#[derive(Logos, Debug, Clone, Copy, PartialEq, Eq)]
enum RawToken {
    // Identifiers: start with a letter, digits permitted after the first
    // character, underscore only inside SCREAMING_SNAKE names (typl reference
    // §2.3). One token kind covers CamelCase, camelCase, and SCREAMING_SNAKE;
    // keyword mapping (typl reference §1.4) happens after lexing.
    #[regex("[A-Za-z][A-Za-z0-9_]*")]
    Ident,

    // Floats must contain a decimal point (typl reference §2.5); no scientific
    // notation. The longer match (digits '.' digits) wins over the bare-digits
    // `IntNumber` pattern, and `0.0..250.0` lexes as float–dotdot–float because
    // the pattern needs a digit after the dot.
    #[regex(r"[0-9]+\.[0-9]+")]
    FloatNumber,

    // Integers: decimal digits (typl reference §2.4). Leading zeros lex as one
    // token here; the parser flags them (FORM-005). A negative literal is a
    // separate `Minus` token followed by digits — the parser combines them.
    #[regex("[0-9]+")]
    IntNumber,

    // String literals (typl reference §2.6): RFC 8259 escapes, no raw newline.
    // The callback scans to the closing quote; an unterminated string is an
    // error token to end of line.
    #[token("\"", lex_string)]
    Str,

    // Doc comments (typl reference §14): `///` line form and `/** ... */` block
    // form. The line form is the longer token, so it outranks the ordinary `//`
    // line comment. The callback scans to the end of the line.
    #[token("///", lex_line_comment)]
    DocCommentLine,

    // Line comment `//` to end of line (typl reference §13).
    #[token("//", lex_line_comment)]
    LineComment,

    // Block comment `/* ... */`, non-nesting (typl reference §13). The callback
    // reports whether it is a doc comment (`/**`) and stops at the first `*/`.
    #[token("/*", lex_block_comment)]
    BlockComment(bool),

    #[token(":")]
    Colon,
    #[token("=")]
    Eq,
    #[token("[")]
    LBracket,
    #[token("]")]
    RBracket,
    #[token("{")]
    LBrace,
    #[token("}")]
    RBrace,
    #[token("(")]
    LParen,
    #[token(")")]
    RParen,
    #[token("..")]
    DotDot,
    #[token(".")]
    Dot,
    #[token("/")]
    Slash,
    #[token(",")]
    Comma,
    #[token("?")]
    Question,
    #[token(";")]
    Semicolon,
    #[token("@")]
    At,
    #[token("|")]
    Pipe,
    #[token("%")]
    Percent,
    #[token("-")]
    Minus,

    // Expression operators (ridl reference §13, expr core). Family tokens
    // like durations: recognised under every profile, rejected by the typl
    // grammar. `logos` picks the longest match, so `<=` outranks `<`, `==`
    // outranks two `=`, and `||` outranks two `|`.
    #[token("<")]
    Lt,
    #[token("<=")]
    Le,
    #[token(">")]
    Gt,
    #[token(">=")]
    Ge,
    #[token("==")]
    EqEq,
    #[token("!=")]
    Neq,
    #[token("&&")]
    AmpAmp,
    #[token("||")]
    PipePipe,
    #[token("!")]
    Bang,
    #[token("+")]
    Plus,
    #[token("*")]
    Star,

    #[regex(r"[ \t\r\n]+")]
    Whitespace,
}

impl From<RawToken> for SyntaxKind {
    fn from(token: RawToken) -> Self {
        match token {
            RawToken::Ident => SyntaxKind::Ident,
            RawToken::FloatNumber => SyntaxKind::FloatNumber,
            RawToken::IntNumber => SyntaxKind::IntNumber,
            RawToken::Str => SyntaxKind::String,
            RawToken::DocCommentLine => SyntaxKind::DocComment,
            RawToken::LineComment => SyntaxKind::LineComment,
            RawToken::BlockComment(is_doc) => {
                if is_doc {
                    SyntaxKind::DocComment
                } else {
                    SyntaxKind::BlockComment
                }
            }
            RawToken::Colon => SyntaxKind::Colon,
            RawToken::Eq => SyntaxKind::Eq,
            RawToken::LBracket => SyntaxKind::LBracket,
            RawToken::RBracket => SyntaxKind::RBracket,
            RawToken::LBrace => SyntaxKind::LBrace,
            RawToken::RBrace => SyntaxKind::RBrace,
            RawToken::LParen => SyntaxKind::LParen,
            RawToken::RParen => SyntaxKind::RParen,
            RawToken::DotDot => SyntaxKind::DotDot,
            RawToken::Dot => SyntaxKind::Dot,
            RawToken::Slash => SyntaxKind::Slash,
            RawToken::Comma => SyntaxKind::Comma,
            RawToken::Question => SyntaxKind::Question,
            RawToken::Semicolon => SyntaxKind::Semicolon,
            RawToken::At => SyntaxKind::At,
            RawToken::Pipe => SyntaxKind::Pipe,
            RawToken::Percent => SyntaxKind::Percent,
            RawToken::Minus => SyntaxKind::Minus,
            RawToken::Lt => SyntaxKind::Lt,
            RawToken::Le => SyntaxKind::Le,
            RawToken::Gt => SyntaxKind::Gt,
            RawToken::Ge => SyntaxKind::Ge,
            RawToken::EqEq => SyntaxKind::EqEq,
            RawToken::Neq => SyntaxKind::Neq,
            RawToken::AmpAmp => SyntaxKind::AmpAmp,
            RawToken::PipePipe => SyntaxKind::PipePipe,
            RawToken::Bang => SyntaxKind::Bang,
            RawToken::Plus => SyntaxKind::Plus,
            RawToken::Star => SyntaxKind::Star,
            RawToken::Whitespace => SyntaxKind::Whitespace,
        }
    }
}

/// A raw token with its byte span, before duration and keyword refinement.
struct Raw {
    kind: SyntaxKind,
    start: usize,
    end: usize,
}

/// Lexes `input` into a flat token stream under `profile`.
///
/// Total: concatenating `token.text` for every returned token reproduces
/// `input` byte for byte. Input the family lexer does not recognise becomes
/// [`SyntaxKind::Error`] tokens rather than aborting the lex.
///
/// The profile decides only which registry words are active keywords: under
/// [`Profile::Typl`] the stream is identical to the E1 lexer's; under
/// [`Profile::Ridl`] the nine ridl words lex to their keyword variants; under
/// [`Profile::Rsdl`] the eight rsdl words do.
/// Everything else — durations, `@`, the expr operators — is a family token
/// in both profiles; the parser draws the profile boundaries.
pub fn lex(input: &str, profile: Profile) -> Vec<Token<'_>> {
    let raws = raw_tokens(input);
    refine(input, &raws, profile)
}

/// Runs `logos` and lifts regex literals out of the raw stream (typl reference
/// §2.7). On a regex literal the scan is driven from the source and lexing
/// resumes from the literal's end, so the pattern bytes are never mis-tokenised
/// as slashes, identifiers, or a stray line comment.
fn raw_tokens(input: &str) -> Vec<Raw> {
    let mut out: Vec<Raw> = Vec::new();
    // The previous significant (non-trivia) token: its kind and its source
    // text. The text is needed to spot the `match` keyword, which is still a
    // raw `Ident` at this stage — keyword mapping happens after the regex lift.
    let mut prev_kind: Option<SyntaxKind> = None;
    let mut prev_text: &str = "";
    let mut base = 0usize;
    'relex: loop {
        let mut lexer = RawToken::lexer(&input[base..]);
        while let Some(result) = lexer.next() {
            let span = lexer.span();
            let start = base + span.start;
            let end = base + span.end;
            let kind = result.map_or(SyntaxKind::Error, SyntaxKind::from);

            if kind == SyntaxKind::Slash && starts_regex(prev_kind, prev_text) {
                let (regex_kind, regex_end) = scan_regex(input, start);
                out.push(Raw {
                    kind: regex_kind,
                    start,
                    end: regex_end,
                });
                prev_kind = Some(regex_kind);
                prev_text = &input[start..regex_end];
                base = regex_end;
                continue 'relex;
            }

            out.push(Raw { kind, start, end });
            if !kind.is_trivia() {
                prev_kind = Some(kind);
                prev_text = &input[start..end];
            }
        }
        break;
    }
    out
}

/// Whether a `/` following this previous significant token begins a regex
/// literal (typl reference §2.7, §5.3): after `=` — an init or const value — or
/// the `match` keyword — a string constraint. `match` is still a raw `Ident` at
/// lexing time, so it is matched by its source text; it is a reserved word, so
/// the text is unambiguous, and typl has no division operator, so neither `= /`
/// nor `match /` can be anything but a regex.
fn starts_regex(prev_kind: Option<SyntaxKind>, prev_text: &str) -> bool {
    prev_kind == Some(SyntaxKind::Eq)
        || (prev_kind == Some(SyntaxKind::Ident) && prev_text == "match")
}

/// Scans a regex literal that begins at the `/` at byte `start`. Returns
/// [`SyntaxKind::Regex`] and the byte just past the closing `/`, or
/// [`SyntaxKind::Error`] and the end of the line when the literal does not
/// terminate on its own line (typl reference §2.7 — a regex cannot contain a
/// raw newline).
fn scan_regex(input: &str, start: usize) -> (SyntaxKind, usize) {
    let bytes = input.as_bytes();
    let mut i = start + 1; // past the opening '/'
    while i < bytes.len() {
        match bytes[i] {
            b'\\' => {
                if i + 1 >= bytes.len() {
                    return (SyntaxKind::Error, bytes.len());
                }
                if is_line_break(bytes, i + 1) {
                    return (SyntaxKind::Error, i + 1);
                }
                i += 2;
            }
            b'/' => return (SyntaxKind::Regex, i + 1),
            _ if is_line_break(bytes, i) => return (SyntaxKind::Error, i),
            _ => i += 1,
        }
    }
    (SyntaxKind::Error, bytes.len())
}

/// Merges duration literals and maps identifiers against the family registry:
/// a keyword `profile` uses becomes its keyword kind, any other registry word
/// becomes [`SyntaxKind::ReservedWord`], everything else stays an `Ident`.
fn refine<'a>(input: &'a str, raws: &[Raw], profile: Profile) -> Vec<Token<'a>> {
    let mut out = Vec::with_capacity(raws.len());
    let mut i = 0;
    while i < raws.len() {
        let raw = &raws[i];

        // Duration: a number immediately followed by a UCUM time atom (typl
        // reference §2.8). "Immediately" means the very next token — whitespace
        // would appear as a separate trivia token in between.
        if matches!(raw.kind, SyntaxKind::IntNumber | SyntaxKind::FloatNumber)
            && let Some(next) = raws.get(i + 1)
            && next.kind == SyntaxKind::Ident
            && is_time_atom(&input[next.start..next.end])
        {
            out.push(Token {
                kind: SyntaxKind::Duration,
                text: &input[raw.start..next.end],
            });
            i += 2;
            continue;
        }

        let text = &input[raw.start..raw.end];
        let kind = if raw.kind == SyntaxKind::Ident {
            keywords::keyword_in(profile, text).unwrap_or(if keywords::is_reserved(text) {
                SyntaxKind::ReservedWord
            } else {
                SyntaxKind::Ident
            })
        } else {
            raw.kind
        };
        out.push(Token { kind, text });
        i += 1;
    }
    out
}

/// The UCUM time atoms a duration literal may use (typl reference §2.8).
fn is_time_atom(text: &str) -> bool {
    matches!(text, "us" | "ms" | "s" | "min" | "h")
}

/// Scans a string literal after the opening `"` (typl reference §2.6). Returns
/// `Ok` on the closing quote; an unterminated string (a raw newline or the end
/// of input before the close) consumes to end of line and becomes an error
/// token.
fn lex_string(lex: &mut logos::Lexer<RawToken>) -> Result<(), ()> {
    let bytes = lex.remainder().as_bytes();
    let mut i = 0;
    while i < bytes.len() {
        match bytes[i] {
            _ if is_line_break(bytes, i) => {
                lex.bump(i);
                return Err(());
            }
            b'\\' => {
                if i + 1 >= bytes.len() || is_line_break(bytes, i + 1) {
                    // Dangling backslash at end of line or input: unterminated.
                    lex.bump(i + 1);
                    return Err(());
                }
                i += 2;
            }
            b'"' => {
                lex.bump(i + 1);
                return Ok(());
            }
            _ => i += 1,
        }
    }
    lex.bump(bytes.len());
    Err(())
}

/// Whether the byte at `i` ends a line (typl reference §2.2): an LF, or a CR
/// that is not followed by an LF. In a CRLF pair the LF ends the line, so the
/// CR stays part of the text before it.
fn is_line_break(bytes: &[u8], i: usize) -> bool {
    match bytes[i] {
        b'\n' => true,
        b'\r' => bytes.get(i + 1) != Some(&b'\n'),
        _ => false,
    }
}

/// Scans a line comment or a doc comment line after its opening slashes
/// (typl reference §13, §14) to the end of the line, which it does not
/// include.
fn lex_line_comment(lex: &mut logos::Lexer<RawToken>) {
    let bytes = lex.remainder().as_bytes();
    let end = (0..bytes.len())
        .find(|&i| is_line_break(bytes, i))
        .unwrap_or(bytes.len());
    lex.bump(end);
}

/// Scans a block comment after the opening `/*` (typl reference §13). Non-
/// nesting: stops at the first `*/`. Returns whether it is a doc comment
/// (`/** ... */`, typl reference §14); an unterminated block comment consumes
/// the rest of the input and becomes an error token.
fn lex_block_comment(lex: &mut logos::Lexer<RawToken>) -> Result<bool, ()> {
    let rest = lex.remainder();
    // A doc comment starts `/**` but the empty comment `/**/` does not.
    let is_doc = rest.starts_with('*') && !rest.starts_with("*/");
    match rest.find("*/") {
        Some(idx) => {
            lex.bump(idx + 2);
            Ok(is_doc)
        }
        None => {
            lex.bump(rest.len());
            Err(())
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn concat(tokens: &[Token<'_>]) -> String {
        tokens.iter().map(|t| t.text).collect()
    }

    /// The kinds of the significant (non-trivia) tokens under `profile`.
    fn kinds_in(profile: Profile, input: &str) -> Vec<SyntaxKind> {
        lex(input, profile)
            .iter()
            .filter(|t| !t.kind.is_trivia())
            .map(|t| t.kind)
            .collect()
    }

    // The nine ridl words lex to their keyword variants under `Profile::Ridl`
    // and stay `ReservedWord` under `Profile::Typl`.
    #[test]
    fn ridl_words_lex_by_profile() {
        for (word, kind) in [
            ("interface", SyntaxKind::InterfaceKw),
            ("service", SyntaxKind::ServiceKw),
            ("signal", SyntaxKind::SignalKw),
            ("event", SyntaxKind::EventKw),
            ("command", SyntaxKind::CommandKw),
            ("query", SyntaxKind::QueryKw),
            ("fixed", SyntaxKind::FixedKw),
            ("require", SyntaxKind::RequireKw),
            ("ensure", SyntaxKind::EnsureKw),
        ] {
            assert_eq!(
                kinds_in(Profile::Ridl, word),
                vec![kind],
                "`{word}` must lex to its keyword variant under Ridl",
            );
            assert_eq!(
                kinds_in(Profile::Typl, word),
                vec![SyntaxKind::ReservedWord],
                "`{word}` must stay ReservedWord under Typl",
            );
        }
    }

    #[test]
    fn other_profile_words_stay_reserved_in_both_profiles() {
        for word in ["states", "model", "component", "automaton"] {
            for profile in [Profile::Typl, Profile::Ridl] {
                assert_eq!(
                    kinds_in(profile, word),
                    vec![SyntaxKind::ReservedWord],
                    "`{word}` must stay ReservedWord under {profile:?}",
                );
            }
        }
    }

    /// The rsdl v0.2 keyword line (rsdl reference §2), paired with the token
    /// kind each word lexes to under `Profile::Rsdl`.
    const RSDL_WORDS: [(&str, SyntaxKind); 8] = [
        ("system", SyntaxKind::SystemKw),
        ("component", SyntaxKind::ComponentKw),
        ("distribution", SyntaxKind::DistributionKw),
        ("deployment", SyntaxKind::DeploymentKw),
        ("machine", SyntaxKind::MachineKw),
        ("offers", SyntaxKind::OffersKw),
        ("requires", SyntaxKind::RequiresKw),
        ("for", SyntaxKind::ForKw),
    ];

    /// The rsdl v0.1 words the v0.2 reference retires (rsdl reference §2). No
    /// other profile uses one, so each leaves the registry.
    const RETIRED_RSDL_WORDS: [&str; 13] = [
        "provides",
        "instance",
        "assurance",
        "target",
        "place",
        "on",
        "transport",
        "bundle",
        "time",
        "base",
        "redundant",
        "supervise",
        "degraded",
    ];

    // Every word of the rsdl keyword line is reserved in every profile: an
    // active keyword under `Profile::Rsdl`, a `ReservedWord` under the other
    // two. `offers`, `distribution` and `machine` are the three the registry
    // gained with rsdl v0.2.
    #[test]
    fn rsdl_words_lex_by_profile() {
        for (word, kind) in RSDL_WORDS {
            assert_eq!(
                kinds_in(Profile::Rsdl, word),
                vec![kind],
                "`{word}` must lex to its keyword variant under Rsdl",
            );
            for profile in [Profile::Typl, Profile::Ridl] {
                assert_eq!(
                    kinds_in(profile, word),
                    vec![SyntaxKind::ReservedWord],
                    "`{word}` must stay ReservedWord under {profile:?}",
                );
            }
        }
    }

    // A retired rsdl word is an ordinary identifier in every profile.
    #[test]
    fn retired_rsdl_words_are_identifiers_in_every_profile() {
        for word in RETIRED_RSDL_WORDS {
            for profile in [Profile::Typl, Profile::Ridl, Profile::Rsdl] {
                assert_eq!(
                    kinds_in(profile, word),
                    vec![SyntaxKind::Ident],
                    "`{word}` must lex to Ident under {profile:?}",
                );
            }
        }
    }

    // The typl words stay active under `Profile::Rsdl`, and a ridl word is a
    // reserved word there.
    #[test]
    fn rsdl_profile_keeps_the_typl_keywords_and_reserves_the_ridl_words() {
        assert_eq!(
            kinds_in(Profile::Rsdl, "package"),
            vec![SyntaxKind::PackageKw]
        );
        assert_eq!(
            kinds_in(Profile::Rsdl, "internal"),
            vec![SyntaxKind::InternalKw]
        );
        assert_eq!(kinds_in(Profile::Rsdl, "true"), vec![SyntaxKind::TrueKw]);
        assert_eq!(
            kinds_in(Profile::Rsdl, "service"),
            vec![SyntaxKind::ReservedWord]
        );
        assert_eq!(
            kinds_in(Profile::Rsdl, "require"),
            vec![SyntaxKind::ReservedWord]
        );
    }

    // Durations and `@` are ordinary tokens under Ridl (the parser draws no
    // TYPL-302 there — see the parser tests).
    #[test]
    fn durations_and_at_lex_clean_under_ridl() {
        assert_eq!(kinds_in(Profile::Ridl, "10ms"), vec![SyntaxKind::Duration]);
        assert_eq!(kinds_in(Profile::Ridl, "@"), vec![SyntaxKind::At]);
    }

    // The expr operator tokens.
    #[test]
    fn expr_operators_lex_under_both_profiles() {
        for profile in [Profile::Typl, Profile::Ridl] {
            assert_eq!(
                kinds_in(profile, "a >= b && !c"),
                vec![
                    SyntaxKind::Ident,
                    SyntaxKind::Ge,
                    SyntaxKind::Ident,
                    SyntaxKind::AmpAmp,
                    SyntaxKind::Bang,
                    SyntaxKind::Ident,
                ],
            );
        }
    }

    #[test]
    fn every_expr_operator_token_lexes() {
        assert_eq!(
            kinds_in(Profile::Ridl, "< <= > >= == != && || ! + *"),
            vec![
                SyntaxKind::Lt,
                SyntaxKind::Le,
                SyntaxKind::Gt,
                SyntaxKind::Ge,
                SyntaxKind::EqEq,
                SyntaxKind::Neq,
                SyntaxKind::AmpAmp,
                SyntaxKind::PipePipe,
                SyntaxKind::Bang,
                SyntaxKind::Plus,
                SyntaxKind::Star,
            ],
        );
    }

    // `T | E` still lexes with the single `Pipe`.
    #[test]
    fn union_pipe_lexes_unchanged() {
        for profile in [Profile::Typl, Profile::Ridl] {
            assert_eq!(
                kinds_in(profile, "T | E"),
                vec![SyntaxKind::Ident, SyntaxKind::Pipe, SyntaxKind::Ident],
            );
        }
    }

    #[test]
    fn fixture_round_trips_with_no_errors() {
        let input = include_str!("../fixtures/walking_skeleton.typl");
        let tokens = lex(input, Profile::Typl);
        assert_eq!(concat(&tokens), input, "token texts must reproduce input");
        assert!(
            tokens.iter().all(|t| t.kind != SyntaxKind::Error),
            "fixture must lex with zero Error tokens, got: {tokens:?}"
        );
    }

    #[test]
    fn range_dots_do_not_get_swallowed_by_float_regex() {
        let tokens = lex("0.0..250.0", Profile::Typl);
        let kinds: Vec<_> = tokens.iter().map(|t| t.kind).collect();
        assert_eq!(
            kinds,
            vec![
                SyntaxKind::FloatNumber,
                SyntaxKind::DotDot,
                SyntaxKind::FloatNumber,
            ]
        );
    }

    #[test]
    fn slash_separated_unit_lexes_as_ident_slash_ident() {
        let tokens = lex("km/h", Profile::Typl);
        let kinds: Vec<_> = tokens.iter().map(|t| t.kind).collect();
        assert_eq!(
            kinds,
            vec![SyntaxKind::Ident, SyntaxKind::Slash, SyntaxKind::Ident]
        );
    }

    #[test]
    fn line_comment_is_one_token() {
        let tokens = lex("// x", Profile::Typl);
        assert_eq!(tokens.len(), 1);
        assert_eq!(tokens[0].kind, SyntaxKind::LineComment);
        assert_eq!(tokens[0].text, "// x");
    }

    #[test]
    fn unknown_byte_yields_error_and_still_round_trips() {
        let input = "$";
        let tokens = lex(input, Profile::Typl);
        assert_eq!(concat(&tokens), input);
        assert_eq!(tokens.len(), 1);
        assert_eq!(tokens[0].kind, SyntaxKind::Error);
    }

    #[test]
    fn keywords_lex_distinctly_from_ident() {
        let tokens = lex("type const step notAKeyword", Profile::Typl);
        let kinds: Vec<_> = tokens
            .iter()
            .filter(|t| t.kind != SyntaxKind::Whitespace)
            .map(|t| t.kind)
            .collect();
        assert_eq!(
            kinds,
            vec![
                SyntaxKind::TypeKw,
                SyntaxKind::ConstKw,
                SyntaxKind::StepKw,
                SyntaxKind::Ident,
            ]
        );
    }

    /// The kinds of the significant (non-trivia) tokens under `Profile::Typl`,
    /// in order — the E1 tests below all pin the typl profile's behavior.
    fn significant_kinds(input: &str) -> Vec<SyntaxKind> {
        kinds_in(Profile::Typl, input)
    }

    // Step (a): every used keyword lexes to its variant; every reserved word to
    // ReservedWord.
    #[test]
    fn every_used_keyword_lexes_to_its_variant() {
        for word in keywords::FAMILY_RESERVED {
            let tokens = lex(word, Profile::Typl);
            assert_eq!(tokens.len(), 1, "`{word}` must lex to one token");
            let expected = keywords::typl_keyword(word).unwrap_or(SyntaxKind::ReservedWord);
            assert_eq!(tokens[0].kind, expected, "`{word}` lexed to the wrong kind");
        }
    }

    #[test]
    fn reserved_word_is_not_an_identifier() {
        assert_eq!(significant_kinds("signal"), vec![SyntaxKind::ReservedWord]);
        assert_eq!(significant_kinds("model"), vec![SyntaxKind::ReservedWord]);
        assert_eq!(significant_kinds("Speed"), vec![SyntaxKind::Ident]);
    }

    // Step (c): `0.0..250.0` lexes float–dotdot–float (kept from E0, extended).
    #[test]
    fn float_range_lexes_as_float_dotdot_float() {
        assert_eq!(
            significant_kinds("0.0..250.0"),
            vec![
                SyntaxKind::FloatNumber,
                SyntaxKind::DotDot,
                SyntaxKind::FloatNumber,
            ]
        );
    }

    // Step (d): `km/h` is ident–slash–ident; a regex literal after `=` is one
    // Regex token.
    #[test]
    fn slash_in_unit_is_not_a_regex() {
        assert_eq!(
            significant_kinds("km/h"),
            vec![SyntaxKind::Ident, SyntaxKind::Slash, SyntaxKind::Ident]
        );
    }

    #[test]
    fn regex_literal_after_eq_is_one_token() {
        let tokens = lex(r"const V = /a\/b/", Profile::Typl);
        let regex: Vec<_> = tokens
            .iter()
            .filter(|t| t.kind == SyntaxKind::Regex)
            .collect();
        assert_eq!(regex.len(), 1, "one regex token expected: {tokens:?}");
        assert_eq!(regex[0].text, r"/a\/b/");
    }

    #[test]
    fn unterminated_regex_is_error_to_end_of_line() {
        let tokens = lex("const V = /abc\n", Profile::Typl);
        let last_significant = tokens
            .iter()
            .rev()
            .find(|t| !t.kind.is_trivia())
            .expect("a significant token");
        assert_eq!(last_significant.kind, SyntaxKind::Error);
        assert_eq!(last_significant.text, "/abc");
    }

    #[test]
    fn inline_match_regex_is_one_token() {
        // The exact `ridl.std` Date declaration (typl reference Appendix A,
        // §5.3): the regex literal follows the `match` keyword, not `=`.
        let line = r"type Date : string [10 match /^\d{4}-\d{2}-\d{2}$/]";
        let tokens = lex(line, Profile::Typl);
        assert!(
            tokens.iter().all(|t| t.kind != SyntaxKind::Error),
            "no error tokens expected: {tokens:?}",
        );
        assert_eq!(
            significant_kinds(line),
            vec![
                SyntaxKind::TypeKw,
                SyntaxKind::Ident,
                SyntaxKind::Colon,
                SyntaxKind::StringKw,
                SyntaxKind::LBracket,
                SyntaxKind::IntNumber,
                SyntaxKind::MatchKw,
                SyntaxKind::Regex,
                SyntaxKind::RBracket,
            ],
        );
        let regex: Vec<_> = tokens
            .iter()
            .filter(|t| t.kind == SyntaxKind::Regex)
            .collect();
        assert_eq!(regex.len(), 1, "one regex token expected: {tokens:?}");
        assert_eq!(regex[0].text, r"/^\d{4}-\d{2}-\d{2}$/");
    }

    // Step (e): durations lex as one Duration; a bare atom is an Ident.
    #[test]
    fn durations_lex_as_one_token() {
        assert_eq!(significant_kinds("10ms"), vec![SyntaxKind::Duration]);
        assert_eq!(significant_kinds("500us"), vec![SyntaxKind::Duration]);
        assert_eq!(significant_kinds("1s"), vec![SyntaxKind::Duration]);
        assert_eq!(significant_kinds("1min"), vec![SyntaxKind::Duration]);
        assert_eq!(significant_kinds("1.5h"), vec![SyntaxKind::Duration]);
    }

    #[test]
    fn bare_time_atom_is_an_identifier() {
        assert_eq!(significant_kinds("min"), vec![SyntaxKind::Ident]);
        // A space breaks the adjacency a duration needs.
        assert_eq!(
            significant_kinds("10 ms"),
            vec![SyntaxKind::IntNumber, SyntaxKind::Ident]
        );
    }

    // Step (f): strings with escapes, and unterminated strings.
    #[test]
    fn string_with_escape_is_one_token() {
        let tokens = lex(r#""a\nb""#, Profile::Typl);
        assert_eq!(tokens.len(), 1);
        assert_eq!(tokens[0].kind, SyntaxKind::String);
        assert_eq!(tokens[0].text, r#""a\nb""#);
    }

    #[test]
    fn unterminated_string_is_error_to_end_of_line() {
        let tokens = lex("\"abc\n", Profile::Typl);
        assert_eq!(tokens[0].kind, SyntaxKind::Error);
        assert_eq!(tokens[0].text, "\"abc");
        // The newline survives as its own trivia token (losslessness).
        assert_eq!(tokens[1].kind, SyntaxKind::Whitespace);
    }

    // Step (g): block comments do not nest.
    #[test]
    fn block_comment_is_one_token() {
        let tokens = lex("/* block */", Profile::Typl);
        assert_eq!(tokens.len(), 1);
        assert_eq!(tokens[0].kind, SyntaxKind::BlockComment);
    }

    #[test]
    fn block_comment_ends_at_first_close() {
        let tokens = lex("/* /* */", Profile::Typl);
        assert_eq!(tokens.len(), 1, "no nesting: {tokens:?}");
        assert_eq!(tokens[0].kind, SyntaxKind::BlockComment);
        assert_eq!(tokens[0].text, "/* /* */");
    }

    #[test]
    fn doc_comments_are_distinct_from_plain_comments() {
        assert_eq!(significant_kinds("/// doc"), Vec::<SyntaxKind>::new());
        let doc_line = lex("/// doc", Profile::Typl);
        assert_eq!(doc_line[0].kind, SyntaxKind::DocComment);
        let doc_block = lex("/** doc */", Profile::Typl);
        assert_eq!(doc_block[0].kind, SyntaxKind::DocComment);
        let plain = lex("// doc", Profile::Typl);
        assert_eq!(plain[0].kind, SyntaxKind::LineComment);
    }

    // Step (h): leading zeros lex as one IntNumber; flagging is the parser's job.
    #[test]
    fn leading_zeros_lex_as_one_integer() {
        let tokens = lex("042", Profile::Typl);
        assert_eq!(tokens.len(), 1);
        assert_eq!(tokens[0].kind, SyntaxKind::IntNumber);
        assert_eq!(tokens[0].text, "042");
    }

    #[test]
    fn minus_is_its_own_token() {
        assert_eq!(
            significant_kinds("-40"),
            vec![SyntaxKind::Minus, SyntaxKind::IntNumber]
        );
    }

    #[test]
    fn all_new_punctuation_lexes() {
        assert_eq!(
            significant_kinds("{}()?;@|%"),
            vec![
                SyntaxKind::LBrace,
                SyntaxKind::RBrace,
                SyntaxKind::LParen,
                SyntaxKind::RParen,
                SyntaxKind::Question,
                SyntaxKind::Semicolon,
                SyntaxKind::At,
                SyntaxKind::Pipe,
                SyntaxKind::Percent,
            ]
        );
    }

    /// The kinds and texts of every token, trivia included.
    fn pairs(input: &str) -> Vec<(SyntaxKind, &str)> {
        lex(input, Profile::Typl)
            .iter()
            .map(|t| (t.kind, t.text))
            .collect()
    }

    // A lone CR is a line break (typl reference §2.2), so a line comment, a
    // doc comment line, an unterminated string and an unterminated regex all
    // end before it, as they do before an LF.
    #[test]
    fn a_lone_cr_ends_a_line_comment() {
        assert_eq!(
            pairs("// a\rx"),
            vec![
                (SyntaxKind::LineComment, "// a"),
                (SyntaxKind::Whitespace, "\r"),
                (SyntaxKind::Ident, "x"),
            ],
        );
    }

    #[test]
    fn a_lone_cr_at_the_end_of_input_ends_a_line_comment() {
        assert_eq!(
            pairs("// a\r"),
            vec![
                (SyntaxKind::LineComment, "// a"),
                (SyntaxKind::Whitespace, "\r"),
            ],
        );
    }

    #[test]
    fn a_lone_cr_ends_a_doc_comment_line() {
        assert_eq!(
            pairs("/// a\rx"),
            vec![
                (SyntaxKind::DocComment, "/// a"),
                (SyntaxKind::Whitespace, "\r"),
                (SyntaxKind::Ident, "x"),
            ],
        );
    }

    #[test]
    fn a_lone_cr_ends_an_unterminated_string() {
        assert_eq!(
            pairs("\"abc\rx"),
            vec![
                (SyntaxKind::Error, "\"abc"),
                (SyntaxKind::Whitespace, "\r"),
                (SyntaxKind::Ident, "x"),
            ],
        );
        // A backslash before a lone CR does not escape the line break.
        assert_eq!(
            pairs("\"a\\\rx"),
            vec![
                (SyntaxKind::Error, "\"a\\"),
                (SyntaxKind::Whitespace, "\r"),
                (SyntaxKind::Ident, "x"),
            ],
        );
    }

    #[test]
    fn a_backslash_and_a_final_lone_cr_end_an_unterminated_string() {
        assert_eq!(
            pairs("\"a\\\r"),
            vec![(SyntaxKind::Error, "\"a\\"), (SyntaxKind::Whitespace, "\r"),],
        );
    }

    #[test]
    fn a_lone_cr_ends_an_unterminated_regex() {
        let tokens = pairs("const V = /abc\rx");
        assert_eq!(
            tokens[tokens.len() - 3..],
            [
                (SyntaxKind::Error, "/abc"),
                (SyntaxKind::Whitespace, "\r"),
                (SyntaxKind::Ident, "x"),
            ],
        );
        let tokens = pairs("const V = /a\\\rx");
        assert_eq!(
            tokens[tokens.len() - 3..],
            [
                (SyntaxKind::Error, "/a\\"),
                (SyntaxKind::Whitespace, "\r"),
                (SyntaxKind::Ident, "x"),
            ],
        );
    }

    // A CRLF pair is one line break, and the tokens around it are the same as
    // before a lone CR became a line break: the CR stays inside the token that
    // precedes the LF.
    #[test]
    fn a_crlf_pair_still_ends_a_line_at_its_lf() {
        assert_eq!(
            pairs("// a\r\nx"),
            vec![
                (SyntaxKind::LineComment, "// a\r"),
                (SyntaxKind::Whitespace, "\n"),
                (SyntaxKind::Ident, "x"),
            ],
        );
        assert_eq!(
            pairs("/// a\r\nx"),
            vec![
                (SyntaxKind::DocComment, "/// a\r"),
                (SyntaxKind::Whitespace, "\n"),
                (SyntaxKind::Ident, "x"),
            ],
        );
        assert_eq!(
            pairs("\"abc\r\nx"),
            vec![
                (SyntaxKind::Error, "\"abc\r"),
                (SyntaxKind::Whitespace, "\n"),
                (SyntaxKind::Ident, "x"),
            ],
        );
        let tokens = pairs("const V = /abc\r\nx");
        assert_eq!(
            tokens[tokens.len() - 3..],
            [
                (SyntaxKind::Error, "/abc\r"),
                (SyntaxKind::Whitespace, "\n"),
                (SyntaxKind::Ident, "x"),
            ],
        );
    }
}