wikioxide 0.1.0

//! Left-to-right inline tokenizer: text runs, strip markers, entities, external links,
//! bare-URL autolinks, magic links and behavior switches. `[[…]]` and `<…>` are handed
//! off to [`super::links`] and [`super::html`].

use crate::config::SiteConfig;
use crate::engine::ast::{Inline, MagicLinkKind, TextRange};
use crate::engine::strip;
use crate::warn::WarningCollector;

use super::{html, links};

/// One tokenizer output: a resolved inline node or an apostrophe run for the quote pass.
#[derive(Debug)]
pub(super) enum Token {
    /// A fully resolved inline node.
    Node(Inline),
    /// A run of two or more apostrophes, resolved later by [`super::quotes`].
    Quotes(TextRange),
}

/// What a special-character handler decided.
pub(super) enum Step {
    /// Emit a token; scanning resumes at the byte offset.
    Emit(Token, usize),
    /// Consume everything up to the byte offset as literal text.
    Text(usize),
    /// Not special after all: the current character is literal text.
    Miss,
}

/// Tokenizes `range` (one line of `flat`) left to right.
pub(super) fn scan(
    flat: &str,
    range: TextRange,
    config: &SiteConfig,
    warnings: &mut WarningCollector,
    depth: u8,
) -> Vec<Token> {
    let start = range.start as usize;
    let end = range.end as usize;
    let mut tokens = Vec::new();
    let mut pos = start;
    let mut text_start = pos;
    while pos < end {
        // `pos` is always a char boundary below `end`, so a char is always there.
        let c = flat[pos..end].chars().next().unwrap_or('\u{0}');
        let step = match c {
            strip::SENTINEL => marker(flat, pos, end),
            '&' => entity(flat, pos, end),
            '[' => bracket(flat, pos, end, config, warnings, depth),
            '<' => html::scan_tag(flat, pos, end, warnings),
            '\'' => quote_run(flat, pos, end),
            '_' => behavior_switch(flat, pos, end, config),
            _ if c.is_ascii_alphabetic() && is_boundary(flat, start, pos) => {
                word(flat, pos, end, config)
            }
            _ => Step::Miss,
        };
        match step {
            Step::Emit(token, next) => {
                flush_text(&mut tokens, text_start, pos);
                tokens.push(token);
                pos = next;
                text_start = next;
            }
            Step::Text(next) => pos = next,
            Step::Miss => pos += c.len_utf8(),
        }
    }
    flush_text(&mut tokens, text_start, end);
    tokens
}

/// Pushes the pending literal-text run `[from, to)` if it is non-empty.
fn flush_text(tokens: &mut Vec<Token>, from: usize, to: usize) {
    if to > from {
        tokens.push(Token::Node(Inline::Text(TextRange::new(
            from as u32,
            to as u32,
        ))));
    }
}

/// Bare URLs and magic links only start where the previous character is not
/// ASCII-alphanumeric (MediaWiki's `(?<![0-9a-zA-Z])`).
fn is_boundary(flat: &str, start: usize, pos: usize) -> bool {
    pos == start
        || flat[start..pos]
            .chars()
            .next_back()
            .is_none_or(|p| !p.is_ascii_alphanumeric())
}

/// Decodes a strip marker at `pos`; a stray sentinel that does not decode (including one
/// truncated by the end of the line) is literal text.
fn marker(flat: &str, pos: usize, end: usize) -> Step {
    match strip::parse_marker(&flat[pos..end]) {
        Some(decoded) => Step::Emit(
            Token::Node(Inline::Marker {
                id: decoded.id,
                kind: decoded.kind,
            }),
            pos + decoded.len,
        ),
        None => Step::Miss,
    }
}

/// The named entities the inline parser decodes. MediaWiki decodes the full HTML5
/// table; this is the common subset the plan pins for M2.
const NAMED_ENTITIES: &[(&str, char)] = &[
    ("Dagger", '‡'),
    ("Prime", '″'),
    ("amp", '&'),
    ("apos", '\''),
    ("bull", '•'),
    ("copy", '©'),
    ("dagger", '†'),
    ("deg", '°'),
    ("gt", '>'),
    ("hellip", '…'),
    ("laquo", '«'),
    ("larr", '←'),
    ("lt", '<'),
    ("mdash", '—'),
    ("middot", '·'),
    ("minus", '−'),
    ("nbsp", '\u{a0}'),
    ("ndash", '–'),
    ("para", '¶'),
    ("plusmn", '±'),
    ("prime", '′'),
    ("quot", '"'),
    ("raquo", '»'),
    ("rarr", '→'),
    ("reg", '®'),
    ("sect", '§'),
    ("shy", '\u{ad}'),
    ("thinsp", '\u{2009}'),
    ("times", '×'),
    ("zwj", '\u{200d}'),
    ("zwnj", '\u{200c}'),
];

/// Decodes `&name;`, `&#NNN;` or `&#xHH;` at `pos`. Unknown names and invalid scalar
/// values stay literal text.
fn entity(flat: &str, pos: usize, end: usize) -> Step {
    /// Longest reference body we accept (`#1114111` and every pinned name fit).
    const MAX_BODY: usize = 12;
    let body = &flat[pos + 1..end];
    let Some(semi) = body.find(';') else {
        return Step::Miss;
    };
    if semi == 0 || semi > MAX_BODY {
        return Step::Miss;
    }
    let name = &body[..semi];
    let decoded = match name.strip_prefix('#') {
        Some(num) => decode_numeric(num),
        None => NAMED_ENTITIES
            .iter()
            .find(|&&(n, _)| n == name)
            .map(|&(_, c)| c),
    };
    match decoded {
        // Decoding to the marker sentinel would forge strip markers downstream, so
        // `&#127;` deliberately stays literal.
        Some(c) if c != strip::SENTINEL => Step::Emit(
            Token::Node(Inline::Entity {
                range: TextRange::new(pos as u32, (pos + semi + 2) as u32),
                decoded: c,
            }),
            pos + semi + 2,
        ),
        _ => Step::Miss,
    }
}

/// Decodes the body of a numeric character reference (`#65` → `decode_numeric("65")`,
/// `#x41` → `decode_numeric("x41")`) into a valid scalar value.
fn decode_numeric(num: &str) -> Option<char> {
    let (digits, radix) = match num.strip_prefix(['x', 'X']) {
        Some(hex) => (hex, 16),
        None => (num, 10),
    };
    // `from_str_radix` accepts a leading `+`, which HTML does not.
    if digits.is_empty() || !digits.chars().all(|c| c.is_digit(radix)) {
        return None;
    }
    u32::from_str_radix(digits, radix)
        .ok()
        .and_then(char::from_u32)
}

/// Groups a run of apostrophes: two or more become a quote token for the second pass.
fn quote_run(flat: &str, pos: usize, end: usize) -> Step {
    let run = flat[pos..end].bytes().take_while(|&b| b == b'\'').count();
    if run >= 2 {
        Step::Emit(
            Token::Quotes(TextRange::new(pos as u32, (pos + run) as u32)),
            pos + run,
        )
    } else {
        Step::Miss
    }
}

/// Recognizes `__WORD__` when the configuration resolves it as a magic word. Extends
/// across inner underscores (`__БЕЗ_ОГЛАВЛЕНИЯ__`) but never across whitespace.
fn behavior_switch(flat: &str, pos: usize, end: usize, config: &SiteConfig) -> Step {
    let rest = &flat[pos..end];
    if !rest.starts_with("__") {
        return Step::Miss;
    }
    let mut search = 2;
    while let Some(rel) = rest[search..].find("__") {
        let close = search + rel;
        let inner = &rest[2..close];
        if inner.is_empty() || inner.chars().any(char::is_whitespace) {
            return Step::Miss;
        }
        if let Some(id) = config.resolve_magic_word(&rest[..close + 2]) {
            return Step::Emit(
                Token::Node(Inline::BehaviorSwitch {
                    id: id.to_owned(),
                    range: TextRange::new(pos as u32, (pos + close + 2) as u32),
                }),
                pos + close + 2,
            );
        }
        search = close + 1;
    }
    Step::Miss
}

/// At a word boundary: tries the RFC/PMID/ISBN magic links first, then a bare-URL
/// autolink.
fn word(flat: &str, pos: usize, end: usize, config: &SiteConfig) -> Step {
    match magic_link(flat, pos, end, config) {
        Step::Miss => autolink(flat, pos, end, config),
        hit => hit,
    }
}

/// Matches `RFC nnn`, `PMID nnn` or `ISBN <10|13 digits>` at `pos`, honoring the
/// per-wiki toggles. The emitted range covers the exact matched text.
fn magic_link(flat: &str, pos: usize, end: usize, config: &SiteConfig) -> Step {
    let rest = &flat[pos..end];
    let links = config.magic_links();
    let matched = if links.rfc && rest.starts_with("RFC") {
        numeric_magic(rest, "RFC").map(|len| (MagicLinkKind::Rfc, len))
    } else if links.pmid && rest.starts_with("PMID") {
        numeric_magic(rest, "PMID").map(|len| (MagicLinkKind::Pmid, len))
    } else if links.isbn && rest.starts_with("ISBN") {
        isbn_magic(rest).map(|len| (MagicLinkKind::Isbn, len))
    } else {
        None
    };
    match matched {
        Some((kind, len)) => Step::Emit(
            Token::Node(Inline::MagicLink {
                kind,
                range: TextRange::new(pos as u32, (pos + len) as u32),
            }),
            pos + len,
        ),
        None => Step::Miss,
    }
}

/// `KEYWORD` + one space + digits, not followed by an ASCII alphanumeric.
fn numeric_magic(rest: &str, keyword: &str) -> Option<usize> {
    let body = rest[keyword.len()..].strip_prefix(' ')?;
    let digits = body.bytes().take_while(u8::is_ascii_digit).count();
    if digits == 0 {
        return None;
    }
    let len = keyword.len() + 1 + digits;
    boundary_after(rest, len).then_some(len)
}

/// `ISBN` + one space + 10 or 13 digits with single `-`/space separators; the final
/// digit may be `X`/`x`. Returns the full matched length.
fn isbn_magic(rest: &str) -> Option<usize> {
    let body = rest[4..].strip_prefix(' ')?;
    let bytes = body.as_bytes();
    let mut digits = 0u32;
    let mut matched = 0; // bytes of `body` up to the last accepted digit
    let mut ended_x = false;
    let mut i = 0;
    while i < bytes.len() && !ended_x {
        let b = bytes[i];
        if b.is_ascii_digit() {
            digits += 1;
            i += 1;
            matched = i;
        } else if (b == b'X' || b == b'x') && digits > 0 {
            digits += 1;
            i += 1;
            matched = i;
            ended_x = true;
        } else if (b == b'-' || b == b' ') && matched == i && digits > 0 {
            // a single separator directly after a digit; a second one in a row leaves
            // `matched < i` and ends the match at the last digit
            i += 1;
        } else {
            break;
        }
    }
    if digits != 10 && digits != 13 {
        return None;
    }
    let len = 5 + matched;
    boundary_after(rest, len).then_some(len)
}

/// True when the character at byte `at` (if any) cannot extend a word.
fn boundary_after(rest: &str, at: usize) -> bool {
    rest[at..]
        .chars()
        .next()
        .is_none_or(|c| !c.is_ascii_alphanumeric())
}

/// Matches a bare URL at `pos`. The protocol-relative `//` never free-links; trailing
/// punctuation is stripped the way MediaWiki does.
fn autolink(flat: &str, pos: usize, end: usize, config: &SiteConfig) -> Step {
    let rest = &flat[pos..end];
    let Some(proto) = match_protocol(rest, config, false) else {
        return Step::Miss;
    };
    let url_stop = rest[proto..]
        .find(|c: char| {
            c.is_whitespace()
                || c == strip::SENTINEL
                || matches!(c, '<' | '>' | '"' | '[' | ']' | '{' | '}')
        })
        .map_or(rest.len(), |i| proto + i);
    if url_stop == proto {
        return Step::Miss;
    }
    let url_end = strip_trailing_punct(&rest[..url_stop], proto);
    if url_end == proto {
        return Step::Miss;
    }
    Step::Emit(
        Token::Node(Inline::AutoUrl {
            url: rest[..url_end].to_owned(),
            range: TextRange::new(pos as u32, (pos + url_end) as u32),
        }),
        pos + url_end,
    )
}

/// MediaWiki strips trailing `,;.:!?»` from free links — and `)` too, unless the URL
/// contains a `(`. Returns the stripped end offset, never below `min`.
fn strip_trailing_punct(url: &str, min: usize) -> usize {
    let strip_paren = !url.contains('(');
    let mut end = url.len();
    for c in url.chars().rev() {
        if end == min {
            break;
        }
        let sep = matches!(c, ',' | ';' | '.' | ':' | '!' | '?' | '»') || (c == ')' && strip_paren);
        if !sep {
            break;
        }
        end -= c.len_utf8();
    }
    end
}

/// The longest external-link protocol at the start of `rest`, matched ASCII
/// case-insensitively. `allow_relative` admits the protocol-relative `//`.
fn match_protocol(rest: &str, config: &SiteConfig, allow_relative: bool) -> Option<usize> {
    config
        .protocols()
        .iter()
        .find(|p| {
            (allow_relative || p.as_str() != "//")
                && rest
                    .get(..p.len())
                    .is_some_and(|head| head.eq_ignore_ascii_case(p))
        })
        .map(String::len)
}

/// `[[` starts an internal link; `[proto://…]` an external one; anything else is
/// a literal bracket.
fn bracket(
    flat: &str,
    pos: usize,
    end: usize,
    config: &SiteConfig,
    warnings: &mut WarningCollector,
    depth: u8,
) -> Step {
    if flat[pos..end].starts_with("[[") {
        links::scan_internal(flat, pos, end, config, warnings, depth)
    } else {
        external(flat, pos, end, config, warnings, depth)
    }
}

/// Parses `[url]` / `[url label]` at `pos`; the bracket stays literal when the protocol
/// is unknown, the URL is empty, or the closing bracket is missing on this line.
fn external(
    flat: &str,
    pos: usize,
    end: usize,
    config: &SiteConfig,
    warnings: &mut WarningCollector,
    depth: u8,
) -> Step {
    let rest = &flat[pos + 1..end];
    let Some(proto) = match_protocol(rest, config, true) else {
        return Step::Miss;
    };
    let url_stop = rest[proto..]
        .find(|c: char| {
            c.is_whitespace() || c == strip::SENTINEL || matches!(c, ']' | '<' | '>' | '"')
        })
        .map_or(rest.len(), |i| proto + i);
    if url_stop == proto {
        return Step::Miss;
    }
    let label_start = url_stop + rest[url_stop..].bytes().take_while(|&b| b == b' ').count();
    let Some(close_rel) = rest[label_start..].find(']') else {
        return Step::Miss;
    };
    let close = label_start + close_rel;
    let label = if close > label_start {
        Some(super::parse_at_depth(
            flat,
            TextRange::new((pos + 1 + label_start) as u32, (pos + 1 + close) as u32),
            config,
            warnings,
            depth + 1,
        ))
    } else {
        None
    };
    Step::Emit(
        Token::Node(Inline::ExternalLink {
            url: rest[..url_stop].to_owned(),
            label,
        }),
        pos + close + 2,
    )
}

#[cfg(test)]
mod tests {
    use super::super::testutil::{check, config_no_magic, dump, parse, parse_with};
    use super::NAMED_ENTITIES;
    use crate::engine::strip::{self, MarkerId, MarkerKind};

    #[test]
    fn named_entity_between_text() {
        check("a&amp;b", "Text(a) Ent(&amp;=&) Text(b)");
    }

    #[test]
    fn every_pinned_named_entity_decodes() {
        assert_eq!(NAMED_ENTITIES.len(), 31);
        assert!(NAMED_ENTITIES.is_sorted_by_key(|&(n, _)| n));
        for &(name, decoded) in NAMED_ENTITIES {
            let src = format!("&{name};");
            let (nodes, warnings) = parse(&src);
            assert_eq!(dump(&src, &nodes), format!("Ent({src}={decoded})"));
            assert_eq!(warnings, Vec::new());
        }
    }

    #[test]
    fn decimal_entity() {
        check("&#65;", "Ent(&#65;=A)");
    }

    #[test]
    fn hex_entity_both_cases() {
        check("&#x41;", "Ent(&#x41;=A)");
        check("&#X42;", "Ent(&#X42;=B)");
    }

    #[test]
    fn invalid_scalar_entities_stay_literal() {
        check("&#1114112;", "Text(&#1114112;)");
        check("&#xD800;", "Text(&#xD800;)");
    }

    #[test]
    fn unknown_entity_stays_literal() {
        check("&foo;", "Text(&foo;)");
    }

    #[test]
    fn malformed_entities_stay_literal() {
        check("&;", "Text(&;)");
        check("&amp", "Text(&amp)");
        check("&#;", "Text(&#;)");
        check("&#x;", "Text(&#x;)");
        check("&#+65;", "Text(&#+65;)");
        check("&#12a;", "Text(&#12a;)");
        check("&waytoolongtobereal;", "Text(&waytoolongtobereal;)");
    }

    #[test]
    fn sentinel_entity_is_refused() {
        check("&#127;", "Text(&#127;)");
        check("&#x7f;", "Text(&#x7f;)");
    }

    #[test]
    fn markers_are_atoms() {
        for (kind, tag) in [
            (MarkerKind::ExtTag, "ExtTag"),
            (MarkerKind::Template, "Template"),
            (MarkerKind::TplArg, "TplArg"),
        ] {
            let mut src = "a".to_owned();
            strip::write_marker(&mut src, kind, MarkerId(7));
            src.push('b');
            let (nodes, warnings) = parse(&src);
            assert_eq!(
                dump(&src, &nodes),
                format!("Text(a) Marker({tag}#7) Text(b)")
            );
            assert_eq!(warnings, Vec::new());
        }
    }

    #[test]
    fn stray_sentinel_is_literal() {
        check("a\u{7f}b", "Text(a\u{7f}b)");
    }

    #[test]
    fn truncated_marker_is_literal() {
        let mut src = String::new();
        strip::write_marker(&mut src, MarkerKind::ExtTag, MarkerId(1));
        src.truncate(src.len() - 3);
        let (nodes, warnings) = parse(&src);
        assert_eq!(dump(&src, &nodes), format!("Text({src})"));
        assert_eq!(warnings, Vec::new());
    }

    #[test]
    fn bare_url_strips_trailing_dot() {
        check(
            "see https://example.com/path. end",
            "Text(see ) Auto(https://example.com/path=https://example.com/path) Text(. end)",
        );
    }

    #[test]
    fn parenthesized_url_with_inner_paren_keeps_closing_paren() {
        check(
            "(https://x/y(z))",
            "Text(() Auto(https://x/y(z))=https://x/y(z)))",
        );
    }

    #[test]
    fn parenthesized_url_without_inner_paren_drops_closing_paren() {
        check(
            "(https://x/y)",
            "Text(() Auto(https://x/y=https://x/y) Text())",
        );
    }

    #[test]
    fn protocol_relative_never_free_links() {
        check("//x.org", "Text(//x.org)");
    }

    #[test]
    fn url_needs_a_word_boundary() {
        check("xhttps://a", "Text(xhttps://a)");
        check("1https://a", "Text(1https://a)");
    }

    #[test]
    fn url_stops_at_reserved_characters() {
        check("https://a<b", "Auto(https://a=https://a) Text(<b)");
        check("https://a{b}", "Auto(https://a=https://a) Text({b})");
    }

    #[test]
    fn url_reduced_to_protocol_is_literal() {
        check("https://,", "Text(https://,)");
        check("https:// x", "Text(https:// x)");
    }

    #[test]
    fn schemeless_protocol_matches_case_insensitively() {
        check("HTTPS://a.org", "Auto(HTTPS://a.org=HTTPS://a.org)");
        check("mailto:a@b.c", "Auto(mailto:a@b.c=mailto:a@b.c)");
    }

    #[test]
    fn rfc_magic_link() {
        check("RFC 7231", "Magic(Rfc:RFC 7231)");
    }

    #[test]
    fn pmid_magic_link() {
        check("PMID 123", "Magic(Pmid:PMID 123)");
    }

    #[test]
    fn rfc_needs_exactly_one_space_and_digits() {
        check("RFC 12a", "Text(RFC 12a)");
        check("RFC  12", "Text(RFC  12)");
        check("RFCX 1", "Text(RFCX 1)");
        check("aRFC 1", "Text(aRFC 1)");
        check("RFC x", "Text(RFC x)");
    }

    #[test]
    fn isbn_thirteen_digits() {
        check(
            "ISBN 978-5-17-118366-9",
            "Magic(Isbn:ISBN 978-5-17-118366-9)",
        );
    }

    #[test]
    fn isbn_ten_digits_with_final_x() {
        check("ISBN 0-306-40615-X", "Magic(Isbn:ISBN 0-306-40615-X)");
        check("ISBN 0-306-40615-x", "Magic(Isbn:ISBN 0-306-40615-x)");
    }

    #[test]
    fn isbn_space_separators() {
        check("ISBN 0 306 40615 5", "Magic(Isbn:ISBN 0 306 40615 5)");
    }

    #[test]
    fn isbn_rejects_bad_shapes() {
        check("ISBN 12345", "Text(ISBN 12345)");
        check("ISBN 1--234567890", "Text(ISBN 1--234567890)");
        check("ISBN 123456789X5", "Text(ISBN 123456789X5)");
        check("ISBN X123456789", "Text(ISBN X123456789)");
        check("ISBN 12X4567890", "Text(ISBN 12X4567890)");
        check("ISBN-978", "Text(ISBN-978)");
    }

    #[test]
    fn magic_links_can_be_disabled() {
        let cfg = config_no_magic();
        for src in ["RFC 7231", "PMID 123", "ISBN 978-5-17-118366-9"] {
            let (nodes, warnings) = parse_with(src, &cfg);
            assert_eq!(dump(src, &nodes), format!("Text({src})"));
            assert_eq!(warnings, Vec::new());
        }
    }

    #[test]
    fn behavior_switch_resolves() {
        check("__NOTOC__", "Sw(notoc:__NOTOC__)");
        check("a__TOC__b", "Text(a) Sw(toc:__TOC__) Text(b)");
    }

    #[test]
    fn behavior_switch_is_case_sensitive_here() {
        check("__notoc__", "Text(__notoc__)");
    }

    #[test]
    fn unknown_switch_stays_literal() {
        check("__UNKNOWN__", "Text(__UNKNOWN__)");
    }

    #[test]
    fn switch_never_crosses_whitespace_or_line_end() {
        check("__NO TOC__", "Text(__NO TOC__)");
        check("__NOTOC", "Text(__NOTOC)");
        check("____", "Text(____)");
        check("_x_", "Text(_x_)");
    }

    #[test]
    fn numbered_external_link() {
        check("[http://x]", "Ext(http://x)");
    }

    #[test]
    fn labeled_external_link() {
        check("[http://x some label]", "Ext(http://x|Text(some label))");
    }

    #[test]
    fn protocol_relative_external_link() {
        check("[//x.org lbl]", "Ext(//x.org|Text(lbl))");
    }

    #[test]
    fn unknown_protocol_stays_literal() {
        check("[foo://x]", "Text([foo://x])");
    }

    #[test]
    fn unclosed_external_link_leaves_bracket_and_free_links() {
        check("[http://x", "Text([) Auto(http://x=http://x)");
    }

    #[test]
    fn external_link_with_empty_url_is_literal() {
        check("[http://]", "Text([http://])");
        check("[http:// lbl]", "Text([http:// lbl])");
    }

    #[test]
    fn external_label_is_inline_parsed() {
        check("[http://x ''i'']", "Ext(http://x|I[Text(i)])");
    }

    #[test]
    fn external_label_without_space_starts_at_url_stop() {
        check("[http://x\"y\"]", "Ext(http://x|Text(\"y\"))");
    }

    #[test]
    fn external_empty_label_is_numbered() {
        check("[http://x ]", "Ext(http://x)");
    }
}