wikioxide 0.1.0

//! Strip markers: opaque placeholders that carry extension-tag bodies — and, before
//! expansion, whole template calls — through the text passes untouched.
//!
//! Marker format mirrors MediaWiki's: `\x7F UNIQ-<kind>-<id:08x>-QINU \x7F` (without the
//! spaces). Input hygiene replaces every raw `0x7F` with U+FFFD, so the sentinel cannot
//! occur in text except where the engine wrote it.

use crate::engine::pp::PpNodeId;
use crate::engine::source::Span;

/// The reserved sentinel byte delimiting markers.
pub(crate) const SENTINEL: char = '\u{7f}';

const PREFIX: &str = "\u{7f}UNIQ-";
const SUFFIX: &str = "-QINU\u{7f}";
const HEX_LEN: usize = 8;

/// Index into [`StripState`].
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub(crate) struct MarkerId(pub(crate) u32);

/// What family of payload a marker points to.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub(crate) enum MarkerKind {
    ExtTag,
    Template,
    TplArg,
}

impl MarkerKind {
    fn code(self) -> &'static str {
        match self {
            MarkerKind::ExtTag => "ext",
            MarkerKind::Template => "tpl",
            MarkerKind::TplArg => "arg",
        }
    }

    fn from_code(code: &str) -> Option<Self> {
        match code {
            "ext" => Some(MarkerKind::ExtTag),
            "tpl" => Some(MarkerKind::Template),
            "arg" => Some(MarkerKind::TplArg),
            _ => None,
        }
    }
}

/// Appends the marker text for `(kind, id)` to `out`.
pub(crate) fn write_marker(out: &mut String, kind: MarkerKind, id: MarkerId) {
    out.push_str(PREFIX);
    out.push_str(kind.code());
    out.push_str(&format!("-{:08x}", id.0));
    out.push_str(SUFFIX);
}

/// A marker decoded from text.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct DecodedMarker {
    pub(crate) kind: MarkerKind,
    pub(crate) id: MarkerId,
    /// Total marker length in bytes.
    pub(crate) len: usize,
}

/// Decodes a marker if `text` starts with one.
pub(crate) fn parse_marker(text: &str) -> Option<DecodedMarker> {
    let rest = text.strip_prefix(PREFIX)?;
    let dash = rest.find('-')?;
    let kind = MarkerKind::from_code(&rest[..dash])?;
    let rest = &rest[dash + 1..];
    let hex = rest.get(..HEX_LEN)?;
    let id = u32::from_str_radix(hex, 16).ok()?;
    if !rest[HEX_LEN..].starts_with(SUFFIX) {
        return None;
    }
    let len = PREFIX.len() + dash + 1 + HEX_LEN + SUFFIX.len();
    Some(DecodedMarker {
        kind,
        id: MarkerId(id),
        len,
    })
}

/// Finds the next marker at or after byte `from`, returning its byte offset.
#[allow(dead_code, reason = "consumed by the unstrip stage landing next")]
pub(crate) fn find_marker(text: &str, from: usize) -> Option<(usize, DecodedMarker)> {
    let mut search = from;
    while let Some(rel) = text[search..].find(SENTINEL) {
        let at = search + rel;
        if let Some(decoded) = parse_marker(&text[at..]) {
            return Some((at, decoded));
        }
        search = at + SENTINEL.len_utf8();
    }
    None
}

/// What a marker stands for.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum StripPayload {
    /// `<tag attrs>body</tag>` captured verbatim at preprocess time.
    ExtTag {
        name: String,
        /// Raw attribute text between the tag name and `>`, if any.
        attributes: Option<Span>,
        /// Raw body between the open and close tags; `None` for self-closing tags.
        body: Option<Span>,
    },
    /// An unexpanded template call (parse-only mode); points into the `PpDom`.
    Template(PpNodeId),
    /// An unexpanded `{{{argument}}}` (parse-only mode); points into the `PpDom`.
    TplArg(PpNodeId),
}

impl StripPayload {
    pub(crate) fn kind(&self) -> MarkerKind {
        match self {
            StripPayload::ExtTag { .. } => MarkerKind::ExtTag,
            StripPayload::Template(_) => MarkerKind::Template,
            StripPayload::TplArg(_) => MarkerKind::TplArg,
        }
    }
}

/// The table markers point into.
#[derive(Debug, Default)]
pub(crate) struct StripState {
    items: Vec<StripPayload>,
}

impl StripState {
    pub(crate) fn new() -> Self {
        Self::default()
    }

    /// Registers a payload and appends its marker to `out`.
    pub(crate) fn add(&mut self, out: &mut String, payload: StripPayload) -> MarkerId {
        let id = MarkerId(u32::try_from(self.items.len()).unwrap_or(u32::MAX));
        let kind = payload.kind();
        self.items.push(payload);
        write_marker(out, kind, id);
        id
    }

    pub(crate) fn get(&self, id: MarkerId) -> Option<&StripPayload> {
        self.items.get(id.0 as usize)
    }

    /// Every registered payload, in registration order. Used to re-parse the bodies of
    /// inline-wikitext extension tags (`<ref>`, `<poem>`) after expansion.
    pub(crate) fn payloads(&self) -> &[StripPayload] {
        &self.items
    }

    #[allow(dead_code, reason = "consumed by the unstrip-stage limits accounting")]
    pub(crate) fn len(&self) -> usize {
        self.items.len()
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::engine::source::SourceId;

    #[test]
    fn markers_round_trip_through_text() {
        let mut state = StripState::new();
        let mut out = String::from("before ");
        let id = state.add(&mut out, StripPayload::Template(PpNodeId(7)));
        out.push_str(" after");

        let (at, decoded) = find_marker(&out, 0).unwrap();
        assert_eq!(at, "before ".len());
        assert_eq!(decoded.kind, MarkerKind::Template);
        assert_eq!(decoded.id, id);
        assert_eq!(state.get(id), Some(&StripPayload::Template(PpNodeId(7))));
        assert_eq!(state.len(), 1);

        let marker_text = &out[at..at + decoded.len];
        assert!(marker_text.starts_with('\u{7f}') && marker_text.ends_with('\u{7f}'));
    }

    #[test]
    fn every_kind_encodes_its_code() {
        let mut state = StripState::new();
        let mut out = String::new();
        state.add(
            &mut out,
            StripPayload::ExtTag {
                name: "nowiki".to_owned(),
                attributes: None,
                body: Some(Span::new(SourceId(0), 2, 10)),
            },
        );
        state.add(&mut out, StripPayload::Template(PpNodeId(0)));
        state.add(&mut out, StripPayload::TplArg(PpNodeId(1)));
        assert!(
            out.contains("UNIQ-ext-") && out.contains("UNIQ-tpl-") && out.contains("UNIQ-arg-")
        );

        let (first, a) = find_marker(&out, 0).unwrap();
        let (second, b) = find_marker(&out, first + a.len).unwrap();
        let (_, c) = find_marker(&out, second + b.len).unwrap();
        assert_eq!(
            [a.kind, b.kind, c.kind],
            [MarkerKind::ExtTag, MarkerKind::Template, MarkerKind::TplArg]
        );
    }

    #[test]
    fn stray_sentinels_and_malformed_markers_do_not_decode() {
        assert_eq!(parse_marker("\u{7f}UNIQ-zzz-00000000-QINU\u{7f}"), None);
        assert_eq!(parse_marker("\u{7f}UNIQ-ext-XYZ-QINU\u{7f}"), None);
        assert_eq!(parse_marker("\u{7f}UNIQ-ext-00000001-XXXX\u{7f}"), None);
        assert_eq!(parse_marker("\u{7f}nope"), None);
        assert_eq!(parse_marker("no sentinel"), None);
        assert_eq!(parse_marker("\u{7f}UNIQ-ext-0000"), None);
        assert_eq!(parse_marker("\u{7f}UNIQ-noDash"), None);
        assert_eq!(find_marker("a \u{7f} b \u{7f}oops", 0), None);
    }

    #[test]
    fn find_skips_stray_sentinel_then_finds_real_marker() {
        let mut state = StripState::new();
        let mut out = String::from("x\u{7f}y ");
        state.add(&mut out, StripPayload::TplArg(PpNodeId(3)));
        let (at, decoded) = find_marker(&out, 0).unwrap();
        assert!(at > 0);
        assert_eq!(decoded.kind, MarkerKind::TplArg);
        assert_eq!(find_marker(&out, at + decoded.len), None);
    }

    #[test]
    fn missing_ids_return_none() {
        let state = StripState::new();
        assert_eq!(state.get(MarkerId(0)), None);
        let debug = format!("{state:?} {:?}", MarkerId(1));
        assert!(debug.contains("StripState"));
    }
}