wikioxide 0.1.0

//! Input hygiene (stage S0) and the span/source-buffer model shared by every stage.
//!
//! All engine spans are byte ranges into a [`SourceStore`] buffer and always lie on
//! char boundaries. Buffer 0 is the page being parsed; later buffers hold template
//! bodies added during expansion.

/// Identifies one buffer in a [`SourceStore`].
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub(crate) struct SourceId(pub(crate) u32);

/// A byte range within one source buffer.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub(crate) struct Span {
    pub(crate) src: SourceId,
    pub(crate) start: u32,
    pub(crate) end: u32,
}

impl Span {
    pub(crate) fn new(src: SourceId, start: u32, end: u32) -> Self {
        debug_assert!(start <= end);
        Self { src, start, end }
    }

    #[allow(
        dead_code,
        reason = "consumed by the expansion-stage limits accounting"
    )]
    pub(crate) fn len(self) -> u32 {
        self.end - self.start
    }

    #[allow(
        dead_code,
        reason = "consumed by the expansion-stage limits accounting"
    )]
    pub(crate) fn is_empty(self) -> bool {
        self.start == self.end
    }
}

/// What hygiene did to a buffer; the caller converts this into warnings.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub(crate) struct HygieneReport {
    pub(crate) truncated: bool,
    pub(crate) sanitized_controls: u32,
}

/// Owns every text buffer of a parse.
#[derive(Debug, Default)]
pub(crate) struct SourceStore {
    buffers: Vec<Box<str>>,
}

impl SourceStore {
    pub(crate) fn new() -> Self {
        Self::default()
    }

    /// Sanitizes `raw` and stores it: CRLF/CR become LF, a leading BOM is stripped,
    /// control characters other than `\n`/`\t` — including `0x7F`, reserved as the
    /// strip-marker sentinel — become U+FFFD, and the buffer is truncated to at most
    /// `max_bytes` on a char boundary.
    pub(crate) fn push(&mut self, raw: &str, max_bytes: usize) -> (SourceId, HygieneReport) {
        let (clean, report) = sanitize(raw, max_bytes);
        let id = SourceId(u32::try_from(self.buffers.len()).unwrap_or(u32::MAX));
        self.buffers.push(clean.into_boxed_str());
        (id, report)
    }

    /// Stores engine-produced text verbatim (no hygiene): the expansion stage parks
    /// `{{#tag:…}}` bodies here so their spans can be sliced like any other. Skipping
    /// sanitization is required — the text may legitimately contain strip-marker
    /// sentinels for already-captured nested tags.
    #[allow(dead_code, reason = "consumed by the expansion engine landing next")]
    pub(crate) fn push_raw(&mut self, text: String) -> SourceId {
        let id = SourceId(u32::try_from(self.buffers.len()).unwrap_or(u32::MAX));
        self.buffers.push(text.into_boxed_str());
        id
    }

    pub(crate) fn text(&self, src: SourceId) -> &str {
        &self.buffers[src.0 as usize]
    }

    pub(crate) fn slice(&self, span: Span) -> &str {
        &self.text(span.src)[span.start as usize..span.end as usize]
    }
}

fn sanitize(raw: &str, max_bytes: usize) -> (String, HygieneReport) {
    let mut report = HygieneReport::default();
    let stripped = raw.strip_prefix('\u{feff}').unwrap_or(raw);
    let mut out = String::with_capacity(stripped.len().min(max_bytes));
    let mut chars = stripped.chars().peekable();
    while let Some(c) = chars.next() {
        let mapped = match c {
            '\r' => {
                if chars.peek() == Some(&'\n') {
                    chars.next();
                }
                '\n'
            }
            c if (c.is_control() || c == '\u{7f}') && c != '\n' && c != '\t' => {
                report.sanitized_controls += 1;
                '\u{fffd}'
            }
            c => c,
        };
        if out.len() + mapped.len_utf8() > max_bytes {
            report.truncated = true;
            break;
        }
        out.push(mapped);
    }
    (out, report)
}

#[cfg(test)]
mod tests {
    use super::*;

    const MAX: usize = 4 * 1024 * 1024;

    fn store_one(raw: &str) -> (SourceStore, SourceId, HygieneReport) {
        let mut store = SourceStore::new();
        let (id, report) = store.push(raw, MAX);
        (store, id, report)
    }

    #[test]
    fn newlines_normalize_and_bom_strips() {
        let (store, id, report) = store_one("\u{feff}a\r\nb\rc\nd");
        assert_eq!(store.text(id), "a\nb\nc\nd");
        assert_eq!(report, HygieneReport::default());
    }

    #[test]
    fn sentinel_and_controls_become_replacement_chars_but_tabs_survive() {
        let (store, id, report) = store_one("a\u{7f}b\u{0}c\td\u{1b}");
        assert_eq!(store.text(id), "a\u{fffd}b\u{fffd}c\td\u{fffd}");
        assert_eq!(report.sanitized_controls, 3);
        assert!(!report.truncated);
    }

    #[test]
    fn truncation_lands_on_a_char_boundary() {
        let mut store = SourceStore::new();
        let (id, report) = store.push("ааа", 5);
        assert_eq!(store.text(id), "аа", "2-byte chars: 5 bytes floor to 4");
        assert!(report.truncated);
    }

    #[test]
    fn trailing_cr_becomes_lf() {
        let (store, id, _) = store_one("a\r");
        assert_eq!(store.text(id), "a\n");
    }

    #[test]
    fn spans_slice_buffers() {
        let (store, id, _) = store_one("пример text");
        let span = Span::new(id, 0, 12);
        assert_eq!(store.slice(span), "пример");
        assert_eq!(span.len(), 12);
        assert!(!span.is_empty());
        assert!(Span::new(id, 3, 3).is_empty());
    }

    #[test]
    fn multiple_buffers_get_sequential_ids() {
        let mut store = SourceStore::new();
        let (a, _) = store.push("page", MAX);
        let (b, _) = store.push("template body", MAX);
        assert_eq!((a, b), (SourceId(0), SourceId(1)));
        assert_eq!(store.text(b), "template body");
        let debug = format!("{store:?} {a:?} {:?}", HygieneReport::default());
        assert!(debug.contains("SourceStore"));
    }

    #[test]
    fn empty_input_is_fine() {
        let (store, id, report) = store_one("");
        assert_eq!(store.text(id), "");
        assert_eq!(report, HygieneReport::default());
    }
}