wikioxide 0.1.0

//! Flatten (stage S1 → S3 bridge): renders a [`PpDom`] to the marker text the line
//! parser consumes — literal text verbatim, comments and mode-ignored content gone,
//! every template/argument/extension-tag capture replaced by an opaque strip marker.

use crate::engine::provenance::ProvMap;
use crate::engine::source::{SourceId, SourceStore};
use crate::engine::strip::{StripPayload, StripState};
use crate::warn::{Warning, WarningCollector};

use super::{PpDom, PpKind};

/// The flattened text plus the map that carries its offsets back to the original source.
pub(crate) struct Flattened {
    /// Marker text the line parser consumes.
    pub(crate) text: String,
    /// Flat→source offset map (verbatim-text runs only; see [`ProvMap`]).
    pub(crate) prov: ProvMap,
}

/// Renders `dom` (whose nodes span source buffer `src`) to flattened text, registering one
/// strip marker per structure node and recording each verbatim-text run in a [`ProvMap`].
///
/// Only the root sequence is walked: children of a template or argument node live
/// behind its marker (the payload carries the node id, so parse-only extraction and
/// the expansion stage descend from there). Extension tags left unclosed at end of
/// input surface here as [`Warning::UnclosedExtensionTag`] — the scanner itself
/// stays silent.
pub(crate) fn flatten(
    dom: &PpDom,
    store: &SourceStore,
    src: SourceId,
    strips: &mut StripState,
    warnings: &mut WarningCollector,
) -> Flattened {
    let mut out = String::new();
    let mut prov = ProvMap::new(store.text(src).len() as u32);
    for &id in dom.root() {
        let node = dom.node(id);
        let flat_start = out.len() as u32;
        match &node.kind {
            PpKind::Text => {
                out.push_str(store.slice(node.span));
                prov.push_text(flat_start, node.span);
            }
            PpKind::Comment | PpKind::Ignored => {}
            PpKind::Template { .. } => {
                strips.add(&mut out, StripPayload::Template(id));
            }
            PpKind::TplArg { .. } => {
                strips.add(&mut out, StripPayload::TplArg(id));
            }
            PpKind::ExtTag {
                name,
                attributes,
                body,
                closed,
            } => {
                let name = store.slice(*name).to_lowercase();
                if !closed {
                    warnings.emit(Warning::UnclosedExtensionTag { name: name.clone() });
                }
                strips.add(
                    &mut out,
                    StripPayload::ExtTag {
                        name,
                        attributes: *attributes,
                        body: *body,
                    },
                );
            }
        }
    }
    prov.seal(out.len() as u32);
    Flattened { text: out, prov }
}

#[cfg(test)]
mod tests {
    use super::super::{InclusionMode, build};
    use super::*;
    use crate::config::{ExtTagContent, NamespaceDef, SiteConfig};
    use crate::engine::source::{SourceId, Span};
    use crate::engine::strip::{MarkerId, MarkerKind, find_marker, write_marker};

    /// Test config matching the builder tests: nowiki/pre opaque, ref inline.
    fn cfg() -> SiteConfig {
        SiteConfig::builder("en", "Test Wiki")
            .namespace(NamespaceDef::main())
            .namespace(NamespaceDef::new(
                10,
                "Template",
                "Template",
                Vec::<String>::new(),
                true,
            ))
            .ext_tag("nowiki", ExtTagContent::Opaque)
            .ext_tag("pre", ExtTagContent::Opaque)
            .ext_tag("ref", ExtTagContent::InlineWikitext)
            .build()
            .unwrap()
    }

    /// Builds `input` (TopLevel) and flattens it, returning everything produced.
    fn run(input: &str) -> (SourceStore, PpDom, StripState, Vec<Warning>, String) {
        let mut store = SourceStore::new();
        let (src, _) = store.push(input, 4 * 1024 * 1024);
        let mut warnings = WarningCollector::new();
        let dom = build(&store, src, &cfg(), InclusionMode::TopLevel, &mut warnings);
        let mut strips = StripState::new();
        let text = flatten(&dom, &store, src, &mut strips, &mut warnings).text;
        (store, dom, strips, warnings.into_vec(), text)
    }

    /// Every marker in `text`, in order.
    fn markers(text: &str) -> Vec<(MarkerKind, MarkerId)> {
        let mut out = Vec::new();
        let mut at = 0;
        while let Some((pos, decoded)) = find_marker(text, at) {
            out.push((decoded.kind, decoded.id));
            at = pos + decoded.len;
        }
        out
    }

    #[test]
    fn pure_text_flattens_verbatim() {
        for input in ["", "plain текст", "a}}b", "{{x}", "a=b|c"] {
            let (_, _, strips, warnings, text) = run(input);
            assert_eq!(text, input, "no bytes lost for {input:?}");
            assert_eq!(strips.len(), 0);
            assert_eq!(warnings, []);
        }
    }

    #[test]
    fn comments_and_ignored_content_vanish() {
        let (_, _, strips, warnings, text) =
            run("a<!-- gone -->b<includeonly>c</includeonly>d<noinclude>e</noinclude>f");
        assert_eq!(text, "abdef", "noinclude tags drop but their content stays");
        assert_eq!(strips.len(), 0);
        assert_eq!(warnings, []);
    }

    #[test]
    fn template_and_tplarg_become_markers_pointing_at_their_nodes() {
        let (_, dom, strips, warnings, text) = run("x{{a|b}}y{{{1}}}z");
        assert_eq!(warnings, []);
        let found = markers(&text);
        assert_eq!(
            found.iter().map(|&(kind, _)| kind).collect::<Vec<_>>(),
            [MarkerKind::Template, MarkerKind::TplArg]
        );
        assert_eq!(
            strips.get(found[0].1),
            Some(&StripPayload::Template(dom.root()[1]))
        );
        assert_eq!(
            strips.get(found[1].1),
            Some(&StripPayload::TplArg(dom.root()[3]))
        );
        // The literal runs survive around the markers, in order.
        let stripped: String = text
            .split('\u{7f}')
            .filter(|chunk| !chunk.starts_with("UNIQ-") && !chunk.ends_with("-QINU"))
            .collect();
        assert_eq!(stripped, "xyz");
    }

    #[test]
    fn flattened_text_matches_the_exact_marker_encoding() {
        let (_, _, _, _, text) = run("a{{b}}c");
        let mut expected = String::from("a");
        write_marker(&mut expected, MarkerKind::Template, MarkerId(0));
        expected.push('c');
        assert_eq!(text, expected);
    }

    #[test]
    fn ext_tag_payload_carries_lowercased_name_attributes_and_body() {
        let (store, _, strips, warnings, text) = run("<ReF name=\"n\">body</rEf>");
        assert_eq!(warnings, []);
        let found = markers(&text);
        assert_eq!(found.len(), 1);
        let attributes = Some(Span::new(SourceId(0), 4, 13));
        let body = Some(Span::new(SourceId(0), 14, 18));
        let expected = StripPayload::ExtTag {
            name: "ref".to_owned(),
            attributes,
            body,
        };
        assert_eq!(strips.get(found[0].1), Some(&expected));
        assert_eq!(attributes.map(|s| store.slice(s)), Some(" name=\"n\""));
        assert_eq!(body.map(|s| store.slice(s)), Some("body"));
    }

    #[test]
    fn self_closing_ext_tag_payload_has_no_body() {
        let (_, _, strips, warnings, text) = run("a<nowiki/>b");
        assert_eq!(warnings, []);
        let found = markers(&text);
        assert_eq!(
            strips.get(found[0].1),
            Some(&StripPayload::ExtTag {
                name: "nowiki".to_owned(),
                attributes: None,
                body: None,
            })
        );
    }

    #[test]
    fn unclosed_ext_tag_warns_at_flatten_time() {
        let (store, _, strips, warnings, text) = run("a<PRE>rest");
        assert_eq!(
            warnings,
            [Warning::UnclosedExtensionTag {
                name: "pre".to_owned()
            }],
            "lower-cased name in the warning"
        );
        let found = markers(&text);
        let body = Some(Span::new(SourceId(0), 6, 10));
        let expected = StripPayload::ExtTag {
            name: "pre".to_owned(),
            attributes: None,
            body,
        };
        assert_eq!(strips.get(found[0].1), Some(&expected));
        assert_eq!(body.map(|s| store.slice(s)), Some("rest"));
    }

    #[test]
    fn markers_appear_in_document_order() {
        let (_, _, strips, _, text) = run("{{a}}<nowiki>x</nowiki>{{{2}}}{{b}}");
        assert_eq!(
            markers(&text)
                .iter()
                .map(|&(kind, id)| (kind, id.0))
                .collect::<Vec<_>>(),
            [
                (MarkerKind::Template, 0),
                (MarkerKind::ExtTag, 1),
                (MarkerKind::TplArg, 2),
                (MarkerKind::Template, 3)
            ]
        );
        assert_eq!(strips.len(), 4);
    }

    #[test]
    fn empty_source_flattens_to_empty_dom_output() {
        let mut store = SourceStore::new();
        let (src, _) = store.push("", 4 * 1024 * 1024);
        let dom = PpDom::new();
        let mut strips = StripState::new();
        let mut warnings = WarningCollector::new();
        assert_eq!(
            flatten(&dom, &store, src, &mut strips, &mut warnings).text,
            ""
        );
        assert_eq!(strips.len(), 0);
    }
}