doc-scraper-rs 0.1.0

/// A group of pages sharing a top-level URL path segment, e.g.
/// `(title: "introduction", pages: [...])`. Produced by
/// [`group_into_sections`] and consumed by [`generate_index`].
#[derive(Debug, Clone)]
pub struct Section {
    pub title: String,
    pub pages: Vec<(String, String, String)>, // (title, relative url, file path relative to dir)
}

/// Groups pages by their top-level URL path segment ("markets/ethena-usde" → "markets")
/// and renders an index.md with one H2 section per group.
pub fn generate_index(sections: &[Section]) -> String {
    let mut out = String::new();
    out.push_str("# Documentation Index\n\n");
    for section in sections {
        out.push_str(&format!("## {}\n\n", section.title));
        for (title, url, _relpath) in &section.pages {
            out.push_str(&format!("- [{}]({})\n", title, url));
        }
        out.push('\n');
    }
    out
}

/// Per-page entry shared between `generate_llms_txt` (catalog) and
/// `generate_llms_full_txt` (concatenated corpus). The `body` is included so
/// the full-txt pass doesn't need to re-read files from disk.
pub type LlmsEntry = (String, String, String, Option<String>); // (title, url, body, descr)

/// Lines: `- [Title](url)` (no description if None).
pub fn generate_llms_txt(pages: &[LlmsEntry]) -> String {
    let mut out = String::new();
    for (title, url, _body, descr) in pages {
        match descr {
            Some(d) if !d.trim().is_empty() => {
                out.push_str(&format!("- [{}]({}): {}\n", title, url, d.trim()));
            }
            _ => out.push_str(&format!("- [{}]({})\n", title, url)),
        }
    }
    out
}

/// Build a single-file `llms-full.txt` per the [llmstxt.org](https://llmstxt.org/)
/// extended-format spec. The output is suitable for direct upload to model-
/// provider file APIs (OpenAI Files, Anthropic prompt-cache attachments, etc.).
///
/// Layout:
/// ```text
/// # {site_title}
///
/// > {site_summary}
///
/// # {page_title}
///
/// URL: {page_url}
///
/// {page_body}
///
/// ---
///
/// # {next_page_title}
/// ...
/// ```
///
/// `pages` is `(title, url, body)` triples. Order is preserved — the sitemap
/// fetch order is the most natural reading order for an LLM.
pub fn generate_llms_full_txt(
    site_title: &str,
    site_summary: &str,
    pages: &[(String, String, String)], // (title, url, body)
) -> String {
    let mut out = String::new();
    out.push_str(&format!("# {}\n\n", site_title));
    out.push_str(&format!("> {}\n\n", site_summary));
    for (title, url, body) in pages {
        out.push_str(&format!("# {}\n\n", title));
        out.push_str(&format!("URL: {}\n\n", url));
        // Body is verbatim. We trim a single trailing newline if present so the
        // separator below stays unambiguous, but otherwise we don't touch the
        // markdown — it has already been through extract_title (which only
        // reads the H1, doesn't mutate) before reaching here.
        let body_trimmed = body.trim_end_matches('\n');
        out.push_str(body_trimmed);
        out.push_str("\n\n---\n\n");
    }
    out
}

/// A section grouping as used by AGENTS.md and `skills/`. The relpath is the
/// on-disk filename (or relative path) so we can build markdown links to it.
pub type AgentSection = (String, Vec<(String, String, String)>); // (section_title, [(title, url, relpath)])

/// Synthesize an AGENTS.md ([agents.md](https://agents.md/)) from the scraped
/// corpus. This file is auto-loaded as project-level context by Claude Code,
/// OpenAI Codex, GitHub Copilot, Cursor, and aider.
///
/// Layout:
/// ```text
/// # {site_title} — Agent Context
///
/// > Auto-generated from {N} pages on {date}.
///
/// ## What this is
/// {overview paragraph, if known}
///
/// ## Sections
/// ### {section_title}
/// - [{page_title}]({relpath})
/// - ...
///
/// ## Per-topic deep dives
/// - [{NN}-{slug}](skills/{NN}-{slug}.md) — {section_title} ({k} pages)
///
/// ## How to use this
/// - ...
/// ```
///
/// `sections` is already grouped + sorted by the caller. `overview_first_para`
/// is the synthesized "what is this codebase" line; pass `None` to omit the
/// block (the header still renders).
pub fn build_agents_md(
    site_title: &str,
    source_url: &str,
    generated_at: &str,
    sections: &[AgentSection],
    overview_first_para: Option<&str>,
) -> String {
    let total_pages: usize = sections.iter().map(|(_, p)| p.len()).sum();
    let mut out = String::new();
    out.push_str(&format!("# {site_title} — Agent Context\n\n"));
    out.push_str(&format!(
        "> Auto-generated by `doc-scraper-rs` from `{source_url}` on {generated_at}. \
         {total_pages} pages across {} sections.\n\n",
        sections.len()
    ));

    if let Some(para) = overview_first_para {
        let trimmed = para.trim();
        if !trimmed.is_empty() {
            out.push_str("## What this is\n\n");
            out.push_str(trimmed);
            out.push_str("\n\n");
        }
    }

    out.push_str("## Sections\n\n");
    for (section_title, pages) in sections {
        out.push_str(&format!("### {section_title}\n\n"));
        for (title, _url, relpath) in pages {
            out.push_str(&format!("- [{title}]({relpath})\n"));
        }
        out.push('\n');
    }

    out.push_str("## Per-topic deep dives\n\n");
    out.push_str(
        "Each top-level section is also a standalone file under `skills/`, \
                  for agents that prefer to load context on demand.\n\n",
    );
    for (idx, (section_title, pages)) in sections.iter().enumerate() {
        let slug = slugify_section(section_title);
        let fname = format!("{:02}-{slug}.md", idx);
        out.push_str(&format!(
            "- [{}](skills/{}) — {} ({} pages)\n",
            fname,
            fname,
            section_title,
            pages.len()
        ));
    }
    out.push('\n');

    out.push_str("## How to use this\n\n");
    out.push_str("- For a quick orientation, read \"What this is\" above.\n");
    out.push_str("- For section-specific questions, read the matching `skills/NN-*.md` file — the numeric prefix matches the order in **Sections** above.\n");
    out.push_str("- For exhaustive context, read `llms-full.txt` (every page concatenated).\n");
    out.push_str("- For a navigable index of pages, read `llms.txt`.\n");
    out.push_str("- For per-page files at the original URLs, browse the directory tree.\n");
    out
}

/// Build one per-section `skills/NN-<slug>.md` file. Same shape as a slice of
/// `llms-full.txt`: a header followed by per-page blocks separated by `---`.
/// Designed to be loaded on-demand by an agent rather than always-on.
pub fn generate_skill_md(
    section_title: &str,
    site_title: &str,
    pages: &[(String, String, String)], // (title, url, body)
) -> String {
    let mut out = String::new();
    out.push_str(&format!("# {section_title} — {site_title}\n\n"));
    out.push_str(&format!(
        "> Topic-scoped context for {site_title}. {} pages in this section.\n\n",
        pages.len()
    ));
    for (title, url, body) in pages {
        out.push_str(&format!("# {title}\n\n"));
        out.push_str(&format!("URL: {url}\n\n"));
        let body_trimmed = body.trim_end_matches('\n');
        out.push_str(body_trimmed);
        out.push_str("\n\n---\n\n");
    }
    out
}

/// Convert a section title to a filename-safe slug. `Risk Framework` →
/// `risk-framework`; `Markets / Ethena` → `markets-ethena`; collapse runs of
/// non-alphanumeric chars into a single `-`, trim leading/trailing dashes,
/// lowercase.
pub fn slugify_section(title: &str) -> String {
    let mut out = String::with_capacity(title.len());
    let mut last_dash = false;
    for ch in title.chars() {
        if ch.is_ascii_alphanumeric() {
            out.push(ch.to_ascii_lowercase());
            last_dash = false;
        } else if !last_dash && !out.is_empty() {
            out.push('-');
            last_dash = true;
        }
    }
    while out.ends_with('-') {
        out.pop();
    }
    if out.is_empty() {
        out.push_str("section");
    }
    out
}

/// Helper that turns a flat page list into Section records grouped by URL top-level.
/// `pages` is (title, full relative url, rel path on disk).
pub fn group_into_sections(pages: Vec<(String, String, String)>) -> Vec<Section> {
    use std::collections::BTreeMap;
    let mut map: BTreeMap<String, Vec<(String, String, String)>> = BTreeMap::new();
    for (title, url, relpath) in pages {
        let top = url
            .trim_start_matches('/')
            .split('/')
            .next()
            .unwrap_or("")
            .to_string();
        let key = if top.is_empty() { "index".into() } else { top };
        map.entry(key).or_default().push((title, url, relpath));
    }
    map.into_iter()
        .map(|(title, pages)| Section { title, pages })
        .collect()
}

#[cfg(test)]
mod tests {
    use super::*;

    fn p(title: &str, url: &str) -> (String, String, String) {
        (title.into(), url.into(), format!("{}.md", url))
    }

    #[test]
    fn group_into_sections_orders_alphabetically() {
        let pages = vec![
            p(
                "Protocol Overview",
                "technical-documentation/protocol-overview",
            ),
            p("Why Strata", "introduction/why-strata"),
            p("Ethena USDe", "markets/ethena-usde"),
            p("Senior Tranche", "introduction/senior-tranche"),
        ];
        let sections = group_into_sections(pages);
        assert_eq!(sections[0].title, "introduction");
        assert_eq!(sections[0].pages.len(), 2);
        assert_eq!(sections[1].title, "markets");
        assert_eq!(sections[2].title, "technical-documentation");
    }

    #[test]
    fn generate_index_emits_h2_per_section() {
        let sections = group_into_sections(vec![p("Foo", "intro/foo"), p("Bar", "markets/bar")]);
        let md = generate_index(&sections);
        assert!(md.starts_with("# Documentation Index\n\n"));
        assert!(md.contains("## intro\n"));
        assert!(md.contains("## markets\n"));
        assert!(md.contains("- [Foo](intro/foo)\n"));
    }

    #[test]
    fn generate_llms_txt_with_and_without_description() {
        let pages: Vec<LlmsEntry> = vec![
            (
                "A".into(),
                "/a".into(),
                String::new(),
                Some("First page".into()),
            ),
            ("B".into(), "/b".into(), String::new(), None),
            ("C".into(), "/c".into(), String::new(), Some("".into())),
        ];
        let s = generate_llms_txt(&pages);
        assert!(s.contains("- [A](/a): First page\n"));
        assert!(s.contains("- [B](/b)\n"));
        assert!(s.contains("- [C](/c)\n"));
    }

    #[test]
    fn generate_llms_full_txt_header_only_for_empty_pages() {
        let s = generate_llms_full_txt("Example", "summary line", &[]);
        assert_eq!(s, "# Example\n\n> summary line\n\n");
    }

    #[test]
    fn generate_llms_full_txt_emits_header_and_per_page_block() {
        let pages = vec![
            (
                "Tranching".into(),
                "https://x/t".into(),
                "Senior tranche absorbs first loss.\n".into(),
            ),
            (
                "Overview".into(),
                "https://x/o".into(),
                "Top-level intro.\n".into(),
            ),
        ];
        let s = generate_llms_full_txt("Pareto", "Concatenated corpus.", &pages);
        // Header
        assert!(s.starts_with("# Pareto\n\n> Concatenated corpus.\n\n"));
        // First page block
        assert!(s.contains(
            "# Tranching\n\nURL: https://x/t\n\nSenior tranche absorbs first loss.\n\n---\n\n"
        ));
        // Second page block, separator between them
        assert!(s.contains("# Overview\n\nURL: https://x/o\n\nTop-level intro.\n\n---\n\n"));
        // Exactly two separators (one per page)
        assert_eq!(s.matches("\n---\n").count(), 2);
    }

    #[test]
    fn generate_llms_full_txt_preserves_in_page_hr() {
        // A page body that itself contains `---` (a markdown horizontal rule)
        // must not break the boundary. Our separator is `\n\n---\n\n` (with
        // blank lines); an in-page `---` is typically surrounded by content,
        // not blank lines on both sides.
        let pages = vec![(
            "Has HR".into(),
            "https://x/h".into(),
            "before\n\n---\n\nafter\n".into(),
        )];
        let s = generate_llms_full_txt("Site", "sum", &pages);
        // Two `---` occurrences: one inside the body, one trailing separator.
        assert_eq!(s.matches("---").count(), 2);
        assert!(s.ends_with("\n\n---\n\n"));
    }

    #[test]
    fn generate_llms_full_txt_handles_empty_body() {
        let pages = vec![("Empty".into(), "https://x/e".into(), String::new())];
        let s = generate_llms_full_txt("Site", "sum", &pages);
        assert!(s.contains("# Empty\n\nURL: https://x/e\n\n\n\n---\n\n"));
    }

    fn section(title: &str, pages: &[(&str, &str, &str)]) -> AgentSection {
        (
            title.into(),
            pages
                .iter()
                .map(|(t, u, rp)| (t.to_string(), u.to_string(), rp.to_string()))
                .collect(),
        )
    }

    #[test]
    fn build_agents_md_empty_sections() {
        let s = build_agents_md("Site", "https://x/", "2026-07-08", &[], None);
        assert!(s.starts_with("# Site — Agent Context\n\n"));
        assert!(s.contains("0 pages across 0 sections"));
        // "What this is" block absent (no overview).
        assert!(!s.contains("## What this is"));
        // Sections and per-topic blocks still render as headers, but empty.
        assert!(s.contains("## Sections\n\n"));
        assert!(s.contains("## Per-topic deep dives\n\n"));
        assert!(s.contains("## How to use this\n\n"));
    }

    #[test]
    fn build_agents_md_with_overview_and_sections() {
        let sections = vec![
            section(
                "introduction",
                &[("Why Strata", "https://x/i/w", "introduction/why-strata.md")],
            ),
            section(
                "markets",
                &[("Ethena", "https://x/m/e", "markets/ethena.md")],
            ),
        ];
        let s = build_agents_md(
            "Strata",
            "https://docs.strata.markets/",
            "2026-07-08",
            &sections,
            Some("Strata is a structured-yield protocol that splits risk into senior and junior tranches."),
        );
        assert!(s.starts_with("# Strata — Agent Context\n\n"));
        assert!(s.contains("2 pages across 2 sections"));
        // Overview block present
        assert!(s.contains("## What this is\n\nStrata is a structured-yield protocol"));
        // Sections listed alphabetically (introduction before markets)
        let intro_pos = s.find("### introduction").unwrap();
        let markets_pos = s.find("### markets").unwrap();
        assert!(intro_pos < markets_pos);
        // Page link present
        assert!(s.contains("- [Why Strata](introduction/why-strata.md)"));
        // Per-topic deep dives use 00-, 01- numeric prefixes
        assert!(s.contains("- [00-introduction.md](skills/00-introduction.md)"));
        assert!(s.contains("- [01-markets.md](skills/01-markets.md)"));
    }

    #[test]
    fn build_agents_md_overview_omitted_when_empty() {
        let s = build_agents_md("Site", "https://x/", "2026-07-08", &[], Some(""));
        assert!(!s.contains("## What this is"));
    }

    #[test]
    fn generate_skill_md_emits_per_page_blocks() {
        let pages = vec![
            (
                "Why Strata".into(),
                "https://x/w".into(),
                "Senior tranche absorbs first loss.\n".into(),
            ),
            (
                "Overview".into(),
                "https://x/o".into(),
                "Top-level intro.\n".into(),
            ),
        ];
        let s = generate_skill_md("introduction", "Strata", &pages);
        assert!(s.starts_with("# introduction — Strata\n\n"));
        assert!(s.contains("> Topic-scoped context for Strata. 2 pages in this section."));
        // Both page blocks present
        assert!(s.contains(
            "# Why Strata\n\nURL: https://x/w\n\nSenior tranche absorbs first loss.\n\n---\n\n"
        ));
        assert!(s.contains("# Overview\n\nURL: https://x/o\n\nTop-level intro.\n\n---\n\n"));
        // Exactly two `---` separators (one per page)
        assert_eq!(s.matches("\n---\n").count(), 2);
    }

    #[test]
    fn generate_skill_md_handles_empty_body() {
        let pages = vec![("Empty".into(), "https://x/e".into(), String::new())];
        let s = generate_skill_md("section", "Site", &pages);
        assert!(s.contains("# Empty\n\nURL: https://x/e\n\n\n\n---\n\n"));
    }

    #[test]
    fn slugify_section_basic() {
        assert_eq!(slugify_section("Risk Framework"), "risk-framework");
        assert_eq!(slugify_section("introduction"), "introduction");
        assert_eq!(slugify_section("UPPER_case"), "upper-case");
    }

    #[test]
    fn slugify_section_collapses_runs_of_separators() {
        assert_eq!(slugify_section("Markets / Ethena"), "markets-ethena");
        assert_eq!(slugify_section("A   B"), "a-b");
        assert_eq!(slugify_section("--leading--"), "leading");
    }

    #[test]
    fn slugify_section_falls_back_when_all_separators() {
        // Pathological input: all separators. We should return *something*
        // rather than empty, so the file `NN-.md` doesn't collide.
        assert_eq!(slugify_section("///"), "section");
        assert_eq!(slugify_section(""), "section");
    }
}