doc-scraper-rs 0.1.0

The fastest, cleanest way to export GitBook docs as markdown.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

use crate::error::{Result, ScraperError};
use url::Url;

/// One entry from `sitemap-pages.xml`: the page URL plus the optional
/// `lastmod` timestamp. The fetch pipeline takes a `Vec<PageRef>` and
/// returns one result per entry.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PageRef {
    pub loc: Url,
    pub lastmod: Option<String>,
}

/// Fetches `https://<host>/sitemap.xml`, follows to `sitemap-pages.xml`, parses it.
pub async fn fetch_sitemap(client: &reqwest::Client, base_url: &Url) -> Result<Vec<PageRef>> {
    let sitemap_url = base_url.join("/sitemap.xml")?;
    let resp = client.get(sitemap_url).send().await?.error_for_status()?;
    let body = resp.text().await?;
    let pages_url = extract_sitemap_pages_url(&body, base_url)?;
    let pages_resp = client.get(pages_url).send().await?.error_for_status()?;
    let pages_body = pages_resp.text().await?;
    parse_sitemap_pages(&pages_body)
}

/// Extracts the `<sitemap><loc>...</loc></sitemap>` URL pointing at the pages sitemap.
pub fn extract_sitemap_pages_url(body: &str, base_url: &Url) -> Result<Url> {
    let doc = roxmltree::Document::parse(body).map_err(|e| ScraperError::Xml(e.to_string()))?;
    let root = doc.root_element();
    // <sitemapindex><sitemap><loc>...</loc></sitemap>...</sitemapindex>
    for node in root.descendants() {
        if node.tag_name().name() == "loc" {
            if let Some(text) = node.text() {
                let url = base_url.join(text.trim())?;
                return Ok(url);
            }
        }
    }
    Err(ScraperError::Sitemap("no <loc> in sitemap index".into()))
}

/// Parses sitemap-pages.xml (a `urlset` of `url` / `loc` records) into PageRef records.
pub fn parse_sitemap_pages(body: &str) -> Result<Vec<PageRef>> {
    let doc = roxmltree::Document::parse(body).map_err(|e| ScraperError::Xml(e.to_string()))?;
    let mut pages = Vec::new();
    for url_node in doc.descendants().filter(|n| n.tag_name().name() == "url") {
        let loc_text = url_node
            .children()
            .find(|c| c.tag_name().name() == "loc")
            .and_then(|c| c.text())
            .ok_or_else(|| ScraperError::Sitemap("<url> missing <loc>".into()))?;
        let loc = Url::parse(loc_text.trim())?;
        let lastmod = url_node
            .children()
            .find(|c| c.tag_name().name() == "lastmod")
            .and_then(|c| c.text())
            .map(str::to_owned);
        pages.push(PageRef { loc, lastmod });
    }
    Ok(pages)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn parse_sitemap_pages_returns_urls_in_document_order() {
        let body = r#"<?xml version="1.0" encoding="UTF-8"?>
            <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
              <url><loc>https://docs.example.com/introduction</loc></url>
              <url><loc>https://docs.example.com/markets</loc><lastmod>2026-01-15</lastmod></url>
              <url><loc>https://docs.example.com/technical/security</loc></url>
            </urlset>"#;
        let pages = parse_sitemap_pages(body).unwrap();
        assert_eq!(pages.len(), 3);
        assert_eq!(
            pages[0].loc.as_str(),
            "https://docs.example.com/introduction"
        );
        assert_eq!(pages[1].loc.as_str(), "https://docs.example.com/markets");
        assert_eq!(pages[1].lastmod.as_deref(), Some("2026-01-15"));
        assert_eq!(
            pages[2].loc.as_str(),
            "https://docs.example.com/technical/security"
        );
        assert_eq!(pages[2].lastmod, None);
    }

    #[test]
    fn parse_sitemap_pages_errors_on_url_missing_loc() {
        let body = r#"<?xml version="1.0"?>
            <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
              <url></url>
            </urlset>"#;
        let err = parse_sitemap_pages(body).unwrap_err();
        matches!(err, ScraperError::Sitemap(_));
    }

    #[test]
    fn parse_sitemap_pages_errors_on_invalid_xml() {
        let body = "<<not xml>>";
        let err = parse_sitemap_pages(body).unwrap_err();
        matches!(err, ScraperError::Xml(_));
    }

    #[test]
    fn extract_sitemap_pages_url_resolves_relative_loc() {
        let body = r#"<?xml version="1.0"?>
            <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
              <sitemap><loc>/sitemap-pages.xml</loc></sitemap>
            </sitemapindex>"#;
        let base = Url::parse("https://docs.example.com/").unwrap();
        let url = extract_sitemap_pages_url(body, &base).unwrap();
        assert_eq!(url.as_str(), "https://docs.example.com/sitemap-pages.xml");
    }

    #[test]
    fn extract_sitemap_pages_url_errors_when_no_loc() {
        let body = r#"<?xml version="1.0"?>
            <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
              <sitemap></sitemap>
            </sitemapindex>"#;
        let base = Url::parse("https://docs.example.com/").unwrap();
        let err = extract_sitemap_pages_url(body, &base).unwrap_err();
        matches!(err, ScraperError::Sitemap(_));
    }
}