scone/
web.rs

1//! URL ingestion (gap-analysis P3): fetch a page, convert to clean
2//! markdown locally, feed the engine. Network lives here in the CLI
3//! layer; scone-core stays offline by construction.
4
5use std::time::Duration;
6
7const MAX_PAGE_BYTES: u64 = 5_000_000;
8const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
9
10/// Fetch a URL and return (markdown_text, domain).
11pub fn fetch_page(url: &str) -> Result<(String, String), String> {
12    let domain = url
13        .split("//")
14        .nth(1)
15        .and_then(|rest| rest.split(['/', ':']).next())
16        .filter(|d| !d.is_empty())
17        .ok_or_else(|| format!("cannot parse a host from {url:?}"))?
18        .to_owned();
19    let mut res = ureq::get(url)
20        .config()
21        .timeout_global(Some(FETCH_TIMEOUT))
22        .build()
23        .call()
24        .map_err(|e| format!("fetch {url}: {e}"))?;
25    let html = res
26        .body_mut()
27        .with_config()
28        .limit(MAX_PAGE_BYTES)
29        .read_to_string()
30        .map_err(|e| format!("read {url}: {e}"))?;
31    let markdown = htmd::HtmlToMarkdown::builder()
32        .skip_tags(vec![
33            "script", "style", "nav", "header", "footer", "aside", "noscript",
34        ])
35        .build()
36        .convert(&html)
37        .map_err(|e| format!("convert {url}: {e}"))?;
38    if markdown.trim().is_empty() {
39        return Err(format!("{url} produced no readable text"));
40    }
41    Ok((markdown, domain))
42}
scone/web.rs

scone/
web.rs
