kowalski_core/tools/internal/
web.rs

1//! In-process **web** helpers: HTTP fetch and a lightweight **HTML → readable Markdown** pass.
2//!
3//! This is the **default** path when a URL returns HTML (non-GitHub or GitHub HTML fallback).
4//! For high-fidelity extraction (readability, paywalls, JS), prefer an **MCP** server or the
5//! [Docker MCP Toolkit](https://docs.docker.com/ai/mcp-catalog-and-toolkit/toolkit/).
6
7use regex::Regex;
8use reqwest::blocking::Client;
9use std::time::Duration;
10
11const FETCH_TIMEOUT_SECS: u64 = 90;
12
13/// Marker for the web helpers module; the agent tools built on it live in `web_tools`.
14#[derive(Debug, Clone, Copy, Default)]
15pub struct WebInternalModule;
16
17fn http_client() -> Result<Client, String> {
18    Client::builder()
19        .timeout(Duration::from_secs(FETCH_TIMEOUT_SECS))
20        .user_agent(concat!("Kowalski/", env!("CARGO_PKG_VERSION")))
21        .build()
22        .map_err(|e| e.to_string())
23}
24
25/// True if the payload is likely HTML (browser page, error HTML, etc.).
26pub fn looks_like_html(s: &str) -> bool {
27    let t = s.trim_start();
28    t.starts_with("<!DOCTYPE")
29        || t.starts_with("<!doctype")
30        || t.starts_with("<html")
31        || t.starts_with("<HTML")
32        || (t.contains('<') && t.contains('>') && t[..t.len().min(512)].contains("</"))
33}
34
35fn flatten_inline_tags(html: &str) -> String {
36    let re_tags = Regex::new(r"<[^>]+>").expect("valid regex");
37    let t = re_tags.replace_all(html, " ");
38    html_entities::decode_html_entities(t.trim())
39}
40
41fn decode_href_entities(url: &str) -> String {
42    html_entities::decode_html_entities(url.trim())
43}
44
45/// Strip scripts/styles and tags; collapse whitespace into Markdown-ish plain text blocks.
46/// Preserves hyperlink targets as `[text](url)` before generic tag stripping so URLs are not lost.
47/// This is **not** a full Readability clone — it makes HTML **usable** in LLM/source bundles.
48pub fn html_body_to_markdown(html: &str) -> String {
49    html_to_markdown_at(html, None)
50}
51
52/// The page's own content: the `<main>` element (else a lone `<article>`) when there is one,
53/// without navigation, header, footer, menus and other page chrome.
54fn main_content(html: &str) -> std::borrow::Cow<'_, str> {
55    // an unclosed <main> (a body cut at the fetch cap) runs to the end of what arrived
56    let re_main = Regex::new(r"(?is)<main\b[^>]*>(.*?)(?:</main\s*>|\z)").expect("valid regex");
57    let re_article = Regex::new(r"(?is)<article\b[^>]*>(.*?)</article\s*>").expect("valid regex");
58    // one <article> is the page's content; many are rows of a list, so keep the page
59    let single_article = || {
60        let mut all = re_article.captures_iter(html);
61        match (all.next(), all.next()) {
62            (Some(c), None) => Some(c),
63            _ => None,
64        }
65    };
66    let picked = re_main
67        .captures(html)
68        .or_else(single_article)
69        .and_then(|c| c.get(1))
70        .map(|m| m.as_str())
71        .filter(|m| m.len() > 200);
72    let mut out = picked.unwrap_or(html).to_string();
73    for tag in ["nav", "header", "footer", "aside", "svg", "noscript", "template", "form", "details", "select", "dialog"] {
74        let re = Regex::new(&format!(r"(?is)<{tag}\b[^>]*>.*?</{tag}\s*>")).expect("valid regex");
75        out = re.replace_all(&out, " ").into_owned();
76    }
77    std::borrow::Cow::Owned(out)
78}
79
80/// [`html_body_to_markdown`] for a page fetched from `base`: relative links become absolute.
81pub fn html_to_markdown_at(html: &str, base: Option<&reqwest::Url>) -> String {
82    let title = Regex::new(r"(?is)<title[^>]*>(.*?)</title>")
83        .expect("valid regex")
84        .captures(html)
85        .and_then(|c| c.get(1))
86        .map(|m| flatten_inline_tags(m.as_str()))
87        .filter(|t| !t.is_empty());
88    let content = main_content(html);
89    let html = content.as_ref();
90    let re_script = Regex::new(r"(?is)<script[^>]*>.*?</script>").expect("valid regex");
91    let re_style = Regex::new(r"(?is)<style[^>]*>.*?</style>").expect("valid regex");
92    let re_anchor = Regex::new(
93        r#"(?is)<a\s[^>]*?\bhref\s*=\s*(?:"(?P<dq>[^"]*)"|'(?P<sq>[^']*)')[^>]*>(?P<inner>.*?)</a>"#,
94    )
95    .expect("valid regex");
96    let re_tags = Regex::new(r"<[^>]+>").expect("valid regex");
97    let re_ws = Regex::new(r"[ \t\r\f\v]+").expect("valid regex");
98    let re_nl = Regex::new(r"\n{3,}").expect("valid regex");
99
100    let s = re_script.replace_all(html, "");
101    let s = re_style.replace_all(&s, "");
102    let s = re_anchor.replace_all(&s, |caps: &regex::Captures| {
103        let url = caps
104            .name("dq")
105            .or_else(|| caps.name("sq"))
106            .map(|m| decode_href_entities(m.as_str()))
107            .unwrap_or_default();
108        let url = match base {
109            Some(b) if !url.is_empty() && !url.starts_with('#') && !url.contains("://") => {
110                b.join(&url).map(|u| u.to_string()).unwrap_or(url)
111            }
112            _ => url,
113        };
114        let inner = caps.name("inner").map(|m| m.as_str()).unwrap_or("");
115        let text = flatten_inline_tags(inner);
116        if url.is_empty() {
117            return text;
118        }
119        let low = url.to_ascii_lowercase();
120        if low.starts_with("javascript:") || low.starts_with("data:") {
121            return text;
122        }
123        if text.is_empty() {
124            return format!("<{url}>");
125        }
126        if url.contains(' ') && !url.starts_with('<') {
127            return format!("[{text}](<{url}>)");
128        }
129        if url.contains(')') {
130            return format!("[{text}](<{url}>)");
131        }
132        format!("[{text}]({url})")
133    });
134    let s = re_tags.replace_all(&s, " ");
135    let s: String = html_entities::decode_html_entities(s.as_ref());
136    let s = re_ws.replace_all(&s, " ");
137    let lines: Vec<&str> = s.lines().map(|l| l.trim()).filter(|l| !l.is_empty()).collect();
138    let body = lines.join("\n\n");
139    let body = re_nl.replace_all(&body, "\n\n");
140    let heading = title.map(|t| format!("# {t}\n\n")).unwrap_or_default();
141    format!(
142        "<!-- converted from HTML (internal web tool; heuristic strip) -->\n\n{heading}{}\n",
143        body.trim()
144    )
145}
146
147// Minimal entity decode without pulling `html-escape` for a few cases
148mod html_entities {
149    pub fn decode_html_entities(s: &str) -> String {
150        s.replace("&nbsp;", " ")
151            .replace("&lt;", "<")
152            .replace("&gt;", ">")
153            .replace("&amp;", "&")
154            .replace("&quot;", "\"")
155            .replace("&#39;", "'")
156    }
157}
158
159/// Plain HTTP GET body as UTF-8 string (no GitHub resolution).
160pub fn fetch_http_body(url: &str) -> Result<String, String> {
161    let client = http_client()?;
162    let resp = client.get(url).send().map_err(|e| e.to_string())?;
163    if !resp.status().is_success() {
164        return Err(format!(
165            "HTTP {} {}",
166            resp.status().as_u16(),
167            resp.status().canonical_reason().unwrap_or("")
168        ));
169    }
170    resp.text().map_err(|e| e.to_string())
171}
172
173/// Fetch URL; if the body looks like HTML, convert to readable Markdown text.
174pub fn fetch_url_as_markdown(url: &str) -> Result<String, String> {
175    let text = fetch_http_body(url)?;
176    if looks_like_html(&text) {
177        Ok(html_to_markdown_at(&text, reqwest::Url::parse(url).ok().as_ref()))
178    } else {
179        Ok(text)
180    }
181}
182
183#[cfg(test)]
184mod tests {
185    use super::*;
186
187    #[test]
188    fn detects_html() {
189        assert!(looks_like_html("<html><body>Hi</body></html>"));
190        assert!(!looks_like_html("# Just markdown"));
191    }
192
193    #[test]
194    fn strips_script_and_tags() {
195        let html = r#"<html><head><script>evil()</script><style>.x{}</style></head>
196            <body><h1>Title</h1><p>Hello <b>world</b></p></body></html>"#;
197        let md = html_body_to_markdown(html);
198        assert!(!md.contains("evil"));
199        assert!(!md.contains("<script"));
200        assert!(md.contains("Title"));
201        assert!(md.contains("world"));
202    }
203
204    #[test]
205    fn keeps_main_content_and_resolves_relative_links() {
206        let html = r#"<html><head><title>Trending Rust</title></head><body>
207            <header><a href="/login">Sign in</a></header><nav>Platform Copilot Actions</nav>
208            <main><h2><a href="/tokio-rs/tokio">tokio-rs / tokio</a></h2>
209            <p>A runtime for writing reliable asynchronous applications with Rust. Stars today: 120.
210            More text so the main element is clearly the page's own content, not a stub.</p></main>
211            <footer>Terms Privacy</footer></body></html>"#;
212        let base = reqwest::Url::parse("https://github.com/trending/rust").unwrap();
213        let md = html_to_markdown_at(html, Some(&base));
214        assert!(md.contains("# Trending Rust"), "{md}");
215        assert!(md.contains("[tokio-rs / tokio](https://github.com/tokio-rs/tokio)"), "{md}");
216        assert!(!md.contains("Sign in") && !md.contains("Copilot") && !md.contains("Privacy"), "{md}");
217    }
218
219    #[test]
220    fn a_main_cut_off_by_the_fetch_cap_still_counts() {
221        let html = format!(
222            "<html><body><nav>Menu Menu</nav><main><p>{}</p><a href=\"/a/b\">a / b</a>",
223            "Repository description text. ".repeat(20)
224        );
225        let md = html_to_markdown_at(&html, Some(&reqwest::Url::parse("https://github.com/x").unwrap()));
226        assert!(md.contains("[a / b](https://github.com/a/b)") && !md.contains("Menu"), "{md}");
227    }
228
229    #[test]
230    fn preserves_anchors_as_markdown_links() {
231        let html = r#"<html><body><p>See <a href="https://example.com/path?q=1&amp;r=2">Example</a> now.</p></body></html>"#;
232        let md = html_body_to_markdown(html);
233        assert!(md.contains("[Example](https://example.com/path?q=1&r=2)"));
234    }
235}
kowalski_core/tools/internal/web.rs

kowalski_core/tools/internal/
web.rs
