rover-fetch 0.2.0

An MCP server for fetching and prepping web content for LLM agents.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

//! Peek `<base href>` from raw HTML before readabilityrs touches it.
//!
//! Per HTML5, `<base>` belongs in `<head>` and the first one wins. We
//! return `None` for relative `<base href>` values (rare, but possible
//! against the document URL) — the caller falls back to the final URL
//! after redirects.

use scraper::{Html, Selector};
use url::Url;

pub fn read_base_href(html: &str) -> Option<Url> {
    let doc = Html::parse_document(html);
    let selector = Selector::parse("head > base[href]").ok()?;
    let first = doc.select(&selector).next()?;
    let href = first.value().attr("href")?.trim();
    if href.is_empty() {
        return None;
    }
    Url::parse(href).ok()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn absolute_base_href_parsed() {
        let html = r#"<!doctype html><html><head>
            <base href="https://x.example/articles/">
            </head><body></body></html>"#;
        assert_eq!(
            read_base_href(html).map(|u| u.to_string()),
            Some("https://x.example/articles/".to_string())
        );
    }

    #[test]
    fn missing_base_returns_none() {
        let html = "<!doctype html><html><head></head><body></body></html>";
        assert_eq!(read_base_href(html), None);
    }

    #[test]
    fn relative_base_returns_none() {
        let html = r#"<!doctype html><html><head>
            <base href="/articles/">
            </head><body></body></html>"#;
        assert_eq!(read_base_href(html), None);
    }

    #[test]
    fn first_base_wins() {
        let html = r#"<!doctype html><html><head>
            <base href="https://first.example/">
            <base href="https://second.example/">
            </head><body></body></html>"#;
        assert_eq!(
            read_base_href(html).map(|u| u.to_string()),
            Some("https://first.example/".to_string())
        );
    }
}