rpi_webfetch/
lib.rs

1use rpi_plugin_sdk::{
2    register_entrypoint, FreeStringFn, PluginApiVt, StableToolSchema, StbString, StbStringRef,
3    StepHandle, StepResult, ToolPartialCb,
4};
5use serde_json::{json, Value};
6use std::ffi::c_void;
7use std::io::Read;
8use std::net::{IpAddr, ToSocketAddrs};
9use std::sync::atomic::{AtomicBool, Ordering};
10use std::time::Duration;
11use url::Url;
12struct Drive {
13    params: Value,
14    cancelled: AtomicBool,
15    done: bool,
16}
17fn validate(input: &str) -> Result<Url, String> {
18    let u = Url::parse(input).map_err(|e| format!("invalid URL: {e}"))?;
19    if !matches!(u.scheme(), "http" | "https") {
20        return Err("only http and https URLs are allowed".into());
21    }
22    if !u.username().is_empty() || u.password().is_some() {
23        return Err("embedded credentials are not allowed".into());
24    }
25    let h = u.host_str().ok_or("URL has no host")?;
26    if h.eq_ignore_ascii_case("localhost") || h.ends_with(".localhost") {
27        return Err("local hosts are not allowed".into());
28    }
29    if let Ok(ip) = h.parse::<IpAddr>() {
30        if blocked_ip(ip) {
31            return Err("private or local IPs are not allowed".into());
32        }
33    } else {
34        // Fail closed for hostnames that resolve to loopback, RFC1918,
35        // link-local, multicast, or other non-public addresses. This catches
36        // the common DNS-based SSRF case that a literal-IP check misses.
37        let port = u.port_or_known_default().unwrap_or(443);
38        if let Ok(addrs) = (h, port).to_socket_addrs() {
39            if addrs.into_iter().any(|addr| blocked_ip(addr.ip())) {
40                return Err("hostname resolves to a private or local IP".into());
41            }
42        }
43    }
44    Ok(u)
45}
46
47fn blocked_ip(ip: IpAddr) -> bool {
48    match ip {
49        IpAddr::V4(v4) => {
50            v4.is_loopback()
51                || v4.is_private()
52                || v4.is_link_local()
53                || v4.is_unspecified()
54                || v4.is_multicast()
55                || v4.octets()[0] == 0
56        }
57        IpAddr::V6(v6) => {
58            v6.is_loopback()
59                || v6.is_unspecified()
60                || v6.is_multicast()
61                || v6.is_unique_local()
62                || v6.is_unicast_link_local()
63        }
64    }
65}
66fn fetch(p: &Value) -> Result<String, String> {
67    let url = validate(
68        p.get("url")
69            .and_then(Value::as_str)
70            .ok_or("url is required")?,
71    )?;
72    let max = p
73        .get("maxChars")
74        .and_then(Value::as_u64)
75        .unwrap_or(20480)
76        .clamp(256, 50000) as usize;
77    let client = reqwest::blocking::Client::builder()
78        .timeout(Duration::from_secs(20))
79        .redirect(reqwest::redirect::Policy::limited(5))
80        .user_agent("rpi-webfetch/0.1")
81        .build()
82        .map_err(|e| e.to_string())?;
83    let mut r = client
84        .get(url)
85        .send()
86        .map_err(|e| format!("web fetch failed: {e}"))?;
87    let status = r.status();
88    let final_url = r.url().to_string();
89    validate(&final_url)?;
90    let ct = r
91        .headers()
92        .get(reqwest::header::CONTENT_TYPE)
93        .and_then(|v| v.to_str().ok())
94        .unwrap_or("")
95        .to_string();
96    let mut b = Vec::new();
97    r.by_ref()
98        .take(1_048_577)
99        .read_to_end(&mut b)
100        .map_err(|e| e.to_string())?;
101    if b.len() > 1_048_576 {
102        return Err("response exceeded 1 MiB".into());
103    }
104    if !(ct.is_empty()
105        || ct.contains("text/")
106        || ct.contains("json")
107        || ct.contains("xml")
108        || ct.contains("html"))
109    {
110        return Ok(json!({"url":final_url,"status":status.as_u16(),"contentType":ct,"skipped":"non-text response"}).to_string());
111    }
112    let raw = String::from_utf8_lossy(&b);
113    let text = if ct.contains("html") {
114        strip_html(&raw)
115    } else {
116        raw.to_string()
117    };
118    let clipped: String = text.chars().take(max).collect();
119    Ok(json!({"url":final_url,"status":status.as_u16(),"contentType":ct,"text":clipped,"truncated":text.chars().count()>max}).to_string())
120}
121fn strip_html(s: &str) -> String {
122    let mut out = s.replace("\r", " ");
123    for tag in [
124        "script", "style", "nav", "footer", "header", "aside", "noscript",
125    ] {
126        let re_start = format!("<{}", tag);
127        while let Some(a) = out.to_ascii_lowercase().find(&re_start) {
128            if let Some(b) = out[a..].find(&format!("</{}>", tag)) {
129                out.replace_range(a..a + b + tag.len() + 3, " ");
130            } else {
131                break;
132            }
133        }
134    }
135    out = out.replace("><", ">\n<");
136    let mut result = String::new();
137    let mut inside = false;
138    for c in out.chars() {
139        match c {
140            '<' => inside = true,
141            '>' => inside = false,
142            '_' if inside => {}
143            c if !inside => result.push(c),
144            _ => {}
145        }
146    }
147    result.split_whitespace().collect::<Vec<_>>().join(" ")
148}
149extern "C" fn execute(
150    _: StbStringRef,
151    params: StbString,
152    free: Option<FreeStringFn>,
153) -> StepHandle {
154    let t = params.to_string_lossy();
155    params.free_with(free);
156    Box::into_raw(Box::new(Drive {
157        params: serde_json::from_str(&t).unwrap_or(Value::Null),
158        cancelled: AtomicBool::new(false),
159        done: false,
160    })) as StepHandle
161}
162extern "C" fn poll(h: StepHandle, _: Option<ToolPartialCb>, _: *mut c_void) -> StepResult {
163    if h.is_null() {
164        return StepResult::err(StbString::from_string("null webfetch handle".into()));
165    }
166    let d = unsafe { &mut *(h as *mut Drive) };
167    if d.cancelled.load(Ordering::SeqCst) {
168        return StepResult::err(StbString::from_string("webfetch cancelled".into()));
169    }
170    if d.done {
171        return StepResult::err(StbString::from_string(
172            "webfetch polled after completion".into(),
173        ));
174    }
175    d.done = true;
176    match fetch(&d.params) {
177        Ok(t) => StepResult::done(StbString::from_string(
178            json!({"content":[{"type":"text","text":t}]}).to_string(),
179        )),
180        Err(e) => StepResult::done(StbString::from_string(
181            json!({
182                "content":[{"type":"text","text":format!("webfetch failed: {e}. Proceed with another reference or retry.")}],
183                "details":{"error":true,"message":e}
184            })
185            .to_string(),
186        )),
187    }
188}
189extern "C" fn cancel(h: StepHandle) {
190    if !h.is_null() {
191        unsafe {
192            (&*(h as *mut Drive))
193                .cancelled
194                .store(true, Ordering::SeqCst);
195        }
196    }
197}
198extern "C" fn destroy(h: StepHandle) {
199    if !h.is_null() {
200        unsafe {
201            drop(Box::from_raw(h as *mut Drive));
202        }
203    }
204}
205extern "C" fn free_string(s: StbString) {
206    if !s.is_empty() && !s.ptr.is_null() {
207        unsafe {
208            let b = std::slice::from_raw_parts(s.ptr as *const u8, s.len);
209            let _ = Box::from_raw(b as *const [u8] as *mut [u8]);
210        }
211    }
212}
213#[no_mangle]
214pub extern "C" fn rpi_plugin_register(api: *const PluginApiVt, abi: u32) -> i32 {
215    register_entrypoint(api, abi, |api| {
216        let Some(register) = api.register_tool else {
217            return 1;
218        };
219        let schema=Box::new(StableToolSchema{name:StbString::from_string("webfetch".into()),description:StbString::from_string("Fetch bounded readable text from a public HTTP(S) URL.".into()),parameters:StbString::from_string(r#"{"type":"object","properties":{"url":{"type":"string"},"maxChars":{"type":"integer","minimum":256,"maximum":50000}},"required":["url"]}"#.into())});
220        let rc = register(&*schema, execute, poll, cancel, destroy, free_string);
221        drop(schema);
222        rc
223    })
224}
225#[cfg(test)]
226mod tests {
227    use super::*;
228    #[test]
229    fn rejects_private() {
230        assert!(validate("http://127.0.0.1").is_err());
231    }
232}
rpi_webfetch/lib.rs

rpi_webfetch/
lib.rs
