rpi_webfetch/
lib.rs

1use rpi_plugin_sdk::{
2    register_entrypoint, FreeStringFn, PluginApiVt, StableToolSchema, StbString, StbStringRef,
3    StepHandle, StepResult, ToolPartialCb,
4};
5use serde_json::{json, Value};
6use std::ffi::c_void;
7use std::io::Read;
8use std::net::IpAddr;
9use std::sync::atomic::{AtomicBool, Ordering};
10use std::time::Duration;
11use url::Url;
12struct Drive {
13    params: Value,
14    cancelled: AtomicBool,
15    done: bool,
16}
17fn validate(input: &str) -> Result<Url, String> {
18    let u = Url::parse(input).map_err(|e| format!("invalid URL: {e}"))?;
19    if !matches!(u.scheme(), "http" | "https") {
20        return Err("only http and https URLs are allowed".into());
21    }
22    if !u.username().is_empty() || u.password().is_some() {
23        return Err("embedded credentials are not allowed".into());
24    }
25    let h = u.host_str().ok_or("URL has no host")?;
26    if h.eq_ignore_ascii_case("localhost") || h.ends_with(".localhost") {
27        return Err("local hosts are not allowed".into());
28    }
29    if let Ok(ip) = h.parse::<IpAddr>() {
30        let blocked = match ip {
31            IpAddr::V4(v4) => {
32                v4.is_loopback()
33                    || v4.is_private()
34                    || v4.is_link_local()
35                    || v4.is_unspecified()
36                    || v4.is_multicast()
37            }
38            IpAddr::V6(v6) => {
39                v6.is_loopback()
40                    || v6.is_unspecified()
41                    || v6.is_multicast()
42                    || v6.is_unique_local()
43                    || v6.is_unicast_link_local()
44            }
45        };
46        if blocked {
47            return Err("private or local IPs are not allowed".into());
48        }
49    }
50    Ok(u)
51}
52fn fetch(p: &Value) -> Result<String, String> {
53    let url = validate(
54        p.get("url")
55            .and_then(Value::as_str)
56            .ok_or("url is required")?,
57    )?;
58    let max = p
59        .get("maxChars")
60        .and_then(Value::as_u64)
61        .unwrap_or(12000)
62        .clamp(256, 50000) as usize;
63    let client = reqwest::blocking::Client::builder()
64        .timeout(Duration::from_secs(20))
65        .redirect(reqwest::redirect::Policy::limited(5))
66        .user_agent("rpi-webfetch/0.1")
67        .build()
68        .map_err(|e| e.to_string())?;
69    let mut r = client
70        .get(url)
71        .send()
72        .map_err(|e| format!("web fetch failed: {e}"))?;
73    let status = r.status();
74    let final_url = r.url().to_string();
75    validate(&final_url)?;
76    let ct = r
77        .headers()
78        .get(reqwest::header::CONTENT_TYPE)
79        .and_then(|v| v.to_str().ok())
80        .unwrap_or("")
81        .to_string();
82    let mut b = Vec::new();
83    r.by_ref()
84        .take(1_048_577)
85        .read_to_end(&mut b)
86        .map_err(|e| e.to_string())?;
87    if b.len() > 1_048_576 {
88        return Err("response exceeded 1 MiB".into());
89    }
90    if !(ct.is_empty()
91        || ct.contains("text/")
92        || ct.contains("json")
93        || ct.contains("xml")
94        || ct.contains("html"))
95    {
96        return Ok(json!({"url":final_url,"status":status.as_u16(),"contentType":ct,"skipped":"non-text response"}).to_string());
97    }
98    let raw = String::from_utf8_lossy(&b);
99    let text = if ct.contains("html") {
100        strip_html(&raw)
101    } else {
102        raw.to_string()
103    };
104    let clipped: String = text.chars().take(max).collect();
105    Ok(json!({"url":final_url,"status":status.as_u16(),"contentType":ct,"text":clipped,"truncated":text.chars().count()>max}).to_string())
106}
107fn strip_html(s: &str) -> String {
108    let mut out = s.replace("\r", " ");
109    for tag in [
110        "script", "style", "nav", "footer", "header", "aside", "noscript",
111    ] {
112        let re_start = format!("<{}", tag);
113        while let Some(a) = out.to_ascii_lowercase().find(&re_start) {
114            if let Some(b) = out[a..].find(&format!("</{}>", tag)) {
115                out.replace_range(a..a + b + tag.len() + 3, " ");
116            } else {
117                break;
118            }
119        }
120    }
121    out = out.replace("><", ">\n<");
122    let mut result = String::new();
123    let mut inside = false;
124    for c in out.chars() {
125        match c {
126            '<' => inside = true,
127            '>' => inside = false,
128            '_' if inside => {}
129            c if !inside => result.push(c),
130            _ => {}
131        }
132    }
133    result.split_whitespace().collect::<Vec<_>>().join(" ")
134}
135extern "C" fn execute(
136    _: StbStringRef,
137    params: StbString,
138    free: Option<FreeStringFn>,
139) -> StepHandle {
140    let t = params.to_string_lossy();
141    params.free_with(free);
142    Box::into_raw(Box::new(Drive {
143        params: serde_json::from_str(&t).unwrap_or(Value::Null),
144        cancelled: AtomicBool::new(false),
145        done: false,
146    })) as StepHandle
147}
148extern "C" fn poll(h: StepHandle, _: Option<ToolPartialCb>, _: *mut c_void) -> StepResult {
149    if h.is_null() {
150        return StepResult::err(StbString::from_string("null webfetch handle".into()));
151    }
152    let d = unsafe { &mut *(h as *mut Drive) };
153    if d.cancelled.load(Ordering::SeqCst) {
154        return StepResult::err(StbString::from_string("webfetch cancelled".into()));
155    }
156    if d.done {
157        return StepResult::err(StbString::from_string(
158            "webfetch polled after completion".into(),
159        ));
160    }
161    d.done = true;
162    match fetch(&d.params) {
163        Ok(t) => StepResult::done(StbString::from_string(
164            json!({"content":[{"type":"text","text":t}]}).to_string(),
165        )),
166        Err(e) => StepResult::err(StbString::from_string(e)),
167    }
168}
169extern "C" fn cancel(h: StepHandle) {
170    if !h.is_null() {
171        unsafe {
172            (&*(h as *mut Drive))
173                .cancelled
174                .store(true, Ordering::SeqCst);
175        }
176    }
177}
178extern "C" fn destroy(h: StepHandle) {
179    if !h.is_null() {
180        unsafe {
181            drop(Box::from_raw(h as *mut Drive));
182        }
183    }
184}
185extern "C" fn free_string(s: StbString) {
186    if !s.is_empty() && !s.ptr.is_null() {
187        unsafe {
188            let b = std::slice::from_raw_parts(s.ptr as *const u8, s.len);
189            let _ = Box::from_raw(b as *const [u8] as *mut [u8]);
190        }
191    }
192}
193#[no_mangle]
194pub extern "C" fn rpi_plugin_register(api: *const PluginApiVt, abi: u32) -> i32 {
195    register_entrypoint(api, abi, |api| {
196        let Some(register) = api.register_tool else {
197            return 1;
198        };
199        let schema=Box::new(StableToolSchema{name:StbString::from_string("webfetch".into()),description:StbString::from_string("Fetch bounded readable text from a public HTTP(S) URL.".into()),parameters:StbString::from_string(r#"{"type":"object","properties":{"url":{"type":"string"},"maxChars":{"type":"integer","minimum":256,"maximum":50000}},"required":["url"]}"#.into())});
200        let rc = register(&*schema, execute, poll, cancel, destroy, free_string);
201        drop(schema);
202        rc
203    })
204}
205#[cfg(test)]
206mod tests {
207    use super::*;
208    #[test]
209    fn rejects_private() {
210        assert!(validate("http://127.0.0.1").is_err());
211    }
212}
rpi_webfetch/lib.rs

rpi_webfetch/
lib.rs
