1use std::collections::{HashMap, VecDeque};
16use std::io::Read;
17use std::path::{Path, PathBuf};
18use std::sync::atomic::{AtomicBool, Ordering};
19use std::sync::mpsc::{self, Receiver, SyncSender, TryRecvError};
20use std::sync::{Arc, OnceLock};
21use std::time::Duration;
22
23use crate::net::{url_host, NetPolicy};
24
25pub const PENDING: u32 = 0;
26pub const HEADERS: u32 = 1;
27pub const DONE: u32 = 2;
28pub const FAILED: u32 = 3;
29
30pub const MAX_REQUESTS: usize = 16;
31pub const MAX_BODY: u64 = 64 << 20;
33pub const MAX_REQUEST_BODY: usize = 16 << 20;
34const MAX_REDIRECTS: usize = 10;
35const CHUNK: usize = 64 << 10;
36const QUEUE: usize = 64;
38
39pub const METHODS: [&str; 7] = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"];
40pub const FORBIDDEN_HEADERS: [&str; 22] = [
44 "accept-charset", "accept-encoding", "access-control-request-headers", "access-control-request-method",
45 "connection", "content-length", "cookie", "cookie2", "date", "dnt", "expect", "host", "keep-alive",
46 "origin", "referer", "set-cookie", "te", "trailer", "transfer-encoding", "upgrade", "user-agent", "via",
47];
48pub const FORBIDDEN_PREFIXES: [&str; 2] = ["proxy-", "sec-"];
49pub const HIDDEN_RESPONSE_HEADERS: [&str; 9] =
53 ["connection", "keep-alive", "proxy-connection", "set-cookie", "set-cookie2", "te", "trailer", "transfer-encoding", "upgrade"];
54
55pub fn header_lines<'a>(headers: impl IntoIterator<Item = (&'a str, String)>) -> String {
61 let mut by_name: std::collections::BTreeMap<String, Vec<String>> = Default::default();
62 for (k, v) in headers {
63 let k = k.to_ascii_lowercase();
64 if !HIDDEN_RESPONSE_HEADERS.contains(&k.as_str()) {
65 by_name.entry(k).or_default().push(v.trim().to_owned());
66 }
67 }
68 if let Some(enc) = by_name.remove("content-encoding") {
69 if enc.iter().any(|e| !e.eq_ignore_ascii_case("identity")) {
70 by_name.remove("content-length");
71 }
72 }
73 by_name.into_iter().map(|(k, v)| format!("{k}: {}\n", v.join(", "))).collect()
74}
75
76#[derive(Clone, Debug, Default)]
78pub enum FetchMode {
79 #[default]
80 Live,
81 Record(PathBuf),
83 Replay(PathBuf),
85}
86
87#[derive(Debug, PartialEq)]
89pub struct Desc {
90 pub method: String,
91 pub url: String,
92 pub headers: Vec<(String, String)>,
93}
94
95pub fn parse_desc(json: &str, body_len: usize) -> Result<Desc, String> {
97 let v: serde_json::Value = serde_json::from_str(json).map_err(|e| format!("invalid JSON: {e}"))?;
98 let o = v.as_object().ok_or("the description must be a JSON object")?;
99 let method = match o.get("method") {
100 None => "GET".to_owned(),
101 Some(m) => m.as_str().ok_or("method must be a string")?.to_ascii_uppercase(),
102 };
103 if !METHODS.contains(&method.as_str()) {
104 return Err(format!("method {method} is not supported"));
105 }
106 let url = o.get("url").and_then(|u| u.as_str()).ok_or("url must be a string")?.to_owned();
107 let lower = url.to_ascii_lowercase();
108 if !(lower.starts_with("http://") || lower.starts_with("https://")) || url_host(&url).is_none() {
109 return Err(format!("not an absolute http(s) URL: {url}"));
110 }
111 if url.chars().any(|c| c.is_ascii_control() || c == ' ') {
112 return Err(format!("invalid URL: {url}"));
113 }
114 let mut headers = Vec::new();
115 if let Some(h) = o.get("headers") {
116 for (k, v) in h.as_object().ok_or("headers must be an object")? {
117 let v = v.as_str().ok_or_else(|| format!("header {k}: the value must be a string"))?;
118 check_header(k, v)?;
119 headers.push((k.to_ascii_lowercase(), v.to_owned()));
120 }
121 }
122 if body_len > 0 && (method == "GET" || method == "HEAD") {
123 return Err(format!("a {method} request can't have a body"));
124 }
125 if body_len > MAX_REQUEST_BODY {
126 return Err(format!("request body over {MAX_REQUEST_BODY} bytes"));
127 }
128 Ok(Desc { method, url, headers })
129}
130
131fn check_header(name: &str, value: &str) -> Result<(), String> {
132 let token = |c: u8| c.is_ascii_alphanumeric() || b"!#$%&'*+-.^_`|~".contains(&c);
133 if name.is_empty() || !name.bytes().all(token) {
134 return Err(format!("invalid header name {name:?}"));
135 }
136 if value.bytes().any(|c| c == b'\r' || c == b'\n' || c == 0) {
137 return Err(format!("invalid value for header {name}"));
138 }
139 let n = name.to_ascii_lowercase();
140 if FORBIDDEN_HEADERS.contains(&n.as_str()) || FORBIDDEN_PREFIXES.iter().any(|p| n.starts_with(p)) {
141 return Err(format!("header {name} can't be set (browsers refuse it)"));
142 }
143 Ok(())
144}
145
146pub fn record_key(method: &str, url: &str, body: &[u8]) -> String {
148 let mut h: u64 = 0xcbf2_9ce4_8422_2325;
149 for b in method.bytes().chain([0]).chain(url.bytes()).chain([0]).chain(body.iter().copied()) {
150 h = (h ^ b as u64).wrapping_mul(0x0000_0100_0000_01b3);
151 }
152 format!("{h:016x}")
153}
154
155enum Event {
156 Head(i32, String),
157 Chunk(Vec<u8>),
158 Done,
159 Fail(String),
160}
161
162struct Req {
163 events: Option<Receiver<Event>>,
164 cancel: Arc<AtomicBool>,
165 state: u32,
166 status: i32,
167 headers: Option<String>,
168 body: VecDeque<u8>,
169 ready_frame: u64,
171}
172
173pub struct Fetch {
174 policy: NetPolicy,
175 user_agent: String,
177 mode: FetchMode,
178 reqs: HashMap<i32, Req>,
179 next: i32,
180 denied: std::collections::HashSet<String>,
182}
183
184impl Default for Fetch {
185 fn default() -> Fetch {
186 Fetch::new(NetPolicy::default(), FetchMode::Live)
187 }
188}
189
190impl Fetch {
191 pub fn new(policy: NetPolicy, mode: FetchMode) -> Fetch {
192 Fetch { policy, user_agent: user_agent(None), mode, reqs: HashMap::new(), next: 1, denied: Default::default() }
193 }
194
195 pub fn set_app_id(&mut self, app_id: Option<&str>) {
197 self.user_agent = user_agent(app_id);
198 }
199
200 pub fn request(&mut self, desc: &str, body: Vec<u8>, frame: u64) -> i32 {
202 let d = match parse_desc(desc, body.len()) {
203 Ok(d) => d,
204 Err(e) => {
205 eprintln!("[gasm] fetch: refused: {e}");
206 return -1;
207 }
208 };
209 if self.reqs.len() >= MAX_REQUESTS {
210 eprintln!("[gasm] fetch: too many requests (max {MAX_REQUESTS}): {}", d.url);
211 return -1;
212 }
213 let cancel = Arc::new(AtomicBool::new(false));
214 let mut req = Req { events: None, cancel: cancel.clone(), state: PENDING, status: 0, headers: None, body: VecDeque::new(), ready_frame: 0 };
215 match &self.mode {
216 FetchMode::Replay(dir) => {
217 req.ready_frame = frame + 1;
218 match load_record(dir, &record_key(&d.method, &d.url, &body)) {
219 Some((status, headers, data)) => {
220 (req.state, req.status, req.headers, req.body) = (DONE, status, Some(headers), data.into());
221 }
222 None => {
223 eprintln!("[gasm] fetch: {} {} is not recorded in {}", d.method, d.url, dir.display());
224 req.state = FAILED;
225 }
226 }
227 }
228 mode => {
229 if let Ok(Some(why)) = self.policy.verdict(&d.url) {
231 let host = url_host(&d.url).unwrap_or_default();
232 if self.denied.insert(host) {
233 eprintln!("[gasm] fetch: denied {} ({why})", d.url);
234 }
235 return -1;
236 }
237 let record = match mode {
238 FetchMode::Record(dir) => Some(dir.join(record_key(&d.method, &d.url, &body))),
239 _ => None,
240 };
241 let (tx, rx) = mpsc::sync_channel(QUEUE);
242 let (policy, ua) = (self.policy.clone(), self.user_agent.clone());
243 std::thread::spawn(move || run(d, body, policy, ua, tx, cancel, record));
244 req.events = Some(rx);
245 }
246 }
247 let h = self.next;
248 self.next += 1;
249 self.reqs.insert(h, req);
250 h
251 }
252
253 fn get(&mut self, h: i32) -> Result<Option<&mut Req>, String> {
255 if h <= 0 || h >= self.next {
256 return Err(format!("gasm:fetch: invalid handle {h}"));
257 }
258 let Some(r) = self.reqs.get_mut(&h) else { return Ok(None) };
259 if let Some(rx) = &r.events {
260 loop {
261 match rx.try_recv() {
262 Ok(Event::Head(status, headers)) => (r.state, r.status, r.headers) = (HEADERS, status, Some(headers)),
263 Ok(Event::Chunk(c)) => r.body.extend(c),
264 Ok(Event::Done) => r.state = DONE,
265 Ok(Event::Fail(e)) => {
266 eprintln!("[gasm] fetch: {e}");
267 (r.state, r.status) = (FAILED, 0);
268 r.body.clear();
269 }
270 Err(TryRecvError::Empty) => break,
271 Err(TryRecvError::Disconnected) => {
272 if r.state < DONE {
273 (r.state, r.status) = (FAILED, 0);
274 }
275 break;
276 }
277 }
278 }
279 if r.state >= DONE {
280 r.events = None;
281 }
282 }
283 Ok(Some(r))
284 }
285
286 fn view(&mut self, h: i32, frame: u64) -> Result<(u32, i32, Option<String>), String> {
288 Ok(match self.get(h)? {
289 None => (FAILED, 0, None),
290 Some(r) if frame < r.ready_frame => (PENDING, 0, None),
291 Some(r) => (r.state, r.status, r.headers.clone()),
292 })
293 }
294
295 pub fn state(&mut self, h: i32, frame: u64) -> Result<u32, String> {
296 Ok(self.view(h, frame)?.0)
297 }
298
299 pub fn status(&mut self, h: i32, frame: u64) -> Result<i32, String> {
300 Ok(self.view(h, frame)?.1)
301 }
302
303 pub fn headers(&mut self, h: i32, frame: u64) -> Result<Option<String>, String> {
304 Ok(self.view(h, frame)?.2)
305 }
306
307 pub fn read(&mut self, h: i32, cap: usize, frame: u64) -> Result<Result<Vec<u8>, ()>, String> {
309 let Some(r) = self.get(h)? else { return Ok(Err(())) };
310 if frame < r.ready_frame || r.state == PENDING {
311 return Ok(Ok(Vec::new()));
312 }
313 if r.body.is_empty() {
314 return Ok(if r.state >= DONE { Err(()) } else { Ok(Vec::new()) });
315 }
316 let n = cap.min(r.body.len());
317 Ok(Ok(r.body.drain(..n).collect()))
318 }
319
320 pub fn close(&mut self, h: i32) -> Result<(), String> {
321 if h <= 0 || h >= self.next {
322 return Err(format!("gasm:fetch: invalid handle {h}"));
323 }
324 if let Some(r) = self.reqs.remove(&h) {
325 r.cancel.store(true, Ordering::Relaxed); }
327 Ok(())
328 }
329}
330
331fn load_record(dir: &Path, key: &str) -> Option<(i32, String, Vec<u8>)> {
332 let meta: serde_json::Value = serde_json::from_slice(&std::fs::read(dir.join(format!("{key}.json"))).ok()?).ok()?;
333 let body = std::fs::read(dir.join(format!("{key}.body"))).ok()?;
334 Some((meta.get("status")?.as_i64()? as i32, meta.get("headers")?.as_str()?.to_owned(), body))
335}
336
337fn save_record(path: &Path, method: &str, url: &str, status: i32, headers: &str, body: &[u8]) {
338 let meta = serde_json::json!({ "method": method, "url": url, "status": status, "headers": headers });
339 let write = || -> std::io::Result<()> {
340 if let Some(d) = path.parent() {
341 std::fs::create_dir_all(d)?;
342 }
343 std::fs::write(path.with_extension("body"), body)?;
344 std::fs::write(path.with_extension("json"), format!("{}\n", serde_json::to_string_pretty(&meta)?))
345 };
346 if let Err(e) = write() {
347 eprintln!("[gasm] fetch: --fetch-record {}: {e}", path.display());
348 }
349}
350
351fn agent() -> &'static ureq::Agent {
352 static AGENT: OnceLock<ureq::Agent> = OnceLock::new();
353 AGENT.get_or_init(|| {
354 let roots = rustls_native_certs::load_native_certs();
356 for e in &roots.errors {
357 eprintln!("[gasm] fetch: certificates: {e}");
358 }
359 let certs = roots.certs.iter().map(|c| ureq::tls::Certificate::from_der(c.as_ref()).to_owned()).collect();
360 let tls = ureq::tls::TlsConfig::builder()
361 .provider(ureq::tls::TlsProvider::Rustls)
362 .root_certs(ureq::tls::RootCerts::Specific(Arc::new(certs)))
363 .unversioned_rustls_crypto_provider(Arc::new(rustls::crypto::ring::default_provider()))
364 .build();
365 ureq::Agent::config_builder()
366 .http_status_as_error(false)
367 .max_redirects(0)
368 .max_redirects_will_error(false)
369 .timeout_connect(Some(Duration::from_secs(30)))
370 .timeout_global(Some(Duration::from_secs(120)))
371 .tls_config(tls)
372 .build()
373 .into()
374 })
375}
376
377pub fn resolve_url(base: &str, location: &str) -> String {
379 resolve(base, location)
380}
381
382fn resolve(base: &str, location: &str) -> String {
383 if location.contains("://") {
384 return location.to_owned();
385 }
386 let (scheme, rest) = base.split_once("://").unwrap_or(("http", base));
387 let authority = rest.split(['/', '?', '#']).next().unwrap_or("");
388 if let Some(l) = location.strip_prefix("//") {
389 return format!("{scheme}://{l}");
390 }
391 if location.starts_with('/') {
392 return format!("{scheme}://{authority}{location}");
393 }
394 let path = rest[authority.len()..].split(['?', '#']).next().unwrap_or("");
395 let dir = &path[..path.rfind('/').map_or(0, |i| i + 1)];
396 format!("{scheme}://{authority}{}{location}", if dir.is_empty() { "/" } else { dir })
397}
398
399pub fn user_agent(app_id: Option<&str>) -> String {
402 let runner = concat!("gasm-run/", env!("CARGO_PKG_VERSION"));
403 match app_id {
404 Some(id) => format!("{id} {runner}"),
405 None => runner.to_owned(),
406 }
407}
408
409pub fn valid_app_id(id: &str) -> bool {
411 !id.is_empty() && id.len() <= 256 && id.bytes().all(|c| (0x20..0x7f).contains(&c))
412}
413
414fn run(d: Desc, body: Vec<u8>, policy: NetPolicy, user_agent: String, tx: SyncSender<Event>, cancel: Arc<AtomicBool>, record: Option<PathBuf>) {
415 let result = (|| -> Result<(), String> {
416 let (mut method, mut url, mut body) = (d.method.clone(), d.url.clone(), body);
417 policy.wait(&url, || cancel.load(Ordering::Relaxed)).map_err(|why| format!("{url}: denied ({why})"))?;
419 let mut hops = 0;
420 let resp = loop {
421 let mut b = ureq::http::Request::builder().method(method.as_str()).uri(url.as_str()).header("user-agent", user_agent.as_str());
422 for (k, v) in &d.headers {
423 b = b.header(k.as_str(), v.as_str());
424 }
425 let req = b.body(body.clone()).map_err(|e| format!("{url}: {e}"))?;
426 let resp = agent().run(req).map_err(|e| format!("{method} {url}: {e}"))?;
427 let status = resp.status().as_u16();
428 let location = resp.headers().get("location").and_then(|v| v.to_str().ok()).map(str::to_owned);
429 match (status, location) {
430 (301 | 302 | 303 | 307 | 308, Some(loc)) => {
431 hops += 1;
432 if hops > MAX_REDIRECTS {
433 return Err(format!("{url}: too many redirects"));
434 }
435 let next = resolve(&url, &loc);
436 if let Err(why) = policy.wait(&next, || cancel.load(Ordering::Relaxed)) {
437 return Err(format!("{url}: redirect to {next} denied ({why})"));
438 }
439 if status == 303 || (matches!(status, 301 | 302) && method == "POST") {
441 if method != "HEAD" {
442 method = "GET".into();
443 }
444 body.clear();
445 }
446 url = next;
447 }
448 _ => break resp,
449 }
450 };
451 let status = resp.status().as_u16() as i32;
452 let headers = header_lines(resp.headers().iter().map(|(k, v)| (k.as_str(), String::from_utf8_lossy(v.as_bytes()).into_owned())));
453 tx.send(Event::Head(status, headers.clone())).map_err(|_| String::new())?;
454 let mut reader = resp.into_body().into_with_config().limit(MAX_BODY).reader();
455 let mut buf = vec![0u8; CHUNK];
456 let mut all = Vec::new();
457 loop {
458 if cancel.load(Ordering::Relaxed) {
459 return Err(String::new());
460 }
461 let n = reader.read(&mut buf).map_err(|e| format!("{url}: {e}"))?;
462 if n == 0 {
463 break;
464 }
465 if record.is_some() {
466 all.extend_from_slice(&buf[..n]);
467 }
468 tx.send(Event::Chunk(buf[..n].to_vec())).map_err(|_| String::new())?;
469 }
470 if let Some(path) = &record {
471 save_record(path, &d.method, &d.url, status, &headers, &all);
472 }
473 tx.send(Event::Done).map_err(|_| String::new())
474 })();
475 if let Err(e) = result {
476 if !e.is_empty() {
477 let _ = tx.send(Event::Fail(e));
478 }
479 }
480}
481
482#[cfg(test)]
483mod tests {
484 use super::*;
485
486 #[test]
487 fn descriptions() {
488 let d = parse_desc(r#"{"url":"https://api.met.no/x?a=1","headers":{"Accept":"application/json"}}"#, 0).unwrap();
489 assert_eq!(d, Desc { method: "GET".into(), url: "https://api.met.no/x?a=1".into(), headers: vec![("accept".into(), "application/json".into())] });
490 assert_eq!(parse_desc(r#"{"method":"post","url":"http://h/"}"#, 3).unwrap().method, "POST");
491 for bad in [
492 r#"[]"#,
493 r#"{"url":"ftp://h/"}"#,
494 r#"{"url":"/relative"}"#,
495 r#"{"method":"TRACE","url":"http://h/"}"#,
496 r#"{"url":"http://h/","headers":{"Cookie":"a"}}"#,
497 r#"{"url":"http://h/","headers":{"sec-fetch-mode":"a"}}"#,
498 r#"{"url":"http://h/","headers":{"x":"a\r\nb"}}"#,
499 r#"{"url":"http://h/","headers":{"bad name":"a"}}"#,
500 ] {
501 assert!(parse_desc(bad, 0).is_err(), "{bad}");
502 }
503 assert!(parse_desc(r#"{"url":"http://h/"}"#, 1).is_err()); }
505
506 #[test]
507 fn response_headers() {
508 let h = header_lines([("Vary", "a".into()), ("content-type", "text/plain".into()), ("Connection", "close".into()), ("vary", " b ".into()), ("set-cookie", "x=1".into())]);
509 assert_eq!(h, "content-type: text/plain\nvary: a, b\n");
510 let gz = header_lines([("content-encoding", "gzip".into()), ("content-length", "10".into()), ("etag", "x".into())]);
511 assert_eq!(gz, "etag: x\n");
512 }
513
514 #[test]
515 fn redirects_resolve() {
516 assert_eq!(resolve("https://a.b/x/y?q", "/z"), "https://a.b/z");
517 assert_eq!(resolve("https://a.b/x/y?q", "z"), "https://a.b/x/z");
518 assert_eq!(resolve("https://a.b", "z"), "https://a.b/z");
519 assert_eq!(resolve("https://a.b/x", "//c.d/e"), "https://c.d/e");
520 assert_eq!(resolve("https://a.b/x", "http://c.d/"), "http://c.d/");
521 }
522
523 #[test]
524 fn keys() {
525 assert_eq!(record_key("GET", "https://example.org/", b""), "a0eecdfd36f8bde7");
527 }
528
529 #[test]
530 fn replay_and_handles() {
531 let dir = std::env::temp_dir().join(format!("gasm-fetch-test-{}", std::process::id()));
532 let _ = std::fs::remove_dir_all(&dir);
533 let key = record_key("GET", "https://example.org/a", b"");
534 save_record(&dir.join(&key), "GET", "https://example.org/a", 200, "content-type: text/plain\n", b"hello");
535 let mut f = Fetch::new(NetPolicy::default(), FetchMode::Replay(dir.clone()));
536 let h = f.request(r#"{"url":"https://example.org/a"}"#, vec![], 5);
537 let miss = f.request(r#"{"url":"https://example.org/b"}"#, vec![], 5);
538 assert_eq!((f.state(h, 5).unwrap(), f.status(h, 5).unwrap(), f.read(h, 9, 5).unwrap()), (PENDING, 0, Ok(vec![])));
539 assert_eq!((f.state(h, 6).unwrap(), f.status(h, 6).unwrap()), (DONE, 200));
540 assert_eq!(f.headers(h, 6).unwrap().as_deref(), Some("content-type: text/plain\n"));
541 assert_eq!(f.read(h, 3, 6).unwrap(), Ok(b"hel".to_vec()));
542 assert_eq!(f.read(h, 9, 6).unwrap(), Ok(b"lo".to_vec()));
543 assert_eq!(f.read(h, 9, 6).unwrap(), Err(()));
544 assert_eq!(f.state(miss, 6).unwrap(), FAILED);
545 f.close(h).unwrap();
546 assert_eq!(f.state(h, 7).unwrap(), FAILED); assert!(f.state(99, 7).is_err()); let _ = std::fs::remove_dir_all(&dir);
549 assert_eq!(Fetch::default().request(r#"{"url":"https://example.org/"}"#, vec![], 0), -1);
551 }
552}