1use std::collections::{HashMap, VecDeque};
16use std::io::Read;
17use std::path::{Path, PathBuf};
18use std::sync::atomic::{AtomicBool, Ordering};
19use std::sync::mpsc::{self, Receiver, SyncSender, TryRecvError};
20use std::sync::{Arc, OnceLock};
21use std::time::Duration;
22
23use crate::net::{url_host, NetPolicy};
24
25pub const PENDING: u32 = 0;
26pub const HEADERS: u32 = 1;
27pub const DONE: u32 = 2;
28pub const FAILED: u32 = 3;
29
30pub const MAX_REQUESTS: usize = 16;
31pub const MAX_BODY: u64 = 64 << 20;
33pub const MAX_REQUEST_BODY: usize = 16 << 20;
34const MAX_REDIRECTS: usize = 10;
35const CHUNK: usize = 64 << 10;
36const QUEUE: usize = 64;
38
39pub const METHODS: [&str; 7] = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"];
40pub const FORBIDDEN_HEADERS: [&str; 22] = [
44 "accept-charset", "accept-encoding", "access-control-request-headers", "access-control-request-method",
45 "connection", "content-length", "cookie", "cookie2", "date", "dnt", "expect", "host", "keep-alive",
46 "origin", "referer", "set-cookie", "te", "trailer", "transfer-encoding", "upgrade", "user-agent", "via",
47];
48pub const FORBIDDEN_PREFIXES: [&str; 2] = ["proxy-", "sec-"];
49pub const HIDDEN_RESPONSE_HEADERS: [&str; 9] =
53 ["connection", "keep-alive", "proxy-connection", "set-cookie", "set-cookie2", "te", "trailer", "transfer-encoding", "upgrade"];
54
55pub fn header_lines<'a>(headers: impl IntoIterator<Item = (&'a str, String)>) -> String {
61 let mut by_name: std::collections::BTreeMap<String, Vec<String>> = Default::default();
62 for (k, v) in headers {
63 let k = k.to_ascii_lowercase();
64 if !HIDDEN_RESPONSE_HEADERS.contains(&k.as_str()) {
65 by_name.entry(k).or_default().push(v.trim().to_owned());
66 }
67 }
68 if let Some(enc) = by_name.remove("content-encoding") {
69 if enc.iter().any(|e| !e.eq_ignore_ascii_case("identity")) {
70 by_name.remove("content-length");
71 }
72 }
73 by_name.into_iter().map(|(k, v)| format!("{k}: {}\n", v.join(", "))).collect()
74}
75
76#[derive(Clone, Debug, Default)]
78pub enum FetchMode {
79 #[default]
80 Live,
81 Record(PathBuf),
83 Replay(PathBuf),
85}
86
87#[derive(Debug, PartialEq)]
89pub struct Desc {
90 pub method: String,
91 pub url: String,
92 pub headers: Vec<(String, String)>,
93}
94
95pub fn parse_desc(json: &str, body_len: usize) -> Result<Desc, String> {
97 let v: serde_json::Value = serde_json::from_str(json).map_err(|e| format!("invalid JSON: {e}"))?;
98 let o = v.as_object().ok_or("the description must be a JSON object")?;
99 let method = match o.get("method") {
100 None => "GET".to_owned(),
101 Some(m) => m.as_str().ok_or("method must be a string")?.to_ascii_uppercase(),
102 };
103 if !METHODS.contains(&method.as_str()) {
104 return Err(format!("method {method} is not supported"));
105 }
106 let url = o.get("url").and_then(|u| u.as_str()).ok_or("url must be a string")?.to_owned();
107 let lower = url.to_ascii_lowercase();
108 if !(lower.starts_with("http://") || lower.starts_with("https://")) || url_host(&url).is_none() {
109 return Err(format!("not an absolute http(s) URL: {url}"));
110 }
111 if url.chars().any(|c| c.is_ascii_control() || c == ' ') {
112 return Err(format!("invalid URL: {url}"));
113 }
114 let mut headers = Vec::new();
115 if let Some(h) = o.get("headers") {
116 for (k, v) in h.as_object().ok_or("headers must be an object")? {
117 let v = v.as_str().ok_or_else(|| format!("header {k}: the value must be a string"))?;
118 check_header(k, v)?;
119 headers.push((k.to_ascii_lowercase(), v.to_owned()));
120 }
121 }
122 if body_len > 0 && (method == "GET" || method == "HEAD") {
123 return Err(format!("a {method} request can't have a body"));
124 }
125 if body_len > MAX_REQUEST_BODY {
126 return Err(format!("request body over {MAX_REQUEST_BODY} bytes"));
127 }
128 Ok(Desc { method, url, headers })
129}
130
131fn check_header(name: &str, value: &str) -> Result<(), String> {
132 let token = |c: u8| c.is_ascii_alphanumeric() || b"!#$%&'*+-.^_`|~".contains(&c);
133 if name.is_empty() || !name.bytes().all(token) {
134 return Err(format!("invalid header name {name:?}"));
135 }
136 if value.bytes().any(|c| c == b'\r' || c == b'\n' || c == 0) {
137 return Err(format!("invalid value for header {name}"));
138 }
139 let n = name.to_ascii_lowercase();
140 if FORBIDDEN_HEADERS.contains(&n.as_str()) || FORBIDDEN_PREFIXES.iter().any(|p| n.starts_with(p)) {
141 return Err(format!("header {name} can't be set (browsers refuse it)"));
142 }
143 Ok(())
144}
145
146pub fn record_key(method: &str, url: &str, body: &[u8]) -> String {
148 let mut h: u64 = 0xcbf2_9ce4_8422_2325;
149 for b in method.bytes().chain([0]).chain(url.bytes()).chain([0]).chain(body.iter().copied()) {
150 h = (h ^ b as u64).wrapping_mul(0x0000_0100_0000_01b3);
151 }
152 format!("{h:016x}")
153}
154
155enum Event {
156 Head(i32, String),
157 Chunk(Vec<u8>),
158 Done,
159 Fail(String),
160}
161
162struct Req {
163 events: Option<Receiver<Event>>,
164 cancel: Arc<AtomicBool>,
165 state: u32,
166 status: i32,
167 headers: Option<String>,
168 body: VecDeque<u8>,
169 ready_frame: u64,
171}
172
173pub struct Fetch {
174 policy: NetPolicy,
175 user_agent: String,
177 mode: FetchMode,
178 reqs: HashMap<i32, Req>,
179 next: i32,
180 denied: std::collections::HashSet<String>,
182}
183
184impl Default for Fetch {
185 fn default() -> Fetch {
186 Fetch::new(NetPolicy::default(), FetchMode::Live)
187 }
188}
189
190impl Fetch {
191 pub fn new(policy: NetPolicy, mode: FetchMode) -> Fetch {
192 Fetch { policy, user_agent: user_agent(None), mode, reqs: HashMap::new(), next: 1, denied: Default::default() }
193 }
194
195 pub fn set_app_id(&mut self, app_id: Option<&str>) {
197 self.user_agent = user_agent(app_id);
198 }
199
200 pub fn request(&mut self, desc: &str, body: Vec<u8>, frame: u64) -> i32 {
202 let d = match parse_desc(desc, body.len()) {
203 Ok(d) => d,
204 Err(e) => {
205 eprintln!("[gasm] fetch: refused: {e}");
206 return -1;
207 }
208 };
209 if self.reqs.len() >= MAX_REQUESTS {
210 eprintln!("[gasm] fetch: too many requests (max {MAX_REQUESTS}): {}", d.url);
211 return -1;
212 }
213 let cancel = Arc::new(AtomicBool::new(false));
214 let mut req = Req { events: None, cancel: cancel.clone(), state: PENDING, status: 0, headers: None, body: VecDeque::new(), ready_frame: 0 };
215 match &self.mode {
216 FetchMode::Replay(dir) => {
217 req.ready_frame = frame + 1;
218 match load_record(dir, &record_key(&d.method, &d.url, &body)) {
219 Some((status, headers, data)) => {
220 (req.state, req.status, req.headers, req.body) = (DONE, status, Some(headers), data.into());
221 }
222 None => {
223 eprintln!("[gasm] fetch: {} {} is not recorded in {}", d.method, d.url, dir.display());
224 req.state = FAILED;
225 }
226 }
227 }
228 mode => {
229 if let Some(why) = self.policy.refusal(&d.url) {
230 let host = url_host(&d.url).unwrap_or_default();
231 if self.denied.insert(host) {
232 eprintln!("[gasm] fetch: denied {} ({why})", d.url);
233 }
234 return -1;
235 }
236 let record = match mode {
237 FetchMode::Record(dir) => Some(dir.join(record_key(&d.method, &d.url, &body))),
238 _ => None,
239 };
240 let (tx, rx) = mpsc::sync_channel(QUEUE);
241 let (policy, ua) = (self.policy.clone(), self.user_agent.clone());
242 std::thread::spawn(move || run(d, body, policy, ua, tx, cancel, record));
243 req.events = Some(rx);
244 }
245 }
246 let h = self.next;
247 self.next += 1;
248 self.reqs.insert(h, req);
249 h
250 }
251
252 fn get(&mut self, h: i32) -> Result<Option<&mut Req>, String> {
254 if h <= 0 || h >= self.next {
255 return Err(format!("gasm:fetch: invalid handle {h}"));
256 }
257 let Some(r) = self.reqs.get_mut(&h) else { return Ok(None) };
258 if let Some(rx) = &r.events {
259 loop {
260 match rx.try_recv() {
261 Ok(Event::Head(status, headers)) => (r.state, r.status, r.headers) = (HEADERS, status, Some(headers)),
262 Ok(Event::Chunk(c)) => r.body.extend(c),
263 Ok(Event::Done) => r.state = DONE,
264 Ok(Event::Fail(e)) => {
265 eprintln!("[gasm] fetch: {e}");
266 (r.state, r.status) = (FAILED, 0);
267 r.body.clear();
268 }
269 Err(TryRecvError::Empty) => break,
270 Err(TryRecvError::Disconnected) => {
271 if r.state < DONE {
272 (r.state, r.status) = (FAILED, 0);
273 }
274 break;
275 }
276 }
277 }
278 if r.state >= DONE {
279 r.events = None;
280 }
281 }
282 Ok(Some(r))
283 }
284
285 fn view(&mut self, h: i32, frame: u64) -> Result<(u32, i32, Option<String>), String> {
287 Ok(match self.get(h)? {
288 None => (FAILED, 0, None),
289 Some(r) if frame < r.ready_frame => (PENDING, 0, None),
290 Some(r) => (r.state, r.status, r.headers.clone()),
291 })
292 }
293
294 pub fn state(&mut self, h: i32, frame: u64) -> Result<u32, String> {
295 Ok(self.view(h, frame)?.0)
296 }
297
298 pub fn status(&mut self, h: i32, frame: u64) -> Result<i32, String> {
299 Ok(self.view(h, frame)?.1)
300 }
301
302 pub fn headers(&mut self, h: i32, frame: u64) -> Result<Option<String>, String> {
303 Ok(self.view(h, frame)?.2)
304 }
305
306 pub fn read(&mut self, h: i32, cap: usize, frame: u64) -> Result<Result<Vec<u8>, ()>, String> {
308 let Some(r) = self.get(h)? else { return Ok(Err(())) };
309 if frame < r.ready_frame || r.state == PENDING {
310 return Ok(Ok(Vec::new()));
311 }
312 if r.body.is_empty() {
313 return Ok(if r.state >= DONE { Err(()) } else { Ok(Vec::new()) });
314 }
315 let n = cap.min(r.body.len());
316 Ok(Ok(r.body.drain(..n).collect()))
317 }
318
319 pub fn close(&mut self, h: i32) -> Result<(), String> {
320 if h <= 0 || h >= self.next {
321 return Err(format!("gasm:fetch: invalid handle {h}"));
322 }
323 if let Some(r) = self.reqs.remove(&h) {
324 r.cancel.store(true, Ordering::Relaxed); }
326 Ok(())
327 }
328}
329
330fn load_record(dir: &Path, key: &str) -> Option<(i32, String, Vec<u8>)> {
331 let meta: serde_json::Value = serde_json::from_slice(&std::fs::read(dir.join(format!("{key}.json"))).ok()?).ok()?;
332 let body = std::fs::read(dir.join(format!("{key}.body"))).ok()?;
333 Some((meta.get("status")?.as_i64()? as i32, meta.get("headers")?.as_str()?.to_owned(), body))
334}
335
336fn save_record(path: &Path, method: &str, url: &str, status: i32, headers: &str, body: &[u8]) {
337 let meta = serde_json::json!({ "method": method, "url": url, "status": status, "headers": headers });
338 let write = || -> std::io::Result<()> {
339 if let Some(d) = path.parent() {
340 std::fs::create_dir_all(d)?;
341 }
342 std::fs::write(path.with_extension("body"), body)?;
343 std::fs::write(path.with_extension("json"), format!("{}\n", serde_json::to_string_pretty(&meta)?))
344 };
345 if let Err(e) = write() {
346 eprintln!("[gasm] fetch: --fetch-record {}: {e}", path.display());
347 }
348}
349
350fn agent() -> &'static ureq::Agent {
351 static AGENT: OnceLock<ureq::Agent> = OnceLock::new();
352 AGENT.get_or_init(|| {
353 let roots = rustls_native_certs::load_native_certs();
355 for e in &roots.errors {
356 eprintln!("[gasm] fetch: certificates: {e}");
357 }
358 let certs = roots.certs.iter().map(|c| ureq::tls::Certificate::from_der(c.as_ref()).to_owned()).collect();
359 let tls = ureq::tls::TlsConfig::builder()
360 .provider(ureq::tls::TlsProvider::Rustls)
361 .root_certs(ureq::tls::RootCerts::Specific(Arc::new(certs)))
362 .unversioned_rustls_crypto_provider(Arc::new(rustls::crypto::ring::default_provider()))
363 .build();
364 ureq::Agent::config_builder()
365 .http_status_as_error(false)
366 .max_redirects(0)
367 .max_redirects_will_error(false)
368 .timeout_connect(Some(Duration::from_secs(30)))
369 .timeout_global(Some(Duration::from_secs(120)))
370 .tls_config(tls)
371 .build()
372 .into()
373 })
374}
375
376fn resolve(base: &str, location: &str) -> String {
378 if location.contains("://") {
379 return location.to_owned();
380 }
381 let (scheme, rest) = base.split_once("://").unwrap_or(("http", base));
382 let authority = rest.split(['/', '?', '#']).next().unwrap_or("");
383 if let Some(l) = location.strip_prefix("//") {
384 return format!("{scheme}://{l}");
385 }
386 if location.starts_with('/') {
387 return format!("{scheme}://{authority}{location}");
388 }
389 let path = rest[authority.len()..].split(['?', '#']).next().unwrap_or("");
390 let dir = &path[..path.rfind('/').map_or(0, |i| i + 1)];
391 format!("{scheme}://{authority}{}{location}", if dir.is_empty() { "/" } else { dir })
392}
393
394pub fn user_agent(app_id: Option<&str>) -> String {
397 let runner = concat!("gasm-run/", env!("CARGO_PKG_VERSION"));
398 match app_id {
399 Some(id) => format!("{id} {runner}"),
400 None => runner.to_owned(),
401 }
402}
403
404pub fn valid_app_id(id: &str) -> bool {
406 !id.is_empty() && id.len() <= 256 && id.bytes().all(|c| (0x20..0x7f).contains(&c))
407}
408
409fn run(d: Desc, body: Vec<u8>, policy: NetPolicy, user_agent: String, tx: SyncSender<Event>, cancel: Arc<AtomicBool>, record: Option<PathBuf>) {
410 let result = (|| -> Result<(), String> {
411 let (mut method, mut url, mut body) = (d.method.clone(), d.url.clone(), body);
412 let mut hops = 0;
413 let resp = loop {
414 let mut b = ureq::http::Request::builder().method(method.as_str()).uri(url.as_str()).header("user-agent", user_agent.as_str());
415 for (k, v) in &d.headers {
416 b = b.header(k.as_str(), v.as_str());
417 }
418 let req = b.body(body.clone()).map_err(|e| format!("{url}: {e}"))?;
419 let resp = agent().run(req).map_err(|e| format!("{method} {url}: {e}"))?;
420 let status = resp.status().as_u16();
421 let location = resp.headers().get("location").and_then(|v| v.to_str().ok()).map(str::to_owned);
422 match (status, location) {
423 (301 | 302 | 303 | 307 | 308, Some(loc)) => {
424 hops += 1;
425 if hops > MAX_REDIRECTS {
426 return Err(format!("{url}: too many redirects"));
427 }
428 let next = resolve(&url, &loc);
429 if let Some(why) = policy.refusal(&next) {
430 return Err(format!("{url}: redirect to {next} denied ({why})"));
431 }
432 if status == 303 || (matches!(status, 301 | 302) && method == "POST") {
434 if method != "HEAD" {
435 method = "GET".into();
436 }
437 body.clear();
438 }
439 url = next;
440 }
441 _ => break resp,
442 }
443 };
444 let status = resp.status().as_u16() as i32;
445 let headers = header_lines(resp.headers().iter().map(|(k, v)| (k.as_str(), String::from_utf8_lossy(v.as_bytes()).into_owned())));
446 tx.send(Event::Head(status, headers.clone())).map_err(|_| String::new())?;
447 let mut reader = resp.into_body().into_with_config().limit(MAX_BODY).reader();
448 let mut buf = vec![0u8; CHUNK];
449 let mut all = Vec::new();
450 loop {
451 if cancel.load(Ordering::Relaxed) {
452 return Err(String::new());
453 }
454 let n = reader.read(&mut buf).map_err(|e| format!("{url}: {e}"))?;
455 if n == 0 {
456 break;
457 }
458 if record.is_some() {
459 all.extend_from_slice(&buf[..n]);
460 }
461 tx.send(Event::Chunk(buf[..n].to_vec())).map_err(|_| String::new())?;
462 }
463 if let Some(path) = &record {
464 save_record(path, &d.method, &d.url, status, &headers, &all);
465 }
466 tx.send(Event::Done).map_err(|_| String::new())
467 })();
468 if let Err(e) = result {
469 if !e.is_empty() {
470 let _ = tx.send(Event::Fail(e));
471 }
472 }
473}
474
475#[cfg(test)]
476mod tests {
477 use super::*;
478
479 #[test]
480 fn descriptions() {
481 let d = parse_desc(r#"{"url":"https://api.met.no/x?a=1","headers":{"Accept":"application/json"}}"#, 0).unwrap();
482 assert_eq!(d, Desc { method: "GET".into(), url: "https://api.met.no/x?a=1".into(), headers: vec![("accept".into(), "application/json".into())] });
483 assert_eq!(parse_desc(r#"{"method":"post","url":"http://h/"}"#, 3).unwrap().method, "POST");
484 for bad in [
485 r#"[]"#,
486 r#"{"url":"ftp://h/"}"#,
487 r#"{"url":"/relative"}"#,
488 r#"{"method":"TRACE","url":"http://h/"}"#,
489 r#"{"url":"http://h/","headers":{"Cookie":"a"}}"#,
490 r#"{"url":"http://h/","headers":{"sec-fetch-mode":"a"}}"#,
491 r#"{"url":"http://h/","headers":{"x":"a\r\nb"}}"#,
492 r#"{"url":"http://h/","headers":{"bad name":"a"}}"#,
493 ] {
494 assert!(parse_desc(bad, 0).is_err(), "{bad}");
495 }
496 assert!(parse_desc(r#"{"url":"http://h/"}"#, 1).is_err()); }
498
499 #[test]
500 fn response_headers() {
501 let h = header_lines([("Vary", "a".into()), ("content-type", "text/plain".into()), ("Connection", "close".into()), ("vary", " b ".into()), ("set-cookie", "x=1".into())]);
502 assert_eq!(h, "content-type: text/plain\nvary: a, b\n");
503 let gz = header_lines([("content-encoding", "gzip".into()), ("content-length", "10".into()), ("etag", "x".into())]);
504 assert_eq!(gz, "etag: x\n");
505 }
506
507 #[test]
508 fn redirects_resolve() {
509 assert_eq!(resolve("https://a.b/x/y?q", "/z"), "https://a.b/z");
510 assert_eq!(resolve("https://a.b/x/y?q", "z"), "https://a.b/x/z");
511 assert_eq!(resolve("https://a.b", "z"), "https://a.b/z");
512 assert_eq!(resolve("https://a.b/x", "//c.d/e"), "https://c.d/e");
513 assert_eq!(resolve("https://a.b/x", "http://c.d/"), "http://c.d/");
514 }
515
516 #[test]
517 fn keys() {
518 assert_eq!(record_key("GET", "https://example.org/", b""), "a0eecdfd36f8bde7");
520 }
521
522 #[test]
523 fn replay_and_handles() {
524 let dir = std::env::temp_dir().join(format!("gasm-fetch-test-{}", std::process::id()));
525 let _ = std::fs::remove_dir_all(&dir);
526 let key = record_key("GET", "https://example.org/a", b"");
527 save_record(&dir.join(&key), "GET", "https://example.org/a", 200, "content-type: text/plain\n", b"hello");
528 let mut f = Fetch::new(NetPolicy::default(), FetchMode::Replay(dir.clone()));
529 let h = f.request(r#"{"url":"https://example.org/a"}"#, vec![], 5);
530 let miss = f.request(r#"{"url":"https://example.org/b"}"#, vec![], 5);
531 assert_eq!((f.state(h, 5).unwrap(), f.status(h, 5).unwrap(), f.read(h, 9, 5).unwrap()), (PENDING, 0, Ok(vec![])));
532 assert_eq!((f.state(h, 6).unwrap(), f.status(h, 6).unwrap()), (DONE, 200));
533 assert_eq!(f.headers(h, 6).unwrap().as_deref(), Some("content-type: text/plain\n"));
534 assert_eq!(f.read(h, 3, 6).unwrap(), Ok(b"hel".to_vec()));
535 assert_eq!(f.read(h, 9, 6).unwrap(), Ok(b"lo".to_vec()));
536 assert_eq!(f.read(h, 9, 6).unwrap(), Err(()));
537 assert_eq!(f.state(miss, 6).unwrap(), FAILED);
538 f.close(h).unwrap();
539 assert_eq!(f.state(h, 7).unwrap(), FAILED); assert!(f.state(99, 7).is_err()); let _ = std::fs::remove_dir_all(&dir);
542 assert_eq!(Fetch::default().request(r#"{"url":"https://example.org/"}"#, vec![], 0), -1);
544 }
545}