1#![cfg(not(target_arch = "wasm32"))]
4
5use anyhow::Result;
6use async_trait::async_trait;
7use wacore::net::{HttpClient, HttpRequest, HttpResponse, StreamingHttpResponse, UploadBody};
8use wacore::stats::HttpResourceReport;
9
10pub const DEFAULT_MAX_BODY_BYTES: u64 = 2 * 1024 * 1024 * 1024;
13
14const INPUT_BUFFER_BYTES: u64 = 16 * 1024;
17const OUTPUT_BUFFER_BYTES: u64 = 16 * 1024;
18const MAX_IDLE_CONNECTIONS: u64 = 3;
20
21#[derive(Debug, Clone)]
24pub struct UreqHttpClient {
25 agent: ureq::Agent,
26 max_body_bytes: u64,
30 pool_report: Option<HttpResourceReport>,
33}
34
35fn default_pool_report() -> HttpResourceReport {
39 HttpResourceReport {
40 pool_connections: Some(MAX_IDLE_CONNECTIONS),
41 pool_buffer_bytes: Some(MAX_IDLE_CONNECTIONS * (INPUT_BUFFER_BYTES + OUTPUT_BUFFER_BYTES)),
42 inflight_bytes: None,
43 }
44}
45
46impl UreqHttpClient {
47 pub fn new() -> Self {
48 Self {
49 agent: build_agent(),
50 max_body_bytes: DEFAULT_MAX_BODY_BYTES,
51 pool_report: Some(default_pool_report()),
52 }
53 }
54
55 pub fn with_agent(agent: ureq::Agent) -> Self {
60 Self {
61 agent,
62 max_body_bytes: DEFAULT_MAX_BODY_BYTES,
63 pool_report: None,
65 }
66 }
67
68 pub fn with_max_body_bytes(mut self, max_body_bytes: u64) -> Self {
72 self.max_body_bytes = max_body_bytes;
73 self
74 }
75}
76
77impl Default for UreqHttpClient {
78 fn default() -> Self {
79 Self::new()
80 }
81}
82
83fn build_agent() -> ureq::Agent {
84 use ureq::config::Config;
85
86 #[allow(unused_mut)]
87 let mut builder = Config::builder()
88 .input_buffer_size(INPUT_BUFFER_BYTES as usize)
91 .output_buffer_size(OUTPUT_BUFFER_BYTES as usize)
92 .max_idle_connections(MAX_IDLE_CONNECTIONS as usize)
93 .max_idle_connections_per_host(2);
94
95 #[cfg(feature = "danger-skip-tls-verify")]
96 {
97 use ureq::tls::TlsConfig;
98 builder = builder.tls_config(TlsConfig::builder().disable_verification(true).build());
99 }
100
101 builder.build().into()
102}
103
104fn status_as_response<Any>(req: ureq::RequestBuilder<Any>) -> ureq::RequestBuilder<Any> {
116 req.config().http_status_as_error(false).build()
117}
118
119const ERROR_BODY_CAP: u64 = 64 * 1024;
128
129fn read_body(response: ureq::http::Response<ureq::Body>, max_body_bytes: u64) -> Result<Vec<u8>> {
143 if response.status().is_success() {
144 return Ok(response
146 .into_body()
147 .into_with_config()
148 .limit(max_body_bytes)
149 .read_to_vec()?);
150 }
151
152 let mut body = Vec::new();
153 let mut reader = std::io::Read::take(
154 response.into_body().into_reader(),
155 max_body_bytes.min(ERROR_BODY_CAP),
156 );
157 let _ = std::io::Read::read_to_end(&mut reader, &mut body);
159 Ok(body)
160}
161
162#[async_trait]
163impl HttpClient for UreqHttpClient {
164 async fn execute(&self, request: HttpRequest) -> Result<HttpResponse> {
165 let agent = self.agent.clone();
166 let max_body_bytes = self.max_body_bytes;
167 tokio::task::spawn_blocking(move || {
169 let response = match request.method.as_str() {
170 "GET" => {
171 let mut req = status_as_response(agent.get(&request.url));
172 for (key, value) in &request.headers {
173 req = req.header(key, value);
174 }
175 req.call()?
176 }
177 "POST" => {
178 let mut req = status_as_response(agent.post(&request.url));
179 for (key, value) in &request.headers {
180 req = req.header(key, value);
181 }
182 if let Some(body) = request.body {
183 req.send(&body[..])?
184 } else {
185 req.send(&[])?
186 }
187 }
188 method => {
189 return Err(anyhow::anyhow!("Unsupported HTTP method: {}", method));
190 }
191 };
192
193 let status_code = response.status().as_u16();
194 let body = read_body(response, max_body_bytes)?;
195
196 Ok(HttpResponse { status_code, body })
197 })
198 .await?
199 }
200
201 fn supports_streaming(&self) -> bool {
202 true
203 }
204
205 fn execute_streaming(&self, request: HttpRequest) -> Result<StreamingHttpResponse> {
206 let response = match request.method.as_str() {
210 "GET" => {
211 let mut req = status_as_response(self.agent.get(&request.url));
212 for (key, value) in &request.headers {
213 req = req.header(key, value);
214 }
215 req.call()?
216 }
217 method => {
218 return Err(anyhow::anyhow!(
219 "Streaming only supports GET, got: {}",
220 method
221 ));
222 }
223 };
224
225 let status_code = response.status().as_u16();
226 let reader = std::io::Read::take(response.into_body().into_reader(), self.max_body_bytes);
233
234 Ok(StreamingHttpResponse {
235 status_code,
236 body: Box::new(reader),
237 })
238 }
239
240 fn supports_upload_streaming(&self) -> bool {
241 true
242 }
243
244 fn execute_upload(
245 &self,
246 request: HttpRequest,
247 body: UploadBody,
248 content_length: u64,
249 ) -> Result<HttpResponse> {
250 if request.method != "POST" {
253 return Err(anyhow::anyhow!(
254 "Upload streaming only supports POST, got: {}",
255 request.method
256 ));
257 }
258
259 let mut req = status_as_response(self.agent.post(&request.url));
260 for (key, value) in &request.headers {
261 req = req.header(key, value);
262 }
263 let content_length = content_length.to_string();
266 req = req.header("content-length", content_length.as_str());
267
268 let response = req.send(ureq::SendBody::from_owned_reader(body))?;
269
270 let status_code = response.status().as_u16();
271 let body = read_body(response, self.max_body_bytes)?;
272
273 Ok(HttpResponse { status_code, body })
274 }
275
276 fn resource_report(&self) -> Option<HttpResourceReport> {
277 self.pool_report
278 }
279}
280
281#[cfg(test)]
282mod tests {
283 use super::*;
284 use std::io::{Read, Write};
285 use std::net::TcpListener;
286 use std::thread;
287
288 fn spawn_fixed_size_server(body_size: usize) -> String {
289 let listener = TcpListener::bind("127.0.0.1:0").expect("bind ephemeral port");
290 let addr = listener.local_addr().unwrap();
291 thread::spawn(move || {
292 let (mut stream, _) = listener.accept().expect("accept");
293 let mut buf = [0u8; 4096];
294 let mut total = Vec::new();
295 loop {
296 let n = stream.read(&mut buf).unwrap_or(0);
297 if n == 0 {
298 return;
299 }
300 total.extend_from_slice(&buf[..n]);
301 if total.windows(4).any(|w| w == b"\r\n\r\n") {
302 break;
303 }
304 }
305 let header = format!(
306 "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
307 body_size
308 );
309 stream.write_all(header.as_bytes()).unwrap();
310 let chunk = vec![0xABu8; 64 * 1024];
311 let mut sent = 0usize;
312 while sent < body_size {
313 let take = chunk.len().min(body_size - sent);
314 stream.write_all(&chunk[..take]).unwrap();
315 sent += take;
316 }
317 });
318 format!("http://{}", addr)
319 }
320
321 #[tokio::test(flavor = "current_thread")]
323 async fn execute_accepts_body_larger_than_ureq_default_limit() {
324 const SIZE: usize = 12 * 1024 * 1024;
325 let url = spawn_fixed_size_server(SIZE);
326 let resp = UreqHttpClient::new()
327 .execute(HttpRequest {
328 method: "GET".into(),
329 url,
330 headers: std::collections::HashMap::new(),
331 body: None,
332 })
333 .await
334 .expect("body must fit under the configured cap");
335 assert_eq!(resp.status_code, 200);
336 assert_eq!(resp.body.len(), SIZE);
337 }
338
339 #[tokio::test(flavor = "current_thread")]
340 async fn with_max_body_bytes_enforces_tighter_cap() {
341 const SIZE: usize = 4 * 1024 * 1024;
342 let url = spawn_fixed_size_server(SIZE);
343 UreqHttpClient::new()
344 .with_max_body_bytes(1024)
345 .execute(HttpRequest {
346 method: "GET".into(),
347 url,
348 headers: std::collections::HashMap::new(),
349 body: None,
350 })
351 .await
352 .expect_err("1 KiB cap must reject a 4 MiB body");
353 }
354
355 #[tokio::test(flavor = "current_thread")]
359 async fn execute_streaming_bounds_body_at_cap() {
360 const SIZE: usize = 4 * 1024 * 1024;
361 const CAP: u64 = 1024;
362 let url = spawn_fixed_size_server(SIZE);
363 let read = tokio::task::spawn_blocking(move || {
364 let mut resp = UreqHttpClient::new()
365 .with_max_body_bytes(CAP)
366 .execute_streaming(HttpRequest {
367 method: "GET".into(),
368 url,
369 headers: std::collections::HashMap::new(),
370 body: None,
371 })
372 .expect("streaming GET should start");
373 let mut sink = std::io::sink();
374 std::io::copy(&mut resp.body, &mut sink).expect("draining the reader should not error")
375 })
376 .await
377 .unwrap();
378 assert_eq!(read, CAP, "streaming body must stop at the cap");
379 }
380
381 fn spawn_capture_server() -> (String, std::sync::mpsc::Receiver<(String, Vec<u8>)>) {
383 let listener = TcpListener::bind("127.0.0.1:0").expect("bind ephemeral port");
384 let addr = listener.local_addr().unwrap();
385 let (tx, rx) = std::sync::mpsc::channel();
386 thread::spawn(move || {
387 let (mut stream, _) = listener.accept().expect("accept");
388 let mut buf = Vec::new();
389 let mut tmp = [0u8; 4096];
390 let header_end = loop {
391 if let Some(pos) = buf.windows(4).position(|w| w == b"\r\n\r\n") {
392 break pos + 4;
393 }
394 let n = stream.read(&mut tmp).unwrap_or(0);
395 if n == 0 {
396 return;
397 }
398 buf.extend_from_slice(&tmp[..n]);
399 };
400 let headers = String::from_utf8_lossy(&buf[..header_end]).to_string();
401 let content_length = headers.lines().find_map(|l| {
402 let (k, v) = l.split_once(':')?;
403 if k.trim().eq_ignore_ascii_case("content-length") {
404 v.trim().parse::<usize>().ok()
405 } else {
406 None
407 }
408 });
409 let mut body = buf[header_end..].to_vec();
410 if let Some(cl) = content_length {
411 while body.len() < cl {
412 let n = stream.read(&mut tmp).unwrap_or(0);
413 if n == 0 {
414 break;
415 }
416 body.extend_from_slice(&tmp[..n]);
417 }
418 }
419 let _ = stream
420 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}");
421 let _ = tx.send((headers, body));
422 });
423 (format!("http://{addr}"), rx)
424 }
425
426 fn parsed_content_length(headers: &str) -> Option<usize> {
427 headers.lines().find_map(|l| {
428 let (k, v) = l.split_once(':')?;
429 k.trim()
430 .eq_ignore_ascii_case("content-length")
431 .then(|| v.trim().parse::<usize>().ok())
432 .flatten()
433 })
434 }
435
436 #[test]
439 fn upload_streaming_sets_content_length_not_chunked() {
440 let (url, rx) = spawn_capture_server();
441 let payload: Vec<u8> = (0..5000u32).map(|i| i as u8).collect();
442 let client = UreqHttpClient::new();
443
444 let resp = client
445 .execute_upload(
446 HttpRequest {
447 method: "POST".into(),
448 url,
449 headers: std::collections::HashMap::new(),
450 body: None,
451 },
452 Box::new(std::io::Cursor::new(payload.clone())),
453 payload.len() as u64,
454 )
455 .expect("upload should succeed");
456 assert_eq!(resp.status_code, 200);
457
458 let (headers, body) = rx
459 .recv_timeout(std::time::Duration::from_secs(5))
460 .expect("server should capture the request");
461 assert_eq!(
462 parsed_content_length(&headers),
463 Some(payload.len()),
464 "exact Content-Length expected, headers:\n{headers}"
465 );
466 assert!(
467 !headers.to_ascii_lowercase().contains("transfer-encoding"),
468 "body must not be chunked, headers:\n{headers}"
469 );
470 assert_eq!(body, payload, "server must receive the exact bytes");
471 }
472
473 #[test]
476 fn upload_streaming_large_body_integrity() {
477 let (url, rx) = spawn_capture_server();
478 let payload: Vec<u8> = (0..200_000usize).map(|i| (i % 251) as u8).collect();
479 let client = UreqHttpClient::new();
480
481 let resp = client
482 .execute_upload(
483 HttpRequest {
484 method: "POST".into(),
485 url,
486 headers: std::collections::HashMap::new(),
487 body: None,
488 },
489 Box::new(std::io::Cursor::new(payload.clone())),
490 payload.len() as u64,
491 )
492 .expect("upload should succeed");
493 assert_eq!(resp.status_code, 200);
494
495 let (headers, body) = rx
496 .recv_timeout(std::time::Duration::from_secs(10))
497 .expect("server should capture the request");
498 assert_eq!(parsed_content_length(&headers), Some(payload.len()));
499 assert_eq!(body, payload);
500 }
501
502 #[test]
505 fn resource_report_estimates_default_pool() {
506 let report = UreqHttpClient::new()
507 .resource_report()
508 .expect("default agent reports a pool estimate");
509 assert_eq!(report.pool_connections, Some(MAX_IDLE_CONNECTIONS));
510 assert_eq!(
511 report.pool_buffer_bytes,
512 Some(MAX_IDLE_CONNECTIONS * (INPUT_BUFFER_BYTES + OUTPUT_BUFFER_BYTES))
513 );
514 assert_eq!(report.inflight_bytes, None);
515 assert!(report.total_bytes() > 0);
516
517 assert!(
519 UreqHttpClient::with_agent(build_agent())
520 .resource_report()
521 .is_none(),
522 "custom-agent client reports no estimate"
523 );
524
525 assert!(
527 UreqHttpClient::new()
528 .with_max_body_bytes(1024)
529 .resource_report()
530 .is_some()
531 );
532 }
533
534 fn spawn_status_server(status: u16, reason: &str) -> String {
535 spawn_status_server_with_body(status, reason, b"denied".to_vec())
536 }
537
538 fn spawn_status_server_with_body(status: u16, reason: &str, body: Vec<u8>) -> String {
541 let listener = TcpListener::bind("127.0.0.1:0").expect("bind ephemeral port");
542 let addr = listener.local_addr().unwrap();
543 let reason = reason.to_string();
544 thread::spawn(move || {
545 let Ok((mut stream, _)) = listener.accept() else {
546 return;
547 };
548 let mut buf = Vec::new();
549 let mut tmp = [0u8; 4096];
550 let header_end = loop {
551 if let Some(pos) = buf.windows(4).position(|w| w == b"\r\n\r\n") {
552 break pos + 4;
553 }
554 match stream.read(&mut tmp) {
555 Ok(0) | Err(_) => return,
556 Ok(n) => buf.extend_from_slice(&tmp[..n]),
557 }
558 };
559 let headers = String::from_utf8_lossy(&buf[..header_end]).to_string();
560 if let Some(cl) = parsed_content_length(&headers) {
561 let mut body_len = buf.len() - header_end;
562 while body_len < cl {
563 match stream.read(&mut tmp) {
564 Ok(0) | Err(_) => break,
565 Ok(n) => body_len += n,
566 }
567 }
568 }
569 let header = format!(
570 "HTTP/1.1 {status} {reason}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
571 body.len()
572 );
573 let _ = stream.write_all(header.as_bytes());
576 let _ = stream.write_all(&body);
577 });
578 format!("http://{addr}")
579 }
580
581 fn get(url: String) -> HttpRequest {
582 HttpRequest {
583 method: "GET".into(),
584 url,
585 headers: std::collections::HashMap::new(),
586 body: None,
587 }
588 }
589
590 #[tokio::test(flavor = "current_thread")]
596 async fn execute_surfaces_non_2xx_status_instead_of_erroring() {
597 for (status, reason) in [
598 (401u16, "Unauthorized"),
599 (403, "Forbidden"),
600 (404, "Not Found"),
601 ] {
602 let url = spawn_status_server(status, reason);
603 let resp = UreqHttpClient::new()
604 .execute(get(url))
605 .await
606 .unwrap_or_else(|e| panic!("{status} must arrive as a response, got error: {e}"));
607 assert_eq!(resp.status_code, status);
608 assert_eq!(resp.body, b"denied");
609 }
610 }
611
612 #[tokio::test(flavor = "current_thread")]
613 async fn execute_post_surfaces_non_2xx_status_instead_of_erroring() {
614 let url = spawn_status_server(403, "Forbidden");
615 let resp = UreqHttpClient::new()
616 .execute(HttpRequest::post(url).with_body(b"payload".to_vec()))
617 .await
618 .expect("403 must arrive as a response, not an error");
619 assert_eq!(resp.status_code, 403);
620 }
621
622 #[tokio::test(flavor = "current_thread")]
624 async fn execute_streaming_surfaces_non_2xx_status_instead_of_erroring() {
625 let url = spawn_status_server(403, "Forbidden");
626 let status = tokio::task::spawn_blocking(move || {
627 UreqHttpClient::new()
628 .execute_streaming(get(url))
629 .expect("403 must arrive as a response, not an error")
630 .status_code
631 })
632 .await
633 .unwrap();
634 assert_eq!(status, 403);
635 }
636
637 #[test]
639 fn execute_upload_surfaces_non_2xx_status_instead_of_erroring() {
640 let url = spawn_status_server(403, "Forbidden");
641 let payload = vec![7u8; 128];
642 let resp = UreqHttpClient::new()
643 .execute_upload(
644 HttpRequest {
645 method: "POST".into(),
646 url,
647 headers: std::collections::HashMap::new(),
648 body: None,
649 },
650 Box::new(std::io::Cursor::new(payload.clone())),
651 payload.len() as u64,
652 )
653 .expect("403 must arrive as a response, not an error");
654 assert_eq!(resp.status_code, 403);
655 }
656
657 #[tokio::test(flavor = "current_thread")]
662 async fn over_cap_error_body_does_not_cost_the_status() {
663 const CAP: u64 = 1024;
664 let url = spawn_status_server_with_body(403, "Forbidden", vec![b'x'; 4 * 1024 * 1024]);
665 let resp = UreqHttpClient::new()
666 .with_max_body_bytes(CAP)
667 .execute(get(url))
668 .await
669 .expect("an over-cap error page must not erase the status it came with");
670 assert_eq!(resp.status_code, 403);
671 assert!(
672 resp.body.len() as u64 <= CAP,
673 "the diagnostic body must stay bounded, got {} bytes",
674 resp.body.len()
675 );
676 }
677
678 #[tokio::test(flavor = "current_thread")]
682 async fn over_cap_success_body_is_still_an_error() {
683 let url = spawn_status_server_with_body(200, "OK", vec![b'x'; 4 * 1024 * 1024]);
684 UreqHttpClient::new()
685 .with_max_body_bytes(1024)
686 .execute(get(url))
687 .await
688 .expect_err("a truncated 2xx payload must never look like a complete one");
689 }
690
691 #[tokio::test(flavor = "current_thread")]
694 async fn custom_agent_also_surfaces_non_2xx_status() {
695 let url = spawn_status_server(403, "Forbidden");
696 let agent: ureq::Agent = ureq::config::Config::builder().build().into();
697 let resp = UreqHttpClient::with_agent(agent)
698 .execute(get(url))
699 .await
700 .expect("403 must arrive as a response even with a custom agent");
701 assert_eq!(resp.status_code, 403);
702 }
703
704 #[test]
705 fn upload_streaming_rejects_non_post() {
706 let client = UreqHttpClient::new();
707 let err = client.execute_upload(
708 HttpRequest {
709 method: "GET".into(),
710 url: "http://127.0.0.1:0/never".into(),
711 headers: std::collections::HashMap::new(),
712 body: None,
713 },
714 Box::new(std::io::Cursor::new(vec![1u8, 2, 3])),
715 3,
716 );
717 assert!(err.is_err(), "only POST is allowed for upload streaming");
718 }
719}