use std::fmt::Write as _;
use async_trait::async_trait;
use bytes::{Bytes, BytesMut};
use cratefield_core::{
ByteStream, HttpClient, HttpError, HttpPolicy, StatusOnly, scrub_request_url, scrub_text,
};
use futures_core::Stream;
use futures_util::StreamExt;
use worker::send::IntoSendFuture;
use worker::{Fetch, Headers, Method, Request as WorkerRequest, RequestInit, RequestRedirect};
pub struct FetchClient;
const INITIAL_READ_CAPACITY: usize = 16 * 1024;
#[async_trait]
impl HttpClient for FetchClient {
async fn send(
&self,
request: http::Request<Bytes>,
) -> Result<http::Response<Bytes>, HttpError> {
let policy = HttpPolicy::of_request(&request);
let status_only = request.extensions().get::<StatusOnly>().is_some();
let (url, worker_request) = build_request(request)?;
let mut response = Fetch::Request(worker_request)
.send()
.into_send()
.await
.map_err(|err| transport(&err, &url))?;
if !status_only {
let declared = response
.headers()
.get("content-length")
.ok()
.flatten()
.and_then(|length| length.trim().parse::<usize>().ok());
if declared.is_some_and(|declared| declared > policy.max_response_bytes) {
return Err(HttpError::ResponseTooLarge {
limit: policy.max_response_bytes,
});
}
}
let headers = response_headers(response.headers(), status_only);
let mut builder = http::Response::builder().status(response.status_code());
if let Some(map) = builder.headers_mut() {
*map = headers;
}
if status_only {
return builder
.body(Bytes::new())
.map_err(|err| transport(&err, &url));
}
let bytes = capped_body(response.stream(), policy.max_response_bytes, &url)
.into_send()
.await?;
builder.body(bytes).map_err(|err| transport(&err, &url))
}
async fn send_streaming(
&self,
request: http::Request<Bytes>,
) -> Result<http::Response<ByteStream>, HttpError> {
let (url, worker_request) = build_request(request)?;
let mut response = Fetch::Request(worker_request)
.send()
.into_send()
.await
.map_err(|err| transport(&err, &url))?;
let headers = response_headers(response.headers(), false);
let mut builder = http::Response::builder().status(response.status_code());
if let Some(map) = builder.headers_mut() {
*map = headers;
}
let body: ByteStream =
match response.stream() {
Ok(stream) => {
let url = url.clone();
Box::pin(crate::send_stream(stream).map(move |chunk| {
chunk.map(Bytes::from).map_err(|err| transport(&err, &url))
}))
}
Err(_) => Box::pin(futures_util::stream::empty()),
};
builder.body(body).map_err(|err| transport(&err, &url))
}
}
fn build_request(request: http::Request<Bytes>) -> Result<(String, WorkerRequest), HttpError> {
let (parts, body) = request.into_parts();
let url = parts.uri.to_string();
let mut init = RequestInit::new();
init.method = Method::from(parts.method.as_str().to_string());
let headers = Headers::new();
for (name, value) in &parts.headers {
let _ = headers.set(name.as_str(), value.to_str().unwrap_or_default());
}
init.headers = headers;
init.with_redirect(RequestRedirect::Manual);
if !body.is_empty() {
let body_text = String::from_utf8(body.to_vec()).map_err(|err| transport(&err, &url))?;
init.with_body(Some(worker::wasm_bindgen::JsValue::from_str(&body_text)));
}
let worker_request =
WorkerRequest::new_with_init(&url, &init).map_err(|err| transport(&err, &url))?;
Ok((url, worker_request))
}
fn response_headers<I>(from: I, status_only: bool) -> http::HeaderMap
where
I: IntoIterator<Item = (String, String)>,
{
let mut headers = http::HeaderMap::new();
for (name, value) in from {
if status_only && is_body_describing(&name) {
continue;
}
if let (Ok(name), Ok(value)) = (
http::HeaderName::try_from(name.as_str()),
http::HeaderValue::try_from(value.as_str()),
) {
headers.insert(name, value);
}
}
headers
}
async fn capped_body<S>(
stream: worker::Result<S>,
limit: usize,
url: &str,
) -> Result<Bytes, HttpError>
where
S: Stream<Item = worker::Result<Vec<u8>>> + Unpin,
{
match stream {
Ok(stream) => read_capped(stream, limit, url).await,
Err(_) => Ok(Bytes::new()),
}
}
fn is_body_describing(name: &str) -> bool {
let Ok(name) = http::HeaderName::try_from(name) else {
return false;
};
matches!(
name,
http::header::CONTENT_LENGTH
| http::header::CONTENT_ENCODING
| http::header::TRANSFER_ENCODING
)
}
async fn read_capped<S>(mut stream: S, limit: usize, url: &str) -> Result<Bytes, HttpError>
where
S: Stream<Item = worker::Result<Vec<u8>>> + Unpin,
{
let mut buffered = BytesMut::with_capacity(limit.min(INITIAL_READ_CAPACITY));
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(|err| transport(&err, url))?;
if buffered.len().saturating_add(chunk.len()) > limit {
return Err(HttpError::ResponseTooLarge { limit });
}
buffered.extend_from_slice(&chunk);
}
Ok(buffered.freeze())
}
fn transport(err: &dyn std::error::Error, url: &str) -> HttpError {
HttpError::Transport(safe_message(err, url))
}
fn safe_message(err: &dyn std::error::Error, url: &str) -> String {
let mut message = err.to_string();
let mut cause = err.source();
while let Some(current) = cause {
let _ = write!(message, ": {current}");
cause = current.source();
}
scrub_text(&scrub_request_url(&message, url))
}
#[cfg(test)]
mod tests {
use super::*;
use futures_util::stream;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
const TOKEN: &str = "0a1b2c3d4e5f60718293a4b5c6d7e8f900112233445566778899aabbccddeeff";
fn apns_url() -> String {
format!("https://api.push.apple.com/3/device/{TOKEN}")
}
fn assert_safe(message: &str) {
assert!(
!message.contains(TOKEN),
"the device token leaked: {message}"
);
assert!(
!message.contains("/3/device"),
"the request path leaked: {message}"
);
}
#[test]
fn a_workerd_fetch_failure_names_the_origin_and_never_the_request_path() {
let url = apns_url();
let err = worker::Error::JsError(format!("TypeError: Fetch API cannot load: {url}."));
let message = safe_message(&err, &url);
assert_safe(&message);
assert!(
message.contains("https://api.push.apple.com"),
"the destination is named: {message}"
);
assert!(
message.contains("Fetch API cannot load"),
"the cause survives the redaction: {message}"
);
}
#[test]
fn the_cause_chain_is_walked_and_scrubbed() {
let url = apns_url();
let inner =
std::io::Error::other(format!("connection refused sending to /3/device/{TOKEN}"));
let err = worker::Error::Io(inner);
assert_eq!(
err.to_string(),
"I/O error",
"if `Display` ever carries the cause, this pass can stop appending it"
);
let message = safe_message(&err, &url);
assert_safe(&message);
assert!(
message.contains("connection refused"),
"the cause is what makes this diagnosable: {message}"
);
}
#[test]
fn a_secret_a_cause_quotes_is_scrubbed_even_when_it_is_not_the_url() {
let url = apns_url();
let err = worker::Error::JsError(
"rejected the Authorization: Bearer abcdefghijklmnop12345 we sent".to_owned(),
);
let message = safe_message(&err, &url);
assert!(
!message.contains("abcdefghijklmnop12345"),
"scrub_text is the second net: {message}"
);
}
#[test]
fn every_error_type_a_failure_site_produces_is_redacted() {
let url = apns_url();
let quoting = |what: &str| format!("{what} at {url}");
let utf8 = String::from_utf8(vec![0xff, 0xfe]).expect_err("not UTF-8");
let js = worker::Error::JsError(quoting("TypeError: Fetch API cannot load:"));
let rust = worker::Error::RustError(quoting("could not construct a request"));
let io = worker::Error::Io(std::io::Error::other(quoting("network connection lost")));
let http_err = http::Response::builder()
.status(200)
.header("x-\u{0}bad", "v")
.body(())
.expect_err("an invalid header name is a builder error");
let sites: [&dyn std::error::Error; 5] = [&utf8, &js, &rust, &io, &http_err];
for err in sites {
let HttpError::Transport(message) = transport(err, &url) else {
panic!("every site reports a transport failure");
};
assert_safe(&message);
}
}
#[test]
fn only_one_place_in_this_module_builds_a_transport_error() {
let source = include_str!("http.rs");
let module = source
.split_once("#[cfg(test)]")
.map_or(source, |(before, _)| before);
assert_eq!(
module.matches("HttpError::Transport(").count(),
1,
"a failure site must call `transport`, not build the error itself"
);
}
#[test]
fn a_redirect_is_never_followed_by_the_transport_request() {
let source = include_str!("http.rs");
let module = source
.split_once("#[cfg(test)]")
.map_or(source, |(before, _)| before);
assert_eq!(
module
.matches("init.with_redirect(RequestRedirect::Manual)")
.count(),
1,
"the init the request is built from must ask for `Manual` exactly once"
);
}
const LIMIT: usize = 4096;
const CHUNK: usize = 1024;
fn counting_chunks(
chunks: usize,
) -> (
impl Stream<Item = worker::Result<Vec<u8>>>,
Arc<AtomicUsize>,
) {
let pulled = Arc::new(AtomicUsize::new(0));
let counter = Arc::clone(&pulled);
let body = stream::iter(0..chunks).map(move |_| {
counter.fetch_add(1, Ordering::SeqCst);
Ok(vec![b'x'; CHUNK])
});
(body, pulled)
}
#[test]
fn a_body_past_the_cap_is_refused_however_it_arrives() {
let url = apns_url();
let (chunked, _) = counting_chunks(6);
let one_big: Vec<worker::Result<Vec<u8>>> = vec![Ok(vec![b'x'; LIMIT + 1])];
let refusals = [
pollster::block_on(read_capped(chunked, LIMIT, &url)),
pollster::block_on(read_capped(stream::iter(one_big), LIMIT, &url)),
];
for got in refusals {
assert!(
matches!(got, Err(HttpError::ResponseTooLarge { limit: LIMIT })),
"a body past the cap must be refused, got {got:?}"
);
}
}
#[test]
fn a_body_under_the_cap_is_read_whole_and_byte_identical() {
let url = apns_url();
let parts: Vec<worker::Result<Vec<u8>>> = vec![
Ok(b"hello ".to_vec()),
Ok(b"capped ".to_vec()),
Ok(b"world".to_vec()),
];
let body = pollster::block_on(read_capped(stream::iter(parts), LIMIT, &url))
.expect("a body under the cap must read");
assert_eq!(body, Bytes::from_static(b"hello capped world"));
}
#[test]
fn the_reader_aborts_at_the_cap_instead_of_draining_the_stream() {
let url = apns_url();
let (chunks, pulled) = counting_chunks(64);
let err = pollster::block_on(read_capped(chunks, LIMIT, &url))
.expect_err("an endless body must be refused, not buffered");
assert!(
matches!(err, HttpError::ResponseTooLarge { limit: LIMIT }),
"got {err}"
);
assert_eq!(
pulled.load(Ordering::SeqCst),
LIMIT / CHUNK + 1,
"reading stops at the chunk that crosses the cap, and nowhere later"
);
}
#[test]
fn a_stream_error_is_reported_through_transport_and_scrubbed() {
let url = apns_url();
let parts: Vec<worker::Result<Vec<u8>>> = vec![
Ok(b"ok".to_vec()),
Err(worker::Error::JsError(format!(
"TypeError: network error reading {url}"
))),
];
let err = pollster::block_on(read_capped(stream::iter(parts), LIMIT, &url))
.expect_err("an unreadable chunk must fail the send");
let HttpError::Transport(message) = &err else {
panic!("a stream error is reported as a transport failure, got {err}");
};
assert_safe(message);
assert!(
message.contains("network error reading"),
"the cause survives the redaction: {message}"
);
}
#[test]
fn a_status_only_response_keeps_every_header_that_does_not_describe_a_body() {
let reply: Vec<(String, String)> = [
("content-length", "4194304"),
("Content-Length", "4194304"),
("content-encoding", "gzip"),
("transfer-encoding", "chunked"),
("content-type", "application/json"),
("location", "https://example.com/hook"),
]
.into_iter()
.map(|(name, value)| (name.to_owned(), value.to_owned()))
.collect();
let probe = response_headers(reply.clone(), true);
for dropped in [
http::header::CONTENT_LENGTH,
http::header::CONTENT_ENCODING,
http::header::TRANSFER_ENCODING,
] {
assert_eq!(
probe.get(&dropped),
None,
"a probe reads no body, so {dropped:?} describes nothing"
);
}
for kept in [
(http::header::CONTENT_TYPE, "application/json"),
(http::header::LOCATION, "https://example.com/hook"),
] {
assert_eq!(
probe.get(kept.0),
Some(&http::HeaderValue::from_static(kept.1)),
"a header that is not about the body is still the caller's"
);
}
let whole = response_headers(reply, false);
assert_eq!(
whole.get(http::header::CONTENT_LENGTH),
Some(&http::HeaderValue::from_static("4194304")),
"a body that is really there is still described"
);
assert_eq!(
whole.get(http::header::CONTENT_ENCODING),
Some(&http::HeaderValue::from_static("gzip"))
);
}
#[test]
fn a_null_body_is_empty_and_a_streamable_one_is_still_read() {
let url = apns_url();
let unstreamable: worker::Result<
futures_util::stream::Iter<std::vec::IntoIter<worker::Result<Vec<u8>>>>,
> = Err(worker::Error::RustError("body is not streamable".into()));
let parts: Vec<worker::Result<Vec<u8>>> =
vec![Ok(b"hello ".to_vec()), Ok(b"world".to_vec())];
let empty = pollster::block_on(capped_body(unstreamable, LIMIT, &url))
.expect("a 204 is a successful answer, not a failed read");
let read = pollster::block_on(capped_body(Ok(stream::iter(parts)), LIMIT, &url))
.expect("a streamable body must read");
assert!(empty.is_empty(), "a null body reads as empty: {empty:?}");
assert_eq!(read, Bytes::from_static(b"hello world"));
}
}