use std::io::{Read, Write};
use std::net::{Shutdown, TcpStream, ToSocketAddrs};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
mod sock;
pub(crate) use sock::Wait;
use sock::Polled;
#[cfg(feature = "net-h2")]
pub mod pool;
pub mod cache;
pub(crate) const MAX_HEAD: usize = 64 * 1024;
pub const MAX_BODY: usize = 256 * 1024 * 1024;
const MAX_REDIRECTS: u32 = 20;
const MAX_CHUNK: u64 = MAX_BODY as u64;
pub(crate) const POLL: Duration = Duration::from_millis(200);
pub static CONNECTS: AtomicU64 = AtomicU64::new(0);
#[cfg(test)]
pub(crate) static TEST_POLL_MS: AtomicU64 = AtomicU64::new(0);
#[cfg(test)]
pub(crate) const TEST_ROOT_PEM: &[u8] = include_bytes!("http/testdata/cert.pem");
#[cfg(test)]
mod stress_tests;
pub const PROTO_H1: &str = "http/1.1";
pub const PROTO_H2: &str = "h2";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ErrorKind {
Refused,
Network,
Protocol,
Aborted,
}
#[derive(Debug, Clone)]
pub struct HttpError {
pub kind: ErrorKind,
pub message: String,
}
impl HttpError {
fn new(kind: ErrorKind, message: impl Into<String>) -> Self {
HttpError { kind, message: message.into() }
}
fn refused(m: impl Into<String>) -> Self {
Self::new(ErrorKind::Refused, m)
}
fn network(m: impl Into<String>) -> Self {
Self::new(ErrorKind::Network, m)
}
fn protocol(m: impl Into<String>) -> Self {
Self::new(ErrorKind::Protocol, m)
}
}
impl std::fmt::Display for HttpError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.message)
}
}
#[derive(Default)]
struct CancelState {
aborted: AtomicBool,
socket: Mutex<Option<TcpStream>>,
#[cfg(feature = "net-h2")]
wake: tokio::sync::Notify,
}
#[derive(Default, Clone)]
pub struct Cancel(Arc<CancelState>);
impl Cancel {
pub fn new() -> Arc<Cancel> {
Arc::new(Cancel::default())
}
pub fn abort(&self) {
self.0.aborted.store(true, Ordering::SeqCst);
if let Some(s) = self.0.socket.lock().unwrap().as_ref() {
let _ = s.shutdown(Shutdown::Both);
}
#[cfg(feature = "net-h2")]
self.0.wake.notify_waiters();
}
pub fn is_aborted(&self) -> bool {
self.0.aborted.load(Ordering::SeqCst)
}
fn check(&self) -> Result<(), HttpError> {
if self.is_aborted() {
return Err(HttpError::new(ErrorKind::Aborted, "the request was aborted"));
}
Ok(())
}
#[cfg(feature = "net-h2")]
pub(crate) async fn aborted(&self) {
loop {
let notified = self.0.wake.notified();
tokio::pin!(notified);
notified.as_mut().enable();
if self.is_aborted() {
return;
}
notified.await;
}
}
fn arm(&self, s: &TcpStream) -> Result<(), HttpError> {
match s.try_clone() {
Ok(dup) => *self.0.socket.lock().unwrap() = Some(dup),
Err(_) => {}
}
self.check()
}
#[cfg_attr(feature = "net-h2", allow(dead_code))]
fn disarm(&self) {
*self.0.socket.lock().unwrap() = None;
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Redirect {
Follow,
Error,
}
#[derive(Debug, Clone)]
pub struct Request {
pub method: String,
pub url: String,
pub headers: Vec<(String, String)>,
pub body: Option<Vec<u8>>,
pub redirect: Redirect,
pub credentials: super::cookies::Credentials,
pub connect_timeout: Duration,
pub io_timeout: Duration,
pub observer: Option<Observer>,
pub cache: cache::CacheMode,
}
pub trait Observe: Send + Sync {
fn upload(&self, sent: u64, total: u64) {
let _ = (sent, total);
}
fn head(&self, head: &ProgressHead<'_>) {
let _ = head;
}
fn body(&self, chunk: &[u8], received: u64, total: Option<u64>) {
let _ = (chunk, received, total);
}
fn streams(&self) -> bool {
false
}
}
pub struct ProgressHead<'a> {
pub status: u16,
pub status_text: &'a str,
pub headers: &'a [(String, String)],
pub url: &'a str,
pub redirected: bool,
pub protocol: &'static str,
}
#[derive(Clone)]
pub struct Observer(pub Arc<dyn Observe>);
impl std::fmt::Debug for Observer {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("Observer")
}
}
pub(crate) fn is_final(status: u16, headers: &[(String, String)]) -> bool {
!(is_redirect(status) && header(headers, "location").is_some())
}
#[derive(Debug, Clone)]
pub struct Response {
pub status: u16,
pub status_text: String,
pub headers: Vec<(String, String)>,
pub url: String,
pub redirected: bool,
pub body: Vec<u8>,
pub protocol: &'static str,
pub reused: bool,
pub connection_id: u64,
pub remote_addr: Option<std::net::SocketAddr>,
pub cache: cache::Outcome,
}
static NEXT_CONNECTION: AtomicU64 = AtomicU64::new(1);
pub(crate) fn next_connection_id() -> u64 {
NEXT_CONNECTION.fetch_add(1, Ordering::Relaxed)
}
pub fn header<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> {
headers.iter().find(|(k, _)| k.eq_ignore_ascii_case(name)).map(|(_, v)| v.as_str())
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Head {
pub status: u16,
pub reason: String,
pub headers: Vec<(String, String)>,
}
pub fn parse_head(buf: &[u8]) -> Result<Head, HttpError> {
let text = String::from_utf8_lossy(buf);
let mut lines = text.split("\r\n");
let status_line = lines.next().unwrap_or("");
let (status, reason) = parse_status_line(status_line)?;
let mut headers: Vec<(String, String)> = Vec::new();
for line in lines {
if line.is_empty() {
continue;
}
if line.starts_with(' ') || line.starts_with('\t') {
if let Some(last) = headers.last_mut() {
last.1.push(' ');
last.1.push_str(line.trim());
continue;
}
return Err(HttpError::protocol("malformed response: header continuation with no header"));
}
let Some(colon) = line.find(':') else {
return Err(HttpError::protocol(format!("malformed response header line: {line:?}")));
};
let name = line[..colon].trim_end();
if name.is_empty() || !name.bytes().all(is_token_byte) {
return Err(HttpError::protocol(format!("malformed response header name: {name:?}")));
}
headers.push((name.to_ascii_lowercase(), line[colon + 1..].trim().to_string()));
}
Ok(Head { status, reason, headers })
}
fn parse_status_line(line: &str) -> Result<(u16, String), HttpError> {
let rest = line
.strip_prefix("HTTP/1.1 ")
.or_else(|| line.strip_prefix("HTTP/1.0 "))
.ok_or_else(|| HttpError::protocol(format!("not an HTTP/1.x response: {:?}", trunc(line, 80))))?;
let code = rest.split(' ').next().unwrap_or("");
if code.len() != 3 || !code.bytes().all(|b| b.is_ascii_digit()) {
return Err(HttpError::protocol(format!("malformed status line: {:?}", trunc(line, 80))));
}
let status: u16 = code.parse().map_err(|_| HttpError::protocol("malformed status code"))?;
let reason = rest[code.len()..].trim().to_string();
Ok((status, reason))
}
fn trunc(s: &str, n: usize) -> &str {
match s.char_indices().nth(n) {
Some((i, _)) => &s[..i],
None => s,
}
}
fn is_token_byte(b: u8) -> bool {
b.is_ascii_alphanumeric() || b"!#$%&'*+-.^_`|~".contains(&b)
}
fn validate_header(name: &str, value: &str) -> Result<(), HttpError> {
if name.is_empty() || !name.bytes().all(is_token_byte) {
return Err(HttpError::refused(format!("invalid header name: {name:?}")));
}
if value.bytes().any(|b| b == b'\r' || b == b'\n' || b == 0) {
return Err(HttpError::refused(format!("invalid header value for {name:?}: contains CR, LF or NUL")));
}
Ok(())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Framing {
None,
Length(usize),
Chunked,
UntilClose,
}
pub fn framing_of(method: &str, status: u16, headers: &[(String, String)]) -> Result<Framing, HttpError> {
if method.eq_ignore_ascii_case("HEAD") || status == 204 || status == 304 || (100..200).contains(&status) {
return Ok(Framing::None);
}
if let Some(te) = header(headers, "transfer-encoding") {
if te.split(',').any(|t| t.trim().eq_ignore_ascii_case("chunked")) {
return Ok(Framing::Chunked);
}
return Err(HttpError::protocol(format!("unsupported Transfer-Encoding: {te:?}")));
}
if let Some(cl) = header(headers, "content-length") {
let mut seen: Option<usize> = None;
for part in cl.split(',') {
let n: usize = part
.trim()
.parse()
.map_err(|_| HttpError::protocol(format!("malformed Content-Length: {cl:?}")))?;
if seen.is_some_and(|s| s != n) {
return Err(HttpError::protocol(format!("conflicting Content-Length values: {cl:?}")));
}
seen = Some(n);
}
let n = seen.ok_or_else(|| HttpError::protocol("empty Content-Length"))?;
if n > MAX_BODY {
return Err(HttpError::refused(format!("response body of {n} bytes exceeds the {MAX_BODY}-byte limit")));
}
return Ok(Framing::Length(n));
}
Ok(Framing::UntilClose)
}
pub fn read_exact_body<R: Read>(r: &mut R, n: usize, prefix: &[u8]) -> Result<Vec<u8>, HttpError> {
let mut out = Vec::with_capacity(n.min(1 << 20));
out.extend_from_slice(&prefix[..prefix.len().min(n)]);
let mut buf = [0u8; 16 * 1024];
while out.len() < n {
let want = (n - out.len()).min(buf.len());
let got = r.read(&mut buf[..want]).map_err(|e| HttpError::network(format!("body read failed: {e}")))?;
if got == 0 {
return Err(HttpError::protocol(format!(
"connection closed after {} of {} declared body bytes",
out.len(),
n
)));
}
out.extend_from_slice(&buf[..got]);
}
Ok(out)
}
pub fn read_until_close<R: Read>(r: &mut R, prefix: &[u8]) -> Result<Vec<u8>, HttpError> {
let mut out = prefix.to_vec();
let mut buf = [0u8; 16 * 1024];
loop {
let got = r.read(&mut buf).map_err(|e| HttpError::network(format!("body read failed: {e}")))?;
if got == 0 {
return Ok(out);
}
if out.len() + got > MAX_BODY {
return Err(HttpError::refused(format!("response body exceeds the {MAX_BODY}-byte limit")));
}
out.extend_from_slice(&buf[..got]);
}
}
pub fn read_chunked<R: Read>(r: &mut R, prefix: &[u8]) -> Result<Vec<u8>, HttpError> {
let mut src = Prefixed::new(r, prefix);
let mut out: Vec<u8> = Vec::new();
loop {
let line = src.read_line()?;
let size_text = line.split(';').next().unwrap_or("").trim();
if size_text.is_empty() {
return Err(HttpError::protocol("malformed chunked body: empty chunk size"));
}
let size = u64::from_str_radix(size_text, 16)
.map_err(|_| HttpError::protocol(format!("malformed chunk size: {size_text:?}")))?;
if size > MAX_CHUNK {
return Err(HttpError::refused(format!("chunk of {size} bytes exceeds the {MAX_CHUNK}-byte limit")));
}
if size == 0 {
loop {
let t = src.read_line()?;
if t.is_empty() {
break;
}
}
return Ok(out);
}
if out.len() as u64 + size > MAX_BODY as u64 {
return Err(HttpError::refused(format!("response body exceeds the {MAX_BODY}-byte limit")));
}
let start = out.len();
out.resize(start + size as usize, 0);
src.read_exact_into(&mut out[start..])?;
let crlf = src.read_line()?;
if !crlf.is_empty() {
return Err(HttpError::protocol("malformed chunked body: missing CRLF after chunk data"));
}
}
}
#[cfg_attr(feature = "net-h2", allow(dead_code))]
fn stream_body<R: Read>(r: &mut R, framing: Framing, prefix: &[u8], sink: &mut dyn FnMut(&[u8])) -> Result<(), HttpError> {
let mut buf = [0u8; 16 * 1024];
let read = |r: &mut R, buf: &mut [u8]| r.read(buf).map_err(|e| HttpError::network(format!("body read failed: {e}")));
match framing {
Framing::None => Ok(()),
Framing::Length(n) => {
let first = &prefix[..prefix.len().min(n)];
if !first.is_empty() {
sink(first);
}
let mut left = n - first.len();
while left > 0 {
let want = left.min(buf.len());
let got = read(r, &mut buf[..want])?;
if got == 0 {
return Err(HttpError::protocol(format!("connection closed after {} of {n} declared body bytes", n - left)));
}
sink(&buf[..got]);
left -= got;
}
Ok(())
}
Framing::UntilClose => {
if !prefix.is_empty() {
sink(prefix);
}
loop {
let got = read(r, &mut buf)?;
if got == 0 {
return Ok(());
}
sink(&buf[..got]);
}
}
Framing::Chunked => {
let mut src = Prefixed::new(r, prefix);
loop {
let line = src.read_line()?;
let size_text = line.split(';').next().unwrap_or("").trim();
let mut size = u64::from_str_radix(size_text, 16)
.map_err(|_| HttpError::protocol(format!("malformed chunk size: {size_text:?}")))?;
if size == 0 {
while !src.read_line()?.is_empty() {}
return Ok(());
}
while size > 0 {
let take = size.min(buf.len() as u64) as usize;
src.read_exact_into(&mut buf[..take])?;
sink(&buf[..take]);
size -= take as u64;
}
if !src.read_line()?.is_empty() {
return Err(HttpError::protocol("malformed chunked body: missing CRLF after chunk data"));
}
}
}
}
}
struct Prefixed<'a, R: Read> {
inner: &'a mut R,
buf: Vec<u8>,
pos: usize,
}
impl<'a, R: Read> Prefixed<'a, R> {
fn new(inner: &'a mut R, prefix: &[u8]) -> Self {
Prefixed { inner, buf: prefix.to_vec(), pos: 0 }
}
fn fill(&mut self) -> Result<bool, HttpError> {
let mut tmp = [0u8; 16 * 1024];
let got = self.inner.read(&mut tmp).map_err(|e| HttpError::network(format!("body read failed: {e}")))?;
if got == 0 {
return Ok(false);
}
if self.pos > 0 {
self.buf.drain(..self.pos);
self.pos = 0;
}
self.buf.extend_from_slice(&tmp[..got]);
Ok(true)
}
fn read_line(&mut self) -> Result<String, HttpError> {
loop {
if let Some(i) = self.buf[self.pos..].windows(2).position(|w| w == b"\r\n") {
let line = String::from_utf8_lossy(&self.buf[self.pos..self.pos + i]).into_owned();
self.pos += i + 2;
return Ok(line);
}
if self.buf.len() - self.pos > MAX_HEAD {
return Err(HttpError::protocol("malformed chunked body: oversized line"));
}
if !self.fill()? {
return Err(HttpError::protocol("connection closed inside a chunked body"));
}
}
}
fn read_exact_into(&mut self, dst: &mut [u8]) -> Result<(), HttpError> {
let mut at = 0;
while at < dst.len() {
let avail = self.buf.len() - self.pos;
if avail == 0 {
if !self.fill()? {
return Err(HttpError::protocol("connection closed inside a chunked body"));
}
continue;
}
let take = avail.min(dst.len() - at);
dst[at..at + take].copy_from_slice(&self.buf[self.pos..self.pos + take]);
self.pos += take;
at += take;
}
Ok(())
}
}
pub fn decode_content_encoding(enc: Option<&str>, bytes: Vec<u8>) -> Result<Vec<u8>, HttpError> {
let Some(enc) = enc else { return Ok(bytes) };
let enc = enc.trim();
if enc.is_empty() || enc.eq_ignore_ascii_case("identity") {
return Ok(bytes);
}
if enc.contains(',') {
return Err(HttpError::refused(format!("stacked Content-Encoding is not supported: {enc:?}")));
}
if bytes.is_empty() {
return Ok(bytes);
}
let mut out = Vec::new();
let r = if enc.eq_ignore_ascii_case("gzip") || enc.eq_ignore_ascii_case("x-gzip") {
flate2::read::GzDecoder::new(&bytes[..]).read_to_end(&mut out)
} else if enc.eq_ignore_ascii_case("deflate") {
match flate2::read::ZlibDecoder::new(&bytes[..]).read_to_end(&mut out) {
Ok(n) => Ok(n),
Err(_) => {
out.clear();
flate2::read::DeflateDecoder::new(&bytes[..]).read_to_end(&mut out)
}
}
} else {
return Err(HttpError::refused(format!("unsupported Content-Encoding: {enc:?}")));
};
r.map_err(|e| HttpError::protocol(format!("Content-Encoding {enc} failed to decode: {e}")))?;
Ok(out)
}
pub fn is_redirect(status: u16) -> bool {
matches!(status, 301 | 302 | 303 | 307 | 308)
}
pub fn redirect_method(status: u16, method: &str) -> (String, bool) {
let upper = method.to_ascii_uppercase();
match status {
303 if upper != "HEAD" => ("GET".into(), false),
301 | 302 if upper == "POST" => ("GET".into(), false),
_ => (upper, true),
}
}
pub fn resolve_location(base: &str, location: &str) -> Result<String, HttpError> {
let base = url::Url::parse(base).map_err(|e| HttpError::protocol(format!("bad base URL: {e}")))?;
base.join(location.trim())
.map(|u| u.to_string())
.map_err(|e| HttpError::protocol(format!("bad Location header {location:?}: {e}")))
}
pub fn same_origin(a: &str, b: &str) -> bool {
let (Ok(a), Ok(b)) = (url::Url::parse(a), url::Url::parse(b)) else { return false };
a.scheme() == b.scheme() && a.host_str() == b.host_str() && a.port_or_known_default() == b.port_or_known_default()
}
fn strip_sensitive(headers: &mut Vec<(String, String)>) {
headers.retain(|(k, _)| {
!matches!(k.to_ascii_lowercase().as_str(), "authorization" | "cookie" | "proxy-authorization" | "www-authenticate")
});
}
pub(crate) fn is_transport_header(lower: &str) -> bool {
matches!(
lower,
"host"
| "content-length"
| "transfer-encoding"
| "connection"
| "keep-alive"
| "proxy-connection"
| "te"
| "trailer"
| "upgrade"
| "http2-settings"
)
}
pub(crate) struct Hops<'a> {
req: &'a Request,
pub(crate) current: String,
pub(crate) method: String,
pub(crate) body: Option<Vec<u8>>,
headers: Vec<(String, String)>,
pub(crate) redirected: bool,
left: u32,
}
pub(crate) struct Hop {
pub(crate) url: url::Url,
pub(crate) headers: Vec<(String, String)>,
cookies: bool,
}
impl<'a> Hops<'a> {
pub(crate) fn new(req: &'a Request) -> Self {
let mut headers = req.headers.clone();
cache::add_mode_headers(req.cache, &mut headers);
Hops {
req,
current: req.url.clone(),
method: req.method.to_ascii_uppercase(),
body: req.body.clone(),
headers,
redirected: false,
left: MAX_REDIRECTS + 1,
}
}
pub(crate) fn next(&mut self) -> Result<Hop, HttpError> {
if self.left == 0 {
return Err(HttpError::network(format!("too many redirects (more than {MAX_REDIRECTS})")));
}
self.left -= 1;
let u = url::Url::parse(&self.current)
.map_err(|e| HttpError::refused(format!("invalid URL {:?}: {e}", self.current)))?;
if u.scheme() != "http" && u.scheme() != "https" {
return Err(HttpError::refused(format!("scheme {:?} is not supported by fetch", u.scheme())));
}
let cookies = self.req.credentials.applies(&u);
let headers = if cookies { super::cookies::with_cookies(&u, &self.headers) } else { None }
.unwrap_or_else(|| self.headers.clone());
Ok(Hop { url: u, headers, cookies })
}
pub(crate) fn after(&mut self, hop: &Hop, status: u16, headers: &[(String, String)]) -> Result<bool, HttpError> {
self.store_cookies(hop, headers);
self.follow(status, headers)
}
pub(crate) fn store_cookies(&self, hop: &Hop, headers: &[(String, String)]) {
if hop.cookies {
super::cookies::store_response(&hop.url, headers);
}
}
pub(crate) fn follow(&mut self, status: u16, headers: &[(String, String)]) -> Result<bool, HttpError> {
if !is_redirect(status) {
return Ok(false);
}
let Some(loc) = header(headers, "location") else { return Ok(false) };
if self.req.redirect == Redirect::Error {
return Err(HttpError::network(format!(
"redirect to {loc:?} refused: the request set redirect: \"error\""
)));
}
let next = resolve_location(&self.current, loc)?;
let (next_method, keep_body) = redirect_method(status, &self.method);
if !same_origin(&self.current, &next) {
strip_sensitive(&mut self.headers);
}
if !keep_body {
self.body = None;
self.headers.retain(|(k, _)| !k.eq_ignore_ascii_case("content-type"));
}
self.method = next_method;
self.current = next;
self.redirected = true;
Ok(true)
}
}
pub type Done = Box<dyn FnOnce(Result<Response, HttpError>) + Send + 'static>;
pub fn start(req: Request, cancel: Cancel, done: Done) -> Result<(), HttpError> {
#[cfg(feature = "net-h2")]
{
pool::start(req, cancel, done)
}
#[cfg(not(feature = "net-h2"))]
{
std::thread::Builder::new()
.name("redjs-fetch".into())
.spawn(move || done(send_blocking(&req, &cancel)))
.map(|_| ())
.map_err(|e| HttpError::network(format!("could not start a request thread: {e}")))
}
}
pub fn send(req: &Request, cancel: &Cancel) -> Result<Response, HttpError> {
#[cfg(feature = "net-h2")]
{
pool::send(req, cancel)
}
#[cfg(not(feature = "net-h2"))]
{
send_blocking(req, cancel)
}
}
pub(crate) enum Stream {
Plain(Polled),
Tls(Box<native_tls::TlsStream<Polled>>),
}
impl Stream {
pub(crate) fn tcp(&self) -> &TcpStream {
match self {
Stream::Plain(s) => s.tcp(),
Stream::Tls(s) => s.get_ref().tcp(),
}
}
pub(crate) fn set_wait(&mut self, wait: Wait) {
match self {
Stream::Plain(s) => s.set_wait(wait),
Stream::Tls(s) => s.get_mut().set_wait(wait),
}
}
}
impl Read for Stream {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
match self {
Stream::Plain(s) => s.read(buf),
Stream::Tls(s) => s.read(buf),
}
}
}
impl Write for Stream {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
match self {
Stream::Plain(s) => s.write(buf),
Stream::Tls(s) => s.write(buf),
}
}
fn flush(&mut self) -> std::io::Result<()> {
match self {
Stream::Plain(s) => s.flush(),
Stream::Tls(s) => s.flush(),
}
}
}
pub(crate) fn dial_host(u: &url::Url) -> Result<String, HttpError> {
match u.host() {
Some(url::Host::Domain(d)) => Ok(d.to_string()),
Some(url::Host::Ipv4(a)) => Ok(a.to_string()),
Some(url::Host::Ipv6(a)) => Ok(a.to_string()),
None => Err(HttpError::refused(format!("URL has no host: {u}"))),
}
}
pub(crate) fn interleave(addrs: Vec<std::net::SocketAddr>) -> Vec<std::net::SocketAddr> {
let Some(first) = addrs.first().copied() else { return addrs };
let (mut a, mut b): (Vec<_>, Vec<_>) = addrs.into_iter().partition(|x| x.is_ipv6() == first.is_ipv6());
a.reverse();
b.reverse();
let mut out = Vec::with_capacity(a.len() + b.len());
loop {
match (a.pop(), b.pop()) {
(None, None) => return out,
(x, y) => {
out.extend(x);
out.extend(y);
}
}
}
}
pub(crate) fn tls_builder() -> native_tls::TlsConnectorBuilder {
#[allow(unused_mut)]
let mut b = native_tls::TlsConnector::builder();
#[cfg(test)]
b.add_root_certificate(native_tls::Certificate::from_pem(TEST_ROOT_PEM).expect("test certificate"));
b
}
pub(crate) fn connect(u: &url::Url, connect_timeout: Duration, cancel: &Cancel) -> Result<Stream, HttpError> {
let host = dial_host(u)?;
let port = u
.port_or_known_default()
.ok_or_else(|| HttpError::refused(format!("URL has no port and no default: {u}")))?;
let secure = u.scheme() == "https" || u.scheme() == "wss";
cancel.check()?;
let addrs: Vec<_> = (host.as_str(), port)
.to_socket_addrs()
.map_err(|e| HttpError::network(format!("could not resolve {host}: {e}")))?
.collect();
if addrs.is_empty() {
return Err(HttpError::network(format!("could not resolve {host}")));
}
let mut last: Option<std::io::Error> = None;
let mut tcp: Option<TcpStream> = None;
for a in interleave(addrs) {
cancel.check()?;
match TcpStream::connect_timeout(&a, connect_timeout) {
Ok(s) => {
tcp = Some(s);
break;
}
Err(e) => last = Some(e),
}
}
let tcp = tcp.ok_or_else(|| {
HttpError::network(match last {
Some(e) => format!("could not connect to {host}:{port}: {e}"),
None => format!("could not connect to {host}:{port}"),
})
})?;
let _ = tcp.set_nodelay(true);
CONNECTS.fetch_add(1, Ordering::SeqCst);
cancel.arm(&tcp)?;
let polled = Polled::new(tcp, cancel.clone(), Wait::Block { budget: connect_timeout })
.map_err(|e| HttpError::network(format!("could not configure the socket: {e}")))?;
if !secure {
return Ok(Stream::Plain(polled));
}
let connector = tls_builder().build().map_err(|e| HttpError::network(format!("TLS setup failed: {e}")))?;
let started = Instant::now();
match connector.connect(&host, polled) {
Ok(s) => Ok(Stream::Tls(Box::new(s))),
Err(_) if cancel.is_aborted() => Err(HttpError::new(ErrorKind::Aborted, "the request was aborted")),
Err(native_tls::HandshakeError::WouldBlock(_)) => {
Err(HttpError::network(format!("TLS handshake with {host} stalled")))
}
Err(native_tls::HandshakeError::Failure(e)) => {
if started.elapsed() >= connect_timeout {
Err(HttpError::network(format!("TLS handshake with {host} timed out")))
} else {
Err(HttpError::network(format!("TLS handshake with {host} failed: {e}")))
}
}
}
}
pub fn serialize_request(
u: &url::Url,
method: &str,
headers: &[(String, String)],
body: Option<&[u8]>,
) -> Result<Vec<u8>, HttpError> {
let mut target = u.path().to_string();
if target.is_empty() {
target.push('/');
}
if let Some(q) = u.query() {
target.push('?');
target.push_str(q);
}
let mut out = Vec::with_capacity(256 + body.map_or(0, |b| b.len()));
out.extend_from_slice(format!("{method} {target} HTTP/1.1\r\n").as_bytes());
let host = u.host_str().unwrap_or("");
let host_header = match u.port() {
Some(p) => format!("{host}:{p}"),
None => host.to_string(),
};
let mut seen: Vec<String> = Vec::new();
for (k, v) in headers {
validate_header(k, v)?;
let lower = k.to_ascii_lowercase();
if is_transport_header(&lower) {
continue;
}
seen.push(lower);
out.extend_from_slice(format!("{k}: {v}\r\n").as_bytes());
}
let has = |n: &str| seen.iter().any(|s| s == n);
out.extend_from_slice(format!("Host: {host_header}\r\n").as_bytes());
if !has("user-agent") {
out.extend_from_slice(format!("User-Agent: {}\r\n", super::navigator::user_agent()).as_bytes());
}
if !has("accept") {
out.extend_from_slice(b"Accept: */*\r\n");
}
if !has("accept-encoding") {
out.extend_from_slice(b"Accept-Encoding: gzip, deflate\r\n");
}
if !has("accept-language") {
out.extend_from_slice(format!("Accept-Language: {}\r\n", super::navigator::accept_language()).as_bytes());
}
if needs_length(method, body) {
out.extend_from_slice(format!("Content-Length: {}\r\n", body.map_or(0, |b| b.len())).as_bytes());
}
out.extend_from_slice(b"\r\n");
if let Some(b) = body {
out.extend_from_slice(b);
}
Ok(out)
}
pub(crate) fn needs_length(method: &str, body: Option<&[u8]>) -> bool {
body.is_some() || matches!(method, "POST" | "PUT" | "PATCH")
}
fn map_io(cancel: &Cancel, e: std::io::Error, what: &str) -> HttpError {
if cancel.is_aborted() {
return HttpError::new(ErrorKind::Aborted, "the request was aborted");
}
match e.kind() {
std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut => {
HttpError::network(format!("{what} timed out"))
}
_ => HttpError::network(format!("{what} failed: {e}")),
}
}
pub(crate) fn read_head<R: Read>(r: &mut R, cancel: &Cancel) -> Result<(Head, Vec<u8>), HttpError> {
let mut acc: Vec<u8> = Vec::with_capacity(2048);
let mut buf = [0u8; 8 * 1024];
loop {
let scan_from = acc.len().saturating_sub(3);
let got = r.read(&mut buf).map_err(|e| map_io(cancel, e, "response read"))?;
if got == 0 {
if acc.is_empty() {
return Err(HttpError::network("the server closed the connection without a response"));
}
return Err(HttpError::protocol("the server closed the connection inside the response head"));
}
acc.extend_from_slice(&buf[..got]);
if let Some(i) = acc[scan_from..].windows(4).position(|w| w == b"\r\n\r\n") {
let end = scan_from + i;
let head = parse_head(&acc[..end])?;
let rest = acc[end + 4..].to_vec();
return Ok((head, rest));
}
if acc.len() > MAX_HEAD {
return Err(HttpError::protocol(format!("response head exceeds {MAX_HEAD} bytes")));
}
}
}
#[allow(clippy::too_many_arguments)]
#[cfg_attr(feature = "net-h2", allow(dead_code))]
fn round_trip(
u: &url::Url,
method: &str,
headers: &[(String, String)],
body: Option<&[u8]>,
req: &Request,
cancel: &Cancel,
url_now: &str,
redirected: bool,
) -> Result<(Head, Vec<u8>, Option<std::net::SocketAddr>), HttpError> {
let mut stream = connect(u, req.connect_timeout, cancel)?;
stream.set_wait(Wait::Block { budget: req.io_timeout });
let remote = stream.tcp().peer_addr().ok();
let wire = serialize_request(u, method, headers, body)?;
let body_len = body.map_or(0, |b| b.len());
let (head_part, body_part) = wire.split_at(wire.len() - body_len);
stream.write_all(head_part).map_err(|e| map_io(cancel, e, "request write"))?;
let mut sent = 0usize;
for piece in body_part.chunks(64 * 1024) {
stream.write_all(piece).map_err(|e| map_io(cancel, e, "request write"))?;
sent += piece.len();
if let Some(o) = &req.observer {
o.0.upload(sent as u64, body_len as u64);
}
}
stream.flush().map_err(|e| map_io(cancel, e, "request write"))?;
let (head, rest) = read_head(&mut stream, cancel)?;
let (head, rest) = if (100..200).contains(&head.status) {
read_head_after_informational(&mut stream, rest, cancel)?
} else {
(head, rest)
};
let observed = req.observer.as_ref().filter(|_| is_final(head.status, &head.headers));
if let Some(o) = observed {
o.0.head(&ProgressHead {
status: head.status,
status_text: &head.reason,
headers: &head.headers,
url: url_now,
redirected,
protocol: PROTO_H1,
});
}
let framing = framing_of(method, head.status, &head.headers)?;
if let Some(o) = observed.filter(|o| o.0.streams()) {
let mut received = 0u64;
let mut sink = |piece: &[u8]| {
received += piece.len() as u64;
o.0.body(piece, received, None);
};
stream_body(&mut stream, framing, &rest, &mut sink).map_err(|e| reclassify(e, cancel))?;
return Ok((head, Vec::new(), remote));
}
let raw = match framing {
Framing::None => Vec::new(),
Framing::Length(n) => read_exact_body(&mut stream, n, &rest).map_err(|e| reclassify(e, cancel))?,
Framing::Chunked => read_chunked(&mut stream, &rest).map_err(|e| reclassify(e, cancel))?,
Framing::UntilClose => read_until_close(&mut stream, &rest).map_err(|e| reclassify(e, cancel))?,
};
if let Some(o) = observed {
if !raw.is_empty() {
let declared = match framing {
Framing::Length(n) => Some(n as u64),
_ => None,
};
o.0.body(&raw, raw.len() as u64, declared);
}
}
let decoded = decode_content_encoding(header(&head.headers, "content-encoding"), raw)?;
Ok((head, decoded, remote))
}
#[cfg_attr(feature = "net-h2", allow(dead_code))]
fn read_head_after_informational<R: Read>(
r: &mut R,
rest: Vec<u8>,
cancel: &Cancel,
) -> Result<(Head, Vec<u8>), HttpError> {
let mut acc = rest;
let mut buf = [0u8; 8 * 1024];
loop {
if let Some(i) = acc.windows(4).position(|w| w == b"\r\n\r\n") {
let head = parse_head(&acc[..i])?;
let tail = acc[i + 4..].to_vec();
if (100..200).contains(&head.status) {
acc = tail;
continue;
}
return Ok((head, tail));
}
let got = r.read(&mut buf).map_err(|e| map_io(cancel, e, "response read"))?;
if got == 0 {
return Err(HttpError::protocol("the server closed the connection after an informational response"));
}
acc.extend_from_slice(&buf[..got]);
if acc.len() > MAX_HEAD {
return Err(HttpError::protocol(format!("response head exceeds {MAX_HEAD} bytes")));
}
}
}
#[cfg_attr(feature = "net-h2", allow(dead_code))]
fn reclassify(e: HttpError, cancel: &Cancel) -> HttpError {
if cancel.is_aborted() {
return HttpError::new(ErrorKind::Aborted, "the request was aborted");
}
e
}
#[cfg_attr(feature = "net-h2", allow(dead_code))]
pub(crate) fn send_blocking(req: &Request, cancel: &Cancel) -> Result<Response, HttpError> {
use cache::{Action, Outcome, Settled};
let mut hops = Hops::new(req);
loop {
cancel.check()?;
let hop = hops.next()?;
let (action, ctx) = cache::plan(&hops.method, hop.url.as_str(), &hop.headers, req.cache).into_parts();
let (validating, conditional) = match action {
Action::Fail(e) => return Err(e),
Action::Serve { entry, revalidate } => {
if revalidate {
cache::revalidate_in_background(req, hop.url.as_str());
}
if hops.follow(entry.status, &entry.headers)? {
continue;
}
return blocking_from_cache(req, &hops, entry, Outcome::Hit, ("", 0, None));
}
Action::Validate { entry, conditional } => (Some(entry), conditional),
Action::Network => (None, Vec::new()),
};
let asked = validating.is_some();
let quiet;
let (sent, headers) = if asked {
quiet = Request { observer: None, ..req.clone() };
let mut h = hop.headers.clone();
h.extend(conditional);
(&quiet, std::borrow::Cow::Owned(h))
} else {
(req, std::borrow::Cow::Borrowed(&hop.headers[..]))
};
let result = round_trip(
&hop.url,
&hops.method,
&headers,
hops.body.as_deref(),
sent,
cancel,
&hops.current,
hops.redirected,
);
cancel.disarm();
let (head, bytes, remote) = result?;
let id = next_connection_id();
match cache::settle(ctx, validating, head.status, &head.reason, &head.headers, &bytes, false) {
Settled::Refreshed(entry) => {
hops.store_cookies(&hop, &head.headers);
if hops.follow(entry.status, &entry.headers)? {
continue;
}
return blocking_from_cache(req, &hops, entry, Outcome::Revalidated, (PROTO_H1, id, remote));
}
Settled::Network(outcome) => {
if hops.after(&hop, head.status, &head.headers)? {
continue;
}
if asked {
let (url, r) = (&hops.current, hops.redirected);
cache::observe(req, url, r, PROTO_H1, head.status, &head.reason, &head.headers, &bytes);
}
return Ok(Response {
status: head.status,
status_text: head.reason,
headers: head.headers,
url: hops.current,
redirected: hops.redirected,
body: bytes,
protocol: PROTO_H1,
reused: false,
connection_id: id,
remote_addr: remote,
cache: outcome,
});
}
}
}
}
#[cfg_attr(feature = "net-h2", allow(dead_code))]
fn blocking_from_cache(
req: &Request,
hops: &Hops<'_>,
entry: cache::Entry,
outcome: cache::Outcome,
net: (&'static str, u64, Option<std::net::SocketAddr>),
) -> Result<Response, HttpError> {
let (url, redirected) = (hops.current.clone(), hops.redirected);
cache::observe(req, &url, redirected, net.0, entry.status, &entry.reason, &entry.headers, &entry.body);
let body = if entry.coded {
decode_content_encoding(header(&entry.headers, "content-encoding"), entry.body)?
} else {
entry.body
};
Ok(Response {
status: entry.status,
status_text: entry.reason,
headers: entry.headers,
url,
redirected,
body,
protocol: net.0,
reused: false,
connection_id: net.1,
remote_addr: net.2,
cache: outcome,
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_normal_head_parses() {
let head = parse_head(b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 2").unwrap();
assert_eq!(head.status, 200);
assert_eq!(head.reason, "OK");
assert_eq!(header(&head.headers, "CONTENT-TYPE"), Some("application/json"));
}
#[test]
fn a_reasonless_status_line_still_parses() {
let head = parse_head(b"HTTP/1.1 204 \r\n").unwrap();
assert_eq!(head.status, 204);
assert_eq!(head.reason, "");
}
#[test]
fn a_non_http_peer_is_a_protocol_error_not_a_default_status() {
for junk in [
&b"<html>hello</html>"[..],
&b"\x16\x03\x01\x02\x00\x01"[..], &b"HTTP/1.1 2O0 OK\r\n"[..], &b"HTTP/9.9 200 OK\r\n"[..],
] {
assert!(parse_head(junk).is_err(), "accepted junk as a response head: {junk:?}");
}
}
#[test]
fn a_malformed_header_line_is_refused() {
assert!(parse_head(b"HTTP/1.1 200 OK\r\nthis-is-not-a-header\r\n").is_err());
assert!(parse_head(b"HTTP/1.1 200 OK\r\nbad header: x\r\n").is_err());
}
#[test]
fn folded_headers_join_onto_the_previous_value() {
let head = parse_head(b"HTTP/1.1 200 OK\r\nX-Long: a\r\n b\r\n").unwrap();
assert_eq!(header(&head.headers, "x-long"), Some("a b"));
}
#[test]
fn framing_follows_the_spec_order() {
let chunked = vec![("transfer-encoding".into(), "chunked".into()), ("content-length".into(), "5".into())];
assert_eq!(framing_of("GET", 200, &chunked).unwrap(), Framing::Chunked);
let len = vec![("content-length".into(), "5".into())];
assert_eq!(framing_of("GET", 200, &len).unwrap(), Framing::Length(5));
assert_eq!(framing_of("GET", 204, &len).unwrap(), Framing::None);
assert_eq!(framing_of("GET", 304, &len).unwrap(), Framing::None);
assert_eq!(framing_of("HEAD", 200, &len).unwrap(), Framing::None);
assert_eq!(framing_of("GET", 200, &[]).unwrap(), Framing::UntilClose);
}
#[test]
fn conflicting_content_lengths_are_refused() {
let bad = vec![("content-length".into(), "5, 9".into())];
assert!(framing_of("GET", 200, &bad).is_err());
let same = vec![("content-length".into(), "5, 5".into())];
assert_eq!(framing_of("GET", 200, &same).unwrap(), Framing::Length(5));
}
#[test]
fn an_unknown_transfer_encoding_is_refused() {
let bad = vec![("transfer-encoding".into(), "gzip".into())];
assert!(framing_of("GET", 200, &bad).is_err());
}
#[test]
fn chunked_decodes_across_reads() {
let wire = b"4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n";
let mut r = &wire[..];
assert_eq!(read_chunked(&mut r, &[]).unwrap(), b"Wikipedia in\r\n\r\nchunks.");
}
#[test]
fn chunked_honours_the_prefix_already_read_off_the_socket() {
let all = b"3\r\nabc\r\n3\r\ndef\r\n0\r\n\r\n";
let (prefix, rest) = all.split_at(7);
let mut r = rest;
assert_eq!(read_chunked(&mut r, prefix).unwrap(), b"abcdef");
}
#[test]
fn a_truncated_chunked_body_is_an_error_not_a_short_body() {
let mut r = &b"3\r\nabc\r\n3\r\nde"[..];
let e = read_chunked(&mut r, &[]).unwrap_err();
assert_eq!(e.kind, ErrorKind::Protocol, "{}", e.message);
}
#[test]
fn chunk_extensions_are_tolerated() {
let mut r = &b"3;name=value\r\nabc\r\n0\r\n\r\n"[..];
assert_eq!(read_chunked(&mut r, &[]).unwrap(), b"abc");
}
#[test]
fn a_chunked_trailer_is_consumed() {
let mut r = &b"3\r\nabc\r\n0\r\nX-Checksum: 1\r\n\r\n"[..];
assert_eq!(read_chunked(&mut r, &[]).unwrap(), b"abc");
}
#[test]
fn a_short_content_length_body_is_an_error() {
let mut r = &b"abc"[..];
let e = read_exact_body(&mut r, 10, &[]).unwrap_err();
assert_eq!(e.kind, ErrorKind::Protocol, "{}", e.message);
}
#[test]
fn a_content_length_body_can_come_entirely_from_the_prefix() {
let mut r = &b""[..];
assert_eq!(read_exact_body(&mut r, 3, b"abcdef").unwrap(), b"abc");
}
#[test]
fn gzip_round_trips_through_the_decoder() {
use flate2::write::GzEncoder;
let mut e = GzEncoder::new(Vec::new(), flate2::Compression::default());
e.write_all(b"hello over the wire").unwrap();
let gz = e.finish().unwrap();
assert_eq!(decode_content_encoding(Some("gzip"), gz).unwrap(), b"hello over the wire");
}
#[test]
fn an_unadvertised_content_encoding_is_refused_by_name() {
let e = decode_content_encoding(Some("br"), vec![1, 2, 3]).unwrap_err();
assert_eq!(e.kind, ErrorKind::Refused);
assert!(e.message.contains("br"), "{}", e.message);
}
#[test]
fn identity_and_absent_encodings_pass_through() {
assert_eq!(decode_content_encoding(None, vec![1, 2]).unwrap(), vec![1, 2]);
assert_eq!(decode_content_encoding(Some("identity"), vec![1, 2]).unwrap(), vec![1, 2]);
}
#[test]
fn redirect_method_rewrites_match_the_fetch_spec() {
assert_eq!(redirect_method(303, "POST"), ("GET".to_string(), false));
assert_eq!(redirect_method(303, "HEAD"), ("HEAD".to_string(), true));
assert_eq!(redirect_method(302, "POST"), ("GET".to_string(), false));
assert_eq!(redirect_method(301, "POST"), ("GET".to_string(), false));
assert_eq!(redirect_method(307, "POST"), ("POST".to_string(), true));
assert_eq!(redirect_method(308, "PUT"), ("PUT".to_string(), true));
assert_eq!(redirect_method(302, "GET"), ("GET".to_string(), true));
}
#[test]
fn location_resolves_relative_absolute_and_scheme_relative() {
let base = "https://api.example.com/v1/thing?x=1";
assert_eq!(resolve_location(base, "/v2/other").unwrap(), "https://api.example.com/v2/other");
assert_eq!(resolve_location(base, "other").unwrap(), "https://api.example.com/v1/other");
assert_eq!(resolve_location(base, "https://cdn.example.net/a").unwrap(), "https://cdn.example.net/a");
assert_eq!(resolve_location(base, "//cdn.example.net/a").unwrap(), "https://cdn.example.net/a");
}
#[test]
fn origin_comparison_accounts_for_default_ports() {
assert!(same_origin("https://a.example/x", "https://a.example:443/y"));
assert!(!same_origin("https://a.example/x", "http://a.example/x"));
assert!(!same_origin("https://a.example/x", "https://b.example/x"));
}
#[test]
fn credentials_are_dropped_on_a_cross_origin_redirect() {
let mut h = vec![
("authorization".to_string(), "Bearer secret".to_string()),
("cookie".to_string(), "sid=1".to_string()),
("accept".to_string(), "*/*".to_string()),
];
strip_sensitive(&mut h);
assert_eq!(h, vec![("accept".to_string(), "*/*".to_string())]);
}
#[test]
fn a_header_cannot_inject_a_request_line() {
let inject = vec![("X-Evil".to_string(), "a\r\nX-Injected: 1".to_string())];
let u = url::Url::parse("http://example.com/").unwrap();
let e = serialize_request(&u, "GET", &inject, None).unwrap_err();
assert_eq!(e.kind, ErrorKind::Refused, "{}", e.message);
let bad_name = vec![("X Evil".to_string(), "1".to_string())];
assert!(serialize_request(&u, "GET", &bad_name, None).is_err());
}
#[test]
fn the_request_line_carries_path_and_query_and_a_host_header() {
let u = url::Url::parse("http://example.com:8080/a/b?q=1&r=2").unwrap();
let wire = serialize_request(&u, "GET", &[], None).unwrap();
let text = String::from_utf8(wire).unwrap();
assert!(text.starts_with("GET /a/b?q=1&r=2 HTTP/1.1\r\n"), "{text}");
assert!(text.contains("Host: example.com:8080\r\n"), "{text}");
assert!(!text.to_ascii_lowercase().contains("connection:"), "{text}");
assert!(text.ends_with("\r\n\r\n"), "{text}");
assert!(!text.contains("Content-Length"), "{text}");
}
#[test]
fn a_body_bearing_method_always_declares_its_length() {
let u = url::Url::parse("https://example.com/submit").unwrap();
let wire = serialize_request(&u, "POST", &[], Some(b"{\"a\":1}")).unwrap();
let text = String::from_utf8(wire).unwrap();
assert!(text.contains("Content-Length: 7\r\n"), "{text}");
assert!(text.ends_with("\r\n\r\n{\"a\":1}"), "{text}");
let empty = String::from_utf8(serialize_request(&u, "POST", &[], None).unwrap()).unwrap();
assert!(empty.contains("Content-Length: 0\r\n"), "{empty}");
}
#[test]
fn caller_framing_headers_are_ignored_rather_than_duplicated() {
let u = url::Url::parse("http://example.com/").unwrap();
let h = vec![
("Content-Length".to_string(), "999".to_string()),
("Transfer-Encoding".to_string(), "chunked".to_string()),
("Host".to_string(), "evil.example".to_string()),
];
let text = String::from_utf8(serialize_request(&u, "POST", &h, Some(b"ab")).unwrap()).unwrap();
assert_eq!(text.matches("Content-Length").count(), 1, "{text}");
assert!(text.contains("Content-Length: 2\r\n"), "{text}");
assert!(!text.contains("chunked"), "{text}");
assert!(text.contains("Host: example.com\r\n"), "{text}");
assert!(!text.contains("evil.example"), "{text}");
}
fn req(url: &str) -> Request {
Request {
method: "GET".into(),
url: url.into(),
headers: vec![],
body: None,
redirect: Redirect::Follow,
credentials: crate::webapi::cookies::Credentials::Omit,
connect_timeout: Duration::from_secs(5),
io_timeout: Duration::from_secs(10),
observer: None,
cache: cache::CacheMode::Default,
}
}
fn cookie_line(request: &[u8]) -> Option<String> {
String::from_utf8_lossy(request)
.lines()
.find_map(|l| l.strip_prefix("cookie: ").or_else(|| l.strip_prefix("Cookie: ")).map(str::to_string))
}
#[test]
fn include_stores_a_redirects_set_cookie_and_sends_it_on_the_next_hop() {
use crate::webapi::cookies;
let _g = cookies::test_lock();
cookies::reset_for_tests();
let (port, seen) = serve(vec![
b"HTTP/1.1 302 Found\r\nSet-Cookie: hop=1; Path=/\r\nLocation: /final\r\nContent-Length: 0\r\n\r\n".to_vec(),
b"HTTP/1.1 200 OK\r\nSet-Cookie: done=2; Path=/\r\nContent-Length: 2\r\n\r\nok".to_vec(),
]);
let mut r = req(&format!("http://127.0.0.1:{port}/start"));
r.credentials = cookies::Credentials::Include;
let resp = send(&r, &Cancel::default()).unwrap();
assert_eq!(resp.status, 200);
assert_eq!(cookie_line(&seen.recv().unwrap()), None, "nothing in the jar for the first hop");
assert_eq!(cookie_line(&seen.recv().unwrap()).as_deref(), Some("hop=1"), "the 302's cookie rode the next hop");
let jar = cookies::header_for(&url::Url::parse(&format!("http://127.0.0.1:{port}/")).unwrap(), cookies::Api::Http);
assert_eq!(jar.as_deref(), Some("hop=1; done=2"));
cookies::reset_for_tests();
}
#[test]
fn a_hop_to_another_host_carries_only_that_hosts_cookies() {
use crate::webapi::cookies;
let _g = cookies::test_lock();
cookies::reset_for_tests();
let (b, seen_b) = serve(vec![b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n".to_vec()]);
let (a, seen_a) = serve(vec![format!(
"HTTP/1.1 302 Found\r\nLocation: http://localhost:{b}/there\r\nContent-Length: 0\r\n\r\n"
)
.into_bytes()]);
cookies::set_from_header(&url::Url::parse("http://127.0.0.1/").unwrap(), "only-ip=1", cookies::Api::Http).unwrap();
cookies::set_from_header(&url::Url::parse("http://localhost/").unwrap(), "only-lh=1", cookies::Api::Http).unwrap();
let mut r = req(&format!("http://127.0.0.1:{a}/start"));
r.credentials = cookies::Credentials::Include;
r.headers.push(("cookie".into(), "script=1".into()));
send(&r, &Cancel::default()).unwrap();
assert_eq!(cookie_line(&seen_a.recv().unwrap()).as_deref(), Some("script=1; only-ip=1"));
assert_eq!(cookie_line(&seen_b.recv().unwrap()).as_deref(), Some("only-lh=1"));
cookies::reset_for_tests();
}
#[test]
fn omit_and_same_origin_neither_send_nor_store() {
use crate::webapi::cookies;
let _g = cookies::test_lock();
cookies::reset_for_tests();
cookies::set_from_header(&url::Url::parse("http://127.0.0.1/").unwrap(), "pre=1", cookies::Api::Http).unwrap();
for mode in [cookies::Credentials::Omit, cookies::Credentials::SameOrigin] {
let (port, seen) = serve(vec![b"HTTP/1.1 200 OK\r\nSet-Cookie: nope=1\r\nContent-Length: 0\r\n\r\n".to_vec()]);
let mut r = req(&format!("http://127.0.0.1:{port}/"));
r.credentials = mode;
send(&r, &Cancel::default()).unwrap();
assert_eq!(cookie_line(&seen.recv().unwrap()), None, "{mode:?} sent the jar");
}
let jar = cookies::header_for(&url::Url::parse("http://127.0.0.1/").unwrap(), cookies::Api::Http);
assert_eq!(jar.as_deref(), Some("pre=1"), "neither mode stored the response's cookie");
cookies::reset_for_tests();
}
#[test]
fn a_non_http_scheme_is_refused_without_touching_the_network() {
let e = send(&req("ftp://example.com/file"), &Cancel::default()).unwrap_err();
assert_eq!(e.kind, ErrorKind::Refused, "{}", e.message);
assert!(e.message.contains("ftp"), "{}", e.message);
}
#[test]
fn a_pre_aborted_request_never_opens_a_socket() {
let cancel = Cancel::default();
cancel.abort();
let mut r = req("http://192.0.2.1:81/never");
r.connect_timeout = Duration::from_secs(30);
let started = std::time::Instant::now();
let e = send(&r, &cancel).unwrap_err();
assert_eq!(e.kind, ErrorKind::Aborted, "{}", e.message);
assert!(started.elapsed() < Duration::from_secs(2), "an aborted request still dialed out");
}
#[test]
fn a_refused_connection_is_a_network_error_not_an_empty_response() {
let e = send(&req("http://127.0.0.1:1/nope"), &Cancel::default()).unwrap_err();
assert_eq!(e.kind, ErrorKind::Network, "{}", e.message);
}
fn serve(replies: Vec<Vec<u8>>) -> (u16, std::sync::mpsc::Receiver<Vec<u8>>) {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
for reply in replies {
let Ok((mut sock, _)) = listener.accept() else { return };
let _ = sock.set_read_timeout(Some(Duration::from_secs(5)));
let mut got = Vec::new();
let mut buf = [0u8; 4096];
loop {
match sock.read(&mut buf) {
Ok(0) => break,
Ok(n) => {
got.extend_from_slice(&buf[..n]);
if let Some(i) = got.windows(4).position(|w| w == b"\r\n\r\n") {
let len = String::from_utf8_lossy(&got[..i])
.lines()
.find_map(|l| {
l.to_ascii_lowercase()
.strip_prefix("content-length:")
.map(|v| v.trim().parse::<usize>().unwrap_or(0))
})
.unwrap_or(0);
if got.len() >= i + 4 + len {
break;
}
}
}
Err(_) => break,
}
}
let _ = tx.send(got);
let _ = sock.write_all(&reply);
let _ = sock.flush();
let _ = sock.shutdown(Shutdown::Write);
}
});
(port, rx)
}
#[test]
fn a_real_request_over_a_real_socket_succeeds() {
let (port, seen) = serve(vec![
b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 17\r\n\r\n{\"instance\":\"ok\"}".to_vec(),
]);
let r = send(&req(&format!("http://127.0.0.1:{port}/resolve")), &Cancel::default()).unwrap();
assert_eq!(r.status, 200);
assert_eq!(r.status_text, "OK");
assert_eq!(header(&r.headers, "content-type"), Some("application/json"));
assert_eq!(String::from_utf8(r.body).unwrap(), "{\"instance\":\"ok\"}");
assert!(!r.redirected);
let request = String::from_utf8(seen.recv().unwrap()).unwrap();
assert!(request.starts_with("GET /resolve HTTP/1.1\r\n"), "{request}");
assert!(request.contains(&format!("Host: 127.0.0.1:{port}\r\n")), "{request}");
}
#[test]
fn a_post_body_and_headers_actually_reach_the_server() {
let (port, seen) = serve(vec![b"HTTP/1.1 204 No Content\r\n\r\n".to_vec()]);
let mut r = req(&format!("http://127.0.0.1:{port}/api/2/envelope/"));
r.method = "POST".into();
r.headers = vec![
("Content-Type".into(), "application/x-sentry-envelope".into()),
("Authorization".into(), "Bearer s3cret".into()),
];
r.body = Some(b"{\"event_id\":\"abc\"}".to_vec());
let resp = send(&r, &Cancel::default()).unwrap();
assert_eq!(resp.status, 204);
assert!(resp.body.is_empty(), "204 must have no body");
let request = String::from_utf8(seen.recv().unwrap()).unwrap();
assert!(request.starts_with("POST /api/2/envelope/ HTTP/1.1\r\n"), "{request}");
assert!(request.contains("Authorization: Bearer s3cret\r\n"), "{request}");
assert!(request.contains("Content-Type: application/x-sentry-envelope\r\n"), "{request}");
assert!(request.contains("Content-Length: 18\r\n"), "{request}");
assert!(request.ends_with("{\"event_id\":\"abc\"}"), "{request}");
}
#[derive(Default)]
struct Seen(std::sync::Mutex<Vec<String>>);
impl Observe for Seen {
fn upload(&self, sent: u64, total: u64) {
self.0.lock().unwrap().push(format!("up {sent}/{total}"));
}
fn head(&self, h: &ProgressHead<'_>) {
self.0.lock().unwrap().push(format!("head {}", h.status));
}
fn body(&self, chunk: &[u8], received: u64, _total: Option<u64>) {
self.0.lock().unwrap().push(format!("body {} {received}", chunk.len()));
}
}
#[test]
fn an_observer_hears_upload_head_and_body_in_order() {
let (port, _seen) = serve(vec![b"HTTP/1.1 201 Created\r\nContent-Length: 7\r\n\r\ncreated".to_vec()]);
let seen = Arc::new(Seen::default());
let mut r = req(&format!("http://127.0.0.1:{port}/up"));
r.method = "PUT".into();
r.body = Some(vec![1u8; 70_000]);
r.observer = Some(Observer(seen.clone()));
let resp = send(&r, &Cancel::default()).unwrap();
assert_eq!(resp.status, 201);
let ev = seen.0.lock().unwrap().clone();
let head = ev.iter().position(|e| e == "head 201").expect("no head event");
assert_eq!(ev[head - 1], "up 70000/70000", "{ev:?}");
assert!(ev[..head].iter().all(|e| e.starts_with("up ")), "{ev:?}");
assert!(ev[head + 1..].iter().all(|e| e.starts_with("body ")), "{ev:?}");
assert!(ev.last().unwrap().ends_with(" 7"), "{ev:?}");
}
#[test]
fn a_chunked_response_arrives_whole() {
let (port, _seen) = serve(vec![
b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n6\r\n world\r\n0\r\n\r\n".to_vec(),
]);
let r = send(&req(&format!("http://127.0.0.1:{port}/stream")), &Cancel::default()).unwrap();
assert_eq!(String::from_utf8(r.body).unwrap(), "hello world");
}
#[test]
fn an_error_status_is_a_response_not_a_rejection() {
let (port, _seen) = serve(vec![
b"HTTP/1.1 404 Not Found\r\nContent-Length: 9\r\n\r\nnot here!".to_vec(),
]);
let r = send(&req(&format!("http://127.0.0.1:{port}/missing")), &Cancel::default()).unwrap();
assert_eq!(r.status, 404);
assert_eq!(String::from_utf8(r.body).unwrap(), "not here!");
}
#[test]
fn redirects_are_followed_and_reported() {
let (port, _seen) = serve(vec![
b"HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 0\r\n\r\n".to_vec(),
b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nthere".to_vec(),
]);
let r = send(&req(&format!("http://127.0.0.1:{port}/start")), &Cancel::default()).unwrap();
assert_eq!(r.status, 200);
assert!(r.redirected);
assert!(r.url.ends_with("/final"), "final URL not reported: {}", r.url);
assert_eq!(String::from_utf8(r.body).unwrap(), "there");
}
#[test]
fn redirect_error_mode_fails_instead_of_following() {
let (port, _seen) = serve(vec![
b"HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 0\r\n\r\n".to_vec(),
]);
let mut r = req(&format!("http://127.0.0.1:{port}/start"));
r.redirect = Redirect::Error;
let e = send(&r, &Cancel::default()).unwrap_err();
assert_eq!(e.kind, ErrorKind::Network, "{}", e.message);
}
#[test]
fn a_peer_that_hangs_up_mid_body_fails_rather_than_truncating() {
let (port, _seen) = serve(vec![b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\nhelo".to_vec()]);
let e = send(&req(&format!("http://127.0.0.1:{port}/short")), &Cancel::default()).unwrap_err();
assert_eq!(e.kind, ErrorKind::Protocol, "{}", e.message);
}
#[test]
fn a_peer_that_is_not_speaking_http_fails_loudly() {
let (port, _seen) = serve(vec![b"I am not an HTTP server\r\n\r\n".to_vec()]);
let e = send(&req(&format!("http://127.0.0.1:{port}/junk")), &Cancel::default()).unwrap_err();
assert_eq!(e.kind, ErrorKind::Protocol, "{}", e.message);
}
#[test]
#[ignore = "requires internet access"]
fn https_against_a_real_server_works() {
let r = send(&req("https://example.com/"), &Cancel::default()).unwrap();
assert_eq!(r.status, 200, "{}", r.status_text);
let text = String::from_utf8_lossy(&r.body).to_ascii_lowercase();
assert!(text.contains("example domain"), "unexpected body: {}", &text[..text.len().min(200)]);
}
#[test]
#[ignore = "requires internet access"]
fn https_json_and_gzip_come_back_decoded() {
let r = send(&req("https://api.github.com/meta"), &Cancel::default()).unwrap();
assert_eq!(r.status, 200, "{}", r.status_text);
let text = String::from_utf8(r.body).expect("gzip left non-UTF-8 bytes behind");
assert!(text.starts_with('{'), "body was not decoded JSON: {}", &text[..text.len().min(120)]);
}
#[test]
#[ignore = "requires internet access"]
fn a_real_redirect_chain_is_followed() {
let r = send(&req("http://github.com/"), &Cancel::default()).unwrap();
assert_eq!(r.status, 200);
assert!(r.redirected);
assert!(r.url.starts_with("https://"), "did not upgrade to https: {}", r.url);
}
#[test]
#[ignore = "requires internet access"]
fn an_invalid_certificate_is_rejected() {
for host in ["expired.badssl.com", "self-signed.badssl.com", "wrong.host.badssl.com", "untrusted-root.badssl.com"] {
let e = send(&req(&format!("https://{host}/")), &Cancel::default())
.err()
.unwrap_or_else(|| panic!("{host} was ACCEPTED — certificate verification is not happening"));
assert_eq!(e.kind, ErrorKind::Network, "{host}: {}", e.message);
assert!(
e.message.to_ascii_lowercase().contains("certificate"),
"{host} failed, but not on certificate validation: {}",
e.message
);
}
}
#[test]
#[ignore = "requires internet access"]
fn a_valid_certificate_chain_is_accepted() {
let r = send(&req("https://sha256.badssl.com/"), &Cancel::default()).unwrap();
assert_eq!(r.status, 200);
}
#[test]
fn abort_mid_flight_shuts_the_socket_down_now() {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
std::thread::spawn(move || {
let held = listener.accept();
std::thread::sleep(Duration::from_secs(20));
drop(held);
});
let cancel = Cancel::new();
let waker = std::sync::Arc::clone(&cancel);
std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(250));
waker.abort();
});
let mut r = req(&format!("http://127.0.0.1:{port}/silent"));
r.io_timeout = Duration::from_secs(30);
let started = std::time::Instant::now();
let e = send(&r, &cancel).unwrap_err();
assert_eq!(e.kind, ErrorKind::Aborted, "{}", e.message);
assert!(
started.elapsed() < Duration::from_secs(5),
"abort waited for the read timeout instead of shutting the socket down ({:?})",
started.elapsed()
);
}
}