use std::{
collections::{BTreeMap, HashSet},
path::Path,
pin::Pin,
task::{Context, Poll},
time::{Duration, Instant},
};
use crate::{
args::Args,
command_auth::{self, Credentials},
error::{Error, Result},
item::{HeaderValue, Item, Items, Upload, Value},
output,
session::Session,
};
use async_fs::File;
use futures_io::AsyncRead;
use futures_util::AsyncReadExt as _;
use serde_json::Value as JsonValue;
use tez::{
Body, Client, Frame, HeaderName, HeaderValue as HttpHeaderValue, HttpBody, Method, Multipart,
Part, Request, RequestBuilder, Response, SizeHint, StatusCode,
auth::Authentication as _,
header::{
ACCEPT, ACCEPT_ENCODING, AUTHORIZATION, CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TYPE,
COOKIE, LOCATION, RANGE, TRANSFER_ENCODING, USER_AGENT,
},
proxy,
};
const FILE_VALUE_LIMIT: u64 = 16 * 1024 * 1024;
const JSON_ACCEPT: &str = "application/json, */*;q=0.5";
const GENERAL_ACCEPT: &str = "*/*";
const USER_AGENT_VALUE: &str = concat!("tez/", env!("CARGO_PKG_VERSION"));
pub(crate) async fn execute(
args: &Args,
stdin_is_terminal: bool,
session: &mut Session,
) -> Result<Outcome> {
let target = args.target()?;
let items = Items::parse(&target.items)?;
let uses_stdin = !stdin_is_terminal && !args.ignore_stdin;
let plan = Plan::new(args, &items, uses_stdin)?;
let resume_from = resume_from(args).await?;
let mut headers = Headers::load(&items).await?;
let command_auth = command_auth::resolve(args.auth_type, args.auth.as_deref(), uses_stdin)?;
session.remember(headers.session_changes(), command_auth.as_ref());
headers.prepend_defaults(session.headers()?);
let auth = (!headers.contains(&AUTHORIZATION))
.then(|| command_auth.or_else(|| session.auth()))
.flatten();
let method = plan.method(target.explicit_method);
let client = client(args, &headers)?;
let mut invocation = Invocation {
client: &client,
args,
items: &items,
plan: &plan,
headers: &headers,
auth,
session,
resume_from,
};
let initial = Attempt {
url: &target.url,
method: method.clone(),
body: plan.body,
include_queries: true,
drop_payload_headers: false,
deadline: None,
};
if args.offline {
return invocation.prepare(initial).await.map(Outcome::Request);
}
if args.follow || args.download {
let response = invocation.send_followed(&target.url, method).await?;
return Ok(Outcome::Response {
response,
resume_from,
});
}
let request = invocation.prepare(initial).await?;
let response = client.send(request).await?;
invocation.session.store_response(&response);
Ok(Outcome::Response {
response,
resume_from,
})
}
pub(crate) enum Outcome {
Request(Request),
Response {
response: Response,
resume_from: Option<u64>,
},
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum BodyKind {
None,
Raw,
File,
Stdin,
Json,
Form,
Multipart,
}
struct Plan {
body: BodyKind,
}
struct Invocation<'a> {
client: &'a Client,
args: &'a Args,
items: &'a Items,
plan: &'a Plan,
headers: &'a Headers,
auth: Option<Credentials>,
session: &'a mut Session,
resume_from: Option<u64>,
}
struct Attempt<'a> {
url: &'a str,
method: Method,
body: BodyKind,
include_queries: bool,
drop_payload_headers: bool,
deadline: Option<Deadline>,
}
#[derive(Clone, Copy)]
struct Deadline {
limit: Duration,
started: Instant,
}
impl Deadline {
fn remaining(self) -> Result<Duration> {
self.limit
.checked_sub(self.started.elapsed())
.filter(|remaining| !remaining.is_zero())
.ok_or_else(Error::timeout)
}
}
impl Plan {
fn new(args: &Args, items: &Items, uses_stdin: bool) -> Result<Self> {
let mut has_fields = false;
let mut has_json_fields = false;
let mut has_uploads = false;
for item in &items.entries {
match item {
Item::Field { .. } => has_fields = true,
Item::Json { .. } => {
has_fields = true;
has_json_fields = true;
}
Item::Upload(_) => has_uploads = true,
Item::Header { .. } | Item::Query { .. } => {}
}
}
if args.json && (args.form || args.multipart) {
return Err(Error::usage(
"JSON, form, and multipart modes cannot be combined",
));
}
if has_uploads && !(args.form || args.multipart) {
return Err(Error::usage("file uploads require --form or --multipart"));
}
if has_json_fields && (args.form || args.multipart) {
return Err(Error::usage("raw JSON fields cannot be used in form data"));
}
let structured = has_fields || has_uploads;
let distinct_sources = usize::from(args.raw.is_some())
+ usize::from(items.body_file.is_some())
+ usize::from(uses_stdin)
+ usize::from(structured);
if distinct_sources > 1 {
return Err(Error::usage(
"request body sources cannot be combined; use --ignore-stdin when needed",
));
}
let body = if args.raw.is_some() {
if args.form || args.multipart {
return Err(Error::usage(
"a raw request body cannot be combined with form data",
));
}
BodyKind::Raw
} else if items.body_file.is_some() {
if args.form || args.multipart {
return Err(Error::usage(
"a request body file cannot be combined with form data",
));
}
BodyKind::File
} else if uses_stdin {
if args.form || args.multipart {
return Err(Error::usage(
"standard input cannot be combined with form data",
));
}
BodyKind::Stdin
} else if has_uploads || args.multipart {
if structured {
BodyKind::Multipart
} else {
BodyKind::None
}
} else if args.form {
if has_fields {
BodyKind::Form
} else {
BodyKind::None
}
} else if has_fields {
BodyKind::Json
} else {
BodyKind::None
};
Ok(Self { body })
}
fn method(&self, explicit: Option<Method>) -> Method {
explicit.unwrap_or_else(|| {
if self.body == BodyKind::None {
Method::GET
} else {
Method::POST
}
})
}
fn uses_json(&self, args: &Args) -> bool {
args.json || self.body == BodyKind::Json
}
}
fn client(args: &Args, headers: &Headers) -> Result<Client> {
let mut builder = Client::builder()
.timeout(None)
.proxy(proxy::Policy::system())
.cookies()
.user_agent(USER_AGENT_VALUE)
.runtime(tez::runtime::Smol::runtime());
if !(args.follow || args.download)
&& let Some(timeout) = args.timeout
{
builder = builder.timeout(timeout);
}
if let Some(proxy) = &args.proxy {
builder = builder.proxy(proxy::Policy::all(proxy));
}
if headers.is_unset(&USER_AGENT) {
builder = builder.no_user_agent();
}
if headers.is_unset(&ACCEPT_ENCODING) {
builder = builder.disable_compression();
}
builder.build().map_err(Into::into)
}
async fn resume_from(args: &Args) -> Result<Option<u64>> {
if !args.continue_download {
return Ok(None);
}
let path = args
.output
.as_ref()
.expect("clap requires --output with --continue");
match async_fs::metadata(path).await {
Ok(metadata) if metadata.is_file() && metadata.len() > 0 => Ok(Some(metadata.len())),
Ok(metadata) if metadata.is_file() => Ok(None),
Ok(_) => Err(Error::usage("download output path is not a file")),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(_) => Err(Error::usage("download output file could not be read")),
}
}
impl Invocation<'_> {
async fn prepare(&self, attempt: Attempt<'_>) -> Result<Request> {
let mut request = self.client.request(attempt.method, attempt.url);
if attempt.include_queries {
request = queries(request, self.items).await?;
}
request = body(request, self.args, self.items, attempt.body).await?;
request = request.header(
ACCEPT,
if self.plan.uses_json(self.args) {
JSON_ACCEPT
} else {
GENERAL_ACCEPT
},
);
if let Some(deadline) = attempt.deadline {
request = request.timeout(deadline.remaining()?);
}
if let Some(offset) = self.resume_from
&& !self.headers.contains(&RANGE)
{
request = request.header(RANGE, format!("bytes={offset}-"));
}
let mut request = request.build()?;
self.headers.apply(&mut request);
if !request.headers().contains_key(AUTHORIZATION)
&& let Some(credentials) = self.auth.as_ref()
{
credentials.authenticate(&mut request).await?;
}
if !request.headers().contains_key(COOKIE)
&& let Some(cookie) = self.session.cookie_header(&request.uri().to_string())
{
request.headers_mut().insert(COOKIE, cookie);
}
if attempt.drop_payload_headers {
for name in [
CONTENT_TYPE,
CONTENT_LENGTH,
CONTENT_ENCODING,
TRANSFER_ENCODING,
] {
request.headers_mut().remove(name);
}
}
if self.args.verbose {
output::write_request_head(&request).await?;
}
Ok(request)
}
async fn send_followed(&mut self, initial_url: &str, mut method: Method) -> Result<Response> {
let deadline = self.args.timeout.map(|limit| Deadline {
limit,
started: Instant::now(),
});
let mut url = url::Url::parse(initial_url)
.map_err(|_| Error::usage("URL must be an absolute HTTP or HTTPS URL"))?;
let mut body = self.plan.body;
let mut include_queries = true;
let mut drop_payload_headers = false;
let mut followed = 0;
loop {
let request = self
.prepare(Attempt {
url: url.as_str(),
method: method.clone(),
body,
include_queries,
drop_payload_headers,
deadline,
})
.await?;
let response = self.client.send(request).await?;
self.session.store_response(&response);
let status = response.status();
let Some(next) = redirect_target(&response)? else {
return Ok(response);
};
if url.origin() != next.origin() || self.args.max_redirects == 0 {
return Ok(response);
}
if followed == self.args.max_redirects {
return Err(Error::redirect("HTTP redirect limit exceeded"));
}
let (next_method, next_body, dropped_body) = redirected_request(status, &method, body);
if next_body == BodyKind::Stdin {
return Err(Error::redirect(
"standard input cannot be replayed across this redirect",
));
}
followed += 1;
url = next;
method = next_method;
body = next_body;
include_queries = false;
drop_payload_headers |= dropped_body;
}
}
}
fn redirect_target(response: &Response) -> Result<Option<url::Url>> {
if !matches!(
response.status(),
StatusCode::MOVED_PERMANENTLY
| StatusCode::FOUND
| StatusCode::SEE_OTHER
| StatusCode::TEMPORARY_REDIRECT
| StatusCode::PERMANENT_REDIRECT
) {
return Ok(None);
}
let Some(location) = response
.headers()
.get(LOCATION)
.and_then(|value| value.to_str().ok())
else {
return Ok(None);
};
let base = url::Url::parse(&response.url().to_string())
.map_err(|_| Error::redirect("redirect response URL is invalid"))?;
let Ok(mut target) = base.join(location) else {
return Ok(None);
};
if !matches!(target.scheme(), "http" | "https") {
return Err(Error::redirect("redirect target scheme is unsupported"));
}
target.set_fragment(None);
Ok(Some(target))
}
fn redirected_request(
status: StatusCode,
method: &Method,
body: BodyKind,
) -> (Method, BodyKind, bool) {
let drops_body = status == StatusCode::SEE_OTHER
|| matches!(status, StatusCode::MOVED_PERMANENTLY | StatusCode::FOUND)
&& *method == Method::POST;
if drops_body {
let method = if *method == Method::HEAD {
Method::HEAD
} else {
Method::GET
};
(method, BodyKind::None, true)
} else {
(method.clone(), body, false)
}
}
async fn queries<'client>(
mut request: RequestBuilder<'client>,
items: &Items,
) -> Result<RequestBuilder<'client>> {
let mut values = Vec::new();
for item in &items.entries {
if let Item::Query { name, value } = item {
values.push((name.clone(), load_value(value).await?));
}
}
if !values.is_empty() {
request = request.query(&values);
}
Ok(request)
}
async fn body<'client>(
request: RequestBuilder<'client>,
args: &Args,
items: &Items,
kind: BodyKind,
) -> Result<RequestBuilder<'client>> {
match kind {
BodyKind::None => Ok(request),
BodyKind::Raw => {
let raw = args
.raw
.as_deref()
.ok_or_else(|| Error::usage("raw request body is missing"))?;
let request = request.body(raw);
Ok(json_content_type(request, args.json))
}
BodyKind::File => {
let path = items
.body_file
.as_deref()
.ok_or_else(|| Error::usage("request body file is missing"))?;
let (body, length) = file_body(path).await?;
let mut request = request.body(body);
if let Some(length) = length {
request = request.header(CONTENT_LENGTH, length.to_string());
}
Ok(json_content_type(request, args.json))
}
BodyKind::Stdin => {
let request = request.body(reader_body(::blocking::Unblock::new(std::io::stdin())));
Ok(json_content_type(request, args.json))
}
BodyKind::Json => {
let mut object = BTreeMap::new();
for item in &items.entries {
match item {
Item::Field { name, value } => {
object.insert(name.clone(), JsonValue::String(load_value(value).await?));
}
Item::Json { name, value } => {
let text = load_value(value).await?;
let value = serde_json::from_str(&text)
.map_err(|_| Error::usage("a JSON field value is invalid"))?;
object.insert(name.clone(), value);
}
Item::Header { .. } | Item::Query { .. } | Item::Upload(_) => {}
}
}
Ok(request.json(&object))
}
BodyKind::Form => {
let mut fields = Vec::new();
for item in &items.entries {
if let Item::Field { name, value } = item {
fields.push((name.clone(), load_value(value).await?));
}
}
Ok(request.form(&fields))
}
BodyKind::Multipart => Ok(request.multipart(multipart(items).await?)),
}
}
fn json_content_type(request: RequestBuilder<'_>, is_json: bool) -> RequestBuilder<'_> {
if is_json {
request.header(CONTENT_TYPE, "application/json")
} else {
request
}
}
async fn multipart(items: &Items) -> Result<Multipart> {
let mut form = Multipart::new();
for item in &items.entries {
match item {
Item::Field { name, value } => {
form = form.text(name.clone(), load_value(value).await?);
}
Item::Upload(upload) => {
form = form.part(upload.name.clone(), upload_part(upload).await?);
}
Item::Header { .. } | Item::Query { .. } | Item::Json { .. } => {}
}
}
Ok(form)
}
async fn upload_part(upload: &Upload) -> Result<Part> {
let (body, _) = file_body(&upload.path).await?;
let file_name = upload
.path
.file_name()
.ok_or_else(|| Error::usage("upload path must name a file"))?
.to_string_lossy()
.into_owned();
let media_type = upload.media_type.clone().unwrap_or_else(|| {
mime_guess::from_path(&upload.path)
.first_or_octet_stream()
.essence_str()
.to_owned()
});
Ok(Part::body(body).file_name(file_name).media_type(media_type))
}
struct Headers {
entries: Vec<LoadedHeader>,
}
struct LoadedHeader {
name: HeaderName,
value: LoadedHeaderValue,
}
enum LoadedHeaderValue {
Unset,
Value(HttpHeaderValue),
}
impl Headers {
async fn load(items: &Items) -> Result<Self> {
let mut entries = Vec::new();
for item in &items.entries {
let Item::Header { name, value } = item else {
continue;
};
let name = name
.parse::<HeaderName>()
.map_err(|_| Error::usage("a request header name is invalid"))?;
let value = match value {
HeaderValue::Unset => LoadedHeaderValue::Unset,
HeaderValue::Empty => LoadedHeaderValue::Value(HttpHeaderValue::from_static("")),
HeaderValue::Value(value) => {
let value = load_value(value).await?;
let value = HttpHeaderValue::try_from(value)
.map_err(|_| Error::usage("a request header value is invalid"))?;
LoadedHeaderValue::Value(value)
}
};
entries.push(LoadedHeader { name, value });
}
Ok(Self { entries })
}
fn contains(&self, name: &HeaderName) -> bool {
self.entries.iter().any(|entry| entry.name == name)
}
fn is_unset(&self, name: &HeaderName) -> bool {
self.entries
.iter()
.rev()
.find(|entry| entry.name == name)
.is_some_and(|entry| matches!(entry.value, LoadedHeaderValue::Unset))
}
fn prepend_defaults(&mut self, defaults: Vec<(HeaderName, HttpHeaderValue)>) {
let overridden = self
.entries
.iter()
.map(|entry| entry.name.clone())
.collect::<HashSet<_>>();
let mut entries = defaults
.into_iter()
.filter(|(name, _)| !overridden.contains(name))
.map(|(name, value)| LoadedHeader {
name,
value: LoadedHeaderValue::Value(value),
})
.collect::<Vec<_>>();
entries.append(&mut self.entries);
self.entries = entries;
}
fn session_changes(&self) -> Vec<(String, Option<String>)> {
self.entries
.iter()
.filter_map(|entry| match &entry.value {
_ if !persistable_header(&entry.name) => None,
LoadedHeaderValue::Unset => Some((entry.name.as_str().to_owned(), None)),
LoadedHeaderValue::Value(value) => value
.to_str()
.ok()
.map(|value| (entry.name.as_str().to_owned(), Some(value.to_owned()))),
})
.collect()
}
fn apply(&self, request: &mut Request) {
let mut has_values = HashSet::new();
for entry in &self.entries {
match &entry.value {
LoadedHeaderValue::Unset => {
request.headers_mut().remove(&entry.name);
has_values.remove(&entry.name);
}
LoadedHeaderValue::Value(value) => {
if has_values.insert(entry.name.clone()) {
request.headers_mut().remove(&entry.name);
}
request
.headers_mut()
.append(entry.name.clone(), value.clone());
}
}
}
}
}
fn persistable_header(name: &HeaderName) -> bool {
[
&AUTHORIZATION,
&COOKIE,
&CONTENT_ENCODING,
&CONTENT_LENGTH,
&CONTENT_TYPE,
&TRANSFER_ENCODING,
]
.into_iter()
.all(|excluded| name != excluded)
}
async fn load_value(value: &Value) -> Result<String> {
match value {
Value::Text(value) => Ok(value.clone()),
Value::File(path) => read_file_value(path).await,
}
}
async fn read_file_value(path: &Path) -> Result<String> {
let file = File::open(path)
.await
.map_err(|_| Error::usage("a request value file could not be opened"))?;
let mut bytes = Vec::new();
file.take(FILE_VALUE_LIMIT + 1)
.read_to_end(&mut bytes)
.await
.map_err(|_| Error::usage("a request value file could not be read"))?;
if bytes.len() as u64 > FILE_VALUE_LIMIT {
return Err(Error::usage(
"request value files cannot be larger than 16 MiB",
));
}
let text = String::from_utf8(bytes)
.map_err(|_| Error::usage("a request value file must contain UTF-8 text"))?;
Ok(text.trim_end_matches(['\r', '\n']).to_owned())
}
async fn file_body(path: &Path) -> Result<(Body, Option<u64>)> {
let file = File::open(path)
.await
.map_err(|_| Error::usage("a request body file could not be opened"))?;
let length = file
.metadata()
.await
.ok()
.filter(std::fs::Metadata::is_file)
.map(|metadata| metadata.len());
let body = match length {
Some(length) => Body::wrap(ExactBody::new(ReaderBody::new(file), length)),
None => reader_body(file),
};
Ok((body, length))
}
fn reader_body(reader: impl AsyncRead + Send + Unpin + 'static) -> Body {
Body::wrap(ReaderBody::new(reader))
}
struct ReaderBody<R> {
reader: R,
buffer: Vec<u8>,
finished: bool,
}
impl<R> ReaderBody<R> {
fn new(reader: R) -> Self {
Self {
reader,
buffer: vec![0; 16 * 1024],
finished: false,
}
}
}
impl<R> HttpBody for ReaderBody<R>
where
R: AsyncRead + Unpin,
{
type Data = bytes::Bytes;
type Error = std::io::Error;
fn poll_frame(
mut self: Pin<&mut Self>,
context: &mut Context<'_>,
) -> Poll<Option<std::result::Result<Frame<Self::Data>, Self::Error>>> {
if self.finished {
return Poll::Ready(None);
}
let this = &mut *self;
match Pin::new(&mut this.reader).poll_read(context, &mut this.buffer) {
Poll::Ready(Ok(0)) => {
this.finished = true;
Poll::Ready(None)
}
Poll::Ready(Ok(read)) => Poll::Ready(Some(Ok(Frame::data(
bytes::Bytes::copy_from_slice(&this.buffer[..read]),
)))),
Poll::Ready(Err(error)) => {
this.finished = true;
Poll::Ready(Some(Err(error)))
}
Poll::Pending => Poll::Pending,
}
}
fn is_end_stream(&self) -> bool {
self.finished
}
}
struct ExactBody<B> {
inner: B,
remaining: u64,
}
impl<B> ExactBody<B> {
const fn new(inner: B, length: u64) -> Self {
Self {
inner,
remaining: length,
}
}
}
impl<B> HttpBody for ExactBody<B>
where
B: HttpBody + Unpin,
B::Data: AsRef<[u8]>,
{
type Data = B::Data;
type Error = B::Error;
fn poll_frame(
self: Pin<&mut Self>,
context: &mut Context<'_>,
) -> Poll<Option<std::result::Result<Frame<Self::Data>, Self::Error>>> {
let this = self.get_mut();
let polled = Pin::new(&mut this.inner).poll_frame(context);
if let Poll::Ready(Some(Ok(frame))) = &polled
&& let Some(data) = frame.data_ref()
{
this.remaining = this.remaining.saturating_sub(data.as_ref().len() as u64);
}
polled
}
fn is_end_stream(&self) -> bool {
self.inner.is_end_stream()
}
fn size_hint(&self) -> SizeHint {
SizeHint::with_exact(self.remaining)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn inferred_method_depends_only_on_a_real_body() {
let empty = Plan {
body: BodyKind::None,
};
let body = Plan {
body: BodyKind::Stdin,
};
assert_eq!(empty.method(None), Method::GET);
assert_eq!(body.method(None), Method::POST);
assert_eq!(body.method(Some(Method::PATCH)), Method::PATCH);
}
#[test]
fn loaded_headers_preserve_repeats_and_unsets() {
let mut request = tez::HttpRequest::builder()
.uri("https://example.com")
.body(Body::empty())
.unwrap();
request
.headers_mut()
.insert("x-mode", HttpHeaderValue::from_static("generated"));
let name = HeaderName::from_static("x-mode");
Headers {
entries: vec![
LoadedHeader {
name: name.clone(),
value: LoadedHeaderValue::Value(HttpHeaderValue::from_static("one")),
},
LoadedHeader {
name: name.clone(),
value: LoadedHeaderValue::Value(HttpHeaderValue::from_static("two")),
},
],
}
.apply(&mut request);
let values = request
.headers()
.get_all("x-mode")
.iter()
.map(|value| value.to_str().unwrap())
.collect::<Vec<_>>();
assert_eq!(values, ["one", "two"]);
Headers {
entries: vec![LoadedHeader {
name,
value: LoadedHeaderValue::Unset,
}],
}
.apply(&mut request);
assert!(!request.headers().contains_key("x-mode"));
}
#[test]
fn only_a_final_unset_disables_a_generated_header() {
let name = HeaderName::from_static("accept-encoding");
let value = LoadedHeaderValue::Value(HttpHeaderValue::from_static("gzip"));
let restored = Headers {
entries: vec![
LoadedHeader {
name: name.clone(),
value: LoadedHeaderValue::Unset,
},
LoadedHeader { name, value },
],
};
assert!(!restored.is_unset(&ACCEPT_ENCODING));
let disabled = Headers {
entries: vec![LoadedHeader {
name: ACCEPT_ENCODING,
value: LoadedHeaderValue::Unset,
}],
};
assert!(disabled.is_unset(&ACCEPT_ENCODING));
}
}