use std::fs::File;
use std::io::{self, IsTerminal, Write};
use std::path::Path;
use std::process::ExitCode;
use std::time::Duration;
use rsurl::{CookieJar, HttpVersionPref, Request, Response, Url};
const VERSION: &str = env!("CARGO_PKG_VERSION");
#[derive(Default, Clone)]
struct Args {
urls: Vec<String>,
output: Option<String>,
include_headers: bool,
head: bool,
verbose: bool,
silent: bool,
method: Option<String>,
headers: Vec<(String, String)>,
data_parts: Vec<DataPart>,
json_parts: Vec<String>,
user_agent: Option<String>,
referer: Option<String>,
http_version: Option<HttpVersionPref>,
follow_redirects: bool,
max_redirs: Option<u32>,
basic_auth: Option<(String, String)>,
insecure: bool,
no_idn: bool,
cacert: Option<String>,
max_time: Option<u64>,
connect_timeout: Option<u64>,
remote_name: bool,
cookie_in: Option<String>,
cookie_jar: Option<String>,
proxy: Option<String>,
proxy_user: Option<(String, String)>,
noproxy: Option<String>,
form_parts: Vec<FormPart>,
form_escape: bool,
upload_file: Option<String>,
continue_at: Option<u64>,
append: bool,
ssh_keys: Vec<String>,
fail: bool,
show_error: bool,
get: bool,
range: Option<String>,
compressed: bool,
dump_header: Option<String>,
remote_time: bool,
create_dirs: bool,
remove_on_error: bool,
no_clobber: bool,
disable_epsv: bool,
ftp_create_dirs: bool,
ftp_port: Option<String>,
max_filesize: Option<u64>,
write_out: Option<String>,
netrc: bool,
netrc_file: Option<String>,
remote_header_name: bool,
retry: u32,
ipv4: bool,
ipv6: bool,
resolve: Vec<(String, u16, std::net::IpAddr)>,
progress_bar: bool,
cert: Option<String>,
key_file: Option<String>,
key_pass: Option<String>,
cert_type_der: bool,
key_type_der: bool,
pinned_pubkey: Option<String>,
capath: Option<String>,
crl_file: Option<String>,
ciphers: Option<String>,
tls13_ciphers: Option<String>,
limit_rate: Option<String>,
speed_limit: Option<String>,
speed_time: Option<String>,
time_cond: Option<String>,
output_dir: Option<String>,
fail_with_body: bool,
proto: Option<String>,
proto_default: Option<String>,
auto_referer: bool,
retry_delay: Option<u64>,
retry_max_time: Option<u64>,
retry_connrefused: bool,
retry_all_errors: bool,
globoff: bool,
location_trusted: bool,
post301: bool,
post302: bool,
post303: bool,
connect_to: Vec<(String, u16, String, u16)>,
unix_socket: Option<String>,
tls_min: Option<rsurl::tls::ProtocolVersion>,
tls_max: Option<rsurl::tls::ProtocolVersion>,
mail_from: Option<String>,
mail_rcpt: Vec<String>,
digest: bool,
bearer: Option<String>,
aws_sigv4: Option<String>,
parallel: bool,
parallel_max: Option<usize>,
}
#[derive(Debug, Clone)]
enum DataPart {
Plain { value: String, at_file_ok: bool },
Binary { value: String },
UrlEncoded { value: String },
}
#[derive(Debug, Clone)]
struct FormPart {
name: String,
body: FormBody,
extras: Vec<FormExtra>,
}
#[derive(Debug, Clone)]
enum FormBody {
Literal(String),
File(String),
FileAsField(String),
LiteralStrict(String),
}
#[derive(Debug, Clone)]
enum FormExtra {
Type(String),
Filename(String),
HeadersFile(String),
}
fn main() -> ExitCode {
let raw: Vec<String> = std::env::args().skip(1).collect();
let raw = expand_short_bundles(&raw);
let expanded = match expand_config(&raw, 0) {
Ok(v) => v,
Err(e) => {
eprintln!("rsurl: {e}");
return ExitCode::from(2);
}
};
let segments = split_operations(&expanded);
let mut ops: Vec<Args> = Vec::with_capacity(segments.len());
for seg in &segments {
match parse_args(seg) {
Ok(a) => ops.push(a),
Err(e) => {
eprintln!("rsurl: {e}");
eprintln!("try 'rsurl --help'");
return ExitCode::from(2);
}
}
}
if ops.iter().all(|a| a.urls.is_empty()) {
print_usage();
return ExitCode::from(2);
}
warn_unsupported(&ops);
let uses_cookies = ops
.iter()
.any(|a| a.cookie_in.is_some() || a.cookie_jar.is_some());
if ops.iter().any(|a| a.parallel) && !uses_cookies {
return ExitCode::from(run_parallel(&ops));
}
let jar_op = ops
.iter()
.find(|a| a.cookie_in.is_some() || a.cookie_jar.is_some())
.unwrap_or(&ops[0]);
let mut jar: Option<CookieJar> = match build_initial_jar(jar_op) {
Ok(j) => j,
Err(e) => {
if show_errors(jar_op) {
eprintln!("rsurl: {e}");
}
return ExitCode::from(2);
}
};
let mut last_failure: u8 = 0;
for op in &ops {
for url in &op.urls {
let expansions = if op.globoff {
vec![(url.clone(), Vec::new())]
} else {
match glob_expand(url) {
Ok(v) => v,
Err(e) => {
if show_errors(op) {
eprintln!("rsurl: {e}");
}
last_failure = 3;
continue;
}
}
};
for (eurl, caps) in expansions {
let code =
if !caps.is_empty() && op.output.as_ref().is_some_and(|o| o.contains('#')) {
let mut op2 = op.clone();
op2.output = op.output.as_ref().map(|o| apply_glob_output(o, &caps));
process_url(&eurl, &op2, jar.as_mut())
} else {
process_url(&eurl, op, jar.as_mut())
};
if code != 0 {
last_failure = code;
}
}
}
}
if let (Some(j), Some(op)) = (jar.as_ref(), ops.iter().find(|a| a.cookie_jar.is_some())) {
let path = op.cookie_jar.as_deref().unwrap();
if let Err(e) = j.save_netscape(path) {
if show_errors(op) {
eprintln!("rsurl: writing cookie jar {path}: {e}");
}
if last_failure == 0 {
last_failure = 23;
}
}
}
ExitCode::from(last_failure)
}
fn streams_to_file(args: &Args) -> bool {
let output_is_file = args.remote_name || args.output.as_deref().is_some_and(|p| p != "-");
output_is_file
&& !args.include_headers
&& !args.fail
&& !args.fail_with_body
&& !args.remote_header_name && !args.digest && args.dump_header.is_none()
}
fn warn_unsupported(ops: &[Args]) {
if !ops.iter().any(show_errors) {
return;
}
if ops
.iter()
.any(|a| (a.speed_limit.is_some() || a.speed_time.is_some()) && !streams_to_file(a))
{
eprintln!(
"rsurl: warning: -y/-Y speed limits are enforced only for file downloads \
(-o FILE / -O)"
);
}
}
fn split_cert_pass(s: &str) -> (&str, Option<&str>) {
let bytes = s.as_bytes();
let mut idx = None;
for (i, &b) in bytes.iter().enumerate() {
if b == b':' {
let is_drive = i == 1 && bytes[0].is_ascii_alphabetic();
if !is_drive {
idx = Some(i);
break;
}
}
}
match idx {
Some(i) => (&s[..i], Some(&s[i + 1..])),
None => (s, None),
}
}
fn parse_cert_type(v: &str, flag: &str) -> Result<bool, String> {
match v.to_ascii_uppercase().as_str() {
"PEM" => Ok(false),
"DER" => Ok(true),
other => Err(format!(
"{flag}: unsupported type {other:?}; only PEM and DER are supported"
)),
}
}
fn short_flag_takes_value(c: char) -> bool {
matches!(
c,
'o' | 'X'
| 'H'
| 'd'
| 'F'
| 'T'
| 'A'
| 'e'
| 'u'
| 'b'
| 'c'
| 'x'
| 'E'
| 'r'
| 'D'
| 'w'
| 'C'
| 'y'
| 'Y'
| 'U'
| 'K'
| 'P'
)
}
fn expand_short_bundles(tokens: &[String]) -> Vec<String> {
let mut out = Vec::new();
for t in tokens {
let is_bundle = t.len() > 2 && t.starts_with('-') && !t.starts_with("--");
if !is_bundle {
out.push(t.clone());
continue;
}
let chars: Vec<char> = t[1..].chars().collect();
let mut i = 0;
while i < chars.len() {
let c = chars[i];
out.push(format!("-{c}"));
if short_flag_takes_value(c) {
let rest: String = chars[i + 1..].iter().collect();
if !rest.is_empty() {
out.push(rest); }
break;
}
i += 1;
}
}
out
}
enum GlobSeg {
Lit(String),
Set(Vec<String>),
}
fn parse_glob(url: &str) -> Result<Vec<GlobSeg>, String> {
let mut segs = Vec::new();
let mut lit = String::new();
let chars: Vec<char> = url.chars().collect();
let mut i = 0;
while i < chars.len() {
match chars[i] {
'\\' if i + 1 < chars.len() => {
lit.push(chars[i + 1]);
i += 2;
}
'{' => {
let close = find_close(&chars, i, '{', '}')
.ok_or_else(|| format!("unmatched '{{' in URL glob: {url:?}"))?;
let inner: String = chars[i + 1..close].iter().collect();
let items: Vec<String> = inner.split(',').map(|s| s.to_string()).collect();
if !lit.is_empty() {
segs.push(GlobSeg::Lit(std::mem::take(&mut lit)));
}
segs.push(GlobSeg::Set(items));
i = close + 1;
}
'[' => {
let close = find_close(&chars, i, '[', ']')
.ok_or_else(|| format!("unmatched '[' in URL glob: {url:?}"))?;
let inner: String = chars[i + 1..close].iter().collect();
let items = expand_range(&inner)
.ok_or_else(|| format!("bad range '[{inner}]' in URL glob"))?;
if !lit.is_empty() {
segs.push(GlobSeg::Lit(std::mem::take(&mut lit)));
}
segs.push(GlobSeg::Set(items));
i = close + 1;
}
c => {
lit.push(c);
i += 1;
}
}
}
if !lit.is_empty() {
segs.push(GlobSeg::Lit(lit));
}
Ok(segs)
}
fn find_close(chars: &[char], open_at: usize, open: char, close: char) -> Option<usize> {
let mut depth = 0;
for (k, &c) in chars.iter().enumerate().skip(open_at) {
if c == open {
depth += 1;
} else if c == close {
depth -= 1;
if depth == 0 {
return Some(k);
}
}
}
None
}
fn expand_range(body: &str) -> Option<Vec<String>> {
let (range, step) = match body.split_once(':') {
Some((r, s)) => (r, s.parse::<usize>().ok().filter(|&s| s > 0)?),
None => (body, 1),
};
let (start, end) = range.split_once('-')?;
if let (Ok(a), Ok(b)) = (start.parse::<u64>(), end.parse::<u64>()) {
let width = if start.starts_with('0') && start.len() > 1 {
start.len()
} else {
0
};
let mut out = Vec::new();
let mut v = a;
while v <= b {
out.push(format!("{v:0width$}"));
v += step as u64;
}
return Some(out);
}
let (sc, ec) = (start.chars().next()?, end.chars().next()?);
if start.chars().count() == 1 && end.chars().count() == 1 && sc <= ec {
let mut out = Vec::new();
let mut c = sc as u32;
while c <= ec as u32 {
if let Some(ch) = char::from_u32(c) {
out.push(ch.to_string());
}
c += step as u32;
}
return Some(out);
}
None
}
fn glob_expand(url: &str) -> Result<Vec<(String, Vec<String>)>, String> {
let segs = parse_glob(url)?;
let mut results = vec![(String::new(), Vec::new())];
for seg in &segs {
match seg {
GlobSeg::Lit(s) => {
for (u, _) in results.iter_mut() {
u.push_str(s);
}
}
GlobSeg::Set(items) => {
let mut next = Vec::with_capacity(results.len() * items.len());
for (u, caps) in &results {
for item in items {
let mut nu = u.clone();
nu.push_str(item);
let mut nc = caps.clone();
nc.push(item.clone());
next.push((nu, nc));
}
}
results = next;
}
}
}
Ok(results)
}
fn apply_glob_output(template: &str, caps: &[String]) -> String {
if caps.is_empty() || !template.contains('#') {
return template.to_string();
}
let mut out = String::with_capacity(template.len());
let mut chars = template.chars().peekable();
while let Some(c) = chars.next() {
if c == '#' {
if let Some(d) = chars.peek().and_then(|d| d.to_digit(10)) {
chars.next();
let idx = d as usize;
if idx >= 1 && idx <= caps.len() {
out.push_str(&caps[idx - 1]);
continue;
}
}
}
out.push(c);
}
out
}
fn run_parallel(ops: &[Args]) -> u8 {
use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering};
struct Item<'a> {
op: &'a Args,
url: String,
caps: Vec<String>,
}
let mut items: Vec<Item> = Vec::new();
for op in ops {
for url in &op.urls {
let expansions = if op.globoff {
vec![(url.clone(), Vec::new())]
} else {
match glob_expand(url) {
Ok(v) => v,
Err(e) => {
if show_errors(op) {
eprintln!("rsurl: {e}");
}
continue;
}
}
};
for (eurl, caps) in expansions {
items.push(Item {
op,
url: eurl,
caps,
});
}
}
}
if items.is_empty() {
return 0;
}
let max = ops
.iter()
.filter_map(|a| a.parallel_max)
.max()
.unwrap_or(50)
.max(1);
let n_threads = max.min(items.len());
let idx = AtomicUsize::new(0);
let worst = AtomicU8::new(0);
std::thread::scope(|s| {
for _ in 0..n_threads {
s.spawn(|| loop {
let i = idx.fetch_add(1, Ordering::Relaxed);
if i >= items.len() {
break;
}
let item = &items[i];
let code = if !item.caps.is_empty()
&& item.op.output.as_ref().is_some_and(|o| o.contains('#'))
{
let mut op2 = item.op.clone();
op2.output = item
.op
.output
.as_ref()
.map(|o| apply_glob_output(o, &item.caps));
process_url(&item.url, &op2, None)
} else {
process_url(&item.url, item.op, None)
};
if code != 0 {
worst.store(code, Ordering::Relaxed);
}
});
}
});
worst.load(Ordering::Relaxed)
}
fn split_operations(toks: &[String]) -> Vec<Vec<String>> {
let mut segs: Vec<Vec<String>> = vec![Vec::new()];
for t in toks {
if t == "--next" || t == "-:" {
segs.push(Vec::new());
} else {
segs.last_mut().unwrap().push(t.clone());
}
}
segs
}
fn expand_config(toks: &[String], depth: u32) -> Result<Vec<String>, String> {
if depth > 16 {
return Err("config files nested too deeply".into());
}
let mut out = Vec::new();
let mut it = toks.iter();
while let Some(t) = it.next() {
if t == "-K" || t == "--config" {
let path = it
.next()
.ok_or_else(|| "--config requires a file".to_string())?;
let text =
std::fs::read_to_string(path).map_err(|e| format!("config file {path}: {e}"))?;
let inner = expand_config(&parse_config_text(&text), depth + 1)?;
out.extend(inner);
} else {
out.push(t.clone());
}
}
Ok(out)
}
fn parse_config_text(text: &str) -> Vec<String> {
let mut out = Vec::new();
for line in text.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
let (opt, val) = match line.find(|c: char| c.is_whitespace() || c == '=' || c == ':') {
Some(i) => {
let rest = line[i..]
.trim_start()
.strip_prefix(['=', ':'])
.unwrap_or_else(|| line[i..].trim_start())
.trim_start();
(&line[..i], Some(rest))
}
None => (line, None),
};
let opt_norm = if opt.starts_with('-') {
opt.to_string()
} else if opt.chars().count() == 1 {
format!("-{opt}")
} else {
format!("--{opt}")
};
out.push(opt_norm);
if let Some(v) = val.filter(|v| !v.is_empty()) {
let v = v
.strip_prefix('"')
.and_then(|s| s.strip_suffix('"'))
.unwrap_or(v);
out.push(v.to_string());
}
}
out
}
fn build_initial_jar(args: &Args) -> Result<Option<CookieJar>, String> {
if args.cookie_in.is_none() && args.cookie_jar.is_none() {
return Ok(None);
}
let mut jar = match args.cookie_in.as_deref() {
Some(s) if !s.contains('=') => CookieJar::load_netscape_or_empty(s)
.map_err(|e| format!("reading cookie file {s}: {e}"))?,
_ => CookieJar::new(),
};
if args.cookie_in.is_none() {
if let Some(path) = args.cookie_jar.as_deref() {
jar = CookieJar::load_netscape_or_empty(path)
.map_err(|e| format!("reading cookie file {path}: {e}"))?;
}
}
Ok(Some(jar))
}
fn resolve_proxy_spec(url: &Url, args: &Args) -> Option<String> {
if let Some(spec) = &args.proxy {
if spec.is_empty() {
return None;
}
return Some(spec.clone());
}
let read = |upper: &str, lower: &str| -> Option<String> {
for k in [upper, lower] {
if let Ok(v) = std::env::var(k) {
if !v.is_empty() {
return Some(v);
}
}
}
None
};
let scheme_proxy = match url.scheme.as_str() {
"https" => read("HTTPS_PROXY", "https_proxy"),
"http" => match std::env::var("http_proxy") {
Ok(v) if !v.is_empty() => Some(v),
_ => None,
},
_ => None,
};
scheme_proxy.or_else(|| read("ALL_PROXY", "all_proxy"))
}
fn resolve_noproxy(args: &Args) -> Option<String> {
if let Some(v) = &args.noproxy {
return Some(v.clone());
}
for k in ["NO_PROXY", "no_proxy"] {
if let Ok(v) = std::env::var(k) {
if !v.is_empty() {
return Some(v);
}
}
}
None
}
fn apply_explicit_cookies(jar: &mut CookieJar, data: &str, request_url: &Url) {
for pair in data.split(';') {
let pair = pair.trim();
if pair.is_empty() {
continue;
}
if let Some((k, v)) = pair.split_once('=') {
let k = k.trim();
let v = v.trim();
if !k.is_empty() {
jar.add_explicit(k, v, request_url);
}
}
}
}
fn read_at_file(path: &str) -> Result<Vec<u8>, String> {
std::fs::read(path).map_err(|e| format!("can't read {path:?}: {e}"))
}
fn strip_newlines(data: Vec<u8>) -> Vec<u8> {
data.into_iter()
.filter(|&b| b != b'\r' && b != b'\n' && b != 0)
.collect()
}
fn percent_encode_form(bytes: &[u8]) -> String {
use std::fmt::Write;
let mut out = String::with_capacity(bytes.len());
for &b in bytes {
match b {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => {
out.push(b as char);
}
b' ' => out.push('+'),
_ => write!(out, "%{b:02X}").expect("write to String"),
}
}
out
}
fn encode_urlencoded(spec: &str) -> Result<Vec<u8>, String> {
if let Some(eq) = spec.find('=') {
let (name, rest) = spec.split_at(eq);
let value = &rest[1..]; let encoded = percent_encode_form(value.as_bytes());
if name.is_empty() {
return Ok(encoded.into_bytes());
}
return Ok(format!("{name}={encoded}").into_bytes());
}
if let Some(at) = spec.find('@') {
let (name, rest) = spec.split_at(at);
let path = &rest[1..]; let bytes = read_at_file(path)?;
let encoded = percent_encode_form(&bytes);
if name.is_empty() {
return Ok(encoded.into_bytes());
}
return Ok(format!("{name}={encoded}").into_bytes());
}
Ok(percent_encode_form(spec.as_bytes()).into_bytes())
}
mod form_parser {
use super::{FormBody, FormExtra, FormPart};
pub(super) fn parse(spec: &str) -> Result<FormPart, String> {
let eq = spec.find('=').ok_or_else(|| {
format!("-F: expected 'name=value', got {spec:?} (use --form-string for literal '=')")
})?;
let name = spec[..eq].to_string();
if name.is_empty() {
return Err(format!("-F: empty field name: {spec:?}"));
}
let rest = &spec[eq + 1..];
let mut tokens = split_semi(rest);
let raw_value = tokens.remove(0);
let body = classify_body(&raw_value);
let mut extras = Vec::new();
for tok in tokens {
extras.push(classify_extra(&tok)?);
}
Ok(FormPart { name, body, extras })
}
fn classify_body(token: &str) -> FormBody {
if let Some(p) = token.strip_prefix('@') {
FormBody::File(p.to_string())
} else if let Some(p) = token.strip_prefix('<') {
FormBody::FileAsField(p.to_string())
} else {
FormBody::Literal(token.to_string())
}
}
fn classify_extra(token: &str) -> Result<FormExtra, String> {
let (k, v) = token
.split_once('=')
.ok_or_else(|| format!("-F: malformed modifier {token:?} (expected key=value)"))?;
let k = k.trim();
let v = v.to_string();
match k.to_ascii_lowercase().as_str() {
"type" => Ok(FormExtra::Type(v)),
"filename" => Ok(FormExtra::Filename(v)),
"headers" => {
let path = v
.strip_prefix('@')
.ok_or_else(|| format!("-F: ;headers= must be @file (got {token:?})"))?;
Ok(FormExtra::HeadersFile(path.to_string()))
}
_ => Err(format!("-F: unknown modifier {k:?}")),
}
}
fn split_semi(s: &str) -> Vec<String> {
let mut out = Vec::new();
let mut cur = String::new();
let mut chars = s.chars().peekable();
let mut in_quote = false;
while let Some(c) = chars.next() {
if in_quote {
match c {
'"' => in_quote = false,
'\\' => match chars.peek() {
Some('"') => {
cur.push('"');
chars.next();
}
Some('\\') => {
cur.push('\\');
chars.next();
}
_ => cur.push('\\'),
},
_ => cur.push(c),
}
} else {
match c {
'"' => in_quote = true,
';' => out.push(std::mem::take(&mut cur)),
_ => cur.push(c),
}
}
}
out.push(cur);
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn simple_literal() {
let p = parse("foo=bar").unwrap();
assert_eq!(p.name, "foo");
assert!(matches!(&p.body, FormBody::Literal(v) if v == "bar"));
assert!(p.extras.is_empty());
}
#[test]
fn at_file_is_file_upload() {
let p = parse("upload=@/tmp/x.bin").unwrap();
assert!(matches!(&p.body, FormBody::File(v) if v == "/tmp/x.bin"));
}
#[test]
fn lt_file_is_field_from_file() {
let p = parse("note=</tmp/x.txt").unwrap();
assert!(matches!(&p.body, FormBody::FileAsField(v) if v == "/tmp/x.txt"));
}
#[test]
fn quoted_value_with_semicolon() {
let p = parse(r#"k="a;b;c""#).unwrap();
assert!(matches!(&p.body, FormBody::Literal(v) if v == "a;b;c"));
}
#[test]
fn quoted_value_with_escapes() {
let p = parse(r#"k="he said \"hi\" \\""#).unwrap();
assert!(matches!(&p.body, FormBody::Literal(v) if v == r#"he said "hi" \"#));
}
#[test]
fn modifiers_type_filename_headers() {
let p = parse("f=@x;type=application/json;filename=other.json;headers=@hdrs").unwrap();
assert!(matches!(&p.body, FormBody::File(p) if p == "x"));
assert_eq!(p.extras.len(), 3);
assert!(matches!(&p.extras[0], FormExtra::Type(v) if v == "application/json"));
assert!(matches!(&p.extras[1], FormExtra::Filename(v) if v == "other.json"));
assert!(matches!(&p.extras[2], FormExtra::HeadersFile(v) if v == "hdrs"));
}
#[test]
fn empty_name_rejected() {
assert!(parse("=value").is_err());
}
#[test]
fn missing_eq_rejected() {
assert!(parse("foo").is_err());
}
#[test]
fn unknown_modifier_rejected() {
assert!(parse("foo=bar;weird=baz").is_err());
}
#[test]
fn headers_missing_at_rejected() {
assert!(parse("foo=bar;headers=hdrs").is_err());
}
}
}
mod multipart {
use super::{FormBody, FormExtra, FormPart};
pub(super) fn build(parts: &[FormPart], escape: bool) -> Result<(String, Vec<u8>), String> {
let boundary = make_boundary();
let mut out = Vec::new();
for part in parts {
out.extend_from_slice(b"--");
out.extend_from_slice(boundary.as_bytes());
out.extend_from_slice(b"\r\n");
write_part(part, escape, &mut out)?;
out.extend_from_slice(b"\r\n");
}
out.extend_from_slice(b"--");
out.extend_from_slice(boundary.as_bytes());
out.extend_from_slice(b"--\r\n");
Ok((boundary, out))
}
fn write_part(part: &FormPart, escape: bool, out: &mut Vec<u8>) -> Result<(), String> {
let (bytes, default_filename, is_upload): (Vec<u8>, Option<String>, bool) = match &part.body
{
FormBody::Literal(s) | FormBody::LiteralStrict(s) => {
(s.as_bytes().to_vec(), None, false)
}
FormBody::File(path) => {
let bytes =
std::fs::read(path).map_err(|e| format!("-F: can't read {path:?}: {e}"))?;
let name = std::path::Path::new(path)
.file_name()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or_else(|| path.clone());
(bytes, Some(name), true)
}
FormBody::FileAsField(path) => {
let bytes =
std::fs::read(path).map_err(|e| format!("-F: can't read {path:?}: {e}"))?;
(bytes, None, false)
}
};
let mut ctype: Option<&str> = None;
let mut filename: Option<String> = default_filename;
let mut extra_headers: Vec<u8> = Vec::new();
let mut promote_to_upload = is_upload;
for ex in &part.extras {
match ex {
FormExtra::Type(t) => ctype = Some(t),
FormExtra::Filename(f) => {
filename = Some(f.clone());
promote_to_upload = true;
}
FormExtra::HeadersFile(path) => {
let raw = std::fs::read(path)
.map_err(|e| format!("-F: can't read headers file {path:?}: {e}"))?;
for line in raw.split(|b| *b == b'\n') {
let mut l = line;
if l.last() == Some(&b'\r') {
l = &l[..l.len() - 1];
}
if l.is_empty() {
continue;
}
extra_headers.extend_from_slice(l);
extra_headers.extend_from_slice(b"\r\n");
}
}
}
}
out.extend_from_slice(b"Content-Disposition: form-data; name=\"");
out.extend_from_slice(encode_attr(&part.name, escape).as_bytes());
out.extend_from_slice(b"\"");
if promote_to_upload || filename.is_some() {
if let Some(fname) = filename.as_deref() {
out.extend_from_slice(b"; filename=\"");
out.extend_from_slice(encode_attr(fname, escape).as_bytes());
out.extend_from_slice(b"\"");
}
}
out.extend_from_slice(b"\r\n");
if let Some(t) = ctype {
out.extend_from_slice(b"Content-Type: ");
out.extend_from_slice(t.as_bytes());
out.extend_from_slice(b"\r\n");
} else if promote_to_upload {
out.extend_from_slice(b"Content-Type: application/octet-stream\r\n");
}
out.extend_from_slice(&extra_headers);
out.extend_from_slice(b"\r\n");
out.extend_from_slice(&bytes);
Ok(())
}
fn encode_attr(s: &str, escape: bool) -> String {
if escape {
let mut out = String::with_capacity(s.len());
for b in s.bytes() {
match b {
b'\r' => out.push_str("%0D"),
b'\n' => out.push_str("%0A"),
b'"' => out.push_str("%22"),
b'\\' => out.push_str("%5C"),
_ => out.push(b as char),
}
}
out
} else {
let mut out = String::with_capacity(s.len());
for c in s.chars() {
match c {
'"' => out.push_str("\\\""),
'\\' => out.push_str("\\\\"),
_ => out.push(c),
}
}
out
}
}
fn make_boundary() -> String {
let mut buf = [0u8; 8];
let ok = std::fs::File::open("/dev/urandom")
.and_then(|mut f| std::io::Read::read_exact(&mut f, &mut buf))
.is_ok();
if !ok {
use std::time::{SystemTime, UNIX_EPOCH};
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
for (i, b) in buf.iter_mut().enumerate() {
*b = ((nanos >> (i * 8)) & 0xFF) as u8;
}
}
let mut hex = String::with_capacity(16 + 19);
hex.push_str("----rsurl-boundary-");
for b in buf {
use std::fmt::Write;
write!(hex, "{b:02x}").expect("write to String");
}
hex
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn literal_part_round_trip() {
let parts = vec![FormPart {
name: "k".into(),
body: FormBody::Literal("v".into()),
extras: vec![],
}];
let (b, bytes) = build(&parts, false).unwrap();
let text = String::from_utf8(bytes).unwrap();
assert!(text.contains(&format!("--{b}\r\n")));
assert!(text.contains("Content-Disposition: form-data; name=\"k\"\r\n"));
assert!(!text.contains("filename="));
assert!(text.contains("\r\n\r\nv\r\n"));
assert!(text.ends_with(&format!("--{b}--\r\n")));
}
#[test]
fn file_part_gets_filename_and_octet_stream() {
let mut tmp = std::env::temp_dir();
tmp.push(format!("rsurl-mp-{}.bin", std::process::id()));
std::fs::write(&tmp, b"FILEBYTES").unwrap();
let path = tmp.to_string_lossy().into_owned();
let basename = tmp.file_name().unwrap().to_string_lossy().into_owned();
let parts = vec![FormPart {
name: "u".into(),
body: FormBody::File(path),
extras: vec![],
}];
let (_, bytes) = build(&parts, false).unwrap();
let _ = std::fs::remove_file(&tmp);
let text = String::from_utf8(bytes).unwrap();
assert!(text.contains(&format!(
"Content-Disposition: form-data; name=\"u\"; filename=\"{basename}\"\r\n"
)));
assert!(text.contains("Content-Type: application/octet-stream\r\n"));
assert!(text.contains("\r\n\r\nFILEBYTES\r\n"));
}
#[test]
fn type_filename_extras_take_effect() {
let parts = vec![FormPart {
name: "x".into(),
body: FormBody::Literal("body".into()),
extras: vec![
FormExtra::Type("application/json".into()),
FormExtra::Filename("over.json".into()),
],
}];
let (_, bytes) = build(&parts, false).unwrap();
let text = String::from_utf8(bytes).unwrap();
assert!(text.contains("name=\"x\"; filename=\"over.json\"\r\n"));
assert!(text.contains("Content-Type: application/json\r\n"));
}
#[test]
fn form_escape_uses_percent_encoding() {
let parts = vec![FormPart {
name: "weird\"name".into(),
body: FormBody::Literal("v".into()),
extras: vec![],
}];
let (_, bytes) = build(&parts, true).unwrap();
let text = String::from_utf8(bytes).unwrap();
assert!(text.contains("name=\"weird%22name\""), "got: {text}");
}
#[test]
fn default_backslash_escape_preserves_curl_behaviour() {
let parts = vec![FormPart {
name: "weird\"name".into(),
body: FormBody::Literal("v".into()),
extras: vec![],
}];
let (_, bytes) = build(&parts, false).unwrap();
let text = String::from_utf8(bytes).unwrap();
assert!(text.contains(r#"name="weird\"name""#), "got: {text}");
}
}
}
type AssembledBody = (Vec<u8>, String, &'static str);
fn build_upload_body(path: &str) -> Result<AssembledBody, String> {
let bytes = std::fs::read(path).map_err(|e| format!("-T: can't read {path:?}: {e}"))?;
Ok((bytes, "application/octet-stream".into(), "PUT"))
}
fn build_multipart_body(parts: &[FormPart], escape: bool) -> Result<AssembledBody, String> {
let (boundary, bytes) = multipart::build(parts, escape)?;
let ctype = format!("multipart/form-data; boundary={boundary}");
Ok((bytes, ctype, "POST"))
}
fn assemble_request_body(args: &Args) -> Result<Option<AssembledBody>, String> {
let n = (!args.data_parts.is_empty()) as u8
+ (!args.form_parts.is_empty()) as u8
+ args.upload_file.is_some() as u8
+ (!args.json_parts.is_empty()) as u8;
if n > 1 {
return Err(
"-d/--data, -F/--form, -T/--upload-file, and --json are mutually exclusive".into(),
);
}
if let Some(path) = &args.upload_file {
return build_upload_body(path).map(Some);
}
if !args.json_parts.is_empty() {
let mut out: Vec<u8> = Vec::new();
for v in &args.json_parts {
if let Some(path) = v.strip_prefix('@') {
out.extend_from_slice(&read_at_file(path)?);
} else {
out.extend_from_slice(v.as_bytes());
}
}
return Ok(Some((out, "application/json".into(), "POST")));
}
if !args.form_parts.is_empty() {
return build_multipart_body(&args.form_parts, args.form_escape).map(Some);
}
if let Some(bytes) = assemble_form_body(&args.data_parts)? {
return Ok(Some((
bytes,
"application/x-www-form-urlencoded".into(),
"POST",
)));
}
Ok(None)
}
fn assemble_form_body(parts: &[DataPart]) -> Result<Option<Vec<u8>>, String> {
if parts.is_empty() {
return Ok(None);
}
let mut out: Vec<u8> = Vec::new();
for part in parts {
if !out.is_empty() {
out.push(b'&');
}
match part {
DataPart::Plain { value, at_file_ok } => {
if *at_file_ok {
if let Some(path) = value.strip_prefix('@') {
out.extend_from_slice(&strip_newlines(read_at_file(path)?));
continue;
}
}
out.extend_from_slice(value.as_bytes());
}
DataPart::Binary { value } => {
if let Some(path) = value.strip_prefix('@') {
out.extend_from_slice(&read_at_file(path)?);
} else {
out.extend_from_slice(value.as_bytes());
}
}
DataPart::UrlEncoded { value } => {
out.extend_from_slice(&encode_urlencoded(value)?);
}
}
}
Ok(Some(out))
}
fn process_url(url: &str, args: &Args, mut jar: Option<&mut CookieJar>) -> u8 {
let scheme_defaulted;
let url: &str = if url.contains("://") {
url
} else {
let scheme = args.proto_default.as_deref().unwrap_or("http");
scheme_defaulted = format!("{scheme}://{url}");
&scheme_defaulted
};
let mut parsed_url = match Url::parse(url) {
Ok(u) => u,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 3;
}
};
if let Err(e) = parsed_url.set_idn(!args.no_idn) {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 3;
}
if let Some(spec) = &args.proto {
if !proto_allowed(&parsed_url.scheme, spec) {
if show_errors(args) {
eprintln!(
"rsurl: protocol \"{}\" not permitted by --proto",
parsed_url.scheme
);
}
return 1;
}
}
if !matches!(parsed_url.scheme.as_str(), "http" | "https") {
if parsed_url.scheme == "rtsp" {
return run_rtsp(&parsed_url, args);
}
if matches!(parsed_url.scheme.as_str(), "smtp" | "smtps") {
return run_smtp(&parsed_url, args);
}
if parsed_url.scheme == "telnet" {
return run_telnet(&parsed_url, args);
}
if matches!(parsed_url.scheme.as_str(), "mqtt" | "mqtts")
&& (!args.data_parts.is_empty() || args.upload_file.is_some())
{
return run_mqtt_publish(&parsed_url, args);
}
if let Some(path) = &args.upload_file {
if matches!(parsed_url.scheme.as_str(), "ftp" | "ftps") {
return run_ftp_upload(&parsed_url, path, args);
}
if parsed_url.scheme == "tftp" {
return run_tftp_upload(&parsed_url, path, args);
}
if matches!(parsed_url.scheme.as_str(), "sftp" | "scp") {
return run_ssh_upload(&parsed_url, path, args);
}
if show_errors(args) {
eprintln!(
"rsurl: -T is only supported for HTTP(S), FTP(S), TFTP, and SFTP/SCP URLs in this build"
);
}
return 2;
}
if matches!(parsed_url.scheme.as_str(), "sftp" | "scp") {
return run_ssh(&parsed_url, args);
}
let to_file = args.remote_name || args.output.as_deref().is_some_and(|p| p != "-");
if to_file {
return run_stream_download(&parsed_url, args);
}
return run_transfer(&parsed_url, args);
}
let mut assembled = match assemble_request_body(args) {
Ok(b) => b,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 2;
}
};
let mut url_owned = url.to_string();
if args.get {
if let Some((bytes, ctype, _)) = &assembled {
if ctype.starts_with("application/x-www-form-urlencoded") {
let q = String::from_utf8_lossy(bytes);
if !q.is_empty() {
url_owned.push(if url_owned.contains('?') { '&' } else { '?' });
url_owned.push_str(&q);
}
assembled = None;
}
}
}
let method = args.method.clone().unwrap_or_else(|| {
if args.head {
"HEAD".to_string()
} else if args.get {
"GET".to_string()
} else if let Some((_, _, m)) = &assembled {
(*m).to_string()
} else {
"GET".to_string()
}
});
let mut req = match Request::new(&method, &url_owned) {
Ok(r) => r,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 3;
}
};
for (k, v) in &args.headers {
req = req.header(k, v);
}
if let Some(ua) = &args.user_agent {
req = req.header("User-Agent", ua);
}
if let Some(rf) = &args.referer {
req = req.header("Referer", rf);
}
if args.auto_referer {
req = req.auto_referer(true);
}
if let Some(tc) = &args.time_cond {
if let Some((hdr, date)) = time_cond_header(tc) {
req = req.header(hdr, &date);
} else if show_errors(args) {
eprintln!("rsurl: warning: could not parse --time-cond {tc:?}");
}
}
let has_header = |name: &str| {
args.headers
.iter()
.any(|(k, _)| k.eq_ignore_ascii_case(name))
};
if args.compressed && !has_header("accept-encoding") {
req = req.header("Accept-Encoding", "gzip, deflate, br, zstd");
}
if !args.json_parts.is_empty() && !has_header("accept") {
req = req.header("Accept", "application/json");
}
if let Some(r) = &args.range {
if !has_header("range") {
let v = if r.contains('=') {
r.clone()
} else {
format!("bytes={r}")
};
req = req.header("Range", &v);
}
}
if let Some((body_bytes, ctype, _method)) = assembled {
if !args
.headers
.iter()
.any(|(k, _)| k.eq_ignore_ascii_case("content-type"))
{
req = req.header("Content-Type", &ctype);
}
req = req.body(body_bytes);
}
match args.http_version {
Some(HttpVersionPref::Http2Only) => req = req.http2_only(),
Some(HttpVersionPref::Http11Only) => req = req.http11_only(),
Some(HttpVersionPref::Http3) => req = req.http3(),
Some(HttpVersionPref::Http3Only) => req = req.http3_only(),
Some(HttpVersionPref::Auto) | None => {}
}
if args.follow_redirects {
req = req.follow_redirects(true);
}
if let Some(n) = args.max_redirs {
req = req.max_redirs(n);
}
if args.location_trusted {
req = req.redirect_trusted(true);
}
if args.post301 {
req = req.keep_post_on(301);
}
if args.post302 {
req = req.keep_post_on(302);
}
if args.post303 {
req = req.keep_post_on(303);
}
for (fh, fp, th, tp) in &args.connect_to {
req = req.connect_to(fh, *fp, th, *tp);
}
if let Some(path) = &args.unix_socket {
#[cfg(unix)]
{
req = req.connector(std::sync::Arc::new(rsurl::net::UnixConnector {
path: path.into(),
}));
}
#[cfg(not(unix))]
{
let _ = path;
if show_errors(args) {
eprintln!("rsurl: --unix-socket is not supported on this platform");
}
return 2;
}
}
if let Some((u, p)) = &args.basic_auth {
req = req.basic_auth(u, p);
} else if args.netrc && parsed_url.userinfo.is_none() {
if let Some((u, p)) = netrc_credentials(args, &parsed_url.host) {
req = req.basic_auth(&u, &p);
}
}
if args.insecure {
req = req.verify_tls(false);
}
if let Some(v) = args.tls_min {
req = req.tls_min_version(v);
}
if let Some(v) = args.tls_max {
req = req.tls_max_version(v);
}
if args.digest {
req = req.digest_auth(true);
}
if let Some(token) = &args.bearer {
if !args
.headers
.iter()
.any(|(k, _)| k.eq_ignore_ascii_case("authorization"))
{
req = req.header("Authorization", &format!("Bearer {token}"));
}
}
if let (Some(spec), Some((ak, sk))) = (&args.aws_sigv4, &args.basic_auth) {
req = req.aws_sigv4(spec, ak, sk);
}
if args.no_idn {
req = req.idn(false);
}
if let Some(path) = &args.cacert {
req = req.ca_bundle(path);
}
if let Some(dir) = &args.capath {
req = req.ca_path(dir);
}
if let Some(spec) = &args.pinned_pubkey {
req = req.pinned_pubkey(spec);
}
if let Some(path) = &args.crl_file {
req = req.crl_file(path);
}
if let Some(list) = &args.ciphers {
req = req.ciphers(list);
}
if let Some(list) = &args.tls13_ciphers {
req = req.tls13_ciphers(list);
}
if let Some(cert) = &args.cert {
let (cert_path, inline_pass) = split_cert_pass(cert);
req = req.client_cert(cert_path);
if let Some(key) = &args.key_file {
req = req.client_key(key);
}
if let Some(pass) = args.key_pass.as_deref().or(inline_pass) {
req = req.client_key_pass(pass);
}
if args.cert_type_der {
req = req.cert_type_der(true);
}
if args.key_type_der {
req = req.key_type_der(true);
}
}
if let Some(secs) = args.max_time {
req = req.max_time(Duration::from_secs(secs));
}
if let Some(secs) = args.connect_timeout {
req = req.connect_timeout(Duration::from_secs(secs));
}
if args.ipv6 {
req = req.ipv6();
} else if args.ipv4 {
req = req.ipv4();
}
for (h, p, ip) in &args.resolve {
req = req.resolve_addr(h, *p, *ip);
}
let proxy_spec = resolve_proxy_spec(&parsed_url, args);
if let Some(spec) = proxy_spec {
req = match req.proxy(&spec) {
Ok(r) => r,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: --proxy: {e}");
}
return 5;
}
};
if let Some((u, p)) = &args.proxy_user {
req = match req.proxy_user(u, p) {
Ok(r) => r,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: --proxy-user: {e}");
}
return 5;
}
};
}
}
if let Some(list) = resolve_noproxy(args) {
req = req.no_proxy(list.split(',').map(str::trim).filter(|s| !s.is_empty()));
}
if let (Some(j), Some(data)) = (jar.as_deref_mut(), args.cookie_in.as_deref()) {
if data.contains('=') {
apply_explicit_cookies(j, data, &parsed_url);
}
}
if streams_to_file(args) {
return run_http_download(req, &parsed_url, args, jar);
}
let started = std::time::Instant::now();
let mut jar = jar;
let mut attempt = 0u32;
let resp = loop {
let attempt_req = req.clone();
let result = match (jar.as_deref_mut(), args.verbose) {
(Some(j), true) => {
let mut err = io::stderr().lock();
attempt_req.send_traced_with_jar(j, &mut err)
}
(Some(j), false) => attempt_req.send_with_jar(j),
(None, true) => {
let mut err = io::stderr().lock();
attempt_req.send_traced(&mut err)
}
(None, false) => attempt_req.send(),
};
let within_budget = args
.retry_max_time
.is_none_or(|m| started.elapsed().as_secs() < m);
match result {
Ok(r) if is_retryable_status(r.status) && attempt < args.retry && within_budget => {
attempt += 1;
if show_errors(args) {
eprintln!(
"rsurl: transient HTTP {} — retry {}/{}",
r.status, attempt, args.retry
);
}
std::thread::sleep(next_retry_delay(attempt, args));
}
Ok(r) => break r,
Err(e) if attempt < args.retry && within_budget && should_retry_err(&e, args) => {
attempt += 1;
if show_errors(args) {
eprintln!("rsurl: {e} — retry {}/{}", attempt, args.retry);
}
std::thread::sleep(next_retry_delay(attempt, args));
}
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return transfer_exit_code(&e);
}
}
};
let time_total = started.elapsed();
if let Some(max) = args.max_filesize {
let declared = resp
.header("content-length")
.and_then(|v| v.trim().parse::<u64>().ok());
if declared.is_some_and(|n| n > max) || resp.body.len() as u64 > max {
if show_errors(args) {
eprintln!("rsurl: Maximum file size exceeded");
}
return 63;
}
}
if let Some(path) = &args.dump_header {
if let Err(e) = dump_headers(&resp, path) {
if show_errors(args) {
eprintln!("rsurl: dump-header {path}: {e}");
}
return 23;
}
}
if args.fail_with_body && resp.status >= 400 {
if show_errors(args) {
eprintln!(
"rsurl: The requested URL returned error: {} {}",
resp.status, resp.reason
);
}
let _ = write_output(&resp, &parsed_url, args);
run_write_out(&resp, &parsed_url, args, time_total, resp.body.len() as u64);
return 22;
}
if args.fail && resp.status >= 400 {
if show_errors(args) {
eprintln!(
"rsurl: The requested URL returned error: {} {}",
resp.status, resp.reason
);
}
run_write_out(&resp, &parsed_url, args, time_total, resp.body.len() as u64);
return 22;
}
if let Err(e) = write_output(&resp, &parsed_url, args) {
if show_errors(args) {
eprintln!("rsurl: write error: {e}");
}
return 23;
}
if args.remote_time {
if let Some(path) = args.output.as_deref().filter(|p| *p != "-") {
set_remote_time(&resp, path);
} else if args.remote_name {
if let Ok(name) = remote_name_from_url(&parsed_url) {
set_remote_time(&resp, &name);
}
}
}
run_write_out(&resp, &parsed_url, args, time_total, resp.body.len() as u64);
0
}
fn parse_args(raw: &[String]) -> Result<Args, String> {
let mut a = Args::default();
let mut it = raw.iter();
while let Some(arg) = it.next() {
match arg.as_str() {
"-h" | "--help" => {
print_usage();
std::process::exit(0);
}
"-V" | "--version" => {
println!("rsurl {VERSION}");
std::process::exit(0);
}
"-o" | "--output" => {
a.output = Some(next_val(&mut it, arg)?);
}
"-i" | "--include" => a.include_headers = true,
"-I" | "--head" => {
a.head = true;
a.include_headers = true;
}
"-v" | "--verbose" => a.verbose = true,
"-s" | "--silent" => a.silent = true,
"-X" | "--request" => a.method = Some(next_val(&mut it, arg)?),
"-H" | "--header" => {
let h = next_val(&mut it, arg)?;
let (k, v) = h
.split_once(':')
.ok_or_else(|| format!("malformed header: {h:?}"))?;
a.headers.push((k.trim().to_string(), v.trim().to_string()));
}
"-d" | "--data" | "--data-ascii" => a.data_parts.push(DataPart::Plain {
value: next_val(&mut it, arg)?,
at_file_ok: true,
}),
"--remove-on-error" => a.remove_on_error = true,
"--no-clobber" => a.no_clobber = true,
"--disable-epsv" => a.disable_epsv = true,
"--ftp-create-dirs" => a.ftp_create_dirs = true,
"-P" | "--ftp-port" => a.ftp_port = Some(next_val(&mut it, arg)?),
"--ftp-pasv" => {}
"--epsv" => a.disable_epsv = false,
"--basic" | "--ftp-skip-pasv-ip" => {}
"--json" => a.json_parts.push(next_val(&mut it, arg)?),
"--oauth2-bearer" => a.bearer = Some(next_val(&mut it, arg)?),
"--aws-sigv4" => a.aws_sigv4 = Some(next_val(&mut it, arg)?),
"--data-raw" => a.data_parts.push(DataPart::Plain {
value: next_val(&mut it, arg)?,
at_file_ok: false,
}),
"--data-binary" => a.data_parts.push(DataPart::Binary {
value: next_val(&mut it, arg)?,
}),
"--data-urlencode" => a.data_parts.push(DataPart::UrlEncoded {
value: next_val(&mut it, arg)?,
}),
"-F" | "--form" => {
let v = next_val(&mut it, arg)?;
a.form_parts.push(form_parser::parse(&v)?);
}
"--form-string" => {
let v = next_val(&mut it, arg)?;
let eq = v
.find('=')
.ok_or_else(|| format!("--form-string: expected 'name=value', got {v:?}"))?;
let name = v[..eq].to_string();
if name.is_empty() {
return Err(format!("--form-string: empty field name: {v:?}"));
}
let value = v[eq + 1..].to_string();
a.form_parts.push(FormPart {
name,
body: FormBody::LiteralStrict(value),
extras: vec![],
});
}
"--form-escape" => a.form_escape = true,
"-T" | "--upload-file" => a.upload_file = Some(next_val(&mut it, arg)?),
"-C" | "--continue-at" => {
let v = next_val(&mut it, arg)?;
if v == "-" {
return Err(
"-C -: automatic resume is not supported; pass an explicit byte offset"
.into(),
);
}
a.continue_at = Some(
v.parse::<u64>()
.map_err(|_| format!("-C/--continue-at: not a byte offset: {v:?}"))?,
);
}
"-a" | "--append" => a.append = true,
"--key" => {
let path = next_val(&mut it, arg)?;
a.key_file = Some(path.clone());
a.ssh_keys.push(path);
}
"-A" | "--user-agent" => a.user_agent = Some(next_val(&mut it, arg)?),
"-e" | "--referer" => {
let v = next_val(&mut it, arg)?;
let (head, auto) = match v.strip_suffix(";auto") {
Some(h) => (h, true),
None => (v.as_str(), false),
};
a.auto_referer = a.auto_referer || auto;
if !head.is_empty() {
a.referer = Some(head.to_string());
}
}
"-z" | "--time-cond" => a.time_cond = Some(next_val(&mut it, arg)?),
"--output-dir" => a.output_dir = Some(next_val(&mut it, arg)?),
"--fail-with-body" => a.fail_with_body = true,
"--proto" => a.proto = Some(next_val(&mut it, arg)?),
"--proto-default" => a.proto_default = Some(next_val(&mut it, arg)?),
"--http2" => a.http_version = Some(HttpVersionPref::Http2Only),
"--http1.1" | "--http1" => a.http_version = Some(HttpVersionPref::Http11Only),
"--http3" => a.http_version = Some(HttpVersionPref::Http3),
"--http3-only" => a.http_version = Some(HttpVersionPref::Http3Only),
"-L" | "--location" => a.follow_redirects = true,
"--max-redirs" => {
let v = next_val(&mut it, arg)?;
a.max_redirs = Some(
v.parse::<u32>()
.map_err(|_| format!("--max-redirs: not a number: {v:?}"))?,
);
}
"-u" | "--user" => {
let v = next_val(&mut it, arg)?;
let (u, p) = match v.split_once(':') {
Some((u, p)) => (u.to_string(), p.to_string()),
None => (v.clone(), String::new()),
};
a.basic_auth = Some((u, p));
}
"-k" | "--insecure" => a.insecure = true,
"--tlsv1" | "--tlsv1.0" | "--tlsv1.1" | "--tlsv1.2" => {
a.tls_min = Some(rsurl::tls::ProtocolVersion::TLSv1_2)
}
"--tlsv1.3" => a.tls_min = Some(rsurl::tls::ProtocolVersion::TLSv1_3),
"--mail-from" => a.mail_from = Some(next_val(&mut it, arg)?),
"--mail-rcpt" => a.mail_rcpt.push(next_val(&mut it, arg)?),
"--digest" => a.digest = true,
"-Z" | "--parallel" => a.parallel = true,
"--parallel-max" => {
a.parallel_max = Some(
next_val(&mut it, arg)?
.parse()
.map_err(|_| "--parallel-max requires a number".to_string())?,
)
}
"--tls-max" => {
let v = next_val(&mut it, arg)?;
a.tls_max = Some(match v.as_str() {
"1.3" => rsurl::tls::ProtocolVersion::TLSv1_3,
"1.0" | "1.1" | "1.2" => rsurl::tls::ProtocolVersion::TLSv1_2,
other => return Err(format!("--tls-max: unsupported version {other:?}")),
});
}
"--no-idn" => a.no_idn = true,
"--cacert" => a.cacert = Some(next_val(&mut it, arg)?),
"--max-time" => {
let v = next_val(&mut it, arg)?;
a.max_time = Some(
v.parse::<u64>()
.map_err(|_| format!("--max-time: not a number: {v:?}"))?,
);
}
"--connect-timeout" => {
let v = next_val(&mut it, arg)?;
a.connect_timeout = Some(
v.parse::<u64>()
.map_err(|_| format!("--connect-timeout: not a number: {v:?}"))?,
);
}
"-O" | "--remote-name" => a.remote_name = true,
"-b" | "--cookie" => a.cookie_in = Some(next_val(&mut it, arg)?),
"-c" | "--cookie-jar" => a.cookie_jar = Some(next_val(&mut it, arg)?),
"-x" | "--proxy" => a.proxy = Some(next_val(&mut it, arg)?),
"--socks4" => a.proxy = Some(format!("socks4://{}", next_val(&mut it, arg)?)),
"--socks4a" => a.proxy = Some(format!("socks4a://{}", next_val(&mut it, arg)?)),
"--socks5" => a.proxy = Some(format!("socks5://{}", next_val(&mut it, arg)?)),
"--socks5-hostname" => a.proxy = Some(format!("socks5h://{}", next_val(&mut it, arg)?)),
"-U" | "--proxy-user" => {
let v = next_val(&mut it, arg)?;
let (u, p) = match v.split_once(':') {
Some((u, p)) => (u.to_string(), p.to_string()),
None => (v.clone(), String::new()),
};
a.proxy_user = Some((u, p));
}
"--noproxy" => a.noproxy = Some(next_val(&mut it, arg)?),
"--url" => a.urls.push(next_val(&mut it, arg)?),
"-f" | "--fail" => a.fail = true,
"-S" | "--show-error" => a.show_error = true,
"-G" | "--get" => a.get = true,
"-r" | "--range" => a.range = Some(next_val(&mut it, arg)?),
"--compressed" => a.compressed = true,
"-D" | "--dump-header" => a.dump_header = Some(next_val(&mut it, arg)?),
"-R" | "--remote-time" => a.remote_time = true,
"--create-dirs" => a.create_dirs = true,
"--max-filesize" => {
a.max_filesize = Some(
next_val(&mut it, arg)?
.parse()
.map_err(|_| "--max-filesize requires a byte count".to_string())?,
)
}
"-w" | "--write-out" => a.write_out = Some(next_val(&mut it, arg)?),
"-n" | "--netrc" => a.netrc = true,
"--netrc-file" => {
a.netrc_file = Some(next_val(&mut it, arg)?);
a.netrc = true;
}
"-J" | "--remote-header-name" => a.remote_header_name = true,
"--retry" => {
a.retry = next_val(&mut it, arg)?
.parse()
.map_err(|_| "--retry requires a count".to_string())?
}
"--retry-delay" => {
a.retry_delay = Some(
next_val(&mut it, arg)?
.parse()
.map_err(|_| "--retry-delay requires seconds".to_string())?,
)
}
"--retry-max-time" => {
a.retry_max_time = Some(
next_val(&mut it, arg)?
.parse()
.map_err(|_| "--retry-max-time requires seconds".to_string())?,
)
}
"--retry-connrefused" => a.retry_connrefused = true,
"--retry-all-errors" => a.retry_all_errors = true,
"-g" | "--globoff" => a.globoff = true,
"--unix-socket" | "--abstract-unix-socket" => {
a.unix_socket = Some(next_val(&mut it, arg)?)
}
"--location-trusted" => {
a.follow_redirects = true;
a.location_trusted = true;
}
"--post301" => a.post301 = true,
"--post302" => a.post302 = true,
"--post303" => a.post303 = true,
"--connect-to" => {
let spec = next_val(&mut it, arg)?;
let p: Vec<&str> = spec.split(':').collect();
if p.len() != 4 {
return Err(format!(
"--connect-to expects HOST1:PORT1:HOST2:PORT2: {spec:?}"
));
}
let port = |s: &str, what: &str| -> Result<u16, String> {
if s.is_empty() {
Ok(0)
} else {
s.parse()
.map_err(|_| format!("--connect-to: bad {what} in {spec:?}"))
}
};
a.connect_to.push((
p[0].to_string(),
port(p[1], "PORT1")?,
p[2].to_string(),
port(p[3], "PORT2")?,
));
}
"-4" | "--ipv4" => a.ipv4 = true,
"-6" | "--ipv6" => a.ipv6 = true,
"-#" | "--progress-bar" => a.progress_bar = true,
"-E" | "--cert" => a.cert = Some(next_val(&mut it, arg)?),
"--pass" => a.key_pass = Some(next_val(&mut it, arg)?),
"--cert-type" => a.cert_type_der = parse_cert_type(&next_val(&mut it, arg)?, arg)?,
"--key-type" => a.key_type_der = parse_cert_type(&next_val(&mut it, arg)?, arg)?,
"--pinnedpubkey" => a.pinned_pubkey = Some(next_val(&mut it, arg)?),
"--capath" => a.capath = Some(next_val(&mut it, arg)?),
"--crlfile" => a.crl_file = Some(next_val(&mut it, arg)?),
"--limit-rate" => a.limit_rate = Some(next_val(&mut it, arg)?),
"-Y" | "--speed-limit" => a.speed_limit = Some(next_val(&mut it, arg)?),
"-y" | "--speed-time" => a.speed_time = Some(next_val(&mut it, arg)?),
"--resolve" => {
let spec = next_val(&mut it, arg)?;
let mut parts = spec.splitn(3, ':');
let host = parts
.next()
.filter(|h| !h.is_empty())
.ok_or_else(|| format!("--resolve: missing host in {spec:?}"))?
.trim_start_matches(['+', '-']);
let port: u16 = parts
.next()
.and_then(|p| p.parse().ok())
.ok_or_else(|| format!("--resolve: bad port in {spec:?}"))?;
let addr_s = parts
.next()
.ok_or_else(|| format!("--resolve: missing address in {spec:?}"))?;
let addr_s = addr_s.trim().trim_start_matches('[').trim_end_matches(']');
let ip: std::net::IpAddr = addr_s
.parse()
.map_err(|_| format!("--resolve: bad IP {addr_s:?}"))?;
a.resolve.push((host.to_string(), port, ip));
}
"-q"
| "--disable"
| "-N"
| "--no-buffer"
| "--no-progress-meter"
| "--styled-output"
| "--no-styled-output" => {}
"--ciphers" => a.ciphers = Some(next_val(&mut it, arg)?),
"--tls13-ciphers" => a.tls13_ciphers = Some(next_val(&mut it, arg)?),
"--cert-status" => {
return Err(
"--cert-status: OCSP-staple validation is not implemented (rsurl does not \
request or require a stapled response)"
.to_string(),
);
}
s if s.starts_with("--") => return Err(format!("unknown option: {s}")),
s if s.starts_with('-') && s.len() > 1 => return Err(format!("unknown option: {s}")),
_ => {
a.urls.push(arg.clone());
}
}
}
Ok(a)
}
fn show_errors(args: &Args) -> bool {
!args.silent || args.show_error
}
fn is_retryable_status(status: u16) -> bool {
matches!(status, 408 | 429 | 500 | 502 | 503 | 504)
}
fn retry_delay(attempt: u32) -> std::time::Duration {
let secs = 1u64
.checked_shl(attempt.saturating_sub(1))
.unwrap_or(60)
.min(60);
std::time::Duration::from_secs(secs)
}
fn next_retry_delay(attempt: u32, args: &Args) -> std::time::Duration {
match args.retry_delay {
Some(s) => std::time::Duration::from_secs(s),
None => retry_delay(attempt),
}
}
fn transfer_exit_code(e: &rsurl::Error) -> u8 {
use std::io::ErrorKind;
match e {
rsurl::Error::InvalidUrl(_) => 3, rsurl::Error::UnsupportedScheme(_) => 1, rsurl::Error::UnexpectedEof => 52, rsurl::Error::Ssh(_) => 79, rsurl::Error::H2NotNegotiated => 7,
rsurl::Error::BadResponse(m) => {
let m = m.to_ascii_lowercase();
if m.contains("timed out") {
28 } else if m.contains("redirect") {
47 } else {
8 }
}
rsurl::Error::Io(io) => match io.kind() {
ErrorKind::TimedOut => 28,
ErrorKind::ConnectionRefused
| ErrorKind::ConnectionReset
| ErrorKind::ConnectionAborted => 7,
_ if io.to_string().contains("failed to lookup") => 6, _ => 7, },
}
}
fn should_retry_err(e: &rsurl::Error, args: &Args) -> bool {
if args.retry_all_errors {
return true;
}
match e {
rsurl::Error::Io(io) => {
let k = io.kind();
matches!(
k,
std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock
) || (args.retry_connrefused && k == std::io::ErrorKind::ConnectionRefused)
}
rsurl::Error::UnexpectedEof => true,
_ => false,
}
}
fn netrc_credentials(args: &Args, host: &str) -> Option<(String, String)> {
let path: std::path::PathBuf = match &args.netrc_file {
Some(p) => p.into(),
None => {
let home = std::env::var_os("HOME")?;
let mut p = std::path::PathBuf::from(home);
p.push(".netrc");
p
}
};
let text = std::fs::read_to_string(&path).ok()?;
netrc_lookup(&text, host)
}
fn netrc_lookup(text: &str, host: &str) -> Option<(String, String)> {
let mut toks = text.split_whitespace();
let mut entries: Vec<(String, Option<String>, Option<String>)> = Vec::new();
while let Some(t) = toks.next() {
match t {
"machine" => {
if let Some(n) = toks.next() {
entries.push((n.to_string(), None, None));
}
}
"default" => entries.push(("\0default".to_string(), None, None)),
"login" => {
if let (Some(v), Some(e)) = (toks.next(), entries.last_mut()) {
e.1 = Some(v.to_string());
}
}
"password" => {
if let (Some(v), Some(e)) = (toks.next(), entries.last_mut()) {
e.2 = Some(v.to_string());
}
}
"account" | "macdef" => {
let _ = toks.next();
}
_ => {}
}
}
let pick = |e: &(String, Option<String>, Option<String>)| {
(
e.1.clone().unwrap_or_default(),
e.2.clone().unwrap_or_default(),
)
};
entries
.iter()
.find(|e| e.0.eq_ignore_ascii_case(host))
.or_else(|| entries.iter().find(|e| e.0 == "\0default"))
.map(pick)
}
fn content_disposition_filename(resp: &Response) -> Option<String> {
let cd = resp.header("content-disposition")?;
for part in cd.split(';') {
let p = part.trim();
let Some(val) = p
.strip_prefix("filename*=")
.or_else(|| p.strip_prefix("filename="))
else {
continue;
};
let val = val.trim().trim_matches('"');
let val = val.rsplit("''").next().unwrap_or(val);
let name = std::path::Path::new(val).file_name()?.to_str()?.to_string();
if name.is_empty() || name == "." || name == ".." {
return None;
}
return Some(name);
}
None
}
fn next_val(it: &mut std::slice::Iter<'_, String>, flag: &str) -> Result<String, String> {
it.next()
.cloned()
.ok_or_else(|| format!("{flag} requires a value"))
}
fn run_ftp_upload(url: &Url, path: &str, args: &Args) -> u8 {
let bytes = match std::fs::read(path) {
Ok(b) => b,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: -T: can't read {path:?}: {e}");
}
return 26;
}
};
let client = match transfer_client(url, args) {
Ok(c) => c,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: --proxy: {e}");
}
return 5;
}
};
let result = if args.append {
client.ftp_append(url, &bytes)
} else {
let (body, resume_at): (&[u8], Option<u64>) = match args.continue_at {
Some(off) => {
let off_usize = off as usize;
if off_usize > bytes.len() {
if show_errors(args) {
eprintln!(
"rsurl: -C {off}: offset is past the end of {path:?} ({} bytes)",
bytes.len()
);
}
return 2;
}
(&bytes[off_usize..], Some(off))
}
None => (&bytes[..], None),
};
client.ftp_store(url, body, resume_at)
};
match result {
Ok(()) => 0,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
transfer_exit_code(&e)
}
}
}
fn run_tftp_upload(url: &Url, path: &str, args: &Args) -> u8 {
let bytes = match std::fs::read(path) {
Ok(b) => b,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: -T: can't read {path:?}: {e}");
}
return 26;
}
};
match rsurl::tftp::store(url, &bytes) {
Ok(()) => 0,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
7
}
}
}
fn build_ssh_options(url: &Url, args: &Args) -> Result<(rsurl::ssh::SshOptions, String), String> {
let (_, url_pass) = rsurl::ssh::userinfo_password(url);
let (cli_user, cli_pass) = match &args.basic_auth {
Some((u, p)) => (
(!u.is_empty()).then(|| u.clone()),
(!p.is_empty()).then(|| p.clone()),
),
None => (None, None),
};
let password = url_pass.or(cli_pass);
let user = rsurl::ssh::resolve_user(url, cli_user.as_deref()).map_err(|e| e.to_string())?;
let opts = rsurl::ssh::SshOptions {
password: password.clone(),
identity_files: args.ssh_keys.iter().map(std::path::PathBuf::from).collect(),
key_passphrase: password,
insecure: args.insecure,
known_hosts_path: None,
timeout: args.max_time.map(Duration::from_secs),
};
Ok((opts, user))
}
fn run_ssh(url: &Url, args: &Args) -> u8 {
let (opts, user) = match build_ssh_options(url, args) {
Ok(x) => x,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 2;
}
};
let result = if args.verbose {
let mut err = io::stderr().lock();
rsurl::ssh::fetch_traced(url, &opts, &user, Some(&mut err))
} else {
rsurl::ssh::fetch(url, &opts, &user)
};
match result {
Ok(bytes) => {
let mut out: Box<dyn Write> = if args.remote_name {
match remote_name_from_url(url) {
Ok(name) => match File::create(&name) {
Ok(f) => Box::new(f),
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: open {name}: {e}");
}
return 23;
}
},
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 23;
}
}
} else {
match &args.output {
Some(path) if path != "-" => match create_output_file(path, args) {
Ok(f) => Box::new(f),
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: open {path}: {e}");
}
return 23;
}
},
_ => Box::new(io::stdout().lock()),
}
};
if let Err(e) = out.write_all(&bytes) {
if show_errors(args) {
eprintln!("rsurl: write error: {e}");
}
return 23;
}
0
}
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
7
}
}
}
fn run_ssh_upload(url: &Url, path: &str, args: &Args) -> u8 {
let bytes = match std::fs::read(path) {
Ok(b) => b,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: -T: can't read {path:?}: {e}");
}
return 26;
}
};
let (opts, user) = match build_ssh_options(url, args) {
Ok(x) => x,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 2;
}
};
let result = if args.verbose {
let mut err = io::stderr().lock();
rsurl::ssh::upload_traced(url, &bytes, &opts, &user, Some(&mut err))
} else {
rsurl::ssh::upload(url, &bytes, &opts, &user)
};
match result {
Ok(()) => 0,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
7
}
}
}
fn run_mqtt_publish(url: &Url, args: &Args) -> u8 {
if args.upload_file.is_some() && !args.data_parts.is_empty() {
if show_errors(args) {
eprintln!("rsurl: -d/--data and -T/--upload-file are mutually exclusive");
}
return 2;
}
let payload: Vec<u8> = if let Some(path) = &args.upload_file {
match std::fs::read(path) {
Ok(b) => b,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: -T: can't read {path:?}: {e}");
}
return 26;
}
}
} else {
match assemble_form_body(&args.data_parts) {
Ok(Some(b)) => b,
Ok(None) => Vec::new(),
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 2;
}
}
};
match rsurl::mqtt::publish(url, &payload, 0) {
Ok(()) => 0,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
7
}
}
}
fn run_rtsp(url: &Url, args: &Args) -> u8 {
let method = args.method.as_deref().unwrap_or("DESCRIBE");
match rsurl::rtsp::run_method(url, method) {
Ok(bytes) => {
let mut out: Box<dyn Write> = match &args.output {
Some(path) if path != "-" => match create_output_file(path, args) {
Ok(f) => Box::new(f),
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: open {path}: {e}");
}
return 23;
}
},
_ => Box::new(io::stdout().lock()),
};
if let Err(e) = out.write_all(&bytes) {
if show_errors(args) {
eprintln!("rsurl: write error: {e}");
}
return 23;
}
0
}
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
7
}
}
}
fn transfer_client(url: &Url, args: &Args) -> rsurl::Result<rsurl::Client> {
let mut c = rsurl::Client::new()
.verify_tls(!args.insecure)
.idn(!args.no_idn)
.ftp_use_epsv(!args.disable_epsv)
.ftp_create_dirs(args.ftp_create_dirs)
.ftp_active(args.ftp_port.is_some());
if let Some(secs) = args.connect_timeout {
c = c.connect_timeout(Some(Duration::from_secs(secs)));
}
if let Some(spec) = resolve_proxy_spec(url, args) {
c = c.proxy(&spec)?;
}
if let Some(list) = resolve_noproxy(args) {
c = c.no_proxy(
list.split(',')
.map(str::trim)
.filter(|s| !s.is_empty())
.map(String::from)
.collect::<Vec<_>>(),
);
}
if let Some(path) = &args.unix_socket {
#[cfg(unix)]
{
c = c.connector(std::sync::Arc::new(rsurl::net::UnixConnector {
path: path.into(),
}));
}
#[cfg(not(unix))]
{
let _ = path;
return Err(rsurl::Error::UnsupportedScheme(
"--unix-socket is not supported on this platform".into(),
));
}
}
Ok(c)
}
fn run_smtp(url: &Url, args: &Args) -> u8 {
let Some(from) = args.mail_from.as_deref() else {
if show_errors(args) {
eprintln!("rsurl: smtp requires --mail-from");
}
return 2;
};
if args.mail_rcpt.is_empty() {
if show_errors(args) {
eprintln!("rsurl: smtp requires at least one --mail-rcpt");
}
return 2;
}
let body: Vec<u8> = if let Some(path) = &args.upload_file {
match std::fs::read(path) {
Ok(b) => b,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: read {path}: {e}");
}
return 2;
}
}
} else if !args.data_parts.is_empty() {
match assemble_request_body(args) {
Ok(Some((b, _, _))) => b,
_ => Vec::new(),
}
} else {
if show_errors(args) {
eprintln!("rsurl: smtp needs a message body (-T <file> or -d)");
}
return 2;
};
let (user, pass) = match url.userinfo.as_deref() {
Some(ui) => match ui.split_once(':') {
Some((u, p)) => (Some(u.to_string()), Some(p.to_string())),
None => (Some(ui.to_string()), None),
},
None => (None, None),
};
let client = match transfer_client(url, args) {
Ok(c) => c,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 5;
}
};
match client.smtp_send(
url,
&body,
from,
&args.mail_rcpt,
user.as_deref(),
pass.as_deref(),
) {
Ok(()) => 0,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
7
}
}
}
fn run_telnet(url: &Url, args: &Args) -> u8 {
let input: Vec<u8> = if let Some(path) = &args.upload_file {
std::fs::read(path).unwrap_or_default()
} else if !args.data_parts.is_empty() {
assemble_request_body(args)
.ok()
.flatten()
.map(|(b, _, _)| b)
.unwrap_or_default()
} else {
Vec::new()
};
let client = match transfer_client(url, args) {
Ok(c) => c,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 5;
}
};
match client.telnet(url, &input) {
Ok(bytes) => {
let mut out: Box<dyn Write> = match &args.output {
Some(path) if path != "-" => match create_output_file(path, args) {
Ok(f) => Box::new(f),
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: open {path}: {e}");
}
return 23;
}
},
_ => Box::new(io::stdout().lock()),
};
if out.write_all(&bytes).is_err() {
return 23;
}
0
}
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
7
}
}
}
fn run_transfer(url: &Url, args: &Args) -> u8 {
let client = match transfer_client(url, args) {
Ok(c) => c,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: --proxy: {e}");
}
return 5;
}
};
match client.transfer_url(url) {
Ok(bytes) => {
let mut out: Box<dyn Write> = match &args.output {
Some(path) if path != "-" => match create_output_file(path, args) {
Ok(f) => Box::new(f),
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: open {path}: {e}");
}
return 23;
}
},
_ => Box::new(io::stdout().lock()),
};
if let Err(e) = out.write_all(&bytes) {
if show_errors(args) {
eprintln!("rsurl: write error: {e}");
}
return 23;
}
0
}
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
transfer_exit_code(&e)
}
}
}
fn run_stream_download(url: &Url, args: &Args) -> u8 {
let client = match transfer_client(url, args) {
Ok(c) => c,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: --proxy: {e}");
}
return 5;
}
};
let name = if args.remote_name {
match remote_name_from_url(url) {
Ok(n) => n,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 23;
}
}
} else {
args.output.clone().unwrap_or_default()
};
let (file, out_path) = match create_output_file_tracked(&name, args) {
Ok(pair) => pair,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: open {name}: {e}");
}
return 23;
}
};
let (speed_limit, speed_time) = {
let lim = args
.speed_limit
.as_deref()
.and_then(|s| s.parse::<u64>().ok());
let tim = args
.speed_time
.as_deref()
.and_then(|s| s.parse::<u64>().ok());
match (lim, tim) {
(None, None) => (None, 30),
(l, t) => (Some(l.unwrap_or(1)), t.unwrap_or(30)),
}
};
let now = std::time::Instant::now();
let mut sink = DownloadSink {
inner: Box::new(file),
written: 0,
max: args.max_filesize,
rate: args.limit_rate.as_deref().and_then(parse_rate),
speed_limit,
speed_time,
started: now,
progress: args.progress_bar,
silent: args.silent,
last_tick: now,
};
let result = client.transfer_url_to(url, &mut sink);
let time_total = now.elapsed();
let written = sink.written;
if args.progress_bar && !args.silent {
eprintln!();
}
match result {
Ok(_) => {
if args.write_out.is_some() {
let resp = rsurl::Response {
status: 0,
reason: String::new(),
version: String::new(),
headers: Vec::new(),
body: Vec::new(),
timing: rsurl::Timing::default(),
};
run_write_out(&resp, url, args, time_total, written);
}
0
}
Err(e) => {
if args.remove_on_error {
drop(sink);
let _ = std::fs::remove_file(&out_path);
}
if e.to_string().contains(MAX_FILESIZE_SENTINEL) {
if show_errors(args) {
eprintln!("rsurl: Maximum file size exceeded");
}
return 63;
}
if e.to_string().contains(LOW_SPEED_SENTINEL) {
if show_errors(args) {
eprintln!(
"rsurl: Operation too slow. Less than {} bytes/sec transferred \
the last {speed_time} seconds",
speed_limit.unwrap_or(1)
);
}
return 28;
}
if show_errors(args) {
eprintln!("rsurl: {e}");
}
transfer_exit_code(&e)
}
}
}
fn sanitize_for_tty(bytes: &[u8]) -> Vec<u8> {
let mut out = Vec::with_capacity(bytes.len());
let mut rest = bytes;
while !rest.is_empty() {
match std::str::from_utf8(rest) {
Ok(s) => {
push_sanitized_str(s, &mut out);
break;
}
Err(e) => {
let valid_up_to = e.valid_up_to();
if valid_up_to > 0 {
let s = unsafe { std::str::from_utf8_unchecked(&rest[..valid_up_to]) };
push_sanitized_str(s, &mut out);
}
let bad = e.error_len().unwrap_or(1);
for &b in &rest[valid_up_to..valid_up_to + bad] {
out.extend_from_slice(format!("\\x{b:02x}").as_bytes());
}
rest = &rest[valid_up_to + bad..];
}
}
}
out
}
fn push_sanitized_str(s: &str, out: &mut Vec<u8>) {
for ch in s.chars() {
let cp = ch as u32;
let dangerous = (cp < 0x20 && ch != '\t') || (0x7f..=0x9f).contains(&cp);
if dangerous {
out.extend_from_slice(format!("\\x{cp:02x}").as_bytes());
} else {
let mut buf = [0u8; 4];
out.extend_from_slice(ch.encode_utf8(&mut buf).as_bytes());
}
}
}
fn body_looks_binary(body: &[u8]) -> bool {
body.contains(&0)
}
fn fmt_secs(d: Option<std::time::Duration>) -> String {
format!("{:.6}", d.map_or(0.0, |d| d.as_secs_f64()))
}
fn run_write_out(
resp: &Response,
url: &Url,
args: &Args,
time_total: std::time::Duration,
size_download: u64,
) {
let Some(fmt) = &args.write_out else { return };
let size_header: usize = resp
.headers
.iter()
.map(|(k, v)| k.len() + v.len() + 4)
.sum::<usize>()
+ resp.version.len()
+ resp.reason.len()
+ 6;
let var = |name: &str| -> String {
match name {
"http_code" | "response_code" => resp.status.to_string(),
"http_version" => resp.version.clone(),
"size_download" => size_download.to_string(),
"size_header" => size_header.to_string(),
"num_headers" => resp.headers.len().to_string(),
"content_type" => resp.header("content-type").unwrap_or("").to_string(),
"ssl_verify_result" => "0".to_string(),
"url_effective" => {
let default = matches!(
(url.scheme.as_str(), url.port),
("http", 80) | ("https", 443)
);
if default {
format!("{}://{}{}", url.scheme, url.host, url.path)
} else {
format!("{}://{}:{}{}", url.scheme, url.host, url.port, url.path)
}
}
"scheme" => url.scheme.to_uppercase(),
"time_total" => format!("{:.6}", time_total.as_secs_f64()),
"time_namelookup" => fmt_secs(None), "time_connect" => fmt_secs(resp.timing.connect),
"time_appconnect" => fmt_secs(resp.timing.appconnect),
"time_pretransfer" => fmt_secs(resp.timing.pretransfer),
"time_starttransfer" => fmt_secs(resp.timing.starttransfer),
_ => String::new(),
}
};
let mut out = String::with_capacity(fmt.len());
let mut chars = fmt.chars().peekable();
while let Some(c) = chars.next() {
match c {
'%' => match chars.peek().copied() {
Some('{') => {
chars.next();
let mut name = String::new();
for nc in chars.by_ref() {
if nc == '}' {
break;
}
name.push(nc);
}
out.push_str(&var(&name));
}
Some('%') => {
chars.next();
out.push('%');
}
_ if chars.clone().take(7).collect::<String>() == "header{" => {
for _ in 0..7 {
chars.next();
}
let mut name = String::new();
for nc in chars.by_ref() {
if nc == '}' {
break;
}
name.push(nc);
}
out.push_str(resp.header(name.trim()).unwrap_or(""));
}
_ => out.push('%'),
},
'\\' => match chars.next() {
Some('n') => out.push('\n'),
Some('t') => out.push('\t'),
Some('r') => out.push('\r'),
Some('\\') => out.push('\\'),
Some(other) => {
out.push('\\');
out.push(other);
}
None => out.push('\\'),
},
other => out.push(other),
}
}
print!("{out}");
let _ = io::stdout().flush();
}
fn httpdate_to_epoch(s: &str) -> Option<u64> {
let rest = s
.trim()
.split_once(", ")
.map(|(_, r)| r)
.unwrap_or(s.trim());
let mut it = rest.split_whitespace();
let day: i64 = it.next()?.parse().ok()?;
let mon: i64 = match it.next()? {
"Jan" => 1,
"Feb" => 2,
"Mar" => 3,
"Apr" => 4,
"May" => 5,
"Jun" => 6,
"Jul" => 7,
"Aug" => 8,
"Sep" => 9,
"Oct" => 10,
"Nov" => 11,
"Dec" => 12,
_ => return None,
};
let year: i64 = it.next()?.parse().ok()?;
let mut hms = it.next()?.split(':');
let hh: i64 = hms.next()?.parse().ok()?;
let mm: i64 = hms.next()?.parse().ok()?;
let ss: i64 = hms.next()?.parse().ok()?;
let y = if mon <= 2 { year - 1 } else { year };
let era = (if y >= 0 { y } else { y - 399 }) / 400;
let yoe = y - era * 400;
let doy = (153 * (if mon > 2 { mon - 3 } else { mon + 9 }) + 2) / 5 + day - 1;
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
let days = era * 146097 + doe - 719468;
let secs = days * 86400 + hh * 3600 + mm * 60 + ss;
u64::try_from(secs).ok()
}
fn epoch_to_httpdate(secs: u64) -> String {
let days = (secs / 86400) as i64;
let rem = (secs % 86400) as i64;
let (hh, mm, ss) = (rem / 3600, (rem % 3600) / 60, rem % 60);
let wd = (days % 7 + 4).rem_euclid(7); let z = days + 719468;
let era = (if z >= 0 { z } else { z - 146096 }) / 146097;
let doe = z - era * 146097;
let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365;
let y = yoe + era * 400;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let d = doy - (153 * mp + 2) / 5 + 1;
let m = if mp < 10 { mp + 3 } else { mp - 9 };
let year = if m <= 2 { y + 1 } else { y };
const WD: [&str; 7] = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
const MON: [&str; 12] = [
"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
];
format!(
"{}, {:02} {} {:04} {:02}:{:02}:{:02} GMT",
WD[wd as usize],
d,
MON[(m - 1) as usize],
year,
hh,
mm,
ss
)
}
fn time_cond_header(spec: &str) -> Option<(&'static str, String)> {
let (unmod, body) = match spec.strip_prefix('-') {
Some(r) => (true, r),
None => (false, spec.strip_prefix('+').unwrap_or(spec)),
};
let date = match std::fs::metadata(body).and_then(|m| m.modified()) {
Ok(mtime) => {
let secs = mtime.duration_since(std::time::UNIX_EPOCH).ok()?.as_secs();
epoch_to_httpdate(secs)
}
Err(_) => body.trim().to_string(),
};
if date.is_empty() {
return None;
}
Some((
if unmod {
"If-Unmodified-Since"
} else {
"If-Modified-Since"
},
date,
))
}
fn proto_allowed(scheme: &str, spec: &str) -> bool {
const ALL: &[&str] = &[
"http", "https", "ftp", "ftps", "sftp", "scp", "imap", "imaps", "pop3", "pop3s", "smtp",
"smtps", "mqtt", "mqtts", "rtsp", "tftp", "ldap", "ldaps", "gopher", "gophers", "dict",
"file", "ws", "wss", "telnet",
];
let mut set: std::collections::HashSet<String> = ALL.iter().map(|s| s.to_string()).collect();
for tok in spec.split(',') {
let tok = tok.trim();
if tok.is_empty() {
continue;
}
let (op, name) = match tok.as_bytes()[0] {
b'=' => ('=', &tok[1..]),
b'+' => ('+', &tok[1..]),
b'-' => ('-', &tok[1..]),
_ => ('+', tok),
};
let names: Vec<String> = if name == "all" {
ALL.iter().map(|s| s.to_string()).collect()
} else {
vec![name.to_ascii_lowercase()]
};
match op {
'=' => {
set.clear();
set.extend(names);
}
'+' => set.extend(names),
'-' => {
for n in names {
set.remove(&n);
}
}
_ => {}
}
}
set.contains(&scheme.to_ascii_lowercase())
}
fn parse_rate(s: &str) -> Option<u64> {
let s = s.trim();
let (num, mult): (&str, u64) = match s.chars().last() {
Some('k') | Some('K') => (&s[..s.len() - 1], 1024),
Some('m') | Some('M') => (&s[..s.len() - 1], 1024 * 1024),
Some('g') | Some('G') => (&s[..s.len() - 1], 1024 * 1024 * 1024),
_ => (s, 1),
};
num.trim()
.parse::<u64>()
.ok()
.map(|n| n.saturating_mul(mult))
}
const MAX_FILESIZE_SENTINEL: &str = "rsurl-max-filesize-exceeded";
const LOW_SPEED_SENTINEL: &str = "rsurl-low-speed-abort";
struct DownloadSink<'a> {
inner: Box<dyn Write + 'a>,
written: u64,
max: Option<u64>,
rate: Option<u64>,
speed_limit: Option<u64>,
speed_time: u64,
started: std::time::Instant,
progress: bool,
silent: bool,
last_tick: std::time::Instant,
}
impl Write for DownloadSink<'_> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
if let Some(max) = self.max {
if self.written + buf.len() as u64 > max {
return Err(io::Error::other(MAX_FILESIZE_SENTINEL));
}
}
if let Some(rate) = self.rate.filter(|r| *r > 0) {
let target = std::time::Duration::from_secs_f64(
(self.written + buf.len() as u64) as f64 / rate as f64,
);
let elapsed = self.started.elapsed();
if target > elapsed {
std::thread::sleep(target - elapsed);
}
}
self.inner.write_all(buf)?;
self.written += buf.len() as u64;
if let Some(limit) = self.speed_limit {
let secs = self.started.elapsed().as_secs();
if secs >= self.speed_time && self.written / secs.max(1) < limit {
return Err(io::Error::other(LOW_SPEED_SENTINEL));
}
}
if self.progress
&& !self.silent
&& self.last_tick.elapsed() >= std::time::Duration::from_millis(100)
{
eprint!("\rrsurl: {} bytes received", self.written);
let _ = io::stderr().flush();
self.last_tick = std::time::Instant::now();
}
Ok(buf.len())
}
fn flush(&mut self) -> io::Result<()> {
self.inner.flush()
}
}
fn run_http_download(
req: rsurl::Request,
url: &Url,
args: &Args,
jar: Option<&mut CookieJar>,
) -> u8 {
let name = if args.remote_name {
match remote_name_from_url(url) {
Ok(n) => n,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 23;
}
}
} else {
args.output.clone().unwrap_or_default()
};
let (file, out_path) = match create_output_file_tracked(&name, args) {
Ok(pair) => pair,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: open {name}: {e}");
}
return 23;
}
};
let (speed_limit, speed_time) = {
let lim = args
.speed_limit
.as_deref()
.and_then(|s| s.parse::<u64>().ok());
let tim = args
.speed_time
.as_deref()
.and_then(|s| s.parse::<u64>().ok());
match (lim, tim) {
(None, None) => (None, 30),
(l, t) => (Some(l.unwrap_or(1)), t.unwrap_or(30)),
}
};
let now = std::time::Instant::now();
let mut sink = DownloadSink {
inner: Box::new(file),
written: 0,
max: args.max_filesize,
rate: args.limit_rate.as_deref().and_then(parse_rate),
speed_limit,
speed_time,
started: now,
progress: args.progress_bar,
silent: args.silent,
last_tick: now,
};
let result = if args.verbose {
let mut err = io::stderr().lock();
req.send_download(&mut sink, jar, &mut err)
} else {
req.send_download(&mut sink, jar, &mut io::sink())
};
let time_total = now.elapsed();
let written = sink.written;
if args.progress_bar && !args.silent {
eprintln!();
}
match result {
Ok(resp) => {
if args.remote_time {
set_remote_time(&resp, &name);
}
run_write_out(&resp, url, args, time_total, written);
0
}
Err(e) => {
if args.remove_on_error {
drop(sink);
let _ = std::fs::remove_file(&out_path);
}
if e.to_string().contains(MAX_FILESIZE_SENTINEL) {
if show_errors(args) {
eprintln!("rsurl: Maximum file size exceeded");
}
return 63;
}
if e.to_string().contains(LOW_SPEED_SENTINEL) {
if show_errors(args) {
eprintln!(
"rsurl: Operation too slow. Less than {} bytes/sec transferred \
the last {speed_time} seconds",
speed_limit.unwrap_or(1)
);
}
return 28;
}
if show_errors(args) {
eprintln!("rsurl: {e}");
}
transfer_exit_code(&e)
}
}
}
fn resolve_output_path(path: &str, args: &Args) -> std::path::PathBuf {
let full = match &args.output_dir {
Some(dir) => std::path::Path::new(dir).join(path),
None => std::path::PathBuf::from(path),
};
if !args.no_clobber || !full.exists() {
return full;
}
let base = full.as_os_str().to_owned();
for n in 1u32.. {
let mut candidate = base.clone();
candidate.push(format!(".{n}"));
let candidate = std::path::PathBuf::from(candidate);
if !candidate.exists() {
return candidate;
}
}
unreachable!("u32 range exhausted picking a --no-clobber name")
}
fn create_output_file_tracked(path: &str, args: &Args) -> io::Result<(File, std::path::PathBuf)> {
let full = resolve_output_path(path, args);
if args.create_dirs {
if let Some(parent) = full.parent() {
if !parent.as_os_str().is_empty() {
std::fs::create_dir_all(parent)?;
}
}
} else if let Some(dir) = &args.output_dir {
std::fs::create_dir_all(dir)?;
}
let file = File::create(&full)?;
Ok((file, full))
}
fn create_output_file(path: &str, args: &Args) -> io::Result<File> {
create_output_file_tracked(path, args).map(|(f, _)| f)
}
fn dump_headers(resp: &Response, path: &str) -> io::Result<()> {
let mut f = File::create(path)?;
write!(f, "{} {} {}\r\n", resp.version, resp.status, resp.reason)?;
for (k, v) in &resp.headers {
write!(f, "{k}: {v}\r\n")?;
}
f.write_all(b"\r\n")
}
fn set_remote_time(resp: &Response, path: &str) {
let Some(lm) = resp.header("last-modified") else {
return;
};
let Some(epoch) = httpdate_to_epoch(lm) else {
return;
};
let mtime = std::time::UNIX_EPOCH + std::time::Duration::from_secs(epoch);
if let Ok(f) = File::options().write(true).open(path) {
let _ = f.set_modified(mtime);
}
}
fn write_output(resp: &Response, url: &Url, args: &Args) -> io::Result<()> {
let mut to_stdout = true;
let mut explicit_stdout = false;
let mut out: Box<dyn Write> = if args.remote_name {
let name = args
.remote_header_name
.then(|| content_disposition_filename(resp))
.flatten()
.map(Ok)
.unwrap_or_else(|| remote_name_from_url(url))
.map_err(|e| io::Error::other(e.to_string()))?;
to_stdout = false;
Box::new(create_output_file(&name, args)?)
} else {
match &args.output {
Some(path) if path != "-" => {
to_stdout = false;
Box::new(create_output_file(path, args)?)
}
Some(_) => {
explicit_stdout = true; Box::new(io::stdout().lock())
}
None => Box::new(io::stdout().lock()),
}
};
let is_tty = to_stdout && io::stdout().is_terminal();
if args.include_headers {
if is_tty {
let line = format!("{} {} ", resp.version, resp.status);
out.write_all(line.as_bytes())?;
out.write_all(&sanitize_for_tty(resp.reason.as_bytes()))?;
out.write_all(b"\r\n")?;
for (k, v) in &resp.headers {
out.write_all(&sanitize_for_tty(k.as_bytes()))?;
out.write_all(b": ")?;
out.write_all(&sanitize_for_tty(v.as_bytes()))?;
out.write_all(b"\r\n")?;
}
} else {
write!(out, "{} {} {}\r\n", resp.version, resp.status, resp.reason)?;
for (k, v) in &resp.headers {
write!(out, "{k}: {v}\r\n")?;
}
}
out.write_all(b"\r\n")?;
}
if is_tty {
if explicit_stdout {
out.write_all(&resp.body)?;
} else if body_looks_binary(&resp.body) {
if show_errors(args) {
eprintln!(
"Warning: Binary output can mess up your terminal. Use \"--output -\" to tell"
);
eprintln!("Warning: rsurl to output it to your terminal anyway, or consider \"-o");
eprintln!("Warning: <FILE>\" to save to a file.");
}
} else {
out.write_all(&sanitize_for_tty(&resp.body))?;
}
} else {
out.write_all(&resp.body)?;
}
Ok(())
}
fn remote_name_from_url(url: &Url) -> Result<String, String> {
let path = url.path.as_str();
let path_no_query = match path.find('?') {
Some(i) => &path[..i],
None => path,
};
let trimmed = path_no_query.trim_end_matches('/');
let last = trimmed.rsplit('/').next().unwrap_or("");
if last.is_empty() {
return Err("Refusing to overwrite stdin".to_string());
}
let basename = Path::new(last)
.file_name()
.and_then(|n| n.to_str())
.ok_or_else(|| "Refusing to overwrite stdin".to_string())?;
if basename.is_empty() {
return Err("Refusing to overwrite stdin".to_string());
}
Ok(basename.to_string())
}
fn print_usage() {
println!(
"rsurl {VERSION} — a pure-Rust curl
Usage: rsurl [options] <url>...
Options:
-o, --output <file> write body to file instead of stdout
-O, --remote-name save body as the URL's last path segment
-i, --include include response headers in the output
-I, --head issue HEAD instead of GET
-v, --verbose print request/response headers to stderr
-s, --silent suppress error messages
-X, --request <method> override HTTP method; for rtsp:// selects the
RTSP method (OPTIONS/DESCRIBE/SETUP/PLAY/TEARDOWN)
-H, --header <line> add a request header (repeatable)
-d, --data <body> POST body (urlencoded); @file reads from disk
and strips CR/LF. Repeatable; joined with '&'.
--data-raw <body> like -d but '@' is taken literally
--data-binary <body> like -d but @file is read verbatim (no strip)
--data-urlencode <s> percent-encode <s> before sending. Forms:
text =text name=text @file name@file
--json <data> POST <data> as application/json and set Accept to
application/json; @file reads verbatim. Repeatable.
-F, --form <name=value> add a multipart/form-data part. Value forms:
text @file (upload) <file (field from file)
Modifiers: ;type= ;filename= ;headers=@hdrfile
--form-string <n=v> like -F but value is taken literally (no @, <, ;)
--form-escape percent-encode names/filenames per RFC 7578 §4.2
(default: backslash-escape, curl-historical)
-T, --upload-file <f> upload the file: HTTP PUT (default Content-Type:
application/octet-stream), FTP/FTPS STOR, TFTP WRQ,
MQTT PUBLISH, or SFTP/SCP write
--key <file> SSH private-key identity file for sftp:// / scp://
public-key auth (repeatable). Without it, the
default ~/.ssh/id_ed25519|id_ecdsa|id_rsa are tried
-C, --continue-at <off> resume at byte <off> (FTP: REST before STOR);
the automatic form '-C -' is not supported
-a, --append FTP/FTPS upload: append (APPE) instead of replacing
(STOR). No-op for other protocols; overrides -C
-A, --user-agent <ua> set User-Agent
-e, --referer <ref> set Referer
-L, --location follow 3xx redirects
--max-redirs <n> cap on redirect hops (default 50)
-u, --user <user:pass> HTTP Basic auth credentials
--digest use HTTP Digest auth with -u credentials
--oauth2-bearer <t> send Authorization: Bearer <token>
--aws-sigv4 <spec> sign with AWS SigV4 (e.g. aws:amz:us-east-1:s3, -u key:secret)
-k, --insecure don't verify the TLS certificate chain
--cacert <file> PEM bundle to use instead of system trust
--tlsv1.2/1.3 require at least this TLS version (floor)
--tls-max <ver> cap the TLS version (1.2 or 1.3)
--mail-from <addr> SMTP envelope sender (smtp://host, body via -T/-d)
--mail-rcpt <addr> SMTP envelope recipient (repeatable)
--no-idn don't convert international (IDN) hostnames to punycode
--max-time <secs> cap on the whole operation's wall time
--connect-timeout <secs>
cap on the TCP connect step
--http2 require HTTP/2 (ALPN h2); error if unavailable
--http1.1 force HTTP/1.1 (alias: --http1)
--http3 try HTTP/3 (QUIC), fall back to HTTP/2/1.1
--http3-only require HTTP/3 (QUIC); no fallback
-b, --cookie <data> cookies to send: \"k=v[; k2=v2]\" or path to a
Netscape cookies.txt file
-c, --cookie-jar <file> write all known cookies to <file> on exit
-x, --proxy <url> route via a proxy. Scheme picks the kind:
http/https/socks4/socks4a/socks5/socks5h (bare
host:port = http). SOCKS5 also tunnels HTTP/3 &
TFTP (UDP). Reads HTTPS_PROXY/http_proxy/ALL_PROXY.
--socks4 <host:port> / --socks4a / --socks5 / --socks5-hostname
shorthands for -x socks4://… etc.
-U, --proxy-user <u:p> credentials for the proxy (Basic / SOCKS5 auth)
--noproxy <hosts> comma-separated host suffixes that bypass the
proxy; \"*\" bypasses everything
-f, --fail on HTTP >= 400, emit no body and exit 22
-S, --show-error show errors even with -s
-G, --get put -d data in the URL query and use GET
-r, --range <range> request a byte range (Range: bytes=<range>)
--compressed ask for a compressed response (decoded anyway)
-D, --dump-header <file> write response headers to <file>
-R, --remote-time set the saved file's mtime from Last-Modified
--create-dirs create missing directories for -o
--remove-on-error delete a partial -o/-O file if the transfer fails
--no-clobber never overwrite an existing -o/-O file (use .1, .2…)
--max-filesize <n> refuse a download larger than <n> bytes
-w, --write-out <fmt> after the transfer, print <fmt> with %{{vars}}
expanded (http_code, size_download, content_type,
url_effective, time_total, time_connect,
time_appconnect, time_pretransfer, time_starttransfer,
ssl_verify_result, %header{{Name}}; phase timers are
HTTP/1.1-only, else 0.000000)
--url <url> add a URL (repeatable; same as a positional arg)
-n, --netrc read credentials from ~/.netrc (when no -u)
--netrc-file <file> read credentials from <file> (implies -n)
-J, --remote-header-name with -O, name the file from Content-Disposition
--retry <n> retry transient failures up to <n> times
--retry-delay <s> fixed delay between retries (else exponential)
--retry-max-time <s> give up retrying after <s> seconds total
--retry-connrefused also retry on connection refused
--retry-all-errors retry on any error
-z, --time-cond <t> If-Modified-Since (or If-Unmodified-Since for
a leading '-'); a filename uses its mtime
--output-dir <dir> directory to prepend to -o/-O output names
--fail-with-body exit 22 on HTTP >= 400 but still write the body
--proto <spec> restrict allowed schemes (e.g. =https,http)
--proto-default <s> scheme for URLs given without one
-g, --globoff disable URL globbing ({{}} and [] taken literally)
-Z, --parallel run this invocation's transfers concurrently
--parallel-max <n> cap on concurrent transfers (default 50)
--location-trusted keep credentials across cross-host redirects
--post301/302/303 keep POST (don't downgrade to GET) on that redirect
--connect-to <spec> dial HOST2:PORT2 for requests to HOST1:PORT1
(keeps the original Host:/SNI)
--unix-socket <path> connect through a Unix-domain socket (Unix only)
-4, --ipv4 connect over IPv4 only
-6, --ipv6 connect over IPv6 only
--resolve <h:p:addr> use <addr> for <host>:<port> (static DNS)
--disable-epsv FTP: skip EPSV, use PASV directly
-P, --ftp-port <addr> FTP: active mode (server connects back); <addr> is
accepted but the control-connection local IP is used
--ftp-create-dirs FTP: create missing remote dirs before upload (-T)
-K, --config <file> read options from a curl-style config file
--next (-:) start a new request with its own options
-#, --progress-bar show progress on streamed file downloads (-o/-O)
-E, --cert <c[:pass]> client certificate for mutual TLS (PEM, or DER with
--cert-type). Optional inline ':password' for the key
--key <file> client TLS private key (also the SSH identity file);
omit when the key is embedded in --cert
--pass <phrase> passphrase for an encrypted --key/--cert key
--cert-type <type> --cert encoding: PEM (default) or DER
--key-type <type> --key encoding: PEM (default) or DER
--pinnedpubkey <h> pin the server pubkey: sha256//BASE64[;sha256//...]
(SHA-256 of the leaf cert's SPKI); fail on mismatch
--capath <dir> trust extra CA certs from every file in <dir>
(in addition to system roots / --cacert)
--crlfile <file> check the server chain against this CRL (PEM/DER;
default backend only)
--ciphers <list> restrict TLS<=1.2 cipher suites (OpenSSL/IANA names,
':'-separated; default backend only)
--tls13-ciphers <l> restrict TLS 1.3 cipher suites (IANA TLS_* names)
--limit-rate <speed> cap download rate (e.g. 200k, 1M) on -o/-O downloads
-y, --speed-time <s> / -Y, --speed-limit <bps>
abort an -o/-O download averaging below <bps>
bytes/sec over <s> seconds (exit 28)
-q, --disable no-op (rsurl reads no config unless -K is given)
-N, --no-buffer no-op (output is already streamed)
--no-progress-meter no-op (no meter is shown by default)
--styled-output, --no-styled-output
no-op (headers are never styled)
-h, --help print this help
-V, --version print version
"
);
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn transfer_exit_code_maps_curl_codes() {
use std::io::{Error as IoError, ErrorKind};
assert_eq!(transfer_exit_code(&rsurl::Error::InvalidUrl("x".into())), 3);
assert_eq!(
transfer_exit_code(&rsurl::Error::UnsupportedScheme("gopher+ssh".into())),
1
);
assert_eq!(transfer_exit_code(&rsurl::Error::UnexpectedEof), 52);
assert_eq!(transfer_exit_code(&rsurl::Error::Ssh("auth".into())), 79);
assert_eq!(
transfer_exit_code(&rsurl::Error::BadResponse("operation timed out".into())),
28
);
assert_eq!(
transfer_exit_code(&rsurl::Error::BadResponse(
"maximum (50) redirects followed".into()
)),
47
);
assert_eq!(
transfer_exit_code(&rsurl::Error::BadResponse("garbage status line".into())),
8
);
assert_eq!(
transfer_exit_code(&rsurl::Error::Io(IoError::from(
ErrorKind::ConnectionRefused
))),
7
);
assert_eq!(
transfer_exit_code(&rsurl::Error::Io(IoError::from(ErrorKind::TimedOut))),
28
);
assert_eq!(
transfer_exit_code(&rsurl::Error::Io(IoError::other(
"failed to lookup address information: Name or service not known"
))),
6
);
}
#[test]
fn proto_allowed_evaluates_specs() {
assert!(proto_allowed("https", "=https,http"));
assert!(!proto_allowed("ftp", "=https,http"));
assert!(proto_allowed("http", "all"));
assert!(!proto_allowed("ftp", "-ftp"));
assert!(proto_allowed("https", "-ftp"));
assert!(proto_allowed("https", "+https"));
}
#[test]
fn epoch_httpdate_roundtrips() {
assert_eq!(
epoch_to_httpdate(784111777),
"Sun, 06 Nov 1994 08:49:37 GMT"
);
assert_eq!(
httpdate_to_epoch("Sun, 06 Nov 1994 08:49:37 GMT"),
Some(784111777)
);
assert_eq!(epoch_to_httpdate(0), "Thu, 01 Jan 1970 00:00:00 GMT");
}
#[test]
fn glob_brace_and_range_expand() {
let urls: Vec<String> = glob_expand("http://h/{a,b}/[1-3]")
.unwrap()
.into_iter()
.map(|(u, _)| u)
.collect();
assert_eq!(
urls,
vec![
"http://h/a/1",
"http://h/a/2",
"http://h/a/3",
"http://h/b/1",
"http://h/b/2",
"http://h/b/3",
]
);
}
#[test]
fn glob_zero_padded_and_step_and_alpha() {
assert_eq!(expand_range("08-11").unwrap(), vec!["08", "09", "10", "11"]);
assert_eq!(expand_range("1-10:3").unwrap(), vec!["1", "4", "7", "10"]);
assert_eq!(expand_range("a-e:2").unwrap(), vec!["a", "c", "e"]);
}
#[test]
fn glob_output_substitution() {
let (_, caps) = &glob_expand("img[1-2].jpg").unwrap()[0];
assert_eq!(apply_glob_output("out-#1.bin", caps), "out-1.bin");
}
#[test]
fn short_bundles_expand() {
let got = expand_short_bundles(&["-sS".into(), "-ofile".into(), "u".into()]);
assert_eq!(got, vec!["-s", "-S", "-o", "file", "u"]);
let got2 = expand_short_bundles(&["--silent".into(), "-".into()]);
assert_eq!(got2, vec!["--silent", "-"]);
}
#[test]
fn sanitize_for_tty_passes_plain_ascii_and_utf8() {
assert_eq!(sanitize_for_tty(b"hello world"), b"hello world");
let utf8 = "café 日本語".as_bytes();
assert_eq!(sanitize_for_tty(utf8), utf8);
}
#[test]
fn sanitize_for_tty_preserves_tab() {
assert_eq!(sanitize_for_tty(b"a\tb"), b"a\tb");
}
#[test]
fn sanitize_for_tty_neutralizes_escape_sequences() {
assert_eq!(sanitize_for_tty(b"\x1b[2J"), b"\\x1b[2J");
assert_eq!(
sanitize_for_tty(b"\x1b]52;c;Zm9v\x07"),
b"\\x1b]52;c;Zm9v\\x07"
);
assert_eq!(sanitize_for_tty(b"\x00\x07\x7f"), b"\\x00\\x07\\x7f");
assert_eq!(sanitize_for_tty("\u{9b}".as_bytes()), b"\\x9b");
assert_eq!(sanitize_for_tty(b"\xa0"), b"\\xa0");
assert_eq!(sanitize_for_tty("\u{a0}".as_bytes()), "\u{a0}".as_bytes());
}
#[test]
fn body_looks_binary_detects_nul() {
assert!(body_looks_binary(b"\x89PNG\x00\x00"));
assert!(!body_looks_binary(b"plain text\n"));
}
#[test]
fn percent_encode_form_passes_unreserved() {
assert_eq!(percent_encode_form(b"abcXYZ012-._~"), "abcXYZ012-._~");
}
#[test]
fn percent_encode_form_space_becomes_plus() {
assert_eq!(percent_encode_form(b"hello world"), "hello+world");
}
#[test]
fn percent_encode_form_special_chars_become_hex() {
assert_eq!(percent_encode_form(b"=&+/?%#"), "%3D%26%2B%2F%3F%25%23",);
}
#[test]
fn percent_encode_form_high_bytes_use_uppercase_hex() {
assert_eq!(percent_encode_form(&[0xC3, 0xA9]), "%C3%A9"); }
#[test]
fn strip_newlines_removes_crlf_and_nul() {
let got = strip_newlines(b"a\r\nb\nc\0d".to_vec());
assert_eq!(got, b"abcd");
}
#[test]
fn strip_newlines_keeps_other_whitespace() {
let got = strip_newlines(b"a\tb c\r\n".to_vec());
assert_eq!(got, b"a\tb c");
}
#[test]
fn encode_urlencoded_plain_content() {
let got = encode_urlencoded("hello world").unwrap();
assert_eq!(got, b"hello+world");
}
#[test]
fn encode_urlencoded_leading_eq_strips_name() {
let got = encode_urlencoded("=hi there").unwrap();
assert_eq!(got, b"hi+there");
}
#[test]
fn encode_urlencoded_name_value() {
let got = encode_urlencoded("name=hello world").unwrap();
assert_eq!(got, b"name=hello+world");
}
#[test]
fn encode_urlencoded_at_file_reads_and_encodes() {
let mut tmp = std::env::temp_dir();
tmp.push("rsurl-urlencode-at-file.txt");
std::fs::write(&tmp, b"hello world").unwrap();
let spec = format!("@{}", tmp.display());
let got = encode_urlencoded(&spec).unwrap();
let _ = std::fs::remove_file(&tmp);
assert_eq!(got, b"hello+world");
}
#[test]
fn encode_urlencoded_name_at_file_reads_and_encodes() {
let mut tmp = std::env::temp_dir();
tmp.push("rsurl-urlencode-name-at.txt");
std::fs::write(&tmp, b"value with spaces").unwrap();
let spec = format!("k@{}", tmp.display());
let got = encode_urlencoded(&spec).unwrap();
let _ = std::fs::remove_file(&tmp);
assert_eq!(got, b"k=value+with+spaces");
}
#[test]
fn encode_urlencoded_eq_wins_over_at() {
let got = encode_urlencoded("x=y@notafile").unwrap();
assert_eq!(got, b"x=y%40notafile");
}
#[test]
fn assemble_empty_is_none() {
assert!(assemble_form_body(&[]).unwrap().is_none());
}
#[test]
fn assemble_joins_with_ampersand() {
let parts = vec![
DataPart::Plain {
value: "a=1".into(),
at_file_ok: true,
},
DataPart::Plain {
value: "b=2".into(),
at_file_ok: true,
},
];
assert_eq!(assemble_form_body(&parts).unwrap().unwrap(), b"a=1&b=2");
}
#[test]
fn assemble_plain_at_file_strips_newlines() {
let mut tmp = std::env::temp_dir();
tmp.push("rsurl-assemble-plain-at.txt");
std::fs::write(&tmp, b"a\r\nb\n").unwrap();
let parts = vec![DataPart::Plain {
value: format!("@{}", tmp.display()),
at_file_ok: true,
}];
let got = assemble_form_body(&parts).unwrap().unwrap();
let _ = std::fs::remove_file(&tmp);
assert_eq!(got, b"ab");
}
#[test]
fn assemble_binary_at_file_keeps_newlines() {
let mut tmp = std::env::temp_dir();
tmp.push("rsurl-assemble-binary-at.txt");
std::fs::write(&tmp, b"a\r\nb\n").unwrap();
let parts = vec![DataPart::Binary {
value: format!("@{}", tmp.display()),
}];
let got = assemble_form_body(&parts).unwrap().unwrap();
let _ = std::fs::remove_file(&tmp);
assert_eq!(got, b"a\r\nb\n");
}
#[test]
fn assemble_data_raw_treats_at_literally() {
let parts = vec![DataPart::Plain {
value: "@literal".into(),
at_file_ok: false,
}];
assert_eq!(assemble_form_body(&parts).unwrap().unwrap(), b"@literal");
}
#[test]
fn assemble_mixes_data_modes() {
let parts = vec![
DataPart::Plain {
value: "n=1".into(),
at_file_ok: true,
},
DataPart::Binary {
value: "rawbytes".into(),
},
DataPart::UrlEncoded {
value: "k=hello world".into(),
},
];
let got = assemble_form_body(&parts).unwrap().unwrap();
assert_eq!(got, b"n=1&rawbytes&k=hello+world");
}
}