use std::fs::File;
use std::io::{self, IsTerminal, Read, Seek, Write};
use std::path::Path;
use std::process::ExitCode;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Duration;
use rsurl::{CookieJar, HttpVersionPref, Request, Response, Url};
const VERSION: &str = env!("CARGO_PKG_VERSION");
#[derive(Default, Clone)]
struct Args {
urls: Vec<String>,
output: Option<String>,
include_headers: bool,
head: bool,
verbose: bool,
verbosity: u8,
silent: bool,
method: Option<String>,
headers: Vec<(String, String)>,
data_parts: Vec<DataPart>,
json_parts: Vec<String>,
user_agent: Option<String>,
referer: Option<String>,
http_version: Option<HttpVersionPref>,
follow_redirects: bool,
max_redirs: Option<u32>,
basic_auth: Option<(String, String)>,
insecure: bool,
no_idn: bool,
cacert: Option<String>,
max_time: Option<u64>,
connect_timeout: Option<u64>,
remote_name: bool,
cookie_in: Option<String>,
cookie_jar: Option<String>,
proxy: Option<String>,
proxy_user: Option<(String, String)>,
noproxy: Option<String>,
form_parts: Vec<FormPart>,
form_escape: bool,
upload_file: Option<String>,
continue_at: Option<u64>,
continue_resume: bool,
append: bool,
ssh_keys: Vec<String>,
fail: bool,
show_error: bool,
get: bool,
range: Option<String>,
compressed: bool,
dump_header: Option<String>,
remote_time: bool,
create_dirs: bool,
remove_on_error: bool,
no_clobber: bool,
disable_epsv: bool,
ssl_reqd: bool,
ftp_create_dirs: bool,
ftp_port: Option<String>,
max_filesize: Option<u64>,
write_out: Option<String>,
netrc: bool,
netrc_file: Option<String>,
remote_header_name: bool,
retry: u32,
ipv4: bool,
ipv6: bool,
resolve: Vec<(String, u16, std::net::IpAddr)>,
progress_bar: bool,
cert: Option<String>,
key_file: Option<String>,
key_pass: Option<String>,
cert_type_der: bool,
key_type_der: bool,
pinned_pubkey: Option<String>,
capath: Option<String>,
crl_file: Option<String>,
ciphers: Option<String>,
tls13_ciphers: Option<String>,
limit_rate: Option<String>,
speed_limit: Option<String>,
speed_time: Option<String>,
time_cond: Option<String>,
output_dir: Option<String>,
fail_with_body: bool,
proto: Option<String>,
proto_default: Option<String>,
auto_referer: bool,
retry_delay: Option<u64>,
retry_max_time: Option<u64>,
retry_connrefused: bool,
retry_all_errors: bool,
globoff: bool,
location_trusted: bool,
post301: bool,
post302: bool,
post303: bool,
connect_to: Vec<(String, u16, String, u16)>,
unix_socket: Option<String>,
tls_min: Option<rsurl::tls::ProtocolVersion>,
tls_max: Option<rsurl::tls::ProtocolVersion>,
mail_from: Option<String>,
mail_rcpt: Vec<String>,
digest: bool,
bearer: Option<String>,
aws_sigv4: Option<String>,
parallel: bool,
parallel_max: Option<usize>,
parallel_segments: Option<usize>,
torrent: bool,
listen_port: Option<u16>,
bt_peers: Vec<String>,
no_dht: bool,
seed: bool,
share_ratio: Option<f64>,
recheck: bool,
bt_info: bool,
bt_save_torrent: bool,
bt_file: Option<String>,
bt_concat: bool,
}
#[derive(Debug, Clone)]
enum DataPart {
Plain { value: String, at_file_ok: bool },
Binary { value: String },
UrlEncoded { value: String },
}
#[derive(Debug, Clone)]
struct FormPart {
name: String,
body: FormBody,
extras: Vec<FormExtra>,
}
#[derive(Debug, Clone)]
enum FormBody {
Literal(String),
File(String),
FileAsField(String),
LiteralStrict(String),
}
#[derive(Debug, Clone)]
enum FormExtra {
Type(String),
Filename(String),
HeadersFile(String),
}
fn main() -> ExitCode {
let raw: Vec<String> = std::env::args().skip(1).collect();
let raw = expand_short_bundles(&raw);
let expanded = match expand_config(&raw, 0) {
Ok(v) => v,
Err(e) => {
eprintln!("rsurl: {e}");
return ExitCode::from(2);
}
};
let segments = split_operations(&expanded);
let mut ops: Vec<Args> = Vec::with_capacity(segments.len());
for seg in &segments {
match parse_args(seg) {
Ok(a) => ops.push(a),
Err(e) => {
eprintln!("rsurl: {e}");
eprintln!("try 'rsurl --help'");
return ExitCode::from(2);
}
}
}
if ops.iter().all(|a| a.urls.is_empty()) {
print_usage();
return ExitCode::from(2);
}
warn_unsupported(&ops);
let uses_cookies = ops
.iter()
.any(|a| a.cookie_in.is_some() || a.cookie_jar.is_some());
if ops.iter().any(|a| a.parallel) && !uses_cookies {
return ExitCode::from(run_parallel(&ops));
}
let jar_op = ops
.iter()
.find(|a| a.cookie_in.is_some() || a.cookie_jar.is_some())
.unwrap_or(&ops[0]);
let mut jar: Option<CookieJar> = match build_initial_jar(jar_op) {
Ok(j) => j,
Err(e) => {
if show_errors(jar_op) {
eprintln!("rsurl: {e}");
}
return ExitCode::from(2);
}
};
let mut last_failure: u8 = 0;
for op in &ops {
for url in &op.urls {
let expansions = if glob_disabled(op, url) {
vec![(url.clone(), Vec::new())]
} else {
match glob_expand(url) {
Ok(v) => v,
Err(e) => {
if show_errors(op) {
eprintln!("rsurl: {e}");
}
last_failure = 3;
continue;
}
}
};
for (eurl, caps) in expansions {
let code =
if !caps.is_empty() && op.output.as_ref().is_some_and(|o| o.contains('#')) {
let mut op2 = op.clone();
op2.output = op.output.as_ref().map(|o| apply_glob_output(o, &caps));
process_url(&eurl, &op2, jar.as_mut())
} else {
process_url(&eurl, op, jar.as_mut())
};
if code != 0 {
last_failure = code;
}
}
}
}
if let (Some(j), Some(op)) = (jar.as_ref(), ops.iter().find(|a| a.cookie_jar.is_some())) {
let path = op.cookie_jar.as_deref().unwrap();
if let Err(e) = j.save_netscape(path) {
if show_errors(op) {
eprintln!("rsurl: writing cookie jar {path}: {e}");
}
if last_failure == 0 {
last_failure = 23;
}
}
}
ExitCode::from(last_failure)
}
fn streams_to_file(args: &Args) -> bool {
let output_is_file = args.remote_name || args.output.as_deref().is_some_and(|p| p != "-");
output_is_file
&& !args.include_headers
&& !args.fail
&& !args.fail_with_body
&& !args.remote_header_name && !args.digest && args.dump_header.is_none()
}
fn warn_unsupported(ops: &[Args]) {
if !ops.iter().any(show_errors) {
return;
}
if ops
.iter()
.any(|a| (a.speed_limit.is_some() || a.speed_time.is_some()) && !streams_to_file(a))
{
eprintln!(
"rsurl: warning: -y/-Y speed limits are enforced only for file downloads \
(-o FILE / -O)"
);
}
}
fn split_cert_pass(s: &str) -> (&str, Option<&str>) {
let bytes = s.as_bytes();
let mut idx = None;
for (i, &b) in bytes.iter().enumerate() {
if b == b':' {
let is_drive = i == 1 && bytes[0].is_ascii_alphabetic();
if !is_drive {
idx = Some(i);
break;
}
}
}
match idx {
Some(i) => (&s[..i], Some(&s[i + 1..])),
None => (s, None),
}
}
fn parse_cert_type(v: &str, flag: &str) -> Result<bool, String> {
match v.to_ascii_uppercase().as_str() {
"PEM" => Ok(false),
"DER" => Ok(true),
other => Err(format!(
"{flag}: unsupported type {other:?}; only PEM and DER are supported"
)),
}
}
fn short_flag_takes_value(c: char) -> bool {
matches!(
c,
'o' | 'X'
| 'H'
| 'd'
| 'F'
| 'T'
| 'A'
| 'e'
| 'u'
| 'b'
| 'c'
| 'x'
| 'E'
| 'r'
| 'D'
| 'w'
| 'C'
| 'y'
| 'Y'
| 'U'
| 'K'
| 'P'
)
}
fn expand_short_bundles(tokens: &[String]) -> Vec<String> {
let mut out = Vec::new();
for t in tokens {
let is_bundle = t.len() > 2 && t.starts_with('-') && !t.starts_with("--");
if !is_bundle {
out.push(t.clone());
continue;
}
let chars: Vec<char> = t[1..].chars().collect();
let mut i = 0;
while i < chars.len() {
let c = chars[i];
out.push(format!("-{c}"));
if short_flag_takes_value(c) {
let rest: String = chars[i + 1..].iter().collect();
if !rest.is_empty() {
out.push(rest); }
break;
}
i += 1;
}
}
out
}
enum GlobSeg {
Lit(String),
Set(Vec<String>),
}
fn parse_glob(url: &str) -> Result<Vec<GlobSeg>, String> {
let mut segs = Vec::new();
let mut lit = String::new();
let chars: Vec<char> = url.chars().collect();
let mut i = 0;
while i < chars.len() {
match chars[i] {
'\\' if i + 1 < chars.len() => {
lit.push(chars[i + 1]);
i += 2;
}
'{' => {
let close = find_close(&chars, i, '{', '}')
.ok_or_else(|| format!("unmatched '{{' in URL glob: {url:?}"))?;
let inner: String = chars[i + 1..close].iter().collect();
let items: Vec<String> = inner.split(',').map(|s| s.to_string()).collect();
if !lit.is_empty() {
segs.push(GlobSeg::Lit(std::mem::take(&mut lit)));
}
segs.push(GlobSeg::Set(items));
i = close + 1;
}
'[' => {
let close = find_close(&chars, i, '[', ']')
.ok_or_else(|| format!("unmatched '[' in URL glob: {url:?}"))?;
let inner: String = chars[i + 1..close].iter().collect();
let items = expand_range(&inner)
.ok_or_else(|| format!("bad range '[{inner}]' in URL glob"))?;
if !lit.is_empty() {
segs.push(GlobSeg::Lit(std::mem::take(&mut lit)));
}
segs.push(GlobSeg::Set(items));
i = close + 1;
}
c => {
lit.push(c);
i += 1;
}
}
}
if !lit.is_empty() {
segs.push(GlobSeg::Lit(lit));
}
Ok(segs)
}
fn find_close(chars: &[char], open_at: usize, open: char, close: char) -> Option<usize> {
let mut depth = 0;
for (k, &c) in chars.iter().enumerate().skip(open_at) {
if c == open {
depth += 1;
} else if c == close {
depth -= 1;
if depth == 0 {
return Some(k);
}
}
}
None
}
fn expand_range(body: &str) -> Option<Vec<String>> {
let (range, step) = match body.split_once(':') {
Some((r, s)) => (r, s.parse::<usize>().ok().filter(|&s| s > 0)?),
None => (body, 1),
};
let (start, end) = range.split_once('-')?;
if let (Ok(a), Ok(b)) = (start.parse::<u64>(), end.parse::<u64>()) {
let width = if start.starts_with('0') && start.len() > 1 {
start.len()
} else {
0
};
let mut out = Vec::new();
let mut v = a;
while v <= b {
out.push(format!("{v:0width$}"));
v += step as u64;
}
return Some(out);
}
let (sc, ec) = (start.chars().next()?, end.chars().next()?);
if start.chars().count() == 1 && end.chars().count() == 1 && sc <= ec {
let mut out = Vec::new();
let mut c = sc as u32;
while c <= ec as u32 {
if let Some(ch) = char::from_u32(c) {
out.push(ch.to_string());
}
c += step as u32;
}
return Some(out);
}
None
}
fn glob_expand(url: &str) -> Result<Vec<(String, Vec<String>)>, String> {
let segs = parse_glob(url)?;
let mut results = vec![(String::new(), Vec::new())];
for seg in &segs {
match seg {
GlobSeg::Lit(s) => {
for (u, _) in results.iter_mut() {
u.push_str(s);
}
}
GlobSeg::Set(items) => {
let mut next = Vec::with_capacity(results.len() * items.len());
for (u, caps) in &results {
for item in items {
let mut nu = u.clone();
nu.push_str(item);
let mut nc = caps.clone();
nc.push(item.clone());
next.push((nu, nc));
}
}
results = next;
}
}
}
Ok(results)
}
fn glob_disabled(op: &Args, url: &str) -> bool {
let torrent_source =
op.torrent || op.bt_info || op.bt_save_torrent || op.bt_file.is_some() || op.bt_concat;
op.globoff
|| url.starts_with("magnet:")
|| (torrent_source && !url.starts_with("http://") && !url.starts_with("https://"))
}
fn apply_glob_output(template: &str, caps: &[String]) -> String {
if caps.is_empty() || !template.contains('#') {
return template.to_string();
}
let mut out = String::with_capacity(template.len());
let mut chars = template.chars().peekable();
while let Some(c) = chars.next() {
if c == '#' {
if let Some(d) = chars.peek().and_then(|d| d.to_digit(10)) {
chars.next();
let idx = d as usize;
if idx >= 1 && idx <= caps.len() {
out.push_str(&caps[idx - 1]);
continue;
}
}
}
out.push(c);
}
out
}
fn run_parallel(ops: &[Args]) -> u8 {
use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering};
struct Item<'a> {
op: &'a Args,
url: String,
caps: Vec<String>,
}
let mut items: Vec<Item> = Vec::new();
for op in ops {
for url in &op.urls {
let expansions = if glob_disabled(op, url) {
vec![(url.clone(), Vec::new())]
} else {
match glob_expand(url) {
Ok(v) => v,
Err(e) => {
if show_errors(op) {
eprintln!("rsurl: {e}");
}
continue;
}
}
};
for (eurl, caps) in expansions {
items.push(Item {
op,
url: eurl,
caps,
});
}
}
}
if items.is_empty() {
return 0;
}
let max = ops
.iter()
.filter_map(|a| a.parallel_max)
.max()
.unwrap_or(50)
.max(1);
let n_threads = max.min(items.len());
let idx = AtomicUsize::new(0);
let worst = AtomicU8::new(0);
std::thread::scope(|s| {
for _ in 0..n_threads {
s.spawn(|| loop {
let i = idx.fetch_add(1, Ordering::Relaxed);
if i >= items.len() {
break;
}
let item = &items[i];
let code = if !item.caps.is_empty()
&& item.op.output.as_ref().is_some_and(|o| o.contains('#'))
{
let mut op2 = item.op.clone();
op2.output = item
.op
.output
.as_ref()
.map(|o| apply_glob_output(o, &item.caps));
process_url(&item.url, &op2, None)
} else {
process_url(&item.url, item.op, None)
};
if code != 0 {
worst.store(code, Ordering::Relaxed);
}
});
}
});
worst.load(Ordering::Relaxed)
}
fn split_operations(toks: &[String]) -> Vec<Vec<String>> {
let mut segs: Vec<Vec<String>> = vec![Vec::new()];
for t in toks {
if t == "--next" || t == "-:" {
segs.push(Vec::new());
} else {
segs.last_mut().unwrap().push(t.clone());
}
}
segs
}
fn expand_config(toks: &[String], depth: u32) -> Result<Vec<String>, String> {
if depth > 16 {
return Err("config files nested too deeply".into());
}
let mut out = Vec::new();
let mut it = toks.iter();
while let Some(t) = it.next() {
if t == "-K" || t == "--config" {
let path = it
.next()
.ok_or_else(|| "--config requires a file".to_string())?;
let text =
std::fs::read_to_string(path).map_err(|e| format!("config file {path}: {e}"))?;
let inner = expand_config(&parse_config_text(&text), depth + 1)?;
out.extend(inner);
} else {
out.push(t.clone());
}
}
Ok(out)
}
fn parse_config_text(text: &str) -> Vec<String> {
let mut out = Vec::new();
for line in text.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
let (opt, val) = match line.find(|c: char| c.is_whitespace() || c == '=' || c == ':') {
Some(i) => {
let rest = line[i..]
.trim_start()
.strip_prefix(['=', ':'])
.unwrap_or_else(|| line[i..].trim_start())
.trim_start();
(&line[..i], Some(rest))
}
None => (line, None),
};
let opt_norm = if opt.starts_with('-') {
opt.to_string()
} else if opt.chars().count() == 1 {
format!("-{opt}")
} else {
format!("--{opt}")
};
out.push(opt_norm);
if let Some(v) = val.filter(|v| !v.is_empty()) {
let v = v
.strip_prefix('"')
.and_then(|s| s.strip_suffix('"'))
.unwrap_or(v);
out.push(v.to_string());
}
}
out
}
fn build_initial_jar(args: &Args) -> Result<Option<CookieJar>, String> {
if args.cookie_in.is_none() && args.cookie_jar.is_none() {
return Ok(None);
}
let mut jar = match args.cookie_in.as_deref() {
Some(s) if !s.contains('=') => CookieJar::load_netscape_or_empty(s)
.map_err(|e| format!("reading cookie file {s}: {e}"))?,
_ => CookieJar::new(),
};
if args.cookie_in.is_none() {
if let Some(path) = args.cookie_jar.as_deref() {
jar = CookieJar::load_netscape_or_empty(path)
.map_err(|e| format!("reading cookie file {path}: {e}"))?;
}
}
Ok(Some(jar))
}
fn resolve_proxy_spec(url: &Url, args: &Args) -> Option<String> {
if let Some(spec) = &args.proxy {
if spec.is_empty() {
return None;
}
return Some(spec.clone());
}
let read = |upper: &str, lower: &str| -> Option<String> {
for k in [upper, lower] {
if let Ok(v) = std::env::var(k) {
if !v.is_empty() {
return Some(v);
}
}
}
None
};
let scheme_proxy = match url.scheme.as_str() {
"https" => read("HTTPS_PROXY", "https_proxy"),
"http" => match std::env::var("http_proxy") {
Ok(v) if !v.is_empty() => Some(v),
_ => None,
},
_ => None,
};
scheme_proxy.or_else(|| read("ALL_PROXY", "all_proxy"))
}
fn resolve_noproxy(args: &Args) -> Option<String> {
if let Some(v) = &args.noproxy {
return Some(v.clone());
}
for k in ["NO_PROXY", "no_proxy"] {
if let Ok(v) = std::env::var(k) {
if !v.is_empty() {
return Some(v);
}
}
}
None
}
fn apply_explicit_cookies(jar: &mut CookieJar, data: &str, request_url: &Url) {
for pair in data.split(';') {
let pair = pair.trim();
if pair.is_empty() {
continue;
}
if let Some((k, v)) = pair.split_once('=') {
let k = k.trim();
let v = v.trim();
if !k.is_empty() {
jar.add_explicit(k, v, request_url);
}
}
}
}
fn read_at_file(path: &str) -> Result<Vec<u8>, String> {
std::fs::read(path).map_err(|e| format!("can't read {path:?}: {e}"))
}
fn strip_newlines(data: Vec<u8>) -> Vec<u8> {
data.into_iter()
.filter(|&b| b != b'\r' && b != b'\n' && b != 0)
.collect()
}
fn percent_encode_form(bytes: &[u8]) -> String {
use std::fmt::Write;
let mut out = String::with_capacity(bytes.len());
for &b in bytes {
match b {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => {
out.push(b as char);
}
b' ' => out.push('+'),
_ => write!(out, "%{b:02X}").expect("write to String"),
}
}
out
}
fn encode_urlencoded(spec: &str) -> Result<Vec<u8>, String> {
if let Some(eq) = spec.find('=') {
let (name, rest) = spec.split_at(eq);
let value = &rest[1..]; let encoded = percent_encode_form(value.as_bytes());
if name.is_empty() {
return Ok(encoded.into_bytes());
}
return Ok(format!("{name}={encoded}").into_bytes());
}
if let Some(at) = spec.find('@') {
let (name, rest) = spec.split_at(at);
let path = &rest[1..]; let bytes = read_at_file(path)?;
let encoded = percent_encode_form(&bytes);
if name.is_empty() {
return Ok(encoded.into_bytes());
}
return Ok(format!("{name}={encoded}").into_bytes());
}
Ok(percent_encode_form(spec.as_bytes()).into_bytes())
}
mod form_parser {
use super::{FormBody, FormExtra, FormPart};
pub(super) fn parse(spec: &str) -> Result<FormPart, String> {
let eq = spec.find('=').ok_or_else(|| {
format!("-F: expected 'name=value', got {spec:?} (use --form-string for literal '=')")
})?;
let name = spec[..eq].to_string();
if name.is_empty() {
return Err(format!("-F: empty field name: {spec:?}"));
}
let rest = &spec[eq + 1..];
let mut tokens = split_semi(rest);
let raw_value = tokens.remove(0);
let body = classify_body(&raw_value);
let mut extras = Vec::new();
for tok in tokens {
extras.push(classify_extra(&tok)?);
}
Ok(FormPart { name, body, extras })
}
fn classify_body(token: &str) -> FormBody {
if let Some(p) = token.strip_prefix('@') {
FormBody::File(p.to_string())
} else if let Some(p) = token.strip_prefix('<') {
FormBody::FileAsField(p.to_string())
} else {
FormBody::Literal(token.to_string())
}
}
fn classify_extra(token: &str) -> Result<FormExtra, String> {
let (k, v) = token
.split_once('=')
.ok_or_else(|| format!("-F: malformed modifier {token:?} (expected key=value)"))?;
let k = k.trim();
let v = v.to_string();
match k.to_ascii_lowercase().as_str() {
"type" => Ok(FormExtra::Type(v)),
"filename" => Ok(FormExtra::Filename(v)),
"headers" => {
let path = v
.strip_prefix('@')
.ok_or_else(|| format!("-F: ;headers= must be @file (got {token:?})"))?;
Ok(FormExtra::HeadersFile(path.to_string()))
}
_ => Err(format!("-F: unknown modifier {k:?}")),
}
}
fn split_semi(s: &str) -> Vec<String> {
let mut out = Vec::new();
let mut cur = String::new();
let mut chars = s.chars().peekable();
let mut in_quote = false;
while let Some(c) = chars.next() {
if in_quote {
match c {
'"' => in_quote = false,
'\\' => match chars.peek() {
Some('"') => {
cur.push('"');
chars.next();
}
Some('\\') => {
cur.push('\\');
chars.next();
}
_ => cur.push('\\'),
},
_ => cur.push(c),
}
} else {
match c {
'"' => in_quote = true,
';' => out.push(std::mem::take(&mut cur)),
_ => cur.push(c),
}
}
}
out.push(cur);
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn simple_literal() {
let p = parse("foo=bar").unwrap();
assert_eq!(p.name, "foo");
assert!(matches!(&p.body, FormBody::Literal(v) if v == "bar"));
assert!(p.extras.is_empty());
}
#[test]
fn at_file_is_file_upload() {
let p = parse("upload=@/tmp/x.bin").unwrap();
assert!(matches!(&p.body, FormBody::File(v) if v == "/tmp/x.bin"));
}
#[test]
fn lt_file_is_field_from_file() {
let p = parse("note=</tmp/x.txt").unwrap();
assert!(matches!(&p.body, FormBody::FileAsField(v) if v == "/tmp/x.txt"));
}
#[test]
fn quoted_value_with_semicolon() {
let p = parse(r#"k="a;b;c""#).unwrap();
assert!(matches!(&p.body, FormBody::Literal(v) if v == "a;b;c"));
}
#[test]
fn quoted_value_with_escapes() {
let p = parse(r#"k="he said \"hi\" \\""#).unwrap();
assert!(matches!(&p.body, FormBody::Literal(v) if v == r#"he said "hi" \"#));
}
#[test]
fn modifiers_type_filename_headers() {
let p = parse("f=@x;type=application/json;filename=other.json;headers=@hdrs").unwrap();
assert!(matches!(&p.body, FormBody::File(p) if p == "x"));
assert_eq!(p.extras.len(), 3);
assert!(matches!(&p.extras[0], FormExtra::Type(v) if v == "application/json"));
assert!(matches!(&p.extras[1], FormExtra::Filename(v) if v == "other.json"));
assert!(matches!(&p.extras[2], FormExtra::HeadersFile(v) if v == "hdrs"));
}
#[test]
fn empty_name_rejected() {
assert!(parse("=value").is_err());
}
#[test]
fn missing_eq_rejected() {
assert!(parse("foo").is_err());
}
#[test]
fn unknown_modifier_rejected() {
assert!(parse("foo=bar;weird=baz").is_err());
}
#[test]
fn headers_missing_at_rejected() {
assert!(parse("foo=bar;headers=hdrs").is_err());
}
}
}
mod multipart {
use super::{FormBody, FormExtra, FormPart};
pub(super) fn build(parts: &[FormPart], escape: bool) -> Result<(String, Vec<u8>), String> {
let boundary = make_boundary();
let mut out = Vec::new();
for part in parts {
out.extend_from_slice(b"--");
out.extend_from_slice(boundary.as_bytes());
out.extend_from_slice(b"\r\n");
write_part(part, escape, &mut out)?;
out.extend_from_slice(b"\r\n");
}
out.extend_from_slice(b"--");
out.extend_from_slice(boundary.as_bytes());
out.extend_from_slice(b"--\r\n");
Ok((boundary, out))
}
fn write_part(part: &FormPart, escape: bool, out: &mut Vec<u8>) -> Result<(), String> {
let (bytes, default_filename, is_upload): (Vec<u8>, Option<String>, bool) = match &part.body
{
FormBody::Literal(s) | FormBody::LiteralStrict(s) => {
(s.as_bytes().to_vec(), None, false)
}
FormBody::File(path) => {
let bytes =
std::fs::read(path).map_err(|e| format!("-F: can't read {path:?}: {e}"))?;
let name = std::path::Path::new(path)
.file_name()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or_else(|| path.clone());
(bytes, Some(name), true)
}
FormBody::FileAsField(path) => {
let bytes =
std::fs::read(path).map_err(|e| format!("-F: can't read {path:?}: {e}"))?;
(bytes, None, false)
}
};
let mut ctype: Option<&str> = None;
let mut filename: Option<String> = default_filename;
let mut extra_headers: Vec<u8> = Vec::new();
let mut promote_to_upload = is_upload;
for ex in &part.extras {
match ex {
FormExtra::Type(t) => ctype = Some(t),
FormExtra::Filename(f) => {
filename = Some(f.clone());
promote_to_upload = true;
}
FormExtra::HeadersFile(path) => {
let raw = std::fs::read(path)
.map_err(|e| format!("-F: can't read headers file {path:?}: {e}"))?;
for line in raw.split(|b| *b == b'\n') {
let mut l = line;
if l.last() == Some(&b'\r') {
l = &l[..l.len() - 1];
}
if l.is_empty() {
continue;
}
extra_headers.extend_from_slice(l);
extra_headers.extend_from_slice(b"\r\n");
}
}
}
}
out.extend_from_slice(b"Content-Disposition: form-data; name=\"");
out.extend_from_slice(encode_attr(&part.name, escape).as_bytes());
out.extend_from_slice(b"\"");
if promote_to_upload || filename.is_some() {
if let Some(fname) = filename.as_deref() {
out.extend_from_slice(b"; filename=\"");
out.extend_from_slice(encode_attr(fname, escape).as_bytes());
out.extend_from_slice(b"\"");
}
}
out.extend_from_slice(b"\r\n");
if let Some(t) = ctype {
out.extend_from_slice(b"Content-Type: ");
out.extend_from_slice(t.as_bytes());
out.extend_from_slice(b"\r\n");
} else if promote_to_upload {
out.extend_from_slice(b"Content-Type: application/octet-stream\r\n");
}
out.extend_from_slice(&extra_headers);
out.extend_from_slice(b"\r\n");
out.extend_from_slice(&bytes);
Ok(())
}
fn encode_attr(s: &str, escape: bool) -> String {
if escape {
let mut out = String::with_capacity(s.len());
for b in s.bytes() {
match b {
b'\r' => out.push_str("%0D"),
b'\n' => out.push_str("%0A"),
b'"' => out.push_str("%22"),
b'\\' => out.push_str("%5C"),
_ => out.push(b as char),
}
}
out
} else {
let mut out = String::with_capacity(s.len());
for c in s.chars() {
match c {
'"' => out.push_str("\\\""),
'\\' => out.push_str("\\\\"),
_ => out.push(c),
}
}
out
}
}
fn make_boundary() -> String {
let mut buf = [0u8; 8];
let ok = std::fs::File::open("/dev/urandom")
.and_then(|mut f| std::io::Read::read_exact(&mut f, &mut buf))
.is_ok();
if !ok {
use std::time::{SystemTime, UNIX_EPOCH};
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
for (i, b) in buf.iter_mut().enumerate() {
*b = ((nanos >> (i * 8)) & 0xFF) as u8;
}
}
let mut hex = String::with_capacity(16 + 19);
hex.push_str("----rsurl-boundary-");
for b in buf {
use std::fmt::Write;
write!(hex, "{b:02x}").expect("write to String");
}
hex
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn literal_part_round_trip() {
let parts = vec![FormPart {
name: "k".into(),
body: FormBody::Literal("v".into()),
extras: vec![],
}];
let (b, bytes) = build(&parts, false).unwrap();
let text = String::from_utf8(bytes).unwrap();
assert!(text.contains(&format!("--{b}\r\n")));
assert!(text.contains("Content-Disposition: form-data; name=\"k\"\r\n"));
assert!(!text.contains("filename="));
assert!(text.contains("\r\n\r\nv\r\n"));
assert!(text.ends_with(&format!("--{b}--\r\n")));
}
#[test]
fn file_part_gets_filename_and_octet_stream() {
let mut tmp = std::env::temp_dir();
tmp.push(format!("rsurl-mp-{}.bin", std::process::id()));
std::fs::write(&tmp, b"FILEBYTES").unwrap();
let path = tmp.to_string_lossy().into_owned();
let basename = tmp.file_name().unwrap().to_string_lossy().into_owned();
let parts = vec![FormPart {
name: "u".into(),
body: FormBody::File(path),
extras: vec![],
}];
let (_, bytes) = build(&parts, false).unwrap();
let _ = std::fs::remove_file(&tmp);
let text = String::from_utf8(bytes).unwrap();
assert!(text.contains(&format!(
"Content-Disposition: form-data; name=\"u\"; filename=\"{basename}\"\r\n"
)));
assert!(text.contains("Content-Type: application/octet-stream\r\n"));
assert!(text.contains("\r\n\r\nFILEBYTES\r\n"));
}
#[test]
fn type_filename_extras_take_effect() {
let parts = vec![FormPart {
name: "x".into(),
body: FormBody::Literal("body".into()),
extras: vec![
FormExtra::Type("application/json".into()),
FormExtra::Filename("over.json".into()),
],
}];
let (_, bytes) = build(&parts, false).unwrap();
let text = String::from_utf8(bytes).unwrap();
assert!(text.contains("name=\"x\"; filename=\"over.json\"\r\n"));
assert!(text.contains("Content-Type: application/json\r\n"));
}
#[test]
fn form_escape_uses_percent_encoding() {
let parts = vec![FormPart {
name: "weird\"name".into(),
body: FormBody::Literal("v".into()),
extras: vec![],
}];
let (_, bytes) = build(&parts, true).unwrap();
let text = String::from_utf8(bytes).unwrap();
assert!(text.contains("name=\"weird%22name\""), "got: {text}");
}
#[test]
fn default_backslash_escape_preserves_curl_behaviour() {
let parts = vec![FormPart {
name: "weird\"name".into(),
body: FormBody::Literal("v".into()),
extras: vec![],
}];
let (_, bytes) = build(&parts, false).unwrap();
let text = String::from_utf8(bytes).unwrap();
assert!(text.contains(r#"name="weird\"name""#), "got: {text}");
}
}
}
type AssembledBody = (Vec<u8>, String, &'static str);
fn build_upload_body(path: &str) -> Result<AssembledBody, String> {
let bytes = std::fs::read(path).map_err(|e| format!("-T: can't read {path:?}: {e}"))?;
Ok((bytes, "application/octet-stream".into(), "PUT"))
}
fn build_multipart_body(parts: &[FormPart], escape: bool) -> Result<AssembledBody, String> {
let (boundary, bytes) = multipart::build(parts, escape)?;
let ctype = format!("multipart/form-data; boundary={boundary}");
Ok((bytes, ctype, "POST"))
}
fn assemble_request_body(args: &Args) -> Result<Option<AssembledBody>, String> {
let n = (!args.data_parts.is_empty()) as u8
+ (!args.form_parts.is_empty()) as u8
+ args.upload_file.is_some() as u8
+ (!args.json_parts.is_empty()) as u8;
if n > 1 {
return Err(
"-d/--data, -F/--form, -T/--upload-file, and --json are mutually exclusive".into(),
);
}
if let Some(path) = &args.upload_file {
return build_upload_body(path).map(Some);
}
if !args.json_parts.is_empty() {
let mut out: Vec<u8> = Vec::new();
for v in &args.json_parts {
if let Some(path) = v.strip_prefix('@') {
out.extend_from_slice(&read_at_file(path)?);
} else {
out.extend_from_slice(v.as_bytes());
}
}
return Ok(Some((out, "application/json".into(), "POST")));
}
if !args.form_parts.is_empty() {
return build_multipart_body(&args.form_parts, args.form_escape).map(Some);
}
if let Some(bytes) = assemble_form_body(&args.data_parts)? {
return Ok(Some((
bytes,
"application/x-www-form-urlencoded".into(),
"POST",
)));
}
Ok(None)
}
fn assemble_form_body(parts: &[DataPart]) -> Result<Option<Vec<u8>>, String> {
if parts.is_empty() {
return Ok(None);
}
let mut out: Vec<u8> = Vec::new();
for part in parts {
if !out.is_empty() {
out.push(b'&');
}
match part {
DataPart::Plain { value, at_file_ok } => {
if *at_file_ok {
if let Some(path) = value.strip_prefix('@') {
out.extend_from_slice(&strip_newlines(read_at_file(path)?));
continue;
}
}
out.extend_from_slice(value.as_bytes());
}
DataPart::Binary { value } => {
if let Some(path) = value.strip_prefix('@') {
out.extend_from_slice(&read_at_file(path)?);
} else {
out.extend_from_slice(value.as_bytes());
}
}
DataPart::UrlEncoded { value } => {
out.extend_from_slice(&encode_urlencoded(value)?);
}
}
}
Ok(Some(out))
}
fn process_url(url: &str, args: &Args, mut jar: Option<&mut CookieJar>) -> u8 {
if args.torrent
|| url.starts_with("magnet:")
|| args.bt_info
|| args.bt_save_torrent
|| args.bt_file.is_some()
|| args.bt_concat
{
#[cfg(feature = "bittorrent")]
{
return run_bittorrent(url, args);
}
#[cfg(not(feature = "bittorrent"))]
{
if show_errors(args) {
eprintln!("rsurl: this build has no BitTorrent support");
}
return 2;
}
}
let scheme_defaulted;
let url: &str = if url.contains("://") {
url
} else {
let scheme = args.proto_default.as_deref().unwrap_or("http");
scheme_defaulted = format!("{scheme}://{url}");
&scheme_defaulted
};
let mut parsed_url = match Url::parse(url) {
Ok(u) => u,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 3;
}
};
if let Err(e) = parsed_url.set_idn(!args.no_idn) {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 3;
}
if let Some(spec) = &args.proto {
if !proto_allowed(&parsed_url.scheme, spec) {
if show_errors(args) {
eprintln!(
"rsurl: protocol \"{}\" not permitted by --proto",
parsed_url.scheme
);
}
return 1;
}
}
if !matches!(parsed_url.scheme.as_str(), "http" | "https") {
if matches!(parsed_url.scheme.as_str(), "ws" | "wss") {
return run_websocket(&parsed_url, args);
}
if parsed_url.scheme == "rtsp" {
return run_rtsp(&parsed_url, args);
}
if matches!(parsed_url.scheme.as_str(), "smtp" | "smtps") {
return run_smtp(&parsed_url, args);
}
if parsed_url.scheme == "telnet" {
return run_telnet(&parsed_url, args);
}
if matches!(parsed_url.scheme.as_str(), "mqtt" | "mqtts")
&& (!args.data_parts.is_empty() || args.upload_file.is_some())
{
return run_mqtt_publish(&parsed_url, args);
}
if let Some(path) = &args.upload_file {
if matches!(parsed_url.scheme.as_str(), "ftp" | "ftps") {
return run_ftp_upload(&parsed_url, path, args);
}
if parsed_url.scheme == "tftp" {
return run_tftp_upload(&parsed_url, path, args);
}
if matches!(parsed_url.scheme.as_str(), "sftp" | "scp") {
#[cfg(feature = "ssh")]
{
return run_ssh_upload(&parsed_url, path, args);
}
#[cfg(not(feature = "ssh"))]
{
if show_errors(args) {
eprintln!("rsurl: this build has no SSH (sftp/scp) support");
}
return 2;
}
}
if show_errors(args) {
eprintln!(
"rsurl: -T is only supported for HTTP(S), FTP(S), TFTP, and SFTP/SCP URLs in this build"
);
}
return 2;
}
if matches!(parsed_url.scheme.as_str(), "sftp" | "scp") {
#[cfg(feature = "ssh")]
{
return run_ssh(&parsed_url, args);
}
#[cfg(not(feature = "ssh"))]
{
if show_errors(args) {
eprintln!("rsurl: this build has no SSH (sftp/scp) support");
}
return 2;
}
}
let to_file = args.remote_name || args.output.as_deref().is_some_and(|p| p != "-");
if to_file {
return run_stream_download(&parsed_url, args);
}
return run_transfer(&parsed_url, args);
}
let mut assembled = match assemble_request_body(args) {
Ok(b) => b,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 2;
}
};
let mut url_owned = url.to_string();
if args.get {
if let Some((bytes, ctype, _)) = &assembled {
if ctype.starts_with("application/x-www-form-urlencoded") {
let q = String::from_utf8_lossy(bytes);
if !q.is_empty() {
url_owned.push(if url_owned.contains('?') { '&' } else { '?' });
url_owned.push_str(&q);
}
assembled = None;
}
}
}
let method = args.method.clone().unwrap_or_else(|| {
if args.head {
"HEAD".to_string()
} else if args.get {
"GET".to_string()
} else if let Some((_, _, m)) = &assembled {
(*m).to_string()
} else {
"GET".to_string()
}
});
let mut req = match Request::new(&method, &url_owned) {
Ok(r) => r,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 3;
}
};
for (k, v) in &args.headers {
req = req.header(k, v);
}
if let Some(ua) = &args.user_agent {
req = req.header("User-Agent", ua);
}
if let Some(rf) = &args.referer {
req = req.header("Referer", rf);
}
if args.auto_referer {
req = req.auto_referer(true);
}
if let Some(tc) = &args.time_cond {
if let Some((hdr, date)) = time_cond_header(tc) {
req = req.header(hdr, &date);
} else if show_errors(args) {
eprintln!("rsurl: warning: could not parse --time-cond {tc:?}");
}
}
let has_header = |name: &str| {
args.headers
.iter()
.any(|(k, _)| k.eq_ignore_ascii_case(name))
};
if args.compressed && !has_header("accept-encoding") {
req = req.header("Accept-Encoding", "gzip, deflate, br, zstd");
}
if !args.json_parts.is_empty() && !has_header("accept") {
req = req.header("Accept", "application/json");
}
if let Some(r) = &args.range {
if !has_header("range") {
let v = if r.contains('=') {
r.clone()
} else {
format!("bytes={r}")
};
req = req.header("Range", &v);
}
}
if let Some((body_bytes, ctype, _method)) = assembled {
if !args
.headers
.iter()
.any(|(k, _)| k.eq_ignore_ascii_case("content-type"))
{
req = req.header("Content-Type", &ctype);
}
req = req.body(body_bytes);
}
match args.http_version {
Some(HttpVersionPref::Http2Only) => req = req.http2_only(),
Some(HttpVersionPref::Http11Only) => req = req.http11_only(),
Some(HttpVersionPref::Http3) => req = req.http3(),
Some(HttpVersionPref::Http3Only) => req = req.http3_only(),
Some(HttpVersionPref::Auto) | None => {}
}
if args.follow_redirects {
req = req.follow_redirects(true);
}
if let Some(n) = args.max_redirs {
req = req.max_redirs(n);
}
if args.location_trusted {
req = req.redirect_trusted(true);
}
if args.post301 {
req = req.keep_post_on(301);
}
if args.post302 {
req = req.keep_post_on(302);
}
if args.post303 {
req = req.keep_post_on(303);
}
for (fh, fp, th, tp) in &args.connect_to {
req = req.connect_to(fh, *fp, th, *tp);
}
if let Some(path) = &args.unix_socket {
#[cfg(unix)]
{
req = req.connector(std::sync::Arc::new(rsurl::net::UnixConnector {
path: path.into(),
}));
}
#[cfg(not(unix))]
{
let _ = path;
if show_errors(args) {
eprintln!("rsurl: --unix-socket is not supported on this platform");
}
return 2;
}
}
if let Some((u, p)) = &args.basic_auth {
req = req.basic_auth(u, p);
} else if args.netrc && parsed_url.userinfo.is_none() {
if let Some((u, p)) = netrc_credentials(args, &parsed_url.host) {
req = req.basic_auth(&u, &p);
}
}
if args.insecure {
req = req.verify_tls(false);
}
if let Some(v) = args.tls_min {
req = req.tls_min_version(v);
}
if let Some(v) = args.tls_max {
req = req.tls_max_version(v);
}
if args.digest {
req = req.digest_auth(true);
}
if let Some(token) = &args.bearer {
if !args
.headers
.iter()
.any(|(k, _)| k.eq_ignore_ascii_case("authorization"))
{
req = req.header("Authorization", &format!("Bearer {token}"));
}
}
if let (Some(spec), Some((ak, sk))) = (&args.aws_sigv4, &args.basic_auth) {
req = req.aws_sigv4(spec, ak, sk);
}
if args.no_idn {
req = req.idn(false);
}
if let Some(path) = &args.cacert {
req = req.ca_bundle(path);
}
if let Some(dir) = &args.capath {
req = req.ca_path(dir);
}
if let Some(spec) = &args.pinned_pubkey {
req = req.pinned_pubkey(spec);
}
if let Some(path) = &args.crl_file {
req = req.crl_file(path);
}
if let Some(list) = &args.ciphers {
req = req.ciphers(list);
}
if let Some(list) = &args.tls13_ciphers {
req = req.tls13_ciphers(list);
}
if let Some(cert) = &args.cert {
let (cert_path, inline_pass) = split_cert_pass(cert);
req = req.client_cert(cert_path);
if let Some(key) = &args.key_file {
req = req.client_key(key);
}
if let Some(pass) = args.key_pass.as_deref().or(inline_pass) {
req = req.client_key_pass(pass);
}
if args.cert_type_der {
req = req.cert_type_der(true);
}
if args.key_type_der {
req = req.key_type_der(true);
}
}
if let Some(secs) = args.max_time {
req = req.max_time(Duration::from_secs(secs));
}
if let Some(secs) = args.connect_timeout {
req = req.connect_timeout(Duration::from_secs(secs));
}
if args.ipv6 {
req = req.ipv6();
} else if args.ipv4 {
req = req.ipv4();
}
for (h, p, ip) in &args.resolve {
req = req.resolve_addr(h, *p, *ip);
}
let proxy_spec = resolve_proxy_spec(&parsed_url, args);
if let Some(spec) = proxy_spec {
req = match req.proxy(&spec) {
Ok(r) => r,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: --proxy: {e}");
}
return 5;
}
};
if let Some((u, p)) = &args.proxy_user {
req = match req.proxy_user(u, p) {
Ok(r) => r,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: --proxy-user: {e}");
}
return 5;
}
};
}
}
if let Some(list) = resolve_noproxy(args) {
req = req.no_proxy(list.split(',').map(str::trim).filter(|s| !s.is_empty()));
}
if let (Some(j), Some(data)) = (jar.as_deref_mut(), args.cookie_in.as_deref()) {
if data.contains('=') {
apply_explicit_cookies(j, data, &parsed_url);
}
}
if streams_to_file(args) {
if args.continue_resume {
if parallel_segments_eligible(args) && jar.is_none() {
if let Some(code) = run_parallel_resume(&req, &parsed_url, args) {
return code;
}
}
return run_http_download(req, &parsed_url, args, jar);
}
if parallel_segments_eligible(args) {
return run_parallel_download(req, &parsed_url, args, jar);
}
return run_http_download(req, &parsed_url, args, jar);
}
let started = std::time::Instant::now();
let mut jar = jar;
let mut attempt = 0u32;
let resp = loop {
let attempt_req = req.clone();
let result = match (jar.as_deref_mut(), args.verbose) {
(Some(j), true) => {
let mut err = io::stderr().lock();
attempt_req.send_traced_with_jar(j, &mut err)
}
(Some(j), false) => attempt_req.send_with_jar(j),
(None, true) => {
let mut err = io::stderr().lock();
attempt_req.send_traced(&mut err)
}
(None, false) => attempt_req.send(),
};
let within_budget = args
.retry_max_time
.is_none_or(|m| started.elapsed().as_secs() < m);
match result {
Ok(r) if is_retryable_status(r.status) && attempt < args.retry && within_budget => {
attempt += 1;
if show_errors(args) {
eprintln!(
"rsurl: transient HTTP {} — retry {}/{}",
r.status, attempt, args.retry
);
}
std::thread::sleep(next_retry_delay(attempt, args));
}
Ok(r) => break r,
Err(e) if attempt < args.retry && within_budget && should_retry_err(&e, args) => {
attempt += 1;
if show_errors(args) {
eprintln!("rsurl: {e} — retry {}/{}", attempt, args.retry);
}
std::thread::sleep(next_retry_delay(attempt, args));
}
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return transfer_exit_code(&e);
}
}
};
let time_total = started.elapsed();
if let Some(max) = args.max_filesize {
let declared = resp
.header("content-length")
.and_then(|v| v.trim().parse::<u64>().ok());
if declared.is_some_and(|n| n > max) || resp.body.len() as u64 > max {
if show_errors(args) {
eprintln!("rsurl: Maximum file size exceeded");
}
return 63;
}
}
if let Some(path) = &args.dump_header {
if let Err(e) = dump_headers(&resp, path) {
if show_errors(args) {
eprintln!("rsurl: dump-header {path}: {e}");
}
return 23;
}
}
if args.fail_with_body && resp.status >= 400 {
if show_errors(args) {
eprintln!(
"rsurl: The requested URL returned error: {} {}",
resp.status, resp.reason
);
}
let _ = write_output(&resp, &parsed_url, args);
run_write_out(&resp, &parsed_url, args, time_total, resp.body.len() as u64);
return 22;
}
if args.fail && resp.status >= 400 {
if show_errors(args) {
eprintln!(
"rsurl: The requested URL returned error: {} {}",
resp.status, resp.reason
);
}
run_write_out(&resp, &parsed_url, args, time_total, resp.body.len() as u64);
return 22;
}
if let Err(e) = write_output(&resp, &parsed_url, args) {
if show_errors(args) {
eprintln!("rsurl: write error: {e}");
}
return 23;
}
if args.remote_time {
if let Some(path) = args.output.as_deref().filter(|p| *p != "-") {
set_remote_time(&resp, path);
} else if args.remote_name {
if let Ok(name) = remote_name_from_url(&parsed_url) {
set_remote_time(&resp, &name);
}
}
}
run_write_out(&resp, &parsed_url, args, time_total, resp.body.len() as u64);
0
}
fn parse_args(raw: &[String]) -> Result<Args, String> {
let mut a = Args::default();
let mut it = raw.iter();
while let Some(arg) = it.next() {
match arg.as_str() {
"-h" | "--help" => {
print_usage();
std::process::exit(0);
}
"-V" | "--version" => {
println!("rsurl {VERSION}");
std::process::exit(0);
}
"-o" | "--output" => {
a.output = Some(next_val(&mut it, arg)?);
}
"-i" | "--include" => a.include_headers = true,
"-I" | "--head" => {
a.head = true;
a.include_headers = true;
}
"-v" | "--verbose" => {
a.verbose = true;
a.verbosity = a.verbosity.saturating_add(1);
}
"-s" | "--silent" => a.silent = true,
"-X" | "--request" => a.method = Some(next_val(&mut it, arg)?),
"-H" | "--header" => {
let h = next_val(&mut it, arg)?;
let (k, v) = h
.split_once(':')
.ok_or_else(|| format!("malformed header: {h:?}"))?;
a.headers.push((k.trim().to_string(), v.trim().to_string()));
}
"-d" | "--data" | "--data-ascii" => a.data_parts.push(DataPart::Plain {
value: next_val(&mut it, arg)?,
at_file_ok: true,
}),
"--remove-on-error" => a.remove_on_error = true,
"--no-clobber" => a.no_clobber = true,
"--disable-epsv" => a.disable_epsv = true,
"--ssl-reqd" => a.ssl_reqd = true,
"--ftp-create-dirs" => a.ftp_create_dirs = true,
"-P" | "--ftp-port" => a.ftp_port = Some(next_val(&mut it, arg)?),
"--ftp-pasv" => {}
"--epsv" => a.disable_epsv = false,
"--basic" | "--ftp-skip-pasv-ip" => {}
"--json" => a.json_parts.push(next_val(&mut it, arg)?),
"--oauth2-bearer" => a.bearer = Some(next_val(&mut it, arg)?),
"--aws-sigv4" => a.aws_sigv4 = Some(next_val(&mut it, arg)?),
"--data-raw" => a.data_parts.push(DataPart::Plain {
value: next_val(&mut it, arg)?,
at_file_ok: false,
}),
"--data-binary" => a.data_parts.push(DataPart::Binary {
value: next_val(&mut it, arg)?,
}),
"--data-urlencode" => a.data_parts.push(DataPart::UrlEncoded {
value: next_val(&mut it, arg)?,
}),
"-F" | "--form" => {
let v = next_val(&mut it, arg)?;
a.form_parts.push(form_parser::parse(&v)?);
}
"--form-string" => {
let v = next_val(&mut it, arg)?;
let eq = v
.find('=')
.ok_or_else(|| format!("--form-string: expected 'name=value', got {v:?}"))?;
let name = v[..eq].to_string();
if name.is_empty() {
return Err(format!("--form-string: empty field name: {v:?}"));
}
let value = v[eq + 1..].to_string();
a.form_parts.push(FormPart {
name,
body: FormBody::LiteralStrict(value),
extras: vec![],
});
}
"--form-escape" => a.form_escape = true,
"-T" | "--upload-file" => a.upload_file = Some(next_val(&mut it, arg)?),
"-C" | "--continue-at" => {
let v = next_val(&mut it, arg)?;
if v == "-" {
a.continue_resume = true;
} else {
a.continue_at = Some(
v.parse::<u64>()
.map_err(|_| format!("-C/--continue-at: not a byte offset: {v:?}"))?,
);
}
}
"-a" | "--append" => a.append = true,
"--key" => {
let path = next_val(&mut it, arg)?;
a.key_file = Some(path.clone());
a.ssh_keys.push(path);
}
"-A" | "--user-agent" => a.user_agent = Some(next_val(&mut it, arg)?),
"-e" | "--referer" => {
let v = next_val(&mut it, arg)?;
let (head, auto) = match v.strip_suffix(";auto") {
Some(h) => (h, true),
None => (v.as_str(), false),
};
a.auto_referer = a.auto_referer || auto;
if !head.is_empty() {
a.referer = Some(head.to_string());
}
}
"-z" | "--time-cond" => a.time_cond = Some(next_val(&mut it, arg)?),
"--output-dir" => a.output_dir = Some(next_val(&mut it, arg)?),
"--fail-with-body" => a.fail_with_body = true,
"--proto" => a.proto = Some(next_val(&mut it, arg)?),
"--proto-default" => a.proto_default = Some(next_val(&mut it, arg)?),
"--http2" => a.http_version = Some(HttpVersionPref::Http2Only),
"--http1.1" | "--http1" => a.http_version = Some(HttpVersionPref::Http11Only),
"--http3" => a.http_version = Some(HttpVersionPref::Http3),
"--http3-only" => a.http_version = Some(HttpVersionPref::Http3Only),
"-L" | "--location" => a.follow_redirects = true,
"--max-redirs" => {
let v = next_val(&mut it, arg)?;
a.max_redirs = Some(
v.parse::<u32>()
.map_err(|_| format!("--max-redirs: not a number: {v:?}"))?,
);
}
"-u" | "--user" => {
let v = next_val(&mut it, arg)?;
let (u, p) = match v.split_once(':') {
Some((u, p)) => (u.to_string(), p.to_string()),
None => (v.clone(), String::new()),
};
a.basic_auth = Some((u, p));
}
"-k" | "--insecure" => a.insecure = true,
"--tlsv1" | "--tlsv1.0" | "--tlsv1.1" | "--tlsv1.2" => {
a.tls_min = Some(rsurl::tls::ProtocolVersion::TLSv1_2)
}
"--tlsv1.3" => a.tls_min = Some(rsurl::tls::ProtocolVersion::TLSv1_3),
"--mail-from" => a.mail_from = Some(next_val(&mut it, arg)?),
"--mail-rcpt" => a.mail_rcpt.push(next_val(&mut it, arg)?),
"--digest" => a.digest = true,
"-Z" | "--parallel" => a.parallel = true,
"--parallel-max" => {
a.parallel_max = Some(
next_val(&mut it, arg)?
.parse()
.map_err(|_| "--parallel-max requires a number".to_string())?,
)
}
"--parallel-segments" => {
let n = match it.clone().next().and_then(|s| s.parse::<usize>().ok()) {
Some(v) => {
it.next();
v
}
None => 4,
};
a.parallel_segments = Some(n);
}
"--torrent" => a.torrent = true,
"--listen-port" => {
a.listen_port = Some(
next_val(&mut it, arg)?
.parse()
.map_err(|_| "--listen-port requires a port number".to_string())?,
)
}
"--bt-peer" => a.bt_peers.push(next_val(&mut it, arg)?),
"--no-dht" => a.no_dht = true,
"--seed" => a.seed = true,
"--recheck" => a.recheck = true,
"--bt-info" => a.bt_info = true,
"--bt-save-torrent" => a.bt_save_torrent = true,
"--bt-file" => a.bt_file = Some(next_val(&mut it, arg)?),
"--bt-concat" => a.bt_concat = true,
"--share-ratio" => {
a.share_ratio = Some(
next_val(&mut it, arg)?
.parse()
.map_err(|_| "--share-ratio requires a number".to_string())?,
)
}
"--tls-max" => {
let v = next_val(&mut it, arg)?;
a.tls_max = Some(match v.as_str() {
"1.3" => rsurl::tls::ProtocolVersion::TLSv1_3,
"1.0" | "1.1" | "1.2" => rsurl::tls::ProtocolVersion::TLSv1_2,
other => return Err(format!("--tls-max: unsupported version {other:?}")),
});
}
"--no-idn" => a.no_idn = true,
"--cacert" => a.cacert = Some(next_val(&mut it, arg)?),
"--max-time" => {
let v = next_val(&mut it, arg)?;
a.max_time = Some(
v.parse::<u64>()
.map_err(|_| format!("--max-time: not a number: {v:?}"))?,
);
}
"--connect-timeout" => {
let v = next_val(&mut it, arg)?;
a.connect_timeout = Some(
v.parse::<u64>()
.map_err(|_| format!("--connect-timeout: not a number: {v:?}"))?,
);
}
"-O" | "--remote-name" => a.remote_name = true,
"-b" | "--cookie" => a.cookie_in = Some(next_val(&mut it, arg)?),
"-c" | "--cookie-jar" => a.cookie_jar = Some(next_val(&mut it, arg)?),
"-x" | "--proxy" => a.proxy = Some(next_val(&mut it, arg)?),
"--socks4" => a.proxy = Some(format!("socks4://{}", next_val(&mut it, arg)?)),
"--socks4a" => a.proxy = Some(format!("socks4a://{}", next_val(&mut it, arg)?)),
"--socks5" => a.proxy = Some(format!("socks5://{}", next_val(&mut it, arg)?)),
"--socks5-hostname" => a.proxy = Some(format!("socks5h://{}", next_val(&mut it, arg)?)),
"-U" | "--proxy-user" => {
let v = next_val(&mut it, arg)?;
let (u, p) = match v.split_once(':') {
Some((u, p)) => (u.to_string(), p.to_string()),
None => (v.clone(), String::new()),
};
a.proxy_user = Some((u, p));
}
"--noproxy" => a.noproxy = Some(next_val(&mut it, arg)?),
"--url" => a.urls.push(next_val(&mut it, arg)?),
"-f" | "--fail" => a.fail = true,
"-S" | "--show-error" => a.show_error = true,
"-G" | "--get" => a.get = true,
"-r" | "--range" => a.range = Some(next_val(&mut it, arg)?),
"--compressed" => a.compressed = true,
"-D" | "--dump-header" => a.dump_header = Some(next_val(&mut it, arg)?),
"-R" | "--remote-time" => a.remote_time = true,
"--create-dirs" => a.create_dirs = true,
"--max-filesize" => {
a.max_filesize = Some(
next_val(&mut it, arg)?
.parse()
.map_err(|_| "--max-filesize requires a byte count".to_string())?,
)
}
"-w" | "--write-out" => a.write_out = Some(next_val(&mut it, arg)?),
"-n" | "--netrc" => a.netrc = true,
"--netrc-file" => {
a.netrc_file = Some(next_val(&mut it, arg)?);
a.netrc = true;
}
"-J" | "--remote-header-name" => a.remote_header_name = true,
"--retry" => {
a.retry = next_val(&mut it, arg)?
.parse()
.map_err(|_| "--retry requires a count".to_string())?
}
"--retry-delay" => {
a.retry_delay = Some(
next_val(&mut it, arg)?
.parse()
.map_err(|_| "--retry-delay requires seconds".to_string())?,
)
}
"--retry-max-time" => {
a.retry_max_time = Some(
next_val(&mut it, arg)?
.parse()
.map_err(|_| "--retry-max-time requires seconds".to_string())?,
)
}
"--retry-connrefused" => a.retry_connrefused = true,
"--retry-all-errors" => a.retry_all_errors = true,
"-g" | "--globoff" => a.globoff = true,
"--unix-socket" | "--abstract-unix-socket" => {
a.unix_socket = Some(next_val(&mut it, arg)?)
}
"--location-trusted" => {
a.follow_redirects = true;
a.location_trusted = true;
}
"--post301" => a.post301 = true,
"--post302" => a.post302 = true,
"--post303" => a.post303 = true,
"--connect-to" => {
let spec = next_val(&mut it, arg)?;
let p: Vec<&str> = spec.split(':').collect();
if p.len() != 4 {
return Err(format!(
"--connect-to expects HOST1:PORT1:HOST2:PORT2: {spec:?}"
));
}
let port = |s: &str, what: &str| -> Result<u16, String> {
if s.is_empty() {
Ok(0)
} else {
s.parse()
.map_err(|_| format!("--connect-to: bad {what} in {spec:?}"))
}
};
a.connect_to.push((
p[0].to_string(),
port(p[1], "PORT1")?,
p[2].to_string(),
port(p[3], "PORT2")?,
));
}
"-4" | "--ipv4" => a.ipv4 = true,
"-6" | "--ipv6" => a.ipv6 = true,
"-#" | "--progress-bar" => a.progress_bar = true,
"-E" | "--cert" => a.cert = Some(next_val(&mut it, arg)?),
"--pass" => a.key_pass = Some(next_val(&mut it, arg)?),
"--cert-type" => a.cert_type_der = parse_cert_type(&next_val(&mut it, arg)?, arg)?,
"--key-type" => a.key_type_der = parse_cert_type(&next_val(&mut it, arg)?, arg)?,
"--pinnedpubkey" => a.pinned_pubkey = Some(next_val(&mut it, arg)?),
"--capath" => a.capath = Some(next_val(&mut it, arg)?),
"--crlfile" => a.crl_file = Some(next_val(&mut it, arg)?),
"--limit-rate" => a.limit_rate = Some(next_val(&mut it, arg)?),
"-Y" | "--speed-limit" => a.speed_limit = Some(next_val(&mut it, arg)?),
"-y" | "--speed-time" => a.speed_time = Some(next_val(&mut it, arg)?),
"--resolve" => {
let spec = next_val(&mut it, arg)?;
let mut parts = spec.splitn(3, ':');
let host = parts
.next()
.filter(|h| !h.is_empty())
.ok_or_else(|| format!("--resolve: missing host in {spec:?}"))?
.trim_start_matches(['+', '-']);
let port: u16 = parts
.next()
.and_then(|p| p.parse().ok())
.ok_or_else(|| format!("--resolve: bad port in {spec:?}"))?;
let addr_s = parts
.next()
.ok_or_else(|| format!("--resolve: missing address in {spec:?}"))?;
let addr_s = addr_s.trim().trim_start_matches('[').trim_end_matches(']');
let ip: std::net::IpAddr = addr_s
.parse()
.map_err(|_| format!("--resolve: bad IP {addr_s:?}"))?;
a.resolve.push((host.to_string(), port, ip));
}
"-q"
| "--disable"
| "-N"
| "--no-buffer"
| "--no-progress-meter"
| "--styled-output"
| "--no-styled-output" => {}
"--ciphers" => a.ciphers = Some(next_val(&mut it, arg)?),
"--tls13-ciphers" => a.tls13_ciphers = Some(next_val(&mut it, arg)?),
"--cert-status" => {
return Err(
"--cert-status: OCSP-staple validation is not implemented (rsurl does not \
request or require a stapled response)"
.to_string(),
);
}
s if s.starts_with("--") => return Err(format!("unknown option: {s}")),
s if s.starts_with('-') && s.len() > 1 => return Err(format!("unknown option: {s}")),
_ => {
a.urls.push(arg.clone());
}
}
}
Ok(a)
}
fn show_errors(args: &Args) -> bool {
!args.silent || args.show_error
}
fn is_retryable_status(status: u16) -> bool {
matches!(status, 408 | 429 | 500 | 502 | 503 | 504)
}
fn retry_delay(attempt: u32) -> std::time::Duration {
let secs = 1u64
.checked_shl(attempt.saturating_sub(1))
.unwrap_or(60)
.min(60);
std::time::Duration::from_secs(secs)
}
fn next_retry_delay(attempt: u32, args: &Args) -> std::time::Duration {
match args.retry_delay {
Some(s) => std::time::Duration::from_secs(s),
None => retry_delay(attempt),
}
}
fn transfer_exit_code(e: &rsurl::Error) -> u8 {
use std::io::ErrorKind;
match e {
rsurl::Error::InvalidUrl(_) => 3, rsurl::Error::UnsupportedScheme(_) => 1, rsurl::Error::UnexpectedEof => 52, rsurl::Error::Ssh(_) => 79, rsurl::Error::H2NotNegotiated => 7,
rsurl::Error::Decode(_) => 23,
rsurl::Error::Status { code, .. } => {
if *code >= 400 {
22 } else {
7
}
}
rsurl::Error::BadResponse(m) => {
let m = m.to_ascii_lowercase();
if m.contains("timed out") {
28 } else if m.contains("redirect") {
47 } else {
8 }
}
rsurl::Error::Io(io) => match io.kind() {
ErrorKind::TimedOut => 28,
ErrorKind::ConnectionRefused
| ErrorKind::ConnectionReset
| ErrorKind::ConnectionAborted => 7,
_ if io.to_string().contains("failed to lookup") => 6, _ => 7, },
rsurl::Error::Cancelled => 42, }
}
fn should_retry_err(e: &rsurl::Error, args: &Args) -> bool {
if args.retry_all_errors {
return true;
}
match e {
rsurl::Error::Io(io) => {
let k = io.kind();
matches!(
k,
std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock
) || (args.retry_connrefused && k == std::io::ErrorKind::ConnectionRefused)
}
rsurl::Error::UnexpectedEof => true,
_ => false,
}
}
fn netrc_credentials(args: &Args, host: &str) -> Option<(String, String)> {
let path: std::path::PathBuf = match &args.netrc_file {
Some(p) => p.into(),
None => {
let home = std::env::var_os("HOME")?;
let mut p = std::path::PathBuf::from(home);
p.push(".netrc");
p
}
};
let text = std::fs::read_to_string(&path).ok()?;
netrc_lookup(&text, host)
}
fn netrc_lookup(text: &str, host: &str) -> Option<(String, String)> {
let mut toks = text.split_whitespace();
let mut entries: Vec<(String, Option<String>, Option<String>)> = Vec::new();
while let Some(t) = toks.next() {
match t {
"machine" => {
if let Some(n) = toks.next() {
entries.push((n.to_string(), None, None));
}
}
"default" => entries.push(("\0default".to_string(), None, None)),
"login" => {
if let (Some(v), Some(e)) = (toks.next(), entries.last_mut()) {
e.1 = Some(v.to_string());
}
}
"password" => {
if let (Some(v), Some(e)) = (toks.next(), entries.last_mut()) {
e.2 = Some(v.to_string());
}
}
"account" | "macdef" => {
let _ = toks.next();
}
_ => {}
}
}
let pick = |e: &(String, Option<String>, Option<String>)| {
(
e.1.clone().unwrap_or_default(),
e.2.clone().unwrap_or_default(),
)
};
entries
.iter()
.find(|e| e.0.eq_ignore_ascii_case(host))
.or_else(|| entries.iter().find(|e| e.0 == "\0default"))
.map(pick)
}
fn content_disposition_filename(resp: &Response) -> Option<String> {
let cd = resp.header("content-disposition")?;
for part in cd.split(';') {
let p = part.trim();
let Some(val) = p
.strip_prefix("filename*=")
.or_else(|| p.strip_prefix("filename="))
else {
continue;
};
let val = val.trim().trim_matches('"');
let val = val.rsplit("''").next().unwrap_or(val);
let name = std::path::Path::new(val).file_name()?.to_str()?.to_string();
if name.is_empty() || name == "." || name == ".." {
return None;
}
return Some(name);
}
None
}
fn next_val(it: &mut std::slice::Iter<'_, String>, flag: &str) -> Result<String, String> {
it.next()
.cloned()
.ok_or_else(|| format!("{flag} requires a value"))
}
fn run_ftp_upload(url: &Url, path: &str, args: &Args) -> u8 {
let bytes = match std::fs::read(path) {
Ok(b) => b,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: -T: can't read {path:?}: {e}");
}
return 26;
}
};
let client = match transfer_client(url, args) {
Ok(c) => c,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: --proxy: {e}");
}
return 5;
}
};
let result = if args.append {
client.ftp_append(url, &bytes)
} else {
let (body, resume_at): (&[u8], Option<u64>) = match args.continue_at {
Some(off) => {
let off_usize = off as usize;
if off_usize > bytes.len() {
if show_errors(args) {
eprintln!(
"rsurl: -C {off}: offset is past the end of {path:?} ({} bytes)",
bytes.len()
);
}
return 2;
}
(&bytes[off_usize..], Some(off))
}
None => (&bytes[..], None),
};
client.ftp_store(url, body, resume_at)
};
match result {
Ok(()) => 0,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
transfer_exit_code(&e)
}
}
}
fn run_tftp_upload(url: &Url, path: &str, args: &Args) -> u8 {
let bytes = match std::fs::read(path) {
Ok(b) => b,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: -T: can't read {path:?}: {e}");
}
return 26;
}
};
match rsurl::tftp::store(url, &bytes) {
Ok(()) => 0,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
7
}
}
}
#[cfg(feature = "ssh")]
fn build_ssh_options(url: &Url, args: &Args) -> Result<(rsurl::ssh::SshOptions, String), String> {
let (_, url_pass) = rsurl::ssh::userinfo_password(url);
let (cli_user, cli_pass) = match &args.basic_auth {
Some((u, p)) => (
(!u.is_empty()).then(|| u.clone()),
(!p.is_empty()).then(|| p.clone()),
),
None => (None, None),
};
let password = url_pass.or(cli_pass);
let user = rsurl::ssh::resolve_user(url, cli_user.as_deref()).map_err(|e| e.to_string())?;
let opts = rsurl::ssh::SshOptions {
password: password.clone(),
identity_files: args.ssh_keys.iter().map(std::path::PathBuf::from).collect(),
key_passphrase: password,
insecure: args.insecure,
known_hosts_path: None,
timeout: args.max_time.map(Duration::from_secs),
};
Ok((opts, user))
}
#[cfg(feature = "ssh")]
fn run_ssh(url: &Url, args: &Args) -> u8 {
let (opts, user) = match build_ssh_options(url, args) {
Ok(x) => x,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 2;
}
};
let result = if args.verbose {
let mut err = io::stderr().lock();
rsurl::ssh::fetch_traced(url, &opts, &user, Some(&mut err))
} else {
rsurl::ssh::fetch(url, &opts, &user)
};
match result {
Ok(bytes) => {
let mut out: Box<dyn Write> = if args.remote_name {
match remote_name_from_url(url) {
Ok(name) => match File::create(&name) {
Ok(f) => Box::new(f),
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: open {name}: {e}");
}
return 23;
}
},
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 23;
}
}
} else {
match &args.output {
Some(path) if path != "-" => match create_output_file(path, args) {
Ok(f) => Box::new(f),
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: open {path}: {e}");
}
return 23;
}
},
_ => Box::new(io::stdout().lock()),
}
};
if let Err(e) = out.write_all(&bytes) {
if show_errors(args) {
eprintln!("rsurl: write error: {e}");
}
return 23;
}
0
}
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
7
}
}
}
#[cfg(feature = "ssh")]
fn run_ssh_upload(url: &Url, path: &str, args: &Args) -> u8 {
let bytes = match std::fs::read(path) {
Ok(b) => b,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: -T: can't read {path:?}: {e}");
}
return 26;
}
};
let (opts, user) = match build_ssh_options(url, args) {
Ok(x) => x,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 2;
}
};
let result = if args.verbose {
let mut err = io::stderr().lock();
rsurl::ssh::upload_traced(url, &bytes, &opts, &user, Some(&mut err))
} else {
rsurl::ssh::upload(url, &bytes, &opts, &user)
};
match result {
Ok(()) => 0,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
7
}
}
}
fn run_mqtt_publish(url: &Url, args: &Args) -> u8 {
if args.upload_file.is_some() && !args.data_parts.is_empty() {
if show_errors(args) {
eprintln!("rsurl: -d/--data and -T/--upload-file are mutually exclusive");
}
return 2;
}
let payload: Vec<u8> = if let Some(path) = &args.upload_file {
match std::fs::read(path) {
Ok(b) => b,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: -T: can't read {path:?}: {e}");
}
return 26;
}
}
} else {
match assemble_form_body(&args.data_parts) {
Ok(Some(b)) => b,
Ok(None) => Vec::new(),
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 2;
}
}
};
match rsurl::mqtt::publish(url, &payload, 0) {
Ok(()) => 0,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
7
}
}
}
fn run_rtsp(url: &Url, args: &Args) -> u8 {
let method = args.method.as_deref().unwrap_or("DESCRIBE");
match rsurl::rtsp::run_method(url, method) {
Ok(bytes) => {
let mut out: Box<dyn Write> = match &args.output {
Some(path) if path != "-" => match create_output_file(path, args) {
Ok(f) => Box::new(f),
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: open {path}: {e}");
}
return 23;
}
},
_ => Box::new(io::stdout().lock()),
};
if let Err(e) = out.write_all(&bytes) {
if show_errors(args) {
eprintln!("rsurl: write error: {e}");
}
return 23;
}
0
}
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
7
}
}
}
fn transfer_client(url: &Url, args: &Args) -> rsurl::Result<rsurl::Client> {
let mut c = rsurl::Client::new()
.verify_tls(!args.insecure)
.idn(!args.no_idn)
.ftp_use_epsv(!args.disable_epsv)
.ftp_create_dirs(args.ftp_create_dirs)
.ftp_active(args.ftp_port.is_some())
.require_tls(args.ssl_reqd);
if let Some(secs) = args.connect_timeout {
c = c.connect_timeout(Some(Duration::from_secs(secs)));
}
if let Some(spec) = resolve_proxy_spec(url, args) {
c = c.proxy(&spec)?;
}
if let Some(list) = resolve_noproxy(args) {
c = c.no_proxy(
list.split(',')
.map(str::trim)
.filter(|s| !s.is_empty())
.map(String::from)
.collect::<Vec<_>>(),
);
}
if let Some(path) = &args.unix_socket {
#[cfg(unix)]
{
c = c.connector(std::sync::Arc::new(rsurl::net::UnixConnector {
path: path.into(),
}));
}
#[cfg(not(unix))]
{
let _ = path;
return Err(rsurl::Error::UnsupportedScheme(
"--unix-socket is not supported on this platform".into(),
));
}
}
Ok(c)
}
fn ws_outgoing(args: &Args) -> std::result::Result<Vec<rsurl::WsMessage>, String> {
if !args.data_parts.is_empty() {
let bytes = assemble_form_body(&args.data_parts)?.unwrap_or_default();
return Ok(vec![bytes_to_ws_message(bytes)]);
}
if !io::stdin().is_terminal() {
let mut buf = Vec::new();
io::stdin()
.read_to_end(&mut buf)
.map_err(|e| format!("reading stdin: {e}"))?;
if buf.is_empty() {
return Ok(Vec::new());
}
return Ok(match std::str::from_utf8(&buf) {
Ok(s) => s
.lines()
.map(|l| rsurl::WsMessage::Text(l.to_string()))
.collect(),
Err(_) => vec![rsurl::WsMessage::Binary(buf)],
});
}
Ok(Vec::new())
}
fn bytes_to_ws_message(bytes: Vec<u8>) -> rsurl::WsMessage {
match String::from_utf8(bytes) {
Ok(s) => rsurl::WsMessage::Text(s),
Err(e) => rsurl::WsMessage::Binary(e.into_bytes()),
}
}
fn run_websocket(url: &Url, args: &Args) -> u8 {
let client = match transfer_client(url, args) {
Ok(c) => c,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: --proxy: {e}");
}
return 5;
}
};
let outgoing = match ws_outgoing(args) {
Ok(m) => m,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 2;
}
};
let subscribe = outgoing.is_empty();
let mut ws = match client.websocket_url(url) {
Ok(w) => w,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return transfer_exit_code(&e);
}
};
if args.verbose {
let path = if url.path.is_empty() { "/" } else { &url.path };
eprintln!(
"* WebSocket connected to {}://{}:{}{path}",
url.scheme, url.host, url.port
);
if ws.compression_enabled() {
eprintln!("* permessage-deflate negotiated");
}
}
for m in &outgoing {
if let Err(e) = ws.send(m) {
if show_errors(args) {
eprintln!("rsurl: send: {e}");
}
return transfer_exit_code(&e);
}
if args.verbose {
let kind = match m {
rsurl::WsMessage::Text(_) => "TEXT",
rsurl::WsMessage::Binary(_) => "BINARY",
};
eprintln!("* > {kind} {} bytes", m.as_bytes().len());
}
}
if !subscribe {
let _ = ws.close();
if args.verbose {
eprintln!("* > CLOSE");
}
}
match args.max_time {
Some(secs) => {
let _ = ws.set_read_timeout(Some(Duration::from_secs(secs)));
}
None if subscribe => {
let _ = ws.set_read_timeout(None);
}
None => {}
}
let mut out: Box<dyn Write> = match &args.output {
Some(path) if path != "-" => match create_output_file(path, args) {
Ok(f) => Box::new(f),
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: open {path}: {e}");
}
return 23;
}
},
_ => Box::new(io::stdout().lock()),
};
let mut code = 0u8;
loop {
match ws.recv_event() {
Ok(rsurl::WsEvent::Text(t)) => {
let _ = out.write_all(t.as_bytes());
let _ = out.write_all(b"\n");
let _ = out.flush();
}
Ok(rsurl::WsEvent::Binary(b)) => {
let _ = out.write_all(&b);
let _ = out.flush();
}
Ok(rsurl::WsEvent::Ping(_)) => {
if args.verbose {
eprintln!("* < PING (auto-pong sent)");
}
}
Ok(rsurl::WsEvent::Pong(_)) => {
if args.verbose {
eprintln!("* < PONG");
}
}
Ok(rsurl::WsEvent::Close(c)) => {
if args.verbose {
match c {
Some(cl) => eprintln!("* < CLOSE {} {}", cl.code, cl.reason),
None => eprintln!("* < CLOSE"),
}
}
break;
}
Err(rsurl::Error::Io(e))
if matches!(
e.kind(),
io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut
) =>
{
if args.verbose {
eprintln!("* idle, closing");
}
break;
}
Err(rsurl::Error::UnexpectedEof) => break,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
code = transfer_exit_code(&e);
break;
}
}
}
let _ = ws.close();
code
}
fn run_smtp(url: &Url, args: &Args) -> u8 {
let Some(from) = args.mail_from.as_deref() else {
if show_errors(args) {
eprintln!("rsurl: smtp requires --mail-from");
}
return 2;
};
if args.mail_rcpt.is_empty() {
if show_errors(args) {
eprintln!("rsurl: smtp requires at least one --mail-rcpt");
}
return 2;
}
let body: Vec<u8> = if let Some(path) = &args.upload_file {
match std::fs::read(path) {
Ok(b) => b,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: read {path}: {e}");
}
return 2;
}
}
} else if !args.data_parts.is_empty() {
match assemble_request_body(args) {
Ok(Some((b, _, _))) => b,
_ => Vec::new(),
}
} else {
if show_errors(args) {
eprintln!("rsurl: smtp needs a message body (-T <file> or -d)");
}
return 2;
};
let (user, pass) = match url.userinfo.as_deref() {
Some(ui) => match ui.split_once(':') {
Some((u, p)) => (Some(u.to_string()), Some(p.to_string())),
None => (Some(ui.to_string()), None),
},
None => (None, None),
};
let client = match transfer_client(url, args) {
Ok(c) => c,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 5;
}
};
match client.smtp_send(
url,
&body,
from,
&args.mail_rcpt,
user.as_deref(),
pass.as_deref(),
) {
Ok(()) => 0,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
7
}
}
}
fn run_telnet(url: &Url, args: &Args) -> u8 {
let input: Vec<u8> = if let Some(path) = &args.upload_file {
std::fs::read(path).unwrap_or_default()
} else if !args.data_parts.is_empty() {
assemble_request_body(args)
.ok()
.flatten()
.map(|(b, _, _)| b)
.unwrap_or_default()
} else {
Vec::new()
};
let client = match transfer_client(url, args) {
Ok(c) => c,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 5;
}
};
match client.telnet(url, &input) {
Ok(bytes) => {
let mut out: Box<dyn Write> = match &args.output {
Some(path) if path != "-" => match create_output_file(path, args) {
Ok(f) => Box::new(f),
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: open {path}: {e}");
}
return 23;
}
},
_ => Box::new(io::stdout().lock()),
};
if out.write_all(&bytes).is_err() {
return 23;
}
0
}
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
7
}
}
}
fn run_transfer(url: &Url, args: &Args) -> u8 {
let client = match transfer_client(url, args) {
Ok(c) => c,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: --proxy: {e}");
}
return 5;
}
};
match client.transfer_url(url) {
Ok(bytes) => {
let mut out: Box<dyn Write> = match &args.output {
Some(path) if path != "-" => match create_output_file(path, args) {
Ok(f) => Box::new(f),
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: open {path}: {e}");
}
return 23;
}
},
_ => Box::new(io::stdout().lock()),
};
if let Err(e) = out.write_all(&bytes) {
if show_errors(args) {
eprintln!("rsurl: write error: {e}");
}
return 23;
}
0
}
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
transfer_exit_code(&e)
}
}
}
fn run_stream_download(url: &Url, args: &Args) -> u8 {
let client = match transfer_client(url, args) {
Ok(c) => c,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: --proxy: {e}");
}
return 5;
}
};
let name = if args.remote_name {
match remote_name_from_url(url) {
Ok(n) => n,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 23;
}
}
} else {
args.output.clone().unwrap_or_default()
};
let (file, out_path) = match create_output_file_tracked(&name, args) {
Ok(pair) => pair,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: open {name}: {e}");
}
return 23;
}
};
let (speed_limit, speed_time) = {
let lim = args
.speed_limit
.as_deref()
.and_then(|s| s.parse::<u64>().ok());
let tim = args
.speed_time
.as_deref()
.and_then(|s| s.parse::<u64>().ok());
match (lim, tim) {
(None, None) => (None, 30),
(l, t) => (Some(l.unwrap_or(1)), t.unwrap_or(30)),
}
};
let now = std::time::Instant::now();
let mut sink = DownloadSink {
inner: Box::new(file),
written: 0,
max: args.max_filesize,
rate: args.limit_rate.as_deref().and_then(parse_rate),
speed_limit,
speed_time,
started: now,
progress: args.progress_bar,
silent: args.silent,
last_tick: now,
resume: None,
};
let result = client.transfer_url_to(url, &mut sink);
let time_total = now.elapsed();
let written = sink.written;
if args.progress_bar && !args.silent {
eprintln!();
}
match result {
Ok(_) => {
if args.write_out.is_some() {
let resp = rsurl::Response {
status: 0,
reason: String::new(),
version: String::new(),
headers: Vec::new(),
body: Vec::new(),
timing: rsurl::Timing::default(),
final_url: String::new(),
tls: None,
};
run_write_out(&resp, url, args, time_total, written);
}
0
}
Err(e) => {
if args.remove_on_error {
drop(sink);
let _ = std::fs::remove_file(&out_path);
}
if e.to_string().contains(MAX_FILESIZE_SENTINEL) {
if show_errors(args) {
eprintln!("rsurl: Maximum file size exceeded");
}
return 63;
}
if e.to_string().contains(LOW_SPEED_SENTINEL) {
if show_errors(args) {
eprintln!(
"rsurl: Operation too slow. Less than {} bytes/sec transferred \
the last {speed_time} seconds",
speed_limit.unwrap_or(1)
);
}
return 28;
}
if show_errors(args) {
eprintln!("rsurl: {e}");
}
transfer_exit_code(&e)
}
}
}
fn sanitize_for_tty(bytes: &[u8]) -> Vec<u8> {
let mut out = Vec::with_capacity(bytes.len());
let mut rest = bytes;
while !rest.is_empty() {
match std::str::from_utf8(rest) {
Ok(s) => {
push_sanitized_str(s, &mut out);
break;
}
Err(e) => {
let valid_up_to = e.valid_up_to();
if valid_up_to > 0 {
let s = unsafe { std::str::from_utf8_unchecked(&rest[..valid_up_to]) };
push_sanitized_str(s, &mut out);
}
let bad = e.error_len().unwrap_or(1);
for &b in &rest[valid_up_to..valid_up_to + bad] {
out.extend_from_slice(format!("\\x{b:02x}").as_bytes());
}
rest = &rest[valid_up_to + bad..];
}
}
}
out
}
fn push_sanitized_str(s: &str, out: &mut Vec<u8>) {
for ch in s.chars() {
let cp = ch as u32;
let dangerous = (cp < 0x20 && ch != '\t') || (0x7f..=0x9f).contains(&cp);
if dangerous {
out.extend_from_slice(format!("\\x{cp:02x}").as_bytes());
} else {
let mut buf = [0u8; 4];
out.extend_from_slice(ch.encode_utf8(&mut buf).as_bytes());
}
}
}
fn body_looks_binary(body: &[u8]) -> bool {
body.contains(&0)
}
fn fmt_secs(d: Option<std::time::Duration>) -> String {
format!("{:.6}", d.map_or(0.0, |d| d.as_secs_f64()))
}
fn run_write_out(
resp: &Response,
url: &Url,
args: &Args,
time_total: std::time::Duration,
size_download: u64,
) {
let Some(fmt) = &args.write_out else { return };
let size_header: usize = resp
.headers
.iter()
.map(|(k, v)| k.len() + v.len() + 4)
.sum::<usize>()
+ resp.version.len()
+ resp.reason.len()
+ 6;
let var = |name: &str| -> String {
match name {
"http_code" | "response_code" => resp.status.to_string(),
"http_version" => resp.version.clone(),
"size_download" => size_download.to_string(),
"size_header" => size_header.to_string(),
"num_headers" => resp.headers.len().to_string(),
"content_type" => resp.header("content-type").unwrap_or("").to_string(),
"ssl_verify_result" => "0".to_string(),
"url_effective" => {
let default = matches!(
(url.scheme.as_str(), url.port),
("http", 80) | ("https", 443)
);
if default {
format!("{}://{}{}", url.scheme, url.host, url.path)
} else {
format!("{}://{}:{}{}", url.scheme, url.host, url.port, url.path)
}
}
"scheme" => url.scheme.to_uppercase(),
"time_total" => format!("{:.6}", time_total.as_secs_f64()),
"time_namelookup" => fmt_secs(None), "time_connect" => fmt_secs(resp.timing.connect),
"time_appconnect" => fmt_secs(resp.timing.appconnect),
"time_pretransfer" => fmt_secs(resp.timing.pretransfer),
"time_starttransfer" => fmt_secs(resp.timing.starttransfer),
_ => String::new(),
}
};
let mut out = String::with_capacity(fmt.len());
let mut chars = fmt.chars().peekable();
while let Some(c) = chars.next() {
match c {
'%' => match chars.peek().copied() {
Some('{') => {
chars.next();
let mut name = String::new();
for nc in chars.by_ref() {
if nc == '}' {
break;
}
name.push(nc);
}
out.push_str(&var(&name));
}
Some('%') => {
chars.next();
out.push('%');
}
_ if chars.clone().take(7).collect::<String>() == "header{" => {
for _ in 0..7 {
chars.next();
}
let mut name = String::new();
for nc in chars.by_ref() {
if nc == '}' {
break;
}
name.push(nc);
}
out.push_str(resp.header(name.trim()).unwrap_or(""));
}
_ => out.push('%'),
},
'\\' => match chars.next() {
Some('n') => out.push('\n'),
Some('t') => out.push('\t'),
Some('r') => out.push('\r'),
Some('\\') => out.push('\\'),
Some(other) => {
out.push('\\');
out.push(other);
}
None => out.push('\\'),
},
other => out.push(other),
}
}
print!("{out}");
let _ = io::stdout().flush();
}
fn httpdate_to_epoch(s: &str) -> Option<u64> {
let rest = s
.trim()
.split_once(", ")
.map(|(_, r)| r)
.unwrap_or(s.trim());
let mut it = rest.split_whitespace();
let day: i64 = it.next()?.parse().ok()?;
let mon: i64 = match it.next()? {
"Jan" => 1,
"Feb" => 2,
"Mar" => 3,
"Apr" => 4,
"May" => 5,
"Jun" => 6,
"Jul" => 7,
"Aug" => 8,
"Sep" => 9,
"Oct" => 10,
"Nov" => 11,
"Dec" => 12,
_ => return None,
};
let year: i64 = it.next()?.parse().ok()?;
let mut hms = it.next()?.split(':');
let hh: i64 = hms.next()?.parse().ok()?;
let mm: i64 = hms.next()?.parse().ok()?;
let ss: i64 = hms.next()?.parse().ok()?;
let y = if mon <= 2 { year - 1 } else { year };
let era = (if y >= 0 { y } else { y - 399 }) / 400;
let yoe = y - era * 400;
let doy = (153 * (if mon > 2 { mon - 3 } else { mon + 9 }) + 2) / 5 + day - 1;
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
let days = era * 146097 + doe - 719468;
let secs = days * 86400 + hh * 3600 + mm * 60 + ss;
u64::try_from(secs).ok()
}
fn epoch_to_httpdate(secs: u64) -> String {
let days = (secs / 86400) as i64;
let rem = (secs % 86400) as i64;
let (hh, mm, ss) = (rem / 3600, (rem % 3600) / 60, rem % 60);
let wd = (days % 7 + 4).rem_euclid(7); let z = days + 719468;
let era = (if z >= 0 { z } else { z - 146096 }) / 146097;
let doe = z - era * 146097;
let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365;
let y = yoe + era * 400;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let d = doy - (153 * mp + 2) / 5 + 1;
let m = if mp < 10 { mp + 3 } else { mp - 9 };
let year = if m <= 2 { y + 1 } else { y };
const WD: [&str; 7] = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
const MON: [&str; 12] = [
"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
];
format!(
"{}, {:02} {} {:04} {:02}:{:02}:{:02} GMT",
WD[wd as usize],
d,
MON[(m - 1) as usize],
year,
hh,
mm,
ss
)
}
fn time_cond_header(spec: &str) -> Option<(&'static str, String)> {
let (unmod, body) = match spec.strip_prefix('-') {
Some(r) => (true, r),
None => (false, spec.strip_prefix('+').unwrap_or(spec)),
};
let date = match std::fs::metadata(body).and_then(|m| m.modified()) {
Ok(mtime) => {
let secs = mtime.duration_since(std::time::UNIX_EPOCH).ok()?.as_secs();
epoch_to_httpdate(secs)
}
Err(_) => body.trim().to_string(),
};
if date.is_empty() {
return None;
}
Some((
if unmod {
"If-Unmodified-Since"
} else {
"If-Modified-Since"
},
date,
))
}
fn proto_allowed(scheme: &str, spec: &str) -> bool {
const ALL: &[&str] = &[
"http", "https", "ftp", "ftps", "sftp", "scp", "imap", "imaps", "pop3", "pop3s", "smtp",
"smtps", "mqtt", "mqtts", "rtsp", "tftp", "ldap", "ldaps", "gopher", "gophers", "dict",
"file", "ws", "wss", "telnet",
];
let mut set: std::collections::HashSet<String> = ALL.iter().map(|s| s.to_string()).collect();
for tok in spec.split(',') {
let tok = tok.trim();
if tok.is_empty() {
continue;
}
let (op, name) = match tok.as_bytes()[0] {
b'=' => ('=', &tok[1..]),
b'+' => ('+', &tok[1..]),
b'-' => ('-', &tok[1..]),
_ => ('+', tok),
};
let names: Vec<String> = if name == "all" {
ALL.iter().map(|s| s.to_string()).collect()
} else {
vec![name.to_ascii_lowercase()]
};
match op {
'=' => {
set.clear();
set.extend(names);
}
'+' => set.extend(names),
'-' => {
for n in names {
set.remove(&n);
}
}
_ => {}
}
}
set.contains(&scheme.to_ascii_lowercase())
}
fn parse_rate(s: &str) -> Option<u64> {
let s = s.trim();
let (num, mult): (&str, u64) = match s.chars().last() {
Some('k') | Some('K') => (&s[..s.len() - 1], 1024),
Some('m') | Some('M') => (&s[..s.len() - 1], 1024 * 1024),
Some('g') | Some('G') => (&s[..s.len() - 1], 1024 * 1024 * 1024),
_ => (s, 1),
};
num.trim()
.parse::<u64>()
.ok()
.map(|n| n.saturating_mul(mult))
}
const MAX_FILESIZE_SENTINEL: &str = "rsurl-max-filesize-exceeded";
const LOW_SPEED_SENTINEL: &str = "rsurl-low-speed-abort";
struct DownloadSink<'a> {
inner: Box<dyn Write + 'a>,
written: u64,
max: Option<u64>,
rate: Option<u64>,
speed_limit: Option<u64>,
speed_time: u64,
started: std::time::Instant,
progress: bool,
silent: bool,
last_tick: std::time::Instant,
resume: Option<ResumeCtx>,
}
struct ResumeCtx {
part: std::path::PathBuf,
total: u64,
validators: Vec<u8>,
last_save: std::time::Instant,
}
fn http_resume_meta(total: u64, done: u64, validators: &[u8]) -> Vec<u8> {
let mut m = Vec::with_capacity(16 + validators.len());
m.extend_from_slice(&total.to_le_bytes());
m.extend_from_slice(&done.to_le_bytes());
m.extend_from_slice(validators);
m
}
fn http_resume_validators(url: &str, etag: &str, last_modified: &str) -> Vec<u8> {
let mut v = Vec::new();
for s in [url, etag, last_modified] {
let b = s.as_bytes();
v.extend_from_slice(&(b.len() as u16).to_le_bytes());
v.extend_from_slice(b);
}
v
}
fn http_resume_parse(meta: &[u8], total: u64, url: &str, etag: &str) -> Option<u64> {
if meta.len() < 16 {
return None;
}
let stored_total = u64::from_le_bytes(meta[0..8].try_into().unwrap());
let done = u64::from_le_bytes(meta[8..16].try_into().unwrap());
if stored_total != total {
return None;
}
let mut p = &meta[16..];
let mut take = || -> Option<String> {
if p.len() < 2 {
return None;
}
let n = u16::from_le_bytes([p[0], p[1]]) as usize;
if p.len() < 2 + n {
return None;
}
let s = String::from_utf8_lossy(&p[2..2 + n]).into_owned();
p = &p[2 + n..];
Some(s)
};
let stored_url = take()?;
let stored_etag = take()?;
let _stored_lm = take()?;
if stored_url != url {
return None;
}
if !etag.is_empty() && !stored_etag.is_empty() && etag != stored_etag {
return None;
}
Some(done)
}
impl Write for DownloadSink<'_> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
if let Some(max) = self.max {
if self.written + buf.len() as u64 > max {
return Err(io::Error::other(MAX_FILESIZE_SENTINEL));
}
}
if let Some(rate) = self.rate.filter(|r| *r > 0) {
let target = std::time::Duration::from_secs_f64(
(self.written + buf.len() as u64) as f64 / rate as f64,
);
let elapsed = self.started.elapsed();
if target > elapsed {
std::thread::sleep(target - elapsed);
}
}
self.inner.write_all(buf)?;
self.written += buf.len() as u64;
if let Some(limit) = self.speed_limit {
let secs = self.started.elapsed().as_secs();
if secs >= self.speed_time && self.written / secs.max(1) < limit {
return Err(io::Error::other(LOW_SPEED_SENTINEL));
}
}
if self.progress
&& !self.silent
&& self.last_tick.elapsed() >= std::time::Duration::from_millis(100)
{
eprint!("\rrsurl: {} bytes received", self.written);
let _ = io::stderr().flush();
self.last_tick = std::time::Instant::now();
}
let written = self.written;
if let Some(rc) = &mut self.resume {
if rc.last_save.elapsed() >= std::time::Duration::from_secs(1) {
let meta = http_resume_meta(rc.total, written, &rc.validators);
let _ = rsurl::resume::write_state(
&rc.part,
rc.total,
rsurl::resume::Kind::HttpStream,
&meta,
);
rc.last_save = std::time::Instant::now();
}
}
Ok(buf.len())
}
fn flush(&mut self) -> io::Result<()> {
self.inner.flush()
}
}
fn parallel_segments_eligible(args: &Args) -> bool {
args.parallel_segments.is_some_and(|n| n > 1)
&& args.continue_at.is_none()
&& args.range.is_none()
&& args.limit_rate.is_none()
&& args.speed_limit.is_none()
&& args.speed_time.is_none()
&& args.upload_file.is_none()
}
const MIN_SEGMENT_BYTES: u64 = 1 << 20; const MAX_SEGMENTS: usize = 16;
struct CountingWriter<W> {
inner: W,
counter: Arc<AtomicU64>,
}
impl<W: Write> Write for CountingWriter<W> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
let n = self.inner.write(buf)?;
self.counter.fetch_add(n as u64, Ordering::Relaxed);
Ok(n)
}
fn flush(&mut self) -> io::Result<()> {
self.inner.flush()
}
}
fn human_bytes(b: u64) -> String {
const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"];
let mut v = b as f64;
let mut i = 0;
while v >= 1024.0 && i < UNITS.len() - 1 {
v /= 1024.0;
i += 1;
}
if i == 0 {
format!("{b} B")
} else {
format!("{v:.1} {}", UNITS[i])
}
}
fn progress_bar(got: u64, total: u64, width: usize) -> String {
let filled = if total == 0 {
width
} else {
((got as u128 * width as u128 / total as u128) as usize).min(width)
};
let mut s = String::with_capacity(width + 2);
s.push('[');
for _ in 0..filled {
s.push('\u{2588}'); }
for _ in filled..width {
s.push('\u{2591}'); }
s.push(']');
s
}
fn render_parallel(
lens: &[u64],
counters: &[Arc<AtomicU64>],
total: u64,
start: std::time::Instant,
tty: bool,
first: &mut bool,
) {
const BAR_W: usize = 24;
let mut err = io::stderr().lock();
let done: u64 = counters.iter().map(|c| c.load(Ordering::Relaxed)).sum();
let secs = start.elapsed().as_secs_f64();
let rate = if secs > 0.0 {
(done as f64 / secs) as u64
} else {
0
};
if tty {
let lines = lens.len() + 1; if *first {
*first = false;
} else {
let _ = write!(err, "\x1b[{lines}A");
}
for (i, (len, ctr)) in lens.iter().zip(counters).enumerate() {
let got = ctr.load(Ordering::Relaxed);
let _ = writeln!(
err,
"\r\x1b[2K seg {:>2}/{:<2} {} {}/{}",
i + 1,
lens.len(),
progress_bar(got, *len, BAR_W),
human_bytes(got),
human_bytes(*len),
);
}
let _ = writeln!(
err,
"\r\x1b[2K total {} {}/{} {}/s",
progress_bar(done, total, BAR_W),
human_bytes(done),
human_bytes(total),
human_bytes(rate),
);
} else {
let _ = write!(
err,
"\rrsurl: {}/{} ({}/s) ",
human_bytes(done),
human_bytes(total),
human_bytes(rate),
);
}
let _ = err.flush();
}
const RESUME_CHUNK: u64 = 4 * 1024 * 1024;
fn bit_get(map: &[u8], i: usize) -> bool {
map[i / 8] & (1 << (i % 8)) != 0
}
fn bit_set(map: &mut [u8], i: usize) {
map[i / 8] |= 1 << (i % 8);
}
fn http_ranged_meta(chunk: u32, total: u64, validators: &[u8], bitmap: &[u8]) -> Vec<u8> {
let mut m = Vec::with_capacity(12 + validators.len() + bitmap.len());
m.extend_from_slice(&chunk.to_le_bytes());
m.extend_from_slice(&total.to_le_bytes());
m.extend_from_slice(&(validators.len() as u32).to_le_bytes());
m.extend_from_slice(validators);
m.extend_from_slice(bitmap);
m
}
fn http_ranged_parse(
meta: &[u8],
chunk: u32,
total: u64,
url: &str,
etag: &str,
) -> Option<Vec<u8>> {
if meta.len() < 16 {
return None;
}
if u32::from_le_bytes(meta[0..4].try_into().unwrap()) != chunk {
return None;
}
if u64::from_le_bytes(meta[4..12].try_into().unwrap()) != total {
return None;
}
let vlen = u32::from_le_bytes(meta[12..16].try_into().unwrap()) as usize;
let rest = meta.get(16..)?;
let validators = rest.get(..vlen)?;
let bitmap = rest.get(vlen..)?.to_vec();
if !validators_match(validators, url, etag) {
return None;
}
Some(bitmap)
}
fn validators_match(stored: &[u8], url: &str, etag: &str) -> bool {
let mut p = stored;
let mut take = || -> Option<String> {
if p.len() < 2 {
return None;
}
let n = u16::from_le_bytes([p[0], p[1]]) as usize;
let s = String::from_utf8_lossy(p.get(2..2 + n)?).into_owned();
p = &p[2 + n..];
Some(s)
};
let (Some(su), Some(se), Some(_lm)) = (take(), take(), take()) else {
return false;
};
su == url && (etag.is_empty() || se.is_empty() || se == etag)
}
fn run_parallel_resume(req: &rsurl::Request, url: &Url, args: &Args) -> Option<u8> {
use std::sync::atomic::{AtomicBool, AtomicUsize};
use std::sync::Mutex;
let name = if args.remote_name {
remote_name_from_url(url).ok()?
} else {
args.output.clone().unwrap_or_default()
};
if name.is_empty() || name == "-" {
return None;
}
let probe = req.clone().method("HEAD").send().ok()?;
if probe.status >= 400 {
return None;
}
let total = probe
.header("content-length")
.and_then(|v| v.trim().parse::<u64>().ok())?;
if total == 0
|| !probe
.header("accept-ranges")
.is_some_and(|v| v.to_ascii_lowercase().contains("bytes"))
{
return None;
}
if let Some(max) = args.max_filesize {
if total > max {
if show_errors(args) {
eprintln!("rsurl: Maximum file size exceeded");
}
return Some(63);
}
}
let etag = probe.header("etag").unwrap_or("").to_string();
let last_mod = probe.header("last-modified").unwrap_or("").to_string();
let url_s = format!("{}://{}:{}{}", url.scheme, url.host, url.port, url.path);
let final_path = resolve_output_path(&name, args);
if args.create_dirs {
if let Some(p) = final_path.parent() {
let _ = std::fs::create_dir_all(p);
}
} else if let Some(dir) = &args.output_dir {
let _ = std::fs::create_dir_all(dir);
}
let part = rsurl::resume::part_path(&final_path);
let num_chunks = total.div_ceil(RESUME_CHUNK) as usize;
let map_len = num_chunks.div_ceil(8);
let validators = http_resume_validators(&url_s, &etag, &last_mod);
let mut bitmap = vec![0u8; map_len];
if let Ok(Some(st)) = rsurl::resume::read_state(&part) {
if st.kind == rsurl::resume::Kind::HttpRanged {
if let Some(bm) = http_ranged_parse(&st.meta, RESUME_CHUNK as u32, total, &url_s, &etag)
{
if bm.len() == map_len {
bitmap = bm;
}
}
}
}
match std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&part)
{
Ok(f) => {
if f.set_len(total).is_err() {
return None; }
}
Err(_) => return None,
}
let _ = rsurl::resume::write_state(
&part,
total,
rsurl::resume::Kind::HttpRanged,
&http_ranged_meta(RESUME_CHUNK as u32, total, &validators, &bitmap),
);
let n_workers = args
.parallel_segments
.unwrap_or(1)
.min(MAX_SEGMENTS)
.min(num_chunks.max(1));
if args.verbose {
let done = (0..num_chunks).filter(|&i| bit_get(&bitmap, i)).count();
eprintln!(
"* resumable parallel: {num_chunks} chunks ({done} already done), {n_workers} workers"
);
}
let bitmap = Arc::new(Mutex::new(bitmap));
let next = Arc::new(AtomicUsize::new(0));
let failed = Arc::new(AtomicBool::new(false));
let mut handles = Vec::with_capacity(n_workers);
for _ in 0..n_workers {
let req = req.clone();
let part = part.clone();
let bitmap = Arc::clone(&bitmap);
let next = Arc::clone(&next);
let failed = Arc::clone(&failed);
let validators = validators.clone();
handles.push(std::thread::spawn(move || {
while !failed.load(Ordering::Relaxed) {
let i = next.fetch_add(1, Ordering::Relaxed);
if i >= num_chunks {
break;
}
if bit_get(&bitmap.lock().unwrap(), i) {
continue; }
let start = i as u64 * RESUME_CHUNK;
let end = (start + RESUME_CHUNK).min(total) - 1;
let r = req.clone().header("Range", &format!("bytes={start}-{end}"));
let mut f = match std::fs::OpenOptions::new().write(true).open(&part) {
Ok(f) => f,
Err(_) => {
failed.store(true, Ordering::Relaxed);
break;
}
};
if f.seek(io::SeekFrom::Start(start)).is_err() {
failed.store(true, Ordering::Relaxed);
break;
}
match r.send_download(&mut f, None, &mut io::sink()) {
Ok(resp) if resp.status == 206 => {}
_ => {
failed.store(true, Ordering::Relaxed);
break;
}
}
let mut bm = bitmap.lock().unwrap();
bit_set(&mut bm, i);
let _ = rsurl::resume::write_state(
&part,
total,
rsurl::resume::Kind::HttpRanged,
&http_ranged_meta(RESUME_CHUNK as u32, total, &validators, &bm),
);
}
}));
}
for h in handles {
let _ = h.join();
}
if failed.load(Ordering::Relaxed) {
if show_errors(args) {
eprintln!("rsurl: parallel resume incomplete; re-run to continue");
}
return Some(18); }
drop(bitmap);
if let Err(e) = rsurl::resume::finalize(&part, &final_path, total) {
if show_errors(args) {
eprintln!("rsurl: finalize {}: {e}", final_path.display());
}
return Some(23);
}
if args.remote_time {
set_remote_time(&probe, &name);
}
Some(0)
}
fn run_parallel_download(
req: rsurl::Request,
url: &Url,
args: &Args,
jar: Option<&mut CookieJar>,
) -> u8 {
if jar.is_some() {
return run_http_download(req, url, args, jar);
}
let probe = match req.clone().method("HEAD").send() {
Ok(r) => r,
Err(_) => return run_http_download(req, url, args, None),
};
let size = probe
.header("content-length")
.and_then(|v| v.trim().parse::<u64>().ok());
let ranges_ok = probe
.header("accept-ranges")
.is_some_and(|v| v.to_ascii_lowercase().contains("bytes"));
let (Some(size), true) = (size, ranges_ok && probe.status < 400) else {
return run_http_download(req, url, args, None);
};
if let Some(max) = args.max_filesize {
if size > max {
if show_errors(args) {
eprintln!("rsurl: Maximum file size exceeded");
}
return 63;
}
}
let requested = args.parallel_segments.unwrap_or(1).min(MAX_SEGMENTS);
let by_size = size.div_ceil(MIN_SEGMENT_BYTES).max(1) as usize;
let n = requested.min(by_size);
if n < 2 {
return run_http_download(req, url, args, None);
}
let name = if args.remote_name {
match remote_name_from_url(url) {
Ok(nm) => nm,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 23;
}
}
} else {
args.output.clone().unwrap_or_default()
};
let (file, out_path) = match create_output_file_tracked(&name, args) {
Ok(pair) => pair,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: open {name}: {e}");
}
return 23;
}
};
if file.set_len(size).is_err() {
drop(file);
return run_http_download(req, url, args, None);
}
drop(file);
if args.verbose {
eprintln!("* Parallel download: {n} segments, {size} bytes");
}
let seg = size / n as u64;
let now = std::time::Instant::now();
let counters: Vec<Arc<AtomicU64>> = (0..n).map(|_| Arc::new(AtomicU64::new(0))).collect();
let mut lens = Vec::with_capacity(n);
let mut handles = Vec::with_capacity(n);
for (i, counter_slot) in counters.iter().enumerate() {
let start = i as u64 * seg;
let end = if i == n - 1 {
size - 1
} else {
(i as u64 + 1) * seg - 1
};
lens.push(end - start + 1);
let seg_req = req.clone().header("Range", &format!("bytes={start}-{end}"));
let path = out_path.clone();
let counter = Arc::clone(counter_slot);
handles.push(std::thread::spawn(move || -> Result<u64, String> {
let mut f = std::fs::OpenOptions::new()
.write(true)
.open(&path)
.map_err(|e| e.to_string())?;
f.seek(io::SeekFrom::Start(start))
.map_err(|e| e.to_string())?;
let mut w = CountingWriter { inner: f, counter };
let mut trace = io::sink();
let resp = seg_req
.send_download(&mut w, None, &mut trace)
.map_err(|e| e.to_string())?;
if resp.status != 206 {
return Err(format!(
"server did not honor Range (status {})",
resp.status
));
}
Ok(end - start + 1)
}));
}
let show_progress = args.progress_bar && !args.silent;
let render = if show_progress {
let lens = lens.clone();
let counters: Vec<Arc<AtomicU64>> = counters.iter().map(Arc::clone).collect();
let done = Arc::new(AtomicBool::new(false));
let done_w = Arc::clone(&done);
let tty = io::stderr().is_terminal();
let handle = std::thread::spawn(move || {
let mut first = true;
while !done_w.load(Ordering::Relaxed) {
render_parallel(&lens, &counters, size, now, tty, &mut first);
std::thread::sleep(std::time::Duration::from_millis(120));
}
render_parallel(&lens, &counters, size, now, tty, &mut first);
eprintln!();
});
Some((done, handle))
} else {
None
};
let mut total = 0u64;
let mut failure: Option<String> = None;
for h in handles {
match h.join() {
Ok(Ok(written)) => total += written,
Ok(Err(e)) => {
failure.get_or_insert(e);
}
Err(_) => {
failure.get_or_insert_with(|| "segment thread panicked".to_string());
}
}
}
if let Some((done, handle)) = render {
done.store(true, Ordering::Relaxed);
let _ = handle.join();
}
if failure.is_none() && total == size {
let elapsed = now.elapsed();
if args.remote_time {
set_remote_time(&probe, &name);
}
run_write_out(&probe, url, args, elapsed, size);
return 0;
}
if args.verbose {
eprintln!(
"* Parallel download incomplete ({}); retrying as a single stream",
failure.as_deref().unwrap_or("size mismatch")
);
}
run_http_download(req, url, args, None)
}
#[cfg(feature = "bittorrent")]
fn bytes_hex(b: &[u8]) -> String {
let mut s = String::with_capacity(b.len() * 2);
for x in b {
s.push_str(&format!("{x:02x}"));
}
s
}
#[cfg(feature = "bittorrent")]
fn json_escape(s: &str) -> String {
let mut o = String::with_capacity(s.len());
for c in s.chars() {
match c {
'"' => o.push_str("\\\""),
'\\' => o.push_str("\\\\"),
'\n' => o.push_str("\\n"),
'\r' => o.push_str("\\r"),
'\t' => o.push_str("\\t"),
c if (c as u32) < 0x20 => o.push_str(&format!("\\u{:04x}", c as u32)),
c => o.push(c),
}
}
o
}
#[cfg(feature = "bittorrent")]
fn metadata_json(meta: &rsurl::bittorrent::Metainfo) -> String {
let mut o = String::new();
o.push_str("{\n");
o.push_str(&format!(" \"name\": \"{}\",\n", json_escape(&meta.name)));
o.push_str(&format!(
" \"info_hash\": \"{}\",\n",
bytes_hex(&meta.info_hash)
));
o.push_str(&format!(" \"piece_length\": {},\n", meta.piece_length));
o.push_str(&format!(" \"num_pieces\": {},\n", meta.num_pieces()));
o.push_str(&format!(" \"total_length\": {},\n", meta.total_length));
o.push_str(&format!(" \"private\": {},\n", meta.private));
o.push_str(" \"trackers\": [");
for (i, t) in meta.trackers.iter().enumerate() {
if i > 0 {
o.push_str(", ");
}
o.push_str(&format!("\"{}\"", json_escape(t)));
}
o.push_str("],\n");
o.push_str(" \"files\": [\n");
let mut off = 0u64;
for (i, f) in meta.files.iter().enumerate() {
if i > 0 {
o.push_str(",\n");
}
o.push_str(&format!(
" {{\"path\": \"{}\", \"length\": {}, \"offset\": {}}}",
json_escape(&f.path.to_string_lossy().replace('\\', "/")),
f.length,
off
));
off += f.length;
}
o.push_str("\n ]\n}\n");
o
}
#[cfg(feature = "bittorrent")]
fn build_torrent(info: &[u8], trackers: &[String]) -> Vec<u8> {
let mut out = Vec::with_capacity(info.len() + 64);
out.push(b'd');
if let Some(first) = trackers.first() {
out.extend_from_slice(b"8:announce");
out.extend_from_slice(format!("{}:", first.len()).as_bytes());
out.extend_from_slice(first.as_bytes());
}
if !trackers.is_empty() {
out.extend_from_slice(b"13:announce-list");
out.push(b'l'); for t in trackers {
out.push(b'l'); out.extend_from_slice(format!("{}:", t.len()).as_bytes());
out.extend_from_slice(t.as_bytes());
out.push(b'e');
}
out.push(b'e');
}
out.extend_from_slice(b"4:info");
out.extend_from_slice(info);
out.push(b'e');
out
}
#[cfg(feature = "bittorrent")]
fn bt_write_torrent(bytes: &[u8], args: &Args) -> u8 {
if let Some(o) = args.output.as_deref().filter(|p| *p != "-") {
let path = resolve_output_path(o, args);
match std::fs::write(&path, bytes) {
Ok(_) => 0,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: write {}: {e}", path.display());
}
23
}
}
} else {
match io::stdout().write_all(bytes) {
Ok(_) => 0,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
23
}
}
}
}
#[cfg(feature = "bittorrent")]
fn bt_resolve_file(
meta: &rsurl::bittorrent::Metainfo,
sel: &str,
) -> std::result::Result<(usize, u64, u64), String> {
let idx = match sel.parse::<usize>() {
Ok(n) if n >= 1 && n <= meta.files.len() => Some(n - 1),
_ => {
let sel = sel.replace('\\', "/"); meta.files.iter().position(|f| {
let p = f.path.to_string_lossy().replace('\\', "/");
p == sel
|| f.path.file_name().and_then(|n| n.to_str()) == Some(sel.as_str())
|| p.ends_with(&sel)
})
}
};
match idx {
Some(i) => {
let offset: u64 = meta.files[..i].iter().map(|f| f.length).sum();
Ok((i, offset, meta.files[i].length))
}
None => Err(format!(
"--bt-file: no file matching {sel:?} (torrent has {} file(s); use --bt-info to list them)",
meta.files.len()
)),
}
}
#[cfg(feature = "bittorrent")]
fn bt_layout(
meta: &rsurl::bittorrent::Metainfo,
args: &Args,
) -> std::result::Result<Vec<(std::path::PathBuf, u64)>, String> {
use std::path::Path;
if args.bt_concat {
return match args.output.as_deref().filter(|p| *p != "-") {
Some(o) => Ok(vec![(resolve_output_path(o, args), meta.total_length)]),
None => Err("--bt-concat needs -o <file>".into()),
};
}
if meta.files.len() == 1 {
if let Some(o) = args.output.as_deref().filter(|p| *p != "-") {
let p = resolve_output_path(o, args);
if p.is_dir() {
Ok(rsurl::bittorrent::file_layout(meta, &p))
} else {
Ok(vec![(p, meta.files[0].length)])
}
} else if let Some(dir) = &args.output_dir {
Ok(rsurl::bittorrent::file_layout(meta, Path::new(dir)))
} else {
Err("torrent download needs -o <file> or --output-dir <dir>".into())
}
} else if let Some(dir) = &args.output_dir {
Ok(rsurl::bittorrent::file_layout(meta, Path::new(dir)))
} else {
Err("multi-file torrent needs --output-dir <dir>".into())
}
}
#[cfg(feature = "bittorrent")]
fn bt_announce_peers(
trackers: &[String],
info_hash: [u8; 20],
peer_id: [u8; 20],
port: u16,
left: u64,
verbose: bool,
) -> Vec<std::net::SocketAddr> {
use rsurl::bittorrent::tracker::{self, AnnounceParams, Event};
use std::time::Duration;
let params = AnnounceParams {
info_hash,
peer_id,
port,
uploaded: 0,
downloaded: 0,
left,
event: Event::Started,
num_want: 100,
key: 0,
};
let mut peers = Vec::new();
for t in trackers {
match tracker::announce(t, ¶ms, Duration::from_secs(10)) {
Ok(resp) => {
if verbose {
eprintln!("* tracker {t}: {} peers", resp.peers.len());
}
peers.extend(resp.peers);
}
Err(e) => {
if verbose {
eprintln!("* tracker {t}: {e}");
}
}
}
if !peers.is_empty() {
break; }
}
peers
}
#[cfg(feature = "bittorrent")]
fn bt_dht_peers(info_hash: [u8; 20], verbose: bool) -> Vec<std::net::SocketAddr> {
use rsurl::bittorrent::dht;
use std::time::Duration;
let bootstrap = dht::default_bootstrap();
if bootstrap.is_empty() {
if verbose {
eprintln!("* dht: no bootstrap nodes resolved");
}
return Vec::new();
}
let node_id = dht::random_node_id();
match dht::find_peers(info_hash, &bootstrap, node_id, Duration::from_secs(20)) {
Ok(peers) => {
if verbose {
eprintln!("* dht: found {} peers", peers.len());
}
peers
}
Err(e) => {
if verbose {
eprintln!("* dht: {e}");
}
Vec::new()
}
}
}
#[cfg(feature = "bittorrent")]
fn run_bittorrent(source: &str, args: &Args) -> u8 {
use rsurl::bittorrent::{self, metadata, Magnet, Metainfo, SeedMode, TorrentOptions};
use std::net::SocketAddr;
use std::time::{Duration, Instant};
let peer_id = match bittorrent::generate_peer_id() {
Ok(p) => p,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 1;
}
};
let listen_port = args.listen_port.unwrap_or(6881);
let seed = match (args.share_ratio, args.seed) {
(Some(r), _) => SeedMode::UntilRatio(r),
(None, true) => SeedMode::Forever,
(None, false) => SeedMode::Off,
};
let opts = TorrentOptions {
peer_id,
listen_port,
seed,
verbosity: args.verbosity,
recheck: args.recheck,
..Default::default()
};
let mut peers: Vec<SocketAddr> = Vec::new();
for p in &args.bt_peers {
match p.parse() {
Ok(a) => peers.push(a),
Err(_) => {
if show_errors(args) {
eprintln!("rsurl: bad --bt-peer {p}");
}
}
}
}
let (meta, trackers, torrent_bytes) = if source.starts_with("magnet:") {
let magnet = match Magnet::parse(source) {
Ok(m) => m,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 2;
}
};
peers.extend(magnet.peers.iter().copied());
if peers.is_empty() {
peers = bt_announce_peers(
&magnet.trackers,
magnet.info_hash,
peer_id,
listen_port,
0,
args.verbose,
);
}
if peers.is_empty() && !args.no_dht {
if !args.silent {
eprintln!("* no tracker peers; trying DHT");
}
peers = bt_dht_peers(magnet.info_hash, args.verbose);
}
peers.sort();
peers.dedup();
if peers.is_empty() {
if show_errors(args) {
eprintln!("rsurl: no peers to fetch magnet metadata from");
}
return 1;
}
if !args.silent {
let label = magnet.display_name.as_deref().unwrap_or("magnet");
eprintln!("* fetching metadata for {label} from {} peers", peers.len());
}
match metadata::fetch_metainfo(
magnet.info_hash,
&peers,
peer_id,
Duration::from_secs(5),
Duration::from_secs(10),
args.verbosity >= 2,
) {
Ok((m, info)) => {
let tb = build_torrent(&info, &magnet.trackers);
(m, magnet.trackers, tb)
}
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: magnet metadata: {e}");
}
return 1;
}
}
} else {
let bytes = if source.starts_with("http://") || source.starts_with("https://") {
match rsurl::Request::get(source).and_then(|r| r.send()) {
Ok(resp) if resp.status == 200 => resp.body,
Ok(resp) => {
if show_errors(args) {
eprintln!("rsurl: fetching torrent: HTTP {}", resp.status);
}
return 1;
}
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: fetching torrent: {e}");
}
return transfer_exit_code(&e);
}
}
} else {
let path = source.strip_prefix("file://").unwrap_or(source);
match std::fs::read(path) {
Ok(b) => b,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: reading torrent {path}: {e}");
}
return 1;
}
}
};
match Metainfo::from_bytes(&bytes) {
Ok(m) => {
let trackers = m.trackers.clone();
(m, trackers, bytes) }
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 1;
}
}
};
if args.bt_info {
print!("{}", metadata_json(&meta));
let _ = io::stdout().flush();
return 0;
}
if args.bt_save_torrent {
return bt_write_torrent(&torrent_bytes, args);
}
let window: Option<(std::path::PathBuf, u64, u64)> = if let Some(sel) = &args.bt_file {
match bt_resolve_file(&meta, sel) {
Ok((idx, off, len)) => {
let out = match args.output.as_deref().filter(|p| *p != "-") {
Some(o) => resolve_output_path(o, args),
None => std::path::PathBuf::from(
meta.files[idx].path.file_name().unwrap_or_default(),
),
};
Some((out, off, off + len))
}
Err(msg) => {
if show_errors(args) {
eprintln!("rsurl: {msg}");
}
return 2;
}
}
} else {
None
};
let layout = if window.is_some() {
Vec::new()
} else {
match bt_layout(&meta, args) {
Ok(l) => l,
Err(msg) => {
if show_errors(args) {
eprintln!("rsurl: {msg}");
}
return 2;
}
}
};
if peers.is_empty() {
peers = bt_announce_peers(
&trackers,
meta.info_hash,
peer_id,
listen_port,
meta.total_length,
args.verbose,
);
}
if peers.is_empty() && !args.no_dht {
if !args.silent {
eprintln!("* no tracker peers; trying DHT");
}
peers = bt_dht_peers(meta.info_hash, args.verbose);
}
peers.sort();
peers.dedup();
if peers.is_empty() {
if show_errors(args) {
eprintln!("rsurl: no peers found (trackers and DHT returned none)");
}
return 1;
}
if !args.silent {
eprintln!(
"* torrent: {} ({}, {} pieces, {} peers)",
meta.name,
human_bytes(meta.total_length),
meta.num_pieces(),
peers.len()
);
match seed {
SeedMode::Forever => eprintln!("* will seed after completion on port {listen_port}"),
SeedMode::UntilRatio(r) => {
eprintln!("* will seed to ratio {r:.2} on port {listen_port}")
}
SeedMode::Off => {}
}
}
let total = match &window {
Some((_, s, e)) => e - s, None => meta.total_length,
};
let show = !args.silent;
let start = Instant::now();
let mut last_tick = start;
let mut cb = |p: &bittorrent::Progress| {
if !show {
return;
}
let complete = p.num_pieces > 0 && p.pieces_complete == p.num_pieces;
if complete && p.uploaded > 0 {
eprint!(
"\r* seeding: uploaded {} (ratio {:.2}) ",
human_bytes(p.uploaded),
p.uploaded as f64 / total.max(1) as f64,
);
let _ = io::stderr().flush();
return;
}
if last_tick.elapsed() >= Duration::from_millis(120) || p.downloaded == total {
let secs = start.elapsed().as_secs_f64();
let rate = if secs > 0.0 {
(p.downloaded as f64 / secs) as u64
} else {
0
};
eprint!(
"\r* {}/{} pieces {}/{} {}/s ",
p.pieces_complete,
p.num_pieces,
human_bytes(p.downloaded),
human_bytes(total),
human_bytes(rate),
);
let _ = io::stderr().flush();
last_tick = Instant::now();
}
};
let result = match window {
Some((out, s, e)) => bittorrent::download_window(&meta, out, &peers, &opts, s, e, &mut cb),
None => bittorrent::download(&meta, layout, &peers, &opts, &mut cb),
};
if show {
eprintln!();
}
match result {
Ok(_) => 0,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
transfer_exit_code(&e)
}
}
}
fn low_speed_params(args: &Args) -> (Option<u64>, u64) {
let lim = args
.speed_limit
.as_deref()
.and_then(|s| s.parse::<u64>().ok());
let tim = args
.speed_time
.as_deref()
.and_then(|s| s.parse::<u64>().ok());
match (lim, tim) {
(None, None) => (None, 30),
(l, t) => (Some(l.unwrap_or(1)), t.unwrap_or(30)),
}
}
fn http_resume(
req: &rsurl::Request,
url: &Url,
args: &Args,
name: &str,
mut jar: Option<&mut CookieJar>,
) -> Option<u8> {
let probe = req.clone().method("HEAD").send().ok()?;
if probe.status >= 400 {
return None;
}
let total = probe
.header("content-length")
.and_then(|v| v.trim().parse::<u64>().ok())?;
if !probe
.header("accept-ranges")
.is_some_and(|v| v.to_ascii_lowercase().contains("bytes"))
{
return None;
}
if let Some(max) = args.max_filesize {
if total > max {
if show_errors(args) {
eprintln!("rsurl: Maximum file size exceeded");
}
return Some(63);
}
}
let etag = probe.header("etag").unwrap_or("").to_string();
let last_mod = probe.header("last-modified").unwrap_or("").to_string();
let url_s = format!("{}://{}:{}{}", url.scheme, url.host, url.port, url.path);
let final_path = resolve_output_path(name, args);
if args.create_dirs {
if let Some(p) = final_path.parent() {
let _ = std::fs::create_dir_all(p);
}
} else if let Some(dir) = &args.output_dir {
let _ = std::fs::create_dir_all(dir);
}
let part = rsurl::resume::part_path(&final_path);
let mut done = match rsurl::resume::read_state(&part) {
Ok(Some(st)) if st.kind == rsurl::resume::Kind::HttpStream => {
http_resume_parse(&st.meta, total, &url_s, &etag).unwrap_or(0)
}
_ => 0,
};
if done > total {
done = 0;
}
if done == total {
if let Err(e) = rsurl::resume::finalize(&part, &final_path, total) {
if show_errors(args) {
eprintln!("rsurl: finalize {}: {e}", final_path.display());
}
return Some(23);
}
return Some(0);
}
let (speed_limit, speed_time) = low_speed_params(args);
let validators = http_resume_validators(&url_s, &etag, &last_mod);
for _ in 0..2 {
let from = done;
let file = match std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&part)
{
Ok(f) => f,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: open {}: {e}", part.display());
}
return Some(23);
}
};
if file.set_len(total).is_err() {
return None; }
let mut f = file;
if f.seek(io::SeekFrom::Start(from)).is_err() {
return None;
}
let now = std::time::Instant::now();
let mut sink = DownloadSink {
inner: Box::new(f),
written: from,
max: args.max_filesize,
rate: args.limit_rate.as_deref().and_then(parse_rate),
speed_limit,
speed_time,
started: now,
progress: args.progress_bar,
silent: args.silent,
last_tick: now,
resume: Some(ResumeCtx {
part: part.clone(),
total,
validators: validators.clone(),
last_save: now,
}),
};
let mut greq = req.clone();
if from > 0 {
greq = greq.header("Range", &format!("bytes={from}-"));
}
let result = if args.verbose {
let mut err = io::stderr().lock();
greq.send_download(&mut sink, jar.as_deref_mut(), &mut err)
} else {
greq.send_download(&mut sink, jar.as_deref_mut(), &mut io::sink())
};
if args.progress_bar && !args.silent {
eprintln!();
}
match result {
Ok(resp) => {
if from > 0 && resp.status == 200 {
if args.verbose {
eprintln!("* server ignored Range; restarting from 0");
}
done = 0;
continue;
}
drop(sink); if let Err(e) = rsurl::resume::finalize(&part, &final_path, total) {
if show_errors(args) {
eprintln!("rsurl: finalize {}: {e}", final_path.display());
}
return Some(23);
}
if args.remote_time {
set_remote_time(&resp, name);
}
run_write_out(&resp, url, args, now.elapsed(), total);
return Some(0);
}
Err(e) => {
let written = sink.written;
drop(sink);
let meta = http_resume_meta(total, written, &validators);
let _ = rsurl::resume::write_state(
&part,
total,
rsurl::resume::Kind::HttpStream,
&meta,
);
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return Some(transfer_exit_code(&e));
}
}
}
Some(0)
}
fn run_http_download(
req: rsurl::Request,
url: &Url,
args: &Args,
mut jar: Option<&mut CookieJar>,
) -> u8 {
let name = if args.remote_name {
match remote_name_from_url(url) {
Ok(n) => n,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: {e}");
}
return 23;
}
}
} else {
args.output.clone().unwrap_or_default()
};
if args.continue_resume && !name.is_empty() && name != "-" {
if let Some(code) = http_resume(&req, url, args, &name, jar.as_deref_mut()) {
return code;
}
}
let (file, out_path) = match create_output_file_tracked(&name, args) {
Ok(pair) => pair,
Err(e) => {
if show_errors(args) {
eprintln!("rsurl: open {name}: {e}");
}
return 23;
}
};
let (speed_limit, speed_time) = low_speed_params(args);
let now = std::time::Instant::now();
let mut sink = DownloadSink {
inner: Box::new(file),
written: 0,
max: args.max_filesize,
rate: args.limit_rate.as_deref().and_then(parse_rate),
speed_limit,
speed_time,
started: now,
progress: args.progress_bar,
silent: args.silent,
last_tick: now,
resume: None,
};
let result = if args.verbose {
let mut err = io::stderr().lock();
req.send_download(&mut sink, jar, &mut err)
} else {
req.send_download(&mut sink, jar, &mut io::sink())
};
let time_total = now.elapsed();
let written = sink.written;
if args.progress_bar && !args.silent {
eprintln!();
}
match result {
Ok(resp) => {
if args.remote_time {
set_remote_time(&resp, &name);
}
run_write_out(&resp, url, args, time_total, written);
0
}
Err(e) => {
if args.remove_on_error {
drop(sink);
let _ = std::fs::remove_file(&out_path);
}
if e.to_string().contains(MAX_FILESIZE_SENTINEL) {
if show_errors(args) {
eprintln!("rsurl: Maximum file size exceeded");
}
return 63;
}
if e.to_string().contains(LOW_SPEED_SENTINEL) {
if show_errors(args) {
eprintln!(
"rsurl: Operation too slow. Less than {} bytes/sec transferred \
the last {speed_time} seconds",
speed_limit.unwrap_or(1)
);
}
return 28;
}
if show_errors(args) {
eprintln!("rsurl: {e}");
}
transfer_exit_code(&e)
}
}
}
fn resolve_output_path(path: &str, args: &Args) -> std::path::PathBuf {
let full = match &args.output_dir {
Some(dir) => std::path::Path::new(dir).join(path),
None => std::path::PathBuf::from(path),
};
if !args.no_clobber || !full.exists() {
return full;
}
let base = full.as_os_str().to_owned();
for n in 1u32.. {
let mut candidate = base.clone();
candidate.push(format!(".{n}"));
let candidate = std::path::PathBuf::from(candidate);
if !candidate.exists() {
return candidate;
}
}
unreachable!("u32 range exhausted picking a --no-clobber name")
}
fn create_output_file_tracked(path: &str, args: &Args) -> io::Result<(File, std::path::PathBuf)> {
let full = resolve_output_path(path, args);
if args.create_dirs {
if let Some(parent) = full.parent() {
if !parent.as_os_str().is_empty() {
std::fs::create_dir_all(parent)?;
}
}
} else if let Some(dir) = &args.output_dir {
std::fs::create_dir_all(dir)?;
}
let file = File::create(&full)?;
Ok((file, full))
}
fn create_output_file(path: &str, args: &Args) -> io::Result<File> {
create_output_file_tracked(path, args).map(|(f, _)| f)
}
fn dump_headers(resp: &Response, path: &str) -> io::Result<()> {
let mut f = File::create(path)?;
write!(f, "{} {} {}\r\n", resp.version, resp.status, resp.reason)?;
for (k, v) in &resp.headers {
write!(f, "{k}: {v}\r\n")?;
}
f.write_all(b"\r\n")
}
fn set_remote_time(resp: &Response, path: &str) {
let Some(lm) = resp.header("last-modified") else {
return;
};
let Some(epoch) = httpdate_to_epoch(lm) else {
return;
};
let mtime = std::time::UNIX_EPOCH + std::time::Duration::from_secs(epoch);
if let Ok(f) = File::options().write(true).open(path) {
let _ = f.set_modified(mtime);
}
}
fn write_output(resp: &Response, url: &Url, args: &Args) -> io::Result<()> {
let mut to_stdout = true;
let mut explicit_stdout = false;
let mut out: Box<dyn Write> = if args.remote_name {
let name = args
.remote_header_name
.then(|| content_disposition_filename(resp))
.flatten()
.map(Ok)
.unwrap_or_else(|| remote_name_from_url(url))
.map_err(|e| io::Error::other(e.to_string()))?;
to_stdout = false;
Box::new(create_output_file(&name, args)?)
} else {
match &args.output {
Some(path) if path != "-" => {
to_stdout = false;
Box::new(create_output_file(path, args)?)
}
Some(_) => {
explicit_stdout = true; Box::new(io::stdout().lock())
}
None => Box::new(io::stdout().lock()),
}
};
let is_tty = to_stdout && io::stdout().is_terminal();
if args.include_headers {
if is_tty {
let line = format!("{} {} ", resp.version, resp.status);
out.write_all(line.as_bytes())?;
out.write_all(&sanitize_for_tty(resp.reason.as_bytes()))?;
out.write_all(b"\r\n")?;
for (k, v) in &resp.headers {
out.write_all(&sanitize_for_tty(k.as_bytes()))?;
out.write_all(b": ")?;
out.write_all(&sanitize_for_tty(v.as_bytes()))?;
out.write_all(b"\r\n")?;
}
} else {
write!(out, "{} {} {}\r\n", resp.version, resp.status, resp.reason)?;
for (k, v) in &resp.headers {
write!(out, "{k}: {v}\r\n")?;
}
}
out.write_all(b"\r\n")?;
}
if is_tty {
if explicit_stdout {
out.write_all(&resp.body)?;
} else if body_looks_binary(&resp.body) {
if show_errors(args) {
eprintln!(
"Warning: Binary output can mess up your terminal. Use \"--output -\" to tell"
);
eprintln!("Warning: rsurl to output it to your terminal anyway, or consider \"-o");
eprintln!("Warning: <FILE>\" to save to a file.");
}
} else {
out.write_all(&sanitize_for_tty(&resp.body))?;
}
} else {
out.write_all(&resp.body)?;
}
Ok(())
}
fn remote_name_from_url(url: &Url) -> Result<String, String> {
let path = url.path.as_str();
let path_no_query = match path.find('?') {
Some(i) => &path[..i],
None => path,
};
let trimmed = path_no_query.trim_end_matches('/');
let last = trimmed.rsplit('/').next().unwrap_or("");
if last.is_empty() {
return Err("Refusing to overwrite stdin".to_string());
}
let basename = Path::new(last)
.file_name()
.and_then(|n| n.to_str())
.ok_or_else(|| "Refusing to overwrite stdin".to_string())?;
if basename.is_empty() {
return Err("Refusing to overwrite stdin".to_string());
}
Ok(basename.to_string())
}
fn print_usage() {
println!(
"rsurl {VERSION} — a pure-Rust curl
Usage: rsurl [options] <url>...
Options:
-o, --output <file> write body to file instead of stdout
-O, --remote-name save body as the URL's last path segment
-i, --include include response headers in the output
-I, --head issue HEAD instead of GET
-v, --verbose print request/response headers to stderr; for a
torrent, -v shows a periodic swarm summary and -vv
adds per-peer connect/unchoke/disconnect detail
-s, --silent suppress error messages
-X, --request <method> override HTTP method; for rtsp:// selects the
RTSP method (OPTIONS/DESCRIBE/SETUP/PLAY/TEARDOWN)
-H, --header <line> add a request header (repeatable)
-d, --data <body> POST body (urlencoded); @file reads from disk
and strips CR/LF. Repeatable; joined with '&'.
--data-raw <body> like -d but '@' is taken literally
--data-binary <body> like -d but @file is read verbatim (no strip)
--data-urlencode <s> percent-encode <s> before sending. Forms:
text =text name=text @file name@file
--json <data> POST <data> as application/json and set Accept to
application/json; @file reads verbatim. Repeatable.
-F, --form <name=value> add a multipart/form-data part. Value forms:
text @file (upload) <file (field from file)
Modifiers: ;type= ;filename= ;headers=@hdrfile
--form-string <n=v> like -F but value is taken literally (no @, <, ;)
--form-escape percent-encode names/filenames per RFC 7578 §4.2
(default: backslash-escape, curl-historical)
-T, --upload-file <f> upload the file: HTTP PUT (default Content-Type:
application/octet-stream), FTP/FTPS STOR, TFTP WRQ,
MQTT PUBLISH, or SFTP/SCP write
--key <file> SSH private-key identity file for sftp:// / scp://
public-key auth (repeatable). Without it, the
default ~/.ssh/id_ed25519|id_ecdsa|id_rsa are tried
-C, --continue-at <off> resume at byte <off> (FTP: REST before STOR);
'-C -' auto-resumes an HTTP download via its
<name>.rsurlpart (needs server Range support; with
--parallel-segments, resumes per chunk)
-a, --append FTP/FTPS upload: append (APPE) instead of replacing
(STOR). No-op for other protocols; overrides -C
-A, --user-agent <ua> set User-Agent
-e, --referer <ref> set Referer
-L, --location follow 3xx redirects
--max-redirs <n> cap on redirect hops (default 50)
-u, --user <user:pass> HTTP Basic auth credentials
--digest use HTTP Digest auth with -u credentials
--oauth2-bearer <t> send Authorization: Bearer <token>
--aws-sigv4 <spec> sign with AWS SigV4 (e.g. aws:amz:us-east-1:s3, -u key:secret)
-k, --insecure don't verify the TLS certificate chain
--cacert <file> PEM bundle to use instead of system trust
--tlsv1.2/1.3 require at least this TLS version (floor)
--tls-max <ver> cap the TLS version (1.2 or 1.3)
--mail-from <addr> SMTP envelope sender (smtp://host, body via -T/-d)
--mail-rcpt <addr> SMTP envelope recipient (repeatable)
--no-idn don't convert international (IDN) hostnames to punycode
--max-time <secs> cap on the whole operation's wall time
--connect-timeout <secs>
cap on the TCP connect step
--http2 require HTTP/2 (ALPN h2); error if unavailable
--http1.1 force HTTP/1.1 (alias: --http1)
--http3 try HTTP/3 (QUIC), fall back to HTTP/2/1.1
--http3-only require HTTP/3 (QUIC); no fallback
-b, --cookie <data> cookies to send: \"k=v[; k2=v2]\" or path to a
Netscape cookies.txt file
-c, --cookie-jar <file> write all known cookies to <file> on exit
-x, --proxy <url> route via a proxy. Scheme picks the kind:
http/https/socks4/socks4a/socks5/socks5h (bare
host:port = http). SOCKS5 also tunnels HTTP/3 &
TFTP (UDP). Reads HTTPS_PROXY/http_proxy/ALL_PROXY.
--socks4 <host:port> / --socks4a / --socks5 / --socks5-hostname
shorthands for -x socks4://… etc.
-U, --proxy-user <u:p> credentials for the proxy (Basic / SOCKS5 auth)
--noproxy <hosts> comma-separated host suffixes that bypass the
proxy; \"*\" bypasses everything
-f, --fail on HTTP >= 400, emit no body and exit 22
-S, --show-error show errors even with -s
-G, --get put -d data in the URL query and use GET
-r, --range <range> request a byte range (Range: bytes=<range>)
--compressed ask for a compressed response (decoded anyway)
-D, --dump-header <file> write response headers to <file>
-R, --remote-time set the saved file's mtime from Last-Modified
--create-dirs create missing directories for -o
--remove-on-error delete a partial -o/-O file if the transfer fails
--no-clobber never overwrite an existing -o/-O file (use .1, .2…)
--max-filesize <n> refuse a download larger than <n> bytes
-w, --write-out <fmt> after the transfer, print <fmt> with %{{vars}}
expanded (http_code, size_download, content_type,
url_effective, time_total, time_connect,
time_appconnect, time_pretransfer, time_starttransfer,
ssl_verify_result, %header{{Name}}; phase timers are
HTTP/1.1-only, else 0.000000)
--url <url> add a URL (repeatable; same as a positional arg)
-n, --netrc read credentials from ~/.netrc (when no -u)
--netrc-file <file> read credentials from <file> (implies -n)
-J, --remote-header-name with -O, name the file from Content-Disposition
--retry <n> retry transient failures up to <n> times
--retry-delay <s> fixed delay between retries (else exponential)
--retry-max-time <s> give up retrying after <s> seconds total
--retry-connrefused also retry on connection refused
--retry-all-errors retry on any error
-z, --time-cond <t> If-Modified-Since (or If-Unmodified-Since for
a leading '-'); a filename uses its mtime
--output-dir <dir> directory to prepend to -o/-O output names
--fail-with-body exit 22 on HTTP >= 400 but still write the body
--proto <spec> restrict allowed schemes (e.g. =https,http)
--proto-default <s> scheme for URLs given without one
-g, --globoff disable URL globbing ({{}} and [] taken literally)
-Z, --parallel run this invocation's transfers concurrently
--parallel-max <n> cap on concurrent transfers (default 50)
--parallel-segments [n] fetch one -o/-O file via n concurrent ranges
(default 4; add -# for a live progress display)
--torrent treat the source (.torrent path/URL or magnet:) as a
torrent; downloads its data to -o/--output-dir
--listen-port <p> port advertised to BitTorrent peers/trackers (6881)
--bt-peer <ip:port> add a torrent peer directly (repeatable)
--no-dht disable the DHT peer-discovery fallback
--seed keep seeding after the torrent completes
--share-ratio <r> seed until uploaded/downloaded reaches r, then exit
--recheck on torrent resume, re-hash on-disk data instead of
trusting the saved .rsurlpart bitfield
--bt-info print torrent metadata as JSON to stdout, then exit
--bt-save-torrent write the .torrent (to -o, else stdout), then exit
--bt-file <N|path> download just one file of a multi-file torrent to -o
--bt-concat download a multi-file torrent as one concatenated -o
--location-trusted keep credentials across cross-host redirects
--post301/302/303 keep POST (don't downgrade to GET) on that redirect
--connect-to <spec> dial HOST2:PORT2 for requests to HOST1:PORT1
(keeps the original Host:/SNI)
--unix-socket <path> connect through a Unix-domain socket (Unix only)
-4, --ipv4 connect over IPv4 only
-6, --ipv6 connect over IPv6 only
--resolve <h:p:addr> use <addr> for <host>:<port> (static DNS)
--disable-epsv FTP: skip EPSV, use PASV directly
--ssl-reqd mail (smtp/imap/pop3): require STARTTLS/STLS upgrade
before sending credentials or data
-P, --ftp-port <addr> FTP: active mode (server connects back); <addr> is
accepted but the control-connection local IP is used
--ftp-create-dirs FTP: create missing remote dirs before upload (-T)
-K, --config <file> read options from a curl-style config file
--next (-:) start a new request with its own options
-#, --progress-bar show progress on streamed file downloads (-o/-O)
-E, --cert <c[:pass]> client certificate for mutual TLS (PEM, or DER with
--cert-type). Optional inline ':password' for the key
--key <file> client TLS private key (also the SSH identity file);
omit when the key is embedded in --cert
--pass <phrase> passphrase for an encrypted --key/--cert key
--cert-type <type> --cert encoding: PEM (default) or DER
--key-type <type> --key encoding: PEM (default) or DER
--pinnedpubkey <h> pin the server pubkey: sha256//BASE64[;sha256//...]
(SHA-256 of the leaf cert's SPKI); fail on mismatch
--capath <dir> trust extra CA certs from every file in <dir>
(in addition to system roots / --cacert)
--crlfile <file> check the server chain against this CRL (PEM/DER;
default backend only)
--ciphers <list> restrict TLS<=1.2 cipher suites (OpenSSL/IANA names,
':'-separated; default backend only)
--tls13-ciphers <l> restrict TLS 1.3 cipher suites (IANA TLS_* names)
--limit-rate <speed> cap download rate (e.g. 200k, 1M) on -o/-O downloads
-y, --speed-time <s> / -Y, --speed-limit <bps>
abort an -o/-O download averaging below <bps>
bytes/sec over <s> seconds (exit 28)
-q, --disable no-op (rsurl reads no config unless -K is given)
-N, --no-buffer no-op (output is already streamed)
--no-progress-meter no-op (no meter is shown by default)
--styled-output, --no-styled-output
no-op (headers are never styled)
-h, --help print this help
-V, --version print version
"
);
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn transfer_exit_code_maps_curl_codes() {
use std::io::{Error as IoError, ErrorKind};
assert_eq!(transfer_exit_code(&rsurl::Error::InvalidUrl("x".into())), 3);
assert_eq!(
transfer_exit_code(&rsurl::Error::UnsupportedScheme("gopher+ssh".into())),
1
);
assert_eq!(transfer_exit_code(&rsurl::Error::UnexpectedEof), 52);
assert_eq!(transfer_exit_code(&rsurl::Error::Ssh("auth".into())), 79);
assert_eq!(
transfer_exit_code(&rsurl::Error::BadResponse("operation timed out".into())),
28
);
assert_eq!(
transfer_exit_code(&rsurl::Error::BadResponse(
"maximum (50) redirects followed".into()
)),
47
);
assert_eq!(
transfer_exit_code(&rsurl::Error::BadResponse("garbage status line".into())),
8
);
assert_eq!(
transfer_exit_code(&rsurl::Error::Io(IoError::from(
ErrorKind::ConnectionRefused
))),
7
);
assert_eq!(
transfer_exit_code(&rsurl::Error::Io(IoError::from(ErrorKind::TimedOut))),
28
);
assert_eq!(
transfer_exit_code(&rsurl::Error::Io(IoError::other(
"failed to lookup address information: Name or service not known"
))),
6
);
}
#[test]
fn proto_allowed_evaluates_specs() {
assert!(proto_allowed("https", "=https,http"));
assert!(!proto_allowed("ftp", "=https,http"));
assert!(proto_allowed("http", "all"));
assert!(!proto_allowed("ftp", "-ftp"));
assert!(proto_allowed("https", "-ftp"));
assert!(proto_allowed("https", "+https"));
}
#[test]
fn epoch_httpdate_roundtrips() {
assert_eq!(
epoch_to_httpdate(784111777),
"Sun, 06 Nov 1994 08:49:37 GMT"
);
assert_eq!(
httpdate_to_epoch("Sun, 06 Nov 1994 08:49:37 GMT"),
Some(784111777)
);
assert_eq!(epoch_to_httpdate(0), "Thu, 01 Jan 1970 00:00:00 GMT");
}
#[test]
fn glob_brace_and_range_expand() {
let urls: Vec<String> = glob_expand("http://h/{a,b}/[1-3]")
.unwrap()
.into_iter()
.map(|(u, _)| u)
.collect();
assert_eq!(
urls,
vec![
"http://h/a/1",
"http://h/a/2",
"http://h/a/3",
"http://h/b/1",
"http://h/b/2",
"http://h/b/3",
]
);
}
#[test]
fn glob_disabled_skips_torrent_paths_and_magnets() {
let torrent = Args {
torrent: true,
..Default::default()
};
assert!(glob_disabled(
&torrent,
r"C:\Users\me\AppData\Local\Temp\x.torrent"
));
assert!(glob_disabled(&torrent, "/tmp/x.torrent"));
assert!(glob_disabled(&Args::default(), "magnet:?xt=urn:btih:abc"));
assert!(!glob_disabled(&torrent, "http://h/file[1-3].torrent"));
assert!(!glob_disabled(&Args::default(), "http://h/{a,b}"));
let off = Args {
globoff: true,
..Default::default()
};
assert!(glob_disabled(&off, "http://h/{a,b}"));
}
#[test]
fn glob_zero_padded_and_step_and_alpha() {
assert_eq!(expand_range("08-11").unwrap(), vec!["08", "09", "10", "11"]);
assert_eq!(expand_range("1-10:3").unwrap(), vec!["1", "4", "7", "10"]);
assert_eq!(expand_range("a-e:2").unwrap(), vec!["a", "c", "e"]);
}
#[test]
fn glob_output_substitution() {
let (_, caps) = &glob_expand("img[1-2].jpg").unwrap()[0];
assert_eq!(apply_glob_output("out-#1.bin", caps), "out-1.bin");
}
#[test]
fn short_bundles_expand() {
let got = expand_short_bundles(&["-sS".into(), "-ofile".into(), "u".into()]);
assert_eq!(got, vec!["-s", "-S", "-o", "file", "u"]);
let got2 = expand_short_bundles(&["--silent".into(), "-".into()]);
assert_eq!(got2, vec!["--silent", "-"]);
}
#[test]
fn sanitize_for_tty_passes_plain_ascii_and_utf8() {
assert_eq!(sanitize_for_tty(b"hello world"), b"hello world");
let utf8 = "café 日本語".as_bytes();
assert_eq!(sanitize_for_tty(utf8), utf8);
}
#[test]
fn sanitize_for_tty_preserves_tab() {
assert_eq!(sanitize_for_tty(b"a\tb"), b"a\tb");
}
#[test]
fn sanitize_for_tty_neutralizes_escape_sequences() {
assert_eq!(sanitize_for_tty(b"\x1b[2J"), b"\\x1b[2J");
assert_eq!(
sanitize_for_tty(b"\x1b]52;c;Zm9v\x07"),
b"\\x1b]52;c;Zm9v\\x07"
);
assert_eq!(sanitize_for_tty(b"\x00\x07\x7f"), b"\\x00\\x07\\x7f");
assert_eq!(sanitize_for_tty("\u{9b}".as_bytes()), b"\\x9b");
assert_eq!(sanitize_for_tty(b"\xa0"), b"\\xa0");
assert_eq!(sanitize_for_tty("\u{a0}".as_bytes()), "\u{a0}".as_bytes());
}
#[test]
fn body_looks_binary_detects_nul() {
assert!(body_looks_binary(b"\x89PNG\x00\x00"));
assert!(!body_looks_binary(b"plain text\n"));
}
#[test]
fn percent_encode_form_passes_unreserved() {
assert_eq!(percent_encode_form(b"abcXYZ012-._~"), "abcXYZ012-._~");
}
#[test]
fn percent_encode_form_space_becomes_plus() {
assert_eq!(percent_encode_form(b"hello world"), "hello+world");
}
#[test]
fn percent_encode_form_special_chars_become_hex() {
assert_eq!(percent_encode_form(b"=&+/?%#"), "%3D%26%2B%2F%3F%25%23",);
}
#[test]
fn percent_encode_form_high_bytes_use_uppercase_hex() {
assert_eq!(percent_encode_form(&[0xC3, 0xA9]), "%C3%A9"); }
#[test]
fn strip_newlines_removes_crlf_and_nul() {
let got = strip_newlines(b"a\r\nb\nc\0d".to_vec());
assert_eq!(got, b"abcd");
}
#[test]
fn strip_newlines_keeps_other_whitespace() {
let got = strip_newlines(b"a\tb c\r\n".to_vec());
assert_eq!(got, b"a\tb c");
}
#[test]
fn encode_urlencoded_plain_content() {
let got = encode_urlencoded("hello world").unwrap();
assert_eq!(got, b"hello+world");
}
#[test]
fn encode_urlencoded_leading_eq_strips_name() {
let got = encode_urlencoded("=hi there").unwrap();
assert_eq!(got, b"hi+there");
}
#[test]
fn encode_urlencoded_name_value() {
let got = encode_urlencoded("name=hello world").unwrap();
assert_eq!(got, b"name=hello+world");
}
#[test]
fn encode_urlencoded_at_file_reads_and_encodes() {
let mut tmp = std::env::temp_dir();
tmp.push("rsurl-urlencode-at-file.txt");
std::fs::write(&tmp, b"hello world").unwrap();
let spec = format!("@{}", tmp.display());
let got = encode_urlencoded(&spec).unwrap();
let _ = std::fs::remove_file(&tmp);
assert_eq!(got, b"hello+world");
}
#[test]
fn encode_urlencoded_name_at_file_reads_and_encodes() {
let mut tmp = std::env::temp_dir();
tmp.push("rsurl-urlencode-name-at.txt");
std::fs::write(&tmp, b"value with spaces").unwrap();
let spec = format!("k@{}", tmp.display());
let got = encode_urlencoded(&spec).unwrap();
let _ = std::fs::remove_file(&tmp);
assert_eq!(got, b"k=value+with+spaces");
}
#[test]
fn encode_urlencoded_eq_wins_over_at() {
let got = encode_urlencoded("x=y@notafile").unwrap();
assert_eq!(got, b"x=y%40notafile");
}
#[test]
fn assemble_empty_is_none() {
assert!(assemble_form_body(&[]).unwrap().is_none());
}
#[test]
fn assemble_joins_with_ampersand() {
let parts = vec![
DataPart::Plain {
value: "a=1".into(),
at_file_ok: true,
},
DataPart::Plain {
value: "b=2".into(),
at_file_ok: true,
},
];
assert_eq!(assemble_form_body(&parts).unwrap().unwrap(), b"a=1&b=2");
}
#[test]
fn assemble_plain_at_file_strips_newlines() {
let mut tmp = std::env::temp_dir();
tmp.push("rsurl-assemble-plain-at.txt");
std::fs::write(&tmp, b"a\r\nb\n").unwrap();
let parts = vec![DataPart::Plain {
value: format!("@{}", tmp.display()),
at_file_ok: true,
}];
let got = assemble_form_body(&parts).unwrap().unwrap();
let _ = std::fs::remove_file(&tmp);
assert_eq!(got, b"ab");
}
#[test]
fn assemble_binary_at_file_keeps_newlines() {
let mut tmp = std::env::temp_dir();
tmp.push("rsurl-assemble-binary-at.txt");
std::fs::write(&tmp, b"a\r\nb\n").unwrap();
let parts = vec![DataPart::Binary {
value: format!("@{}", tmp.display()),
}];
let got = assemble_form_body(&parts).unwrap().unwrap();
let _ = std::fs::remove_file(&tmp);
assert_eq!(got, b"a\r\nb\n");
}
#[test]
fn assemble_data_raw_treats_at_literally() {
let parts = vec![DataPart::Plain {
value: "@literal".into(),
at_file_ok: false,
}];
assert_eq!(assemble_form_body(&parts).unwrap().unwrap(), b"@literal");
}
#[test]
fn assemble_mixes_data_modes() {
let parts = vec![
DataPart::Plain {
value: "n=1".into(),
at_file_ok: true,
},
DataPart::Binary {
value: "rawbytes".into(),
},
DataPart::UrlEncoded {
value: "k=hello world".into(),
},
];
let got = assemble_form_body(&parts).unwrap().unwrap();
assert_eq!(got, b"n=1&rawbytes&k=hello+world");
}
}