//! Convert http requests with or without its payload
//! into a valid curl command.
use std::borrow::Cow;
use std::fmt::{self, Write};
use std::process::Command;
use crate::header::ACCEPT_ENCODING;
use crate::headers::{HeaderEncode, ProxyAuthorization};
use crate::{HeaderName, Method, Version, request};
use rama_core::bytes::Bytes;
use rama_http_types::HttpRequestParts;
use rama_net::address::ProxyAddress;
use rama_net::mode::{ConnectIpMode, DnsResolveIpMode};
use rama_net::uri::Uri;
use rama_net::user::ProxyCredential;
use rama_net::{AuthorityInputExt, ProtocolInputExt};
/// Create a `curl` command string for the given [`HttpRequestParts`].
pub fn cmd_string_for_request_parts(
parts: &(impl HttpRequestParts + AuthorityInputExt + ProtocolInputExt),
) -> String {
let mut cmd = "curl".to_owned();
write_curl_command_for_request_parts(&mut cmd, parts, None);
cmd
}
/// Create a `curl` command string for the given [`HttpRequestParts`] and payload bytes.
pub fn cmd_string_for_request_parts_and_payload(parts: &request::Parts, payload: &Bytes) -> String {
let mut cmd = "curl".to_owned();
write_curl_command_for_request_parts(&mut cmd, parts, Some(payload));
cmd
}
/// Create a `curl` [`Command`] for the given [`HttpRequestParts`].
pub fn cmd_for_request_parts(
parts: &(impl HttpRequestParts + AuthorityInputExt + ProtocolInputExt),
) -> Command {
let mut cmd = Command::new("curl");
write_curl_command_for_request_parts(&mut cmd, parts, None);
cmd
}
/// Create a `curl` [`Command`] for the given [`HttpRequestParts`] and payload bytes.
pub fn cmd_for_request_parts_and_payload(parts: &request::Parts, payload: &Bytes) -> Command {
let mut cmd = Command::new("curl");
write_curl_command_for_request_parts(&mut cmd, parts, Some(payload));
cmd
}
trait CurlCommandWriter {
fn write_uri(&mut self, uri: Uri) -> &mut Self;
fn write_single(&mut self, one: impl fmt::Display) -> &mut Self;
fn write_tuple(
&mut self,
one: impl fmt::Display,
two: impl fmt::Display,
quote_value: bool,
) -> &mut Self;
fn write_header(&mut self, key: HeaderName, value: Cow<'_, str>) -> &mut Self;
}
impl CurlCommandWriter for Command {
// `Command::arg` passes each value as a distinct argv element straight to
// the curl process (no shell), so values must NOT be quoted: any surrounding
// quotes would become literal bytes of the argument. This is the dual of the
// `String` writer below, which builds shell text and therefore must quote.
fn write_uri(&mut self, uri: Uri) -> &mut Self {
self.arg(uri.to_string())
}
fn write_single(&mut self, one: impl fmt::Display) -> &mut Self {
self.arg(one.to_string())
}
fn write_tuple(
&mut self,
one: impl fmt::Display,
two: impl fmt::Display,
_quote_value: bool,
) -> &mut Self {
// `quote_value` only governs shell-text quoting; argv needs none.
self.arg(one.to_string()).arg(two.to_string())
}
fn write_header(&mut self, key: HeaderName, value: Cow<'_, str>) -> &mut Self {
self.arg("-H").arg(format!("{key}: {value}"))
}
}
impl CurlCommandWriter for String {
fn write_uri(&mut self, uri: Uri) -> &mut Self {
self.push(' ');
write_shell_single_quoted(self, uri);
self
}
fn write_single(&mut self, one: impl fmt::Display) -> &mut Self {
_ = write!(self, " \\{} {one}", rama_utils::str::NATIVE_NEWLINE);
self
}
fn write_tuple(
&mut self,
one: impl fmt::Display,
two: impl fmt::Display,
quote_value: bool,
) -> &mut Self {
_ = write!(self, " \\{} {one} ", rama_utils::str::NATIVE_NEWLINE);
if quote_value {
write_shell_single_quoted(self, two);
} else {
_ = write!(self, "{two}");
}
self
}
fn write_header(&mut self, key: HeaderName, value: Cow<'_, str>) -> &mut Self {
_ = write!(self, " \\{} -H ", rama_utils::str::NATIVE_NEWLINE);
write_shell_single_quoted(self, format_args!("{key}: {value}"));
self
}
}
fn write_shell_single_quoted(out: &mut String, value: impl fmt::Display) {
struct ShellSingleQuoted<'a>(&'a mut String);
impl fmt::Write for ShellSingleQuoted<'_> {
fn write_str(&mut self, value: &str) -> fmt::Result {
let mut start = 0;
for (idx, ch) in value.char_indices() {
if ch == '\'' {
self.0.push_str(&value[start..idx]);
self.0.push_str("'\\''");
start = idx + ch.len_utf8();
}
}
self.0.push_str(&value[start..]);
Ok(())
}
}
out.push('\'');
_ = write!(&mut ShellSingleQuoted(out), "{value}");
out.push('\'');
}
fn write_curl_command_for_request_parts(
writer: &mut impl CurlCommandWriter,
parts: &(impl HttpRequestParts + AuthorityInputExt + ProtocolInputExt),
payload: Option<&Bytes>,
) {
let mut uri = parts.uri().clone();
// Origin-form requests carry only a path; reconstruct the full URL for curl
// from the request context's authority (+ scheme). Requests that already
// carry an authority (absolute- or authority-form) are rendered as-is.
if uri.authority().is_none()
&& let Some(authority) = parts.authority()
{
let protocol = parts.protocol();
uri.set_authority(authority.without_default_port_for(protocol).into());
if uri.scheme().is_none()
&& let Some(protocol) = protocol
{
uri.set_scheme(protocol.clone());
}
}
writer.write_uri(uri);
if parts.headers().contains_key(ACCEPT_ENCODING) {
writer.write_single("--compressed");
}
if parts.method() != Method::GET {
writer.write_tuple("-X", parts.method(), false);
}
match parts.version() {
Version::HTTP_09 => {
writer.write_single("--http0.9");
}
Version::HTTP_10 => {
writer.write_single("--http1.0");
}
Version::HTTP_11 => {
writer.write_single("--http1.1");
}
Version::HTTP_2 => {
writer.write_single("--http2");
}
Version::HTTP_3 => {
writer.write_single("--http3");
}
}
if let Some(proxy_addr) = parts
.extensions()
.get_ref::<ProxyAddress>()
.or_else(|| parts.extensions().get_ref())
{
writer.write_tuple("-x", proxy_addr, true);
if let Some(ProxyCredential::Bearer(bearer)) = &proxy_addr.credential
&& let Some(value) = ProxyAuthorization(bearer.clone()).encode_to_value()
{
let s_value = String::from_utf8_lossy(value.as_bytes());
writer.write_header(crate::header::PROXY_AUTHORIZATION, s_value);
}
}
match (
parts.extensions().get_ref::<DnsResolveIpMode>(),
parts.extensions().get_ref::<ConnectIpMode>(),
) {
(Some(DnsResolveIpMode::SingleIpV4), _)
| (
None | Some(DnsResolveIpMode::DualPreferIpV4 | DnsResolveIpMode::Dual),
Some(ConnectIpMode::Ipv4),
) => {
// force ipv4
writer.write_single("-4");
}
(Some(DnsResolveIpMode::SingleIpV6), _)
| (
None | Some(DnsResolveIpMode::DualPreferIpV4 | DnsResolveIpMode::Dual),
Some(ConnectIpMode::Ipv6),
) => {
// force ipv6
writer.write_single("-6");
}
_ => (), // nothing that can be done
}
for (key, value) in parts.headers().ordered_iter() {
if matches!(
key.standard(),
Some(
crate::header::StandardHeader::Host
| crate::header::StandardHeader::ContentLength
| crate::header::StandardHeader::TransferEncoding
)
) {
// ignore content headers as we are not sending a payload
continue;
}
let s_value = String::from_utf8_lossy(value.as_bytes());
writer.write_header(key.clone(), s_value);
}
if let Some(payload) = payload
&& !payload.is_empty()
{
writer.write_tuple(
"--data-raw",
String::from_utf8_lossy(payload.as_ref()),
true,
);
}
}
#[cfg(test)]
mod tests {
use rama_net::Protocol;
use rama_net::address::HostWithPort;
use rama_net::user::credentials::{basic, bearer};
use crate::body::util::BodyExt;
use crate::layer::har;
use super::*;
#[tokio::test]
async fn test_cmd_string_for_request_parts_from_har() {
struct TestCase {
description: &'static str,
input_har_request: &'static str,
expected_cmd_string: String,
}
for test_case in [
TestCase {
description: "GET example.com",
input_har_request: r##"{
"bodySize": 0,
"method": "GET",
"url": "https://example.com/",
"httpVersion": "HTTP/2",
"headers": [
{
"name": "Host",
"value": "example.com"
},
{
"name": "User-Agent",
"value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:142.0) Gecko/20100101 Firefox/142.0"
},
{
"name": "Accept",
"value": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
},
{
"name": "Accept-Language",
"value": "en-US,en;q=0.5"
},
{
"name": "Accept-Encoding",
"value": "gzip, deflate, br, zstd"
},
{
"name": "Sec-GPC",
"value": "1"
},
{
"name": "Upgrade-Insecure-Requests",
"value": "1"
},
{
"name": "Connection",
"value": "keep-alive"
},
{
"name": "Sec-Fetch-Dest",
"value": "document"
},
{
"name": "Sec-Fetch-Mode",
"value": "navigate"
},
{
"name": "Sec-Fetch-Site",
"value": "none"
},
{
"name": "Sec-Fetch-User",
"value": "?1"
},
{
"name": "Priority",
"value": "u=0, i"
},
{
"name": "Pragma",
"value": "no-cache"
},
{
"name": "Cache-Control",
"value": "no-cache"
}
],
"cookies": [],
"queryString": [],
"headersSize": 504
}"##,
expected_cmd_string: format!(
r##"curl 'https://example.com/' \{NL} --compressed \{NL} --http2 \{NL} -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:142.0) Gecko/20100101 Firefox/142.0' \{NL} -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' \{NL} -H 'Accept-Language: en-US,en;q=0.5' \{NL} -H 'Accept-Encoding: gzip, deflate, br, zstd' \{NL} -H 'Sec-GPC: 1' \{NL} -H 'Upgrade-Insecure-Requests: 1' \{NL} -H 'Connection: keep-alive' \{NL} -H 'Sec-Fetch-Dest: document' \{NL} -H 'Sec-Fetch-Mode: navigate' \{NL} -H 'Sec-Fetch-Site: none' \{NL} -H 'Sec-Fetch-User: ?1' \{NL} -H 'Priority: u=0, i' \{NL} -H 'Pragma: no-cache' \{NL} -H 'Cache-Control: no-cache'"##,
NL = rama_utils::str::NATIVE_NEWLINE
),
},
TestCase {
description: "POST form request for ramaproxy FP",
input_har_request: r##"{
"bodySize": 19,
"method": "POST",
"url": "https://fp.ramaproxy.org/form",
"httpVersion": "HTTP/2",
"headers": [
{
"name": "Host",
"value": "fp.ramaproxy.org"
},
{
"name": "User-Agent",
"value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:142.0) Gecko/20100101 Firefox/142.0"
},
{
"name": "Accept",
"value": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
},
{
"name": "Accept-Language",
"value": "en-US,en;q=0.5"
},
{
"name": "Accept-Encoding",
"value": "gzip, deflate, br, zstd"
},
{
"name": "Content-Type",
"value": "application/x-www-form-urlencoded"
},
{
"name": "Content-Length",
"value": "19"
},
{
"name": "Origin",
"value": "https://fp.ramaproxy.org"
},
{
"name": "Sec-GPC",
"value": "1"
},
{
"name": "Connection",
"value": "keep-alive"
},
{
"name": "Referer",
"value": "https://fp.ramaproxy.org/report"
},
{
"name": "Cookie",
"value": "rama-fp=ready"
},
{
"name": "Upgrade-Insecure-Requests",
"value": "1"
},
{
"name": "Sec-Fetch-Dest",
"value": "document"
},
{
"name": "Sec-Fetch-Mode",
"value": "navigate"
},
{
"name": "Sec-Fetch-Site",
"value": "same-origin"
},
{
"name": "Sec-Fetch-User",
"value": "?1"
},
{
"name": "Priority",
"value": "u=0, i"
},
{
"name": "Pragma",
"value": "no-cache"
},
{
"name": "Cache-Control",
"value": "no-cache"
},
{
"name": "TE",
"value": "trailers"
}
],
"cookies": [
{
"name": "rama-fp",
"value": "ready"
}
],
"queryString": [],
"headersSize": 689,
"postData": {
"mimeType": "application/x-www-form-urlencoded",
"params": [
{
"name": "source",
"value": "web"
},
{
"name": "rating",
"value": "3"
}
],
"text": "source=web&rating=3"
}
}"##,
expected_cmd_string: format!(
r##"curl 'https://fp.ramaproxy.org/form' \{NL} --compressed \{NL} -X POST \{NL} --http2 \{NL} -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:142.0) Gecko/20100101 Firefox/142.0' \{NL} -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' \{NL} -H 'Accept-Language: en-US,en;q=0.5' \{NL} -H 'Accept-Encoding: gzip, deflate, br, zstd' \{NL} -H 'Content-Type: application/x-www-form-urlencoded' \{NL} -H 'Origin: https://fp.ramaproxy.org' \{NL} -H 'Sec-GPC: 1' \{NL} -H 'Connection: keep-alive' \{NL} -H 'Referer: https://fp.ramaproxy.org/report' \{NL} -H 'Cookie: rama-fp=ready' \{NL} -H 'Upgrade-Insecure-Requests: 1' \{NL} -H 'Sec-Fetch-Dest: document' \{NL} -H 'Sec-Fetch-Mode: navigate' \{NL} -H 'Sec-Fetch-Site: same-origin' \{NL} -H 'Sec-Fetch-User: ?1' \{NL} -H 'Priority: u=0, i' \{NL} -H 'Pragma: no-cache' \{NL} -H 'Cache-Control: no-cache' \{NL} -H 'TE: trailers' \{NL} --data-raw 'source=web&rating=3'"##,
NL = rama_utils::str::NATIVE_NEWLINE
),
},
] {
// put input together
let har_request: har::spec::Request = serde_json::from_str(test_case.input_har_request)
.unwrap_or_else(|err| {
panic!(
"expect testcase '{}' har request to deserialize: {err}",
test_case.description
)
});
let request: crate::Request = har_request.try_into().unwrap_or_else(|err| {
panic!(
"expect testcase '{}' har request to convert into a http request: {err}",
test_case.description
)
});
let (parts, body) = request.into_parts();
let payload = body.collect().await.unwrap().to_bytes();
let cmd_string = if payload.is_empty() {
cmd_string_for_request_parts(&parts)
} else {
cmd_string_for_request_parts_and_payload(&parts, &payload)
};
assert_eq!(
test_case.expected_cmd_string, cmd_string,
"testcase '{}'",
test_case.description
);
}
}
#[test]
fn test_cmd_string_for_request_with_http_proxy_no_auth() {
let (parts, _) = crate::Request::builder()
.uri(Uri::parse_authority_form("example.com").unwrap())
.body(())
.unwrap()
.into_parts();
parts.extensions.insert(ProxyAddress {
protocol: None,
address: HostWithPort::local_ipv4(8080),
credential: None,
});
let s = cmd_string_for_request_parts(&&parts);
assert_eq!(
s,
format!(
r##"curl 'example.com' \{NL} --http1.1 \{NL} -x '127.0.0.1:8080'"##,
NL = rama_utils::str::NATIVE_NEWLINE
),
);
}
#[test]
fn test_cmd_string_for_request_with_ipv4_preference() {
let (parts, _) = crate::Request::builder()
.uri(Uri::parse_authority_form("example.com").unwrap())
.body(())
.unwrap()
.into_parts();
parts.extensions.insert(DnsResolveIpMode::SingleIpV4);
let s = cmd_string_for_request_parts(&&parts);
assert_eq!(
s,
format!(
r##"curl 'example.com' \{NL} --http1.1 \{NL} -4"##,
NL = rama_utils::str::NATIVE_NEWLINE
),
);
}
#[test]
fn test_cmd_string_for_request_with_ipv6_preference() {
let (parts, _) = crate::Request::builder()
.uri(Uri::parse_authority_form("example.com").unwrap())
.body(())
.unwrap()
.into_parts();
parts.extensions.insert(DnsResolveIpMode::SingleIpV6);
let s = cmd_string_for_request_parts(&&parts);
assert_eq!(
s,
format!(
r##"curl 'example.com' \{NL} --http1.1 \{NL} -6"##,
NL = rama_utils::str::NATIVE_NEWLINE
),
);
}
#[test]
fn test_cmd_string_for_request_with_http_proxy_with_auth_basic_only_username() {
let (parts, _) = crate::Request::builder()
.uri(Uri::parse_authority_form("example.com").unwrap())
.body(())
.unwrap()
.into_parts();
parts.extensions.insert(ProxyAddress {
protocol: None,
address: HostWithPort::local_ipv4(8080),
credential: Some(ProxyCredential::Basic(basic!("john"))),
});
let s = cmd_string_for_request_parts(&&parts);
assert_eq!(
s,
format!(
r##"curl 'example.com' \{NL} --http1.1 \{NL} -x 'john@127.0.0.1:8080'"##,
NL = rama_utils::str::NATIVE_NEWLINE
),
);
}
#[test]
fn test_cmd_string_for_request_with_http_proxy_with_auth_basic() {
let (parts, _) = crate::Request::builder()
.uri(Uri::parse_authority_form("example.com").unwrap())
.body(())
.unwrap()
.into_parts();
parts.extensions.insert(ProxyAddress {
protocol: None,
address: HostWithPort::local_ipv4(8080),
credential: Some(ProxyCredential::Basic(basic!("john", "secret"))),
});
let s = cmd_string_for_request_parts(&&parts);
assert_eq!(
s,
format!(
r##"curl 'example.com' \{NL} --http1.1 \{NL} -x 'john:secret@127.0.0.1:8080'"##,
NL = rama_utils::str::NATIVE_NEWLINE
),
)
}
#[test]
fn test_cmd_string_for_request_with_http_proxy_with_auth_bearer() {
let (parts, _) = crate::Request::builder()
.uri(Uri::parse_authority_form("example.com").unwrap())
.body(())
.unwrap()
.into_parts();
parts.extensions.insert(ProxyAddress {
protocol: None,
address: HostWithPort::local_ipv4(8080),
credential: Some(ProxyCredential::Bearer(bearer!("abc123"))),
});
let s = cmd_string_for_request_parts(&&parts);
assert_eq!(
s,
format!(
r##"curl 'example.com' \{NL} --http1.1 \{NL} -x '127.0.0.1:8080' \{NL} -H 'proxy-authorization: Bearer abc123'"##,
NL = rama_utils::str::NATIVE_NEWLINE
),
);
}
#[test]
fn test_cmd_string_for_request_with_socks5_proxy() {
let (parts, _) = crate::Request::builder()
.version(Version::HTTP_3)
.uri(Uri::parse_authority_form("example.com").unwrap())
.body(())
.unwrap()
.into_parts();
parts.extensions.insert(ProxyAddress {
protocol: Some(Protocol::SOCKS5),
address: HostWithPort::local_ipv4(8080),
credential: Some(ProxyCredential::Basic(basic!("user", "pass"))),
});
let s = cmd_string_for_request_parts(&&parts);
assert_eq!(
s,
format!(
r##"curl 'example.com' \{NL} --http3 \{NL} -x 'socks5://user:pass@127.0.0.1:8080'"##,
NL = rama_utils::str::NATIVE_NEWLINE
),
);
}
#[test]
fn test_cmd_string_for_request_shell_escapes_quoted_values() {
let (parts, _) = crate::Request::builder()
.method(Method::POST)
.uri("https://example.com/")
.header("x-test", "a'b")
.body(())
.unwrap()
.into_parts();
let payload = Bytes::from_static(b"source='web'&rating=3");
let s = cmd_string_for_request_parts_and_payload(&parts, &payload);
assert_eq!(
s,
format!(
r##"curl 'https://example.com/' \{NL} -X POST \{NL} --http1.1 \{NL} -H 'x-test: a'\''b' \{NL} --data-raw 'source='\''web'\''&rating=3'"##,
NL = rama_utils::str::NATIVE_NEWLINE
),
);
}
#[test]
fn test_cmd_string_for_request_shell_escapes_uri() {
// A single quote is a valid RFC 3986 sub-delim and reaches the URI
// writer verbatim, so it must be shell-escaped like any other value.
let (parts, _) = crate::Request::builder()
.uri("http://example.com/a'b?x=y'z")
.body(())
.unwrap()
.into_parts();
let s = cmd_string_for_request_parts(&&parts);
assert_eq!(
s,
format!(
r##"curl 'http://example.com/a'\''b?x=y'\''z' \{NL} --http1.1"##,
NL = rama_utils::str::NATIVE_NEWLINE
),
);
}
#[test]
fn test_cmd_for_request_passes_unquoted_argv() {
// The `Command` path executes curl directly (no shell), so values must
// be passed as bare argv elements: surrounding shell quotes would become
// literal bytes of the argument and break curl.
let (parts, _) = crate::Request::builder()
.method(Method::POST)
.uri("https://example.com/path?q=a'b")
.header("x-test", "a'b")
.body(())
.unwrap()
.into_parts();
let cmd = cmd_for_request_parts_and_payload(&parts, &Bytes::from_static(b"source='web'"));
let args: Vec<String> = cmd
.get_args()
.map(|a| a.to_string_lossy().into_owned())
.collect();
// URI, header value and --data-raw payload are all unquoted argv elements.
assert!(
args.contains(&"https://example.com/path?q=a'b".to_owned()),
"uri must be a bare argv element, got {args:?}",
);
assert!(
args.contains(&"x-test: a'b".to_owned()),
"header must be a bare argv element, got {args:?}",
);
assert!(
args.contains(&"source='web'".to_owned()),
"payload must be a bare argv element, got {args:?}",
);
// No argv element should be wrapped in literal shell quotes.
assert!(
!args
.iter()
.any(|a| a.starts_with('\'') && a.ends_with('\'')),
"no argv element may be shell-quoted, got {args:?}",
);
}
}