use std::borrow::Cow;
use std::io::{self, BufRead, BufReader, Read, Write};
use std::time::Instant;
use encoding_rs::Encoding;
use encoding_rs_io::DecodeReaderBytesBuilder;
use mime::Mime;
use reqwest::blocking::{Body, Request, Response};
use reqwest::cookie::CookieStore;
use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, CONTENT_LENGTH, CONTENT_TYPE, COOKIE, HOST};
use url::Url;
use crate::formatting::headers::HeaderFormatter;
use crate::utils::reason_phrase;
use crate::{
buffer::Buffer,
cli::FormatOptions,
cli::{Pretty, Theme},
decoder::{decompress, get_compression_type},
formatting::serde_json_format,
formatting::{get_json_formatter, Highlighter},
middleware::ResponseExt,
utils::{copy_largebuf, test_mode, BUFFER_SIZE},
};
const BINARY_SUPPRESSOR: &str = concat!(
"+-----------------------------------------+\n",
"| NOTE: binary data not shown in terminal |\n",
"+-----------------------------------------+\n",
"\n"
);
struct BinaryGuard<'a, T: Read> {
reader: BufReader<&'a mut T>,
buffer: Vec<u8>,
checked: bool,
}
#[derive(Debug)]
struct FoundBinaryData;
impl std::fmt::Display for FoundBinaryData {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("binary data not shown in terminal")
}
}
impl std::error::Error for FoundBinaryData {}
impl<'a, T: Read> BinaryGuard<'a, T> {
fn new(reader: &'a mut T, checked: bool) -> Self {
Self {
reader: BufReader::with_capacity(BUFFER_SIZE, reader),
buffer: Vec::new(),
checked,
}
}
fn read_lines(&mut self) -> io::Result<Option<&[u8]>> {
self.buffer.clear();
loop {
let buf = match self.reader.fill_buf() {
Ok(buf) => buf,
Err(e) if e.kind() == io::ErrorKind::Interrupted => continue,
Err(e) => return Err(e),
};
if self.checked && buf.contains(&b'\0') {
return Err(io::Error::new(io::ErrorKind::InvalidData, FoundBinaryData));
} else if buf.is_empty() {
if self.buffer.is_empty() {
return Ok(None);
} else {
return Ok(Some(&self.buffer));
}
} else if let Some(ind) = memchr::memrchr(b'\n', buf) {
self.buffer.extend_from_slice(&buf[..=ind]);
self.reader.consume(ind + 1);
return Ok(Some(&self.buffer));
} else {
self.buffer.extend_from_slice(buf);
let n = buf.len(); self.reader.consume(n);
}
}
}
}
pub struct Printer {
format_json: bool,
json_indent_level: usize,
sort_headers: bool,
color: bool,
theme: Theme,
stream: Option<bool>,
buffer: Buffer,
}
impl Printer {
pub fn new(
pretty: Pretty,
theme: Theme,
stream: impl Into<Option<bool>>,
buffer: Buffer,
format_options: FormatOptions,
) -> Self {
Printer {
format_json: format_options.json_format.unwrap_or(pretty.format()),
json_indent_level: format_options.json_indent.unwrap_or(4),
sort_headers: format_options.headers_sort.unwrap_or(pretty.format()),
color: pretty.color(),
stream: stream.into(),
theme,
buffer,
}
}
fn get_highlighter(&mut self, syntax: &'static str) -> Highlighter<'_> {
Highlighter::new(syntax, self.theme, &mut self.buffer)
}
fn get_header_formatter(&mut self) -> HeaderFormatter<'_, Buffer> {
let is_terminal = self.buffer.is_terminal();
HeaderFormatter::new(
&mut self.buffer,
self.color.then(|| self.theme.as_syntect_theme()),
is_terminal,
self.sort_headers,
)
}
fn print_colorized_text(&mut self, text: &str, syntax: &'static str) -> io::Result<()> {
self.get_highlighter(syntax).highlight(text)
}
fn print_syntax_text(&mut self, text: &str, syntax: &'static str) -> io::Result<()> {
if self.color {
self.print_colorized_text(text, syntax)
} else {
self.buffer.print(text)
}
}
fn print_json_text(&mut self, text: &str, check_valid: bool) -> io::Result<()> {
if !self.format_json {
return self.print_syntax_text(text, "json");
}
if check_valid && !valid_json(text) {
return self.print_syntax_text(text, "json");
}
if self.color {
let mut buf = Vec::new();
serde_json_format(self.json_indent_level, text, &mut buf)?;
buf.write_all(b"\n\n")?;
let text = String::from_utf8_lossy(&buf);
self.print_colorized_text(&text, "json")
} else {
serde_json_format(self.json_indent_level, text, &mut self.buffer)?;
self.buffer.write_all(b"\n\n")?;
self.buffer.flush()?;
Ok(())
}
}
fn print_body_text(&mut self, content_type: ContentType, body: &str) -> io::Result<()> {
match content_type {
ContentType::Json => self.print_json_text(body, true),
ContentType::Xml => self.print_syntax_text(body, "xml"),
ContentType::Html => self.print_syntax_text(body, "html"),
ContentType::Css => self.print_syntax_text(body, "css"),
ContentType::Text | ContentType::JavaScript if valid_json(body) => {
self.print_json_text(body, false)
}
ContentType::JavaScript => self.print_syntax_text(body, "js"),
_ => self.buffer.print(body),
}
}
fn print_stream(&mut self, reader: &mut impl Read) -> io::Result<()> {
if !self.buffer.is_terminal() {
return copy_largebuf(reader, &mut self.buffer, true);
}
let mut guard = BinaryGuard::new(reader, true);
while let Some(lines) = guard.read_lines()? {
self.buffer.write_all(lines)?;
self.buffer.flush()?;
}
Ok(())
}
fn print_colorized_stream(
&mut self,
stream: &mut impl Read,
syntax: &'static str,
) -> io::Result<()> {
let mut guard = BinaryGuard::new(stream, self.buffer.is_terminal());
let mut highlighter = self.get_highlighter(syntax);
while let Some(lines) = guard.read_lines()? {
for line in lines.split_inclusive(|&b| b == b'\n') {
highlighter.highlight_bytes(line)?;
}
highlighter.flush()?;
}
Ok(())
}
fn print_syntax_stream(
&mut self,
stream: &mut impl Read,
syntax: &'static str,
) -> io::Result<()> {
if self.color {
self.print_colorized_stream(stream, syntax)
} else {
self.print_stream(stream)
}
}
fn print_json_stream(&mut self, stream: &mut impl Read) -> io::Result<()> {
if !self.format_json {
self.print_syntax_stream(stream, "json")
} else if self.color {
let mut guard = BinaryGuard::new(stream, self.buffer.is_terminal());
let mut formatter = get_json_formatter(self.json_indent_level);
let mut highlighter = self.get_highlighter("json");
let mut buf = Vec::new();
while let Some(lines) = guard.read_lines()? {
formatter.format_buf(lines, &mut buf)?;
for line in buf.split_inclusive(|&b| b == b'\n') {
highlighter.highlight_bytes(line)?;
}
highlighter.flush()?;
buf.clear();
}
Ok(())
} else {
let mut formatter = get_json_formatter(self.json_indent_level);
if !self.buffer.is_terminal() {
let mut buf = vec![0; BUFFER_SIZE];
loop {
match stream.read(&mut buf) {
Ok(0) => return Ok(()),
Ok(n) => {
formatter.format_buf(&buf[0..n], &mut self.buffer)?;
self.buffer.flush()?;
}
Err(e) if e.kind() == io::ErrorKind::Interrupted => continue,
Err(e) => return Err(e),
}
}
}
let mut guard = BinaryGuard::new(stream, true);
while let Some(lines) = guard.read_lines()? {
formatter.format_buf(lines, &mut self.buffer)?;
self.buffer.flush()?;
}
Ok(())
}
}
fn print_body_stream(
&mut self,
content_type: ContentType,
body: &mut impl Read,
) -> io::Result<()> {
match content_type {
ContentType::Json => self.print_json_stream(body),
ContentType::Xml => self.print_syntax_stream(body, "xml"),
ContentType::Html => self.print_syntax_stream(body, "html"),
ContentType::Css => self.print_syntax_stream(body, "css"),
ContentType::JavaScript => self.print_syntax_stream(body, "js"),
_ => self.print_stream(body),
}
}
pub fn print_separator(&mut self) -> io::Result<()> {
self.buffer.print("\n")?;
self.buffer.flush()?;
Ok(())
}
pub fn print_request_headers<T>(&mut self, request: &Request, cookie_jar: &T) -> io::Result<()>
where
T: CookieStore,
{
let url = request.url();
let version = request.version();
let mut headers = request.headers().clone();
headers
.entry(ACCEPT)
.or_insert_with(|| HeaderValue::from_static("*/*"));
if let Some(cookie) = cookie_jar.cookies(url) {
headers.insert(COOKIE, cookie);
}
if let Some(body) = request.body().and_then(Body::as_bytes) {
headers
.entry(CONTENT_LENGTH)
.or_insert_with(|| body.len().into());
}
if let Some(host) = request.url().host_str() {
headers.entry(HOST).or_insert_with(|| {
if test_mode() {
HeaderValue::from_str("http.mock")
} else if let Some(port) = request.url().port() {
HeaderValue::from_str(&format!("{host}:{port}"))
} else {
HeaderValue::from_str(host)
}
.expect("hostname should already be validated/parsed")
});
}
self.get_header_formatter().print_request_headers(
request.method(),
request.url(),
version,
&headers,
)?;
self.buffer.print("\n")?;
self.buffer.flush()?;
Ok(())
}
pub fn print_response_headers(&mut self, response: &Response) -> io::Result<()> {
self.get_header_formatter().print_response_headers(
response.version(),
response.status(),
&reason_phrase(response),
response.headers(),
)?;
self.buffer.print("\n")?;
self.buffer.flush()?;
Ok(())
}
pub fn print_request_body(&mut self, request: &mut Request) -> anyhow::Result<()> {
let content_type = get_content_type(request.headers());
if let Some(body) = request.body_mut() {
let body = body.buffer()?;
if body.contains(&b'\0') {
self.buffer.print(BINARY_SUPPRESSOR)?;
} else {
self.print_body_text(content_type, &String::from_utf8_lossy(body))?;
self.buffer.print("\n")?;
}
self.buffer.print("\n")?;
self.buffer.flush()?;
}
Ok(())
}
pub fn print_response_body(
&mut self,
response: &mut Response,
encoding: Option<&'static Encoding>,
mime: Option<&str>,
) -> anyhow::Result<()> {
let starting_time = Instant::now();
let url = response.url().clone();
let content_type =
mime.map_or_else(|| get_content_type(response.headers()), ContentType::from);
let encoding = encoding.or_else(|| get_charset(response));
let compression_type = get_compression_type(response.headers());
let mut body = decompress(response, compression_type);
let stream = self.stream.unwrap_or(content_type.is_stream());
if !self.buffer.is_terminal() {
if (self.color || self.format_json) && content_type.is_text() {
if stream {
self.print_body_stream(
content_type,
&mut decode_stream(&mut body, encoding, &url)?,
)?;
} else {
let mut buf = Vec::new();
body.read_to_end(&mut buf)?;
let text = decode_blob_unconditional(&buf, encoding, &url);
self.print_body_text(content_type, &text)?;
}
} else if stream {
copy_largebuf(&mut body, &mut self.buffer, true)?;
} else {
let mut buf = Vec::new();
body.read_to_end(&mut buf)?;
self.buffer.write_all(&buf)?;
}
} else if stream {
match self
.print_body_stream(content_type, &mut decode_stream(&mut body, encoding, &url)?)
{
Ok(_) => {
self.buffer.print("\n")?;
}
Err(err) if err.get_ref().is_some_and(|err| err.is::<FoundBinaryData>()) => {
self.buffer.print(BINARY_SUPPRESSOR)?;
}
Err(err) => return Err(err.into()),
}
} else {
let mut buf = Vec::new();
body.read_to_end(&mut buf)?;
match decode_blob(&buf, encoding, &url) {
None => {
self.buffer.print(BINARY_SUPPRESSOR)?;
}
Some(text) => {
self.print_body_text(content_type, &text)?;
self.buffer.print("\n")?;
}
};
}
self.buffer.flush()?;
drop(body); response.meta_mut().content_download_duration = Some(starting_time.elapsed());
Ok(())
}
pub fn print_response_meta(&mut self, response: &Response) -> anyhow::Result<()> {
let meta = response.meta();
let mut total_elapsed_time = meta.request_duration.as_secs_f64();
if let Some(content_download_duration) = meta.content_download_duration {
total_elapsed_time += content_download_duration.as_secs_f64();
}
self.buffer
.print(&format!("Elapsed time: {total_elapsed_time:.5}s\n"))?;
if let Some(remote_addr) = response.remote_addr() {
self.buffer
.print(&format!("Remote address: {remote_addr:?}\n"))?;
}
self.buffer.print("\n")?;
Ok(())
}
}
enum ContentType {
Json,
Html,
Xml,
JavaScript,
Css,
Text,
UrlencodedForm,
Multipart,
EventStream,
Unknown,
}
impl ContentType {
fn is_text(&self) -> bool {
match self {
ContentType::Unknown | ContentType::UrlencodedForm | ContentType::Multipart => false,
ContentType::Json
| ContentType::Html
| ContentType::Xml
| ContentType::JavaScript
| ContentType::Css
| ContentType::Text
| ContentType::EventStream => true,
}
}
fn is_stream(&self) -> bool {
match self {
ContentType::EventStream => true,
ContentType::Json
| ContentType::Html
| ContentType::Xml
| ContentType::JavaScript
| ContentType::Css
| ContentType::Text
| ContentType::UrlencodedForm
| ContentType::Multipart
| ContentType::Unknown => false,
}
}
}
impl From<&str> for ContentType {
fn from(content_type: &str) -> Self {
if content_type.contains("json") {
ContentType::Json
} else if content_type.contains("html") {
ContentType::Html
} else if content_type.contains("xml") {
ContentType::Xml
} else if content_type.contains("multipart") {
ContentType::Multipart
} else if content_type.contains("x-www-form-urlencoded") {
ContentType::UrlencodedForm
} else if content_type.contains("javascript") {
ContentType::JavaScript
} else if content_type.contains("css") {
ContentType::Css
} else if content_type.contains("event-stream") {
ContentType::EventStream
} else if content_type.contains("text") {
ContentType::Text
} else {
ContentType::Unknown
}
}
}
fn get_content_type(headers: &HeaderMap) -> ContentType {
headers
.get(CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.map_or(ContentType::Unknown, ContentType::from)
}
fn valid_json(text: &str) -> bool {
serde_json::from_str::<serde::de::IgnoredAny>(text).is_ok()
}
fn decode_blob<'a>(
raw: &'a [u8],
encoding: Option<&'static Encoding>,
url: &Url,
) -> Option<Cow<'a, str>> {
let encoding = encoding.unwrap_or_else(|| detect_encoding(raw, true, url));
if encoding.is_ascii_compatible() && raw.contains(&0) {
return None;
}
let text = encoding.decode_with_bom_removal(raw).0;
if !encoding.is_ascii_compatible() && text.contains('\0') {
None
} else {
Some(text)
}
}
fn decode_blob_unconditional<'a>(
raw: &'a [u8],
encoding: Option<&'static Encoding>,
url: &Url,
) -> Cow<'a, str> {
let encoding = encoding.unwrap_or_else(|| detect_encoding(raw, true, url));
encoding.decode_with_bom_removal(raw).0
}
fn decode_stream<'a>(
stream: &'a mut impl Read,
encoding: Option<&'static Encoding>,
url: &Url,
) -> io::Result<impl Read + 'a> {
let capacity = if encoding.is_some() { 16 } else { 16 * 1024 };
let mut reader = BufReader::with_capacity(capacity, stream);
let encoding = match encoding {
Some(encoding) => encoding,
None => {
let peek = reader.fill_buf()?;
detect_encoding(peek, false, url)
}
};
let reader = DecodeReaderBytesBuilder::new()
.encoding(Some(encoding))
.build(reader);
Ok(reader)
}
fn detect_encoding(mut bytes: &[u8], mut complete: bool, url: &Url) -> &'static Encoding {
if bytes.starts_with(b"\xEF\xBB\xBF") {
return encoding_rs::UTF_8;
} else if bytes.starts_with(b"\xFF\xFE") {
return encoding_rs::UTF_16LE;
} else if bytes.starts_with(b"\xFE\xFF") {
return encoding_rs::UTF_16BE;
}
const CHARDET_PEEK_SIZE: usize = 64 * 1024;
if bytes.len() > CHARDET_PEEK_SIZE {
bytes = &bytes[..CHARDET_PEEK_SIZE];
complete = false;
}
let mut detector = chardetng::EncodingDetector::new();
detector.feed(bytes, complete);
let tld = url.domain().and_then(get_tld).map(str::as_bytes);
detector.guess(tld, true)
}
fn get_tld(domain: &str) -> Option<&str> {
domain.trim_end_matches('.').rsplit('.').next()
}
fn get_charset(response: &Response) -> Option<&'static Encoding> {
let content_type = response.headers().get(CONTENT_TYPE)?.to_str().ok()?;
let mime: Mime = content_type.parse().ok()?;
let encoding_name = mime.get_param("charset")?.as_str();
Encoding::for_label(encoding_name.as_bytes())
}
#[cfg(test)]
mod tests {
use crate::utils::random_string;
use crate::{buffer::Buffer, cli::Cli, vec_of_strings};
use super::*;
fn run_cmd(args: impl IntoIterator<Item = String>, is_stdout_tty: bool) -> Printer {
let args = Cli::try_parse_from(args).unwrap();
let theme = args.style.unwrap_or_default();
let buffer = Buffer::new(args.download, args.output.as_deref(), is_stdout_tty).unwrap();
let pretty = args.pretty.unwrap_or_else(|| buffer.guess_pretty());
Printer::new(pretty, theme, false, buffer, FormatOptions::default())
}
fn temp_path() -> String {
let mut dir = std::env::temp_dir();
let filename = random_string();
dir.push(filename);
dir.to_str().unwrap().to_owned()
}
#[test]
fn terminal_mode() {
let p = run_cmd(vec_of_strings!["xh", "httpbin.org/get"], true);
assert_eq!(p.color, true);
assert!(p.buffer.is_stdout());
}
#[test]
fn redirect_mode() {
let p = run_cmd(vec_of_strings!["xh", "httpbin.org/get"], false);
assert_eq!(p.color, false);
assert!(p.buffer.is_redirect());
}
#[test]
fn terminal_mode_with_output_file() {
let output = temp_path();
let p = run_cmd(vec_of_strings!["xh", "httpbin.org/get", "-o", output], true);
assert_eq!(p.color, false);
assert!(p.buffer.is_file());
}
#[test]
fn redirect_mode_with_output_file() {
let output = temp_path();
let p = run_cmd(
vec_of_strings!["xh", "httpbin.org/get", "-o", output],
false,
);
assert_eq!(p.color, false);
assert!(p.buffer.is_file());
}
#[test]
fn terminal_mode_download() {
let p = run_cmd(vec_of_strings!["xh", "httpbin.org/get", "-d"], true);
assert_eq!(p.color, true);
assert!(p.buffer.is_stderr());
}
#[test]
fn redirect_mode_download() {
let p = run_cmd(vec_of_strings!["xh", "httpbin.org/get", "-d"], false);
assert_eq!(p.color, true);
assert!(p.buffer.is_stderr());
}
#[test]
fn terminal_mode_download_with_output_file() {
let output = temp_path();
let p = run_cmd(
vec_of_strings!["xh", "httpbin.org/get", "-d", "-o", output],
true,
);
assert_eq!(p.color, true);
assert!(p.buffer.is_stderr());
}
#[test]
fn redirect_mode_download_with_output_file() {
let output = temp_path();
let p = run_cmd(
vec_of_strings!["xh", "httpbin.org/get", "-d", "-o", output],
false,
);
assert_eq!(p.color, true);
assert!(p.buffer.is_stderr());
}
}