use anyhow::{Error, Result};
use log::debug;
use std::sync::Arc;
use super::types::ResponseData;
use crate::domain::extract_domain;
use crate::fetch::request::{extract_http_headers, extract_security_headers};
use crate::fetch::stream::{stream_bytes_with_limit, OnLimit, StreamedBytes};
use crate::utils::sha256_hex;
fn compute_body_sha256(body: &str) -> Option<String> {
if body.is_empty() {
return None;
}
Some(sha256_hex(body.as_bytes()))
}
fn format_http_version(version: reqwest::Version) -> String {
match version {
reqwest::Version::HTTP_09 => "HTTP/0.9".to_string(),
reqwest::Version::HTTP_10 => "HTTP/1.0".to_string(),
reqwest::Version::HTTP_11 => "HTTP/1.1".to_string(),
reqwest::Version::HTTP_2 => "HTTP/2".to_string(),
reqwest::Version::HTTP_3 => "HTTP/3".to_string(),
other => format!("{other:?}"),
}
}
fn decode_body_with_charset(bytes: &[u8], content_type: Option<&str>) -> String {
crate::fetch::charset::decode_body(bytes, content_type, true)
}
fn decode_streamed_body(
streamed: StreamedBytes,
content_type: Option<&str>,
final_domain: &str,
) -> (String, bool) {
match streamed {
StreamedBytes::Complete(bytes) => (decode_body_with_charset(&bytes, content_type), false),
StreamedBytes::Truncated(bytes) => {
debug!("Body exceeded limit for {final_domain}, parsing truncated prefix");
(decode_body_with_charset(&bytes, content_type), true)
}
}
}
fn is_scannable_content_type(ct_lower: &str) -> bool {
let mime = ct_lower.split(';').next().unwrap_or("").trim();
if mime.starts_with("text/") {
return true;
}
matches!(
mime,
"application/xhtml+xml"
| "application/javascript"
| "application/ecmascript"
| "application/x-javascript"
| "application/json"
| "application/manifest+json"
| "application/ld+json"
| "application/xml"
| "application/atom+xml"
| "application/rss+xml"
| "application/soap+xml"
) || mime.ends_with("+json")
|| mime.ends_with("+xml")
}
#[allow(clippy::too_many_lines)] pub(crate) async fn extract_response_data(
response: reqwest::Response,
original_url: &str,
_final_url_str: &str,
) -> Result<Option<ResponseData>, Error> {
let final_url = response.url().to_string();
debug!("Final url after redirects: {final_url}");
let initial_domain = extract_domain(original_url)?;
let final_domain = extract_domain(&final_url)?;
debug!("Initial domain: {initial_domain}, Final domain: {final_domain}");
let parsed_url = reqwest::Url::parse(&final_url)?;
let host = parsed_url
.host_str()
.ok_or_else(|| anyhow::Error::msg("Failed to extract host"))?
.to_string();
let status = response.status();
let status_desc = status
.canonical_reason()
.unwrap_or("Unknown Status Code")
.to_string();
let headers = response.headers().clone();
let http_version = Some(format_http_version(response.version()));
let content_type = headers
.get(reqwest::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.map(std::string::ToString::to_string);
let security_headers = extract_security_headers(&headers);
let http_headers = extract_http_headers(&headers);
if let Some(ct) = headers.get(reqwest::header::CONTENT_TYPE) {
let ct_lc = ct.to_str().unwrap_or("").to_lowercase();
if !is_scannable_content_type(&ct_lc) {
log::debug!("Skipping {final_domain} - non-scannable content-type: {ct_lc}");
return Ok(None);
}
} else {
debug!("No Content-Type header for {final_domain}, continuing anyway");
}
if let Some(encoding) = headers.get(reqwest::header::CONTENT_ENCODING) {
debug!("Content-Encoding for {final_domain}: {encoding:?}");
}
let streamed = stream_bytes_with_limit(
response,
crate::config::MAX_RESPONSE_BODY_SIZE,
OnLimit::Truncate,
&final_domain,
)
.await;
let (body, body_truncated) = match streamed {
Ok(s) => decode_streamed_body(s, content_type.as_deref(), &final_domain),
Err(e) => {
log::warn!("Failed to read response body for {final_domain}: {e}");
(String::new(), false)
}
};
if body.is_empty() {
log::debug!("Empty response body for {final_domain}, recording metadata only");
return Ok(Some(ResponseData {
initial_url: original_url.to_string(),
final_url,
initial_domain,
final_domain,
host,
status: status.as_u16(),
status_desc,
headers,
security_headers,
http_headers,
body: Arc::<str>::from(""),
body_sha256: None,
body_truncated,
content_length: None,
http_version: http_version.clone(),
content_type: content_type.clone(),
}));
}
log::debug!("Body length for {final_domain}: {} bytes", body.len());
if body.contains("<title") || body.contains("<TITLE") {
log::debug!("Title tag found in raw HTML for {final_domain}");
} else {
log::debug!("No title tag found in raw HTML for {final_domain}");
}
let body_sha256 = compute_body_sha256(&body);
#[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)]
let content_length = Some(body.len() as i64);
Ok(Some(ResponseData {
initial_url: original_url.to_string(),
final_url,
initial_domain,
final_domain,
host,
status: status.as_u16(),
status_desc,
headers,
security_headers,
http_headers,
body: Arc::<str>::from(body),
body_sha256,
body_truncated,
content_length,
http_version,
content_type,
}))
}
#[cfg(test)]
mod tests {
use super::*;
use httptest::{matchers::*, responders::*, Expectation, Server};
#[test]
fn test_decode_body_with_charset_uses_content_type_label() {
let bytes = b"\x93hello\x94";
let decoded = decode_body_with_charset(bytes, Some("text/html; charset=windows-1252"));
assert!(decoded.contains('\u{201C}'), "got {decoded:?}");
assert!(!decoded.contains('\u{FFFD}'), "got {decoded:?}");
}
#[test]
fn test_decode_body_with_charset_falls_back_to_meta_tag() {
let mut bytes = Vec::new();
bytes.extend_from_slice(
b"<html><head><meta charset=\"windows-1252\"><title>x</title></head><body>",
);
bytes.push(0x93);
bytes.extend_from_slice(b"text");
bytes.push(0x94);
bytes.extend_from_slice(b"</body></html>");
let decoded = decode_body_with_charset(&bytes, Some("text/html"));
assert!(decoded.contains('\u{201C}'), "got {decoded:?}");
}
#[test]
fn test_decode_body_with_charset_meta_http_equiv() {
let mut bytes = Vec::new();
bytes.extend_from_slice(
b"<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=windows-1252\"></head><body>",
);
bytes.push(0x93);
bytes.extend_from_slice(b"</body></html>");
let decoded = decode_body_with_charset(&bytes, None);
assert!(decoded.contains('\u{201C}'), "got {decoded:?}");
}
#[test]
fn test_decode_body_with_charset_bom_utf16() {
let bytes: &[u8] = &[0xFF, 0xFE, b'h', 0x00, b'i', 0x00];
let decoded = decode_body_with_charset(bytes, Some("text/html"));
assert_eq!(decoded, "hi");
}
#[test]
fn test_decode_body_with_charset_default_utf8() {
let bytes = "hello world".as_bytes();
let decoded = decode_body_with_charset(bytes, None);
assert_eq!(decoded, "hello world");
}
#[test]
fn test_decode_body_with_charset_unknown_label_falls_through() {
let bytes = "hello".as_bytes();
let decoded = decode_body_with_charset(bytes, Some("text/html; charset=not-a-real-thing"));
assert_eq!(decoded, "hello");
}
#[test]
fn test_decode_body_with_charset_preserves_secret_through_smart_quotes() {
let mut bytes = Vec::new();
bytes.extend_from_slice(b"var key = ");
bytes.push(0x91);
bytes.extend_from_slice(b"abcdef0123456789abcdef0123456789");
bytes.push(0x92);
bytes.extend_from_slice(b";");
let decoded = decode_body_with_charset(&bytes, Some("text/html; charset=windows-1252"));
assert!(decoded.contains("abcdef0123456789abcdef0123456789"));
}
#[test]
fn test_is_scannable_content_type_text_family() {
assert!(is_scannable_content_type("text/html"));
assert!(is_scannable_content_type("text/html; charset=utf-8"));
assert!(is_scannable_content_type("text/plain"));
assert!(is_scannable_content_type("text/javascript"));
assert!(is_scannable_content_type("text/xml"));
assert!(is_scannable_content_type("text/css"));
assert!(is_scannable_content_type("text/csv"));
}
#[test]
fn test_is_scannable_content_type_application_specific() {
assert!(is_scannable_content_type("application/javascript"));
assert!(is_scannable_content_type("application/x-javascript"));
assert!(is_scannable_content_type("application/ecmascript"));
assert!(is_scannable_content_type("application/xhtml+xml"));
assert!(is_scannable_content_type("application/json"));
assert!(is_scannable_content_type("application/manifest+json"));
assert!(is_scannable_content_type("application/ld+json"));
assert!(is_scannable_content_type("application/xml"));
assert!(is_scannable_content_type("application/atom+xml"));
}
#[test]
fn test_is_scannable_content_type_plus_suffix_match() {
assert!(is_scannable_content_type("application/vnd.api+json"));
assert!(is_scannable_content_type("application/hal+json"));
assert!(is_scannable_content_type("application/problem+json"));
assert!(is_scannable_content_type("application/vnd.opc.cmdb+xml"));
}
#[test]
fn test_is_scannable_content_type_rejects_binary() {
assert!(!is_scannable_content_type("image/png"));
assert!(!is_scannable_content_type("image/jpeg"));
assert!(!is_scannable_content_type("application/pdf"));
assert!(!is_scannable_content_type("application/zip"));
assert!(!is_scannable_content_type("application/octet-stream"));
assert!(!is_scannable_content_type("video/mp4"));
assert!(!is_scannable_content_type("audio/mpeg"));
assert!(!is_scannable_content_type("font/woff2"));
}
#[test]
fn test_is_scannable_content_type_handles_parameters() {
assert!(is_scannable_content_type(
"application/json ; charset=utf-8"
));
assert!(is_scannable_content_type(
"text/html; charset=windows-1252; boundary=foo"
));
}
#[tokio::test]
async fn test_extract_response_data_success() {
let server = Server::run();
let server_url = server.url("/test").to_string();
let test_url = "https://example.com/test";
server.expect(
Expectation::matching(request::method_path("GET", "/test")).respond_with(
status_code(200)
.insert_header("Content-Type", "text/html; charset=utf-8")
.insert_header("Content-Security-Policy", "default-src 'self'")
.insert_header("Server", "nginx/1.18.0")
.body("<html><head><title>Test</title></head><body>Hello</body></html>"),
),
);
let client = reqwest::Client::new();
let response = client.get(&server_url).send().await.unwrap();
let result = extract_response_data(response, test_url, &server_url).await;
assert!(
result.is_ok(),
"extract_response_data should succeed with IP final URL, got: {result:?}"
);
}
#[tokio::test]
async fn test_extract_response_data_non_html_content_type() {
let server = Server::run();
let server_url = server.url("/test").to_string();
let test_url = "https://example.com/test";
server.expect(
Expectation::matching(request::method_path("GET", "/test")).respond_with(
status_code(200)
.insert_header("Content-Type", "application/json")
.body(r#"{"key": "value"}"#),
),
);
let client = reqwest::Client::new();
let response = client.get(&server_url).send().await.unwrap();
let result = extract_response_data(response, test_url, &server_url).await;
let data = result
.expect("application/json is scannable")
.expect("should return Some ResponseData");
assert_eq!(data.content_type.as_deref(), Some("application/json"));
assert!(data.body.contains("key"));
}
#[tokio::test]
async fn test_extract_response_data_missing_content_type() {
let server = Server::run();
let server_url = server.url("/test").to_string();
let test_url = "https://example.com/test";
server.expect(
Expectation::matching(request::method_path("GET", "/test")).respond_with(
status_code(200).body("<html><head><title>Test</title></head></html>"),
),
);
let client = reqwest::Client::new();
let response = client.get(&server_url).send().await.unwrap();
let result = extract_response_data(response, test_url, &server_url).await;
if let Err(e) = &result {
let msg = e.to_string();
assert!(
!msg.contains("IP addresses do not have registrable domains"),
"IP domain rejection should be gone, got: {msg}"
);
}
}
#[tokio::test]
async fn test_extract_response_data_empty_body() {
let server = Server::run();
let server_url = server.url("/test").to_string();
let test_url = "https://example.com/test";
server.expect(
Expectation::matching(request::method_path("GET", "/test")).respond_with(
status_code(200)
.insert_header("Content-Type", "text/html")
.body(""),
),
);
let client = reqwest::Client::new();
let response = client.get(&server_url).send().await.unwrap();
let result = extract_response_data(response, test_url, &server_url).await;
assert!(
result.is_ok(),
"empty HTML body with IP final URL should extract, got: {result:?}"
);
}
#[test]
fn test_extract_response_data_domain_extraction_logic() {
let original_url = "https://example.com/page";
let final_url = "https://example.org/page";
assert_eq!(extract_domain(original_url).unwrap(), "example.com");
assert_eq!(extract_domain(final_url).unwrap(), "example.org");
}
#[tokio::test]
async fn test_extract_response_data_security_headers_extraction() {
let server = Server::run();
let server_url = server.url("/test").to_string();
let test_url = "https://example.com/test";
server.expect(
Expectation::matching(request::method_path("GET", "/test")).respond_with(
status_code(200)
.insert_header("Content-Type", "text/html")
.insert_header("Content-Security-Policy", "default-src 'self'")
.insert_header("Strict-Transport-Security", "max-age=31536000")
.body("<html><body>Test</body></html>"),
),
);
let client = reqwest::Client::new();
let response = client.get(&server_url).send().await.unwrap();
let result = extract_response_data(response, test_url, &server_url).await;
assert!(
result.is_ok(),
"expected Ok after IP domain support, got: {result:?}"
);
}
#[tokio::test]
async fn test_extract_response_data_http_headers_extraction() {
let server = Server::run();
let server_url = server.url("/test").to_string();
let test_url = "https://example.com/test";
server.expect(
Expectation::matching(request::method_path("GET", "/test")).respond_with(
status_code(200)
.insert_header("Content-Type", "text/html")
.insert_header("Server", "nginx/1.18.0")
.insert_header("X-Powered-By", "PHP/7.4")
.body("<html><body>Test</body></html>"),
),
);
let client = reqwest::Client::new();
let response = client.get(&server_url).send().await.unwrap();
let result = extract_response_data(response, test_url, &server_url).await;
assert!(
result.is_ok(),
"expected Ok after IP domain support, got: {result:?}"
);
}
#[tokio::test]
async fn test_extract_response_data_status_code_extraction() {
let server = Server::run();
let server_url = server.url("/test").to_string();
let test_url = "https://example.com/test";
server.expect(
Expectation::matching(request::method_path("GET", "/test")).respond_with(
status_code(404)
.insert_header("Content-Type", "text/html")
.body("<html><body>Not Found</body></html>"),
),
);
let client = reqwest::Client::new();
let response = client.get(&server_url).send().await.unwrap();
let result = extract_response_data(response, test_url, &server_url).await;
assert!(
result.is_ok(),
"expected Ok after IP domain support, got: {result:?}"
);
}
#[test]
fn test_content_type_filtering_logic() {
let valid_types = vec![
"text/html",
"text/html; charset=utf-8",
"text/html;charset=utf-8",
"TEXT/HTML", "text/html; charset=ISO-8859-1",
];
for ct in valid_types {
let ct_lower = ct.to_lowercase();
assert!(
ct_lower.starts_with("text/html"),
"Content type '{}' should be recognized as HTML",
ct
);
}
let invalid_types = vec![
"application/json",
"text/plain",
"application/xml",
"image/png",
"text/css",
"application/javascript",
];
for ct in invalid_types {
let ct_lower = ct.to_lowercase();
assert!(
!ct_lower.starts_with("text/html"),
"Content type '{}' should NOT be recognized as HTML",
ct
);
}
}
#[test]
fn test_body_size_limit_logic() {
const MAX_SIZE: usize = 2 * 1024 * 1024;
assert_eq!(MAX_SIZE, 2_097_152, "MAX_RESPONSE_BODY_SIZE should be 2MB");
let body_at_limit = "x".repeat(MAX_SIZE);
assert_eq!(body_at_limit.len(), MAX_SIZE);
assert!(body_at_limit.len() <= MAX_SIZE);
let body_over_limit = "x".repeat(MAX_SIZE + 1);
assert_eq!(body_over_limit.len(), MAX_SIZE + 1);
assert!(body_over_limit.len() > MAX_SIZE);
assert_eq!("".len(), 0);
}
#[test]
fn test_decode_streamed_body_sets_truncated_flag() {
let complete = StreamedBytes::Complete(b"<html>ok</html>".to_vec());
let (body, truncated) = decode_streamed_body(complete, Some("text/html"), "example.com");
assert!(!truncated);
assert!(body.contains("ok"));
let over = StreamedBytes::Truncated(b"x".repeat(100));
let (body2, truncated2) = decode_streamed_body(over, Some("text/html"), "example.com");
assert!(
truncated2,
"Truncated stream must set body_truncated for FN awareness"
);
assert_eq!(body2.len(), 100);
}
#[tokio::test]
async fn test_extract_response_data_content_type_case_insensitive() {
let server = Server::run();
let server_url = server.url("/case").to_string();
let test_url = "https://example.com/case";
server.expect(
Expectation::matching(request::method_path("GET", "/case")).respond_with(
status_code(200)
.insert_header("Content-Type", "TEXT/HTML; CHARSET=UTF-8")
.body("<html><body>Test</body></html>"),
),
);
let client = reqwest::Client::new();
let response = client.get(&server_url).send().await.unwrap();
let result = extract_response_data(response, test_url, &server_url).await;
assert!(
result.is_ok(),
"expected Ok after IP domain support, got: {result:?}"
);
}
#[tokio::test]
async fn test_extract_response_data_malformed_content_type_header() {
let server = Server::run();
let server_url = server.url("/malformed").to_string();
let test_url = "https://example.com/malformed";
server.expect(
Expectation::matching(request::method_path("GET", "/malformed")).respond_with(
status_code(200)
.insert_header("Content-Type", "text/html")
.body("<html><body>Test</body></html>"),
),
);
let client = reqwest::Client::new();
let response = client.get(&server_url).send().await.unwrap();
let result = extract_response_data(response, test_url, &server_url).await;
assert!(
result.is_ok(),
"expected Ok after IP domain support, got: {result:?}"
);
}
#[tokio::test]
async fn test_extract_response_data_body_read_failure_handled() {
let server = Server::run();
let server_url = server.url("/body-error").to_string();
let test_url = "https://example.com/body-error";
server.expect(
Expectation::matching(request::method_path("GET", "/body-error")).respond_with(
status_code(200)
.insert_header("Content-Type", "text/html; charset=utf-8")
.body("<html><body>Test</body></html>"),
),
);
let client = reqwest::Client::new();
let response = client.get(&server_url).send().await.unwrap();
let result = extract_response_data(response, test_url, &server_url).await;
assert!(
result.is_ok(),
"expected Ok after IP domain support, got: {result:?}"
);
}
#[test]
fn test_content_type_with_charset_variations() {
let variations = vec![
("text/html; charset=utf-8", true),
("text/html;charset=utf-8", true), ("text/html; charset=UTF-8", true), ("text/html; charset=ISO-8859-1", true),
("text/html; charset=\"utf-8\"", true), ("text/html; boundary=something", true), ("text/html", true), ("application/json; charset=utf-8", false), ];
for (content_type, should_pass) in variations {
let ct_lower = content_type.to_lowercase();
let is_html = ct_lower.starts_with("text/html");
assert_eq!(
is_html,
should_pass,
"Content type '{}' should {} be recognized as HTML",
content_type,
if should_pass { "" } else { "NOT" }
);
}
}
#[test]
fn test_host_extraction_edge_cases() {
let test_cases = vec![
("https://example.com/path", "example.com"),
("https://www.example.com/path", "www.example.com"),
(
"http://subdomain.example.com:8080/path",
"subdomain.example.com",
),
("https://example.com:443/path", "example.com"),
];
for (url, expected_host) in test_cases {
let parsed = reqwest::Url::parse(url).unwrap();
let host = parsed.host_str().unwrap();
assert_eq!(
host, expected_host,
"Host extraction failed for URL: {}",
url
);
}
}
#[test]
fn test_host_extraction_failure_handling() {
let file_url = reqwest::Url::parse("file:///path/to/file").unwrap();
let host_result = file_url.host_str();
assert!(
host_result.is_none(),
"file:// URLs should not have host_str()"
);
}
#[tokio::test]
async fn test_extract_response_data_body_read_error_handling_path() {
let server = Server::run();
let server_url = server.url("/body-error-path").to_string();
let test_url = "https://example.com/body-error-path";
server.expect(
Expectation::matching(request::method_path("GET", "/body-error-path")).respond_with(
status_code(200)
.insert_header("Content-Type", "text/html; charset=utf-8")
.body("<html><body>Test</body></html>"),
),
);
let client = reqwest::Client::new();
let response = client.get(&server_url).send().await.unwrap();
let result = extract_response_data(response, test_url, &server_url).await;
assert!(
result.is_ok(),
"expected Ok after IP domain support, got: {result:?}"
);
}
#[tokio::test]
async fn test_extract_response_data_content_encoding_handled() {
let server = Server::run();
let server_url = server.url("/compressed").to_string();
let test_url = "https://example.com/compressed";
server.expect(
Expectation::matching(request::method_path("GET", "/compressed")).respond_with(
status_code(200)
.insert_header("Content-Type", "text/html; charset=utf-8")
.insert_header("Content-Encoding", "gzip")
.body("<html><body>Test</body></html>"),
),
);
let client = reqwest::Client::new();
let response = client.get(&server_url).send().await.unwrap();
let result = extract_response_data(response, test_url, &server_url).await;
assert!(
result.is_ok(),
"expected Ok after IP domain support, got: {result:?}"
);
}
#[test]
fn test_extract_response_data_status_code_reason_extraction() {
use reqwest::StatusCode;
let test_cases = vec![
(StatusCode::OK, "OK"),
(StatusCode::NOT_FOUND, "Not Found"),
(StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error"),
(StatusCode::MOVED_PERMANENTLY, "Moved Permanently"),
(StatusCode::FORBIDDEN, "Forbidden"),
];
for (status, expected_reason) in test_cases {
let reason = status
.canonical_reason()
.unwrap_or("Unknown Status Code")
.to_string();
assert!(
reason == expected_reason || reason == "Unknown Status Code",
"Status {} should have reason '{}' or 'Unknown Status Code', got '{}'",
status.as_u16(),
expected_reason,
reason
);
}
}
#[test]
fn test_extract_response_data_body_size_limit_enforcement() {
use crate::config::MAX_RESPONSE_BODY_SIZE;
let body_at_limit = "x".repeat(MAX_RESPONSE_BODY_SIZE);
assert_eq!(body_at_limit.len(), MAX_RESPONSE_BODY_SIZE);
assert!(body_at_limit.len() <= MAX_RESPONSE_BODY_SIZE);
let body_over_limit = "x".repeat(MAX_RESPONSE_BODY_SIZE + 1);
assert_eq!(body_over_limit.len(), MAX_RESPONSE_BODY_SIZE + 1);
assert!(body_over_limit.len() > MAX_RESPONSE_BODY_SIZE);
assert_eq!(
MAX_RESPONSE_BODY_SIZE,
2 * 1024 * 1024,
"MAX_RESPONSE_BODY_SIZE should be 2MB"
);
}
#[test]
fn test_extract_response_data_html_preview_logic() {
use crate::config::MAX_HTML_PREVIEW_CHARS;
let short_body = "<html><body>Short</body></html>";
let preview_short: String = short_body.chars().take(MAX_HTML_PREVIEW_CHARS).collect();
assert_eq!(preview_short, short_body);
assert!(preview_short.len() <= MAX_HTML_PREVIEW_CHARS);
let long_body = "x".repeat(MAX_HTML_PREVIEW_CHARS * 2);
let preview_long: String = long_body.chars().take(MAX_HTML_PREVIEW_CHARS).collect();
assert_eq!(preview_long.len(), MAX_HTML_PREVIEW_CHARS);
assert!(preview_long.len() < long_body.len());
assert_eq!(
MAX_HTML_PREVIEW_CHARS, 500,
"MAX_HTML_PREVIEW_CHARS should be 500"
);
}
#[tokio::test]
async fn test_stream_body_with_limit_within_limit() {
let server = Server::run();
let server_url = server.url("/stream-small").to_string();
let body_content = "Hello, World!";
server.expect(
Expectation::matching(request::method_path("GET", "/stream-small"))
.respond_with(status_code(200).body(body_content)),
);
let client = reqwest::Client::new();
let response = client.get(&server_url).send().await.unwrap();
let result = stream_bytes_with_limit(response, 1024, OnLimit::Truncate, "test.com").await;
match result.unwrap() {
StreamedBytes::Complete(bytes) => {
assert_eq!(String::from_utf8(bytes).unwrap(), body_content);
}
StreamedBytes::Truncated(_) => panic!("small body must not be truncated"),
}
}
#[tokio::test]
async fn test_stream_body_with_limit_exceeds_limit() {
let server = Server::run();
let server_url = server.url("/stream-large").to_string();
let large_body = "x".repeat(2000);
server.expect(
Expectation::matching(request::method_path("GET", "/stream-large"))
.respond_with(status_code(200).body(large_body)),
);
let client = reqwest::Client::new();
let response = client.get(&server_url).send().await.unwrap();
let result = stream_bytes_with_limit(response, 1000, OnLimit::Truncate, "test.com").await;
match result.unwrap() {
StreamedBytes::Truncated(bytes) => {
assert_eq!(bytes.len(), 1000, "prefix must be exactly the cap");
assert!(bytes.iter().all(|&b| b == b'x'));
}
StreamedBytes::Complete(_) => {
panic!("oversized body must be reported as truncated")
}
}
}
#[tokio::test]
async fn test_stream_body_with_limit_exactly_at_limit() {
let server = Server::run();
let server_url = server.url("/stream-exact").to_string();
let exact_body = "x".repeat(1000);
server.expect(
Expectation::matching(request::method_path("GET", "/stream-exact"))
.respond_with(status_code(200).body(exact_body.clone())),
);
let client = reqwest::Client::new();
let response = client.get(&server_url).send().await.unwrap();
let result = stream_bytes_with_limit(response, 1000, OnLimit::Truncate, "test.com").await;
match result.unwrap() {
StreamedBytes::Complete(bytes) => assert_eq!(bytes.len(), 1000),
StreamedBytes::Truncated(_) => {
panic!("body exactly at limit should be complete")
}
}
}
#[tokio::test]
async fn test_stream_body_with_limit_empty_body() {
let server = Server::run();
let server_url = server.url("/stream-empty").to_string();
server.expect(
Expectation::matching(request::method_path("GET", "/stream-empty"))
.respond_with(status_code(200).body("")),
);
let client = reqwest::Client::new();
let response = client.get(&server_url).send().await.unwrap();
let result = stream_bytes_with_limit(response, 1000, OnLimit::Truncate, "test.com").await;
match result.unwrap() {
StreamedBytes::Complete(bytes) => assert!(bytes.is_empty()),
StreamedBytes::Truncated(_) => panic!("empty body must not be truncated"),
}
}
#[test]
fn test_stream_body_prevents_oom_attack() {
use crate::config::MAX_RESPONSE_BODY_SIZE;
assert_eq!(MAX_RESPONSE_BODY_SIZE, 2 * 1024 * 1024);
}
}