use std::collections::HashMap;
use serde::{Serialize, Serializer};
use crate::common::CacheContext;
use crate::engines::engine_client::{CapturedResponse, ScrapeOptions, ScrapeResponse};
use crate::workers::request_identity::request_fingerprint;
#[must_use]
pub fn generate_scrape_cache_key(ctx: &CacheContext, options: &ScrapeOptions) -> String {
let fingerprint = request_fingerprint(options);
format!(
"scrape:{}:{:?}:{}?fp={:016x}",
ctx.team_id, ctx.method, ctx.url, fingerprint
)
}
const SENSITIVE_RESPONSE_HEADERS: &[&str] = &[
"set-cookie",
"cookie",
"authorization",
"proxy-authorization",
"www-authenticate",
"x-api-key",
"x-auth-token",
"x-session-id",
];
pub fn filter_sensitive_headers(headers: &mut HashMap<String, String>) {
headers.retain(|k, _| {
!SENSITIVE_RESPONSE_HEADERS
.iter()
.any(|s| k.eq_ignore_ascii_case(s))
});
}
#[derive(Serialize)]
pub struct SanitizedScrapeResponse<'a> {
pub status_code: u16,
pub content: &'a str,
pub screenshot: Option<&'a String>,
pub content_type: &'a str,
#[serde(serialize_with = "serialize_sanitized_headers")]
pub headers: &'a HashMap<String, String>,
pub response_time_ms: u64,
pub final_url: Option<&'a String>,
pub markdown: Option<&'a String>,
#[serde(skip_serializing_if = "<[CapturedResponse]>::is_empty")]
pub captured_responses: &'a [CapturedResponse],
#[serde(skip_serializing_if = "Option::is_none")]
pub js_degraded: &'a Option<bool>,
}
impl<'a> SanitizedScrapeResponse<'a> {
#[must_use]
pub fn from_response(response: &'a ScrapeResponse) -> Self {
Self {
status_code: response.status_code,
content: &response.content,
screenshot: response.screenshot.as_ref(),
content_type: &response.content_type,
headers: &response.headers,
response_time_ms: response.response_time_ms,
final_url: response.final_url.as_ref(),
markdown: response.markdown.as_ref(),
captured_responses: &response.captured_responses,
js_degraded: &response.js_degraded,
}
}
}
fn serialize_sanitized_headers<S>(
headers: &HashMap<String, String>,
serializer: S,
) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
use serde::ser::SerializeMap;
let mut map = serializer.serialize_map(Some(headers.len()))?;
for (k, v) in headers {
if !SENSITIVE_RESPONSE_HEADERS
.iter()
.any(|s| k.eq_ignore_ascii_case(s))
{
map.serialize_entry(k, v)?;
}
}
map.end()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::common::CacheMode;
use crate::domain::id::Id;
use crate::engines::engine_client::HttpMethod;
fn test_team() -> Id {
Id::from_u128(0x7e57_0000_0000_0000_0000_0000_0000_0001)
}
fn other_team() -> Id {
Id::from_u128(0x7e57_0000_0000_0000_0000_0000_0000_0002)
}
fn make_ctx(url: &str, method: HttpMethod) -> CacheContext {
make_ctx_for(test_team(), url, method)
}
fn make_ctx_for(team_id: Id, url: &str, method: HttpMethod) -> CacheContext {
CacheContext {
team_id,
url: url.to_string(),
method,
mode: CacheMode::Enabled,
}
}
#[test]
fn test_generate_scrape_cache_key_format() {
let ctx = make_ctx("https://example.com", HttpMethod::Get);
let opts = ScrapeOptions::default();
let key = generate_scrape_cache_key(&ctx, &opts);
let prefix = format!("scrape:{}:Get:https://example.com?fp=", test_team());
assert!(key.starts_with(&prefix));
assert_eq!(key.len(), prefix.len() + 16);
}
#[test]
fn test_cache_key_differs_by_team_id() {
let url = "https://example.com/private";
let opts = ScrapeOptions::default();
let key_a =
generate_scrape_cache_key(&make_ctx_for(test_team(), url, HttpMethod::Get), &opts);
let key_b =
generate_scrape_cache_key(&make_ctx_for(other_team(), url, HttpMethod::Get), &opts);
assert_ne!(
key_a, key_b,
"different team_id must produce different cache keys"
);
}
#[test]
fn test_cache_key_fingerprint_matches_shared_request_fingerprint() {
let mut opts = ScrapeOptions {
needs_js: true,
..Default::default()
};
opts.headers
.insert("X-Trace".to_string(), "abc".to_string());
let key =
generate_scrape_cache_key(&make_ctx("https://example.com", HttpMethod::Get), &opts);
let expected = request_fingerprint(&opts);
assert!(
key.ends_with(&format!("?fp={:016x}", expected)),
"cache key fingerprint must reuse workers::request_identity::request_fingerprint"
);
}
#[test]
fn test_cache_key_ignores_cache_mode_control_switch() {
let ctx = make_ctx("https://example.com/cache-mode", HttpMethod::Get);
let baseline = generate_scrape_cache_key(
&ctx,
&ScrapeOptions {
cache_mode: None,
..Default::default()
},
);
for mode in [
CacheMode::Enabled,
CacheMode::Disabled,
CacheMode::ReadOnly,
CacheMode::Bypass,
] {
let opts = ScrapeOptions {
cache_mode: Some(mode),
..Default::default()
};
assert_eq!(
generate_scrape_cache_key(&ctx, &opts),
baseline,
"cache key must ignore cache_mode ({mode:?})"
);
}
}
#[test]
fn test_generate_scrape_cache_key_different_methods_produce_different_keys() {
let url = "https://example.com";
let opts = ScrapeOptions::default();
let get_key = generate_scrape_cache_key(&make_ctx(url, HttpMethod::Get), &opts);
let post_key = generate_scrape_cache_key(&make_ctx(url, HttpMethod::Post), &opts);
assert_ne!(get_key, post_key);
}
#[test]
fn test_generate_scrape_cache_key_different_urls_produce_different_keys() {
let opts = ScrapeOptions::default();
let key1 = generate_scrape_cache_key(&make_ctx("https://a.com", HttpMethod::Get), &opts);
let key2 = generate_scrape_cache_key(&make_ctx("https://b.com", HttpMethod::Get), &opts);
assert_ne!(key1, key2);
}
#[test]
fn test_cache_key_differs_by_headers() {
let ctx = make_ctx("https://example.com", HttpMethod::Get);
let mut opts_zh = ScrapeOptions::default();
opts_zh
.headers
.insert("Accept-Language".to_string(), "zh".to_string());
let mut opts_en = ScrapeOptions::default();
opts_en
.headers
.insert("Accept-Language".to_string(), "en".to_string());
let key_zh = generate_scrape_cache_key(&ctx, &opts_zh);
let key_en = generate_scrape_cache_key(&ctx, &opts_en);
assert_ne!(
key_zh, key_en,
"different headers must produce different cache keys"
);
}
#[test]
fn test_cache_key_same_team_differs_by_needs_js() {
let ctx = make_ctx("https://example.com", HttpMethod::Get);
let opts_js = ScrapeOptions {
needs_js: true,
..ScrapeOptions::default()
};
let opts_no_js = ScrapeOptions {
needs_js: false,
..ScrapeOptions::default()
};
let key_js = generate_scrape_cache_key(&ctx, &opts_js);
let key_no_js = generate_scrape_cache_key(&ctx, &opts_no_js);
assert_ne!(
key_js, key_no_js,
"needs_js=true vs false must produce different keys"
);
}
#[test]
fn test_cache_key_differs_by_session_id() {
let ctx = make_ctx("https://example.com", HttpMethod::Get);
let opts_s1 = ScrapeOptions {
session_id: Some("session-abc".to_string()),
..ScrapeOptions::default()
};
let opts_s2 = ScrapeOptions {
session_id: Some("session-xyz".to_string()),
..ScrapeOptions::default()
};
let key_s1 = generate_scrape_cache_key(&ctx, &opts_s1);
let key_s2 = generate_scrape_cache_key(&ctx, &opts_s2);
assert_ne!(
key_s1, key_s2,
"different session_id must produce different keys"
);
}
#[test]
fn test_cache_key_same_options_produce_same_key() {
let ctx = make_ctx("https://example.com", HttpMethod::Get);
let opts1 = ScrapeOptions::default();
let opts2 = ScrapeOptions::default();
let key1 = generate_scrape_cache_key(&ctx, &opts1);
let key2 = generate_scrape_cache_key(&ctx, &opts2);
assert_eq!(key1, key2);
}
#[test]
fn test_cache_key_headers_order_independent() {
let ctx = make_ctx("https://example.com", HttpMethod::Get);
let mut opts_a = ScrapeOptions::default();
opts_a
.headers
.insert("X-Custom".to_string(), "v1".to_string());
opts_a
.headers
.insert("Accept".to_string(), "json".to_string());
let mut opts_b = ScrapeOptions::default();
opts_b
.headers
.insert("Accept".to_string(), "json".to_string());
opts_b
.headers
.insert("X-Custom".to_string(), "v1".to_string());
let key_a = generate_scrape_cache_key(&ctx, &opts_a);
let key_b = generate_scrape_cache_key(&ctx, &opts_b);
assert_eq!(key_a, key_b, "headers order must not affect cache key");
}
#[test]
fn test_filter_sensitive_headers_removes_known_sensitive() {
let mut headers = HashMap::new();
headers.insert("Set-Cookie".to_string(), "session=abc".to_string());
headers.insert("Authorization".to_string(), "Bearer xyz".to_string());
headers.insert("X-API-Key".to_string(), "key123".to_string());
headers.insert("Content-Type".to_string(), "text/html".to_string());
filter_sensitive_headers(&mut headers);
assert!(!headers.contains_key("Set-Cookie"));
assert!(!headers.contains_key("Authorization"));
assert!(!headers.contains_key("X-API-Key"));
assert!(headers.contains_key("Content-Type"));
}
#[test]
fn test_filter_sensitive_headers_case_insensitive() {
let mut headers = HashMap::new();
headers.insert("set-cookie".to_string(), "v".to_string());
headers.insert("SET-COOKIE".to_string(), "v".to_string());
headers.insert("Set-Cookie".to_string(), "v".to_string());
filter_sensitive_headers(&mut headers);
assert!(headers.is_empty(), "all case variants should be removed");
}
#[test]
fn test_filter_sensitive_headers_empty_map_no_op() {
let mut headers = HashMap::new();
filter_sensitive_headers(&mut headers);
assert!(headers.is_empty());
}
#[test]
fn test_sanitized_serialize_skips_sensitive_headers() {
let mut headers = HashMap::new();
headers.insert("Set-Cookie".to_string(), "secret".to_string());
headers.insert("Content-Type".to_string(), "text/html".to_string());
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
status_code: 200,
content: "<html></html>".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers,
response_time_ms: 42,
final_url: Some("https://example.com".to_string()),
markdown: None,
};
let sanitized = SanitizedScrapeResponse::from_response(&response);
let json = serde_json::to_string(&sanitized).expect("serialize should succeed");
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
let headers_obj = parsed["headers"]
.as_object()
.expect("headers should be object");
assert!(!headers_obj.contains_key("Set-Cookie"));
assert!(headers_obj.contains_key("Content-Type"));
}
#[test]
fn test_sanitized_serialize_round_trip_restores_response() {
let mut headers = HashMap::new();
headers.insert("Content-Type".to_string(), "text/html".to_string());
headers.insert("X-Trace-Id".to_string(), "abc-123".to_string());
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
status_code: 200,
content: "hello".to_string(),
screenshot: Some("base64data".to_string()),
content_type: "text/html".to_string(),
headers,
response_time_ms: 100,
final_url: None,
markdown: Some("# Title".to_string()),
};
let sanitized = SanitizedScrapeResponse::from_response(&response);
let json = serde_json::to_string(&sanitized).unwrap();
let restored: ScrapeResponse = serde_json::from_str(&json).unwrap();
assert_eq!(restored.status_code, 200);
assert_eq!(restored.content, "hello");
assert_eq!(restored.screenshot.as_deref(), Some("base64data"));
assert_eq!(restored.content_type, "text/html");
assert_eq!(restored.headers.len(), 2);
assert_eq!(
restored.headers.get("Content-Type").map(|v| v.as_str()),
Some("text/html")
);
assert_eq!(restored.response_time_ms, 100);
assert_eq!(restored.markdown.as_deref(), Some("# Title"));
assert!(restored.captured_responses.is_empty());
}
#[test]
fn test_sanitized_roundtrip_preserves_captured_responses() {
let response = ScrapeResponse {
js_degraded: None,
captured_responses: vec![CapturedResponse {
url: "https://example.com/api/data".to_string(),
status: 200,
mime_type: "application/json".to_string(),
body: "{\"ok\":true}".to_string(),
}],
status_code: 200,
content: "hello".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
markdown: None,
};
let sanitized = SanitizedScrapeResponse::from_response(&response);
let json = serde_json::to_string(&sanitized).unwrap();
let restored: ScrapeResponse = serde_json::from_str(&json).unwrap();
assert_eq!(restored.captured_responses.len(), 1);
assert_eq!(
restored.captured_responses[0].url,
"https://example.com/api/data"
);
assert_eq!(restored.captured_responses[0].status, 200);
assert_eq!(restored.captured_responses[0].body, "{\"ok\":true}");
}
#[test]
fn test_sanitized_serialize_does_not_modify_original() {
let mut headers = HashMap::new();
headers.insert("Set-Cookie".to_string(), "secret".to_string());
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
status_code: 200,
content: "test".to_string(),
screenshot: None,
content_type: "text/plain".to_string(),
headers,
response_time_ms: 1,
final_url: None,
markdown: None,
};
let sanitized = SanitizedScrapeResponse::from_response(&response);
let _ = serde_json::to_string(&sanitized).unwrap();
assert_eq!(response.headers.len(), 1);
assert!(response.headers.contains_key("Set-Cookie"));
}
}