use chrono::NaiveDateTime;
use serde::{Deserialize, Serialize};
use serde_json::Value;
const SENSITIVE_HEADERS: &[&str] = &[
"set-cookie",
"authorization",
"x-auth-token",
"x-api-key",
"proxy-authorization",
"cookie",
];
#[derive(Debug, Deserialize, Serialize)]
pub struct ScrapeResponseDto {
pub id: crate::domain::id::Id,
pub url: String,
#[serde(default)]
pub credits_used: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub captured_responses: Option<Vec<CapturedResponseDto>>,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct CapturedResponseDto {
pub url: String,
pub status: u16,
pub mime_type: String,
pub body: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ScrapeResultDto {
pub content: String,
pub status_code: u16,
pub content_type: Option<String>,
pub response_time_ms: i64,
pub headers: Option<Value>,
pub meta_data: Option<Value>,
pub screenshot: Option<String>,
pub created_at: NaiveDateTime,
}
impl ScrapeResultDto {
pub fn filter_sensitive_headers(&mut self) {
if let Some(Value::Object(ref mut map)) = self.headers {
let sensitive: Vec<String> = map
.keys()
.filter(|key| {
SENSITIVE_HEADERS
.iter()
.any(|s| s.eq_ignore_ascii_case(key))
})
.cloned()
.collect();
for key in sensitive {
map.insert(key, Value::String("[REDACTED]".to_string()));
}
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ScrapeStatusResponseDto {
pub id: crate::domain::id::Id,
pub status: String,
pub url: String,
pub created_at: NaiveDateTime,
pub completed_at: Option<NaiveDateTime>,
pub result: Option<ScrapeResultDto>,
pub metadata: Option<Value>,
pub error: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CancelScrapeResponseDto {
pub message: String,
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn redacts_sensitive_headers_regardless_of_case() {
let mut dto = ScrapeResultDto {
content: String::new(),
status_code: 200,
content_type: None,
response_time_ms: 0,
headers: Some(json!({
"Set-Cookie": "session=abc",
"X-Api-Key": "k-123",
"AUTHORIZATION": "Bearer t",
"Content-Type": "text/html",
})),
meta_data: None,
screenshot: None,
created_at: NaiveDateTime::default(),
};
dto.filter_sensitive_headers();
let map = dto.headers.as_ref().unwrap().as_object().unwrap();
assert_eq!(map.get("Set-Cookie").unwrap(), "[REDACTED]");
assert_eq!(map.get("X-Api-Key").unwrap(), "[REDACTED]");
assert_eq!(map.get("AUTHORIZATION").unwrap(), "[REDACTED]");
assert_eq!(map.get("Content-Type").unwrap(), "text/html");
}
#[test]
fn redacts_headers_when_no_headers_present() {
let mut dto = ScrapeResultDto {
content: String::new(),
status_code: 200,
content_type: None,
response_time_ms: 0,
headers: None,
meta_data: None,
screenshot: None,
created_at: NaiveDateTime::default(),
};
dto.filter_sensitive_headers();
assert!(dto.headers.is_none());
}
}