use crate::common::CacheMode;
use log::warn;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::time::Duration;
pub use crate::common::HttpMethod;
#[derive(Debug, Clone)]
pub struct ScrapeRequest {
pub url: String,
pub options: ScrapeOptions,
}
impl ScrapeRequest {
pub fn new(url: impl Into<String>) -> Self {
Self {
url: url.into(),
options: ScrapeOptions::default(),
}
}
pub fn with_options(mut self, options: ScrapeOptions) -> Self {
self.options = options;
self
}
pub fn needs_js(mut self) -> Self {
self.options.needs_js = true;
self
}
pub fn needs_screenshot(mut self) -> Self {
self.options.needs_screenshot = true;
self
}
pub fn mobile(mut self) -> Self {
self.options.mobile = true;
self
}
pub fn timeout(mut self, duration: Duration) -> Self {
self.options.timeout = duration;
self
}
pub fn needs_mllm(mut self) -> Self {
self.options.needs_mllm = true;
self
}
}
pub const MAX_SESSION_ID_LEN: usize = 128;
#[must_use]
pub fn validate_session_id(session_id: &str) -> bool {
if session_id.len() > MAX_SESSION_ID_LEN {
return false;
}
session_id.bytes().all(|b| (0x20..=0x7E).contains(&b))
}
#[derive(Debug, Clone, PartialEq, Default, Serialize)]
pub enum WaitFor {
#[default]
NetworkIdle,
Selector(String),
DomStable(Duration),
ResponseUrl(String),
ResponseBody(String),
}
#[derive(Debug, Clone, Serialize)]
pub struct ScrapeOptions {
pub method: HttpMethod,
pub needs_js: bool,
pub needs_screenshot: bool,
pub mobile: bool,
pub timeout: Duration,
pub body: Option<String>,
pub sync_wait_ms: u32,
pub actions: Vec<PageAction>,
pub screenshot_config: Option<ScreenshotConfig>,
pub proxy: Option<String>,
pub skip_tls_verification: bool,
pub headers: HashMap<String, String>,
pub needs_tls_fingerprint: bool,
pub use_fire_engine: bool,
pub block_ads: bool,
pub block_media: bool,
pub session_id: Option<String>,
pub cache_mode: Option<CacheMode>,
pub wait_for: Option<WaitFor>,
pub needs_mllm: bool,
}
impl Default for ScrapeOptions {
fn default() -> Self {
Self {
method: HttpMethod::Get,
needs_js: false,
needs_screenshot: false,
mobile: false,
timeout: Duration::from_secs(30),
body: None,
sync_wait_ms: 0,
actions: Vec::new(),
screenshot_config: None,
proxy: None,
skip_tls_verification: false,
headers: HashMap::new(),
needs_tls_fingerprint: false,
use_fire_engine: false,
block_ads: false,
block_media: false,
session_id: None,
cache_mode: None,
wait_for: None,
needs_mllm: false,
}
}
}
impl ScrapeOptions {
pub fn builder() -> ScrapeOptionsBuilder {
ScrapeOptionsBuilder::default()
}
}
#[derive(Debug, Clone, Default)]
pub struct ScrapeOptionsBuilder(ScrapeOptions);
impl ScrapeOptionsBuilder {
pub fn method(mut self, method: HttpMethod) -> Self {
self.0.method = method;
self
}
pub fn needs_js(mut self, enabled: bool) -> Self {
self.0.needs_js = enabled;
self
}
pub fn needs_screenshot(mut self, enabled: bool) -> Self {
self.0.needs_screenshot = enabled;
self
}
pub fn mobile(mut self, enabled: bool) -> Self {
self.0.mobile = enabled;
self
}
pub fn timeout(mut self, duration: Duration) -> Self {
self.0.timeout = duration;
self
}
pub fn body(mut self, body: impl Into<String>) -> Self {
self.0.body = Some(body.into());
self
}
pub fn sync_wait_ms(mut self, ms: u32) -> Self {
self.0.sync_wait_ms = ms;
self
}
pub fn proxy(mut self, proxy: impl Into<String>) -> Self {
self.0.proxy = Some(proxy.into());
self
}
pub fn skip_tls_verification(mut self, skip: bool) -> Self {
if skip {
let env = std::env::var("APP_ENVIRONMENT")
.or_else(|_| std::env::var("CRAWLRS_ENV"))
.unwrap_or_else(|_| "development".to_string());
let is_production =
env.eq_ignore_ascii_case("production") || env.eq_ignore_ascii_case("prod");
if is_production {
warn!(
target: "security",
"SECURITY ALERT: Attempt to skip TLS verification in production environment '{}' - DENIED. \
TLS verification will remain enabled to prevent man-in-the-middle attacks.",
env
);
return self;
}
warn!(
target: "security",
"TLS certificate verification disabled in '{}' environment. \
This should ONLY be used for testing purposes. \
NEVER disable TLS verification in production as it enables man-in-the-middle attacks.",
env
);
}
self.0.skip_tls_verification = skip;
self
}
pub fn headers(mut self, headers: HashMap<String, String>) -> Self {
self.0.headers = headers;
self
}
pub fn needs_tls_fingerprint(mut self, enabled: bool) -> Self {
self.0.needs_tls_fingerprint = enabled;
self
}
pub fn use_fire_engine(mut self, enabled: bool) -> Self {
self.0.use_fire_engine = enabled;
self
}
pub fn screenshot_config(mut self, config: ScreenshotConfig) -> Self {
self.0.screenshot_config = Some(config);
self
}
pub fn block_ads(mut self, enabled: bool) -> Self {
self.0.block_ads = enabled;
self
}
pub fn block_media(mut self, enabled: bool) -> Self {
self.0.block_media = enabled;
self
}
pub fn session_id(mut self, session_id: impl Into<String>) -> Self {
let sid = session_id.into();
if validate_session_id(&sid) {
self.0.session_id = Some(sid);
} else {
warn!(
"session_id rejected (length={}, max={}, must be printable ASCII); \
falling back to None (RoundRobin)",
sid.len(),
MAX_SESSION_ID_LEN
);
}
self
}
pub fn build(self) -> ScrapeOptions {
self.0
}
pub fn needs_mllm(mut self, enabled: bool) -> Self {
self.0.needs_mllm = enabled;
self
}
pub fn cache_mode(mut self, mode: impl Into<Option<CacheMode>>) -> Self {
self.0.cache_mode = mode.into();
self
}
#[must_use]
pub fn wait_for(mut self, wait: impl Into<Option<WaitFor>>) -> Self {
self.0.wait_for = wait.into();
self
}
}
#[derive(Debug, Clone, Serialize)]
pub enum PageAction {
Wait { milliseconds: u64 },
Click { selector: String },
Scroll { direction: ScrollDirection },
Input { selector: String, text: String },
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize)]
pub enum ScrollDirection {
#[default]
Down,
Up,
Bottom,
Top,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct ScreenshotConfig {
pub full_page: bool,
pub selector: Option<String>,
pub quality: Option<u8>,
pub format: Option<String>,
}
impl Default for ScreenshotConfig {
fn default() -> Self {
Self {
full_page: true,
selector: None,
quality: Some(80),
format: Some("jpeg".to_string()),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CapturedResponse {
pub url: String,
pub status: u16,
pub mime_type: String,
pub body: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ScrapeResponse {
pub status_code: u16,
pub content: String,
pub screenshot: Option<String>,
pub content_type: String,
pub headers: HashMap<String, String>,
pub response_time_ms: u64,
pub final_url: Option<String>,
pub markdown: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub captured_responses: Vec<CapturedResponse>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub js_degraded: Option<bool>,
}
impl ScrapeResponse {
pub fn new(
status_code: u16,
content: impl Into<String>,
content_type: impl Into<String>,
) -> Self {
Self {
status_code,
content: content.into(),
screenshot: None,
content_type: content_type.into(),
headers: HashMap::new(),
response_time_ms: 0,
final_url: None,
markdown: None,
captured_responses: Vec::new(),
js_degraded: None,
}
}
pub fn is_success(&self) -> bool {
(200..300).contains(&self.status_code)
}
}
#[derive(Debug, Clone)]
pub struct InternalScrapeRequest {
pub url: String,
pub method: HttpMethod,
pub headers: HashMap<String, String>,
pub timeout: Duration,
pub needs_js: bool,
pub needs_screenshot: bool,
pub screenshot_config: Option<InternalScreenshotConfig>,
pub mobile: bool,
pub proxy: Option<String>,
pub skip_tls_verification: bool,
pub needs_tls_fingerprint: bool,
pub use_fire_engine: bool,
pub actions: Vec<InternalPageAction>,
pub body: Option<String>,
pub sync_wait_ms: u32,
pub block_ads: bool,
pub block_media: bool,
pub session_id: Option<String>,
pub wait_for: Option<WaitFor>,
pub needs_mllm: bool,
}
#[derive(Debug, Clone)]
pub struct InternalScreenshotConfig {
pub full_page: bool,
pub selector: Option<String>,
pub quality: Option<u8>,
pub format: Option<String>,
}
#[derive(Debug, Clone)]
pub enum InternalPageAction {
Wait { milliseconds: u64 },
Click { selector: String },
Scroll { direction: String },
Input { selector: String, text: String },
Screenshot { full_page: bool },
}
#[derive(Debug, Clone, Default)]
pub struct InternalScrapeResponse {
pub status_code: u16,
pub content: String,
pub screenshot: Option<String>,
pub content_type: String,
pub headers: HashMap<String, String>,
pub response_time_ms: u64,
pub final_url: Option<String>,
pub captured_responses: Vec<InternalCapturedResponse>,
pub js_degraded: bool,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct InternalCapturedResponse {
pub url: String,
pub status: u16,
pub mime_type: String,
pub body: String,
}
impl ScrapeRequest {
#[inline]
pub(crate) fn to_internal(
&self,
) -> Result<InternalScrapeRequest, crate::engines::engine_client::EngineError> {
if let Some(crate::engines::engine_client::WaitFor::ResponseUrl(pattern)) =
&self.options.wait_for
{
if let Err(e) = regex::Regex::new(pattern) {
return Err(crate::engines::engine_client::EngineError::InvalidUrl(
format!("wait_for_response_url regex invalid: {}", e),
));
}
}
let options = &self.options;
let internal = InternalScrapeRequest {
url: self.url.clone(),
method: options.method,
headers: options.headers.clone(),
timeout: options.timeout,
needs_js: options.needs_js,
needs_screenshot: options.needs_screenshot,
screenshot_config: options
.screenshot_config
.as_ref()
.map(convert_screenshot_config),
mobile: options.mobile,
proxy: options.proxy.clone(),
skip_tls_verification: options.skip_tls_verification,
needs_tls_fingerprint: options.needs_tls_fingerprint,
use_fire_engine: options.use_fire_engine,
actions: convert_page_actions(&options.actions),
body: options.body.clone(),
sync_wait_ms: options.sync_wait_ms,
block_ads: options.block_ads,
block_media: options.block_media,
session_id: resolve_session_id(&options.session_id),
wait_for: options.wait_for.clone(),
needs_mllm: options.needs_mllm,
};
Ok(internal)
}
}
fn convert_page_actions(actions: &[PageAction]) -> Vec<InternalPageAction> {
actions
.iter()
.map(|action| match action {
PageAction::Wait { milliseconds } => InternalPageAction::Wait {
milliseconds: *milliseconds,
},
PageAction::Click { selector } => InternalPageAction::Click {
selector: selector.clone(),
},
PageAction::Scroll { direction } => {
let direction_str = match direction {
ScrollDirection::Down => "down",
ScrollDirection::Up => "up",
ScrollDirection::Bottom => "bottom",
ScrollDirection::Top => "top",
};
InternalPageAction::Scroll {
direction: direction_str.to_string(),
}
}
PageAction::Input { selector, text } => InternalPageAction::Input {
selector: selector.clone(),
text: text.clone(),
},
})
.collect()
}
fn convert_screenshot_config(config: &ScreenshotConfig) -> InternalScreenshotConfig {
InternalScreenshotConfig {
full_page: config.full_page,
selector: config.selector.clone(),
quality: config.quality,
format: config.format.clone(),
}
}
fn resolve_session_id(session_id: &Option<String>) -> Option<String> {
match session_id {
Some(sid) if validate_session_id(sid) => Some(sid.clone()),
Some(sid) => {
warn!(
"session_id rejected in to_internal (length={}, max={}, \
must be printable ASCII); falling back to None (RoundRobin)",
sid.len(),
MAX_SESSION_ID_LEN
);
None
}
None => None,
}
}
impl InternalScrapeResponse {
#[inline]
pub fn to_public(&self, _original_url: &str) -> ScrapeResponse {
ScrapeResponse {
status_code: self.status_code,
content: self.content.clone(),
screenshot: self.screenshot.clone(),
content_type: self.content_type.clone(),
headers: self.headers.clone(),
response_time_ms: self.response_time_ms,
final_url: self.final_url.clone(),
markdown: None,
captured_responses: self
.captured_responses
.iter()
.map(|c| CapturedResponse {
url: c.url.clone(),
status: c.status,
mime_type: c.mime_type.clone(),
body: c.body.clone(),
})
.collect(),
js_degraded: if self.js_degraded { Some(true) } else { None },
}
}
}