use anyhow::{Context, Result};
use chrono::Utc;
use dashmap::DashMap;
use log::{debug, error, info, warn};
use serde_json::{json, Value};
use std::collections::HashMap;
use std::sync::atomic::{AtomicI64, Ordering};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::OwnedSemaphorePermit;
use tokio::time::sleep;
use crate::application::dto::crawl_request::CrawlConfigDto;
use crate::application::dto::scrape_request::{ScrapeOptionsDto, ScrapeRequestDto};
use crate::application::use_cases::create_scrape::CreateScrapeUseCaseTrait;
use crate::config::settings::Settings;
use crate::domain::id::Id;
use crate::domain::models::{Task, TaskStatus};
use crate::domain::repositories::crawl_repository::CrawlRepository;
use crate::domain::repositories::credits_repository::CreditsRepository;
use crate::domain::repositories::scrape_result_repository::ScrapeResultRepository;
use crate::domain::repositories::task_repository::TaskRepository;
use crate::domain::services::extraction_service::ExtractionServiceTrait;
use crate::domain::services::retry_handler::RetryHandler;
use crate::domain::services::webhook_service::WebhookService;
use crate::utils::dedup::Deduplicator;
use crate::common::CacheContext;
use crate::common::CacheMode;
use crate::common::HttpMethod;
use crate::domain::services::team_semaphore::TeamSemaphore;
use crate::engines::engine_client::{
EngineClient, EngineError, PageAction, ScrapeOptions, ScrapeRequest, ScrapeResponse,
ScreenshotConfig, ScrollDirection, WaitFor,
};
use crate::infrastructure::oxcache::CacheService;
use crate::infrastructure::security::ssrf::is_internal_url;
use crate::queue::task_queue::TaskQueue;
use crate::workers::coalesce_coordinator::CoalesceCoordinator;
use crate::workers::shutdown::ShutdownCoordinator;
use crate::workers::cache_utils;
use crate::workers::coalesce_coordinator::CoalesceKey;
use crate::utils::url_redaction::redact_url_for_log;
use crate::utils::retry_policy::RetryPolicy;
#[cfg(feature = "content")]
use crate::workers::markdown_post_processor::MarkdownPostProcessor;
use crate::utils::retry::{RetryDirective, RetryTracker};
use crate::utils::robots::RobotsCheckerTrait;
#[cfg(feature = "metrics")]
use crate::workers::scheduler::memory_scheduler::MemoryScheduler;
pub(super) use super::crawl_link_extractor::{
check_robots_txt as check_robots_txt_fn, extract_and_queue_links as extract_and_queue_links_fn,
update_crawl_completion_status as update_crawl_completion_status_fn,
};
pub(super) use super::scrape_executor::{
process_text_encoding, save_result, try_read_scrape_cache, try_write_scrape_cache,
};
pub(super) use super::scrape_response_builder::{
build_crawl_request as build_crawl_request_fn,
build_extract_request as build_extract_request_fn, parse_crawl_payload, parse_extract_payload,
};
#[cfg(test)]
use crate::application::dto::extract_request::ExtractRequestDto;
pub struct ScrapeWorker {
repository: Arc<dyn TaskRepository>,
result_repository: Arc<dyn ScrapeResultRepository>,
crawl_repository: Arc<dyn CrawlRepository>,
webhook_service: Arc<dyn WebhookService>,
credits_repository: Arc<dyn CreditsRepository>,
engine_client: Arc<EngineClient>,
_create_scrape_use_case: Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore: Arc<TeamSemaphore>,
coalesce_coordinator: Arc<CoalesceCoordinator>,
#[cfg(feature = "content")]
markdown_post_processor: MarkdownPostProcessor,
token_usage: Arc<DashMap<crate::domain::id::Id, AtomicI64>>,
robots_checker: Arc<dyn RobotsCheckerTrait>,
settings: Arc<Settings>,
worker_id: crate::domain::id::Id,
default_concurrency_limit: usize,
retry_handler: RetryHandler,
extraction_service: Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "metrics")]
memory_scheduler: Arc<MemoryScheduler>,
deduplicator: Arc<parking_lot::RwLock<Deduplicator>>,
crawl_terminal_cache: DashMap<crate::domain::id::Id, (bool, std::time::Instant)>,
crawl_terminal_cache_inserts: std::sync::atomic::AtomicUsize,
crawl_blacklist_cache:
DashMap<crate::domain::id::Id, (std::sync::Arc<Vec<String>>, std::time::Instant)>,
crawl_blacklist_cache_inserts: std::sync::atomic::AtomicUsize,
cache_service: Arc<dyn CacheService>,
shutdown_coordinator: Arc<ShutdownCoordinator>,
}
impl std::fmt::Debug for ScrapeWorker {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ScrapeWorker")
.field("worker_id", &self.worker_id)
.field("default_concurrency_limit", &self.default_concurrency_limit)
.finish_non_exhaustive()
}
}
impl ScrapeWorker {
pub fn worker_id(&self) -> crate::domain::id::Id {
self.worker_id
}
pub fn new(deps: ScrapeWorkerDeps) -> Self {
let retry_policy = RetryPolicy::slow(); let retry_handler = RetryHandler::new(deps.repository.clone(), retry_policy.clone());
Self {
repository: deps.repository,
result_repository: deps.result_repository,
crawl_repository: deps.crawl_repository,
webhook_service: deps.webhook_service,
credits_repository: deps.credits_repository,
engine_client: deps.engine_client,
_create_scrape_use_case: deps.create_scrape_use_case,
team_semaphore: deps.team_semaphore,
coalesce_coordinator: deps.coalesce_coordinator,
#[cfg(feature = "content")]
markdown_post_processor: MarkdownPostProcessor::new(
crate::domain::services::markdown::build_converter(&deps.settings.markdown),
Some(Arc::clone(&deps.content_extractor)),
),
token_usage: Arc::new(DashMap::new()),
robots_checker: deps.robots_checker,
settings: deps.settings,
worker_id: crate::domain::id::Id::generate(),
default_concurrency_limit: deps.default_concurrency_limit,
retry_handler,
extraction_service: deps.extraction_service,
#[cfg(feature = "metrics")]
memory_scheduler: deps.memory_scheduler,
deduplicator: Arc::new(parking_lot::RwLock::new(Deduplicator::new())),
crawl_terminal_cache: DashMap::new(),
crawl_terminal_cache_inserts: std::sync::atomic::AtomicUsize::new(0),
crawl_blacklist_cache: DashMap::with_capacity(1024),
crawl_blacklist_cache_inserts: std::sync::atomic::AtomicUsize::new(0),
cache_service: deps.cache_service,
shutdown_coordinator: Arc::new(ShutdownCoordinator::default()),
}
}
pub fn with_shutdown_coordinator(mut self, coordinator: Arc<ShutdownCoordinator>) -> Self {
self.shutdown_coordinator = coordinator;
self
}
pub async fn run(&self, queue: Arc<dyn TaskQueue>) {
info!("Scrape worker {} started", self.worker_id);
loop {
if self.shutdown_coordinator.is_shutting_down() {
info!(
"Scrape worker {} received shutdown, exiting after current task",
self.worker_id
);
break;
}
match self.process_next_task(&queue).await {
Ok(processed) => {
if !processed {
sleep(Duration::from_secs(1)).await;
}
}
Err(e) => {
error!("Error processing task: {}", e);
sleep(Duration::from_secs(1)).await;
}
}
}
}
pub(crate) fn with_deduplicator_opt(
mut self,
dedup: Option<Arc<parking_lot::RwLock<Deduplicator>>>,
) -> Self {
if let Some(d) = dedup {
self.deduplicator = d;
}
self
}
#[cfg(test)]
pub(crate) fn deduplicator_for_test(&self) -> Arc<parking_lot::RwLock<Deduplicator>> {
self.deduplicator.clone()
}
async fn process_next_task(&self, queue: &dyn TaskQueue) -> Result<bool> {
let task_opt = queue.dequeue(self.worker_id).await?;
if let Some(task) = task_opt {
self.process_task(task).await?;
return Ok(true);
}
Ok(false)
}
fn acquire_concurrency_permit(&self, task: &Task) -> Option<OwnedSemaphorePermit> {
self.team_semaphore.try_acquire(task.team_id)
}
async fn process_task(&self, task: Task) -> Result<()> {
debug!(
"process_task: task_id={}, url={}, task_type={}",
task.id, task.url, task.task_type
);
let _lock_heartbeat = crate::workers::lock_heartbeat::LockHeartbeat::start(
self.repository.clone(),
task.id,
self.worker_id,
self.settings.concurrency.task_lock_duration_seconds,
);
info!("Processing task");
if self.handle_expired_task(&task).await? {
return Ok(());
}
#[cfg(feature = "metrics")]
if self.admit_task_by_memory(&task).await? {
return Ok(());
}
let _permit = match self.acquire_permit_or_reschedule(&task).await? {
Some(p) => p,
None => return Ok(()),
};
let task_type = task.task_type;
let result = match task_type.as_str() {
"scrape" => self.process_scrape_task(task).await,
"crawl" => self.process_crawl_task(task).await,
"extract" => self.process_extract_task(task).await,
_ => return Err(anyhow::anyhow!("Unknown task type: {}", task_type)),
};
if let Err(ref e) = result {
debug!("error: {}", e);
} else {
debug!("Task processing completed successfully");
}
result
}
pub(crate) fn parse_scrape_request_dto(task: &Task) -> Result<ScrapeRequestDto> {
serde_json::from_value(task.payload.clone()).context("Failed to parse task payload")
}
pub(crate) fn build_scrape_request_from_dto(dto: &ScrapeRequestDto) -> Result<ScrapeRequest> {
let options = dto.options.as_ref();
let headers = build_dto_headers(options);
let needs_js = dto_needs_js(dto, options);
let screenshot_config = build_dto_screenshot_config(options);
Ok(ScrapeRequest {
url: dto.url.clone(),
options: ScrapeOptions {
method: HttpMethod::Get,
body: None,
headers,
timeout: Duration::from_secs(options.and_then(|o| o.timeout).unwrap_or(30)),
needs_js,
needs_screenshot: options.and_then(|o| o.screenshot).unwrap_or(false),
screenshot_config,
mobile: options.and_then(|o| o.mobile).unwrap_or(false),
proxy: options.and_then(|o| o.proxy.clone()),
skip_tls_verification: options
.and_then(|o| o.skip_tls_verification)
.unwrap_or(false),
needs_tls_fingerprint: options
.and_then(|o| o.needs_tls_fingerprint)
.unwrap_or(false),
use_fire_engine: options.and_then(|o| o.use_fire_engine).unwrap_or(false),
actions: map_dto_actions(dto),
sync_wait_ms: dto.sync_wait_ms.unwrap_or(0),
needs_mllm: resolve_needs_mllm(options)?,
block_ads: options.and_then(|o| o.block_ads).unwrap_or(false),
block_media: options.and_then(|o| o.block_media).unwrap_or(false),
session_id: None,
cache_mode: map_dto_cache_mode(options),
wait_for: map_wait_for(options.and_then(|o| o.wait_for)),
},
})
}
pub fn build_scrape_request(task: &Task) -> Result<ScrapeRequest> {
let dto = Self::parse_scrape_request_dto(task)?;
Self::build_scrape_request_from_dto(&dto)
}
}
fn build_dto_headers(options: Option<&ScrapeOptionsDto>) -> HashMap<String, String> {
let mut headers = HashMap::with_capacity(16);
if let Some(opts) = options {
if let Some(h) = &opts.headers {
if let Some(obj) = h.as_object() {
for (k, v) in obj {
if k.eq_ignore_ascii_case("host") {
log::warn!(
"ignoring custom Host header in scrape options (target host is fixed by URL)"
);
continue;
}
if let Some(s) = v.as_str() {
headers.insert(k.clone(), s.to_string());
}
}
}
}
}
headers
}
fn dto_needs_js(dto: &ScrapeRequestDto, options: Option<&ScrapeOptionsDto>) -> bool {
dto.actions.as_ref().map(|a| !a.is_empty()).unwrap_or(false)
|| options.and_then(|o| o.js_rendering).unwrap_or(false)
}
fn build_dto_screenshot_config(options: Option<&ScrapeOptionsDto>) -> Option<ScreenshotConfig> {
options.and_then(|o| {
o.screenshot_options.as_ref().map(|so| ScreenshotConfig {
full_page: so.full_page.unwrap_or(false),
selector: so.selector.clone(),
quality: so.quality,
format: so.format.clone(),
})
})
}
fn map_dto_actions(dto: &ScrapeRequestDto) -> Vec<PageAction> {
dto.actions
.clone()
.unwrap_or_default()
.into_iter()
.filter_map(|a| match a {
crate::application::dto::scrape_request::ScrapeActionDto::Wait { milliseconds } => {
Some(PageAction::Wait { milliseconds })
}
crate::application::dto::scrape_request::ScrapeActionDto::Click { selector } => {
Some(PageAction::Click { selector })
}
crate::application::dto::scrape_request::ScrapeActionDto::Scroll { direction } => {
let dir = map_scroll_direction(&direction);
Some(PageAction::Scroll { direction: dir })
}
crate::application::dto::scrape_request::ScrapeActionDto::Screenshot { .. } => {
None
}
crate::application::dto::scrape_request::ScrapeActionDto::Input { selector, text } => {
Some(PageAction::Input { selector, text })
}
})
.collect()
}
fn map_scroll_direction(direction: &str) -> ScrollDirection {
match direction.to_lowercase().as_str() {
"up" => ScrollDirection::Up,
"top" => ScrollDirection::Top,
"bottom" => ScrollDirection::Bottom,
_ => ScrollDirection::Down,
}
}
fn resolve_needs_mllm(options: Option<&ScrapeOptionsDto>) -> Result<bool> {
let requested = options.and_then(|o| o.needs_mllm).unwrap_or(false);
if requested {
#[cfg(not(feature = "engine-mllm"))]
anyhow::bail!("task requested needs_mllm but this build lacks the `engine-mllm` feature");
#[cfg(feature = "engine-mllm")]
log::debug!("task requests MLLM analysis");
}
Ok(requested)
}
fn map_dto_cache_mode(options: Option<&ScrapeOptionsDto>) -> Option<CacheMode> {
let bypass = options.and_then(|o| o.bypass_cache).unwrap_or(false);
if bypass {
Some(CacheMode::Bypass)
} else {
options.and_then(|o| o.cache_mode)
}
}
fn extract_cache_validators(headers: &serde_json::Value) -> Option<(String, Option<String>)> {
let map = headers.as_object()?;
let lookup = |name: &str| {
map.iter()
.find(|(k, _)| k.eq_ignore_ascii_case(name))
.and_then(|(_, v)| v.as_str())
.map(str::to_string)
};
let etag = lookup("etag");
let last_modified = lookup("last-modified");
if etag.is_some() || last_modified.is_some() {
Some((etag.unwrap_or_default(), last_modified))
} else {
None
}
}
fn not_modified_response(
prior: &crate::domain::models::ScrapeResult,
response: &ScrapeResponse,
) -> ScrapeResponse {
let mut headers = response.headers.clone();
headers.insert("x-crawlrs-not-modified".to_string(), "true".to_string());
for (k, v) in prior.headers.as_object().into_iter().flatten() {
if let Some(v_str) = v.as_str() {
headers
.entry(k.clone())
.or_insert_with(|| v_str.to_string());
}
}
ScrapeResponse {
js_degraded: None,
content: prior.content.clone(),
status_code: 200,
screenshot: None,
content_type: prior.content_type.clone(),
headers,
response_time_ms: response.response_time_ms,
final_url: response.final_url.clone(),
markdown: None,
captured_responses: Vec::new(),
}
}
#[cfg(test)]
mod incremental_scrape_tests {
use super::*;
use serde_json::json;
#[test]
fn extract_validators_reads_etag_and_last_modified_case_insensitive() {
let headers =
json!({"ETag": "\"abc123\"", "Last-Modified": "Wed, 24 Sep 2026 08:00:00 GMT"});
let (etag, lm) = extract_cache_validators(&headers).expect("validators present");
assert_eq!(etag, "\"abc123\"");
assert_eq!(lm.as_deref(), Some("Wed, 24 Sep 2026 08:00:00 GMT"));
let headers = json!({"etag": "\"x\"", "last-modified": "Wed, 24 Sep 2026 09:00:00 GMT"});
let (etag, _) = extract_cache_validators(&headers).expect("lowercase keys");
assert_eq!(etag, "\"x\"");
}
#[test]
fn extract_validators_none_when_missing() {
assert!(extract_cache_validators(&json!({"content-type": "text/html"})).is_none());
assert!(extract_cache_validators(&serde_json::Value::Null).is_none());
assert!(extract_cache_validators(&json!("not an object")).is_none());
}
#[test]
fn not_modified_response_reuses_prior_content_with_marker() {
use crate::domain::models::ScrapeResult;
let prior = ScrapeResult {
id: crate::domain::id::Id::generate(),
task_id: crate::domain::id::Id::generate(),
url: "https://example.com/a".to_string(),
status_code: 200,
content: "<html>original body</html>".to_string(),
content_type: "text/html".to_string(),
headers: json!({"etag": "\"v1\"", "content-type": "text/html"}),
meta_data: serde_json::Value::Null,
screenshot: None,
response_time_ms: 42,
created_at: chrono::Utc::now(),
};
let not_modified = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
status_code: 304,
content: String::new(),
content_type: String::new(),
headers: {
let mut h = std::collections::HashMap::new();
h.insert("etag".to_string(), "\"v1\"".to_string());
h
},
screenshot: None,
response_time_ms: 5,
final_url: None,
markdown: None,
};
let reused = not_modified_response(&prior, ¬_modified);
assert_eq!(reused.status_code, 200, "304 must be rewritten as 200");
assert_eq!(reused.content, "<html>original body</html>");
assert_eq!(reused.content_type, "text/html");
assert_eq!(
reused
.headers
.get("x-crawlrs-not-modified")
.map(String::as_str),
Some("true"),
"not-modified marker must be present"
);
assert_eq!(
reused.headers.get("etag").map(String::as_str),
Some("\"v1\"")
);
}
#[test]
fn conditional_request_dto_deserializes_all_branches() {
use crate::application::dto::scrape_request::ScrapeRequestDto;
let on: ScrapeRequestDto = serde_json::from_value(
json!({"url": "https://x.com", "options": {"conditional_request": true}}),
)
.expect("valid");
assert_eq!(
on.options.as_ref().and_then(|o| o.conditional_request),
Some(true)
);
let off: ScrapeRequestDto = serde_json::from_value(
json!({"url": "https://x.com", "options": {"conditional_request": false}}),
)
.expect("valid");
assert_eq!(
off.options.as_ref().and_then(|o| o.conditional_request),
Some(false)
);
let absent: ScrapeRequestDto =
serde_json::from_value(json!({"url": "https://x.com"})).expect("valid");
assert_eq!(
absent.options.as_ref().and_then(|o| o.conditional_request),
None,
"field absent must default to None (disabled)"
);
}
}
fn map_wait_for(ms: Option<u64>) -> Option<WaitFor> {
ms.map(|ms| WaitFor::DomStable(std::time::Duration::from_millis(ms.min(10_000))))
}
#[cfg(test)]
impl ScrapeWorker {
async fn parse_crawl_payload(
&self,
task: &Task,
) -> Result<(crate::domain::id::Id, u32, CrawlConfigDto)> {
parse_crawl_payload(task)
}
async fn check_robots_txt(&self, task: &Task) -> bool {
check_robots_txt_fn(
task,
self.robots_checker.as_ref(),
self.settings.robots.user_agent.as_str(),
)
.await
}
fn build_crawl_request(&self, task: &Task, config: &CrawlConfigDto) -> ScrapeRequest {
build_crawl_request_fn(
task,
config,
self.settings.timeouts.engines.default_timeout_seconds,
)
}
async fn update_crawl_completion_status(&self, crawl_id: crate::domain::id::Id) {
update_crawl_completion_status_fn(crawl_id, self.crawl_repository.as_ref()).await
}
async fn parse_extract_payload(&self, task: &Task) -> Result<(ExtractRequestDto, String)> {
parse_extract_payload(task)
}
fn build_extract_request(&self, url: &str) -> ScrapeRequest {
build_extract_request_fn(url, self.settings.timeouts.engines.default_timeout_seconds)
}
async fn try_read_scrape_cache(
&self,
ctx: &CacheContext,
key: &str,
) -> Result<Option<ScrapeResponse>> {
try_read_scrape_cache(ctx, key, self.cache_service.as_ref()).await
}
async fn try_write_scrape_cache(
&self,
ctx: &CacheContext,
key: &str,
response: &ScrapeResponse,
) -> Result<()> {
try_write_scrape_cache(
ctx,
key,
response,
self.cache_service.as_ref(),
self.settings.cache.types.search.ttl_seconds,
)
.await
}
async fn save_result(
&self,
task: &Task,
response: &ScrapeResponse,
extra_data: Option<Value>,
) -> Result<()> {
save_result(task, response, extra_data, self.result_repository.as_ref()).await
}
#[cfg(test)]
async fn extract_and_queue_links(
&self,
task: &Task,
response: &ScrapeResponse,
crawl_id: crate::domain::id::Id,
current_depth: u32,
config: &CrawlConfigDto,
) -> Result<()> {
extract_and_queue_links_fn(
task,
response,
crawl_id,
current_depth,
config,
&[],
self.repository.as_ref(),
self.crawl_repository.as_ref(),
&self.deduplicator,
)
.await
}
async fn process_text_encoding<'a>(
&self,
task: &Task,
response: &'a ScrapeResponse,
) -> Result<std::borrow::Cow<'a, str>> {
crate::workers::scrape_executor::process_text_encoding(task, response).await
}
}
mod task_lifecycle;
mod scrape_pipeline;
mod extraction_billing;
mod success_finalization;
mod crawl_task;
mod extract_task;
mod builder;
pub use builder::ScrapeWorkerBuilder;
mod deps;
pub use deps::ScrapeWorkerDeps;
#[cfg(test)]
#[path = "../tests/scrape_worker_test.rs"]
mod tests;