mod link_filters;
use link_filters::url_accepted;
use std::collections::HashSet;
use std::sync::Arc;
use anyhow::Result;
use log::{debug, error, info, warn};
use scraper::{Html, Selector};
use serde_json::json;
use url::Url;
use chrono::Utc;
use crate::application::dto::crawl_request::CrawlConfigDto;
use crate::domain::models::{Task, TaskStatus, TaskType};
use crate::domain::repositories::crawl_repository::CrawlRepository;
use crate::domain::repositories::task_repository::TaskRepository;
use crate::engines::engine_client::ScrapeResponse;
use crate::i18n::tr_log_args;
use crate::utils::dedup::{DedupResult, Deduplicator};
use crate::utils::robots::RobotsCheckerTrait;
use crate::workers::crawl::{
DomainFilter, FilterContext, Frontier, PathDepthScorer, ScoringContext, UrlPatternFilter,
UrlScorer,
};
use crate::workers::errors::ScrapeWorkerError;
pub async fn check_robots_txt(
task: &Task,
robots_checker: &dyn RobotsCheckerTrait,
user_agent: &str,
) -> bool {
if !robots_checker
.is_allowed(&task.url, user_agent)
.await
.unwrap_or(true)
{
info!("Access denied by robots.txt for {}", task.url);
return false;
}
if let Some(delay) = robots_checker
.get_crawl_delay(&task.url, user_agent)
.await
.unwrap_or(None)
{
info!("Respecting crawl delay of {:?} for {}", delay, task.url);
tokio::time::sleep(delay).await;
}
true
}
pub async fn update_crawl_completion_status(
crawl_id: crate::domain::id::Id,
crawl_repository: &dyn CrawlRepository,
) {
match crawl_repository.finalize_if_all_terminal(crawl_id).await {
Ok(true) => info!(
"Crawl {} marked terminal (all tasks completed/failed)",
crawl_id
),
Ok(false) => debug!("crawl {} not eligible for terminal transition", crawl_id),
Err(e) => error!("Failed to finalize crawl {}: {}", crawl_id, e),
}
}
#[allow(clippy::too_many_arguments)]
pub async fn extract_and_queue_links(
task: &Task,
response: &ScrapeResponse,
crawl_id: crate::domain::id::Id,
current_depth: u32,
config: &CrawlConfigDto,
domain_blacklist: &[String],
repository: &dyn TaskRepository,
crawl_repository: &dyn CrawlRepository,
deduplicator: &Arc<parking_lot::RwLock<Deduplicator>>,
) -> Result<()> {
if !response.content_type.contains("text/html") {
return Ok(());
}
let LinkFilters {
pattern_filter,
filter_ctx,
blacklist_filter,
empty_include_short_circuit,
} = build_link_filters(task, config, domain_blacklist);
let unique_links = collect_unique_links(
task,
response,
&pattern_filter,
empty_include_short_circuit,
&blacklist_filter,
&filter_ctx,
)?;
info!("Found {} unique links on {}", unique_links.len(), task.url);
let (mut to_enqueue, db_check) = partition_by_dedup(&unique_links, deduplicator)?;
let mut new_urls: Vec<String> = Vec::with_capacity(unique_links.len());
new_urls.extend(std::mem::take(&mut to_enqueue));
if !db_check.is_empty() {
new_urls.extend(resolve_db_check(db_check, repository, deduplicator).await?);
}
if !new_urls.is_empty() {
score_and_enqueue(
new_urls,
task,
crawl_id,
current_depth,
config,
repository,
crawl_repository,
)
.await?;
}
Ok(())
}
struct LinkFilters {
pattern_filter: Option<UrlPatternFilter>,
filter_ctx: FilterContext,
blacklist_filter: Option<DomainFilter>,
empty_include_short_circuit: bool,
}
fn build_link_filters(
task: &Task,
config: &CrawlConfigDto,
domain_blacklist: &[String],
) -> LinkFilters {
let empty_include_short_circuit =
matches!(&config.include_patterns, Some(patterns) if patterns.is_empty());
let pattern_filter = if empty_include_short_circuit {
None
} else {
Some(UrlPatternFilter::new(
config.include_patterns.clone().unwrap_or_default(),
config.exclude_patterns.clone().unwrap_or_default(),
))
};
let filter_ctx = FilterContext::default();
let blacklist_filter = if domain_blacklist.is_empty() {
None
} else {
info!(
"Crawl task {} enforcing domain blacklist with {} entries",
task.id,
domain_blacklist.len()
);
Some(DomainFilter::denylist_normalized(domain_blacklist.to_vec()))
};
LinkFilters {
pattern_filter,
filter_ctx,
blacklist_filter,
empty_include_short_circuit,
}
}
fn collect_unique_links(
task: &Task,
response: &ScrapeResponse,
pattern_filter: &Option<UrlPatternFilter>,
empty_include_short_circuit: bool,
blacklist_filter: &Option<DomainFilter>,
filter_ctx: &FilterContext,
) -> Result<HashSet<String>> {
let document = Html::parse_document(&response.content);
let selector =
Selector::parse("a").map_err(|e| ScrapeWorkerError::SelectorError(e.to_string()))?;
let join_base = response
.final_url
.as_deref()
.and_then(|u| Url::parse(u).ok())
.unwrap_or(Url::parse(&task.url)?);
let seed_host = Url::parse(&task.url)
.ok()
.and_then(|u| u.host_str().map(str::to_ascii_lowercase));
let mut allowed_hosts: Vec<String> = seed_host.clone().into_iter().collect();
if let Some(final_host) = response
.final_url
.as_deref()
.and_then(|u| Url::parse(u).ok())
.and_then(|u| u.host_str().map(str::to_ascii_lowercase))
{
let related = seed_host
.as_ref()
.map(|seed| host_matches(&final_host, seed) || host_matches(seed, &final_host))
.unwrap_or(false);
if related {
allowed_hosts.push(final_host);
} else {
log::debug!(
"redirect final host {} unrelated to seed host; not expanding crawl scope",
final_host
);
}
}
allowed_hosts.sort();
allowed_hosts.dedup();
let mut links = HashSet::new();
for element in document.select(&selector) {
if let Some(href) = element.value().attr("href") {
if let Ok(absolute_url) = join_base.join(href) {
let same_host = absolute_url
.host_str()
.map(str::to_ascii_lowercase)
.map(|host| allowed_hosts.iter().any(|root| host_matches(&host, root)))
.unwrap_or(false);
if !same_host {
continue;
}
let url_str = absolute_url.to_string();
if url_str.len() > 2048 {
log::debug!("skip oversized child url ({} bytes)", url_str.len());
continue;
}
if url_accepted(
&url_str,
&task.url,
pattern_filter,
empty_include_short_circuit,
blacklist_filter,
filter_ctx,
) {
links.insert(url_str);
}
}
}
}
Ok(links)
}
fn host_matches(host: &str, root: &str) -> bool {
host == root
|| (host.len() > root.len() + 1
&& host.ends_with(root)
&& host.as_bytes()[host.len() - root.len() - 1] == b'.')
}
fn partition_by_dedup(
unique_links: &HashSet<String>,
deduplicator: &Arc<parking_lot::RwLock<Deduplicator>>,
) -> Result<(Vec<String>, Vec<String>)> {
let mut to_enqueue: Vec<String> = Vec::with_capacity(unique_links.len());
let mut db_check: Vec<String> = Vec::with_capacity(unique_links.len());
{
let mut dedup = deduplicator.write();
for link in unique_links {
match dedup.check_and_insert(link) {
Ok(DedupResult::DefinitelyNew { normalized }) => {
to_enqueue.push(normalized);
}
Ok(DedupResult::MaybeExisting { normalized, .. }) => {
db_check.push(normalized);
}
Err(e) => {
return Err(anyhow::anyhow!(
"URL dedup check failed for {}: {}",
link,
e
));
}
}
}
}
Ok((to_enqueue, db_check))
}
async fn resolve_db_check(
mut db_check: Vec<String>,
repository: &dyn TaskRepository,
deduplicator: &Arc<parking_lot::RwLock<Deduplicator>>,
) -> Result<Vec<String>> {
db_check.sort_unstable();
db_check.dedup();
let existing_urls = repository.find_existing_urls(&db_check).await?;
let existing_url_set: HashSet<String> = existing_urls.into_iter().collect();
let mut to_db_insert: Vec<&String> = Vec::with_capacity(db_check.len());
for normalized in &db_check {
if !existing_url_set.contains(normalized) {
to_db_insert.push(normalized);
}
}
let mut to_db_enqueue: Vec<String> = Vec::with_capacity(to_db_insert.len());
if !to_db_insert.is_empty() {
let mut dedup = deduplicator.write();
for normalized in &to_db_insert {
dedup.insert(normalized);
to_db_enqueue.push(normalized.to_string());
}
}
Ok(to_db_enqueue)
}
#[allow(clippy::too_many_arguments)]
async fn score_and_enqueue(
new_urls: Vec<String>,
task: &Task,
crawl_id: crate::domain::id::Id,
current_depth: u32,
config: &CrawlConfigDto,
repository: &dyn TaskRepository,
crawl_repository: &dyn CrawlRepository,
) -> Result<()> {
if let Ok(Some(progress)) = crawl_repository.find_progress_by_id(crawl_id).await {
if crate::application::use_cases::crawl_state_machine::is_terminal(&progress.status) {
log::info!(
"crawl {} is terminal; skip enqueueing {} discovered link(s)",
crawl_id,
new_urls.len()
);
return Ok(());
}
}
let frontier = build_scored_frontier(&new_urls, task);
let is_dfs_strategy = config
.strategy
.as_deref()
.map(|s| s.to_lowercase() == "dfs")
.unwrap_or(false);
while let Some(scored) = frontier.pop() {
if scored.url.len() > 2048 {
log::debug!(
"skip oversized normalized child url: {} bytes (limit 2048)",
scored.url.len()
);
continue;
}
let priority = child_priority(task.priority, is_dfs_strategy, current_depth);
let new_task =
build_child_crawl_task(task, crawl_id, current_depth, config, priority, scored.url);
match repository.create(&new_task).await {
Ok(_) => {
crawl_repository.increment_total_tasks(crawl_id).await?;
}
Err(e) => {
if e.is_unique_conflict() {
log::debug!("skip duplicate child task url={}", new_task.url);
} else {
return Err(e.into());
}
}
}
}
Ok(())
}
fn build_scored_frontier(new_urls: &[String], task: &Task) -> Frontier {
let scorer = PathDepthScorer::new();
let scoring_ctx = ScoringContext::default();
let frontier = Frontier::new();
for url in new_urls {
let base_score = scorer.score(url, &scoring_ctx);
let score = (base_score as f64 * scoring_ctx.kg_priority_boost).clamp(0.0, 1.0) as f32;
match crate::workers::crawl::ScoredUrl::new(url.clone(), score) {
Ok(scored) => frontier.push(scored),
Err(e) => {
warn!(
"{}",
tr_log_args(
"link-score-skip",
&[
("task_id", task.id.to_string()),
("error", e.to_string()),
("url", url.as_str().to_string()),
],
)
);
}
}
}
info!(
"{}",
tr_log_args(
"link-frontier-enqueued",
&[
("task_id", task.id.to_string()),
("count", frontier.len().to_string()),
("domains", frontier.domain_count().to_string()),
],
)
);
frontier
}
fn child_priority(parent_priority: i32, is_dfs_strategy: bool, current_depth: u32) -> i32 {
if is_dfs_strategy {
let depth_key = (current_depth.min(1_000_000) as i32).saturating_add(1);
parent_priority.saturating_sub(depth_key)
} else {
parent_priority
}
}
fn build_child_crawl_task(
task: &Task,
crawl_id: crate::domain::id::Id,
current_depth: u32,
config: &CrawlConfigDto,
priority: i32,
url: String,
) -> Task {
Task {
id: crate::domain::id::Id::generate(),
task_type: TaskType::Crawl,
status: TaskStatus::Queued,
priority,
team_id: task.team_id,
api_key_id: task.api_key_id,
url,
payload: json!({
"crawl_id": crawl_id.to_string(),
"depth": current_depth + 1,
"config": config,
}),
retry_count: 0,
attempt_count: 0,
max_retries: 3,
scheduled_at: None,
created_at: Utc::now(),
started_at: None,
completed_at: None,
crawl_id: Some(crawl_id),
updated_at: Utc::now(),
lock_token: None,
lock_expires_at: None,
expires_at: None,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::models::Crawl;
use crate::domain::models::CrawlStatus;
use crate::domain::repositories::task_repository::RepositoryError;
use std::sync::atomic::{AtomicU32, Ordering};
fn make_task(url: &str) -> Task {
Task {
id: crate::domain::id::Id::generate(),
task_type: TaskType::Crawl,
status: TaskStatus::Queued,
priority: 0,
team_id: crate::domain::id::Id::generate(),
api_key_id: crate::domain::id::Id::generate(),
url: url.to_string(),
payload: json!({}),
retry_count: 0,
attempt_count: 0,
max_retries: 3,
scheduled_at: None,
created_at: Utc::now(),
started_at: None,
completed_at: None,
crawl_id: None,
updated_at: Utc::now(),
lock_token: None,
lock_expires_at: None,
expires_at: None,
}
}
struct MockRobotsChecker {
allowed: bool,
}
#[async_trait::async_trait]
impl RobotsCheckerTrait for MockRobotsChecker {
async fn is_allowed(&self, _url: &str, _user_agent: &str) -> Result<bool> {
Ok(self.allowed)
}
async fn get_crawl_delay(
&self,
_url: &str,
_user_agent: &str,
) -> Result<Option<std::time::Duration>> {
Ok(None)
}
}
#[derive(Default)]
struct MockCrawlRepo {
update_status_count: AtomicU32,
}
struct ConfigurableCrawlRepo {
crawl: Option<Crawl>,
update_status_count: AtomicU32,
last_status: std::sync::Mutex<Option<CrawlStatus>>,
}
impl ConfigurableCrawlRepo {
fn with_crawl(c: Crawl) -> Self {
Self {
crawl: Some(c),
update_status_count: AtomicU32::new(0),
last_status: std::sync::Mutex::new(None),
}
}
fn last_status(&self) -> Option<CrawlStatus> {
*self.last_status.lock().expect("status mutex poisoned")
}
}
#[async_trait::async_trait]
impl CrawlRepository for ConfigurableCrawlRepo {
async fn create(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
Ok(crawl.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Crawl>, RepositoryError> {
Ok(self.crawl.clone())
}
async fn update(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
Ok(crawl.clone())
}
async fn increment_completed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn increment_failed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn update_status(
&self,
_id: crate::domain::id::Id,
status: CrawlStatus,
) -> Result<(), RepositoryError> {
*self.last_status.lock().expect("status mutex poisoned") = Some(status);
self.update_status_count.fetch_add(1, Ordering::SeqCst);
Ok(())
}
async fn update_status_guarded(
&self,
_id: crate::domain::id::Id,
status: CrawlStatus,
) -> Result<bool, RepositoryError> {
if self.crawl.as_ref().is_some_and(|c| {
matches!(
c.status,
CrawlStatus::Completed | CrawlStatus::Failed | CrawlStatus::Cancelled
)
}) {
return Ok(false);
}
*self.last_status.lock().expect("status mutex poisoned") = Some(status);
self.update_status_count.fetch_add(1, Ordering::SeqCst);
Ok(true)
}
async fn finalize_if_all_terminal(
&self,
_id: crate::domain::id::Id,
) -> Result<bool, RepositoryError> {
let Some(c) = self.crawl.as_ref() else {
return Ok(false);
};
if matches!(
c.status,
crate::domain::models::CrawlStatus::Completed
| crate::domain::models::CrawlStatus::Failed
| crate::domain::models::CrawlStatus::Cancelled
) {
return Ok(false);
}
if c.total_tasks() > 0 && c.completed_tasks() + c.failed_tasks() == c.total_tasks() {
let status = if c.completed_tasks() == 0 {
crate::domain::models::CrawlStatus::Failed
} else {
crate::domain::models::CrawlStatus::Completed
};
*self.last_status.lock().expect("status mutex poisoned") = Some(status);
self.update_status_count.fetch_add(1, Ordering::SeqCst);
Ok(true)
} else {
Ok(false)
}
}
async fn increment_total_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn find_by_team_id_paginated(
&self,
_team_id: crate::domain::id::Id,
_limit: u32,
_offset: u32,
) -> Result<Vec<Crawl>, RepositoryError> {
Ok(vec![])
}
async fn count_by_team_id(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
}
#[async_trait::async_trait]
impl CrawlRepository for MockCrawlRepo {
async fn create(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
Ok(crawl.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Crawl>, RepositoryError> {
Ok(None)
}
async fn update(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
Ok(crawl.clone())
}
async fn increment_completed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn increment_failed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn update_status(
&self,
_id: crate::domain::id::Id,
_status: CrawlStatus,
) -> Result<(), RepositoryError> {
self.update_status_count.fetch_add(1, Ordering::SeqCst);
Ok(())
}
async fn increment_total_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn find_by_team_id_paginated(
&self,
_team_id: crate::domain::id::Id,
_limit: u32,
_offset: u32,
) -> Result<Vec<Crawl>, RepositoryError> {
Ok(vec![])
}
async fn count_by_team_id(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
}
#[tokio::test]
async fn test_check_robots_txt_allowed_returns_true() {
let task = make_task("https://example.com");
let checker = MockRobotsChecker { allowed: true };
assert!(check_robots_txt(&task, &checker, "crawlrs-bot/1.0").await);
}
#[tokio::test]
async fn test_check_robots_txt_denied_returns_false() {
let task = make_task("https://example.com");
let checker = MockRobotsChecker { allowed: false };
assert!(!check_robots_txt(&task, &checker, "crawlrs-bot/1.0").await);
}
#[tokio::test]
async fn test_update_crawl_completion_status_crawl_not_found() {
let repo = MockCrawlRepo::default();
update_crawl_completion_status(crate::domain::id::Id::generate(), &repo).await;
}
use crate::application::dto::crawl_request::CrawlConfigDto;
use crate::domain::repositories::crawl_repository::CrawlRepository;
use crate::domain::repositories::task_repository::TaskQueryParams;
use crate::engines::engine_client::ScrapeResponse;
use crate::utils::dedup::Deduplicator;
use std::collections::HashSet;
struct CountingTaskRepo {
create_count: AtomicU32,
captured_priorities: std::sync::Mutex<Vec<i32>>,
}
impl CountingTaskRepo {
fn new() -> Self {
Self {
create_count: AtomicU32::new(0),
captured_priorities: std::sync::Mutex::new(Vec::new()),
}
}
}
#[async_trait::async_trait]
impl TaskRepository for CountingTaskRepo {
async fn create(&self, task: &Task) -> Result<Task, RepositoryError> {
self.create_count.fetch_add(1, Ordering::SeqCst);
self.captured_priorities.lock().unwrap().push(task.priority);
Ok(Task::new(
crate::domain::id::Id::generate(),
TaskType::Crawl,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
String::new(),
json!({}),
))
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn update(&self, _task: &Task) -> Result<Task, RepositoryError> {
Ok(Task::new(
crate::domain::id::Id::generate(),
TaskType::Crawl,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
String::new(),
json!({}),
))
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
_urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
Ok(HashSet::new())
}
async fn reset_stuck_tasks(
&self,
_timeout: chrono::Duration,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
Ok(vec![])
}
async fn query_tasks(
&self,
_params: TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
Ok((vec![], 0))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((vec![], vec![]))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
fn make_response(content_type: &str, content: &str) -> ScrapeResponse {
ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: content.to_string(),
status_code: 200,
content_type: content_type.to_string(),
headers: Default::default(),
screenshot: None,
response_time_ms: 100,
final_url: None,
markdown: None,
}
}
fn default_config() -> CrawlConfigDto {
CrawlConfigDto {
max_depth: 2,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
}
}
#[tokio::test]
async fn test_extract_links_non_html_returns_early() {
let task = make_task("https://example.com");
let response = make_response("application/json", r#"{"key": "value"}"#);
let repo = CountingTaskRepo::new();
let crawl_repo = MockCrawlRepo::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let config = default_config();
extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
&[],
&repo,
&crawl_repo,
&dedup,
)
.await
.unwrap();
assert_eq!(
repo.create_count.load(Ordering::SeqCst),
0,
"non-HTML should not extract links"
);
}
#[tokio::test]
async fn test_extract_links_html_with_links() {
let task = make_task("https://example.com/page");
let html = r#"<html><body>
<a href="https://example.com/other">Other</a>
<a href="https://example.com/another">Another</a>
</body></html>"#;
let response = make_response("text/html", html);
let repo = CountingTaskRepo::new();
let crawl_repo = MockCrawlRepo::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let config = default_config();
extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
&[],
&repo,
&crawl_repo,
&dedup,
)
.await
.unwrap();
assert!(
repo.create_count.load(Ordering::SeqCst) > 0,
"should enqueue found links"
);
}
#[tokio::test]
async fn test_extract_links_skips_self_link() {
let task = make_task("https://example.com/page");
let html = r#"<html><body>
<a href="https://example.com/page">Self</a>
<a href="https://example.com/other">Other</a>
</body></html>"#;
let response = make_response("text/html", html);
let repo = CountingTaskRepo::new();
let crawl_repo = MockCrawlRepo::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let config = default_config();
extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
&[],
&repo,
&crawl_repo,
&dedup,
)
.await
.unwrap();
assert_eq!(
repo.create_count.load(Ordering::SeqCst),
1,
"self-link should be skipped"
);
}
#[tokio::test]
async fn test_extract_links_empty_include_patterns_short_circuit() {
let task = make_task("https://example.com/page");
let html = r#"<html><body><a href="https://example.com/other">Other</a></body></html>"#;
let response = make_response("text/html", html);
let repo = CountingTaskRepo::new();
let crawl_repo = MockCrawlRepo::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let mut config = default_config();
config.include_patterns = Some(vec![]);
extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
&[],
&repo,
&crawl_repo,
&dedup,
)
.await
.unwrap();
assert_eq!(
repo.create_count.load(Ordering::SeqCst),
0,
"empty include_patterns should block all links"
);
}
#[tokio::test]
async fn test_extract_links_with_dfs_strategy() {
let task = make_task("https://example.com/page");
let html = r#"<html><body>
<a href="https://example.com/other">Other</a>
</body></html>"#;
let response = make_response("text/html", html);
let repo = CountingTaskRepo::new();
let crawl_repo = MockCrawlRepo::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let mut config = default_config();
config.strategy = Some("dfs".to_string());
extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
&[],
&repo,
&crawl_repo,
&dedup,
)
.await
.unwrap();
assert!(
repo.create_count.load(Ordering::SeqCst) > 0,
"DFS strategy should enqueue links"
);
}
#[tokio::test]
async fn test_dfs_child_priority_below_parent_and_bfs_above() {
async fn run_with_strategy(strategy: Option<&str>, html: &str) -> Vec<i32> {
let mut task = make_task("https://example.com/page");
task.priority = 5;
let response = make_response("text/html", html);
let repo = CountingTaskRepo::new();
let crawl_repo = MockCrawlRepo::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let mut config = default_config();
config.strategy = strategy.map(str::to_string);
extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
&[],
&repo,
&crawl_repo,
&dedup,
)
.await
.unwrap();
let priorities = repo.captured_priorities.lock().unwrap().clone();
priorities
}
let html = r#"<html><body>
<a href="https://example.com/child">Child</a>
</body></html>"#;
let dfs = run_with_strategy(Some("dfs"), html).await;
assert!(
dfs.iter().all(|&p| p < 5),
"dfs child priority must be BELOW parent (5), got {dfs:?}"
);
let bfs = run_with_strategy(Some("bfs"), html).await;
assert!(
bfs.iter().all(|&p| p >= 5),
"bfs child priority must be >= parent (5), got {bfs:?}"
);
}
#[tokio::test]
async fn test_extract_links_with_exclude_patterns() {
let task = make_task("https://example.com/page");
let html = r#"<html><body>
<a href="https://example.com/good">Good</a>
<a href="https://example.com/bad">Bad</a>
</body></html>"#;
let response = make_response("text/html", html);
let repo = CountingTaskRepo::new();
let crawl_repo = MockCrawlRepo::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let mut config = default_config();
config.exclude_patterns = Some(vec![".*bad.*".to_string()]);
extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
&[],
&repo,
&crawl_repo,
&dedup,
)
.await
.unwrap();
assert_eq!(
repo.create_count.load(Ordering::SeqCst),
1,
"excluded pattern links should be filtered"
);
}
#[tokio::test]
async fn test_update_crawl_completion_status_all_done_marks_completed() {
let crawl = Crawl::with_all_fields(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
"test".into(),
"https://example.com".into(),
"https://example.com".into(),
CrawlStatus::Processing,
json!({}),
5,
3,
2,
Utc::now(),
Utc::now(),
None,
);
let repo = ConfigurableCrawlRepo::with_crawl(crawl);
update_crawl_completion_status(crate::domain::id::Id::generate(), &repo).await;
assert_eq!(
repo.update_status_count.load(Ordering::SeqCst),
1,
"should mark completed when all tasks done"
);
}
#[tokio::test]
async fn test_update_crawl_completion_status_all_failed_marks_failed() {
let crawl = Crawl::with_all_fields(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
"test".into(),
"https://example.com".into(),
"https://example.com".into(),
CrawlStatus::Processing,
json!({}),
5,
0,
5,
Utc::now(),
Utc::now(),
None,
);
let repo = ConfigurableCrawlRepo::with_crawl(crawl);
update_crawl_completion_status(crate::domain::id::Id::generate(), &repo).await;
assert_eq!(repo.update_status_count.load(Ordering::SeqCst), 1);
assert_eq!(
repo.last_status(),
Some(CrawlStatus::Failed),
"all-failed crawl must be marked Failed"
);
}
#[tokio::test]
async fn test_update_crawl_completion_status_not_yet_done() {
let crawl = Crawl::with_all_fields(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
"test".into(),
"https://example.com".into(),
"https://example.com".into(),
CrawlStatus::Processing,
json!({}),
5,
1,
0,
Utc::now(),
Utc::now(),
None,
);
let repo = ConfigurableCrawlRepo::with_crawl(crawl);
update_crawl_completion_status(crate::domain::id::Id::generate(), &repo).await;
assert_eq!(
repo.update_status_count.load(Ordering::SeqCst),
0,
"should not mark completed when tasks pending"
);
}
#[tokio::test]
async fn test_update_crawl_completion_status_find_error() {
struct ErrorCrawlRepo;
#[async_trait::async_trait]
impl CrawlRepository for ErrorCrawlRepo {
async fn create(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
Ok(crawl.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Crawl>, RepositoryError> {
Err(RepositoryError::Database(anyhow::anyhow!("db error")))
}
async fn update(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
Ok(crawl.clone())
}
async fn increment_completed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn increment_failed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn update_status(
&self,
_id: crate::domain::id::Id,
_status: CrawlStatus,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn increment_total_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn find_by_team_id_paginated(
&self,
_team_id: crate::domain::id::Id,
_limit: u32,
_offset: u32,
) -> Result<Vec<Crawl>, RepositoryError> {
Ok(vec![])
}
async fn count_by_team_id(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
}
let repo = ErrorCrawlRepo;
update_crawl_completion_status(crate::domain::id::Id::generate(), &repo).await;
}
struct ExistingUrlTaskRepo {
create_count: AtomicU32,
existing_urls: HashSet<String>,
}
impl ExistingUrlTaskRepo {
fn new_with_existing(urls: Vec<String>) -> Self {
Self {
create_count: AtomicU32::new(0),
existing_urls: urls.into_iter().collect(),
}
}
}
#[async_trait::async_trait]
impl TaskRepository for ExistingUrlTaskRepo {
async fn create(&self, _task: &Task) -> Result<Task, RepositoryError> {
self.create_count.fetch_add(1, Ordering::SeqCst);
Ok(Task::new(
crate::domain::id::Id::generate(),
TaskType::Crawl,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
String::new(),
json!({}),
))
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn update(&self, _task: &Task) -> Result<Task, RepositoryError> {
Ok(Task::new(
crate::domain::id::Id::generate(),
TaskType::Crawl,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
String::new(),
json!({}),
))
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
Ok(urls
.iter()
.filter(|u| self.existing_urls.contains(u.as_str()))
.cloned()
.collect())
}
async fn reset_stuck_tasks(
&self,
_timeout: chrono::Duration,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
Ok(vec![])
}
async fn query_tasks(
&self,
_params: TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
Ok((vec![], 0))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((vec![], vec![]))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
#[tokio::test]
async fn test_extract_links_db_check_path_with_preinserted_urls() {
let task = make_task("https://example.com/page");
let html = r#"<html><body>
<a href="https://example.com/new-link">New</a>
</body></html>"#;
let response = make_response("text/html", html);
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
{
let mut d = dedup.write();
d.insert("https://example.com/new-link");
}
let repo =
ExistingUrlTaskRepo::new_with_existing(
vec!["https://example.com/new-link".to_string()],
);
let crawl_repo = MockCrawlRepo::default();
let config = default_config();
extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
&[],
&repo,
&crawl_repo,
&dedup,
)
.await
.unwrap();
assert_eq!(
repo.create_count.load(Ordering::SeqCst),
0,
"existing URLs should not be enqueued via DB check path"
);
}
#[tokio::test]
async fn test_extract_links_db_check_path_existing_url_skipped() {
let task = make_task("https://example.com/page");
let html = r#"<html><body>
<a href="https://example.com/existing-link">Existing</a>
</body></html>"#;
let response = make_response("text/html", html);
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
{
let mut d = dedup.write();
d.insert("https://example.com/existing-link");
}
let repo = ExistingUrlTaskRepo::new_with_existing(vec![
"https://example.com/existing-link".to_string(),
]);
let crawl_repo = MockCrawlRepo::default();
let config = default_config();
extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
&[],
&repo,
&crawl_repo,
&dedup,
)
.await
.unwrap();
assert_eq!(
repo.create_count.load(Ordering::SeqCst),
0,
"existing URLs should not be enqueued"
);
}
#[tokio::test]
async fn test_extract_links_bloom_false_positive_enqueued_after_db_miss() {
let task = make_task("https://example.com/page");
let html = r#"<html><body>
<a href="https://example.com/fresh-link">Fresh</a>
</body></html>"#;
let response = make_response("text/html", html);
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
{
let mut d = dedup.write();
d.insert("https://example.com/fresh-link");
}
let repo = ExistingUrlTaskRepo::new_with_existing(vec![]);
let crawl_repo = MockCrawlRepo::default();
let config = default_config();
extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
&[],
&repo,
&crawl_repo,
&dedup,
)
.await
.unwrap();
assert_eq!(
repo.create_count.load(Ordering::SeqCst),
1,
"DB-confirmed-new URL must be enqueued despite bloom positive (false positive must not cause missed URLs)"
);
}
}