use super::{SeedTaskContext, SitemapFeedStats};
use crate::domain::models::{Task, TaskStatus, TaskType};
use crate::domain::repositories::task_repository::TaskRepository;
use crate::utils::dedup::{DedupResult, Deduplicator};
use crate::workers::crawl::{DomainFilter, FilterContext, UrlFilter, UrlPatternFilter};
use serde_json::json;
use std::collections::HashSet;
use std::sync::Arc;
pub(super) fn filter_batch_urls(
batch: &mut Vec<String>,
stats: &mut SitemapFeedStats,
pattern_filter: Option<&UrlPatternFilter>,
blacklist_filter: Option<&DomainFilter>,
filter_ctx: &FilterContext,
empty_include_short_circuit: bool,
) -> Vec<String> {
let mut to_enqueue: Vec<String> = Vec::new();
for url in batch.drain(..) {
if let Some(bf) = blacklist_filter {
if !bf.accept(&url, filter_ctx) {
stats.filtered_out += 1;
continue;
}
}
if empty_include_short_circuit {
stats.filtered_out += 1;
continue;
}
if let Some(filter) = pattern_filter {
if !filter.accept(&url, filter_ctx) {
stats.filtered_out += 1;
continue;
}
}
to_enqueue.push(url);
}
to_enqueue
}
pub(super) async fn dedup_filtered_batch(
to_enqueue: Vec<String>,
dedup: &Arc<parking_lot::RwLock<Deduplicator>>,
task_repo: &dyn TaskRepository,
stats: &mut SitemapFeedStats,
) -> anyhow::Result<Vec<String>> {
let mut new_urls: Vec<String> = Vec::new();
let mut db_check: Vec<String> = Vec::new();
{
let mut dedup = dedup.write();
for url in to_enqueue {
match dedup.check_and_insert(&url) {
Ok(DedupResult::DefinitelyNew { normalized }) => new_urls.push(normalized),
Ok(DedupResult::MaybeExisting { normalized, .. }) => db_check.push(normalized),
Err(e) => return Err(anyhow::anyhow!("URL dedup check failed for {url}: {e}")),
}
}
}
if !db_check.is_empty() {
db_check.sort_unstable();
db_check.dedup();
let existing: HashSet<String> = task_repo.find_existing_urls(&db_check).await?;
for normalized in db_check {
if existing.contains(&normalized) {
stats.duplicates_skipped += 1;
} else {
dedup.write().insert(&normalized);
new_urls.push(normalized);
}
}
}
new_urls.sort_unstable();
new_urls.dedup();
Ok(new_urls)
}
pub(super) fn build_sitemap_task(url: &str, seed: &SeedTaskContext) -> Task {
Task {
id: crate::domain::id::Id::generate(),
task_type: TaskType::Crawl,
status: TaskStatus::Queued,
priority: seed.priority,
team_id: seed.team_id,
api_key_id: seed.api_key_id,
url: url.to_string(),
payload: json!({
"crawl_id": seed.crawl_id,
"depth": 1,
"config": seed.config_json,
}),
retry_count: 0,
attempt_count: 0,
max_retries: 3,
scheduled_at: None,
created_at: chrono::Utc::now(),
started_at: None,
completed_at: None,
crawl_id: Some(seed.crawl_id),
updated_at: chrono::Utc::now(),
lock_token: None,
lock_expires_at: None,
expires_at: None,
}
}