mod sitemap_batch;
use sitemap_batch::{build_sitemap_task, dedup_filtered_batch, filter_batch_urls};
use std::sync::Arc;
use serde::{Deserialize, Serialize};
use url::Url;
use crate::application::dto::map_request::MapRequestDto;
use crate::application::use_cases::map_use_case::{
MapUseCase, SitemapFetcher, SitemapKind, SitemapStream, SitemapStreamOptions,
};
use crate::domain::repositories::crawl_repository::CrawlRepository;
use crate::domain::repositories::task_repository::TaskRepository;
use crate::infrastructure::oxcache::CacheService;
use crate::utils::dedup::Deduplicator;
use crate::workers::crawl::{DomainFilter, FilterContext, UrlPatternFilter};
#[cfg(test)]
use crate::domain::models::{Task, TaskStatus, TaskType};
#[cfg(test)]
use serde_json::json;
#[cfg(test)]
use std::collections::HashSet;
const FEED_BATCH_SIZE: usize = 500;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SitemapFeedPointer {
pub sitemap_url: String,
pub consumed: usize,
}
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct SitemapFeedStats {
pub enqueued: usize,
pub filtered_out: usize,
pub duplicates_skipped: usize,
pub consumed: usize,
pub resumed_from: Option<usize>,
}
#[derive(Debug, Clone)]
pub struct SeedTaskContext {
pub team_id: crate::domain::id::Id,
pub api_key_id: crate::domain::id::Id,
pub crawl_id: crate::domain::id::Id,
pub priority: i32,
pub config_json: serde_json::Value,
}
pub fn sitemap_feed_pointer_key(crawl_id: crate::domain::id::Id) -> String {
format!("crawl:sitemap-feed:{crawl_id}")
}
pub fn derive_sitemap_url(seed_url: &str) -> Option<String> {
let parsed = Url::parse(seed_url).ok()?;
let host = parsed.host_str()?;
let origin = match parsed.port() {
Some(port) => format!("{}://{host}:{port}", parsed.scheme()),
None => format!("{}://{host}", parsed.scheme()),
};
Some(format!("{origin}/sitemap.xml"))
}
async fn load_pointer(
cache: &dyn CacheService,
crawl_id: crate::domain::id::Id,
sitemap_url: &str,
) -> Option<usize> {
let raw = cache
.get(&sitemap_feed_pointer_key(crawl_id))
.await
.ok()??;
match serde_json::from_str::<SitemapFeedPointer>(&raw) {
Ok(p) if p.sitemap_url == sitemap_url => Some(p.consumed),
Ok(p) => {
log::warn!(
"{}",
crate::i18n::tr_log_args(
"sitemap-pointer-mismatch",
&[
("crawl_id", crawl_id.to_string(),),
("pointer_url", p.sitemap_url.as_str().to_string(),),
("sitemap_url", sitemap_url.to_string()),
],
)
);
None
}
Err(e) => {
log::warn!(
"{}",
crate::i18n::tr_log_args(
"sitemap-pointer-corrupt",
&[
("crawl_id", crawl_id.to_string(),),
("error", e.to_string()),
],
)
);
None
}
}
}
#[allow(clippy::too_many_arguments)]
pub async fn feed_sitemap_as_tasks(
fetcher: Arc<dyn SitemapFetcher>,
sitemap_url: String,
options: SitemapStreamOptions,
include_patterns: Option<Vec<String>>,
exclude_patterns: Option<Vec<String>>,
domain_blacklist: Vec<String>,
seed: SeedTaskContext,
dedup: &Arc<parking_lot::RwLock<Deduplicator>>,
task_repo: &dyn TaskRepository,
crawl_repo: &dyn CrawlRepository,
cache: &dyn CacheService,
pointer_ttl_secs: u64,
) -> anyhow::Result<SitemapFeedStats> {
let stream = open_sitemap_stream(fetcher, &sitemap_url, options).await?;
let mut rx = stream.rx;
let mut stats = SitemapFeedStats::default();
let seek =
resolve_seek_position(cache, seed.crawl_id, &sitemap_url, stream.kind, &mut stats).await;
let (pattern_filter, blacklist_filter, filter_ctx, empty_include_short_circuit) =
build_feed_filters(include_patterns, exclude_patterns, domain_blacklist);
feed_sitemap_stream(
&mut rx,
seek,
&mut stats,
pattern_filter.as_ref(),
blacklist_filter.as_ref(),
&filter_ctx,
empty_include_short_circuit,
&seed,
&sitemap_url,
dedup,
task_repo,
crawl_repo,
cache,
pointer_ttl_secs,
)
.await?;
if stats.consumed > 0 {
let _ = cache.delete(&sitemap_feed_pointer_key(seed.crawl_id)).await;
}
Ok(stats)
}
async fn open_sitemap_stream(
fetcher: Arc<dyn SitemapFetcher>,
sitemap_url: &str,
options: SitemapStreamOptions,
) -> anyhow::Result<SitemapStream> {
let use_case = MapUseCase::new(fetcher);
let dto = MapRequestDto {
url: sitemap_url.to_string(),
include_patterns: None,
exclude_patterns: None,
limit: None,
};
use_case
.execute_streaming(&dto, options)
.await
.map_err(|e| anyhow::anyhow!(e.to_string()))
}
async fn resolve_seek_position(
cache: &dyn CacheService,
crawl_id: crate::domain::id::Id,
sitemap_url: &str,
kind: SitemapKind,
stats: &mut SitemapFeedStats,
) -> Option<usize> {
match load_pointer(cache, crawl_id, sitemap_url).await {
Some(consumed) if consumed > 0 => {
if kind == SitemapKind::Urlset {
stats.resumed_from = Some(consumed);
Some(consumed)
} else {
log::info!(
"{}",
crate::i18n::tr_log_args(
"sitemap-resume-without-seek",
&[
("crawl_id", crawl_id.to_string(),),
("consumed", (consumed as i64).to_string()),
],
)
);
None
}
}
_ => None,
}
}
fn build_feed_filters(
include_patterns: Option<Vec<String>>,
exclude_patterns: Option<Vec<String>>,
domain_blacklist: Vec<String>,
) -> (
Option<UrlPatternFilter>,
Option<DomainFilter>,
FilterContext,
bool,
) {
let empty_include_short_circuit =
matches!(&include_patterns, Some(patterns) if patterns.is_empty());
let pattern_filter = if empty_include_short_circuit {
None
} else {
Some(UrlPatternFilter::new(
include_patterns.unwrap_or_default(),
exclude_patterns.unwrap_or_default(),
))
};
let blacklist_filter = if domain_blacklist.is_empty() {
None
} else {
Some(DomainFilter::denylist_normalized(domain_blacklist))
};
let filter_ctx = FilterContext::default();
(
pattern_filter,
blacklist_filter,
filter_ctx,
empty_include_short_circuit,
)
}
#[allow(clippy::too_many_arguments)]
async fn feed_sitemap_stream(
rx: &mut tokio::sync::mpsc::Receiver<String>,
seek: Option<usize>,
stats: &mut SitemapFeedStats,
pattern_filter: Option<&UrlPatternFilter>,
blacklist_filter: Option<&DomainFilter>,
filter_ctx: &FilterContext,
empty_include_short_circuit: bool,
seed: &SeedTaskContext,
sitemap_url: &str,
dedup: &Arc<parking_lot::RwLock<Deduplicator>>,
task_repo: &dyn TaskRepository,
crawl_repo: &dyn CrawlRepository,
cache: &dyn CacheService,
pointer_ttl_secs: u64,
) -> anyhow::Result<()> {
let mut batch: Vec<String> = Vec::with_capacity(FEED_BATCH_SIZE.min(64));
while let Some(loc) = rx.recv().await {
if let Some(seek_n) = seek {
if stats.consumed < seek_n {
stats.consumed += 1;
continue;
}
}
stats.consumed += 1;
batch.push(loc);
if batch.len() >= FEED_BATCH_SIZE {
flush_batch(
&mut batch,
stats,
pattern_filter,
blacklist_filter,
filter_ctx,
empty_include_short_circuit,
seed,
sitemap_url,
dedup,
task_repo,
crawl_repo,
cache,
pointer_ttl_secs,
)
.await?;
}
}
if !batch.is_empty() {
flush_batch(
&mut batch,
stats,
pattern_filter,
blacklist_filter,
filter_ctx,
empty_include_short_circuit,
seed,
sitemap_url,
dedup,
task_repo,
crawl_repo,
cache,
pointer_ttl_secs,
)
.await?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
async fn flush_batch(
batch: &mut Vec<String>,
stats: &mut SitemapFeedStats,
pattern_filter: Option<&UrlPatternFilter>,
blacklist_filter: Option<&DomainFilter>,
filter_ctx: &FilterContext,
empty_include_short_circuit: bool,
seed: &SeedTaskContext,
sitemap_url: &str,
dedup: &Arc<parking_lot::RwLock<Deduplicator>>,
task_repo: &dyn TaskRepository,
crawl_repo: &dyn CrawlRepository,
cache: &dyn CacheService,
pointer_ttl_secs: u64,
) -> anyhow::Result<()> {
if let Ok(Some(progress)) = crawl_repo.find_progress_by_id(seed.crawl_id).await {
if crate::application::use_cases::crawl_state_machine::is_terminal(&progress.status) {
log::info!(
"crawl {} is terminal; skip enqueueing sitemap batch",
seed.crawl_id
);
return Ok(());
}
}
let new_urls = dedup_filtered_batch(
filter_batch_urls(
batch,
stats,
pattern_filter,
blacklist_filter,
filter_ctx,
empty_include_short_circuit,
),
dedup,
task_repo,
stats,
)
.await?;
for url in &new_urls {
if url.len() > 2048 {
log::debug!("skip oversized sitemap task url ({} bytes)", url.len());
continue;
}
let task = build_sitemap_task(url, seed);
match task_repo.create(&task).await {
Ok(_) => {
crawl_repo.increment_total_tasks(seed.crawl_id).await?;
stats.enqueued += 1;
}
Err(e) => {
if e.is_unique_conflict() {
log::debug!("skip duplicate sitemap task url={}", task.url);
} else {
return Err(e.into());
}
}
}
}
let pointer = SitemapFeedPointer {
sitemap_url: sitemap_url.to_string(),
consumed: stats.consumed,
};
cache
.set(
&sitemap_feed_pointer_key(seed.crawl_id),
&serde_json::to_string(&pointer)?,
pointer_ttl_secs,
)
.await?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::application::use_cases::map_use_case::MapError;
use crate::domain::repositories::task_repository::RepositoryError;
use async_trait::async_trait;
use std::collections::HashMap;
use std::sync::atomic::{AtomicU32, Ordering};
#[derive(Default)]
struct MockCache {
store: parking_lot::Mutex<HashMap<String, String>>,
set_keys: parking_lot::Mutex<Vec<String>>,
deleted: parking_lot::Mutex<Vec<String>>,
}
impl MockCache {
fn set_calls(&self, key: &str) -> usize {
self.set_keys.lock().iter().filter(|k| *k == key).count()
}
}
#[async_trait::async_trait]
impl CacheService for MockCache {
fn get(
&self,
key: &str,
) -> std::pin::Pin<
Box<dyn std::future::Future<Output = anyhow::Result<Option<String>>> + Send + '_>,
> {
let v = self.store.lock().get(key).cloned();
Box::pin(async move { Ok(v) })
}
fn set(
&self,
key: &str,
value: &str,
_ttl_seconds: u64,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + Send + '_>>
{
self.store.lock().insert(key.to_string(), value.to_string());
self.set_keys.lock().push(key.to_string());
Box::pin(async { Ok(()) })
}
fn delete(
&self,
key: &str,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + Send + '_>>
{
self.store.lock().remove(key);
self.deleted.lock().push(key.to_string());
Box::pin(async { Ok(()) })
}
fn exists(
&self,
key: &str,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<bool>> + Send + '_>>
{
let v = self.store.lock().contains_key(key);
Box::pin(async move { Ok(v) })
}
}
struct CountingTaskRepo {
created: parking_lot::Mutex<Vec<Task>>,
existing_urls: HashSet<String>,
}
impl CountingTaskRepo {
fn new() -> Self {
Self {
created: parking_lot::Mutex::new(Vec::new()),
existing_urls: HashSet::new(),
}
}
fn new_with_existing(urls: &[&str]) -> Self {
Self {
created: parking_lot::Mutex::new(Vec::new()),
existing_urls: urls.iter().map(|s| s.to_string()).collect(),
}
}
fn created_urls(&self) -> Vec<String> {
self.created.lock().iter().map(|t| t.url.clone()).collect()
}
}
#[async_trait]
impl TaskRepository for CountingTaskRepo {
async fn create(&self, task: &Task) -> Result<Task, RepositoryError> {
self.created.lock().push(task.clone());
Ok(task.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn update(&self, task: &Task) -> Result<Task, RepositoryError> {
Ok(task.clone())
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
Ok(urls
.iter()
.filter(|u| self.existing_urls.contains(*u))
.cloned()
.collect())
}
async fn list_distinct_urls(
&self,
_limit: u32,
_after_url: Option<String>,
) -> Result<Vec<String>, RepositoryError> {
Ok(vec![])
}
async fn reset_stuck_tasks(
&self,
_timeout: chrono::Duration,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
Ok(vec![])
}
async fn query_tasks(
&self,
_params: crate::domain::repositories::task_repository::TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
Ok((vec![], 0))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((vec![], vec![]))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
#[derive(Default)]
struct CountingCrawlRepo {
total_added: AtomicU32,
}
#[async_trait]
impl CrawlRepository for CountingCrawlRepo {
async fn create(
&self,
crawl: &crate::domain::models::Crawl,
) -> Result<crate::domain::models::Crawl, RepositoryError> {
Ok(crawl.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<crate::domain::models::Crawl>, RepositoryError> {
Ok(None)
}
async fn update(
&self,
crawl: &crate::domain::models::Crawl,
) -> Result<crate::domain::models::Crawl, RepositoryError> {
Ok(crawl.clone())
}
async fn increment_completed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn increment_failed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn update_status(
&self,
_id: crate::domain::id::Id,
_status: crate::domain::models::CrawlStatus,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn increment_total_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
self.total_added.fetch_add(1, Ordering::SeqCst);
Ok(())
}
async fn find_by_team_id_paginated(
&self,
_team_id: crate::domain::id::Id,
_limit: u32,
_offset: u32,
) -> Result<Vec<crate::domain::models::Crawl>, RepositoryError> {
Ok(vec![])
}
async fn count_by_team_id(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
}
struct StaticSitemapFetcher {
responses: HashMap<String, (u16, String)>,
}
fn urlset_xml(locs: &[String]) -> String {
let inner: String = locs
.iter()
.map(|l| format!("<url><loc>{l}</loc></url>"))
.collect();
format!(
r#"<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">{inner}</urlset>"#
)
}
#[async_trait]
impl SitemapFetcher for StaticSitemapFetcher {
async fn fetch(
&self,
url: &str,
) -> Result<crate::application::use_cases::map_use_case::SitemapFetch, MapError> {
match self.responses.get(url) {
Some((status, body)) => {
Ok(crate::application::use_cases::map_use_case::SitemapFetch {
status: *status,
body: body.clone(),
})
}
None => Err(MapError::TargetUnreachable(format!("no mock for {url}"))),
}
}
}
fn seed_ctx(crawl_id: crate::domain::id::Id) -> SeedTaskContext {
SeedTaskContext {
team_id: crate::domain::id::Id::generate(),
api_key_id: crate::domain::id::Id::generate(),
crawl_id,
priority: 100,
config_json: json!({"max_depth": 2}),
}
}
fn options(max_urls: usize) -> SitemapStreamOptions {
SitemapStreamOptions {
domain_blacklist: Vec::new(),
channel_capacity: 16,
max_urls,
max_concurrent_sub_sitemaps: 2,
}
}
#[allow(clippy::too_many_arguments)]
async fn feed(
fetcher: StaticSitemapFetcher,
sitemap_url: &str,
include: Option<Vec<String>>,
exclude: Option<Vec<String>>,
blacklist: Vec<String>,
seed: SeedTaskContext,
dedup: &Arc<parking_lot::RwLock<Deduplicator>>,
task_repo: &CountingTaskRepo,
crawl_repo: &CountingCrawlRepo,
cache: &MockCache,
) -> anyhow::Result<SitemapFeedStats> {
feed_sitemap_as_tasks(
Arc::new(fetcher),
sitemap_url.to_string(),
options(50_000),
include,
exclude,
blacklist,
seed,
dedup,
task_repo,
crawl_repo,
cache,
7 * 24 * 3600,
)
.await
}
const SITEMAP_URL: &str = "https://a.com/sitemap.xml";
#[tokio::test]
async fn feed_enqueues_task_rows_and_clears_pointer() {
let locs: Vec<String> = (0..5).map(|i| format!("https://a.com/{i}")).collect();
let fetcher = StaticSitemapFetcher {
responses: HashMap::from([(SITEMAP_URL.to_string(), (200u16, urlset_xml(&locs)))]),
};
let crawl_id = crate::domain::id::Id::generate();
let task_repo = CountingTaskRepo::new();
let crawl_repo = CountingCrawlRepo::default();
let cache = MockCache::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let stats = feed(
fetcher,
SITEMAP_URL,
None,
None,
vec![],
seed_ctx(crawl_id),
&dedup,
&task_repo,
&crawl_repo,
&cache,
)
.await
.expect("feed");
assert_eq!(stats.enqueued, 5);
assert_eq!(stats.consumed, 5);
assert_eq!(stats.filtered_out, 0);
assert_eq!(task_repo.created_urls().len(), 5);
assert_eq!(crawl_repo.total_added.load(Ordering::SeqCst), 5);
let created = task_repo.created.lock();
let first = &created[0];
assert_eq!(first.task_type, TaskType::Crawl);
assert_eq!(first.status, TaskStatus::Queued);
assert_eq!(first.crawl_id, Some(crawl_id));
assert_eq!(first.payload["crawl_id"], crawl_id.to_string());
assert_eq!(first.payload["depth"], 1);
assert_eq!(first.payload["config"], json!({"max_depth": 2}));
assert_eq!(first.team_id, first.team_id);
assert_eq!(
cache.deleted.lock().len(),
1,
"pointer must be deleted after full feed"
);
assert!(!cache
.store
.lock()
.contains_key(&sitemap_feed_pointer_key(crawl_id)));
}
#[tokio::test]
async fn feed_applies_include_exclude_filters() {
let locs: Vec<String> = vec![
"https://a.com/blog/1".into(),
"https://a.com/blog/tag/x".into(),
"https://a.com/page/1".into(),
"https://a.com/blog/2".into(),
];
let fetcher = StaticSitemapFetcher {
responses: HashMap::from([(SITEMAP_URL.to_string(), (200u16, urlset_xml(&locs)))]),
};
let task_repo = CountingTaskRepo::new();
let crawl_repo = CountingCrawlRepo::default();
let cache = MockCache::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let stats = feed(
fetcher,
SITEMAP_URL,
Some(vec!["a\\.com/blog/.*".to_string()]),
Some(vec!["a\\.com/blog/tag/.*".to_string()]),
vec![],
seed_ctx(crate::domain::id::Id::generate()),
&dedup,
&task_repo,
&crawl_repo,
&cache,
)
.await
.expect("feed");
assert_eq!(stats.enqueued, 2, "only /blog/* minus /tag/* enqueue");
assert_eq!(stats.filtered_out, 2);
let mut urls = task_repo.created_urls();
urls.sort();
assert_eq!(urls, vec!["https://a.com/blog/1", "https://a.com/blog/2"]);
}
#[tokio::test]
async fn feed_empty_include_blocks_all() {
let locs: Vec<String> = (0..3).map(|i| format!("https://a.com/{i}")).collect();
let fetcher = StaticSitemapFetcher {
responses: HashMap::from([(SITEMAP_URL.to_string(), (200u16, urlset_xml(&locs)))]),
};
let task_repo = CountingTaskRepo::new();
let crawl_repo = CountingCrawlRepo::default();
let cache = MockCache::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let stats = feed(
fetcher,
SITEMAP_URL,
Some(vec![]),
None,
vec![],
seed_ctx(crate::domain::id::Id::generate()),
&dedup,
&task_repo,
&crawl_repo,
&cache,
)
.await
.expect("feed");
assert_eq!(stats.enqueued, 0);
assert_eq!(stats.filtered_out, 3);
}
#[tokio::test]
async fn feed_domain_blacklist_blocks() {
let locs: Vec<String> = vec!["https://a.com/ok".into(), "https://evil.com/bad".into()];
let fetcher = StaticSitemapFetcher {
responses: HashMap::from([(SITEMAP_URL.to_string(), (200u16, urlset_xml(&locs)))]),
};
let task_repo = CountingTaskRepo::new();
let crawl_repo = CountingCrawlRepo::default();
let cache = MockCache::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let stats = feed(
fetcher,
SITEMAP_URL,
None,
None,
vec!["evil.com".to_string()],
seed_ctx(crate::domain::id::Id::generate()),
&dedup,
&task_repo,
&crawl_repo,
&cache,
)
.await
.expect("feed");
assert_eq!(task_repo.created_urls(), vec!["https://a.com/ok"]);
assert_eq!(stats.filtered_out, 1);
}
#[tokio::test]
async fn feed_dedup_layers_skip_known_urls() {
let locs: Vec<String> = vec![
"https://a.com/in-process".into(), "https://a.com/bloom-fp".into(), "https://a.com/fresh".into(), ];
let fetcher = StaticSitemapFetcher {
responses: HashMap::from([(SITEMAP_URL.to_string(), (200u16, urlset_xml(&locs)))]),
};
let task_repo = CountingTaskRepo::new_with_existing(&["https://a.com/in-process"]);
let crawl_repo = CountingCrawlRepo::default();
let cache = MockCache::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
{
let mut d = dedup.write();
d.insert("https://a.com/in-process");
d.insert("https://a.com/bloom-fp");
}
let stats = feed(
fetcher,
SITEMAP_URL,
None,
None,
vec![],
seed_ctx(crate::domain::id::Id::generate()),
&dedup,
&task_repo,
&crawl_repo,
&cache,
)
.await
.expect("feed");
assert_eq!(
stats.enqueued, 2,
"in-process URL skipped, bloom false positive enqueued"
);
assert_eq!(stats.duplicates_skipped, 1);
let mut urls = task_repo.created_urls();
urls.sort();
assert_eq!(urls, vec!["https://a.com/bloom-fp", "https://a.com/fresh"]);
}
#[tokio::test]
async fn feed_resumes_from_pointer_seek() {
let locs: Vec<String> = (0..10).map(|i| format!("https://a.com/{i}")).collect();
let xml = urlset_xml(&locs);
let crawl_id = crate::domain::id::Id::generate();
let cache = MockCache::default();
let pointer = SitemapFeedPointer {
sitemap_url: SITEMAP_URL.to_string(),
consumed: 3,
};
cache
.set(
&sitemap_feed_pointer_key(crawl_id),
&serde_json::to_string(&pointer).unwrap(),
3600,
)
.await
.unwrap();
let fetcher = StaticSitemapFetcher {
responses: HashMap::from([(SITEMAP_URL.to_string(), (200u16, xml))]),
};
let task_repo = CountingTaskRepo::new();
let crawl_repo = CountingCrawlRepo::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let stats = feed(
fetcher,
SITEMAP_URL,
None,
None,
vec![],
seed_ctx(crawl_id),
&dedup,
&task_repo,
&crawl_repo,
&cache,
)
.await
.expect("resumed feed");
assert_eq!(stats.resumed_from, Some(3), "must seek past consumed=3");
assert_eq!(stats.consumed, 10, "total consumed includes sought locs");
assert_eq!(stats.enqueued, 7, "only locs 3..10 enqueue after resume");
assert_eq!(
task_repo.created_urls(),
(3..10)
.map(|i| format!("https://a.com/{i}"))
.collect::<Vec<_>>()
);
assert!(!cache
.store
.lock()
.contains_key(&sitemap_feed_pointer_key(crawl_id)));
}
#[tokio::test]
async fn feed_ignores_pointer_for_different_sitemap_url() {
let locs: Vec<String> = vec!["https://a.com/x".into(), "https://a.com/y".into()];
let fetcher = StaticSitemapFetcher {
responses: HashMap::from([(SITEMAP_URL.to_string(), (200u16, urlset_xml(&locs)))]),
};
let crawl_id = crate::domain::id::Id::generate();
let cache = MockCache::default();
let pointer = SitemapFeedPointer {
sitemap_url: "https://a.com/old-sitemap.xml".to_string(),
consumed: 99,
};
cache
.set(
&sitemap_feed_pointer_key(crawl_id),
&serde_json::to_string(&pointer).unwrap(),
3600,
)
.await
.unwrap();
let task_repo = CountingTaskRepo::new();
let crawl_repo = CountingCrawlRepo::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let stats = feed(
fetcher,
SITEMAP_URL,
None,
None,
vec![],
seed_ctx(crawl_id),
&dedup,
&task_repo,
&crawl_repo,
&cache,
)
.await
.expect("feed");
assert_eq!(
stats.resumed_from, None,
"mismatched pointer must be ignored"
);
assert_eq!(stats.enqueued, 2);
}
#[tokio::test]
async fn feed_survives_corrupt_pointer() {
let locs: Vec<String> = vec!["https://a.com/only".into()];
let fetcher = StaticSitemapFetcher {
responses: HashMap::from([(SITEMAP_URL.to_string(), (200u16, urlset_xml(&locs)))]),
};
let crawl_id = crate::domain::id::Id::generate();
let cache = MockCache::default();
cache
.set(&sitemap_feed_pointer_key(crawl_id), "{{not json", 3600)
.await
.unwrap();
let task_repo = CountingTaskRepo::new();
let crawl_repo = CountingCrawlRepo::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let stats = feed(
fetcher,
SITEMAP_URL,
None,
None,
vec![],
seed_ctx(crawl_id),
&dedup,
&task_repo,
&crawl_repo,
&cache,
)
.await
.expect("corrupt pointer must not fail the feed");
assert_eq!(stats.resumed_from, None);
assert_eq!(stats.enqueued, 1);
}
#[tokio::test]
async fn feed_sitemap_404_is_legal_empty_result() {
let fetcher = StaticSitemapFetcher {
responses: HashMap::from([(SITEMAP_URL.to_string(), (404u16, "not found".into()))]),
};
let task_repo = CountingTaskRepo::new();
let crawl_repo = CountingCrawlRepo::default();
let cache = MockCache::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let stats = feed(
fetcher,
SITEMAP_URL,
None,
None,
vec![],
seed_ctx(crate::domain::id::Id::generate()),
&dedup,
&task_repo,
&crawl_repo,
&cache,
)
.await
.expect("404 must be Ok (legal empty)");
assert_eq!(stats.enqueued, 0);
assert_eq!(stats.consumed, 0);
}
#[tokio::test]
async fn feed_5xx_propagates_error() {
let fetcher = StaticSitemapFetcher {
responses: HashMap::from([(SITEMAP_URL.to_string(), (500u16, "boom".into()))]),
};
let task_repo = CountingTaskRepo::new();
let crawl_repo = CountingCrawlRepo::default();
let cache = MockCache::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let result = feed(
fetcher,
SITEMAP_URL,
None,
None,
vec![],
seed_ctx(crate::domain::id::Id::generate()),
&dedup,
&task_repo,
&crawl_repo,
&cache,
)
.await;
assert!(result.is_err(), "5xx must propagate as error");
}
#[tokio::test]
async fn feed_respects_max_urls_cap() {
let locs: Vec<String> = (0..50).map(|i| format!("https://a.com/{i}")).collect();
let fetcher = StaticSitemapFetcher {
responses: HashMap::from([(SITEMAP_URL.to_string(), (200u16, urlset_xml(&locs)))]),
};
let task_repo = CountingTaskRepo::new();
let crawl_repo = CountingCrawlRepo::default();
let cache = MockCache::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let stats = feed_sitemap_as_tasks(
Arc::new(fetcher),
SITEMAP_URL.to_string(),
options(10),
None,
None,
vec![],
seed_ctx(crate::domain::id::Id::generate()),
&dedup,
&task_repo,
&crawl_repo,
&cache,
3600,
)
.await
.expect("feed");
assert_eq!(stats.enqueued, 10);
assert_eq!(stats.consumed, 10);
}
#[tokio::test]
async fn feed_persists_pointer_with_consumed_progress() {
let locs: Vec<String> = (0..7).map(|i| format!("https://a.com/{i}")).collect();
let fetcher = StaticSitemapFetcher {
responses: HashMap::from([(SITEMAP_URL.to_string(), (200u16, urlset_xml(&locs)))]),
};
let crawl_id = crate::domain::id::Id::generate();
let task_repo = CountingTaskRepo::new();
let crawl_repo = CountingCrawlRepo::default();
let cache = MockCache::default();
let dedup = Arc::new(parking_lot::RwLock::new(Deduplicator::new()));
let _ = feed(
fetcher,
SITEMAP_URL,
None,
None,
vec![],
seed_ctx(crawl_id),
&dedup,
&task_repo,
&crawl_repo,
&cache,
)
.await
.expect("feed");
let key = sitemap_feed_pointer_key(crawl_id);
assert!(
cache.set_calls(&key) >= 1,
"pointer must be persisted at least once"
);
assert!(cache.deleted.lock().contains(&key));
}
#[test]
fn derive_sitemap_url_from_seed() {
assert_eq!(
derive_sitemap_url("https://a.com"),
Some("https://a.com/sitemap.xml".to_string())
);
assert_eq!(
derive_sitemap_url("https://a.com/some/page?q=1"),
Some("https://a.com/sitemap.xml".to_string())
);
assert_eq!(
derive_sitemap_url("http://localhost:8899"),
Some("http://localhost:8899/sitemap.xml".to_string())
);
assert_eq!(derive_sitemap_url("not-a-url"), None);
}
#[test]
fn explicit_sitemap_url_takes_precedence_by_caller_contract() {
let explicit = Some("https://cdn.b.com/sitemap-index.xml".to_string());
let resolved = explicit
.or_else(|| derive_sitemap_url("https://a.com"))
.expect("explicit wins");
assert_eq!(resolved, "https://cdn.b.com/sitemap-index.xml");
}
}