use crate::application::dto::crawl_request::CrawlRequestDto;
use crate::domain::models::{Crawl, CrawlStatus, Task, TaskStatus, TaskType};
use crate::domain::repositories::crawl_repository::CrawlRepository;
use chrono::{DateTime, Utc};
use serde_json::json;
use std::fmt::Display;
#[derive(Debug)]
pub enum PersistCrawlError<E> {
Repository(crate::domain::repositories::task_repository::RepositoryError),
Enqueue(E),
}
impl<E: Display> Display for PersistCrawlError<E> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Repository(e) => write!(f, "crawl create failed: {e}"),
Self::Enqueue(e) => write!(f, "initial task enqueue failed: {e}"),
}
}
}
pub fn build_initial_crawl_task(
team_id: crate::domain::id::Id,
api_key_id: crate::domain::id::Id,
crawl_id: crate::domain::id::Id,
dto: &CrawlRequestDto,
now: DateTime<Utc>,
) -> Task {
Task {
id: crate::domain::id::Id::generate(),
task_type: TaskType::Crawl,
status: TaskStatus::Queued,
priority: 100,
team_id,
api_key_id,
url: dto.url.clone(),
payload: json!({
"crawl_id": crawl_id,
"depth": 0,
"config": dto.config,
"sitemap_seed": dto.sitemap_seed,
"sitemap_url": dto.sitemap_url,
}),
retry_count: 0,
attempt_count: 0,
max_retries: 3,
scheduled_at: None,
created_at: now,
started_at: None,
completed_at: None,
crawl_id: Some(crawl_id),
updated_at: now,
lock_token: None,
lock_expires_at: None,
expires_at: dto.expires_at,
}
}
pub fn build_crawl_entity_with_root(
crawl_id: crate::domain::id::Id,
team_id: crate::domain::id::Id,
dto: &CrawlRequestDto,
root_url: &str,
now: DateTime<Utc>,
) -> Crawl {
Crawl::with_all_fields(
crawl_id,
team_id,
dto.name
.clone()
.unwrap_or_else(|| "Untitled Crawl".to_string()),
root_url.to_string(),
dto.url.clone(),
CrawlStatus::Queued,
json!(dto.config),
1, 0, 0, now,
now,
None,
)
}
#[allow(clippy::too_many_arguments)] pub async fn persist_crawl_with_initial_task<F, Fut, E>(
crawl_repo: &dyn CrawlRepository,
team_id: crate::domain::id::Id,
api_key_id: crate::domain::id::Id,
dto: &CrawlRequestDto,
root_url: &str,
domain_blacklist: Vec<String>,
now: DateTime<Utc>,
enqueue: F,
) -> Result<Crawl, PersistCrawlError<E>>
where
F: FnOnce(Task) -> Fut,
Fut: std::future::Future<Output = Result<Task, E>>,
E: Display,
{
let crawl_id = crate::domain::id::Id::generate();
let mut crawl = build_crawl_entity_with_root(crawl_id, team_id, dto, root_url, now);
if let Some(obj) = crawl.config_mut().as_object_mut() {
obj.insert(
"domain_blacklist".to_string(),
serde_json::json!(domain_blacklist),
);
} else {
log::error!(
"crawl {} config is not a JSON object; domain_blacklist snapshot not stored",
crawl_id
);
}
crawl_repo
.create(&crawl)
.await
.map_err(PersistCrawlError::Repository)?;
let task = build_initial_crawl_task(team_id, api_key_id, crawl_id, dto, now);
if let Err(e) = enqueue(task).await {
if let Err(update_err) = crawl_repo.finalize_failed_after_enqueue(crawl_id).await {
log::error!(
"Failed to mark crawl {} failed during enqueue compensation: {}",
crawl_id,
update_err
);
}
return Err(PersistCrawlError::Enqueue(e));
}
Ok(crawl)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::application::dto::crawl_request::CrawlConfigDto;
fn make_dto() -> CrawlRequestDto {
CrawlRequestDto {
url: "https://example.com".to_string(),
validated_url: None,
name: Some("Test Crawl".to_string()),
config: CrawlConfigDto {
max_depth: 3,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
},
sync_wait_ms: None,
expires_at: None,
sitemap_seed: None,
sitemap_url: None,
}
}
#[test]
fn test_build_initial_crawl_task() {
let team_id = crate::domain::id::Id::generate();
let api_key_id = crate::domain::id::Id::generate();
let crawl_id = crate::domain::id::Id::generate();
let dto = make_dto();
let now = Utc::now();
let task = build_initial_crawl_task(team_id, api_key_id, crawl_id, &dto, now);
assert_eq!(task.team_id, team_id);
assert_eq!(task.api_key_id, api_key_id);
assert_eq!(task.task_type, TaskType::Crawl);
assert_eq!(task.status, TaskStatus::Queued);
assert_eq!(task.crawl_id, Some(crawl_id));
assert_eq!(task.url, "https://example.com");
assert_eq!(task.payload["crawl_id"], crawl_id.to_string());
assert_eq!(task.payload["depth"], 0);
}
#[test]
fn test_build_initial_crawl_task_transfers_sitemap_source_fields() {
let mut dto = make_dto();
dto.sitemap_seed = Some(true);
dto.sitemap_url = Some("https://cdn.b.com/sitemap-index.xml".to_string());
let task = build_initial_crawl_task(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
&dto,
Utc::now(),
);
assert_eq!(task.payload["sitemap_seed"], true);
assert_eq!(
task.payload["sitemap_url"],
"https://cdn.b.com/sitemap-index.xml"
);
}
#[test]
fn test_build_crawl_entity() {
let crawl_id = crate::domain::id::Id::generate();
let team_id = crate::domain::id::Id::generate();
let dto = make_dto();
let now = Utc::now();
let crawl = build_crawl_entity_with_root(crawl_id, team_id, &dto, &dto.url, now);
assert_eq!(crawl.id, crawl_id);
assert_eq!(crawl.team_id, team_id);
assert_eq!(crawl.status, CrawlStatus::Queued);
}
#[test]
fn test_build_crawl_entity_default_name() {
let crawl_id = crate::domain::id::Id::generate();
let team_id = crate::domain::id::Id::generate();
let mut dto = make_dto();
dto.name = None;
let now = Utc::now();
let crawl = build_crawl_entity_with_root(crawl_id, team_id, &dto, &dto.url, now);
assert_eq!(crawl.id, crawl_id);
}
}