#[cfg(feature = "auth")]
pub mod api_key_handler;
pub mod audit_handler;
pub mod crawl_commands;
pub mod crawl_handler;
pub mod crawl_queries;
mod credits;
pub mod extract_handler;
#[cfg(feature = "sitemap")]
pub mod map_handler;
pub mod metrics_handler;
pub mod response_builder;
pub mod scrape_commands;
pub mod scrape_handler;
pub mod scrape_queries;
pub mod search_handler;
pub mod searxng_handler;
pub mod task_commands;
pub mod task_handler;
pub mod task_queries;
#[cfg(feature = "teams")]
pub mod team_handler;
#[cfg(feature = "webhook")]
pub mod webhook_handler;
use crate::domain::models::Task;
use axum::http::StatusCode;
use axum::response::Response;
use std::collections::HashMap;
use crate::infrastructure::security::ssrf::validate_url;
use crate::presentation::handlers::response_builder::errors;
#[inline]
pub fn extract_task_ids(tasks: &[Task]) -> Vec<crate::domain::id::Id> {
tasks.iter().map(|task| task.id).collect()
}
#[inline]
pub fn tasks_to_id_map(tasks: Vec<Task>) -> HashMap<crate::domain::id::Id, Task> {
tasks.into_iter().map(|task| (task.id, task)).collect()
}
pub async fn check_ssrf_url(
url: &str,
team_id: crate::domain::id::Id,
api_key_id: crate::domain::id::Id,
) -> Option<Response> {
match validate_url(url).await {
Ok(validated) => {
log::trace!(
"URL passed SSRF validation url={} team_id={} resolved_ips={:?}",
crate::utils::url_redaction::redact_url_for_log(url),
team_id,
validated.resolved_ips
);
None
}
Err(e) => {
log::warn!(
"SSRF attack attempt blocked url={} team_id={} api_key_id={} error={}",
crate::utils::url_redaction::redact_url_for_log(url),
team_id,
api_key_id,
e
);
Some(errors::bad_request(format!(
"SSRF protection: {}",
e.kind()
)))
}
}
}
pub async fn check_ssrf_urls_batch(
urls: &[String],
team_id: crate::domain::id::Id,
api_key_id: crate::domain::id::Id,
) -> Option<Response> {
let results = futures::future::join_all(urls.iter().map(|url| validate_url(url))).await;
for (url, result) in urls.iter().zip(results) {
if let Err(e) = result {
log::warn!(
"SSRF attack attempt blocked url={} team_id={} api_key_id={} error={}",
crate::utils::url_redaction::redact_url_for_log(url),
team_id,
api_key_id,
e
);
return Some(errors::bad_request(format!(
"SSRF protection: {}",
e.kind()
)));
}
}
None
}
#[inline]
pub fn sync_wait_status_code(sync_wait_ms: u32, is_timeout: bool) -> StatusCode {
if sync_wait_ms > 0 && is_timeout {
StatusCode::ACCEPTED
} else {
StatusCode::CREATED
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::models::{Task, TaskStatus, TaskType};
use chrono::Utc;
fn make_task(id: crate::domain::id::Id) -> Task {
let now = Utc::now();
Task {
id,
task_type: TaskType::Scrape,
status: TaskStatus::Queued,
priority: 0,
team_id: crate::domain::id::Id::generate(),
api_key_id: crate::domain::id::Id::generate(),
url: "https://example.com".to_string(),
payload: serde_json::json!({}),
retry_count: 0,
attempt_count: 0,
max_retries: 3,
scheduled_at: None,
expires_at: None,
created_at: now,
started_at: None,
completed_at: None,
crawl_id: None,
updated_at: now,
lock_token: None,
lock_expires_at: None,
}
}
#[test]
fn test_extract_task_ids_returns_ids_in_order() {
let id1 = crate::domain::id::Id::generate();
let id2 = crate::domain::id::Id::generate();
let id3 = crate::domain::id::Id::generate();
let tasks = vec![make_task(id1), make_task(id2), make_task(id3)];
let ids = extract_task_ids(&tasks);
assert_eq!(ids, vec![id1, id2, id3]);
}
#[test]
fn test_extract_task_ids_empty_slice_returns_empty_vec() {
let tasks: Vec<Task> = vec![];
let ids = extract_task_ids(&tasks);
assert!(ids.is_empty());
}
#[test]
fn test_extract_task_ids_single_element() {
let id = crate::domain::id::Id::generate();
let tasks = vec![make_task(id)];
let ids = extract_task_ids(&tasks);
assert_eq!(ids, vec![id]);
}
#[test]
fn test_extract_task_ids_does_not_consume_input() {
let id1 = crate::domain::id::Id::generate();
let id2 = crate::domain::id::Id::generate();
let tasks = vec![make_task(id1), make_task(id2)];
let ids = extract_task_ids(&tasks);
assert_eq!(ids.len(), 2);
assert_eq!(tasks.len(), 2);
assert_eq!(tasks[0].id, id1);
assert_eq!(tasks[1].id, id2);
}
#[test]
fn test_tasks_to_id_map_builds_correct_mapping() {
let id1 = crate::domain::id::Id::generate();
let id2 = crate::domain::id::Id::generate();
let tasks = vec![make_task(id1), make_task(id2)];
let map = tasks_to_id_map(tasks);
assert_eq!(map.len(), 2);
assert!(map.contains_key(&id1));
assert!(map.contains_key(&id2));
assert_eq!(map.get(&id1).unwrap().id, id1);
assert_eq!(map.get(&id2).unwrap().id, id2);
}
#[test]
fn test_tasks_to_id_map_empty_input_returns_empty_map() {
let tasks: Vec<Task> = vec![];
let map = tasks_to_id_map(tasks);
assert!(map.is_empty());
}
#[test]
fn test_tasks_to_id_map_single_element() {
let id = crate::domain::id::Id::generate();
let tasks = vec![make_task(id)];
let map = tasks_to_id_map(tasks);
assert_eq!(map.len(), 1);
assert_eq!(map.get(&id).unwrap().id, id);
}
#[test]
fn test_tasks_to_id_map_duplicate_ids_last_wins() {
let id = crate::domain::id::Id::generate();
let mut task1 = make_task(id);
task1.url = "https://first.com".to_string();
let mut task2 = make_task(id);
task2.url = "https://second.com".to_string();
let tasks = vec![task1, task2];
let map = tasks_to_id_map(tasks);
assert_eq!(map.len(), 1);
assert_eq!(map.get(&id).unwrap().url, "https://second.com");
}
}