pub(crate) use super::crawl_commands::{cancel_crawl, create_crawl};
pub(crate) use super::crawl_queries::{get_crawl, get_crawl_results};
#[cfg(test)]
mod tests {
use super::*;
use crate::application::dto::crawl_request::{CrawlConfigDto, CrawlRequestDto};
use crate::application::use_cases::crawl_use_case::CrawlUseCaseError;
use crate::common::constants::crawl_task::{CRAWL_TASK_CREDITS_COST, DEFAULT_TIMEOUT_MS};
use crate::common::test_helpers::create_test_db_pool;
use crate::domain::repositories::task_repository::RepositoryError;
use crate::i18n::{I18nBundle, Locale};
use crate::presentation::handlers::task_queries::SyncWaitResult;
use crate::presentation::middleware::auth_middleware::AuthState;
use crate::presentation::state::CrawlHandlerState;
use axum::{
extract::{ConnectInfo, Extension, Path},
http::StatusCode,
response::IntoResponse,
Json,
};
use chrono::Datelike;
use sdforge::validator::Validate;
use std::net::SocketAddr;
use std::sync::Arc;
#[test]
fn test_validation_error_maps_to_bad_request() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let err = CrawlUseCaseError::ValidationError("max_depth exceeds limit".to_string());
let (status, msg) = <(StatusCode, String)>::from(err);
assert_eq!(status, StatusCode::BAD_REQUEST);
assert_eq!(msg, "max_depth exceeds limit");
}
#[test]
fn test_repository_database_error_maps_to_internal_server_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let err = CrawlUseCaseError::Repository(RepositoryError::Database(anyhow::anyhow!(
"connection refused"
)));
let (status, msg) = <(StatusCode, String)>::from(err);
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(msg, "Internal server error");
}
#[test]
fn test_repository_not_found_maps_to_internal_server_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let err = CrawlUseCaseError::Repository(RepositoryError::NotFound);
let (status, _msg) = <(StatusCode, String)>::from(err);
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR);
}
#[test]
fn test_not_found_maps_to_404() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let err = CrawlUseCaseError::NotFound;
let (status, msg) = <(StatusCode, String)>::from(err);
assert_eq!(status, StatusCode::NOT_FOUND);
assert_eq!(msg, "Crawl not found");
}
#[test]
fn test_anyhow_error_maps_to_internal_server_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let err = CrawlUseCaseError::Anyhow(anyhow::anyhow!("unexpected failure"));
let (status, msg) = <(StatusCode, String)>::from(err);
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(msg, "Internal server error");
}
#[test]
fn test_anyhow_error_with_context_preserved() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let err = CrawlUseCaseError::Anyhow(
anyhow::anyhow!("base error").context("with additional context"),
);
let (status, msg) = <(StatusCode, String)>::from(err);
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(msg, "Internal server error");
}
#[test]
fn test_crawl_request_dto_minimal() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{
"url": "https://example.com",
"config": {"max_depth": 2}
}"#;
let dto: CrawlRequestDto = serde_json::from_str(json).unwrap();
assert_eq!(dto.url, "https://example.com");
assert_eq!(dto.config.max_depth, 2);
assert!(dto.name.is_none());
assert!(dto.sync_wait_ms.is_none());
assert!(dto.expires_at.is_none());
}
#[test]
fn test_crawl_request_dto_full() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{
"url": "https://example.com",
"name": "My Crawl",
"config": {
"max_depth": 3,
"include_patterns": ["*/blog/*"],
"exclude_patterns": ["*/admin/*"],
"strategy": "bfs",
"crawl_delay_ms": 1000,
"max_concurrency": 5,
"proxy": "http://proxy:8080",
"headers": {"X-Custom": "value"}
},
"sync_wait_ms": 5000
}"#;
let dto: CrawlRequestDto = serde_json::from_str(json).unwrap();
assert_eq!(dto.url, "https://example.com");
assert_eq!(dto.name.as_deref(), Some("My Crawl"));
assert_eq!(dto.config.max_depth, 3);
assert!(dto.config.include_patterns.is_some());
assert!(dto.config.exclude_patterns.is_some());
assert_eq!(dto.config.strategy.as_deref(), Some("bfs"));
assert_eq!(dto.config.crawl_delay_ms, Some(1000));
assert_eq!(dto.config.max_concurrency, Some(5));
assert!(dto.config.proxy.is_some());
assert!(dto.config.headers.is_some());
assert_eq!(dto.sync_wait_ms, Some(5000));
}
#[test]
fn test_crawl_request_dto_with_max_depth_zero() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"url":"https://example.com","config":{"max_depth":0}}"#;
let dto: CrawlRequestDto = serde_json::from_str(json).unwrap();
assert_eq!(dto.config.max_depth, 0);
}
#[test]
fn test_crawl_request_dto_with_max_depth_five() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"url":"https://example.com","config":{"max_depth":5}}"#;
let dto: CrawlRequestDto = serde_json::from_str(json).unwrap();
assert_eq!(dto.config.max_depth, 5);
}
#[test]
fn test_crawl_request_dto_deny_unknown_fields() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"url":"https://example.com","config":{"max_depth":2},"unknown_field":42}"#;
let result: Result<CrawlRequestDto, _> = serde_json::from_str(json);
assert!(result.is_err(), "unknown fields should be rejected");
}
#[test]
fn test_crawl_config_dto_deny_unknown_fields() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"max_depth":2,"unknown":true}"#;
let result: Result<CrawlConfigDto, _> = serde_json::from_str(json);
assert!(result.is_err());
}
#[test]
fn test_max_depth_at_boundary_passes() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let config = CrawlConfigDto {
max_depth: 5,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
};
assert!(config.max_depth <= 5, "max_depth of 5 should pass");
}
#[test]
fn test_max_depth_exceeds_limit_fails() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let config = CrawlConfigDto {
max_depth: 6,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
};
assert!(config.max_depth > 5, "max_depth of 6 should fail");
}
#[test]
fn test_max_depth_zero_passes() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let config = CrawlConfigDto {
max_depth: 0,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
};
assert!(config.max_depth <= 5);
}
#[test]
fn test_crawl_config_dto_clone() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let config = CrawlConfigDto {
max_depth: 3,
include_patterns: Some(vec!["/blog/*".to_string()]),
exclude_patterns: Some(vec!["/admin/*".to_string()]),
strategy: Some("dfs".to_string()),
crawl_delay_ms: Some(500),
max_concurrency: Some(10),
max_pages: None,
proxy: Some("http://proxy:8080".to_string()),
headers: Some(serde_json::json!({"Accept": "text/html"})),
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
};
let cloned = config.clone();
assert_eq!(cloned.max_depth, 3);
assert_eq!(cloned.include_patterns, config.include_patterns);
assert_eq!(cloned.exclude_patterns, config.exclude_patterns);
assert_eq!(cloned.strategy, config.strategy);
assert_eq!(cloned.crawl_delay_ms, config.crawl_delay_ms);
assert_eq!(cloned.max_concurrency, config.max_concurrency);
assert_eq!(cloned.proxy, config.proxy);
}
#[test]
fn test_crawl_config_dto_serialization_roundtrip() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let config = CrawlConfigDto {
max_depth: 2,
include_patterns: Some(vec!["/api/*".to_string()]),
exclude_patterns: None,
strategy: Some("bfs".to_string()),
crawl_delay_ms: None,
max_concurrency: Some(20),
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
};
let json = serde_json::to_string(&config).unwrap();
let deserialized: CrawlConfigDto = serde_json::from_str(&json).unwrap();
assert_eq!(deserialized.max_depth, 2);
assert_eq!(deserialized.include_patterns, config.include_patterns);
assert_eq!(deserialized.strategy, config.strategy);
assert_eq!(deserialized.max_concurrency, config.max_concurrency);
}
#[test]
fn test_crawl_config_dto_debug() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let config = CrawlConfigDto {
max_depth: 1,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
};
let debug = format!("{:?}", config);
assert!(debug.contains("CrawlConfigDto"));
assert!(debug.contains("max_depth"));
assert!(debug.contains("1"));
}
#[test]
fn test_crawl_request_dto_validate_success() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = CrawlRequestDto {
url: "https://example.com".to_string(),
validated_url: None,
name: Some("test".to_string()),
config: CrawlConfigDto {
max_depth: 3,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
},
sync_wait_ms: Some(5000),
expires_at: None,
sitemap_seed: None,
sitemap_url: None,
};
assert!(dto.validate().is_ok());
}
#[test]
fn test_crawl_request_dto_validate_empty_url_fails() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = CrawlRequestDto {
url: "".to_string(),
validated_url: None,
name: None,
config: CrawlConfigDto {
max_depth: 1,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
},
sync_wait_ms: None,
expires_at: None,
sitemap_seed: None,
sitemap_url: None,
};
assert!(dto.validate().is_err());
}
#[test]
fn test_crawl_request_dto_validate_sync_wait_ms_too_large_fails() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = CrawlRequestDto {
url: "https://example.com".to_string(),
validated_url: None,
name: None,
config: CrawlConfigDto {
max_depth: 1,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
},
sync_wait_ms: Some(30001),
expires_at: None,
sitemap_seed: None,
sitemap_url: None,
};
assert!(dto.validate().is_err());
}
#[test]
fn test_crawl_request_dto_validate_sync_wait_ms_zero_passes() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = CrawlRequestDto {
url: "https://example.com".to_string(),
validated_url: None,
name: None,
config: CrawlConfigDto {
max_depth: 1,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
},
sync_wait_ms: Some(0),
expires_at: None,
sitemap_seed: None,
sitemap_url: None,
};
assert!(dto.validate().is_ok());
}
#[test]
fn test_sync_wait_result_timeout() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let result = SyncWaitResult {
waited_time_ms: 5000,
is_timeout: true,
};
assert_eq!(result.waited_time_ms, 5000);
assert!(result.is_timeout);
}
#[test]
fn test_sync_wait_result_no_timeout() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let result = SyncWaitResult {
waited_time_ms: 0,
is_timeout: false,
};
assert_eq!(result.waited_time_ms, 0);
assert!(!result.is_timeout);
}
#[test]
fn test_status_code_accepted_when_timeout() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let sync_wait_ms = 5000u32;
let wait_result = SyncWaitResult {
waited_time_ms: 5000,
is_timeout: true,
};
let status_code = if sync_wait_ms > 0 && wait_result.is_timeout {
StatusCode::ACCEPTED
} else {
StatusCode::CREATED
};
assert_eq!(status_code, StatusCode::ACCEPTED);
}
#[test]
fn test_status_code_created_when_no_timeout() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let sync_wait_ms = 5000u32;
let wait_result = SyncWaitResult {
waited_time_ms: 1000,
is_timeout: false,
};
let status_code = if sync_wait_ms > 0 && wait_result.is_timeout {
StatusCode::ACCEPTED
} else {
StatusCode::CREATED
};
assert_eq!(status_code, StatusCode::CREATED);
}
#[test]
fn test_status_code_created_when_sync_wait_zero() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let sync_wait_ms = 0u32;
let wait_result = SyncWaitResult {
waited_time_ms: 0,
is_timeout: true, };
let status_code = if sync_wait_ms > 0 && wait_result.is_timeout {
StatusCode::ACCEPTED
} else {
StatusCode::CREATED
};
assert_eq!(status_code, StatusCode::CREATED);
}
#[test]
fn test_validation_error_empty_message() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let err = CrawlUseCaseError::ValidationError(String::new());
let (status, msg) = <(StatusCode, String)>::from(err);
assert_eq!(status, StatusCode::BAD_REQUEST);
assert!(msg.is_empty());
}
#[test]
fn test_anyhow_error_chained_context() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let err = CrawlUseCaseError::Anyhow(
anyhow::anyhow!("root cause")
.context("middle context")
.context("outer context"),
);
let (status, msg) = <(StatusCode, String)>::from(err);
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(msg, "Internal server error");
}
#[test]
fn test_repository_database_error_with_complex_message() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let err = CrawlUseCaseError::Repository(RepositoryError::Database(anyhow::anyhow!(
"connection pool exhausted after 30s timeout"
)));
let (status, msg) = <(StatusCode, String)>::from(err);
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(msg, "Internal server error");
}
#[test]
fn test_crawl_task_credits_cost_value() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
assert_eq!(CRAWL_TASK_CREDITS_COST, 10);
}
#[test]
fn test_default_timeout_ms_constant() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
assert_eq!(DEFAULT_TIMEOUT_MS, 5000);
}
#[test]
#[allow(clippy::unnecessary_literal_unwrap)]
fn test_sync_wait_ms_defaults_to_default_timeout_when_none() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let payload_sync_wait_ms: Option<u32> = None;
let sync_wait_ms = payload_sync_wait_ms.unwrap_or(DEFAULT_TIMEOUT_MS as u32);
assert_eq!(sync_wait_ms, DEFAULT_TIMEOUT_MS as u32);
assert_eq!(sync_wait_ms, 5000);
}
#[test]
fn test_sync_wait_ms_uses_custom_value_when_some() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let sync_wait_ms = 10000;
assert_eq!(sync_wait_ms, 10000);
}
#[test]
fn test_sync_wait_ms_zero_uses_zero() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let sync_wait_ms = 0;
assert_eq!(sync_wait_ms, 0);
}
#[test]
fn test_max_depth_six_fails_handler_check() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let max_depth: u32 = 6;
assert!(max_depth > 5, "max_depth of 6 should fail handler check");
}
#[test]
fn test_max_depth_five_passes_handler_check() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let max_depth: u32 = 5;
assert!(max_depth <= 5, "max_depth of 5 should pass handler check");
}
#[test]
fn test_max_depth_zero_passes_handler_check() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let max_depth: u32 = 0;
assert!(max_depth <= 5);
}
#[test]
fn test_max_depth_one_passes_handler_check() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let max_depth: u32 = 1;
assert!(max_depth <= 5);
}
#[test]
fn test_crawl_request_dto_with_expires_at() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{
"url": "https://example.com",
"config": {"max_depth": 2},
"expires_at": "2025-12-31T23:59:59Z"
}"#;
let dto: CrawlRequestDto = serde_json::from_str(json).unwrap();
assert!(dto.expires_at.is_some());
let expires = dto.expires_at.unwrap();
assert_eq!(expires.year(), 2025);
assert_eq!(expires.month(), 12);
assert_eq!(expires.day(), 31);
}
#[test]
fn test_crawl_request_dto_with_name() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{
"url": "https://example.com",
"name": "My Crawl Task",
"config": {"max_depth": 1}
}"#;
let dto: CrawlRequestDto = serde_json::from_str(json).unwrap();
assert_eq!(dto.name.as_deref(), Some("My Crawl Task"));
}
#[test]
fn test_crawl_config_dto_minimal() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"max_depth": 1}"#;
let config: CrawlConfigDto = serde_json::from_str(json).unwrap();
assert_eq!(config.max_depth, 1);
assert!(config.include_patterns.is_none());
assert!(config.exclude_patterns.is_none());
assert!(config.strategy.is_none());
assert!(config.crawl_delay_ms.is_none());
assert!(config.max_concurrency.is_none());
assert!(config.proxy.is_none());
assert!(config.headers.is_none());
assert!(config.extraction_rules.is_none());
}
#[test]
fn test_crawl_config_dto_with_patterns() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{
"max_depth": 3,
"include_patterns": ["/blog/*", "/news/*"],
"exclude_patterns": ["/admin/*", "/login/*"]
}"#;
let config: CrawlConfigDto = serde_json::from_str(json).unwrap();
assert_eq!(config.max_depth, 3);
assert_eq!(config.include_patterns.as_ref().unwrap().len(), 2);
assert_eq!(config.exclude_patterns.as_ref().unwrap().len(), 2);
}
#[test]
fn test_crawl_config_dto_with_headers_and_proxy() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{
"max_depth": 2,
"headers": {"Authorization": "Bearer token"},
"proxy": "socks5://proxy:1080"
}"#;
let config: CrawlConfigDto = serde_json::from_str(json).unwrap();
assert!(config.headers.is_some());
assert_eq!(config.proxy.as_deref(), Some("socks5://proxy:1080"));
}
#[test]
fn test_crawl_use_case_error_validation_display() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let err = CrawlUseCaseError::ValidationError("invalid input".to_string());
let display = format!("{}", err);
assert!(display.contains("Validation failed"));
assert!(display.contains("invalid input"));
}
#[test]
fn test_crawl_use_case_error_repository_display() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let err = CrawlUseCaseError::Repository(RepositoryError::NotFound);
let display = format!("{}", err);
assert!(display.contains("Repository error"));
}
#[test]
fn test_crawl_use_case_error_not_found_display() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let err = CrawlUseCaseError::NotFound;
let display = format!("{}", err);
assert!(display.contains("Crawl not found"));
}
#[test]
fn test_crawl_use_case_error_anyhow_display() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let err = CrawlUseCaseError::Anyhow(anyhow::anyhow!("something went wrong"));
let display = format!("{}", err);
assert!(display.contains("something went wrong"));
}
#[test]
fn test_sync_wait_result_default_when_no_tasks() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let result = SyncWaitResult {
waited_time_ms: 0,
is_timeout: false,
};
assert_eq!(result.waited_time_ms, 0);
assert!(!result.is_timeout);
}
#[test]
fn test_sync_wait_result_default_on_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let result = SyncWaitResult {
waited_time_ms: 0,
is_timeout: false,
};
assert!(!result.is_timeout);
}
#[test]
fn test_sync_wait_result_timeout_with_waited_time() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let sync_wait_ms = 5000u32;
let result = SyncWaitResult {
waited_time_ms: sync_wait_ms as u64,
is_timeout: true,
};
assert_eq!(result.waited_time_ms, 5000);
assert!(result.is_timeout);
}
#[test]
fn test_no_sync_wait_returns_created() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let sync_wait_ms: u32 = 0;
let wait_result = SyncWaitResult {
waited_time_ms: 0,
is_timeout: true, };
let status_code = if sync_wait_ms > 0 && wait_result.is_timeout {
StatusCode::ACCEPTED
} else {
StatusCode::CREATED
};
assert_eq!(status_code, StatusCode::CREATED);
}
#[test]
fn test_sync_wait_with_timeout_returns_accepted() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let sync_wait_ms: u32 = 3000;
let wait_result = SyncWaitResult {
waited_time_ms: 3000,
is_timeout: true,
};
let status_code = if sync_wait_ms > 0 && wait_result.is_timeout {
StatusCode::ACCEPTED
} else {
StatusCode::CREATED
};
assert_eq!(status_code, StatusCode::ACCEPTED);
}
#[test]
fn test_sync_wait_without_timeout_returns_created() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let sync_wait_ms: u32 = 3000;
let wait_result = SyncWaitResult {
waited_time_ms: 1000,
is_timeout: false,
};
let status_code = if sync_wait_ms > 0 && wait_result.is_timeout {
StatusCode::ACCEPTED
} else {
StatusCode::CREATED
};
assert_eq!(status_code, StatusCode::CREATED);
}
#[test]
fn test_crawl_config_dto_with_empty_extraction_rules() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{
"max_depth": 1,
"extraction_rules": {}
}"#;
let config: CrawlConfigDto = serde_json::from_str(json).unwrap();
assert!(config.extraction_rules.is_some());
assert_eq!(config.extraction_rules.as_ref().unwrap().len(), 0);
}
#[test]
fn test_crawl_request_dto_serialization_roundtrip() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = CrawlRequestDto {
url: "https://example.com".to_string(),
validated_url: None,
name: Some("Test Crawl".to_string()),
config: CrawlConfigDto {
max_depth: 3,
include_patterns: Some(vec!["/api/*".to_string()]),
exclude_patterns: None,
strategy: Some("bfs".to_string()),
crawl_delay_ms: Some(500),
max_concurrency: Some(5),
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
},
sync_wait_ms: Some(5000),
expires_at: None,
sitemap_seed: None,
sitemap_url: None,
};
let json = serde_json::to_string(&dto).unwrap();
let deserialized: CrawlRequestDto = serde_json::from_str(&json).unwrap();
assert_eq!(deserialized.url, dto.url);
assert_eq!(deserialized.name, dto.name);
assert_eq!(deserialized.config.max_depth, dto.config.max_depth);
assert_eq!(deserialized.sync_wait_ms, dto.sync_wait_ms);
}
#[test]
fn test_crawl_request_dto_validated_url_is_skipped_in_serialization() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = CrawlRequestDto {
url: "https://example.com".to_string(),
validated_url: None,
name: None,
config: CrawlConfigDto {
max_depth: 1,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
},
sync_wait_ms: None,
expires_at: None,
sitemap_seed: None,
sitemap_url: None,
};
let json = serde_json::to_string(&dto).unwrap();
assert!(!json.contains("validated_url"));
}
use crate::domain::auth::ApiKeyScope;
use crate::domain::models::ScrapeResult;
use crate::domain::models::{Crawl, CrawlStatus, Task, TaskStatus, TaskType};
#[cfg(feature = "webhook")]
use crate::domain::models::Webhook;
use crate::domain::repositories::crawl_repository::CrawlRepository;
#[cfg(feature = "teams")]
use crate::domain::repositories::geo_restriction_repository::{
GeoRestrictionRepository, GeoRestrictionRepositoryError,
};
use crate::domain::repositories::scrape_result_repository::ScrapeResultRepository;
use crate::domain::repositories::task_repository::{TaskQueryParams, TaskRepository};
#[cfg(feature = "webhook")]
use crate::domain::repositories::webhook_repository::WebhookRepository;
#[cfg(feature = "teams")]
use crate::domain::services::geo_location::{GeoLocation, GeoLocationService};
use crate::domain::services::rate_limiting_service::{
BacklogService, ConcurrencyConfig, ConcurrencyControlService, ConcurrencyResult,
QuotaService, RateLimitConfig, RateLimitResult, RateLimitService, RateLimitingError,
RateLimitingService,
};
#[cfg(feature = "teams")]
use crate::domain::services::team_service::{TeamGeoRestrictions, TeamService};
use async_trait::async_trait;
use std::collections::HashSet;
#[cfg(feature = "teams")]
use std::net::IpAddr;
use std::sync::Mutex;
struct MockCrawlRepository {
stored_crawl: Mutex<Option<Crawl>>,
find_should_fail: bool,
create_should_fail: bool,
update_should_fail: bool,
}
impl MockCrawlRepository {
fn new() -> Self {
Self {
stored_crawl: Mutex::new(None),
find_should_fail: false,
create_should_fail: false,
update_should_fail: false,
}
}
fn with_crawl(crawl: Crawl) -> Self {
Self {
stored_crawl: Mutex::new(Some(crawl)),
find_should_fail: false,
create_should_fail: false,
update_should_fail: false,
}
}
fn failing_find() -> Self {
Self {
find_should_fail: true,
..Self::new()
}
}
fn failing_create() -> Self {
Self {
create_should_fail: true,
..Self::new()
}
}
fn failing_update_with_crawl(crawl: Crawl) -> Self {
Self {
stored_crawl: Mutex::new(Some(crawl)),
update_should_fail: true,
..Self::new()
}
}
}
#[async_trait]
impl CrawlRepository for MockCrawlRepository {
async fn update_status_guarded(
&self,
_id: crate::domain::id::Id,
status: CrawlStatus,
) -> Result<bool, RepositoryError> {
if self.update_should_fail {
return Err(RepositoryError::Database(anyhow::anyhow!("update failed")));
}
let mut guard = self.stored_crawl.lock().unwrap();
let Some(crawl) = guard.as_mut() else {
return Ok(false);
};
if matches!(
crawl.status,
CrawlStatus::Completed | CrawlStatus::Failed | CrawlStatus::Cancelled
) {
return Ok(false);
}
crawl.status = status;
crawl.updated_at = chrono::Utc::now();
if matches!(
status,
CrawlStatus::Completed | CrawlStatus::Failed | CrawlStatus::Cancelled
) {
crawl.completed_at = Some(chrono::Utc::now());
}
Ok(true)
}
async fn create(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
if self.create_should_fail {
return Err(RepositoryError::Database(anyhow::anyhow!("create failed")));
}
Ok(crawl.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Crawl>, RepositoryError> {
if self.find_should_fail {
return Err(RepositoryError::Database(anyhow::anyhow!(
"find_by_id failed"
)));
}
Ok(self.stored_crawl.lock().unwrap().clone())
}
async fn update(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
if self.update_should_fail {
return Err(RepositoryError::Database(anyhow::anyhow!("update failed")));
}
Ok(crawl.clone())
}
async fn increment_completed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn increment_failed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn update_status(
&self,
_id: crate::domain::id::Id,
_status: crate::domain::models::CrawlStatus,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn increment_total_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn find_by_team_id_paginated(
&self,
_team_id: crate::domain::id::Id,
_limit: u32,
_offset: u32,
) -> Result<Vec<Crawl>, RepositoryError> {
Ok(vec![])
}
async fn count_by_team_id(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
}
struct MockTaskRepository {
tasks: Vec<Task>,
find_by_crawl_should_fail: bool,
}
impl MockTaskRepository {
fn new() -> Self {
Self {
tasks: vec![],
find_by_crawl_should_fail: false,
}
}
fn with_tasks(tasks: Vec<Task>) -> Self {
Self {
tasks,
find_by_crawl_should_fail: false,
}
}
fn failing_find_by_crawl() -> Self {
Self {
tasks: vec![],
find_by_crawl_should_fail: true,
}
}
}
#[async_trait]
impl TaskRepository for MockTaskRepository {
async fn create(&self, task: &Task) -> Result<Task, RepositoryError> {
Ok(task.clone())
}
async fn find_by_id(
&self,
id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(self.tasks.iter().find(|t| t.id == id).cloned())
}
async fn update(&self, task: &Task) -> Result<Task, RepositoryError> {
Ok(task.clone())
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
_urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
Ok(HashSet::new())
}
async fn reset_stuck_tasks(
&self,
_timeout: chrono::Duration,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
if self.find_by_crawl_should_fail {
return Err(RepositoryError::Database(anyhow::anyhow!(
"find_by_crawl_id failed"
)));
}
Ok(self.tasks.clone())
}
async fn query_tasks(
&self,
params: TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
let result: Vec<Task> = if let Some(ref task_ids) = params.task_ids {
self.tasks
.iter()
.filter(|t| task_ids.contains(&t.id))
.cloned()
.collect()
} else {
self.tasks.clone()
};
let count = result.len() as u64;
Ok((result, count))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((vec![], vec![]))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
#[cfg(feature = "webhook")]
struct MockWebhookRepository;
#[cfg(feature = "webhook")]
#[async_trait]
impl WebhookRepository for MockWebhookRepository {
async fn create(&self, webhook: &Webhook) -> Result<Webhook, RepositoryError> {
Ok(webhook.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Webhook>, RepositoryError> {
Ok(None)
}
async fn find_by_team_id(
&self,
_team_id: crate::domain::id::Id,
) -> Result<Vec<Webhook>, RepositoryError> {
Ok(vec![])
}
}
struct MockScrapeResultRepository {
results: Vec<ScrapeResult>,
find_should_fail: bool,
}
impl MockScrapeResultRepository {
fn new() -> Self {
Self {
results: vec![],
find_should_fail: false,
}
}
fn failing() -> Self {
Self {
results: vec![],
find_should_fail: true,
}
}
}
#[async_trait]
impl ScrapeResultRepository for MockScrapeResultRepository {
async fn save(&self, _result: ScrapeResult) -> anyhow::Result<()> {
Ok(())
}
async fn find_by_task_id(
&self,
_task_id: crate::domain::id::Id,
) -> anyhow::Result<Option<ScrapeResult>> {
Ok(None)
}
async fn find_by_task_ids(
&self,
task_ids: &[crate::domain::id::Id],
) -> anyhow::Result<Vec<ScrapeResult>> {
if self.find_should_fail {
return Err(anyhow::anyhow!("find_by_task_ids failed"));
}
Ok(self
.results
.iter()
.filter(|r| task_ids.contains(&r.task_id))
.cloned()
.collect())
}
async fn get_team_avg_response_time(
&self,
_team_id: crate::domain::id::Id,
) -> anyhow::Result<f64> {
Ok(0.0)
}
async fn cleanup_expired(&self, _retention_days: i64) -> anyhow::Result<u64> {
Ok(0)
}
}
#[cfg(feature = "teams")]
struct MockGeoRestrictionRepository {
restrictions: TeamGeoRestrictions,
get_should_fail: bool,
}
#[cfg(feature = "teams")]
impl MockGeoRestrictionRepository {
fn new() -> Self {
Self {
restrictions: TeamGeoRestrictions::default(),
get_should_fail: false,
}
}
}
#[cfg(feature = "teams")]
#[async_trait]
impl GeoRestrictionRepository for MockGeoRestrictionRepository {
async fn get_team_restrictions(
&self,
_team_id: crate::domain::id::Id,
) -> Result<TeamGeoRestrictions, GeoRestrictionRepositoryError> {
if self.get_should_fail {
return Err(GeoRestrictionRepositoryError::Database(
"get_team_restrictions failed".to_string(),
));
}
Ok(self.restrictions.clone())
}
async fn update_team_restrictions(
&self,
_team_id: crate::domain::id::Id,
_restrictions: &TeamGeoRestrictions,
) -> Result<(), GeoRestrictionRepositoryError> {
Ok(())
}
async fn log_geo_restriction_action(
&self,
_team_id: crate::domain::id::Id,
_ip_address: &str,
_country_code: &str,
_action: &str,
_reason: &str,
) -> Result<(), GeoRestrictionRepositoryError> {
Ok(())
}
}
#[cfg(feature = "teams")]
struct MockGeoLocationService;
#[cfg(feature = "teams")]
#[async_trait]
impl GeoLocationService for MockGeoLocationService {
async fn get_location(&self, _ip: &IpAddr) -> anyhow::Result<GeoLocation> {
Ok(GeoLocation::default())
}
}
struct MockRateLimitingService {
rate_limit_result: RateLimitResult,
quota_should_fail: bool,
}
impl MockRateLimitingService {
fn new_allowed() -> Self {
Self {
rate_limit_result: RateLimitResult::Allowed,
quota_should_fail: false,
}
}
fn new_denied() -> Self {
Self {
rate_limit_result: RateLimitResult::Denied {
reason: "Too many requests".to_string(),
},
quota_should_fail: false,
}
}
fn new_quota_exceeded() -> Self {
Self {
rate_limit_result: RateLimitResult::Allowed,
quota_should_fail: true,
}
}
}
#[async_trait]
impl RateLimitService for MockRateLimitingService {
async fn check_rate_limit(
&self,
_api_key: &str,
_endpoint: &str,
) -> Result<RateLimitResult, RateLimitingError> {
Ok(self.rate_limit_result.clone())
}
async fn get_team_rate_limit_config(
&self,
_team_id: crate::domain::id::Id,
) -> Result<RateLimitConfig, RateLimitingError> {
Ok(RateLimitConfig::default())
}
async fn update_team_rate_limit_config(
&self,
_team_id: crate::domain::id::Id,
_config: RateLimitConfig,
) -> Result<(), RateLimitingError> {
Ok(())
}
async fn cleanup_expired_rate_limits(&self) -> Result<u64, RateLimitingError> {
Ok(0)
}
}
#[async_trait]
impl ConcurrencyControlService for MockRateLimitingService {
async fn check_team_concurrency(
&self,
_team_id: crate::domain::id::Id,
_task_id: crate::domain::id::Id,
) -> Result<ConcurrencyResult, RateLimitingError> {
Ok(ConcurrencyResult::Allowed)
}
async fn release_team_concurrency_slot(
&self,
_team_id: crate::domain::id::Id,
_task_id: crate::domain::id::Id,
) -> Result<(), RateLimitingError> {
Ok(())
}
async fn get_team_current_concurrency(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u32, RateLimitingError> {
Ok(0)
}
async fn get_team_concurrency_config(
&self,
_team_id: crate::domain::id::Id,
) -> Result<ConcurrencyConfig, RateLimitingError> {
Ok(ConcurrencyConfig::default())
}
async fn update_team_concurrency_config(
&self,
_team_id: crate::domain::id::Id,
_config: ConcurrencyConfig,
) -> Result<(), RateLimitingError> {
Ok(())
}
}
#[async_trait]
impl BacklogService for MockRateLimitingService {
async fn process_backlog_tasks(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u32, RateLimitingError> {
Ok(0)
}
}
#[async_trait]
impl QuotaService for MockRateLimitingService {
async fn check_and_deduct_quota(
&self,
_team_id: crate::domain::id::Id,
_amount: i64,
_transaction_type: crate::domain::models::CreditsTransactionType,
_description: String,
_reference_id: Option<crate::domain::id::Id>,
) -> Result<(), RateLimitingError> {
if self.quota_should_fail {
return Err(RateLimitingError::CreditsError);
}
Ok(())
}
async fn get_quota_balance(
&self,
_team_id: crate::domain::id::Id,
) -> Result<i64, RateLimitingError> {
Ok(1000)
}
async fn refund_quota(
&self,
_team_id: crate::domain::id::Id,
_amount: i64,
_description: String,
_reference_id: Option<crate::domain::id::Id>,
) -> Result<(), RateLimitingError> {
Ok(())
}
}
impl RateLimitingService for MockRateLimitingService {}
fn make_auth_state() -> AuthState {
AuthState::new(
create_test_db_pool(),
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
ApiKeyScope::default(),
)
}
fn make_auth_state_with_team(team_id: crate::domain::id::Id) -> AuthState {
AuthState::new(
create_test_db_pool(),
team_id,
crate::domain::id::Id::generate(),
ApiKeyScope::default(),
)
}
fn test_locale() -> Locale {
"en-US".parse().unwrap()
}
fn test_bundle() -> Arc<I18nBundle> {
Arc::new(I18nBundle::load("en-US", &["en-US", "zh-CN"]).unwrap())
}
fn make_socket_addr() -> SocketAddr {
"203.0.113.1:8080".parse().expect("valid SocketAddr")
}
fn make_crawl(team_id: crate::domain::id::Id, status: CrawlStatus) -> Crawl {
let now = chrono::Utc::now();
Crawl::with_all_fields(
crate::domain::id::Id::generate(),
team_id,
"Test Crawl".to_string(),
"https://example.com".to_string(),
"https://example.com".to_string(),
status,
serde_json::json!({"max_depth": 2}),
1,
0,
0,
now,
now,
None,
)
}
fn make_task(
crawl_id: crate::domain::id::Id,
team_id: crate::domain::id::Id,
status: TaskStatus,
) -> Task {
let now = chrono::Utc::now();
Task {
id: crate::domain::id::Id::generate(),
task_type: TaskType::Crawl,
status,
priority: 100,
team_id,
api_key_id: crate::domain::id::Id::generate(),
url: "https://example.com".to_string(),
payload: serde_json::json!({"crawl_id": crawl_id, "depth": 0}),
retry_count: 0,
attempt_count: 0,
max_retries: 3,
scheduled_at: None,
created_at: now,
started_at: None,
completed_at: None,
crawl_id: Some(crawl_id),
updated_at: now,
lock_token: None,
lock_expires_at: None,
expires_at: None,
}
}
fn make_crawl_request_dto(
url: &str,
max_depth: u32,
sync_wait_ms: Option<u32>,
max_concurrency: Option<u32>,
) -> CrawlRequestDto {
CrawlRequestDto {
url: url.to_string(),
validated_url: None,
name: Some("Test Crawl".to_string()),
config: CrawlConfigDto {
max_depth,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
},
sync_wait_ms,
expires_at: None,
sitemap_seed: None,
sitemap_url: None,
}
}
fn test_settings() -> Arc<crate::config::settings::Settings> {
Arc::new(crate::config::settings::Settings::default())
}
fn test_settings_with_sitemap(enabled: bool) -> Arc<crate::config::settings::Settings> {
let mut settings = crate::config::settings::Settings::default();
settings.sitemap.enabled = enabled;
Arc::new(settings)
}
fn build_handler_state(
crawl_repo: MockCrawlRepository,
task_repo: MockTaskRepository,
scrape_result_repo: MockScrapeResultRepository,
rate_limiting_service: MockRateLimitingService,
) -> Arc<CrawlHandlerState> {
let crawl_repo: Arc<dyn CrawlRepository> = Arc::new(crawl_repo);
let task_repo: Arc<dyn TaskRepository> = Arc::new(task_repo);
let scrape_result_repo: Arc<dyn ScrapeResultRepository> = Arc::new(scrape_result_repo);
let rate_limiting_service: Arc<dyn RateLimitingService> = Arc::new(rate_limiting_service);
#[cfg(feature = "teams")]
let geo_restriction_repo: Arc<dyn GeoRestrictionRepository> =
Arc::new(MockGeoRestrictionRepository::new());
#[cfg(feature = "teams")]
let team_service = Arc::new(TeamService::new(
Arc::new(MockGeoLocationService),
geo_restriction_repo.clone(),
));
#[cfg(feature = "webhook")]
let webhook_repo: Arc<dyn WebhookRepository> = Arc::new(MockWebhookRepository);
Arc::new(CrawlHandlerState::new(
crawl_repo,
task_repo,
scrape_result_repo,
rate_limiting_service,
#[cfg(feature = "webhook")]
webhook_repo,
#[cfg(feature = "teams")]
geo_restriction_repo,
#[cfg(feature = "teams")]
team_service,
))
}
#[tokio::test]
async fn test_create_crawl_rejects_sitemap_seed_when_master_switch_disabled() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let state = build_handler_state(
MockCrawlRepository::new(),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state();
let mut payload = make_crawl_request_dto("https://example.com", 2, Some(0), None);
payload.sitemap_seed = Some(true);
let response = create_crawl(
Extension(state),
Extension(test_settings()), Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
ConnectInfo(make_socket_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(
response.status(),
StatusCode::UNPROCESSABLE_ENTITY,
"sitemap_seed must be rejected when master switch is off"
);
let body = axum::body::to_bytes(response.into_body(), usize::MAX)
.await
.expect("body");
let text = String::from_utf8_lossy(&body);
assert!(
text.contains("CRAWLRS__SITEMAP__ENABLED"),
"rejection must name the master switch env var, got: {text}"
);
}
#[tokio::test]
async fn test_create_crawl_accepts_sitemap_seed_when_master_switch_enabled() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let state = build_handler_state(
MockCrawlRepository::new(),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state();
let mut payload = make_crawl_request_dto("https://example.com", 2, Some(0), None);
payload.sitemap_seed = Some(true);
payload.sitemap_url = Some("https://example.com/sitemap.xml".to_string());
let response = create_crawl(
Extension(state),
Extension(test_settings_with_sitemap(true)),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
ConnectInfo(make_socket_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(
response.status(),
StatusCode::CREATED,
"sitemap_seed must be accepted when master switch is on"
);
}
#[tokio::test]
async fn test_create_crawl_success_no_sync_wait() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let state = build_handler_state(
MockCrawlRepository::new(),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state();
let payload = make_crawl_request_dto("https://example.com", 2, Some(0), None);
let response = create_crawl(
Extension(state),
Extension(test_settings()),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
ConnectInfo(make_socket_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::CREATED);
}
#[tokio::test]
async fn test_create_crawl_success_sync_wait_empty_tasks() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let state = build_handler_state(
MockCrawlRepository::new(),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state();
let payload = make_crawl_request_dto("https://example.com", 2, Some(100), None);
let response = create_crawl(
Extension(state),
Extension(test_settings()),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
ConnectInfo(make_socket_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::CREATED);
}
#[tokio::test]
async fn test_create_crawl_success_sync_wait_completed() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let task = make_task(
crate::domain::id::Id::generate(),
team_id,
TaskStatus::Completed,
);
let state = build_handler_state(
MockCrawlRepository::new(),
MockTaskRepository::with_tasks(vec![task]),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state_with_team(team_id);
let payload = make_crawl_request_dto("https://example.com", 2, Some(5000), None);
let response = create_crawl(
Extension(state),
Extension(test_settings()),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
ConnectInfo(make_socket_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::CREATED);
}
#[tokio::test]
async fn test_create_crawl_sync_wait_timeout_returns_accepted() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let task = make_task(
crate::domain::id::Id::generate(),
team_id,
TaskStatus::Queued,
);
let state = build_handler_state(
MockCrawlRepository::new(),
MockTaskRepository::with_tasks(vec![task]),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state_with_team(team_id);
let payload = make_crawl_request_dto("https://example.com", 2, Some(100), None);
let response = create_crawl(
Extension(state),
Extension(test_settings()),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
ConnectInfo(make_socket_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::ACCEPTED);
}
#[tokio::test]
async fn test_create_crawl_sync_wait_find_error_returns_created() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let state = build_handler_state(
MockCrawlRepository::new(),
MockTaskRepository::failing_find_by_crawl(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state();
let payload = make_crawl_request_dto("https://example.com", 2, Some(100), None);
let response = create_crawl(
Extension(state),
Extension(test_settings()),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
ConnectInfo(make_socket_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::CREATED);
}
#[tokio::test]
async fn test_create_crawl_max_depth_exceeds_returns_unprocessable_entity() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let state = build_handler_state(
MockCrawlRepository::new(),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state();
let payload = make_crawl_request_dto("https://example.com", 6, Some(0), None);
let response = create_crawl(
Extension(state),
Extension(test_settings()),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
ConnectInfo(make_socket_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
}
#[tokio::test]
async fn test_create_crawl_ssrf_private_ip_returns_bad_request() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let state = build_handler_state(
MockCrawlRepository::new(),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state();
let payload = make_crawl_request_dto("http://127.0.0.1", 2, Some(0), None);
let response = create_crawl(
Extension(state),
Extension(test_settings()),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
ConnectInfo(make_socket_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn test_create_crawl_rate_limited_returns_too_many_requests() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let state = build_handler_state(
MockCrawlRepository::new(),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_denied(),
);
let auth = make_auth_state();
let payload = make_crawl_request_dto("https://example.com", 2, Some(0), None);
let response = create_crawl(
Extension(state),
Extension(test_settings()),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
ConnectInfo(make_socket_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS);
}
#[tokio::test]
async fn test_create_crawl_quota_exceeded_returns_payment_required() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let state = build_handler_state(
MockCrawlRepository::new(),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_quota_exceeded(),
);
let auth = make_auth_state();
let payload = make_crawl_request_dto("https://example.com", 2, Some(0), None);
let response = create_crawl(
Extension(state),
Extension(test_settings()),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
ConnectInfo(make_socket_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::PAYMENT_REQUIRED);
}
#[tokio::test]
async fn test_create_crawl_use_case_validation_error_returns_bad_request() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let state = build_handler_state(
MockCrawlRepository::new(),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state();
let payload = make_crawl_request_dto("https://example.com", 2, Some(0), Some(200));
let response = create_crawl(
Extension(state),
Extension(test_settings()),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
ConnectInfo(make_socket_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
}
#[tokio::test]
async fn test_create_crawl_use_case_repository_error_returns_internal_server_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let state = build_handler_state(
MockCrawlRepository::failing_create(),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state();
let payload = make_crawl_request_dto("https://example.com", 2, Some(0), None);
let response = create_crawl(
Extension(state),
Extension(test_settings()),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
ConnectInfo(make_socket_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
}
#[tokio::test]
async fn test_get_crawl_success_returns_ok() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let crawl = make_crawl(team_id, CrawlStatus::Queued);
let crawl_id = crawl.id;
let state = build_handler_state(
MockCrawlRepository::with_crawl(crawl),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state_with_team(team_id);
let response = get_crawl(
Extension(state),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
Path(crawl_id),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::OK);
}
#[tokio::test]
async fn test_get_crawl_not_found_returns_404() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let state = build_handler_state(
MockCrawlRepository::new(),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state();
let response = get_crawl(
Extension(state),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
Path(crate::domain::id::Id::generate()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn test_get_crawl_wrong_team_returns_404() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let crawl = make_crawl(crate::domain::id::Id::generate(), CrawlStatus::Queued);
let crawl_id = crawl.id;
let state = build_handler_state(
MockCrawlRepository::with_crawl(crawl),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state_with_team(crate::domain::id::Id::generate());
let response = get_crawl(
Extension(state),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
Path(crawl_id),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn test_get_crawl_repo_error_returns_internal_server_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let state = build_handler_state(
MockCrawlRepository::failing_find(),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state();
let response = get_crawl(
Extension(state),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
Path(crate::domain::id::Id::generate()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
}
#[tokio::test]
async fn test_get_crawl_results_success_returns_ok() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let crawl = make_crawl(team_id, CrawlStatus::Completed);
let crawl_id = crawl.id;
let task = make_task(crawl_id, team_id, TaskStatus::Completed);
let state = build_handler_state(
MockCrawlRepository::with_crawl(crawl),
MockTaskRepository::with_tasks(vec![task]),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state_with_team(team_id);
let response = get_crawl_results(
Extension(state),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
Path(crawl_id),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::OK);
}
#[tokio::test]
async fn test_get_crawl_results_crawl_not_found_returns_404() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let state = build_handler_state(
MockCrawlRepository::new(),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state();
let response = get_crawl_results(
Extension(state),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
Path(crate::domain::id::Id::generate()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn test_get_crawl_results_repo_error_returns_internal_server_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let crawl = make_crawl(team_id, CrawlStatus::Completed);
let crawl_id = crawl.id;
let task = make_task(crawl_id, team_id, TaskStatus::Completed);
let state = build_handler_state(
MockCrawlRepository::with_crawl(crawl),
MockTaskRepository::with_tasks(vec![task]),
MockScrapeResultRepository::failing(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state_with_team(team_id);
let response = get_crawl_results(
Extension(state),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
Path(crawl_id),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
}
#[tokio::test]
async fn test_cancel_crawl_success_returns_no_content() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let crawl = make_crawl(team_id, CrawlStatus::Queued);
let crawl_id = crawl.id;
let state = build_handler_state(
MockCrawlRepository::with_crawl(crawl),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state_with_team(team_id);
let response = cancel_crawl(
Extension(state),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
Path(crawl_id),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::NO_CONTENT);
}
#[tokio::test]
async fn test_cancel_crawl_not_found_returns_404() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let state = build_handler_state(
MockCrawlRepository::new(),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state();
let response = cancel_crawl(
Extension(state),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
Path(crate::domain::id::Id::generate()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn test_cancel_crawl_wrong_team_returns_404() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let crawl = make_crawl(crate::domain::id::Id::generate(), CrawlStatus::Queued);
let crawl_id = crawl.id;
let state = build_handler_state(
MockCrawlRepository::with_crawl(crawl),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state_with_team(crate::domain::id::Id::generate());
let response = cancel_crawl(
Extension(state),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
Path(crawl_id),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn test_cancel_crawl_already_completed_returns_no_content() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let crawl = make_crawl(team_id, CrawlStatus::Completed);
let crawl_id = crawl.id;
let state = build_handler_state(
MockCrawlRepository::with_crawl(crawl),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state_with_team(team_id);
let response = cancel_crawl(
Extension(state),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
Path(crawl_id),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::NO_CONTENT);
}
#[tokio::test]
async fn test_cancel_crawl_repo_error_returns_internal_server_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let state = build_handler_state(
MockCrawlRepository::failing_find(),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state();
let response = cancel_crawl(
Extension(state),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
Path(crate::domain::id::Id::generate()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
}
#[tokio::test]
async fn test_cancel_crawl_update_error_returns_internal_server_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let crawl = make_crawl(team_id, CrawlStatus::Queued);
let crawl_id = crawl.id;
let state = build_handler_state(
MockCrawlRepository::failing_update_with_crawl(crawl),
MockTaskRepository::new(),
MockScrapeResultRepository::new(),
MockRateLimitingService::new_allowed(),
);
let auth = make_auth_state_with_team(team_id);
let response = cancel_crawl(
Extension(state),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
Path(crawl_id),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
}
}