use axum::{
extract::{ConnectInfo, Extension},
http::StatusCode,
response::{IntoResponse, Response},
Json,
};
use log::error;
use std::net::SocketAddr;
use crate::application::dto::extract_request::ExtractRequestDto;
use crate::common::constants::crawl_task;
use crate::config::settings::Settings;
use crate::domain::models::{Task, TaskType};
#[cfg(feature = "teams")]
use crate::domain::repositories::geo_restriction_repository::GeoRestrictionRepository;
use crate::domain::repositories::task_repository::TaskRepository;
use crate::domain::services::rate_limiting_service::RateLimitingService;
#[cfg(feature = "teams")]
use crate::domain::services::team_service::TeamService;
use crate::presentation::handlers::credits::{CreditCharge, CreditFlow};
use crate::presentation::handlers::response_builder::{
error_response, json_rejection_response, success_response,
};
use crate::presentation::handlers::task_handler::wait_for_tasks_completion;
use crate::presentation::handlers::{check_ssrf_urls_batch, sync_wait_status_code};
use crate::presentation::middleware::auth_middleware::AuthState;
use crate::queue::task_queue::TaskQueue;
use sdforge::validator::Validate as _;
use std::sync::Arc;
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct ExtractResponseDto {
pub id: crate::domain::id::Id,
pub status: String,
}
#[cfg(feature = "teams")]
#[allow(clippy::too_many_arguments)]
pub async fn extract<GR>(
Extension(queue): Extension<Arc<dyn TaskQueue>>,
Extension(_settings): Extension<Arc<Settings>>,
Extension(task_repository): Extension<Arc<dyn TaskRepository>>,
Extension(rate_limiting_service): Extension<Arc<dyn RateLimitingService>>,
Extension(geo_restriction_repo): Extension<Arc<GR>>,
Extension(team_service): Extension<Arc<TeamService>>,
Extension(auth_state): Extension<AuthState>,
ConnectInfo(addr): ConnectInfo<SocketAddr>,
payload: Result<Json<ExtractRequestDto>, axum::extract::rejection::JsonRejection>,
) -> impl IntoResponse
where
GR: GeoRestrictionRepository + 'static,
{
let Json(payload) = match payload {
Ok(parsed) => parsed,
Err(ref rej) => return json_rejection_response(rej),
};
let team_id = auth_state.team_id;
if let Some(response) = validate_extract_fields(&payload) {
return response;
}
if let Some(response) = validate_extract_sync_wait(&payload) {
return response;
}
if let Err(verrs) = payload.validate() {
return error_response(
StatusCode::UNPROCESSABLE_ENTITY,
crate::presentation::handlers::response_builder::validation_error_message(&verrs),
);
}
if payload.urls.iter().any(|url| url.len() > 2048) {
return error_response(
StatusCode::UNPROCESSABLE_ENTITY,
"each url must be at most 2048 bytes",
);
}
if let Some(response) =
check_ssrf_urls_batch(&payload.urls, team_id, auth_state.api_key_id).await
{
return response;
}
let client_ip = addr.ip().to_string();
let restrictions = match geo_restriction_repo.get_team_restrictions(team_id).await {
Ok(restrictions) => restrictions,
Err(e) => {
error!("Failed to get team restrictions: {:?}", e);
return error_response(
StatusCode::INTERNAL_SERVER_ERROR,
"Failed to validate geographic access",
);
}
};
if let Some(response) = validate_extract_geo_access(
&geo_restriction_repo,
&team_service,
team_id,
&client_ip,
&restrictions,
)
.await
{
return response;
}
let task = build_extract_task(&payload, team_id, auth_state.api_key_id);
if let Some(response) = CreditFlow::new(rate_limiting_service.as_ref(), team_id)
.deduct(CreditCharge::Extract {
url_count: payload.urls.len(),
})
.await
{
return response;
}
enqueue_extract(
queue.as_ref(),
task,
task_repository.as_ref(),
rate_limiting_service.as_ref(),
team_id,
&payload,
)
.await
}
#[cfg(not(feature = "teams"))]
#[allow(clippy::too_many_arguments)]
pub async fn extract(
Extension(queue): Extension<Arc<dyn TaskQueue>>,
Extension(_settings): Extension<Arc<Settings>>,
Extension(task_repository): Extension<Arc<dyn TaskRepository>>,
Extension(rate_limiting_service): Extension<Arc<dyn RateLimitingService>>,
Extension(auth_state): Extension<AuthState>,
ConnectInfo(_addr): ConnectInfo<SocketAddr>,
payload: Result<Json<ExtractRequestDto>, axum::extract::rejection::JsonRejection>,
) -> impl IntoResponse {
let Json(payload) = match payload {
Ok(parsed) => parsed,
Err(ref rej) => return json_rejection_response(rej),
};
let team_id = auth_state.team_id;
if let Some(response) = preflight_extract(
&payload,
team_id,
auth_state.api_key_id,
rate_limiting_service.as_ref(),
)
.await
{
return response;
}
let task = build_extract_task(&payload, team_id, auth_state.api_key_id);
enqueue_extract(
queue.as_ref(),
task,
task_repository.as_ref(),
rate_limiting_service.as_ref(),
team_id,
&payload,
)
.await
}
async fn enqueue_extract(
queue: &dyn TaskQueue,
task: Task,
task_repository: &dyn TaskRepository,
rate_limiting_service: &dyn RateLimitingService,
team_id: crate::domain::id::Id,
payload: &ExtractRequestDto,
) -> Response {
match queue.enqueue(task.clone()).await {
Ok(_) => {
let sync_wait_ms = payload
.sync_wait_ms
.unwrap_or(crawl_task::DEFAULT_TIMEOUT_MS as u32);
let waited_time_ms =
await_extract_completion(task_repository, task.id, team_id, sync_wait_ms).await;
extract_accepted_response(task.id, sync_wait_ms, waited_time_ms)
}
Err(e) => {
CreditFlow::new(rate_limiting_service, team_id)
.refund(
CreditCharge::Extract {
url_count: payload.urls.len(),
},
Some(task.id),
)
.await;
error_response(StatusCode::INTERNAL_SERVER_ERROR, e.to_string())
}
}
}
#[cfg(not(feature = "teams"))]
async fn preflight_extract(
payload: &ExtractRequestDto,
team_id: crate::domain::id::Id,
api_key_id: crate::domain::id::Id,
rate_limiting_service: &dyn RateLimitingService,
) -> Option<Response> {
if let Some(response) = validate_extract_fields(payload) {
return Some(response);
}
if let Some(response) = validate_extract_sync_wait(payload) {
return Some(response);
}
if payload.urls.iter().any(|url| url.len() > 2048) {
return Some(error_response(
StatusCode::UNPROCESSABLE_ENTITY,
"each url must be at most 2048 bytes",
));
}
if let Err(errors) = payload.validate() {
return Some(error_response(
StatusCode::UNPROCESSABLE_ENTITY,
crate::presentation::handlers::response_builder::validation_error_message(&errors),
));
}
if let Some(response) = check_ssrf_urls_batch(&payload.urls, team_id, api_key_id).await {
return Some(response);
}
CreditFlow::new(rate_limiting_service, team_id)
.deduct(CreditCharge::Extract {
url_count: payload.urls.len(),
})
.await
}
#[cfg(feature = "teams")]
async fn validate_extract_geo_access<GR>(
geo_restriction_repo: &Arc<GR>,
team_service: &Arc<TeamService>,
team_id: crate::domain::id::Id,
client_ip: &str,
restrictions: &crate::domain::services::team_service::TeamGeoRestrictions,
) -> Option<Response>
where
GR: GeoRestrictionRepository + 'static,
{
match team_service
.validate_geographic_restriction(team_id, client_ip, restrictions)
.await
{
Ok(crate::domain::services::team_service::GeoRestrictionResult::Allowed) => {
if let Err(e) = geo_restriction_repo
.log_geo_restriction_action(
team_id,
client_ip,
"",
"ALLOWED",
"Extract request - Geographic restriction check passed",
)
.await
{
error!("Failed to log geographic restriction action: {}", e);
}
None
}
Ok(crate::domain::services::team_service::GeoRestrictionResult::Denied(reason)) => {
if let Err(e) = geo_restriction_repo
.log_geo_restriction_action(
team_id,
client_ip,
"",
"DENIED",
&format!("Extract request - {}", reason),
)
.await
{
error!("Failed to log geographic restriction action: {}", e);
}
Some(error_response(
StatusCode::FORBIDDEN,
format!("Access denied due to geographic restrictions: {}", reason),
))
}
Err(e) => {
error!("Failed to validate geographic restrictions: {:?}", e);
Some(error_response(
StatusCode::INTERNAL_SERVER_ERROR,
"Failed to validate geographic access",
))
}
}
}
fn validate_extract_fields(payload: &ExtractRequestDto) -> Option<Response> {
if payload.urls.is_empty() {
return Some(error_response(
StatusCode::UNPROCESSABLE_ENTITY,
"At least one URL is required",
));
}
if payload.prompt.is_none() && payload.schema.is_none() && payload.rules.is_none() {
return Some(error_response(
StatusCode::UNPROCESSABLE_ENTITY,
"Either prompt, schema, or rules is required",
));
}
None
}
fn validate_extract_sync_wait(payload: &ExtractRequestDto) -> Option<Response> {
if let Some(ms) = payload.sync_wait_ms {
if ms > crawl_task::MAX_SYNC_WAIT_MS {
return Some(error_response(
StatusCode::BAD_REQUEST,
format!("sync_wait_ms must be <= {}", crawl_task::MAX_SYNC_WAIT_MS),
));
}
}
None
}
fn build_extract_task(
payload: &ExtractRequestDto,
team_id: crate::domain::id::Id,
api_key_id: crate::domain::id::Id,
) -> Task {
Task::new(
crate::domain::id::Id::generate(),
TaskType::Extract,
team_id,
api_key_id,
payload
.urls
.first()
.expect("URLs already validated as non-empty")
.clone(),
serde_json::to_value(payload).unwrap_or_default(),
)
}
async fn await_extract_completion(
task_repository: &dyn TaskRepository,
task_id: crate::domain::id::Id,
team_id: crate::domain::id::Id,
sync_wait_ms: u32,
) -> u64 {
let mut waited_time_ms = 0u64;
if sync_wait_ms > 0 {
let wait_start = std::time::Instant::now();
match wait_for_tasks_completion(
task_repository,
&[task_id],
team_id,
sync_wait_ms,
crawl_task::BASE_POLL_INTERVAL_MS,
)
.await
{
Ok(_) => {
waited_time_ms = wait_start.elapsed().as_millis() as u64;
}
Err(e) => {
error!("Failed to wait for task completion: {:?}", e);
}
}
}
waited_time_ms
}
fn extract_accepted_response(
task_id: crate::domain::id::Id,
sync_wait_ms: u32,
waited_time_ms: u64,
) -> Response {
let status_code = sync_wait_status_code(sync_wait_ms, waited_time_ms >= sync_wait_ms as u64);
success_response(
status_code,
ExtractResponseDto {
id: task_id,
status: "pending".to_string(),
},
)
}
#[cfg(all(test, feature = "teams"))]
mod tests {
use super::*;
use crate::domain::models::TaskStatus;
use crate::domain::models::CreditsTransactionType;
#[allow(unused_imports)]
use crate::domain::services::rate_limiting_service::{
BacklogService, ConcurrencyConfig, ConcurrencyControlService, ConcurrencyResult,
QuotaService, RateLimitConfig, RateLimitResult, RateLimitService, RateLimitingError,
};
struct MockAllAllowRateLimitingService;
#[async_trait::async_trait]
impl RateLimitService for MockAllAllowRateLimitingService {
async fn check_rate_limit(
&self,
_api_key: &str,
_endpoint: &str,
) -> Result<RateLimitResult, RateLimitingError> {
Ok(RateLimitResult::Allowed)
}
async fn get_team_rate_limit_config(
&self,
_team_id: crate::domain::id::Id,
) -> Result<RateLimitConfig, RateLimitingError> {
Ok(RateLimitConfig::default())
}
async fn update_team_rate_limit_config(
&self,
_team_id: crate::domain::id::Id,
_config: RateLimitConfig,
) -> Result<(), RateLimitingError> {
Ok(())
}
async fn cleanup_expired_rate_limits(&self) -> Result<u64, RateLimitingError> {
Ok(0)
}
}
#[async_trait::async_trait]
impl ConcurrencyControlService for MockAllAllowRateLimitingService {
async fn check_team_concurrency(
&self,
_team_id: crate::domain::id::Id,
_task_id: crate::domain::id::Id,
) -> Result<ConcurrencyResult, RateLimitingError> {
Ok(ConcurrencyResult::Allowed)
}
async fn release_team_concurrency_slot(
&self,
_team_id: crate::domain::id::Id,
_task_id: crate::domain::id::Id,
) -> Result<(), RateLimitingError> {
Ok(())
}
async fn get_team_current_concurrency(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u32, RateLimitingError> {
Ok(0)
}
async fn get_team_concurrency_config(
&self,
_team_id: crate::domain::id::Id,
) -> Result<ConcurrencyConfig, RateLimitingError> {
Ok(ConcurrencyConfig::default())
}
async fn update_team_concurrency_config(
&self,
_team_id: crate::domain::id::Id,
_config: ConcurrencyConfig,
) -> Result<(), RateLimitingError> {
Ok(())
}
}
#[async_trait::async_trait]
impl BacklogService for MockAllAllowRateLimitingService {
async fn process_backlog_tasks(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u32, RateLimitingError> {
Ok(0)
}
}
#[async_trait::async_trait]
impl QuotaService for MockAllAllowRateLimitingService {
async fn check_and_deduct_quota(
&self,
_team_id: crate::domain::id::Id,
_amount: i64,
_transaction_type: CreditsTransactionType,
_description: String,
_reference_id: Option<crate::domain::id::Id>,
) -> Result<(), RateLimitingError> {
Ok(())
}
async fn refund_quota(
&self,
_team_id: crate::domain::id::Id,
_amount: i64,
_description: String,
_reference_id: Option<crate::domain::id::Id>,
) -> Result<(), RateLimitingError> {
Ok(())
}
async fn get_quota_balance(
&self,
_team_id: crate::domain::id::Id,
) -> Result<i64, RateLimitingError> {
Ok(1000)
}
}
impl RateLimitingService for MockAllAllowRateLimitingService {}
fn make_rate_limiting_service() -> Arc<dyn RateLimitingService> {
Arc::new(MockAllAllowRateLimitingService)
}
#[test]
fn test_extract_response_dto_serialization() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let task_id = crate::domain::id::Id::generate();
let dto = ExtractResponseDto {
id: task_id,
status: "pending".to_string(),
};
let json = serde_json::to_string(&dto).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(parsed["id"], task_id.to_string());
assert_eq!(parsed["status"], "pending");
}
#[test]
fn test_extract_response_dto_deserialization() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let task_id = crate::domain::id::Id::generate();
let json = format!(r#"{{"id":"{}","status":"completed"}}"#, task_id);
let dto: ExtractResponseDto = serde_json::from_str(&json).unwrap();
assert_eq!(dto.id, task_id);
assert_eq!(dto.status, "completed");
}
#[test]
fn test_extract_response_dto_accepted_status() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = ExtractResponseDto {
id: crate::domain::id::Id::generate(),
status: "accepted".to_string(),
};
let json = serde_json::to_string(&dto).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(parsed["status"], "accepted");
}
#[test]
fn test_extract_request_dto_minimal_with_prompt() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"urls":["https://example.com"],"prompt":"Extract title"}"#;
let dto: ExtractRequestDto = serde_json::from_str(json).unwrap();
assert_eq!(dto.urls.len(), 1);
assert_eq!(dto.prompt.as_deref(), Some("Extract title"));
assert!(dto.schema.is_none());
assert!(dto.rules.is_none());
}
#[test]
fn test_extract_request_dto_with_schema() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"urls":["https://example.com"],"schema":{"type":"object"}}"#;
let dto: ExtractRequestDto = serde_json::from_str(json).unwrap();
assert!(dto.schema.is_some());
assert!(dto.prompt.is_none());
}
#[test]
fn test_extract_request_dto_with_sync_wait_ms() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"urls":["https://example.com"],"prompt":"test","sync_wait_ms":10000}"#;
let dto: ExtractRequestDto = serde_json::from_str(json).unwrap();
assert_eq!(dto.sync_wait_ms, Some(10000));
}
#[test]
fn test_extract_request_dto_multiple_urls() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"urls":["https://a.com","https://b.com","https://c.com"],"prompt":"test"}"#;
let dto: ExtractRequestDto = serde_json::from_str(json).unwrap();
assert_eq!(dto.urls.len(), 3);
}
#[test]
fn test_extract_request_dto_empty_urls() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"urls":[],"prompt":"test"}"#;
let dto: ExtractRequestDto = serde_json::from_str(json).unwrap();
assert!(dto.urls.is_empty());
}
#[test]
fn test_max_sync_wait_ms_value() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
assert_eq!(crawl_task::MAX_SYNC_WAIT_MS, 30000);
}
#[test]
fn test_default_timeout_ms_value() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
assert_eq!(crawl_task::DEFAULT_TIMEOUT_MS, 5000);
}
#[test]
fn test_base_poll_interval_ms_value() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
assert_eq!(crawl_task::BASE_POLL_INTERVAL_MS, 1000);
}
#[test]
fn test_extract_response_dto_clone() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let task_id = crate::domain::id::Id::generate();
let dto = ExtractResponseDto {
id: task_id,
status: "pending".to_string(),
};
let cloned = dto.clone();
assert_eq!(dto.id, cloned.id);
assert_eq!(dto.status, cloned.status);
}
#[test]
fn test_extract_response_dto_debug() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let task_id = crate::domain::id::Id::generate();
let dto = ExtractResponseDto {
id: task_id,
status: "completed".to_string(),
};
let debug = format!("{:?}", dto);
assert!(debug.contains("ExtractResponseDto"));
assert!(debug.contains(&task_id.to_string()));
assert!(debug.contains("completed"));
}
#[test]
fn test_extract_response_dto_round_trip() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let task_id = crate::domain::id::Id::generate();
let original = ExtractResponseDto {
id: task_id,
status: "queued".to_string(),
};
let json = serde_json::to_string(&original).unwrap();
let deserialized: ExtractResponseDto = serde_json::from_str(&json).unwrap();
assert_eq!(original.id, deserialized.id);
assert_eq!(original.status, deserialized.status);
}
#[test]
fn test_extract_request_dto_with_rules() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let mut rules = std::collections::HashMap::new();
rules.insert(
"title".to_string(),
crate::domain::services::extraction_service::ExtractionRule {
selector: Some("h1".to_string()),
attr: None,
is_array: false,
use_llm: None,
llm_prompt: None,
output_format: None,
},
);
let dto = ExtractRequestDto {
urls: vec!["https://example.com".to_string()],
prompt: None,
schema: None,
model: None,
rules: Some(rules),
sync_wait_ms: None,
};
assert!(dto.rules.is_some());
assert_eq!(dto.rules.as_ref().unwrap().len(), 1);
}
#[test]
fn test_extract_request_dto_with_model() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"urls":["https://example.com"],"prompt":"test","model":"gpt-4"}"#;
let dto: ExtractRequestDto = serde_json::from_str(json).unwrap();
assert_eq!(dto.model.as_deref(), Some("gpt-4"));
}
#[test]
fn test_extract_request_dto_full_payload() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{
"urls": ["https://a.com", "https://b.com"],
"prompt": "Extract data",
"schema": {"type": "object", "properties": {}},
"model": "gpt-4",
"sync_wait_ms": 5000
}"#;
let dto: ExtractRequestDto = serde_json::from_str(json).unwrap();
assert_eq!(dto.urls.len(), 2);
assert!(dto.prompt.is_some());
assert!(dto.schema.is_some());
assert!(dto.model.is_some());
assert_eq!(dto.sync_wait_ms, Some(5000));
assert!(dto.rules.is_none());
}
#[test]
fn test_extract_request_dto_minimal_with_only_schema() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"urls":["https://example.com"],"schema":{"type":"object"}}"#;
let dto: ExtractRequestDto = serde_json::from_str(json).unwrap();
assert!(dto.prompt.is_none());
assert!(dto.schema.is_some());
assert!(dto.rules.is_none());
}
#[test]
fn test_extract_request_dto_minimal_with_only_rules() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"urls":["https://example.com"],"rules":{}}"#;
let dto: ExtractRequestDto = serde_json::from_str(json).unwrap();
assert!(dto.prompt.is_none());
assert!(dto.schema.is_none());
assert!(dto.rules.is_some());
assert!(dto.rules.as_ref().unwrap().is_empty());
}
#[test]
fn test_extract_request_dto_no_extraction_method() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"urls":["https://example.com"]}"#;
let dto: ExtractRequestDto = serde_json::from_str(json).unwrap();
assert!(dto.prompt.is_none());
assert!(dto.schema.is_none());
assert!(dto.rules.is_none());
}
#[test]
fn test_extract_request_dto_sync_wait_ms_zero() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"urls":["https://example.com"],"prompt":"test","sync_wait_ms":0}"#;
let dto: ExtractRequestDto = serde_json::from_str(json).unwrap();
assert_eq!(dto.sync_wait_ms, Some(0));
}
#[test]
fn test_validation_empty_urls_fails() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = ExtractRequestDto {
urls: vec![],
prompt: Some("test".to_string()),
schema: None,
model: None,
rules: None,
sync_wait_ms: None,
};
assert!(dto.urls.is_empty(), "empty urls should trigger BAD_REQUEST");
}
#[test]
fn test_validation_no_extraction_method_fails() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = ExtractRequestDto {
urls: vec!["https://example.com".to_string()],
prompt: None,
schema: None,
model: None,
rules: None,
sync_wait_ms: None,
};
let has_extraction_method =
dto.prompt.is_some() || dto.schema.is_some() || dto.rules.is_some();
assert!(
!has_extraction_method,
"no extraction method should trigger BAD_REQUEST"
);
}
#[test]
fn test_validation_has_prompt_passes() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = ExtractRequestDto {
urls: vec!["https://example.com".to_string()],
prompt: Some("test".to_string()),
schema: None,
model: None,
rules: None,
sync_wait_ms: None,
};
let has_extraction_method =
dto.prompt.is_some() || dto.schema.is_some() || dto.rules.is_some();
assert!(has_extraction_method);
assert!(!dto.urls.is_empty());
}
#[test]
fn test_validation_has_schema_passes() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = ExtractRequestDto {
urls: vec!["https://example.com".to_string()],
prompt: None,
schema: Some(serde_json::json!({"type": "object"})),
model: None,
rules: None,
sync_wait_ms: None,
};
let has_extraction_method =
dto.prompt.is_some() || dto.schema.is_some() || dto.rules.is_some();
assert!(has_extraction_method);
}
#[test]
fn test_validation_has_rules_passes() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = ExtractRequestDto {
urls: vec!["https://example.com".to_string()],
prompt: None,
schema: None,
model: None,
rules: Some(std::collections::HashMap::new()),
sync_wait_ms: None,
};
let has_extraction_method =
dto.prompt.is_some() || dto.schema.is_some() || dto.rules.is_some();
assert!(has_extraction_method);
}
#[test]
fn test_validation_sync_wait_ms_at_max_passes() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = ExtractRequestDto {
urls: vec!["https://example.com".to_string()],
prompt: Some("test".to_string()),
schema: None,
model: None,
rules: None,
sync_wait_ms: Some(crawl_task::MAX_SYNC_WAIT_MS),
};
if let Some(ms) = dto.sync_wait_ms {
assert!(ms <= crawl_task::MAX_SYNC_WAIT_MS);
}
}
#[test]
fn test_validation_sync_wait_ms_exceeds_max() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = ExtractRequestDto {
urls: vec!["https://example.com".to_string()],
prompt: Some("test".to_string()),
schema: None,
model: None,
rules: None,
sync_wait_ms: Some(crawl_task::MAX_SYNC_WAIT_MS + 1),
};
if let Some(ms) = dto.sync_wait_ms {
assert!(ms > crawl_task::MAX_SYNC_WAIT_MS, "should exceed max");
}
}
#[test]
fn test_task_construction_for_extract() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let task_id = crate::domain::id::Id::generate();
let team_id = crate::domain::id::Id::generate();
let api_key_id = crate::domain::id::Id::generate();
let now = chrono::Utc::now();
let task = Task {
id: task_id,
task_type: TaskType::Extract,
status: TaskStatus::Queued,
priority: 0,
team_id,
api_key_id,
url: "https://example.com".to_string(),
payload: serde_json::Value::Null,
retry_count: 0,
attempt_count: 0,
max_retries: 3,
scheduled_at: None,
expires_at: None,
created_at: now,
started_at: None,
completed_at: None,
crawl_id: None,
updated_at: now,
lock_token: None,
lock_expires_at: None,
};
assert_eq!(task.task_type, TaskType::Extract);
assert_eq!(task.status, TaskStatus::Queued);
assert_eq!(task.max_retries, 3);
}
#[test]
fn test_extract_task_credits_cost() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
assert_eq!(crawl_task::EXTRACT_TASK_CREDITS_COST, 8);
}
#[test]
fn test_scrape_task_credits_cost() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
assert_eq!(crawl_task::SCRAPE_TASK_CREDITS_COST, 5);
}
#[test]
fn test_crawl_task_credits_cost() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
assert_eq!(crawl_task::CRAWL_TASK_CREDITS_COST, 10);
}
#[tokio::test]
async fn test_error_response_builds_correct_status() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
use axum::body::to_bytes;
let response = error_response(StatusCode::BAD_REQUEST, "test error");
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert!(json["success"].is_boolean());
assert_eq!(json["success"], false);
}
#[tokio::test]
async fn test_error_response_internal_server_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
use axum::body::to_bytes;
let response = error_response(StatusCode::INTERNAL_SERVER_ERROR, "Something went wrong");
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(json["success"], false);
}
#[tokio::test]
async fn test_error_response_forbidden() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let response = error_response(
StatusCode::FORBIDDEN,
"Access denied due to geographic restrictions: blocked region",
);
assert_eq!(response.status(), StatusCode::FORBIDDEN);
}
use crate::domain::auth::ApiKeyScope;
use crate::domain::repositories::geo_restriction_repository::GeoRestrictionRepositoryError;
use crate::domain::repositories::task_repository::{RepositoryError, TaskQueryParams};
use crate::domain::services::geo_location::{GeoLocation, GeoLocationService};
use crate::domain::services::team_service::TeamGeoRestrictions;
use crate::queue::task_queue::QueueError;
use async_trait::async_trait;
use axum::body::to_bytes;
use dbnexus::DbPool;
use std::collections::HashSet;
use std::net::IpAddr;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::Mutex;
struct MockTaskQueue {
enqueue_should_fail: bool,
enqueue_count: AtomicU32,
}
impl MockTaskQueue {
fn succeeding() -> Self {
Self {
enqueue_should_fail: false,
enqueue_count: AtomicU32::new(0),
}
}
fn failing() -> Self {
Self {
enqueue_should_fail: true,
enqueue_count: AtomicU32::new(0),
}
}
}
#[async_trait]
impl TaskQueue for MockTaskQueue {
async fn enqueue(&self, task: Task) -> Result<Task, QueueError> {
self.enqueue_count.fetch_add(1, Ordering::SeqCst);
if self.enqueue_should_fail {
return Err(QueueError::Repository(RepositoryError::Database(
anyhow::anyhow!("queue enqueue down"),
)));
}
Ok(task)
}
async fn dequeue(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, QueueError> {
Ok(None)
}
async fn complete(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
async fn fail(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
async fn cancel(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
}
struct MockTaskRepository {
created_count: AtomicU32,
should_fail_query: bool,
completed_task: Mutex<Option<Task>>,
}
impl MockTaskRepository {
fn succeeding() -> Self {
Self {
created_count: AtomicU32::new(0),
should_fail_query: false,
completed_task: Mutex::new(None),
}
}
fn failing_query() -> Self {
Self {
created_count: AtomicU32::new(0),
should_fail_query: true,
completed_task: Mutex::new(None),
}
}
fn with_completed_task(task: Task) -> Self {
Self {
created_count: AtomicU32::new(0),
should_fail_query: false,
completed_task: Mutex::new(Some(task)),
}
}
}
#[async_trait]
impl TaskRepository for MockTaskRepository {
async fn create(&self, task: &Task) -> Result<Task, RepositoryError> {
self.created_count.fetch_add(1, Ordering::SeqCst);
Ok(task.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn update(&self, task: &Task) -> Result<Task, RepositoryError> {
Ok(task.clone())
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
_urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
Ok(HashSet::new())
}
async fn reset_stuck_tasks(
&self,
_timeout: chrono::Duration,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
Ok(vec![])
}
async fn query_tasks(
&self,
_params: TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
if self.should_fail_query {
return Err(RepositoryError::Database(anyhow::anyhow!(
"query_tasks down"
)));
}
let guard = self.completed_task.lock().unwrap();
if let Some(ref task) = *guard {
return Ok((vec![task.clone()], 1));
}
Ok((vec![], 0))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((vec![], vec![]))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
struct MockGeoRestrictionRepository {
restrictions: Mutex<TeamGeoRestrictions>,
get_should_fail: bool,
log_should_fail: bool,
log_count: AtomicU32,
}
impl MockGeoRestrictionRepository {
fn with_restrictions(restrictions: TeamGeoRestrictions) -> Self {
Self {
restrictions: Mutex::new(restrictions),
get_should_fail: false,
log_should_fail: false,
log_count: AtomicU32::new(0),
}
}
fn failing_get() -> Self {
Self {
restrictions: Mutex::new(TeamGeoRestrictions::default()),
get_should_fail: true,
log_should_fail: false,
log_count: AtomicU32::new(0),
}
}
fn with_failing_log(restrictions: TeamGeoRestrictions) -> Self {
Self {
restrictions: Mutex::new(restrictions),
get_should_fail: false,
log_should_fail: true,
log_count: AtomicU32::new(0),
}
}
}
#[async_trait]
impl GeoRestrictionRepository for MockGeoRestrictionRepository {
async fn get_team_restrictions(
&self,
_team_id: crate::domain::id::Id,
) -> Result<TeamGeoRestrictions, GeoRestrictionRepositoryError> {
if self.get_should_fail {
return Err(GeoRestrictionRepositoryError::Database(
"geo restriction repo down".to_string(),
));
}
Ok(self.restrictions.lock().unwrap().clone())
}
async fn update_team_restrictions(
&self,
_team_id: crate::domain::id::Id,
_restrictions: &TeamGeoRestrictions,
) -> Result<(), GeoRestrictionRepositoryError> {
Ok(())
}
async fn log_geo_restriction_action(
&self,
_team_id: crate::domain::id::Id,
_ip_address: &str,
_country_code: &str,
_action: &str,
_reason: &str,
) -> Result<(), GeoRestrictionRepositoryError> {
self.log_count.fetch_add(1, Ordering::SeqCst);
if self.log_should_fail {
return Err(GeoRestrictionRepositoryError::Other(
"log action down".to_string(),
));
}
Ok(())
}
}
struct MockGeoLocationService {
should_fail: bool,
country_code: String,
}
impl MockGeoLocationService {
fn succeeding_with_country(code: &str) -> Self {
Self {
should_fail: false,
country_code: code.to_string(),
}
}
fn failing() -> Self {
Self {
should_fail: true,
country_code: "US".to_string(),
}
}
}
#[async_trait]
impl GeoLocationService for MockGeoLocationService {
async fn get_location(&self, _ip: &IpAddr) -> anyhow::Result<GeoLocation> {
if self.should_fail {
return Err(anyhow::anyhow!("geolocation service down"));
}
Ok(GeoLocation {
country_code: self.country_code.clone(),
..GeoLocation::default()
})
}
}
fn make_test_db_pool() -> Arc<DbPool> {
crate::common::test_helpers::create_test_db_pool()
}
fn make_test_auth_state() -> AuthState {
AuthState::new(
make_test_db_pool(),
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
ApiKeyScope::default(),
)
}
fn make_test_settings() -> Arc<Settings> {
Arc::new(Settings::default())
}
fn make_addr() -> SocketAddr {
SocketAddr::from(([127, 0, 0, 1], 8080))
}
fn make_valid_payload() -> ExtractRequestDto {
ExtractRequestDto {
urls: vec!["https://example.com".to_string()],
prompt: Some("Extract title".to_string()),
schema: None,
model: None,
rules: None,
sync_wait_ms: Some(0),
}
}
fn make_team_service(geo_loc_service: Arc<dyn GeoLocationService>) -> Arc<TeamService> {
let geo_repo: Arc<dyn GeoRestrictionRepository> = Arc::new(
MockGeoRestrictionRepository::with_restrictions(TeamGeoRestrictions::default()),
);
Arc::new(TeamService::new(geo_loc_service, geo_repo))
}
async fn assert_response(
response: axum::response::Response,
expected_status: StatusCode,
) -> serde_json::Value {
assert_eq!(response.status(), expected_status);
let bytes = to_bytes(response.into_body(), usize::MAX)
.await
.expect("failed to read response body");
serde_json::from_slice(&bytes).expect("response body is not valid JSON")
}
#[tokio::test]
async fn test_extract_empty_urls_returns_unprocessable() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
TeamGeoRestrictions::default(),
));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let payload = ExtractRequestDto {
urls: vec![],
prompt: Some("test".to_string()),
schema: None,
model: None,
rules: None,
sync_wait_ms: None,
};
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
let json = assert_response(response, StatusCode::UNPROCESSABLE_ENTITY).await;
assert_eq!(json["success"], false);
}
#[tokio::test]
async fn test_extract_no_extraction_method_returns_unprocessable() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
TeamGeoRestrictions::default(),
));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let payload = ExtractRequestDto {
urls: vec!["https://example.com".to_string()],
prompt: None,
schema: None,
model: None,
rules: None,
sync_wait_ms: None,
};
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
let json = assert_response(response, StatusCode::UNPROCESSABLE_ENTITY).await;
assert_eq!(json["success"], false);
}
#[tokio::test]
async fn test_extract_geo_repo_error_returns_internal_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let geo_repo = Arc::new(MockGeoRestrictionRepository::failing_get());
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(make_valid_payload())),
)
.await
.into_response();
let json = assert_response(response, StatusCode::INTERNAL_SERVER_ERROR).await;
assert_eq!(json["success"], false);
}
#[tokio::test]
async fn test_extract_sync_wait_ms_exceeds_max_returns_bad_request() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
TeamGeoRestrictions::default(),
));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let payload = ExtractRequestDto {
urls: vec!["https://example.com".to_string()],
prompt: Some("test".to_string()),
schema: None,
model: None,
rules: None,
sync_wait_ms: Some(crawl_task::MAX_SYNC_WAIT_MS + 1),
};
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
let json = assert_response(response, StatusCode::BAD_REQUEST).await;
assert_eq!(json["success"], false);
}
#[tokio::test]
async fn test_extract_geo_denied_returns_forbidden() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let restrictions = TeamGeoRestrictions {
enable_geo_restrictions: true,
allowed_countries: None,
blocked_countries: Some(vec!["US".to_string()]),
ip_whitelist: None,
domain_blacklist: None,
};
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
restrictions,
));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(make_valid_payload())),
)
.await
.into_response();
let json = assert_response(response, StatusCode::FORBIDDEN).await;
assert_eq!(json["success"], false);
assert!(
json["error"]["message"]
.as_str()
.unwrap_or("")
.contains("geographic restrictions"),
"error message should mention geographic restrictions"
);
}
#[tokio::test]
async fn test_extract_geo_validation_error_returns_internal_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let restrictions = TeamGeoRestrictions {
enable_geo_restrictions: true,
allowed_countries: None,
blocked_countries: Some(vec!["CN".to_string()]),
ip_whitelist: None,
domain_blacklist: None,
};
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
restrictions,
));
let team_service = make_team_service(Arc::new(MockGeoLocationService::failing()));
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(make_valid_payload())),
)
.await
.into_response();
let json = assert_response(response, StatusCode::INTERNAL_SERVER_ERROR).await;
assert_eq!(json["success"], false);
}
#[tokio::test]
async fn test_extract_success_returns_created() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
TeamGeoRestrictions::default(),
));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let payload = make_valid_payload();
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
let json = assert_response(response, StatusCode::CREATED).await;
assert_eq!(json["success"], true);
assert!(json["data"]["id"].is_string());
assert_eq!(json["data"]["status"], "pending");
}
#[tokio::test]
async fn test_extract_with_sync_wait_returns_accepted() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
TeamGeoRestrictions::default(),
));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let payload = ExtractRequestDto {
urls: vec!["https://example.com".to_string()],
prompt: Some("test".to_string()),
schema: None,
model: None,
rules: None,
sync_wait_ms: Some(1),
};
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
let json = assert_response(response, StatusCode::ACCEPTED).await;
assert_eq!(json["success"], true);
assert!(json["data"]["id"].is_string());
assert_eq!(json["data"]["status"], "pending");
}
#[tokio::test]
async fn test_extract_enqueue_failure_returns_internal_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::failing());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
TeamGeoRestrictions::default(),
));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(make_valid_payload())),
)
.await
.into_response();
let json = assert_response(response, StatusCode::INTERNAL_SERVER_ERROR).await;
assert_eq!(json["success"], false);
}
#[tokio::test]
async fn test_extract_allowed_country_passes_geo_check() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let restrictions = TeamGeoRestrictions {
enable_geo_restrictions: true,
allowed_countries: Some(vec!["US".to_string()]),
blocked_countries: None,
ip_whitelist: None,
domain_blacklist: None,
};
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
restrictions,
));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(make_valid_payload())),
)
.await
.into_response();
let json = assert_response(response, StatusCode::CREATED).await;
assert_eq!(json["success"], true);
}
#[tokio::test]
async fn test_extract_ip_whitelist_bypasses_country_check() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let restrictions = TeamGeoRestrictions {
enable_geo_restrictions: true,
allowed_countries: Some(vec!["CN".to_string()]),
blocked_countries: None,
ip_whitelist: Some(vec!["127.0.0.0/8".to_string()]),
domain_blacklist: None,
};
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
restrictions,
));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(make_valid_payload())),
)
.await
.into_response();
let json = assert_response(response, StatusCode::CREATED).await;
assert_eq!(json["success"], true);
}
#[tokio::test]
async fn test_extract_allowed_with_failing_log() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_failing_log(
TeamGeoRestrictions::default(),
));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo.clone()),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(make_valid_payload())),
)
.await
.into_response();
let json = assert_response(response, StatusCode::CREATED).await;
assert_eq!(json["success"], true);
assert_eq!(geo_repo.log_count.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_extract_denied_with_failing_log() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let restrictions = TeamGeoRestrictions {
enable_geo_restrictions: true,
allowed_countries: None,
blocked_countries: Some(vec!["US".to_string()]),
ip_whitelist: None,
domain_blacklist: None,
};
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_failing_log(restrictions));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo.clone()),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(make_valid_payload())),
)
.await
.into_response();
let json = assert_response(response, StatusCode::FORBIDDEN).await;
assert_eq!(json["success"], false);
assert_eq!(geo_repo.log_count.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_extract_sync_wait_with_query_failure() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::failing_query());
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
TeamGeoRestrictions::default(),
));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let payload = ExtractRequestDto {
urls: vec!["https://example.com".to_string()],
prompt: Some("test".to_string()),
schema: None,
model: None,
rules: None,
sync_wait_ms: Some(1000),
};
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
let json = assert_response(response, StatusCode::CREATED).await;
assert_eq!(json["success"], true);
assert!(json["data"]["id"].is_string());
}
#[tokio::test]
async fn test_extract_sync_wait_completes_returns_created() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let completed_task = Task {
id: crate::domain::id::Id::generate(),
task_type: TaskType::Extract,
status: TaskStatus::Completed,
priority: 0,
team_id: crate::domain::id::Id::generate(),
api_key_id: crate::domain::id::Id::generate(),
url: "https://example.com".to_string(),
payload: serde_json::Value::Null,
retry_count: 0,
attempt_count: 0,
max_retries: 3,
scheduled_at: None,
expires_at: None,
created_at: chrono::Utc::now(),
started_at: None,
completed_at: None,
crawl_id: None,
updated_at: chrono::Utc::now(),
lock_token: None,
lock_expires_at: None,
};
let task_repo: Arc<dyn TaskRepository> =
Arc::new(MockTaskRepository::with_completed_task(completed_task));
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
TeamGeoRestrictions::default(),
));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let payload = ExtractRequestDto {
urls: vec!["https://example.com".to_string()],
prompt: Some("test".to_string()),
schema: None,
model: None,
rules: None,
sync_wait_ms: Some(500),
};
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
let json = assert_response(response, StatusCode::CREATED).await;
assert_eq!(json["success"], true);
assert!(json["data"]["id"].is_string());
assert_eq!(json["data"]["status"], "pending");
}
use crate::test_utils::ensure_debug_logger;
#[tokio::test]
async fn test_extract_no_extraction_method_log_evaluated() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
ensure_debug_logger();
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
TeamGeoRestrictions::default(),
));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let payload = ExtractRequestDto {
urls: vec!["https://example.com".to_string()],
prompt: None,
schema: None,
model: None,
rules: None,
sync_wait_ms: None,
};
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
let json = assert_response(response, StatusCode::UNPROCESSABLE_ENTITY).await;
assert_eq!(json["success"], false);
}
#[tokio::test]
async fn test_extract_geo_repo_error_log_evaluated() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
ensure_debug_logger();
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let geo_repo = Arc::new(MockGeoRestrictionRepository::failing_get());
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(make_valid_payload())),
)
.await
.into_response();
let json = assert_response(response, StatusCode::INTERNAL_SERVER_ERROR).await;
assert_eq!(json["success"], false);
}
#[tokio::test]
async fn test_extract_sync_wait_ms_exceeds_max_log_evaluated() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
ensure_debug_logger();
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
TeamGeoRestrictions::default(),
));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let payload = ExtractRequestDto {
urls: vec!["https://example.com".to_string()],
prompt: Some("test".to_string()),
schema: None,
model: None,
rules: None,
sync_wait_ms: Some(crawl_task::MAX_SYNC_WAIT_MS + 1),
};
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
let json = assert_response(response, StatusCode::BAD_REQUEST).await;
assert_eq!(json["success"], false);
}
#[tokio::test]
async fn test_extract_geo_denied_log_evaluated() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
ensure_debug_logger();
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let restrictions = TeamGeoRestrictions {
enable_geo_restrictions: true,
allowed_countries: None,
blocked_countries: Some(vec!["US".to_string()]),
ip_whitelist: None,
domain_blacklist: None,
};
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
restrictions,
));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(make_valid_payload())),
)
.await
.into_response();
let json = assert_response(response, StatusCode::FORBIDDEN).await;
assert_eq!(json["success"], false);
}
#[tokio::test]
async fn test_extract_geo_validation_error_log_evaluated() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
ensure_debug_logger();
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let restrictions = TeamGeoRestrictions {
enable_geo_restrictions: true,
allowed_countries: None,
blocked_countries: Some(vec!["CN".to_string()]),
ip_whitelist: None,
domain_blacklist: None,
};
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
restrictions,
));
let team_service = make_team_service(Arc::new(MockGeoLocationService::failing()));
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(make_valid_payload())),
)
.await
.into_response();
let json = assert_response(response, StatusCode::INTERNAL_SERVER_ERROR).await;
assert_eq!(json["success"], false);
}
#[tokio::test]
async fn test_extract_sync_wait_log_evaluated() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
ensure_debug_logger();
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::succeeding());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::failing_query());
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
TeamGeoRestrictions::default(),
));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let payload = ExtractRequestDto {
urls: vec!["https://example.com".to_string()],
prompt: Some("test".to_string()),
schema: None,
model: None,
rules: None,
sync_wait_ms: Some(1000),
};
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
let json = assert_response(response, StatusCode::CREATED).await;
assert_eq!(json["success"], true);
}
#[tokio::test]
async fn test_extract_enqueue_failure_log_evaluated() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
ensure_debug_logger();
let queue: Arc<dyn TaskQueue> = Arc::new(MockTaskQueue::failing());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::succeeding());
let geo_repo = Arc::new(MockGeoRestrictionRepository::with_restrictions(
TeamGeoRestrictions::default(),
));
let team_service = make_team_service(Arc::new(
MockGeoLocationService::succeeding_with_country("US"),
));
let response = extract::<MockGeoRestrictionRepository>(
Extension(queue),
Extension(make_test_settings()),
Extension(task_repo),
Extension(make_rate_limiting_service()),
Extension(geo_repo),
Extension(team_service),
Extension(make_test_auth_state()),
ConnectInfo(make_addr()),
Ok(Json(make_valid_payload())),
)
.await
.into_response();
let json = assert_response(response, StatusCode::INTERNAL_SERVER_ERROR).await;
assert_eq!(json["success"], false);
}
}