pub(crate) use super::scrape_commands::{cancel_scrape, create_scrape};
pub(crate) use super::scrape_queries::get_scrape_status;
#[cfg(test)]
mod tests {
use super::*;
use crate::application::dto::scrape_request::{ScrapeActionDto, ScrapeRequestDto};
use crate::application::dto::scrape_response::{
CancelScrapeResponseDto, CapturedResponseDto, ScrapeResponseDto, ScrapeResultDto,
ScrapeStatusResponseDto,
};
use crate::common::constants::crawl_task::MAX_SYNC_WAIT_MS;
use crate::config::settings::Settings;
use crate::domain::models::{Task, TaskStatus, TaskType};
use crate::domain::repositories::geo_restriction_repository::GeoRestrictionRepository;
use crate::domain::services::team_service::TeamService;
use axum::extract::ConnectInfo;
use std::net::SocketAddr;
fn make_addr() -> SocketAddr {
"127.0.0.1:8080".parse().expect("valid socket addr")
}
use crate::domain::repositories::scrape_result_repository::ScrapeResultRepository;
use crate::domain::repositories::task_repository::TaskRepository;
use crate::domain::services::rate_limiting_service::RateLimitingService;
use crate::i18n::{I18nBundle, Locale};
use crate::presentation::extractors::CrawlRsDeps;
use crate::presentation::middleware::auth_middleware::AuthState;
use axum::{
extract::{Extension, Path},
http::StatusCode,
response::IntoResponse,
Json,
};
use chrono::NaiveDateTime;
use sdforge::validator::Validate;
use std::sync::Arc;
#[test]
fn test_scrape_request_dto_minimal_valid() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"url":"https://example.com"}"#;
let dto: ScrapeRequestDto = serde_json::from_str(json).unwrap();
assert_eq!(dto.url, "https://example.com");
assert!(dto.formats.is_none());
assert!(dto.sync_wait_ms.is_none());
}
#[test]
fn test_scrape_request_dto_with_all_fields() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{
"url": "https://example.com",
"formats": ["html", "markdown"],
"include_tags": ["div", "span"],
"exclude_tags": ["script"],
"sync_wait_ms": 5000,
"options": {
"headers": {},
"timeout": 30,
"js_rendering": true
}
}"#;
let dto: ScrapeRequestDto = serde_json::from_str(json).unwrap();
assert_eq!(dto.url, "https://example.com");
assert_eq!(dto.formats.as_ref().unwrap().len(), 2);
assert_eq!(dto.sync_wait_ms, Some(5000));
assert!(dto.options.is_some());
}
#[test]
fn test_scrape_request_dto_rejects_unknown_fields() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"url":"https://example.com","unknown_field":"value"}"#;
let result: Result<ScrapeRequestDto, _> = serde_json::from_str(json);
assert!(result.is_err());
}
#[test]
fn test_scrape_request_dto_empty_url_fails_validation() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"url":""}"#;
let dto: ScrapeRequestDto = serde_json::from_str(json).unwrap();
let validation = dto.validate();
assert!(validation.is_err());
}
#[test]
fn test_scrape_request_dto_sync_wait_ms_at_max() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"url":"https://example.com","sync_wait_ms":30000}"#;
let dto: ScrapeRequestDto = serde_json::from_str(json).unwrap();
assert!(dto.validate().is_ok());
}
#[test]
fn test_scrape_request_dto_sync_wait_ms_exceeds_max() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"url":"https://example.com","sync_wait_ms":30001}"#;
let dto: ScrapeRequestDto = serde_json::from_str(json).unwrap();
assert!(dto.validate().is_err());
}
#[test]
fn test_scrape_request_dto_sync_wait_ms_zero_ok() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"url":"https://example.com","sync_wait_ms":0}"#;
let dto: ScrapeRequestDto = serde_json::from_str(json).unwrap();
assert!(dto.validate().is_ok());
}
#[test]
fn test_scrape_response_dto_serialization() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let task_id = crate::domain::id::Id::generate();
let dto = ScrapeResponseDto {
id: task_id,
url: "https://example.com".to_string(),
credits_used: 1,
captured_responses: None,
};
let json = serde_json::to_string(&dto).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(parsed["id"], task_id.to_string());
assert_eq!(parsed["url"], "https://example.com");
assert_eq!(parsed["credits_used"], 1);
}
#[test]
fn test_scrape_response_dto_empty_capture_absent_in_json() {
let dto = ScrapeResponseDto {
id: crate::domain::id::Id::generate(),
url: "https://example.com".to_string(),
credits_used: 1,
captured_responses: None,
};
let json = serde_json::to_string(&dto).unwrap();
assert!(
!json.contains("captured_responses"),
"None captures must be absent from JSON, got: {json}"
);
let parsed: ScrapeResponseDto =
serde_json::from_str(&format!(r#"{{"id":"{}","url":"https://x.com"}}"#, dto.id))
.unwrap();
assert!(parsed.captured_responses.is_none());
}
#[test]
fn test_scrape_response_dto_non_empty_capture_complete_in_json() {
let dto = ScrapeResponseDto {
id: crate::domain::id::Id::generate(),
url: "https://example.com".to_string(),
credits_used: 1,
captured_responses: Some(vec![CapturedResponseDto {
url: "https://example.com/api/data".to_string(),
status: 200,
mime_type: "application/json".to_string(),
body: r#"{"ok":true}"#.to_string(),
}]),
};
let json = serde_json::to_string(&dto).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
let captures = parsed["captured_responses"].as_array().unwrap();
assert_eq!(captures.len(), 1);
assert_eq!(captures[0]["url"], "https://example.com/api/data");
assert_eq!(captures[0]["status"], 200);
assert_eq!(captures[0]["mime_type"], "application/json");
assert_eq!(captures[0]["body"], r#"{"ok":true}"#);
}
#[test]
fn test_scrape_response_dto_deserialization() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = format!(
r#"{{"id":"{}","url":"https://test.com","credits_used":5}}"#,
crate::domain::id::Id::generate()
);
let dto: ScrapeResponseDto = serde_json::from_str(&json).unwrap();
assert_eq!(dto.url, "https://test.com");
assert_eq!(dto.credits_used, 5);
}
#[test]
fn test_scrape_response_dto_default_credits_used() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = format!(
r#"{{"id":"{}","url":"https://test.com"}}"#,
crate::domain::id::Id::generate()
);
let dto: ScrapeResponseDto = serde_json::from_str(&json).unwrap();
assert_eq!(dto.credits_used, 0);
}
#[test]
fn test_scrape_result_dto_serialization() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = ScrapeResultDto {
content: "<html>test</html>".to_string(),
status_code: 200,
content_type: Some("text/html".to_string()),
response_time_ms: 150,
headers: Some(serde_json::json!({"content-length": "100"})),
meta_data: Some(serde_json::json!({"key": "value"})),
screenshot: None,
created_at: NaiveDateTime::parse_from_str("2025-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S")
.unwrap(),
};
let json = serde_json::to_string(&dto).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(parsed["status_code"], 200);
assert_eq!(parsed["content"], "<html>test</html>");
assert_eq!(parsed["response_time_ms"], 150);
assert_eq!(parsed["content_type"], "text/html");
assert!(parsed["screenshot"].is_null());
}
#[test]
fn test_scrape_result_dto_with_screenshot() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = ScrapeResultDto {
content: "content".to_string(),
status_code: 200,
content_type: None,
response_time_ms: 0,
headers: None,
meta_data: None,
screenshot: Some("base64data".to_string()),
created_at: NaiveDateTime::parse_from_str("2025-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S")
.unwrap(),
};
let json = serde_json::to_string(&dto).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(parsed["screenshot"], "base64data");
}
#[test]
fn test_scrape_status_response_dto_pending() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let task_id = crate::domain::id::Id::generate();
let dto = ScrapeStatusResponseDto {
id: task_id,
status: "queued".to_string(),
url: "https://example.com".to_string(),
created_at: NaiveDateTime::parse_from_str("2025-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S")
.unwrap(),
completed_at: None,
result: None,
metadata: None,
error: None,
};
let json = serde_json::to_string(&dto).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(parsed["status"], "queued");
assert!(parsed["completed_at"].is_null());
assert!(parsed["result"].is_null());
}
#[test]
fn test_scrape_status_response_dto_failed_with_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = ScrapeStatusResponseDto {
id: crate::domain::id::Id::generate(),
status: "failed".to_string(),
url: "https://example.com".to_string(),
created_at: NaiveDateTime::parse_from_str("2025-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S")
.unwrap(),
completed_at: Some(
NaiveDateTime::parse_from_str("2025-01-01T00:01:00", "%Y-%m-%dT%H:%M:%S").unwrap(),
),
result: None,
metadata: None,
error: Some("Connection timeout".to_string()),
};
let json = serde_json::to_string(&dto).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(parsed["status"], "failed");
assert_eq!(parsed["error"], "Connection timeout");
assert!(parsed["completed_at"].is_string());
}
#[test]
fn test_cancel_scrape_response_dto_serialization() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = CancelScrapeResponseDto {
message: "Scrape task cancelled".to_string(),
};
let json = serde_json::to_string(&dto).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(parsed["message"], "Scrape task cancelled");
}
#[test]
fn test_cancel_scrape_response_dto_deserialization() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"message":"done"}"#;
let dto: CancelScrapeResponseDto = serde_json::from_str(json).unwrap();
assert_eq!(dto.message, "done");
}
#[test]
fn test_scrape_action_wait_deserialization() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"type":"wait","milliseconds":1000}"#;
let action: ScrapeActionDto = serde_json::from_str(json).unwrap();
match action {
ScrapeActionDto::Wait { milliseconds } => assert_eq!(milliseconds, 1000),
_ => panic!("Expected Wait action"),
}
}
#[test]
fn test_scrape_action_click_deserialization() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r##"{"type":"click","selector":"#button"}"##;
let action: ScrapeActionDto = serde_json::from_str(json).unwrap();
match action {
ScrapeActionDto::Click { selector } => assert_eq!(selector, "#button"),
_ => panic!("Expected Click action"),
}
}
#[test]
fn test_scrape_action_scroll_deserialization() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"type":"scroll","direction":"down"}"#;
let action: ScrapeActionDto = serde_json::from_str(json).unwrap();
match action {
ScrapeActionDto::Scroll { direction } => assert_eq!(direction, "down"),
_ => panic!("Expected Scroll action"),
}
}
#[test]
fn test_scrape_action_screenshot_with_full_page() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"type":"screenshot","full_page":true}"#;
let action: ScrapeActionDto = serde_json::from_str(json).unwrap();
match action {
ScrapeActionDto::Screenshot { full_page } => assert_eq!(full_page, Some(true)),
_ => panic!("Expected Screenshot action"),
}
}
#[test]
fn test_scrape_action_input_deserialization() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r##"{"type":"input","selector":"#search","text":"hello"}"##;
let action: ScrapeActionDto = serde_json::from_str(json).unwrap();
match action {
ScrapeActionDto::Input { selector, text } => {
assert_eq!(selector, "#search");
assert_eq!(text, "hello");
}
_ => panic!("Expected Input action"),
}
}
#[test]
fn test_scrape_action_wait_serialization_roundtrip() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let action = ScrapeActionDto::Wait { milliseconds: 2000 };
let json = serde_json::to_string(&action).unwrap();
let deserialized: ScrapeActionDto = serde_json::from_str(&json).unwrap();
match deserialized {
ScrapeActionDto::Wait { milliseconds } => assert_eq!(milliseconds, 2000),
_ => panic!("Expected Wait action"),
}
}
#[test]
fn test_scrape_action_click_serialization_roundtrip() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let action = ScrapeActionDto::Click {
selector: ".btn".to_string(),
};
let json = serde_json::to_string(&action).unwrap();
let deserialized: ScrapeActionDto = serde_json::from_str(&json).unwrap();
match deserialized {
ScrapeActionDto::Click { selector } => assert_eq!(selector, ".btn"),
_ => panic!("Expected Click action"),
}
}
#[test]
fn test_scrape_action_scroll_serialization_roundtrip() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let action = ScrapeActionDto::Scroll {
direction: "up".to_string(),
};
let json = serde_json::to_string(&action).unwrap();
let deserialized: ScrapeActionDto = serde_json::from_str(&json).unwrap();
match deserialized {
ScrapeActionDto::Scroll { direction } => assert_eq!(direction, "up"),
_ => panic!("Expected Scroll action"),
}
}
#[test]
fn test_scrape_action_screenshot_without_full_page() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"type":"screenshot"}"#;
let action: ScrapeActionDto = serde_json::from_str(json).unwrap();
match action {
ScrapeActionDto::Screenshot { full_page } => assert_eq!(full_page, None),
_ => panic!("Expected Screenshot action"),
}
}
#[test]
fn test_scrape_action_screenshot_full_page_false() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"type":"screenshot","full_page":false}"#;
let action: ScrapeActionDto = serde_json::from_str(json).unwrap();
match action {
ScrapeActionDto::Screenshot { full_page } => assert_eq!(full_page, Some(false)),
_ => panic!("Expected Screenshot action"),
}
}
#[test]
fn test_scrape_action_unknown_type_fails() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"type":"unknown","data":123}"#;
let result: Result<ScrapeActionDto, _> = serde_json::from_str(json);
assert!(result.is_err());
}
#[test]
fn test_scrape_action_camel_case_tag() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"type":"wait","milliseconds":100}"#;
let action: ScrapeActionDto = serde_json::from_str(json).unwrap();
match action {
ScrapeActionDto::Wait { milliseconds } => assert_eq!(milliseconds, 100),
_ => panic!("Expected Wait action"),
}
}
#[test]
fn test_scrape_options_dto_minimal() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"headers":{}}"#;
let dto: crate::application::dto::scrape_request::ScrapeOptionsDto =
serde_json::from_str(json).unwrap();
assert!(dto.headers.is_some());
assert!(dto.timeout.is_none());
assert!(dto.js_rendering.is_none());
assert!(dto.screenshot.is_none());
assert!(dto.proxy.is_none());
}
#[test]
fn test_scrape_options_dto_full() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{
"headers": {"User-Agent": "test"},
"wait_for": 1000,
"timeout": 60,
"js_rendering": true,
"screenshot": true,
"mobile": true,
"proxy": "http://proxy:8080",
"skip_tls_verification": false,
"needs_tls_fingerprint": true,
"use_fire_engine": true
}"#;
let dto: crate::application::dto::scrape_request::ScrapeOptionsDto =
serde_json::from_str(json).unwrap();
assert_eq!(dto.timeout, Some(60));
assert_eq!(dto.wait_for, Some(1000));
assert_eq!(dto.js_rendering, Some(true));
assert_eq!(dto.screenshot, Some(true));
assert_eq!(dto.mobile, Some(true));
assert_eq!(dto.proxy.as_deref(), Some("http://proxy:8080"));
assert_eq!(dto.skip_tls_verification, Some(false));
assert_eq!(dto.needs_tls_fingerprint, Some(true));
assert_eq!(dto.use_fire_engine, Some(true));
}
#[test]
fn test_scrape_options_dto_deny_unknown_fields() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{"headers":{},"unknown":1}"#;
let result: Result<crate::application::dto::scrape_request::ScrapeOptionsDto, _> =
serde_json::from_str(json);
assert!(result.is_err());
}
#[test]
fn test_scrape_options_dto_serialization_roundtrip() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = crate::application::dto::scrape_request::ScrapeOptionsDto {
headers: Some(serde_json::json!({"X-Test": "value"})),
wait_for: Some(500),
timeout: Some(30),
js_rendering: Some(true),
screenshot: Some(false),
screenshot_options: None,
mobile: Some(false),
proxy: Some("http://proxy:9090".to_string()),
skip_tls_verification: None,
needs_tls_fingerprint: None,
use_fire_engine: None,
..Default::default()
};
let json = serde_json::to_string(&dto).unwrap();
let deserialized: crate::application::dto::scrape_request::ScrapeOptionsDto =
serde_json::from_str(&json).unwrap();
assert_eq!(deserialized.timeout, Some(30));
assert_eq!(deserialized.wait_for, Some(500));
assert_eq!(deserialized.js_rendering, Some(true));
assert_eq!(deserialized.proxy.as_deref(), Some("http://proxy:9090"));
}
#[test]
fn test_screenshot_options_dto_minimal() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{}"#;
let dto: crate::application::dto::scrape_request::ScreenshotOptionsDto =
serde_json::from_str(json).unwrap();
assert!(dto.full_page.is_none());
assert!(dto.selector.is_none());
assert!(dto.quality.is_none());
assert!(dto.format.is_none());
}
#[test]
fn test_screenshot_options_dto_full() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r##"{
"full_page": true,
"selector": "#content",
"quality": 80,
"format": "jpeg"
}"##;
let dto: crate::application::dto::scrape_request::ScreenshotOptionsDto =
serde_json::from_str(json).unwrap();
assert_eq!(dto.full_page, Some(true));
assert_eq!(dto.selector.as_deref(), Some("#content"));
assert_eq!(dto.quality, Some(80));
assert_eq!(dto.format.as_deref(), Some("jpeg"));
}
#[test]
fn test_screenshot_options_dto_serialization_roundtrip() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = crate::application::dto::scrape_request::ScreenshotOptionsDto {
full_page: Some(false),
selector: Some("div.main".to_string()),
quality: Some(100),
format: Some("png".to_string()),
};
let json = serde_json::to_string(&dto).unwrap();
let deserialized: crate::application::dto::scrape_request::ScreenshotOptionsDto =
serde_json::from_str(&json).unwrap();
assert_eq!(deserialized.full_page, Some(false));
assert_eq!(deserialized.selector.as_deref(), Some("div.main"));
assert_eq!(deserialized.quality, Some(100));
assert_eq!(deserialized.format.as_deref(), Some("png"));
}
#[test]
fn test_max_sync_wait_ms_constant() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
assert_eq!(MAX_SYNC_WAIT_MS, 30000);
}
#[test]
fn test_sync_wait_ms_at_max_passes_handler_check() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let ms: u32 = 30000;
assert!(ms <= MAX_SYNC_WAIT_MS, "ms at max should pass");
}
#[test]
fn test_sync_wait_ms_above_max_fails_handler_check() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let ms: u32 = 30001;
assert!(ms > MAX_SYNC_WAIT_MS, "ms above max should fail");
}
#[test]
fn test_sync_wait_ms_none_skips_check() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let ms: Option<u32> = None;
assert!(ms.is_none());
}
#[test]
fn test_sync_wait_ms_zero_passes_handler_check() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let ms: u32 = 0;
assert!(ms <= MAX_SYNC_WAIT_MS);
}
#[test]
fn test_status_code_async_mode_returns_created() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let sync_wait_ms: u32 = 0;
let is_timeout = false;
let status_code = if sync_wait_ms > 0 {
if is_timeout {
StatusCode::ACCEPTED
} else {
StatusCode::CREATED
}
} else {
StatusCode::CREATED
};
assert_eq!(status_code, StatusCode::CREATED);
}
#[test]
fn test_status_code_sync_mode_timeout_returns_accepted() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let sync_wait_ms: u32 = 5000;
let is_timeout = true;
let status_code = if sync_wait_ms > 0 {
if is_timeout {
StatusCode::ACCEPTED
} else {
StatusCode::CREATED
}
} else {
StatusCode::CREATED
};
assert_eq!(status_code, StatusCode::ACCEPTED);
}
#[test]
fn test_status_code_sync_mode_completed_returns_created() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let sync_wait_ms: u32 = 5000;
let is_timeout = false;
let status_code = if sync_wait_ms > 0 {
if is_timeout {
StatusCode::ACCEPTED
} else {
StatusCode::CREATED
}
} else {
StatusCode::CREATED
};
assert_eq!(status_code, StatusCode::CREATED);
}
#[test]
fn test_scrape_result_dto_deserialization() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{
"content": "<html></html>",
"status_code": 200,
"content_type": "text/html",
"response_time_ms": 100,
"headers": {"x-custom": "val"},
"meta_data": {"key": "value"},
"screenshot": null,
"created_at": "2025-01-01T00:00:00"
}"#;
let dto: ScrapeResultDto = serde_json::from_str(json).unwrap();
assert_eq!(dto.content, "<html></html>");
assert_eq!(dto.status_code, 200);
assert_eq!(dto.content_type.as_deref(), Some("text/html"));
assert_eq!(dto.response_time_ms, 100);
assert!(dto.screenshot.is_none());
}
#[test]
fn test_scrape_result_dto_deserialization_minimal() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{
"content": "text",
"status_code": 404,
"response_time_ms": 50,
"created_at": "2025-06-01T12:00:00"
}"#;
let dto: ScrapeResultDto = serde_json::from_str(json).unwrap();
assert_eq!(dto.status_code, 404);
assert_eq!(dto.response_time_ms, 50);
assert!(dto.content_type.is_none());
assert!(dto.headers.is_none());
assert!(dto.meta_data.is_none());
assert!(dto.screenshot.is_none());
}
#[test]
fn test_scrape_status_response_dto_deserialization() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let task_id = crate::domain::id::Id::generate();
let json = format!(
r#"{{
"id": "{}",
"status": "completed",
"url": "https://example.com",
"created_at": "2025-01-01T00:00:00",
"completed_at": "2025-01-01T00:01:00",
"result": null,
"metadata": {{"page": 1}},
"error": null
}}"#,
task_id
);
let dto: ScrapeStatusResponseDto = serde_json::from_str(&json).unwrap();
assert_eq!(dto.id, task_id);
assert_eq!(dto.status, "completed");
assert_eq!(dto.url, "https://example.com");
assert!(dto.result.is_none());
assert!(dto.error.is_none());
assert!(dto.completed_at.is_some());
assert!(dto.metadata.is_some());
}
#[test]
fn test_scrape_status_response_dto_with_result() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{
"id": "00000000-0000-0000-0000-000000000001",
"status": "completed",
"url": "https://test.com",
"created_at": "2025-01-01T00:00:00",
"completed_at": null,
"result": {
"content": "data",
"status_code": 200,
"content_type": "application/json",
"response_time_ms": 42,
"headers": null,
"meta_data": null,
"screenshot": null,
"created_at": "2025-01-01T00:00:01"
},
"metadata": null,
"error": null
}"#;
let dto: ScrapeStatusResponseDto = serde_json::from_str(json).unwrap();
assert!(dto.result.is_some());
let result = dto.result.unwrap();
assert_eq!(result.content, "data");
assert_eq!(result.status_code, 200);
assert_eq!(result.response_time_ms, 42);
}
#[test]
fn test_scrape_response_dto_roundtrip() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let dto = ScrapeResponseDto {
id: crate::domain::id::Id::generate(),
url: "https://roundtrip.com".to_string(),
credits_used: 3,
captured_responses: None,
};
let json = serde_json::to_string(&dto).unwrap();
let deserialized: ScrapeResponseDto = serde_json::from_str(&json).unwrap();
assert_eq!(deserialized.id, dto.id);
assert_eq!(deserialized.url, dto.url);
assert_eq!(deserialized.credits_used, dto.credits_used);
}
#[test]
fn test_task_status_display_for_scrape_response() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
assert_eq!(TaskStatus::Queued.to_string(), "queued");
assert_eq!(TaskStatus::Active.to_string(), "active");
assert_eq!(TaskStatus::Completed.to_string(), "completed");
assert_eq!(TaskStatus::Failed.to_string(), "failed");
assert_eq!(TaskStatus::Cancelled.to_string(), "cancelled");
}
#[test]
fn test_task_construction_for_scrape() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let api_key_id = crate::domain::id::Id::generate();
let now = chrono::Utc::now();
let task = Task {
id: crate::domain::id::Id::generate(),
task_type: TaskType::Scrape,
status: TaskStatus::Queued,
priority: 0,
team_id,
api_key_id,
url: "https://example.com".to_string(),
payload: serde_json::Value::Null,
retry_count: 0,
attempt_count: 0,
max_retries: 3,
scheduled_at: None,
expires_at: None,
created_at: now,
started_at: None,
completed_at: None,
crawl_id: None,
updated_at: now,
lock_token: None,
lock_expires_at: None,
};
assert_eq!(task.task_type, TaskType::Scrape);
assert_eq!(task.status, TaskStatus::Queued);
assert_eq!(task.priority, 0);
assert_eq!(task.max_retries, 3);
assert_eq!(task.team_id, team_id);
assert_eq!(task.api_key_id, api_key_id);
assert!(task.scheduled_at.is_none());
assert!(task.expires_at.is_none());
assert!(task.started_at.is_none());
assert!(task.completed_at.is_none());
}
#[test]
fn test_scrape_request_dto_with_options_and_actions() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r##"{
"url": "https://example.com",
"formats": ["html"],
"actions": [
{"type":"wait","milliseconds":500},
{"type":"click","selector":"#load-more"}
],
"options": {
"timeout": 30,
"js_rendering": true
}
}"##;
let dto: ScrapeRequestDto = serde_json::from_str(json).unwrap();
assert!(dto.actions.is_some());
assert_eq!(dto.actions.as_ref().unwrap().len(), 2);
assert!(dto.options.is_some());
assert_eq!(dto.options.as_ref().unwrap().timeout, Some(30));
}
#[test]
fn test_scrape_request_dto_with_metadata() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{
"url": "https://example.com",
"metadata": {"user_id": 123, "session": "abc"}
}"#;
let dto: ScrapeRequestDto = serde_json::from_str(json).unwrap();
assert!(dto.metadata.is_some());
let meta = dto.metadata.unwrap();
assert_eq!(meta["user_id"], 123);
assert_eq!(meta["session"], "abc");
}
#[test]
fn test_scrape_request_dto_with_webhook() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let json = r#"{
"url": "https://example.com",
"webhook": "https://hooks.example.com/abc"
}"#;
let dto: ScrapeRequestDto = serde_json::from_str(json).unwrap();
assert_eq!(
dto.webhook.as_deref(),
Some("https://hooks.example.com/abc")
);
}
#[test]
fn test_scrape_request_dto_url_too_long_fails_validation() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let long_url = "https://example.com/".to_string() + &"a".repeat(2048);
let json = serde_json::json!({
"url": long_url
})
.to_string();
let dto: ScrapeRequestDto = serde_json::from_str(&json).unwrap();
assert!(dto.validate().is_err());
}
#[test]
fn test_scrape_request_dto_url_at_max_length_passes() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let base = "https://example.com/";
let padding = "a".repeat(2048 - base.len());
let url = format!("{}{}", base, padding);
assert_eq!(url.len(), 2048);
let json = serde_json::json!({ "url": url }).to_string();
let dto: ScrapeRequestDto = serde_json::from_str(&json).unwrap();
assert!(dto.validate().is_ok());
}
use crate::common::test_helpers::create_test_db_pool;
use crate::domain::auth::ApiKeyScope;
use crate::domain::models::ScrapeResult;
use crate::domain::repositories::task_repository::{RepositoryError, TaskQueryParams};
use crate::domain::services::rate_limiting_service::{
BacklogService, ConcurrencyConfig, ConcurrencyControlService, ConcurrencyResult,
QuotaService, RateLimitConfig, RateLimitResult, RateLimitService, RateLimitingError,
};
use crate::queue::task_queue::{QueueError, TaskQueue};
use async_trait::async_trait;
use std::collections::{HashMap, HashSet};
use std::sync::Mutex;
type SharedTasks = Arc<Mutex<HashMap<crate::domain::id::Id, Task>>>;
struct MockTaskQueue {
enqueue_should_fail: bool,
}
impl MockTaskQueue {
fn new_success() -> Self {
Self {
enqueue_should_fail: false,
}
}
fn new_failing() -> Self {
Self {
enqueue_should_fail: true,
}
}
}
#[async_trait]
impl TaskQueue for MockTaskQueue {
async fn enqueue(&self, task: Task) -> Result<Task, QueueError> {
if self.enqueue_should_fail {
return Err(QueueError::Repository(RepositoryError::Database(
anyhow::anyhow!("enqueue failed"),
)));
}
Ok(task)
}
async fn dequeue(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, QueueError> {
Ok(None)
}
async fn complete(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
async fn fail(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
async fn cancel(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
}
struct MockTaskQueueLinkedCompleted {
shared: SharedTasks,
}
impl MockTaskQueueLinkedCompleted {
fn new(shared: SharedTasks) -> Self {
Self { shared }
}
}
#[async_trait]
impl TaskQueue for MockTaskQueueLinkedCompleted {
async fn enqueue(&self, mut task: Task) -> Result<Task, QueueError> {
task.status = TaskStatus::Completed;
task.completed_at = Some(chrono::Utc::now());
self.shared.lock().unwrap().insert(task.id, task.clone());
Ok(task)
}
async fn dequeue(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, QueueError> {
Ok(None)
}
async fn complete(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
async fn fail(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
async fn cancel(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
}
struct MockTaskRepository {
tasks: SharedTasks,
find_should_fail: bool,
mark_cancelled_should_fail: bool,
}
impl MockTaskRepository {
fn new() -> Self {
Self {
tasks: Arc::new(Mutex::new(HashMap::new())),
find_should_fail: false,
mark_cancelled_should_fail: false,
}
}
fn with_task(task: Task) -> Self {
let id = task.id;
let map = Arc::new(Mutex::new(HashMap::new()));
map.lock().unwrap().insert(id, task);
Self {
tasks: map,
find_should_fail: false,
mark_cancelled_should_fail: false,
}
}
fn new_linked(shared: SharedTasks) -> Self {
Self {
tasks: shared,
find_should_fail: false,
mark_cancelled_should_fail: false,
}
}
fn failing_find() -> Self {
Self {
tasks: Arc::new(Mutex::new(HashMap::new())),
find_should_fail: true,
mark_cancelled_should_fail: false,
}
}
fn failing_mark_cancelled() -> Self {
Self {
tasks: Arc::new(Mutex::new(HashMap::new())),
find_should_fail: false,
mark_cancelled_should_fail: true,
}
}
}
#[async_trait]
impl TaskRepository for MockTaskRepository {
async fn create(&self, task: &Task) -> Result<Task, RepositoryError> {
Ok(task.clone())
}
async fn find_by_id(
&self,
id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
if self.find_should_fail {
return Err(RepositoryError::Database(anyhow::anyhow!(
"find_by_id failed"
)));
}
Ok(self.tasks.lock().unwrap().get(&id).cloned())
}
async fn update(&self, task: &Task) -> Result<Task, RepositoryError> {
Ok(task.clone())
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
if self.mark_cancelled_should_fail {
return Err(RepositoryError::Database(anyhow::anyhow!(
"mark_cancelled failed"
)));
}
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
_urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
Ok(HashSet::new())
}
async fn reset_stuck_tasks(
&self,
_timeout: chrono::Duration,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
Ok(vec![])
}
async fn query_tasks(
&self,
params: TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
let tasks = self.tasks.lock().unwrap();
let mut result: Vec<Task> = Vec::new();
if let Some(ref task_ids) = params.task_ids {
for id in task_ids {
if let Some(task) = tasks.get(id) {
result.push(task.clone());
}
}
}
let count = result.len() as u64;
Ok((result, count))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((vec![], vec![]))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
struct MockRateLimitingService {
rate_limit_result: RateLimitResult,
quota_should_fail: bool,
}
impl MockRateLimitingService {
fn new_allowed() -> Self {
Self {
rate_limit_result: RateLimitResult::Allowed,
quota_should_fail: false,
}
}
fn new_denied() -> Self {
Self {
rate_limit_result: RateLimitResult::Denied {
reason: "Too many requests".to_string(),
},
quota_should_fail: false,
}
}
fn new_retry_after() -> Self {
Self {
rate_limit_result: RateLimitResult::RetryAfter {
retry_after_seconds: 30,
},
quota_should_fail: false,
}
}
fn new_quota_exceeded() -> Self {
Self {
rate_limit_result: RateLimitResult::Allowed,
quota_should_fail: true,
}
}
}
#[async_trait]
impl RateLimitService for MockRateLimitingService {
async fn check_rate_limit(
&self,
_api_key: &str,
_endpoint: &str,
) -> Result<RateLimitResult, RateLimitingError> {
Ok(self.rate_limit_result.clone())
}
async fn get_team_rate_limit_config(
&self,
_team_id: crate::domain::id::Id,
) -> Result<RateLimitConfig, RateLimitingError> {
Ok(RateLimitConfig::default())
}
async fn update_team_rate_limit_config(
&self,
_team_id: crate::domain::id::Id,
_config: RateLimitConfig,
) -> Result<(), RateLimitingError> {
Ok(())
}
async fn cleanup_expired_rate_limits(&self) -> Result<u64, RateLimitingError> {
Ok(0)
}
}
#[async_trait]
impl ConcurrencyControlService for MockRateLimitingService {
async fn check_team_concurrency(
&self,
_team_id: crate::domain::id::Id,
_task_id: crate::domain::id::Id,
) -> Result<ConcurrencyResult, RateLimitingError> {
Ok(ConcurrencyResult::Allowed)
}
async fn release_team_concurrency_slot(
&self,
_team_id: crate::domain::id::Id,
_task_id: crate::domain::id::Id,
) -> Result<(), RateLimitingError> {
Ok(())
}
async fn get_team_current_concurrency(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u32, RateLimitingError> {
Ok(0)
}
async fn get_team_concurrency_config(
&self,
_team_id: crate::domain::id::Id,
) -> Result<ConcurrencyConfig, RateLimitingError> {
Ok(ConcurrencyConfig::default())
}
async fn update_team_concurrency_config(
&self,
_team_id: crate::domain::id::Id,
_config: ConcurrencyConfig,
) -> Result<(), RateLimitingError> {
Ok(())
}
}
#[async_trait]
impl BacklogService for MockRateLimitingService {
async fn process_backlog_tasks(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u32, RateLimitingError> {
Ok(0)
}
}
#[async_trait]
impl QuotaService for MockRateLimitingService {
async fn check_and_deduct_quota(
&self,
_team_id: crate::domain::id::Id,
_amount: i64,
_transaction_type: crate::domain::models::CreditsTransactionType,
_description: String,
_reference_id: Option<crate::domain::id::Id>,
) -> Result<(), RateLimitingError> {
if self.quota_should_fail {
return Err(RateLimitingError::CreditsError);
}
Ok(())
}
async fn get_quota_balance(
&self,
_team_id: crate::domain::id::Id,
) -> Result<i64, RateLimitingError> {
Ok(1000)
}
async fn refund_quota(
&self,
_team_id: crate::domain::id::Id,
_amount: i64,
_description: String,
_reference_id: Option<crate::domain::id::Id>,
) -> Result<(), RateLimitingError> {
Ok(())
}
}
#[async_trait]
impl RateLimitingService for MockRateLimitingService {}
struct MockScrapeResultRepository {
result: Option<ScrapeResult>,
should_fail: bool,
}
impl MockScrapeResultRepository {
fn new_empty() -> Self {
Self {
result: None,
should_fail: false,
}
}
fn with_result(result: ScrapeResult) -> Self {
Self {
result: Some(result),
should_fail: false,
}
}
fn failing() -> Self {
Self {
result: None,
should_fail: true,
}
}
}
#[async_trait]
impl ScrapeResultRepository for MockScrapeResultRepository {
async fn save(&self, _result: ScrapeResult) -> anyhow::Result<()> {
Ok(())
}
async fn find_by_task_id(
&self,
_task_id: crate::domain::id::Id,
) -> anyhow::Result<Option<ScrapeResult>> {
if self.should_fail {
return Err(anyhow::anyhow!("scrape result repo down"));
}
Ok(self.result.clone())
}
async fn find_by_task_ids(
&self,
_task_ids: &[crate::domain::id::Id],
) -> anyhow::Result<Vec<ScrapeResult>> {
Ok(vec![])
}
async fn get_team_avg_response_time(
&self,
_team_id: crate::domain::id::Id,
) -> anyhow::Result<f64> {
Ok(0.0)
}
async fn cleanup_expired(&self, _retention_days: i64) -> anyhow::Result<u64> {
Ok(0)
}
}
fn make_auth_state() -> AuthState {
AuthState::new(
create_test_db_pool(),
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
ApiKeyScope::default(),
)
}
fn test_locale() -> Locale {
"en-US".parse().unwrap()
}
fn test_bundle() -> Arc<I18nBundle> {
Arc::new(I18nBundle::load("en-US", &["en-US", "zh-CN"]).unwrap())
}
fn make_auth_state_with_team(team_id: crate::domain::id::Id) -> AuthState {
AuthState::new(
create_test_db_pool(),
team_id,
crate::domain::id::Id::generate(),
ApiKeyScope::default(),
)
}
fn make_task(team_id: crate::domain::id::Id, status: TaskStatus) -> Task {
let now = chrono::Utc::now();
Task {
id: crate::domain::id::Id::generate(),
task_type: TaskType::Scrape,
status,
priority: 0,
team_id,
api_key_id: crate::domain::id::Id::generate(),
url: "https://example.com".to_string(),
payload: serde_json::Value::Null,
retry_count: 0,
attempt_count: 0,
max_retries: 3,
scheduled_at: None,
expires_at: None,
created_at: now,
started_at: None,
completed_at: None,
crawl_id: None,
updated_at: now,
lock_token: None,
lock_expires_at: None,
}
}
fn make_scrape_result(task_id: crate::domain::id::Id) -> ScrapeResult {
ScrapeResult {
id: crate::domain::id::Id::generate(),
task_id,
url: "https://example.com".to_string(),
status_code: 200,
content: "<html>test</html>".to_string(),
content_type: "text/html".to_string(),
response_time_ms: 100,
created_at: chrono::Utc::now(),
headers: serde_json::json!({"content-length": "100"}),
meta_data: serde_json::json!({"key": "value"}),
screenshot: None,
}
}
fn make_scrape_request_dto(url: &str, sync_wait_ms: Option<u32>) -> ScrapeRequestDto {
ScrapeRequestDto {
url: url.to_string(),
formats: None,
include_tags: None,
exclude_tags: None,
webhook: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
actions: None,
options: None,
metadata: None,
sync_wait_ms,
}
}
#[tokio::test]
async fn test_create_scrape_success_async() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue = Arc::new(MockTaskQueue::new_success());
let task_repo = Arc::new(MockTaskRepository::new());
let rate_limit = Arc::new(MockRateLimitingService::new_allowed());
let settings = Arc::new(Settings::default());
let auth = make_auth_state();
let payload = make_scrape_request_dto("https://example.com", None);
let response = create_scrape(
CrawlRsDeps {
queue,
settings,
task_repo,
rate_limiting_service: rate_limit,
auth_state: auth,
},
Extension(None::<Arc<dyn GeoRestrictionRepository>>),
Extension(None::<Arc<TeamService>>),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::CREATED);
}
#[tokio::test]
async fn test_create_scrape_sync_wait_ms_exceeds_max() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue = Arc::new(MockTaskQueue::new_success());
let task_repo = Arc::new(MockTaskRepository::new());
let rate_limit = Arc::new(MockRateLimitingService::new_allowed());
let settings = Arc::new(Settings::default());
let auth = make_auth_state();
let payload = make_scrape_request_dto("https://example.com", Some(MAX_SYNC_WAIT_MS + 1));
let response = create_scrape(
CrawlRsDeps {
queue,
settings,
task_repo,
rate_limiting_service: rate_limit,
auth_state: auth,
},
Extension(None::<Arc<dyn GeoRestrictionRepository>>),
Extension(None::<Arc<TeamService>>),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
}
#[tokio::test]
async fn test_create_scrape_ssrf_blocked_localhost() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue = Arc::new(MockTaskQueue::new_success());
let task_repo = Arc::new(MockTaskRepository::new());
let rate_limit = Arc::new(MockRateLimitingService::new_allowed());
let settings = Arc::new(Settings::default());
let auth = make_auth_state();
let payload = make_scrape_request_dto("http://localhost", None);
let response = create_scrape(
CrawlRsDeps {
queue,
settings,
task_repo,
rate_limiting_service: rate_limit,
auth_state: auth,
},
Extension(None::<Arc<dyn GeoRestrictionRepository>>),
Extension(None::<Arc<TeamService>>),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn test_create_scrape_ssrf_blocked_127001() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue = Arc::new(MockTaskQueue::new_success());
let task_repo = Arc::new(MockTaskRepository::new());
let rate_limit = Arc::new(MockRateLimitingService::new_allowed());
let settings = Arc::new(Settings::default());
let auth = make_auth_state();
let payload = make_scrape_request_dto("http://127.0.0.1", None);
let response = create_scrape(
CrawlRsDeps {
queue,
settings,
task_repo,
rate_limiting_service: rate_limit,
auth_state: auth,
},
Extension(None::<Arc<dyn GeoRestrictionRepository>>),
Extension(None::<Arc<TeamService>>),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn test_create_scrape_rate_limited_denied() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue = Arc::new(MockTaskQueue::new_success());
let task_repo = Arc::new(MockTaskRepository::new());
let rate_limit = Arc::new(MockRateLimitingService::new_denied());
let settings = Arc::new(Settings::default());
let auth = make_auth_state();
let payload = make_scrape_request_dto("https://example.com", None);
let response = create_scrape(
CrawlRsDeps {
queue,
settings,
task_repo,
rate_limiting_service: rate_limit,
auth_state: auth,
},
Extension(None::<Arc<dyn GeoRestrictionRepository>>),
Extension(None::<Arc<TeamService>>),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS);
}
#[tokio::test]
async fn test_create_scrape_rate_limited_retry_after() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue = Arc::new(MockTaskQueue::new_success());
let task_repo = Arc::new(MockTaskRepository::new());
let rate_limit = Arc::new(MockRateLimitingService::new_retry_after());
let settings = Arc::new(Settings::default());
let auth = make_auth_state();
let payload = make_scrape_request_dto("https://example.com", None);
let response = create_scrape(
CrawlRsDeps {
queue,
settings,
task_repo,
rate_limiting_service: rate_limit,
auth_state: auth,
},
Extension(None::<Arc<dyn GeoRestrictionRepository>>),
Extension(None::<Arc<TeamService>>),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS);
}
#[tokio::test]
async fn test_create_scrape_quota_exceeded() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue = Arc::new(MockTaskQueue::new_success());
let task_repo = Arc::new(MockTaskRepository::new());
let rate_limit = Arc::new(MockRateLimitingService::new_quota_exceeded());
let settings = Arc::new(Settings::default());
let auth = make_auth_state();
let payload = make_scrape_request_dto("https://example.com", None);
let response = create_scrape(
CrawlRsDeps {
queue,
settings,
task_repo,
rate_limiting_service: rate_limit,
auth_state: auth,
},
Extension(None::<Arc<dyn GeoRestrictionRepository>>),
Extension(None::<Arc<TeamService>>),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::PAYMENT_REQUIRED);
}
#[tokio::test]
async fn test_create_scrape_enqueue_failure() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let queue = Arc::new(MockTaskQueue::new_failing());
let task_repo = Arc::new(MockTaskRepository::new());
let rate_limit = Arc::new(MockRateLimitingService::new_allowed());
let settings = Arc::new(Settings::default());
let auth = make_auth_state();
let payload = make_scrape_request_dto("https://example.com", None);
let response = create_scrape(
CrawlRsDeps {
queue,
settings,
task_repo,
rate_limiting_service: rate_limit,
auth_state: auth,
},
Extension(None::<Arc<dyn GeoRestrictionRepository>>),
Extension(None::<Arc<TeamService>>),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
}
#[tokio::test]
async fn test_create_scrape_sync_mode_completed() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let shared: SharedTasks = Arc::new(Mutex::new(HashMap::new()));
let task_repo = Arc::new(MockTaskRepository::new_linked(Arc::clone(&shared)));
let queue = Arc::new(MockTaskQueueLinkedCompleted::new(Arc::clone(&shared)));
let rate_limit = Arc::new(MockRateLimitingService::new_allowed());
let settings = Arc::new(Settings::default());
let auth = make_auth_state_with_team(team_id);
let payload = make_scrape_request_dto("https://example.com", Some(5000));
let response = create_scrape(
CrawlRsDeps {
queue,
settings,
task_repo,
rate_limiting_service: rate_limit,
auth_state: auth,
},
Extension(None::<Arc<dyn GeoRestrictionRepository>>),
Extension(None::<Arc<TeamService>>),
ConnectInfo(make_addr()),
Ok(Json(payload)),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::CREATED);
}
#[tokio::test]
async fn test_cancel_scrape_success() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let task = make_task(team_id, TaskStatus::Queued);
let task_id = task.id;
let repo = Arc::new(MockTaskRepository::with_task(task));
let auth = make_auth_state_with_team(team_id);
let response = cancel_scrape(
Path(task_id),
Extension(repo),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::OK);
}
#[tokio::test]
async fn test_cancel_scrape_not_found() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let repo = Arc::new(MockTaskRepository::new());
let auth = make_auth_state();
let task_id = crate::domain::id::Id::generate();
let response = cancel_scrape(
Path(task_id),
Extension(repo),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn test_cancel_scrape_forbidden() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let task = make_task(crate::domain::id::Id::generate(), TaskStatus::Queued);
let task_id = task.id;
let repo = Arc::new(MockTaskRepository::with_task(task));
let auth = make_auth_state_with_team(crate::domain::id::Id::generate());
let response = cancel_scrape(
Path(task_id),
Extension(repo),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::FORBIDDEN);
}
#[tokio::test]
async fn test_cancel_scrape_find_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let repo = Arc::new(MockTaskRepository::failing_find());
let auth = make_auth_state();
let task_id = crate::domain::id::Id::generate();
let response = cancel_scrape(
Path(task_id),
Extension(repo),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
}
#[tokio::test]
async fn test_cancel_scrape_mark_cancelled_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let task = make_task(team_id, TaskStatus::Queued);
let task_id = task.id;
let repo = Arc::new(MockTaskRepository::failing_mark_cancelled());
{
let mut tasks = repo.tasks.lock().unwrap();
tasks.insert(task_id, task);
}
let auth = make_auth_state_with_team(team_id);
let response = cancel_scrape(
Path(task_id),
Extension(repo),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
}
#[tokio::test]
async fn test_get_scrape_status_in_progress() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let task = make_task(team_id, TaskStatus::Active);
let task_id = task.id;
let task_repo = Arc::new(MockTaskRepository::with_task(task));
let result_repo = Arc::new(MockScrapeResultRepository::new_empty());
let auth = make_auth_state_with_team(team_id);
let response = get_scrape_status(
Path(task_id),
Extension(task_repo),
Extension(result_repo),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::OK);
}
#[tokio::test]
async fn test_get_scrape_status_completed_with_result() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let mut task = make_task(team_id, TaskStatus::Completed);
task.completed_at = Some(chrono::Utc::now());
let task_id = task.id;
let result = make_scrape_result(task_id);
let task_repo = Arc::new(MockTaskRepository::with_task(task));
let result_repo = Arc::new(MockScrapeResultRepository::with_result(result));
let auth = make_auth_state_with_team(team_id);
let response = get_scrape_status(
Path(task_id),
Extension(task_repo),
Extension(result_repo),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::OK);
}
#[tokio::test]
async fn test_get_scrape_status_completed_no_result() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let mut task = make_task(team_id, TaskStatus::Completed);
task.completed_at = Some(chrono::Utc::now());
let task_id = task.id;
let task_repo = Arc::new(MockTaskRepository::with_task(task));
let result_repo = Arc::new(MockScrapeResultRepository::new_empty());
let auth = make_auth_state_with_team(team_id);
let response = get_scrape_status(
Path(task_id),
Extension(task_repo),
Extension(result_repo),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::OK);
}
#[tokio::test]
async fn test_get_scrape_status_completed_result_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let mut task = make_task(team_id, TaskStatus::Completed);
task.completed_at = Some(chrono::Utc::now());
let task_id = task.id;
let task_repo = Arc::new(MockTaskRepository::with_task(task));
let result_repo = Arc::new(MockScrapeResultRepository::failing());
let auth = make_auth_state_with_team(team_id);
let response = get_scrape_status(
Path(task_id),
Extension(task_repo),
Extension(result_repo),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::OK);
}
#[tokio::test]
async fn test_get_scrape_status_failed_task() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let mut task = make_task(team_id, TaskStatus::Failed);
task.completed_at = Some(chrono::Utc::now());
task.payload = serde_json::json!({"error": "Connection timeout"});
let task_id = task.id;
let task_repo = Arc::new(MockTaskRepository::with_task(task));
let result_repo = Arc::new(MockScrapeResultRepository::new_empty());
let auth = make_auth_state_with_team(team_id);
let response = get_scrape_status(
Path(task_id),
Extension(task_repo),
Extension(result_repo),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::OK);
}
#[tokio::test]
async fn test_get_scrape_status_failed_task_no_error_field() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let team_id = crate::domain::id::Id::generate();
let mut task = make_task(team_id, TaskStatus::Failed);
task.completed_at = Some(chrono::Utc::now());
let task_id = task.id;
let task_repo = Arc::new(MockTaskRepository::with_task(task));
let result_repo = Arc::new(MockScrapeResultRepository::new_empty());
let auth = make_auth_state_with_team(team_id);
let response = get_scrape_status(
Path(task_id),
Extension(task_repo),
Extension(result_repo),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::OK);
}
#[tokio::test]
async fn test_get_scrape_status_not_found() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let task_repo = Arc::new(MockTaskRepository::new());
let result_repo = Arc::new(MockScrapeResultRepository::new_empty());
let auth = make_auth_state();
let task_id = crate::domain::id::Id::generate();
let response = get_scrape_status(
Path(task_id),
Extension(task_repo),
Extension(result_repo),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn test_get_scrape_status_forbidden() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let task = make_task(crate::domain::id::Id::generate(), TaskStatus::Active);
let task_id = task.id;
let task_repo = Arc::new(MockTaskRepository::with_task(task));
let result_repo = Arc::new(MockScrapeResultRepository::new_empty());
let auth = make_auth_state_with_team(crate::domain::id::Id::generate());
let response = get_scrape_status(
Path(task_id),
Extension(task_repo),
Extension(result_repo),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::FORBIDDEN);
}
#[tokio::test]
async fn test_get_scrape_status_find_error() {
if crate::common::test_helpers::skip_if_no_test_db() {
return;
}
let task_repo = Arc::new(MockTaskRepository::failing_find());
let result_repo = Arc::new(MockScrapeResultRepository::new_empty());
let auth = make_auth_state();
let task_id = crate::domain::id::Id::generate();
let response = get_scrape_status(
Path(task_id),
Extension(task_repo),
Extension(result_repo),
Extension(auth),
Extension(test_locale()),
Extension(test_bundle()),
)
.await
.into_response();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
}
}