use crate::config::settings::Settings;
use crate::domain::models::Crawl;
use crate::domain::models::CreditsTransactionType;
use crate::domain::models::{Task, TaskType};
use crate::domain::repositories::crawl_repository::CrawlRepository;
use crate::domain::repositories::credits_repository::CreditsRepository;
use crate::domain::repositories::credits_repository::CreditsRepositoryError;
use crate::domain::repositories::task_repository::TaskRepository;
use chrono::Utc;
use serde_json::json;
use std::sync::Arc;
use thiserror::Error;
#[derive(Debug, Clone)]
pub struct SearchQuery {
pub query: String,
pub limit: Option<u32>,
pub lang: Option<String>,
pub country: Option<String>,
pub engine: Option<String>,
pub sources: Option<Vec<String>>,
pub crawl_results: Option<bool>,
pub crawl_config: Option<SearchCrawlConfig>,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct SearchCrawlConfig {
pub max_depth: u32,
pub include_patterns: Option<Vec<String>>,
pub exclude_patterns: Option<Vec<String>>,
pub strategy: String,
pub crawl_delay_ms: Option<u64>,
pub max_concurrency: u32,
pub headers: Option<serde_json::Value>,
pub proxy: Option<String>,
pub extraction_rules: Option<
std::collections::HashMap<
String,
crate::domain::services::extraction_service::ExtractionRule,
>,
>,
#[serde(default, skip_serializing)]
pub domain_blacklist: Vec<String>,
}
impl Default for SearchCrawlConfig {
fn default() -> Self {
Self {
max_depth: 1,
include_patterns: None,
exclude_patterns: None,
strategy: "bfs".to_string(),
crawl_delay_ms: None,
max_concurrency: 10,
headers: None,
proxy: None,
extraction_rules: None,
domain_blacklist: Vec::new(),
}
}
}
#[derive(Debug, Clone)]
pub struct SearchResult {
pub title: String,
pub url: String,
pub description: Option<String>,
pub engine: String,
pub score: Option<f64>,
}
#[derive(Debug, Clone)]
pub struct SearchResponse {
pub query: String,
pub results: Vec<SearchResult>,
pub crawl_id: Option<crate::domain::id::Id>,
pub credits_used: u32,
}
#[derive(Error, Debug)]
pub enum SearchServiceError {
#[error("Validation failed: {0}")]
ValidationError(String),
#[error("Repository error: {0}")]
Repository(#[from] crate::domain::repositories::task_repository::RepositoryError),
#[error("Credits repository error: {0}")]
CreditsRepository(#[from] CreditsRepositoryError),
#[error("Search engine error: {0}")]
SearchEngine(String),
#[error("Insufficient credits: available {available}, required {required}")]
InsufficientCredits { available: i64, required: i64 },
#[error("unknown search engine: {0}")]
UnknownEngine(String),
#[error("smart search: {0}")]
SmartSearchFailed(String),
}
impl From<String> for SearchServiceError {
fn from(msg: String) -> Self {
SearchServiceError::ValidationError(msg)
}
}
impl From<&str> for SearchServiceError {
fn from(msg: &str) -> Self {
SearchServiceError::ValidationError(msg.to_string())
}
}
impl From<anyhow::Error> for SearchServiceError {
fn from(err: anyhow::Error) -> Self {
SearchServiceError::SearchEngine(err.to_string())
}
}
use crate::domain::services::rerank::SearchReranker;
use crate::search::client::SearchClientTrait;
use crate::search::engine_trait::SearchEngine;
#[async_trait::async_trait]
pub trait SearchServiceTrait: Send + Sync {
async fn search(
&self,
team_id: crate::domain::id::Id,
api_key_id: crate::domain::id::Id,
query: SearchQuery,
) -> Result<SearchResponse, SearchServiceError>;
}
pub const MAX_SEARCH_RESULTS: u32 = 200;
pub const MAX_SEARCH_CRAWL_DEPTH: u32 = 5;
pub struct SearchService {
crawl_repo: Arc<dyn CrawlRepository>,
task_repo: Arc<dyn TaskRepository>,
credits_repo: Arc<dyn CreditsRepository>,
search_client: Arc<dyn SearchClientTrait>,
smart_engine: Option<Arc<dyn SearchEngine>>,
max_results: u32,
reranker: Arc<SearchReranker>,
}
impl SearchService {
pub fn new(
crawl_repo: Arc<dyn CrawlRepository>,
task_repo: Arc<dyn TaskRepository>,
credits_repo: Arc<dyn CreditsRepository>,
settings: Arc<Settings>,
search_client: Arc<dyn SearchClientTrait>,
reranker: Arc<SearchReranker>,
) -> Self {
Self {
crawl_repo,
task_repo,
credits_repo,
search_client,
smart_engine: None,
max_results: settings.search.max_results,
reranker,
}
}
#[must_use]
pub fn with_smart_engine(mut self, engine: Arc<dyn SearchEngine>) -> Self {
self.smart_engine = Some(engine);
self
}
pub async fn search(
&self,
team_id: crate::domain::id::Id,
api_key_id: crate::domain::id::Id,
query: SearchQuery,
) -> Result<SearchResponse, SearchServiceError> {
if query.query.trim().is_empty() {
return Err(SearchServiceError::ValidationError(
"Query cannot be empty".to_string(),
));
}
let current_balance = self.credits_repo.get_balance(team_id).await?;
let search_cost = 1i64;
if current_balance < search_cost {
return Err(SearchServiceError::InsufficientCredits {
available: current_balance,
required: search_cost,
});
}
let engine_param = if let Some(sources) = &query.sources {
if sources.len() == 1 {
Some(sources[0].as_str())
} else {
None
}
} else {
query.engine.as_deref()
};
let results = self
.perform_search(
&query.query,
query
.limit
.unwrap_or(10)
.min(self.max_results)
.min(MAX_SEARCH_RESULTS),
query.lang.as_deref(),
query.country.as_deref(),
engine_param,
)
.await?;
let mut crawl_id = None;
let credits_used = search_cost;
if query.crawl_results.unwrap_or(false) && !results.is_empty() {
let cid = crate::domain::id::Id::generate();
let _now = Utc::now();
let config = query.crawl_config.unwrap_or_default();
if config.max_depth > MAX_SEARCH_CRAWL_DEPTH {
return Err(SearchServiceError::ValidationError(format!(
"crawl_config.max_depth must be <= {MAX_SEARCH_CRAWL_DEPTH}"
)));
}
let now = chrono::Utc::now();
let mut seen_urls = std::collections::HashSet::new();
let normalized_blacklist =
crate::domain::services::team_service::normalize_denylist_entries(
&config.domain_blacklist,
);
let mut eligible = Vec::new();
for result in &results {
if eligible.len() >= MAX_SEARCH_RESULTS as usize {
log::warn!(
"search crawl truncated at {} results (hard cap)",
MAX_SEARCH_RESULTS
);
break;
}
if result.url.len() > 2048 {
log::warn!(
"skip oversized search result url ({} bytes)",
result.url.len()
);
#[cfg(feature = "metrics")]
metrics::counter!("crawlrs_search_oversized_url_total").increment(1);
continue;
}
if let Some(reason) =
crate::domain::services::team_service::domain_in_normalized_denylist(
&result.url,
&normalized_blacklist,
)
{
log::warn!("skip blacklisted search result: {}", reason);
#[cfg(feature = "metrics")]
metrics::counter!("crawlrs_search_blacklisted_url_total").increment(1);
continue;
}
if seen_urls.insert(result.url.clone()) {
eligible.push(result);
}
}
if !eligible.is_empty() {
let mut crawl_config = json!(config);
if let Some(obj) = crawl_config.as_object_mut() {
obj.insert("domain_blacklist".to_string(), json!(normalized_blacklist));
}
let crawl = Crawl::with_all_fields(
cid,
team_id,
format!("Search Crawl: {}", query.query),
"search://".to_string() + &query.query,
"search://".to_string() + &query.query,
crate::domain::models::CrawlStatus::Queued,
crawl_config,
eligible.len() as i32,
0,
0,
now,
now,
None,
);
self.crawl_repo.create(&crawl).await?;
for result in eligible {
let mut task = Task::new(
crate::domain::id::Id::generate(),
TaskType::Crawl,
team_id,
api_key_id,
result.url.clone(),
json!({
"crawl_id": cid,
"depth": 0,
"config": config,
}),
);
task.priority = 100;
task.crawl_id = Some(cid);
if let Err(e) = self.task_repo.create(&task).await {
log::error!("Failed to enqueue search crawl task (crawl {}): {}", cid, e);
if let Err(mark_err) =
self.crawl_repo.finalize_failed_after_enqueue(cid).await
{
log::error!(
"Failed to mark incomplete search crawl {} as Failed: {}",
cid,
mark_err
);
}
return Err(e.into());
}
}
crawl_id = Some(cid);
}
}
self.credits_repo
.deduct_credits(
team_id,
search_cost,
CreditsTransactionType::Search,
format!("Search query: {}", query.query),
None,
)
.await?;
Ok(SearchResponse {
query: query.query,
results,
crawl_id,
credits_used: credits_used as u32,
})
}
async fn perform_search(
&self,
query: &str,
limit: u32,
_lang: Option<&str>,
_country: Option<&str>,
engine: Option<&str>,
) -> Result<Vec<SearchResult>, SearchServiceError> {
if let Some(results) = self.try_smart_search(query, limit, engine).await? {
return Ok(results);
}
if let Some(name) = engine {
if name != "smart" && crate::search::types::SearchEngineType::from_name(name).is_none()
{
return Err(SearchServiceError::UnknownEngine(name.to_string()));
}
}
let mut command = self.search_client.search(query).await;
command = command.limit(limit);
if let Some(engine_name) = engine {
command = command.with_engine(engine_name);
}
let response = command
.execute()
.await
.map_err(|e| SearchServiceError::SearchEngine(e.to_string()))?;
let filtered_results: Vec<SearchResult> = response
.items
.into_iter()
.take(limit as usize)
.map(into_search_result)
.collect();
let filtered_results = self.reranker.rerank_results(query, filtered_results).await;
Ok(filtered_results)
}
async fn try_smart_search(
&self,
query: &str,
limit: u32,
engine: Option<&str>,
) -> Result<Option<Vec<SearchResult>>, SearchServiceError> {
let use_smart = engine.is_none() || engine == Some("smart");
if !use_smart {
return Ok(None);
}
if let Some(smart) = &self.smart_engine {
let request = crate::search::engine_trait::SearchRequest::new(query).with_limit(limit);
let response = smart
.search(&request)
.await
.map_err(|e| SearchServiceError::SmartSearchFailed(e.to_string()))?;
let results: Vec<SearchResult> = response
.items
.into_iter()
.take(limit as usize)
.map(into_search_result)
.collect();
return Ok(Some(self.reranker.rerank_results(query, results).await));
}
if engine.is_none() {
log::warn!("{}", crate::i18n::tr_log("search-smart-not-injected"));
}
Ok(None)
}
}
fn into_search_result(item: crate::search::response::ResponseItem) -> SearchResult {
SearchResult {
title: item.title,
url: item.url,
description: Some(item.description),
engine: item.engine.name().to_string(),
score: None,
}
}
#[async_trait::async_trait]
impl SearchServiceTrait for SearchService {
async fn search(
&self,
team_id: crate::domain::id::Id,
api_key_id: crate::domain::id::Id,
query: SearchQuery,
) -> Result<SearchResponse, SearchServiceError> {
Self::search(self, team_id, api_key_id, query).await
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::models::{CrawlStatus, CreditsTransaction, CreditsTransactionType};
use crate::domain::repositories::crawl_repository::CrawlRepository;
use crate::domain::repositories::credits_repository::CreditsRepositoryError;
use crate::domain::repositories::task_repository::{
RepositoryError, TaskQueryParams, TaskRepository,
};
use crate::search::client::{SearchClient, SearchClientTrait, SearchCommand};
use crate::search::engine_trait::{SearchEngine, SearchRequest};
use crate::search::error::SearchError;
use crate::search::response::{Response, ResponseItem};
use crate::search::types::SearchEngineType;
use async_trait::async_trait;
use std::collections::HashSet;
use std::sync::Mutex;
struct MockCreditsRepo {
balance: i64,
get_balance_fails: bool,
deduct_fails: bool,
deduct_calls: Mutex<Vec<(crate::domain::id::Id, i64)>>,
}
impl MockCreditsRepo {
fn with_balance(balance: i64) -> Self {
Self {
balance,
get_balance_fails: false,
deduct_fails: false,
deduct_calls: Mutex::new(Vec::new()),
}
}
fn failing() -> Self {
Self {
balance: 0,
get_balance_fails: true,
deduct_fails: false,
deduct_calls: Mutex::new(Vec::new()),
}
}
}
#[async_trait]
impl CreditsRepository for MockCreditsRepo {
async fn get_balance(
&self,
_team_id: crate::domain::id::Id,
) -> Result<i64, CreditsRepositoryError> {
if self.get_balance_fails {
return Err(CreditsRepositoryError::DatabaseError(
"mock get_balance failure".to_string(),
));
}
Ok(self.balance)
}
async fn deduct_credits(
&self,
team_id: crate::domain::id::Id,
amount: i64,
_transaction_type: CreditsTransactionType,
_description: String,
_reference_id: Option<crate::domain::id::Id>,
) -> Result<(), CreditsRepositoryError> {
if self.deduct_fails {
return Err(CreditsRepositoryError::DatabaseError(
"mock deduct failure".to_string(),
));
}
self.deduct_calls.lock().unwrap().push((team_id, amount));
Ok(())
}
async fn add_credits(
&self,
_team_id: crate::domain::id::Id,
_amount: i64,
_transaction_type: CreditsTransactionType,
_description: String,
_reference_id: Option<crate::domain::id::Id>,
) -> Result<i64, CreditsRepositoryError> {
Ok(0)
}
async fn get_transaction_history(
&self,
_team_id: crate::domain::id::Id,
_limit: Option<u32>,
) -> Result<Vec<CreditsTransaction>, CreditsRepositoryError> {
Ok(vec![])
}
async fn initialize_team_credits(
&self,
_team_id: crate::domain::id::Id,
_initial_balance: i64,
) -> Result<i64, CreditsRepositoryError> {
Ok(0)
}
}
struct MockTaskRepo {
create_fails: bool,
created: Mutex<Vec<Task>>,
}
impl MockTaskRepo {
fn new() -> Self {
Self {
create_fails: false,
created: Mutex::new(Vec::new()),
}
}
fn failing() -> Self {
Self {
create_fails: true,
created: Mutex::new(Vec::new()),
}
}
fn created_count(&self) -> usize {
self.created.lock().unwrap().len()
}
fn created_urls(&self) -> Vec<String> {
self.created
.lock()
.unwrap()
.iter()
.map(|task| task.url.clone())
.collect()
}
}
#[async_trait]
impl TaskRepository for MockTaskRepo {
async fn create(&self, task: &Task) -> Result<Task, RepositoryError> {
if self.create_fails {
return Err(RepositoryError::Database(anyhow::anyhow!(
"mock task create failure"
)));
}
self.created.lock().unwrap().push(task.clone());
Ok(task.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn update(&self, task: &Task) -> Result<Task, RepositoryError> {
Ok(task.clone())
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
_urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
Ok(HashSet::new())
}
async fn reset_stuck_tasks(
&self,
_timeout: chrono::Duration,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
Ok(vec![])
}
async fn query_tasks(
&self,
_params: TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
Ok((vec![], 0))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((vec![], vec![]))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
struct MockCrawlRepo {
create_fails: bool,
created: Mutex<Vec<Crawl>>,
compensations: Mutex<u32>,
}
impl MockCrawlRepo {
fn new() -> Self {
Self {
create_fails: false,
created: Mutex::new(Vec::new()),
compensations: Mutex::new(0),
}
}
fn failing() -> Self {
Self {
create_fails: true,
created: Mutex::new(Vec::new()),
compensations: Mutex::new(0),
}
}
fn created_count(&self) -> usize {
self.created.lock().unwrap().len()
}
fn compensation_count(&self) -> u32 {
*self.compensations.lock().unwrap()
}
fn last_created_config(&self) -> Option<serde_json::Value> {
self.created
.lock()
.unwrap()
.last()
.map(|c| c.config().clone())
}
}
#[async_trait]
impl CrawlRepository for MockCrawlRepo {
async fn create(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
if self.create_fails {
return Err(RepositoryError::Database(anyhow::anyhow!(
"mock crawl create failure"
)));
}
self.created.lock().unwrap().push(crawl.clone());
Ok(crawl.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Crawl>, RepositoryError> {
Ok(None)
}
async fn finalize_failed_after_enqueue(
&self,
_id: crate::domain::id::Id,
) -> Result<bool, RepositoryError> {
*self.compensations.lock().unwrap() += 1;
Ok(true)
}
async fn update(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
Ok(crawl.clone())
}
async fn increment_completed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn increment_failed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn update_status(
&self,
_id: crate::domain::id::Id,
_status: CrawlStatus,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn increment_total_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn find_by_team_id_paginated(
&self,
_team_id: crate::domain::id::Id,
_limit: u32,
_offset: u32,
) -> Result<Vec<Crawl>, RepositoryError> {
Ok(vec![])
}
async fn count_by_team_id(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
}
struct MockSearchEngine {
engine_type: SearchEngineType,
items: Vec<ResponseItem>,
fail: bool,
last_request: Mutex<Option<SearchRequest>>,
}
impl MockSearchEngine {
fn success_with_items(engine_type: SearchEngineType, items: Vec<ResponseItem>) -> Self {
Self {
engine_type,
items,
fail: false,
last_request: Mutex::new(None),
}
}
fn failing(engine_type: SearchEngineType) -> Self {
Self {
engine_type,
items: Vec::new(),
fail: true,
last_request: Mutex::new(None),
}
}
#[allow(dead_code)]
fn last_request(&self) -> Option<SearchRequest> {
self.last_request.lock().unwrap().clone()
}
}
#[async_trait]
impl SearchEngine for MockSearchEngine {
fn name(&self) -> &'static str {
match self.engine_type {
SearchEngineType::Bing => "MockBing",
SearchEngineType::Baidu => "MockBaidu",
SearchEngineType::Sogou => "MockSogou",
_ => "MockEngine",
}
}
fn engine_type(&self) -> SearchEngineType {
self.engine_type
}
fn health(&self) -> crate::search::types::EngineHealth {
crate::search::types::EngineHealth::Healthy
}
async fn search(
&self,
request: &SearchRequest,
) -> Result<Response<ResponseItem>, SearchError> {
*self.last_request.lock().unwrap() = Some(request.clone());
if self.fail {
return Err(SearchError::NoEngineAvailable);
}
let total = self.items.len() as u64;
Ok(Response {
items: self.items.clone(),
total_results: Some(total),
engine: self.engine_type,
})
}
}
fn make_response_item(title: &str, url: &str, engine: SearchEngineType) -> ResponseItem {
ResponseItem {
title: title.to_string(),
url: url.to_string(),
description: format!("desc for {}", title),
engine,
}
}
struct MockSearchClient {
inner: SearchClient,
}
impl MockSearchClient {
fn new_with_items(items: Vec<ResponseItem>) -> Self {
let engine: Arc<dyn SearchEngine> = Arc::new(MockSearchEngine::success_with_items(
SearchEngineType::Bing,
items,
));
Self {
inner: SearchClient::new_with_engines(vec![engine], SearchEngineType::Bing),
}
}
fn failing() -> Self {
let engine: Arc<dyn SearchEngine> =
Arc::new(MockSearchEngine::failing(SearchEngineType::Bing));
Self {
inner: SearchClient::new_with_engines(vec![engine], SearchEngineType::Bing),
}
}
fn new() -> Self {
Self::new_with_items(vec![
make_response_item("Result 1", "https://example.com/1", SearchEngineType::Bing),
make_response_item("Result 2", "https://example.com/2", SearchEngineType::Bing),
])
}
}
#[async_trait]
impl SearchClientTrait for MockSearchClient {
async fn search(&self, query: &str) -> SearchCommand {
self.inner.search(query)
}
async fn search_with_engine(
&self,
_query: &str,
_engine: SearchEngineType,
) -> Result<Response<ResponseItem>, SearchError> {
Err(SearchError::NoEngineAvailable)
}
fn default_engine(&self) -> SearchEngineType {
SearchEngineType::Bing
}
}
fn make_service_with_smart(
search_client: MockSearchClient,
smart: Arc<dyn SearchEngine>,
) -> SearchService {
SearchService {
crawl_repo: Arc::new(MockCrawlRepo::new()),
task_repo: Arc::new(MockTaskRepo::new()),
credits_repo: Arc::new(MockCreditsRepo::with_balance(100)),
search_client: Arc::new(search_client),
smart_engine: Some(smart),
max_results: 50,
reranker: Arc::new(SearchReranker::new(None, 25)),
}
}
#[tokio::test]
async fn test_engine_none_routes_to_smart_engine() {
let smart: Arc<dyn SearchEngine> = Arc::new(MockSearchEngine::success_with_items(
SearchEngineType::Smart,
vec![make_response_item(
"smart consensus",
"https://smart.example/1",
SearchEngineType::Smart,
)],
));
let client = MockSearchClient::new_with_items(vec![make_response_item(
"bing single",
"https://bing.example/1",
SearchEngineType::Bing,
)]);
let service = make_service_with_smart(client, smart);
let results = service
.perform_search("rust async", 10, None, None, None)
.await
.expect("perform_search");
assert!(
results.iter().all(|r| r.title == "smart consensus"),
"engine=None must route to smart engine, got: {:?}",
results.iter().map(|r| r.title.clone()).collect::<Vec<_>>()
);
}
#[tokio::test]
async fn test_explicit_engine_routes_to_single_engine() {
let smart: Arc<dyn SearchEngine> = Arc::new(MockSearchEngine::success_with_items(
SearchEngineType::Smart,
vec![make_response_item(
"smart consensus",
"https://smart.example/1",
SearchEngineType::Smart,
)],
));
let client = MockSearchClient::new_with_items(vec![make_response_item(
"bing single",
"https://bing.example/1",
SearchEngineType::Bing,
)]);
let service = make_service_with_smart(client, smart);
let results = service
.perform_search("rust async", 10, None, None, Some("bing"))
.await
.expect("perform_search");
assert!(
results.iter().all(|r| r.title == "bing single"),
"explicit engine must bypass smart aggregation, got: {:?}",
results.iter().map(|r| r.title.clone()).collect::<Vec<_>>()
);
}
#[tokio::test]
async fn test_unknown_engine_name_returns_validation_error() {
let service = make_service_with_smart(
MockSearchClient::new(),
Arc::new(MockSearchEngine::success_with_items(
SearchEngineType::Smart,
vec![],
)),
);
let result = service
.perform_search("rust async", 10, None, None, Some("google"))
.await;
match result {
Err(SearchServiceError::UnknownEngine(name)) => {
assert_eq!(
name, "google",
"error should carry the unknown engine name: {name}"
);
}
other => panic!("expected UnknownEngine, got {other:?}"),
}
}
fn make_service(credits: Arc<dyn CreditsRepository>) -> SearchService {
SearchService {
crawl_repo: Arc::new(MockCrawlRepo::new()),
task_repo: Arc::new(MockTaskRepo::new()),
credits_repo: credits,
search_client: Arc::new(MockSearchClient::new()),
smart_engine: None,
max_results: 50,
reranker: Arc::new(SearchReranker::new(None, 25)),
}
}
fn make_query(text: &str) -> SearchQuery {
SearchQuery {
query: text.to_string(),
limit: None,
lang: None,
country: None,
engine: None,
sources: None,
crawl_results: None,
crawl_config: None,
}
}
fn make_service_with_recording(
credits: Arc<dyn CreditsRepository>,
search_client: Arc<dyn SearchClientTrait>,
) -> (SearchService, Arc<MockCrawlRepo>, Arc<MockTaskRepo>) {
let crawl_repo = Arc::new(MockCrawlRepo::new());
let task_repo = Arc::new(MockTaskRepo::new());
let service = SearchService {
crawl_repo: crawl_repo.clone(),
task_repo: task_repo.clone(),
credits_repo: credits,
search_client,
smart_engine: None,
max_results: 50,
reranker: Arc::new(SearchReranker::new(None, 25)),
};
(service, crawl_repo, task_repo)
}
fn make_failing_search_service(credits: Arc<dyn CreditsRepository>) -> SearchService {
SearchService {
crawl_repo: Arc::new(MockCrawlRepo::new()),
task_repo: Arc::new(MockTaskRepo::new()),
credits_repo: credits,
search_client: Arc::new(MockSearchClient::failing()),
smart_engine: None,
max_results: 50,
reranker: Arc::new(SearchReranker::new(None, 25)),
}
}
struct PrefixScoreReranker {
fail: bool,
}
#[async_trait::async_trait]
impl crate::domain::services::rerank::RerankProvider for PrefixScoreReranker {
async fn rerank(
&self,
_query: &str,
documents: &[String],
_top_k: Option<usize>,
) -> Result<
Vec<crate::domain::services::rerank::RerankScore>,
crate::domain::services::rerank::RerankError,
> {
if self.fail {
return Err(crate::domain::services::rerank::RerankError::Provider(
"down".to_string(),
));
}
Ok(documents
.iter()
.enumerate()
.map(|(index, doc)| {
let score = if doc.starts_with("boost") { 0.99 } else { 0.01 };
crate::domain::services::rerank::RerankScore { index, score }
})
.collect())
}
fn name(&self) -> &str {
"prefix-mock"
}
}
#[tokio::test]
async fn test_perform_search_applies_rerank_and_scores() {
let items = vec![
make_response_item("plain one", "https://example.com/1", SearchEngineType::Bing),
make_response_item("boost me", "https://example.com/2", SearchEngineType::Bing),
];
let service = SearchService {
crawl_repo: Arc::new(MockCrawlRepo::new()),
task_repo: Arc::new(MockTaskRepo::new()),
credits_repo: Arc::new(MockCreditsRepo::with_balance(100)),
search_client: Arc::new(MockSearchClient::new_with_items(items)),
smart_engine: None,
max_results: 50,
reranker: Arc::new(SearchReranker::new(
Some(Arc::new(PrefixScoreReranker { fail: false })),
25,
)),
};
let results = service
.perform_search("anything", 10, None, None, None)
.await
.expect("perform_search");
assert_eq!(results[0].title, "boost me");
assert!((results[0].score.unwrap() - 0.99).abs() < 1e-6);
assert_eq!(results[1].title, "plain one");
assert!((results[1].score.unwrap() - 0.01).abs() < 1e-6);
}
#[tokio::test]
async fn test_perform_search_rerank_failure_fails_open() {
let items = vec![
make_response_item("Result 1", "https://example.com/1", SearchEngineType::Bing),
make_response_item("Result 2", "https://example.com/2", SearchEngineType::Bing),
];
let service = SearchService {
crawl_repo: Arc::new(MockCrawlRepo::new()),
task_repo: Arc::new(MockTaskRepo::new()),
credits_repo: Arc::new(MockCreditsRepo::with_balance(100)),
search_client: Arc::new(MockSearchClient::new_with_items(items)),
smart_engine: None,
max_results: 50,
reranker: Arc::new(SearchReranker::new(
Some(Arc::new(PrefixScoreReranker { fail: true })),
25,
)),
};
let results = service
.perform_search("anything", 10, None, None, None)
.await
.expect("perform_search");
assert_eq!(results[0].title, "Result 1");
assert_eq!(results[1].title, "Result 2");
assert!(results.iter().all(|r| r.score.is_none()));
}
#[test]
fn test_search_query_construction_with_all_fields() {
let query = SearchQuery {
query: "rust web scraping".to_string(),
limit: Some(20),
lang: Some("en".to_string()),
country: Some("us".to_string()),
engine: Some("google".to_string()),
sources: Some(vec!["google".to_string(), "bing".to_string()]),
crawl_results: Some(true),
crawl_config: Some(SearchCrawlConfig::default()),
};
assert_eq!(query.query, "rust web scraping");
assert_eq!(query.limit, Some(20));
assert_eq!(query.lang.as_deref(), Some("en"));
assert_eq!(query.country.as_deref(), Some("us"));
assert_eq!(query.engine.as_deref(), Some("google"));
assert_eq!(query.sources.as_ref().unwrap().len(), 2);
assert_eq!(query.crawl_results, Some(true));
assert!(query.crawl_config.is_some());
}
#[test]
fn test_search_query_with_minimal_fields() {
let query = make_query("test");
assert_eq!(query.query, "test");
assert!(query.limit.is_none());
assert!(query.lang.is_none());
assert!(query.crawl_results.is_none());
}
#[test]
fn test_search_query_empty_string() {
let query = make_query("");
assert!(query.query.is_empty());
}
#[test]
fn test_search_crawl_config_default_values() {
let config = SearchCrawlConfig::default();
assert_eq!(config.max_depth, 1, "default max_depth should be 1");
assert_eq!(config.strategy, "bfs", "default strategy should be bfs");
assert_eq!(
config.max_concurrency, 10,
"default max_concurrency should be 10"
);
assert!(config.include_patterns.is_none());
assert!(config.exclude_patterns.is_none());
assert!(config.crawl_delay_ms.is_none());
assert!(config.headers.is_none());
assert!(config.proxy.is_none());
assert!(config.extraction_rules.is_none());
}
#[test]
fn test_search_crawl_config_serde_roundtrip() {
let config = SearchCrawlConfig {
max_depth: 3,
include_patterns: Some(vec!["*/blog/*".to_string()]),
exclude_patterns: Some(vec!["*/admin/*".to_string()]),
strategy: "bfs".to_string(),
crawl_delay_ms: Some(500),
max_concurrency: 10,
headers: Some(serde_json::json!({"User-Agent": "test"})),
proxy: Some("http://proxy:8080".to_string()),
extraction_rules: None,
domain_blacklist: Vec::new(),
};
let json = serde_json::to_string(&config).expect("should serialize");
let back: SearchCrawlConfig = serde_json::from_str(&json).expect("should deserialize");
assert_eq!(back.max_depth, 3);
assert_eq!(back.strategy, "bfs");
assert_eq!(back.max_concurrency, 10);
assert_eq!(back.crawl_delay_ms, Some(500));
assert_eq!(
back.include_patterns.as_ref().unwrap(),
&vec!["*/blog/*".to_string()]
);
}
#[test]
fn test_search_crawl_config_serde_minimal() {
let config = SearchCrawlConfig::default();
let json = serde_json::to_string(&config).expect("should serialize");
let back: SearchCrawlConfig = serde_json::from_str(&json).expect("should deserialize");
assert_eq!(config.max_depth, back.max_depth);
assert_eq!(config.strategy, back.strategy);
}
#[test]
fn test_search_result_construction() {
let result = SearchResult {
title: "Rust Programming".to_string(),
url: "https://www.rust-lang.org".to_string(),
description: Some(
"A language empowering everyone to build reliable software.".to_string(),
),
engine: "Google".to_string(),
score: None,
};
assert_eq!(result.title, "Rust Programming");
assert_eq!(result.url, "https://www.rust-lang.org");
assert!(result.description.is_some());
assert_eq!(result.engine, "Google");
}
#[test]
fn test_search_result_with_none_description() {
let result = SearchResult {
title: "Test".to_string(),
url: "https://example.com".to_string(),
description: None,
engine: "Bing".to_string(),
score: None,
};
assert!(result.description.is_none());
}
#[test]
fn test_search_response_construction_with_crawl() {
let response = SearchResponse {
query: "rust scraping".to_string(),
results: vec![SearchResult {
title: "Result 1".to_string(),
url: "https://example.com".to_string(),
description: None,
engine: "Google".to_string(),
score: None,
}],
crawl_id: Some(crate::domain::id::Id::generate()),
credits_used: 1,
};
assert_eq!(response.query, "rust scraping");
assert_eq!(response.results.len(), 1);
assert!(response.crawl_id.is_some());
assert_eq!(response.credits_used, 1);
}
#[test]
fn test_search_response_construction_without_crawl() {
let response = SearchResponse {
query: "test".to_string(),
results: vec![],
crawl_id: None,
credits_used: 1,
};
assert!(response.results.is_empty());
assert!(response.crawl_id.is_none());
}
#[test]
fn test_search_service_error_from_string() {
let err: SearchServiceError = "bad query".to_string().into();
match err {
SearchServiceError::ValidationError(msg) => {
assert_eq!(msg, "bad query");
}
other => panic!("expected ValidationError, got {:?}", other),
}
}
#[test]
fn test_search_service_error_from_str_ref() {
let err: SearchServiceError = "invalid input".into();
match err {
SearchServiceError::ValidationError(msg) => {
assert_eq!(msg, "invalid input");
}
other => panic!("expected ValidationError, got {:?}", other),
}
}
#[test]
fn test_search_service_error_from_anyhow_error() {
let err: SearchServiceError = anyhow::anyhow!("engine crashed").into();
match err {
SearchServiceError::SearchEngine(msg) => {
assert!(msg.contains("engine crashed"), "msg: {}", msg);
}
other => panic!("expected SearchEngine, got {:?}", other),
}
}
#[test]
fn test_search_service_error_display_validation() {
let err = SearchServiceError::ValidationError("missing field".to_string());
let msg = err.to_string();
assert!(msg.contains("Validation failed"));
assert!(msg.contains("missing field"));
}
#[test]
fn test_search_service_error_display_insufficient_credits() {
let err = SearchServiceError::InsufficientCredits {
available: 0,
required: 1,
};
let msg = err.to_string();
assert!(msg.contains("Insufficient credits"));
assert!(msg.contains("available 0"));
assert!(msg.contains("required 1"));
}
#[test]
fn test_search_service_error_display_search_engine() {
let err = SearchServiceError::SearchEngine("timeout".to_string());
let msg = err.to_string();
assert!(msg.contains("Search engine error"));
assert!(msg.contains("timeout"));
}
#[test]
fn search_crawl_payload_is_worker_compatible() {
let config = SearchCrawlConfig {
max_depth: 1,
include_patterns: None,
exclude_patterns: None,
strategy: "bfs".to_string(),
crawl_delay_ms: None,
max_concurrency: 10,
headers: None,
proxy: None,
extraction_rules: None,
domain_blacklist: vec!["blocked.com".to_string()],
};
let config_json = serde_json::to_value(&config).expect("serialize config");
assert!(
config_json.get("domain_blacklist").is_none(),
"domain_blacklist must not be serialized into CrawlConfigDto"
);
let crawl_id = crate::domain::id::Id::generate();
let mut task = Task::new(
crate::domain::id::Id::generate(),
TaskType::Crawl,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
"https://example.com/".to_string(),
json!({
"crawl_id": crawl_id.to_string(),
"depth": 0,
"config": config_json,
}),
);
task.crawl_id = Some(crawl_id);
let parsed = crate::workers::scrape_response_builder::parse_crawl_payload(&task)
.expect("search crawl payload must parse as CrawlConfigDto");
assert_eq!(parsed.1, 0);
assert_eq!(parsed.2.max_depth, 1);
assert!(
task.payload.get("domain_blacklist").is_none(),
"domain_blacklist must not be copied into task payload"
);
}
#[tokio::test]
async fn search_crawl_stores_normalized_blacklist_in_crawl_config() {
let credits = Arc::new(MockCreditsRepo::with_balance(100));
let client = MockSearchClient::new_with_items(vec![
make_response_item(
"Blocked",
"https://Blocked.COM/page",
SearchEngineType::Bing,
),
make_response_item(
"Allowed",
"https://allowed.example/page",
SearchEngineType::Bing,
),
]);
let (service, crawl_repo, task_repo) =
make_service_with_recording(credits, Arc::new(client));
let config = SearchCrawlConfig {
domain_blacklist: vec![" Blocked.com. ".to_string()],
..SearchCrawlConfig::default()
};
let query = SearchQuery {
crawl_results: Some(true),
crawl_config: Some(config),
..make_query("blacklist storage")
};
let resp = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await
.expect("search should succeed");
assert_eq!(resp.results.len(), 2, "search response keeps all results");
let created = crawl_repo.created.lock().unwrap();
let crawl = created.last().expect("crawl record should exist");
assert_eq!(crawl.total_tasks(), 1, "total counts eligible results only");
let stored = crawl.config().clone();
assert_eq!(
stored["domain_blacklist"],
json!(["blocked.com"]),
"crawl.config must carry the normalized blacklist"
);
assert_eq!(task_repo.created_count(), 1, "one task for allowed result");
assert_eq!(
task_repo.created_urls(),
vec!["https://allowed.example/page".to_string()],
"blacklisted URL must not be enqueued"
);
}
#[tokio::test]
async fn test_search_crawl_config_depth_above_limit_is_rejected() {
let credits = Arc::new(MockCreditsRepo::with_balance(100));
let (service, crawl_repo, task_repo) =
make_service_with_recording(credits, Arc::new(MockSearchClient::new()));
let query = SearchQuery {
crawl_results: Some(true),
crawl_config: Some(SearchCrawlConfig {
max_depth: MAX_SEARCH_CRAWL_DEPTH + 1,
..SearchCrawlConfig::default()
}),
..make_query("too deep")
};
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(matches!(
result,
Err(SearchServiceError::ValidationError(_))
));
assert_eq!(crawl_repo.created_count(), 0, "no crawl on invalid depth");
assert_eq!(task_repo.created_count(), 0, "no tasks on invalid depth");
}
#[tokio::test]
async fn test_search_empty_query_returns_validation_error() {
let service = make_service(Arc::new(MockCreditsRepo::with_balance(100)));
let query = make_query("");
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_err());
match result.unwrap_err() {
SearchServiceError::ValidationError(msg) => {
assert!(msg.contains("empty"), "error should mention empty: {}", msg);
}
other => panic!("expected ValidationError, got {:?}", other),
}
}
#[tokio::test]
async fn test_search_whitespace_only_query_returns_validation_error() {
let service = make_service(Arc::new(MockCreditsRepo::with_balance(100)));
let query = make_query(" ");
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_err(), "whitespace-only query should fail");
match result.unwrap_err() {
SearchServiceError::ValidationError(_) => {}
other => panic!("expected ValidationError, got {:?}", other),
}
}
#[tokio::test]
async fn test_search_insufficient_credits_returns_error() {
let service = make_service(Arc::new(MockCreditsRepo::with_balance(0)));
let query = make_query("rust scraping");
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_err());
match result.unwrap_err() {
SearchServiceError::InsufficientCredits {
available,
required,
} => {
assert_eq!(available, 0);
assert_eq!(required, 1);
}
other => panic!("expected InsufficientCredits, got {:?}", other),
}
}
#[tokio::test]
async fn test_search_exact_balance_succeeds_validation() {
let service = make_service(Arc::new(MockCreditsRepo::with_balance(1)));
let query = make_query("rust scraping");
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(
result.is_ok(),
"should pass credit check and perform_search, got {:?}",
result.err()
);
let resp = result.unwrap();
assert_eq!(resp.credits_used, 1, "search cost is 1 credit");
}
#[tokio::test]
async fn test_search_with_single_source_passes_source_as_engine() {
let service = make_service(Arc::new(MockCreditsRepo::with_balance(10)));
let query = SearchQuery {
sources: Some(vec!["bing".to_string()]),
..make_query("rust scraping")
};
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(
result.is_ok(),
"should reach perform_search and succeed, got {:?}",
result.err()
);
}
#[tokio::test]
async fn test_search_with_multiple_sources_uses_aggregator() {
let service = make_service(Arc::new(MockCreditsRepo::with_balance(10)));
let query = SearchQuery {
sources: Some(vec!["bing".to_string(), "baidu".to_string()]),
..make_query("rust scraping")
};
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(
result.is_ok(),
"aggregator path should reach perform_search and succeed, got {:?}",
result.err()
);
}
#[tokio::test]
async fn test_search_with_engine_specified_passes_engine() {
let service = make_service(Arc::new(MockCreditsRepo::with_balance(10)));
let query = SearchQuery {
engine: Some("exa".to_string()),
..make_query("rust scraping")
};
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_err());
match result.unwrap_err() {
SearchServiceError::ValidationError(_) => {
panic!("should pass validation");
}
SearchServiceError::InsufficientCredits { .. } => {
panic!("should pass credit check");
}
_ => { }
}
}
#[tokio::test]
async fn test_search_with_limit_specified_passes_to_perform_search() {
let service = make_service(Arc::new(MockCreditsRepo::with_balance(10)));
let query = SearchQuery {
limit: Some(5),
lang: Some("en".to_string()),
country: Some("us".to_string()),
..make_query("rust scraping")
};
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(
result.is_ok(),
"should reach perform_search and succeed, got {:?}",
result.err()
);
let resp = result.unwrap();
assert_eq!(
resp.results.len(),
2,
"mock returns 2 items, limit=5 keeps both"
);
}
#[tokio::test]
async fn test_search_get_balance_failure_returns_credits_error() {
let service = make_service(Arc::new(MockCreditsRepo::failing()));
let query = make_query("rust scraping");
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_err());
match result.unwrap_err() {
SearchServiceError::CreditsRepository(err) => match err {
CreditsRepositoryError::DatabaseError(msg) => {
assert!(msg.contains("mock get_balance failure"));
}
other => panic!("expected DatabaseError, got {:?}", other),
},
other => panic!("expected CreditsRepository, got {:?}", other),
}
}
#[tokio::test]
async fn test_search_negative_balance_returns_insufficient_credits() {
let service = make_service(Arc::new(MockCreditsRepo::with_balance(-5)));
let query = make_query("rust scraping");
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_err());
match result.unwrap_err() {
SearchServiceError::InsufficientCredits {
available,
required,
} => {
assert_eq!(available, -5, "should report negative balance");
assert_eq!(required, 1);
}
other => panic!("expected InsufficientCredits, got {:?}", other),
}
}
#[tokio::test]
async fn test_search_service_trait_forwards_empty_query_error() {
let service = make_service(Arc::new(MockCreditsRepo::with_balance(100)));
let query = make_query("");
let result: Result<SearchResponse, SearchServiceError> = SearchServiceTrait::search(
&service,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_err());
match result.unwrap_err() {
SearchServiceError::ValidationError(_) => {}
other => panic!("expected ValidationError, got {:?}", other),
}
}
#[tokio::test]
async fn test_search_success_returns_results() {
let credits = Arc::new(MockCreditsRepo::with_balance(100));
let (service, _, _) =
make_service_with_recording(credits, Arc::new(MockSearchClient::new()));
let query = make_query("rust web scraping");
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_ok(), "expected ok, got {:?}", result.err());
let resp = result.unwrap();
assert_eq!(resp.query, "rust web scraping");
assert_eq!(
resp.results.len(),
2,
"MockSearchClient::new returns 2 items"
);
assert_eq!(resp.credits_used, 1, "search cost is 1 credit");
assert!(resp.crawl_id.is_none(), "crawl_results not set");
}
#[tokio::test]
async fn test_search_success_deducts_credits() {
let credits = Arc::new(MockCreditsRepo::with_balance(100));
let credits_handle: Arc<MockCreditsRepo> = credits.clone();
let (service, _, _) =
make_service_with_recording(credits, Arc::new(MockSearchClient::new()));
let query = make_query("rust");
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_ok());
let calls = credits_handle.deduct_calls.lock().unwrap().clone();
assert_eq!(calls.len(), 1, "deduct_credits should be called once");
assert_eq!(calls[0].1, 1, "deducted amount should be 1");
}
#[tokio::test]
async fn test_search_success_with_crawl_creates_tasks_and_crawl() {
let (service, crawl_repo, task_repo) = make_service_with_recording(
Arc::new(MockCreditsRepo::with_balance(100)),
Arc::new(MockSearchClient::new()),
);
let query = SearchQuery {
crawl_results: Some(true),
..make_query("rust scraping")
};
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_ok());
let resp = result.unwrap();
assert!(resp.crawl_id.is_some(), "crawl_id should be set");
assert_eq!(
crawl_repo.created_count(),
1,
"exactly one crawl record should be created"
);
assert_eq!(
task_repo.created_count(),
resp.results.len(),
"one task per search result should be created"
);
}
#[tokio::test]
async fn test_search_success_with_crawl_skips_when_no_results() {
let credits = Arc::new(MockCreditsRepo::with_balance(100));
let (service, crawl_repo, task_repo) = make_service_with_recording(
credits,
Arc::new(MockSearchClient::new_with_items(vec![])),
);
let query = SearchQuery {
crawl_results: Some(true),
..make_query("empty")
};
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_ok());
let resp = result.unwrap();
assert!(resp.crawl_id.is_none(), "no crawl when results empty");
assert_eq!(crawl_repo.created_count(), 0);
assert_eq!(task_repo.created_count(), 0);
}
#[tokio::test]
async fn test_search_respects_limit_truncates_results() {
let credits = Arc::new(MockCreditsRepo::with_balance(100));
let (service, _, _) = make_service_with_recording(
credits,
Arc::new(MockSearchClient::new_with_items(vec![
make_response_item("a", "https://a.example", SearchEngineType::Bing),
make_response_item("b", "https://b.example", SearchEngineType::Bing),
make_response_item("c", "https://c.example", SearchEngineType::Bing),
make_response_item("d", "https://d.example", SearchEngineType::Bing),
])),
);
let query = SearchQuery {
limit: Some(2),
..make_query("limited")
};
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_ok());
let resp = result.unwrap();
assert_eq!(
resp.results.len(),
2,
"limit=2 should truncate to 2 results"
);
}
#[tokio::test]
async fn test_search_perform_search_failure_returns_search_engine_error() {
let service = make_failing_search_service(Arc::new(MockCreditsRepo::with_balance(100)));
let query = make_query("will fail");
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_err());
match result.unwrap_err() {
SearchServiceError::SearchEngine(msg) => {
assert!(!msg.is_empty(), "error message should be non-empty");
}
other => panic!("expected SearchEngine error, got {:?}", other),
}
}
#[tokio::test]
async fn test_search_crawl_repo_failure_propagates() {
let credits: Arc<dyn CreditsRepository> = Arc::new(MockCreditsRepo::with_balance(100));
let service = SearchService {
crawl_repo: Arc::new(MockCrawlRepo::failing()),
task_repo: Arc::new(MockTaskRepo::new()),
credits_repo: credits,
search_client: Arc::new(MockSearchClient::new()),
smart_engine: None,
max_results: 50,
reranker: Arc::new(SearchReranker::new(None, 25)),
};
let query = SearchQuery {
crawl_results: Some(true),
..make_query("crawl fails")
};
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_err());
match result.unwrap_err() {
SearchServiceError::Repository(_) => { }
other => panic!("expected Repository error, got {:?}", other),
}
}
#[tokio::test]
async fn test_search_task_repo_failure_propagates_and_compensates() {
let credits: Arc<dyn CreditsRepository> = Arc::new(MockCreditsRepo::with_balance(100));
let crawl_repo = Arc::new(MockCrawlRepo::new());
let service = SearchService {
crawl_repo: crawl_repo.clone(),
task_repo: Arc::new(MockTaskRepo::failing()),
credits_repo: credits,
search_client: Arc::new(MockSearchClient::new()),
smart_engine: None,
max_results: 50,
reranker: Arc::new(SearchReranker::new(None, 25)),
};
let query = SearchQuery {
crawl_results: Some(true),
..make_query("task fails")
};
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_err());
match result.unwrap_err() {
SearchServiceError::Repository(_) => { }
other => panic!("expected Repository error, got {:?}", other),
}
assert_eq!(
crawl_repo.compensation_count(),
1,
"enqueue failure must mark the crawl terminal (not leave it queued)"
);
}
#[tokio::test]
async fn test_search_deduct_credits_failure_propagates() {
let credits = Arc::new(MockCreditsRepo {
balance: 100,
get_balance_fails: false,
deduct_fails: true,
deduct_calls: Mutex::new(Vec::new()),
});
let (service, _, _) =
make_service_with_recording(credits, Arc::new(MockSearchClient::new()));
let query = make_query("deduct fails");
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_err());
match result.unwrap_err() {
SearchServiceError::CreditsRepository(_) => { }
other => panic!("expected CreditsRepository error, got {:?}", other),
}
}
#[tokio::test]
async fn test_search_trait_delegates_to_success_path() {
let service = make_service(Arc::new(MockCreditsRepo::with_balance(100)));
let query = make_query("trait delegation");
let result: Result<SearchResponse, SearchServiceError> = SearchServiceTrait::search(
&service,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_ok(), "trait should forward to success path");
let resp = result.unwrap();
assert_eq!(resp.results.len(), 2);
}
#[tokio::test]
async fn test_mock_credits_repo_unused_methods_return_expected_defaults() {
let credits = MockCreditsRepo::with_balance(10);
let team_id = crate::domain::id::Id::generate();
assert_eq!(
credits
.add_credits(
team_id,
5,
CreditsTransactionType::Search,
"add".to_string(),
None
)
.await
.unwrap(),
0
);
assert!(credits
.get_transaction_history(team_id, None)
.await
.unwrap()
.is_empty());
assert_eq!(
credits.initialize_team_credits(team_id, 100).await.unwrap(),
0
);
}
#[tokio::test]
async fn test_mock_task_repo_unused_methods_return_expected_defaults() {
let task_repo = MockTaskRepo::new();
let task_id = crate::domain::id::Id::generate();
assert!(task_repo.find_by_id(task_id).await.unwrap().is_none());
assert!(task_repo
.acquire_next(crate::domain::id::Id::generate())
.await
.unwrap()
.is_none());
assert!(!task_repo
.exists_by_url("https://example.com")
.await
.unwrap());
task_repo.mark_completed(task_id, None).await.unwrap();
task_repo.mark_failed(task_id, None).await.unwrap();
task_repo.mark_cancelled(task_id).await.unwrap();
assert_eq!(task_repo.expire_tasks().await.unwrap(), 0);
assert_eq!(
task_repo
.cancel_tasks_by_crawl_id(crate::domain::id::Id::generate())
.await
.unwrap(),
0
);
}
#[tokio::test]
async fn test_mock_crawl_repo_unused_methods_return_expected_defaults() {
let crawl_repo = MockCrawlRepo::new();
let crawl_id = crate::domain::id::Id::generate();
assert!(crawl_repo.find_by_id(crawl_id).await.unwrap().is_none());
crawl_repo
.increment_completed_tasks(crawl_id)
.await
.unwrap();
crawl_repo.increment_failed_tasks(crawl_id).await.unwrap();
crawl_repo.increment_total_tasks(crawl_id).await.unwrap();
crawl_repo
.update_status(crawl_id, CrawlStatus::Completed)
.await
.unwrap();
assert!(crawl_repo
.find_by_team_id_paginated(crawl_id, 10, 0)
.await
.unwrap()
.is_empty());
assert_eq!(crawl_repo.count_by_team_id(crawl_id).await.unwrap(), 0);
}
#[tokio::test]
async fn test_mock_task_repo_remaining_methods_return_defaults() {
let task_repo = MockTaskRepo::new();
let task = Task::new(
crate::domain::id::Id::generate(),
TaskType::Scrape,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
"https://example.com".to_string(),
json!({}),
);
let updated = task_repo.update(&task).await.unwrap();
assert_eq!(updated.id, task.id);
let existing = task_repo
.find_existing_urls(&["https://a.com".to_string()])
.await
.unwrap();
assert!(existing.is_empty());
assert_eq!(
task_repo
.reset_stuck_tasks(chrono::Duration::minutes(5))
.await
.unwrap(),
0
);
assert!(task_repo
.find_by_crawl_id(crate::domain::id::Id::generate())
.await
.unwrap()
.is_empty());
let (tasks, count) = task_repo
.query_tasks(crate::domain::repositories::task_repository::TaskQueryParams::default())
.await
.unwrap();
assert!(tasks.is_empty());
assert_eq!(count, 0);
let (cancelled, failed) = task_repo
.batch_cancel(
vec![crate::domain::id::Id::generate()],
crate::domain::id::Id::generate(),
false,
)
.await
.unwrap();
assert!(cancelled.is_empty());
assert!(failed.is_empty());
}
#[tokio::test]
async fn test_mock_crawl_repo_update_echoes_back() {
let crawl_repo = MockCrawlRepo::new();
let crawl = Crawl::new(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
"Test".to_string(),
"https://example.com".to_string(),
"https://example.com".to_string(),
json!({}),
);
let result = crawl_repo.update(&crawl).await.unwrap();
assert_eq!(result.id, crawl.id);
}
#[test]
fn test_mock_search_engine_name_engine_type_health_all_variants() {
for engine_type in [
SearchEngineType::Bing,
SearchEngineType::Bing,
SearchEngineType::Baidu,
SearchEngineType::Sogou,
] {
let engine = MockSearchEngine::success_with_items(engine_type, vec![]);
assert_eq!(engine.engine_type(), engine_type);
let _ = engine.name();
assert!(matches!(
engine.health(),
crate::search::types::EngineHealth::Healthy
));
}
for engine_type in [
SearchEngineType::Smart,
SearchEngineType::Exa,
SearchEngineType::Tavily,
] {
let engine = MockSearchEngine::success_with_items(engine_type, vec![]);
assert_eq!(engine.engine_type(), engine_type);
assert_eq!(engine.name(), "MockEngine");
assert!(matches!(
engine.health(),
crate::search::types::EngineHealth::Healthy
));
}
}
#[test]
fn test_mock_search_engine_last_request_is_none_initially() {
let engine = MockSearchEngine::success_with_items(SearchEngineType::Bing, vec![]);
assert!(engine.last_request().is_none());
}
fn make_test_settings_for_search() -> Settings {
use crate::config::settings::*;
Settings {
server: ServerSettings::default(),
robots: RobotsSettings::default(),
database: DatabaseSettings::default(),
cors: CorsSettings::default(),
rate_limiting: RateLimitingSettings::default(),
concurrency: ConcurrencySettings::default(),
retention: RetentionSettings::default(),
webhook: WebhookSettings::default(),
bing_search: BingSearchSettings::default(),
search: SearchSettings::default(),
llm: LLMSettings::default(),
rag: RagSettings::default(),
fluxweir: FluxweirSettings::default(),
proxy: ProxySettings::default(),
engines: EngineSettings::default(),
markdown: Default::default(),
logging: LoggingSettings::default(),
workers: WorkerSettings::default(),
sitemap: SitemapSettings::default(),
crawl: CrawlSettings::default(),
timeouts: TimeoutSettings::default(),
cache: CacheSettings::default(),
websocket: WebSocketSettings::default(),
trusted_proxies: TrustedProxySettings::default(),
observability: ObservabilitySettings::default(),
auth: AuthSettings::default(),
i18n: I18nSettings::default(),
politeness: PolitenessSettings::default(),
session: SessionSettings::default(),
content_extraction: ContentExtractionSettings::default(),
id: IdSettings::default(),
}
}
#[test]
fn test_search_service_new_constructor_assigns_dependencies() {
let settings = make_test_settings_for_search();
let service = SearchService::new(
Arc::new(MockCrawlRepo::new()),
Arc::new(MockTaskRepo::new()),
Arc::new(MockCreditsRepo::with_balance(10)),
Arc::new(settings),
Arc::new(MockSearchClient::new()),
Arc::new(SearchReranker::new(None, 25)),
);
let rt = tokio::runtime::Runtime::new().unwrap();
let err = rt
.block_on(service.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
make_query(""),
))
.unwrap_err();
assert!(matches!(
err,
SearchServiceError::ValidationError(ref msg) if msg.contains("empty")
));
}
#[tokio::test]
async fn test_search_with_crawl_results_and_custom_crawl_config() {
let credits = Arc::new(MockCreditsRepo::with_balance(100));
let (service, crawl_repo, task_repo) =
make_service_with_recording(credits, Arc::new(MockSearchClient::new()));
let custom_config = SearchCrawlConfig {
max_depth: 5,
include_patterns: Some(vec!["*/blog/*".to_string()]),
exclude_patterns: Some(vec!["*/admin/*".to_string()]),
strategy: "dfs".to_string(),
crawl_delay_ms: Some(1000),
max_concurrency: 20,
headers: Some(json!({"User-Agent": "CrawlrsBot"})),
proxy: Some("http://proxy:8080".to_string()),
extraction_rules: None,
domain_blacklist: Vec::new(),
};
let query = SearchQuery {
crawl_results: Some(true),
crawl_config: Some(custom_config),
..make_query("custom crawl config")
};
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_ok(), "expected ok, got {:?}", result.err());
let resp = result.unwrap();
assert!(resp.crawl_id.is_some(), "crawl_id should be set");
assert_eq!(crawl_repo.created_count(), 1, "one crawl record created");
assert_eq!(
task_repo.created_count(),
resp.results.len(),
"one task per result"
);
let config = crawl_repo
.last_created_config()
.expect("crawl record should exist");
assert_eq!(config["max_depth"], 5, "custom max_depth should be 5");
assert_eq!(config["strategy"], "dfs", "custom strategy should be dfs");
assert_eq!(
config["max_concurrency"], 20,
"custom max_concurrency should be 20"
);
assert_eq!(
config["crawl_delay_ms"], 1000,
"custom crawl_delay_ms should be 1000"
);
assert_eq!(
config["proxy"], "http://proxy:8080",
"custom proxy should be set"
);
assert_eq!(
config["include_patterns"][0], "*/blog/*",
"custom include_patterns should be set"
);
assert_eq!(
config["exclude_patterns"][0], "*/admin/*",
"custom exclude_patterns should be set"
);
}
#[tokio::test]
async fn test_search_with_crawl_results_default_config_uses_bfs_strategy() {
let credits = Arc::new(MockCreditsRepo::with_balance(100));
let (service, crawl_repo, _) =
make_service_with_recording(credits, Arc::new(MockSearchClient::new()));
let query = SearchQuery {
crawl_results: Some(true),
crawl_config: None,
..make_query("default crawl config")
};
let result = service
.search(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
query,
)
.await;
assert!(result.is_ok());
let resp = result.unwrap();
assert!(resp.crawl_id.is_some());
let config = crawl_repo
.last_created_config()
.expect("crawl record should exist");
assert_eq!(config["max_depth"], 1, "default max_depth should be 1");
assert_eq!(config["strategy"], "bfs", "default strategy should be bfs");
assert_eq!(
config["max_concurrency"], 10,
"default max_concurrency should be 10"
);
}
#[test]
fn test_search_service_error_display_repository() {
let inner = RepositoryError::Database(anyhow::anyhow!("db connection lost"));
let err = SearchServiceError::Repository(inner);
let msg = err.to_string();
assert!(msg.contains("Repository error"));
assert!(msg.contains("db connection lost"));
}
#[test]
fn test_search_service_error_display_credits_repository() {
let inner = CreditsRepositoryError::DatabaseError("db timeout".to_string());
let err = SearchServiceError::CreditsRepository(inner);
let msg = err.to_string();
assert!(msg.contains("Credits repository error"));
assert!(msg.contains("db timeout"));
}
}