#![allow(unused_variables)]
use crate::application::dto::task_query_request::{
ScrapeResultInfoDto, TaskInfoDto, TaskQueryDataDto, TaskQueryRequestDto,
};
use crate::common::constants::crawl_task;
use crate::common::constants::server_config;
use crate::domain::models::TaskStatus;
use crate::domain::repositories::scrape_result_repository::ScrapeResultRepository;
use crate::domain::repositories::task_repository::{TaskQueryParams, TaskRepository};
use crate::infrastructure::repositories::scrape_result_repo_impl::ScrapeResultRepositoryImpl;
use crate::presentation::errors::CrawlRsError;
use crate::presentation::handlers::extract_task_ids;
use crate::presentation::handlers::response_builder::{json_rejection_error, ApiResponse};
use crate::presentation::middleware::auth_middleware::AuthState;
use anyhow;
use axum::{extract::Extension, Json};
use chrono::{TimeZone, Utc};
use sdforge::validator::Validate;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::time::sleep;
pub async fn wait_for_tasks_completion(
task_repo: &dyn TaskRepository,
task_ids: &[crate::domain::id::Id],
team_id: crate::domain::id::Id,
sync_wait_ms: u32,
base_poll_interval_ms: u64,
) -> Result<(), CrawlRsError> {
let start_time = Instant::now();
let timeout_duration = Duration::from_millis(sync_wait_ms as u64);
let min_interval = 500u64;
let max_interval = 2000u64;
let mut current_interval = base_poll_interval_ms.clamp(min_interval, max_interval);
let mut last_completion_rate = 0.0f64;
let mut poll_count = 0u32;
while start_time.elapsed() < timeout_duration {
poll_count += 1;
if poll_count_exceeded(poll_count, crawl_task::MAX_POLL_COUNT) {
return Ok(());
}
let tasks = query_tasks_for_poll(task_repo, team_id, task_ids).await?;
let completion_rate = calculate_completion_rate(&tasks, task_ids);
if completion_rate >= 1.0 {
return Ok(());
}
let new_interval = calculate_next_interval(
completion_rate,
last_completion_rate,
current_interval,
min_interval,
max_interval,
);
current_interval = new_interval;
last_completion_rate = completion_rate;
let remaining_time = timeout_duration.saturating_sub(start_time.elapsed());
let wait_duration = Duration::from_millis(current_interval).min(remaining_time);
if !wait_duration.is_zero() {
sleep(wait_duration).await;
}
}
Ok(())
}
#[inline]
pub(crate) fn poll_count_exceeded(count: u32, max_count: u32) -> bool {
if count >= max_count {
log::debug!("Reached max poll count ({}) for task completion", max_count);
true
} else {
false
}
}
pub(crate) async fn query_tasks_for_poll(
task_repo: &dyn TaskRepository,
team_id: crate::domain::id::Id,
task_ids: &[crate::domain::id::Id],
) -> Result<Vec<crate::domain::models::Task>, CrawlRsError> {
let (tasks, _) = task_repo
.query_tasks(TaskQueryParams {
team_id,
task_ids: Some(task_ids.to_vec()),
limit: task_ids.len() as u32,
..Default::default()
})
.await?;
Ok(tasks)
}
#[inline]
pub(crate) fn calculate_completion_rate(
tasks: &[crate::domain::models::Task],
task_ids: &[crate::domain::id::Id],
) -> f64 {
if task_ids.is_empty() {
return 1.0;
}
let completed_count = tasks
.iter()
.filter(|task| {
matches!(
task.status,
TaskStatus::Completed | TaskStatus::Failed | TaskStatus::Cancelled
)
})
.count();
completed_count as f64 / task_ids.len() as f64
}
#[inline]
pub(crate) fn calculate_next_interval(
completion_rate: f64,
last_rate: f64,
current_interval: u64,
min_interval: u64,
max_interval: u64,
) -> u64 {
let progress = completion_rate - last_rate;
let rate_based = min_interval + ((max_interval - min_interval) as f64 * completion_rate) as u64;
match progress {
p if p > 0.0 => ((current_interval as f64 * 1.2).max(rate_based as f64) as u64)
.clamp(min_interval, max_interval),
p if p < 0.0 => ((current_interval as f64 * 0.8).min(rate_based as f64) as u64)
.clamp(min_interval, max_interval),
_ => rate_based.clamp(min_interval, max_interval),
}
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct TaskQueryResponseMeta {
pub status: String,
pub credits_used: u32,
pub response_time_ms: u64,
}
pub async fn query_tasks<T: TaskRepository>(
Extension(auth_state): Extension<AuthState>,
Extension(task_repo): Extension<Arc<T>>,
Extension(scrape_result_repo): Extension<Arc<ScrapeResultRepositoryImpl>>,
payload: Result<Json<TaskQueryRequestDto>, axum::extract::rejection::JsonRejection>,
) -> Result<Json<ApiResponse<TaskQueryDataDto>>, CrawlRsError> {
let request = extract_query_request(payload)?;
let team_id = auth_state.team_id;
let start_time = Instant::now();
validate_request(&request)?;
let (limit, offset, include_results, sync_wait_ms) = apply_defaults(&request);
let task_types_clone = request.task_types.clone();
let statuses_clone = request.statuses.clone();
let (mut tasks, total) =
execute_task_query(task_repo.as_ref(), team_id, &request, limit, offset).await?;
if sync_wait_ms > 0 && !tasks.is_empty() {
tasks = sync_wait_and_refresh(
task_repo.as_ref(),
team_id,
&request,
tasks,
limit,
offset,
sync_wait_ms,
)
.await?;
}
let task_id_to_result = if include_results && !tasks.is_empty() {
fetch_scrape_results(scrape_result_repo.as_ref(), &tasks).await?
} else {
None
};
let task_infos = build_task_infos(&tasks, task_id_to_result.as_ref());
let has_more = (offset + limit) < total as u32;
Ok(Json(ApiResponse::success(TaskQueryDataDto {
tasks: task_infos,
total,
has_more,
})))
}
fn extract_query_request(
payload: Result<Json<TaskQueryRequestDto>, axum::extract::rejection::JsonRejection>,
) -> Result<TaskQueryRequestDto, CrawlRsError> {
match payload {
Ok(Json(request)) => Ok(request),
Err(rej) => Err(json_rejection_error(rej)),
}
}
async fn sync_wait_and_refresh<T: TaskRepository>(
task_repo: &T,
team_id: crate::domain::id::Id,
request: &TaskQueryRequestDto,
mut tasks: Vec<crate::domain::models::Task>,
limit: u32,
offset: u32,
sync_wait_ms: u32,
) -> Result<Vec<crate::domain::models::Task>, CrawlRsError> {
let waited_time_ms = handle_sync_wait(task_repo, &tasks, team_id, sync_wait_ms).await?;
if waited_time_ms > 0 {
(tasks, _) = execute_task_query(task_repo, team_id, request, limit, offset).await?;
}
Ok(tasks)
}
pub(crate) fn validate_request(request: &TaskQueryRequestDto) -> Result<(), CrawlRsError> {
if let Err(errors) = request.validate() {
Err(CrawlRsError::Unprocessable(
crate::presentation::handlers::response_builder::validation_error_message(&errors),
))
} else {
Ok(())
}
}
pub(crate) fn apply_defaults(request: &TaskQueryRequestDto) -> (u32, u32, bool, u32) {
(
request
.limit
.unwrap_or(server_config::DEFAULT_PAGE_LIMIT)
.min(server_config::MAX_PAGE_LIMIT),
request.offset.unwrap_or(0),
request.include_results.unwrap_or(false),
request
.sync_wait_ms
.unwrap_or(crawl_task::DEFAULT_TIMEOUT_MS as u32),
)
}
pub(crate) async fn execute_task_query<T: TaskRepository>(
task_repo: &T,
team_id: crate::domain::id::Id,
request: &TaskQueryRequestDto,
limit: u32,
offset: u32,
) -> Result<(Vec<crate::domain::models::Task>, u64), CrawlRsError> {
task_repo
.query_tasks(TaskQueryParams {
team_id,
task_ids: request.task_ids.clone(),
task_types: request.task_types.clone(),
statuses: request.statuses.clone(),
created_after: request.created_after.map(|dt| dt.with_timezone(&Utc)),
created_before: request.created_before.map(|dt| dt.with_timezone(&Utc)),
crawl_id: request.crawl_id,
limit,
offset,
cursor: None,
cursor_id: None,
})
.await
.map_err(|e| CrawlRsError::from(anyhow::anyhow!("Query failed: {:?}", e)))
}
pub(crate) async fn handle_sync_wait<T: TaskRepository>(
task_repo: &T,
tasks: &[crate::domain::models::Task],
team_id: crate::domain::id::Id,
sync_wait_ms: u32,
) -> Result<u64, CrawlRsError> {
let task_ids = extract_task_ids(tasks);
let wait_start = Instant::now();
wait_for_tasks_completion(
task_repo,
&task_ids,
team_id,
sync_wait_ms,
crawl_task::BASE_POLL_INTERVAL_MS,
)
.await?;
Ok(wait_start.elapsed().as_millis() as u64)
}
pub struct SyncWaitResult {
pub waited_time_ms: u64,
pub is_timeout: bool,
}
pub async fn handle_sync_wait_and_get_status(
task_repo: &dyn TaskRepository,
task_ids: &[crate::domain::id::Id],
team_id: crate::domain::id::Id,
sync_wait_ms: u32,
) -> Result<SyncWaitResult, CrawlRsError> {
if sync_wait_ms == 0 || task_ids.is_empty() {
return Ok(SyncWaitResult {
waited_time_ms: 0,
is_timeout: false,
});
}
let wait_start = Instant::now();
match wait_for_tasks_completion(
task_repo,
task_ids,
team_id,
sync_wait_ms,
crawl_task::BASE_POLL_INTERVAL_MS,
)
.await
{
Ok(_) => {
let waited_time_ms = wait_start.elapsed().as_millis() as u64;
Ok(SyncWaitResult {
waited_time_ms,
is_timeout: waited_time_ms >= sync_wait_ms as u64,
})
}
Err(e) => {
log::error!("Failed to wait for task completion: {:?}", e);
let waited_time_ms = wait_start.elapsed().as_millis() as u64;
Ok(SyncWaitResult {
waited_time_ms,
is_timeout: waited_time_ms >= sync_wait_ms as u64,
})
}
}
}
pub(crate) async fn fetch_scrape_results(
scrape_result_repo: &ScrapeResultRepositoryImpl,
tasks: &[crate::domain::models::Task],
) -> Result<
Option<std::collections::HashMap<crate::domain::id::Id, crate::domain::models::ScrapeResult>>,
CrawlRsError,
> {
let task_ids = extract_task_ids(tasks);
let results = scrape_result_repo.find_by_task_ids(&task_ids).await?;
let mut map = std::collections::HashMap::with_capacity(results.len());
for result in results {
map.insert(result.task_id, result);
}
Ok(Some(map))
}
pub(crate) fn build_task_infos(
tasks: &[crate::domain::models::Task],
results_map: Option<
&std::collections::HashMap<crate::domain::id::Id, crate::domain::models::ScrapeResult>,
>,
) -> Vec<TaskInfoDto> {
tasks
.iter()
.map(|task| {
let result = results_map
.and_then(|m| m.get(&task.id))
.map(build_scrape_result_json);
TaskInfoDto {
id: task.id,
task_type: task.task_type,
status: task.status,
priority: task.priority,
url: task.url.clone(),
attempt_count: task.attempt_count,
max_retries: task.max_retries,
created_at: chrono::FixedOffset::east_opt(0)
.unwrap()
.from_utc_datetime(&task.created_at.naive_utc()),
started_at: task.started_at.as_ref().map(|dt| {
chrono::FixedOffset::east_opt(0)
.unwrap()
.from_utc_datetime(&dt.naive_utc())
}),
completed_at: task.completed_at.as_ref().map(|dt| {
chrono::FixedOffset::east_opt(0)
.unwrap()
.from_utc_datetime(&dt.naive_utc())
}),
crawl_id: task.crawl_id,
result,
}
})
.collect()
}
pub(crate) fn build_scrape_result_json(
scrape_result: &crate::domain::models::ScrapeResult,
) -> ScrapeResultInfoDto {
let escaped_content = html_escape::encode_text(&scrape_result.content);
ScrapeResultInfoDto {
id: scrape_result.id,
status_code: scrape_result.status_code as u16,
content: escaped_content.to_string(),
metadata: Some(scrape_result.meta_data.clone()),
}
}