use axum::{
extract::{ConnectInfo, Extension, Path},
http::StatusCode,
response::{IntoResponse, Response},
Json,
};
use std::net::SocketAddr;
use std::sync::Arc;
use crate::application::dto::crawl_request::CrawlRequestDto;
use crate::application::use_cases::crawl_use_case::CrawlUseCaseError;
use crate::common::constants::crawl_task::DEFAULT_TIMEOUT_MS;
use crate::config::settings::Settings;
use crate::i18n::{I18nBundle, Locale};
use crate::presentation::handlers::credits::{CreditCharge, CreditFlow};
use crate::presentation::handlers::response_builder::errors;
use crate::presentation::handlers::response_builder::{
error_response, error_response_locale, errors_locale, json_rejection_response, success_response,
};
use crate::presentation::handlers::task_queries::handle_sync_wait_and_get_status;
use crate::presentation::handlers::task_queries::SyncWaitResult;
use crate::presentation::handlers::{check_ssrf_url, extract_task_ids, sync_wait_status_code};
use crate::presentation::helpers::rate_limit_helper::check_rate_limit;
use crate::presentation::middleware::auth_middleware::AuthState;
use crate::presentation::state::CrawlHandlerState;
use log::error;
fn sitemap_seed_rejection(payload: &CrawlRequestDto, settings: &Settings) -> Option<&'static str> {
#[cfg(not(feature = "sitemap"))]
let _ = settings;
if payload.sitemap_seed != Some(true) {
return None;
}
#[cfg(not(feature = "sitemap"))]
let disabled_reason: Option<&'static str> = Some("api-sitemap-feature-required");
#[cfg(feature = "sitemap")]
let disabled_reason: Option<&'static str> = if settings.sitemap.enabled {
None
} else {
Some("api-sitemap-source-disabled")
};
disabled_reason
}
pub async fn create_crawl(
Extension(state): Extension<Arc<CrawlHandlerState>>,
Extension(settings): Extension<Arc<Settings>>,
Extension(auth_state): Extension<AuthState>,
Extension(locale): Extension<Locale>,
Extension(bundle): Extension<Arc<I18nBundle>>,
ConnectInfo(addr): ConnectInfo<SocketAddr>,
payload: Result<Json<CrawlRequestDto>, axum::extract::rejection::JsonRejection>,
) -> impl IntoResponse {
let Json(payload) = match payload {
Ok(parsed) => parsed,
Err(ref rej) => return json_rejection_response(rej),
};
let team_id = auth_state.team_id;
let sync_wait_ms = payload.sync_wait_ms.unwrap_or(DEFAULT_TIMEOUT_MS as u32);
if let Some(response) =
preflight_crawl(&state, &payload, &settings, &auth_state, &locale, &bundle).await
{
return response;
}
let use_case = state.create_use_case();
let client_ip = addr.ip().to_string();
let crawl_url = payload.url.clone();
match use_case
.create_crawl(team_id, auth_state.api_key_id, payload, &client_ip)
.await
{
Ok(crawl) => {
let wait_result =
await_crawl_sync_wait(state.as_ref(), crawl.id, team_id, sync_wait_ms).await;
let status_code = sync_wait_status_code(sync_wait_ms, wait_result.is_timeout);
success_response(status_code, crawl)
}
Err(e) => {
CreditFlow::new(state.rate_limiting_service.as_ref(), team_id)
.refund(CreditCharge::Crawl { url: &crawl_url }, None)
.await;
crawl_error_response(e, &locale, &bundle)
}
}
}
async fn preflight_crawl(
state: &CrawlHandlerState,
payload: &CrawlRequestDto,
settings: &Settings,
auth_state: &AuthState,
locale: &Locale,
bundle: &Arc<I18nBundle>,
) -> Option<Response> {
if let Some(response) = validate_crawl_request(payload, settings, locale, bundle) {
return Some(response);
}
if let Err(response) = check_rate_limit(
state.rate_limiting_service.as_ref(),
auth_state.api_key_id,
"/v1/crawl",
)
.await
{
return Some(response);
}
if let Some(response) =
check_crawl_ssrf_targets(payload, auth_state.team_id, auth_state.api_key_id).await
{
return Some(response);
}
if let Some(response) =
CreditFlow::new(state.rate_limiting_service.as_ref(), auth_state.team_id)
.deduct(CreditCharge::Crawl { url: &payload.url })
.await
{
return Some(response);
}
None
}
fn crawl_error_response(
e: CrawlUseCaseError,
locale: &Locale,
bundle: &Arc<I18nBundle>,
) -> Response {
match e {
CrawlUseCaseError::NotFound => {
errors_locale::not_found(locale, bundle, "api-crawl-not-found")
}
e => {
let (status, msg): (StatusCode, String) = e.into();
error_response(status, msg)
}
}
}
fn validate_crawl_request(
payload: &CrawlRequestDto,
settings: &Settings,
locale: &Locale,
bundle: &Arc<I18nBundle>,
) -> Option<Response> {
if payload.url.len() > 2048 {
return Some(errors::unprocessable_entity(
"url must be at most 2048 bytes",
));
}
if payload.config.max_depth > crate::application::dto::crawl_request::MAX_CRAWL_DEPTH {
return Some(errors::unprocessable_entity(
"max_depth must be between 0 and 5",
));
}
if let Some(concurrency) = payload.config.max_concurrency {
if concurrency > crate::application::dto::crawl_request::MAX_CONCURRENCY {
return Some(errors::unprocessable_entity(
"max_concurrency must be between 1 and 50",
));
}
}
if let Some(key) = sitemap_seed_rejection(payload, settings) {
return Some(error_response_locale(
StatusCode::UNPROCESSABLE_ENTITY,
locale,
bundle,
key,
));
}
None
}
async fn check_crawl_ssrf_targets(
payload: &CrawlRequestDto,
team_id: crate::domain::id::Id,
api_key_id: crate::domain::id::Id,
) -> Option<Response> {
if let Some(response) = check_ssrf_url(&payload.url, team_id, api_key_id).await {
return Some(response);
}
if let Some(ref proxy_url) = payload.config.proxy {
if let Some(response) = check_ssrf_url(proxy_url, team_id, api_key_id).await {
return Some(response);
}
}
if let Some(ref sitemap_url) = payload.sitemap_url {
if let Some(response) = check_ssrf_url(sitemap_url, team_id, api_key_id).await {
return Some(response);
}
}
None
}
async fn await_crawl_sync_wait(
state: &CrawlHandlerState,
crawl_id: crate::domain::id::Id,
team_id: crate::domain::id::Id,
sync_wait_ms: u32,
) -> SyncWaitResult {
if sync_wait_ms == 0 {
return SyncWaitResult {
waited_time_ms: 0,
is_timeout: false,
};
}
match state.task_repo.find_by_crawl_id(crawl_id).await {
Ok(tasks) => {
if tasks.is_empty() {
SyncWaitResult {
waited_time_ms: 0,
is_timeout: false,
}
} else {
let task_ids = extract_task_ids(&tasks);
handle_sync_wait_and_get_status(
state.task_repo.as_ref(),
&task_ids,
team_id,
sync_wait_ms,
)
.await
.unwrap_or(SyncWaitResult {
waited_time_ms: sync_wait_ms as u64,
is_timeout: true,
})
}
}
Err(e) => {
error!("Failed to find tasks for crawl {}: {:?}", crawl_id, e);
SyncWaitResult {
waited_time_ms: 0,
is_timeout: false,
}
}
}
}
pub async fn cancel_crawl(
Extension(state): Extension<Arc<CrawlHandlerState>>,
Extension(auth_state): Extension<AuthState>,
Extension(locale): Extension<Locale>,
Extension(bundle): Extension<Arc<I18nBundle>>,
Path(crawl_id): Path<crate::domain::id::Id>,
) -> impl IntoResponse {
let team_id = auth_state.team_id;
let use_case = state.create_use_case();
match use_case.cancel_crawl(crawl_id, team_id).await {
Ok(_) => StatusCode::NO_CONTENT.into_response(),
Err(e) => match e {
CrawlUseCaseError::NotFound => {
errors_locale::not_found(&locale, &bundle, "api-crawl-not-found")
}
e => {
let (status, msg): (StatusCode, String) = e.into();
error_response(status, msg)
}
},
}
}
impl From<CrawlUseCaseError> for (StatusCode, String) {
fn from(err: CrawlUseCaseError) -> Self {
match err {
CrawlUseCaseError::ValidationError(msg) => (StatusCode::BAD_REQUEST, msg),
CrawlUseCaseError::Repository(_) => (
StatusCode::INTERNAL_SERVER_ERROR,
"Internal server error".to_string(),
),
CrawlUseCaseError::NotFound => (StatusCode::NOT_FOUND, "Crawl not found".to_string()),
CrawlUseCaseError::Anyhow(_) => (
StatusCode::INTERNAL_SERVER_ERROR,
"Internal server error".to_string(),
),
}
}
}