use axum::{
extract::ConnectInfo,
extract::{Extension, Json, Path},
http::StatusCode,
response::{IntoResponse, Response},
};
use log::error;
use sdforge::validator::Validate as _;
use std::sync::Arc;
use crate::{
application::dto::scrape_request::ScrapeRequestDto,
application::dto::scrape_response::{CancelScrapeResponseDto, ScrapeResponseDto},
common::constants::crawl_task::MAX_SYNC_WAIT_MS,
domain::models::{Task, TaskType},
domain::repositories::geo_restriction_repository::GeoRestrictionRepository,
domain::repositories::task_repository::TaskRepository,
domain::services::rate_limiting_service::RateLimitingService,
domain::services::team_service::TeamService,
i18n::{I18nBundle, Locale},
presentation::extractors::CrawlRsDeps,
presentation::handlers::credits::{CreditCharge, CreditFlow},
presentation::handlers::response_builder::{
errors, errors_locale, json_rejection_response, success_response, ApiResponse,
},
presentation::handlers::task_queries::handle_sync_wait_and_get_status,
presentation::handlers::{check_ssrf_url, sync_wait_status_code},
presentation::helpers::rate_limit_helper::check_rate_limit,
presentation::middleware::auth_middleware::AuthState,
queue::task_queue::TaskQueue,
};
use std::net::SocketAddr;
pub async fn create_scrape(
CrawlRsDeps {
queue,
settings: _settings,
task_repo: task_repository,
rate_limiting_service,
auth_state,
}: CrawlRsDeps,
Extension(geo_restriction_repo): Extension<Option<Arc<dyn GeoRestrictionRepository>>>,
Extension(team_service): Extension<Option<Arc<TeamService>>>,
ConnectInfo(addr): ConnectInfo<SocketAddr>,
payload: Result<Json<ScrapeRequestDto>, axum::extract::rejection::JsonRejection>,
) -> impl IntoResponse {
let Json(payload) = match payload {
Ok(parsed) => parsed,
Err(ref rej) => return json_rejection_response(rej),
};
let team_id = auth_state.team_id;
if let Some(response) = preflight_scrape(
&payload,
rate_limiting_service.as_ref(),
&auth_state,
geo_restriction_repo.as_ref(),
team_service.as_ref(),
addr.ip(),
)
.await
{
return response;
}
let task = build_scrape_task(&payload, team_id, auth_state.api_key_id);
let sync_wait_ms = payload.sync_wait_ms.unwrap_or(0);
enqueue_scrape(
queue.as_ref(),
task,
task_repository.as_ref(),
rate_limiting_service.as_ref(),
team_id,
&payload,
sync_wait_ms,
)
.await
}
fn build_scrape_task(
payload: &ScrapeRequestDto,
team_id: crate::domain::id::Id,
api_key_id: crate::domain::id::Id,
) -> Task {
Task::new(
crate::domain::id::Id::generate(),
TaskType::Scrape,
team_id,
api_key_id,
payload.url.clone(),
serde_json::to_value(payload).unwrap_or_default(),
)
}
async fn preflight_scrape(
payload: &ScrapeRequestDto,
rate_limiting_service: &dyn RateLimitingService,
auth_state: &AuthState,
geo_restriction_repo: Option<&Arc<dyn GeoRestrictionRepository>>,
team_service: Option<&Arc<TeamService>>,
client_ip: std::net::IpAddr,
) -> Option<Response> {
let team_id = auth_state.team_id;
let api_key_id = auth_state.api_key_id;
if let Some(ms) = payload.sync_wait_ms {
if ms > MAX_SYNC_WAIT_MS {
return Some(errors::unprocessable_entity(format!(
"sync_wait_ms must be <= {}",
MAX_SYNC_WAIT_MS
)));
}
}
if let Err(response) = check_rate_limit(rate_limiting_service, api_key_id, "/v1/scrape").await {
return Some(response);
}
if let Some(response) = check_scrape_ssrf_targets(payload, team_id, api_key_id).await {
return Some(response);
}
if let Some(response) =
enforce_scrape_geo_restrictions(geo_restriction_repo, team_service, team_id, client_ip)
.await
{
return Some(response);
}
if payload.url.len() > 2048 {
return Some(errors::unprocessable_entity(
"url must be at most 2048 bytes",
));
}
if let Err(errors) = payload.validate() {
return Some(errors::unprocessable_entity(
crate::presentation::handlers::response_builder::validation_error_message(&errors),
));
}
CreditFlow::new(rate_limiting_service, team_id)
.deduct(CreditCharge::Scrape { url: &payload.url })
.await
}
async fn enqueue_scrape(
queue: &dyn TaskQueue,
task: Task,
task_repository: &dyn TaskRepository,
rate_limiting_service: &dyn RateLimitingService,
team_id: crate::domain::id::Id,
payload: &ScrapeRequestDto,
sync_wait_ms: u32,
) -> Response {
match queue.enqueue(task.clone()).await {
Ok(_) => respond_scrape_enqueued(task_repository, task, team_id, sync_wait_ms).await,
Err(e) => {
error!(
"Failed to enqueue task for team {}: {}. Payload: {:?}",
team_id, e, payload
);
CreditFlow::new(rate_limiting_service, team_id)
.refund(CreditCharge::Scrape { url: &payload.url }, Some(task.id))
.await;
errors::internal_server_error(e.to_string())
}
}
}
async fn check_scrape_ssrf_targets(
payload: &ScrapeRequestDto,
team_id: crate::domain::id::Id,
api_key_id: crate::domain::id::Id,
) -> Option<Response> {
if let Some(response) = check_ssrf_url(&payload.url, team_id, api_key_id).await {
return Some(response);
}
if let Some(ref options) = payload.options {
if let Some(ref proxy_url) = options.proxy {
if let Some(response) = check_ssrf_url(proxy_url, team_id, api_key_id).await {
return Some(response);
}
}
}
if let Some(ref webhook_url) = payload.webhook {
if let Some(response) = check_ssrf_url(webhook_url, team_id, api_key_id).await {
return Some(response);
}
}
None
}
async fn enforce_scrape_geo_restrictions(
geo_restriction_repo: Option<&Arc<dyn GeoRestrictionRepository>>,
team_service: Option<&Arc<TeamService>>,
team_id: crate::domain::id::Id,
client_ip: std::net::IpAddr,
) -> Option<Response> {
let (geo_repo, team_service) = match (geo_restriction_repo, team_service) {
(Some(geo_repo), Some(team_service)) => (geo_repo, team_service),
_ => return None,
};
let client_ip = client_ip.to_string();
let restrictions = match geo_repo.get_team_restrictions(team_id).await {
Ok(r) => r,
Err(e) => {
error!("Failed to get team restrictions: {:?}", e);
return Some(errors::internal_server_error(
"Failed to validate geographic access",
));
}
};
match team_service
.validate_geographic_restriction(team_id, &client_ip, &restrictions)
.await
{
Ok(crate::domain::services::team_service::GeoRestrictionResult::Allowed) => {
if let Err(e) = geo_repo
.log_geo_restriction_action(
team_id,
&client_ip,
"",
"ALLOWED",
"Scrape request - geographic restriction check passed",
)
.await
{
error!("Failed to log geographic restriction action: {:?}", e);
}
None
}
Ok(crate::domain::services::team_service::GeoRestrictionResult::Denied(reason)) => {
if let Err(e) = geo_repo
.log_geo_restriction_action(team_id, &client_ip, "", "DENIED", &reason)
.await
{
error!("Failed to log geographic restriction action: {:?}", e);
}
Some(errors::forbidden(reason))
}
Err(e) => {
error!("Geographic restriction validation error: {:?}", e);
Some(errors::internal_server_error(
"Failed to validate geographic access",
))
}
}
}
async fn respond_scrape_enqueued(
task_repository: &dyn TaskRepository,
task: Task,
team_id: crate::domain::id::Id,
sync_wait_ms: u32,
) -> Response {
let wait_result =
handle_sync_wait_and_get_status(task_repository, &[task.id], team_id, sync_wait_ms)
.await
.unwrap_or({
crate::presentation::handlers::task_queries::SyncWaitResult {
waited_time_ms: 0,
is_timeout: false,
}
});
let response = ScrapeResponseDto {
id: task.id,
url: task.url,
credits_used: 1,
captured_responses: None,
};
let status_code = sync_wait_status_code(sync_wait_ms, wait_result.is_timeout);
success_response(status_code, response)
}
pub async fn cancel_scrape(
Path(id): Path<crate::domain::id::Id>,
Extension(repository): Extension<Arc<dyn TaskRepository>>,
Extension(auth_state): Extension<AuthState>,
Extension(locale): Extension<Locale>,
Extension(bundle): Extension<Arc<I18nBundle>>,
) -> impl IntoResponse {
let team_id = auth_state.team_id;
match repository.find_by_id(id).await {
Ok(Some(task)) => {
if task.team_id != team_id {
return errors_locale::forbidden(&locale, &bundle, "api-access-denied");
}
match repository.mark_cancelled(id).await {
Ok(_) => {
let response = CancelScrapeResponseDto {
message: crate::i18n::t(&locale, &bundle, "api-scrape-cancelled"),
};
(StatusCode::OK, Json(ApiResponse::success(response))).into_response()
}
Err(e) => {
error!("Failed to cancel task {}: {}", id, e);
errors_locale::internal_server_error(&locale, &bundle, "api-internal-error")
}
}
}
Ok(None) => errors_locale::not_found(&locale, &bundle, "api-task-not-found"),
Err(e) => {
error!("Failed to get task {} for cancellation: {}", id, e);
errors_locale::internal_server_error(&locale, &bundle, "api-internal-error")
}
}
}