#![allow(dead_code)]
use sdforge::prelude::*;
use std::sync::Arc;
use crate::domain::models::{Task, TaskStatus, TaskType};
use crate::domain::repositories::crawl_repository::CrawlRepository;
use crate::domain::services::search_service::{SearchQuery, SearchServiceTrait};
use crate::i18n::{t, I18nBundle, Locale};
use crate::presentation::middleware::auth_middleware::AuthState;
use crate::queue::task_queue::TaskQueue;
const MAX_TASK_URL_BYTES: usize = 2048;
#[derive(serde::Deserialize)]
pub struct SdkSearchRequest {
pub query: String,
pub limit: Option<u32>,
}
#[derive(serde::Serialize)]
pub struct SdkSearchResult {
pub title: String,
pub url: String,
pub description: Option<String>,
pub engine: String,
}
#[derive(serde::Serialize)]
pub struct SdkSearchResponse {
pub query: String,
pub results: Vec<SdkSearchResult>,
pub credits_used: u32,
}
#[derive(serde::Deserialize)]
pub struct SdkCreateTaskRequest {
pub url: String,
pub task_type: String,
}
#[derive(serde::Serialize)]
pub struct SdkCreateTaskResponse {
pub id: crate::domain::id::Id,
pub url: String,
pub status: String,
}
#[derive(serde::Deserialize)]
pub struct SdkScrapeRequest {
pub url: String,
}
#[derive(serde::Serialize)]
pub struct SdkScrapeResponse {
pub id: crate::domain::id::Id,
pub url: String,
}
#[derive(serde::Deserialize)]
pub struct SdkCreateCrawlRequest {
pub name: String,
pub url: String,
pub seed_url: String,
#[serde(default)]
pub max_depth: Option<u32>,
}
#[derive(serde::Serialize)]
pub struct SdkCrawlResponse {
pub id: crate::domain::id::Id,
pub status: String,
pub url: String,
}
#[forge(
name = "sdk_search",
version = "v1",
path = "/sdk/search",
method = "POST",
description = "SDK search endpoint"
)]
async fn sdk_search(
#[state] search_service: Arc<dyn SearchServiceTrait>,
#[state] auth_state: AuthState,
#[state] locale: Locale,
#[state] i18n: Arc<I18nBundle>,
req: SdkSearchRequest,
) -> Result<SdkSearchResponse, ApiError> {
if req.query.trim().is_empty() {
return Err(ApiError::InvalidInput {
message: t(&locale, &i18n, "sdk-query-empty"),
field: Some("query".to_string()),
value: None,
});
}
let query = SearchQuery {
query: req.query.clone(),
limit: req.limit,
lang: None,
country: None,
engine: None,
sources: None,
crawl_results: None,
crawl_config: None,
};
let resp = search_service
.search(auth_state.team_id, auth_state.api_key_id, query)
.await
.map_err(|_e| ApiError::Internal {
message: "Internal server error".to_string(),
error_id: crate::domain::id::Id::generate().to_string(),
source: None,
context: None,
})?;
Ok(SdkSearchResponse {
query: resp.query,
results: resp
.results
.into_iter()
.map(|r| SdkSearchResult {
title: r.title,
url: r.url,
description: r.description,
engine: r.engine,
})
.collect(),
credits_used: resp.credits_used,
})
}
#[forge(
name = "sdk_create_task",
version = "v1",
path = "/sdk/tasks",
method = "POST",
description = "SDK create task endpoint"
)]
async fn sdk_create_task(
#[state] queue: Arc<dyn TaskQueue>,
#[state] auth_state: AuthState,
#[state] locale: Locale,
#[state] i18n: Arc<I18nBundle>,
req: SdkCreateTaskRequest,
) -> Result<SdkCreateTaskResponse, ApiError> {
if req.url.trim().is_empty() {
return Err(ApiError::InvalidInput {
message: t(&locale, &i18n, "sdk-url-empty"),
field: Some("url".to_string()),
value: None,
});
}
if req.url.len() > MAX_TASK_URL_BYTES {
return Err(ApiError::InvalidInput {
message: "url must be at most 2048 bytes".to_string(),
field: Some("url".to_string()),
value: None,
});
}
let task_type = match req.task_type.as_str() {
"scrape" => TaskType::Scrape,
"crawl" => TaskType::Crawl,
_ => {
return Err(ApiError::InvalidInput {
message: format!("unsupported task_type: {}", req.task_type),
field: Some("task_type".to_string()),
value: Some(serde_json::json!(req.task_type)),
});
}
};
let now = chrono::Utc::now();
let task = Task {
id: crate::domain::id::Id::generate(),
task_type,
status: TaskStatus::Queued,
priority: 0,
team_id: auth_state.team_id,
api_key_id: auth_state.api_key_id,
url: req.url.clone(),
payload: serde_json::json!({"url": req.url}),
retry_count: 0,
attempt_count: 0,
max_retries: 3,
scheduled_at: None,
expires_at: None,
created_at: now,
started_at: None,
completed_at: None,
crawl_id: None,
updated_at: now,
lock_token: None,
lock_expires_at: None,
};
let created = queue.enqueue(task).await.map_err(|_e| ApiError::Internal {
message: "Internal server error".to_string(),
error_id: crate::domain::id::Id::generate().to_string(),
source: None,
context: None,
})?;
Ok(SdkCreateTaskResponse {
id: created.id,
url: created.url,
status: format!("{:?}", created.status),
})
}
#[forge(
name = "sdk_scrape",
version = "v1",
path = "/sdk/scrape",
method = "POST",
description = "SDK scrape endpoint"
)]
async fn sdk_scrape(
#[state] queue: Arc<dyn TaskQueue>,
#[state] auth_state: AuthState,
#[state] locale: Locale,
#[state] i18n: Arc<I18nBundle>,
req: SdkScrapeRequest,
) -> Result<SdkScrapeResponse, ApiError> {
if req.url.trim().is_empty() {
return Err(ApiError::InvalidInput {
message: t(&locale, &i18n, "sdk-url-empty"),
field: Some("url".to_string()),
value: None,
});
}
if req.url.len() > MAX_TASK_URL_BYTES {
return Err(ApiError::InvalidInput {
message: "url must be at most 2048 bytes".to_string(),
field: Some("url".to_string()),
value: None,
});
}
let now = chrono::Utc::now();
let task = Task {
id: crate::domain::id::Id::generate(),
task_type: TaskType::Scrape,
status: TaskStatus::Queued,
priority: 0,
team_id: auth_state.team_id,
api_key_id: auth_state.api_key_id,
url: req.url.clone(),
payload: serde_json::json!({"url": req.url}),
retry_count: 0,
attempt_count: 0,
max_retries: 3,
scheduled_at: None,
expires_at: None,
created_at: now,
started_at: None,
completed_at: None,
crawl_id: None,
updated_at: now,
lock_token: None,
lock_expires_at: None,
};
let created = queue.enqueue(task).await.map_err(|_e| ApiError::Internal {
message: "Internal server error".to_string(),
error_id: crate::domain::id::Id::generate().to_string(),
source: None,
context: None,
})?;
Ok(SdkScrapeResponse {
id: created.id,
url: created.url,
})
}
#[forge(
name = "sdk_create_crawl",
version = "v1",
path = "/sdk/crawl",
method = "POST",
description = "SDK create crawl endpoint"
)]
#[allow(clippy::too_many_arguments)] async fn sdk_create_crawl(
#[state] crawl_repo: Arc<dyn CrawlRepository>,
#[state] queue: Arc<dyn TaskQueue>,
#[state] auth_state: AuthState,
#[state] locale: Locale,
#[state] i18n: Arc<I18nBundle>,
#[state] geo_repo: Option<
Arc<dyn crate::domain::repositories::geo_restriction_repository::GeoRestrictionRepository>,
>,
#[state] rate_limiting_service: Option<
Arc<dyn crate::domain::services::rate_limiting_service::RateLimitingService>,
>,
req: SdkCreateCrawlRequest,
) -> Result<SdkCrawlResponse, ApiError> {
for (field, value) in [
("url", req.url.as_str()),
("seed_url", req.seed_url.as_str()),
] {
if value.trim().is_empty() {
return Err(ApiError::InvalidInput {
message: t(&locale, &i18n, "sdk-url-empty"),
field: Some(field.to_string()),
value: None,
});
}
if value.len() > MAX_TASK_URL_BYTES {
return Err(ApiError::InvalidInput {
message: format!("{field} must be at most 2048 bytes"),
field: Some(field.to_string()),
value: None,
});
}
if crate::application::dto::crawl_request::is_http_url(value).is_err() {
return Err(ApiError::InvalidInput {
message: format!("{field} must start with http:// or https://"),
field: Some(field.to_string()),
value: None,
});
}
}
let max_depth = req.max_depth.unwrap_or(2);
if max_depth > crate::application::dto::crawl_request::MAX_CRAWL_DEPTH {
return Err(ApiError::InvalidInput {
message: format!(
"max_depth must be between 0 and {}",
crate::application::dto::crawl_request::MAX_CRAWL_DEPTH
),
field: Some("max_depth".to_string()),
value: Some(serde_json::json!(max_depth)),
});
}
if req.name.trim().is_empty() || req.name.chars().count() > 255 {
return Err(ApiError::InvalidInput {
message: "name must be 1-255 characters".to_string(),
field: Some("name".to_string()),
value: None,
});
}
if let Some(service) = rate_limiting_service.as_ref() {
use crate::domain::services::rate_limiting_service::RateLimitResult;
match service
.check_rate_limit(&auth_state.api_key_id.to_string(), "/sdk/crawl")
.await
{
Ok(RateLimitResult::Allowed) => {}
Ok(RateLimitResult::Denied { reason }) => {
log::warn!(
"SDK crawl rate limited for key {}: {}",
auth_state.api_key_id,
reason
);
return Err(ApiError::RateLimitExceeded {
limit: 0,
window_seconds: 60,
});
}
Ok(RateLimitResult::RetryAfter {
retry_after_seconds,
}) => {
return Err(ApiError::ServiceUnavailable {
service: "rate_limit".to_string(),
retry_after: Some(retry_after_seconds),
source: None,
});
}
Err(e) => {
log::error!("SDK crawl rate limit check failed: {}", e);
return Err(ApiError::Internal {
message: "Internal server error".to_string(),
error_id: crate::domain::id::Id::generate().to_string(),
source: None,
context: None,
});
}
}
}
let restrictions = match geo_repo.as_ref() {
Some(repo) => repo
.get_team_restrictions(auth_state.team_id)
.await
.map_err(|e| {
log::error!(
"Failed to load geo restrictions for team {} on SDK crawl: {}",
auth_state.team_id,
e
);
ApiError::Internal {
message: "Internal server error".to_string(),
error_id: crate::domain::id::Id::generate().to_string(),
source: None,
context: None,
}
})?,
None => crate::domain::services::team_service::TeamGeoRestrictions::default(),
};
if let Some(reason) =
crate::domain::services::team_service::domain_blacklist_denial(&req.seed_url, &restrictions)
{
return Err(ApiError::InvalidInput {
message: reason,
field: Some("seed_url".to_string()),
value: None,
});
}
let domain_blacklist = restrictions
.effective_domain_blacklist()
.map_err(|message| ApiError::InvalidInput {
message,
field: Some("domain_blacklist".to_string()),
value: None,
})?;
match tokio::time::timeout(
std::time::Duration::from_secs(5),
crate::presentation::handlers::check_ssrf_url(
&req.seed_url,
auth_state.team_id,
auth_state.api_key_id,
),
)
.await
{
Ok(None) => {}
Ok(Some(_)) => {
return Err(ApiError::InvalidInput {
message: "seed_url is not allowed (SSRF protection)".to_string(),
field: Some("seed_url".to_string()),
value: None,
});
}
Err(_) => {
return Err(ApiError::ServiceUnavailable {
service: "ssrf_check".to_string(),
retry_after: Some(1),
source: None,
});
}
}
let dto = crate::application::dto::crawl_request::CrawlRequestDto {
url: req.seed_url,
validated_url: None,
name: Some(req.name),
config: crate::application::dto::crawl_request::CrawlConfigDto {
max_depth,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
},
sync_wait_ms: None,
expires_at: None,
sitemap_seed: None,
sitemap_url: None,
};
let now = chrono::Utc::now();
let crawl =
crate::application::use_cases::crawl_link_processor::persist_crawl_with_initial_task(
crawl_repo.as_ref(),
auth_state.team_id,
auth_state.api_key_id,
&dto,
&req.url,
domain_blacklist,
now,
|task| {
let q = queue.clone();
async move { q.enqueue(task).await }
},
)
.await
.map_err(|e| {
log::error!("SDK crawl create failed: {}", e);
ApiError::Internal {
message: "Internal server error".to_string(),
error_id: crate::domain::id::Id::generate().to_string(),
source: None,
context: None,
}
})?;
Ok(SdkCrawlResponse {
id: crawl.id,
status: format!("{:?}", crate::domain::models::CrawlStatus::Queued),
url: crawl.root_url,
})
}
pub fn build_sdk_router() -> axum::Router {
let router = sdforge::http::build();
#[cfg(feature = "openapi")]
let router = router.route("/openapi.json", axum::routing::get(openapi_json_handler));
router
}
#[cfg(feature = "openapi")]
async fn openapi_json_handler() -> axum::Json<utoipa::openapi::OpenApi> {
axum::Json(sdforge::openapi::generate_openapi_spec())
}
#[cfg(any(test, feature = "test-mocks"))]
pub mod mocks;
#[cfg(all(test, feature = "test-mocks"))]
mod tests;