crw_server/routes/v2/
map.rs

1//! `POST /v2/map` — discover URLs, returning v2 link OBJECTS (the headline
2//! v1→v2 delta: v1 returned `links: string[]`, v2 returns
3//! `links: [{url, title?, description?}]`).
4
5use std::time::Duration;
6
7use axum::Json;
8use axum::extract::State;
9use axum::extract::rejection::JsonRejection;
10use serde::{Deserialize, Serialize};
11
12use crw_core::error::CrwError;
13use crw_crawl::crawl::{DiscoverOptions, discover_urls};
14
15use super::adapters::V2Link;
16use crate::error::AppError;
17use crate::state::AppState;
18
19#[derive(Debug, Deserialize)]
20#[serde(rename_all = "camelCase")]
21pub struct V2MapRequest {
22    pub url: String,
23    #[serde(default)]
24    pub limit: Option<usize>,
25    #[serde(default)]
26    pub include_paths: Vec<String>,
27    #[serde(default)]
28    pub exclude_paths: Vec<String>,
29    #[serde(default)]
30    pub search: Option<String>,
31    /// "include" (default) | "only" | "skip".
32    #[serde(default = "default_sitemap")]
33    pub sitemap: String,
34    #[serde(default)]
35    pub max_discovery_depth: Option<u32>,
36    /// v2 `timeout` is milliseconds.
37    #[serde(default)]
38    pub timeout: Option<u64>,
39}
40
41fn default_sitemap() -> String {
42    "include".to_string()
43}
44
45#[derive(Debug, Serialize)]
46pub struct V2MapResponse {
47    pub success: bool,
48    pub links: Vec<V2Link>,
49}
50
51pub async fn map(
52    State(state): State<AppState>,
53    body: Result<Json<V2MapRequest>, JsonRejection>,
54) -> Result<Json<V2MapResponse>, AppError> {
55    let Json(req) = body.map_err(AppError::from)?;
56    let parsed_url = url::Url::parse(&req.url)
57        .map_err(|e| CrwError::InvalidRequest(format!("Invalid URL: {e}")))?;
58    crw_core::url_safety::validate_safe_url_resolved(&parsed_url)
59        .await
60        .map_err(CrwError::InvalidRequest)?;
61
62    let use_sitemap = !req.sitemap.eq_ignore_ascii_case("skip");
63    let crawl_fallback = !req.sitemap.eq_ignore_ascii_case("only");
64    let max_depth = req
65        .max_discovery_depth
66        .unwrap_or(state.config.crawler.default_max_depth);
67    let timeout_secs = req
68        .timeout
69        .map(|ms| (ms / 1000).max(1))
70        .unwrap_or(120)
71        .min(300);
72
73    let fut = discover_urls(DiscoverOptions {
74        base_url: &req.url,
75        max_depth,
76        use_sitemap,
77        renderer: &state.renderer,
78        max_concurrency: state.config.crawler.max_concurrency,
79        requests_per_second: state.config.crawler.requests_per_second,
80        user_agent: &state.config.crawler.user_agent,
81        proxy: state.config.crawler.proxy.clone(),
82        deadline_ms_per_page: state.config.effective_deadline_ms(None, None),
83        per_host_max_concurrent: state.config.crawler.per_host_max_concurrent,
84        crawl_fallback,
85        url_filter: state.url_filter.clone(),
86        // Push the caller's limit INTO discovery so a large `limit` actually
87        // discovers that many URLs. The post-filter `truncate` below stays as a
88        // safety net (and trims when include/exclude/search narrow the set).
89        max_urls: req
90            .limit
91            .unwrap_or(crw_crawl::crawl::DEFAULT_MAX_DISCOVERED_URLS),
92    });
93
94    let result = match tokio::time::timeout(Duration::from_secs(timeout_secs), fut).await {
95        Ok(r) => r?,
96        Err(_) => return Err(AppError(CrwError::Timeout(timeout_secs * 1000))),
97    };
98
99    let mut urls = result.urls;
100    if !req.include_paths.is_empty() {
101        urls.retain(|u| req.include_paths.iter().any(|p| u.contains(p.as_str())));
102    }
103    if !req.exclude_paths.is_empty() {
104        urls.retain(|u| !req.exclude_paths.iter().any(|p| u.contains(p.as_str())));
105    }
106    if let Some(s) = req.search.as_ref().filter(|s| !s.is_empty()) {
107        let needle = s.to_lowercase();
108        urls.retain(|u| u.to_lowercase().contains(&needle));
109    }
110    // `0` means unbounded (matches discovery); only truncate for a positive cap.
111    if let Some(limit) = req.limit.filter(|l| *l > 0) {
112        urls.truncate(limit);
113    }
114
115    let links = urls
116        .into_iter()
117        .map(|url| V2Link {
118            url,
119            title: None,
120            description: None,
121        })
122        .collect();
123
124    Ok(Json(V2MapResponse {
125        success: true,
126        links,
127    }))
128}
crw_server/routes/v2/map.rs

crw_server/routes/v2/
map.rs
