crw_server/routes/
extract.rs

1//! Native `POST /v1/extract` (+ `GET /v1/extract/{id}`). Multi-URL structured
2//! extraction as an async job. Unlike the FC-legacy `/v2/extract` (which merges
3//! every URL's JSON into one object, last-write-wins), the native route returns
4//! a **per-URL array** (`results:[{url,status,data,error,llmUsage}]`) that keeps
5//! each URL's object distinct and carries per-URL LLM usage for downstream
6//! billing. No FC envelope (`success`/`urlTrace`/deprecation warning).
7
8use axum::Json;
9use axum::extract::rejection::JsonRejection;
10use axum::extract::{Path, State};
11use serde::{Deserialize, Serialize};
12use serde_json::Value;
13use uuid::Uuid;
14
15use crw_core::error::CrwError;
16use crw_core::types::{ExtractOptions, LlmUsage, OutputFormat, ScrapeRequest};
17
18use crate::error::AppError;
19use crate::routes::v2::adapters::expires_at_rfc3339;
20use crate::state::{AppState, PreparedUrl, UrlResult};
21
22/// Native extract request. camelCase like every other v1 public type.
23/// NOTE: no `#[derive(Debug)]` — `llm_api_key` is a secret and must never land
24/// in a `{:?}` log line.
25#[derive(Deserialize)]
26#[serde(rename_all = "camelCase")]
27pub struct ExtractRequest {
28    #[serde(default)]
29    pub urls: Vec<String>,
30    /// Free-text extraction objective (LLM infers the shape). Wired into the
31    /// extractor's `extract.prompt` slot — the field JSON extraction actually
32    /// reads (NOT `summary_prompt`, which only drives the summary format).
33    #[serde(default)]
34    pub prompt: Option<String>,
35    /// JSON Schema constraining the output.
36    #[serde(default)]
37    pub schema: Option<Value>,
38    // BYOK passthrough.
39    #[serde(default)]
40    pub llm_api_key: Option<String>,
41    #[serde(default)]
42    pub llm_provider: Option<String>,
43    #[serde(default)]
44    pub llm_model: Option<String>,
45    // `base_url` is parsed only so we can REJECT it with a clear 400 instead of
46    // silently ignoring it (which would route a BYOK key to the wrong endpoint).
47    // It flows unvalidated into the LLM client (`build_byok_llm_config`), an SSRF
48    // vector shared engine-wide; not accepted here until that path validates it.
49    #[serde(default)]
50    pub base_url: Option<String>,
51    /// Per-field attribution (Phase 2b). Parsed now; until 2b ships, `true` is
52    /// rejected so callers never believe an inert flag is honored.
53    #[serde(default)]
54    pub basis: Option<bool>,
55}
56
57#[derive(Debug, Serialize)]
58#[serde(rename_all = "camelCase")]
59pub struct ExtractStartResponse {
60    pub id: String,
61    pub status: String,
62    /// Count of URLs actually enqueued for fetch (preflight-failed URLs are in
63    /// the status `results`, not this count).
64    pub urls: usize,
65}
66
67pub async fn start_extract(
68    State(state): State<AppState>,
69    body: Result<Json<ExtractRequest>, JsonRejection>,
70) -> Result<Json<ExtractStartResponse>, AppError> {
71    let Json(req) = body.map_err(AppError::from)?;
72    let prepared = prepare_extract(&state, req).await?;
73    let urls = prepared.valid_count;
74    let id = state
75        .start_extract_job(prepared.entries, prepared.template)
76        .await;
77    Ok(Json(ExtractStartResponse {
78        id: id.to_string(),
79        status: "processing".to_string(),
80        urls,
81    }))
82}
83
84/// Validated + SSRF-preflighted extract inputs, ready for `start_extract_job`.
85pub(crate) struct PreparedExtract {
86    pub entries: Vec<PreparedUrl>,
87    pub template: ScrapeRequest,
88    /// Count of URLs enqueued for fetch (preflight-failed URLs excluded).
89    pub valid_count: usize,
90}
91
92/// Shared validation, SSRF preflight, and template build for the HTTP route and
93/// the MCP `crw_extract` tool. Returns `CrwError::InvalidRequest` (→ 400) on any
94/// rejected input so both callers get identical semantics.
95pub(crate) async fn prepare_extract(
96    state: &AppState,
97    req: ExtractRequest,
98) -> Result<PreparedExtract, CrwError> {
99    if req.urls.is_empty() {
100        return Err(CrwError::InvalidRequest(
101            "`urls` is required and must be non-empty".into(),
102        ));
103    }
104    let cap = state.config.crawler.max_extract_urls;
105    if req.urls.len() > cap {
106        return Err(CrwError::InvalidRequest(format!(
107            "too many urls: {} exceeds the per-request limit of {cap}",
108            req.urls.len()
109        )));
110    }
111    // A whitespace-only prompt is treated as absent (the extractor filters it to
112    // empty anyway) so we reject upfront instead of fetching then failing.
113    let has_prompt = req.prompt.as_deref().is_some_and(|p| !p.trim().is_empty());
114    if !has_prompt && req.schema.is_none() {
115        return Err(CrwError::InvalidRequest(
116            "nothing to extract: provide a non-empty `prompt`, a `schema`, or both".into(),
117        ));
118    }
119    if req.basis == Some(true) {
120        return Err(CrwError::InvalidRequest(
121            "`basis` (per-field attribution) is not yet supported".into(),
122        ));
123    }
124    if req.base_url.is_some() {
125        return Err(CrwError::InvalidRequest(
126            "`baseUrl` is not supported on /v1/extract; configure the LLM endpoint \
127             server-side ([extraction.llm.base_url])"
128                .into(),
129        ));
130    }
131
132    // LLM-availability guards, upfront (cheaper than failing in the worker).
133    // Mirror /v1/scrape's BYOK-header guard: the worker reaches the LLM directly,
134    // bypassing the scrape handler's check.
135    if let Some(cfg) = state.config.extraction.llm.as_ref()
136        && cfg.require_byok_header.is_some()
137        && req.llm_api_key.is_none()
138    {
139        return Err(CrwError::InvalidRequest(
140            "LLM features require a per-request llm_api_key (BYOK header guard active)".into(),
141        ));
142    }
143    if state.config.extraction.llm.is_none() && req.llm_api_key.is_none() {
144        return Err(CrwError::InvalidRequest(
145            "extraction requires an LLM: set [extraction.llm] in server config or pass \
146             llm_api_key in the request body"
147                .into(),
148        ));
149    }
150
151    // Per-URL preflight in original order. Bad parse / SSRF failures become
152    // `failed` results (surfaced, not dropped); valid URLs are enqueued.
153    let mut entries = Vec::with_capacity(req.urls.len());
154    let mut valid_count = 0usize;
155    for u in &req.urls {
156        match url::Url::parse(u) {
157            Ok(parsed) => match crw_core::url_safety::validate_safe_url_resolved(&parsed).await {
158                Ok(()) => {
159                    valid_count += 1;
160                    entries.push(PreparedUrl {
161                        url: u.clone(),
162                        preflight_error: None,
163                    });
164                }
165                Err(e) => entries.push(PreparedUrl {
166                    url: u.clone(),
167                    preflight_error: Some(e),
168                }),
169            },
170            Err(e) => entries.push(PreparedUrl {
171                url: u.clone(),
172                preflight_error: Some(format!("invalid URL: {e}")),
173            }),
174        }
175    }
176    if valid_count == 0 {
177        return Err(CrwError::InvalidRequest(
178            "no valid URLs to extract (all failed URL parsing or the SSRF safety check)".into(),
179        ));
180    }
181
182    let template = ScrapeRequest {
183        formats: vec![OutputFormat::Json],
184        json_schema: req.schema.clone(),
185        // `extract.prompt` is the field JSON extraction reads (single.rs).
186        extract: Some(ExtractOptions {
187            schema: None,
188            prompt: req.prompt.clone(),
189        }),
190        llm_api_key: req.llm_api_key.clone(),
191        llm_provider: req.llm_provider.clone(),
192        llm_model: req.llm_model.clone(),
193        ..Default::default()
194    };
195
196    Ok(PreparedExtract {
197        entries,
198        template,
199        valid_count,
200    })
201}
202
203#[derive(Debug, Serialize)]
204#[serde(rename_all = "camelCase")]
205pub struct ExtractUrlResult {
206    pub url: String,
207    pub status: String,
208    #[serde(skip_serializing_if = "Option::is_none")]
209    pub data: Option<Value>,
210    #[serde(skip_serializing_if = "Option::is_none")]
211    pub error: Option<String>,
212    #[serde(skip_serializing_if = "Option::is_none")]
213    pub llm_usage: Option<LlmUsage>,
214}
215
216impl From<UrlResult> for ExtractUrlResult {
217    fn from(r: UrlResult) -> Self {
218        ExtractUrlResult {
219            url: r.url,
220            status: r.status.as_str().to_string(),
221            data: r.data,
222            error: r.error,
223            llm_usage: r.llm_usage,
224        }
225    }
226}
227
228#[derive(Debug, Serialize)]
229#[serde(rename_all = "camelCase")]
230pub struct ExtractStatusResponse {
231    pub id: String,
232    pub status: String,
233    #[serde(skip_serializing_if = "Vec::is_empty")]
234    pub results: Vec<ExtractUrlResult>,
235    /// Job-level error, set only when every URL failed.
236    #[serde(skip_serializing_if = "Option::is_none")]
237    pub error: Option<String>,
238    pub expires_at: String,
239    pub credits_used: u32,
240    pub tokens_used: u32,
241}
242
243pub async fn get_extract(
244    State(state): State<AppState>,
245    Path(id): Path<Uuid>,
246) -> Result<Json<ExtractStatusResponse>, AppError> {
247    let rec = {
248        let jobs = state.extract_jobs.read().await;
249        jobs.get(&id)
250            .cloned()
251            .ok_or_else(|| CrwError::NotFound(format!("Extract job {id} not found")))?
252    };
253    let expires_at = expires_at_rfc3339(rec.created_at, state.config.crawler.job_ttl_secs);
254    Ok(Json(ExtractStatusResponse {
255        id: id.to_string(),
256        status: rec.status.as_str().to_string(),
257        results: rec
258            .per_url
259            .into_iter()
260            .map(ExtractUrlResult::from)
261            .collect(),
262        error: rec.error,
263        expires_at,
264        credits_used: rec.credits_used,
265        tokens_used: rec.tokens_used,
266    }))
267}
crw_server/routes/extract.rs

crw_server/routes/
extract.rs
