crw_server/routes/
change_tracking.rs

1//! `POST /v1/change-tracking/diff` — stateless change-tracking diff endpoint.
2//!
3//! This is the crawl-path workhorse: the SaaS monitor reconciler scrapes pages
4//! (via `/v1/crawl`), then calls this endpoint with each page's current
5//! markdown/json plus the prior snapshot to get a per-page diff. opencore
6//! stores nothing — `previous` is supplied by the caller.
7//!
8//! Two wire shapes on one route, discriminated by the presence of the `batch`
9//! key (no `deny_unknown_fields`, so a Single body's extra fields and a Batch
10//! body's shared fields never reject each other):
11//!   - Single: `{ current, previous?, modes, schema?, prompt?, contentType?, tag? }`
12//!   - Batch:  `{ batch: [ { url?, current, previous?, ... } ], modes, schema?, ... }`
13//!     where top-level `modes/schema/prompt/contentType` are shared defaults
14//!     each item may override.
15//!
16//! `goal` / `judgeEnabled` are accepted and ignored: this endpoint is
17//! deterministic and never calls an LLM. The judge runs only on `/v1/scrape`,
18//! opt-in via `goal` + `judgeEnabled: true` alongside the `changeTracking`
19//! format.
20
21use axum::Json;
22use axum::extract::State;
23use axum::extract::rejection::JsonRejection;
24use crw_core::error::CrwError;
25use crw_core::types::{
26    ApiResponse, ChangeTrackingMode, ChangeTrackingOptions, ChangeTrackingResult,
27    ChangeTrackingSnapshot,
28};
29use serde::Deserialize;
30use serde_json::Value;
31
32use crate::error::AppError;
33use crate::state::AppState;
34
35/// The current scrape content for one page.
36#[derive(Debug, Clone, Deserialize)]
37#[serde(rename_all = "camelCase")]
38pub struct DiffCurrent {
39    #[serde(default)]
40    pub markdown: Option<String>,
41    #[serde(default)]
42    pub json: Option<Value>,
43}
44
45/// One page to diff (single body, or one entry of a batch).
46#[derive(Debug, Clone, Deserialize)]
47#[serde(rename_all = "camelCase")]
48pub struct DiffItem {
49    #[serde(default)]
50    pub url: Option<String>,
51    #[serde(default)]
52    pub current: Option<DiffCurrent>,
53    #[serde(default)]
54    pub previous: Option<ChangeTrackingSnapshot>,
55    #[serde(default)]
56    pub modes: Option<Vec<ChangeTrackingMode>>,
57    #[serde(default)]
58    pub schema: Option<Value>,
59    #[serde(default)]
60    pub prompt: Option<String>,
61    #[serde(default, alias = "content_type")]
62    pub content_type: Option<String>,
63    #[serde(default)]
64    pub tag: Option<String>,
65    // Accepted and ignored: this endpoint is deterministic and never calls an
66    // LLM. The judge runs only on /v1/scrape (goal + judgeEnabled: true).
67    #[serde(default)]
68    pub goal: Option<String>,
69    #[serde(default, alias = "judge_enabled")]
70    pub judge_enabled: Option<bool>,
71}
72
73/// Request body. The presence of `batch` selects batch mode. Single-mode
74/// fields are flattened onto the same struct; in batch mode `modes/schema/
75/// prompt/contentType` act as shared defaults for items that omit them.
76#[derive(Debug, Clone, Deserialize)]
77#[serde(rename_all = "camelCase")]
78pub struct DiffRequest {
79    #[serde(default)]
80    pub batch: Option<Vec<DiffItem>>,
81    // ---- single-mode (and batch shared-default) fields ----
82    #[serde(default)]
83    pub current: Option<DiffCurrent>,
84    #[serde(default)]
85    pub previous: Option<ChangeTrackingSnapshot>,
86    #[serde(default)]
87    pub modes: Option<Vec<ChangeTrackingMode>>,
88    #[serde(default)]
89    pub schema: Option<Value>,
90    #[serde(default)]
91    pub prompt: Option<String>,
92    #[serde(default, alias = "content_type")]
93    pub content_type: Option<String>,
94    #[serde(default)]
95    pub tag: Option<String>,
96    #[serde(default)]
97    pub goal: Option<String>,
98    #[serde(default, alias = "judge_enabled")]
99    pub judge_enabled: Option<bool>,
100}
101
102fn default_modes() -> Vec<ChangeTrackingMode> {
103    vec![ChangeTrackingMode::GitDiff]
104}
105
106/// Build options + run the diff for one item, applying shared defaults.
107fn diff_one(
108    item: &DiffItem,
109    shared_modes: &Option<Vec<ChangeTrackingMode>>,
110    shared_schema: &Option<Value>,
111    shared_prompt: &Option<String>,
112    shared_content_type: &Option<String>,
113) -> Result<ChangeTrackingResult, CrwError> {
114    let current = item.current.as_ref().ok_or_else(|| {
115        CrwError::InvalidRequest("each diff item requires a 'current' object".into())
116    })?;
117
118    let modes = item
119        .modes
120        .clone()
121        .or_else(|| shared_modes.clone())
122        .unwrap_or_else(default_modes);
123
124    let opts = ChangeTrackingOptions {
125        modes,
126        schema: item.schema.clone().or_else(|| shared_schema.clone()),
127        prompt: item.prompt.clone().or_else(|| shared_prompt.clone()),
128        previous: item.previous.clone(),
129        tag: item.tag.clone(),
130        content_type: item
131            .content_type
132            .clone()
133            .or_else(|| shared_content_type.clone()),
134    };
135
136    let markdown = current.markdown.as_deref().unwrap_or("");
137    Ok(crw_diff::compute_change_tracking(
138        &opts,
139        markdown,
140        current.json.as_ref(),
141        opts.content_type.as_deref(),
142    ))
143}
144
145pub async fn diff(
146    State(_state): State<AppState>,
147    body: Result<Json<DiffRequest>, JsonRejection>,
148) -> Result<Json<ApiResponse<Value>>, AppError> {
149    let Json(req) = body.map_err(AppError::from)?;
150
151    // Batch mode: presence of `batch` wins.
152    if let Some(items) = &req.batch {
153        if items.is_empty() {
154            return Err(AppError::from(CrwError::InvalidRequest(
155                "'batch' must contain at least one item".into(),
156            )));
157        }
158        let mut results: Vec<ChangeTrackingResult> = Vec::with_capacity(items.len());
159        for item in items {
160            results.push(diff_one(
161                item,
162                &req.modes,
163                &req.schema,
164                &req.prompt,
165                &req.content_type,
166            )?);
167        }
168        let data = serde_json::to_value(results)
169            .map_err(|e| CrwError::Internal(format!("failed to serialize diff results: {e}")))?;
170        return Ok(Json(ApiResponse::ok(data)));
171    }
172
173    // Single mode.
174    let single = DiffItem {
175        url: None,
176        current: req.current.clone(),
177        previous: req.previous.clone(),
178        modes: req.modes.clone(),
179        schema: req.schema.clone(),
180        prompt: req.prompt.clone(),
181        content_type: req.content_type.clone(),
182        tag: req.tag.clone(),
183        goal: req.goal.clone(),
184        judge_enabled: req.judge_enabled,
185    };
186    let result = diff_one(&single, &None, &None, &None, &None)?;
187    let data = serde_json::to_value(result)
188        .map_err(|e| CrwError::Internal(format!("failed to serialize diff result: {e}")))?;
189    Ok(Json(ApiResponse::ok(data)))
190}
crw_server/routes/change_tracking.rs

crw_server/routes/
change_tracking.rs
