writ_client/
models.rs

1//! Wire models for the Writ agent API.
2//!
3//! Field-level shapes mirror the daemon source (`writ-agent/src/local/`),
4//! never invented. Only stable scalar fields are strongly typed; everything else —
5//! including any field a future daemon adds — lands in the `extra` map via
6//! `#[serde(flatten)]`, so unknown fields never break deserialization. Boolean-ish
7//! daemon columns arrive as SQLite `0/1` integers and are typed `Option<i64>` here
8//! to match the wire exactly.
9
10use serde::{Deserialize, Serialize};
11use serde_json::{Map, Value};
12
13/// Catch-all for wire fields this SDK does not type. Keyed by field name.
14pub type Extra = Map<String, Value>;
15
16/// `GET /v1/agent` — lightweight daemon status (`server.rs::agent_status`).
17#[derive(Debug, Clone, Default, Serialize, Deserialize)]
18#[serde(default)]
19pub struct AgentStatus {
20    pub status: String,
21    pub version: Option<String>,
22    pub active_runs: Option<i64>,
23    pub encrypted: Option<bool>,
24    pub due_monitors: Option<i64>,
25    pub last_tick_at: Option<String>,
26    pub warm_browser: Option<bool>,
27    #[serde(flatten)]
28    pub extra: Extra,
29}
30
31/// `GET /v1/health` — deep health probe (`server.rs::health`).
32#[derive(Debug, Clone, Default, Serialize, Deserialize)]
33#[serde(default)]
34pub struct Health {
35    pub status: String,
36    pub version: Option<String>,
37    pub cipher_present: Option<bool>,
38    pub db_ok: Option<bool>,
39    pub keyring_ok: Option<bool>,
40    pub active_runs: Option<i64>,
41    pub warm_browser: Option<bool>,
42    /// `{ last_tick_at, due_monitors }`.
43    pub scheduler: Option<Value>,
44    /// `{ linked, account_id }`.
45    pub cloud_link: Option<Value>,
46    #[serde(flatten)]
47    pub extra: Extra,
48}
49
50/// A workflow row as returned by the API (`store/workflows.rs::Workflow`, shaped by
51/// `api/v1/workflows.rs::redact`): `credentials_encrypted` never appears; the daemon
52/// adds `has_credentials`, `credential_keys`, `placeholders`, `has_login` and
53/// re-hydrates JSON-TEXT columns (`steps`, `functions`, …) into real JSON.
54#[derive(Debug, Clone, Default, Serialize, Deserialize)]
55#[serde(default)]
56pub struct Workflow {
57    pub id: i64,
58    pub name: String,
59    pub description: Option<String>,
60    pub workflow_type: Option<String>,
61    pub entry_url: Option<String>,
62    /// Recorded steps, re-hydrated to a JSON array by the daemon.
63    pub steps: Option<Value>,
64    pub form_data: Option<Value>,
65    pub functions: Option<Value>,
66    pub is_active: Option<i64>,
67    pub is_verified: Option<i64>,
68    pub timeout_ms: Option<i64>,
69    pub retry_count: Option<i64>,
70    pub headless: Option<i64>,
71    pub schedule_enabled: Option<i64>,
72    pub schedule_interval_ms: Option<i64>,
73    /// `interval` (default) | `daily` | `weekly`.
74    pub schedule_kind: Option<String>,
75    /// "HH:MM" local wall-clock fire time (daily/weekly).
76    pub schedule_time: Option<String>,
77    /// JSON array *string* of ISO weekday ints (weekly only) — not re-hydrated.
78    pub schedule_days: Option<String>,
79    pub schedule_tz: Option<String>,
80    pub last_scheduled_at: Option<String>,
81    pub next_scheduled_at: Option<String>,
82    pub default_persona_id: Option<i64>,
83    pub http_capable: Option<i64>,
84    pub usage_count: Option<i64>,
85    pub total_run_count: Option<i64>,
86    pub total_failure_count: Option<i64>,
87    pub consecutive_failures: Option<i64>,
88    pub last_run_at: Option<String>,
89    pub last_run_status: Option<String>,
90    pub last_run_duration_ms: Option<i64>,
91    pub last_failure_at: Option<String>,
92    pub last_failure_error: Option<String>,
93    pub cloud_callable: Option<i64>,
94    pub execution_target: Option<String>,
95    pub marketplace_slug: Option<String>,
96    pub created_at: Option<String>,
97    pub updated_at: Option<String>,
98    // redact()-computed additions:
99    pub has_credentials: Option<bool>,
100    /// Name-only view of the sealed credential map's keys — never values.
101    pub credential_keys: Option<Vec<String>>,
102    /// `{key, label, field_type}` run-input descriptors.
103    pub placeholders: Option<Vec<Value>>,
104    pub has_login: Option<bool>,
105    #[serde(flatten)]
106    pub extra: Extra,
107}
108
109/// `POST /v1/workflows/:id/run` → `202 {run_id, status:"running"}`.
110#[derive(Debug, Clone, Default, Serialize, Deserialize)]
111#[serde(default)]
112pub struct RunStarted {
113    pub run_id: i64,
114    pub status: String,
115    #[serde(flatten)]
116    pub extra: Extra,
117}
118
119/// Terminal run document returned by `workflows().run_wait(..)` (`?wait=true`).
120///
121/// A FAILED run arrives here as a normal value with `status == "failed"`, NOT as an
122/// error: the call succeeded in REPORTING the outcome. Check `status`.
123#[derive(Debug, Clone, Deserialize, Serialize)]
124pub struct RunCompleted {
125    pub run_id: i64,
126    /// One of `success` | `failed` | `timeout` | `cancelled`.
127    pub status: String,
128    #[serde(default)]
129    pub done: bool,
130    /// The run's result payload, when it produced one.
131    #[serde(default)]
132    pub data: Option<Value>,
133    /// Why it failed. Present for non-success terminal states.
134    #[serde(default)]
135    pub error: Option<String>,
136    #[serde(default)]
137    pub duration_ms: Option<i64>,
138    /// Populated only on the 504 (still-running) body.
139    #[serde(default)]
140    pub status_url: Option<String>,
141    #[serde(default)]
142    pub events_url: Option<String>,
143    #[serde(flatten)]
144    pub extra: Extra,
145}
146
147/// Enriched run-feed item (`api/v1/runs.rs::RunFeedItem`).
148///
149/// `id` is a composite string `"<run_type>-<row_id>"` (e.g. `"workflow-3"`); the
150/// numeric id for `runs().get/cancel/events` is [`RunFeedItem::row_id`].
151#[derive(Debug, Clone, Default, Serialize, Deserialize)]
152#[serde(default)]
153pub struct RunFeedItem {
154    /// Composite id, unique across the feed: `"<run_type>-<row_id>"`.
155    pub id: String,
156    /// `workflow` | `check` | `automation` (open set).
157    pub run_type: Option<String>,
158    pub entity_id: Option<i64>,
159    pub entity_name: Option<String>,
160    /// `running | success | failed | cancelled | timeout | captcha_required |
161    /// twofa_required` — treat as an open string enum.
162    pub status: String,
163    pub started_at: Option<String>,
164    pub finished_at: Option<String>,
165    pub duration_ms: Option<i64>,
166    pub trigger_source: Option<String>,
167    pub error: Option<String>,
168    pub detail_url_hint: Option<String>,
169    pub data_url_hint: Option<String>,
170    /// Workflow runs only: extracted record count.
171    pub rows_extracted: Option<i64>,
172    /// Check runs only: whether the check detected a change.
173    pub change_detected: Option<bool>,
174    /// Execution lane: `http` | `browser` | `hybrid` (workflow runs).
175    pub engine: Option<String>,
176    #[serde(flatten)]
177    pub extra: Extra,
178}
179
180impl RunFeedItem {
181    /// The numeric run row id parsed out of the composite `id`
182    /// (`"workflow-3"` → `3`). A purely numeric `id` also parses.
183    pub fn row_id(&self) -> Option<i64> {
184        self.id
185            .rsplit_once('-')
186            .and_then(|(_, tail)| tail.parse().ok())
187            .or_else(|| self.id.parse().ok())
188    }
189
190    /// True while the run is still in flight.
191    pub fn is_running(&self) -> bool {
192        self.status == "running"
193    }
194}
195
196/// `GET /v1/runs/:id/results` → `{run_id, status, result}`.
197#[derive(Debug, Clone, Default, Serialize, Deserialize)]
198#[serde(default)]
199pub struct RunResults {
200    pub run_id: i64,
201    pub status: String,
202    /// The run's raw `result_data` JSON (`null` when the run produced none).
203    pub result: Value,
204    #[serde(flatten)]
205    pub extra: Extra,
206}
207
208/// `GET /v1/runs/:id/data` → `{run_id, status, data}` (default JSON lane).
209#[derive(Debug, Clone, Default, Serialize, Deserialize)]
210#[serde(default)]
211pub struct RunData {
212    pub run_id: i64,
213    pub status: String,
214    /// The run's extracted data (`result_data.extracted_data`, falling back to the
215    /// whole result payload).
216    pub data: Value,
217    #[serde(flatten)]
218    pub extra: Extra,
219}
220
221/// Outcome of a cancel call. A `202` answers `{run_id, status:"cancel_requested"}`;
222/// a `409` answers `{status:"not_running", ...}` — per DESIGN.md §7 the 409 is a
223/// valid result, not an error, so both land here.
224#[derive(Debug, Clone, Default, Serialize, Deserialize)]
225#[serde(default)]
226pub struct CancelOutcome {
227    /// The run row id (present on run-scoped cancels and signalled workflow cancels).
228    pub run_id: Option<i64>,
229    /// The workflow id (workflow-scoped cancels only).
230    pub id: Option<i64>,
231    /// `cancel_requested` | `not_running`.
232    pub status: String,
233    /// Run-scoped 409s include the run's actual terminal status here.
234    pub run_status: Option<String>,
235    #[serde(flatten)]
236    pub extra: Extra,
237}
238
239impl CancelOutcome {
240    /// True when a live run was actually signalled.
241    pub fn cancel_requested(&self) -> bool {
242        self.status == "cancel_requested"
243    }
244}
245
246/// One frame of the `GET /v1/runs/:id/events` SSE stream
247/// (`engine/events.rs::RunEvent`, tagged on `"event"`, snake_case).
248///
249/// `Finished` and `Error` are stream-closing. Unrecognized frames (a future daemon
250/// vocabulary) decode as [`RunEvent::Unknown`] instead of failing.
251#[derive(Debug, Clone, PartialEq)]
252pub enum RunEvent {
253    /// The run is registered and about to drive its steps.
254    Started { run_id: i64, total_steps: u64 },
255    /// A step transitioned; `status` ∈ `running|succeeded|failed|skipped` (open set).
256    Step {
257        run_id: i64,
258        index: u64,
259        step_type: String,
260        status: String,
261    },
262    /// Coarse progress hint.
263    Progress {
264        run_id: i64,
265        completed: u64,
266        total: u64,
267    },
268    /// Terminal: final status (`success | failed | cancelled | timeout |
269    /// captcha_required | twofa_required`, open set). Stream-closing.
270    Finished { run_id: i64, status: String },
271    /// Terminal: the run failed around the step loop. Stream-closing.
272    Error { run_id: i64, message: String },
273    /// A frame this SDK version does not recognize (raw payload preserved).
274    Unknown(Value),
275}
276
277/// Private tagged mirror of the daemon's known event vocabulary.
278#[derive(Deserialize)]
279#[serde(tag = "event", rename_all = "snake_case")]
280enum TaggedRunEvent {
281    Started {
282        run_id: i64,
283        total_steps: u64,
284    },
285    Step {
286        run_id: i64,
287        index: u64,
288        step_type: String,
289        status: String,
290    },
291    Progress {
292        run_id: i64,
293        completed: u64,
294        total: u64,
295    },
296    Finished {
297        run_id: i64,
298        status: String,
299    },
300    Error {
301        run_id: i64,
302        message: String,
303    },
304}
305
306impl From<TaggedRunEvent> for RunEvent {
307    fn from(ev: TaggedRunEvent) -> Self {
308        match ev {
309            TaggedRunEvent::Started {
310                run_id,
311                total_steps,
312            } => RunEvent::Started {
313                run_id,
314                total_steps,
315            },
316            TaggedRunEvent::Step {
317                run_id,
318                index,
319                step_type,
320                status,
321            } => RunEvent::Step {
322                run_id,
323                index,
324                step_type,
325                status,
326            },
327            TaggedRunEvent::Progress {
328                run_id,
329                completed,
330                total,
331            } => RunEvent::Progress {
332                run_id,
333                completed,
334                total,
335            },
336            TaggedRunEvent::Finished { run_id, status } => RunEvent::Finished { run_id, status },
337            TaggedRunEvent::Error { run_id, message } => RunEvent::Error { run_id, message },
338        }
339    }
340}
341
342impl<'de> Deserialize<'de> for RunEvent {
343    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
344    where
345        D: serde::Deserializer<'de>,
346    {
347        let value = Value::deserialize(deserializer)?;
348        Ok(
349            match serde_json::from_value::<TaggedRunEvent>(value.clone()) {
350                Ok(known) => known.into(),
351                Err(_) => RunEvent::Unknown(value),
352            },
353        )
354    }
355}
356
357impl RunEvent {
358    /// Parse one SSE `data:` payload. Never fails: unparseable/unknown frames become
359    /// [`RunEvent::Unknown`] (with the raw text wrapped as a JSON string when the
360    /// payload is not even JSON).
361    pub fn parse(data: &str) -> RunEvent {
362        match serde_json::from_str::<RunEvent>(data) {
363            Ok(ev) => ev,
364            Err(_) => RunEvent::Unknown(Value::String(data.to_string())),
365        }
366    }
367
368    /// The run id this event belongs to, when carried.
369    pub fn run_id(&self) -> Option<i64> {
370        match self {
371            RunEvent::Started { run_id, .. }
372            | RunEvent::Step { run_id, .. }
373            | RunEvent::Progress { run_id, .. }
374            | RunEvent::Finished { run_id, .. }
375            | RunEvent::Error { run_id, .. } => Some(*run_id),
376            RunEvent::Unknown(v) => v.get("run_id").and_then(Value::as_i64),
377        }
378    }
379
380    /// True for the two stream-closing variants (`Finished` / `Error`).
381    pub fn is_terminal(&self) -> bool {
382        matches!(self, RunEvent::Finished { .. } | RunEvent::Error { .. })
383    }
384}
385
386/// Final answer of [`crate::resources::Workflows::run_and_wait`].
387#[derive(Debug, Clone)]
388pub struct RunOutcome {
389    /// The final run row (fetched once after the terminal event).
390    pub run: RunFeedItem,
391    /// `runs().results()` payload when `RunOptions::include_results` was set.
392    pub results: Option<RunResults>,
393}
394
395/// A monitor (target) row, enriched with live check state
396/// (`api/v1/monitors.rs::enrich` over `store/targets.rs`).
397#[derive(Debug, Clone, Default, Serialize, Deserialize)]
398#[serde(default)]
399pub struct Monitor {
400    pub id: i64,
401    pub url: Option<String>,
402    pub name: Option<String>,
403    /// `content` | `uptime`.
404    pub check_type: Option<String>,
405    pub enabled: Option<i64>,
406    pub check_period_ms: Option<i64>,
407    pub requires_playwright: Option<i64>,
408    // Live-state enrichment:
409    pub state: Option<String>,
410    pub last_checked_at: Option<String>,
411    pub status_code: Option<i64>,
412    pub is_up: Option<bool>,
413    pub last_change_at: Option<String>,
414    pub state_updated_at: Option<String>,
415    pub changes_count: Option<i64>,
416    pub selector_count: Option<i64>,
417    pub created_at: Option<String>,
418    pub updated_at: Option<String>,
419    #[serde(flatten)]
420    pub extra: Extra,
421}
422
423/// `GET /v1/monitors/:id/changes` — paginated change + uptime history.
424#[derive(Debug, Clone, Default, Serialize, Deserialize)]
425#[serde(default)]
426pub struct MonitorHistory {
427    pub monitor_id: i64,
428    pub limit: Option<i64>,
429    pub offset: Option<i64>,
430    pub has_more: Option<bool>,
431    /// Content-diff history rows, newest first.
432    pub changes: Vec<Value>,
433    /// Up/down + SSL samples, newest first.
434    pub uptime_checks: Vec<Value>,
435    #[serde(flatten)]
436    pub extra: Extra,
437}
438
439/// A content selector under a monitor (`store/target_selectors.rs`).
440#[derive(Debug, Clone, Default, Serialize, Deserialize)]
441#[serde(default)]
442pub struct Selector {
443    pub id: i64,
444    pub target_id: Option<i64>,
445    pub name: Option<String>,
446    pub selector: Option<String>,
447    pub description: Option<String>,
448    pub enabled: Option<i64>,
449    /// `text` | `html` | `visual`.
450    pub content_type: Option<String>,
451    pub ignore_regex: Option<String>,
452    pub priority: Option<i64>,
453    pub created_at: Option<String>,
454    pub updated_at: Option<String>,
455    #[serde(flatten)]
456    pub extra: Extra,
457}
458
459/// A field extractor under a selector (`store/selector_extractors.rs`).
460#[derive(Debug, Clone, Default, Serialize, Deserialize)]
461#[serde(default)]
462pub struct Extractor {
463    pub id: i64,
464    pub target_selector_id: Option<i64>,
465    pub name: Option<String>,
466    pub output_name: Option<String>,
467    pub enabled: Option<i64>,
468    pub extract_type: Option<String>,
469    /// JSON-TEXT on the wire (string) — kept loose.
470    pub config: Option<Value>,
471    pub is_array: Option<i64>,
472    pub default_value: Option<String>,
473    pub created_at: Option<String>,
474    pub updated_at: Option<String>,
475    #[serde(flatten)]
476    pub extra: Extra,
477}
478
479/// An automation (event→action rule) row, JSON-TEXT columns parsed by the daemon
480/// (`api/v1/automations.rs::automation_response`).
481#[derive(Debug, Clone, Default, Serialize, Deserialize)]
482#[serde(default)]
483pub struct Automation {
484    pub id: i64,
485    pub name: String,
486    pub enabled: Option<i64>,
487    pub event_type: Option<String>,
488    pub conditions: Option<Value>,
489    pub actions: Option<Value>,
490    pub blocks: Option<Value>,
491    pub created_at: Option<String>,
492    pub updated_at: Option<String>,
493    #[serde(flatten)]
494    pub extra: Extra,
495}
496
497/// A persona in its redacted wire form (`api/v1/personas.rs::shape`): secrets
498/// collapse to `has_*` booleans + `linked_secrets` names; no `*_encrypted` column
499/// ever appears.
500#[derive(Debug, Clone, Default, Serialize, Deserialize)]
501#[serde(default)]
502pub struct Persona {
503    pub id: i64,
504    pub name: Option<String>,
505    pub description: Option<String>,
506    pub target_domain: Option<String>,
507    pub login_username: Option<String>,
508    pub has_password: Option<bool>,
509    pub twofa_method: Option<String>,
510    pub has_totp_seed: Option<bool>,
511    pub email_otp_mode: Option<String>,
512    pub has_fingerprint: Option<bool>,
513    pub has_proxy: Option<bool>,
514    pub is_active: Option<bool>,
515    pub validation_status: Option<String>,
516    pub has_warm_session: Option<bool>,
517    pub session_expires_at: Option<String>,
518    pub last_login_at: Option<String>,
519    pub last_used_at: Option<String>,
520    pub created_at: Option<String>,
521    pub updated_at: Option<String>,
522    /// `[{id, name}]` workflows pinned to this persona.
523    pub linked_workflows: Option<Vec<Value>>,
524    pub linked_secrets: Option<Value>,
525    #[serde(flatten)]
526    pub extra: Extra,
527}
528
529/// Secret **metadata** (`api/v1/secrets.rs::meta`) — the value is never returned.
530#[derive(Debug, Clone, Default, Serialize, Deserialize)]
531#[serde(default)]
532pub struct SecretMeta {
533    pub id: Option<i64>,
534    /// The unique TEXT key the secret is addressed by.
535    pub key: String,
536    /// Cloud-contract alias of `key`.
537    pub name: Option<String>,
538    pub description: Option<String>,
539    /// e.g. `credentials` | `card` | free-form.
540    pub category: Option<String>,
541    pub is_credential: Option<bool>,
542    pub is_card: Option<bool>,
543    /// Credential secrets only: the (non-secret) username half.
544    pub username: Option<String>,
545    /// Card secrets only: last 4 digits.
546    pub card_last4: Option<String>,
547    pub created_at: Option<String>,
548    pub updated_at: Option<String>,
549    pub last_used_at: Option<String>,
550    pub use_count: Option<i64>,
551    #[serde(flatten)]
552    pub extra: Extra,
553}
554
555/// `GET /v1/vault/status` → `{enabled, locked, idle_timeout_secs}`.
556#[derive(Debug, Clone, Default, Serialize, Deserialize)]
557#[serde(default)]
558pub struct VaultStatus {
559    pub enabled: bool,
560    pub locked: bool,
561    pub idle_timeout_secs: Option<i64>,
562    #[serde(flatten)]
563    pub extra: Extra,
564}
565
566/// A stored-file handle in its OpenAI-style wire form
567/// (`api/v1/files.rs::WireStoredFile`).
568#[derive(Debug, Clone, Default, Serialize, Deserialize)]
569#[serde(default)]
570pub struct StoredFile {
571    /// TEXT id, `file_<hex>`.
572    pub id: String,
573    pub object: Option<String>,
574    pub filename: Option<String>,
575    pub content_type: Option<String>,
576    /// Size in bytes.
577    pub bytes: Option<i64>,
578    /// UNIX epoch seconds.
579    pub created_at: Option<i64>,
580    pub status: Option<String>,
581    /// `upload` | `api` | `workflow_output`.
582    pub source: Option<String>,
583    pub purpose: Option<String>,
584    #[serde(flatten)]
585    pub extra: Extra,
586}
587
588/// A scoped `wlk_` API-key record (`store/local_api_keys.rs`; `key_hash` is never
589/// serialized by the daemon). The plaintext `key` appears **only** in the create
590/// response — capture it immediately.
591#[derive(Debug, Clone, Default, Serialize, Deserialize)]
592#[serde(default)]
593pub struct ApiKey {
594    pub id: i64,
595    pub name: Option<String>,
596    /// `wlk_` + first 6 chars — safe to display.
597    pub prefix: Option<String>,
598    /// CSV of scopes: `read|run|admin`.
599    pub scopes: Option<String>,
600    pub enabled: Option<i64>,
601    pub last_used_at: Option<String>,
602    pub created_at: Option<String>,
603    pub revoked_at: Option<String>,
604    /// The one-time plaintext key (create response only; never recoverable later).
605    pub key: Option<String>,
606    #[serde(flatten)]
607    pub extra: Extra,
608}
609
610/// `POST /v1/ws-ticket` → `{ticket, expires_in_secs}`.
611#[derive(Debug, Clone, Default, Serialize, Deserialize)]
612#[serde(default)]
613pub struct WsTicket {
614    /// Single-use `wtk_…` connect ticket.
615    pub ticket: String,
616    pub expires_in_secs: u64,
617    #[serde(flatten)]
618    pub extra: Extra,
619}
620
621/// A Dragnet crawl-job status view (`api/v1/crawl.rs::to_view` over
622/// `store/crawl_jobs.rs::CrawlJob`). One crawl fans a seed URL across a bounded
623/// in-process worker pool; extracted pages aggregate under the synthetic
624/// [`CrawlJob::data_workflow_id`] workflow, read back through the Data API. As with
625/// monitor rows, the boolean columns arrive as SQLite `0/1` ints and stay typed as
626/// `i64` (not coerced).
627#[derive(Debug, Clone, Default, Serialize, Deserialize)]
628#[serde(default)]
629pub struct CrawlJob {
630    pub id: i64,
631    pub name: String,
632    pub seed_url: String,
633    /// Path-regex allowlist (parsed from JSON-TEXT into an array by the daemon view).
634    pub include_paths: Vec<String>,
635    /// Path-regex denylist.
636    pub exclude_paths: Vec<String>,
637    pub max_depth: i64,
638    /// boolean as `0/1`.
639    pub same_domain: i64,
640    /// boolean as `0/1`.
641    pub allow_subdomains: i64,
642    /// `markdown` | `schema`.
643    pub extract_mode: String,
644    /// The JSON schema object driving `schema` extraction (`null` for markdown).
645    pub extract_schema: Option<Value>,
646    pub persona_id: Option<i64>,
647    /// boolean as `0/1`.
648    pub respect_robots: i64,
649    pub delay_ms: i64,
650    pub max_concurrent: i64,
651    pub page_budget: i64,
652    /// The synthetic per-crawl workflow the extracted pages aggregate under.
653    pub workflow_id: Option<i64>,
654    /// Alias of `workflow_id` the view adds for the Data API
655    /// (`/v1/workflows/{data_workflow_id}/data`).
656    pub data_workflow_id: Option<i64>,
657    pub concierge_session_id: Option<i64>,
658    /// `queued | mapping | crawling | stopping | completed | failed | cancelled`
659    /// (terminal: the last three) — treat as an open string enum.
660    pub status: String,
661    pub pages_discovered: i64,
662    pub pages_done: i64,
663    pub pages_failed: i64,
664    pub pages_skipped: i64,
665    pub workers_active: i64,
666    pub current_depth: i64,
667    pub error: Option<String>,
668    /// boolean as `0/1`.
669    pub cancel_requested: i64,
670    /// Always `"Dragnet"`.
671    pub brand: String,
672    /// Daemon-computed convenience: true for the terminal states.
673    pub is_terminal: bool,
674    pub created_at: String,
675    pub updated_at: Option<String>,
676    pub started_at: Option<String>,
677    pub completed_at: Option<String>,
678    #[serde(flatten)]
679    pub extra: Extra,
680}
681
682/// Body for `POST /v1/crawl` — start a Dragnet whole-site crawl. Only `url` is
683/// required (empty → the daemon `400`s); every unset optional field is **omitted**
684/// from the wire body so the daemon fills its documented default.
685#[derive(Debug, Clone, Default, Serialize)]
686pub struct CrawlStartParams {
687    /// Seed URL to crawl from (required).
688    pub url: String,
689    /// Human label for the crawl (defaults daemon-side).
690    #[serde(skip_serializing_if = "Option::is_none")]
691    pub name: Option<String>,
692    /// `"markdown"` (default) | `"schema"`.
693    #[serde(skip_serializing_if = "Option::is_none")]
694    pub extract_mode: Option<String>,
695    /// JSON schema object driving `schema` extraction.
696    #[serde(skip_serializing_if = "Option::is_none")]
697    pub extract_schema: Option<Value>,
698    /// Persona to crawl as.
699    #[serde(skip_serializing_if = "Option::is_none")]
700    pub persona_id: Option<i64>,
701    /// Path-regex allowlist.
702    #[serde(skip_serializing_if = "Option::is_none")]
703    pub include_paths: Option<Vec<String>>,
704    /// Path-regex denylist.
705    #[serde(skip_serializing_if = "Option::is_none")]
706    pub exclude_paths: Option<Vec<String>>,
707    /// Max link depth from the seed (default 3).
708    #[serde(skip_serializing_if = "Option::is_none")]
709    pub max_depth: Option<i64>,
710    /// Hard cap on pages fetched (default 500).
711    #[serde(skip_serializing_if = "Option::is_none")]
712    pub page_budget: Option<i64>,
713    /// In-process worker cap (default 4).
714    #[serde(skip_serializing_if = "Option::is_none")]
715    pub max_concurrent: Option<i64>,
716    /// Politeness delay between fetches, ms (default 250).
717    #[serde(skip_serializing_if = "Option::is_none")]
718    pub delay_ms: Option<i64>,
719    /// Honor `robots.txt` (default true).
720    #[serde(skip_serializing_if = "Option::is_none")]
721    pub respect_robots: Option<bool>,
722    /// Stay on the seed's domain (default true).
723    #[serde(skip_serializing_if = "Option::is_none")]
724    pub same_domain: Option<bool>,
725    /// Allow subdomains of the seed domain (default true).
726    #[serde(skip_serializing_if = "Option::is_none")]
727    pub allow_subdomains: Option<bool>,
728}
729
730/// `GET /v1/crawl` → `{crawls: [CrawlJob…]}`. **Not** a [`crate::Page`]: this
731/// endpoint answers a named object, mirroring the daemon's other non-envelope list
732/// (`/v1/data`).
733#[derive(Debug, Clone, Default, Serialize, Deserialize)]
734#[serde(default)]
735pub struct CrawlList {
736    pub crawls: Vec<CrawlJob>,
737    #[serde(flatten)]
738    pub extra: Extra,
739}
740
741/// `POST /v1/crawl/:id/cancel` → the refreshed [`CrawlJob`] view plus
742/// `cancel_requested_now` (true iff this call flipped a live crawl to `stopping`;
743/// false when it was already terminal). Never a 409 — always the view.
744#[derive(Debug, Clone, Default, Serialize, Deserialize)]
745#[serde(default)]
746pub struct CrawlCancel {
747    /// The refreshed crawl view.
748    #[serde(flatten)]
749    pub job: CrawlJob,
750    /// True iff this call is the one that requested cancellation.
751    pub cancel_requested_now: bool,
752}
753
754/// One row of `GET /v1/datasets` — a dataset that has accumulated extracted data,
755/// sourced from either a crawl or a workflow.
756#[derive(Debug, Clone, Default, Serialize, Deserialize)]
757#[serde(default)]
758pub struct Dataset {
759    pub id: i64,
760    pub name: String,
761    /// `"crawl"` | `"workflow"` — the dataset's origin lane.
762    pub source_type: String,
763    pub run_count: i64,
764    pub last_updated: Option<String>,
765    pub origin: Option<String>,
766    #[serde(flatten)]
767    pub extra: Extra,
768}
769
770/// `GET /v1/datasets` → `{datasets: [Dataset…]}`. **Not** a [`crate::Page`]: like
771/// [`CrawlList`], this endpoint answers a named object rather than the list
772/// envelope (unwrap its `datasets` field).
773#[derive(Debug, Clone, Default, Serialize, Deserialize)]
774#[serde(default)]
775pub struct DatasetList {
776    pub datasets: Vec<Dataset>,
777    #[serde(flatten)]
778    pub extra: Extra,
779}
780
781/// `GET /v1/datasets/:id` → one dataset's metadata + schema. `columns`/`facets`
782/// are query-engine-driven, so they stay loosely-typed [`Value`]s.
783#[derive(Debug, Clone, Default, Serialize, Deserialize)]
784#[serde(default)]
785pub struct DatasetMeta {
786    pub id: i64,
787    pub name: String,
788    /// `"crawl"` | `"workflow"`.
789    pub source_type: String,
790    /// Column descriptors (shape driven by the query engine).
791    pub columns: Value,
792    /// Per-column facet values.
793    pub facets: Value,
794    pub row_count: i64,
795    pub run_count: i64,
796    pub truncated: bool,
797    #[serde(flatten)]
798    pub extra: Extra,
799}
800
801/// The dataset a search hit belongs to — the identifying subset the search
802/// endpoints echo back per result.
803#[derive(Debug, Clone, Default, Serialize, Deserialize)]
804#[serde(default)]
805pub struct DatasetRef {
806    pub id: i64,
807    pub name: Option<String>,
808    /// `"crawl"` | `"workflow"`.
809    pub source_type: String,
810    #[serde(flatten)]
811    pub extra: Extra,
812}
813
814/// One hit from `GET /v1/datasets/search` or `GET /v1/datasets/:id/search`.
815/// `fields`/`highlight` are query-engine-driven (dynamic columns), so they stay
816/// loosely-typed [`Value`]s.
817#[derive(Debug, Clone, Default, Serialize, Deserialize)]
818#[serde(default)]
819pub struct DatasetSearchHit {
820    pub dataset: DatasetRef,
821    pub run_id: Option<i64>,
822    pub run_at: Option<String>,
823    /// The matched row's fields (shape driven by the query engine).
824    pub fields: Value,
825    /// Per-field highlight fragments (shape driven by the query engine).
826    pub highlight: Value,
827    #[serde(flatten)]
828    pub extra: Extra,
829}
830
831/// Output shape for a dataset read (the `?format=` query param).
832///
833/// `Json` is the documented envelope. The rest render TEXT and are CONTENT-AWARE:
834/// a dataset whose records carry long-form content (a crawl's pages have
835/// `markdown`) renders as documents, anything else as a table. Because they are
836/// not JSON they are served by the `*_text` methods on [`crate::resources::Datasets`].
837#[derive(Debug, Clone, Copy, PartialEq, Eq)]
838pub enum DatasetFormat {
839    /// The documented JSON envelope (the API default).
840    Json,
841    /// Comma-separated table.
842    Csv,
843    /// Readable prose — documents for a crawl, a table otherwise.
844    Markdown,
845    /// A standalone HTML document (meant to be saved/viewed, not parsed).
846    Html,
847}
848
849impl DatasetFormat {
850    /// The wire value for the `format` query param.
851    pub fn as_str(self) -> &'static str {
852        match self {
853            DatasetFormat::Json => "json",
854            DatasetFormat::Csv => "csv",
855            DatasetFormat::Markdown => "markdown",
856            DatasetFormat::Html => "html",
857        }
858    }
859}
860
861/// `GET /v1/datasets/search` and `GET /v1/datasets/:id/search` → full-text search
862/// results across the unified dataset index.
863#[derive(Debug, Clone, Default, Serialize, Deserialize)]
864#[serde(default)]
865pub struct DatasetSearchResult {
866    pub query: String,
867    pub terms: Vec<String>,
868    pub results: Vec<DatasetSearchHit>,
869    pub total: i64,
870    pub truncated: bool,
871    pub scanned_runs: i64,
872    #[serde(flatten)]
873    pub extra: Extra,
874}
875
876#[cfg(test)]
877mod tests {
878    use super::*;
879    use serde_json::json;
880
881    #[test]
882    fn run_feed_item_row_id_parses_composite() {
883        let item: RunFeedItem =
884            serde_json::from_value(json!({"id": "workflow-3", "status": "success"})).unwrap();
885        assert_eq!(item.row_id(), Some(3));
886        let item: RunFeedItem =
887            serde_json::from_value(json!({"id": "check-142", "status": "running"})).unwrap();
888        assert_eq!(item.row_id(), Some(142));
889        assert!(item.is_running());
890        let bare: RunFeedItem =
891            serde_json::from_value(json!({"id": "7", "status": "success"})).unwrap();
892        assert_eq!(bare.row_id(), Some(7));
893    }
894
895    #[test]
896    fn run_event_parses_known_and_unknown() {
897        let ev = RunEvent::parse(r#"{"event":"started","run_id":9,"total_steps":4}"#);
898        assert_eq!(
899            ev,
900            RunEvent::Started {
901                run_id: 9,
902                total_steps: 4
903            }
904        );
905        assert!(!ev.is_terminal());
906
907        let ev = RunEvent::parse(
908            r#"{"event":"step","run_id":9,"index":1,"step_type":"click","status":"succeeded"}"#,
909        );
910        assert_eq!(ev.run_id(), Some(9));
911
912        let ev = RunEvent::parse(r#"{"event":"finished","run_id":9,"status":"success"}"#);
913        assert!(ev.is_terminal());
914
915        let ev = RunEvent::parse(r#"{"event":"error","run_id":9,"message":"navigation failed"}"#);
916        assert!(ev.is_terminal());
917
918        // Future vocabulary degrades to Unknown, keeping run_id readable.
919        let ev = RunEvent::parse(r#"{"event":"warp","run_id":9,"factor":5}"#);
920        assert!(matches!(ev, RunEvent::Unknown(_)));
921        assert_eq!(ev.run_id(), Some(9));
922        assert!(!ev.is_terminal());
923
924        // Non-JSON payload wraps the raw text.
925        let ev = RunEvent::parse("not json");
926        assert_eq!(ev, RunEvent::Unknown(Value::String("not json".into())));
927    }
928
929    #[test]
930    fn crawl_cancel_flattens_job_and_splits_cancel_flag() {
931        // The cancel view is the CrawlJob fields plus a sibling `cancel_requested_now`;
932        // the flag must land on the outer struct, not get swallowed by CrawlJob.extra.
933        let c: CrawlCancel = serde_json::from_value(json!({
934            "id": 5, "name": "Dragnet: example.com", "seed_url": "https://example.com",
935            "include_paths": ["^/docs"], "exclude_paths": [], "status": "stopping",
936            "brand": "Dragnet", "is_terminal": false, "workflow_id": 77,
937            "data_workflow_id": 77, "cancel_requested_now": true, "some_future": 1
938        }))
939        .unwrap();
940        assert!(c.cancel_requested_now);
941        assert_eq!(c.job.id, 5);
942        assert_eq!(c.job.status, "stopping");
943        assert_eq!(c.job.brand, "Dragnet");
944        assert_eq!(c.job.data_workflow_id, Some(77));
945        assert_eq!(c.job.include_paths, vec!["^/docs".to_string()]);
946        // Unknown fields still land in CrawlJob.extra, and the flag is NOT among them.
947        assert_eq!(c.job.extra["some_future"], 1);
948        assert!(!c.job.extra.contains_key("cancel_requested_now"));
949    }
950
951    #[test]
952    fn crawl_start_params_omit_unset_fields() {
953        let body = serde_json::to_value(CrawlStartParams {
954            url: "https://example.com".into(),
955            max_depth: Some(2),
956            respect_robots: Some(true),
957            ..Default::default()
958        })
959        .unwrap();
960        assert_eq!(body["url"], "https://example.com");
961        assert_eq!(body["max_depth"], 2);
962        assert_eq!(body["respect_robots"], true);
963        // Unset optionals are omitted, not sent as null.
964        assert!(body.get("name").is_none());
965        assert!(body.get("persona_id").is_none());
966        assert!(body.get("page_budget").is_none());
967        assert!(body.get("include_paths").is_none());
968    }
969
970    #[test]
971    fn workflow_unknown_fields_land_in_extra() {
972        let wf: Workflow = serde_json::from_value(json!({
973            "id": 5, "name": "scrape", "steps": [], "some_future_field": {"x": 1}
974        }))
975        .unwrap();
976        assert_eq!(wf.id, 5);
977        assert_eq!(wf.extra["some_future_field"]["x"], 1);
978    }
979}
writ_client/models.rs

writ_client/
models.rs
