pagesift_api/types/
mod.rs

1//! Firecrawl v1/v2 request and response types.
2
3pub mod agent;
4pub mod attachment;
5pub mod interact;
6pub mod locate;
7pub mod parse;
8pub mod request;
9pub mod response;
10
11pub use agent::{
12    AgentBudget, AgentEffort, AgentEvidence, AgentFailure, AgentOutcome, AgentRequest, AgentSource,
13    AgentStep, AgentUsage, FieldSource,
14};
15pub use attachment::Attachment;
16pub use interact::{
17    InteractRequest, InteractResponse, InteractStopped, LIVE_VIEW_UNAVAILABLE, SESSION_REPLAYED,
18    instance_tag, is_scrape_id, issued_elsewhere, scrape_id, session_gone,
19};
20pub use locate::{
21    ObserveRequest, ObserveResponse, Observed, SimilarData, SimilarItem, SimilarRequest,
22    SimilarResponse,
23};
24pub use parse::{PdfParser, Upload, UploadKind};
25
26pub use request::{
27    AdaptiveOptions, BatchScrapeRequest, CrawlRequest, CrawlScope, ExtractRequest, Format,
28    ImageAssetMode, Location, MapRequest, QueryFilter, RESPOND_TIMINGS, ScrapeOptions,
29    ScrapeRequest, SearchRequest,
30};
31pub use response::{
32    ChangeTracking, CrawlSummary, CrawlTrace, DocChunk, Document, ImageAssetResult,
33    ImageAssetStatus, JobError, JobErrors, JobStarted, JobStatus, MapLink, MapResponse, Metadata,
34    OcrMeta, PageMarkdown, PagesiftMeta, PlanStep, PlanTrace, QueryFilterStats, ScrapeResponse,
35};
36
37#[cfg(test)]
38mod tests {
39    use super::*;
40    use serde_json::json;
41
42    #[test]
43    fn formats_accept_strings_and_objects() {
44        let r: ScrapeRequest = serde_json::from_value(json!({
45            "url": "https://example.com",
46            "formats": ["markdown", {"type": "screenshot", "fullPage": true}],
47        }))
48        .unwrap();
49        assert_eq!(r.options.formats[0], Format::new("markdown"));
50        assert_eq!(r.options.formats[1].kind, "screenshot");
51        assert_eq!(r.options.formats[1].options["fullPage"], json!(true));
52    }
53
54    #[test]
55    fn defaults_follow_firecrawl_and_unknown_fields_are_ignored() {
56        let r: ScrapeRequest = serde_json::from_value(json!({
57            "url": "https://example.com",
58            "origin": "python-sdk",
59            "integration": null,
60            "zeroDataRetention": false,
61        }))
62        .unwrap();
63        let o = r.options;
64        assert_eq!(o.formats, vec![Format::new("markdown")]);
65        assert!(o.only_main_content && o.block_ads && o.store_in_cache && o.remove_base64_images);
66        assert_eq!(o.wait_for, 0);
67        assert!(!o.ignore_robots_txt);
68    }
69
70    #[test]
71    fn crawl_and_map_defaults() {
72        let c: CrawlRequest =
73            serde_json::from_value(json!({"url": "https://example.com"})).unwrap();
74        assert_eq!((c.limit, c.sitemap.as_str()), (10_000, "include"));
75        assert!(c.scrape_options.only_main_content);
76        let m: MapRequest = serde_json::from_value(json!({"url": "https://example.com"})).unwrap();
77        assert_eq!(m.limit, 5_000);
78        assert!(m.include_subdomains && m.ignore_query_parameters);
79    }
80
81    #[test]
82    fn query_filter_accepts_string_or_object() {
83        let r: ScrapeRequest = serde_json::from_value(json!({
84            "url": "https://example.com", "queryFilter": "refund policy",
85        }))
86        .unwrap();
87        let f = r.options.query_filter.unwrap();
88        assert_eq!(
89            (
90                f.query.as_str(),
91                f.threshold,
92                f.relative,
93                f.top_k,
94                f.headings
95            ),
96            ("refund policy", None, Some(0.5), None, true)
97        );
98        let r: ScrapeRequest = serde_json::from_value(json!({
99            "url": "https://example.com",
100            "queryFilter": {"query": "q", "threshold": 0.5, "topK": 3, "headings": false},
101        }))
102        .unwrap();
103        let f = r.options.query_filter.unwrap();
104        assert_eq!(
105            (f.threshold, f.relative, f.top_k, f.headings),
106            (Some(0.5), None, Some(3), false)
107        );
108        let r: ScrapeRequest = serde_json::from_value(json!({
109            "url": "https://example.com",
110            "queryFilter": {"query": "q", "maxTokens": 800},
111        }))
112        .unwrap();
113        let f = r.options.query_filter.unwrap();
114        assert_eq!((f.relative, f.max_tokens), (Some(0.5), Some(800)));
115        assert!(ScrapeOptions::default().query_filter.is_none());
116    }
117
118    #[test]
119    fn crawl_query_and_adaptive_options() {
120        let c: CrawlRequest =
121            serde_json::from_value(json!({"url": "https://example.com"})).unwrap();
122        assert_eq!((c.query, c.adaptive), (None, None));
123        let c: CrawlRequest = serde_json::from_value(json!({
124            "url": "https://example.com", "query": ["refund", "policy"], "adaptive": true,
125        }))
126        .unwrap();
127        assert_eq!(c.query.as_deref(), Some("refund policy"));
128        assert_eq!(c.adaptive, Some(AdaptiveOptions::default()));
129        let c: CrawlRequest = serde_json::from_value(json!({
130            "url": "https://example.com", "query": "  ", "adaptive": {"patience": 2},
131        }))
132        .unwrap();
133        assert_eq!(c.query, None, "blank query is no query");
134        assert_eq!(c.adaptive.unwrap().patience, 2);
135        let c: CrawlRequest = serde_json::from_value(json!({
136            "url": "https://example.com", "query": null, "adaptive": false,
137        }))
138        .unwrap();
139        assert_eq!((c.query, c.adaptive), (None, None));
140    }
141
142    #[test]
143    fn change_tracking_serializes_firecrawl_fields() {
144        let doc = Document {
145            change_tracking: Some(ChangeTracking {
146                previous_scrape_at: None,
147                change_status: "new".into(),
148                visibility: "visible".into(),
149                diff: None,
150                json: None,
151                pagesift: None,
152            }),
153            ..Default::default()
154        };
155        let v = serde_json::to_value(&doc).unwrap();
156        assert_eq!(
157            v["changeTracking"],
158            json!({"previousScrapeAt": null, "changeStatus": "new", "visibility": "visible"})
159        );
160        assert!(
161            serde_json::to_value(Document::default())
162                .unwrap()
163                .get("changeTracking")
164                .is_none()
165        );
166    }
167
168    #[test]
169    fn metadata_serializes_firecrawl_names_and_extension() {
170        let mut meta = Metadata {
171            source_url: Some("https://example.com/a".into()),
172            status_code: 200,
173            proxy_used: "basic".into(),
174            pagesift: Some(PagesiftMeta {
175                content_hash: Some("ab".into()),
176                ..Default::default()
177            }),
178            ..Default::default()
179        };
180        meta.extra
181            .insert("ogImage".into(), json!("https://example.com/i.png"));
182        let v = serde_json::to_value(&meta).unwrap();
183        assert_eq!(v["sourceURL"], "https://example.com/a");
184        assert_eq!(v["statusCode"], 200);
185        assert_eq!(v["proxyUsed"], "basic");
186        assert_eq!(v["ogImage"], "https://example.com/i.png");
187        assert_eq!(v["pagesift"]["contentHash"], "ab");
188        assert!(v.get("title").is_none());
189    }
190}
191mod asset_cache;
192pub use asset_cache::AssetFetchCache;
193mod progress;
194pub use progress::{MarkdownPreview, MarkdownProgress};
pagesift_api/types/mod.rs

pagesift_api/types/
mod.rs
