pagesift_api/types/
mod.rs

1//! Firecrawl v1/v2 request and response types.
2
3pub mod request;
4pub mod response;
5
6pub use request::{
7    AdaptiveOptions, BatchScrapeRequest, CrawlRequest, ExtractRequest, Format, Location,
8    MapRequest, QueryFilter, ScrapeOptions, ScrapeRequest, SearchRequest,
9};
10pub use response::{
11    ChangeTracking, CrawlSummary, DocChunk, Document, JobError, JobErrors, JobStarted, JobStatus,
12    MapLink, MapResponse, Metadata, PagesiftMeta, QueryFilterStats, ScrapeResponse,
13};
14
15#[cfg(test)]
16mod tests {
17    use super::*;
18    use serde_json::json;
19
20    #[test]
21    fn formats_accept_strings_and_objects() {
22        let r: ScrapeRequest = serde_json::from_value(json!({
23            "url": "https://example.com",
24            "formats": ["markdown", {"type": "screenshot", "fullPage": true}],
25        }))
26        .unwrap();
27        assert_eq!(r.options.formats[0], Format::new("markdown"));
28        assert_eq!(r.options.formats[1].kind, "screenshot");
29        assert_eq!(r.options.formats[1].options["fullPage"], json!(true));
30    }
31
32    #[test]
33    fn defaults_follow_firecrawl_and_unknown_fields_are_ignored() {
34        let r: ScrapeRequest = serde_json::from_value(json!({
35            "url": "https://example.com",
36            "origin": "python-sdk",
37            "integration": null,
38            "zeroDataRetention": false,
39        }))
40        .unwrap();
41        let o = r.options;
42        assert_eq!(o.formats, vec![Format::new("markdown")]);
43        assert!(o.only_main_content && o.block_ads && o.store_in_cache && o.remove_base64_images);
44        assert_eq!(o.wait_for, 0);
45        assert!(!o.ignore_robots_txt);
46    }
47
48    #[test]
49    fn crawl_and_map_defaults() {
50        let c: CrawlRequest =
51            serde_json::from_value(json!({"url": "https://example.com"})).unwrap();
52        assert_eq!((c.limit, c.sitemap.as_str()), (10_000, "include"));
53        assert!(c.scrape_options.only_main_content);
54        let m: MapRequest = serde_json::from_value(json!({"url": "https://example.com"})).unwrap();
55        assert_eq!(m.limit, 5_000);
56        assert!(m.include_subdomains && m.ignore_query_parameters);
57    }
58
59    #[test]
60    fn query_filter_accepts_string_or_object() {
61        let r: ScrapeRequest = serde_json::from_value(json!({
62            "url": "https://example.com", "queryFilter": "refund policy",
63        }))
64        .unwrap();
65        let f = r.options.query_filter.unwrap();
66        assert_eq!(
67            (
68                f.query.as_str(),
69                f.threshold,
70                f.relative,
71                f.top_k,
72                f.headings
73            ),
74            ("refund policy", None, Some(0.5), None, true)
75        );
76        let r: ScrapeRequest = serde_json::from_value(json!({
77            "url": "https://example.com",
78            "queryFilter": {"query": "q", "threshold": 0.5, "topK": 3, "headings": false},
79        }))
80        .unwrap();
81        let f = r.options.query_filter.unwrap();
82        assert_eq!(
83            (f.threshold, f.relative, f.top_k, f.headings),
84            (Some(0.5), None, Some(3), false)
85        );
86        let r: ScrapeRequest = serde_json::from_value(json!({
87            "url": "https://example.com",
88            "queryFilter": {"query": "q", "maxTokens": 800},
89        }))
90        .unwrap();
91        let f = r.options.query_filter.unwrap();
92        assert_eq!((f.relative, f.max_tokens), (Some(0.5), Some(800)));
93        assert!(ScrapeOptions::default().query_filter.is_none());
94    }
95
96    #[test]
97    fn crawl_query_and_adaptive_options() {
98        let c: CrawlRequest =
99            serde_json::from_value(json!({"url": "https://example.com"})).unwrap();
100        assert_eq!((c.query, c.adaptive), (None, None));
101        let c: CrawlRequest = serde_json::from_value(json!({
102            "url": "https://example.com", "query": ["refund", "policy"], "adaptive": true,
103        }))
104        .unwrap();
105        assert_eq!(c.query.as_deref(), Some("refund policy"));
106        assert_eq!(c.adaptive, Some(AdaptiveOptions::default()));
107        let c: CrawlRequest = serde_json::from_value(json!({
108            "url": "https://example.com", "query": "  ", "adaptive": {"patience": 2},
109        }))
110        .unwrap();
111        assert_eq!(c.query, None, "blank query is no query");
112        assert_eq!(c.adaptive.unwrap().patience, 2);
113        let c: CrawlRequest = serde_json::from_value(json!({
114            "url": "https://example.com", "query": null, "adaptive": false,
115        }))
116        .unwrap();
117        assert_eq!((c.query, c.adaptive), (None, None));
118    }
119
120    #[test]
121    fn change_tracking_serializes_firecrawl_fields() {
122        let doc = Document {
123            change_tracking: Some(ChangeTracking {
124                previous_scrape_at: None,
125                change_status: "new".into(),
126                visibility: "visible".into(),
127                diff: None,
128                json: None,
129                pagesift: None,
130            }),
131            ..Default::default()
132        };
133        let v = serde_json::to_value(&doc).unwrap();
134        assert_eq!(
135            v["changeTracking"],
136            json!({"previousScrapeAt": null, "changeStatus": "new", "visibility": "visible"})
137        );
138        assert!(
139            serde_json::to_value(Document::default())
140                .unwrap()
141                .get("changeTracking")
142                .is_none()
143        );
144    }
145
146    #[test]
147    fn metadata_serializes_firecrawl_names_and_extension() {
148        let mut meta = Metadata {
149            source_url: Some("https://example.com/a".into()),
150            status_code: 200,
151            proxy_used: "basic".into(),
152            pagesift: Some(PagesiftMeta {
153                content_hash: Some("ab".into()),
154                ..Default::default()
155            }),
156            ..Default::default()
157        };
158        meta.extra
159            .insert("ogImage".into(), json!("https://example.com/i.png"));
160        let v = serde_json::to_value(&meta).unwrap();
161        assert_eq!(v["sourceURL"], "https://example.com/a");
162        assert_eq!(v["statusCode"], 200);
163        assert_eq!(v["proxyUsed"], "basic");
164        assert_eq!(v["ogImage"], "https://example.com/i.png");
165        assert_eq!(v["pagesift"]["contentHash"], "ab");
166        assert!(v.get("title").is_none());
167    }
168}
pagesift_api/types/mod.rs

pagesift_api/types/
mod.rs
