pagesift_api/types/
mod.rs

1//! Firecrawl v1/v2 request and response types.
2
3pub mod parse;
4pub mod request;
5pub mod response;
6
7pub use parse::{PdfParser, Upload, UploadKind};
8
9pub use request::{
10    AdaptiveOptions, BatchScrapeRequest, CrawlRequest, CrawlScope, ExtractRequest, Format,
11    Location, MapRequest, QueryFilter, ScrapeOptions, ScrapeRequest, SearchRequest,
12};
13pub use response::{
14    ChangeTracking, CrawlSummary, DocChunk, Document, JobError, JobErrors, JobStarted, JobStatus,
15    MapLink, MapResponse, Metadata, PageMarkdown, PagesiftMeta, QueryFilterStats, ScrapeResponse,
16};
17
18#[cfg(test)]
19mod tests {
20    use super::*;
21    use serde_json::json;
22
23    #[test]
24    fn formats_accept_strings_and_objects() {
25        let r: ScrapeRequest = serde_json::from_value(json!({
26            "url": "https://example.com",
27            "formats": ["markdown", {"type": "screenshot", "fullPage": true}],
28        }))
29        .unwrap();
30        assert_eq!(r.options.formats[0], Format::new("markdown"));
31        assert_eq!(r.options.formats[1].kind, "screenshot");
32        assert_eq!(r.options.formats[1].options["fullPage"], json!(true));
33    }
34
35    #[test]
36    fn defaults_follow_firecrawl_and_unknown_fields_are_ignored() {
37        let r: ScrapeRequest = serde_json::from_value(json!({
38            "url": "https://example.com",
39            "origin": "python-sdk",
40            "integration": null,
41            "zeroDataRetention": false,
42        }))
43        .unwrap();
44        let o = r.options;
45        assert_eq!(o.formats, vec![Format::new("markdown")]);
46        assert!(o.only_main_content && o.block_ads && o.store_in_cache && o.remove_base64_images);
47        assert_eq!(o.wait_for, 0);
48        assert!(!o.ignore_robots_txt);
49    }
50
51    #[test]
52    fn crawl_and_map_defaults() {
53        let c: CrawlRequest =
54            serde_json::from_value(json!({"url": "https://example.com"})).unwrap();
55        assert_eq!((c.limit, c.sitemap.as_str()), (10_000, "include"));
56        assert!(c.scrape_options.only_main_content);
57        let m: MapRequest = serde_json::from_value(json!({"url": "https://example.com"})).unwrap();
58        assert_eq!(m.limit, 5_000);
59        assert!(m.include_subdomains && m.ignore_query_parameters);
60    }
61
62    #[test]
63    fn query_filter_accepts_string_or_object() {
64        let r: ScrapeRequest = serde_json::from_value(json!({
65            "url": "https://example.com", "queryFilter": "refund policy",
66        }))
67        .unwrap();
68        let f = r.options.query_filter.unwrap();
69        assert_eq!(
70            (
71                f.query.as_str(),
72                f.threshold,
73                f.relative,
74                f.top_k,
75                f.headings
76            ),
77            ("refund policy", None, Some(0.5), None, true)
78        );
79        let r: ScrapeRequest = serde_json::from_value(json!({
80            "url": "https://example.com",
81            "queryFilter": {"query": "q", "threshold": 0.5, "topK": 3, "headings": false},
82        }))
83        .unwrap();
84        let f = r.options.query_filter.unwrap();
85        assert_eq!(
86            (f.threshold, f.relative, f.top_k, f.headings),
87            (Some(0.5), None, Some(3), false)
88        );
89        let r: ScrapeRequest = serde_json::from_value(json!({
90            "url": "https://example.com",
91            "queryFilter": {"query": "q", "maxTokens": 800},
92        }))
93        .unwrap();
94        let f = r.options.query_filter.unwrap();
95        assert_eq!((f.relative, f.max_tokens), (Some(0.5), Some(800)));
96        assert!(ScrapeOptions::default().query_filter.is_none());
97    }
98
99    #[test]
100    fn crawl_query_and_adaptive_options() {
101        let c: CrawlRequest =
102            serde_json::from_value(json!({"url": "https://example.com"})).unwrap();
103        assert_eq!((c.query, c.adaptive), (None, None));
104        let c: CrawlRequest = serde_json::from_value(json!({
105            "url": "https://example.com", "query": ["refund", "policy"], "adaptive": true,
106        }))
107        .unwrap();
108        assert_eq!(c.query.as_deref(), Some("refund policy"));
109        assert_eq!(c.adaptive, Some(AdaptiveOptions::default()));
110        let c: CrawlRequest = serde_json::from_value(json!({
111            "url": "https://example.com", "query": "  ", "adaptive": {"patience": 2},
112        }))
113        .unwrap();
114        assert_eq!(c.query, None, "blank query is no query");
115        assert_eq!(c.adaptive.unwrap().patience, 2);
116        let c: CrawlRequest = serde_json::from_value(json!({
117            "url": "https://example.com", "query": null, "adaptive": false,
118        }))
119        .unwrap();
120        assert_eq!((c.query, c.adaptive), (None, None));
121    }
122
123    #[test]
124    fn change_tracking_serializes_firecrawl_fields() {
125        let doc = Document {
126            change_tracking: Some(ChangeTracking {
127                previous_scrape_at: None,
128                change_status: "new".into(),
129                visibility: "visible".into(),
130                diff: None,
131                json: None,
132                pagesift: None,
133            }),
134            ..Default::default()
135        };
136        let v = serde_json::to_value(&doc).unwrap();
137        assert_eq!(
138            v["changeTracking"],
139            json!({"previousScrapeAt": null, "changeStatus": "new", "visibility": "visible"})
140        );
141        assert!(
142            serde_json::to_value(Document::default())
143                .unwrap()
144                .get("changeTracking")
145                .is_none()
146        );
147    }
148
149    #[test]
150    fn metadata_serializes_firecrawl_names_and_extension() {
151        let mut meta = Metadata {
152            source_url: Some("https://example.com/a".into()),
153            status_code: 200,
154            proxy_used: "basic".into(),
155            pagesift: Some(PagesiftMeta {
156                content_hash: Some("ab".into()),
157                ..Default::default()
158            }),
159            ..Default::default()
160        };
161        meta.extra
162            .insert("ogImage".into(), json!("https://example.com/i.png"));
163        let v = serde_json::to_value(&meta).unwrap();
164        assert_eq!(v["sourceURL"], "https://example.com/a");
165        assert_eq!(v["statusCode"], 200);
166        assert_eq!(v["proxyUsed"], "basic");
167        assert_eq!(v["ogImage"], "https://example.com/i.png");
168        assert_eq!(v["pagesift"]["contentHash"], "ab");
169        assert!(v.get("title").is_none());
170    }
171}
pagesift_api/types/mod.rs

pagesift_api/types/
mod.rs
