pagesift_api/types/
mod.rs

1//! Firecrawl v1/v2 request and response types.
2
3pub mod attachment;
4pub mod parse;
5pub mod request;
6pub mod response;
7
8pub use attachment::Attachment;
9pub use parse::{PdfParser, Upload, UploadKind};
10
11pub use request::{
12    AdaptiveOptions, BatchScrapeRequest, CrawlRequest, CrawlScope, ExtractRequest, Format,
13    Location, MapRequest, QueryFilter, ScrapeOptions, ScrapeRequest, SearchRequest,
14};
15pub use response::{
16    ChangeTracking, CrawlSummary, CrawlTrace, DocChunk, Document, JobError, JobErrors, JobStarted,
17    JobStatus, MapLink, MapResponse, Metadata, PageMarkdown, PagesiftMeta, PlanStep, PlanTrace,
18    QueryFilterStats, ScrapeResponse,
19};
20
21#[cfg(test)]
22mod tests {
23    use super::*;
24    use serde_json::json;
25
26    #[test]
27    fn formats_accept_strings_and_objects() {
28        let r: ScrapeRequest = serde_json::from_value(json!({
29            "url": "https://example.com",
30            "formats": ["markdown", {"type": "screenshot", "fullPage": true}],
31        }))
32        .unwrap();
33        assert_eq!(r.options.formats[0], Format::new("markdown"));
34        assert_eq!(r.options.formats[1].kind, "screenshot");
35        assert_eq!(r.options.formats[1].options["fullPage"], json!(true));
36    }
37
38    #[test]
39    fn defaults_follow_firecrawl_and_unknown_fields_are_ignored() {
40        let r: ScrapeRequest = serde_json::from_value(json!({
41            "url": "https://example.com",
42            "origin": "python-sdk",
43            "integration": null,
44            "zeroDataRetention": false,
45        }))
46        .unwrap();
47        let o = r.options;
48        assert_eq!(o.formats, vec![Format::new("markdown")]);
49        assert!(o.only_main_content && o.block_ads && o.store_in_cache && o.remove_base64_images);
50        assert_eq!(o.wait_for, 0);
51        assert!(!o.ignore_robots_txt);
52    }
53
54    #[test]
55    fn crawl_and_map_defaults() {
56        let c: CrawlRequest =
57            serde_json::from_value(json!({"url": "https://example.com"})).unwrap();
58        assert_eq!((c.limit, c.sitemap.as_str()), (10_000, "include"));
59        assert!(c.scrape_options.only_main_content);
60        let m: MapRequest = serde_json::from_value(json!({"url": "https://example.com"})).unwrap();
61        assert_eq!(m.limit, 5_000);
62        assert!(m.include_subdomains && m.ignore_query_parameters);
63    }
64
65    #[test]
66    fn query_filter_accepts_string_or_object() {
67        let r: ScrapeRequest = serde_json::from_value(json!({
68            "url": "https://example.com", "queryFilter": "refund policy",
69        }))
70        .unwrap();
71        let f = r.options.query_filter.unwrap();
72        assert_eq!(
73            (
74                f.query.as_str(),
75                f.threshold,
76                f.relative,
77                f.top_k,
78                f.headings
79            ),
80            ("refund policy", None, Some(0.5), None, true)
81        );
82        let r: ScrapeRequest = serde_json::from_value(json!({
83            "url": "https://example.com",
84            "queryFilter": {"query": "q", "threshold": 0.5, "topK": 3, "headings": false},
85        }))
86        .unwrap();
87        let f = r.options.query_filter.unwrap();
88        assert_eq!(
89            (f.threshold, f.relative, f.top_k, f.headings),
90            (Some(0.5), None, Some(3), false)
91        );
92        let r: ScrapeRequest = serde_json::from_value(json!({
93            "url": "https://example.com",
94            "queryFilter": {"query": "q", "maxTokens": 800},
95        }))
96        .unwrap();
97        let f = r.options.query_filter.unwrap();
98        assert_eq!((f.relative, f.max_tokens), (Some(0.5), Some(800)));
99        assert!(ScrapeOptions::default().query_filter.is_none());
100    }
101
102    #[test]
103    fn crawl_query_and_adaptive_options() {
104        let c: CrawlRequest =
105            serde_json::from_value(json!({"url": "https://example.com"})).unwrap();
106        assert_eq!((c.query, c.adaptive), (None, None));
107        let c: CrawlRequest = serde_json::from_value(json!({
108            "url": "https://example.com", "query": ["refund", "policy"], "adaptive": true,
109        }))
110        .unwrap();
111        assert_eq!(c.query.as_deref(), Some("refund policy"));
112        assert_eq!(c.adaptive, Some(AdaptiveOptions::default()));
113        let c: CrawlRequest = serde_json::from_value(json!({
114            "url": "https://example.com", "query": "  ", "adaptive": {"patience": 2},
115        }))
116        .unwrap();
117        assert_eq!(c.query, None, "blank query is no query");
118        assert_eq!(c.adaptive.unwrap().patience, 2);
119        let c: CrawlRequest = serde_json::from_value(json!({
120            "url": "https://example.com", "query": null, "adaptive": false,
121        }))
122        .unwrap();
123        assert_eq!((c.query, c.adaptive), (None, None));
124    }
125
126    #[test]
127    fn change_tracking_serializes_firecrawl_fields() {
128        let doc = Document {
129            change_tracking: Some(ChangeTracking {
130                previous_scrape_at: None,
131                change_status: "new".into(),
132                visibility: "visible".into(),
133                diff: None,
134                json: None,
135                pagesift: None,
136            }),
137            ..Default::default()
138        };
139        let v = serde_json::to_value(&doc).unwrap();
140        assert_eq!(
141            v["changeTracking"],
142            json!({"previousScrapeAt": null, "changeStatus": "new", "visibility": "visible"})
143        );
144        assert!(
145            serde_json::to_value(Document::default())
146                .unwrap()
147                .get("changeTracking")
148                .is_none()
149        );
150    }
151
152    #[test]
153    fn metadata_serializes_firecrawl_names_and_extension() {
154        let mut meta = Metadata {
155            source_url: Some("https://example.com/a".into()),
156            status_code: 200,
157            proxy_used: "basic".into(),
158            pagesift: Some(PagesiftMeta {
159                content_hash: Some("ab".into()),
160                ..Default::default()
161            }),
162            ..Default::default()
163        };
164        meta.extra
165            .insert("ogImage".into(), json!("https://example.com/i.png"));
166        let v = serde_json::to_value(&meta).unwrap();
167        assert_eq!(v["sourceURL"], "https://example.com/a");
168        assert_eq!(v["statusCode"], 200);
169        assert_eq!(v["proxyUsed"], "basic");
170        assert_eq!(v["ogImage"], "https://example.com/i.png");
171        assert_eq!(v["pagesift"]["contentHash"], "ab");
172        assert!(v.get("title").is_none());
173    }
174}
pagesift_api/types/mod.rs

pagesift_api/types/
mod.rs
