pagesift_api/types/
mod.rs

1//! Firecrawl v1/v2 request and response types.
2
3pub mod parse;
4pub mod request;
5pub mod response;
6
7pub use parse::{PdfParser, Upload, UploadKind};
8
9pub use request::{
10    AdaptiveOptions, BatchScrapeRequest, CrawlRequest, CrawlScope, ExtractRequest, Format,
11    Location, MapRequest, QueryFilter, ScrapeOptions, ScrapeRequest, SearchRequest,
12};
13pub use response::{
14    ChangeTracking, CrawlSummary, CrawlTrace, DocChunk, Document, JobError, JobErrors, JobStarted,
15    JobStatus, MapLink, MapResponse, Metadata, PageMarkdown, PagesiftMeta, QueryFilterStats,
16    ScrapeResponse,
17};
18
19#[cfg(test)]
20mod tests {
21    use super::*;
22    use serde_json::json;
23
24    #[test]
25    fn formats_accept_strings_and_objects() {
26        let r: ScrapeRequest = serde_json::from_value(json!({
27            "url": "https://example.com",
28            "formats": ["markdown", {"type": "screenshot", "fullPage": true}],
29        }))
30        .unwrap();
31        assert_eq!(r.options.formats[0], Format::new("markdown"));
32        assert_eq!(r.options.formats[1].kind, "screenshot");
33        assert_eq!(r.options.formats[1].options["fullPage"], json!(true));
34    }
35
36    #[test]
37    fn defaults_follow_firecrawl_and_unknown_fields_are_ignored() {
38        let r: ScrapeRequest = serde_json::from_value(json!({
39            "url": "https://example.com",
40            "origin": "python-sdk",
41            "integration": null,
42            "zeroDataRetention": false,
43        }))
44        .unwrap();
45        let o = r.options;
46        assert_eq!(o.formats, vec![Format::new("markdown")]);
47        assert!(o.only_main_content && o.block_ads && o.store_in_cache && o.remove_base64_images);
48        assert_eq!(o.wait_for, 0);
49        assert!(!o.ignore_robots_txt);
50    }
51
52    #[test]
53    fn crawl_and_map_defaults() {
54        let c: CrawlRequest =
55            serde_json::from_value(json!({"url": "https://example.com"})).unwrap();
56        assert_eq!((c.limit, c.sitemap.as_str()), (10_000, "include"));
57        assert!(c.scrape_options.only_main_content);
58        let m: MapRequest = serde_json::from_value(json!({"url": "https://example.com"})).unwrap();
59        assert_eq!(m.limit, 5_000);
60        assert!(m.include_subdomains && m.ignore_query_parameters);
61    }
62
63    #[test]
64    fn query_filter_accepts_string_or_object() {
65        let r: ScrapeRequest = serde_json::from_value(json!({
66            "url": "https://example.com", "queryFilter": "refund policy",
67        }))
68        .unwrap();
69        let f = r.options.query_filter.unwrap();
70        assert_eq!(
71            (
72                f.query.as_str(),
73                f.threshold,
74                f.relative,
75                f.top_k,
76                f.headings
77            ),
78            ("refund policy", None, Some(0.5), None, true)
79        );
80        let r: ScrapeRequest = serde_json::from_value(json!({
81            "url": "https://example.com",
82            "queryFilter": {"query": "q", "threshold": 0.5, "topK": 3, "headings": false},
83        }))
84        .unwrap();
85        let f = r.options.query_filter.unwrap();
86        assert_eq!(
87            (f.threshold, f.relative, f.top_k, f.headings),
88            (Some(0.5), None, Some(3), false)
89        );
90        let r: ScrapeRequest = serde_json::from_value(json!({
91            "url": "https://example.com",
92            "queryFilter": {"query": "q", "maxTokens": 800},
93        }))
94        .unwrap();
95        let f = r.options.query_filter.unwrap();
96        assert_eq!((f.relative, f.max_tokens), (Some(0.5), Some(800)));
97        assert!(ScrapeOptions::default().query_filter.is_none());
98    }
99
100    #[test]
101    fn crawl_query_and_adaptive_options() {
102        let c: CrawlRequest =
103            serde_json::from_value(json!({"url": "https://example.com"})).unwrap();
104        assert_eq!((c.query, c.adaptive), (None, None));
105        let c: CrawlRequest = serde_json::from_value(json!({
106            "url": "https://example.com", "query": ["refund", "policy"], "adaptive": true,
107        }))
108        .unwrap();
109        assert_eq!(c.query.as_deref(), Some("refund policy"));
110        assert_eq!(c.adaptive, Some(AdaptiveOptions::default()));
111        let c: CrawlRequest = serde_json::from_value(json!({
112            "url": "https://example.com", "query": "  ", "adaptive": {"patience": 2},
113        }))
114        .unwrap();
115        assert_eq!(c.query, None, "blank query is no query");
116        assert_eq!(c.adaptive.unwrap().patience, 2);
117        let c: CrawlRequest = serde_json::from_value(json!({
118            "url": "https://example.com", "query": null, "adaptive": false,
119        }))
120        .unwrap();
121        assert_eq!((c.query, c.adaptive), (None, None));
122    }
123
124    #[test]
125    fn change_tracking_serializes_firecrawl_fields() {
126        let doc = Document {
127            change_tracking: Some(ChangeTracking {
128                previous_scrape_at: None,
129                change_status: "new".into(),
130                visibility: "visible".into(),
131                diff: None,
132                json: None,
133                pagesift: None,
134            }),
135            ..Default::default()
136        };
137        let v = serde_json::to_value(&doc).unwrap();
138        assert_eq!(
139            v["changeTracking"],
140            json!({"previousScrapeAt": null, "changeStatus": "new", "visibility": "visible"})
141        );
142        assert!(
143            serde_json::to_value(Document::default())
144                .unwrap()
145                .get("changeTracking")
146                .is_none()
147        );
148    }
149
150    #[test]
151    fn metadata_serializes_firecrawl_names_and_extension() {
152        let mut meta = Metadata {
153            source_url: Some("https://example.com/a".into()),
154            status_code: 200,
155            proxy_used: "basic".into(),
156            pagesift: Some(PagesiftMeta {
157                content_hash: Some("ab".into()),
158                ..Default::default()
159            }),
160            ..Default::default()
161        };
162        meta.extra
163            .insert("ogImage".into(), json!("https://example.com/i.png"));
164        let v = serde_json::to_value(&meta).unwrap();
165        assert_eq!(v["sourceURL"], "https://example.com/a");
166        assert_eq!(v["statusCode"], 200);
167        assert_eq!(v["proxyUsed"], "basic");
168        assert_eq!(v["ogImage"], "https://example.com/i.png");
169        assert_eq!(v["pagesift"]["contentHash"], "ab");
170        assert!(v.get("title").is_none());
171    }
172}
pagesift_api/types/mod.rs

pagesift_api/types/
mod.rs
