pagesift_api/types/
response.rs

1//! Firecrawl v1/v2 response types and metadata structures.
2
3use serde::{Deserialize, Serialize};
4use serde_json::Value;
5use std::collections::BTreeMap;
6
7/// Explanation when a query-directed crawl completes (`pagesift` field in job status).
8#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
9#[serde(rename_all = "camelCase")]
10pub struct CrawlSummary {
11    /// `coverage_reached`, `saturated`, `limit_reached`, `exhausted`, `cancelled`.
12    pub stop_reason: String,
13    /// Coverage percentage (0.0 to 1.0) when adaptive crawl completes.
14    #[serde(skip_serializing_if = "Option::is_none")]
15    pub coverage: Option<f64>,
16    /// Number of distinct evidence blocks gathered for each term.
17    #[serde(skip_serializing_if = "BTreeMap::is_empty", default)]
18    pub term_evidence: BTreeMap<String, usize>,
19    /// Actual pages fetched.
20    pub pages: usize,
21}
22
23/// PageSift extensions inside `metadata.pagesift`.
24/// 带查询的爬取里一页的来历（`metadata.pagesift.crawl`）。
25#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
26#[serde(rename_all = "camelCase")]
27pub struct CrawlTrace {
28    /// 从种子页算起的深度；种子是 0。
29    pub depth: usize,
30    /// 发现这个链接的页面；种子和 sitemap 里的 URL 没有。
31    #[serde(skip_serializing_if = "Option::is_none")]
32    pub parent: Option<String>,
33    /// 链接入队时的分数（越高越先抓）；种子没有。
34    #[serde(skip_serializing_if = "Option::is_none")]
35    pub score: Option<f32>,
36    /// 附件（PDF、Office 文档）的来源页；普通页面没有。
37    #[serde(skip_serializing_if = "Option::is_none")]
38    pub attachment_of: Option<String>,
39}
40
41#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
42#[serde(rename_all = "camelCase")]
43pub struct PagesiftMeta {
44    #[serde(skip_serializing_if = "Option::is_none")]
45    pub content_hash: Option<String>,
46    #[serde(skip_serializing_if = "Option::is_none")]
47    pub raw_hash: Option<String>,
48    #[serde(skip_serializing_if = "Option::is_none")]
49    pub text_hash: Option<String>,
50    #[serde(skip_serializing_if = "Vec::is_empty", default)]
51    pub redirect_chain: Vec<String>,
52    #[serde(skip_serializing_if = "Option::is_none")]
53    pub etag: Option<String>,
54    #[serde(skip_serializing_if = "Option::is_none")]
55    pub last_modified: Option<String>,
56    #[serde(skip_serializing_if = "std::ops::Not::not", default)]
57    pub not_modified: bool,
58    /// `http` or `browser`.
59    #[serde(skip_serializing_if = "Option::is_none")]
60    pub engine: Option<String>,
61    /// `lightpanda`, `moli`, or `chrome`.
62    #[serde(skip_serializing_if = "Option::is_none")]
63    pub browser_engine: Option<String>,
64    #[serde(skip_serializing_if = "Vec::is_empty", default)]
65    pub warnings: Vec<String>,
66    /// Timing per phase in milliseconds.
67    #[serde(skip_serializing_if = "BTreeMap::is_empty", default)]
68    pub timings: BTreeMap<String, f64>,
69    /// Duplicate canonical reference URL.
70    #[serde(skip_serializing_if = "Option::is_none")]
71    pub duplicate_of: Option<String>,
72    /// Query relevance score (0.0 to 1.0).
73    #[serde(skip_serializing_if = "Option::is_none")]
74    pub relevance: Option<f32>,
75    /// 带查询的爬取里这一页是怎么来的：深度、父页、入队分数、附件的来源页。
76    #[serde(skip_serializing_if = "Option::is_none")]
77    pub crawl: Option<CrawlTrace>,
78    /// Query filter statistics.
79    #[serde(skip_serializing_if = "Option::is_none")]
80    pub query_filter: Option<QueryFilterStats>,
81    /// JSON-LD blocks found on the page, parsed and passed through unchanged.
82    #[serde(skip_serializing_if = "Vec::is_empty", default)]
83    pub json_ld: Vec<Value>,
84}
85
86#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
87#[serde(rename_all = "camelCase")]
88pub struct QueryFilterStats {
89    pub kept_blocks: usize,
90    pub total_blocks: usize,
91    pub tokens_before: usize,
92    pub tokens_after: usize,
93}
94
95/// Firecrawl `changeTracking` response object.
96#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
97#[serde(rename_all = "camelCase")]
98pub struct ChangeTracking {
99    pub previous_scrape_at: Option<String>,
100    /// `new`, `same`, `changed`, `removed`.
101    pub change_status: String,
102    /// `visible` or `hidden`.
103    pub visibility: String,
104    #[serde(skip_serializing_if = "Option::is_none")]
105    pub diff: Option<Value>,
106    #[serde(skip_serializing_if = "Option::is_none")]
107    pub json: Option<Value>,
108    #[serde(skip_serializing_if = "Option::is_none")]
109    pub pagesift: Option<Value>,
110}
111
112#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
113#[serde(rename_all = "camelCase")]
114pub struct Metadata {
115    #[serde(skip_serializing_if = "Option::is_none")]
116    pub title: Option<String>,
117    #[serde(skip_serializing_if = "Option::is_none")]
118    pub description: Option<String>,
119    #[serde(skip_serializing_if = "Option::is_none")]
120    pub language: Option<String>,
121    #[serde(skip_serializing_if = "Option::is_none")]
122    pub keywords: Option<String>,
123    #[serde(skip_serializing_if = "Option::is_none")]
124    pub robots: Option<String>,
125    #[serde(skip_serializing_if = "Option::is_none")]
126    pub favicon: Option<String>,
127    #[serde(skip_serializing_if = "Option::is_none")]
128    pub published_time: Option<String>,
129    #[serde(skip_serializing_if = "Option::is_none")]
130    pub author: Option<String>,
131    #[serde(skip_serializing_if = "Option::is_none")]
132    pub modified_time: Option<String>,
133    #[serde(skip_serializing_if = "Option::is_none")]
134    pub url: Option<String>,
135    #[serde(rename = "sourceURL", skip_serializing_if = "Option::is_none")]
136    pub source_url: Option<String>,
137    pub status_code: u16,
138    #[serde(skip_serializing_if = "Option::is_none")]
139    pub content_type: Option<String>,
140    #[serde(skip_serializing_if = "Option::is_none")]
141    pub error: Option<String>,
142    pub proxy_used: String,
143    #[serde(skip_serializing_if = "Option::is_none")]
144    pub cache_state: Option<String>,
145    #[serde(skip_serializing_if = "Option::is_none")]
146    pub cached_at: Option<String>,
147    #[serde(flatten)]
148    pub extra: BTreeMap<String, Value>,
149    #[serde(skip_serializing_if = "Option::is_none")]
150    pub pagesift: Option<PagesiftMeta>,
151}
152
153#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
154#[serde(rename_all = "camelCase")]
155pub struct Document {
156    #[serde(skip_serializing_if = "Option::is_none")]
157    pub markdown: Option<String>,
158    /// 每页的 Markdown：只在 `/v2/parse` 的 PDF 请求了 `parsers: [{type: "pdf", pages: true}]` 时给。
159    #[serde(skip_serializing_if = "Option::is_none")]
160    pub pages: Option<Vec<PageMarkdown>>,
161    #[serde(skip_serializing_if = "Option::is_none")]
162    pub html: Option<String>,
163    #[serde(skip_serializing_if = "Option::is_none")]
164    pub raw_html: Option<String>,
165    #[serde(skip_serializing_if = "Option::is_none")]
166    pub links: Option<Vec<String>>,
167    /// `links` 里每个链接的锚文本（同序），爬虫给链接打分用；不进 JSON。
168    #[serde(skip)]
169    pub link_text: Option<Vec<String>>,
170    /// 页面声明的其他语言版本（`hreflang`）：(语言, 地址)。爬虫找英文版用；不进 JSON。
171    #[serde(skip)]
172    pub alternates: Vec<(String, String)>,
173    #[serde(skip_serializing_if = "Option::is_none")]
174    pub images: Option<Vec<String>>,
175    #[serde(skip_serializing_if = "Option::is_none")]
176    pub screenshot: Option<String>,
177    #[serde(skip_serializing_if = "Option::is_none")]
178    pub json: Option<Value>,
179    #[serde(skip_serializing_if = "Option::is_none")]
180    pub summary: Option<String>,
181    #[serde(skip_serializing_if = "Option::is_none")]
182    pub actions: Option<Value>,
183    #[serde(skip_serializing_if = "Option::is_none")]
184    pub warning: Option<String>,
185    #[serde(skip_serializing_if = "Option::is_none")]
186    pub change_tracking: Option<ChangeTracking>,
187    #[serde(skip_serializing_if = "Option::is_none")]
188    pub chunks: Option<Vec<DocChunk>>,
189    pub metadata: Metadata,
190}
191
192/// Firecrawl `data.pages[]`。
193#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
194#[serde(rename_all = "camelCase")]
195pub struct PageMarkdown {
196    /// 从 1 开始的页码。
197    pub page_number: usize,
198    pub markdown: String,
199}
200
201#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
202#[serde(rename_all = "camelCase")]
203pub struct DocChunk {
204    pub text: String,
205    pub heading_path: Vec<String>,
206    pub start: usize,
207    pub end: usize,
208    pub tokens: usize,
209}
210
211#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
212pub struct MapLink {
213    pub url: String,
214    #[serde(skip_serializing_if = "Option::is_none")]
215    pub title: Option<String>,
216    #[serde(skip_serializing_if = "Option::is_none")]
217    pub description: Option<String>,
218}
219
220#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
221pub struct ScrapeResponse {
222    pub success: bool,
223    pub data: Document,
224    #[serde(skip_serializing_if = "Option::is_none")]
225    pub warning: Option<String>,
226}
227
228#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
229#[serde(rename_all = "camelCase")]
230pub struct JobStarted {
231    pub success: bool,
232    pub id: String,
233    pub url: String,
234    #[serde(rename = "invalidURLs", skip_serializing_if = "Option::is_none")]
235    pub invalid_urls: Option<Vec<String>>,
236}
237
238#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
239#[serde(rename_all = "camelCase")]
240pub struct JobStatus {
241    pub success: bool,
242    pub status: String,
243    pub completed: usize,
244    pub total: usize,
245    pub credits_used: usize,
246    pub expires_at: String,
247    #[serde(skip_serializing_if = "Option::is_none")]
248    pub next: Option<String>,
249    pub data: Vec<Document>,
250    #[serde(skip_serializing_if = "Option::is_none")]
251    pub pagesift: Option<CrawlSummary>,
252}
253
254#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
255pub struct JobError {
256    pub id: String,
257    pub timestamp: String,
258    pub url: String,
259    pub code: String,
260    pub error: String,
261}
262
263#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
264#[serde(rename_all = "camelCase")]
265pub struct JobErrors {
266    pub errors: Vec<JobError>,
267    pub robots_blocked: Vec<String>,
268}
269
270#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
271pub struct MapResponse {
272    pub success: bool,
273    pub links: Vec<MapLink>,
274}
pagesift_api/types/response.rs

pagesift_api/types/
response.rs
