pagesift_api/types/
response.rs

1//! Firecrawl v1/v2 response types and metadata structures.
2
3use serde::{Deserialize, Serialize};
4use serde_json::Value;
5use std::collections::BTreeMap;
6
7/// Explanation when a query-directed crawl completes (`pagesift` field in job status).
8#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
9#[serde(rename_all = "camelCase")]
10pub struct CrawlSummary {
11    /// `coverage_reached`, `saturated`, `no_relevant_pages` (adaptive: nothing relevant found and
12    /// no lead left to follow), `limit_reached`, `exhausted`, `cancelled`.
13    pub stop_reason: String,
14    /// Coverage percentage (0.0 to 1.0) when adaptive crawl completes.
15    #[serde(skip_serializing_if = "Option::is_none")]
16    pub coverage: Option<f64>,
17    /// Number of distinct evidence blocks gathered for each term.
18    #[serde(skip_serializing_if = "BTreeMap::is_empty", default)]
19    pub term_evidence: BTreeMap<String, usize>,
20    /// Actual pages fetched.
21    pub pages: usize,
22}
23
24/// PageSift extensions inside `metadata.pagesift`.
25/// 带查询的爬取里一页的来历（`metadata.pagesift.crawl`）。
26#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
27#[serde(rename_all = "camelCase")]
28pub struct CrawlTrace {
29    /// 从种子页算起的深度；种子是 0。
30    pub depth: usize,
31    /// 发现这个链接的页面；种子和 sitemap 里的 URL 没有。
32    #[serde(skip_serializing_if = "Option::is_none")]
33    pub parent: Option<String>,
34    /// 链接入队时的分数（越高越先抓）；种子没有。
35    #[serde(skip_serializing_if = "Option::is_none")]
36    pub score: Option<f32>,
37    /// 附件（PDF、Office 文档）的来源页；普通页面没有。
38    #[serde(skip_serializing_if = "Option::is_none")]
39    pub attachment_of: Option<String>,
40}
41
42/// `metadata.pagesift.ocr`。
43#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
44#[serde(rename_all = "camelCase")]
45pub struct OcrMeta {
46    pub engine: String,
47    pub model: String,
48    /// `cpu` 或 `cuda:N`。
49    pub device: String,
50    /// 文字来自 OCR 的页数。
51    pub pages: usize,
52    /// 想用 GPU 却退回 CPU 时为真（原因在 `gpu_fallback` 警告里）。
53    pub fallback: bool,
54}
55
56#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
57#[serde(rename_all = "camelCase")]
58pub struct PagesiftMeta {
59    #[serde(skip_serializing_if = "Option::is_none")]
60    pub content_hash: Option<String>,
61    #[serde(skip_serializing_if = "Option::is_none")]
62    pub raw_hash: Option<String>,
63    #[serde(skip_serializing_if = "Option::is_none")]
64    pub text_hash: Option<String>,
65    #[serde(skip_serializing_if = "Vec::is_empty", default)]
66    pub redirect_chain: Vec<String>,
67    #[serde(skip_serializing_if = "Option::is_none")]
68    pub etag: Option<String>,
69    #[serde(skip_serializing_if = "Option::is_none")]
70    pub last_modified: Option<String>,
71    #[serde(skip_serializing_if = "std::ops::Not::not", default)]
72    pub not_modified: bool,
73    /// `http` or `browser`.
74    #[serde(skip_serializing_if = "Option::is_none")]
75    pub engine: Option<String>,
76    /// `lightpanda`, `moli`, or `chrome`.
77    #[serde(skip_serializing_if = "Option::is_none")]
78    pub browser_engine: Option<String>,
79    /// 这次 HTTP 抓取确实按这个档案发（如 `chrome_156`）：站点看到的是这个浏览器，不是 pagesift。
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub browser_fingerprint: Option<String>,
82    #[serde(skip_serializing_if = "Vec::is_empty", default)]
83    pub warnings: Vec<String>,
84    /// Timing per phase in milliseconds.
85    #[serde(skip_serializing_if = "BTreeMap::is_empty", default)]
86    pub timings: BTreeMap<String, f64>,
87    /// OCR：哪个引擎和设备识别了多少页。
88    #[serde(skip_serializing_if = "Option::is_none")]
89    pub ocr: Option<OcrMeta>,
90    /// Duplicate canonical reference URL.
91    #[serde(skip_serializing_if = "Option::is_none")]
92    pub duplicate_of: Option<String>,
93    /// Query relevance score (0.0 to 1.0).
94    #[serde(skip_serializing_if = "Option::is_none")]
95    pub relevance: Option<f32>,
96    /// 带查询的爬取里这一页是怎么来的：深度、父页、入队分数、附件的来源页。
97    #[serde(skip_serializing_if = "Option::is_none")]
98    pub crawl: Option<CrawlTrace>,
99    /// Query filter statistics.
100    #[serde(skip_serializing_if = "Option::is_none")]
101    pub query_filter: Option<QueryFilterStats>,
102    /// JSON-LD blocks found on the page, parsed and passed through unchanged.
103    #[serde(skip_serializing_if = "Vec::is_empty", default)]
104    pub json_ld: Vec<Value>,
105    /// How the result was produced: one entry per decision (render or not, merge or not) with its inputs.
106    /// Only present when the request asks for an explanation.
107    #[serde(skip_serializing_if = "Option::is_none")]
108    pub plan: Option<PlanTrace>,
109}
110
111/// `metadata.pagesift.plan`：每一步决策的依据。
112#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
113#[serde(rename_all = "camelCase")]
114pub struct PlanTrace {
115    pub steps: Vec<PlanStep>,
116}
117
118#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
119#[serde(rename_all = "camelCase")]
120pub struct PlanStep {
121    /// `render`（是否开浏览器）；`adopt`（是否采用渲染结果）或 `merge`（采用的是 HTTP 与渲染按块合并的结果）。
122    pub action: String,
123    pub chosen: bool,
124    /// 依据来自学到的后验；`false` 表示冷启动时沿用的规则。
125    pub learned: bool,
126    /// 用来决策的概率（Thompson 抽样值）。
127    #[serde(skip_serializing_if = "Option::is_none")]
128    pub p: Option<f64>,
129    /// 可得的价值（质量单位）。
130    #[serde(skip_serializing_if = "Option::is_none")]
131    pub value: Option<f64>,
132    /// 资源与时间的代价（质量单位）。
133    #[serde(skip_serializing_if = "Option::is_none")]
134    pub charge: Option<f64>,
135    /// 这一步之后结果的校准质量分。
136    #[serde(skip_serializing_if = "Option::is_none")]
137    pub quality: Option<f64>,
138    /// 合并时各块的来源：两边都有、只来自 HTTP、只来自渲染。
139    #[serde(skip_serializing_if = "Option::is_none")]
140    pub blocks: Option<[usize; 3]>, // [推导] 来源种类数
141}
142
143#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
144#[serde(rename_all = "camelCase")]
145pub struct QueryFilterStats {
146    pub kept_blocks: usize,
147    pub total_blocks: usize,
148    pub tokens_before: usize,
149    pub tokens_after: usize,
150}
151
152/// Firecrawl `changeTracking` response object.
153#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
154#[serde(rename_all = "camelCase")]
155pub struct ChangeTracking {
156    pub previous_scrape_at: Option<String>,
157    /// `new`, `same`, `changed`, `removed`.
158    pub change_status: String,
159    /// `visible` or `hidden`.
160    pub visibility: String,
161    #[serde(skip_serializing_if = "Option::is_none")]
162    pub diff: Option<Value>,
163    #[serde(skip_serializing_if = "Option::is_none")]
164    pub json: Option<Value>,
165    #[serde(skip_serializing_if = "Option::is_none")]
166    pub pagesift: Option<Value>,
167}
168
169#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
170#[serde(rename_all = "camelCase")]
171pub struct Metadata {
172    /// `/v2/scrape`：`/v2/scrape/{id}/interact` 用的 id。
173    #[serde(skip_serializing_if = "Option::is_none")]
174    pub scrape_id: Option<String>,
175    #[serde(skip_serializing_if = "Option::is_none")]
176    pub title: Option<String>,
177    #[serde(skip_serializing_if = "Option::is_none")]
178    pub description: Option<String>,
179    #[serde(skip_serializing_if = "Option::is_none")]
180    pub language: Option<String>,
181    #[serde(skip_serializing_if = "Option::is_none")]
182    pub keywords: Option<String>,
183    #[serde(skip_serializing_if = "Option::is_none")]
184    pub robots: Option<String>,
185    #[serde(skip_serializing_if = "Option::is_none")]
186    pub favicon: Option<String>,
187    #[serde(skip_serializing_if = "Option::is_none")]
188    pub published_time: Option<String>,
189    #[serde(skip_serializing_if = "Option::is_none")]
190    pub author: Option<String>,
191    #[serde(skip_serializing_if = "Option::is_none")]
192    pub modified_time: Option<String>,
193    #[serde(skip_serializing_if = "Option::is_none")]
194    pub url: Option<String>,
195    #[serde(rename = "sourceURL", skip_serializing_if = "Option::is_none")]
196    pub source_url: Option<String>,
197    pub status_code: u16,
198    #[serde(skip_serializing_if = "Option::is_none")]
199    pub content_type: Option<String>,
200    #[serde(skip_serializing_if = "Option::is_none")]
201    pub error: Option<String>,
202    pub proxy_used: String,
203    #[serde(skip_serializing_if = "Option::is_none")]
204    pub cache_state: Option<String>,
205    #[serde(skip_serializing_if = "Option::is_none")]
206    pub cached_at: Option<String>,
207    #[serde(flatten)]
208    pub extra: BTreeMap<String, Value>,
209    #[serde(skip_serializing_if = "Option::is_none")]
210    pub pagesift: Option<PagesiftMeta>,
211}
212
213#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
214#[serde(rename_all = "camelCase")]
215pub struct Document {
216    #[serde(skip_serializing_if = "Option::is_none")]
217    pub markdown: Option<String>,
218    /// 每页的 Markdown：只在 `/v2/parse` 的 PDF 请求了 `parsers: [{type: "pdf", pages: true}]` 时给。
219    #[serde(skip_serializing_if = "Option::is_none")]
220    pub pages: Option<Vec<PageMarkdown>>,
221    #[serde(skip_serializing_if = "Option::is_none")]
222    pub html: Option<String>,
223    #[serde(skip_serializing_if = "Option::is_none")]
224    pub raw_html: Option<String>,
225    #[serde(skip_serializing_if = "Option::is_none")]
226    pub links: Option<Vec<String>>,
227    /// `links` 里每个链接的锚文本（同序），爬虫给链接打分用；不进 JSON。
228    #[serde(skip)]
229    pub link_text: Option<Vec<String>>,
230    /// 页面声明的其他语言版本（`hreflang`）：(语言, 地址)。爬虫找英文版用；不进 JSON。
231    #[serde(skip)]
232    pub alternates: Vec<(String, String)>,
233    /// 引擎内部用的校准相对质量分（`pagesift_plan::quality`：只用来比较同一页的结果，不是绝对概率），
234    /// 采用渲染结果和按块合并时用；不进 JSON、不进缓存。
235    #[serde(skip)]
236    pub quality: Option<f64>,
237    #[serde(skip_serializing_if = "Option::is_none")]
238    pub images: Option<Vec<String>>,
239    #[serde(skip_serializing_if = "Option::is_none")]
240    pub screenshot: Option<String>,
241    #[serde(skip_serializing_if = "Option::is_none")]
242    pub json: Option<Value>,
243    /// PageSift extension: items extracted with the `extract` format's CSS schema.
244    #[serde(skip_serializing_if = "Option::is_none")]
245    pub extract: Option<Value>,
246    #[serde(skip_serializing_if = "Option::is_none")]
247    pub summary: Option<String>,
248    #[serde(skip_serializing_if = "Option::is_none")]
249    pub actions: Option<Value>,
250    #[serde(skip_serializing_if = "Option::is_none")]
251    pub warning: Option<String>,
252    #[serde(skip_serializing_if = "Option::is_none")]
253    pub change_tracking: Option<ChangeTracking>,
254    #[serde(skip_serializing_if = "Option::is_none")]
255    pub chunks: Option<Vec<DocChunk>>,
256    /// PageSift extension: exported files (`exportImages`). JSON lists `{id, format, bytes, hash}`;
257    /// the content is downloaded from `GET /v2/attachments/{id}`.
258    #[serde(skip_serializing_if = "Vec::is_empty", default)]
259    pub attachments: Vec<super::Attachment>,
260    pub metadata: Metadata,
261}
262
263/// Firecrawl `data.pages[]`。
264#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
265#[serde(rename_all = "camelCase")]
266pub struct PageMarkdown {
267    /// 从 1 开始的页码。
268    pub page_number: usize,
269    pub markdown: String,
270}
271
272#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
273#[serde(rename_all = "camelCase")]
274pub struct DocChunk {
275    pub text: String,
276    pub heading_path: Vec<String>,
277    pub start: usize,
278    pub end: usize,
279    pub tokens: usize,
280}
281
282#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
283pub struct MapLink {
284    pub url: String,
285    #[serde(skip_serializing_if = "Option::is_none")]
286    pub title: Option<String>,
287    #[serde(skip_serializing_if = "Option::is_none")]
288    pub description: Option<String>,
289}
290
291#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
292pub struct ScrapeResponse {
293    pub success: bool,
294    pub data: Document,
295    #[serde(skip_serializing_if = "Option::is_none")]
296    pub warning: Option<String>,
297}
298
299#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
300#[serde(rename_all = "camelCase")]
301pub struct JobStarted {
302    pub success: bool,
303    pub id: String,
304    pub url: String,
305    #[serde(rename = "invalidURLs", skip_serializing_if = "Option::is_none")]
306    pub invalid_urls: Option<Vec<String>>,
307    /// Request fields that were accepted but ignored (`ignored_fields: a, b`).
308    #[serde(default, skip_serializing_if = "Vec::is_empty")]
309    pub warnings: Vec<String>,
310}
311
312#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
313#[serde(rename_all = "camelCase")]
314pub struct JobStatus {
315    pub success: bool,
316    pub status: String,
317    pub completed: usize,
318    pub total: usize,
319    pub credits_used: usize,
320    pub expires_at: String,
321    #[serde(skip_serializing_if = "Option::is_none")]
322    pub next: Option<String>,
323    pub data: Vec<Document>,
324    #[serde(skip_serializing_if = "Option::is_none")]
325    pub pagesift: Option<CrawlSummary>,
326}
327
328#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
329pub struct JobError {
330    pub id: String,
331    pub timestamp: String,
332    pub url: String,
333    pub code: String,
334    pub error: String,
335}
336
337#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
338#[serde(rename_all = "camelCase")]
339pub struct JobErrors {
340    pub errors: Vec<JobError>,
341    pub robots_blocked: Vec<String>,
342}
343
344#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
345pub struct MapResponse {
346    pub success: bool,
347    pub links: Vec<MapLink>,
348    /// Request fields that were accepted but ignored (`ignored_fields: a, b`).
349    #[serde(default, skip_serializing_if = "Vec::is_empty")]
350    pub warnings: Vec<String>,
351}
pagesift_api/types/response.rs

pagesift_api/types/
response.rs
