pagesift_api/types/
response.rs

1//! Firecrawl v1/v2 response types and metadata structures.
2
3use serde::{Deserialize, Serialize};
4use serde_json::Value;
5use std::collections::BTreeMap;
6
7/// Explanation when a query-directed crawl completes (`pagesift` field in job status).
8#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
9#[serde(rename_all = "camelCase")]
10pub struct CrawlSummary {
11    /// `coverage_reached`, `saturated`, `no_relevant_pages` (adaptive: nothing relevant found and
12    /// no lead left to follow), `limit_reached`, `exhausted`, `cancelled`.
13    pub stop_reason: String,
14    /// Coverage percentage (0.0 to 1.0) when adaptive crawl completes.
15    #[serde(skip_serializing_if = "Option::is_none")]
16    pub coverage: Option<f64>,
17    /// Number of distinct evidence blocks gathered for each term.
18    #[serde(skip_serializing_if = "BTreeMap::is_empty", default)]
19    pub term_evidence: BTreeMap<String, usize>,
20    /// Actual pages fetched.
21    pub pages: usize,
22}
23
24/// PageSift extensions inside `metadata.pagesift`.
25/// 带查询的爬取里一页的来历（`metadata.pagesift.crawl`）。
26#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
27#[serde(rename_all = "camelCase")]
28pub struct CrawlTrace {
29    /// 从种子页算起的深度；种子是 0。
30    pub depth: usize,
31    /// 发现这个链接的页面；种子和 sitemap 里的 URL 没有。
32    #[serde(skip_serializing_if = "Option::is_none")]
33    pub parent: Option<String>,
34    /// 链接入队时的分数（越高越先抓）；种子没有。
35    #[serde(skip_serializing_if = "Option::is_none")]
36    pub score: Option<f32>,
37    /// 附件（PDF、Office 文档）的来源页；普通页面没有。
38    #[serde(skip_serializing_if = "Option::is_none")]
39    pub attachment_of: Option<String>,
40}
41
42/// `metadata.pagesift.ocr`。
43#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
44#[serde(rename_all = "camelCase")]
45pub struct OcrMeta {
46    pub engine: String,
47    pub model: String,
48    /// `cpu` 或 `cuda:N`。
49    pub device: String,
50    /// 文字来自 OCR 的页数。
51    pub pages: usize,
52    /// 想用 GPU 却退回 CPU 时为真（原因在 `gpu_fallback` 警告里）。
53    pub fallback: bool,
54}
55
56#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
57#[serde(rename_all = "camelCase")]
58pub struct PagesiftMeta {
59    #[serde(skip_serializing_if = "Option::is_none")]
60    pub content_hash: Option<String>,
61    #[serde(skip_serializing_if = "Option::is_none")]
62    pub raw_hash: Option<String>,
63    #[serde(skip_serializing_if = "Option::is_none")]
64    pub text_hash: Option<String>,
65    #[serde(skip_serializing_if = "Vec::is_empty", default)]
66    pub redirect_chain: Vec<String>,
67    #[serde(skip_serializing_if = "Option::is_none")]
68    pub etag: Option<String>,
69    #[serde(skip_serializing_if = "Option::is_none")]
70    pub last_modified: Option<String>,
71    #[serde(skip_serializing_if = "std::ops::Not::not", default)]
72    pub not_modified: bool,
73    /// `http` or `browser`.
74    #[serde(skip_serializing_if = "Option::is_none")]
75    pub engine: Option<String>,
76    /// `lightpanda`, `moli`, or `chrome`.
77    #[serde(skip_serializing_if = "Option::is_none")]
78    pub browser_engine: Option<String>,
79    /// 这次 HTTP 抓取确实按这个档案发（如 `chrome_156`）：站点看到的是这个浏览器，不是 pagesift。
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub browser_fingerprint: Option<String>,
82    /// Access-barrier recovery attempts, when itna is enabled.
83    #[serde(skip_serializing_if = "Option::is_none")]
84    pub itna: Option<Value>,
85    #[serde(skip_serializing_if = "Vec::is_empty", default)]
86    pub warnings: Vec<String>,
87    /// Timing per phase in milliseconds.
88    #[serde(skip_serializing_if = "BTreeMap::is_empty", default)]
89    pub timings: BTreeMap<String, f64>,
90    /// OCR：哪个引擎和设备识别了多少页。
91    #[serde(skip_serializing_if = "Option::is_none")]
92    pub ocr: Option<OcrMeta>,
93    /// Duplicate canonical reference URL.
94    #[serde(skip_serializing_if = "Option::is_none")]
95    pub duplicate_of: Option<String>,
96    /// Query relevance score (0.0 to 1.0).
97    #[serde(skip_serializing_if = "Option::is_none")]
98    pub relevance: Option<f32>,
99    /// 带查询的爬取里这一页是怎么来的：深度、父页、入队分数、附件的来源页。
100    #[serde(skip_serializing_if = "Option::is_none")]
101    pub crawl: Option<CrawlTrace>,
102    /// Query filter statistics.
103    #[serde(skip_serializing_if = "Option::is_none")]
104    pub query_filter: Option<QueryFilterStats>,
105    /// JSON-LD blocks found on the page, parsed and passed through unchanged.
106    #[serde(skip_serializing_if = "Vec::is_empty", default)]
107    pub json_ld: Vec<Value>,
108    /// How the result was produced: one entry per decision (render or not, merge or not) with its inputs.
109    /// Only present when the request asks for an explanation.
110    #[serde(skip_serializing_if = "Option::is_none")]
111    pub plan: Option<PlanTrace>,
112}
113
114/// `metadata.pagesift.plan`：每一步决策的依据。
115#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
116#[serde(rename_all = "camelCase")]
117pub struct PlanTrace {
118    pub steps: Vec<PlanStep>,
119}
120
121#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
122#[serde(rename_all = "camelCase")]
123pub struct PlanStep {
124    /// `render`（是否开浏览器）；`adopt`（是否采用渲染结果）或 `merge`（采用的是 HTTP 与渲染按块合并的结果）。
125    pub action: String,
126    pub chosen: bool,
127    /// 依据来自学到的后验；`false` 表示冷启动时沿用的规则。
128    pub learned: bool,
129    /// 用来决策的概率（Thompson 抽样值）。
130    #[serde(skip_serializing_if = "Option::is_none")]
131    pub p: Option<f64>,
132    /// 可得的价值（质量单位）。
133    #[serde(skip_serializing_if = "Option::is_none")]
134    pub value: Option<f64>,
135    /// 资源与时间的代价（质量单位）。
136    #[serde(skip_serializing_if = "Option::is_none")]
137    pub charge: Option<f64>,
138    /// 这一步之后结果的校准质量分。
139    #[serde(skip_serializing_if = "Option::is_none")]
140    pub quality: Option<f64>,
141    /// 合并时各块的来源：两边都有、只来自 HTTP、只来自渲染。
142    #[serde(skip_serializing_if = "Option::is_none")]
143    pub blocks: Option<[usize; 3]>, // [推导] 来源种类数
144}
145
146#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
147#[serde(rename_all = "camelCase")]
148pub struct QueryFilterStats {
149    pub kept_blocks: usize,
150    pub total_blocks: usize,
151    pub tokens_before: usize,
152    pub tokens_after: usize,
153}
154
155/// Firecrawl `changeTracking` response object.
156#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
157#[serde(rename_all = "camelCase")]
158pub struct ChangeTracking {
159    pub previous_scrape_at: Option<String>,
160    /// `new`, `same`, `changed`, `removed`.
161    pub change_status: String,
162    /// `visible` or `hidden`.
163    pub visibility: String,
164    #[serde(skip_serializing_if = "Option::is_none")]
165    pub diff: Option<Value>,
166    #[serde(skip_serializing_if = "Option::is_none")]
167    pub json: Option<Value>,
168    #[serde(skip_serializing_if = "Option::is_none")]
169    pub pagesift: Option<Value>,
170}
171
172#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
173#[serde(rename_all = "camelCase")]
174pub struct Metadata {
175    /// `/v2/scrape`：`/v2/scrape/{id}/interact` 用的 id。
176    #[serde(skip_serializing_if = "Option::is_none")]
177    pub scrape_id: Option<String>,
178    #[serde(skip_serializing_if = "Option::is_none")]
179    pub title: Option<String>,
180    #[serde(skip_serializing_if = "Option::is_none")]
181    pub description: Option<String>,
182    #[serde(skip_serializing_if = "Option::is_none")]
183    pub language: Option<String>,
184    #[serde(skip_serializing_if = "Option::is_none")]
185    pub keywords: Option<String>,
186    #[serde(skip_serializing_if = "Option::is_none")]
187    pub robots: Option<String>,
188    #[serde(skip_serializing_if = "Option::is_none")]
189    pub favicon: Option<String>,
190    #[serde(skip_serializing_if = "Option::is_none")]
191    pub published_time: Option<String>,
192    #[serde(skip_serializing_if = "Option::is_none")]
193    pub author: Option<String>,
194    #[serde(skip_serializing_if = "Option::is_none")]
195    pub modified_time: Option<String>,
196    #[serde(skip_serializing_if = "Option::is_none")]
197    pub url: Option<String>,
198    #[serde(rename = "sourceURL", skip_serializing_if = "Option::is_none")]
199    pub source_url: Option<String>,
200    pub status_code: u16,
201    /// 响应带 `Retry-After`。只用于拦截分类（429/503 的限速判定），不进 JSON、不进缓存。
202    #[serde(skip)]
203    pub retry_after: bool,
204    #[serde(skip_serializing_if = "Option::is_none")]
205    pub content_type: Option<String>,
206    #[serde(skip_serializing_if = "Option::is_none")]
207    pub error: Option<String>,
208    pub proxy_used: String,
209    #[serde(skip_serializing_if = "Option::is_none")]
210    pub cache_state: Option<String>,
211    #[serde(skip_serializing_if = "Option::is_none")]
212    pub cached_at: Option<String>,
213    #[serde(flatten)]
214    pub extra: BTreeMap<String, Value>,
215    #[serde(skip_serializing_if = "Option::is_none")]
216    pub pagesift: Option<PagesiftMeta>,
217}
218
219#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
220#[serde(rename_all = "camelCase")]
221pub struct Document {
222    #[serde(skip_serializing_if = "Option::is_none")]
223    pub markdown: Option<String>,
224    /// 每页的 Markdown：只在 `/v2/parse` 的 PDF 请求了 `parsers: [{type: "pdf", pages: true}]` 时给。
225    #[serde(skip_serializing_if = "Option::is_none")]
226    pub pages: Option<Vec<PageMarkdown>>,
227    #[serde(skip_serializing_if = "Option::is_none")]
228    pub html: Option<String>,
229    #[serde(skip_serializing_if = "Option::is_none")]
230    pub raw_html: Option<String>,
231    #[serde(skip_serializing_if = "Option::is_none")]
232    pub links: Option<Vec<String>>,
233    /// `links` 里每个链接的锚文本（同序），爬虫给链接打分用；不进 JSON。
234    #[serde(skip)]
235    pub link_text: Option<Vec<String>>,
236    /// 页面声明的其他语言版本（`hreflang`）：(语言, 地址)。爬虫找英文版用；不进 JSON。
237    #[serde(skip)]
238    pub alternates: Vec<(String, String)>,
239    /// 引擎内部用的校准相对质量分（`pagesift_plan::quality`：只用来比较同一页的结果，不是绝对概率），
240    /// 采用渲染结果和按块合并时用；不进 JSON、不进缓存。
241    #[serde(skip)]
242    pub quality: Option<f64>,
243    #[serde(skip_serializing_if = "Option::is_none")]
244    pub images: Option<Vec<String>>,
245    #[serde(skip_serializing_if = "Option::is_none")]
246    pub screenshot: Option<String>,
247    #[serde(skip_serializing_if = "Option::is_none")]
248    pub json: Option<Value>,
249    /// PageSift extension: items extracted with the `extract` format's CSS schema.
250    #[serde(skip_serializing_if = "Option::is_none")]
251    pub extract: Option<Value>,
252    #[serde(skip_serializing_if = "Option::is_none")]
253    pub summary: Option<String>,
254    #[serde(skip_serializing_if = "Option::is_none")]
255    pub actions: Option<Value>,
256    #[serde(skip_serializing_if = "Option::is_none")]
257    pub warning: Option<String>,
258    #[serde(skip_serializing_if = "Option::is_none")]
259    pub change_tracking: Option<ChangeTracking>,
260    #[serde(skip_serializing_if = "Option::is_none")]
261    pub chunks: Option<Vec<DocChunk>>,
262    /// PageSift extension: exported files (`exportImages`). JSON lists `{id, format, bytes, hash}`;
263    /// the content is downloaded from `GET /v2/attachments/{id}`.
264    #[serde(skip_serializing_if = "Vec::is_empty", default)]
265    pub attachments: Vec<super::Attachment>,
266    /// Outcome for each requested image, including failed and skipped images.
267    #[serde(default, skip_serializing_if = "Vec::is_empty")]
268    pub image_assets: Vec<ImageAssetResult>,
269    pub metadata: Metadata,
270}
271
272/// Per-source outcome. A failed or skipped image has no content-derived id.
273#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
274#[serde(rename_all = "camelCase")]
275pub struct ImageAssetResult {
276    pub source: String,
277    pub status: ImageAssetStatus,
278    #[serde(skip_serializing_if = "Option::is_none")]
279    pub id: Option<String>,
280    #[serde(skip_serializing_if = "Option::is_none")]
281    pub mime: Option<String>,
282    #[serde(skip_serializing_if = "Option::is_none")]
283    pub bytes: Option<u64>,
284    #[serde(skip_serializing_if = "Option::is_none")]
285    pub reason: Option<String>,
286}
287
288#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
289#[serde(rename_all = "kebab-case")]
290pub enum ImageAssetStatus {
291    Ok,
292    SkippedLimit,
293    Failed,
294}
295
296/// Firecrawl `data.pages[]`。
297#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
298#[serde(rename_all = "camelCase")]
299pub struct PageMarkdown {
300    /// 从 1 开始的页码。
301    pub page_number: usize,
302    pub markdown: String,
303}
304
305#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
306#[serde(rename_all = "camelCase")]
307pub struct DocChunk {
308    pub text: String,
309    pub heading_path: Vec<String>,
310    pub start: usize,
311    pub end: usize,
312    pub tokens: usize,
313}
314
315#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
316pub struct MapLink {
317    pub url: String,
318    #[serde(skip_serializing_if = "Option::is_none")]
319    pub title: Option<String>,
320    #[serde(skip_serializing_if = "Option::is_none")]
321    pub description: Option<String>,
322}
323
324#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
325pub struct ScrapeResponse {
326    pub success: bool,
327    pub data: Document,
328    #[serde(skip_serializing_if = "Option::is_none")]
329    pub warning: Option<String>,
330}
331
332#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
333#[serde(rename_all = "camelCase")]
334pub struct JobStarted {
335    pub success: bool,
336    pub id: String,
337    pub url: String,
338    #[serde(rename = "invalidURLs", skip_serializing_if = "Option::is_none")]
339    pub invalid_urls: Option<Vec<String>>,
340    /// Request fields that were accepted but ignored (`ignored_fields: a, b`).
341    #[serde(default, skip_serializing_if = "Vec::is_empty")]
342    pub warnings: Vec<String>,
343}
344
345#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
346#[serde(rename_all = "camelCase")]
347pub struct JobStatus {
348    pub success: bool,
349    pub status: String,
350    pub completed: usize,
351    pub total: usize,
352    pub credits_used: usize,
353    pub expires_at: String,
354    #[serde(skip_serializing_if = "Option::is_none")]
355    pub next: Option<String>,
356    pub data: Vec<Document>,
357    #[serde(skip_serializing_if = "Option::is_none")]
358    pub pagesift: Option<CrawlSummary>,
359}
360
361#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
362pub struct JobError {
363    pub id: String,
364    pub timestamp: String,
365    pub url: String,
366    pub code: String,
367    pub error: String,
368}
369
370#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
371#[serde(rename_all = "camelCase")]
372pub struct JobErrors {
373    pub errors: Vec<JobError>,
374    pub robots_blocked: Vec<String>,
375}
376
377#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
378pub struct MapResponse {
379    pub success: bool,
380    pub links: Vec<MapLink>,
381    /// Request fields that were accepted but ignored (`ignored_fields: a, b`).
382    #[serde(default, skip_serializing_if = "Vec::is_empty")]
383    pub warnings: Vec<String>,
384}
pagesift_api/types/response.rs

pagesift_api/types/
response.rs
