pagesift_api/types/
response.rs

1//! Firecrawl v1/v2 response types and metadata structures.
2
3use serde::{Deserialize, Serialize};
4use serde_json::Value;
5use std::collections::BTreeMap;
6
7/// Explanation when a query-directed crawl completes (`pagesift` field in job status).
8#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
9#[serde(rename_all = "camelCase")]
10pub struct CrawlSummary {
11    /// `coverage_reached`, `saturated`, `no_relevant_pages` (adaptive: nothing relevant found and
12    /// no lead left to follow), `limit_reached`, `exhausted`, `cancelled`.
13    pub stop_reason: String,
14    /// Coverage percentage (0.0 to 1.0) when adaptive crawl completes.
15    #[serde(skip_serializing_if = "Option::is_none")]
16    pub coverage: Option<f64>,
17    /// Number of distinct evidence blocks gathered for each term.
18    #[serde(skip_serializing_if = "BTreeMap::is_empty", default)]
19    pub term_evidence: BTreeMap<String, usize>,
20    /// Actual pages fetched.
21    pub pages: usize,
22}
23
24/// PageSift extensions inside `metadata.pagesift`.
25/// 带查询的爬取里一页的来历（`metadata.pagesift.crawl`）。
26#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
27#[serde(rename_all = "camelCase")]
28pub struct CrawlTrace {
29    /// 从种子页算起的深度；种子是 0。
30    pub depth: usize,
31    /// 发现这个链接的页面；种子和 sitemap 里的 URL 没有。
32    #[serde(skip_serializing_if = "Option::is_none")]
33    pub parent: Option<String>,
34    /// 链接入队时的分数（越高越先抓）；种子没有。
35    #[serde(skip_serializing_if = "Option::is_none")]
36    pub score: Option<f32>,
37    /// 附件（PDF、Office 文档）的来源页；普通页面没有。
38    #[serde(skip_serializing_if = "Option::is_none")]
39    pub attachment_of: Option<String>,
40}
41
42/// `metadata.pagesift.ocr`。
43#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
44#[serde(rename_all = "camelCase")]
45pub struct OcrMeta {
46    pub engine: String,
47    pub model: String,
48    /// `cpu` 或 `cuda:N`。
49    pub device: String,
50    /// 文字来自 OCR 的页数。
51    pub pages: usize,
52    /// 想用 GPU 却退回 CPU 时为真（原因在 `gpu_fallback` 警告里）。
53    pub fallback: bool,
54}
55
56#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
57#[serde(rename_all = "camelCase")]
58pub struct PagesiftMeta {
59    #[serde(skip_serializing_if = "Option::is_none")]
60    pub content_hash: Option<String>,
61    #[serde(skip_serializing_if = "Option::is_none")]
62    pub raw_hash: Option<String>,
63    #[serde(skip_serializing_if = "Option::is_none")]
64    pub text_hash: Option<String>,
65    #[serde(skip_serializing_if = "Vec::is_empty", default)]
66    pub redirect_chain: Vec<String>,
67    #[serde(skip_serializing_if = "Option::is_none")]
68    pub etag: Option<String>,
69    #[serde(skip_serializing_if = "Option::is_none")]
70    pub last_modified: Option<String>,
71    #[serde(skip_serializing_if = "std::ops::Not::not", default)]
72    pub not_modified: bool,
73    /// `http` or `browser`.
74    #[serde(skip_serializing_if = "Option::is_none")]
75    pub engine: Option<String>,
76    /// `lightpanda`, `moli`, or `chrome`.
77    #[serde(skip_serializing_if = "Option::is_none")]
78    pub browser_engine: Option<String>,
79    /// 这次 HTTP 抓取确实按这个档案发（如 `chrome_156`）：站点看到的是这个浏览器，不是 pagesift。
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub browser_fingerprint: Option<String>,
82    /// Access-barrier recovery attempts, when itna is enabled.
83    #[serde(skip_serializing_if = "Option::is_none")]
84    pub itna: Option<Value>,
85    #[serde(skip_serializing_if = "Vec::is_empty", default)]
86    pub warnings: Vec<String>,
87    /// Timing per phase in milliseconds.
88    #[serde(skip_serializing_if = "BTreeMap::is_empty", default)]
89    pub timings: BTreeMap<String, f64>,
90    /// OCR：哪个引擎和设备识别了多少页。
91    #[serde(skip_serializing_if = "Option::is_none")]
92    pub ocr: Option<OcrMeta>,
93    /// Duplicate canonical reference URL.
94    #[serde(skip_serializing_if = "Option::is_none")]
95    pub duplicate_of: Option<String>,
96    /// Query relevance score (0.0 to 1.0).
97    #[serde(skip_serializing_if = "Option::is_none")]
98    pub relevance: Option<f32>,
99    /// 带查询的爬取里这一页是怎么来的：深度、父页、入队分数、附件的来源页。
100    #[serde(skip_serializing_if = "Option::is_none")]
101    pub crawl: Option<CrawlTrace>,
102    /// Query filter statistics.
103    #[serde(skip_serializing_if = "Option::is_none")]
104    pub query_filter: Option<QueryFilterStats>,
105    /// JSON-LD blocks found on the page, parsed and passed through unchanged.
106    #[serde(skip_serializing_if = "Vec::is_empty", default)]
107    pub json_ld: Vec<Value>,
108    /// How the result was produced: one entry per decision (render or not, merge or not) with its inputs.
109    /// Only present when the request asks for an explanation.
110    #[serde(skip_serializing_if = "Option::is_none")]
111    pub plan: Option<PlanTrace>,
112}
113
114/// `metadata.pagesift.plan`：每一步决策的依据。
115#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
116#[serde(rename_all = "camelCase")]
117pub struct PlanTrace {
118    pub steps: Vec<PlanStep>,
119}
120
121#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
122#[serde(rename_all = "camelCase")]
123pub struct PlanStep {
124    /// `render`（是否开浏览器）；`adopt`（是否采用渲染结果）或 `merge`（采用的是 HTTP 与渲染按块合并的结果）。
125    pub action: String,
126    pub chosen: bool,
127    /// 依据来自学到的后验；`false` 表示冷启动时沿用的规则。
128    pub learned: bool,
129    /// 用来决策的概率（Thompson 抽样值）。
130    #[serde(skip_serializing_if = "Option::is_none")]
131    pub p: Option<f64>,
132    /// 可得的价值（质量单位）。
133    #[serde(skip_serializing_if = "Option::is_none")]
134    pub value: Option<f64>,
135    /// 资源与时间的代价（质量单位）。
136    #[serde(skip_serializing_if = "Option::is_none")]
137    pub charge: Option<f64>,
138    /// 这一步之后结果的校准质量分。
139    #[serde(skip_serializing_if = "Option::is_none")]
140    pub quality: Option<f64>,
141    /// 合并时各块的来源：两边都有、只来自 HTTP、只来自渲染。
142    #[serde(skip_serializing_if = "Option::is_none")]
143    pub blocks: Option<[usize; 3]>, // [推导] 来源种类数
144}
145
146#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
147#[serde(rename_all = "camelCase")]
148pub struct QueryFilterStats {
149    pub kept_blocks: usize,
150    pub total_blocks: usize,
151    pub tokens_before: usize,
152    pub tokens_after: usize,
153}
154
155/// Firecrawl `changeTracking` response object.
156#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
157#[serde(rename_all = "camelCase")]
158pub struct ChangeTracking {
159    pub previous_scrape_at: Option<String>,
160    /// `new`, `same`, `changed`, `removed`.
161    pub change_status: String,
162    /// `visible` or `hidden`.
163    pub visibility: String,
164    #[serde(skip_serializing_if = "Option::is_none")]
165    pub diff: Option<Value>,
166    #[serde(skip_serializing_if = "Option::is_none")]
167    pub json: Option<Value>,
168    #[serde(skip_serializing_if = "Option::is_none")]
169    pub pagesift: Option<Value>,
170}
171
172#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
173#[serde(rename_all = "camelCase")]
174pub struct Metadata {
175    /// `/v2/scrape`：`/v2/scrape/{id}/interact` 用的 id。
176    #[serde(skip_serializing_if = "Option::is_none")]
177    pub scrape_id: Option<String>,
178    #[serde(skip_serializing_if = "Option::is_none")]
179    pub title: Option<String>,
180    #[serde(skip_serializing_if = "Option::is_none")]
181    pub description: Option<String>,
182    #[serde(skip_serializing_if = "Option::is_none")]
183    pub language: Option<String>,
184    #[serde(skip_serializing_if = "Option::is_none")]
185    pub keywords: Option<String>,
186    #[serde(skip_serializing_if = "Option::is_none")]
187    pub robots: Option<String>,
188    #[serde(skip_serializing_if = "Option::is_none")]
189    pub favicon: Option<String>,
190    #[serde(skip_serializing_if = "Option::is_none")]
191    pub published_time: Option<String>,
192    #[serde(skip_serializing_if = "Option::is_none")]
193    pub author: Option<String>,
194    #[serde(skip_serializing_if = "Option::is_none")]
195    pub modified_time: Option<String>,
196    #[serde(skip_serializing_if = "Option::is_none")]
197    pub url: Option<String>,
198    #[serde(rename = "sourceURL", skip_serializing_if = "Option::is_none")]
199    pub source_url: Option<String>,
200    pub status_code: u16,
201    #[serde(skip_serializing_if = "Option::is_none")]
202    pub content_type: Option<String>,
203    #[serde(skip_serializing_if = "Option::is_none")]
204    pub error: Option<String>,
205    pub proxy_used: String,
206    #[serde(skip_serializing_if = "Option::is_none")]
207    pub cache_state: Option<String>,
208    #[serde(skip_serializing_if = "Option::is_none")]
209    pub cached_at: Option<String>,
210    #[serde(flatten)]
211    pub extra: BTreeMap<String, Value>,
212    #[serde(skip_serializing_if = "Option::is_none")]
213    pub pagesift: Option<PagesiftMeta>,
214}
215
216#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
217#[serde(rename_all = "camelCase")]
218pub struct Document {
219    #[serde(skip_serializing_if = "Option::is_none")]
220    pub markdown: Option<String>,
221    /// 每页的 Markdown：只在 `/v2/parse` 的 PDF 请求了 `parsers: [{type: "pdf", pages: true}]` 时给。
222    #[serde(skip_serializing_if = "Option::is_none")]
223    pub pages: Option<Vec<PageMarkdown>>,
224    #[serde(skip_serializing_if = "Option::is_none")]
225    pub html: Option<String>,
226    #[serde(skip_serializing_if = "Option::is_none")]
227    pub raw_html: Option<String>,
228    #[serde(skip_serializing_if = "Option::is_none")]
229    pub links: Option<Vec<String>>,
230    /// `links` 里每个链接的锚文本（同序），爬虫给链接打分用；不进 JSON。
231    #[serde(skip)]
232    pub link_text: Option<Vec<String>>,
233    /// 页面声明的其他语言版本（`hreflang`）：(语言, 地址)。爬虫找英文版用；不进 JSON。
234    #[serde(skip)]
235    pub alternates: Vec<(String, String)>,
236    /// 引擎内部用的校准相对质量分（`pagesift_plan::quality`：只用来比较同一页的结果，不是绝对概率），
237    /// 采用渲染结果和按块合并时用；不进 JSON、不进缓存。
238    #[serde(skip)]
239    pub quality: Option<f64>,
240    #[serde(skip_serializing_if = "Option::is_none")]
241    pub images: Option<Vec<String>>,
242    #[serde(skip_serializing_if = "Option::is_none")]
243    pub screenshot: Option<String>,
244    #[serde(skip_serializing_if = "Option::is_none")]
245    pub json: Option<Value>,
246    /// PageSift extension: items extracted with the `extract` format's CSS schema.
247    #[serde(skip_serializing_if = "Option::is_none")]
248    pub extract: Option<Value>,
249    #[serde(skip_serializing_if = "Option::is_none")]
250    pub summary: Option<String>,
251    #[serde(skip_serializing_if = "Option::is_none")]
252    pub actions: Option<Value>,
253    #[serde(skip_serializing_if = "Option::is_none")]
254    pub warning: Option<String>,
255    #[serde(skip_serializing_if = "Option::is_none")]
256    pub change_tracking: Option<ChangeTracking>,
257    #[serde(skip_serializing_if = "Option::is_none")]
258    pub chunks: Option<Vec<DocChunk>>,
259    /// PageSift extension: exported files (`exportImages`). JSON lists `{id, format, bytes, hash}`;
260    /// the content is downloaded from `GET /v2/attachments/{id}`.
261    #[serde(skip_serializing_if = "Vec::is_empty", default)]
262    pub attachments: Vec<super::Attachment>,
263    pub metadata: Metadata,
264}
265
266/// Firecrawl `data.pages[]`。
267#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
268#[serde(rename_all = "camelCase")]
269pub struct PageMarkdown {
270    /// 从 1 开始的页码。
271    pub page_number: usize,
272    pub markdown: String,
273}
274
275#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
276#[serde(rename_all = "camelCase")]
277pub struct DocChunk {
278    pub text: String,
279    pub heading_path: Vec<String>,
280    pub start: usize,
281    pub end: usize,
282    pub tokens: usize,
283}
284
285#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
286pub struct MapLink {
287    pub url: String,
288    #[serde(skip_serializing_if = "Option::is_none")]
289    pub title: Option<String>,
290    #[serde(skip_serializing_if = "Option::is_none")]
291    pub description: Option<String>,
292}
293
294#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
295pub struct ScrapeResponse {
296    pub success: bool,
297    pub data: Document,
298    #[serde(skip_serializing_if = "Option::is_none")]
299    pub warning: Option<String>,
300}
301
302#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
303#[serde(rename_all = "camelCase")]
304pub struct JobStarted {
305    pub success: bool,
306    pub id: String,
307    pub url: String,
308    #[serde(rename = "invalidURLs", skip_serializing_if = "Option::is_none")]
309    pub invalid_urls: Option<Vec<String>>,
310    /// Request fields that were accepted but ignored (`ignored_fields: a, b`).
311    #[serde(default, skip_serializing_if = "Vec::is_empty")]
312    pub warnings: Vec<String>,
313}
314
315#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
316#[serde(rename_all = "camelCase")]
317pub struct JobStatus {
318    pub success: bool,
319    pub status: String,
320    pub completed: usize,
321    pub total: usize,
322    pub credits_used: usize,
323    pub expires_at: String,
324    #[serde(skip_serializing_if = "Option::is_none")]
325    pub next: Option<String>,
326    pub data: Vec<Document>,
327    #[serde(skip_serializing_if = "Option::is_none")]
328    pub pagesift: Option<CrawlSummary>,
329}
330
331#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
332pub struct JobError {
333    pub id: String,
334    pub timestamp: String,
335    pub url: String,
336    pub code: String,
337    pub error: String,
338}
339
340#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
341#[serde(rename_all = "camelCase")]
342pub struct JobErrors {
343    pub errors: Vec<JobError>,
344    pub robots_blocked: Vec<String>,
345}
346
347#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
348pub struct MapResponse {
349    pub success: bool,
350    pub links: Vec<MapLink>,
351    /// Request fields that were accepted but ignored (`ignored_fields: a, b`).
352    #[serde(default, skip_serializing_if = "Vec::is_empty")]
353    pub warnings: Vec<String>,
354}
pagesift_api/types/response.rs

pagesift_api/types/
response.rs
