pagesift_api/types/
response.rs

1//! Firecrawl v1/v2 response types and metadata structures.
2
3use serde::{Deserialize, Serialize};
4use serde_json::Value;
5use std::collections::BTreeMap;
6
7/// Explanation when a query-directed crawl completes (`pagesift` field in job status).
8#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
9#[serde(rename_all = "camelCase")]
10pub struct CrawlSummary {
11    /// `coverage_reached`, `saturated`, `no_relevant_pages` (adaptive: nothing relevant found and
12    /// no lead left to follow), `limit_reached`, `exhausted`, `cancelled`.
13    pub stop_reason: String,
14    /// Coverage percentage (0.0 to 1.0) when adaptive crawl completes.
15    #[serde(skip_serializing_if = "Option::is_none")]
16    pub coverage: Option<f64>,
17    /// Number of distinct evidence blocks gathered for each term.
18    #[serde(skip_serializing_if = "BTreeMap::is_empty", default)]
19    pub term_evidence: BTreeMap<String, usize>,
20    /// Actual pages fetched.
21    pub pages: usize,
22}
23
24/// PageSift extensions inside `metadata.pagesift`.
25/// 带查询的爬取里一页的来历（`metadata.pagesift.crawl`）。
26#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
27#[serde(rename_all = "camelCase")]
28pub struct CrawlTrace {
29    /// 从种子页算起的深度；种子是 0。
30    pub depth: usize,
31    /// 发现这个链接的页面；种子和 sitemap 里的 URL 没有。
32    #[serde(skip_serializing_if = "Option::is_none")]
33    pub parent: Option<String>,
34    /// 链接入队时的分数（越高越先抓）；种子没有。
35    #[serde(skip_serializing_if = "Option::is_none")]
36    pub score: Option<f32>,
37    /// 附件（PDF、Office 文档）的来源页；普通页面没有。
38    #[serde(skip_serializing_if = "Option::is_none")]
39    pub attachment_of: Option<String>,
40}
41
42#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
43#[serde(rename_all = "camelCase")]
44pub struct PagesiftMeta {
45    #[serde(skip_serializing_if = "Option::is_none")]
46    pub content_hash: Option<String>,
47    #[serde(skip_serializing_if = "Option::is_none")]
48    pub raw_hash: Option<String>,
49    #[serde(skip_serializing_if = "Option::is_none")]
50    pub text_hash: Option<String>,
51    #[serde(skip_serializing_if = "Vec::is_empty", default)]
52    pub redirect_chain: Vec<String>,
53    #[serde(skip_serializing_if = "Option::is_none")]
54    pub etag: Option<String>,
55    #[serde(skip_serializing_if = "Option::is_none")]
56    pub last_modified: Option<String>,
57    #[serde(skip_serializing_if = "std::ops::Not::not", default)]
58    pub not_modified: bool,
59    /// `http` or `browser`.
60    #[serde(skip_serializing_if = "Option::is_none")]
61    pub engine: Option<String>,
62    /// `lightpanda`, `moli`, or `chrome`.
63    #[serde(skip_serializing_if = "Option::is_none")]
64    pub browser_engine: Option<String>,
65    #[serde(skip_serializing_if = "Vec::is_empty", default)]
66    pub warnings: Vec<String>,
67    /// Timing per phase in milliseconds.
68    #[serde(skip_serializing_if = "BTreeMap::is_empty", default)]
69    pub timings: BTreeMap<String, f64>,
70    /// Duplicate canonical reference URL.
71    #[serde(skip_serializing_if = "Option::is_none")]
72    pub duplicate_of: Option<String>,
73    /// Query relevance score (0.0 to 1.0).
74    #[serde(skip_serializing_if = "Option::is_none")]
75    pub relevance: Option<f32>,
76    /// 带查询的爬取里这一页是怎么来的：深度、父页、入队分数、附件的来源页。
77    #[serde(skip_serializing_if = "Option::is_none")]
78    pub crawl: Option<CrawlTrace>,
79    /// Query filter statistics.
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub query_filter: Option<QueryFilterStats>,
82    /// JSON-LD blocks found on the page, parsed and passed through unchanged.
83    #[serde(skip_serializing_if = "Vec::is_empty", default)]
84    pub json_ld: Vec<Value>,
85    /// How the result was produced: one entry per decision (render or not, merge or not) with its inputs.
86    /// Only present when the request asks for an explanation.
87    #[serde(skip_serializing_if = "Option::is_none")]
88    pub plan: Option<PlanTrace>,
89}
90
91/// `metadata.pagesift.plan`：每一步决策的依据。
92#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
93#[serde(rename_all = "camelCase")]
94pub struct PlanTrace {
95    pub steps: Vec<PlanStep>,
96}
97
98#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
99#[serde(rename_all = "camelCase")]
100pub struct PlanStep {
101    /// `render`（是否开浏览器）；`adopt`（是否采用渲染结果）或 `merge`（采用的是 HTTP 与渲染按块合并的结果）。
102    pub action: String,
103    pub chosen: bool,
104    /// 依据来自学到的后验；`false` 表示冷启动时沿用的规则。
105    pub learned: bool,
106    /// 用来决策的概率（Thompson 抽样值）。
107    #[serde(skip_serializing_if = "Option::is_none")]
108    pub p: Option<f64>,
109    /// 可得的价值（质量单位）。
110    #[serde(skip_serializing_if = "Option::is_none")]
111    pub value: Option<f64>,
112    /// 资源与时间的代价（质量单位）。
113    #[serde(skip_serializing_if = "Option::is_none")]
114    pub charge: Option<f64>,
115    /// 这一步之后结果的校准质量分。
116    #[serde(skip_serializing_if = "Option::is_none")]
117    pub quality: Option<f64>,
118    /// 合并时各块的来源：两边都有、只来自 HTTP、只来自渲染。
119    #[serde(skip_serializing_if = "Option::is_none")]
120    pub blocks: Option<[usize; 3]>, // [推导] 来源种类数
121}
122
123#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
124#[serde(rename_all = "camelCase")]
125pub struct QueryFilterStats {
126    pub kept_blocks: usize,
127    pub total_blocks: usize,
128    pub tokens_before: usize,
129    pub tokens_after: usize,
130}
131
132/// Firecrawl `changeTracking` response object.
133#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
134#[serde(rename_all = "camelCase")]
135pub struct ChangeTracking {
136    pub previous_scrape_at: Option<String>,
137    /// `new`, `same`, `changed`, `removed`.
138    pub change_status: String,
139    /// `visible` or `hidden`.
140    pub visibility: String,
141    #[serde(skip_serializing_if = "Option::is_none")]
142    pub diff: Option<Value>,
143    #[serde(skip_serializing_if = "Option::is_none")]
144    pub json: Option<Value>,
145    #[serde(skip_serializing_if = "Option::is_none")]
146    pub pagesift: Option<Value>,
147}
148
149#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
150#[serde(rename_all = "camelCase")]
151pub struct Metadata {
152    #[serde(skip_serializing_if = "Option::is_none")]
153    pub title: Option<String>,
154    #[serde(skip_serializing_if = "Option::is_none")]
155    pub description: Option<String>,
156    #[serde(skip_serializing_if = "Option::is_none")]
157    pub language: Option<String>,
158    #[serde(skip_serializing_if = "Option::is_none")]
159    pub keywords: Option<String>,
160    #[serde(skip_serializing_if = "Option::is_none")]
161    pub robots: Option<String>,
162    #[serde(skip_serializing_if = "Option::is_none")]
163    pub favicon: Option<String>,
164    #[serde(skip_serializing_if = "Option::is_none")]
165    pub published_time: Option<String>,
166    #[serde(skip_serializing_if = "Option::is_none")]
167    pub author: Option<String>,
168    #[serde(skip_serializing_if = "Option::is_none")]
169    pub modified_time: Option<String>,
170    #[serde(skip_serializing_if = "Option::is_none")]
171    pub url: Option<String>,
172    #[serde(rename = "sourceURL", skip_serializing_if = "Option::is_none")]
173    pub source_url: Option<String>,
174    pub status_code: u16,
175    #[serde(skip_serializing_if = "Option::is_none")]
176    pub content_type: Option<String>,
177    #[serde(skip_serializing_if = "Option::is_none")]
178    pub error: Option<String>,
179    pub proxy_used: String,
180    #[serde(skip_serializing_if = "Option::is_none")]
181    pub cache_state: Option<String>,
182    #[serde(skip_serializing_if = "Option::is_none")]
183    pub cached_at: Option<String>,
184    #[serde(flatten)]
185    pub extra: BTreeMap<String, Value>,
186    #[serde(skip_serializing_if = "Option::is_none")]
187    pub pagesift: Option<PagesiftMeta>,
188}
189
190#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
191#[serde(rename_all = "camelCase")]
192pub struct Document {
193    #[serde(skip_serializing_if = "Option::is_none")]
194    pub markdown: Option<String>,
195    /// 每页的 Markdown：只在 `/v2/parse` 的 PDF 请求了 `parsers: [{type: "pdf", pages: true}]` 时给。
196    #[serde(skip_serializing_if = "Option::is_none")]
197    pub pages: Option<Vec<PageMarkdown>>,
198    #[serde(skip_serializing_if = "Option::is_none")]
199    pub html: Option<String>,
200    #[serde(skip_serializing_if = "Option::is_none")]
201    pub raw_html: Option<String>,
202    #[serde(skip_serializing_if = "Option::is_none")]
203    pub links: Option<Vec<String>>,
204    /// `links` 里每个链接的锚文本（同序），爬虫给链接打分用；不进 JSON。
205    #[serde(skip)]
206    pub link_text: Option<Vec<String>>,
207    /// 页面声明的其他语言版本（`hreflang`）：(语言, 地址)。爬虫找英文版用；不进 JSON。
208    #[serde(skip)]
209    pub alternates: Vec<(String, String)>,
210    /// 引擎内部用的校准相对质量分（`pagesift_plan::quality`：只用来比较同一页的结果，不是绝对概率），
211    /// 采用渲染结果和按块合并时用；不进 JSON、不进缓存。
212    #[serde(skip)]
213    pub quality: Option<f64>,
214    #[serde(skip_serializing_if = "Option::is_none")]
215    pub images: Option<Vec<String>>,
216    #[serde(skip_serializing_if = "Option::is_none")]
217    pub screenshot: Option<String>,
218    #[serde(skip_serializing_if = "Option::is_none")]
219    pub json: Option<Value>,
220    #[serde(skip_serializing_if = "Option::is_none")]
221    pub summary: Option<String>,
222    #[serde(skip_serializing_if = "Option::is_none")]
223    pub actions: Option<Value>,
224    #[serde(skip_serializing_if = "Option::is_none")]
225    pub warning: Option<String>,
226    #[serde(skip_serializing_if = "Option::is_none")]
227    pub change_tracking: Option<ChangeTracking>,
228    #[serde(skip_serializing_if = "Option::is_none")]
229    pub chunks: Option<Vec<DocChunk>>,
230    /// PageSift extension: exported files (`exportImages`). JSON lists `{id, format, bytes, hash}`;
231    /// the content is downloaded from `GET /v2/attachments/{id}`.
232    #[serde(skip_serializing_if = "Vec::is_empty", default)]
233    pub attachments: Vec<super::Attachment>,
234    pub metadata: Metadata,
235}
236
237/// Firecrawl `data.pages[]`。
238#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
239#[serde(rename_all = "camelCase")]
240pub struct PageMarkdown {
241    /// 从 1 开始的页码。
242    pub page_number: usize,
243    pub markdown: String,
244}
245
246#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
247#[serde(rename_all = "camelCase")]
248pub struct DocChunk {
249    pub text: String,
250    pub heading_path: Vec<String>,
251    pub start: usize,
252    pub end: usize,
253    pub tokens: usize,
254}
255
256#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
257pub struct MapLink {
258    pub url: String,
259    #[serde(skip_serializing_if = "Option::is_none")]
260    pub title: Option<String>,
261    #[serde(skip_serializing_if = "Option::is_none")]
262    pub description: Option<String>,
263}
264
265#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
266pub struct ScrapeResponse {
267    pub success: bool,
268    pub data: Document,
269    #[serde(skip_serializing_if = "Option::is_none")]
270    pub warning: Option<String>,
271}
272
273#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
274#[serde(rename_all = "camelCase")]
275pub struct JobStarted {
276    pub success: bool,
277    pub id: String,
278    pub url: String,
279    #[serde(rename = "invalidURLs", skip_serializing_if = "Option::is_none")]
280    pub invalid_urls: Option<Vec<String>>,
281}
282
283#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
284#[serde(rename_all = "camelCase")]
285pub struct JobStatus {
286    pub success: bool,
287    pub status: String,
288    pub completed: usize,
289    pub total: usize,
290    pub credits_used: usize,
291    pub expires_at: String,
292    #[serde(skip_serializing_if = "Option::is_none")]
293    pub next: Option<String>,
294    pub data: Vec<Document>,
295    #[serde(skip_serializing_if = "Option::is_none")]
296    pub pagesift: Option<CrawlSummary>,
297}
298
299#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
300pub struct JobError {
301    pub id: String,
302    pub timestamp: String,
303    pub url: String,
304    pub code: String,
305    pub error: String,
306}
307
308#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
309#[serde(rename_all = "camelCase")]
310pub struct JobErrors {
311    pub errors: Vec<JobError>,
312    pub robots_blocked: Vec<String>,
313}
314
315#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
316pub struct MapResponse {
317    pub success: bool,
318    pub links: Vec<MapLink>,
319}
pagesift_api/types/response.rs

pagesift_api/types/
response.rs
