pagesift_api/types/
response.rs

1//! Firecrawl v1/v2 response types and metadata structures.
2
3use serde::{Deserialize, Serialize};
4use serde_json::Value;
5use std::collections::BTreeMap;
6
7/// Explanation when a query-directed crawl completes (`pagesift` field in job status).
8#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
9#[serde(rename_all = "camelCase")]
10pub struct CrawlSummary {
11    /// `coverage_reached`, `saturated`, `no_relevant_pages` (adaptive: nothing relevant found and
12    /// no lead left to follow), `limit_reached`, `exhausted`, `cancelled`.
13    pub stop_reason: String,
14    /// Coverage percentage (0.0 to 1.0) when adaptive crawl completes.
15    #[serde(skip_serializing_if = "Option::is_none")]
16    pub coverage: Option<f64>,
17    /// Number of distinct evidence blocks gathered for each term.
18    #[serde(skip_serializing_if = "BTreeMap::is_empty", default)]
19    pub term_evidence: BTreeMap<String, usize>,
20    /// Actual pages fetched.
21    pub pages: usize,
22}
23
24/// PageSift extensions inside `metadata.pagesift`.
25/// 带查询的爬取里一页的来历（`metadata.pagesift.crawl`）。
26#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
27#[serde(rename_all = "camelCase")]
28pub struct CrawlTrace {
29    /// 从种子页算起的深度；种子是 0。
30    pub depth: usize,
31    /// 发现这个链接的页面；种子和 sitemap 里的 URL 没有。
32    #[serde(skip_serializing_if = "Option::is_none")]
33    pub parent: Option<String>,
34    /// 链接入队时的分数（越高越先抓）；种子没有。
35    #[serde(skip_serializing_if = "Option::is_none")]
36    pub score: Option<f32>,
37    /// 附件（PDF、Office 文档）的来源页；普通页面没有。
38    #[serde(skip_serializing_if = "Option::is_none")]
39    pub attachment_of: Option<String>,
40}
41
42#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
43#[serde(rename_all = "camelCase")]
44pub struct PagesiftMeta {
45    #[serde(skip_serializing_if = "Option::is_none")]
46    pub content_hash: Option<String>,
47    #[serde(skip_serializing_if = "Option::is_none")]
48    pub raw_hash: Option<String>,
49    #[serde(skip_serializing_if = "Option::is_none")]
50    pub text_hash: Option<String>,
51    #[serde(skip_serializing_if = "Vec::is_empty", default)]
52    pub redirect_chain: Vec<String>,
53    #[serde(skip_serializing_if = "Option::is_none")]
54    pub etag: Option<String>,
55    #[serde(skip_serializing_if = "Option::is_none")]
56    pub last_modified: Option<String>,
57    #[serde(skip_serializing_if = "std::ops::Not::not", default)]
58    pub not_modified: bool,
59    /// `http` or `browser`.
60    #[serde(skip_serializing_if = "Option::is_none")]
61    pub engine: Option<String>,
62    /// `lightpanda`, `moli`, or `chrome`.
63    #[serde(skip_serializing_if = "Option::is_none")]
64    pub browser_engine: Option<String>,
65    #[serde(skip_serializing_if = "Vec::is_empty", default)]
66    pub warnings: Vec<String>,
67    /// Timing per phase in milliseconds.
68    #[serde(skip_serializing_if = "BTreeMap::is_empty", default)]
69    pub timings: BTreeMap<String, f64>,
70    /// Duplicate canonical reference URL.
71    #[serde(skip_serializing_if = "Option::is_none")]
72    pub duplicate_of: Option<String>,
73    /// Query relevance score (0.0 to 1.0).
74    #[serde(skip_serializing_if = "Option::is_none")]
75    pub relevance: Option<f32>,
76    /// 带查询的爬取里这一页是怎么来的：深度、父页、入队分数、附件的来源页。
77    #[serde(skip_serializing_if = "Option::is_none")]
78    pub crawl: Option<CrawlTrace>,
79    /// Query filter statistics.
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub query_filter: Option<QueryFilterStats>,
82    /// JSON-LD blocks found on the page, parsed and passed through unchanged.
83    #[serde(skip_serializing_if = "Vec::is_empty", default)]
84    pub json_ld: Vec<Value>,
85}
86
87#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
88#[serde(rename_all = "camelCase")]
89pub struct QueryFilterStats {
90    pub kept_blocks: usize,
91    pub total_blocks: usize,
92    pub tokens_before: usize,
93    pub tokens_after: usize,
94}
95
96/// Firecrawl `changeTracking` response object.
97#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
98#[serde(rename_all = "camelCase")]
99pub struct ChangeTracking {
100    pub previous_scrape_at: Option<String>,
101    /// `new`, `same`, `changed`, `removed`.
102    pub change_status: String,
103    /// `visible` or `hidden`.
104    pub visibility: String,
105    #[serde(skip_serializing_if = "Option::is_none")]
106    pub diff: Option<Value>,
107    #[serde(skip_serializing_if = "Option::is_none")]
108    pub json: Option<Value>,
109    #[serde(skip_serializing_if = "Option::is_none")]
110    pub pagesift: Option<Value>,
111}
112
113#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
114#[serde(rename_all = "camelCase")]
115pub struct Metadata {
116    #[serde(skip_serializing_if = "Option::is_none")]
117    pub title: Option<String>,
118    #[serde(skip_serializing_if = "Option::is_none")]
119    pub description: Option<String>,
120    #[serde(skip_serializing_if = "Option::is_none")]
121    pub language: Option<String>,
122    #[serde(skip_serializing_if = "Option::is_none")]
123    pub keywords: Option<String>,
124    #[serde(skip_serializing_if = "Option::is_none")]
125    pub robots: Option<String>,
126    #[serde(skip_serializing_if = "Option::is_none")]
127    pub favicon: Option<String>,
128    #[serde(skip_serializing_if = "Option::is_none")]
129    pub published_time: Option<String>,
130    #[serde(skip_serializing_if = "Option::is_none")]
131    pub author: Option<String>,
132    #[serde(skip_serializing_if = "Option::is_none")]
133    pub modified_time: Option<String>,
134    #[serde(skip_serializing_if = "Option::is_none")]
135    pub url: Option<String>,
136    #[serde(rename = "sourceURL", skip_serializing_if = "Option::is_none")]
137    pub source_url: Option<String>,
138    pub status_code: u16,
139    #[serde(skip_serializing_if = "Option::is_none")]
140    pub content_type: Option<String>,
141    #[serde(skip_serializing_if = "Option::is_none")]
142    pub error: Option<String>,
143    pub proxy_used: String,
144    #[serde(skip_serializing_if = "Option::is_none")]
145    pub cache_state: Option<String>,
146    #[serde(skip_serializing_if = "Option::is_none")]
147    pub cached_at: Option<String>,
148    #[serde(flatten)]
149    pub extra: BTreeMap<String, Value>,
150    #[serde(skip_serializing_if = "Option::is_none")]
151    pub pagesift: Option<PagesiftMeta>,
152}
153
154#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
155#[serde(rename_all = "camelCase")]
156pub struct Document {
157    #[serde(skip_serializing_if = "Option::is_none")]
158    pub markdown: Option<String>,
159    /// 每页的 Markdown：只在 `/v2/parse` 的 PDF 请求了 `parsers: [{type: "pdf", pages: true}]` 时给。
160    #[serde(skip_serializing_if = "Option::is_none")]
161    pub pages: Option<Vec<PageMarkdown>>,
162    #[serde(skip_serializing_if = "Option::is_none")]
163    pub html: Option<String>,
164    #[serde(skip_serializing_if = "Option::is_none")]
165    pub raw_html: Option<String>,
166    #[serde(skip_serializing_if = "Option::is_none")]
167    pub links: Option<Vec<String>>,
168    /// `links` 里每个链接的锚文本（同序），爬虫给链接打分用；不进 JSON。
169    #[serde(skip)]
170    pub link_text: Option<Vec<String>>,
171    /// 页面声明的其他语言版本（`hreflang`）：(语言, 地址)。爬虫找英文版用；不进 JSON。
172    #[serde(skip)]
173    pub alternates: Vec<(String, String)>,
174    #[serde(skip_serializing_if = "Option::is_none")]
175    pub images: Option<Vec<String>>,
176    #[serde(skip_serializing_if = "Option::is_none")]
177    pub screenshot: Option<String>,
178    #[serde(skip_serializing_if = "Option::is_none")]
179    pub json: Option<Value>,
180    #[serde(skip_serializing_if = "Option::is_none")]
181    pub summary: Option<String>,
182    #[serde(skip_serializing_if = "Option::is_none")]
183    pub actions: Option<Value>,
184    #[serde(skip_serializing_if = "Option::is_none")]
185    pub warning: Option<String>,
186    #[serde(skip_serializing_if = "Option::is_none")]
187    pub change_tracking: Option<ChangeTracking>,
188    #[serde(skip_serializing_if = "Option::is_none")]
189    pub chunks: Option<Vec<DocChunk>>,
190    pub metadata: Metadata,
191}
192
193/// Firecrawl `data.pages[]`。
194#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
195#[serde(rename_all = "camelCase")]
196pub struct PageMarkdown {
197    /// 从 1 开始的页码。
198    pub page_number: usize,
199    pub markdown: String,
200}
201
202#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
203#[serde(rename_all = "camelCase")]
204pub struct DocChunk {
205    pub text: String,
206    pub heading_path: Vec<String>,
207    pub start: usize,
208    pub end: usize,
209    pub tokens: usize,
210}
211
212#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
213pub struct MapLink {
214    pub url: String,
215    #[serde(skip_serializing_if = "Option::is_none")]
216    pub title: Option<String>,
217    #[serde(skip_serializing_if = "Option::is_none")]
218    pub description: Option<String>,
219}
220
221#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
222pub struct ScrapeResponse {
223    pub success: bool,
224    pub data: Document,
225    #[serde(skip_serializing_if = "Option::is_none")]
226    pub warning: Option<String>,
227}
228
229#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
230#[serde(rename_all = "camelCase")]
231pub struct JobStarted {
232    pub success: bool,
233    pub id: String,
234    pub url: String,
235    #[serde(rename = "invalidURLs", skip_serializing_if = "Option::is_none")]
236    pub invalid_urls: Option<Vec<String>>,
237}
238
239#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
240#[serde(rename_all = "camelCase")]
241pub struct JobStatus {
242    pub success: bool,
243    pub status: String,
244    pub completed: usize,
245    pub total: usize,
246    pub credits_used: usize,
247    pub expires_at: String,
248    #[serde(skip_serializing_if = "Option::is_none")]
249    pub next: Option<String>,
250    pub data: Vec<Document>,
251    #[serde(skip_serializing_if = "Option::is_none")]
252    pub pagesift: Option<CrawlSummary>,
253}
254
255#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
256pub struct JobError {
257    pub id: String,
258    pub timestamp: String,
259    pub url: String,
260    pub code: String,
261    pub error: String,
262}
263
264#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
265#[serde(rename_all = "camelCase")]
266pub struct JobErrors {
267    pub errors: Vec<JobError>,
268    pub robots_blocked: Vec<String>,
269}
270
271#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
272pub struct MapResponse {
273    pub success: bool,
274    pub links: Vec<MapLink>,
275}
pagesift_api/types/response.rs

pagesift_api/types/
response.rs
