1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
// Copyright (c) 2025-2026 Kirky.X🌠
// SPDX-License-Identifier: Apache-2.0
//! 成功收尾与失败处理 — 从 `ScrapeWorker` 拆分的 partial impl block
//!
//! 包含抓取成功收尾(结果落库 + 任务终结 + webhook)、webhook 触发与
//! 失败重试分派。
use super::*;
impl ScrapeWorker {
pub(super) async fn handle_scrape_success(
&self,
task: &Task,
scrape_request_dto: Option<&ScrapeRequestDto>,
response: ScrapeResponse,
) -> Result<bool> {
debug!("task_id: {}", task.id);
// 文本编码处理 - 集成文本处理功能
// process_text_encoding 返回 Cow<'_, str>,禁用路径零 clone
let processed_content = Self::process_content_with_fallback(task, &response).await;
// PERF- 复用 process_scrape_task 已解析的 ScrapeRequestDto 引用,
// 不再在 handle_scrape_success 内部二次 from_value(task.payload.clone())。
let parsed_req = scrape_request_dto;
// 创建处理后的响应用于后续处理
// 调用 MarkdownPostProcessor(独立组件)
// 若 formats 含 "markdown" 则生成 Markdown,否则返回 Ok(None)
//
// (错误显性化):generate() 现返回 Result<Option<String>, _>,
// 区分"未请求 markdown"(Ok(None))与"转换失败/空结果"(Err)。
// 调用方策略:markdown 为增强字段,失败不阻断基础抓取结果,
// 错误时记录告警并继续(generated_markdown = None)。
// generate() 改为 async,支持 only_main_content 前置正文提取
let generated_markdown = self
.generate_markdown_or_warn(task, parsed_req, &processed_content)
.await;
// handle_scrape_success 改为 owned ScrapeResponse,
// 构造 processed_response 时直接 move 字段,避免 clone screenshot(100KB+)/headers/...
let processed_response = ScrapeResponse {
js_degraded: response.js_degraded,
content: processed_content,
status_code: response.status_code,
screenshot: response.screenshot,
content_type: response.content_type,
headers: response.headers,
response_time_ms: response.response_time_ms,
final_url: response.final_url,
markdown: generated_markdown.or(response.markdown),
captured_responses: response.captured_responses,
};
// Token 扣费延后:先收集用量,仅在赢得 mark_completed 守卫后统一扣除,
// 防止丢守卫的陈旧 worker 与认领方对同一次任务双扣费
let (extracted_data, pending_token_usages) = self
.run_extraction_stage(task, parsed_req, &processed_response.content)
.await;
// 先落库结果(save_result 按 task_id 幂等 upsert),再以锁守卫终结任务:
// 中间崩溃的窗口退化为"结果已存、任务可被僵尸恢复重跑",重跑后幂等覆盖
// 不会产生双结果;而旧顺序(先 Completed 后存结果)中间崩溃会留下
// "任务 Completed 但结果永久缺失"的不可恢复状态。
save_result(
task,
&processed_response,
extracted_data,
self.result_repository.as_ref(),
)
.await?;
let marked = self
.repository
.mark_completed(task.id, Some(self.worker_id))
.await?;
if marked == 0 {
warn!(
"Task {} no longer owned by worker {}, skipping token/feature credit \
deduction and webhook delivery",
task.id, self.worker_id
);
return Ok(false);
}
// 守卫胜利:统一执行延后的 token 扣费(仅本 worker 认领期间产生的用量)
for usage in &pending_token_usages {
self.deduct_token_credits(task.team_id, task.id, usage, "Tokens used for extraction")
.await;
}
debug!(
"task_id: {}, Successfully marked task as completed",
task.id
);
self.trigger_webhook(task, None).await;
Ok(true)
}
/// 文本编码处理并保留失败回退(自 [`Self::handle_scrape_success`] 提取):
/// `process_text_encoding` 返回 `Cow<'_, str>`,禁用路径零 clone;
/// 失败时 warn 并回退原始 content。
async fn process_content_with_fallback(task: &Task, response: &ScrapeResponse) -> String {
match process_text_encoding(task, response).await {
Ok(content) => content.into_owned(),
Err(e) => {
warn!(
"{}",
crate::i18n::tr_log_args(
"scrape-task-encoding-fallback",
&[("error", e.to_string())],
)
);
response.content.clone()
}
}
}
/// 按请求生成 Markdown;失败降级为 `None` 并告警(自
/// [`Self::handle_scrape_success`] 提取)。
#[cfg(feature = "content")]
async fn generate_markdown_or_warn(
&self,
task: &Task,
parsed_req: Option<&ScrapeRequestDto>,
processed_content: &str,
) -> Option<String> {
if let Some(req) = parsed_req {
self.markdown_post_processor
.generate(task.id, req, processed_content)
.await
.unwrap_or_else(|e| {
warn!(
"task_id: {}, markdown post-processing failed: {}",
task.id, e
);
None
})
} else {
None
}
}
/// content 特性关闭时不生成 Markdown(自
/// [`Self::handle_scrape_success`] 提取)。
#[cfg(not(feature = "content"))]
async fn generate_markdown_or_warn(
&self,
_task: &Task,
_parsed_req: Option<&ScrapeRequestDto>,
_processed_content: &str,
) -> Option<String> {
None
}
// `maybe_generate_markdown` 方法已迁移至 `MarkdownPostProcessor`(独立组件)。
// 原 `HtmdMarkdownService` 调用已替换为 `self.markdown_post_processor.generate(...)`,
// 由 `handle_scrape_success` 中调用。
pub(super) async fn trigger_webhook(&self, task: &Task, error_msg: Option<String>) {
let result = match error_msg {
Some(msg) => self.webhook_service.trigger_failure(task, msg).await,
None => self.webhook_service.trigger_completion(task).await,
};
if let Err(e) = result {
error!("Failed to trigger webhook for task {}: {}", task.id, e);
}
}
/// 返回重试器结论:`Retried` 表示任务已回队(**不是终态**),
/// `Failed` 表示重试预算耗尽/不可重试(终态)。crawl 层据此只在
/// 终态失败时累加 failed_tasks,避免把“首次可重试失败”当终态
/// (会提前把 crawl 标 Failed 并取消重试)。
pub(super) async fn handle_failure(
&self,
task: &mut Task,
error: &str,
) -> Result<crate::domain::services::retry_handler::HandleFailureResult> {
match self.retry_handler.handle_failure(task, error).await {
outcome @ crate::domain::services::retry_handler::HandleFailureResult::Retried {
..
} => Ok(outcome),
outcome @ crate::domain::services::retry_handler::HandleFailureResult::Failed => {
Ok(outcome)
}
outcome @ crate::domain::services::retry_handler::HandleFailureResult::Stale => {
Ok(outcome)
}
crate::domain::services::retry_handler::HandleFailureResult::Error(e) => Err(e),
}
}
}