use anyhow::{Context, Result};
use chrono::Utc;
use log::{debug, info, warn};
use serde_json::Value;
use crate::common::CacheContext;
use crate::domain::models::{ScrapeResult, Task};
use crate::domain::repositories::scrape_result_repository::ScrapeResultRepository;
use crate::engines::engine_client::ScrapeResponse;
use crate::i18n::tr_log_args;
use crate::infrastructure::oxcache::CacheService;
use crate::utils::crawl_text_integration::{
CrawlTextIntegration, ProcessedScrapeResponse, ScrapeResponseInput,
};
use crate::utils::robots::RobotsCheckerTrait;
use crate::utils::url_redaction::redact_url_for_log;
use crate::workers::cache_utils::SanitizedScrapeResponse;
pub async fn process_text_encoding<'a>(
task: &Task,
response: &'a ScrapeResponse,
) -> Result<std::borrow::Cow<'a, str>> {
info!(
"{}",
tr_log_args(
"scrape-exec-encoding-started",
&[
("task_id", task.id.to_string()),
("url", task.url.as_str().to_string()),
],
)
);
let text_integration = CrawlTextIntegration::new(false);
if !text_integration.is_enabled() {
return Ok(std::borrow::Cow::Borrowed(&response.content));
}
let input = ScrapeResponseInput {
content: response.content.as_bytes().to_vec(),
url: task.url.clone(),
content_type: Some(response.content_type.clone()),
status_code: response.status_code,
};
match text_integration
.process_scrape_response(
&input.content,
&input.url,
input.content_type.as_deref(),
input.status_code,
)
.await
{
Ok(processed_response) => encoding_result_to_cow(processed_response),
Err(e) => {
warn!(
"{}",
tr_log_args("scrape-exec-encoding-error", &[("error", e.to_string())],)
);
Err(anyhow::anyhow!("Text encoding processing exception: {}", e))
}
}
}
fn encoding_result_to_cow<'a>(
processed_response: ProcessedScrapeResponse,
) -> Result<std::borrow::Cow<'a, str>> {
if processed_response.processing_success {
info!(
"{}",
tr_log_args(
"scrape-exec-encoding-succeeded",
&[
(
"encoding",
processed_response
.encoding_detected
.clone()
.unwrap_or_else(|| "none".to_string())
.to_string()
),
("elapsed", processed_response.processing_success.to_string()),
(
"quality",
processed_response.processing_error.is_none().to_string()
),
],
)
);
Ok(std::borrow::Cow::Owned(
processed_response.processed_content,
))
} else {
let error_msg = processed_response
.processing_error
.unwrap_or_else(|| "Unknown error".to_string());
warn!(
"{}",
tr_log_args(
"scrape-exec-encoding-failed",
&[("error", error_msg.clone().to_string())],
)
);
Err(anyhow::anyhow!(
"Text encoding processing failed: {}",
error_msg
))
}
}
pub async fn save_result(
task: &Task,
response: &ScrapeResponse,
extra_data: Option<Value>,
result_repository: &dyn ScrapeResultRepository,
) -> Result<()> {
let mut meta_data = Value::Null;
if let Some(data) = extra_data {
meta_data = data;
}
if let Some(ref markdown) = response.markdown {
merge_meta_field(&mut meta_data, "markdown", Value::String(markdown.clone()));
}
if !response.captured_responses.is_empty() {
let captured = serde_json::to_value(&response.captured_responses).unwrap_or(Value::Null);
merge_meta_field(&mut meta_data, "captured_responses", captured);
}
let content_to_store = response.content.clone();
let result = ScrapeResult {
id: crate::domain::id::Id::generate(),
task_id: task.id,
url: task.url.clone(),
status_code: response.status_code as i32,
content: content_to_store,
content_type: response.content_type.clone(),
headers: serde_json::to_value(&response.headers).unwrap_or(Value::Null),
meta_data,
screenshot: response.screenshot.clone(),
response_time_ms: response.response_time_ms as i64,
created_at: Utc::now(),
};
result_repository.save(result).await?;
Ok(())
}
fn merge_meta_field(meta_data: &mut Value, key: &str, value: Value) {
match meta_data {
Value::Null => {
let mut map = serde_json::Map::new();
map.insert(key.to_string(), value);
*meta_data = Value::Object(map);
}
Value::Object(map) => {
map.insert(key.to_string(), value);
}
_ => {
let original = std::mem::replace(meta_data, Value::Null);
let mut map = serde_json::Map::new();
map.insert("extracted".to_string(), original);
map.insert(key.to_string(), value);
*meta_data = Value::Object(map);
}
}
}
pub async fn check_scrape_robots_allowed(
task: &Task,
respect_override: Option<bool>,
default_respect: bool,
user_agent: &str,
robots_checker: &dyn RobotsCheckerTrait,
) -> Result<bool> {
let respect = respect_override.unwrap_or(default_respect);
if !respect {
return Ok(true);
}
match robots_checker.is_allowed(&task.url, user_agent).await {
Ok(allowed) => Ok(allowed),
Err(e) => {
warn!(
"robots.txt fetch failed for task {} (fail-open): {}",
task.id, e
);
Ok(true)
}
}
}
pub async fn try_read_scrape_cache(
ctx: &CacheContext,
key: &str,
cache_service: &dyn CacheService,
) -> Result<Option<ScrapeResponse>> {
match cache_service.get(key).await {
Ok(Some(json)) => match serde_json::from_str::<ScrapeResponse>(&json) {
Ok(resp) => Ok(Some(resp)),
Err(e) => {
Err(anyhow::anyhow!(
"Cache deserialize failed url={} error={}",
redact_url_for_log(&ctx.url),
e
))
}
},
Ok(None) => Ok(None),
Err(e) => Err(anyhow::anyhow!("cache get failed: {}", e)),
}
}
pub async fn try_write_scrape_cache(
ctx: &CacheContext,
key: &str,
response: &ScrapeResponse,
cache_service: &dyn CacheService,
ttl_seconds: u64,
) -> Result<()> {
let sanitized = SanitizedScrapeResponse::from_response(response);
let json = serde_json::to_string(&sanitized)
.context("Failed to serialize ScrapeResponse for cache")?;
cache_service
.set(key, &json, ttl_seconds)
.await
.map_err(|e| anyhow::anyhow!("cache set failed: {}", e))?;
if log::log_enabled!(log::Level::Debug) {
debug!(
"Cache written url={} ttl={}s mode={:?}",
redact_url_for_log(&ctx.url),
ttl_seconds,
ctx.mode
);
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::common::CacheMode;
use crate::domain::models::{TaskStatus, TaskType};
use crate::domain::repositories::scrape_result_repository::ScrapeResultRepository;
use crate::engines::types::CapturedResponse;
use crate::infrastructure::oxcache::CacheService;
use chrono::Utc;
use serde_json::json;
use std::collections::HashMap;
use std::sync::atomic::{AtomicU32, Ordering};
fn make_task(url: &str) -> Task {
Task {
id: crate::domain::id::Id::generate(),
task_type: TaskType::Scrape,
status: TaskStatus::Queued,
priority: 0,
team_id: crate::domain::id::Id::generate(),
api_key_id: crate::domain::id::Id::generate(),
url: url.to_string(),
payload: json!({}),
retry_count: 0,
attempt_count: 0,
max_retries: 3,
scheduled_at: None,
created_at: Utc::now(),
started_at: None,
completed_at: None,
crawl_id: None,
updated_at: Utc::now(),
lock_token: None,
lock_expires_at: None,
expires_at: None,
}
}
struct FakeRobotsChecker {
allowed: bool,
err: Option<String>,
last_ua: std::sync::Mutex<Option<String>>,
}
impl FakeRobotsChecker {
fn new(allowed: bool) -> Self {
Self {
allowed,
err: None,
last_ua: std::sync::Mutex::new(None),
}
}
fn erroring(message: &str) -> Self {
Self {
allowed: false,
err: Some(message.to_string()),
last_ua: std::sync::Mutex::new(None),
}
}
fn last_ua(&self) -> Option<String> {
self.last_ua.lock().unwrap().clone()
}
}
#[async_trait::async_trait]
impl RobotsCheckerTrait for FakeRobotsChecker {
async fn is_allowed(&self, _url_str: &str, user_agent: &str) -> Result<bool> {
*self.last_ua.lock().unwrap() = Some(user_agent.to_string());
match &self.err {
Some(m) => Err(anyhow::anyhow!("robots fetch failed: {}", m)),
None => Ok(self.allowed),
}
}
async fn get_crawl_delay(
&self,
_url_str: &str,
_user_agent: &str,
) -> Result<Option<std::time::Duration>> {
Ok(None)
}
}
#[tokio::test]
async fn robots_disabled_by_default_allows_even_disallowed() {
let task = make_task("https://blocked.example.com/x");
let checker = FakeRobotsChecker::new(false);
let ok = check_scrape_robots_allowed(&task, None, false, "ua/1", &checker).await;
assert!(ok.unwrap());
assert_eq!(checker.last_ua(), None, "未启用时不应查询 robots");
}
#[tokio::test]
async fn robots_global_default_on_blocks_disallowed() {
let task = make_task("https://blocked.example.com/x");
let checker = FakeRobotsChecker::new(false);
let ok = check_scrape_robots_allowed(&task, None, true, "ua/1", &checker).await;
assert!(!ok.unwrap(), "Disallow 站点应被拦下");
assert_eq!(
checker.last_ua().as_deref(),
Some("ua/1"),
"匹配 UA 来自配置"
);
}
#[tokio::test]
async fn robots_request_true_overrides_global_off() {
let task = make_task("https://blocked.example.com/x");
let checker = FakeRobotsChecker::new(false);
let ok = check_scrape_robots_allowed(&task, Some(true), false, "ua/1", &checker).await;
assert!(!ok.unwrap(), "请求级 true 在全局关闭时仍生效");
}
#[tokio::test]
async fn robots_request_false_overrides_global_on() {
let task = make_task("https://blocked.example.com/x");
let checker = FakeRobotsChecker::new(false);
let ok = check_scrape_robots_allowed(&task, Some(false), true, "ua/1", &checker).await;
assert!(ok.unwrap(), "请求级 false 在全局开启时仍放行");
}
#[tokio::test]
async fn robots_checker_error_fails_open() {
let task = make_task("https://any.example.com/x");
let checker = FakeRobotsChecker::erroring("dns down");
let ok = check_scrape_robots_allowed(&task, None, true, "ua/1", &checker).await;
assert!(ok.unwrap(), "robots 获取失败 fail-open 放行");
}
#[tokio::test]
async fn robots_allowed_site_passes() {
let task = make_task("https://news.example.com/");
let checker = FakeRobotsChecker::new(true);
let ok = check_scrape_robots_allowed(&task, None, true, "ua/1", &checker).await;
assert!(ok.unwrap());
assert_eq!(checker.last_ua().as_deref(), Some("ua/1"));
}
fn make_response(content: &str) -> ScrapeResponse {
ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: content.to_string(),
status_code: 200,
content_type: "text/html".to_string(),
headers: HashMap::new(),
screenshot: None,
response_time_ms: 100,
final_url: None,
markdown: None,
}
}
#[derive(Default)]
struct MockResultRepo {
save_count: AtomicU32,
saved: std::sync::Mutex<Vec<ScrapeResult>>,
}
#[async_trait::async_trait]
impl ScrapeResultRepository for MockResultRepo {
async fn save(&self, result: ScrapeResult) -> Result<()> {
self.save_count.fetch_add(1, Ordering::SeqCst);
self.saved.lock().expect("repo lock").push(result);
Ok(())
}
async fn find_by_task_id(
&self,
_task_id: crate::domain::id::Id,
) -> Result<Option<ScrapeResult>> {
Ok(None)
}
async fn find_by_task_ids(
&self,
_task_ids: &[crate::domain::id::Id],
) -> Result<Vec<ScrapeResult>> {
Ok(vec![])
}
async fn get_team_avg_response_time(&self, _team_id: crate::domain::id::Id) -> Result<f64> {
Ok(0.0)
}
async fn cleanup_expired(&self, _retention_days: i64) -> anyhow::Result<u64> {
Ok(0)
}
}
use std::future::Future;
use std::pin::Pin;
#[derive(Default)]
struct MockCacheService {
get_count: AtomicU32,
set_count: AtomicU32,
}
#[async_trait::async_trait]
impl CacheService for MockCacheService {
fn get(
&self,
_key: &str,
) -> Pin<Box<dyn Future<Output = anyhow::Result<Option<String>>> + Send + '_>> {
self.get_count.fetch_add(1, Ordering::SeqCst);
Box::pin(async { Ok(None) })
}
fn set(
&self,
_key: &str,
_value: &str,
_ttl_seconds: u64,
) -> Pin<Box<dyn Future<Output = anyhow::Result<()>> + Send + '_>> {
self.set_count.fetch_add(1, Ordering::SeqCst);
Box::pin(async { Ok(()) })
}
fn delete(
&self,
_key: &str,
) -> Pin<Box<dyn Future<Output = anyhow::Result<()>> + Send + '_>> {
Box::pin(async { Ok(()) })
}
fn exists(
&self,
_key: &str,
) -> Pin<Box<dyn Future<Output = anyhow::Result<bool>> + Send + '_>> {
Box::pin(async { Ok(false) })
}
}
#[tokio::test]
async fn test_process_text_encoding_returns_borrowed_when_disabled() {
let task = make_task("https://example.com");
let response = make_response("Hello World");
let result = process_text_encoding(&task, &response).await.unwrap();
assert_eq!(result.as_ref(), "Hello World");
}
#[tokio::test]
async fn test_save_result_stores_in_repository() {
let task = make_task("https://example.com");
let response = make_response("<html>test</html>");
let repo = MockResultRepo::default();
save_result(&task, &response, None, &repo).await.unwrap();
assert_eq!(repo.save_count.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_save_result_with_extra_data() {
let task = make_task("https://example.com");
let response = make_response("content");
let repo = MockResultRepo::default();
let extra = json!({"key": "value"});
save_result(&task, &response, Some(extra), &repo)
.await
.unwrap();
assert_eq!(repo.save_count.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_save_result_with_markdown() {
let task = make_task("https://example.com");
let mut response = make_response("content");
response.markdown = Some("# Title".to_string());
let repo = MockResultRepo::default();
save_result(&task, &response, None, &repo).await.unwrap();
assert_eq!(repo.save_count.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_save_result_with_markdown_and_object_extra_data() {
let task = make_task("https://example.com");
let mut response = make_response("content");
response.markdown = Some("# MD".to_string());
let repo = MockResultRepo::default();
let extra = json!({"key": "value"});
save_result(&task, &response, Some(extra), &repo)
.await
.unwrap();
assert_eq!(repo.save_count.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn save_result_persists_captured_responses_into_meta_data() {
let task = make_task("https://example.com");
let mut response = make_response("content");
response.captured_responses = vec![CapturedResponse {
url: "https://api.example.com/data".to_string(),
status: 200,
mime_type: "application/json".to_string(),
body: "{\"ok\":true}".to_string(),
}];
let repo = MockResultRepo::default();
save_result(&task, &response, None, &repo).await.unwrap();
let saved = repo.saved.lock().expect("repo lock");
assert_eq!(saved.len(), 1);
assert_eq!(
saved[0].meta_data,
json!({"captured_responses": [{
"url": "https://api.example.com/data",
"status": 200,
"mime_type": "application/json",
"body": "{\"ok\":true}",
}]}),
"captured_responses 必须持久化进 meta_data(API 经 GET 结果透出)"
);
}
#[tokio::test]
async fn save_result_captured_coexists_with_markdown_and_extracted() {
let task = make_task("https://example.com");
let mut response = make_response("content");
response.markdown = Some("# MD".to_string());
response.captured_responses = vec![CapturedResponse {
url: "https://api.example.com/x".to_string(),
status: 204,
mime_type: "text/plain".to_string(),
body: "x".to_string(),
}];
let repo = MockResultRepo::default();
let extra = json!({"author": "a"});
save_result(&task, &response, Some(extra), &repo)
.await
.unwrap();
let saved = repo.saved.lock().expect("repo lock");
let meta = &saved[0].meta_data;
assert_eq!(meta["author"], "a");
assert_eq!(meta["markdown"], "# MD");
assert_eq!(meta["captured_responses"][0]["status"], 204);
assert_eq!(
meta["captured_responses"][0]["url"],
"https://api.example.com/x"
);
assert!(
meta.get("extracted").is_none(),
"对象型 extra 不得再包 extracted 层"
);
}
#[tokio::test]
async fn save_result_without_captured_keeps_meta_data_free_of_key() {
let task = make_task("https://example.com");
let response = make_response("content");
let repo = MockResultRepo::default();
save_result(&task, &response, None, &repo).await.unwrap();
let saved = repo.saved.lock().expect("repo lock");
assert_eq!(
saved[0].meta_data,
Value::Null,
"空捕获不得引入 captured_responses 键"
);
}
#[tokio::test]
async fn test_save_result_with_markdown_and_non_object_extra_data() {
let task = make_task("https://example.com");
let mut response = make_response("content");
response.markdown = Some("# MD".to_string());
let repo = MockResultRepo::default();
let extra = json!(["item1", "item2"]);
save_result(&task, &response, Some(extra), &repo)
.await
.unwrap();
assert_eq!(repo.save_count.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_try_read_scrape_cache_miss_returns_none() {
let ctx = CacheContext {
team_id: crate::domain::id::Id::nil(),
url: "https://example.com".to_string(),
method: crate::common::HttpMethod::Get,
mode: CacheMode::Enabled,
};
let cache = MockCacheService::default();
let result = try_read_scrape_cache(&ctx, "key", &cache).await.unwrap();
assert!(result.is_none());
assert_eq!(cache.get_count.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_try_write_scrape_cache_writes_serialized_response() {
let ctx = CacheContext {
team_id: crate::domain::id::Id::nil(),
url: "https://example.com".to_string(),
method: crate::common::HttpMethod::Get,
mode: CacheMode::Enabled,
};
let response = make_response("content");
let cache = MockCacheService::default();
try_write_scrape_cache(&ctx, "key", &response, &cache, 300)
.await
.unwrap();
assert_eq!(cache.set_count.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_try_write_scrape_cache_key_matches_read_key() {
let ctx = CacheContext {
team_id: crate::domain::id::Id::nil(),
url: "https://example.com".to_string(),
method: crate::common::HttpMethod::Get,
mode: CacheMode::Enabled,
};
let options = crate::engines::engine_client::ScrapeOptions {
method: crate::common::HttpMethod::Get,
body: None,
headers: std::collections::HashMap::new(),
timeout: std::time::Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: vec![],
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
cache_mode: None,
wait_for: None,
needs_mllm: false,
};
let key = crate::workers::cache_utils::generate_scrape_cache_key(&ctx, &options);
assert!(key.contains("example.com"));
}
struct ErrorCacheService {
get_err: bool,
set_err: bool,
}
impl ErrorCacheService {
fn get_error() -> Self {
Self {
get_err: true,
set_err: false,
}
}
fn set_error() -> Self {
Self {
get_err: false,
set_err: true,
}
}
}
#[async_trait::async_trait]
impl CacheService for ErrorCacheService {
fn get(
&self,
_key: &str,
) -> std::pin::Pin<
Box<dyn std::future::Future<Output = anyhow::Result<Option<String>>> + Send + '_>,
> {
if self.get_err {
Box::pin(async { Err(anyhow::anyhow!("cache get error")) })
} else {
Box::pin(async { Ok(None) })
}
}
fn set(
&self,
_key: &str,
_value: &str,
_ttl: u64,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + Send + '_>>
{
if self.set_err {
Box::pin(async { Err(anyhow::anyhow!("cache set error")) })
} else {
Box::pin(async { Ok(()) })
}
}
fn delete(
&self,
_key: &str,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + Send + '_>>
{
Box::pin(async { Ok(()) })
}
fn exists(
&self,
_key: &str,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<bool>> + Send + '_>>
{
Box::pin(async { Ok(false) })
}
}
struct CorruptCacheService;
#[async_trait::async_trait]
impl CacheService for CorruptCacheService {
fn get(
&self,
_key: &str,
) -> std::pin::Pin<
Box<dyn std::future::Future<Output = anyhow::Result<Option<String>>> + Send + '_>,
> {
Box::pin(async { Ok(Some("not-valid-json".to_string())) })
}
fn set(
&self,
_key: &str,
_value: &str,
_ttl: u64,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + Send + '_>>
{
Box::pin(async { Ok(()) })
}
fn delete(
&self,
_key: &str,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + Send + '_>>
{
Box::pin(async { Ok(()) })
}
fn exists(
&self,
_key: &str,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<bool>> + Send + '_>>
{
Box::pin(async { Ok(false) })
}
}
#[tokio::test]
async fn test_try_read_scrape_cache_get_error() {
let ctx = CacheContext {
team_id: crate::domain::id::Id::nil(),
url: "https://example.com".to_string(),
method: crate::common::HttpMethod::Get,
mode: CacheMode::Enabled,
};
let cache = ErrorCacheService::get_error();
let result = try_read_scrape_cache(&ctx, "key", &cache).await;
assert!(result.is_err(), "cache get error should propagate");
}
#[tokio::test]
async fn test_try_read_scrape_cache_corrupt_data() {
let ctx = CacheContext {
team_id: crate::domain::id::Id::nil(),
url: "https://example.com".to_string(),
method: crate::common::HttpMethod::Get,
mode: CacheMode::Enabled,
};
let cache = CorruptCacheService;
let result = try_read_scrape_cache(&ctx, "key", &cache).await;
assert!(
result.is_err(),
"corrupt cache data should return deserialization error"
);
}
#[tokio::test]
async fn test_try_write_scrape_cache_set_error() {
let ctx = CacheContext {
team_id: crate::domain::id::Id::nil(),
url: "https://example.com".to_string(),
method: crate::common::HttpMethod::Get,
mode: CacheMode::Enabled,
};
let response = make_response("content");
let cache = ErrorCacheService::set_error();
let result = try_write_scrape_cache(&ctx, "key", &response, &cache, 300).await;
assert!(result.is_err(), "cache set error should propagate");
}
}