use super::extraction_billing::{ExtractionSettlement, ExtractionSpec};
use super::ScrapeWorker;
use super::{build_extract_request_fn, parse_extract_payload};
use crate::domain::models::{ScrapeResult, Task, TaskStatus};
use crate::engines::engine_client::ScrapeResponse;
use anyhow::Result;
use chrono::Utc;
use log::{debug, info, warn};
use serde_json::{json, Value};
use std::collections::HashMap;
use crate::workers::scrape_executor::process_text_encoding;
impl ScrapeWorker {
pub(super) async fn process_extract_task(&self, mut task: Task) -> Result<()> {
info!("Processing extract task {}", task.id);
let (payload, url) = parse_extract_payload(&task)?;
debug!("has_rules: {}", payload.rules.is_some());
if let Some(ref rules) = payload.rules {
debug!("rules_count: {}", rules.len());
}
let scrape_req =
build_extract_request_fn(&url, self.settings.timeouts.engines.default_timeout_seconds);
let scrape_resp = self.engine_client.scrape(&scrape_req).await?;
let processed_content = match process_text_encoding(&task, &scrape_resp).await {
Ok(content) => content.into_owned(),
Err(e) => {
warn!(
"{}",
crate::i18n::tr_log_args(
"scrape-task-encoding-fallback",
&[("error", e.to_string())],
)
);
scrape_resp.content.clone()
}
};
let processed_scrape_resp = ScrapeResponse {
content: processed_content,
..scrape_resp
};
if let Some(rules) = payload.rules {
return self
.handle_rules_extraction(&mut task, &processed_scrape_resp, &rules, &url)
.await;
}
if let Some(prompt) = payload.prompt {
return self
.handle_prompt_extraction(&mut task, &processed_scrape_resp, prompt, &url)
.await;
}
if let Some(schema) = payload.schema {
return self
.handle_schema_extraction(&mut task, &processed_scrape_resp, &schema, &url)
.await;
}
self.save_extract_result(&mut task, &processed_scrape_resp, None, &url)
.await
}
pub(super) async fn handle_rules_extraction(
&self,
task: &mut Task,
response: &ScrapeResponse,
rules: &HashMap<String, crate::domain::services::extraction_service::ExtractionRule>,
url: &str,
) -> Result<()> {
debug!("rules: {:?}", rules);
let extracted_data = self
.dispatch_extraction(
task,
&response.content,
url,
&ExtractionSpec::Rules(rules),
ExtractionSettlement::Immediate {
description: "Tokens used for extraction rules",
},
)
.await?;
self.save_extract_result(task, response, Some(extracted_data), url)
.await
}
pub(super) async fn handle_prompt_extraction(
&self,
task: &mut Task,
response: &ScrapeResponse,
prompt: String,
url: &str,
) -> Result<()> {
let extracted_data = self
.dispatch_extraction(
task,
&response.content,
url,
&ExtractionSpec::Prompt(prompt.as_str()),
ExtractionSettlement::Immediate {
description: "Tokens used for extraction",
},
)
.await?;
self.save_extract_result(task, response, Some(extracted_data), url)
.await
}
pub(super) async fn handle_schema_extraction(
&self,
task: &mut Task,
response: &ScrapeResponse,
schema: &serde_json::Value,
url: &str,
) -> Result<()> {
let extracted_data = self
.dispatch_extraction(
task,
&response.content,
url,
&ExtractionSpec::Schema(schema),
ExtractionSettlement::Immediate {
description: "Tokens used for schema extraction",
},
)
.await?;
self.save_extract_result(task, response, Some(extracted_data), url)
.await
}
pub(super) async fn save_extract_result(
&self,
task: &mut Task,
response: &ScrapeResponse,
extracted_data: Option<Value>,
url: &str,
) -> Result<()> {
let meta_data = extracted_data
.map(|data| json!({ "extracted_data": data }))
.unwrap_or(json!({}));
let scrape_result = ScrapeResult {
id: crate::domain::id::Id::generate(),
task_id: task.id,
url: url.to_string(),
status_code: response.status_code as i32,
content: response.content.clone(),
content_type: response.content_type.clone(),
headers: serde_json::to_value(&response.headers).unwrap_or(json!({})),
meta_data,
screenshot: None,
response_time_ms: response.response_time_ms as i64,
created_at: Utc::now(),
};
self.result_repository.save(scrape_result).await?;
task.status = TaskStatus::Completed;
self.repository.update(task).await?;
self.trigger_webhook(task, None).await;
Ok(())
}
}