crawlrs 0.2.0

crawlrs - Enterprise-grade web scraping and crawling platform built with Rust
Documentation
// Copyright (c) 2025-2026 Kirky.X🌠
// SPDX-License-Identifier: Apache-2.0

//! Extract 任务处理方法
//!
//! 从 `ScrapeWorker` 提取的 extract 任务处理相关方法(partial impl block)。
//! 包含任务分发、规则/Prompt/Schema 提取和结果保存。

use super::extraction_billing::{ExtractionSettlement, ExtractionSpec};
use super::ScrapeWorker;
use super::{build_extract_request_fn, parse_extract_payload};
use crate::domain::models::{ScrapeResult, Task, TaskStatus};
use crate::engines::engine_client::ScrapeResponse;
use anyhow::Result;
use chrono::Utc;
use log::{debug, info, warn};
use serde_json::{json, Value};
use std::collections::HashMap;

use crate::workers::scrape_executor::process_text_encoding;

impl ScrapeWorker {
    pub(super) async fn process_extract_task(&self, mut task: Task) -> Result<()> {
        info!("Processing extract task {}", task.id);

        let (payload, url) = parse_extract_payload(&task)?;
        debug!("has_rules: {}", payload.rules.is_some());
        if let Some(ref rules) = payload.rules {
            debug!("rules_count: {}", rules.len());
        }

        // 2. 构建并执行 Scrape 请求
        let scrape_req =
            build_extract_request_fn(&url, self.settings.timeouts.engines.default_timeout_seconds);
        let scrape_resp = self.engine_client.scrape(&scrape_req).await?;

        // 3. 文本编码处理
        let processed_content = match process_text_encoding(&task, &scrape_resp).await {
            Ok(content) => content.into_owned(),
            Err(e) => {
                warn!(
                    "{}",
                    crate::i18n::tr_log_args(
                        "scrape-task-encoding-fallback",
                        &[("error", e.to_string())],
                    )
                );
                scrape_resp.content.clone()
            }
        };

        let processed_scrape_resp = ScrapeResponse {
            content: processed_content,
            ..scrape_resp
        };

        // 4. 根据不同的提取方式处理
        if let Some(rules) = payload.rules {
            return self
                .handle_rules_extraction(&mut task, &processed_scrape_resp, &rules, &url)
                .await;
        }

        if let Some(prompt) = payload.prompt {
            return self
                .handle_prompt_extraction(&mut task, &processed_scrape_resp, prompt, &url)
                .await;
        }

        if let Some(schema) = payload.schema {
            return self
                .handle_schema_extraction(&mut task, &processed_scrape_resp, &schema, &url)
                .await;
        }

        // Fallback: 无提取规则时保存原始结果
        self.save_extract_result(&mut task, &processed_scrape_resp, None, &url)
            .await
    }

    /// 处理基于规则的提取(Immediate 结算:提取成功即扣费,保持既有扣费点)
    pub(super) async fn handle_rules_extraction(
        &self,
        task: &mut Task,
        response: &ScrapeResponse,
        rules: &HashMap<String, crate::domain::services::extraction_service::ExtractionRule>,
        url: &str,
    ) -> Result<()> {
        debug!("rules: {:?}", rules);

        let extracted_data = self
            .dispatch_extraction(
                task,
                &response.content,
                url,
                &ExtractionSpec::Rules(rules),
                ExtractionSettlement::Immediate {
                    description: "Tokens used for extraction rules",
                },
            )
            .await?;

        self.save_extract_result(task, response, Some(extracted_data), url)
            .await
    }

    /// 处理基于 Prompt 的提取(Immediate 结算:提取成功即扣费,保持既有扣费点)
    pub(super) async fn handle_prompt_extraction(
        &self,
        task: &mut Task,
        response: &ScrapeResponse,
        prompt: String,
        url: &str,
    ) -> Result<()> {
        let extracted_data = self
            .dispatch_extraction(
                task,
                &response.content,
                url,
                &ExtractionSpec::Prompt(prompt.as_str()),
                ExtractionSettlement::Immediate {
                    description: "Tokens used for extraction",
                },
            )
            .await?;

        self.save_extract_result(task, response, Some(extracted_data), url)
            .await
    }

    /// 处理基于 Schema 的提取(Immediate 结算:提取成功即扣费,保持既有扣费点)
    pub(super) async fn handle_schema_extraction(
        &self,
        task: &mut Task,
        response: &ScrapeResponse,
        schema: &serde_json::Value,
        url: &str,
    ) -> Result<()> {
        let extracted_data = self
            .dispatch_extraction(
                task,
                &response.content,
                url,
                &ExtractionSpec::Schema(schema),
                ExtractionSettlement::Immediate {
                    description: "Tokens used for schema extraction",
                },
            )
            .await?;

        self.save_extract_result(task, response, Some(extracted_data), url)
            .await
    }

    /// 保存提取结果
    pub(super) async fn save_extract_result(
        &self,
        task: &mut Task,
        response: &ScrapeResponse,
        extracted_data: Option<Value>,
        url: &str,
    ) -> Result<()> {
        let meta_data = extracted_data
            .map(|data| json!({ "extracted_data": data }))
            .unwrap_or(json!({}));

        let scrape_result = ScrapeResult {
            id: crate::domain::id::Id::generate(),
            task_id: task.id,
            url: url.to_string(),
            status_code: response.status_code as i32,
            content: response.content.clone(),
            content_type: response.content_type.clone(),
            headers: serde_json::to_value(&response.headers).unwrap_or(json!({})),
            meta_data,
            screenshot: None,
            response_time_ms: response.response_time_ms as i64,
            created_at: Utc::now(),
        };

        self.result_repository.save(scrape_result).await?;

        task.status = TaskStatus::Completed;
        self.repository.update(task).await?;

        self.trigger_webhook(task, None).await;

        Ok(())
    }
}