mocra-core 0.4.0

The mocra crawler framework runtime: errors, cache, utilities, domain models, downloader, data-plane queue, coordination, scheduler and engine.
Documentation
use crate::cacheable::CacheAble;
use crate::common::model::Cookies;
use crate::common::model::ExecutionMark;
use crate::common::model::meta::MetaData;
use uuid::Uuid;

use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Response {
    pub id: Uuid,
    pub platform: String,
    pub account: String,
    pub module: String,
    pub status_code: u16,
    pub cookies: Cookies,
    pub content: Vec<u8>,
    pub storage_path: Option<String>,
    pub headers: Vec<(String, String)>,
    pub task_retry_times: usize,
    pub metadata: MetaData,
    pub download_middleware: Vec<String>,
    pub data_middleware: Vec<String>,
    pub task_finished: bool,
    pub context: ExecutionMark,
    pub run_id: Uuid,
    pub prefix_request: Uuid,
    pub request_hash: Option<String>,
    #[serde(default)]
    pub priority: crate::common::model::Priority,
}
impl Response {
    /// 响应体按 UTF-8 解码为字符串(严格;无效字节返回错误)。
    pub fn text(&self) -> crate::errors::Result<&str> {
        std::str::from_utf8(&self.content).map_err(|e| {
            crate::errors::Error::new(
                crate::errors::ErrorKind::Response,
                Some(format!("response body is not valid UTF-8: {e}")),
            )
        })
    }

    /// 响应体按 UTF-8 解码(无效字节以 U+FFFD 替换)。
    pub fn text_lossy(&self) -> std::borrow::Cow<'_, str> {
        String::from_utf8_lossy(&self.content)
    }

    /// 响应体反序列化为 JSON。
    pub fn json<T: serde::de::DeserializeOwned>(&self) -> crate::errors::Result<T> {
        serde_json::from_slice(&self.content).map_err(|e| {
            crate::errors::Error::new(
                crate::errors::ErrorKind::Response,
                Some(format!("response body is not valid JSON: {e}")),
            )
        })
    }

    pub fn task_id(&self) -> String {
        format!("{}-{}", self.account, self.platform)
    }
    pub fn module_id(&self) -> String {
        format!("{}-{}-{}", self.account, self.platform, self.module)
    }
    /// Run-scoped task identifier for error tracking.
    pub fn task_runtime_id(&self) -> String {
        format!("{}:{}:{}", self.platform, self.account, self.run_id)
    }
    /// Run-scoped module identifier for error tracking.
    pub fn module_runtime_id(&self) -> String {
        format!(
            "{}-{}-{}-{}",
            self.account, self.platform, self.module, self.run_id
        )
    }
    pub fn get_meta<T>(&self, key: &str) -> Option<T>
    where
        T: for<'de> Deserialize<'de>,
    {
        self.metadata.get_trait_config::<T>(key)
    }
    pub fn get_login_config<T>(&self, key: &str) -> Option<T>
    where
        T: for<'de> Deserialize<'de>,
    {
        self.metadata.get_login_config::<T>(key)
    }
    pub fn get_module_config<T>(&self, key: &str) -> Option<T>
    where
        T: for<'de> Deserialize<'de>,
    {
        self.metadata.get_module_config::<T>(key)
    }
    pub fn get_task_config<T>(&self, key: &str) -> Option<T>
    where
        T: for<'de> Deserialize<'de>,
    {
        self.metadata.get_task_config::<T>(key)
    }
    pub fn get_context(&self) -> &ExecutionMark {
        &self.context
    }
    pub fn with_context(mut self, ctx: ExecutionMark) -> Self {
        self.context = ctx;
        self
    }
}

impl CacheAble for Response {
    fn field() -> impl AsRef<str> {
        "response"
    }

    fn serialized_size_hint(&self) -> Option<usize> {
        Some(self.content.len() + self.headers.len() * 64)
    }

    fn clone_for_serialize(&self) -> Option<Self> {
        Some(self.clone())
    }
}

impl crate::common::model::priority::Prioritizable for Response {
    fn get_priority(&self) -> crate::common::model::Priority {
        self.priority
    }
}

use async_trait::async_trait;
#[async_trait]
impl crate::common::interface::storage::Offloadable for Response {
    fn should_offload(&self, threshold: usize) -> bool {
        self.content.len() > threshold && self.storage_path.is_none()
    }

    async fn offload(
        &mut self,
        storage: &std::sync::Arc<dyn crate::common::interface::storage::BlobStorage>,
    ) -> crate::errors::Result<()> {
        if self.content.is_empty() {
            return Ok(());
        }
        let key = format!("response/{}/{}.bin", self.run_id, self.id);
        let path = storage.put(&key, &self.content).await?;
        self.storage_path = Some(path);
        self.content.clear();
        self.content.shrink_to_fit();
        Ok(())
    }

    async fn reload(
        &mut self,
        storage: &std::sync::Arc<dyn crate::common::interface::storage::BlobStorage>,
    ) -> crate::errors::Result<()> {
        if let Some(path) = &self.storage_path {
            // If content is already present (e.g. reloaded twice), skip?
            // But if we want to ensure full content, we should check if empty.
            if self.content.is_empty() {
                let data = storage.get(path).await?;
                self.content = data;
            }
        }
        Ok(())
    }
}