kumo 0.3.9

An async web crawling framework for Rust - Scrapy for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

pub mod dedup;
pub mod filter;

pub use dedup::DropDuplicates;
pub use filter::{FilterPipeline, RequireFields};

use crate::error::KumoError;

/// Processing stage applied to every item before it reaches the `ItemStore`.
///
/// Pipelines are applied in registration order. Each stage receives the item
/// produced by the previous stage (or the raw scraped item for the first stage).
///
/// # Return values
///
/// - `Ok(Some(item))` — pass the item to the next stage / store.
/// - `Ok(None)` — **drop** the item silently (no error, no store write).
/// - `Err(e)` — log the error and drop the item; the crawl continues.
///
/// # Example
///
/// ```rust,ignore
/// use kumo::prelude::*;
/// use async_trait::async_trait;
///
/// struct PriceFilter;
///
/// #[async_trait]
/// impl Pipeline for PriceFilter {
///     async fn process(
///         &self,
///         item: serde_json::Value,
///     ) -> Result<Option<serde_json::Value>, KumoError> {
///         // Drop items where price is missing or zero.
///         let price = item["price"].as_f64().unwrap_or(0.0);
///         if price <= 0.0 { return Ok(None); }
///         Ok(Some(item))
///     }
/// }
/// ```
#[async_trait::async_trait]
pub trait Pipeline: Send + Sync {
    async fn process(
        &self,
        item: serde_json::Value,
    ) -> Result<Option<serde_json::Value>, KumoError>;
}