spider_lib/pipelines/mod.rs
1//! Module for `spider-lib` item pipeline implementations.
2//!
3//! This module serves as a container for various concrete implementations
4//! of the `Pipeline` trait. Each submodule within this module provides
5//! a specific mechanism for processing, storing, or transforming
6//! `ScrapedItem`s after they have been extracted by a spider.
7//!
8//! It re-exports several built-in pipelines such as:
9//! - `console_writer`: For printing items to the console (debugging).
10//! - `csv_exporter`: For exporting items to CSV files.
11//! - `deduplication`: For filtering out duplicate items.
12//! - `json_writer`: For exporting items to a single JSON file.
13//! - `jsonl_writer`: For exporting items to JSON Lines files.
14//! - `sqlite_writer`: For persisting items to a SQLite database.
15
16pub mod console_writer;
17pub mod deduplication;
18
19#[cfg(feature = "pipeline-csv")]
20pub mod csv_exporter;
21
22#[cfg(feature = "pipeline-json")]
23pub mod json_writer;
24
25#[cfg(feature = "pipeline-json")]
26pub mod jsonl_writer;
27
28#[cfg(feature = "pipeline-sqlite")]
29pub mod sqlite_writer;