pipeflow/
lib.rs

1//! pipeflow - A lightweight, configuration-driven data pipeline framework
2//!
3//! # Overview
4//!
5//! pipeflow follows the classic ETL pattern:
6//!
7//! ```text
8//! Source → Transform → Sink
9//! ```
10//!
11//! Pipelines are defined in YAML configuration files, requiring no code to set up
12//! common data processing workflows.
13//!
14//! # Quick Start
15//!
16//! ```yaml
17//! # pipeline.yaml
18//!
19//! pipeline:
20//!   sources:
21//!     - id: api_poller
22//!       type: http_client
23//!       config:
24//!         url: "https://httpbin.org/json"
25//!         interval: "10s" # supports "30s", "5m", "1h 30m", etc.
26//!
27//!   sinks:
28//!     - id: console_out
29//!       type: console
30//!       inputs: [api_poller]
31//!       config:
32//!         format: pretty
33//! ```
34//!
35//! # Features
36//!
37//! - **Configuration-driven**: Define pipelines in YAML
38//! - **Backpressure**: Bounded channels with configurable capacity
39//! - **Fan-out**: One source/transform can feed multiple downstream nodes
40
41#![warn(missing_docs)]
42#![warn(clippy::all)]
43#![deny(unsafe_code)]
44
45pub mod config;
46pub mod engine;
47pub mod error;
48pub mod message;
49pub mod sink;
50pub mod source;
51pub mod transform;
52
53// Re-exports
54pub use error::{Error, Result};
55pub use message::{DeadLetter, Message, MessageMeta, SharedMessage};
56
57/// Prelude module for convenient imports
58pub mod prelude {
59    pub use crate::config::Config;
60    pub use crate::engine::Engine;
61    pub use crate::error::{Error, Result};
62    pub use crate::message::{DeadLetter, Message, MessageMeta, SharedMessage};
63    pub use crate::sink::Sink;
64    pub use crate::source::Source;
65    pub use crate::transform::Transform;
66}
pipeflow/lib.rs

pipeflow/
lib.rs
