pipeflow 0.0.4

A lightweight, configuration-driven data pipeline framework
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

[package]
name = "pipeflow"
version = "0.0.4"
edition = "2024"
rust-version = "1.92"
authors = ["Telepair <dev@telepair.io>"]
description = "A lightweight, configuration-driven data pipeline framework"
license = "MIT"
repository = "https://github.com/telepair/pipeflow"
documentation = "https://docs.rs/pipeflow"
readme = "README.md"
keywords = ["pipeline", "etl", "data", "streaming", "async"]
categories = ["asynchronous", "data-structures", "command-line-utilities"]

[[bin]]
name = "pipeflow"
path = "src/main.rs"

[lib]
name = "pipeflow"
path = "src/lib.rs"

[dependencies]
# CLI
clap = { version = "4", features = ["derive"] }
# Async runtime
tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time", "fs", "io-util", "signal"] }
tokio-stream = { version = "0.1", features = ["sync"] }

# Serialization
serde = { version = "1", features = ["derive"] }
serde_json = "1"
serde_yaml = "0.9"

# HTTP
reqwest = { version = "0.12", features = ["json"], optional = true }
axum = { version = "0.7", optional = true }
lettre = { version = "0.11", default-features = false, features = ["tokio1", "tokio1-native-tls", "smtp-transport", "builder"], optional = true }

# Database
sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite", "postgres", "json"], optional = true }

# Redis
redis = { version = "0.27", default-features = false, features = ["tokio-comp", "connection-manager"], optional = true }

# Utilities
uuid = { version = "1", features = ["v7", "serde"] }
chrono = { version = "0.4", features = ["serde"] }
chrono-tz = "0.10"
cron = { version = "0.12", optional = true }
humantime-serde = "1"
thiserror = "2"
async-trait = "0.1"
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
rand = "0.9"
futures = "0.3.31"
regex = "1.12.2"  # Required for filter step `matches` operator
digest = "0.10"    # Common trait for hash algorithms
md-5 = "0.10.6"
sha1 = "0.10.5"
sha2 = "0.10.8"
hex = "0.4.3"
base64 = "0.22.1"
humantime = "2.3.0"

[features]
default = ["http-client", "file", "notify"]
full = ["http-client", "http-server", "database", "file", "redis", "notify"]

# Sources
http-client = ["dep:reqwest", "dep:cron"]
http-server = ["dep:axum", "dep:reqwest"]
database = ["dep:sqlx", "dep:cron"]
file = []
redis = ["dep:redis", "dep:cron"]
notify = ["http-client", "dep:lettre"]

# Testing utilities (exposes internal APIs for testing)
test-utils = []

[dev-dependencies]
wiremock = "0.6"
tokio-test = "0.4"
pipeflow = { path = ".", features = ["test-utils"] }
futures = "0.3.31"
tempfile = "3.24.0"
anyhow = "1.0.100"