[package]
edition = "2024"
rust-version = "1.94"
name = "spate"
version = "0.1.0"
authors = ["Marcus Kainth <marcus@kainth.dev>"]
build = false
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "High-performance at-least-once ETL pipeline framework with a chaining operator API, checkpoint-driven source commits, sharded asynchronous sinks, backpressure, and Prometheus metrics."
homepage = "https://spate.kainth.dev"
readme = "README.md"
keywords = [
"etl",
"pipeline",
"streaming",
"kafka",
"clickhouse",
]
categories = [
"asynchronous",
"data-structures",
]
license = "Apache-2.0"
repository = "https://github.com/spate-etl/spate"
[package.metadata.docs.rs]
all-features = true
[features]
avro = ["dep:spate-avro"]
clickhouse = ["dep:spate-clickhouse"]
clickhouse-chrono = [
"clickhouse",
"spate-clickhouse/chrono",
]
clickhouse-rust-decimal = [
"clickhouse",
"spate-clickhouse/rust_decimal",
]
clickhouse-time = [
"clickhouse",
"spate-clickhouse/time",
]
clickhouse-uuid = [
"clickhouse",
"spate-clickhouse/uuid",
]
coordination = ["dep:spate-coordination"]
coordination-nats = [
"coordination",
"spate-coordination/nats",
]
default = []
full = [
"avro",
"json",
"kafka",
"clickhouse",
"coordination-nats",
"s3",
]
json = ["dep:spate-json"]
json-arbitrary-precision = [
"json",
"spate-json/arbitrary-precision",
]
json-float-roundtrip = [
"json",
"spate-json/float-roundtrip",
]
json-raw-value = [
"json",
"spate-json/raw-value",
]
json-simd = [
"json",
"spate-json/simd",
]
kafka = ["dep:spate-kafka"]
kafka-tls = [
"kafka",
"spate-kafka/tls",
]
s3 = ["dep:spate-s3"]
[lib]
name = "spate"
path = "src/lib.rs"
[[example]]
name = "clickhouse_aggregating_mv"
path = "examples/clickhouse_aggregating_mv.rs"
[[example]]
name = "coordinated_pipeline"
path = "examples/coordinated_pipeline.rs"
required-features = ["coordination"]
[[example]]
name = "custom_metrics"
path = "examples/custom_metrics.rs"
[[example]]
name = "custom_operator"
path = "examples/custom_operator.rs"
[[example]]
name = "custom_source_sink"
path = "examples/custom_source_sink.rs"
[[example]]
name = "json_ndjson_memory"
path = "examples/json_ndjson_memory.rs"
required-features = ["json"]
[[example]]
name = "kafka_avro_flatmap_clickhouse"
path = "examples/kafka_avro_flatmap_clickhouse.rs"
required-features = ["full"]
[[example]]
name = "kafka_avro_to_clickhouse"
path = "examples/kafka_avro_to_clickhouse.rs"
required-features = ["full"]
[[example]]
name = "kafka_to_kafka_split"
path = "examples/kafka_to_kafka_split.rs"
required-features = ["kafka"]
[[example]]
name = "memory_pipeline"
path = "examples/memory_pipeline.rs"
[[example]]
name = "multi_table_split"
path = "examples/multi_table_split.rs"
required-features = ["full"]
[[example]]
name = "s3_backfill"
path = "examples/s3_backfill.rs"
required-features = [
"s3",
"json",
]
[[example]]
name = "s3_coordinated_backfill"
path = "examples/s3_coordinated_backfill.rs"
required-features = [
"s3",
"json",
"coordination",
]
[[test]]
name = "e2e_drain_outage"
path = "tests/e2e_drain_outage.rs"
[[test]]
name = "e2e_drain_restart"
path = "tests/e2e_drain_restart.rs"
required-features = ["full"]
[[test]]
name = "e2e_happy"
path = "tests/e2e_happy.rs"
required-features = ["full"]
[[test]]
name = "e2e_poison"
path = "tests/e2e_poison.rs"
required-features = ["full"]
[[test]]
name = "e2e_rebalance"
path = "tests/e2e_rebalance.rs"
required-features = ["full"]
[[test]]
name = "e2e_sink_outage"
path = "tests/e2e_sink_outage.rs"
required-features = ["full"]
[dependencies.spate-avro]
version = "=0.1.0"
optional = true
[dependencies.spate-clickhouse]
version = "=0.1.0"
optional = true
[dependencies.spate-coordination]
version = "=0.1.0"
optional = true
default-features = false
[dependencies.spate-core]
version = "=0.1.0"
[dependencies.spate-json]
version = "=0.1.0"
optional = true
[dependencies.spate-kafka]
version = "=0.1.0"
optional = true
[dependencies.spate-s3]
version = "=0.1.0"
optional = true
[dev-dependencies.apache-avro]
version = "0.21"
[dev-dependencies.bytes]
version = "1.11.1"
[dev-dependencies.clickhouse]
version = "0.15"
[dev-dependencies.flate2]
version = "1.1"
[dev-dependencies.http-body-util]
version = "0.1.2"
[dev-dependencies.hyper]
version = "1.10"
features = [
"http1",
"server",
]
[dev-dependencies.hyper-util]
version = "0.1.12"
[dev-dependencies.metrics]
version = "0.24.2"
[dev-dependencies.rdkafka]
version = "0.39"
[dev-dependencies.serde]
version = "1.0.228"
features = ["derive"]
[dev-dependencies.serde_json]
version = "1.0.145"
[dev-dependencies.spate-test]
version = "=0.1.0"
[dev-dependencies.tempfile]
version = "3"
[dev-dependencies.testcontainers]
version = "0.27"
features = ["blocking"]
[dev-dependencies.testcontainers-modules]
version = "0.15"
features = ["kafka"]
[dev-dependencies.tokio]
version = "1.52"
features = [
"rt-multi-thread",
"macros",
"signal",
"time",
]
default-features = false
[dev-dependencies.tracing]
version = "0.1.44"
[lints.clippy]
dbg_macro = "warn"
print_stderr = "warn"
print_stdout = "warn"
undocumented_unsafe_blocks = "warn"
[lints.clippy.all]
level = "warn"
priority = -1
[lints.rust]
missing_debug_implementations = "warn"
missing_docs = "warn"
unreachable_pub = "warn"
unsafe_op_in_unsafe_fn = "deny"
[lints.rust.rust_2018_idioms]
level = "warn"
priority = -1
[lints.rust.unexpected_cfgs]
level = "warn"
priority = 0
check-cfg = ["cfg(loom)"]