spate 0.1.0

High-performance at-least-once ETL pipeline framework with a chaining operator API, checkpoint-driven source commits, sharded asynchronous sinks, backpressure, and Prometheus metrics.
Documentation
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.

[package]
edition = "2024"
rust-version = "1.94"
name = "spate"
version = "0.1.0"
authors = ["Marcus Kainth <marcus@kainth.dev>"]
build = false
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "High-performance at-least-once ETL pipeline framework with a chaining operator API, checkpoint-driven source commits, sharded asynchronous sinks, backpressure, and Prometheus metrics."
homepage = "https://spate.kainth.dev"
readme = "README.md"
keywords = [
    "etl",
    "pipeline",
    "streaming",
    "kafka",
    "clickhouse",
]
categories = [
    "asynchronous",
    "data-structures",
]
license = "Apache-2.0"
repository = "https://github.com/spate-etl/spate"

[package.metadata.docs.rs]
all-features = true

[features]
avro = ["dep:spate-avro"]
clickhouse = ["dep:spate-clickhouse"]
clickhouse-chrono = [
    "clickhouse",
    "spate-clickhouse/chrono",
]
clickhouse-rust-decimal = [
    "clickhouse",
    "spate-clickhouse/rust_decimal",
]
clickhouse-time = [
    "clickhouse",
    "spate-clickhouse/time",
]
clickhouse-uuid = [
    "clickhouse",
    "spate-clickhouse/uuid",
]
coordination = ["dep:spate-coordination"]
coordination-nats = [
    "coordination",
    "spate-coordination/nats",
]
default = []
full = [
    "avro",
    "json",
    "kafka",
    "clickhouse",
    "coordination-nats",
    "s3",
]
json = ["dep:spate-json"]
json-arbitrary-precision = [
    "json",
    "spate-json/arbitrary-precision",
]
json-float-roundtrip = [
    "json",
    "spate-json/float-roundtrip",
]
json-raw-value = [
    "json",
    "spate-json/raw-value",
]
json-simd = [
    "json",
    "spate-json/simd",
]
kafka = ["dep:spate-kafka"]
kafka-tls = [
    "kafka",
    "spate-kafka/tls",
]
s3 = ["dep:spate-s3"]

[lib]
name = "spate"
path = "src/lib.rs"

[[example]]
name = "clickhouse_aggregating_mv"
path = "examples/clickhouse_aggregating_mv.rs"

[[example]]
name = "coordinated_pipeline"
path = "examples/coordinated_pipeline.rs"
required-features = ["coordination"]

[[example]]
name = "custom_metrics"
path = "examples/custom_metrics.rs"

[[example]]
name = "custom_operator"
path = "examples/custom_operator.rs"

[[example]]
name = "custom_source_sink"
path = "examples/custom_source_sink.rs"

[[example]]
name = "json_ndjson_memory"
path = "examples/json_ndjson_memory.rs"
required-features = ["json"]

[[example]]
name = "kafka_avro_flatmap_clickhouse"
path = "examples/kafka_avro_flatmap_clickhouse.rs"
required-features = ["full"]

[[example]]
name = "kafka_avro_to_clickhouse"
path = "examples/kafka_avro_to_clickhouse.rs"
required-features = ["full"]

[[example]]
name = "kafka_to_kafka_split"
path = "examples/kafka_to_kafka_split.rs"
required-features = ["kafka"]

[[example]]
name = "memory_pipeline"
path = "examples/memory_pipeline.rs"

[[example]]
name = "multi_table_split"
path = "examples/multi_table_split.rs"
required-features = ["full"]

[[example]]
name = "s3_backfill"
path = "examples/s3_backfill.rs"
required-features = [
    "s3",
    "json",
]

[[example]]
name = "s3_coordinated_backfill"
path = "examples/s3_coordinated_backfill.rs"
required-features = [
    "s3",
    "json",
    "coordination",
]

[[test]]
name = "e2e_drain_outage"
path = "tests/e2e_drain_outage.rs"

[[test]]
name = "e2e_drain_restart"
path = "tests/e2e_drain_restart.rs"
required-features = ["full"]

[[test]]
name = "e2e_happy"
path = "tests/e2e_happy.rs"
required-features = ["full"]

[[test]]
name = "e2e_poison"
path = "tests/e2e_poison.rs"
required-features = ["full"]

[[test]]
name = "e2e_rebalance"
path = "tests/e2e_rebalance.rs"
required-features = ["full"]

[[test]]
name = "e2e_sink_outage"
path = "tests/e2e_sink_outage.rs"
required-features = ["full"]

[dependencies.spate-avro]
version = "=0.1.0"
optional = true

[dependencies.spate-clickhouse]
version = "=0.1.0"
optional = true

[dependencies.spate-coordination]
version = "=0.1.0"
optional = true
default-features = false

[dependencies.spate-core]
version = "=0.1.0"

[dependencies.spate-json]
version = "=0.1.0"
optional = true

[dependencies.spate-kafka]
version = "=0.1.0"
optional = true

[dependencies.spate-s3]
version = "=0.1.0"
optional = true

[dev-dependencies.apache-avro]
version = "0.21"

[dev-dependencies.bytes]
version = "1.11.1"

[dev-dependencies.clickhouse]
version = "0.15"

[dev-dependencies.flate2]
version = "1.1"

[dev-dependencies.http-body-util]
version = "0.1.2"

[dev-dependencies.hyper]
version = "1.10"
features = [
    "http1",
    "server",
]

[dev-dependencies.hyper-util]
version = "0.1.12"

[dev-dependencies.metrics]
version = "0.24.2"

[dev-dependencies.rdkafka]
version = "0.39"

[dev-dependencies.serde]
version = "1.0.228"
features = ["derive"]

[dev-dependencies.serde_json]
version = "1.0.145"

[dev-dependencies.spate-test]
version = "=0.1.0"

[dev-dependencies.tempfile]
version = "3"

[dev-dependencies.testcontainers]
version = "0.27"
features = ["blocking"]

[dev-dependencies.testcontainers-modules]
version = "0.15"
features = ["kafka"]

[dev-dependencies.tokio]
version = "1.52"
features = [
    "rt-multi-thread",
    "macros",
    "signal",
    "time",
]
default-features = false

[dev-dependencies.tracing]
version = "0.1.44"

[lints.clippy]
dbg_macro = "warn"
print_stderr = "warn"
print_stdout = "warn"
undocumented_unsafe_blocks = "warn"

[lints.clippy.all]
level = "warn"
priority = -1

[lints.rust]
missing_debug_implementations = "warn"
missing_docs = "warn"
unreachable_pub = "warn"
unsafe_op_in_unsafe_fn = "deny"

[lints.rust.rust_2018_idioms]
level = "warn"
priority = -1

[lints.rust.unexpected_cfgs]
level = "warn"
priority = 0
check-cfg = ["cfg(loom)"]