spate 0.2.0

High-performance at-least-once ETL pipeline framework with a chaining operator API, checkpoint-driven source commits, sharded asynchronous sinks, backpressure, and Prometheus metrics.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

# Multi-sink split: Kafka → Avro → split into one ClickHouse table per event
# kind. The `sinks:` map replaces `sink:` — each entry is an ordinary
# single-key connector section, keyed by the name the chain resolves via
# `ctx.sink("<name>")`.

pipeline:
  name: storefront-split
  io_threads: 2

checkpoint:
  interval: 5s
  max_pending_batches: 1024
  drain_timeout: 25s

backpressure:
  max_inflight_bytes: 1GiB

admin:
  listen: 0.0.0.0:9090  # /metrics, /healthz, /readyz

metrics:
  exporter: prometheus

source:
  kafka:
    brokers: ${KAFKA_BROKERS:-localhost:9092}
    topic: ${KAFKA_TOPIC:-storefront-events}
    group_id: ${KAFKA_GROUP:-storefront-split-etl}
    commit_interval: 5s
    rdkafka:
      auto.offset.reset: earliest

deserializer:
  avro:
    # A top-level union of the three event records: the branch index on the
    # wire selects the Rust enum's variant positionally, so the variant order
    # in the example must match the order below.
    mode: raw
    schema:
      inline: |
        [
          {"type":"record","name":"OrderPlaced","namespace":"spate.datagen","fields":[
            {"name":"order_id","type":"long"},
            {"name":"customer_id","type":"int"},
            {"name":"region","type":"string"},
            {"name":"placed_at","type":{"type":"long","logicalType":"timestamp-millis"}},
            {"name":"lines","type":{"type":"array","items":
              {"type":"record","name":"OrderLine","fields":[
                {"name":"sku","type":"string"},
                {"name":"qty","type":"int"},
                {"name":"unit_cents","type":"int"}]}}}]},
          {"type":"record","name":"PaymentCaptured","namespace":"spate.datagen","fields":[
            {"name":"order_id","type":"long"},
            {"name":"amount_cents","type":"long"}]},
          {"type":"record","name":"RefundIssued","namespace":"spate.datagen","fields":[
            {"name":"order_id","type":"long"},
            {"name":"amount_cents","type":"long"},
            {"name":"reason","type":"string"}]}
        ]

# One sink per destination table. Each is a full ClickHouse sink section — its
# own columns, format, shards, and batch/linger — so per-table part sizing is
# tuned independently.
sinks:
  payments:
    clickhouse:
      table: ${CLICKHOUSE_PAYMENTS_TABLE:-payments}
      columns: [order_id, amount_cents]
      format: native
      validate_schema: full
      shards:
        - replicas: ["${CLICKHOUSE_URL:-http://localhost:8123}"]
      user: ${CLICKHOUSE_USER:-default}
      password: ${CLICKHOUSE_PASSWORD:-}
      batch:
        max_rows: 500000
        max_bytes: 128MiB
        linger: 1s
      inflight:
        max_per_shard: 2

  refunds:
    clickhouse:
      table: ${CLICKHOUSE_REFUNDS_TABLE:-refunds}
      columns: [order_id, amount_cents, reason]
      format: native
      validate_schema: full
      shards:
        - replicas: ["${CLICKHOUSE_URL:-http://localhost:8123}"]
      user: ${CLICKHOUSE_USER:-default}
      password: ${CLICKHOUSE_PASSWORD:-}
      # A low-volume table: linger longer to build bigger parts. The cost is
      # checkpoint lag — a Kafka batch with even one refund in it holds its
      # offsets until this fires (docs/user-guide/02-concepts/06-multi-sink.md).
      batch:
        max_rows: 500000
        max_bytes: 128MiB
        linger: 5s
      inflight:
        max_per_shard: 2