udb 0.3.1

Universal Data Broker — a Rust gRPC broker over multiple databases (Postgres, MySQL, SQLite, MongoDB, ClickHouse, Cassandra, MSSQL, Redis, Qdrant, S3, Neo4j, …) with per-tenant RLS, 2PC, sagas, and CDC.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157

services:
  postgres:
    build:
      context: .
      dockerfile: docker/postgres-pg-partman/Dockerfile
    image: udb-postgres-pg-partman:16
    command:
      - postgres
      - -c
      - wal_level=logical
      - -c
      - max_replication_slots=10
      - -c
      - max_wal_senders=10
    environment:
      POSTGRES_USER: udb
      POSTGRES_PASSWORD: udb
      POSTGRES_DB: udb
    ports:
      - "55432:5432"
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U udb -d udb"]
      interval: 5s
      timeout: 3s
      retries: 20

  kafka:
    image: apache/kafka:3.9.0
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
      # Two broker listeners: INTERNAL (kafka:9092) for in-Docker clients (the udb
      # broker), EXTERNAL (localhost:59192) advertised so host clients — the
      # integration tests + Spark on the host — can both bootstrap AND reach the
      # advertised address. Single-advertised-listener setups break host clients
      # because the broker advertises its Docker-internal name.
      KAFKA_LISTENERS: INTERNAL://0.0.0.0:9092,CONTROLLER://kafka:9093,EXTERNAL://0.0.0.0:59192
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9092,EXTERNAL://localhost:59192
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
      KAFKA_COMPRESSION_TYPE: lz4
    ports:
      - "59192:59192"
    healthcheck:
      test: ["CMD-SHELL", "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null 2>&1"]
      interval: 10s
      timeout: 5s
      retries: 20

  redis:
    image: redis:7-alpine
    ports:
      - "56379:6379"
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      timeout: 3s
      retries: 20

  udb:
    profiles: ["broker"]
    build:
      context: ../..
      dockerfile: src/udb/Dockerfile
    environment:
      UDB_PG_DSN: postgres://udb:udb@postgres:5432/udb
      UDB_REDIS_DSN: redis://redis:6379/0
      UDB_QDRANT_URL: http://qdrant:6333
      UDB_MINIO_ENDPOINT: http://minio:9000
      UDB_MINIO_ACCESS_KEY: minio
      UDB_MINIO_SECRET_KEY: minio123
      UDB_KAFKA_BROKERS: kafka:9092
      UDB_ABAC_DEFAULT_ALLOW: "true"
      RUST_LOG: info
    ports:
      - "50051:50051"
      - "50052:50052"
    depends_on:
      postgres:
        condition: service_healthy
      redis:
        condition: service_healthy
      qdrant:
        condition: service_healthy
      minio:
        condition: service_healthy
      kafka:
        condition: service_healthy
    healthcheck:
      test: ["CMD", "/usr/local/bin/grpc_health_probe", "-addr=127.0.0.1:50051"]
      interval: 10s
      timeout: 5s
      start_period: 30s
      retries: 6

  qdrant:
    image: qdrant/qdrant:v1.13.4
    ports:
      - "56333:6333"
    healthcheck:
      test: ["CMD-SHELL", "true"]
      interval: 5s
      timeout: 3s
      retries: 20

  minio:
    image: minio/minio:RELEASE.2025-01-20T14-49-07Z
    command: server /data --console-address ":9001"
    environment:
      MINIO_ROOT_USER: minio
      MINIO_ROOT_PASSWORD: minio123
    ports:
      - "59000:9000"
      - "59001:9001"
    healthcheck:
      test: ["CMD", "mc", "ready", "local"]
      interval: 5s
      timeout: 3s
      retries: 20

  # Apache Spark consumer of the native auth event stream (Kafka → Spark).
  # `--profile spark` runs the Structured Streaming job in `analytics/spark`
  # against the integration Kafka, materializing per-tenant auth metrics. Use
  # SPARK_RUN_ONCE=1 (Trigger.AvailableNow) for a bounded end-to-end test run.
  spark:
    profiles: ["spark"]
    image: apache/spark:3.5.1-scala2.12-java17-python3-ubuntu
    user: root
    depends_on:
      kafka:
        condition: service_healthy
    environment:
      UDB_KAFKA_BOOTSTRAP: kafka:9092
      UDB_SPARK_OUTPUT: /opt/udb/out/auth_metrics
      UDB_SPARK_CHECKPOINT: /opt/udb/out/_chk
      SPARK_RUN_ONCE: ${SPARK_RUN_ONCE:-0}
    volumes:
      - ./analytics/spark:/opt/udb/spark:ro
      - ./analytics/spark/_out:/opt/udb/out
    command:
      - /opt/spark/bin/spark-submit
      - --packages
      - org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.1
      - /opt/udb/spark/auth_events_streaming.py
      - --bootstrap
      - kafka:9092
      - --output
      - /opt/udb/out/auth_metrics
      - --checkpoint
      - /opt/udb/out/_chk
      - --console