danube-connect-core 0.5.0

Core SDK for building Danube connectors
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141

# Danube Docker Compose Setup for Connector Testing
# This setup provides a complete Danube cluster for testing connectors
# Includes: ETCD (metadata), MinIO (S3 storage), and Danube Brokers

services:
  # ETCD service for metadata storage
  etcd:
    container_name: "danube-etcd"
    image: quay.io/coreos/etcd:v3.5.9
    environment:
      - ETCDCTL_API=3
      - ETCD_NAME=danube-etcd
      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
      - ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379
      - ETCD_INITIAL_CLUSTER=danube-etcd=http://etcd:2380
      - ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380
      - ETCD_INITIAL_ADVERTISE_PEER_URLS=http://etcd:2380
      - ETCD_INITIAL_CLUSTER_STATE=new
      - ETCD_INITIAL_CLUSTER_TOKEN=danube-cluster
    ports:
      - "2379:2379"
      - "2380:2380"
    networks:
      - danube_net
    volumes:
      - etcd_data:/etcd-data
    healthcheck:
      test:
        [
          "CMD",
          "etcdctl",
          "--endpoints=http://127.0.0.1:2379",
          "endpoint",
          "health",
        ]
      interval: 5s
      timeout: 5s
      retries: 10
      start_period: 10s

  # MinIO service for S3-compatible object storage
  minio:
    container_name: "danube-minio"
    image: minio/minio:RELEASE.2025-07-23T15-54-02Z
    environment:
      - MINIO_ROOT_USER=minioadmin
      - MINIO_ROOT_PASSWORD=minioadmin123
      - MINIO_DOMAIN=minio
    ports:
      - "9000:9000" # MinIO API
      - "9001:9001" # MinIO Console
    command: ["server", "/data", "--console-address", ":9001"]
    networks:
      - danube_net
    volumes:
      - minio_data:/data
    healthcheck:
      test: ["CMD", "curl", "-f", "http://minio:9000/minio/health/live"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 10s

  # MinIO client to create required buckets
  mc:
    container_name: "danube-mc"
    image: minio/mc:RELEASE.2024-09-16T17-43-14Z
    depends_on:
      minio:
        condition: service_healthy
    networks:
      - danube_net
    entrypoint: >
      /bin/sh -c "
      echo 'Waiting for MinIO to be ready...';
      until (/usr/bin/mc alias set danube-minio http://minio:9000 minioadmin minioadmin123) do 
        echo 'MinIO not ready, waiting...' && sleep 2; 
      done;
      echo 'Creating Danube buckets...';
      /usr/bin/mc rm -r --force danube-minio/danube-messages || true;
      /usr/bin/mc rm -r --force danube-minio/danube-wal || true;
      /usr/bin/mc mb danube-minio/danube-messages;
      /usr/bin/mc mb danube-minio/danube-wal;
      echo 'Buckets created successfully';
      echo 'MinIO setup complete - keeping container alive';
      tail -f /dev/null
      "

  # Danube Broker 1
  broker1:
    container_name: "danube-broker1"
    image: ghcr.io/danube-messaging/danube-broker:latest
    depends_on:
      etcd:
        condition: service_healthy
      minio:
        condition: service_healthy
      mc:
        condition: service_started
    environment:
      - RUST_LOG=danube_broker=info,danube_core=info,danube_persistent_storage=info
      - AWS_ACCESS_KEY_ID=minioadmin
      - AWS_SECRET_ACCESS_KEY=minioadmin123
      - AWS_REGION=us-east-1
    volumes:
      - ./danube_broker.yml:/etc/danube_broker.yml:ro
      - broker1_wal:/data/wal
    networks:
      - danube_net
    ports:
      - "6650:6650" # Broker gRPC
      - "50051:50051" # Admin API
      - "9040:9040" # Prometheus metrics
    command:
      [
        "--config-file",
        "/etc/danube_broker.yml",
        "--broker-addr",
        "0.0.0.0:6650",
        "--admin-addr",
        "0.0.0.0:50051",
        "--prom-exporter",
        "0.0.0.0:9040",
        "--advertised-addr",
        "broker1:6650",
      ]
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:9040/metrics || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 30s

networks:
  danube_net:
    driver: bridge

volumes:
  etcd_data:
  minio_data:
  broker1_wal: