distributed 4.9.0

CQRS/ES framework for Rust using Plain Old Rust Structs — append-only events, replay, snapshots, outbox, service bus, and pluggable infrastructure
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279

name: celld (live + e2e-celld)

# Reusable workflow: referenced via `uses: ./.github/workflows/integration-celld.yaml`
# from both the PR-quality and push-to-main pipelines.
#
# Local parity:
#   make -C tests/e2e-celld test
#   make -C tests/e2e-ui up && make -C tests/e2e-ui up-celld-nats
#   WATCH=0 WATCH_WORKER=0 make -C tests/e2e-celld run
#   E2E_UI_ORIGIN=http://localhost:5180 npx --prefix tests/e2e-ui playwright test \
#     todos.user.spec.ts chat.user.spec.ts --project chromium-user
#
# Default `cargo test` (quality) still runs fixture-only celld checks and
# skips live HTTP unless CELLD_URL is set. This job sets CELLD_URL / NATS_URL.
on:
  workflow_call:

env:
  CARGO_TERM_COLOR: always
  # Zitadel owns :18080 in the browser topology.
  CELLD_HTTP_PORT: "18880"
  CELLD_URL: http://127.0.0.1:18880
  NATS_PORT: "14222"
  NATS_URL: nats://127.0.0.1:14222
  DISTRIBUTED_INTERNAL_SECRET: test-only-internal-secret-change-me-2026

jobs:
  e2e-celld:
    name: e2e-celld workspace tests
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: tests/e2e-celld
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
        with:
          persist-credentials: false
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8
        with:
          toolchain: stable
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: tests/e2e-celld -> target
          shared-key: e2e-celld-workspace
      - name: Run e2e-celld workspace tests
        run: cargo test --workspace --verbose

  live:
    name: celld 0.4 live (local store + Queue + NATS)
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
        with:
          persist-credentials: false

      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8
        with:
          toolchain: stable
          targets: wasm32-unknown-unknown

      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: |
            . -> target
            tests/e2e-celld -> target
            tests/celld/worker -> tests/celld/worker/target
          shared-key: celld-live

      - uses: actions/setup-node@v4
        with:
          node-version: "22"
          cache: npm
          cache-dependency-path: |
            js/package-lock.json
            tests/e2e-ui/package-lock.json
            tests/e2e-ui/ui/package-lock.json

      - name: Install host tools
        run: sudo apt-get update && sudo apt-get install -y jq openssl curl

      - name: Install esbuild
        run: npm install -g esbuild

      - name: Install worker-build
        run: cargo install worker-build --locked || cargo install worker-build

      - name: Install celld CLI
        run: |
          curl -fsSL https://celld.dev/install.sh | sh
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"

      - name: Bring up Postgres + Zitadel and bootstrap OIDC
        run: make -C tests/e2e-ui up

      # Finish every compile before celld starts watching the Worker project.
      # The live phase should only execute already-built test binaries and the
      # application lifecycle commands, so a cold cache cannot trigger celld
      # hot reloads in the middle of an HTTP assertion.
      - name: Build Distributed application and live test binaries
        run: |
          cargo build -p distributed_cli --bin distributed
          target/debug/distributed build tests/e2e-ui --output json
          target/debug/distributed build tests/e2e-celld --output json
          cargo test --test celld --no-run
          cargo test --test e2e_ui_celld_nats_profile \
            --features graphql,http,sqlite --no-run
          npm install --prefix tests/e2e-ui

      - name: Bring up celld 0.4 + Queue relay + NATS
        run: |
          command -v celld
          command -v worker-build
          make -C tests/e2e-ui up-celld-nats

      - name: Live celld HTTP + NATS profile tests
        run: make -C tests/e2e-ui test-celld

      - name: Start e2e-celld application with Distributed dev
        run: |
          set -euo pipefail
          set -a
          # shellcheck disable=SC1091
          . tests/e2e-ui/e2e-ui.env
          set +a
          export BIND=127.0.0.1:8791
          export CELLD_URL=http://127.0.0.1:18880
          export NATS_URL=nats://127.0.0.1:14222
          export AUTH_URL=http://localhost:5180
          export AUTH_USE_SECURE_COOKIES=false
          export AUTH_TRUST_HOST=true
          export PUBLIC_E2E_PROFILE=celld-nats
          export UI_BIND=localhost
          export UI_URL=http://localhost:5180

          target/debug/distributed dev tests/e2e-celld \
            > tests/e2e-celld/.ci-distributed-dev.log 2>&1 &
          echo $! > tests/e2e-celld/.ci-distributed-dev.pid

          ok=0
          for i in $(seq 1 240); do
            if grep -Fq 'lifecycle dev: ready generation=' \
              tests/e2e-celld/.ci-distributed-dev.log; then
              ok=1
              break
            fi
            if ! kill -0 "$(cat tests/e2e-celld/.ci-distributed-dev.pid)" 2>/dev/null; then
              echo "distributed dev exited before e2e-celld readiness"
              tail -200 tests/e2e-celld/.ci-distributed-dev.log
              exit 1
            fi
            sleep 0.5
          done
          if [ "$ok" != "1" ]; then
            echo "distributed dev failed to make e2e-celld ready"
            tail -200 tests/e2e-celld/.ci-distributed-dev.log
            exit 1
          fi
          grep -F 'lifecycle dev: process api ready http://127.0.0.1:8791' \
            tests/e2e-celld/.ci-distributed-dev.log
          grep -F 'lifecycle dev: process ui ready http://localhost:5180' \
            tests/e2e-celld/.ci-distributed-dev.log

          ok=0
          for i in $(seq 1 60); do
            code=$(curl -s -o /dev/null -w '%{http_code}' \
              "http://localhost:5180/" 2>/dev/null || echo 000)
            if [ "$code" = "200" ] || [ "$code" = "302" ] || [ "$code" = "303" ]; then
              ok=1
              break
            fi
            sleep 0.5
          done
          if [ "$ok" != "1" ]; then
            echo "e2e-celld UI failed to become ready (last HTTP $code)"
            tail -200 tests/e2e-celld/.ci-distributed-dev.log
            exit 1
          fi

      - name: Install Playwright + Chromium
        working-directory: tests/e2e-ui
        run: npx playwright install chromium --with-deps

      - name: Red-team private HTTP boundaries
        run: |
          set -euo pipefail
          assert_status() {
            expected="$1"
            shift
            actual=$(curl -sS -o /tmp/celld-red-team-response -w '%{http_code}' "$@")
            if [ "$actual" != "$expected" ]; then
              echo "expected HTTP $expected, got $actual"
              sed -n '1,40p' /tmp/celld-red-team-response
              exit 1
            fi
          }

          assert_status 401 -X POST \
            http://127.0.0.1:8791/internal/celld-queue/relay \
            -H 'content-type: application/json' \
            -d '{}'
          assert_status 401 -X POST \
            http://127.0.0.1:8791/internal/celld-queue/relay \
            -H 'content-type: application/json' \
            -H 'x-distributed-internal-secret: forged-red-team-secret-000000' \
            -d '{}'
          assert_status 422 -X POST \
            http://127.0.0.1:8791/internal/celld-queue/relay \
            -H 'content-type: application/json' \
            -H "x-distributed-internal-secret: $DISTRIBUTED_INTERNAL_SECRET" \
            -d '{}'
          assert_status 401 -X POST \
            http://127.0.0.1:8791/zitadel.scrape.v1 \
            -H 'content-type: application/json' \
            -d '{}'
          assert_status 401 \
            http://127.0.0.1:18880/todo/red-team
          assert_status 404 -X POST \
            http://127.0.0.1:18880/todo/red-team/outbox.complete \
            -H 'content-type: application/json' \
            -H "x-distributed-internal-secret: $DISTRIBUTED_INTERNAL_SECRET" \
            -d '{"ids":["forged"]}'

      - name: Todo + Chat browser lifecycle through celld
        working-directory: tests/e2e-ui
        run: >-
          npx playwright test todos.user.spec.ts chat.user.spec.ts
          --project chromium-user
        env:
          E2E_UI_ORIGIN: http://localhost:5180
          E2E_API_ORIGIN: http://127.0.0.1:8791
          CI: true

      - name: Upload celld Playwright report
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: celld-playwright-report
          path: |
            tests/e2e-ui/playwright-report
            tests/e2e-ui/test-results
          if-no-files-found: ignore
          retention-days: 7

      - name: Dump logs on failure
        if: failure()
        run: |
          echo '=== celld ==='
          tail -200 tests/e2e-ui/.make-celld.log || true
          echo '=== celld Queue relay registration ==='
          tail -120 tests/e2e-ui/.make-celld-relay.log || true
          echo '=== NATS profile compose ==='
          docker compose -f tests/e2e-ui/celld-nats-profile/docker-compose.yml ps -a || true
          docker compose -f tests/e2e-ui/celld-nats-profile/docker-compose.yml logs --tail=80 || true
          echo '=== e2e-celld API ==='
          tail -240 tests/e2e-celld/.ci-distributed-dev.log || true
          echo '=== e2e-celld UI ==='
          grep -E 'lifecycle dev: process ui|vite|Local:' \
            tests/e2e-celld/.ci-distributed-dev.log | tail -100 || true
          echo '=== Postgres + Zitadel ==='
          docker compose -f tests/e2e-ui/docker/docker-compose.yml ps -a || true
          docker compose -f tests/e2e-ui/docker/docker-compose.yml logs --tail=100 || true

      - name: Tear down
        if: always()
        run: |
          if [ -f tests/e2e-celld/.ci-distributed-dev.pid ]; then
            pid=$(cat tests/e2e-celld/.ci-distributed-dev.pid)
            kill -INT "$pid" 2>/dev/null || true
            for _ in $(seq 1 40); do
              kill -0 "$pid" 2>/dev/null || break
              sleep 0.25
            done
          fi
          lsof -ti:5180 2>/dev/null | xargs -r kill -9 2>/dev/null || true
          lsof -ti:8791 2>/dev/null | xargs -r kill -9 2>/dev/null || true
          make -C tests/e2e-ui down-celld-nats || true
          make -C tests/e2e-ui down-celld || true
          docker compose -f tests/e2e-ui/docker/docker-compose.yml down -v || true