1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
name: Deploy
# Trunk-based staging deploy. Fires when the Build workflow completes
# successfully on the trunk (master/main) or a `release/**` branch, so
# every green push to the trunk ships to staging automatically — staging
# always reflects the head of master (the yoii-mcs model). release/**
# also deploys to staging as a release candidate; production ships from
# the v* tag via release.yml. Gating on Build success keeps a red build
# from deploying, without adding a test step to this fast path.
#
# What "deploy" means here:
# 1. Build this commit's broker image and push it to the in-LAN registry
# as zakuro-broker:<sha7>.
# 2. Commit that tag into zakuro-infra's
# k8s/zc-broker-fleet-staging/statefulset.yaml.
# ArgoCD app zc-broker-fleet-staging (automated, prune + selfHeal) syncs the
# change to the staging cluster. zakuro-infra's argocd-refresh workflow pokes
# ArgoCD on every push under k8s/**, so the rollout starts within seconds.
#
# This workflow used to POST a `deploy-staging` repository_dispatch to
# zakuro-infra and stop at the HTTP 204. Nothing there listens for that
# event (its only dispatch receiver is deploy-prod.yml). So every run went
# green while the fleet stayed on whatever tag someone last bumped by hand
# (zakuro-infra#114). Each step below now fails the job unless its effect
# is checked: the image is in the registry, and the bump commit is on
# zakuro-infra main.
#
# Every broker fleet follows this deploy, not only the staging statefulset:
# the GPU fleet's node brokers (k8s/gpu-fleet/node) used to be bumped by
# hand and fell behind. A broker refuses peers older than itself
# (ZAKURO_MIN_PEER_VERSION defaults to its own version), so a fleet left on
# 0.0.34 was cut off the moment one v0.0.37 broker joined the mesh. Two
# rules keep the fleets at or above every released broker:
# * a v* tag deploys: the release commit lives on release/**, never on
# master, and is the first build that reports the new version;
# * a build whose Cargo version is older than the newest v* tag is pushed
# but NOT promoted. Master's version string lags behind a release until
# its "sync master version" PR lands, so promoting it would downgrade
# the fleets below the released brokers' floor.
# The fleets restart only once their brokers are idle: each ArgoCD app runs
# a PreSync idle gate and retries until it passes (see zakuro-infra
# k8s/gpu-fleet/node/idle-gate.yaml).
on:
workflow_run:
workflows:
types:
branches:
- master
- main
- 'release/**'
push:
tags:
workflow_dispatch:
inputs:
ref:
description: "Tag, branch or SHA to deploy (default: the dispatched ref), e.g. v0.0.37"
required: false
default: ""
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
REGISTRY: registry.local.zakuro-ai.com
# The image the staging statefulset runs, built from docker/Dockerfile.
# Its runtime stage is the contract the fleet manifest relies on: zc at
# /usr/local/bin/zc, user zc (uid 1001), and /bin/sh + sed for the
# peer-list wrapper. Tags are the 7-char zc commit, the style the
# hand-made bumps used (e.g. e2be662).
IMAGE: registry.local.zakuro-ai.com/zakuroai/zakuro-broker
# Every manifest that runs a broker. Each must hold at least one
# `image: ${IMAGE}:<tag>` line, and all of them are moved together.
FLEET_MANIFESTS: >-
k8s/zc-broker-fleet-staging/statefulset.yaml
k8s/gpu-fleet/node/zakuro-node.yaml
permissions:
contents: read
jobs:
deploy-staging:
# A v* tag push deploys directly: release.yml builds and tests the tag.
if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
# amd64 nodes only (zakuro-infra#302). On the arm64 Mac runners in `cpu`,
# the buildx docker-container builder emulates linux/amd64 with qemu and
# rustc segfaults ("qemu: uncaught target signal 11", run 41465114504).
runs-on: cpu-amd64
# An uncached image build recompiles the whole dependency graph
# (~15-20 min, see docker/Dockerfile). Reading docker.yml's GHA cache
# usually cuts it to a few minutes.
timeout-minutes: 45
# The staging fleet is one shared environment behind one manifest line.
# Deploy one commit at a time, in order, and never cancel a deploy that
# has started.
concurrency:
group: zc-deploy-staging
cancel-in-progress: false
env:
# In the workflow_run context, github.sha resolves to the default
# branch. Pin to the SHA that triggered Build, so that the image and
# the manifest bump point at the right commit.
DEPLOY_SHA: ${{ github.event.workflow_run.head_sha || inputs.ref || github.sha }}
DEPLOY_REF: ${{ github.event.workflow_run.head_branch || inputs.ref || github.ref_name }}
steps:
- name: Check secrets
env:
# The zakuro-infra credential: the deploy App's key once
# DEPLOY_APP_CLIENT_ID is set (INFRA-0012), DEPLOY_PAT until then.
INFRA_CREDENTIAL: ${{ vars.DEPLOY_APP_CLIENT_ID != '' && secrets.DEPLOY_APP_PRIVATE_KEY || secrets.DEPLOY_PAT }}
DRIVE_DEPLOY_KEY: ${{ secrets.DRIVE_DEPLOY_KEY }}
run: |
set -euo pipefail
# Fail loudly on a missing secret rather than deploying nothing. An
# unset DEPLOY_PAT once made this job report successful deploys
# that never happened.
missing=0
for name in INFRA_CREDENTIAL DRIVE_DEPLOY_KEY; do
if [ -z "${!name:-}" ]; then
echo "::error::${name} is not set on zakuro-ai/zc — cannot deploy staging."
missing=1
fi
done
[ "$missing" = 0 ]
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ env.DEPLOY_SHA }}
persist-credentials: false
# The tags, for the newest-release check below.
fetch-depth: 0
fetch-tags: true
- name: Decide whether this build may be promoted
run: |
set -euo pipefail
# DEPLOY_SHA may be a tag name (dispatch input): tag the image with
# the commit it resolved to.
echo "TAG=$(git rev-parse --short=7 HEAD)" >> "$GITHUB_ENV"
VERSION=$(sed -n '/^\[package\]/,/^\[/s/^version = "\(.*\)"/\1/p' Cargo.toml | head -1)
LATEST=$(git tag -l 'v*' | sed 's/^v//' | sort -V | tail -1)
if [ -z "$VERSION" ]; then
echo "::error::no [package] version in Cargo.toml"
exit 1
fi
# sort -V puts the older version first: promote unless this build
# is strictly older than the newest release.
if [ -n "$LATEST" ] && [ "$VERSION" != "$LATEST" ] \
&& [ "$(printf '%s\n%s\n' "$VERSION" "$LATEST" | sort -V | head -1)" = "$VERSION" ]; then
echo "::notice::zc ${VERSION} is older than the newest release v${LATEST}: image pushed, fleets NOT moved (brokers on v${LATEST} would refuse it)."
echo "PROMOTE=false" >> "$GITHUB_ENV"
else
echo "zc ${VERSION} (newest release v${LATEST:-none}): promoting."
echo "PROMOTE=true" >> "$GITHUB_ENV"
fi
# zakuro-client is a git dep on the private zakuro-ai/zakuro-drive
# repo. This uses the same read-only deploy key and ssh-agent
# forwarding as docker.yml, so the key never lands in an image layer.
- name: Start ssh-agent with the drive deploy key
env:
DRIVE_DEPLOY_KEY: ${{ secrets.DRIVE_DEPLOY_KEY }}
run: |
eval "$(ssh-agent -s)"
echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV"
echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV"
printf '%s\n' "$DRIVE_DEPLOY_KEY" | ssh-add -
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4
- name: Build and push the broker image
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7
with:
context: .
file: docker/Dockerfile
# The staging node is amd64.
platforms: linux/amd64
ssh: default=${{ env.SSH_AUTH_SOCK }}
tags: ${{ env.IMAGE }}:${{ env.TAG }}
push: true
# Read the shared amd64 layer cache (written below on each deploy and by
# docker.yml on release builds), so unchanged dependencies are not recompiled.
cache-from: type=gha,scope=linux/amd64
# docker.yml no longer builds on master pushes, so the staging
# build maintains the shared amd64 layer cache itself.
cache-to: type=gha,scope=linux/amd64,mode=max
- name: Verify the image is in the registry
run: |
set -euo pipefail
code=$(curl -sS -o /dev/null -w '%{http_code}' \
-H 'Accept: application/vnd.oci.image.index.v1+json' \
-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
-H 'Accept: application/vnd.oci.image.manifest.v1+json' \
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
"https://${REGISTRY}/v2/zakuroai/zakuro-broker/manifests/${TAG}")
if [ "$code" != "200" ]; then
echo "::error::${IMAGE}:${TAG} is not in the registry (HTTP $code) — staging was NOT deployed."
exit 1
fi
echo "${IMAGE}:${TAG} is in the registry."
# Minted right before its consumer: an App token lives one hour and
# the image build above can take most of the job's 45 minutes.
- name: Mint deploy App token
id: infra-token
if: env.PROMOTE == 'true' && vars.DEPLOY_APP_CLIENT_ID != ''
uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0
with:
client-id: ${{ vars.DEPLOY_APP_CLIENT_ID }}
private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }}
owner: zakuro-ai
repositories: zakuro-infra
permission-contents: write
- name: Promote the image in zakuro-infra
if: env.PROMOTE == 'true'
env:
INFRA_TOKEN: ${{ steps.infra-token.outputs.token || secrets.DEPLOY_PAT }}
run: |
set -euo pipefail
INFRA_DIR="$(mktemp -d)/zakuro-infra"
git clone --depth=1 \
"https://x-access-token:${INFRA_TOKEN}@github.com/zakuro-ai/zakuro-infra.git" \
"$INFRA_DIR"
cd "$INFRA_DIR"
# Require at least one broker image line per manifest. None means
# the manifest changed shape, and a blind sed would do nothing.
# Image lines are anchored on IMAGE, so a hand pin to another
# repository (zakuroai/zc:5fd74b1, #123) still fails here.
OLD=""
for m in $FLEET_MANIFESTS; do
tags=$(sed -n -E "s|^[[:space:]]*image: ${IMAGE}:([^[:space:]]+).*|\1|p" "$m")
if [ -z "$tags" ]; then
echo "::error::no '${IMAGE}:<tag>' line in ${m}"
exit 1
fi
OLD=$(printf '%s\n%s\n' "$OLD" "$tags" | grep . | sort -u | paste -sd, -)
sed -i -E "s|^([[:space:]]*image: ${IMAGE}:)[^[:space:]]+|\1${TAG}|" "$m"
grep -n "image: ${IMAGE}:" "$m"
done
if git diff --quiet; then
echo "The fleets are already on ${TAG}; nothing to promote."
exit 0
fi
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
# shellcheck disable=SC2086 # a space-separated list, split on purpose
git add $FLEET_MANIFESTS
git commit \
-m "deploy(staging): broker image ${OLD} -> ${TAG}" \
-m "Source: zakuro-ai/zc@${TAG} on ${DEPLOY_REF}. ArgoCD apps zc-broker-fleet-staging and gpu-fleet-*-node auto-sync once their brokers are idle."
pushed=false
for attempt in 1 2 3 4 5; do
if git push origin HEAD:main; then pushed=true; break; fi
echo "push rejected, rebasing (attempt $attempt)"
git pull --rebase origin main
sleep 3
done
if [ "$pushed" != true ]; then
echo "::error::could not push the tag bump to zakuro-infra — staging was NOT deployed."
exit 1
fi
# Check the effect, not the exit code: the bump must be on main.
git fetch -q origin main
if ! git merge-base --is-ancestor HEAD origin/main; then
echo "::error::the bump commit $(git rev-parse --short HEAD) is not on zakuro-infra main — staging was NOT deployed."
exit 1
fi
echo "Promoted ${IMAGE}:${TAG} in zakuro-infra ($(git rev-parse --short HEAD)); the fleets roll as soon as their brokers are idle."