zc2 0.0.46

P2P compute broker with yen-denominated billing, WAL, and broker mesh support
name: Deploy

# Trunk-based staging deploy. Fires when the Build workflow completes
# successfully on the trunk (master/main) or a `release/**` branch, so
# every green push to the trunk ships to staging automatically — staging
# always reflects the head of master (the yoii-mcs model). release/**
# also deploys to staging as a release candidate; production ships from
# the v* tag via release.yml. Gating on Build success keeps a red build
# from deploying, without adding a test step to this fast path.
#
# What "deploy" means here:
#   1. Build this commit's broker image and push it to the in-LAN registry
#      as zakuro-broker:<sha7>.
#   2. Commit that tag into zakuro-infra's
#      k8s/zc-broker-fleet-staging/statefulset.yaml.
# ArgoCD app zc-broker-fleet-staging (automated, prune + selfHeal) syncs the
# change to the staging cluster. zakuro-infra's argocd-refresh workflow pokes
# ArgoCD on every push under k8s/**, so the rollout starts within seconds.
#
# This workflow used to POST a `deploy-staging` repository_dispatch to
# zakuro-infra and stop at the HTTP 204. Nothing there listens for that
# event (its only dispatch receiver is deploy-prod.yml). So every run went
# green while the fleet stayed on whatever tag someone last bumped by hand
# (zakuro-infra#114). Each step below now fails the job unless its effect
# is checked: the image is in the registry, and the bump commit is on
# zakuro-infra main.
#
# Every broker fleet follows this deploy, not only the staging statefulset:
# the GPU fleet's node brokers (k8s/gpu-fleet/node) used to be bumped by
# hand and fell behind. A broker refuses peers older than itself
# (ZAKURO_MIN_PEER_VERSION defaults to its own version), so a fleet left on
# 0.0.34 was cut off the moment one v0.0.37 broker joined the mesh. Two
# rules keep the fleets at or above every released broker:
#   * a v* tag deploys: the release commit lives on release/**, never on
#     master, and is the first build that reports the new version;
#   * a build whose Cargo version is older than the newest v* tag is pushed
#     but NOT promoted. Master's version string lags behind a release until
#     its "sync master version" PR lands, so promoting it would downgrade
#     the fleets below the released brokers' floor.
# The fleets restart only once their brokers are idle: each ArgoCD app runs
# a PreSync idle gate and retries until it passes (see zakuro-infra
# k8s/gpu-fleet/node/idle-gate.yaml).

on:
  workflow_run:
    workflows: ["Build"]
    types: [completed]
    branches:
      - master
      - main
      - 'release/**'
  push:
    tags: ['v*']
  workflow_dispatch:
    inputs:
      ref:
        description: "Tag, branch or SHA to deploy (default: the dispatched ref), e.g. v0.0.37"
        required: false
        default: ""

env:
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
  REGISTRY: registry.local.zakuro-ai.com
  # The image the staging statefulset runs, built from docker/Dockerfile.
  # Its runtime stage is the contract the fleet manifest relies on: zc at
  # /usr/local/bin/zc, user zc (uid 1001), and /bin/sh + sed for the
  # peer-list wrapper. Tags are the 7-char zc commit, the style the
  # hand-made bumps used (e.g. e2be662).
  IMAGE: registry.local.zakuro-ai.com/zakuroai/zakuro-broker
  # Every manifest that runs a broker. Each must hold at least one
  # `image: ${IMAGE}:<tag>` line, and all of them are moved together.
  FLEET_MANIFESTS: >-
    k8s/zc-broker-fleet-staging/statefulset.yaml
    k8s/gpu-fleet/node/zakuro-node.yaml

permissions:
  contents: read

jobs:
  deploy-staging:
    # A v* tag push deploys directly: release.yml builds and tests the tag.
    if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
    # amd64 nodes only (zakuro-infra#302). On the arm64 Mac runners in `cpu`,
    # the buildx docker-container builder emulates linux/amd64 with qemu and
    # rustc segfaults ("qemu: uncaught target signal 11", run 41465114504).
    runs-on: cpu-amd64
    # An uncached image build recompiles the whole dependency graph
    # (~15-20 min, see docker/Dockerfile). Reading docker.yml's GHA cache
    # usually cuts it to a few minutes.
    timeout-minutes: 45
    # The staging fleet is one shared environment behind one manifest line.
    # Deploy one commit at a time, in order, and never cancel a deploy that
    # has started.
    concurrency:
      group: zc-deploy-staging
      cancel-in-progress: false
    env:
      # In the workflow_run context, github.sha resolves to the default
      # branch. Pin to the SHA that triggered Build, so that the image and
      # the manifest bump point at the right commit.
      DEPLOY_SHA: ${{ github.event.workflow_run.head_sha || inputs.ref || github.sha }}
      DEPLOY_REF: ${{ github.event.workflow_run.head_branch || inputs.ref || github.ref_name }}
    steps:
      - name: Check secrets
        env:
          # The zakuro-infra credential: the deploy App's key once
          # DEPLOY_APP_CLIENT_ID is set (INFRA-0012), DEPLOY_PAT until then.
          INFRA_CREDENTIAL: ${{ vars.DEPLOY_APP_CLIENT_ID != '' && secrets.DEPLOY_APP_PRIVATE_KEY || secrets.DEPLOY_PAT }}
          DRIVE_DEPLOY_KEY: ${{ secrets.DRIVE_DEPLOY_KEY }}
        run: |
          set -euo pipefail
          # Fail loudly on a missing secret rather than deploying nothing. An
          # unset DEPLOY_PAT once made this job report successful deploys
          # that never happened.
          missing=0
          for name in INFRA_CREDENTIAL DRIVE_DEPLOY_KEY; do
            if [ -z "${!name:-}" ]; then
              echo "::error::${name} is not set on zakuro-ai/zc — cannot deploy staging."
              missing=1
            fi
          done
          [ "$missing" = 0 ]

      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
        with:
          ref: ${{ env.DEPLOY_SHA }}
          persist-credentials: false
          # The tags, for the newest-release check below.
          fetch-depth: 0
          fetch-tags: true

      - name: Decide whether this build may be promoted
        run: |
          set -euo pipefail
          # DEPLOY_SHA may be a tag name (dispatch input): tag the image with
          # the commit it resolved to.
          echo "TAG=$(git rev-parse --short=7 HEAD)" >> "$GITHUB_ENV"
          VERSION=$(sed -n '/^\[package\]/,/^\[/s/^version = "\(.*\)"/\1/p' Cargo.toml | head -1)
          LATEST=$(git tag -l 'v*' | sed 's/^v//' | sort -V | tail -1)
          if [ -z "$VERSION" ]; then
            echo "::error::no [package] version in Cargo.toml"
            exit 1
          fi
          # sort -V puts the older version first: promote unless this build
          # is strictly older than the newest release.
          if [ -n "$LATEST" ] && [ "$VERSION" != "$LATEST" ] \
             && [ "$(printf '%s\n%s\n' "$VERSION" "$LATEST" | sort -V | head -1)" = "$VERSION" ]; then
            echo "::notice::zc ${VERSION} is older than the newest release v${LATEST}: image pushed, fleets NOT moved (brokers on v${LATEST} would refuse it)."
            echo "PROMOTE=false" >> "$GITHUB_ENV"
          else
            echo "zc ${VERSION} (newest release v${LATEST:-none}): promoting."
            echo "PROMOTE=true" >> "$GITHUB_ENV"
          fi

      # zakuro-client is a git dep on the private zakuro-ai/zakuro-drive
      # repo. This uses the same read-only deploy key and ssh-agent
      # forwarding as docker.yml, so the key never lands in an image layer.
      - name: Start ssh-agent with the drive deploy key
        env:
          DRIVE_DEPLOY_KEY: ${{ secrets.DRIVE_DEPLOY_KEY }}
        run: |
          eval "$(ssh-agent -s)"
          echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV"
          echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV"
          printf '%s\n' "$DRIVE_DEPLOY_KEY" | ssh-add -

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5  # v4

      - name: Build and push the broker image
        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf  # v7
        with:
          context: .
          file: docker/Dockerfile
          # The staging node is amd64.
          platforms: linux/amd64
          ssh: default=${{ env.SSH_AUTH_SOCK }}
          tags: ${{ env.IMAGE }}:${{ env.TAG }}
          push: true
          # Read the shared amd64 layer cache (written below on each deploy and by
          # docker.yml on release builds), so unchanged dependencies are not recompiled.
          cache-from: type=gha,scope=linux/amd64
          # docker.yml no longer builds on master pushes, so the staging
          # build maintains the shared amd64 layer cache itself.
          cache-to: type=gha,scope=linux/amd64,mode=max

      - name: Verify the image is in the registry
        run: |
          set -euo pipefail
          code=$(curl -sS -o /dev/null -w '%{http_code}' \
            -H 'Accept: application/vnd.oci.image.index.v1+json' \
            -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
            -H 'Accept: application/vnd.oci.image.manifest.v1+json' \
            -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
            "https://${REGISTRY}/v2/zakuroai/zakuro-broker/manifests/${TAG}")
          if [ "$code" != "200" ]; then
            echo "::error::${IMAGE}:${TAG} is not in the registry (HTTP $code) — staging was NOT deployed."
            exit 1
          fi
          echo "${IMAGE}:${TAG} is in the registry."

      # Minted right before its consumer: an App token lives one hour and
      # the image build above can take most of the job's 45 minutes.
      - name: Mint deploy App token
        id: infra-token
        if: env.PROMOTE == 'true' && vars.DEPLOY_APP_CLIENT_ID != ''
        uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510  # v3.3.0
        with:
          client-id: ${{ vars.DEPLOY_APP_CLIENT_ID }}
          private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }}
          owner: zakuro-ai
          repositories: zakuro-infra
          permission-contents: write

      - name: Promote the image in zakuro-infra
        if: env.PROMOTE == 'true'
        env:
          INFRA_TOKEN: ${{ steps.infra-token.outputs.token || secrets.DEPLOY_PAT }}
        run: |
          set -euo pipefail
          INFRA_DIR="$(mktemp -d)/zakuro-infra"
          git clone --depth=1 \
            "https://x-access-token:${INFRA_TOKEN}@github.com/zakuro-ai/zakuro-infra.git" \
            "$INFRA_DIR"
          cd "$INFRA_DIR"

          # Require at least one broker image line per manifest. None means
          # the manifest changed shape, and a blind sed would do nothing.
          # Image lines are anchored on IMAGE, so a hand pin to another
          # repository (zakuroai/zc:5fd74b1, #123) still fails here.
          OLD=""
          for m in $FLEET_MANIFESTS; do
            tags=$(sed -n -E "s|^[[:space:]]*image: ${IMAGE}:([^[:space:]]+).*|\1|p" "$m")
            if [ -z "$tags" ]; then
              echo "::error::no '${IMAGE}:<tag>' line in ${m}"
              exit 1
            fi
            OLD=$(printf '%s\n%s\n' "$OLD" "$tags" | grep . | sort -u | paste -sd, -)
            sed -i -E "s|^([[:space:]]*image: ${IMAGE}:)[^[:space:]]+|\1${TAG}|" "$m"
            grep -n "image: ${IMAGE}:" "$m"
          done
          if git diff --quiet; then
            echo "The fleets are already on ${TAG}; nothing to promote."
            exit 0
          fi

          git config user.name  "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          # shellcheck disable=SC2086  # a space-separated list, split on purpose
          git add $FLEET_MANIFESTS
          git commit \
            -m "deploy(staging): broker image ${OLD} -> ${TAG}" \
            -m "Source: zakuro-ai/zc@${TAG} on ${DEPLOY_REF}. ArgoCD apps zc-broker-fleet-staging and gpu-fleet-*-node auto-sync once their brokers are idle."

          pushed=false
          for attempt in 1 2 3 4 5; do
            if git push origin HEAD:main; then pushed=true; break; fi
            echo "push rejected, rebasing (attempt $attempt)"
            git pull --rebase origin main
            sleep 3
          done
          if [ "$pushed" != true ]; then
            echo "::error::could not push the tag bump to zakuro-infra — staging was NOT deployed."
            exit 1
          fi

          # Check the effect, not the exit code: the bump must be on main.
          git fetch -q origin main
          if ! git merge-base --is-ancestor HEAD origin/main; then
            echo "::error::the bump commit $(git rev-parse --short HEAD) is not on zakuro-infra main — staging was NOT deployed."
            exit 1
          fi
          echo "Promoted ${IMAGE}:${TAG} in zakuro-infra ($(git rev-parse --short HEAD)); the fleets roll as soon as their brokers are idle."