systemg 0.63.1

name: CI

on:
  push:
    branches:
      - main
      - develop
  pull_request:
    branches:
      - main
      - develop
  release:
    types:
      - published
  workflow_dispatch:
    inputs:
      test-release:
        description: "Simulate a release"
        required: false
        default: "true"

# Cancel in-progress workflows when a new commit is pushed to the same branch
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  RUSTC_VERSION: 1.96.0

jobs:
  cancel-previous:
    name: Cancel Previous Runs
    runs-on: ubuntu-latest
    steps:
      - name: Cancel Previous Runs
        uses: styfle/cancel-workflow-action@0.12.1
        with:
          access_token: ${{ github.token }}

  check-commit-message:
    name: Check Commit Message
    runs-on: ubuntu-latest
    needs: cancel-previous
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4
        with:
          # On PRs, check the PR's head commit, not the synthetic merge commit
          # GitHub checks out by default (whose subject is "Merge ... into ...").
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Validate commit message format
        run: ci/check-commit-message.sh

  cargo-audit:
    name: Cargo Audit
    runs-on: ubuntu-latest
    needs: check-commit-message
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Audit Dependencies for CVEs
        uses: rustsec/audit-check@v2
        with:
          token: ${{ github.token }}

  cargo-deny:
    name: Cargo Deny
    runs-on: ubuntu-latest
    needs: check-commit-message
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Check Advisories, Bans, Licenses, Sources
        uses: EmbarkStudios/cargo-deny-action@v2
        with:
          command: check

  cargo-vet:
    name: Cargo Vet
    runs-on: ubuntu-latest
    needs: check-commit-message
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Install cargo-vet
        run: cargo install cargo-vet --locked

      - name: Vet Dependencies
        run: cargo vet --locked

  changes:
    name: Detect Changes
    runs-on: ubuntu-latest
    needs: cargo-audit
    outputs:
      docs: ${{ steps.filter.outputs.docs }}
      rust: ${{ steps.filter.outputs.rust }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: recursive

      - name: Detect changed projects
        id: filter
        uses: dorny/paths-filter@v3
        with:
          filters: |
            docs:
              - 'docs/**'
              - '.github/workflows/ci.yaml'
            rust:
              - 'Cargo.toml'
              - 'Cargo.lock'
              - 'src/**'
              - 'tests/**'
              - 'examples/**'
              - 'ci/**'
              - 'clippy.toml'
              - 'rustfmt.toml'
              - 'rust-toolchain.toml'
              - 'Dockerfile.builder'
              - 'config.yaml'
              - '.github/workflows/ci.yaml'

  extract-version:
    name: Extract Version
    runs-on: ubuntu-latest
    needs: cargo-audit
    outputs:
      current_version: ${{ steps.extract.outputs.current_version }}
      tag: ${{ steps.extract.outputs.tag }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Set Up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.13"

      - name: Extract current version
        id: extract
        shell: bash
        run: |
          set -euo pipefail
          current_version=$(python ci/extract_version.py)
          echo "current_version=$current_version" >> "$GITHUB_OUTPUT"
          echo "tag=v$current_version" >> "$GITHUB_OUTPUT"

  detect-version-bump:
    name: Detect Version Bump
    runs-on: ubuntu-latest
    needs: [cargo-audit, extract-version]
    outputs:
      bumped: ${{ steps.detect.outputs.bumped }}
      current_version: ${{ steps.detect.outputs.current_version }}
      previous_version: ${{ steps.detect.outputs.previous_version }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set Up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.13"

      # Compares against the PUBLISHED release, not the previous commit.
      #
      # Diffing Cargo.toml across commits made a release un-repeatable: when a
      # release run failed on something unrelated to the code (a flaky job, a
      # CI-only fix), the retry pushed a commit whose previous commit already
      # carried the same version, so `bumped` came out false and the version
      # never shipped at all. The only way out was to burn a patch number on a
      # fix that changed nothing about the release.
      #
      # A release is a tag, and `create-github-release` already creates OR
      # updates one, so asking "is this version published yet?" makes a retry
      # idempotent: it republishes the same version until it succeeds, and
      # still refuses to re-cut a version that already shipped.
      - name: Detect unpublished version on main push
        id: detect
        shell: bash
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail

          current_version="${{ needs.extract-version.outputs.current_version }}"
          current_tag="${{ needs.extract-version.outputs.tag }}"

          previous_version=""
          bumped="false"

          if [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then
            previous_version="$(gh release view --json tagName -q .tagName 2>/dev/null || true)"

            if [[ -z "$previous_version" ]]; then
              echo "No published release found; treating ${current_tag} as unpublished."
              bumped="true"
            elif gh release view "$current_tag" >/dev/null 2>&1; then
              echo "${current_tag} is already published; nothing to release."
            else
              echo "${current_tag} is not published yet (latest is ${previous_version})."
              bumped="true"
            fi
          fi

          echo "current_version=$current_version" >> "$GITHUB_OUTPUT"
          echo "previous_version=$previous_version" >> "$GITHUB_OUTPUT"
          echo "bumped=$bumped" >> "$GITHUB_OUTPUT"

          echo "Current version: $current_version"
          echo "Latest published: ${previous_version:-<none>}"
          echo "Needs release: $bumped"

  docs:
    name: Mintlify Docs
    runs-on: ubuntu-latest
    needs: [cancel-previous, changes]
    if: ${{ needs.changes.outputs.docs == 'true' }}

    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Set Up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: "20.17.0"

      - name: Install Mintlify CLI
        # Pin Mintlify to keep docs validation reproducible across CI runs.
        run: npm install --global mintlify@4.0.373

      - name: Validate Docs
        working-directory: docs
        run: mintlify broken-links

  rustdoc:
    name: Rust Doc
    runs-on: ubuntu-latest
    needs: [cancel-previous, changes]
    if: ${{ needs.changes.outputs.rust == 'true' }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - name: Cache Cargo Dependencies
        uses: Swatinem/rust-cache@v2

      - name: Check Documentation
        run: cargo rustdoc --lib -- -D missing_docs

  cargo-sort:
    name: Cargo Sort
    runs-on: ubuntu-latest
    needs: [changes, rustdoc]
    if: ${{ needs.changes.outputs.rust == 'true' }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - name: Cache Cargo Dependencies
        uses: Swatinem/rust-cache@v2

      - name: Install cargo-sort
        run: cargo install cargo-sort

      - name: Check Cargo.toml Formatting
        run: cargo sort --check --workspace

  rustfmt:
    name: Rustfmt
    runs-on: ubuntu-latest
    needs: [changes, rustdoc]
    if: ${{ needs.changes.outputs.rust == 'true' }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Install Rust Nightly
        uses: dtolnay/rust-toolchain@nightly
        with:
          components: rustfmt

      - name: Cache Cargo Dependencies
        uses: Swatinem/rust-cache@v2

      - name: Run Rustfmt
        run: cargo +nightly fmt -- --check

  clippy:
    name: Clippy
    runs-on: ubuntu-latest
    needs: [changes, cargo-sort, rustfmt]
    if: ${{ needs.changes.outputs.rust == 'true' }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - name: Cache Cargo Dependencies
        uses: Swatinem/rust-cache@v2

      - name: Install clippy
        run: rustup component add clippy

      - name: Run Clippy
        run: cargo clippy --all-targets --all-features -- -D warnings

  cargo-build:
    name: Cargo Build
    runs-on: ubuntu-latest
    needs: [changes, clippy]
    if: ${{ needs.changes.outputs.rust == 'true' }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - name: Cache Cargo Dependencies
        uses: Swatinem/rust-cache@v2

      - name: Cargo build
        run: cargo build --locked

  # These tests run in parallel after clippy passes
  cargo-test:
    name: Cargo Test
    runs-on: ubuntu-latest
    needs: [changes, clippy]
    if: ${{ needs.changes.outputs.rust == 'true' }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - name: Cache Cargo Dependencies
        uses: Swatinem/rust-cache@v2

      - name: Run Tests
        run: cargo test --verbose

  # macOS system-mode smoke test on a native runner (no Docker parity lane
  # exists for Darwin): /Library path selection, rootless validate, and typed
  # refusal of Linux-only kernel features.
  macos-system-mode:
    name: macOS System Mode
    runs-on: macos-latest
    needs: [changes, clippy]
    if: ${{ needs.changes.outputs.rust == 'true' }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - name: Cache Cargo Dependencies
        uses: Swatinem/rust-cache@v2

      - name: Build release binary
        run: cargo build --release

      - name: Run macOS system-mode smoke test
        run: tests/macos/system_mode_smoke.sh

  # Use-case tests: real end-to-end runs of the sysg binary in Docker.
  # Each case boots a config via the CLI and asserts on ps / on-disk state /
  # `sysg status` output — the coverage the in-process façade tests never had.
  docker-usecase-tests:
    name: Docker Use-Case Tests (${{ matrix.base.name }})
    runs-on: ubuntu-latest
    needs: [changes, clippy]
    if: ${{ needs.changes.outputs.rust == 'true' }}
    timeout-minutes: 45
    strategy:
      fail-fast: false
      matrix:
        base:
          - name: glibc
            dockerfile: tests/docker/usecase/Dockerfile.base
            cases: ""
          - name: alpine-musl
            dockerfile: tests/docker/usecase/Dockerfile.base.alpine
            cases: >-
              parity-start-status-stop parity-restart-reconcile
              parity-logs-follow-and-purge parity-cron-lifecycle
              parity-inspect-json parity-supervisor-recycle
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Run use-case suite
        env:
          USECASE_BASE_DOCKERFILE: ${{ matrix.base.dockerfile }}
        run: tests/docker/usecase/run_all.sh ${{ matrix.base.cases }}

  # Container-init (PID 1) tests: the container's ENTRYPOINT is `sysg init`,
  # so these run in their own harness (docker exec assertions, teardown verdict
  # from the container exit code) rather than the use-case runner.
  docker-pid1-tests:
    name: Docker PID1 Tests (${{ matrix.base.name }})
    runs-on: ubuntu-latest
    needs: [changes, clippy]
    if: ${{ needs.changes.outputs.rust == 'true' }}
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        base:
          - name: glibc
            dockerfile: tests/docker/usecase/Dockerfile.base
          - name: alpine-musl
            dockerfile: tests/docker/usecase/Dockerfile.base.alpine
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Run PID1 suite
        run: tests/docker/pid1/run_pid1_tests.sh ${{ matrix.base.dockerfile }}

  # Kernel-enforced sandbox tests (Phase 3): Landlock confinement and the
  # schema-v3 fail-closed refusal, in a privileged container. Enforcement
  # assertions self-skip on kernels without Landlock; the fail-closed refusals
  # always run.
  docker-sandbox-tests:
    name: Docker Sandbox Tests
    runs-on: ubuntu-latest
    needs: [changes, clippy]
    if: ${{ needs.changes.outputs.rust == 'true' }}
    timeout-minutes: 30
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build base and sandbox images
        run: |
          docker build -f tests/docker/usecase/Dockerfile.base -t sysg-usecase-base .
          docker build -f tests/docker/sandbox/Dockerfile -t sysg-sandbox .

      - name: Print runner kernel
        run: uname -r

      - name: Run sandbox suite (privileged, Landlock required)
        run: |
          docker run --rm --privileged \
            --security-opt apparmor:unconfined \
            --security-opt seccomp=unconfined \
            -e REQUIRE_LANDLOCK=1 \
            sysg-sandbox

  # Native-kernel enforcement lane: runs the kernel-facing sandbox checks
  # directly on the GitHub runner's real kernel (Ubuntu 24.04 ships 6.x with
  # Landlock enabled), against the release binary — no Docker, no VM. This is
  # where Landlock ENFORCEMENT actually runs (the Docker Desktop kernel used in
  # local dev lacks Landlock, so those assertions self-skip there).
  # REQUIRE_LANDLOCK=1 makes a missing Landlock a CI failure, not a silent skip.
  kernel-native:
    name: Kernel Native Enforcement
    runs-on: ubuntu-24.04
    needs: [changes, clippy]
    if: ${{ needs.changes.outputs.rust == 'true' }}
    timeout-minutes: 30
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - name: Cache Cargo Dependencies
        uses: Swatinem/rust-cache@v2

      - name: Print runner kernel
        run: uname -r

      - name: Build release binary
        run: cargo build --release

      - name: Run kernel-facing sandbox checks (Landlock required)
        env:
          REQUIRE_LANDLOCK: "1"
        run: sudo -E env "PATH=$PATH" REQUIRE_LANDLOCK=1 tests/kernel/run_in_vm.sh

  # Model-based sequence harness: drives random lifecycle operation sequences
  # against a real system-mode supervisor and asserts the invariant oracle is
  # clean and the model matches reality after EVERY step. Finds race/ordering
  # bugs no fixed UAT enumerates. Fixed seeds keep failures reproducible.
  docker-model-tests:
    name: Docker Model Tests (seed ${{ matrix.seed }})
    runs-on: ubuntu-latest
    needs: [changes, clippy]
    if: ${{ needs.changes.outputs.rust == 'true' }}
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        seed: [1, 2, 7, 42]
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build base and model images
        run: |
          docker build -f tests/docker/usecase/Dockerfile.base -t sysg-usecase-base .
          docker build -f tests/docker/model/Dockerfile -t sysg-model .

      - name: Run model sequence harness
        run: docker run --rm -e SEED=${{ matrix.seed }} -e MODEL_STEPS=50 sysg-model

  # Docker UAT tests run in parallel with cargo-test
  # Tests both user mode and kernel mode scenarios
  # Legacy UAT suites (tests/docker/user, tests/docker/kernel). Disabled: their
  # fixtures predate the v2 manifest schema (environment:, instances:,
  # {{.Instance}}, top-level health_check) and their scenarios are superseded by
  # the parity / pid1 / sandbox / model / kernel-matrix lanes. Kept for their
  # unique scenarios (--sys --drop-privileges contract, rolling deploy,
  # cron-under-sys, HA recovery), to be ported into the new lanes before revival.
  docker-uat-tests:
    name: Docker UAT Tests (legacy, disabled)
    runs-on: ubuntu-latest
    needs: [changes, clippy]
    if: false
    timeout-minutes: 120
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: user-linux-x86_64
            test-mode: user
            docker-platform: linux/amd64
            runtime-image: ubuntu:22.04
          - name: kernel-linux-x86_64
            test-mode: kernel
            docker-platform: linux/amd64
            runtime-image: ubuntu:22.04
          - name: kernel-linux-arm64
            test-mode: kernel
            docker-platform: linux/arm64
            runtime-image: ubuntu:22.04
          - name: kernel-debian-x86_64
            test-mode: kernel
            docker-platform: linux/amd64
            runtime-image: debian:bookworm-slim
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - name: Cache Cargo Dependencies
        uses: Swatinem/rust-cache@v2

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Build Docker image (${{ matrix.test-mode }})
        run: |
          set -euo pipefail
          # Build the Docker image using CI-specific Dockerfile
          timeout 45m docker build \
            --platform "${{ matrix.docker-platform }}" \
            --build-arg RUNTIME_IMAGE="${{ matrix.runtime-image }}" \
            -f tests/docker/${{ matrix.test-mode }}/Dockerfile.ci \
            -t systemg-${{ matrix.name }}:ci \
            .

      - name: Run Docker UAT tests (${{ matrix.test-mode }})
        id: run-tests
        run: |
          set -euo pipefail

          # Create a directory for test output
          mkdir -p test-output

          # Run the Docker container with appropriate settings
          if [ "${{ matrix.test-mode }}" = "kernel" ]; then
            # Kernel mode needs privileged access
            timeout 45m docker run \
              --rm \
              --platform "${{ matrix.docker-platform }}" \
              --privileged \
              --cap-add SYS_ADMIN \
              --cap-add NET_ADMIN \
              --security-opt apparmor:unconfined \
              -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
              -v $(pwd)/test-output:/test-output \
              systemg-${{ matrix.name }}:ci | tee test-output/${{ matrix.name }}.log
          else
            # User mode can run without special privileges
            timeout 45m docker run \
              --rm \
              --platform "${{ matrix.docker-platform }}" \
              -v $(pwd)/test-output:/test-output \
              systemg-${{ matrix.name }}:ci | tee test-output/${{ matrix.name }}.log
          fi

          # Extract test results for job summary
          if grep -q "All.*tests passed successfully" test-output/${{ matrix.name }}.log; then
            echo "${{ matrix.name }}: All tests passed" >> $GITHUB_STEP_SUMMARY
          else
            echo "${{ matrix.name }}: Some tests failed" >> $GITHUB_STEP_SUMMARY
            exit 1
          fi

      - name: Upload test logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: docker-${{ matrix.name }}-logs
          path: test-output/${{ matrix.name }}.log

  sync-plugin-version:
    name: Sync Claude Plugin Version
    runs-on: ubuntu-latest
    # The Claude plugin is version-stamped and pushed only AFTER the crate is
    # published to crates.io, so the plugin never advertises a version that is
    # not yet installable.
    needs:
      - extract-version
      - detect-version-bump
      - cargo-publish
    permissions:
      contents: write
    if: >-
      github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.detect-version-bump.outputs.bumped == 'true' && needs.cargo-publish.result == 'success'
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4
        with:
          ref: main

      - name: Set Up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.13"

      - name: Stamp plugin.json with release version
        shell: bash
        run: |
          set -euo pipefail
          python ci/sync_plugin_version.py
          if git diff --quiet .claude-plugin/plugin.json; then
            echo "Plugin version already in sync."
            exit 0
          fi
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add .claude-plugin/plugin.json
          git commit -m "chore(plugin): sync claude plugin version to ${{ needs.extract-version.outputs.tag }}"
          git push origin main

  create-github-release:
    name: Create GitHub Release
    runs-on: ubuntu-latest
    needs:
      - cargo-test
      - cargo-deny
      - cargo-vet
      - docker-usecase-tests
      - docker-pid1-tests
      - docker-sandbox-tests
      - docker-model-tests
      - kernel-native
      - macos-system-mode
      - extract-version
      - detect-version-bump
    permissions:
      contents: write
    if: >-
      github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.detect-version-bump.outputs.bumped == 'true'
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Create or Update release
        uses: softprops/action-gh-release@v2
        with:
          tag_name: ${{ needs.extract-version.outputs.tag }}
          name: ${{ needs.extract-version.outputs.tag }}
          target_commitish: ${{ github.sha }}
          generate_release_notes: true
          append_body: false
          make_latest: true
          fail_on_unmatched_files: false
        env:
          GITHUB_TOKEN: ${{ github.token }}

  upload-release-binaries:
    name: Upload Release Binaries
    runs-on: ${{ matrix.job.os }}
    permissions:
      contents: write
    needs:
      - cargo-test
      - detect-version-bump
      - create-github-release
      - extract-version
    if: |
      !cancelled() &&
      needs.cargo-test.result == 'success' &&
      (
        (needs.create-github-release.result == 'success') ||
        (needs.create-github-release.result == 'skipped' && github.event_name == 'release' && github.event.action == 'published') ||
        (github.event_name == 'workflow_dispatch')
      )
    strategy:
      matrix:
        job:
          - os: ubuntu-latest
            platform: linux
            target: x86_64-unknown-linux-gnu
            cross_image: x86_64-linux-gnu
            cross_dockerfile: ci/Dockerfile.x86_64-unknown-linux-gnu-clang
            features: linux
          - os: ubuntu-latest
            platform: linux-arm
            target: aarch64-unknown-linux-gnu
            cross_image: aarch64-linux-gnu
            cross_dockerfile: ci/Dockerfile.aarch64-unknown-linux-gnu-clang
            features: linux
          - os: ubuntu-latest
            platform: debian
            target: x86_64-unknown-linux-gnu
            cross_image: debian-x86_64-linux-gnu
            cross_dockerfile: ci/Dockerfile.debian-x86_64-unknown-linux-gnu-clang
            features: linux
            artifact_suffix: -debian
          - os: ubuntu-latest
            platform: alpine
            target: x86_64-unknown-linux-musl
            cross_image: x86_64-linux-musl
            cross_dockerfile: ci/Dockerfile.x86_64-unknown-linux-musl-clang
            features: linux
            artifact_suffix: -alpine
          - os: ubuntu-latest
            platform: alpine-arm
            target: aarch64-unknown-linux-musl
            cross_image: aarch64-linux-musl
            cross_dockerfile: ci/Dockerfile.aarch64-unknown-linux-musl-clang
            features: linux
            artifact_suffix: -alpine
          - os: macos-latest
            platform: darwin
            target: x86_64-apple-darwin
          - os: macos-latest
            platform: darwin-arm
            target: aarch64-apple-darwin

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        if: matrix.job.cross_image
        uses: docker/setup-buildx-action@v3

      - name: Setup custom cross env ${{ matrix.job.cross_image }}
        if: matrix.job.cross_image
        uses: docker/build-push-action@v5
        with:
          context: ci
          file: ${{ matrix.job.cross_dockerfile }}
          tags: ${{ matrix.job.cross_image }}:latest
          load: true
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - name: Install packages (macOS)
        if: matrix.job.os == 'macos-latest'
        run: |
          ci/macos-install-packages.sh

      - name: Install toolchain
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: ${{ env.RUSTC_VERSION }}
          targets: ${{ matrix.job.target }}

      - name: Install cross
        uses: baptiste0928/cargo-install@v3
        with:
          crate: cross
          cache-key: ${{ matrix.job.target }}
          version: "0.2.5"

      - name: Build sysg
        run: |
          cross \
            build \
            --profile=release \
            --target ${{ matrix.job.target }}

      - name: Strip release binary linux x86_64
        if: matrix.job.target == 'x86_64-unknown-linux-gnu'
        run: >
          strip "target/${{ matrix.job.target }}/release/sysg"

      - name: Strip release binary aarch64-linux-gnu
        if: matrix.job.target == 'aarch64-unknown-linux-gnu'
        run: |
          docker run --rm -v \
          "$PWD/target:/target:Z" \
          aarch64-linux-gnu:latest \
          aarch64-linux-gnu-strip \
          /target/aarch64-unknown-linux-gnu/release/sysg

      - name: Strip release binary mac
        if: matrix.job.os == 'macos-latest'
        run: |
          strip -x "target/${{ matrix.job.target }}/release/sysg"

      - name: Prepare Binary Artifact
        env:
          PLATFORM_NAME: ${{ matrix.job.platform }}
          TARGET: ${{ matrix.job.target }}
          ARTIFACT_SUFFIX: ${{ matrix.job.artifact_suffix || '' }}
        run: >
          # Get version from either the tag (for auto-release) or GITHUB_REF (for manual releases)

          if [ "${{ needs.extract-version.outputs.tag }}" != "" ]; then
            SYSG_VERSION="${{ needs.extract-version.outputs.tag }}"
          else
            SYSG_VERSION="${GITHUB_REF#refs/tags/}"
          fi

          # optionally trim v from tag prefix

          SYSG_VERSION="${SYSG_VERSION#v}"

          echo "version is: $SYSG_VERSION"

          # setup artifact filename

          SYSG_ARTIFACT="sysg-$SYSG_VERSION-$TARGET$ARTIFACT_SUFFIX"

          SYSG_ZIP_FILE_NAME="$SYSG_ARTIFACT.tar.gz"

          echo "SYSG_ZIP_FILE_NAME=$SYSG_ZIP_FILE_NAME" >> $GITHUB_ENV

          # create zip file

          mkdir -pv "$SYSG_ARTIFACT"

          cp "target/${{ matrix.job.target }}/release/sysg" "$SYSG_ARTIFACT"

          tar -czvf "$SYSG_ZIP_FILE_NAME" "$SYSG_ARTIFACT"

      - name: Upload Sysg Artifact
        uses: softprops/action-gh-release@v1
        if: github.event_name == 'release' || needs.create-github-release.result == 'success'
        env:
          GITHUB_TOKEN: ${{ github.token }}
        with:
          tag_name: ${{ needs.extract-version.outputs.tag }}
          files: ./${{ env.SYSG_ZIP_FILE_NAME }}

#      - name: Notify if Job Fails
#        uses: ravsamhq/notify-slack-action@v2
#        if: always() && (github.ref == 'refs/heads/master' || github.ref_type == 'tag')
#          && matrix.job.os != 'macos-latest'
#        with:
#          status: ${{ job.status }}
#          token: ${{ secrets.GITHUB_TOKEN }}
#          notification_title: "{workflow} has {status_message}"
#          message_format: "{emoji} *{workflow}* {status_message} in <{repo_url}|{repo}> :
#            <{run_url}|View Run Results>"
#          footer: ""
#          notify_when: failure
#        env:
#          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_NOTIFY_BUILD }}

  upload-s3:
    name: Upload Binaries to S3
    needs:
      - upload-release-binaries
      - extract-version
    if: |
      !cancelled() &&
      needs.upload-release-binaries.result == 'success'
    environment: ci
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Download and upload release assets to AWS S3
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          RELEASE_TAG: ${{ needs.extract-version.outputs.tag }}
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: us-east-2
          CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.SH_SYSG_DEV_CLOUDFRONT_DISTRIBUTION_ID }}
          S3_BUCKET: sh.sysg.dev
        run: |
          set -euo pipefail

          VERSION="${RELEASE_TAG#v}"
          PUBLIC_BASE_URL="https://${S3_BUCKET}"

          # Objects fall into two classes with opposite caching needs, and uploading
          # them without any Cache-Control leaves CloudFront to apply its 24h default
          # TTL to everything -- which is how a stale `latest/VERSION` gets served for
          # a full day after a release.
          #
          #   MUTABLE   pointers that change every release (latest/*, the install
          #             script). Must revalidate against the origin before serving, or
          #             `curl | sh` hands users the previous release's binary.
          #   IMMUTABLE version-pinned artifacts that never change once published.
          #             Cache them at the edge forever; they never need invalidating.
          CACHE_MUTABLE="no-cache, max-age=0, must-revalidate"
          CACHE_IMMUTABLE="public, max-age=31536000, immutable"

          # An empty distribution ID silently skipped invalidation, which is what kept
          # the stale-TTL bug hidden. Correctness no longer depends on invalidation
          # running, but a release that cannot invalidate is still a release that
          # serves stale bytes at the edge for longer than it should. Fail loudly.
          if [ -z "${CLOUDFRONT_DISTRIBUTION_ID:-}" ]; then
            echo "CLOUDFRONT_DISTRIBUTION_ID is empty; refusing to publish a release that cannot invalidate the CDN." >&2
            exit 1
          fi

          aws s3 cp scripts/index.sh "s3://${S3_BUCKET}/index.sh" \
            --content-type "text/x-shellscript" \
            --cache-control "$CACHE_MUTABLE"
          aws s3 cp scripts/index.sh "s3://${S3_BUCKET}/index.html" \
            --content-type "text/x-shellscript" \
            --cache-control "$CACHE_MUTABLE"

          printf '%s\n' "$VERSION" > VERSION
          aws s3 cp VERSION "s3://${S3_BUCKET}/latest/VERSION" \
            --content-type "text/plain" \
            --cache-control "$CACHE_MUTABLE"

          # Get release ID from tag (works for both auto-release and manual releases)
          if [ -n "${{ github.event.release.id }}" ]; then
            RELEASE_ID="${{ github.event.release.id }}"
          else
            # Fetch release ID by tag for auto-releases
            RELEASE_ID=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
              -H "Accept: application/vnd.github.v3+json" \
              "https://api.github.com/repos/${{ github.repository }}/releases/tags/$RELEASE_TAG" \
              | jq -r '.id')
          fi

          echo "Fetching .tar.gz assets for release ID $RELEASE_ID ..."

          while read -r url name; do
            echo "Downloading $name from $url ..."
            curl -sSL -H "Authorization: token $GITHUB_TOKEN" -o "$name" "$url"

            # The version-pinned key never changes once published, so it can be cached
            # forever. The same bytes under latest/ are a moving pointer and must be
            # revalidated, otherwise the edge keeps serving the previous release.
            echo "Uploading $name to s3://${S3_BUCKET}/${name} ..."
            aws s3 cp "$name" "s3://${S3_BUCKET}/${name}" \
              --content-type "application/gzip" \
              --cache-control "$CACHE_IMMUTABLE"

            echo "Uploading $name to s3://${S3_BUCKET}/latest/${name} ..."
            aws s3 cp "$name" "s3://${S3_BUCKET}/latest/${name}" \
              --content-type "application/gzip" \
              --cache-control "$CACHE_MUTABLE"

            echo "Uploaded $name successfully"
          done < <(
            curl -s -H "Authorization: token $GITHUB_TOKEN" \
                -H "Accept: application/vnd.github.v3+json" \
                "https://api.github.com/repos/${{ github.repository }}/releases/$RELEASE_ID/assets" \
            | jq -r '.[] | select(.name | endswith(".tar.gz")) | "\(.browser_download_url) \(.name)"'
          )

          invalidation_id="$(aws cloudfront create-invalidation \
            --distribution-id "$CLOUDFRONT_DISTRIBUTION_ID" \
            --paths "/" "/index.sh" "/index.html" "/latest/*" "/sysg-${VERSION}-*.tar.gz" \
            --query 'Invalidation.Id' \
            --output text)"
          aws cloudfront wait invalidation-completed \
            --distribution-id "$CLOUDFRONT_DISTRIBUTION_ID" \
            --id "$invalidation_id"

          # Check the origin first. This asserts what we actually published, and
          # cannot be fooled by whatever an edge happens to be holding.
          origin_version="$(aws s3 cp "s3://${S3_BUCKET}/latest/VERSION" - | sed 's/^v//' | awk 'NR==1 {print $1; exit}')"
          if [ "$origin_version" != "$VERSION" ]; then
            echo "S3 origin version mismatch: got '$origin_version', expected '$VERSION'" >&2
            exit 1
          fi

          # Then confirm the edge caught up. Invalidation completing does not guarantee
          # every PoP has converged, so retry rather than failing the release on a
          # single unlucky read.
          public_version=""
          for attempt in $(seq 1 10); do
            public_version="$(curl -fsSL -H 'Cache-Control: no-cache' "${PUBLIC_BASE_URL}/latest/VERSION" | sed 's/^v//' | awk 'NR==1 {print $1; exit}')"
            if [ "$public_version" = "$VERSION" ]; then
              break
            fi
            echo "Attempt ${attempt}: CDN still serving '${public_version}', want '${VERSION}'; retrying ..."
            sleep 15
          done

          if [ "$public_version" != "$VERSION" ]; then
            echo "Public latest version mismatch after invalidation: got '$public_version', expected '$VERSION'" >&2
            exit 1
          fi

  cargo-publish:
    name: Publish to crates.io
    runs-on: ubuntu-latest
    needs:
      - cargo-test
      - detect-version-bump
      - create-github-release
      - upload-s3
    if: |
      !cancelled() &&
      needs.cargo-test.result == 'success' &&
      needs.upload-s3.result == 'success' &&
      (
        (
          github.event_name == 'push' &&
          github.ref == 'refs/heads/main' &&
          needs.create-github-release.result == 'success'
        ) ||
        (github.event_name == 'workflow_dispatch')
      )
    environment: ci
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - name: Publish Crate
        env:
          CARGO_REGISTRY_TOKEN: ${{ secrets.CRATESIO_TOKEN }}
        run: |
          if [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ github.event.inputs.test-release }}" == "true" ]; then
            cargo publish --dry-run --locked --no-verify
          else
            cargo publish --locked --no-verify
          fi