kache 1.2.2

name: Bench

# Clone benchmark — builds kache from THIS ref and runs the cross-clone
# cold/warm scenarios and the repeated hk/eza/aube suites, then uploads reports. This is the heavy, hours-long, tens-of-GB-disk job, so
# it lives in its own workflow and never blocks a push or PR. The per-PR perf
# gate (perf-gate.yml) measures hk, eza and aube separately.
#
# Triggers:
#   - schedule  : nightly, runs every arm in the matrix below except the
#                 Firefox sccache and mbx arms, which bench-sccache.yml
#                 dispatches on its own weekly cadence.
#   - dispatch  : manual, pick a scenario (default all, excluding Firefox).
#                 Extra flags depend on the selected recipe.
#
# The two Firefox/Windows benchmarks live in bench-firefox-windows.yml. They
# take five to six hours each, and a run is finished only when its slowest job
# is, so keeping them here held every other arm's telemetry behind them.
#
# Runs on the kunobi self-hosted Linux ARC runners in
# tenant-int-dev/zur1-builder1: `kunobi-runners-large`, runners with a 120Gi
# ephemeral-storage request and 32Gi of memory for the heavy arms, and
# `kunobi-runners`, the six-slot general pool at 60Gi and 16Gi. Each arm picks
# by its disk floor. The large pool is not limited to one job at a time: llvm,
# surrealdb and firefox-pull all started on it together on 2026-09-11, so heavy
# arms can overlap and their wall times are not isolated. A gha-runner-scale-set
# is targeted by its INSTALLATION NAME as a single `runs-on` label
# (`runs-on: kunobi-runners`) — NOT by a `[self-hosted, Linux, X64, …]` label
# set (that scheme only matches the legacy multi-label runners, e.g.
# kunobi-windows). These are EPHEMERAL pods (fresh per job,
# `_work` is an emptyDir): there is no stale tool state to isolate, but there is
# also no persistent clone-ref cache, so every run re-clones the upstream repo,
# and Firefox's `./mach bootstrap` re-provisions ~/.mozbuild each time. The image
# is minimal, so the base build toolchain is installed below.
on:
  schedule:
    # 03:37 Zurich time nightly. Keeping :37 avoids top-of-hour cron congestion.
    - cron: "37 3 * * *"
      timezone: "Europe/Zurich"
  workflow_dispatch:
    inputs:
      scenario:
        description: "Which scenario(s) to run"
        type: choice
        options: [all, hk-comparison, surrealdb, lance, opendal, kobe, cuda-oxide, cuda-oxide-patched, eza, aube, llvm, hk, hk-pull-perf, hk-mbx, hk-pull-mbx, opendal-mbx, lance-mbx, surrealdb-mbx, firefox-sccache, firefox-mbx]
        default: all
      extra_args:
        description: "Extra recipe arguments (hk/eza/aube: bench-short flags; other scenarios: --skip-clone, --retry)"
        type: string
        default: ""

concurrency:
  # One bench run per scenario+ref at a time; don't cancel a multi-hour run.
  #
  # The scenario is part of the key because that is what the sentence above
  # says, and for a while it was not: the key was workflow+ref alone, so every
  # scenario on a ref shared ONE slot. An 8-minute `hk` dispatch queued behind
  # an 8-hour `firefox-pull`, which is the exact thing this block exists to
  # prevent. Splitting the runner pool does not help - three runners are no use
  # to a job Actions will not even assign.
  #
  # A scheduled run has no inputs and keys on 'all', so the nightly still
  # serialises against another full sweep, which is what we want.
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs.scenario || 'all' }}
  cancel-in-progress: false

permissions:
  contents: read

env:
  CARGO_TERM_COLOR: always

jobs:
  # Build the bench matrix from the trigger. Selection must happen HERE, not in
  # a job-level `if` on `bench`: the `matrix` context is NOT available in a
  # job-level `if` (it is evaluated before the matrix is expanded, so it reads
  # empty) — a dispatch picking a single scenario would otherwise run zero arms.
  # This tiny job uses the validation runner so a skipped scenario never
  # claims a benchmark runner. The schedule includes Firefox; manual all
  # excludes it. Per-arm timeout/min_free_gb travel with each entry.
  plan:
    if: (github.repository == 'kunobi-ninja/kache' || vars.ENABLE_BENCHMARKS == 'true')
    name: Plan scenarios
    runs-on: ${{ fromJSON(!github.event.repository.private && '"ubuntu-latest"' || vars.CI_RUNNER_LINUX) }}
    outputs:
      matrix: ${{ steps.pick.outputs.matrix }}
      # Avoid starting a matrix job if no Linux scenarios were selected.
      has_linux: ${{ steps.pick.outputs.has_linux }}
    steps:
      - id: pick
        shell: bash
        env:
          # schedule has no inputs -> treat as `all`; dispatch honours the input.
          EVENT_NAME: ${{ github.event_name }}
          SCENARIO: ${{ github.event_name == 'schedule' && 'all' || github.event.inputs.scenario }}
        run: |
          # Each arm carries: name (display + artifact label), cmd (the `just`
          # recipe + profile to run), dir (scratch subdir under tmp/bench to
          # upload), timeout (benchmark-step minutes), job_timeout (adds time
          # for setup and the final telemetry upload), min_free_gb (precheck
          # floor — fail the arm if free disk is below this before we start,
          # since disk-full mid-build is the worst failure mode; emptyDir is
          # node-backed, so the precheck also catches a too-small node).
          #
          # SurrealDB's `surreal` server binary at default features — a large
          # pure-Rust workspace whose default features also drag in native C/C++
          # deps (rocksdb, grpcio, quickjs) that build outside kache. Comparable
          # native-prerequisite set (already installed below).
          surrealdb='{"name":"surrealdb","cmd":"bench surrealdb","dir":"bench-surrealdb","timeout":180,"job_timeout":225,"min_free_gb":80,"runner":"kunobi-runners-large"}'
          # Lance's `lance` crate at default features — a large pure-Rust
          # workspace (Arrow, DataFusion, object-store backends, prost codegen).
          # Needs protoc + openssl headers (already installed below). Lighter
          # than surrealdb: no rocksdb/grpcio/quickjs native compile.
          #
          # Lance, OpenDAL and LLVM run three samples through bench-short: one
          # cold build and three of each warm build per tool, Kache and mbx
          # alternating in one pod. A single warm sample of Lance has ranged
          # from 5 to 420 s across nights; repeats reuse the cold build, so they
          # cost minutes. No contention stage: it knows only hk, eza and aube.
          # Both tools keep a checkout and cold snapshot, hence the disk floors.
          lance='{"name":"lance","cmd":"bench-short lance 3 --mbx mbx","dir":"bench-lance","timeout":180,"job_timeout":225,"min_free_gb":100,"runner":"kunobi-runners-large"}'
          # OpenDAL core (apache/opendal `core/` workspace) with the portable
          # service-feature set — mid-size pure-Rust, lighter than surrealdb.
          # aws-lc-sys (rustls) compiles outside kache; cmake + a C compiler
          # are already in the apt step below.
          opendal='{"name":"opendal","cmd":"bench-short opendal 3 --mbx mbx","dir":"bench-opendal","timeout":120,"job_timeout":165,"min_free_gb":60,"runner":"kunobi-runners"}'
          # kobe (kunobi-ninja/kobe): our own Kubernetes operator and CLI, a
          # four-package workspace over about 790 locked crates. Minutes, and
          # a C compiler is all it needs.
          kobe='{"name":"kobe","cmd":"bench kobe --warm-same-tree","dir":"bench-kobe","timeout":60,"job_timeout":90,"min_free_gb":40,"runner":"kunobi-runners"}'
          # NVlabs/cuda-oxide's examples-compile job: its rustc codegen backend,
          # the shared host dependency tree, and 200+ example workspaces built
          # into one target dir. Needs the CUDA toolkit headers and LLVM 22,
          # installed only for this arm below.
          cuda_oxide='{"name":"cuda-oxide","cmd":"bench cuda-oxide --warm-same-tree","dir":"bench-cuda-oxide","timeout":120,"job_timeout":180,"min_free_gb":40,"runner":"kunobi-runners"}'
          # The same job with a cuda-oxide patch that declares every backend
          # output, so kache can trust and cache the backend compiles.
          cuda_oxide_patched='{"name":"cuda-oxide-patched","cmd":"bench cuda-oxide-patched --warm-same-tree","dir":"bench-cuda-oxide-patched","timeout":120,"job_timeout":180,"min_free_gb":40,"runner":"kunobi-runners"}'
          # Full cold Firefox build is the long pole — hours. `bench firefox`
          # resolves to the exact `bench-firefox` scenario: `--profile firefox`
          # is a name SUBSTRING match that also catches `bench-firefox-windows`,
          # but discovery prefers the exact-name hit (#458), so no `os:` tag is
          # needed to pin this host-native arm. (The Windows variant runs in
          # bench-firefox-windows.yml, on the self-hosted Windows runner.)
          firefox='{"name":"firefox","cmd":"bench firefox","dir":"bench-firefox","timeout":360,"job_timeout":405,"min_free_gb":120,"runner":"kunobi-runners-large"}'
          # Same Firefox shape through sccache for a side-by-side comparison.
          firefox_sccache='{"name":"firefox-sccache","cmd":"bench-sccache firefox","dir":"bench-firefox-sccache","timeout":360,"job_timeout":405,"min_free_gb":120,"runner":"kunobi-runners-large"}'
          # Firefox through `mbx exec`, which caches its C and C++ compiles
          # and leaves the Rust crates to compile in every phase (see the
          # scenario file).
          firefox_mbx='{"name":"firefox-mbx","cmd":"bench-mbx firefox","dir":"bench-firefox-mbx","timeout":360,"job_timeout":405,"min_free_gb":120,"runner":"kunobi-runners-large"}'
          # Almost-pure C/C++ CMake build (X86-only Release) — lighter than Firefox.
          llvm='{"name":"llvm","cmd":"bench-short llvm 3","dir":"bench-llvm","timeout":240,"job_timeout":285,"min_free_gb":100,"runner":"kunobi-runners-large"}'
          # Daily-pull (TEMPORAL axis, #477): one clone, build ref A then rebuild
          # ref B in the SAME worktree — two full sequential Firefox builds, so a
          # longer timeout than `firefox`. Only ONE worktree (no cross-clone
          # clone-b), so disk is comparable, not double; keep the 120 GB floor.
          # `bench firefox-pull` resolves to the exact `bench-firefox-pull`
          # scenario (substring also catches `bench-firefox-pull-windows`, but
          # discovery prefers the exact-name hit). The Windows variant runs in
          # bench-firefox-windows.yml, on the self-hosted Windows runner.
          firefox_pull='{"name":"firefox-pull","cmd":"bench firefox-pull","dir":"bench-firefox-pull","timeout":480,"job_timeout":525,"min_free_gb":120,"runner":"kunobi-runners-large"}'
          # hk (jdx/hk): a mid-size Rust CLI whose build scripts compile ~750
          # C/asm objects through cc-rs (aws-lc-sys, vendored libgit2), wired
          # the way `kache init` does it so both compiler families are measured.
          # `--warm-same-tree` adds the same-path warm phase between cold and the
          # cross-clone warm. Minutes, not hours; a C compiler is all it needs.
          #
          # The three short subjects (hk, eza, aube) pass the same flags: six
          # kache samples, three of mbx (the daily reading against it, so it is
          # a median too, not one sample), and three warm contention batches. Six
          # of everything is the per-PR gate's setting for a head-against-base
          # test, which needs five pairs; a nightly has no base. sccache is
          # pinned and runs only in the weekly reference (bench-sccache.yml);
          # under contention it took a third of the aube arm.
          hk='{"name":"hk","cmd":"bench-short hk 6 --context-samples 3 --contention-samples 3 --mbx mbx","dir":"bench-hk","timeout":120,"job_timeout":150,"min_free_gb":40,"runner":"kunobi-runners"}'
          # The same hk shape through mbx, for a side-by-side comparison
          # (as firefox-sccache is for firefox). The workflow installs the
          # latest mbx release; the run records the version it measured.
          hk_mbx='{"name":"hk-mbx","cmd":"bench-mbx hk --warm-same-tree","dir":"bench-hk-mbx","timeout":60,"job_timeout":90,"min_free_gb":40,"runner":"kunobi-runners"}'
          # Next commit in the same worktree: cold at the parent, warm on the
          # child. Predictions on, verification off. #1000 made that the
          # default, so the predictions-off and verify-always arms are gone.
          hk_pull_perf='{"name":"hk-pull-perf","cmd":"bench hk-pull-perf","dir":"bench-hk-pull-perf","timeout":60,"job_timeout":90,"min_free_gb":40,"runner":"kunobi-runners"}'
          hk_pull_mbx='{"name":"hk-pull-mbx","cmd":"bench-mbx hk-pull","dir":"bench-hk-pull-mbx","timeout":60,"job_timeout":90,"min_free_gb":40,"runner":"kunobi-runners"}'
          # eza: the smallest subject and the only one built through a fat-LTO
          # release profile, so the final unit is one whole-program codegen
          # instead of many small ones. Its bundled libgit2 and zlib put the
          # majority of its cacheable units on the C side (199 of 319 in a local
          # run). The cheapest arm in the sweep: minutes, and under a gigabyte
          # of build output.
          eza='{"name":"eza","cmd":"bench-short eza 6 --context-samples 3 --contention-samples 3 --mbx mbx","dir":"bench-eza","timeout":60,"job_timeout":90,"min_free_gb":40,"runner":"kunobi-runners"}'
          # aube: a 15-crate workspace with C dependencies (aws-lc-sys,
          # mimalloc, tree-sitter through cc-rs; libz-ng-sys through cmake).
          # Where hk is one crate over a large graph, aube's time goes to
          # workspace units that depend on each other. At six samples of
          # every tool this arm needs about 146 minutes.
          aube='{"name":"aube","cmd":"bench-short aube 6 --context-samples 3 --contention-samples 3 --mbx mbx","dir":"bench-aube","timeout":120,"job_timeout":150,"min_free_gb":40,"runner":"kunobi-runners"}'
          # Subjects through mbx's Cargo front, which caches both Rust and the
          # C a build script compiles. Builds Cargo does not drive go through
          # `mbx exec` instead and get C/C++ caching only: firefox has such an
          # arm (firefox_mbx above), llvm does not. The nightly measures
          # opendal and lance through mbx inside their bench-short arms above;
          # these single runs stay for a manual dispatch.
          opendal_mbx='{"name":"opendal-mbx","cmd":"bench-mbx opendal","dir":"bench-opendal-mbx","timeout":120,"job_timeout":165,"min_free_gb":40,"runner":"kunobi-runners"}'
          lance_mbx='{"name":"lance-mbx","cmd":"bench-mbx lance","dir":"bench-lance-mbx","timeout":180,"job_timeout":225,"min_free_gb":60,"runner":"kunobi-runners"}'
          surrealdb_mbx='{"name":"surrealdb-mbx","cmd":"bench-mbx surrealdb","dir":"bench-surrealdb-mbx","timeout":180,"job_timeout":225,"min_free_gb":80,"runner":"kunobi-runners-large"}'
          case "$SCENARIO" in
            surrealdb)            inc="[$surrealdb]" ;;
            lance)                inc="[$lance]" ;;
            opendal)              inc="[$opendal]" ;;
            kobe)                 inc="[$kobe]" ;;
            cuda-oxide)           inc="[$cuda_oxide]" ;;
            cuda-oxide-patched)   inc="[$cuda_oxide_patched]" ;;
            llvm)                 inc="[$llvm]" ;;
            hk-comparison)       inc="[$hk,$hk_pull_perf,$hk_pull_mbx]" ;;
            hk)                   inc="[$hk]" ;;
            hk-mbx)               inc="[$hk_mbx]" ;;
            hk-pull-perf)         inc="[$hk_pull_perf]" ;;
            hk-pull-mbx)          inc="[$hk_pull_mbx]" ;;
            eza)                  inc="[$eza]" ;;
            aube)                 inc="[$aube]" ;;
            opendal-mbx)          inc="[$opendal_mbx]" ;;
            lance-mbx)            inc="[$lance_mbx]" ;;
            surrealdb-mbx)        inc="[$surrealdb_mbx]" ;;
            # Not in the nightly: sccache is pinned in mise.toml, so this arm
            # measures the same binary every night while only kache moves, and
            # kache against kache is what the per-PR perf gate reports.
            # `bench-sccache.yml` gives it its own run — weekly, and on a pin
            # bump — so the nightly never waits for it and its results reach
            # the collector on their own schedule.
            firefox-sccache)      inc="[$firefox_sccache]" ;;
            # Weekly too, from the same dispatcher. It tracks the latest mbx
            # release, and a Firefox build is too long for the nightly.
            firefox-mbx)          inc="[$firefox_mbx]" ;;
            # Windows-only scenarios are served by bench-firefox-windows.yml,
            # so the Linux matrix is EMPTY for them (an empty include
            # skips the `bench` job cleanly). Without this they'd fall to `*)`
            # and wastefully run the whole Linux `all` matrix alongside.
            *)                    inc="[$surrealdb,$lance,$opendal,$kobe,$cuda_oxide,$cuda_oxide_patched,$llvm,$hk,$hk_pull_perf,$hk_pull_mbx,$eza,$aube,$surrealdb_mbx]" ;;
          esac
          if [ "$EVENT_NAME" = "schedule" ]; then
            inc="$(jq -c --argjson extra "[$firefox,$firefox_pull]" '. + $extra' <<<"$inc")"
          fi
          echo "matrix={\"include\":$inc}" >> "$GITHUB_OUTPUT"
          if [ "$inc" = "[]" ]; then
            echo "has_linux=false" >> "$GITHUB_OUTPUT"
          else
            echo "has_linux=true" >> "$GITHUB_OUTPUT"
          fi

  bench:
    name: Bench (${{ matrix.name }})
    needs: plan
    # Skip cleanly when the Linux matrix is empty (a windows-only dispatch) — an
    # empty matrix would otherwise fail the run.
    if: needs.plan.outputs.has_linux == 'true'
    # gha-runner-scale-set: match by the scale-set's installation name (single
    # label), not a `[self-hosted, …]` set. See header comment.
    #
    # Which pool is per arm, from the disk floor beside it in the matrix: an
    # arm that fits the general pool's 80Gi ceiling runs there, several at a
    # time, and the heavy ones keep the reserved node and its 120 GB floor.
    #
    # Read the ceiling and the floor together. The ceiling is a pod limit on a
    # node-backed emptyDir, and crossing it evicts the runner mid-build. The
    # floor is a free-space precheck against the node filesystem, which is
    # several times larger, so it passes long before the ceiling is reached
    # and cannot catch that case on its own.
    runs-on: >-
      ${{ fromJSON(
        matrix.runner == 'kunobi-runners-large' &&
        (vars.BENCH_RUNNER_LINUX_LARGE ||
         (github.repository == 'kunobi-ninja/kache' && '"kunobi-runners-large"') || 'null') ||
        vars.BENCH_RUNNER_LINUX ||
        (github.repository == 'kunobi-ninja/kache' && '"kunobi-runners"')
      ) }}
    timeout-minutes: ${{ matrix.job_timeout }}
    strategy:
      # Arms run concurrently (up to the kunobi-runners scale-set's maxRunners).
      # They previously starved the 120 GB free-disk precheck (103 GB, #447) by
      # co-locating Firefox-class arms on one node — work is a node-backed
      # emptyDir and the scheduler is disk-blind. Two things handle that
      # runner-side (Zondax/tenant-int-dev), so neither needs a `max-parallel`
      # cap here: the general pool carries a podAntiAffinity that spreads its
      # concurrent runners onto distinct nodes, giving each light arm its own
      # node's disk. The heavy arms sit on `kunobi-runners-large`, which was
      # assumed to take one job at a time but has started three together (see
      # the header); the free-disk precheck is what protects them there.
      # Matrix is generated by `plan` (see above) — selection can't live in a
      # job-level `if` because `matrix` isn't available there.
      fail-fast: false
      matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
    defaults:
      run:
        shell: bash
    steps:
      # For the job's own duration, reported by the last step.
      - name: Record the job start
        run: echo "BENCH_JOB_STARTED=$(date +%s)" >> "$GITHUB_ENV"

      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        timeout-minutes: 10

      - name: Set up Node for Kartero
        uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0
        with:
          node-version: 24

      # Seed a valid failure artifact before prerequisites or benchmark work.
      # A completed benchmark overwrites it with the full metric set. Any early
      # exit leaves verdict.ok=0 for the final always-running upload.
      - name: Seed failure telemetry
        run: |
          mkdir -p tmp/bench
          npx --yes @kunobi/kartero@0.4.14 gauge \
            --name kache.bench.verdict.ok \
            --value 0 \
            --unit 1 \
            --attribute kache.bench.project=${{ matrix.dir }} \
            --attribute kache.bench.cache_tool=${{ endsWith(matrix.name, '-sccache') && 'sccache' || endsWith(matrix.name, '-mbx') && 'mbx' || 'kache' }} \
            --output tmp/bench/${{ matrix.dir }}
        env:
          NPM_CONFIG_CACHE: ${{ runner.temp }}/kartero-npm-cache

      # The ARC runner image is minimal. Install the base toolchain the bench
      # needs around the build: SurrealDB links native dependencies and needs
      # protoc + clang + cmake + openssl headers (SurrealDB's default features
      # need the same set — rocksdb bindgen wants libclang, storage-tikv/grpcio
      # wants protoc; Lance's prost-build wants protoc and a few default-feature
      # backends want openssl headers; OpenDAL's rustls/aws-lc-sys wants cmake +
      # a C compiler); Firefox's `./mach bootstrap`
      # pulls its own clang/rust toolchain but still needs python3 + common
      # archive/file utilities present first; the LLVM scenario drives CMake
      # with the Ninja generator, so it needs ninja-build. git/curl are needed by
      # the clone and by mise. (Firefox also installs further packages itself via
      # bootstrap, which uses passwordless sudo on this image.)
      - name: Install build prerequisites (apt)
        run: |
          export DEBIAN_FRONTEND=noninteractive
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends \
            build-essential pkg-config \
            protobuf-compiler clang libclang-dev cmake ninja-build \
            libssl-dev \
            git curl wget ca-certificates file unzip bzip2 xz-utils \
            python3 python3-dev python3-pip

      # cuda-oxide only. Its bindgen step needs cuda.h and curand.h, and its
      # device pipeline shells out to llc-22 for PTX. Same packages as the
      # upstream examples-compile job: CUDA 13.1 nvcc, cudart-dev, curand-dev
      # and nvjitlink-dev, plus LLVM 22 from apt.llvm.org.
      - name: Install CUDA toolkit and LLVM 22 (cuda-oxide only)
        if: startsWith(matrix.name, 'cuda-oxide')
        timeout-minutes: 15
        run: |
          set -euo pipefail
          source /etc/os-release
          export DEBIAN_FRONTEND=noninteractive
          distro="ubuntu${VERSION_ID//./}"
          wget -q -O "$RUNNER_TEMP/cuda-keyring.deb" \
            "https://developer.download.nvidia.com/compute/cuda/repos/${distro}/x86_64/cuda-keyring_1.1-1_all.deb"
          sudo dpkg -i "$RUNNER_TEMP/cuda-keyring.deb"
          sudo install -d -m 0755 /etc/apt/keyrings
          wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key \
            | sudo tee /etc/apt/keyrings/apt-llvm-org.asc >/dev/null
          echo "deb [signed-by=/etc/apt/keyrings/apt-llvm-org.asc] https://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-22 main" \
            | sudo tee /etc/apt/sources.list.d/llvm-22.list
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends \
            cuda-nvcc-13-1 cuda-cudart-dev-13-1 libcurand-dev-13-1 libnvjitlink-dev-13-1 \
            llvm-22
          test -f /usr/local/cuda-13.1/include/cuda.h
          llc-22 --version | head -2
          {
            echo "CUDA_TOOLKIT_PATH=/usr/local/cuda-13.1"
            echo "CUDA_OXIDE_LLC=/usr/bin/llc-22"
          } >> "$GITHUB_ENV"

      # mise installs rust (per rust-toolchain.toml) and just (the bench recipe
      # is `just bench`) for every arm, and sccache only for the sccache arms
      # and a short suite dispatched with `--sccache` (the weekly reference),
      # version pinned by mise.toml. A
      # tool an arm does not use must never be able to fail its install step:
      # on 2026-09-10 a GitHub attestation 503 on the sccache download failed
      # five arms that never run it. A transient failure gets one retry below.
      # On the ephemeral Linux pod there's no stale state and no competing
      # system Rust, so mise-action puts the tools on PATH directly; none of the
      # macOS isolation/PATH workarounds from ci.yml are needed.
      - name: Install tools via mise
        id: mise
        continue-on-error: true
        uses: zondax/actions/setup-mise@14e656f2dcc6012c36298e3a6e95dc4ae0f022f0 # v1.5.3
        with:
          install: true
          # Pin mise CLI — see ci.yml (v2026.6.10 regression).
          version: 2026.6.9
          install_args: --force rust github:casey/just${{ (endsWith(matrix.name, '-sccache') || contains(github.event.inputs.extra_args, '--sccache')) && ' github:mozilla/sccache' || '' }}
          cache: false
          reshim: false

      - name: Install tools via mise (retry)
        if: steps.mise.outcome == 'failure'
        uses: zondax/actions/setup-mise@14e656f2dcc6012c36298e3a6e95dc4ae0f022f0 # v1.5.3
        with:
          install: true
          version: 2026.6.9
          install_args: --force rust github:casey/just${{ (endsWith(matrix.name, '-sccache') || contains(github.event.inputs.extra_args, '--sccache')) && ' github:mozilla/sccache' || '' }}
          cache: false
          reshim: false

      # The comparison tool for the `-mbx` arms, installed from its GitHub
      # release rather than through the shared mise step: the pinned mise CLI
      # is older than that tool's registry entry, and a tool the other arms do
      # not use must never be able to fail their install step. Latest release
      # on purpose — the arm tracks the tool as it ships — with the checksum
      # verified and the version recorded in the log.
      - name: Install mbx (comparison arms only)
        if: endsWith(matrix.name, '-mbx') || contains(matrix.cmd, '--mbx')
        timeout-minutes: 10
        run: |
          bash scripts/install-bench-mbx.sh

      - name: Authenticate git for third-party clones
        env:
          GH_JOB_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail
          header="$(printf 'x-access-token:%s' "$GH_JOB_TOKEN" | base64 -w0)"
          git config --global --replace-all \
            "http.https://github.com/.extraheader" "AUTHORIZATION: basic $header"

      # Fail fast on a full disk — disk-full mid-build is the worst failure mode
      # on the emptyDir-backed work volume.
      - name: Free-disk precheck
        run: |
          avail_gb=$(df -BG --output=avail . | awk 'NR==2 {gsub(/G/,"",$1); print $1}')
          echo "Free disk: ${avail_gb} GB (need >= ${{ matrix.min_free_gb }} GB)"
          if [ "${avail_gb:-0}" -lt "${{ matrix.min_free_gb }}" ]; then
            echo "::error::Insufficient free disk (${avail_gb} GB < ${{ matrix.min_free_gb }} GB) for the ${{ matrix.name }} bench"
            exit 1
          fi

      - name: Run bench (${{ matrix.name }})
        timeout-minutes: ${{ matrix.timeout }}
        # `just bench` builds release kache + the scenario engine, then runs cold
        # then warm against one cache and writes reports under
        # tmp/bench/bench-<scenario>/. The engine self-diagnoses: a non-zero exit
        # means the run did not validly exercise kache, so we surface it as a
        # failure rather than uploading a misleading green report.
        run: |
          rustc --version && cargo --version && just --version
          # The short suites (hk, eza, aube) stop five minutes before this
          # step's limit and report what they finished; a killed step
          # reports nothing. Other arms ignore it.
          export BENCH_DEADLINE_AT=$(( $(date +%s) + (${{ matrix.timeout }} - 5) * 60 ))
          # EXTRA_ARGS is passed via env (not interpolated into the command line)
          # so a dispatch input can't inject shell. Unquoted on purpose: the
          # bench flags (e.g. `--skip-clone --retry`) must word-split. matrix.cmd
          # is the full recipe + profile, e.g. "bench firefox" or
          # "bench-sccache firefox".
          # shellcheck disable=SC2086
          just ${{ matrix.cmd }} $EXTRA_ARGS
        env:
          # Empty on schedule; the dispatch input otherwise (`--skip-clone`, …).
          EXTRA_ARGS: ${{ github.event.inputs.extra_args }}
          # bench-short alternates tools between samples but measures cold in
          # the first; rotating its order by night keeps one tool from always
          # building cold first on a cold page cache.
          BENCH_ORDER_SEED: ${{ github.run_number }}
          # The bench measures kache itself; it must not run under a wrapper.
          RUSTC_WRAPPER: ""
          # The subject uses its own pinned Rust toolchain.
          RUSTUP_TOOLCHAIN: ""

      - name: Upload bench reports
        # always(): a failed/leaky run's reports + build/wrapper logs are the most
        # useful artifacts to inspect. NO hashFiles() gate: hashFiles globs the
        # WHOLE scratch tree (multi-GB clone worktrees, .git, the engine's live
        # unix socket) and THROWS on the socket/special files — which errored the
        # upload even on a valid run and discarded build-cold.log. `if-no-files-found:
        # warn` already makes an empty match (e.g. after a precheck failure) a no-op.
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: bench-${{ matrix.name }}
          # Reports, per-phase JSON/MD, Chrome traces, key-diff, ALL top-level
          # logs (build-cold/build-warm/wrapper-* — the build logs hold the
          # actual compile error on a failed run), and sccache's
          # *.sccache-adv.txt stats for the sccache arm. These are flat globs
          # in the scratch dir, so they exclude the multi-GB clone worktrees /
          # objdirs / cache. The two `!` lines drop scratch the contention
          # driver removes on exit: a timed-out step leaves it behind with
          # builds still running in it, and files vanishing mid-walk fail the
          # whole upload.
          path: |
            tmp/bench/${{ matrix.dir }}/*.json
            tmp/bench/${{ matrix.dir }}/*.md
            tmp/bench/${{ matrix.dir }}/report-*.json
            tmp/bench/${{ matrix.dir }}/report-*.md
            tmp/bench/${{ matrix.dir }}/trace-*.json
            tmp/bench/${{ matrix.dir }}/key-diff.*
            tmp/bench/${{ matrix.dir }}/*.log
            tmp/bench/${{ matrix.dir }}/*.txt
            tmp/bench/${{ matrix.dir }}/logs/
            tmp/bench/${{ matrix.dir }}/contention/
            tmp/bench/${{ matrix.dir }}/contention.log
            !tmp/bench/${{ matrix.dir }}/contention/work/**
            !tmp/bench/${{ matrix.dir }}/contention/cargo-home/**
          retention-days: 30
          if-no-files-found: warn

      # OTLP JSON for kartero. Separate name and retention from the diagnostic
      # bundle above: kartero pulls `telemetry-otlp-v1*` and nothing else. 14
      # days, so a collector outage shorter than two weeks loses no night. The suffix is required because upload-artifact v4
      # rejects two jobs in one run sharing an identical artifact name.
      - name: Validate telemetry
        if: always()
        timeout-minutes: 5
        run: npx --yes @kunobi/kartero@0.4.14 validate --input tmp/bench/${{ matrix.dir }}
        env:
          NPM_CONFIG_CACHE: ${{ runner.temp }}/kartero-npm-cache

      - name: Upload telemetry
        if: always()
        timeout-minutes: 10
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: telemetry-otlp-v1-${{ matrix.name }}
          path: |
            tmp/bench/${{ matrix.dir }}/metrics.otlp.json
            tmp/bench/${{ matrix.dir }}/schema_version
            tmp/bench/${{ matrix.dir }}/${{ matrix.dir }}.json
          retention-days: 14
          if-no-files-found: error

      # Daemon/store counters (`kache.cache.*`), not the bench gauges above.
      # Separate artifacts so Kartero imports them as different payloads.
      # Checked like the bench payload: since 0.4.14 the CLI refuses what the
      # collector would refuse, and it has no metric-name allowlist to fail on.
      - name: Validate cache telemetry
        if: always()
        timeout-minutes: 5
        run: |
          for dir in tmp/bench/${{ matrix.dir }}/cache-otlp-*/; do
            [ -f "$dir/metrics.otlp.json" ] || continue
            npx --yes @kunobi/kartero@0.4.14 validate --input "$dir"
          done
        env:
          NPM_CONFIG_CACHE: ${{ runner.temp }}/kartero-npm-cache

      - name: Upload cache telemetry (cold)
        if: always()
        timeout-minutes: 10
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: telemetry-otlp-v1-cache-${{ matrix.name }}-cold
          path: tmp/bench/${{ matrix.dir }}/cache-otlp-cold/
          retention-days: 14
          if-no-files-found: warn

      - name: Upload cache telemetry (warm)
        if: always()
        timeout-minutes: 10
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: telemetry-otlp-v1-cache-${{ matrix.name }}-warm
          path: tmp/bench/${{ matrix.dir }}/cache-otlp-warm/
          retention-days: 14
          if-no-files-found: warn

      # Only the arms that pass `--warm-same-tree` write this directory; the
      # step is skipped elsewhere so a missing phase never reads as a warning.
      - name: Upload cache telemetry (warm-same-tree)
        if: always() && (contains(matrix.cmd, '--warm-same-tree') || startsWith(matrix.cmd, 'bench-short')) && !endsWith(matrix.name, '-mbx')
        timeout-minutes: 10
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: telemetry-otlp-v1-cache-${{ matrix.name }}-warm-same-tree
          path: tmp/bench/${{ matrix.dir }}/cache-otlp-warm-same-tree/
          retention-days: 14
          if-no-files-found: warn

      # Last, so the duration covers the uploads and the conclusion covers every
      # step before. A job killed at its own limit runs no further steps, so a
      # missing point here is itself the signal. The ARC image may lack Python
      # if setup failed early; there is nothing to report then either.
      - name: Write job telemetry
        if: always()
        env:
          JOB_STATUS: ${{ job.status }}
          CACHE_TOOL: ${{ endsWith(matrix.name, '-sccache') && 'sccache' || endsWith(matrix.name, '-mbx') && 'mbx' || 'kache' }}
        run: |
          if command -v python3 >/dev/null && [ -n "${BENCH_JOB_STARTED:-}" ]; then
            python3 scripts/bench/job_telemetry.py \
              --project "${{ matrix.dir }}" --cache-tool "$CACHE_TOOL" \
              --started "$BENCH_JOB_STARTED" --status "$JOB_STATUS" \
              --output "tmp/bench-job/${{ matrix.dir }}"
          fi

      - name: Upload job telemetry
        if: always()
        timeout-minutes: 10
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: telemetry-otlp-v1-job-${{ matrix.name }}
          path: tmp/bench-job/${{ matrix.dir }}/
          retention-days: 14
          if-no-files-found: warn