kache 0.9.0

name: Bench

# Clone benchmark — builds kache from THIS ref and runs the cross-clone
# cold/warm scenarios (Firefox, Substrate, SurrealDB, LLVM) against one shared
# cache, then
# uploads the reports. This is the heavy, hours-long, tens-of-GB-disk job the
# bench engine's doc-comment says is "intentionally NOT wired into [PR] CI": it
# lives in its own workflow so it never blocks a push/PR.
#
# Triggers:
#   - schedule  : nightly, runs BOTH scenarios.
#   - dispatch  : manual, pick a scenario (default all) and pass extra `just
#                 bench` flags (e.g. `--skip-clone`, `--retry`).
#
# Runs on the kunobi self-hosted Linux ARC runner: the `kunobi-runners`
# gha-runner-scale-set in tenant-int-dev/zur1-builder1. A gha-runner-scale-set
# is targeted by its INSTALLATION NAME as a single `runs-on` label
# (`runs-on: kunobi-runners`) — NOT by a `[self-hosted, Linux, X64, …]` label
# set (that scheme only matches the legacy multi-label runners, e.g.
# kunobi-windows). These are EPHEMERAL pods (fresh per job,
# `_work` is an emptyDir): there is no stale tool state to isolate, but there is
# also no persistent clone-ref cache, so every run re-clones the upstream repo,
# and Firefox's `./mach bootstrap` re-provisions ~/.mozbuild each time. The image
# is minimal, so the base build toolchain is installed below.
on:
  schedule:
    # ~04:37 UTC nightly. Off the top-of-hour to dodge GitHub's cron congestion.
    - cron: "37 4 * * *"
  workflow_dispatch:
    inputs:
      scenario:
        description: "Which scenario(s) to run"
        type: choice
        options: [all, substrate, surrealdb, firefox, firefox-sccache, llvm, firefox-windows]
        default: all
      extra_args:
        description: "Extra args appended to `just bench <profile>` (e.g. --skip-clone, --retry)"
        type: string
        default: ""

concurrency:
  # One bench run per scenario+ref at a time; don't cancel a multi-hour run.
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: false

permissions:
  contents: read

env:
  CARGO_TERM_COLOR: always

jobs:
  # Build the bench matrix from the trigger. Selection must happen HERE, not in
  # a job-level `if` on `bench`: the `matrix` context is NOT available in a
  # job-level `if` (it is evaluated before the matrix is expanded, so it reads
  # empty) — a dispatch picking a single scenario would otherwise run zero arms.
  # This tiny job runs on a GitHub-hosted runner so a skipped scenario never
  # claims a self-hosted kunobi runner. schedule + `all` -> both; otherwise the
  # one picked scenario. Per-arm timeout/min_free_gb travel with each entry.
  plan:
    name: Plan scenarios
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.pick.outputs.matrix }}
    steps:
      - id: pick
        shell: bash
        env:
          # schedule has no inputs -> treat as `all`; dispatch honours the input.
          SCENARIO: ${{ github.event_name == 'schedule' && 'all' || github.event.inputs.scenario }}
        run: |
          # Each arm carries: name (display + artifact label), cmd (the `just`
          # recipe + profile to run), dir (scratch subdir under tmp/bench to
          # upload), timeout (minutes), min_free_gb (precheck floor — fail the
          # arm if free disk is below this before we start, since disk-full
          # mid-build is the worst failure mode; emptyDir is node-backed, so the
          # precheck also catches a too-small node).
          #
          # Smaller than Firefox but still tens of minutes cold.
          substrate='{"name":"substrate","cmd":"bench substrate","dir":"bench-substrate","timeout":180,"min_free_gb":60}'
          # SurrealDB's `surreal` server binary at default features — a large
          # pure-Rust workspace whose default features also drag in native C/C++
          # deps (rocksdb, grpcio, quickjs) that build outside kache. Comparable
          # weight to substrate; same native-prereq set (already installed below).
          surrealdb='{"name":"surrealdb","cmd":"bench surrealdb","dir":"bench-surrealdb","timeout":180,"min_free_gb":80}'
          # Full cold Firefox build is the long pole — hours. `bench firefox`
          # resolves to the exact `bench-firefox` scenario: `--profile firefox`
          # is a name SUBSTRING match that also catches `bench-firefox-windows`,
          # but discovery prefers the exact-name hit (#458), so no `os:` tag is
          # needed to pin this host-native arm. (The Windows variant runs in its
          # own job below, on the self-hosted Windows runner.)
          firefox='{"name":"firefox","cmd":"bench firefox","dir":"bench-firefox","timeout":360,"min_free_gb":120}'
          # Same Firefox shape through sccache for a side-by-side comparison.
          firefox_sccache='{"name":"firefox-sccache","cmd":"bench-sccache firefox","dir":"bench-firefox-sccache","timeout":360,"min_free_gb":120}'
          # Almost-pure C/C++ CMake build (X86-only Release) — lighter than Firefox.
          llvm='{"name":"llvm","cmd":"bench llvm","dir":"bench-llvm","timeout":240,"min_free_gb":100}'
          case "$SCENARIO" in
            substrate)       inc="[$substrate]" ;;
            surrealdb)       inc="[$surrealdb]" ;;
            firefox)         inc="[$firefox]" ;;
            firefox-sccache) inc="[$firefox_sccache]" ;;
            llvm)            inc="[$llvm]" ;;
            *)               inc="[$substrate,$surrealdb,$firefox,$firefox_sccache,$llvm]" ;;
          esac
          echo "matrix={\"include\":$inc}" >> "$GITHUB_OUTPUT"

  bench:
    name: Bench (${{ matrix.name }})
    needs: plan
    # gha-runner-scale-set: match by the scale-set's installation name (single
    # label), not a `[self-hosted, …]` set. See header comment.
    runs-on: kunobi-runners
    timeout-minutes: ${{ matrix.timeout }}
    strategy:
      # Arms run concurrently (up to the kunobi-runners scale-set's maxRunners).
      # They previously starved the 120 GB free-disk precheck (103 GB, #447) by
      # co-locating Firefox-class arms on one node — work is a node-backed
      # emptyDir and the scheduler is disk-blind. That is now handled runner-side
      # by a podAntiAffinity on the kunobi-runners scale-set
      # (Zondax/tenant-int-dev) that spreads the concurrent runner pods onto
      # distinct nodes, so each arm gets its own node's disk WITHOUT a
      # `max-parallel` cap here (no need to serialize / slow the nightly).
      # Matrix is generated by `plan` (see above) — selection can't live in a
      # job-level `if` because `matrix` isn't available there.
      fail-fast: false
      matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        timeout-minutes: 10

      # The ARC runner image is minimal. Install the base toolchain the bench
      # needs around the build: substrate links rocksdb/secp256k1 and needs
      # protoc + clang + cmake + openssl headers (SurrealDB's default features
      # need the same set — rocksdb bindgen wants libclang, storage-tikv/grpcio
      # wants protoc); Firefox's `./mach bootstrap`
      # pulls its own clang/rust toolchain but still needs python3 + common
      # archive/file utilities present first; the LLVM scenario drives CMake
      # with the Ninja generator, so it needs ninja-build. git/curl are needed by
      # the clone and by mise. (Firefox also installs further packages itself via
      # bootstrap, which uses passwordless sudo on this image.)
      - name: Install build prerequisites (apt)
        run: |
          export DEBIAN_FRONTEND=noninteractive
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends \
            build-essential pkg-config \
            protobuf-compiler clang libclang-dev cmake ninja-build \
            libssl-dev \
            git curl wget ca-certificates file unzip bzip2 xz-utils \
            python3 python3-dev python3-pip

      # mise installs rust (per rust-toolchain.toml) + just (the bench recipe is
      # `just bench`) + sccache (the firefox-sccache arm wires sccache as the
      # compiler cache; the version is pinned by mise.toml). On the ephemeral
      # Linux pod there's no stale state and no competing system Rust, so
      # mise-action puts the tools on PATH directly — none of the macOS
      # isolation/PATH workarounds from ci.yml are needed.
      - name: Install tools via mise
        uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0
        with:
          # Pin mise CLI — see ci.yml (v2026.6.10 regression).
          version: 2026.6.9
          install_args: --force rust github:casey/just github:mozilla/sccache
          cache: false
          reshim: false

      # Fail fast on a full disk — disk-full mid-build is the worst failure mode
      # on the emptyDir-backed work volume.
      - name: Free-disk precheck
        run: |
          avail_gb=$(df -BG --output=avail . | awk 'NR==2 {gsub(/G/,"",$1); print $1}')
          echo "Free disk: ${avail_gb} GB (need >= ${{ matrix.min_free_gb }} GB)"
          if [ "${avail_gb:-0}" -lt "${{ matrix.min_free_gb }}" ]; then
            echo "::error::Insufficient free disk (${avail_gb} GB < ${{ matrix.min_free_gb }} GB) for the ${{ matrix.name }} bench"
            exit 1
          fi

      - name: Run bench (${{ matrix.name }})
        # `just bench` builds release kache + the scenario engine, then runs cold
        # then warm against one cache and writes reports under
        # tmp/bench/bench-<scenario>/. The engine self-diagnoses: a non-zero exit
        # means the run did not validly exercise kache, so we surface it as a
        # failure rather than uploading a misleading green report.
        run: |
          rustc --version && cargo --version && just --version
          # EXTRA_ARGS is passed via env (not interpolated into the command line)
          # so a dispatch input can't inject shell. Unquoted on purpose: the
          # bench flags (e.g. `--skip-clone --retry`) must word-split. matrix.cmd
          # is the full recipe + profile, e.g. "bench firefox" or
          # "bench-sccache firefox".
          # shellcheck disable=SC2086
          just ${{ matrix.cmd }} $EXTRA_ARGS
        env:
          # Empty on schedule; the dispatch input otherwise (`--skip-clone`, …).
          EXTRA_ARGS: ${{ github.event.inputs.extra_args }}
          # The bench measures kache itself; it must not run under a wrapper.
          RUSTC_WRAPPER: ""
          # Unset the toolchain pin mise exports. The cloned projects pin their
          # OWN Rust via rust-toolchain.toml (polkadot-sdk -> 1.93.0); a forced
          # RUSTUP_TOOLCHAIN overrides that and breaks substrate's wasm-runtime
          # build. With it cleared, rustup honours each tree's rust-toolchain.toml
          # (kache builds with its pin, substrate with 1.93.0).
          RUSTUP_TOOLCHAIN: ""

      - name: Upload bench reports
        # always(): a failed/leaky run's reports + build/wrapper logs are the most
        # useful artifacts to inspect. NO hashFiles() gate: hashFiles globs the
        # WHOLE scratch tree (multi-GB clone worktrees, .git, the engine's live
        # unix socket) and THROWS on the socket/special files — which errored the
        # upload even on a valid run and discarded build-cold.log. `if-no-files-found:
        # warn` already makes an empty match (e.g. after a precheck failure) a no-op.
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: bench-${{ matrix.name }}
          # Reports, per-phase JSON/MD, key-diff, ALL top-level logs
          # (build-cold/build-warm/wrapper-* — the build logs hold the actual
          # compile error on a failed run), and sccache's *.sccache-adv.txt stats
          # for the sccache arm. These are flat globs in the scratch dir, so they
          # exclude the multi-GB clone worktrees / objdirs / cache.
          path: |
            tmp/bench/${{ matrix.dir }}/*.json
            tmp/bench/${{ matrix.dir }}/*.md
            tmp/bench/${{ matrix.dir }}/report-*.json
            tmp/bench/${{ matrix.dir }}/report-*.md
            tmp/bench/${{ matrix.dir }}/key-diff.*
            tmp/bench/${{ matrix.dir }}/*.log
            tmp/bench/${{ matrix.dir }}/*.txt
          retention-days: 30
          if-no-files-found: warn

  # Firefox compile-cache bench on the self-hosted Windows runner (clang-cl +
  # MSVC under MozillaBuild). Separate from the `bench` matrix above: that runs
  # on the ephemeral Linux ARC (`kunobi-runners`); this needs the persistent
  # Windows runner (`kunobi-windows`, same as ci.yml's test-windows), a
  # different prereq set (no apt; MozillaBuild + VS Build Tools), and `bash` via
  # Git/MozillaBuild `sh.exe`.
  #
  # Runs nightly (schedule) like the other bench arms, plus on a dispatch that
  # selects firefox-windows or all. Until the runner is provisioned with
  # MozillaBuild the nightly arm fails at the prereq precheck — the same
  # self-diagnosing "this bench could not validly run" signal the other arms
  # give, not a silent green.
  bench-firefox-windows:
    name: Bench Firefox (Windows)
    if: >-
      github.event_name == 'schedule' ||
      (github.event_name == 'workflow_dispatch' &&
       (github.event.inputs.scenario == 'firefox-windows' || github.event.inputs.scenario == 'all'))
    runs-on: ${{ fromJSON('["self-hosted", "Windows", "X64", "kunobi-windows"]') }}
    timeout-minutes: 360
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        timeout-minutes: 10

      # Fail fast with an explicit checklist if a Firefox build prerequisite is
      # missing. MozillaBuild provides mach's bash/python env; clang-cl (cc/c++),
      # nasm, and MSVC + Windows SDK back the compile. clang/rust are pulled by
      # `./mach bootstrap`, so they are NOT pre-required here. Keep in sync with
      # ans-runners/tasks/windows/setup_packages.yml.
      - name: Verify Firefox/Windows build prerequisites
        run: |
          missing=
          if [ -z "${MOZILLABUILD:-}" ] && [ ! -d "/c/mozilla-build" ]; then
            echo "MISSING: MozillaBuild (set MOZILLABUILD or install to C:\\mozilla-build) — provision it (ans-runners/tasks/windows/setup_packages.yml)"
            missing=1
          fi
          for t in cc c++ nasm; do
            if ! command -v "$t" >/dev/null 2>&1; then
              echo "MISSING on runner PATH: $t — provision it (ans-runners/tasks/windows/setup_packages.yml)"
              missing=1
            fi
          done
          [ -z "$missing" ] || exit 1
          echo "all Firefox/Windows prerequisites present"

      # Disk-full mid-build is the worst failure mode. One ref clone + two
      # worktrees + two objdirs + the cache is large.
      - name: Free-disk precheck
        run: |
          avail_gb=$(df -BG . 2>/dev/null | awk 'NR==2 {gsub(/G/,"",$4); print $4}')
          echo "Free disk: ${avail_gb:-?} GB (need >= 120 GB)"
          if [ "${avail_gb:-0}" -lt 120 ]; then
            echo "::error::Insufficient free disk (${avail_gb} GB < 120 GB) for the Firefox/Windows bench"
            exit 1
          fi

      # Isolate tool state on the persistent runner: shared rustup/mise go stale
      # and wedge with `Access is denied (os error 5)`. Mirrors ci.yml's
      # test-windows so one run can't poison the next.
      - name: Isolate tool state
        run: |
          mkdir -p \
            "$RUNNER_TEMP/rustup" "$RUNNER_TEMP/cargo" \
            "$RUNNER_TEMP/mise-data/bin" "$RUNNER_TEMP/mise-data/shims" \
            "$RUNNER_TEMP/mise-cache" "$RUNNER_TEMP/mise-state"
          safe_runner=$(printf '%s' "$RUNNER_NAME" | tr -c 'A-Za-z0-9_.-' '_')
          rm -rf "$RUNNER_TEMP"/mise-dl-"$safe_runner"-* 2>/dev/null || true
          mise_dl="$RUNNER_TEMP/mise-dl-$safe_runner-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
          mkdir -p "$mise_dl"
          {
            echo "MISE_DL_TMPDIR=$mise_dl"
            echo "RUSTUP_HOME=$RUNNER_TEMP/rustup"
            echo "CARGO_HOME=$RUNNER_TEMP/cargo"
            echo "MISE_DATA_DIR=$RUNNER_TEMP/mise-data"
            echo "MISE_CACHE_DIR=$RUNNER_TEMP/mise-cache"
            echo "MISE_STATE_DIR=$RUNNER_TEMP/mise-state"
          } >> "$GITHUB_ENV"

      - name: Install tools via mise
        uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0
        with:
          # Pin mise CLI — see ci.yml (v2026.6.10 regression).
          version: 2026.6.9
          install_args: --force rust github:casey/just
          cache: false
          reshim: false

      - name: Run bench (firefox-windows)
        # The fuller `firefox-windows` profile is a unique substring, so it
        # selects bench-firefox-windows directly — no `os:` workaround needed
        # (the `bench` recipe already ANDs suite:bench + backend:kache).
        run: |
          rustc --version && cargo --version && just --version
          just bench firefox-windows
        env:
          # The bench measures kache itself; it must not run under a wrapper.
          RUSTC_WRAPPER: ""
          # Let each tree honour its own rust-toolchain.toml (see Linux arm).
          RUSTUP_TOOLCHAIN: ""
          # Append `--no-system-changes` to the scenario's `./mach bootstrap`
          # (see scenarios/bench-firefox-windows/scenario.toml). The runner runs
          # as NT AUTHORITY\NETWORK SERVICE (non-elevated), so mach's own Windows
          # SDK acquisition via `msiexec /a` fails with exit 1601; the runner
          # already has MSVC + the Windows SDK (provisioned by ans-runners), so
          # bootstrap must not try to install them. A local run leaves this unset
          # and gets a full, dependency-installing bootstrap.
          MACH_BOOTSTRAP_FLAGS: "--no-system-changes"

      - name: Upload bench reports
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: bench-firefox-windows
          path: |
            tmp/bench/bench-firefox-windows/*.json
            tmp/bench/bench-firefox-windows/*.md
            tmp/bench/bench-firefox-windows/report-*.json
            tmp/bench/bench-firefox-windows/report-*.md
            tmp/bench/bench-firefox-windows/key-diff.*
            tmp/bench/bench-firefox-windows/*.log
            tmp/bench/bench-firefox-windows/*.txt
          retention-days: 30
          if-no-files-found: warn