kache 0.18.0

name: Bench

# Clone benchmark — builds kache from THIS ref and runs the cross-clone
# cold/warm scenarios (Firefox, Substrate, SurrealDB, Lance, OpenDAL, eza, LLVM)
# against one shared cache, then
# uploads the reports. This is the heavy, hours-long, tens-of-GB-disk job the
# bench engine's doc-comment says is "intentionally NOT wired into [PR] CI": it
# lives in its own workflow so it never blocks a push/PR.
#
# Triggers:
#   - schedule  : nightly, runs BOTH scenarios.
#   - dispatch  : manual, pick a scenario (default all) and pass extra `just
#                 bench` flags (e.g. `--skip-clone`, `--retry`).
#
# Runs on the kunobi self-hosted Linux ARC runners in
# tenant-int-dev/zur1-builder1: `kunobi-runners-large`, one runner on a
# reserved node with a 120Gi ephemeral-storage request and 32Gi of memory, and
# `kunobi-runners`, the six-slot general pool at 60Gi and 16Gi. Each arm picks
# by its disk floor. A gha-runner-scale-set
# is targeted by its INSTALLATION NAME as a single `runs-on` label
# (`runs-on: kunobi-runners`) — NOT by a `[self-hosted, Linux, X64, …]` label
# set (that scheme only matches the legacy multi-label runners, e.g.
# kunobi-windows). These are EPHEMERAL pods (fresh per job,
# `_work` is an emptyDir): there is no stale tool state to isolate, but there is
# also no persistent clone-ref cache, so every run re-clones the upstream repo,
# and Firefox's `./mach bootstrap` re-provisions ~/.mozbuild each time. The image
# is minimal, so the base build toolchain is installed below.
on:
  schedule:
    # 03:37 Zurich time nightly. Keeping :37 avoids top-of-hour cron congestion.
    - cron: "37 3 * * *"
      timezone: "Europe/Zurich"
  workflow_dispatch:
    inputs:
      scenario:
        description: "Which scenario(s) to run"
        type: choice
        options: [all, substrate, surrealdb, lance, opendal, eza, firefox, firefox-sccache, llvm, firefox-windows, firefox-pull, firefox-pull-windows, hk, hk-pull, hk-mbx, opendal-mbx, lance-mbx, substrate-mbx, surrealdb-mbx]
        default: all
      extra_args:
        description: "Extra args appended to `just bench <profile>` (e.g. --skip-clone, --retry)"
        type: string
        default: ""

concurrency:
  # One bench run per scenario+ref at a time; don't cancel a multi-hour run.
  #
  # The scenario is part of the key because that is what the sentence above
  # says, and for a while it was not: the key was workflow+ref alone, so every
  # scenario on a ref shared ONE slot. An 8-minute `hk` dispatch queued behind
  # an 8-hour `firefox-pull`, which is the exact thing this block exists to
  # prevent. Splitting the runner pool does not help - three runners are no use
  # to a job Actions will not even assign.
  #
  # A scheduled run has no inputs and keys on 'all', so the nightly still
  # serialises against another full sweep, which is what we want.
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs.scenario || 'all' }}
  cancel-in-progress: false

permissions:
  contents: read

env:
  CARGO_TERM_COLOR: always

jobs:
  # Build the bench matrix from the trigger. Selection must happen HERE, not in
  # a job-level `if` on `bench`: the `matrix` context is NOT available in a
  # job-level `if` (it is evaluated before the matrix is expanded, so it reads
  # empty) — a dispatch picking a single scenario would otherwise run zero arms.
  # This tiny job uses the validation runner so a skipped scenario never
  # claims a benchmark runner. schedule + `all` -> both; otherwise the
  # one picked scenario. Per-arm timeout/min_free_gb travel with each entry.
  plan:
    if: (github.repository == 'kunobi-ninja/kache' || vars.ENABLE_BENCHMARKS == 'true')
    name: Plan scenarios
    runs-on: ${{ fromJSON(!github.event.repository.private && '"ubuntu-latest"' || vars.CI_RUNNER_LINUX) }}
    outputs:
      matrix: ${{ steps.pick.outputs.matrix }}
      # Whether the Linux matrix has any arm. Windows-only scenarios produce an
      # EMPTY matrix, which GitHub treats as an invalid job and fails the whole
      # run — so the `bench` job is gated on this to skip cleanly instead.
      has_linux: ${{ steps.pick.outputs.has_linux }}
    steps:
      - id: pick
        shell: bash
        env:
          # schedule has no inputs -> treat as `all`; dispatch honours the input.
          SCENARIO: ${{ github.event_name == 'schedule' && 'all' || github.event.inputs.scenario }}
        run: |
          # Each arm carries: name (display + artifact label), cmd (the `just`
          # recipe + profile to run), dir (scratch subdir under tmp/bench to
          # upload), timeout (benchmark-step minutes), job_timeout (adds time
          # for setup and the final telemetry upload), min_free_gb (precheck
          # floor — fail the arm if free disk is below this before we start,
          # since disk-full mid-build is the worst failure mode; emptyDir is
          # node-backed, so the precheck also catches a too-small node).
          #
          # Smaller than Firefox but still tens of minutes cold.
          substrate='{"name":"substrate","cmd":"bench substrate","dir":"bench-substrate","timeout":180,"job_timeout":225,"min_free_gb":60,"runner":"kunobi-runners-large"}'
          # SurrealDB's `surreal` server binary at default features — a large
          # pure-Rust workspace whose default features also drag in native C/C++
          # deps (rocksdb, grpcio, quickjs) that build outside kache. Comparable
          # weight to substrate; same native-prereq set (already installed below).
          surrealdb='{"name":"surrealdb","cmd":"bench surrealdb","dir":"bench-surrealdb","timeout":180,"job_timeout":225,"min_free_gb":80,"runner":"kunobi-runners-large"}'
          # Lance's `lance` crate at default features — a large pure-Rust
          # workspace (Arrow, DataFusion, object-store backends, prost codegen).
          # Needs protoc + openssl headers (already installed below). Lighter
          # than surrealdb: no rocksdb/grpcio/quickjs native compile.
          lance='{"name":"lance","cmd":"bench lance","dir":"bench-lance","timeout":180,"job_timeout":225,"min_free_gb":60,"runner":"kunobi-runners-large"}'
          # OpenDAL core (apache/opendal `core/` workspace) with the portable
          # service-feature set — mid-size pure-Rust, lighter than surrealdb.
          # aws-lc-sys (rustls) compiles outside kache; cmake + a C compiler
          # are already in the apt step below.
          opendal='{"name":"opendal","cmd":"bench opendal","dir":"bench-opendal","timeout":120,"job_timeout":165,"min_free_gb":40,"runner":"kunobi-runners"}'
          # Full cold Firefox build is the long pole — hours. `bench firefox`
          # resolves to the exact `bench-firefox` scenario: `--profile firefox`
          # is a name SUBSTRING match that also catches `bench-firefox-windows`,
          # but discovery prefers the exact-name hit (#458), so no `os:` tag is
          # needed to pin this host-native arm. (The Windows variant runs in its
          # own job below, on the self-hosted Windows runner.)
          firefox='{"name":"firefox","cmd":"bench firefox","dir":"bench-firefox","timeout":360,"job_timeout":405,"min_free_gb":120,"runner":"kunobi-runners-large"}'
          # Same Firefox shape through sccache for a side-by-side comparison.
          firefox_sccache='{"name":"firefox-sccache","cmd":"bench-sccache firefox","dir":"bench-firefox-sccache","timeout":360,"job_timeout":405,"min_free_gb":120,"runner":"kunobi-runners-large"}'
          # Almost-pure C/C++ CMake build (X86-only Release) — lighter than Firefox.
          llvm='{"name":"llvm","cmd":"bench llvm","dir":"bench-llvm","timeout":240,"job_timeout":285,"min_free_gb":100,"runner":"kunobi-runners-large"}'
          # Daily-pull (TEMPORAL axis, #477): one clone, build ref A then rebuild
          # ref B in the SAME worktree — two full sequential Firefox builds, so a
          # longer timeout than `firefox`. Only ONE worktree (no cross-clone
          # clone-b), so disk is comparable, not double; keep the 120 GB floor.
          # `bench firefox-pull` resolves to the exact `bench-firefox-pull`
          # scenario (substring also catches `bench-firefox-pull-windows`, but
          # discovery prefers the exact-name hit). The Windows variant runs in its
          # own job below, on the self-hosted Windows runner.
          firefox_pull='{"name":"firefox-pull","cmd":"bench firefox-pull","dir":"bench-firefox-pull","timeout":480,"job_timeout":525,"min_free_gb":120,"runner":"kunobi-runners-large"}'
          # hk (jdx/hk): a mid-size Rust CLI whose build scripts compile ~750
          # C/asm objects through cc-rs (aws-lc-sys, vendored libgit2), wired
          # the way `kache init` does it so both compiler families are measured.
          # `--warm-same-tree` adds the same-path warm phase between cold and the
          # cross-clone warm. Minutes, not hours; a C compiler is all it needs.
          hk='{"name":"hk","cmd":"bench hk --warm-same-tree","dir":"bench-hk","timeout":60,"job_timeout":90,"min_free_gb":30,"runner":"kunobi-runners"}'
          # Same subject on the temporal axis: cold at the parent commit, then
          # rebuild the child in the same worktree with the store warm.
          hk_pull='{"name":"hk-pull","cmd":"bench hk-pull","dir":"bench-hk-pull","timeout":60,"job_timeout":90,"min_free_gb":30,"runner":"kunobi-runners"}'
          # The same hk shape through mbx, for a side-by-side comparison
          # (as firefox-sccache is for firefox). mbx comes from mise at its
          # latest release; the run records the version it measured.
          hk_mbx='{"name":"hk-mbx","cmd":"bench-mbx hk","dir":"bench-hk-mbx","timeout":60,"job_timeout":90,"min_free_gb":30,"runner":"kunobi-runners"}'
          # eza: the smallest subject and the only one built through a fat-LTO
          # release profile, so the final unit is one whole-program codegen
          # instead of many small ones. Its bundled libgit2 and zlib put the
          # majority of its cacheable units on the C side (199 of 319 in a local
          # run). The cheapest arm in the sweep: minutes, and under a gigabyte
          # of build output.
          eza='{"name":"eza","cmd":"bench eza --warm-same-tree","dir":"bench-eza","timeout":60,"job_timeout":90,"min_free_gb":20,"runner":"kunobi-runners"}'
          # Two more subjects on the same side-by-side axis. mbx fronts Cargo,
          # so the arms it can serve at all are exactly the cargo-driven ones:
          # firefox (mach/gn/ninja) and llvm (cmake) are out of reach, which is
          # why firefox's comparison arm is sccache instead. Both sit on the
          # general pool — the reserved runner takes one job at a time and is
          # already carrying the multi-hour arms.
          opendal_mbx='{"name":"opendal-mbx","cmd":"bench-mbx opendal","dir":"bench-opendal-mbx","timeout":120,"job_timeout":165,"min_free_gb":40,"runner":"kunobi-runners"}'
          lance_mbx='{"name":"lance-mbx","cmd":"bench-mbx lance","dir":"bench-lance-mbx","timeout":180,"job_timeout":225,"min_free_gb":60,"runner":"kunobi-runners"}'
          # The two heavyweights complete the set: these five are every subject
          # mbx can serve at all. They keep their kache arms' disk floors, and
          # 80Gi is exactly the general pool's per-runner ceiling with no
          # headroom, so they run on the reserved runner as their kache
          # counterparts do.
          substrate_mbx='{"name":"substrate-mbx","cmd":"bench-mbx substrate","dir":"bench-substrate-mbx","timeout":180,"job_timeout":225,"min_free_gb":60,"runner":"kunobi-runners-large"}'
          surrealdb_mbx='{"name":"surrealdb-mbx","cmd":"bench-mbx surrealdb","dir":"bench-surrealdb-mbx","timeout":180,"job_timeout":225,"min_free_gb":80,"runner":"kunobi-runners-large"}'
          case "$SCENARIO" in
            substrate)            inc="[$substrate]" ;;
            surrealdb)            inc="[$surrealdb]" ;;
            lance)                inc="[$lance]" ;;
            opendal)              inc="[$opendal]" ;;
            firefox)              inc="[$firefox]" ;;
            firefox-sccache)      inc="[$firefox_sccache]" ;;
            llvm)                 inc="[$llvm]" ;;
            firefox-pull)         inc="[$firefox_pull]" ;;
            hk)                   inc="[$hk]" ;;
            hk-pull)              inc="[$hk_pull]" ;;
            hk-mbx)               inc="[$hk_mbx]" ;;
            eza)                  inc="[$eza]" ;;
            opendal-mbx)          inc="[$opendal_mbx]" ;;
            lance-mbx)            inc="[$lance_mbx]" ;;
            substrate-mbx)        inc="[$substrate_mbx]" ;;
            surrealdb-mbx)        inc="[$surrealdb_mbx]" ;;
            # Windows-only scenarios are served by their own self-hosted jobs
            # below, so the Linux matrix is EMPTY for them (an empty include
            # skips the `bench` job cleanly). Without this they'd fall to `*)`
            # and wastefully run the whole Linux `all` matrix alongside.
            firefox-windows)      inc="[]" ;;
            firefox-pull-windows) inc="[]" ;;
            *)                    inc="[$substrate,$surrealdb,$lance,$opendal,$firefox,$firefox_sccache,$llvm,$firefox_pull,$hk,$hk_pull,$hk_mbx,$eza,$opendal_mbx,$lance_mbx,$substrate_mbx,$surrealdb_mbx]" ;;
          esac
          echo "matrix={\"include\":$inc}" >> "$GITHUB_OUTPUT"
          if [ "$inc" = "[]" ]; then
            echo "has_linux=false" >> "$GITHUB_OUTPUT"
          else
            echo "has_linux=true" >> "$GITHUB_OUTPUT"
          fi

  bench:
    name: Bench (${{ matrix.name }})
    needs: plan
    # Skip cleanly when the Linux matrix is empty (a windows-only dispatch) — an
    # empty matrix would otherwise fail the run.
    if: needs.plan.outputs.has_linux == 'true'
    # gha-runner-scale-set: match by the scale-set's installation name (single
    # label), not a `[self-hosted, …]` set. See header comment.
    #
    # Which pool is per arm, from the disk floor beside it in the matrix: an
    # arm that fits the general pool's 80Gi ceiling runs there, several at a
    # time, and the heavy ones keep the reserved node and its 120 GB floor.
    #
    # Read the ceiling and the floor together. The ceiling is a pod limit on a
    # node-backed emptyDir, and crossing it evicts the runner mid-build. The
    # floor is a free-space precheck against the node filesystem, which is
    # several times larger, so it passes long before the ceiling is reached
    # and cannot catch that case on its own.
    runs-on: >-
      ${{ fromJSON(
        matrix.runner == 'kunobi-runners-large' &&
        (vars.BENCH_RUNNER_LINUX_LARGE ||
         (github.repository == 'kunobi-ninja/kache' && '"kunobi-runners-large"') || 'null') ||
        vars.BENCH_RUNNER_LINUX ||
        (github.repository == 'kunobi-ninja/kache' && '"kunobi-runners"')
      ) }}
    timeout-minutes: ${{ matrix.job_timeout }}
    strategy:
      # Arms run concurrently (up to the kunobi-runners scale-set's maxRunners).
      # They previously starved the 120 GB free-disk precheck (103 GB, #447) by
      # co-locating Firefox-class arms on one node — work is a node-backed
      # emptyDir and the scheduler is disk-blind. Two things handle that
      # runner-side (Zondax/tenant-int-dev), so neither needs a `max-parallel`
      # cap here: the general pool carries a podAntiAffinity that spreads its
      # concurrent runners onto distinct nodes, giving each light arm its own
      # node's disk, and the heavy arms sit on a reserved runner that takes one
      # job at a time.
      # Matrix is generated by `plan` (see above) — selection can't live in a
      # job-level `if` because `matrix` isn't available there.
      fail-fast: false
      matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        timeout-minutes: 10

      - name: Set up Node for Kartero
        uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0
        with:
          node-version: 24

      # Seed a valid failure artifact before prerequisites or benchmark work.
      # A completed benchmark overwrites it with the full metric set. Any early
      # exit leaves verdict.ok=0 for the final always-running upload.
      - name: Seed failure telemetry
        run: |
          mkdir -p tmp/bench
          npx --yes @kunobi/kartero@0.3.0 gauge \
            --name kache.bench.verdict.ok \
            --value 0 \
            --unit 1 \
            --attribute kache.bench.project=${{ matrix.dir }} \
            --attribute kache.bench.cache_tool=${{ endsWith(matrix.name, '-sccache') && 'sccache' || endsWith(matrix.name, '-mbx') && 'mbx' || 'kache' }} \
            --output tmp/bench/${{ matrix.dir }}
        env:
          NPM_CONFIG_CACHE: ${{ runner.temp }}/kartero-npm-cache

      # The ARC runner image is minimal. Install the base toolchain the bench
      # needs around the build: substrate links rocksdb/secp256k1 and needs
      # protoc + clang + cmake + openssl headers (SurrealDB's default features
      # need the same set — rocksdb bindgen wants libclang, storage-tikv/grpcio
      # wants protoc; Lance's prost-build wants protoc and a few default-feature
      # backends want openssl headers; OpenDAL's rustls/aws-lc-sys wants cmake +
      # a C compiler); Firefox's `./mach bootstrap`
      # pulls its own clang/rust toolchain but still needs python3 + common
      # archive/file utilities present first; the LLVM scenario drives CMake
      # with the Ninja generator, so it needs ninja-build. git/curl are needed by
      # the clone and by mise. (Firefox also installs further packages itself via
      # bootstrap, which uses passwordless sudo on this image.)
      - name: Install build prerequisites (apt)
        run: |
          export DEBIAN_FRONTEND=noninteractive
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends \
            build-essential pkg-config \
            protobuf-compiler clang libclang-dev cmake ninja-build \
            libssl-dev \
            git curl wget ca-certificates file unzip bzip2 xz-utils \
            python3 python3-dev python3-pip

      # mise installs rust (per rust-toolchain.toml) + just (the bench recipe is
      # `just bench`) + sccache (the firefox-sccache arm wires sccache as the
      # compiler cache; the version is pinned by mise.toml). On the ephemeral
      # Linux pod there's no stale state and no competing system Rust, so
      # mise-action puts the tools on PATH directly — none of the macOS
      # isolation/PATH workarounds from ci.yml are needed.
      - name: Install tools via mise
        uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0
        with:
          # Pin mise CLI — see ci.yml (v2026.6.10 regression).
          version: 2026.6.9
          install_args: --force rust github:casey/just github:mozilla/sccache
          cache: false
          reshim: false

      # The comparison tool for the `-mbx` arms, installed from its GitHub
      # release rather than through the shared mise step: the pinned mise CLI
      # is older than that tool's registry entry, and a tool the other arms do
      # not use must never be able to fail their install step. Latest release
      # on purpose — the arm tracks the tool as it ships — with the checksum
      # verified and the version recorded in the log.
      - name: Install mbx (comparison arms only)
        if: endsWith(matrix.name, '-mbx')
        timeout-minutes: 10
        run: |
          set -euo pipefail
          # curl, not `gh`: the ARC runner image ships no GitHub CLI.
          # /releases/latest redirects to the tag, so the newest release is
          # one HEAD request and needs no token or JSON parsing.
          repo=https://github.com/jdx/mr-boxington
          archive=mbx-x86_64-unknown-linux-gnu.tar.gz
          dir="$RUNNER_TEMP/mbx"
          mkdir -p "$dir"
          tag="$(curl -fsSLI -o /dev/null -w '%{url_effective}' "$repo/releases/latest" | sed 's#.*/tag/##')"
          case "$tag" in
            v[0-9]*) ;;
            *) echo "::error::could not resolve the latest release tag (got '$tag')"; exit 1 ;;
          esac
          curl -fsSL -o "$dir/$archive" "$repo/releases/download/$tag/$archive"
          curl -fsSL -o "$dir/SHA256SUMS" "$repo/releases/download/$tag/SHA256SUMS"
          (cd "$dir" && grep "  $archive$" SHA256SUMS | sha256sum --check --strict -)
          tar -xzf "$dir/$archive" -C "$dir"
          echo "$dir" >> "$GITHUB_PATH"
          echo "mbx release $tag"
          "$dir/mbx" --version

      # The harness clones third-party subjects over https. Anonymous
      # requests share this cluster's egress quota, and when it is exhausted
      # GitHub answers 401, which git reports as "could not read Username"
      # and the bench dies before it builds anything. Authenticating with the
      # job's own read-only token moves those fetches onto the much larger
      # per-token limit. Ephemeral pod, so the config dies with the job.
      - name: Authenticate git for third-party clones
        env:
          GH_JOB_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail
          header="$(printf 'x-access-token:%s' "$GH_JOB_TOKEN" | base64 -w0)"
          git config --global --replace-all \
            "http.https://github.com/.extraheader" "AUTHORIZATION: basic $header"

      # Fail fast on a full disk — disk-full mid-build is the worst failure mode
      # on the emptyDir-backed work volume.
      - name: Free-disk precheck
        run: |
          avail_gb=$(df -BG --output=avail . | awk 'NR==2 {gsub(/G/,"",$1); print $1}')
          echo "Free disk: ${avail_gb} GB (need >= ${{ matrix.min_free_gb }} GB)"
          if [ "${avail_gb:-0}" -lt "${{ matrix.min_free_gb }}" ]; then
            echo "::error::Insufficient free disk (${avail_gb} GB < ${{ matrix.min_free_gb }} GB) for the ${{ matrix.name }} bench"
            exit 1
          fi

      - name: Run bench (${{ matrix.name }})
        timeout-minutes: ${{ matrix.timeout }}
        # `just bench` builds release kache + the scenario engine, then runs cold
        # then warm against one cache and writes reports under
        # tmp/bench/bench-<scenario>/. The engine self-diagnoses: a non-zero exit
        # means the run did not validly exercise kache, so we surface it as a
        # failure rather than uploading a misleading green report.
        run: |
          rustc --version && cargo --version && just --version
          # EXTRA_ARGS is passed via env (not interpolated into the command line)
          # so a dispatch input can't inject shell. Unquoted on purpose: the
          # bench flags (e.g. `--skip-clone --retry`) must word-split. matrix.cmd
          # is the full recipe + profile, e.g. "bench firefox" or
          # "bench-sccache firefox".
          # shellcheck disable=SC2086
          just ${{ matrix.cmd }} $EXTRA_ARGS
        env:
          # Empty on schedule; the dispatch input otherwise (`--skip-clone`, …).
          EXTRA_ARGS: ${{ github.event.inputs.extra_args }}
          # The bench measures kache itself; it must not run under a wrapper.
          RUSTC_WRAPPER: ""
          # Unset the toolchain pin mise exports. The cloned projects pin their
          # OWN Rust via rust-toolchain.toml (polkadot-sdk -> 1.93.0); a forced
          # RUSTUP_TOOLCHAIN overrides that and breaks substrate's wasm-runtime
          # build. With it cleared, rustup honours each tree's rust-toolchain.toml
          # (kache builds with its pin, substrate with 1.93.0).
          RUSTUP_TOOLCHAIN: ""

      - name: Upload bench reports
        # always(): a failed/leaky run's reports + build/wrapper logs are the most
        # useful artifacts to inspect. NO hashFiles() gate: hashFiles globs the
        # WHOLE scratch tree (multi-GB clone worktrees, .git, the engine's live
        # unix socket) and THROWS on the socket/special files — which errored the
        # upload even on a valid run and discarded build-cold.log. `if-no-files-found:
        # warn` already makes an empty match (e.g. after a precheck failure) a no-op.
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: bench-${{ matrix.name }}
          # Reports, per-phase JSON/MD, key-diff, ALL top-level logs
          # (build-cold/build-warm/wrapper-* — the build logs hold the actual
          # compile error on a failed run), and sccache's *.sccache-adv.txt stats
          # for the sccache arm. These are flat globs in the scratch dir, so they
          # exclude the multi-GB clone worktrees / objdirs / cache.
          path: |
            tmp/bench/${{ matrix.dir }}/*.json
            tmp/bench/${{ matrix.dir }}/*.md
            tmp/bench/${{ matrix.dir }}/report-*.json
            tmp/bench/${{ matrix.dir }}/report-*.md
            tmp/bench/${{ matrix.dir }}/key-diff.*
            tmp/bench/${{ matrix.dir }}/*.log
            tmp/bench/${{ matrix.dir }}/*.txt
          retention-days: 30
          if-no-files-found: warn

      # Short-lived OTLP JSON for kartero. Separate name and retention from
      # the diagnostic bundle above: kartero pulls `telemetry-otlp-v1*` and
      # nothing else. The suffix is required because upload-artifact v4
      # rejects two jobs in one run sharing an identical artifact name.
      - name: Validate telemetry
        if: always()
        timeout-minutes: 5
        run: npx --yes @kunobi/kartero@0.3.0 validate --input tmp/bench/${{ matrix.dir }}
        env:
          NPM_CONFIG_CACHE: ${{ runner.temp }}/kartero-npm-cache

      - name: Upload telemetry
        if: always()
        timeout-minutes: 10
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: telemetry-otlp-v1-${{ matrix.name }}
          path: |
            tmp/bench/${{ matrix.dir }}/metrics.otlp.json
            tmp/bench/${{ matrix.dir }}/schema_version
            tmp/bench/${{ matrix.dir }}/${{ matrix.dir }}.json
          retention-days: 3
          if-no-files-found: error

      # Daemon/store counters (`kache.cache.*`), not the bench gauges above.
      # Separate artifacts so Kartero imports them as different payloads.
      # Do not validate with @kunobi/kartero@0.3.0: that CLI's published
      # allowlist does not include kache.cache.* yet.
      - name: Upload cache telemetry (cold)
        if: always()
        timeout-minutes: 10
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: telemetry-otlp-v1-cache-${{ matrix.name }}-cold
          path: tmp/bench/${{ matrix.dir }}/cache-otlp-cold/
          retention-days: 3
          if-no-files-found: warn

      - name: Upload cache telemetry (warm)
        if: always()
        timeout-minutes: 10
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: telemetry-otlp-v1-cache-${{ matrix.name }}-warm
          path: tmp/bench/${{ matrix.dir }}/cache-otlp-warm/
          retention-days: 3
          if-no-files-found: warn

      # Only the arms that pass `--warm-same-tree` write this directory; the
      # step is skipped elsewhere so a missing phase never reads as a warning.
      - name: Upload cache telemetry (warm-same-tree)
        if: always() && contains(matrix.cmd, '--warm-same-tree')
        timeout-minutes: 10
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: telemetry-otlp-v1-cache-${{ matrix.name }}-warm-same-tree
          path: tmp/bench/${{ matrix.dir }}/cache-otlp-warm-same-tree/
          retention-days: 3
          if-no-files-found: warn

  # Firefox compile-cache bench on the self-hosted Windows runner (clang-cl +
  # MSVC under MozillaBuild). Separate from the `bench` matrix above: that runs
  # on the ephemeral Linux ARC pools; this needs the persistent
  # Windows runner (`kunobi-windows`), a different prereq set (no apt;
  # MozillaBuild + VS Build Tools), and `bash` via
  # Git/MozillaBuild `sh.exe`.
  #
  # Runs nightly (schedule) like the other bench arms, plus on a dispatch that
  # selects firefox-windows or all. Until the runner is provisioned with
  # MozillaBuild the nightly arm fails at the prereq precheck — the same
  # self-diagnosing "this bench could not validly run" signal the other arms
  # give, not a silent green.
  bench-firefox-windows:
    name: Bench Firefox (Windows)
    if: >-
      (github.repository == 'kunobi-ninja/kache' || vars.ENABLE_BENCHMARKS == 'true') &&
      (github.event_name == 'schedule' ||
       (github.event_name == 'workflow_dispatch' &&
        (github.event.inputs.scenario == 'firefox-windows' || github.event.inputs.scenario == 'all')))
    runs-on: ${{ fromJSON(vars.BENCH_RUNNER_WINDOWS || (github.repository == 'kunobi-ninja/kache' && '["self-hosted","Windows","X64","kunobi-windows"]')) }}
    timeout-minutes: 420
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        timeout-minutes: 10

      - name: Set up Node for Kartero
        uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0
        with:
          node-version: 24

      - name: Seed failure telemetry
        run: |
          output="$(printf '%s' "$RUNNER_TEMP" | tr '\\' '/')/f"
          npx --yes @kunobi/kartero@0.3.0 gauge \
            --name kache.bench.verdict.ok \
            --value 0 \
            --unit 1 \
            --attribute kache.bench.project=bench-firefox-windows \
            --attribute kache.bench.cache_tool=kache \
            --output "$output"
        env:
          NPM_CONFIG_CACHE: ${{ runner.temp }}/kartero-npm-cache

      # Fail fast with an explicit checklist if a Firefox build prerequisite is
      # missing. MozillaBuild provides mach's bash/python env; clang-cl, nasm,
      # and MSVC + Windows SDK back the compile. Mirrors the pull job (#728):
      # check clang-cl by absolute path — it is NOT on the default PATH, and
      # `cc`/`c++` are MinGW gcc/g++ on this runner, the wrong compiler for
      # Firefox, so their presence proves nothing. Keep in sync with
      # ans-runners/tasks/windows/setup_packages.yml.
      - name: Verify Firefox/Windows build prerequisites
        run: |
          missing=
          if [ -z "${MOZILLABUILD:-}" ] && [ ! -d "/c/mozilla-build" ]; then
            echo "MISSING: MozillaBuild (set MOZILLABUILD or install to C:\\mozilla-build) — provision it (ans-runners/tasks/windows/setup_packages.yml)"
            missing=1
          fi
          if [ ! -f "$CLANG_CL_BIN/clang-cl.exe" ]; then
            echo "MISSING: clang-cl at $CLANG_CL_BIN — provision it (ans-runners/tasks/windows/setup_packages.yml)"
            missing=1
          fi
          if ! command -v nasm >/dev/null 2>&1; then
            echo "MISSING on runner PATH: nasm — provision it (ans-runners/tasks/windows/setup_packages.yml)"
            missing=1
          fi
          [ -z "$missing" ] || exit 1
          echo "all Firefox/Windows prerequisites present"
        env:
          # VS-integrated clang-cl dir (ans-runners installs the VC.Llvm.Clang
          # component into VS Build Tools). Kept in sync with the run step.
          CLANG_CL_BIN: "/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/Llvm/x64/bin"

      # Disk-full mid-build is the worst failure mode. One ref clone + two
      # worktrees + two objdirs + the cache is large.
      - name: Free-disk precheck
        run: |
          avail_gb=$(df -BG . 2>/dev/null | awk 'NR==2 {gsub(/G/,"",$4); print $4}')
          echo "Free disk: ${avail_gb:-?} GB (need >= 120 GB)"
          if [ "${avail_gb:-0}" -lt 120 ]; then
            echo "::error::Insufficient free disk (${avail_gb} GB < 120 GB) for the Firefox/Windows bench"
            exit 1
          fi

      # Isolate tool state on the persistent runner: shared rustup/mise go stale
      # and wedge with `Access is denied (os error 5)`. Mirrors ci.yml's
      # test-windows so one run can't poison the next.
      - name: Isolate tool state
        run: |
          mkdir -p \
            "$RUNNER_TEMP/rustup" "$RUNNER_TEMP/cargo" \
            "$RUNNER_TEMP/mise-data/bin" "$RUNNER_TEMP/mise-data/shims" \
            "$RUNNER_TEMP/mise-cache" "$RUNNER_TEMP/mise-state"
          safe_runner=$(printf '%s' "$RUNNER_NAME" | tr -c 'A-Za-z0-9_.-' '_')
          rm -rf "$RUNNER_TEMP"/mise-dl-"$safe_runner"-* 2>/dev/null || true
          mise_dl="$RUNNER_TEMP/mise-dl-$safe_runner-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
          mkdir -p "$mise_dl"
          {
            echo "MISE_DL_TMPDIR=$mise_dl"
            echo "RUSTUP_HOME=$RUNNER_TEMP/rustup"
            echo "CARGO_HOME=$RUNNER_TEMP/cargo"
            echo "MISE_DATA_DIR=$RUNNER_TEMP/mise-data"
            echo "MISE_CACHE_DIR=$RUNNER_TEMP/mise-cache"
            echo "MISE_STATE_DIR=$RUNNER_TEMP/mise-state"
          } >> "$GITHUB_ENV"

      - name: Install tools via mise
        uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0
        with:
          # Pin mise CLI — see ci.yml (v2026.6.10 regression).
          version: 2026.6.9
          # node@lts: Firefox's build needs Node.js (bundling / some codegen). It
          # is normally provided by `mach bootstrap`, which the no-update
          # bootstrap skips, so install it here (user-level, no elevation).
          install_args: --force rust github:casey/just node@lts
          cache: false
          reshim: false

      - name: Run bench (firefox-windows)
        timeout-minutes: 360
        # The fuller `firefox-windows` profile is a unique substring, so it
        # selects bench-firefox-windows directly — no `os:` workaround needed
        # (the `bench` recipe already ANDs suite:bench + backend:kache).
        #
        # SHORT work dir: Firefox's `./mach build` refuses a source directory
        # path longer than 62 chars on Windows. The default bench work dir
        # lives under the runner checkout (`…\_work\kache\kache\tmp\bench\
        # <scenario>\clone-a` — 84 chars) and trips that guard (#728). Put the
        # clone under `$RUNNER_TEMP\f` instead (service-writable, already used
        # by the isolate-tool-state step; `f` keeps it distinct from the pull
        # job's `b` when inspecting a runner) so the source dir is ~45 chars.
        # Reports land under that same dir — see the upload paths below.
        # $RUNNER_TEMP is a Windows path (C:\…\_temp); convert to forward
        # slashes so the value survives the just/sh layers (see the pull job).
        run: |
          rustc --version && cargo --version && just --version
          # Firefox needs clang-cl, which ans-runners installs into VS Build Tools
          # but is NOT on the default PATH. Prepend its dir so mach discovers it
          # (mach finds the MSVC toolchain + Windows SDK itself via vswhere, so no
          # vcvars is needed). The scenario's `--enable-bootstrap=no-update`
          # mozconfig stops mach from trying to fetch its OWN clang/VS (the fetch
          # needs an elevated msiexec the NETWORK SERVICE account can't run —
          # exit 1601). Mirrors the pull job (#728).
          export PATH="$CLANG_CL_BIN:$PATH"
          clang-cl --version | head -1 || { echo "clang-cl not on PATH"; exit 1; }
          # cbindgen (Rust->C header generator): another user-level Firefox build
          # tool `mach bootstrap` would install and the no-update bootstrap skips.
          # Install it into the isolated CARGO_HOME (already on PATH) if absent.
          command -v cbindgen >/dev/null 2>&1 || cargo install cbindgen --locked
          cbindgen --version || { echo "cbindgen install failed"; exit 1; }
          # node is installed by mise (node@lts), but mach invokes `node` as a
          # deep subprocess where mise isn't activated, so the mise SHIM fails
          # ("cannot find binary path"). Worse, node isn't pinned as the ACTIVE
          # version, so `mise which node` returns nothing either. Pin it global,
          # then resolve the REAL binary; fall back to globbing the install dir.
          mise use -g node@lts >/dev/null 2>&1 || true
          node_real="$(mise which node 2>/dev/null || true)"
          if [ -z "$node_real" ]; then
            mdd="$(cygpath -u "$MISE_DATA_DIR" 2>/dev/null || printf '%s' "$MISE_DATA_DIR")"
            node_real="$(find "$mdd/installs/node" -name node.exe 2>/dev/null | head -1)"
          fi
          [ -n "$node_real" ] || { echo "could not resolve node binary"; exit 1; }
          # Prepend the real node dir to PATH and set NODEJS (mach's documented
          # override) so mach uses the binary directly, not the failing shim.
          export PATH="$(dirname "$node_real"):$PATH"
          export NODEJS="$(cygpath -w "$node_real" 2>/dev/null || printf '%s' "$node_real")"
          node --version || { echo "node not resolvable"; exit 1; }
          echo "[bench] node: $node_real"
          work="$(printf '%s' "$RUNNER_TEMP" | tr '\\' '/')/f"
          mkdir -p "$work"
          echo "[bench] work dir: $work"
          just bench firefox-windows --work-dir "$work"
        env:
          # Shared mach toolchain pre-provisioned (elevated) by ans-runners
          # (tag: firefox-toolchain). Pointing MOZBUILD_STATE_PATH here makes the
          # no-update-bootstrap build consume the fetched VS / Win SDK / Windows
          # App SDK / clang / cbindgen / node instead of trying (and failing) to
          # acquire them as the non-elevated runner service.
          MOZBUILD_STATE_PATH: 'C:\mozbuild-shared'
          # VS-integrated clang-cl dir (ans-runners installs the VC.Llvm.Clang
          # component). Prepended to PATH above; kept in sync with the prereq step.
          CLANG_CL_BIN: "/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/Llvm/x64/bin"
          # The bench measures kache itself; it must not run under a wrapper.
          RUSTC_WRAPPER: ""
          # Let each tree honour its own rust-toolchain.toml (see Linux arm).
          RUSTUP_TOOLCHAIN: ""

      - name: Upload bench reports
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: bench-firefox-windows
          # Reports live under the SHORT work dir (`$RUNNER_TEMP/f`), not
          # tmp/bench, because of the mach 62-char source-path guard (see the
          # run step). The flat globs pick up summaries/reports/logs; clone
          # worktrees/objdirs/cache are subdirs and excluded.
          path: |
            ${{ runner.temp }}/f/*.json
            ${{ runner.temp }}/f/*.md
            ${{ runner.temp }}/f/report-*.json
            ${{ runner.temp }}/f/report-*.md
            ${{ runner.temp }}/f/key-diff.*
            ${{ runner.temp }}/f/*.log
            ${{ runner.temp }}/f/*.txt
          retention-days: 30
          if-no-files-found: warn

      - name: Validate telemetry
        if: always()
        timeout-minutes: 5
        run: npx --yes @kunobi/kartero@0.3.0 validate --input "${{ runner.temp }}/f"
        env:
          NPM_CONFIG_CACHE: ${{ runner.temp }}/kartero-npm-cache

      - name: Upload telemetry
        if: always()
        timeout-minutes: 10
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: telemetry-otlp-v1-firefox-windows
          path: |
            ${{ runner.temp }}/f/metrics.otlp.json
            ${{ runner.temp }}/f/schema_version
            ${{ runner.temp }}/f/bench-firefox-windows.json
          retention-days: 3
          if-no-files-found: error

      - name: Upload cache telemetry (cold)
        if: always()
        timeout-minutes: 10
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: telemetry-otlp-v1-cache-firefox-windows-cold
          path: ${{ runner.temp }}/f/cache-otlp-cold/
          retention-days: 3
          if-no-files-found: warn

      - name: Upload cache telemetry (warm)
        if: always()
        timeout-minutes: 10
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: telemetry-otlp-v1-cache-firefox-windows-warm
          path: ${{ runner.temp }}/f/cache-otlp-warm/
          retention-days: 3
          if-no-files-found: warn

  # Firefox DAILY-PULL (temporal, #477) compile-cache bench on the self-hosted
  # Windows runner — the axis that matches the reported issue (clang-cl on
  # Windows, same checkout, source moves a day forward). Mirrors
  # `bench-firefox-windows` step-for-step; the only differences are the scenario
  # profile (`firefox-pull-windows`), a longer timeout (two full sequential
  # Firefox builds — cold at ref A, then a from-scratch rebuild at ref B — vs
  # the cross-clone job's cold+warm), and the artifact name. Runs nightly and on
  # a dispatch selecting firefox-pull-windows or all.
  bench-firefox-pull-windows:
    name: Bench Firefox Pull (Windows)
    if: >-
      (github.repository == 'kunobi-ninja/kache' || vars.ENABLE_BENCHMARKS == 'true') &&
      (github.event_name == 'schedule' ||
       (github.event_name == 'workflow_dispatch' &&
        (github.event.inputs.scenario == 'firefox-pull-windows' || github.event.inputs.scenario == 'all')))
    runs-on: ${{ fromJSON(vars.BENCH_RUNNER_WINDOWS || (github.repository == 'kunobi-ninja/kache' && '["self-hosted","Windows","X64","kunobi-windows"]')) }}
    timeout-minutes: 540
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        timeout-minutes: 10

      - name: Set up Node for Kartero
        uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0
        with:
          node-version: 24

      - name: Seed failure telemetry
        run: |
          output="$(printf '%s' "$RUNNER_TEMP" | tr '\\' '/')/b"
          npx --yes @kunobi/kartero@0.3.0 gauge \
            --name kache.bench.verdict.ok \
            --value 0 \
            --unit 1 \
            --attribute kache.bench.project=bench-firefox-pull-windows \
            --attribute kache.bench.cache_tool=kache \
            --output "$output"
        env:
          NPM_CONFIG_CACHE: ${{ runner.temp }}/kartero-npm-cache

      # Fail fast with an explicit checklist if a Firefox build prerequisite is
      # missing. MozillaBuild provides mach's bash/python env; clang-cl (cc/c++),
      # nasm, and MSVC + Windows SDK back the compile. clang/rust are pulled by
      # `./mach bootstrap`, so they are NOT pre-required here. Keep in sync with
      # ans-runners/tasks/windows/setup_packages.yml.
      - name: Verify Firefox/Windows build prerequisites
        run: |
          missing=
          if [ -z "${MOZILLABUILD:-}" ] && [ ! -d "/c/mozilla-build" ]; then
            echo "MISSING: MozillaBuild (set MOZILLABUILD or install to C:\\mozilla-build) — provision it (ans-runners/tasks/windows/setup_packages.yml)"
            missing=1
          fi
          # Firefox's `./mach build` REQUIRES clang-cl. It's provisioned into VS
          # Build Tools by ans-runners (tasks/windows/setup_packages.yml) but is
          # NOT on the default PATH (the run step below prepends its dir), so
          # check the binary by absolute path. Do NOT rely on `cc`/`c++`: those
          # are MinGW gcc/g++ on the runner — the wrong compiler for Firefox.
          if [ ! -f "$CLANG_CL_BIN/clang-cl.exe" ]; then
            echo "MISSING: clang-cl at $CLANG_CL_BIN — provision it (ans-runners/tasks/windows/setup_packages.yml)"
            missing=1
          fi
          if ! command -v nasm >/dev/null 2>&1; then
            echo "MISSING on runner PATH: nasm — provision it (ans-runners/tasks/windows/setup_packages.yml)"
            missing=1
          fi
          [ -z "$missing" ] || exit 1
          echo "all Firefox/Windows prerequisites present"
        env:
          # VS-integrated clang-cl dir (ans-runners installs the VC.Llvm.Clang
          # component into VS Build Tools). Kept in sync with the run step.
          CLANG_CL_BIN: "/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/Llvm/x64/bin"

      # Disk-full mid-build is the worst failure mode. The pull scenario uses ONE
      # ref clone + ONE worktree, but rebuilds its objdir from scratch for the
      # pull phase and keeps the cold cache — still Firefox-class, so hold the
      # 120 GB floor.
      - name: Free-disk precheck
        run: |
          avail_gb=$(df -BG . 2>/dev/null | awk 'NR==2 {gsub(/G/,"",$4); print $4}')
          echo "Free disk: ${avail_gb:-?} GB (need >= 120 GB)"
          if [ "${avail_gb:-0}" -lt 120 ]; then
            echo "::error::Insufficient free disk (${avail_gb} GB < 120 GB) for the Firefox/Windows daily-pull bench"
            exit 1
          fi

      # Isolate tool state on the persistent runner: shared rustup/mise go stale
      # and wedge with `Access is denied (os error 5)`. Mirrors ci.yml's
      # test-windows so one run can't poison the next.
      - name: Isolate tool state
        run: |
          mkdir -p \
            "$RUNNER_TEMP/rustup" "$RUNNER_TEMP/cargo" \
            "$RUNNER_TEMP/mise-data/bin" "$RUNNER_TEMP/mise-data/shims" \
            "$RUNNER_TEMP/mise-cache" "$RUNNER_TEMP/mise-state"
          safe_runner=$(printf '%s' "$RUNNER_NAME" | tr -c 'A-Za-z0-9_.-' '_')
          rm -rf "$RUNNER_TEMP"/mise-dl-"$safe_runner"-* 2>/dev/null || true
          mise_dl="$RUNNER_TEMP/mise-dl-$safe_runner-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
          mkdir -p "$mise_dl"
          {
            echo "MISE_DL_TMPDIR=$mise_dl"
            echo "RUSTUP_HOME=$RUNNER_TEMP/rustup"
            echo "CARGO_HOME=$RUNNER_TEMP/cargo"
            echo "MISE_DATA_DIR=$RUNNER_TEMP/mise-data"
            echo "MISE_CACHE_DIR=$RUNNER_TEMP/mise-cache"
            echo "MISE_STATE_DIR=$RUNNER_TEMP/mise-state"
          } >> "$GITHUB_ENV"

      - name: Install tools via mise
        uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0
        with:
          # Pin mise CLI — see ci.yml (v2026.6.10 regression).
          version: 2026.6.9
          # node@lts: Firefox's build needs Node.js (bundling / some codegen). It
          # is normally provided by `mach bootstrap`, which --disable-bootstrap
          # skips, so install it here (user-level, no elevation).
          install_args: --force rust github:casey/just node@lts
          cache: false
          reshim: false

      - name: Run bench (firefox-pull-windows)
        timeout-minutes: 480
        # `firefox-pull-windows` is a unique substring, so it selects
        # bench-firefox-pull-windows directly (the `bench` recipe already ANDs
        # suite:bench + backend:kache).
        #
        # SHORT work dir: Firefox's `./mach build` refuses a source directory
        # path longer than 62 chars on Windows. The default bench work dir lives
        # under the runner checkout (`…\_work\kache\kache\tmp\bench\<scenario>\
        # clone-a` — 89 chars) and trips that guard. Put the clone under
        # `$RUNNER_TEMP\b` instead (service-writable, already used by the
        # isolate-tool-state step) so the source dir is ~47 chars. Reports land
        # under that same dir — see the upload paths below.
        run: |
          rustc --version && cargo --version && just --version
          # Firefox needs clang-cl, which ans-runners installs into VS Build Tools
          # but is NOT on the default PATH. Prepend its dir so mach discovers it
          # (mach finds the MSVC toolchain + Windows SDK itself via vswhere, so no
          # vcvars is needed). `--disable-bootstrap` in the mozconfig stops mach
          # from trying to fetch its OWN clang/VS (the fetch needs an elevated
          # msiexec the NETWORK SERVICE account can't run — exit 1601).
          export PATH="$CLANG_CL_BIN:$PATH"
          clang-cl --version | head -1 || { echo "clang-cl not on PATH"; exit 1; }
          # cbindgen (Rust->C header generator): another user-level Firefox build
          # tool `mach bootstrap` installs and --disable-bootstrap skips. Install
          # it into the isolated CARGO_HOME (already on PATH) if absent.
          command -v cbindgen >/dev/null 2>&1 || cargo install cbindgen --locked
          cbindgen --version || { echo "cbindgen install failed"; exit 1; }
          # node is installed by mise (node@lts), but mach invokes `node` as a
          # deep subprocess where mise isn't activated, so the mise SHIM fails
          # ("cannot find binary path"). Worse, node isn't pinned as the ACTIVE
          # version, so `mise which node` returns nothing either. Pin it global,
          # then resolve the REAL binary; fall back to globbing the install dir.
          mise use -g node@lts >/dev/null 2>&1 || true
          node_real="$(mise which node 2>/dev/null || true)"
          if [ -z "$node_real" ]; then
            mdd="$(cygpath -u "$MISE_DATA_DIR" 2>/dev/null || printf '%s' "$MISE_DATA_DIR")"
            node_real="$(find "$mdd/installs/node" -name node.exe 2>/dev/null | head -1)"
          fi
          [ -n "$node_real" ] || { echo "could not resolve node binary"; exit 1; }
          # Prepend the real node dir to PATH and set NODEJS (mach's documented
          # override) so mach uses the binary directly, not the failing shim.
          export PATH="$(dirname "$node_real"):$PATH"
          export NODEJS="$(cygpath -w "$node_real" 2>/dev/null || printf '%s' "$node_real")"
          node --version || { echo "node not resolvable"; exit 1; }
          echo "[bench] node: $node_real"
          # $RUNNER_TEMP is a Windows path (C:\…\_temp). Its backslashes get
          # eaten as escapes when the value passes through just/sh, collapsing it
          # to a bogus RELATIVE path that then joins onto the checkout cwd (still
          # >62 chars). Convert to forward slashes (C:/…/_temp) so it survives the
          # shell layers and Rust canonicalizes the intended short absolute path.
          work="$(printf '%s' "$RUNNER_TEMP" | tr '\\' '/')/b"
          mkdir -p "$work"
          echo "[bench] work dir: $work"
          # The full mach toolchain (VS / Win SDK / Windows App SDK / clang /
          # cbindgen / node / …) is pre-provisioned into the shared
          # MOZBUILD_STATE_PATH by ans-runners (elevated, where `msiexec /a`
          # works — the non-elevated runner service can't do it, exit 1601).
          # The build runs --disable-bootstrap and consumes that shared toolchain,
          # so the bench NEVER runs `mach bootstrap` itself (no --force-setup).
          just bench firefox-pull-windows --work-dir "$work"
        env:
          # Shared mach toolchain pre-provisioned (elevated) by ans-runners
          # (tag: firefox-toolchain). Pointing MOZBUILD_STATE_PATH here makes the
          # --disable-bootstrap build consume the fetched VS / Win SDK / Windows
          # App SDK / clang / cbindgen / node instead of trying (and failing) to
          # acquire them as the non-elevated runner service.
          MOZBUILD_STATE_PATH: 'C:\mozbuild-shared'
          # VS-integrated clang-cl dir (ans-runners installs the VC.Llvm.Clang
          # component). Prepended to PATH above; kept in sync with the prereq step.
          CLANG_CL_BIN: "/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/Llvm/x64/bin"
          # The bench measures kache itself; it must not run under a wrapper.
          RUSTC_WRAPPER: ""
          # Let each tree honour its own rust-toolchain.toml (see Linux arm).
          RUSTUP_TOOLCHAIN: ""
          # Append `--no-system-changes` to the scenario's `./mach bootstrap`
          # (see scenarios/bench-firefox-pull-windows/scenario.toml) — the runner
          # runs as NT AUTHORITY\NETWORK SERVICE (non-elevated), so mach's own
          # Windows SDK acquisition via `msiexec /a` fails with exit 1601; the
          # runner already has MSVC + the Windows SDK (provisioned by
          # ans-runners), so bootstrap must not try to install them.
          MACH_BOOTSTRAP_FLAGS: "--no-system-changes"

      - name: Upload bench reports
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: bench-firefox-pull-windows
          # Reports live under the SHORT work dir (`$RUNNER_TEMP/b`), not
          # tmp/bench, because of the mach 62-char source-path guard (see the run
          # step). `${{ runner.temp }}/b/` is the flat scratch dir the engine
          # writes summaries/reports/logs into (clone worktrees/objdirs/cache are
          # subdirs and excluded by these flat globs).
          path: |
            ${{ runner.temp }}/b/*.json
            ${{ runner.temp }}/b/*.md
            ${{ runner.temp }}/b/report-*.json
            ${{ runner.temp }}/b/report-*.md
            ${{ runner.temp }}/b/key-diff.*
            ${{ runner.temp }}/b/*.log
            ${{ runner.temp }}/b/*.txt
          retention-days: 30
          if-no-files-found: warn

      - name: Validate telemetry
        if: always()
        timeout-minutes: 5
        run: npx --yes @kunobi/kartero@0.3.0 validate --input "${{ runner.temp }}/b"
        env:
          NPM_CONFIG_CACHE: ${{ runner.temp }}/kartero-npm-cache

      - name: Upload telemetry
        if: always()
        timeout-minutes: 10
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: telemetry-otlp-v1-firefox-pull-windows
          path: |
            ${{ runner.temp }}/b/metrics.otlp.json
            ${{ runner.temp }}/b/schema_version
            ${{ runner.temp }}/b/bench-firefox-pull-windows.json
          retention-days: 3
          if-no-files-found: error

      - name: Upload cache telemetry (cold)
        if: always()
        timeout-minutes: 10
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: telemetry-otlp-v1-cache-firefox-pull-windows-cold
          path: ${{ runner.temp }}/b/cache-otlp-cold/
          retention-days: 3
          if-no-files-found: warn

      - name: Upload cache telemetry (pull)
        if: always()
        timeout-minutes: 10
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: telemetry-otlp-v1-cache-firefox-pull-windows-pull
          path: ${{ runner.temp }}/b/cache-otlp-pull/
          retention-days: 3
          if-no-files-found: warn