toko-tooling 0.45.0

Shared development tools, CI helpers and engineering rules for Toko Studios
Documentation
name: CI

on:
  pull_request:
  push:
    branches:
      - main

permissions:
  contents: read

defaults:
  run:
    shell: /bin/bash --noprofile --norc -e -o pipefail {0}

concurrency:
  # A newer run replaces queued and running checks for this workflow and ref.
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  change-scope:
    runs-on: ubuntu-24.04
    outputs:
      scope: ${{ steps.classify.outputs.scope }}
      hosts: ${{ steps.classify.outputs.hosts }}
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
        with:
          fetch-depth: 0
          persist-credentials: false
      - name: Classify the exact tested change
        id: classify
        env:
          CHANGE_EVENT: ${{ github.event_name }}
          CHANGE_REF: ${{ github.event_name == 'pull_request' && format('refs/heads/{0}', github.base_ref) || github.ref }}
          CHANGE_BASE: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
          CHANGE_HEAD: ${{ github.sha }}
        run: |
          scope="$(perl scripts/ci/classify-ci-change.pl --event "$CHANGE_EVENT" --ref "$CHANGE_REF" \
            --base "$CHANGE_BASE" --head "$CHANGE_HEAD" --policy ci/docs-only-paths.txt \
            --allow-version-bump)"
          case "$scope" in docs|full) ;; *) exit 1 ;; esac
          if [[ "$scope" == docs ]]; then
            git diff --check "$CHANGE_BASE...$CHANGE_HEAD" --
          fi
          echo "scope=$scope" >> "$GITHUB_OUTPUT"
          if [[ "$scope" == docs ]]; then
            echo 'hosts=["ubuntu-24.04"]' >> "$GITHUB_OUTPUT"
          else
            echo 'hosts=["ubuntu-24.04","macos-15","macos-15-intel"]' >> "$GITHUB_OUTPUT"
          fi

  lint-and-security:
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
        with:
          fetch-depth: 0
          persist-credentials: false
      - name: Restore pinned tool downloads
        uses: ./.github/actions/cache-tool-downloads
      - name: Install pinned validation tools
        shell: bash
        run: |
          tool_dir="$RUNNER_TEMP/toko-tooling-bin"
          actionlint_bin="$(bash scripts/ci/install-ci-tool.sh actionlint \
            --install-dir "$tool_dir")"
          shellcheck_bin="$(bash scripts/ci/install-ci-tool.sh shellcheck \
            --install-dir "$tool_dir")"
          gitleaks_bin="$(bash scripts/ci/install-ci-tool.sh gitleaks \
            --install-dir "$tool_dir")"
          make --no-print-directory install-host-tools
          make --no-print-directory host-tools-check
          echo "$GITHUB_WORKSPACE/.tools/host/bin" >> "$GITHUB_PATH"
          {
            echo "ACTIONLINT_BIN=$actionlint_bin"
            echo "SHELLCHECK_BIN=$shellcheck_bin"
            echo "GITLEAKS_BIN=$gitleaks_bin"
          } >>"$GITHUB_ENV"
      - name: Save pinned tool downloads
        uses: ./.github/actions/cache-tool-downloads
        with:
          mode: save
      - name: Prepare pinned compiler for dependency checks
        id: lint-toolchain
        run: bash scripts/dev/prepare-rust-toolchain.sh --toolchain-file rust-toolchain.toml
      - name: Check shell and workflow sources
        shell: bash
        run: |
          make check-shell SHELLCHECK="$SHELLCHECK_BIN"
          "$ACTIONLINT_BIN" .github/workflows/*.yml
          make check-no-symlinks
          make check-pins
          make check-doc-links
          make check-version
      - name: Qualify independent downloaded-evidence readers
        run: perl scripts/ci/test-failure-evidence.pl
      - name: Scan repository history for secrets
        shell: bash
        run: |
          "$GITLEAKS_BIN" git \
            --redact=100 \
            --no-banner \
            --no-color \
            --verbose \
            --log-level warn \
            "$GITHUB_WORKSPACE"

  portable-regression:
    needs: change-scope
    if: ${{ !cancelled() }}
    permissions:
      contents: read
      actions: read
    strategy:
      fail-fast: false
      matrix:
        os: ${{ fromJSON(needs.change-scope.outputs.hosts || '["ubuntu-24.04","macos-15","macos-15-intel"]') }}
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
        with:
          # Snapshot-impact qualification follows committed changelog ancestry.
          fetch-depth: 0
          persist-credentials: false
      - name: Require successful change classification
        env:
          CLASSIFICATION_RESULT: ${{ needs.change-scope.result }}
        run: test "$CLASSIFICATION_RESULT" = success
      - name: Check documentation-only changes
        if: needs.change-scope.outputs.scope == 'docs'
        run: |
          make check-doc-links
      - name: Install fixture dependencies on Linux
        if: ${{ needs.change-scope.outputs.scope != 'docs' && (runner.os == 'Linux') }}
        shell: bash
        run: |
          sudo apt-get update
          sudo apt-get install -y xz-utils zip unzip
      - name: Select system Bash on macOS
        if: ${{ needs.change-scope.outputs.scope != 'docs' && (runner.os == 'macOS') }}
        shell: bash
        run: |
          echo /bin >> "$GITHUB_PATH"
      - name: Restore pinned tool downloads
        if: needs.change-scope.outputs.scope != 'docs'
        uses: ./.github/actions/cache-tool-downloads
      - name: Prepare host tools and identify Cargo build cache
        if: needs.change-scope.outputs.scope != 'docs'
        id: tool-cache-key
        shell: bash
        run: |
          make --no-print-directory install-host-tools
          make --no-print-directory host-tools-check
          export PATH="$GITHUB_WORKSPACE/.tools/host/bin:$PATH"
          bash scripts/dev/prepare-rust-toolchain.sh --toolchain-file rust-toolchain.toml
          bash scripts/ci/cargo-tool-cache-key.sh ci/tool-versions.env >> "$GITHUB_OUTPUT"
      - name: Restore Cargo tool builds
        if: needs.change-scope.outputs.scope != 'docs'
        id: tool-cache
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        timeout-minutes: 3
        continue-on-error: true
        env:
          SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2
        with:
          path: |
            ${{ steps.tool-cache-key.outputs.root }}/cargo-cache
            ${{ steps.tool-cache-key.outputs.root }}/rust-*
          key: cargo-tools-${{ runner.os }}-${{ runner.arch }}-${{ steps.tool-cache-key.outputs.key }}-${{ hashFiles('Makefile') }}
      - name: Prepare and check the complete common toolset
        if: needs.change-scope.outputs.scope != 'docs'
        run: bash scripts/ci/prepare-common-tools.sh
      - name: Save pinned tool downloads
        if: needs.change-scope.outputs.scope != 'docs'
        uses: ./.github/actions/cache-tool-downloads
        with:
          mode: save
      - name: Publish the selected IC tools on PATH
        if: needs.change-scope.outputs.scope != 'docs'
        run: echo "$GITHUB_WORKSPACE/.tools/ic/bin" >> "$GITHUB_PATH"
      - name: Qualify native tool reuse across checkouts without downloads
        id: native-tool-reuse
        if: needs.change-scope.outputs.scope != 'docs'
        shell: bash
        run: |
          sharing="$RUNNER_TEMP/portable-fixtures/shared-tools"
          mkdir -p "$sharing/consumer" "$sharing/bin"
          # The installer selects the consumer's compiler. Without this pin,
          # the scratch checkout selects the runner default outside our tree.
          cp rust-toolchain.toml "$sharing/consumer/rust-toolchain.toml"
          printf '#!/bin/sh\necho unexpected-download >&2\nexit 91\n' > "$sharing/bin/curl"
          chmod +x "$sharing/bin/curl"
          for kind in host ic; do
            pins=(--versions "$GITHUB_WORKSPACE/ci/tool-versions.env")
            if [[ "$kind" == ic ]]; then
              pins=(--pins "$GITHUB_WORKSPACE/ci/ic-tools.tsv"
                    --tool didc --tool ic-wasm --tool icp --tool quill --tool wasm-opt)
            fi
            for mode in install check; do
              options=()
              if [[ "$mode" == check ]]; then options=(--check); fi
              PATH="$sharing/bin:$PATH" bash "scripts/dev/install-$kind-tools.sh" \
                --consumer "$sharing/consumer" "${pins[@]}" ${options[@]+"${options[@]}"} \
                > "$sharing/$kind-$mode.log" 2>&1
            done
            test "$(cd ".tools/$kind" && pwd -P)" = "$(cd "$sharing/consumer/.tools/$kind" && pwd -P)"
          done
          # Cargo sharing uses the same prepared compiler and standard build
          # environment. A second checkout must select the exact existing bytes.
          CARGO_NET_OFFLINE=true bash scripts/dev/install-rust-tools.sh \
            --consumer "$sharing/consumer" --versions "$GITHUB_WORKSPACE/ci/tool-versions.env" \
            > "$sharing/rust-install.log" 2>&1
          CARGO_NET_OFFLINE=true bash scripts/dev/install-rust-tools.sh \
            --consumer "$sharing/consumer" --versions "$GITHUB_WORKSPACE/ci/tool-versions.env" --check \
            > "$sharing/rust-check.log" 2>&1
          records="$(perl scripts/ci/tool-catalog-records.pl rust "$GITHUB_WORKSPACE/ci/tool-versions.env")"
          while IFS=$'\t' read -r name _; do
            tool="$(printf '%s' "$name" | tr '[:upper:]_' '[:lower:]-')"
            original=".tools/rust/common/bin/$tool"
            reused="$sharing/consumer/.tools/rust/common/bin/$tool"
            test -f "$original"
            test -x "$original"
            test -f "$reused"
            test -x "$reused"
            test "$(perl -MCwd=abs_path -e 'my $p=abs_path(shift); defined $p or exit 1; print $p' "$original")" = \
                 "$(perl -MCwd=abs_path -e 'my $p=abs_path(shift); defined $p or exit 1; print $p' "$reused")"
          done <<< "$records"
      # Save immediately after real setup/reuse checks: a later cancelled test
      # run must not force the next push to rebuild identical tools. PRs only read.
      - name: Save verified Cargo tool builds
        if: ${{ needs.change-scope.outputs.scope != 'docs' && (github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.tool-cache.outputs.cache-hit != 'true') }}
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        timeout-minutes: 3
        continue-on-error: true
        with:
          path: |
            ${{ steps.tool-cache-key.outputs.root }}/cargo-cache
            ${{ steps.tool-cache-key.outputs.root }}/rust-*
          key: cargo-tools-${{ runner.os }}-${{ runner.arch }}-${{ steps.tool-cache-key.outputs.key }}-${{ hashFiles('Makefile') }}
      - name: Select the frontend fixture runtime
        if: needs.change-scope.outputs.scope != 'docs'
        uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
        with:
          node-version-file: ci/frontend/package.json
          package-manager-cache: false
      - name: Prepare the locked frontend fixture
        if: needs.change-scope.outputs.scope != 'docs'
        run: |
          test "$(npm --version)" = "$(jq -er '.engines.npm' ci/frontend/package.json)"
          npm ci --prefix ci/frontend
      - name: Qualify native runtime scanner compilation and semantics
        if: needs.change-scope.outputs.scope != 'docs'
        id: runtime-scanner
        run: |
          compiler="$(yq -r '.toolchain.channel' rust-toolchain.toml)"
          bash scripts/ci/qualify-runtime-panics.sh "$compiler" "$RUNNER_TEMP/portable-fixtures/runtime-scanner"
      - name: Qualify repository dependency maintenance
        if: needs.change-scope.outputs.scope != 'docs'
        run: perl scripts/ci/test-cargo-dependencies.pl
      - name: Qualify Rust 1.100 lint and warning policy
        if: needs.change-scope.outputs.scope != 'docs'
        run: perl scripts/ci/test-rust-1100-policy.pl
      - name: Qualify public documentation identities
        if: needs.change-scope.outputs.scope != 'docs'
        run: make test-public-docs
      - name: Run portable regression set with prepared tools
        id: portable-tests
        if: needs.change-scope.outputs.scope != 'docs'
        shell: bash
        env:
          TMPDIR: ${{ runner.temp }}/portable-fixtures
        run: |
          mkdir -p "$TMPDIR"
          bash --version | head -n 1
          make test-portable 2>&1 | tee "$RUNNER_TEMP/portable-regression.log"
      - name: Qualify the real frontend formatter and Rust hook
        if: needs.change-scope.outputs.scope != 'docs'
        env:
          TMPDIR: ${{ runner.temp }}/portable-fixtures
        run: |
          bash scripts/ci/test-frontend-formatting.sh 2>&1 | tee "$RUNNER_TEMP/portable-fixtures/frontend-formatting.log"
      - name: Exercise expected native installer and check failures
        if: needs.change-scope.outputs.scope != 'docs'
        id: retention
        continue-on-error: true
        run: bash scripts/ci/qualify-native-failure-retention.sh "$RUNNER_TEMP/portable-fixtures/native-retention"
      - name: Upload the native failure qualification evidence
        if: ${{ needs.change-scope.outputs.scope != 'docs' && (!cancelled() && steps.retention.outcome == 'failure') }}
        id: retention-upload
        uses: ./.github/actions/retain-failure-evidence
        env:
          TOKO_TOOLING_STORE: ${{ runner.temp }}/portable-fixtures/native-retention/store
        with:
          name: native-retention-${{ matrix.os }}-${{ github.run_attempt }}
          temp-root: ${{ runner.temp }}/portable-fixtures/native-retention/temp
          repository-root: ${{ runner.temp }}/portable-fixtures/native-retention/consumer
      - name: Download native failure evidence by exact ID
        if: ${{ needs.change-scope.outputs.scope != 'docs' && (!cancelled() && steps.retention-upload.outputs.artifact-id != '') }}
        timeout-minutes: 3
        env:
          GH_TOKEN: ${{ github.token }}
          EVIDENCE_REPOSITORY: ${{ github.repository }}
          EVIDENCE_RUN: ${{ github.run_id }}
          EVIDENCE_ARTIFACT: ${{ steps.retention-upload.outputs.artifact-id }}
          EVIDENCE_DIGEST: ${{ steps.retention-upload.outputs.artifact-digest }}
        run: bash scripts/ci/download-evidence.sh "$EVIDENCE_REPOSITORY" "$EVIDENCE_RUN" "$EVIDENCE_ARTIFACT" "$EVIDENCE_DIGEST" "$RUNNER_TEMP/native-retention-downloaded"
      - name: Verify native failure status and uploaded payload
        if: ${{ needs.change-scope.outputs.scope != 'docs' && (!cancelled() && steps.retention.outcome != 'skipped') }}
        env:
          RETENTION_OUTCOME: ${{ steps.retention.outcome }}
          RETENTION_QUALIFIED: ${{ steps.retention.outputs.qualified }}
        run: |
          test "$RETENTION_OUTCOME" = failure
          test "$RETENTION_QUALIFIED" = true
          cd "$RUNNER_TEMP/native-retention-downloaded"
          mkdir payload
          tar -xzf evidence.tar.gz -C payload
          cd payload
          bash "$GITHUB_WORKSPACE/scripts/ci/verify-evidence-checksums.sh" "$RUNNER_TEMP/portable-fixtures/native-retention/expected.sha256"
          cmp "$RUNNER_TEMP/portable-fixtures/native-retention/temp/portable-fixtures/"$'line\nbreak:payload' "portable-fixtures/"$'line\nbreak:payload'
          test "$(perl -e 'printf "%o", (stat($ARGV[0]))[2] & 0777' "portable-fixtures/"$'line\nbreak:payload')" = 640
          test -x .tools/ic-set.check/bin/quill
          test -L portable-fixtures/link
          test "$(readlink portable-fixtures/link)" = ../outside-evidence
          test ! -e portable-fixtures/link
      - name: Prepare failed candidate alongside verified native toolsets
        if: ${{ needs.change-scope.outputs.scope != 'docs' && (!cancelled() && steps.retention.outputs.qualified == 'true') }}
        id: compact-control
        shell: bash
        run: |
          candidate="$(mktemp -d "$GITHUB_WORKSPACE/.tools/ic-set.evidenceXXXXXX")"
          cp -R "$RUNNER_TEMP/portable-fixtures/native-retention/consumer/.tools/ic-set.check/." "$candidate/"
          printf 'candidate=%s\n' "${candidate##*/}" >> "$GITHUB_OUTPUT"
      - name: Upload compact native tool evidence
        if: ${{ needs.change-scope.outputs.scope != 'docs' && (!cancelled() && steps.compact-control.outputs.candidate != '') }}
        id: compact-upload
        uses: ./.github/actions/retain-failure-evidence
        with:
          name: native-compact-tools-${{ matrix.os }}-${{ github.run_attempt }}
          temp-root: ${{ runner.temp }}/portable-fixtures/native-retention/temp
          repository-root: ${{ github.workspace }}
          compact-successful-tools: 'true'
          ic-tools: 'didc ic-wasm icp quill wasm-opt'
      - name: Download compact evidence by exact ID
        if: ${{ needs.change-scope.outputs.scope != 'docs' && (!cancelled() && steps.compact-upload.outputs.artifact-id != '') }}
        timeout-minutes: 3
        env:
          GH_TOKEN: ${{ github.token }}
          EVIDENCE_REPOSITORY: ${{ github.repository }}
          EVIDENCE_RUN: ${{ github.run_id }}
          EVIDENCE_ARTIFACT: ${{ steps.compact-upload.outputs.artifact-id }}
          EVIDENCE_DIGEST: ${{ steps.compact-upload.outputs.artifact-digest }}
        run: bash scripts/ci/download-evidence.sh "$EVIDENCE_REPOSITORY" "$EVIDENCE_RUN" "$EVIDENCE_ARTIFACT" "$EVIDENCE_DIGEST" "$RUNNER_TEMP/compact-tools-downloaded"
      - name: Verify compact native tool evidence and retained candidate
        id: verify-compact-tools
        if: ${{ needs.change-scope.outputs.scope != 'docs' && (!cancelled() && steps.compact-upload.outputs.artifact-id != '') }}
        env:
          EVIDENCE_CANDIDATE: ${{ steps.compact-control.outputs.candidate }}
        shell: bash
        run: |
          payload="$RUNNER_TEMP/compact-tools-downloaded/payload"
          mkdir "$payload"
          tar -xzf "$RUNNER_TEMP/compact-tools-downloaded/evidence.tar.gz" -C "$payload"
          shopt -s nullglob
          diagnostics=("$payload"/tool-evidence.*)
          test "${#diagnostics[@]}" = 1
          for kind in host ic; do
            selection="$(cat "${diagnostics[0]}/$kind/selection.txt")"
            test ! -e "$payload/.tools/${selection##*/}"
            test -s "${diagnostics[0]}/$kind/check.log"
          done
          cmp ci/tool-versions.env "${diagnostics[0]}/host/caller-pins"
          cmp ci/ic-tools.tsv "${diagnostics[0]}/ic/caller-pins"
          for receipt in pins.tsv tools host files.sha256; do
            cmp ".tools/ic/$receipt" "${diagnostics[0]}/ic/$receipt"
          done
          cmp ".tools/$EVIDENCE_CANDIDATE/bin/quill" "$payload/.tools/$EVIDENCE_CANDIDATE/bin/quill"
          test -x "$payload/.tools/$EVIDENCE_CANDIDATE/bin/quill"
          for log in ic-tools-install ic-tools-check rust-tools-install rust-tools-check; do
            cmp "$RUNNER_TEMP/portable-fixtures/native-retention/temp/$log.log" "$payload/$log.log"
          done
      - name: Retain failed validation and installer evidence
        if: ${{ needs.change-scope.outputs.scope != 'docs' && (failure()) }}
        uses: ./.github/actions/retain-failure-evidence
        with:
          name: portable-failure-${{ matrix.os }}
          temp-root: ${{ runner.temp }}
          repository-root: ${{ github.workspace }}
          compact-successful-tools: 'true'
          ic-tools: 'didc ic-wasm icp quill wasm-opt'