name: CI
on:
push:
branches:
- main
pull_request:
permissions:
contents: read
concurrency:
group: ci-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
fmt:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Check out repository
uses: actions/checkout@v6
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt
- name: Cache Rust dependencies
uses: Swatinem/rust-cache@v2
- name: Check formatting
run: cargo fmt --check
docs-lint:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Check out repository
uses: actions/checkout@v6
- name: Install Node.js
uses: actions/setup-node@v6
with:
node-version: 22
package-manager-cache: false
- name: Lint markdown docs
run: npm run docs:lint
secret-scan:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Check out repository
uses: actions/checkout@v6
- name: Scan repository for leaked credentials
shell: bash
run: |
set -euo pipefail
docker run --rm \
-v "${PWD}:/repo" \
ghcr.io/gitleaks/gitleaks:v8.30.1 \
detect --source /repo --no-git --redact --no-banner --config /repo/.gitleaks.toml
supply-chain:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Check out repository
uses: actions/checkout@v6
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
components: clippy
- name: Cache Rust dependencies
uses: Swatinem/rust-cache@v2
with:
cache-bin: false
- name: Cache cargo supply-chain tools
id: cargo-supply-chain-cache
uses: actions/cache@v5
with:
path: |
~/.cargo/bin/cargo-audit
~/.cargo/bin/cargo-deny
key: ${{ runner.os }}-cargo-supply-chain-tools-audit-0.22.1-deny-0.19.0
- name: Check locked compile graph
run: cargo check --locked --all-targets --all-features
- name: Run clippy warning gate
run: cargo clippy --locked --all-targets --all-features -- -D warnings
- name: Install cargo-audit
shell: bash
run: |
set -euo pipefail
if ! command -v cargo-audit >/dev/null 2>&1; then
cargo install cargo-audit --locked --version 0.22.1
fi
- name: Run cargo audit
shell: bash
run: |
set -euo pipefail
for attempt in 1 2 3; do
if cargo audit; then
exit 0
fi
if [ "$attempt" -eq 3 ]; then
exit 1
fi
sleep "$((attempt * 5))"
done
- name: Install cargo-deny
shell: bash
run: |
set -euo pipefail
if ! command -v cargo-deny >/dev/null 2>&1; then
cargo install cargo-deny --locked --version 0.19.0
fi
- name: Run cargo deny checks
shell: bash
run: |
set -euo pipefail
for attempt in 1 2 3; do
if cargo deny check advisories sources; then
exit 0
fi
if [ "$attempt" -eq 3 ]; then
exit 1
fi
sleep "$((attempt * 5))"
done
release-sync:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Check out repository
uses: actions/checkout@v6
- name: Verify release/version sync
shell: bash
run: |
set -euo pipefail
npm run npm:sync-version
git diff --exit-code -- Cargo.toml npm README.md QUICKSTART.md scripts/npm
npm-package-smoke:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Check out repository
uses: actions/checkout@v6
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Cache Rust dependencies
uses: Swatinem/rust-cache@v2
- name: Install Node.js
uses: actions/setup-node@v6
with:
node-version: 22
package-manager-cache: false
- name: Build host release binary
run: cargo build --release --locked --target x86_64-unknown-linux-gnu
- name: Smoke staged npm package
run: node scripts/ci/npm-package-smoke.mjs --binary-dir target/x86_64-unknown-linux-gnu/release
auto-rotate:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Check out repository
uses: actions/checkout@v6
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Cache Rust dependencies
uses: Swatinem/rust-cache@v2
- name: Run auto-rotate integration tests
run: cargo test --test auto_rotate
profile-commands-internal:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Check out repository
uses: actions/checkout@v6
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Cache Rust dependencies
uses: Swatinem/rust-cache@v2
- name: Run profile command internal tests
run: |
cargo test --lib profile_commands_internal_tests:: -- --test-threads=1
main-internal-core:
name: Main internal core (${{ matrix.label }})
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
include:
- shard: 0
shard_count: 4
label: shard 1 of 4
- shard: 1
shard_count: 4
label: shard 2 of 4
- shard: 2
shard_count: 4
label: shard 3 of 4
- shard: 3
shard_count: 4
label: shard 4 of 4
steps:
- name: Check out repository
uses: actions/checkout@v6
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Cache Rust dependencies
uses: Swatinem/rust-cache@v2
- name: Run main internal core tests ${{ matrix.label }}
shell: bash
env:
SHARD_INDEX: ${{ matrix.shard }}
SHARD_COUNT: ${{ matrix.shard_count }}
RUST_BACKTRACE: "1"
CARGO_TERM_COLOR: always
run: |
set -euo pipefail
mapfile -t core_tests < <(
cargo test --lib main_internal_tests:: -- --list \
| awk '/^main_internal_tests::.*: test$/ { sub(/: test$/, ""); if ($0 !~ /runtime_proxy_/) print }'
)
if [ "${#core_tests[@]}" -eq 0 ]; then
echo "No main internal core tests found"
exit 1
fi
selected_tests=()
for index in "${!core_tests[@]}"; do
if [ "$((index % SHARD_COUNT))" -eq "${SHARD_INDEX}" ]; then
selected_tests+=("${core_tests[$index]}")
fi
done
if [ "${#selected_tests[@]}" -eq 0 ]; then
echo "No tests selected for main internal core ${SHARD_INDEX}/${SHARD_COUNT}"
exit 1
fi
printf 'Running %s of %s main internal core tests in shard %s/%s\n' \
"${#selected_tests[@]}" \
"${#core_tests[@]}" \
"${SHARD_INDEX}" \
"${SHARD_COUNT}"
for test_name in "${selected_tests[@]}"; do
cargo test --lib "${test_name}" -- --exact --test-threads=1
done
env-sensitive-parallel-guard:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Check out repository
uses: actions/checkout@v6
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Cache Rust dependencies
uses: Swatinem/rust-cache@v2
- name: Run env-sensitive parallel guard
run: node scripts/ci/runtime-env-parallel.mjs --runs 2 --test-threads 4
main-internal-runtime-proxy:
name: Main internal runtime proxy (${{ matrix.label }})
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
include:
- suite: root
label: root proxy helpers
filters: |
broker|main_internal_tests::runtime_proxy_broker_
log-paths|main_internal_tests::runtime_proxy_log_paths_
worker-count|main_internal_tests::runtime_proxy_worker_count_
endpoint-child|main_internal_tests::runtime_proxy_endpoint_child_
claude-launch-root|main_internal_tests::runtime_proxy_claude_launch_
claude-caveman-root|main_internal_tests::prepare_runtime_proxy_claude_
- suite: selection
label: selection and quota
filters: |
selection|main_internal_tests::runtime_proxy_selection_and_pressure::selection::
- suite: rotation
label: rotation and affinity
filters: |
rotation|main_internal_tests::runtime_proxy_selection_and_pressure::rotation::
- suite: state
label: state persistence
filters: |
state|main_internal_tests::runtime_proxy_selection_and_pressure::state::
- suite: admission
label: admission and request paths
filters: |
admission|main_internal_tests::runtime_proxy_selection_and_pressure::admission::
- suite: health
label: health and pressure
filters: |
health|main_internal_tests::runtime_proxy_selection_and_pressure::health::
pressure|main_internal_tests::runtime_proxy_selection_and_pressure::pressure::
- suite: persistence
label: persisted backoff selection
filters: |
persistence|main_internal_tests::runtime_proxy_selection_and_pressure::persistence::
- suite: doctor
label: doctor and incidents
filters: |
doctor|main_internal_tests::runtime_proxy_selection_and_pressure::doctor::
incidents|main_internal_tests::runtime_proxy_selection_and_pressure::incidents::
- suite: continuations
label: continuation behavior
filters: |
continuations|main_internal_tests::runtime_proxy_continuations::
- suite: anthropic-launch
label: anthropic launch
filters: |
lane-and-launch|main_internal_tests::runtime_proxy_claude_and_anthropic::lane_and_launch::
- suite: anthropic-request
label: anthropic request translation
filters: |
request-translation|main_internal_tests::runtime_proxy_claude_and_anthropic::request_translation::
- suite: anthropic-response
label: anthropic response translation
filters: |
response-translation|main_internal_tests::runtime_proxy_claude_and_anthropic::response_translation::
- suite: anthropic-runtime
label: anthropic runtime behavior
filters: |
runtime-behavior|main_internal_tests::runtime_proxy_claude_and_anthropic::runtime_proxy_behavior::
env:
RUNTIME_PROXY_ARTIFACT_DIR: target/ci/runtime-proxy/${{ matrix.suite }}
CARGO_TERM_COLOR: always
RUST_BACKTRACE: "1"
steps:
- name: Check out repository
uses: actions/checkout@v6
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Cache Rust dependencies
uses: Swatinem/rust-cache@v2
- name: Validate runtime CI manifest
if: matrix.suite == 'root'
run: npm run ci:runtime-manifest
- name: Run runtime proxy internal test shard ${{ matrix.label }}
shell: bash
run: |
set -euo pipefail
while IFS= read -r shard; do
if [ -z "${shard}" ]; then
continue
fi
shard_label="${shard%%|*}"
filter="${shard#*|}"
if [ -z "${shard_label}" ] || [ -z "${filter}" ] || [ "${shard_label}" = "${filter}" ]; then
echo "Invalid runtime proxy shard entry: ${shard}"
exit 1
fi
node scripts/ci/runtime-proxy-shard.mjs \
--filter "${filter}" \
--test-threads 1 \
--artifact-dir "${RUNTIME_PROXY_ARTIFACT_DIR}/${shard_label}"
done <<< "${{ matrix.filters }}"
- name: Upload runtime proxy shard diagnostics
if: failure()
uses: actions/upload-artifact@v7
with:
name: runtime-proxy-${{ matrix.suite }}-diagnostics
path: ${{ env.RUNTIME_PROXY_ARTIFACT_DIR }}
if-no-files-found: ignore
runtime-proxy-bench-smoke:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Check out repository
uses: actions/checkout@v6
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Cache Rust dependencies
uses: Swatinem/rust-cache@v2
- name: Check runtime proxy benchmark regression thresholds
env:
PRODEX_RUNTIME_PROXY_BENCH_CHECK: "1"
PRODEX_RUNTIME_PROXY_BENCH_THRESHOLD_FILE: scripts/ci/runtime-proxy-bench-thresholds.json
run: cargo bench --locked --features bench-support --bench runtime_proxy_hot_paths
runtime-stress:
name: Runtime stress (${{ matrix.label }})
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
include:
- suite: stress
label: broad shard
- suite: serialized
label: serialized shard
- suite: continuation
label: continuation shard
env:
RUNTIME_STRESS_ARTIFACT_DIR: target/ci/runtime-stress/${{ matrix.suite }}
PRODEX_RUNTIME_LOG_DIR: target/ci/runtime-stress/${{ matrix.suite }}/runtime-logs
CARGO_TERM_COLOR: always
RUST_BACKTRACE: "1"
steps:
- name: Check out repository
uses: actions/checkout@v6
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Cache Rust dependencies
uses: Swatinem/rust-cache@v2
- name: Prepare runtime stress diagnostics
shell: bash
run: |
set -euo pipefail
rm -rf "${RUNTIME_STRESS_ARTIFACT_DIR}"
mkdir -p "${PRODEX_RUNTIME_LOG_DIR}"
{
echo "suite=${{ matrix.suite }}"
echo "label=${{ matrix.label }}"
echo "artifact_dir=${RUNTIME_STRESS_ARTIFACT_DIR}"
echo "runtime_log_dir=${PRODEX_RUNTIME_LOG_DIR}"
echo "started_at=$(date -u +%FT%TZ)"
echo "command=npm run ci:runtime-stress -- --suite ${{ matrix.suite }}"
} > "${RUNTIME_STRESS_ARTIFACT_DIR}/metadata.txt"
- name: Run runtime stress ${{ matrix.label }}
shell: bash
run: |
set -euo pipefail
npm run ci:runtime-stress -- --suite "${{ matrix.suite }}" 2>&1 | tee "${RUNTIME_STRESS_ARTIFACT_DIR}/runtime-stress.log"
- name: Collect runtime stress diagnostics
if: failure()
shell: bash
run: |
set -euo pipefail
mkdir -p "${RUNTIME_STRESS_ARTIFACT_DIR}" "${PRODEX_RUNTIME_LOG_DIR}"
diagnostics_path="${RUNTIME_STRESS_ARTIFACT_DIR}/diagnostics.txt"
pointer_path="${PRODEX_RUNTIME_LOG_DIR}/prodex-runtime-latest.path"
latest_log_path=""
{
echo "suite=${{ matrix.suite }}"
echo "label=${{ matrix.label }}"
echo "artifact_dir=${RUNTIME_STRESS_ARTIFACT_DIR}"
echo "runtime_log_dir=${PRODEX_RUNTIME_LOG_DIR}"
echo "finished_at=$(date -u +%FT%TZ)"
echo "runtime_log_pointer=${pointer_path}"
} > "${diagnostics_path}"
if [[ -f "${pointer_path}" ]]; then
latest_log_path="$(tr -d '\r\n' < "${pointer_path}")"
echo "runtime_log_pointer_target=${latest_log_path}" >> "${diagnostics_path}"
else
echo "runtime_log_pointer_target=missing" >> "${diagnostics_path}"
fi
if [[ -z "${latest_log_path}" || ! -f "${latest_log_path}" ]]; then
latest_log_path="$(
find "${PRODEX_RUNTIME_LOG_DIR}" -maxdepth 1 -type f -name 'prodex-runtime*.log' -printf '%T@ %p\n' \
| sort -nr \
| head -n 1 \
| cut -d' ' -f2-
)"
fi
find "${PRODEX_RUNTIME_LOG_DIR}" -maxdepth 1 -type f -print \
| sort \
| sed 's/^/log_entry=/' >> "${diagnostics_path}"
if [[ -n "${latest_log_path}" && -f "${latest_log_path}" ]]; then
echo "latest_runtime_log=${latest_log_path}" >> "${diagnostics_path}"
cp "${latest_log_path}" "${RUNTIME_STRESS_ARTIFACT_DIR}/latest-runtime.log"
tail -n 200 "${latest_log_path}" | tee "${RUNTIME_STRESS_ARTIFACT_DIR}/latest-runtime-log-tail.txt"
else
echo "latest_runtime_log=missing" >> "${diagnostics_path}"
fi
- name: Upload runtime stress diagnostics
if: failure()
uses: actions/upload-artifact@v7
with:
name: runtime-stress-${{ matrix.suite }}-diagnostics
path: ${{ env.RUNTIME_STRESS_ARTIFACT_DIR }}
if-no-files-found: ignore