name: Benchmark Impact Engines
on:
workflow_dispatch:
pull_request:
paths:
- "src/engine/**"
- "scripts/bench-impact-engines.sh"
- "bench-fixtures/**"
- ".github/workflows/bench.yml"
permissions:
contents: read
concurrency:
group: bench-impact-${{ github.ref }}
cancel-in-progress: true
jobs:
bench-impact:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install rust-analyzer component
run: rustup component add rust-analyzer
- name: Build release binary
run: cargo build --release --locked -q
- name: Run TS vs Auto(strict-if-available) benchmark
id: bench
continue-on-error: true
timeout-minutes: 12
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
BASE="origin/${{ github.base_ref }}"
RUNS=1
else
BASE="origin/main"
RUNS=2
fi
echo "base=$BASE runs=$RUNS compare=auto(strict-if-available)" > bench-report.txt
timeout 10m scripts/bench-impact-engines.sh \
--base "$BASE" \
--runs "$RUNS" \
--direction callers \
--lang rust \
--compare-auto-strict-if-available \
--rpc-counts \
--save-ts-json bench-rust-ts.json \
--save-lsp-json bench-rust-auto-strict-if-available.json \
| tee -a bench-report.txt
- name: Publish benchmark summary
if: always()
run: |
{
echo "## Impact engine benchmark"
echo
if [ -f bench-report.txt ]; then
sed 's/^/`/;s/$/`/' bench-report.txt
else
echo "benchmark report missing"
fi
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload benchmark artifact
if: always()
uses: actions/upload-artifact@v7
with:
name: bench-impact-engines
path: |
bench-report.txt
bench-rust-ts.json
bench-rust-auto-strict-if-available.json
if-no-files-found: warn
bench-go-strict-lsp:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- uses: actions/setup-go@v6
with:
go-version: "1.22.x"
- name: Install gopls
run: |
GOTOOLCHAIN=auto go install golang.org/x/tools/gopls@latest
echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
- name: Build release binary
run: cargo build --release --locked -q
- name: Run Go strict-LSP benchmark
timeout-minutes: 12
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
RUNS=1
else
RUNS=2
fi
echo "runs=$RUNS lang=go fixture=bench-fixtures/go-heavy.diff" > bench-go-report.txt
timeout 10m scripts/bench-impact-engines.sh \
--diff-file bench-fixtures/go-heavy.diff \
--runs "$RUNS" \
--direction callers \
--lang go \
--min-lsp-changed 6 \
--min-lsp-impacted 15 \
--save-ts-json bench-go-ts.json \
--save-lsp-json bench-go-lsp.json \
| tee -a bench-go-report.txt
- name: Publish Go benchmark summary
if: always()
run: |
{
echo "## Go strict-LSP benchmark"
echo
if [ -f bench-go-report.txt ]; then
sed 's/^/`/;s/$/`/' bench-go-report.txt
else
echo "go benchmark report missing"
fi
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload Go benchmark artifact
if: always()
uses: actions/upload-artifact@v7
with:
name: bench-go-strict-lsp
path: |
bench-go-report.txt
bench-go-ts.json
bench-go-lsp.json
if-no-files-found: warn
bench-java-strict-lsp:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- uses: actions/setup-java@v4
with:
distribution: temurin
java-version: "17"
- name: Install jdtls
run: |
mkdir -p "$HOME/.local/jdtls" "$HOME/.local/bin"
curl -fsSL "https://download.eclipse.org/jdtls/snapshots/jdt-language-server-latest.tar.gz" -o /tmp/jdtls.tar.gz
tar -xzf /tmp/jdtls.tar.gz -C "$HOME/.local/jdtls"
JDTLS_BIN="$(find "$HOME/.local/jdtls" -type f -name jdtls | head -n 1)"
if [ -z "$JDTLS_BIN" ]; then
echo "jdtls binary not found after extraction" >&2
exit 1
fi
chmod +x "$JDTLS_BIN"
ln -sf "$JDTLS_BIN" "$HOME/.local/bin/jdtls"
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Build release binary
run: cargo build --release --locked -q
- name: Run Java strict-LSP benchmark
timeout-minutes: 12
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
RUNS=1
else
RUNS=2
fi
echo "runs=$RUNS lang=java fixture=bench-fixtures/java-heavy.diff" > bench-java-report.txt
timeout 10m scripts/bench-impact-engines.sh \
--diff-file bench-fixtures/java-heavy.diff \
--runs "$RUNS" \
--direction callers \
--lang java \
--min-lsp-changed 7 \
--min-lsp-impacted 15 \
--save-ts-json bench-java-ts.json \
--save-lsp-json bench-java-lsp.json \
| tee -a bench-java-report.txt
- name: Publish Java benchmark summary
if: always()
run: |
{
echo "## Java strict-LSP benchmark"
echo
if [ -f bench-java-report.txt ]; then
sed 's/^/`/;s/$/`/' bench-java-report.txt
else
echo "java benchmark report missing"
fi
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload Java benchmark artifact
if: always()
uses: actions/upload-artifact@v7
with:
name: bench-java-strict-lsp
path: |
bench-java-report.txt
bench-java-ts.json
bench-java-lsp.json
if-no-files-found: warn
bench-python-strict-lsp:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- uses: actions/setup-node@v6
with:
node-version: "20"
- name: Install Python LSP server (robust + fallback)
id: install_python_lsp
run: |
set -euo pipefail
INSTALL_LOG="bench-python-install.log"
INSTALL_SUMMARY="bench-python-install-summary.txt"
: > "$INSTALL_LOG"
exec > >(tee -a "$INSTALL_LOG") 2>&1
select_server() {
local server="$1"
echo "python_lsp=$server" >> "$GITHUB_OUTPUT"
echo "INSTALL_RESULT=success python_lsp=$server" > "$INSTALL_SUMMARY"
echo "[python-lsp] selected=$server"
}
echo "[python-lsp] node=$(node -v) npm=$(npm -v) python=$(python3 --version 2>&1)"
# Primary: pyright
if npm install -g pyright; then
if command -v pyright-langserver >/dev/null 2>&1 && pyright --version >/tmp/pyright-version.log 2>&1; then
select_server pyright
cat /tmp/pyright-version.log
exit 0
fi
echo "[python-lsp] pyright executable check failed, trying fallback" >&2
cat /tmp/pyright-version.log >&2 || true
else
echo "[python-lsp] npm install pyright failed, trying fallback" >&2
fi
# Fallback-1: basedpyright
if npm install -g basedpyright; then
if command -v basedpyright-langserver >/dev/null 2>&1 && basedpyright --version >/tmp/basedpyright-version.log 2>&1; then
select_server basedpyright
cat /tmp/basedpyright-version.log
exit 0
fi
echo "[python-lsp] basedpyright executable check failed, trying fallback-2" >&2
cat /tmp/basedpyright-version.log >&2 || true
else
echo "[python-lsp] npm install basedpyright failed, trying fallback-2" >&2
fi
# Fallback-2: pylsp
if python3 -m pip install --user --disable-pip-version-check python-lsp-server; then
export PATH="$HOME/.local/bin:$PATH"
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
if command -v pylsp >/dev/null 2>&1 && pylsp --version >/tmp/pylsp-version.log 2>&1; then
select_server pylsp
cat /tmp/pylsp-version.log
exit 0
fi
cat /tmp/pylsp-version.log >&2 || true
fi
echo "INSTALL_RESULT=failure reason=no-usable-server candidates=pyright,basedpyright,pylsp" > "$INSTALL_SUMMARY"
echo "::error::[python-lsp] failed to install a usable python LSP server (pyright/basedpyright/pylsp)" >&2
exit 1
- name: Build release binary
run: cargo build --release --locked -q
- name: Run Python strict-LSP benchmark
timeout-minutes: 12
env:
DIMPACT_PYTHON_LSP: ${{ steps.install_python_lsp.outputs.python_lsp }}
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
RUNS=1
else
RUNS=2
fi
echo "runs=$RUNS lang=python fixture=bench-fixtures/python-heavy.diff python_lsp=${DIMPACT_PYTHON_LSP}" > bench-python-report.txt
timeout 10m scripts/bench-impact-engines.sh \
--diff-file bench-fixtures/python-heavy.diff \
--runs "$RUNS" \
--direction callers \
--lang python \
--min-lsp-changed 3 \
--min-lsp-impacted 5 \
--save-ts-json bench-python-ts.json \
--save-lsp-json bench-python-lsp.json \
| tee -a bench-python-report.txt
- name: Publish Python benchmark summary
if: always()
run: |
{
echo "## Python strict-LSP benchmark"
echo
echo "install step conclusion: `${{ steps.install_python_lsp.conclusion }}`"
if [ -f bench-python-install-summary.txt ]; then
echo
echo "### install summary"
sed 's/^/`/;s/$/`/' bench-python-install-summary.txt
fi
if [ -f bench-python-install.log ]; then
echo
echo "### install log (tail -n 80)"
echo '```'
tail -n 80 bench-python-install.log
echo '```'
fi
echo
if [ -f bench-python-report.txt ]; then
sed 's/^/`/;s/$/`/' bench-python-report.txt
else
echo "python benchmark report missing"
fi
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload Python benchmark artifact
if: always()
uses: actions/upload-artifact@v7
with:
name: bench-python-strict-lsp
path: |
bench-python-install.log
bench-python-install-summary.txt
bench-python-report.txt
bench-python-ts.json
bench-python-lsp.json
if-no-files-found: warn
bench-typescript-strict-lsp:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- uses: actions/setup-node@v6
with:
node-version: "20"
- name: Install TypeScript language server
run: |
npm install -g typescript typescript-language-server
typescript-language-server --help >/dev/null
- name: Build release binary
run: cargo build --release --locked -q
- name: Run TypeScript strict-LSP benchmark
timeout-minutes: 12
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
RUNS=1
else
RUNS=2
fi
echo "runs=$RUNS lang=typescript fixture=bench-fixtures/ts-heavy.diff" > bench-typescript-report.txt
timeout 10m scripts/bench-impact-engines.sh \
--diff-file bench-fixtures/ts-heavy.diff \
--runs "$RUNS" \
--direction callers \
--lang typescript \
--min-lsp-changed 6 \
--min-lsp-impacted 15 \
--save-ts-json bench-typescript-ts.json \
--save-lsp-json bench-typescript-lsp.json \
| tee -a bench-typescript-report.txt
- name: Publish TypeScript benchmark summary
if: always()
run: |
{
echo "## TypeScript strict-LSP benchmark"
echo
if [ -f bench-typescript-report.txt ]; then
sed 's/^/`/;s/$/`/' bench-typescript-report.txt
else
echo "typescript benchmark report missing"
fi
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload TypeScript benchmark artifact
if: always()
uses: actions/upload-artifact@v7
with:
name: bench-typescript-strict-lsp
path: |
bench-typescript-report.txt
bench-typescript-ts.json
bench-typescript-lsp.json
if-no-files-found: warn
bench-javascript-strict-lsp:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- uses: actions/setup-node@v6
with:
node-version: "20"
- name: Install JavaScript language server
run: |
npm install -g typescript typescript-language-server
typescript-language-server --help >/dev/null
- name: Build release binary
run: cargo build --release --locked -q
- name: Run JavaScript strict-LSP benchmark
timeout-minutes: 12
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
RUNS=1
else
RUNS=2
fi
echo "runs=$RUNS lang=javascript fixture=bench-fixtures/js-heavy.diff" > bench-javascript-report.txt
timeout 10m scripts/bench-impact-engines.sh \
--diff-file bench-fixtures/js-heavy.diff \
--runs "$RUNS" \
--direction callers \
--lang javascript \
--min-lsp-changed 6 \
--min-lsp-impacted 15 \
--save-ts-json bench-javascript-ts.json \
--save-lsp-json bench-javascript-lsp.json \
| tee -a bench-javascript-report.txt
- name: Publish JavaScript benchmark summary
if: always()
run: |
{
echo "## JavaScript strict-LSP benchmark"
echo
if [ -f bench-javascript-report.txt ]; then
sed 's/^/`/;s/$/`/' bench-javascript-report.txt
else
echo "javascript benchmark report missing"
fi
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload JavaScript benchmark artifact
if: always()
uses: actions/upload-artifact@v7
with:
name: bench-javascript-strict-lsp
path: |
bench-javascript-report.txt
bench-javascript-ts.json
bench-javascript-lsp.json
if-no-files-found: warn
bench-ruby-strict-lsp:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- uses: ruby/setup-ruby@v1
with:
ruby-version: "3.2"
- name: Install ruby-lsp
run: |
gem install ruby-lsp --no-document
ruby-lsp --help >/dev/null
- name: Build release binary
run: cargo build --release --locked -q
- name: Run Ruby strict-LSP benchmark
timeout-minutes: 12
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
RUNS=1
else
RUNS=2
fi
echo "runs=$RUNS lang=ruby fixture=bench-fixtures/ruby-heavy.diff" > bench-ruby-report.txt
timeout 10m scripts/bench-impact-engines.sh \
--diff-file bench-fixtures/ruby-heavy.diff \
--runs "$RUNS" \
--direction callers \
--lang ruby \
--min-lsp-changed 3 \
--min-lsp-impacted 5 \
--save-ts-json bench-ruby-ts.json \
--save-lsp-json bench-ruby-lsp.json \
| tee -a bench-ruby-report.txt
- name: Publish Ruby benchmark summary
if: always()
run: |
{
echo "## Ruby strict-LSP benchmark"
echo
if [ -f bench-ruby-report.txt ]; then
sed 's/^/`/;s/$/`/' bench-ruby-report.txt
else
echo "ruby benchmark report missing"
fi
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload Ruby benchmark artifact
if: always()
uses: actions/upload-artifact@v7
with:
name: bench-ruby-strict-lsp
path: |
bench-ruby-report.txt
bench-ruby-ts.json
bench-ruby-lsp.json
if-no-files-found: warn
bench-guardrail-language-summary:
runs-on: ubuntu-latest
if: always()
needs:
- bench-go-strict-lsp
- bench-java-strict-lsp
- bench-python-strict-lsp
- bench-typescript-strict-lsp
- bench-javascript-strict-lsp
- bench-ruby-strict-lsp
steps:
- name: Download language benchmark artifacts
uses: actions/download-artifact@v8
with:
pattern: bench-*-strict-lsp
path: bench-artifacts
merge-multiple: false
if-no-files-found: warn
- name: Publish language guardrail summary
env:
GO_JOB_RESULT: ${{ needs['bench-go-strict-lsp'].result }}
JAVA_JOB_RESULT: ${{ needs['bench-java-strict-lsp'].result }}
PYTHON_JOB_RESULT: ${{ needs['bench-python-strict-lsp'].result }}
TYPESCRIPT_JOB_RESULT: ${{ needs['bench-typescript-strict-lsp'].result }}
JAVASCRIPT_JOB_RESULT: ${{ needs['bench-javascript-strict-lsp'].result }}
RUBY_JOB_RESULT: ${{ needs['bench-ruby-strict-lsp'].result }}
run: |
set -euo pipefail
fmt_delta() {
local d="$1"
if (( d >= 0 )); then
echo "+$d"
else
echo "$d"
fi
}
extract_metric() {
local report="$1"
local metric="$2"
local line
line=$(grep -E "^\[threshold\] (PASS|FAIL) ${metric} actual=[0-9]+ min=[0-9]+$" "$report" | tail -n 1 || true)
if [[ -z "$line" ]]; then
echo ""
return 0
fi
local state actual minv delta
state=$(echo "$line" | sed -E 's/^\[threshold\] (PASS|FAIL).*/\1/')
actual=$(echo "$line" | sed -E 's/.*actual=([0-9]+).*/\1/')
minv=$(echo "$line" | sed -E 's/.*min=([0-9]+).*/\1/')
delta=$((actual - minv))
echo "$state $actual $minv $delta"
}
extract_note() {
local report="$1"
local note=""
note=$(grep -E "^\[threshold-fail-at-a-glance\]" "$report" | tail -n 1 || true)
if [[ -z "$note" ]]; then
note=$(grep -E "^Error:" "$report" | head -n 1 || true)
fi
if [[ -z "$note" ]]; then
note=$(grep -E "^\[threshold-check\] RESULT=FAIL" "$report" | tail -n 1 || true)
fi
note=${note//|/\\|}
echo "$note"
}
append_row() {
local language="$1"
local job_result="$2"
local report="$3"
local guardrail="UNKNOWN"
local changed_cell="-"
local impacted_cell="-"
local note=""
if [[ ! -f "$report" ]]; then
if [[ "$job_result" == "success" ]]; then
guardrail="MISSING"
else
guardrail="BLOCKED"
fi
note="report missing"
echo "| ${language} | ${job_result} | ${guardrail} | ${changed_cell} | ${impacted_cell} | ${note} |" >> "$SUMMARY_TMP"
return 0
fi
local changed_info impacted_info
changed_info=$(extract_metric "$report" "lsp.changed")
impacted_info=$(extract_metric "$report" "lsp.impacted")
if [[ -n "$changed_info" ]]; then
read -r c_state c_actual c_min c_delta <<<"$changed_info"
changed_cell="${c_actual}/${c_min} (Δ$(fmt_delta "$c_delta"))"
fi
if [[ -n "$impacted_info" ]]; then
read -r i_state i_actual i_min i_delta <<<"$impacted_info"
impacted_cell="${i_actual}/${i_min} (Δ$(fmt_delta "$i_delta"))"
fi
if [[ -n "$changed_info" && -n "$impacted_info" ]]; then
if [[ "$c_state" == "PASS" && "$i_state" == "PASS" ]]; then
guardrail="PASS"
else
guardrail="FAIL"
fi
elif grep -q '^Error:' "$report"; then
guardrail="BLOCKED"
elif grep -q '^\[threshold-check\] RESULT=FAIL' "$report"; then
guardrail="FAIL"
elif [[ "$job_result" != "success" ]]; then
guardrail="BLOCKED"
else
guardrail="NO_METRICS"
fi
note=$(extract_note "$report")
if [[ -z "$note" && "$guardrail" == "NO_METRICS" ]]; then
note="threshold lines not emitted"
fi
echo "| ${language} | ${job_result} | ${guardrail} | ${changed_cell} | ${impacted_cell} | ${note} |" >> "$SUMMARY_TMP"
}
SUMMARY_TMP="$(mktemp)"
{
echo "## Strict-LSP language guardrail summary"
echo
echo "| Language | Job result | Guardrail | changed (actual/min, Δ) | impacted (actual/min, Δ) | Note |"
echo "|---|---|---|---|---|---|"
} > "$SUMMARY_TMP"
append_row "TypeScript" "$TYPESCRIPT_JOB_RESULT" "bench-artifacts/bench-typescript-strict-lsp/bench-typescript-report.txt"
append_row "JavaScript" "$JAVASCRIPT_JOB_RESULT" "bench-artifacts/bench-javascript-strict-lsp/bench-javascript-report.txt"
append_row "Ruby" "$RUBY_JOB_RESULT" "bench-artifacts/bench-ruby-strict-lsp/bench-ruby-report.txt"
append_row "Go" "$GO_JOB_RESULT" "bench-artifacts/bench-go-strict-lsp/bench-go-report.txt"
append_row "Java" "$JAVA_JOB_RESULT" "bench-artifacts/bench-java-strict-lsp/bench-java-report.txt"
append_row "Python" "$PYTHON_JOB_RESULT" "bench-artifacts/bench-python-strict-lsp/bench-python-report.txt"
cat "$SUMMARY_TMP" >> "$GITHUB_STEP_SUMMARY"
cat "$SUMMARY_TMP"