name: Smoke Test
on:
workflow_dispatch:
schedule:
- cron: "0 4 * * 1"
env:
CARGO_TERM_COLOR: always
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
debug-format-repo-scan:
name: Scan Repositories
runs-on: ubuntu-latest
timeout-minutes: 120
permissions:
contents: read
outputs:
failure_count: ${{ steps.summarize.outputs.failure_count }}
env:
TARGET_REPOS: |
HoTT/book
dalcde/cam-notes
latex3/latex2e
latex3/latex3
matze/mtheme
mml-book/mml-book.github.io
pgf-tikz/pgf
posquit0/Awesome-CV
stacks/stacks-project
vEnhance/napkin
ALLOWLIST: |
# Catcode-rebinding bootstrap files: general \catcode handling is an
# explicit non-goal (AGENTS.md decision #1), so these stay format-error.
latex3/latex3|texmf/tex/latex/base/doc.sty|format-error
latex3/latex3|texmf/tex/latex/base/doc-2016-02-15.sty|format-error
latex3/latex3|texmf/tex/latex/base/doc-2021-06-01.sty|format-error
latex3/latex3|texmf/tex/latex/base/source2edoc.cls|format-error
latex3/latex3|texmf/tex/latex/base/shortvrb.sty|format-error
latex3/latex3|texmf/tex/latex/base/ltxdoc.cls|format-error
latex3/latex3|support/docstrip.tex|format-error
latex3/latex3|texmf/tex/latex/base/docstrip.tex|format-error
# latex2e is the upstream source repo for the same doc/docstrip sources
# (doc.sty is generated from doc.dtx). Same non-goal roots: the doc.sty
# variants and doc.dtx carry the `\macro@code` catcode-gymnastics region
# (`|` made the escape, `!` a comment, `{`/`}` catcode 12); docstrip.dtx,
# ltdefns.dtx, and ltcmd.dtx lean on `\iffalse}\fi`/`\iffalse{\fi`
# editor-balance tricks in doc-margin prose (a brace hidden in a skipped
# conditional branch). Issue #71.
latex3/latex2e|base/doc-2016-02-15.sty|format-error
latex3/latex2e|base/doc-2021-06-01.sty|format-error
latex3/latex2e|base/doc.dtx|format-error
latex3/latex2e|base/docstrip.dtx|format-error
latex3/latex2e|base/ltdefns.dtx|format-error
latex3/latex2e|base/ltcmd.dtx|format-error
# source2edoc.cls closes doc.dtx's `oldcomments` from inside its own
# catcode-swapped region (`/gdef/oldc< \end{oldcomments}>`: `/` is the
# escape, `<`/`>` the group delimiters). preload.dtx's `xii}` closes a
# brace opened on a *different* docstrip guard variant, so only running
# docstrip's guard selection balances it. Issue #71.
latex3/latex2e|base/source2edoc.cls|format-error
latex3/latex2e|base/preload.dtx|format-error
# expl3-code.tex rebinds the *structural* catcodes through expl3's own
# API: `\char_set_catcode_group_begin:N \^^@` / `_group_end:N` make
# `^^@` a brace for a region, so the file's real grouping is invisible
# to a `{`/`}`-based reader. Same non-goal as the `\catcode` bootstrap
# files above (AGENTS.md decision #1). Issue #71.
latex3/latex2e|texmf/tex/latex/l3kernel/expl3-code.tex|format-error
# Genuinely broken upstream — one typo each, and badness reads both
# exactly as docstrip and doc.sty do. latex-lab-enumitem.dtx opens a
# `\changes{` on a `%%` line (the second `%` comments the line out,
# stranding the `}` on the next doc line); latex-lab-new-or-2.dtx comments
# out an `\end{plugdecl}` as `% % \end{plugdecl}`. Issue #71.
latex3/latex2e|required/latex-lab/latex-lab-enumitem.dtx|format-error
latex3/latex2e|required/latex-lab/latex-lab-new-or-2.dtx|format-error
latex3/latex3|l3kernel/testfiles-backend/support/driver.tex|format-error
latex3/latex3|texmf/tex/latex/base/nfssfont.tex|format-error
latex3/latex3|xpackages/xinitials/xinitials-test.tex|format-error
latex3/latex3|xpackages/xor/xo-pfloat.tex|format-error
latex3/latex3|xpackages/xor/xo-balance.tex|format-error
# array.dtx hides an `\end{macrocode}` with no opener inside a doc-margin
# `%\iffalse % this is a METACOMMENT !` block: doc.sty skips the whole
# conditional, so the imbalance never executes. Balance hidden in a
# skipped conditional, like docstrip.dtx/ltdefns.dtx above. Issue #71.
latex3/latex2e|required/tools/array.dtx|format-error
# mathcolor.dtx typesets a deliberately *invalid* example in doc prose:
# `\[ \left\{ … \mathcolor{red}{\right\}} \]`, whose own surrounding
# sentence says the construct "is not achievable at all because \left and
# \right form a group". TeX rejects it for exactly the reason badness
# does, so no parser change is correct here. Issue #71.
latex3/latex2e|required/graphics/mathcolor.dtx|format-error
# lscape.dtx wraps its whole manual in a doc-prose `\begin{landscape}`
# whose closer is `\MaybeStop{\end{landscape}}` — doc.sty's finishing-code
# macro (the `\StopEventually` successor) runs its argument at TeX time,
# so the `\end{landscape}` lives inside a macro-argument brace group. The
# brace-group gate rightly reads that `\end` as macro code (like
# `\StopEventually{\end{document}}` in the gate itself), which leaves the
# top-level `\begin` with no reachable closer. Pairing them needs to know
# `\MaybeStop` executes its argument — macro semantics, a TeX engine. The
# symmetric begin-side demotion is out of reach structurally: a top-level
# forward scan cannot tell a macrocode-chunk brace from a real group (why
# `environment_escapes_group` bails at `group_depth == 0`). Issue #71.
latex3/latex2e|required/graphics/lscape.dtx|format-error
# Genuinely broken upstream (a commented-out balancing \end).
latex3/latex3|xpackages/xor/xo-page.dtx|format-error
latex3/latex3|xpackages/galley/galley2.dtx|format-error
# pgfmanual fragments: these are \input chunks of pgfmanual.tex, not
# standalone documents. They lean on the parent preamble
# (pgfmanual-en-macros.tex) that makes `|` an active short-verb char
# (\gdef|{...\verb|...}) and defines the `codeexample` verbatim
# environment. Neither is visible when badness parses a fragment in
# isolation, and resolving external catcode/short-verb state is a
# non-goal (AGENTS.md decision #1), so the unbalanced `{`/environments
# in verbatim spans stay format-error. See issue #63.
pgf-tikz/pgf|doc/generic/pgf/pgfmanual-en-base-arrows.tex|format-error
pgf-tikz/pgf|doc/generic/pgf/pgfmanual-en-base-layers.tex|format-error
pgf-tikz/pgf|doc/generic/pgf/pgfmanual-en-dv-axes.tex|format-error
pgf-tikz/pgf|doc/generic/pgf/pgfmanual-en-gd-display-layer.tex|format-error
pgf-tikz/pgf|doc/generic/pgf/pgfmanual-en-library-external.tex|format-error
pgf-tikz/pgf|doc/generic/pgf/pgfmanual-en-library-matrices.tex|format-error
pgf-tikz/pgf|doc/generic/pgf/pgfmanual-en-library-petri.tex|format-error
pgf-tikz/pgf|doc/generic/pgf/pgfmanual-en-library-profiler.tex|format-error
pgf-tikz/pgf|doc/generic/pgf/pgfmanual-en-library-rdf.tex|format-error
pgf-tikz/pgf|doc/generic/pgf/pgfmanual-en-tikz-coordinates.tex|format-error
pgf-tikz/pgf|doc/generic/pgf/pgfmanual-en-tikz-shapes.tex|format-error
pgf-tikz/pgf|doc/generic/pgf/pgfmanual-en-tutorial-map.tex|format-error
pgf-tikz/pgf|doc/generic/pgf/pgfmanual-en-tutorial-nodes.tex|format-error
# pgf source: catcode-rebinding bootstrap regions (`|`/`/` rebound to
# the escape char with `\` made `other`, `[`/`]`/`%`/`"` reassigned)
# unbalance normal brace/escape lexing; general \catcode handling is a
# non-goal (AGENTS.md decision #1). tikzlibrarygraphs registers active
# `\]`/`\)` as literal data (`\tikz@lib@prepare@active{\]}{...}`), which
# draws a conservative orphan-closer diagnostic. Issue #63.
pgf-tikz/pgf|tex/generic/pgf/basiclayer/pgfcoreshade.code.tex|format-error
pgf-tikz/pgf|tex/generic/pgf/frontendlayer/tikz/libraries/graphs/tikzlibrarygraphs.code.tex|format-error
pgf-tikz/pgf|tex/generic/pgf/frontendlayer/tikz/libraries/tikzexternalshared.code.tex|format-error
pgf-tikz/pgf|tex/generic/pgf/libraries/pgflibraryprofiler.code.tex|format-error
pgf-tikz/pgf|tex/generic/pgf/modules/pgfmoduleplot.code.tex|format-error
pgf-tikz/pgf|tex/generic/pgf/utilities/pgfutil-common.tex|format-error
pgf-tikz/pgf|tex/latex/pgf/doc/pgfmanual-en-macros.tex|format-error
pgf-tikz/pgf|tex/latex/pgf/doc/pgfmanual.prettyprinter.code.tex|format-error
steps:
- uses: actions/checkout@v7
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
- name: Cache Rust artifacts
uses: Swatinem/rust-cache@v2
with:
cache-bin: "false"
prefix-key: "v1-rust"
- name: Build badness binary
run: cargo build --release
- name: Scan repositories for debug-format regressions
id: scan
shell: bash
run: |
set -euo pipefail
BADNESS_BIN="$GITHUB_WORKSPACE/target/release/badness"
BADNESS_SHA="$(git rev-parse HEAD)"
BADNESS_VERSION="$("$BADNESS_BIN" --version | head -n 1 || true)"
RESULTS_DIR="$RUNNER_TEMP/badness-debug-format-scan"
REPOS_DIR="$RESULTS_DIR/repos"
LOGS_DIR="$RESULTS_DIR/logs"
mkdir -p "$REPOS_DIR" "$LOGS_DIR"
FAILURES_TSV="$RESULTS_DIR/failures.tsv"
{
printf 'repo\tfailure_type\tfile\tlog_path\treport_path\trepo_sha\tbadness_sha\tbadness_version\tidempotency_input_path\tidempotency_once_path\tidempotency_twice_path\n'
} > "$FAILURES_TSV"
# Normalize the allowlist into exact `repo|path|type` lines (drop blank
# and `#` comment lines). A recorded failure whose key matches one of
# these is suppressed (see `ALLOWLIST` in the job env, issue #60).
ALLOWLIST_ENTRIES="$(printf '%s\n' "$ALLOWLIST" | sed 's/[[:space:]]*$//' | grep -Ev '^[[:space:]]*(#|$)' || true)"
SUPPRESSED_COUNT=0
is_allowlisted() {
# args: repo file failure_type
grep -Fxq "$1|$2|$3" <<< "$ALLOWLIST_ENTRIES"
}
while IFS= read -r repo; do
if [ -z "$repo" ]; then
continue
fi
repo_dir="$REPOS_DIR/${repo//\//__}"
git clone --depth 1 "https://github.com/${repo}.git" "$repo_dir"
repo_sha="$(git -C "$repo_dir" rev-parse HEAD)"
while IFS= read -r -d '' rel_file; do
# Skip dotfile basenames like a bare `.tex`: git's `*.tex` glob
# matches them, but badness's file discovery rejects a stemless
# dotfile, which would bucket as a bogus `unknown` failure
# (issue #59).
case "$(basename "$rel_file")" in
.*) continue ;;
esac
file_key="$(printf '%s' "${repo}:${rel_file}" | sha256sum | awk '{print $1}')"
log_path="$LOGS_DIR/$file_key.log"
report_path="$LOGS_DIR/$file_key.report.md"
pass_dir="$LOGS_DIR/$file_key.passes"
config_args=()
status=0
(cd "$repo_dir" && timeout 120 "$BADNESS_BIN" debug format --checks all --dump-dir "$pass_dir" "$rel_file") >"$log_path" 2>&1 || status=$?
if [ "$status" -ne 0 ]; then
# A broken badness.toml in the target repo aborts every file
# before any checks run (config errors always name the config
# path). That is the target repo's breakage, not a badness
# regression, so retry against built-in defaults; only failures
# that persist without the repo's config are worth reporting.
if grep -Fq 'badness.toml' "$log_path"; then
config_args=(--no-config)
retry_status=0
(cd "$repo_dir" && timeout 120 "$BADNESS_BIN" debug format --checks all "${config_args[@]}" --dump-dir "$pass_dir" "$rel_file") >"$log_path" 2>&1 || retry_status=$?
if [ "$retry_status" -eq 0 ]; then
echo "notice: $repo has an invalid badness config; $rel_file passed with --no-config"
continue
fi
status=$retry_status
fi
# Generate human-friendly markdown report for issue snippets.
(cd "$repo_dir" && timeout 120 "$BADNESS_BIN" debug format --checks all "${config_args[@]}" --report "$rel_file") >"$report_path" 2>&1 || true
matched=0
failure_types=""
idempotency_input_rel=""
idempotency_once_rel=""
idempotency_twice_rel=""
safe_rel_file="$(printf '%s' "$rel_file" | sed 's/[^[:alnum:]._-]/_/g')"
if [ -f "$pass_dir/$safe_rel_file.idempotency.input.txt" ]; then
idempotency_input_rel="logs/$file_key.passes/$safe_rel_file.idempotency.input.txt"
fi
if [ -f "$pass_dir/$safe_rel_file.idempotency.once.txt" ]; then
idempotency_once_rel="logs/$file_key.passes/$safe_rel_file.idempotency.once.txt"
fi
if [ -f "$pass_dir/$safe_rel_file.idempotency.twice.txt" ]; then
idempotency_twice_rel="logs/$file_key.passes/$safe_rel_file.idempotency.twice.txt"
fi
add_failure_type() {
local candidate="$1"
if ! grep -Fxq "$candidate" <<< "$failure_types"; then
if [ -z "$failure_types" ]; then
failure_types="$candidate"
else
failure_types+=$'\n'"$candidate"
fi
fi
}
if [ "$status" -eq 124 ]; then
add_failure_type "timeout"
fi
if grep -Eiq 'idempot' "$log_path" "$report_path" 2>/dev/null; then
add_failure_type "idempotency"
fi
if grep -Eiq 'lossless' "$log_path" "$report_path" 2>/dev/null; then
add_failure_type "losslessness"
fi
if grep -Eiq 'format-error' "$log_path" "$report_path" 2>/dev/null; then
add_failure_type "format-error"
fi
if [ -f "$report_path" ]; then
while IFS= read -r parsed_type; do
[ -n "$parsed_type" ] && add_failure_type "$parsed_type"
done < <(grep -Eo '\((idempotency|losslessness|format-error)\)' "$report_path" | tr -d '()' | sort -u)
fi
if [ -n "$failure_types" ]; then
matched=1
while IFS= read -r failure_type; do
[ -z "$failure_type" ] && continue
if is_allowlisted "$repo" "$rel_file" "$failure_type"; then
echo "notice: allowlisted $failure_type for $repo:$rel_file (issue #60, record-only)"
SUPPRESSED_COUNT=$((SUPPRESSED_COUNT + 1))
continue
fi
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
"$repo" \
"$failure_type" \
"$rel_file" \
"logs/$file_key.log" \
"logs/$file_key.report.md" \
"$repo_sha" \
"$BADNESS_SHA" \
"$BADNESS_VERSION" \
"$idempotency_input_rel" \
"$idempotency_once_rel" \
"$idempotency_twice_rel" \
>> "$FAILURES_TSV"
done <<< "$failure_types"
fi
if [ "$matched" -eq 0 ]; then
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
"$repo" \
"unknown" \
"$rel_file" \
"logs/$file_key.log" \
"logs/$file_key.report.md" \
"$repo_sha" \
"$BADNESS_SHA" \
"$BADNESS_VERSION" \
"$idempotency_input_rel" \
"$idempotency_once_rel" \
"$idempotency_twice_rel" \
>> "$FAILURES_TSV"
fi
fi
done < <(git -C "$repo_dir" ls-files -z -- '*.tex' '*.sty' '*.cls' '*.dtx' '*.ins' '*.bib')
done <<< "$TARGET_REPOS"
# Hand the suppressed-record tally to the summarize step (separate
# shells share only the results dir, which is also the uploaded
# artifact).
echo "$SUPPRESSED_COUNT" > "$RESULTS_DIR/suppressed_count"
- name: Summarize scan results
id: summarize
shell: bash
run: |
set -euo pipefail
RESULTS_DIR="$RUNNER_TEMP/badness-debug-format-scan"
FAILURES_TSV="$RESULTS_DIR/failures.tsv"
failure_count="$(($(wc -l < "$FAILURES_TSV") - 1))"
if [ "$failure_count" -lt 0 ]; then
failure_count=0
fi
suppressed_count=0
if [ -f "$RESULTS_DIR/suppressed_count" ]; then
suppressed_count="$(cat "$RESULTS_DIR/suppressed_count")"
fi
echo "failure_count=$failure_count" >> "$GITHUB_OUTPUT"
{
echo "### Badness debug format repo scan"
echo ""
echo "- Failure records: $failure_count"
echo "- Suppressed (allowlisted, record-only): $suppressed_count"
echo "- Scanned extensions: \`.tex\`, \`.sty\`, \`.cls\`, \`.dtx\`, \`.ins\`, \`.bib\`"
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload scan results artifact
uses: actions/upload-artifact@v7
with:
name: debug-format-repo-scan-results
path: ${{ runner.temp }}/badness-debug-format-scan
if-no-files-found: error
debug-format-repo-scan-issues:
name: Create Regression Issues
needs: debug-format-repo-scan
runs-on: ubuntu-latest
permissions:
issues: write
contents: read
steps:
- name: Download scan results artifact
uses: actions/download-artifact@v8
with:
name: debug-format-repo-scan-results
path: ${{ runner.temp }}/badness-debug-format-scan
- name: Create or update regression issues
if: needs.debug-format-repo-scan.outputs.failure_count != '0'
env:
GH_TOKEN: ${{ github.token }}
shell: bash
run: |
set -euo pipefail
RESULTS_DIR="$RUNNER_TEMP/badness-debug-format-scan"
FAILURES_TSV="$RESULTS_DIR/failures.tsv"
RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
existing_labels="$(gh label list --repo "$GITHUB_REPOSITORY" --limit 500 --json name --jq '.[].name')"
has_label() {
local target="$1"
grep -Fxq "$target" <<< "$existing_labels"
}
ensure_label() {
local name="$1"
local color="$2"
local description="$3"
if has_label "$name"; then
return 0
fi
if gh label create "$name" --repo "$GITHUB_REPOSITORY" --color "$color" --description "$description" >/dev/null 2>&1; then
existing_labels+=$'\n'"$name"
else
echo "warning: could not create label '$name'; issues will be created without that label"
fi
}
tail -n +2 "$FAILURES_TSV" | awk -F '\t' '{print $1 "\t" $2}' | sort -u | while IFS=$'\t' read -r repo failure_type; do
if [ -z "$repo" ] || [ -z "$failure_type" ]; then
continue
fi
marker="badness-debug-format-key:repo=${repo};type=${failure_type}"
title="CI: debug-format regression in ${repo} (${failure_type})"
count="$(awk -F '\t' -v r="$repo" -v t="$failure_type" '$1 == r && $2 == t {c++} END {print c+0}' "$FAILURES_TSV")"
samples="$(awk -F '\t' -v r="$repo" -v t="$failure_type" '$1 == r && $2 == t {printf "- `%s`\n", $3; c++; if (c == 10) exit}' "$FAILURES_TSV")"
sample_file="$(awk -F '\t' -v r="$repo" -v t="$failure_type" '$1 == r && $2 == t {print $3; exit}' "$FAILURES_TSV")"
sample_log_rel="$(awk -F '\t' -v r="$repo" -v t="$failure_type" '$1 == r && $2 == t {print $4; exit}' "$FAILURES_TSV")"
sample_report_rel="$(awk -F '\t' -v r="$repo" -v t="$failure_type" '$1 == r && $2 == t {print $5; exit}' "$FAILURES_TSV")"
sample_repo_sha="$(awk -F '\t' -v r="$repo" -v t="$failure_type" '$1 == r && $2 == t {print $6; exit}' "$FAILURES_TSV")"
sample_badness_sha="$(awk -F '\t' -v r="$repo" -v t="$failure_type" '$1 == r && $2 == t {print $7; exit}' "$FAILURES_TSV")"
sample_badness_version="$(awk -F '\t' -v r="$repo" -v t="$failure_type" '$1 == r && $2 == t {print $8; exit}' "$FAILURES_TSV")"
sample_idempotency_input_rel="$(awk -F '\t' -v r="$repo" -v t="$failure_type" '$1 == r && $2 == t {print $9; exit}' "$FAILURES_TSV")"
sample_idempotency_once_rel="$(awk -F '\t' -v r="$repo" -v t="$failure_type" '$1 == r && $2 == t {print $10; exit}' "$FAILURES_TSV")"
sample_idempotency_twice_rel="$(awk -F '\t' -v r="$repo" -v t="$failure_type" '$1 == r && $2 == t {print $11; exit}' "$FAILURES_TSV")"
if [ -n "$sample_report_rel" ]; then
sample_report_path="$RESULTS_DIR/$sample_report_rel"
else
sample_report_path=""
fi
report_excerpt=""
sample_diff_line=""
if [ -n "$sample_report_path" ] && [ -f "$sample_report_path" ]; then
report_excerpt="$(sed -n '1,120p' "$sample_report_path")"
sample_diff_line="$(grep -Eo 'Approx\. diff start line: [0-9]+' "$sample_report_path" | head -n 1 | awk '{print $NF}' || true)"
fi
ISSUE_TITLE="$title"
export ISSUE_TITLE
issue_number="$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --limit 200 --json number,title --jq '.[] | select(.title == env.ISSUE_TITLE) | .number' | head -n 1)"
if [ -n "$issue_number" ]; then
comment_file="$RUNNER_TEMP/badness-issue-comment.md"
{
echo "Regression still reproduces for \`$repo\` (\`$failure_type\`)."
echo ""
echo "- New failure records: $count"
echo "- Workflow run: $RUN_URL"
echo "- Badness command: \`badness debug format --checks all --report <FILE>\`"
if [ -n "$sample_repo_sha" ]; then
echo "- Target repository commit: \`$sample_repo_sha\`"
fi
if [ -n "$sample_badness_version" ]; then
echo "- Badness version used in scan: \`$sample_badness_version\`"
fi
if [ -n "$sample_badness_sha" ]; then
echo "- Badness commit used in scan: \`$sample_badness_sha\`"
fi
if [ -n "$sample_log_rel" ]; then
echo "- Sample log path: \`$sample_log_rel\`"
fi
if [ -n "$sample_report_rel" ]; then
echo "- Sample report path: \`$sample_report_rel\`"
fi
if [ -n "$sample_idempotency_input_rel" ]; then
echo "- Sample idempotency input artifact: \`$sample_idempotency_input_rel\`"
fi
if [ -n "$sample_idempotency_once_rel" ]; then
echo "- Sample idempotency pass1 artifact: \`$sample_idempotency_once_rel\`"
fi
if [ -n "$sample_idempotency_twice_rel" ]; then
echo "- Sample idempotency pass2 artifact: \`$sample_idempotency_twice_rel\`"
fi
if [ -n "$sample_diff_line" ]; then
echo "- Approx. diff start line: $sample_diff_line"
fi
echo ""
echo "Sample files:"
echo "$samples"
if [ -n "$sample_file" ] && [ -n "$sample_repo_sha" ]; then
echo ""
echo "Reproduce locally:"
echo ""
echo '```bash'
echo "git clone https://github.com/$repo.git"
echo "cd $(basename "$repo")"
echo "git checkout $sample_repo_sha"
echo "badness debug format --checks all --report \"$sample_file\""
echo '```'
fi
if [ -n "$report_excerpt" ]; then
echo ""
echo "<details>"
echo "<summary>Sample report excerpt</summary>"
echo ""
echo '````text'
printf '%s\n' "$report_excerpt"
echo '````'
echo "</details>"
fi
} > "$comment_file"
gh issue comment "$issue_number" --repo "$GITHUB_REPOSITORY" --body-file "$comment_file"
else
body_file="$RUNNER_TEMP/badness-issue-body.md"
{
echo "<!-- $marker -->"
echo "# Debug-format regression detected"
echo ""
echo "- Target repository: \`$repo\`"
echo "- Failure type: \`$failure_type\`"
echo "- Failure records in this run: $count"
echo "- Workflow run: $RUN_URL"
echo "- Badness command: \`badness debug format --checks all --report <FILE>\`"
if [ -n "$sample_repo_sha" ]; then
echo "- Target repository commit: \`$sample_repo_sha\`"
fi
if [ -n "$sample_badness_version" ]; then
echo "- Badness version used in scan: \`$sample_badness_version\`"
fi
if [ -n "$sample_badness_sha" ]; then
echo "- Badness commit used in scan: \`$sample_badness_sha\`"
fi
if [ -n "$sample_file" ]; then
echo "- Sample file: \`$sample_file\`"
fi
if [ -n "$sample_log_rel" ]; then
echo "- Sample log path: \`$sample_log_rel\`"
fi
if [ -n "$sample_report_rel" ]; then
echo "- Sample report path: \`$sample_report_rel\`"
fi
if [ -n "$sample_idempotency_input_rel" ]; then
echo "- Sample idempotency input artifact: \`$sample_idempotency_input_rel\`"
fi
if [ -n "$sample_idempotency_once_rel" ]; then
echo "- Sample idempotency pass1 artifact: \`$sample_idempotency_once_rel\`"
fi
if [ -n "$sample_idempotency_twice_rel" ]; then
echo "- Sample idempotency pass2 artifact: \`$sample_idempotency_twice_rel\`"
fi
if [ -n "$sample_diff_line" ]; then
echo "- Approx. diff start line: $sample_diff_line"
fi
echo ""
echo "Sample files:"
echo "$samples"
if [ -n "$sample_file" ] && [ -n "$sample_repo_sha" ]; then
echo ""
echo "Reproduce locally:"
echo ""
echo '```bash'
echo "git clone https://github.com/$repo.git"
echo "cd $(basename "$repo")"
echo "git checkout $sample_repo_sha"
echo "badness debug format --checks all --report \"$sample_file\""
echo '```'
fi
if [ -n "$report_excerpt" ]; then
echo ""
echo "<details>"
echo "<summary>Sample report excerpt</summary>"
echo ""
echo '````text'
printf '%s\n' "$report_excerpt"
echo '````'
echo "</details>"
fi
echo ""
echo "See artifact \`debug-format-repo-scan-results\` for full logs."
} > "$body_file"
ensure_label "ci" "0366D6" "Continuous integration"
ensure_label "debug-format" "5319E7" "Badness debug format scan regressions"
case "$failure_type" in
idempotency) failure_color="B60205" ;;
losslessness) failure_color="D93F0B" ;;
format-error) failure_color="FBCA04" ;;
timeout) failure_color="1D76DB" ;;
unknown) failure_color="6E7781" ;;
*) failure_color="6E7781" ;;
esac
ensure_label "$failure_type" "$failure_color" "Regression category from debug format scan"
label_args=()
for label in "ci" "debug-format" "$failure_type"; do
if has_label "$label"; then
label_args+=(--label "$label")
fi
done
gh issue create \
--repo "$GITHUB_REPOSITORY" \
--title "$title" \
"${label_args[@]}" \
--body-file "$body_file"
fi
done
- name: Comment on issues that no longer reproduce
env:
GH_TOKEN: ${{ github.token }}
shell: bash
run: |
set -euo pipefail
RESULTS_DIR="$RUNNER_TEMP/badness-debug-format-scan"
FAILURES_TSV="$RESULTS_DIR/failures.tsv"
RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
# Build the set of (repo, failure_type) pairs that failed in this run.
current_keys="$RUNNER_TEMP/badness-current-keys.txt"
: > "$current_keys"
if [ -f "$FAILURES_TSV" ]; then
tail -n +2 "$FAILURES_TSV" \
| awk -F '\t' 'NF >= 2 && $1 != "" && $2 != "" {printf "badness-debug-format-key:repo=%s;type=%s\n", $1, $2}' \
| sort -u > "$current_keys"
fi
# List open issues that carry our marker, then check each one.
open_issues_json="$(gh issue list \
--repo "$GITHUB_REPOSITORY" \
--state open \
--search 'in:body "badness-debug-format-key:"' \
--limit 500 \
--json number,body)"
echo "$open_issues_json" | jq -c '.[]' | while IFS= read -r issue; do
number="$(jq -r '.number' <<< "$issue")"
body="$(jq -r '.body' <<< "$issue")"
marker="$(grep -Eo 'badness-debug-format-key:repo=[^;]+;type=[^[:space:]<>]+' <<< "$body" | head -n 1 || true)"
if [ -z "$marker" ]; then
continue
fi
if grep -Fxq "$marker" "$current_keys"; then
# Still reproducing — handled by the create/update step above.
continue
fi
# Avoid spamming: skip if the latest comment already reports a green run.
latest_comment="$(gh issue view "$number" --repo "$GITHUB_REPOSITORY" --json comments \
--jq '.comments | last | .body // ""')"
if grep -Fq 'No longer reproducing' <<< "$latest_comment"; then
continue
fi
comment_file="$RUNNER_TEMP/badness-issue-green-comment.md"
{
echo "No longer reproducing in the latest scan."
echo ""
echo "- Marker: \`$marker\`"
echo "- Workflow run: $RUN_URL"
echo ""
echo "If this stays green over the next few weekly scans, this issue can be closed."
} > "$comment_file"
gh issue comment "$number" --repo "$GITHUB_REPOSITORY" --body-file "$comment_file"
done
- name: Fail workflow when regressions are detected
if: needs.debug-format-repo-scan.outputs.failure_count != '0'
shell: bash
run: |
echo "Debug format regressions detected: ${{ needs.debug-format-repo-scan.outputs.failure_count }}"
exit 1