name: cargo-echoes
description: Find duplicated Rust doctests, functions and code blocks, and review the copies a pull request adds
branding:
icon: 'copy'
color: 'orange'
inputs:
threshold:
description: Report duplicate groups with at least this many sites
required: false
default: '2'
min-tokens:
description: Skip doctest bodies with fewer tokens than this
required: false
default: '0'
functions:
description: Also check for duplicated functions
required: false
default: 'false'
fn-min-tokens:
description: Skip functions whose body counts fewer tokens than this, a path or an operator counting as one
required: false
default: ''
context-blocks:
description: Also check for duplicated context blocks
required: false
default: 'false'
context-min-tokens:
description: Skip context blocks with fewer tokens than this
required: false
default: ''
context-select:
description: Select the local contexts to check, empty keeps the whole workspace
required: false
default: ''
context-search:
description: Restrict the comparison corpus to one context scope
required: false
default: ''
context-dependency-depth:
description: Search dependency sources to this shortest-path depth, zero stays local
required: false
default: '0'
context-dev-dependencies:
description: Include development dependency edges
required: false
default: 'false'
context-build-dependencies:
description: Include build dependency edges
required: false
default: 'false'
context-all-optional-dependencies:
description: Explore optional dependency paths outside the selected feature configuration
required: false
default: 'false'
context-features:
description: Cargo features enabled for dependency discovery
required: false
default: ''
context-all-features:
description: Enable all features for dependency discovery
required: false
default: 'false'
context-no-default-features:
description: Disable default features for dependency discovery
required: false
default: 'false'
context-target:
description: Target triple used for dependency discovery
required: false
default: ''
context-cache-dir:
description: Persistent dependency index directory, empty for a run-local directory
required: false
default: ''
context-check-latest:
description: Check the latest published release of indexed dependencies
required: false
default: 'true'
context-fail-on-reuse:
description: Make dependency reuse findings fail the duplicate gate
required: false
default: 'false'
all-targets:
description: Also scan bin and example targets
required: false
default: 'false'
package:
description: Restrict the scan to one workspace member by name
required: false
default: ''
version:
description: The echoes version to install, empty for the latest
required: false
default: ''
install:
description: Install echoes, false uses the cargo echoes already on the path
required: false
default: 'true'
cwd:
description: The workspace directory to scan
required: false
default: ''
pr-number:
description: Post the findings as a pull request review on this PR instead of failing the job
required: false
default: ''
dry-run:
description: Render the review instead of posting it
required: false
default: 'false'
only-new:
description: In pull request mode report only duplicates this pull request introduces
required: false
default: 'true'
annotations:
description: Annotate each copy to remove, which needs no token
required: false
default: 'true'
mode:
description: "review posts the findings in one job, while prepare and post split it so a workflow_run job can review a fork's pull request"
required: false
default: 'review'
payload:
description: Directory holding the prepared review, written by prepare and read by post
required: false
default: ''
max-comments:
description: Inline comments posted per run, the omitted copies go to the review body and the run summary
required: false
default: '50'
organizations:
description: Scan the crates of these GitHub organizations or users instead of the workspace, one per line
required: false
default: ''
repositories:
description: Scan exactly these OWNER/REPO repositories instead of the workspace, one per line
required: false
default: ''
runs:
using: 'composite'
steps:
- name: Check the repository selection
if: ${{ inputs.organizations != '' || inputs.repositories != '' }}
shell: bash
env:
ORGANIZATIONS: ${{ inputs.organizations }}
REPOSITORIES: ${{ inputs.repositories }}
PR_NUMBER: ${{ inputs.pr-number }}
DRY_RUN: ${{ inputs.dry-run }}
MODE: ${{ inputs.mode }}
run: |
if [ -n "${ORGANIZATIONS//[[:space:]]/}" ] && [ -n "${REPOSITORIES//[[:space:]]/}" ]; then
echo "::error title=echoes::organizations and repositories are alternatives, set one"
exit 2
fi
if [ -n "$PR_NUMBER" ] || [ "$DRY_RUN" = "true" ] || [ "$MODE" != "review" ]; then
echo "::error title=echoes::organizations and repositories only run as a gate, without pr-number, dry-run or mode"
exit 2
fi
- name: Install echoes
if: ${{ inputs.mode != 'post' && inputs.install == 'true' }}
shell: bash
env:
GH_TOKEN: ${{ github.token }}
VERSION: ${{ inputs.version }}
run: |
set -euo pipefail
repo="LucaCappelletti94/cargo-echoes"
tag=''
[ -n "$VERSION" ] && tag="v$VERSION"
# Keep these targets in sync with the release matrix in .github/workflows/release.yml.
case "$RUNNER_OS/$RUNNER_ARCH" in
Linux/X64) target=x86_64-unknown-linux-gnu ;;
Linux/ARM64) target=aarch64-unknown-linux-gnu ;;
macOS/X64) target=x86_64-apple-darwin ;;
macOS/ARM64) target=aarch64-apple-darwin ;;
Windows/X64) target=x86_64-pc-windows-msvc ;;
*) target='' ;;
esac
if [ -n "$target" ]; then
asset="cargo-echoes-$target.tar.gz"
dest="$RUNNER_TEMP/echoes-bin"
mkdir -p "$dest"
# Action series releases carry no binaries and GitHub marks the newest one Latest,
# so resolve the newest release that actually ships the asset instead of trusting Latest.
if [ -z "$tag" ]; then
tag="$(gh api "repos/$repo/releases?per_page=30" --paginate \
--jq ".[] | select(.draft == false and .prerelease == false) | select([.assets[].name] | index(\"$asset\")) | .tag_name" 2>/dev/null | head -n1)" || true
fi
args=(--repo "$repo" --pattern "$asset" --dir "$dest" --clobber)
[ -n "$tag" ] && args=("$tag" "${args[@]}")
if err="$(gh release download "${args[@]}" 2>&1)"; then
bin=cargo-echoes
[ "$RUNNER_OS" = Windows ] && bin=cargo-echoes.exe
tar -xzf "$dest/$asset" -C "$dest"
chmod +x "$dest/$bin" 2>/dev/null || true
echo "$dest" >> "$GITHUB_PATH"
echo "echoes: installed the prebuilt $asset from $tag"
exit 0
fi
echo "echoes: could not download $asset, falling back to source"
printf '%s\n' "$err" | sed 's/^/echoes: /'
else
echo "echoes: no prebuilt binary for $RUNNER_OS/$RUNNER_ARCH, falling back to source"
fi
install=(echoes)
[ -n "$tag" ] && install+=(--version "$VERSION")
cargo install "${install[@]}"
- name: Resolve the dependency index directory
id: cache-dir
if: ${{ inputs.mode != 'post' && inputs.context-dependency-depth > 0 }}
shell: bash
env:
CONTEXT_CACHE_DIR: ${{ inputs.context-cache-dir }}
run: |
dir="${CONTEXT_CACHE_DIR:-$RUNNER_TEMP/echoes-context-index}"
mkdir -p "$dir/public"
echo "dir=$dir" >> "$GITHUB_OUTPUT"
echo "echoes-context-index-dir=$dir" >> "$GITHUB_ENV"
- name: Restore the dependency index
if: ${{ inputs.mode != 'post' && inputs.context-dependency-depth > 0 }}
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830
with:
path: ${{ env.echoes-context-index-dir }}/public
key: echoes-context-index-${{ runner.os }}-${{ inputs.context-target != '' && inputs.context-target || 'any' }}-v1
restore-keys: echoes-context-index-${{ runner.os }}-
- name: Scan for duplicates
id: scan
if: ${{ inputs.mode != 'post' }}
shell: bash
working-directory: ${{ inputs.cwd }}
env:
ALL_TARGETS: ${{ inputs.all-targets }}
PACKAGE: ${{ inputs.package }}
PR_NUMBER: ${{ inputs.pr-number }}
DRY_RUN: ${{ inputs.dry-run }}
THRESHOLD: ${{ inputs.threshold }}
MIN_TOKENS: ${{ inputs.min-tokens }}
FUNCTIONS: ${{ inputs.functions }}
FN_MIN_TOKENS: ${{ inputs.fn-min-tokens }}
ANNOTATIONS: ${{ inputs.annotations }}
CONTEXT_BLOCKS: ${{ inputs.context-blocks }}
CONTEXT_MIN_TOKENS: ${{ inputs.context-min-tokens }}
CONTEXT_SELECT: ${{ inputs.context-select }}
CONTEXT_SEARCH: ${{ inputs.context-search }}
CONTEXT_DEPTH: ${{ inputs.context-dependency-depth }}
CONTEXT_DEV: ${{ inputs.context-dev-dependencies }}
CONTEXT_BUILD: ${{ inputs.context-build-dependencies }}
CONTEXT_ALL_OPT: ${{ inputs.context-all-optional-dependencies }}
CONTEXT_FEATURES: ${{ inputs.context-features }}
CONTEXT_ALL_FEATURES: ${{ inputs.context-all-features }}
CONTEXT_NO_DEFAULT: ${{ inputs.context-no-default-features }}
CONTEXT_TARGET: ${{ inputs.context-target }}
CONTEXT_CACHE_DIR: ${{ env.echoes-context-index-dir }}
CONTEXT_CHECK_LATEST: ${{ inputs.context-check-latest }}
CONTEXT_FAIL: ${{ inputs.context-fail-on-reuse }}
ORGANIZATIONS: ${{ inputs.organizations }}
REPOSITORIES: ${{ inputs.repositories }}
WORKFLOW_TOKEN: ${{ github.token }}
run: |
# The runner drops step summaries over 1 MiB, so a step keeps its
# summary writes within a safe total, the truncated tail gets a note.
sum_append() {
local text="$1" file="$GITHUB_STEP_SUMMARY"
local cap=$((900 * 1024)) size=0 room len
[ -f "$file" ] && size="$(wc -c < "$file")"
room=$((cap - size))
len="$(printf '%s' "$text" | wc -c)"
if [ "$room" -gt 200 ] && [ "$len" -le "$((room - 200))" ]; then
printf '%s\n' "$text" >> "$file"
elif [ "$room" -gt 200 ]; then
head -c "$((room - 200))" <<< "$text" >> "$file"
printf '\nechoes: the run summary is full, the rest is truncated\n' >> "$file"
fi
}
selection=()
while read -r name; do
[ -n "$name" ] && selection+=(--organization "$name")
done <<< "$ORGANIZATIONS"
while read -r name; do
[ -n "$name" ] && selection+=(--repository "$name")
done <<< "$REPOSITORIES"
prefix=''
if [ "${#selection[@]}" -gt 0 ]; then
# A job token reaches private repositories, the workflow token the public ones.
export GH_TOKEN="${GH_TOKEN:-$WORKFLOW_TOKEN}"
else
# Report paths are relative to the workspace root cargo resolves, the cwd or one of
# its ancestors. GitHub reads, matches and links paths from the repository root, so
# one prefix from that root projects every report path for the later steps.
root="$(cargo metadata --no-deps --format-version 1 2>/dev/null | jq -r '.workspace_root')" || root=''
if [ -n "$root" ] && [ -n "${GITHUB_WORKSPACE:-}" ]; then
prefix="$(realpath --relative-to="$GITHUB_WORKSPACE" "$root" 2>/dev/null)" || prefix=''
[ "$prefix" = "." ] && prefix=''
fi
fi
printf '%s' "$prefix" > "$RUNNER_TEMP/echoes-prefix"
printf '%s\n' 'if $prefix == "" then . else ((.groups[]?.sites[]?, .context_groups[]?.sites[]?) |= (.file = "\($prefix)/\(.file)")) | (if .reuse then (.reuse.groups[]?.local |= (.file = "\($prefix)/\(.file)")) else . end) end' \
> "$RUNNER_TEMP/echoes-root.jq"
set --
[ "$ALL_TARGETS" = "true" ] && set -- --all-targets "$@"
[ -n "$PACKAGE" ] && set -- --package "$PACKAGE" "$@"
[ "$FUNCTIONS" = "true" ] && set -- --functions "$@"
[ -n "$FN_MIN_TOKENS" ] && set -- --fn-min-tokens "$FN_MIN_TOKENS" "$@"
[ "$CONTEXT_BLOCKS" = "true" ] && set -- --context-blocks "$@"
[ -n "$CONTEXT_MIN_TOKENS" ] && set -- --context-min-tokens "$CONTEXT_MIN_TOKENS" "$@"
[ -n "$CONTEXT_SELECT" ] && set -- --context-select "$CONTEXT_SELECT" "$@"
[ -n "$CONTEXT_SEARCH" ] && set -- --context-search "$CONTEXT_SEARCH" "$@"
if [ "$CONTEXT_DEPTH" != "0" ]; then
set -- --context-dependency-depth "$CONTEXT_DEPTH" "$@"
[ "$CONTEXT_DEV" = "true" ] && set -- --context-dev-dependencies "$@"
[ "$CONTEXT_BUILD" = "true" ] && set -- --context-build-dependencies "$@"
[ "$CONTEXT_ALL_OPT" = "true" ] && set -- --context-all-optional-dependencies "$@"
[ -n "$CONTEXT_FEATURES" ] && set -- --context-features "$CONTEXT_FEATURES" "$@"
[ "$CONTEXT_ALL_FEATURES" = "true" ] && set -- --context-all-features "$@"
[ "$CONTEXT_NO_DEFAULT" = "true" ] && set -- --context-no-default-features "$@"
[ -n "$CONTEXT_TARGET" ] && set -- --context-target "$CONTEXT_TARGET" "$@"
[ -n "$CONTEXT_CACHE_DIR" ] && set -- --context-cache-dir "$CONTEXT_CACHE_DIR" "$@"
[ "$CONTEXT_CHECK_LATEST" = "false" ] && set -- --context-check-latest false "$@"
[ "$CONTEXT_FAIL" = "true" ] && set -- --context-fail-on-reuse "$@"
fi
set -- "$@" "${selection[@]}"
if [ -n "$PR_NUMBER" ] || [ "$DRY_RUN" = "true" ]; then
report="$RUNNER_TEMP/echoes-report.json"
status=0
cargo echoes \
--threshold "$THRESHOLD" \
--min-tokens "$MIN_TOKENS" \
--no-fail --json "$@" > "$report.raw" || status=$?
if [ ! -s "$report.raw" ]; then
echo "echoes: the scan produced no report"
exit 2
fi
jq --arg prefix "$prefix" -f "$RUNNER_TEMP/echoes-root.jq" "$report.raw" > "$report"
rm -f "$report.raw"
jq -r '. as $r | "\($r.total) doctests, \($r.unique) unique, \($r.functions // 0) functions, \($r.unique_functions // 0) unique functions, \($r.groups | length) duplicated groups"' "$report"
# The whole scan in the run summary, which needs no token, so a
# fork pull request has it whatever the review can do.
sum_append "$(
jq -r '"### echoes\n\n\(.total) doctests, \(.unique) unique, \(.functions // 0) functions, \(.unique_functions // 0) unique functions, \(.groups | length) duplicated groups"' "$report"
jq -r '
.groups[] | "\n- group `\(.id)`, \(.kind // "doctest"), \(.sites | length) copies\n"
+ (.sites | map(" - `" + .file + ":" + (.line | tostring) + "` " + .item)
| join("\n"))' "$report"
)"
# Context groups and dependency reuse, grouped by local context with
# the accessible APIs first, in the same public report.
context_summary="$(jq -r '
def sp: .file;
def site_md: " - `" + sp + ":" + (.line | tostring) + "` " + .item;
def cand_md:
" - `\(.package) \(.version)`"
+ (if .depth > 0 then " (depth \(.depth))" else "" end)
+ (if (.conditions | length) > 0 then " (needs \(.conditions | join(", ")))" else "" end)
+ (if .action == "reuse-api" then ": the public item `\(.api_path // .site.item)` can be reused"
elif .action == "upgrade-api" then ": the public item `\(.api_path // .site.item)` is available in a newer published release"
elif .action == "request-extraction" then ": the internal item `\(.site.item)` can be requested for extraction"
elif .action == "unresolved" then ": source analysis does not establish accessibility"
else ": the reuse path is unresolved" end)
+ (if .source_url
then (if .api_path and .api_path != .site.item then " [source (\(.site.item))](\(.source_url))" else " [source](\(.source_url))" end)
else "" end)
+ (if .existing_issue then " [existing request \(.existing_issue.state)](\(.existing_issue.url))"
elif .issue_url then " [request extraction](\(.issue_url))"
elif .action == "request-extraction" then " (the upstream repository is not declared, so no issue link is offered)"
else "" end)
+ (if .lookup_warning then "\n lookup: \(.lookup_warning)" else "" end);
(if (.context_groups | length) > 0
then "### Context groups\n\n"
+ ([ .context_groups[]
| "- group `\(.id)`, \(.sites | length) copies\n"
+ ([ .sites[] | site_md ] | join("\n"))
] | join("\n"))
else "" end)
+ (if ((.reuse // {}) | (.groups // []) | length) > 0
then "### Dependency reuse\n\n"
+ ([ (.reuse.groups // [])[]
| "- local `" + (.local | sp) + ":" + (.local.line | tostring) + "` " + .local.item + "\n"
+ ([ .candidates[] | cand_md ] | join("\n"))
] | join("\n"))
+ "Dependency coverage: \(.reuse.coverage.packages) packages, \(.reuse.coverage.indexed_files) files indexed, \(.reuse.coverage.cached_files) from cache"
+ (if .reuse.coverage.complete then "" else "\n\nThe requested dependency analysis is incomplete, the run fails\n" + ([ .reuse.coverage.failures[] | "- " + . ] | join("\n")) end)
+ (if (.reuse.coverage.warnings | length) > 0 then "\n" + ([ .reuse.coverage.warnings[] | "- " + . ] | join("\n")) else "" end)
else "" end)' "$report")"
[ -n "$context_summary" ] && sum_append "$context_summary"
if [ "$(jq '(.groups | length) + (.context_groups | length) + ((.reuse // {}).groups // [] | length)' "$report")" -gt 0 ]; then
echo "found=true" >> "$GITHUB_OUTPUT"
fi
if [ "$CONTEXT_DEPTH" != "0" ]; then
index_ok=false
[ "$(jq -r '.reuse.coverage.complete // false' "$report")" = "true" ] && index_ok=true
echo "index-ok=$index_ok" >> "$GITHUB_OUTPUT"
fi
exit "$status"
else
out="$RUNNER_TEMP/echoes-gate.txt"
status=0
gate=()
# Annotations anchor to this repository's files, which a selection does not scan.
[ "$ANNOTATIONS" = "true" ] && [ "${#selection[@]}" -eq 0 ] && gate+=(--github)
cargo echoes \
--threshold "$THRESHOLD" \
--min-tokens "$MIN_TOKENS" \
"${gate[@]}" "$@" > "$out" || status=$?
# GitHub resolves an annotation path from the repository root.
if [ -n "$prefix" ]; then
while IFS= read -r line; do
case "$line" in
::*" file="*) printf '%s file=%s/%s\n' "${line%% file=*}" "$prefix" "${line#* file=}" ;;
*) printf '%s\n' "$line" ;;
esac
done < "$out" > "$out.rooted"
mv "$out.rooted" "$out"
fi
cat "$out"
sum_append "$(
echo '### echoes'
echo
echo '```'
grep -v '^::' "$out"
echo '```'
)"
if [ "$CONTEXT_DEPTH" != "0" ]; then
case "$status" in
0|1) echo "index-ok=true" >> "$GITHUB_OUTPUT" ;;
*) echo "index-ok=false" >> "$GITHUB_OUTPUT" ;;
esac
fi
exit "$status"
fi
- name: Scan the base commit
if: ${{ inputs.pr-number != '' && inputs.only-new == 'true' && inputs.mode != 'post' }}
shell: bash
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ inputs.pr-number }}
CWD: ${{ inputs.cwd }}
ALL_TARGETS: ${{ inputs.all-targets }}
PACKAGE: ${{ inputs.package }}
THRESHOLD: ${{ inputs.threshold }}
MIN_TOKENS: ${{ inputs.min-tokens }}
FUNCTIONS: ${{ inputs.functions }}
FN_MIN_TOKENS: ${{ inputs.fn-min-tokens }}
CONTEXT_BLOCKS: ${{ inputs.context-blocks }}
CONTEXT_MIN_TOKENS: ${{ inputs.context-min-tokens }}
CONTEXT_SELECT: ${{ inputs.context-select }}
CONTEXT_SEARCH: ${{ inputs.context-search }}
CONTEXT_DEPTH: ${{ inputs.context-dependency-depth }}
CONTEXT_DEV: ${{ inputs.context-dev-dependencies }}
CONTEXT_BUILD: ${{ inputs.context-build-dependencies }}
CONTEXT_ALL_OPT: ${{ inputs.context-all-optional-dependencies }}
CONTEXT_FEATURES: ${{ inputs.context-features }}
CONTEXT_ALL_FEATURES: ${{ inputs.context-all-features }}
CONTEXT_NO_DEFAULT: ${{ inputs.context-no-default-features }}
CONTEXT_TARGET: ${{ inputs.context-target }}
CONTEXT_CACHE_DIR: ${{ env.echoes-context-index-dir }}
CONTEXT_CHECK_LATEST: ${{ inputs.context-check-latest }}
CONTEXT_FAIL: ${{ inputs.context-fail-on-reuse }}
run: |
# The base report lets the review ignore duplicates that already exist on the
# base commit. Any failure degrades to reviewing the full report.
base_sha="$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" --jq '.base.sha' 2>/dev/null)" || base_sha=''
if [ -z "$base_sha" ]; then
echo "echoes: cannot read the base commit, the full report is reviewed"
exit 0
fi
if ! git -C "$GITHUB_WORKSPACE" fetch -q --depth 1 origin "$base_sha" \
|| ! git -C "$GITHUB_WORKSPACE" worktree add -q --detach "$RUNNER_TEMP/echoes-base" "$base_sha"; then
echo "echoes: cannot check out the base commit, the full report is reviewed"
exit 0
fi
printf '%s\n' "$base_sha" > "$RUNNER_TEMP/echoes-base-sha"
dir="$RUNNER_TEMP/echoes-base"
[ -n "$CWD" ] && dir="$dir/$CWD"
set --
[ "$ALL_TARGETS" = "true" ] && set -- --all-targets "$@"
[ -n "$PACKAGE" ] && set -- --package "$PACKAGE" "$@"
[ "$FUNCTIONS" = "true" ] && set -- --functions "$@"
[ -n "$FN_MIN_TOKENS" ] && set -- --fn-min-tokens "$FN_MIN_TOKENS" "$@"
[ "$CONTEXT_BLOCKS" = "true" ] && set -- --context-blocks "$@"
[ -n "$CONTEXT_MIN_TOKENS" ] && set -- --context-min-tokens "$CONTEXT_MIN_TOKENS" "$@"
[ -n "$CONTEXT_SELECT" ] && set -- --context-select "$CONTEXT_SELECT" "$@"
[ -n "$CONTEXT_SEARCH" ] && set -- --context-search "$CONTEXT_SEARCH" "$@"
if [ "$CONTEXT_DEPTH" != "0" ]; then
set -- --context-dependency-depth "$CONTEXT_DEPTH" "$@"
[ "$CONTEXT_DEV" = "true" ] && set -- --context-dev-dependencies "$@"
[ "$CONTEXT_BUILD" = "true" ] && set -- --context-build-dependencies "$@"
[ "$CONTEXT_ALL_OPT" = "true" ] && set -- --context-all-optional-dependencies "$@"
[ -n "$CONTEXT_FEATURES" ] && set -- --context-features "$CONTEXT_FEATURES" "$@"
[ "$CONTEXT_ALL_FEATURES" = "true" ] && set -- --context-all-features "$@"
[ "$CONTEXT_NO_DEFAULT" = "true" ] && set -- --context-no-default-features "$@"
[ -n "$CONTEXT_TARGET" ] && set -- --context-target "$CONTEXT_TARGET" "$@"
[ -n "$CONTEXT_CACHE_DIR" ] && set -- --context-cache-dir "$CONTEXT_CACHE_DIR" "$@"
[ "$CONTEXT_CHECK_LATEST" = "false" ] && set -- --context-check-latest false "$@"
[ "$CONTEXT_FAIL" = "true" ] && set -- --context-fail-on-reuse "$@"
fi
# Threshold 1 inventories every base site, a group the pull request
# completes from a single existing copy must still be reported.
# The base tree has the head's layout, so the head's prefix roots its paths.
base_report="$RUNNER_TEMP/echoes-base-report.json"
if ! (cd "$dir" && cargo echoes \
--threshold 1 \
--min-tokens "$MIN_TOKENS" \
--no-fail --json "$@") > "$base_report.raw" \
|| ! jq --rawfile prefix "$RUNNER_TEMP/echoes-prefix" -f "$RUNNER_TEMP/echoes-root.jq" \
"$base_report.raw" > "$base_report"; then
rm -f "$base_report"
echo "echoes: the base scan failed, the full report is reviewed"
fi
rm -f "$base_report.raw"
- name: Review the findings on the pull request
if: ${{ inputs.mode == 'post' || (steps.scan.outputs.found == 'true' && (inputs.pr-number != '' || inputs.dry-run == 'true')) }}
shell: bash
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ inputs.pr-number }}
DRY_RUN: ${{ inputs.dry-run }}
ANNOTATIONS: ${{ inputs.annotations }}
MODE: ${{ inputs.mode }}
PAYLOAD: ${{ inputs.payload }}
MAX_COMMENTS: ${{ inputs.max-comments }}
run: |
# The runner drops step summaries over 1 MiB, so a step keeps its
# summary writes within a safe total, the truncated tail gets a note.
sum_append() {
local text="$1" file="$GITHUB_STEP_SUMMARY"
local cap=$((900 * 1024)) size=0 room len
[ -f "$file" ] && size="$(wc -c < "$file")"
room=$((cap - size))
len="$(printf '%s' "$text" | wc -c)"
if [ "$room" -gt 200 ] && [ "$len" -le "$((room - 200))" ]; then
printf '%s\n' "$text" >> "$file"
elif [ "$room" -gt 200 ]; then
head -c "$((room - 200))" <<< "$text" >> "$file"
printf '\nechoes: the run summary is full, the rest is truncated\n' >> "$file"
fi
}
# A 429 is a rate limit, and a 403 naming a rate limit is one too.
rate_limited_error() {
local err="$1"
if printf '%s' "$err" | grep -qE 'HTTP 429|"status": "429"'; then
return 0
fi
if printf '%s' "$err" | grep -qE 'HTTP 403|"status": "403"'; then
if printf '%s' "$err" | grep -qiE 'rate limit|x-ratelimit-remaining: 0'; then
return 0
fi
fi
return 1
}
# Whole lines within a byte budget, for the review body truncation.
fit_lines() {
LC_ALL=C awk -v cap="$1" 'BEGIN { b = 0 } { if (b < cap && b + length($0) + 1 <= cap) { print; b += length($0) + 1 } }'
}
# No set -e here. Each step checks its own status and writes an outcome file, so a
# failed api call degrades to a skipped review instead of aborting the run.
pr="$PR_NUMBER"
dry="$DRY_RUN"
repo="$GITHUB_REPOSITORY"
report="$RUNNER_TEMP/echoes-report.json"
base_report="$RUNNER_TEMP/echoes-base-report.json"
body="$RUNNER_TEMP/echoes-review.md"
status="$RUNNER_TEMP/echoes-post-status"
inline_file="$RUNNER_TEMP/echoes-inline.tsv"
mode="$MODE"
payload="${PAYLOAD:-$RUNNER_TEMP/echoes-payload}"
comments="$payload/comments.ndjson"
meta="$payload/review.json"
reuse_tmp="$RUNNER_TEMP/echoes-reuse.ndjson"
if [ "$mode" = "post" ]; then
reuse_tmp="$payload/reuse.ndjson"
fi
review_event=''
mkdir -p "$payload"
failed_sites=''
out_of_range=''
omitted=0
rate_omitted=0
omitted_file="$RUNNER_TEMP/echoes-omitted.txt"
[ "$mode" = "post" ] && omitted_file="$payload/omitted.txt"
onlynew=0
[ -s "$base_report" ] && onlynew=1
base_sha="$(cat "$RUNNER_TEMP/echoes-base-sha" 2>/dev/null || echo '')"
run_url="$GITHUB_SERVER_URL/$repo/actions/runs/$GITHUB_RUN_ID"
# Post mode consumes a payload another job prepared, so it reads no
# source tree and skips straight to the api calls.
if [ "$mode" = "post" ]; then
if [ ! -s "$meta" ]; then
echo "echoes: the prepared review is missing from $payload"
echo "failed" > "$status"
exit 1
fi
pr="$(jq -r '.pr' "$meta")"
head_sha="$(jq -r '.head_sha' "$meta")"
out_of_range="$(jq -r '.out_of_range' "$meta")"
omitted="$(jq -r '.omitted // 0' "$meta")"
findings="$(jq -r '.findings' "$meta")"
review_event="$(jq -r '.event // empty' "$meta")"
findings_span=0
grep -qE " (on another type|and can't be deleted|under another cfg|and is public API)\$" <<< "$findings" && findings_span=1
else
# The head SHA and changed file list, empty when there is no pull request.
head_sha=''
changed_files=''
if [ -n "$pr" ]; then
head_sha="$(gh api "repos/$repo/pulls/$pr" --jq '.head.sha' 2>/dev/null)" || head_sha=''
if [ -n "$head_sha" ]; then
changed_files="$(gh api "repos/$repo/pulls/$pr/files" --paginate --jq '.[].filename' 2>/dev/null)" || changed_files=''
fi
fi
if [ -n "$pr" ] && [ -z "$head_sha" ] && [ "$dry" != "true" ]; then
echo "echoes: cannot read pull request $pr, the review is not posted"
echo "failed" > "$status"
exit 1
fi
if [ "$dry" = "true" ]; then
# A dry run renders the review as if every scanned site were in
# the diff, so the step summary shows each comment in full.
changed_json="$(jq -r '
([.groups[].sites[].file]
+ [(.context_groups // [])[] | .sites[].file]
+ [((.reuse // {}).groups // [])[] | .local.file]) | unique' "$report")"
elif [ -n "$pr" ] && [ -n "$head_sha" ] && [ -n "$changed_files" ]; then
changed_json="$(printf '%s\n' "$changed_files" | jq -R . | jq -s .)"
else
changed_json='[]'
fi
# Base site identity, a site exists on the base when the same hash, item and
# path were already there, and the first base copy is the kept original.
if [ "$onlynew" = "1" ]; then
base_json="$(jq '
{ keys: ([.groups[] | .hash as $h | .sites[] | "\($h)\t\(.item)\t\(.file)"]
+ [(.context_groups // [])[] | .hash as $h | .sites[] | "\($h)\t\(.item)\t\(.file)"]
+ [((.reuse // {}).groups // [])[] | .hash as $h | .local as $l | "\($h)\t\($l.item)\t\($l.file)"]),
keep: (reduce .groups[] as $gr ({};
.[$gr.hash] = ($gr.sites[0].file + ":" + ($gr.sites[0].line | tostring)))),
keep_ctx: (reduce (.context_groups // [])[] as $gr ({};
.[$gr.hash] = ($gr.sites[0].file + ":" + ($gr.sites[0].line | tostring)))),
keep_end: (reduce .groups[] as $gr ({}; .[$gr.hash] = ($gr.sites[0].end // null))),
keep_item: (reduce .groups[] as $gr ({}; .[$gr.hash] = $gr.sites[0].item)) }' "$base_report")"
else
base_json='{"keys": [], "keep": {}, "keep_ctx": {}, "keep_end": {}, "keep_item": {}}'
fi
# Sites to comment on. In only-new mode a site must be introduced by this
# pull request, a new hash, item or path relative to the base report. A
# site is inline when its file is in the pull request diff.
onlynew_mode=false
[ "$onlynew" = "1" ] && onlynew_mode=true
# One command line argument holds at most 128 KiB, a large pull request's
# file list exceeds it, so jq reads both lists from files.
changed_file="$RUNNER_TEMP/echoes-changed.json"
base_file="$RUNNER_TEMP/echoes-base-keys.json"
printf '%s' "$changed_json" > "$changed_file"
printf '%s' "$base_json" > "$base_file"
jq -r --slurpfile changed "$changed_file" --slurpfile base "$base_file" \
--argjson onlynew_mode "$onlynew_mode" '
$changed[0] as $changed
| $base[0] as $base
| . as $r
| [ range(0; ($r.groups | length)) as $gi
| $r.groups[$gi]
| ($gi + 1) as $g
| (.sites | length) as $n
| .hash as $h
| (.sites[0].file + ":" + (.sites[0].line | tostring)) as $first
| (.sites[0].end // null) as $fend
| (.kind // "doctest") as $kind
| ([.sites[].self_type] as $t | ($t[0]) as $f | ($t[1:] | map(. != $f) | any)) as $spans
# A binary older than 0.4.1 sends no remedy, its rule only told types apart.
| (if $kind == "function" then (.remedy // (if $spans then "generic-or-macro" else "delete" end)) else "delete" end) as $remedy
| (reduce (.sites[] | (.self_type // "free function")) as $t ([]; if index($t) == null then . + [$t] else . end) | join(", ")) as $selftypes
| (.sites[0].item) as $firstitem
| ($base.keep[$h] // "") as $basekeep
| ($base.keep_item[$h] // "") as $baseitem
| ([ .sites[] | ([$h, .item, .file] | join("\t")) ]) as $ks
| [ .sites | to_entries[]
| (([$h, .value.item, .value.file] | join("\t")) as $k
| (([ $ks[: .key ][] | select(. == $k) ] | length) + 1) as $idx
| ([ $base.keys[] | select(. == $k) ] | length) as $bcount
| { g: $g, c: (.key + 1), n: $n, first: $first, fend: $fend, s: .value,
p: .value.file,
isNew: ($idx > $bcount),
basekeep: $basekeep, bend: ($base.keep_end[$h] // null),
kind: $kind, remedy: $remedy, selftypes: $selftypes, firstitem: $firstitem,
baseitem: $baseitem }) ] as $rows
| ($rows | map(select(.isNew)) | length) as $knew
| $rows[] | . + { knew: $knew }
]
| .[]
| . as $row
| select(($changed | index($row.p)) != null)
| select(if $onlynew_mode then $row.isNew else true end)
| [$row.p, (.s.line | tostring), .s.item, (.g | tostring), (.c | tostring),
(.n | tostring), .first, (if .isNew then "1" else "0" end),
(if .basekeep == "" then "-" else .basekeep end), (.knew | tostring),
(.s.end // "-" | tostring), (.fend // "-" | tostring), (.bend // "-" | tostring),
.kind, .remedy, .firstitem,
(if .baseitem == "" then "-" else .baseitem end), .selftypes,
(if .s.public then "1" else "0" end)]
| @tsv' "$report" > "$inline_file"
# Neutral reuse and context entries, one row per local site that survives
# the same changed-file and only-new filters as the local duplicates.
jq -c --slurpfile changed "$changed_file" --slurpfile base "$base_file" \
--argjson onlynew_mode "$onlynew_mode" '
def spath: .file;
def site_md: " - `" + spath + ":" + (.line | tostring) + "` " + .item;
def cand_md:
" - `\(.package) \(.version)`"
+ (if .depth > 0 then " (depth \(.depth))" else "" end)
+ (if (.conditions | length) > 0 then " (needs \(.conditions | join(", ")))" else "" end)
+ (if .action == "reuse-api" then ": the public item `\(.api_path // .site.item)` can be reused"
elif .action == "upgrade-api" then ": the public item `\(.api_path // .site.item)` is available in a newer published release"
elif .action == "request-extraction" then ": the internal item `\(.site.item)` can be requested for extraction"
elif .action == "unresolved" then ": source analysis does not establish accessibility"
else ": the reuse path is unresolved" end)
+ (if .source_url
then (if .api_path and .api_path != .site.item then " [source (\(.site.item))](\(.source_url))" else " [source](\(.source_url))" end)
else "" end)
+ (if .existing_issue then " [existing request \(.existing_issue.state)](\(.existing_issue.url))"
elif .issue_url then " [request extraction](\(.issue_url))"
elif .action == "request-extraction" then " (the upstream repository is not declared, so no issue link is offered)"
else "" end)
+ (if .lookup_warning then "\n lookup: \(.lookup_warning)" else "" end);
$changed[0] as $changed
| $base[0] as $base
| [ ( (.context_groups // [])
| .[] as $gr
| ($base.keep_ctx[$gr.hash] // "") as $basekeep
| ($gr.sites | length) as $n
| range(0; $n) as $i
| (if $i == 0 and (($basekeep == "") or ($onlynew_mode | not)) then empty else $gr.sites[$i] end) as $s
| ($gr.hash + "\t" + $s.item + "\t" + ($s | spath)) as $k
| { path: ($s | spath), line: $s.line, end: $s.end, item: $s.item,
kind: "context", hash: $gr.hash,
isNew: (([ $base.keys[] | select(. == $k) ] | length) == 0),
others: ([ $gr.sites | to_entries[] | select(.key != $i) | .value | site_md ]) }
)
, ( ((.reuse // {}).groups // [])
| .[] as $gr
| $gr.local as $l
| ($gr.hash + "\t" + $l.item + "\t" + ($l | spath)) as $k
| { path: ($l | spath), line: $l.line, end: $l.end, item: $l.item,
kind: "reuse", hash: $gr.hash,
isNew: (([ $base.keys[] | select(. == $k) ] | length) == 0),
candidates_md: ([ $gr.candidates[] | cand_md ] | join("\n")) } )
]
| map(. as $e | select(($changed | index($e.path)) != null))
| map(select(if $onlynew_mode then .isNew else true end))
| .[]
' "$report" > "$reuse_tmp"
# The inline posts, live mode only. The first post is the permission probe.
# Each comment links to a prefilled wrong-review issue for its group.
uri() { jq -rn --arg s "$1" '$s | @uri'; }
version="$(cargo echoes --version | awk '{ print $NF }')"
report_link() {
local sites
sites="$(jq -r --argjson g "$1" '
.groups[$g - 1].sites as $s
| [$s[:12][] | "\(.file):\(.line) \(.item)"]
+ (if ($s | length) > 12 then ["and \(($s | length) - 12) more"] else [] end)
| join("\n")' "$report")"
printf '[👎 Report a wrong review](https://github.com/LucaCappelletti94/cargo-echoes/issues/new?template=wrong-review.yml&title=%s&version=%s&review=%s&group=%s)' \
"$(uri "Wrong review on $repo#$pr, group $1")" \
"$(uri "$version")" \
"$(uri "$GITHUB_SERVER_URL/$repo/pull/$pr")" \
"$(uri "$sites")"
}
support_link='[❤️ Support echoes](https://github.com/sponsors/LucaCappelletti94)'
# A permalink to lines $3 to $4 of path $2 at commit $1, GitHub shows
# and highlights those lines. Without a sha it falls back to plain text.
blob_link() {
local anchor label
if [ -n "${4:-}" ] && [ "$4" != "$3" ]; then
anchor="#L$3-L$4"
label="$3-$4"
else
anchor="#L$3"
label="$3"
fi
if [ -n "$1" ]; then
printf '[`%s:%s`](%s/%s/blob/%s/%s%s)' "$2" "$label" "$GITHUB_SERVER_URL" "$repo" "$1" "$2" "$anchor"
else
printf '`%s:%s`' "$2" "$label"
fi
}
# The bare permalink url GitHub expands into a code preview of the
# cited lines, empty when no commit is known.
blob_url() {
[ -n "$1" ] || return 0
if [ -n "${4:-}" ] && [ "$4" != "$3" ]; then
printf '%s/%s/blob/%s/%s#L%s-L%s' "$GITHUB_SERVER_URL" "$repo" "$1" "$2" "$3" "$4"
else
printf '%s/%s/blob/%s/%s#L%s' "$GITHUB_SERVER_URL" "$repo" "$1" "$2" "$3"
fi
}
# The inline text location of a block, `path:start` or `path:start-end`.
loc_text() {
if [ -n "${3:-}" ] && [ "$3" != "$2" ]; then
printf '`%s:%s-%s`' "$1" "$2" "$3"
else
printf '`%s:%s`' "$1" "$2"
fi
}
# The permalink wrapped in a markdown link when it exists.
loc_md() {
if [ -n "$2" ]; then
printf '[%s](%s)' "$1" "$2"
else
printf '%s' "$1"
fi
}
# The bare url GitHub previews as code, collapsed in a details block
# when the block spans more than ten lines.
preview_of() {
local n="${3:-$2}" label="${4:-the duplicated doctest}"
if [ $((n - $2 + 1)) -gt 10 ]; then
printf '<details>\n<summary>Show %s</summary>\n\n%s\n\n</details>' "$label" "$1"
else
printf '%s' "$1"
fi
}
# Workflow command escaping, a property also ends at `,` or `:`.
escape_data() {
local s="${1//%/%25}"
s="${s//$'\r'/%0D}"
printf '%s' "${s//$'\n'/%0A}"
}
escape_property() {
local s
s="$(escape_data "$1")"
s="${s//:/%3A}"
printf '%s' "${s//,/%2C}"
}
rendered=''
findings=''
findings_span=0
# One pass builds every comment, live posts them, the dry run renders them.
# The end line of each block comes from the report, `-` when the crate has
# none, a block spanning an include or an escaped literal, or an older binary.
declare -A seen_first=() seen_first_end=()
while IFS=$'\t' read -r p line item g c n first new basekeep knew end fend bend kind remedy firstitem baseitem selftypes public; do
[ -n "$p" ] || continue
# A public copy that already shipped is called or deprecated, deleting it breaks callers.
if [ "$remedy" = "delete" ] && [ "$kind" = "function" ] && [ "$public" = "1" ] && [ "$onlynew" != "1" ]; then
remedy=public
fi
site="$p:$line"
[ "$basekeep" = "-" ] && basekeep=''
[ "$end" = "-" ] && end=''
[ "$fend" = "-" ] && fend=''
[ "$bend" = "-" ] && bend=''
[ "$baseitem" = "-" ] && baseitem=''
kept_item="$firstitem"
[ -n "$baseitem" ] && kept_item="$baseitem"
if [ "$new" = "1" ] && [ -z "${seen_first[$g]:-}" ]; then
seen_first[$g]="$site"
seen_first_end[$g]="$end"
fi
if [ "$onlynew" = "1" ]; then
if [ -n "$basekeep" ] || [ "$site" != "${seen_first[$g]}" ]; then removable=1; else removable=0; fi
elif [ "$c" != "1" ]; then
removable=1
else
removable=0
fi
keepat="$first"
keepat_end="$fend"
if [ "$new" = "1" ]; then
keepat="${seen_first[$g]:-$first}"
keepat_end="${seen_first_end[$g]:-$fend}"
fi
# Comments only land on copies the author should act on, a kept
# copy stays silent.
[ "$removable" = "1" ] || continue
# The annotation carries the same finding without a token, so a
# fork pull request shows it anchored in the diff.
if [ "$ANNOTATIONS" = "true" ] && [ "$dry" != "true" ]; then
keep="$basekeep"
[ -z "$keep" ] && keep="$keepat"
if [ "$kind" = "function" ]; then
case "$remedy" in
generic-or-macro) where='on another type' share='a generic or a macro can share it' ;;
helper-or-macro) where="and can't be deleted" share='a helper or a macro can share it' ;;
merge-cfg) where='under another cfg' share='one function under both cfgs can replace them' ;;
public) where='and is public API' share='make it call the kept copy or deprecate it' ;;
*) where='' share='' ;;
esac
if [ -n "$where" ]; then
printf '::error file=%s,line=%s,title=echoes::%s repeats %s at %s %s, %s, group %s\n' \
"$(escape_property "$p")" "$line" "$(escape_data "$item")" "$(escape_data "$kept_item")" "$(escape_data "$keep")" "$where" "$share" "$g"
findings="$findings- \`$p:$line\` $item, repeats \`$kept_item\` at \`$keep\` $where"$'\n'
findings_span=1
else
printf '::error file=%s,line=%s,title=echoes::%s duplicates the function %s at %s, group %s\n' \
"$(escape_property "$p")" "$line" "$(escape_data "$item")" "$(escape_data "$kept_item")" "$(escape_data "$keep")" "$g"
findings="$findings- \`$p:$line\` $item, duplicates the function \`$kept_item\` at \`$keep\`"$'\n'
fi
else
printf '::error file=%s,line=%s,title=echoes::%s duplicates the doctest at %s, group %s\n' \
"$(escape_property "$p")" "$line" "$(escape_data "$item")" "$(escape_data "$keep")" "$g"
findings="$findings- \`$p:$line\` $item, duplicates \`$keep\`"$'\n'
fi
fi
close_line="$end"
if [ -z "$close_line" ]; then
out_of_range="$out_of_range- group $g $(blob_link "$head_sha" "$p" "$line") $item"$'\n'
continue
fi
# The suggestion deletes the block plus the empty line right after
# it, which would become a stray once the copy is gone.
delete_end="$close_line"
if [ "$remedy" = "delete" ]; then
nx="$(awk -v l="$close_line" 'NR == l + 1 { print "x" $0; exit }' "$p" 2>/dev/null)"
if [ -n "$nx" ] && [ -z "$(printf '%s' "${nx#x}" | tr -d '[:space:]')" ]; then
delete_end=$((close_line + 1))
fi
fi
url=''
loc=''
pv_s=''
pv_e=''
if [ "$onlynew" = "1" ] && [ -n "$basekeep" ]; then
bp="${basekeep%:*}"
bl="${basekeep##*:}"
ce="$bend"
url="$(blob_url "$base_sha" "$bp" "$bl" "$ce")"
pv_s="$bl"
pv_e="$ce"
loc="$(loc_md "$(loc_text "$bp" "$bl" "$ce")" "$url")"
elif [ "$onlynew" = "1" ]; then
kp="${keepat%:*}"
kl="${keepat##*:}"
ce="$keepat_end"
url="$(blob_url "$head_sha" "$kp" "$kl" "$ce")"
pv_s="$kl"
pv_e="$ce"
loc="$(loc_md "$(loc_text "$kp" "$kl" "$ce")" "$url")"
else
fp="${first%:*}"
fl="${first##*:}"
ce="$fend"
url="$(blob_url "$head_sha" "$fp" "$fl" "$ce")"
pv_s="$fl"
pv_e="$ce"
loc="$(loc_md "$(loc_text "$fp" "$fl" "$ce")" "$url")"
fi
echoes_link='[echoes](https://github.com/LucaCappelletti94/cargo-echoes)'
if [ "$kind" = "function" ]; then
if [ "$remedy" != "delete" ]; then
preview='the repeated function'
else
preview='the duplicated function'
fi
else
preview='the duplicated doctest'
fi
if [ "$remedy" = "generic-or-macro" ]; then
note="This method repeats \`$kept_item\` at $loc on another type ($selftypes). A generic or a macro can share one body."
elif [ "$remedy" = "helper-or-macro" ]; then
note="This function repeats \`$kept_item\` at $loc. A trait or an exported symbol needs it, so it can't simply be deleted, a helper or a macro can share one body."
elif [ "$remedy" = "public" ]; then
note="This function repeats \`$kept_item\` at $loc. It is public API, so deleting it breaks its callers, make it call the kept copy or deprecate it."
elif [ "$remedy" = "merge-cfg" ]; then
note="This function repeats \`$kept_item\` at $loc under another \`cfg\`. Deleting either copy drops a configuration, one function under both conditions can replace them."
elif [ "$kind" = "function" ]; then
if [ "$onlynew" = "1" ] && [ -n "$basekeep" ]; then
note="This function was added by this pull request and duplicates a copy that already exists at $loc, $echoes_link suggests deleting the added copy, or updating it if it was meant to differ."
elif [ "$onlynew" = "1" ]; then
note="This pull request adds $knew copies of the same function, $echoes_link suggests keeping the first added copy at $loc and deleting this one, or updating it if it was meant to differ."
else
note="This function appears $n times in this module, this is copy $c. $echoes_link suggests keeping the first copy at $loc and deleting this one, or updating it if it was meant to differ."
fi
elif [ "$onlynew" = "1" ] && [ -n "$basekeep" ]; then
note="This doctest was added by this pull request and duplicates a copy that already exists at $loc, [echoes](https://github.com/LucaCappelletti94/cargo-echoes) suggests removing the added copy."
elif [ "$onlynew" = "1" ]; then
note="This pull request adds $knew copies of the same doctest, [echoes](https://github.com/LucaCappelletti94/cargo-echoes) suggests keeping the first added copy at $loc and removing this one."
else
note="This doctest appears $n times in this scan, this is copy $c. [echoes](https://github.com/LucaCappelletti94/cargo-echoes) suggests keeping the first copy at $loc and removing this one."
fi
comment="$note"
[ -n "$url" ] && comment="$comment"$'\n\n'"$(preview_of "$url" "$pv_s" "$pv_e" "$preview")"
if [ "$remedy" = "delete" ]; then
comment="$comment"$'\n\n```suggestion\n```'
fi
comment="$comment"$'\n\n'"$(report_link "$g") $support_link"
if [ "$dry" = "true" ]; then
rendered+="$comment"$'\n\n'
continue
fi
# One row per comment, so a later job can post it without the
# source tree it was computed from.
jq -cn --arg path "$p" --arg start "$line" --arg stop "$delete_end" \
--arg group "$g" --arg item "$item" --arg close "$close_line" --arg body "$comment" \
'{path: $path, start_line: ($start | tonumber), line: ($stop | tonumber),
group: $group, item: $item, close_line: ($close | tonumber), body: $body}' \
>> "$comments"
done < "$inline_file"
# The neutral reuse and context entries, the local comment minus the
# removal suggestion, so approximate and upstream matches stay advice.
reuse_rendered=''
if [ -s "$reuse_tmp" ]; then
reuse_built="$RUNNER_TEMP/echoes-reuse-built.ndjson"
: > "$reuse_built"
while IFS= read -r row; do
[ -n "$row" ] || continue
p="$(printf '%s' "$row" | jq -r '.path')"
line="$(printf '%s' "$row" | jq -r '.line')"
end="$(printf '%s' "$row" | jq -r '.end')"
item="$(printf '%s' "$row" | jq -r '.item')"
kind="$(printf '%s' "$row" | jq -r '.kind')"
others="$(printf '%s' "$row" | jq -r '.others // [] | join("\n")')"
candidates_md="$(printf '%s' "$row" | jq -r '.candidates_md // empty')"
if [ "$kind" = "context" ]; then
note="This context is repeated in this scan. A shared helper can carry it once. The other copies are at"
[ -n "$others" ] && note="$note"$'\n'"$others"
else
note="This context matches an upstream dependency context. The accessible APIs are listed first"
note="$note"$'\n'"$candidates_md"
fi
note="$note"$'\n\n'"Type compatibility and behavior equivalence need human review."
r_url="$(blob_url "$head_sha" "$p" "$line" "$end")"
comment="$note"
[ -n "$r_url" ] && comment="$comment"$'\n\n'"$(preview_of "$r_url" "$line" "$end" "the local context")"
comment="$comment"$'\n\n'"$support_link"
if [ "$dry" = "true" ]; then
reuse_rendered+="$comment"$'\n\n'
continue
fi
jq -cn --arg path "$p" --arg start "$line" --arg stop "$end" \
--arg item "$item" --arg kind "$kind" --arg body "$comment" \
'{path: $path, start_line: ($start | tonumber), line: ($stop | tonumber),
item: $item, kind: $kind, body: $body}' >> "$reuse_built"
done < "$reuse_tmp"
mv "$reuse_built" "$reuse_tmp"
fi
if [ "$dry" = "true" ]; then
stats="$(jq -r '"\(.total) doctests, \(.unique) unique, \(.functions // 0) functions, \(.unique_functions // 0) unique functions, \(.groups | length) duplicated groups"' "$report")"
groups_block="$(jq -r '
.groups | to_entries | map(
"Group \(.key + 1), \(.value.kind // "doctest"), \(.value.sites | length) copies\n\n"
+ "```\n\(.value.sites[0].code)\n```\n\n"
+ (.value.sites | to_entries | map(
(if .key == 0 then "first " else "copy " end)
+ .value.file + ":" + (.value.line | tostring) + " " + .value.item
+ (if (.value.info | length) > 0 then " (" + (.value.info | join(", ")) + ")" else "" end)
) | join("\n")) + "\n\n"
) | join("")' "$report")"
{
title='Duplicated doctests found by echoes'
if [ "$(jq '[.groups[] | select(.kind == "function")] | length' "$report")" -gt 0 ]; then
title='Duplicates found by echoes'
fi
printf '%s\n' "$title"
echo 'echoes dry run, the review is not posted'
echo
printf '%s\n' "$stats"
echo
printf '%s\n\n' "$groups_block"
echo 'Inline comments echoes would post'
echo
printf '%s' "$rendered"
if [ -n "$reuse_rendered" ]; then
echo
echo 'Neutral reuse and context advice echoes would post'
echo
printf '%s' "$reuse_rendered"
fi
if [ -n "$out_of_range" ]; then
echo
echo 'Sites outside the lines GitHub shows in the diff, the review body would list them'
echo
printf '%s' "$out_of_range"
fi
echo
echo
echo "Run $run_url"
} > "$body"
sum_append "$(cat "$body")"
echo "echoes dry run, the review is not posted"
cat "$body"
echo "dry-run" > "$status"
exit 0
fi
# The inline posts are capped per run, the omitted copies go to the
# review body and the run summary.
case "$MAX_COMMENTS" in
'' | *[!0-9]*) max_comments=50 ;;
*) max_comments="$MAX_COMMENTS" ;;
esac
: > "$omitted_file"
touch "$comments" "$reuse_tmp"
local_n="$(wc -l < "$comments")"
reuse_n="$(wc -l < "$reuse_tmp")"
total_n=$((local_n + reuse_n))
if [ "$total_n" -gt "$max_comments" ]; then
omitted=$((total_n - max_comments))
keep_local="$max_comments"
[ "$keep_local" -gt "$local_n" ] && keep_local="$local_n"
keep_reuse=$((max_comments - keep_local))
{ tail -n +$((keep_local + 1)) "$comments"
tail -n +$((keep_reuse + 1)) "$reuse_tmp"; } |
jq -rs 'map("- " + .path + ":" + (.start_line | tostring) + " " + .item
+ (if .group then " (group " + (.group | tostring) + ")" else " (" + (.kind // "context") + ")" end))
| join("\n")' > "$omitted_file"
head -n "$keep_local" "$comments" > "$comments.capped"
head -n "$keep_reuse" "$reuse_tmp" > "$reuse_tmp.capped"
mv "$comments.capped" "$comments"
mv "$reuse_tmp.capped" "$reuse_tmp"
fi
# Both lists grow with the pull request, past one argument's 128 KiB.
printf '%s' "$out_of_range" > "$RUNNER_TEMP/echoes-out-of-range.txt"
printf '%s' "$findings" > "$RUNNER_TEMP/echoes-findings.txt"
event='REQUEST_CHANGES'
[ -s "$comments" ] || [ -n "$out_of_range" ] || event='COMMENT'
jq -n --arg pr "$pr" --arg head "$head_sha" --arg event "$event" \
--argjson omitted "$omitted" \
--rawfile oor "$RUNNER_TEMP/echoes-out-of-range.txt" --rawfile f "$RUNNER_TEMP/echoes-findings.txt" \
'{pr: $pr, head_sha: $head, event: $event, omitted: $omitted, out_of_range: $oor, findings: $f}' > "$meta"
fi
# Nothing to post, in prepare mode the payload still exists so the
# posting job has something to download.
if [ ! -s "$comments" ] && [ -z "$out_of_range" ] && [ ! -s "$reuse_tmp" ] && [ "$omitted" -eq 0 ]; then
if [ "$onlynew" = "1" ]; then
echo "echoes: no duplicate was introduced by this pull request, the review is not posted"
echo "skipped-existing" > "$status"
else
echo "echoes: no duplicate is in the pull request diff, the review is not posted"
echo "skipped-nodiff" > "$status"
fi
exit 0
fi
# The reported copies in the run summary, beside the whole scan the
# scan step wrote there.
if [ -n "$findings" ] && [ "$dry" != "true" ]; then
findings_title='#### Copies to remove'
[ "$findings_span" = "1" ] && findings_title='#### Findings'
sum_append "$(printf '\n%s\n\n%s' "$findings_title" "$findings")"
fi
# The omitted copies and the rate limit stay in the run summary, the
# public record of what the review did not post inline.
if [ "$omitted" -gt 0 ]; then
omitted_block="$(printf '#### Inline copies not posted\n\n%s inline copies were not posted, they are listed in this run summary\n\n' "$omitted")"
[ -s "$omitted_file" ] && omitted_block="$omitted_block$(head -c 400000 "$omitted_file")"$'\n'
sum_append "$omitted_block"
fi
if [ "$mode" = "prepare" ]; then
cp "$reuse_tmp" "$payload/reuse.ndjson"
cp "$omitted_file" "$payload/omitted.txt"
echo "echoes: the review is prepared in $payload for a later job"
echo "prepared" > "$status"
exit 0
fi
readonly_notice() {
echo "echoes: the GITHUB_TOKEN is read-only for this pull request, the review is not posted"
echo "echoes: the annotations and the run summary carry the findings, the removal suggestion needs a review comment"
}
# The first post is the permission probe.
readonly=0
rate_limited=0
posted_local=0
posted_reuse=0
planned_n="$(cat "$comments" "$reuse_tmp" | wc -l)"
post_one() {
local row="$1" p line err item g
p="$(printf '%s' "$row" | jq -r '.path')"
line="$(printf '%s' "$row" | jq -r '.start_line')"
if ! err="$(printf '%s' "$row" | jq -r '.body' | gh api "repos/$repo/pulls/$pr/comments" \
-f "commit_id=$head_sha" -f "path=$p" \
-F "start_line=$line" -F "line=$(printf '%s' "$row" | jq -r '.line')" \
-f start_side=RIGHT -f side=RIGHT -F 'body=@-' 2>&1)"; then
if rate_limited_error "$err"; then
rate_limited=1
return 1
fi
if printf '%s' "$err" | grep -qE 'HTTP 403|"status": "403"'; then
readonly=1
return 1
fi
item="$(printf '%s' "$row" | jq -r '.item')"
g="$(printf '%s' "$row" | jq -r '.group // .kind')"
printf 'echoes: %s:%s %s\n' "$p" "$line" "$item"
printf '%s\n' "$err" | sed 's/^/echoes: /'
if printf '%s' "$err" | grep -qE 'HTTP 422|"status": "422"'; then
out_of_range="$out_of_range- group $g $(printf '`%s:%s`' "$p" "$line") $item"$'\n'
else
failed_sites="$failed_sites group $g $p:$line $item"$'\n'
fi
fi
return 0
}
while IFS= read -r row; do
[ -n "$row" ] || continue
post_one "$row" || break
posted_local=$((posted_local + 1))
done < "$comments"
if [ "$readonly" != "1" ] && [ "$rate_limited" != "1" ]; then
while IFS= read -r row; do
[ -n "$row" ] || continue
post_one "$row" || break
posted_reuse=$((posted_reuse + 1))
done < "$reuse_tmp"
fi
if [ "$readonly" = "1" ]; then
readonly_notice
echo "skipped-readonly" > "$status"
exit 0
fi
# The rows the rate limit left unposted, listed like the omitted ones.
if [ "$rate_limited" = "1" ]; then
rate_omitted=$((planned_n - posted_local - posted_reuse))
rate_unposted="$(
{ tail -n +$((posted_local + 1)) "$comments"
tail -n +$((posted_reuse + 1)) "$reuse_tmp"; } |
jq -rs 'map("- " + .path + ":" + (.start_line | tostring) + " " + .item
+ (if .group then " (group " + (.group | tostring) + ")" else " (" + (.kind // "context") + ")" end))
| join("\n")'
)"
echo "echoes: the api rate limit was reached, $rate_omitted inline copies were not posted"
fi
if [ "$rate_omitted" -gt 0 ]; then
rate_block="$(printf '#### Rate limit\n\n%s inline copies were not posted because the api rate limit was reached, they are listed in this run summary\n\n' "$rate_omitted")"
rate_block="$rate_block$rate_unposted"$'\n'
sum_append "$rate_block"
fi
# A review with no comment of its own needs a body. Sites GitHub could not
# anchor inline are listed with permalinks to the head commit.
if [ -z "$review_event" ]; then
review_event='REQUEST_CHANGES'
[ -s "$comments" ] || [ -n "$out_of_range" ] || review_event='COMMENT'
fi
review=(-f "event=$review_event")
review_body=''
if [ -n "$out_of_range" ]; then
review_body="$(printf 'These duplicates are not on lines GitHub shows in this pull request diff, the inline comment cannot attach there. The links point at the copies on the head commit.\n\n%s' "$out_of_range")"
fi
if [ -n "$failed_sites" ]; then
review_body="$review_body$(printf '\n\nDuplicates echoes could not comment inline\n\n```\n%s```' "$failed_sites")"
fi
if [ "$omitted" -gt 0 ]; then
review_body="$review_body$(printf '\n\n%s inline copies were not posted, they are listed in the run summary %s' "$omitted" "$run_url")"
fi
if [ "$rate_omitted" -gt 0 ]; then
review_body="$review_body$(printf '\n\n%s inline copies were not posted because the api rate limit was reached, they are listed in the run summary %s' "$rate_omitted" "$run_url")"
fi
# The api rejects a review body over 65,536 characters, so the lists
# are truncated with a pointer to the run summary.
if [ "${#review_body}" -gt 65000 ]; then
budget=$((65000 / 2 - 400))
oor_part=''
if [ -n "$out_of_range" ]; then
oor_kept="$(printf '%s' "$out_of_range" | fit_lines "$budget")"
oor_more=$(($(printf '%s' "$out_of_range" | wc -l) - $(printf '%s' "$oor_kept" | wc -l)))
oor_part='These duplicates are not on lines GitHub shows in this pull request diff, the inline comment cannot attach there. The links point at the copies on the head commit.'
oor_part="$oor_part"$'\n\n'"$oor_kept"
[ "$oor_more" -gt 0 ] && oor_part="$oor_part"$'\n'"and $oor_more more, they are in the run summary $run_url"
fi
failed_part=''
if [ -n "$failed_sites" ]; then
failed_kept="$(printf '%s' "$failed_sites" | fit_lines "$budget")"
failed_more=$(($(printf '%s' "$failed_sites" | wc -l) - $(printf '%s' "$failed_kept" | wc -l)))
failed_part="$(printf '\n\nDuplicates echoes could not comment inline\n\n```\n%s```' "$failed_kept")"
[ "$failed_more" -gt 0 ] && failed_part="$failed_part"$'\n'"and $failed_more more, they are in the run summary $run_url"
fi
review_body="$oor_part$failed_part"
[ "$omitted" -gt 0 ] && review_body="$review_body$(printf '\n\n%s inline copies were not posted, they are listed in the run summary %s' "$omitted" "$run_url")"
[ "$rate_omitted" -gt 0 ] && review_body="$review_body$(printf '\n\n%s inline copies were not posted because the api rate limit was reached, they are listed in the run summary %s' "$rate_omitted" "$run_url")"
fi
if [ -n "$review_body" ]; then
review+=(-f "body=$review_body")
fi
if ! resp="$(gh api -X POST "repos/$repo/pulls/$pr/reviews" "${review[@]}" 2>&1)"; then
if rate_limited_error "$resp"; then
echo "echoes: the api rate limit was reached, the review was not posted"
printf '%s\n' "$resp"
echo "failed" > "$status"
exit 1
fi
if printf '%s' "$resp" | grep -qE 'HTTP 403|"status": "403"'; then
readonly_notice
echo "skipped-readonly" > "$status"
exit 0
fi
echo "echoes: posting the review failed"
printf '%s\n' "$resp"
echo "failed" > "$status"
exit 1
fi
posted_state="$(printf '%s' "$resp" | jq -r '.state' 2>/dev/null)" || posted_state=''
expected_state='CHANGES_REQUESTED'
[ "$review_event" = "COMMENT" ] && expected_state='COMMENTED'
if [ "$posted_state" != "$expected_state" ]; then
echo "echoes: the review response is unexpected"
printf '%s\n' "$resp"
echo "failed" > "$status"
exit 1
fi
if [ -n "$failed_sites" ]; then
echo "echoes: the inline comments failed for the following sites"
printf '%s' "$failed_sites"
fi
echo "echoes: posted the review on pull request $pr"
echo "posted" > "$status"
- name: Save the dependency index
if: ${{ always() && !cancelled() && inputs.mode != 'post' && inputs.context-dependency-depth > 0 && steps.scan.outputs.index-ok == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830
with:
path: ${{ env.echoes-context-index-dir }}/public
key: echoes-context-index-${{ runner.os }}-${{ inputs.context-target != '' && inputs.context-target || 'any' }}-v1-${{ github.run_id }}-${{ github.run_attempt }}