name: dejadoc
description: >-
Install dejadoc, fail the job when doctests are duplicated
across the workspace, or post the findings as a pull request review
branding:
icon: 'copy'
color: 'orange'
inputs:
threshold:
description: Report duplicate groups with at least this many sites
required: false
default: '2'
min-tokens:
description: Skip doctest bodies with fewer tokens than this
required: false
default: '0'
all-targets:
description: Also scan bin and example targets
required: false
default: 'false'
package:
description: Restrict the scan to one workspace member by name
required: false
default: ''
version:
description: The dejadoc version to install, empty for the latest
required: false
default: ''
cwd:
description: The workspace directory to scan
required: false
default: ''
pr-number:
description: Post the findings as a pull request review on this PR instead of failing the job
required: false
default: ''
dry-run:
description: Render the review instead of posting it
required: false
default: 'false'
only-new:
description: In pull request mode report only duplicates this pull request introduces
required: false
default: 'true'
runs:
using: 'composite'
steps:
- name: Install dejadoc
shell: bash
env:
GH_TOKEN: ${{ github.token }}
VERSION: ${{ inputs.version }}
run: |
set -euo pipefail
repo="LucaCappelletti94/cargo-dejadoc"
tag=''
[ -n "$VERSION" ] && tag="v$VERSION"
# Keep these targets in sync with the release matrix in .github/workflows/release.yml.
case "$RUNNER_OS/$RUNNER_ARCH" in
Linux/X64) target=x86_64-unknown-linux-gnu ;;
Linux/ARM64) target=aarch64-unknown-linux-gnu ;;
macOS/X64) target=x86_64-apple-darwin ;;
macOS/ARM64) target=aarch64-apple-darwin ;;
Windows/X64) target=x86_64-pc-windows-msvc ;;
*) target='' ;;
esac
if [ -n "$target" ]; then
asset="cargo-dejadoc-$target.tar.gz"
dest="$RUNNER_TEMP/dejadoc-bin"
mkdir -p "$dest"
args=(--repo "$repo" --pattern "$asset" --dir "$dest" --clobber)
[ -n "$tag" ] && args=("$tag" "${args[@]}")
if err="$(gh release download "${args[@]}" 2>&1)"; then
bin=cargo-dejadoc
[ "$RUNNER_OS" = Windows ] && bin=cargo-dejadoc.exe
tar -xzf "$dest/$asset" -C "$dest"
chmod +x "$dest/$bin" 2>/dev/null || true
echo "$dest" >> "$GITHUB_PATH"
echo "dejadoc: installed the prebuilt $asset from ${tag:-the latest release}"
exit 0
fi
echo "dejadoc: could not download $asset, falling back to source"
printf '%s\n' "$err" | sed 's/^/dejadoc: /'
else
echo "dejadoc: no prebuilt binary for $RUNNER_OS/$RUNNER_ARCH, falling back to source"
fi
install=(dejadoc)
[ -n "$tag" ] && install+=(--version "$VERSION")
cargo install "${install[@]}"
- name: Scan for duplicated doctests
id: scan
shell: bash
working-directory: ${{ inputs.cwd }}
env:
ALL_TARGETS: ${{ inputs.all-targets }}
PACKAGE: ${{ inputs.package }}
PR_NUMBER: ${{ inputs.pr-number }}
DRY_RUN: ${{ inputs.dry-run }}
THRESHOLD: ${{ inputs.threshold }}
MIN_TOKENS: ${{ inputs.min-tokens }}
run: |
set --
[ "$ALL_TARGETS" = "true" ] && set -- --all-targets "$@"
[ -n "$PACKAGE" ] && set -- --package "$PACKAGE" "$@"
if [ -n "$PR_NUMBER" ] || [ "$DRY_RUN" = "true" ]; then
report="$RUNNER_TEMP/dejadoc-report.json"
cargo dejadoc \
--threshold "$THRESHOLD" \
--min-tokens "$MIN_TOKENS" \
--no-fail --json "$@" > "$report"
jq -r '. as $r | "\($r.total) doctests, \($r.unique) unique, \($r.groups | length) duplicated groups"' "$report"
if [ "$(jq '.groups | length' "$report")" -gt 0 ]; then
echo "found=true" >> "$GITHUB_OUTPUT"
fi
else
cargo dejadoc \
--threshold "$THRESHOLD" \
--min-tokens "$MIN_TOKENS" \
"$@"
fi
- name: Scan the base commit
if: ${{ inputs.pr-number != '' && inputs.only-new == 'true' }}
shell: bash
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ inputs.pr-number }}
CWD: ${{ inputs.cwd }}
ALL_TARGETS: ${{ inputs.all-targets }}
PACKAGE: ${{ inputs.package }}
THRESHOLD: ${{ inputs.threshold }}
MIN_TOKENS: ${{ inputs.min-tokens }}
run: |
# The base report lets the review ignore duplicates that already exist on the
# base commit. Any failure degrades to reviewing the full report.
base_sha="$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" --jq '.base.sha' 2>/dev/null)" || base_sha=''
if [ -z "$base_sha" ]; then
echo "dejadoc: cannot read the base commit, the full report is reviewed"
exit 0
fi
if ! git -C "$GITHUB_WORKSPACE" fetch -q --depth 1 origin "$base_sha" \
|| ! git -C "$GITHUB_WORKSPACE" worktree add -q --detach "$RUNNER_TEMP/dejadoc-base" "$base_sha"; then
echo "dejadoc: cannot check out the base commit, the full report is reviewed"
exit 0
fi
printf '%s\n' "$base_sha" > "$RUNNER_TEMP/dejadoc-base-sha"
dir="$RUNNER_TEMP/dejadoc-base"
[ -n "$CWD" ] && dir="$dir/$CWD"
set --
[ "$ALL_TARGETS" = "true" ] && set -- --all-targets "$@"
# Threshold 1 inventories every base doctest, a group the pull request
# completes from a single existing copy must still be reported.
if ! (cd "$dir" && cargo dejadoc \
--threshold 1 \
--min-tokens "$MIN_TOKENS" \
--no-fail --json "$@") > "$RUNNER_TEMP/dejadoc-base-report.json"; then
rm -f "$RUNNER_TEMP/dejadoc-base-report.json"
echo "dejadoc: the base scan failed, the full report is reviewed"
fi
- name: Post the findings as a pull request review
if: ${{ steps.scan.outputs.found == 'true' && (inputs.pr-number != '' || inputs.dry-run == 'true') }}
shell: bash
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ inputs.pr-number }}
DRY_RUN: ${{ inputs.dry-run }}
CWD: ${{ inputs.cwd }}
run: |
# No set -e here. Each step checks its own status and writes an outcome file, so a
# failed api call degrades to a skipped review instead of aborting the run.
pr="$PR_NUMBER"
dry="$DRY_RUN"
cwd="$CWD"
repo="$GITHUB_REPOSITORY"
report="$RUNNER_TEMP/dejadoc-report.json"
base_report="$RUNNER_TEMP/dejadoc-base-report.json"
body="$RUNNER_TEMP/dejadoc-review.md"
status="$RUNNER_TEMP/dejadoc-post-status"
inline_file="$RUNNER_TEMP/dejadoc-inline.tsv"
onlynew=0
[ -s "$base_report" ] && onlynew=1
base_sha="$(cat "$RUNNER_TEMP/dejadoc-base-sha" 2>/dev/null || echo '')"
# The head SHA and changed file list, empty when there is no pull request.
head_sha=''
changed_files=''
if [ -n "$pr" ]; then
head_sha="$(gh api "repos/$repo/pulls/$pr" --jq '.head.sha' 2>/dev/null)" || head_sha=''
if [ -n "$head_sha" ]; then
changed_files="$(gh api "repos/$repo/pulls/$pr/files" --paginate --jq '.[].filename' 2>/dev/null)" || changed_files=''
fi
fi
if [ -n "$pr" ] && [ -z "$head_sha" ] && [ "$dry" != "true" ]; then
echo "dejadoc: cannot read pull request $pr, the review is not posted"
echo "failed" > "$status"
exit 1
fi
if [ "$dry" = "true" ]; then
# A dry run renders the review as if every scanned site were in
# the diff, so the step summary shows each comment in full.
changed_json="$(jq -r --arg cwd "$cwd" '
def sp: if $cwd == "" then .file else "\($cwd)/\(.file)" end;
[.groups[].sites[] | sp] | unique' "$report")"
elif [ -n "$pr" ] && [ -n "$head_sha" ] && [ -n "$changed_files" ]; then
changed_json="$(printf '%s\n' "$changed_files" | jq -R . | jq -s .)"
else
changed_json='[]'
fi
# Base site identity, a site exists on the base when the same hash, item and
# path were already there, and the first base copy is the kept original.
if [ "$onlynew" = "1" ]; then
base_json="$(jq --arg cwd "$cwd" '
def sp: if $cwd == "" then .file else "\($cwd)/\(.file)" end;
{ keys: [.groups[] | .hash as $h | .sites[] | "\($h)\t\(.item)\t\(sp)"],
keep: (reduce .groups[] as $gr ({};
.[$gr.hash] = (($gr.sites[0] | sp) + ":" + ($gr.sites[0].line | tostring)))) }' "$base_report")"
else
base_json='{"keys": [], "keep": {}}'
fi
# Sites to comment on. In only-new mode a site must be introduced by this
# pull request, a new hash, item or path relative to the base report. A
# site is inline when its file is in the pull request diff.
onlynew_mode=false
[ "$onlynew" = "1" ] && onlynew_mode=true
jq -r --arg cwd "$cwd" --argjson changed "$changed_json" --argjson base "$base_json" \
--argjson onlynew_mode "$onlynew_mode" '
def spath: if $cwd == "" then .file else "\($cwd)/\(.file)" end;
. as $r
| [ range(0; ($r.groups | length)) as $gi
| $r.groups[$gi]
| ($gi + 1) as $g
| (.sites | length) as $n
| .hash as $h
| ((.sites[0] | spath) + ":" + (.sites[0].line | tostring)) as $first
| ($base.keep[$h] // "") as $basekeep
| ([ .sites[] | ([$h, .item, (spath)] | join("\t")) ]) as $ks
| [ .sites | to_entries[]
| (([$h, .value.item, (.value | spath)] | join("\t")) as $k
| (([ $ks[: .key ][] | select(. == $k) ] | length) + 1) as $idx
| ([ $base.keys[] | select(. == $k) ] | length) as $bcount
| { g: $g, c: (.key + 1), n: $n, first: $first, s: .value,
p: (.value | spath),
isNew: ($idx > $bcount),
basekeep: $basekeep }) ] as $rows
| ($rows | map(select(.isNew)) | length) as $knew
| $rows[] | . + { knew: $knew }
]
| .[]
| . as $row
| select(($changed | index($row.p)) != null)
| select(if $onlynew_mode then $row.isNew else true end)
| [$row.p, (.s.line | tostring), .s.item, (.g | tostring), (.c | tostring),
(.n | tostring), .first, (if .isNew then "1" else "0" end),
(if .basekeep == "" then "-" else .basekeep end), (.knew | tostring)]
| @tsv' "$report" > "$inline_file"
# The inline posts, live mode only. The first post is the permission probe.
# Each comment links to a prefilled wrong-review issue for its group.
uri() { jq -rn --arg s "$1" '$s | @uri'; }
version="$(cargo dejadoc --version | awk '{ print $NF }')"
report_link() {
local sites
sites="$(jq -r --arg cwd "$cwd" --argjson g "$1" '
def sp: if $cwd == "" then .file else "\($cwd)/\(.file)" end;
.groups[$g - 1].sites as $s
| [$s[:12][] | "\(sp):\(.line) \(.item)"]
+ (if ($s | length) > 12 then ["and \(($s | length) - 12) more"] else [] end)
| join("\n")' "$report")"
printf '[👎 Report a wrong review](https://github.com/LucaCappelletti94/cargo-dejadoc/issues/new?template=wrong-review.yml&title=%s&version=%s&review=%s&group=%s)' \
"$(uri "Wrong review on $repo#$pr, group $1")" \
"$(uri "$version")" \
"$(uri "$GITHUB_SERVER_URL/$repo/pull/$pr")" \
"$(uri "$sites")"
}
support_link='[❤️ Support dejadoc](https://github.com/sponsors/LucaCappelletti94)'
# The line closing the doctest opened at line $2 of file $1, empty
# when no closing fence is found, the live posting eligibility.
close_line_of() {
awk -v l="$2" 'NR > l && $0 ~ /^[[:space:]]*(\/\/\/|\/\/!|\/\/)?[[:space:]]*```[[:space:]]*$/ { print NR; exit }' "$1" 2>/dev/null
}
# A permalink to lines $3 to $4 of path $2 at commit $1, GitHub shows
# and highlights those lines. Without a sha it falls back to plain text.
blob_link() {
local anchor label
if [ -n "${4:-}" ] && [ "$4" != "$3" ]; then
anchor="#L$3-L$4"
label="$3-$4"
else
anchor="#L$3"
label="$3"
fi
if [ -n "$1" ]; then
printf '[`%s:%s`](%s/%s/blob/%s/%s%s)' "$2" "$label" "$GITHUB_SERVER_URL" "$repo" "$1" "$2" "$anchor"
else
printf '`%s:%s`' "$2" "$label"
fi
}
# The line ending the doctest block started at line $2 of path $1,
# read from file $3 when given, from the head tree otherwise.
block_end() {
local f
f="$1"
[ -n "${3:-}" ] && f="$3"
close_line_of "$f" "$2"
}
# The bare permalink url GitHub expands into a code preview of the
# cited lines, empty when no commit is known.
blob_url() {
[ -n "$1" ] || return 0
if [ -n "${4:-}" ] && [ "$4" != "$3" ]; then
printf '%s/%s/blob/%s/%s#L%s-L%s' "$GITHUB_SERVER_URL" "$repo" "$1" "$2" "$3" "$4"
else
printf '%s/%s/blob/%s/%s#L%s' "$GITHUB_SERVER_URL" "$repo" "$1" "$2" "$3"
fi
}
# The inline text location of a block, `path:start` or `path:start-end`.
loc_text() {
if [ -n "${3:-}" ] && [ "$3" != "$2" ]; then
printf '`%s:%s-%s`' "$1" "$2" "$3"
else
printf '`%s:%s`' "$1" "$2"
fi
}
# The permalink wrapped in a markdown link when it exists.
loc_md() {
if [ -n "$2" ]; then
printf '[%s](%s)' "$1" "$2"
else
printf '%s' "$1"
fi
}
# The bare url GitHub previews as code, collapsed in a details block
# when the block spans more than ten lines.
preview_of() {
local n="${3:-$2}"
if [ $((n - $2 + 1)) -gt 10 ]; then
printf '<details>\n<summary>Show the duplicated doctest</summary>\n\n%s\n\n</details>' "$1"
else
printf '%s' "$1"
fi
}
# Base tree file for a repo-relative path, empty when the tree is absent.
base_file() {
if [ -f "$RUNNER_TEMP/dejadoc-base/$1" ]; then
printf '%s' "$RUNNER_TEMP/dejadoc-base/$1"
fi
}
readonly=0
failed_sites=''
out_of_range=''
rendered=''
# One pass builds every comment, live posts them, the dry run renders them.
declare -A seen_first=()
while IFS=$'\t' read -r p line item g c n first new basekeep knew; do
[ -n "$p" ] || continue
site="$p:$line"
[ "$basekeep" = "-" ] && basekeep=''
if [ "$new" = "1" ] && [ -z "${seen_first[$g]:-}" ]; then
seen_first[$g]="$site"
fi
if [ "$onlynew" = "1" ]; then
if [ -n "$basekeep" ] || [ "$site" != "${seen_first[$g]}" ]; then removable=1; else removable=0; fi
elif [ "$c" != "1" ]; then
removable=1
else
removable=0
fi
keepat="$first"
[ "$new" = "1" ] && keepat="${seen_first[$g]:-$first}"
# Comments only land on copies the author should act on, a kept
# copy stays silent.
[ "$removable" = "1" ] || continue
close_line="$(close_line_of "$p" "$line")"
if [ -z "$close_line" ]; then
out_of_range="$out_of_range- group $g $(blob_link "$head_sha" "$p" "$line") $item"$'\n'
continue
fi
# An empty line right after the block becomes a stray once the
# copy is gone, it joins the deleted range.
delete_end="$close_line"
nx="$(awk -v l="$close_line" 'NR == l + 1 { print "x" $0; exit }' "$p" 2>/dev/null)"
if [ -n "$nx" ] && [ -z "$(printf '%s' "${nx#x}" | tr -d '[:space:]')" ]; then
delete_end=$((close_line + 1))
fi
url=''
pv_s=''
pv_e=''
if [ "$onlynew" = "1" ]; then
if [ -n "$basekeep" ]; then
bp="${basekeep%:*}"
bl="${basekeep##*:}"
ce="$(block_end "$bp" "$bl" "$(base_file "$bp")")"
url="$(blob_url "$base_sha" "$bp" "$bl" "$ce")"
pv_s="$bl"
pv_e="$ce"
note="This doctest was added by this pull request and duplicates a copy that already exists at $(loc_md "$(loc_text "$bp" "$bl" "$ce")" "$url"), [dejadoc](https://github.com/LucaCappelletti94/cargo-dejadoc) suggests removing the added copy."
else
kp="${keepat%:*}"
kl="${keepat##*:}"
ce="$(block_end "$kp" "$kl")"
url="$(blob_url "$head_sha" "$kp" "$kl" "$ce")"
pv_s="$kl"
pv_e="$ce"
note="This pull request adds $knew copies of the same doctest, [dejadoc](https://github.com/LucaCappelletti94/cargo-dejadoc) suggests keeping the first added copy at $(loc_md "$(loc_text "$kp" "$kl" "$ce")" "$url") and removing this one."
fi
else
fp="${first%:*}"
fl="${first##*:}"
ce="$(block_end "$fp" "$fl")"
url="$(blob_url "$head_sha" "$fp" "$fl" "$ce")"
pv_s="$fl"
pv_e="$ce"
note="This doctest appears $n times in this scan, this is copy $c. [dejadoc](https://github.com/LucaCappelletti94/cargo-dejadoc) suggests keeping the first copy at $(loc_md "$(loc_text "$fp" "$fl" "$ce")" "$url") and removing this one."
fi
comment="$note"
[ -n "$url" ] && comment="$comment"$'\n\n'"$(preview_of "$url" "$pv_s" "$pv_e")"
comment="$comment"$'\n\n```suggestion\n```'
comment="$comment"$'\n\n'"$(report_link "$g") $support_link"
if [ "$dry" = "true" ]; then
rendered+="$comment"$'\n\n'
continue
fi
fields=(-F "start_line=$line" -F "line=$delete_end" -f start_side=RIGHT -f side=RIGHT)
if ! err="$(gh api "repos/$repo/pulls/$pr/comments" \
-f "commit_id=$head_sha" -f "path=$p" "${fields[@]}" \
-f "body=$comment" 2>&1)"; then
if printf '%s' "$err" | grep -qE 'HTTP 403|"status": "403"'; then
readonly=1
break
fi
printf 'dejadoc: %s:%s %s\n' "$p" "$line" "$item"
printf '%s\n' "$err" | sed 's/^/dejadoc: /'
if printf '%s' "$err" | grep -qE 'HTTP 422|"status": "422"'; then
out_of_range="$out_of_range- group $g $(blob_link "$head_sha" "$p" "$line" "$close_line") $item"$'\n'
else
failed_sites="$failed_sites group $g $p:$line $item"$'\n'
fi
fi
done < "$inline_file"
run_url="$GITHUB_SERVER_URL/$repo/actions/runs/$GITHUB_RUN_ID"
if [ "$dry" = "true" ]; then
stats="$(jq -r '"\(.total) doctests, \(.unique) unique, \(.groups | length) duplicated groups"' "$report")"
groups_block="$(jq -r --arg cwd "$cwd" '
def sp: if $cwd == "" then .file else "\($cwd)/\(.file)" end;
.groups | to_entries | map(
"Group \(.key + 1), \(.value.sites | length) copies\n\n"
+ "```\n\(.value.sites[0].code)\n```\n\n"
+ (.value.sites | to_entries | map(
(if .key == 0 then "first " else "copy " end)
+ (.value | sp) + ":" + (.value.line | tostring) + " " + .value.item
+ (if (.value.info | length) > 0 then " (" + (.value.info | join(", ")) + ")" else "" end)
) | join("\n")) + "\n\n"
) | join("")' "$report")"
{
echo 'Duplicated doctests found by dejadoc'
echo 'dejadoc dry run, the review is not posted'
echo
printf '%s\n' "$stats"
echo
printf '%s\n\n' "$groups_block"
echo 'Inline comments dejadoc would post'
echo
printf '%s' "$rendered"
if [ -n "$out_of_range" ]; then
echo
echo 'Sites outside the lines GitHub shows in the diff, the review body would list them'
echo
printf '%s' "$out_of_range"
fi
echo
echo
echo "Run $run_url"
} > "$body"
{ echo; cat "$body"; } >> "$GITHUB_STEP_SUMMARY"
echo "dejadoc dry run, the review is not posted"
cat "$body"
echo "dry-run" > "$status"
exit 0
fi
if [ ! -s "$inline_file" ]; then
if [ "$onlynew" = "1" ]; then
echo "dejadoc: no duplicate was introduced by this pull request, the review is not posted"
echo "skipped-existing" > "$status"
else
echo "dejadoc: no duplicated doctest is in the pull request diff, the review is not posted"
echo "skipped-nodiff" > "$status"
fi
exit 0
fi
readonly_notice() {
echo "dejadoc: the GITHUB_TOKEN is read-only for this pull request, the review is not posted"
echo "dejadoc: to review fork pull requests, trigger on pull_request_target and check out the pull request head, or on a private repository enable 'Send write tokens to workflows from pull requests'"
}
if [ "$readonly" = "1" ]; then
readonly_notice
echo "skipped-readonly" > "$status"
exit 0
fi
# A review with no comment of its own needs a body. Sites GitHub could not
# anchor inline are listed with permalinks to the head commit.
review=(-f event=REQUEST_CHANGES)
review_body=''
if [ -n "$out_of_range" ]; then
review_body="$(printf 'These duplicated doctests are not on lines GitHub shows in this pull request diff, the inline comment cannot attach there. The links point at the copies on the head commit.\n\n%s' "$out_of_range")"
fi
if [ -n "$failed_sites" ]; then
failed_block="$(printf '\n\nDuplicated doctests dejadoc could not comment inline\n\n```\n%s```' "$failed_sites")"
review_body="$review_body$failed_block"
fi
if [ -n "$review_body" ]; then
review+=(-f "body=$review_body")
fi
if ! resp="$(gh api -X POST "repos/$repo/pulls/$pr/reviews" "${review[@]}" 2>&1)"; then
if printf '%s' "$resp" | grep -qE 'HTTP 403|"status": "403"'; then
readonly_notice
echo "skipped-readonly" > "$status"
exit 0
fi
echo "dejadoc: posting the review failed"
printf '%s\n' "$resp"
echo "failed" > "$status"
exit 1
fi
posted_state="$(printf '%s' "$resp" | jq -r '.state' 2>/dev/null)" || posted_state=''
if [ "$posted_state" != "CHANGES_REQUESTED" ]; then
echo "dejadoc: the review response is unexpected"
printf '%s\n' "$resp"
echo "failed" > "$status"
exit 1
fi
if [ -n "$failed_sites" ]; then
echo "dejadoc: the inline comments failed for the following sites"
printf '%s' "$failed_sites"
fi
echo "dejadoc: posted the review on pull request $pr"
echo "posted" > "$status"