name: Fuzzing
env:
CARGO_INCREMENTAL: "0"
on:
pull_request:
push:
branches:
- '*'
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
uufuzz-examples:
name: Build and test uufuzz examples
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7.0.1
with:
persist-credentials: false
- name: Build uufuzz library
run: |
cd fuzz/uufuzz
cargo build --release
- name: Run uufuzz tests
run: |
cd fuzz/uufuzz
cargo test --lib
- name: Build and run uufuzz examples
run: |
cd fuzz/uufuzz
echo "Building all examples..."
cargo build --examples --release
# Run all examples except integration_testing (which has FD issues in CI)
for example in examples/*.rs; do
example_name=$(basename "$example" .rs)
if [ "$example_name" != "integration_testing" ]; then
cargo run --example "$example_name" --release
fi
done
fuzz-build:
name: Build the fuzzers
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7.0.1
with:
persist-credentials: false
- name: Install `cargo-fuzz`
uses: taiki-e/install-action@v2
with:
tool: cargo-fuzz
- name: Emulate a nightly toolchain
run: |
echo "RUSTC_BOOTSTRAP=1" >> "${GITHUB_ENV}"
- name: Run `cargo-fuzz build`
run: cargo fuzz build --target host-tuple
fuzz-run:
needs: fuzz-build
name: Fuzz (${{ matrix.group.name }})
runs-on: ubuntu-latest
timeout-minutes: 15
env:
RUN_FOR: 60
strategy:
matrix:
group:
- name: smoke
targets: |
fuzz_test:true
fuzz_date:true
fuzz_expr:true
fuzz_printf:true
fuzz_echo:true
- name: parsers
targets: |
fuzz_parse_glob:true
fuzz_parse_size:false
fuzz_parse_time:false
fuzz_seq_parse_number:false
- name: commands-a
targets: |
fuzz_seq:false
fuzz_sort:false
fuzz_wc:false
fuzz_cut:false
fuzz_split:false
- name: commands-b
targets: |
fuzz_tr:false
fuzz_env:false
fuzz_cksum:false
fuzz_non_utf8_paths:true
fuzz_dirname:true
steps:
- uses: actions/checkout@v7.0.1
with:
persist-credentials: false
- name: Install `cargo-fuzz`
uses: taiki-e/install-action@v2
with:
tool: cargo-fuzz
- name: Emulate a nightly toolchain
run: |
echo "RUSTC_BOOTSTRAP=1" >> "${GITHUB_ENV}"
- name: Run ${{ matrix.group.name }} fuzz targets
shell: bash
run: |
set -o pipefail
mkdir -p fuzz/stats
REQUIRED_TARGET_FAILED=0
while IFS=: read -r TARGET SHOULD_PASS; do
[ -n "$TARGET" ] || continue
STATS_FILE="fuzz/stats/${TARGET}.txt"
# Force the correct target
# https://github.com/rust-fuzz/cargo-fuzz/issues/398
set +e
cargo fuzz run --target host-tuple "$TARGET" -- -max_total_time=${{ env.RUN_FOR }} -timeout=${{ env.RUN_FOR }} -detect_leaks=0 -print_final_stats=1 2>&1 | tee "$STATS_FILE"
FUZZ_STATUS=${PIPESTATUS[0]}
set -e
# Save should_pass value to file for summary job to use
echo "$SHOULD_PASS" > "fuzz/stats/${TARGET}.should_pass"
# Print stats to job output for immediate visibility
echo "----------------------------------------"
echo "FUZZING STATISTICS FOR $TARGET"
echo "----------------------------------------"
echo "Runs: $(grep -q "stat::number_of_executed_units" "$STATS_FILE" && grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}' || echo "unknown")"
echo "Execution Rate: $(grep -q "stat::average_exec_per_sec" "$STATS_FILE" && grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}' || echo "unknown") execs/sec"
echo "New Units: $(grep -q "stat::new_units_added" "$STATS_FILE" && grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}' || echo "unknown")"
echo "Expected: $SHOULD_PASS"
echo "Exit code: $FUZZ_STATUS"
if grep -q "SUMMARY: " "$STATS_FILE"; then
echo "Status: $(grep "SUMMARY: " "$STATS_FILE" | head -1)"
else
echo "Status: Completed"
fi
echo "----------------------------------------"
# Add summary to GitHub step summary
echo "### Fuzzing Results for $TARGET" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Metric | Value |" >> $GITHUB_STEP_SUMMARY
echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY
if grep -q "stat::number_of_executed_units" "$STATS_FILE"; then
echo "| Runs | $(grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}') |" >> $GITHUB_STEP_SUMMARY
fi
if grep -q "stat::average_exec_per_sec" "$STATS_FILE"; then
echo "| Execution Rate | $(grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}') execs/sec |" >> $GITHUB_STEP_SUMMARY
fi
if grep -q "stat::new_units_added" "$STATS_FILE"; then
echo "| New Units | $(grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}') |" >> $GITHUB_STEP_SUMMARY
fi
echo "| Should pass | $SHOULD_PASS |" >> $GITHUB_STEP_SUMMARY
echo "| Exit code | $FUZZ_STATUS |" >> $GITHUB_STEP_SUMMARY
if grep -q "SUMMARY: " "$STATS_FILE"; then
echo "| Status | $(grep "SUMMARY: " "$STATS_FILE" | head -1) |" >> $GITHUB_STEP_SUMMARY
else
echo "| Status | Completed |" >> $GITHUB_STEP_SUMMARY
fi
echo "" >> $GITHUB_STEP_SUMMARY
if [ "$SHOULD_PASS" = "true" ] && [ "$FUZZ_STATUS" -ne 0 ]; then
REQUIRED_TARGET_FAILED=1
fi
done <<< "${{ matrix.group.targets }}"
exit "$REQUIRED_TARGET_FAILED"
- name: Upload Stats
if: always()
uses: actions/upload-artifact@v7
with:
name: fuzz-stats-${{ matrix.group.name }}
path: |
fuzz/stats/*.txt
fuzz/stats/*.should_pass
retention-days: 5
fuzz-summary:
needs: fuzz-run
name: Fuzzing Summary
runs-on: ubuntu-latest
if: always()
steps:
- uses: actions/checkout@v7.0.1
with:
persist-credentials: false
- name: Download all stats
uses: actions/download-artifact@v8
with:
path: fuzz/stats-artifacts
pattern: fuzz-stats-*
merge-multiple: true
- name: Prepare stats directory
run: |
mkdir -p fuzz/stats
# Debug: List content of stats-artifacts directory
echo "Contents of stats-artifacts directory:"
find fuzz/stats-artifacts -type f | sort
# Extract files from the artifact directories - handle nested directories
find fuzz/stats-artifacts -type f -name "*.txt" -exec cp {} fuzz/stats/ \;
find fuzz/stats-artifacts -type f -name "*.should_pass" -exec cp {} fuzz/stats/ \;
# Debug information
echo "Contents of stats directory after extraction:"
ls -la fuzz/stats/
echo "Contents of should_pass files (if any):"
cat fuzz/stats/*.should_pass 2>/dev/null || echo "No should_pass files found"
- name: Generate Summary
run: |
echo "# Fuzzing Summary" > fuzzing_summary.md
echo "" >> fuzzing_summary.md
echo "| Target | Runs | Exec/sec | New Units | Should pass | Status |" >> fuzzing_summary.md
echo "|--------|------|----------|-----------|-------------|--------|" >> fuzzing_summary.md
TOTAL_RUNS=0
TOTAL_NEW_UNITS=0
for stat_file in fuzz/stats/*.txt; do
TARGET=$(basename "$stat_file" .txt)
SHOULD_PASS_FILE="${stat_file%.*}.should_pass"
# Get expected status
if [ -f "$SHOULD_PASS_FILE" ]; then
EXPECTED=$(cat "$SHOULD_PASS_FILE")
else
EXPECTED="unknown"
fi
# Extract runs
if grep -q "stat::number_of_executed_units" "$stat_file"; then
RUNS=$(grep "stat::number_of_executed_units" "$stat_file" | awk '{print $2}')
TOTAL_RUNS=$((TOTAL_RUNS + RUNS))
else
RUNS="unknown"
fi
# Extract execution rate
if grep -q "stat::average_exec_per_sec" "$stat_file"; then
EXEC_RATE=$(grep "stat::average_exec_per_sec" "$stat_file" | awk '{print $2}')
else
EXEC_RATE="unknown"
fi
# Extract new units added
if grep -q "stat::new_units_added" "$stat_file"; then
NEW_UNITS=$(grep "stat::new_units_added" "$stat_file" | awk '{print $2}')
if [[ "$NEW_UNITS" =~ ^[0-9]+$ ]]; then
TOTAL_NEW_UNITS=$((TOTAL_NEW_UNITS + NEW_UNITS))
fi
else
NEW_UNITS="unknown"
fi
# Extract status
if grep -q "SUMMARY: " "$stat_file"; then
STATUS=$(grep "SUMMARY: " "$stat_file" | head -1)
else
STATUS="Completed"
fi
echo "| $TARGET | $RUNS | $EXEC_RATE | $NEW_UNITS | $EXPECTED | $STATUS |" >> fuzzing_summary.md
done
echo "" >> fuzzing_summary.md
echo "## Overall Statistics" >> fuzzing_summary.md
echo "" >> fuzzing_summary.md
echo "- **Total runs:** $TOTAL_RUNS" >> fuzzing_summary.md
echo "- **Total new units discovered:** $TOTAL_NEW_UNITS" >> fuzzing_summary.md
echo "- **Average execution rate:** $(grep -h "stat::average_exec_per_sec" fuzz/stats/*.txt | awk '{sum += $2; count++} END {if (count > 0) print sum/count " execs/sec"; else print "unknown"}')" >> fuzzing_summary.md
# Add count by expected status
echo "- **Tests expected to pass:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "true")" >> fuzzing_summary.md
echo "- **Tests expected to fail:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "false")" >> fuzzing_summary.md
# Write to GitHub step summary
cat fuzzing_summary.md >> $GITHUB_STEP_SUMMARY
- name: Show Summary
run: |
cat fuzzing_summary.md
- name: Upload Summary
uses: actions/upload-artifact@v7
with:
name: fuzzing-summary
path: fuzzing_summary.md
retention-days: 5