wikiwho 0.3.3

name: CI

on:
  push:
    branches: [main]
  pull_request:

# Cancel superseded runs for the same ref.
concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
  # Cap proptest case counts so the 3×100k-case suites in optimized_str stay fast in CI.
  # Known failures are still always replayed from the committed *.proptest-regressions files.
  PROPTEST_CASES: "4096"

jobs:
  fmt:
    name: rustfmt
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with: { lfs: false }
      - uses: dtolnay/rust-toolchain@stable
        with: { components: rustfmt }
      - run: cargo fmt --all --check

  clippy:
    name: clippy
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        # Python-free feature combinations (python-diff is linted in the parity job).
        features:
          - "--no-default-features"
          - "" # default
          - "--features serde"
          - "--features cli"
          - "--features serde,cli,strict,optimized-str,optimized-lowercase"
    steps:
      - uses: actions/checkout@v4
        with: { lfs: false }
      - uses: dtolnay/rust-toolchain@stable
        with: { components: clippy }
      - uses: Swatinem/rust-cache@v2
        with:
          key: clippy-${{ matrix.features }}
      - run: cargo clippy --all-targets ${{ matrix.features }} -- -D warnings

  test:
    name: test (lib + doc)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with: { lfs: false }
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      # Superset of non-python features runs every unit test (incl. serde round-trips and
      # both optimized/naive string paths).
      - run: cargo test --lib --features serde,cli,strict,optimized-str,optimized-lowercase
      - run: cargo test --doc --features serde

  doc:
    name: doc
    runs-on: ubuntu-latest
    env:
      RUSTDOCFLAGS: "-D warnings"
    steps:
      - uses: actions/checkout@v4
        with: { lfs: false }
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      # --all-features pulls in python-diff; pyo3's build script only needs the
      # interpreter (preinstalled on the runner), not a venv, to build docs.
      - run: cargo doc --no-deps --all-features

  msrv:
    name: msrv (1.94.1)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with: { lfs: false }
      - uses: dtolnay/rust-toolchain@1.94.1
      - uses: Swatinem/rust-cache@v2
      - run: cargo check --all-features

  coverage:
    name: coverage
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with: { lfs: false }
      - uses: dtolnay/rust-toolchain@stable
        with: { components: llvm-tools-preview }
      - uses: Swatinem/rust-cache@v2
      - uses: taiki-e/install-action@cargo-llvm-cov
      - name: Collect coverage (pure-Rust lib tests)
        run: cargo llvm-cov --lib --features serde,cli,strict,optimized-str,optimized-lowercase --lcov --output-path lcov.info
      - uses: codecov/codecov-action@v4
        with:
          files: lcov.info
          fail_ci_if_error: false
          token: ${{ secrets.CODECOV_TOKEN }}

  package:
    name: cargo package
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with: { lfs: false }
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - run: cargo package --no-verify

  semver:
    name: semver (all features)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with: { lfs: false }
      # cargo-semver-checks compares the public API against the latest crates.io release and
      # fails if the version in Cargo.toml doesn't reflect the change (0.x rules understood) —
      # so a PR that makes a breaking change must bump the version (the minor field, for 0.x).
      # --all-features covers the whole surface, incl. the python-diff `use_python_diff` toggle;
      # pyo3 only needs a Python interpreter to *build* (not the WikiWho venv — rustdoc is
      # compiled, never run). This is also what enforces SemVer for releases: release.yml
      # requires this workflow to be green for the tagged commit.
      - uses: actions/setup-python@v5
        with: { python-version: "3.12" }
      - uses: obi1kenobi/cargo-semver-checks-action@v2
        with:
          feature-group: all-features

  changelog:
    name: changelog entry
    # PRs must record their change under [Unreleased] in CHANGELOG.md. Skipped for Dependabot
    # and for PRs carrying the `skip-changelog` label (CI/docs/refactors that don't warrant one).
    if: github.event_name == 'pull_request' && github.actor != 'dependabot[bot]' && !contains(github.event.pull_request.labels.*.name, 'skip-changelog')
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with: { fetch-depth: 0, lfs: false }
      - name: Require a CHANGELOG.md update
        run: |
          git fetch --no-tags origin "$GITHUB_BASE_REF"
          if git diff --name-only "origin/$GITHUB_BASE_REF...HEAD" | grep -qx 'CHANGELOG.md'; then
            echo "CHANGELOG.md updated."
          else
            echo "::error::Add an entry under [Unreleased] in CHANGELOG.md, or apply the 'skip-changelog' label for changes that don't warrant one (CI, docs, refactors)."
            exit 1
          fi

  # --- Headline: prove the Rust implementation matches the reference Python WikiWho ---

  parity-deterministic:
    name: parity vs Python (deterministic)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with: { lfs: false }
      # Selective LFS: fetch only the small representative subset, never the 808 MB dump.
      - name: Fetch representative dump subset (LFS)
        run: git lfs pull --include="dev-data/reference-dumps/dewiktionary-20240901-ci-subset.xml.zst"
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - uses: actions/setup-python@v5
        with: { python-version: "3.12" }
      - name: Set up reference WikiWho venv
        run: |
          python -m venv .venv
          ./.venv/bin/pip install --upgrade pip
          ./.venv/bin/pip install -r requirements.txt
          # "Activate" the venv for later steps so pyo3's auto-initialized interpreter
          # finds the installed WikiWho module (VIRTUAL_ENV is what site.py keys on).
          echo "VIRTUAL_ENV=$PWD/.venv" >> "$GITHUB_ENV"
          echo "PYO3_PYTHON=$PWD/.venv/bin/python" >> "$GITHUB_ENV"
          echo "$PWD/.venv/bin" >> "$GITHUB_PATH"
      - name: Deterministic parity (synthetic fixtures + real subset pages)
        run: |
          SUBSET="$PWD/dev-data/reference-dumps/dewiktionary-20240901-ci-subset.xml.zst"
          if [ -f "$SUBSET" ]; then
            export WIKIWHO_TEST_DUMP="$SUBSET"
            echo "Using representative subset: $SUBSET"
          else
            echo "::warning::subset not available; real-page parity tests will self-skip"
          fi
          cargo test --features python-diff,serde --test algorithm_exact_tests -- \
            known_bad_example_anontalkpagetext \
            known_bad_example_familia \
            known_bad_example_hallo \
            random_pages_100 \
            test_case_
      - name: Deep parity on full dump (push to main only)
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          if python3 tools/fetch_test_data.py --which full --dest dev-data/reference-dumps; then
            export WIKIWHO_TEST_DUMP="$PWD/dev-data/reference-dumps/dewiktionary-20240901-pages-meta-history.xml.zst"
            cargo test --features python-diff,serde --test algorithm_exact_tests -- \
              known_bad_example_familia known_bad_example_hallo random_pages_100
          else
            echo "::warning::full dump unavailable from wikiwho-data; skipping deep parity"
          fi

  gold-standard:
    name: gold standard accuracy (pure Rust)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with: { lfs: false }
      # Selective LFS: fetch only the cached gold-standard article pages, never the dumps.
      - name: Fetch gold-standard article cache (LFS)
        run: git lfs pull --include="dev-data/article-cache/*.json.zst"
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      # The CSV is fetched from a Wayback snapshot and SHA-256-verified by the script. Cache
      # it (keyed on the script, which pins the expected hash) so an occasional archive.org
      # hiccup can't fail unrelated PRs.
      - name: Cache gold-standard CSV
        id: gold-csv
        uses: actions/cache@v4
        with:
          path: dev-data/gold_standard.partial.newnames.csv
          key: gold-standard-csv-${{ hashFiles('tools/fetch_gold_standard.py') }}
      - name: Fetch the archived gold-standard CSV
        if: steps.gold-csv.outputs.cache-hit != 'true'
        run: python3 tools/fetch_gold_standard.py # stdlib only; the runner's python3 suffices
      - name: Precision vs. paper gold standard (imara-diff / pure Rust)
        run: |
          cargo test --features serde --test algorithm_statistic_tests -- \
            --ignored --nocapture gold_standard_precision_rust

  gold-standard-parity:
    name: gold standard vs Python (push to main)
    # The python-diff statistical baselines need the reference WikiWho venv; like the deep
    # parity run, they're reserved for push-to-main to keep PRs fast.
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with: { lfs: false }
      - name: Fetch gold-standard article cache (LFS)
        run: git lfs pull --include="dev-data/article-cache/*.json.zst"
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - uses: actions/setup-python@v5
        with: { python-version: "3.12" }
      - name: Set up reference WikiWho venv
        run: |
          python -m venv .venv
          ./.venv/bin/pip install --upgrade pip
          ./.venv/bin/pip install -r requirements.txt
          # "Activate" the venv for later steps so pyo3's auto-initialized interpreter
          # finds the installed WikiWho module (VIRTUAL_ENV is what site.py keys on).
          echo "VIRTUAL_ENV=$PWD/.venv" >> "$GITHUB_ENV"
          echo "PYO3_PYTHON=$PWD/.venv/bin/python" >> "$GITHUB_ENV"
          echo "$PWD/.venv/bin" >> "$GITHUB_PATH"
      - name: Cache gold-standard CSV
        id: gold-csv
        uses: actions/cache@v4
        with:
          path: dev-data/gold_standard.partial.newnames.csv
          key: gold-standard-csv-${{ hashFiles('tools/fetch_gold_standard.py') }}
      - name: Fetch the archived gold-standard CSV
        if: steps.gold-csv.outputs.cache-hit != 'true'
        run: python3 tools/fetch_gold_standard.py
      - name: Python-diff precision + Rust/Python divergence
        run: |
          cargo test --features python-diff,serde --test algorithm_statistic_tests -- \
            --ignored --nocapture \
            gold_standard_precision_python_diff \
            divergence_rate_gold_standard_articles