efficient_pca 0.1.8

name: Rust CI

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  push:
    branches: [main]
    if: "!contains(github.event.head_commit.message, 'Merge pull request #')"
  pull_request:
    branches: [main]
    types: [opened, synchronize, reopened]

env:
  CARGO_TERM_COLOR: always

permissions:
  contents: read
  issues: write

jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        backend: [openblas, openblas-system, mkl, faer]
        include:
          - backend: openblas
            feature: backend_openblas
            rustflags: ""
          - backend: openblas-system
            feature: backend_openblas_system
            rustflags: ""
          - backend: mkl
            feature: backend_mkl
            rustflags: "-L/opt/intel/oneapi/mkl/latest/lib/intel64"
          - backend: faer # This name is just a label for the matrix row
            feature: "faer_links_ndarray_static_openblas"
            rustflags: ""
          - backend: openblas-diag # New entry for diagnostic tests
            feature: "backend_openblas,enable-eigensnp-diagnostics"
            rustflags: ""

    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@nightly
        with:
          components: rustfmt, clippy

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.13"

      - name: Cache pip dependencies
        uses: actions/cache@v4
        with:
          path: ~/.local/lib/python${{ steps.setup-python.outputs.python-version }}/site-packages
          key: ${{ runner.os }}-pip-installed-packages-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/rust.yml') }}
          restore-keys: |
            ${{ runner.os }}-pip-installed-packages-${{ steps.setup-python.outputs.python-version }}-

      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip
          pip install numpy scikit-learn scipy pandas matplotlib seaborn

      - name: Cache gfortran apt package
        uses: actions/cache@v4
        id: cache-gfortran
        if: matrix.backend == 'openblas' || matrix.backend == 'openblas-system' || matrix.backend == 'faer' # Only run this step if gfortran is needed
        with:
          path: |
            /var/cache/apt/archives
            /var/lib/apt/lists
          key: ${{ runner.os }}-apt-gfortran-key-gfortran-v1
          restore-keys: |
            ${{ runner.os }}-apt-gfortran-key-gfortran-v1

      - name: Install Fortran compiler
        if: matrix.backend == 'openblas' || matrix.backend == 'openblas-system' || matrix.backend == 'faer'
        run: |
          if [[ "${{ steps.cache-gfortran.outputs.cache-hit }}" != 'true' ]]; then
            sudo apt-get update -qq
          fi
          sudo apt-get install -y gfortran

      - name: Install system OpenBLAS
        if: matrix.backend == 'openblas-system'
        run: |
          sudo apt-get update -qq
          sudo apt-get install -y libopenblas-dev

      - name: Cache MKL apt packages
        uses: actions/cache@v4
        id: cache-mkl-apt
        if: matrix.backend == 'mkl'
        with:
          path: |
            /var/cache/apt/archives
            /var/lib/apt/lists
          key: ${{ runner.os }}-apt-mkl-key-v2-${{ hashFiles('.github/workflows/rust.yml') }}
          restore-keys: |
            ${{ runner.os }}-apt-mkl-key-v2-

      - name: Setup Intel MKL Repository
        if: matrix.backend == 'mkl'
        run: |
          wget -qO- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | 
            sudo gpg --dearmor --output /usr/share/keyrings/intel-archive-keyring.gpg
          echo "deb [signed-by=/usr/share/keyrings/intel-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | 
            sudo tee /etc/apt/sources.list.d/oneAPI.list

      - name: Install Intel MKL
        if: matrix.backend == 'mkl'
        run: |
          if [[ "${{ steps.cache-mkl-apt.outputs.cache-hit }}" != 'true' ]]; then
            echo "MKL installation cache MISS. Installing MKL via apt..."
            # apt-get update is needed here because "Setup Intel MKL Repository" (which runs before this)
            # adds a new repository, and apt needs to be aware of its contents.
            sudo apt-get update -qq 
            sudo apt-get install -y intel-oneapi-mkl-devel
          else
            echo "MKL installation cache HIT. Skipping apt install."
          fi
          # These environment variables must always be set for MKL builds.
          echo "Setting MKL environment variables."
          echo "MKL_ROOT=/opt/intel/oneapi/mkl/latest" >> "$GITHUB_ENV"
          echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" >> "$GITHUB_ENV"

      - name: Cache Cargo global directories (registry, git DBs)
        uses: actions/cache@v4
        id: cache-cargo-global
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
          key: ${{ runner.os }}-cargo-global-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-global-

      - name: Cache Cargo build artifacts (target directory)
        uses: actions/cache@v4
        id: cache-cargo-target
        with:
          path: target
          key: ${{ runner.os }}-cargo-target-deps-${{ matrix.backend }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('**/Cargo.toml') }}
          restore-keys: |
            ${{ runner.os }}-cargo-target-deps-${{ matrix.backend }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('**/Cargo.toml') }}
            ${{ runner.os }}-cargo-target-deps-${{ matrix.backend }}-${{ hashFiles('**/Cargo.lock') }}-
            ${{ runner.os }}-cargo-target-deps-${{ matrix.backend }}-

      - name: Build
        id: build_step
        run: |
          if [[ "${{ matrix.feature }}" == "backend_openblas" ]] || [[ "${{ matrix.feature }}" == "backend_openblas,enable-eigensnp-diagnostics" ]]; then
            cargo build --release --features ${{ matrix.feature }}
          else
            cargo build --release --no-default-features --features ${{ matrix.feature }}
          fi
        env:
          RUSTFLAGS: ${{ matrix.rustflags }}

      - name: Mark build failure
        if: steps.build_step.outcome == 'failure'
        run: |
          mkdir -p /home/runner/work/_temp/build_failure_flags
          touch /home/runner/work/_temp/build_failure_flags/build_failed_${{ matrix.backend }}.flag
      - name: Upload build failure flag
        if: steps.build_step.outcome == 'failure'
        uses: actions/upload-artifact@v4
        with:
          name: build-failure-flag-${{ matrix.backend }}
          path: /home/runner/work/_temp/build_failure_flags/build_failed_${{ matrix.backend }}.flag
          retention-days: 1

      - name: Run core test suite
        run: |
          set -euo pipefail

          FEATURE="${MATRIX_FEATURE}"
          if [[ "$FEATURE" == "backend_openblas" ]] || [[ "$FEATURE" == "backend_openblas,enable-eigensnp-diagnostics" ]]; then
            FEATURE_ARGS=(--features "$FEATURE")
          else
            FEATURE_ARGS=(--no-default-features --features "$FEATURE")
          fi

          echo "Running cargo test for binary and example targets"
          cargo test "${FEATURE_ARGS[@]}" --bins --examples

          echo "Discovering non-eigensnp integration tests"
          readarray -t NON_EIGENSNP_TESTS < <(find tests -maxdepth 1 -type f -name '*.rs' ! -name 'eigensnp*.rs' -exec basename {} .rs \; | sort)

          if (( ${#NON_EIGENSNP_TESTS[@]} == 0 )); then
            echo "No non-eigensnp integration tests detected"
          else
            for test_target in "${NON_EIGENSNP_TESTS[@]}"; do
              echo "Running integration test target: ${test_target}"
              cargo test "${FEATURE_ARGS[@]}" --test "${test_target}"
            done
          fi
        env:
          RUSTFLAGS: ${{ matrix.rustflags }}
          RUST_BACKTRACE: 1
          MATRIX_FEATURE: ${{ matrix.feature }}

      - name: Run eigensnp test suite (non-blocking)
        run: |
          set -euo pipefail

          FEATURE="${MATRIX_FEATURE}"
          if [[ "$FEATURE" == "backend_openblas" ]] || [[ "$FEATURE" == "backend_openblas,enable-eigensnp-diagnostics" ]]; then
            FEATURE_ARGS=(--features "$FEATURE")
          else

            FEATURE_ARGS=(--no-default-features --features "$FEATURE")
          fi

          echo "Discovering eigensnp-specific integration tests"
          readarray -t EIGENSNP_TESTS < <(find tests -maxdepth 1 -type f -name 'eigensnp*.rs' -exec basename {} .rs \; | sort)

          declare -a FAILED_TASKS=()

          if (( ${#EIGENSNP_TESTS[@]} == 0 )); then
            echo "No eigensnp integration tests detected"
          else
            for test_target in "${EIGENSNP_TESTS[@]}"; do
              echo "Running eigensnp integration test target: ${test_target}"
              if ! cargo test "${FEATURE_ARGS[@]}" --test "${test_target}"; then
                echo "::warning::Eigensnp integration test '${test_target}' failed (permitted)."
                FAILED_TASKS+=("integration:${test_target}")
              fi
            done
          fi

          echo "Running documentation tests (treated as eigensnp coverage)"
          if ! cargo test "${FEATURE_ARGS[@]}" --doc; then
            echo "::warning::Documentation tests failed (permitted as eigensnp coverage)."
            FAILED_TASKS+=("doc-tests")
          fi

          if (( ${#FAILED_TASKS[@]} > 0 )); then
            echo "::warning::The following eigensnp checks failed but will not block CI: ${FAILED_TASKS[*]}"
          fi
        env:
          RUSTFLAGS: ${{ matrix.rustflags }}
          RUST_BACKTRACE: 1
          MATRIX_FEATURE: ${{ matrix.feature }}

      - name: Verify contents of target/test_artifacts before upload
        if: always() # Run even if prior steps failed, to see what's there
        run: |
          echo "Listing target/test_artifacts directory for ${{ matrix.backend }}:"
          if [ -d "target/test_artifacts/" ]; then
            ls -R target/test_artifacts/
          else
            echo "target/test_artifacts/ directory does not exist"
            mkdir -p target/test_artifacts/
            echo "Created empty target/test_artifacts/ directory"
          fi
          echo "-----------------------------------------"
          echo "Searching for eigensnp_summary_results.tsv in target/test_artifacts for ${{ matrix.backend }}:"
          find target/test_artifacts/ -name "eigensnp_summary_results.tsv" -print || echo "No eigensnp_summary_results.tsv files found"
          echo "-----------------------------------------"

      - name: Upload eigensnp test summary
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: eigensnp-test-artifacts-${{ matrix.backend }}
          path: target/test_artifacts/
          retention-days: 7 # Optional: Keep for 7 days

      - name: Benchmark
        run: |
          if [[ "${{ matrix.feature }}" == "backend_openblas" ]] || [[ "${{ matrix.feature }}" == "backend_openblas,enable-eigensnp-diagnostics" ]]; then
            cargo bench --features ${{ matrix.feature }},jemalloc
          else
            cargo bench --no-default-features --features ${{ matrix.feature }},jemalloc
          fi
        env:
          RUSTFLAGS: ${{ matrix.rustflags }}
          RUST_BACKTRACE: 1

      - name: Upload Raw Benchmark Results
        if: success()
        uses: actions/upload-artifact@v4
        with:
          name: raw-benchmark-results-${{ matrix.backend }}.tsv
          path: benchmark_raw_results.tsv

      - name: Generate failure report
        if: steps.build_step.outcome == 'failure'
        run: |
          echo ">>> Generate failure report step triggered for ${{ matrix.backend }}."
          set -e
          cargo install getdoc --locked
          getdoc --features ${{ matrix.feature }}
          echo "DEBUG: Checking for report.md after getdoc:"
          ls -al report.md || echo "report.md not found"
          mv report.md failure-report-${{ matrix.backend }}.md
          echo "DEBUG: Checking for failure-report-${{ matrix.backend }}.md after mv:"
          ls -al failure-report-${{ matrix.backend }}.md || echo "failure-report-${{ matrix.backend }}.md not found"

      - name: Upload failure report
        if: steps.build_step.outcome == 'failure'
        uses: actions/upload-artifact@v4
        with:
          name: failure-report-${{ matrix.backend }}
          path: failure-report-${{ matrix.backend }}.md

  consolidate-failures:
    if: always() # Changed: Job now runs always, decision logic is in steps
    needs: test
    runs-on: ubuntu-latest
    steps:
      - name: Download build failure flags
        uses: actions/download-artifact@v4
        id: download-build-flags
        with:
          pattern: build-failure-flag-*
          path: /home/runner/work/_temp/build-failure-flags
          # merge-multiple is true by default for patterns
        continue-on-error: true # Important: don't fail if no flags are found

      - name: List downloaded build flags
        run: |
          mkdir -p /home/runner/work/_temp/build-failure-flags
          echo "Listing contents of /home/runner/work/_temp/build-failure-flags after download:"
          ls -R /home/runner/work/_temp/build-failure-flags
          echo "Looking for *.flag files specifically:"
          find /home/runner/work/_temp/build-failure-flags -type f -name '*.flag' -print || echo "No .flag files found by find"

      - name: Check if any build failure occurred
        id: check-build-failure
        run: |
          mkdir -p /home/runner/work/_temp/build-failure-flags
          if [[ -n "$(find /home/runner/work/_temp/build-failure-flags -type f -name 'build_failed_*.flag' -print -quit)" ]]; then
            echo "Build failure detected."
            echo "BUILD_FAILED=true" >> $GITHUB_OUTPUT
          else
            echo "No build failure detected."
            echo "BUILD_FAILED=false" >> $GITHUB_OUTPUT
          fi

      - name: Download failure reports
        uses: actions/download-artifact@v4
        with:
          pattern: failure-report-*
          path: reports
          merge-multiple: true

      - name: Consolidate reports
        if: steps.check-build-failure.outputs.BUILD_FAILED == 'true' # Changed: Conditioned on build failure
        run: |
          exec > consolidated-report.md
          cat << EOF
          # CI Failure Report
          
          **Run ID:** ${{ github.run_id }}
          **Commit:** ${{ github.sha }}
          **Branch:** ${{ github.ref_name }}
          **Timestamp:** $(date -u +"%Y-%m-%d %H:%M:%S UTC")
          
          EOF
          
          for report in reports/failure-report-*.md; do
            if [[ -f "$report" ]]; then
              backend=$(basename "$report" .md | sed 's/failure-report-//')
              cat << EOF
          
          ## Backend: ${backend}
          
          EOF
              cat "$report"
              echo -e "\n---\n"
            fi
          done

      - name: Upload consolidated report
        if: steps.check-build-failure.outputs.BUILD_FAILED == 'true' # Changed: Conditioned on build failure
        uses: actions/upload-artifact@v4
        with:
          name: consolidated-failure-report
          path: consolidated-report.md

      - name: Close stale issues
        if: github.event_name == 'push' && github.ref == 'refs/heads/main' # Unchanged: Retains original condition
        uses: actions/github-script@v7
        with:
          script: |
            const cutoff = new Date(Date.now() - 24 * 60 * 60 * 1000);
            const { data: issues } = await github.rest.issues.listForRepo({
              owner: context.repo.owner,
              repo: context.repo.repo,
              state: 'open',
              labels: 'ci-failure,automated',
            });
            
            for (const issue of issues) {
              if (new Date(issue.created_at) < cutoff && 
                  issue.title.includes('CI Failure Report')) {
                await github.rest.issues.update({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  issue_number: issue.number,
                  state: 'closed'
                });
              }
            }

      - name: Create failure issue
        if: steps.check-build-failure.outputs.BUILD_FAILED == 'true' # Changed: Conditioned on build failure
        uses: peter-evans/create-issue-from-file@v5
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          title: "CI Failure Report - Run ${{ github.run_id }}"
          content-filepath: consolidated-report.md
          labels: ci-failure,automated

  analyze_benchmarks:
    runs-on: ubuntu-latest
    needs: test
    if: always() # 'test' job (all matrix legs) completed successfully
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.13' # Matched version in 'test' job

      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip
          pip install pandas matplotlib seaborn scipy numpy

      - name: Download benchmark TSV artifacts
        uses: actions/download-artifact@v4
        with:
          # Pattern to download all raw benchmark TSVs from different backends
          pattern: raw-benchmark-results-*.tsv 
          path: benches/benchmark_artifacts 
          merge-multiple: true # Explicitly add this line
          # 'merge-multiple' is true by default if 'name' is not given and 'pattern' is.
          # This should place all downloaded files into the 'path' directory.

      - name: Run benchmark analysis script
        run: python benches/analyze_benchmarks.py

      - name: Upload analysis results
        uses: actions/upload-artifact@v4
        with:
          name: consolidated-benchmark-analysis-results
          path: benches/output/ # Uploads the whole directory

  analyze_results:
    name: Analyze Test Results
    runs-on: ubuntu-latest
    needs: test # Ensures it runs after all matrix legs of the 'test' job
    if: always() # Run even if some test legs failed
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.13" # Match the version in the 'test' job

      - name: Install analysis dependencies
        run: |
          python -m pip install --upgrade pip
          pip install pandas matplotlib seaborn scipy numpy tabulate # Dependencies for the analysis script

      - name: Download all test artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: eigensnp-test-artifacts-* # Downloads all artifacts starting with this pattern
          path: artifacts/ # All downloaded artifacts will be in subdirectories under 'artifacts/'
                           # e.g., artifacts/eigensnp-test-artifacts-openblas/target/test_artifacts/...

      - name: List downloaded artifact structure
        run: |
          echo "Listing contents of artifacts/ directory:"
          ls -R artifacts/
          echo "-----------------------------------------"
          echo "Searching for eigensnp_summary_results.tsv files:"
          find artifacts/ -name "eigensnp_summary_results.tsv" -print
          echo "-----------------------------------------"
          echo "Searching for any .tsv files:"
          find artifacts/ -name "*.tsv" -print
          echo "-----------------------------------------"

      - name: Run analysis script
        run: |
          python tests/analyze_eigensnp_results.py --input-dir artifacts/ --output-dir analysis_output/

      - name: Upload analysis report
        uses: actions/upload-artifact@v4
        with:
          name: eigensnp-analysis-report
          path: analysis_output/
          retention-days: 7