article_scraper 3.0.0-alpha1

//! Review report for fixture changes.
//!
//! `cargo test -p article_scraper --all-features -- --ignored fixture_report` runs every FTR and
//! readability fixture, compares the output with `expected.html` in canonical form and writes
//! `target/fixture-review/index.html` with one entry per changed fixture:
//!
//! - expected and actual output side by side in sandboxed iframes (no scripts),
//! - a unified diff of the canonical forms,
//! - how similar the text content is, with the largest changes sorted to the top.
//!
//! Review the report, then update the accepted fixtures with
//! `UPDATE_FIXTURES=1 cargo test -p article_scraper --all-features <test filter>`.

use super::readability::tests::{fixture_path as readability_path, readability_fixture_output};
use super::tests::{FTR_FIXTURES, fixture_path as ftr_path, ftr_fixture_output};
use crate::test_util::canonicalize;
use dom_query::Document;
use similar::TextDiff;
use std::fmt::Write;
use std::path::PathBuf;
use std::time::Duration;

const DIFF_TIMEOUT: Duration = Duration::from_secs(2);

struct Entry {
    kind: &'static str,
    name: String,
    expected: Option<String>,
    outcome: Result<String, String>,
}

enum Status {
    Unchanged,
    Changed { similarity: f32, diff: String },
    New,
    Failed(String),
}

impl Entry {
    fn status(&self) -> Status {
        let actual = match &self.outcome {
            Ok(actual) => actual,
            Err(error) => return Status::Failed(error.clone()),
        };
        let Some(expected) = &self.expected else {
            return Status::New;
        };

        let expected_canonical = canonicalize(expected);
        let actual_canonical = canonicalize(actual);
        if expected_canonical == actual_canonical {
            return Status::Unchanged;
        }

        let diff = TextDiff::configure()
            .timeout(DIFF_TIMEOUT)
            .diff_lines(&expected_canonical, &actual_canonical)
            .unified_diff()
            .context_radius(3)
            .header("expected", "actual")
            .to_string();

        let expected_text = text_content(expected);
        let actual_text = text_content(actual);
        let similarity = TextDiff::configure()
            .timeout(DIFF_TIMEOUT)
            .diff_words(&expected_text, &actual_text)
            .ratio();

        Status::Changed { similarity, diff }
    }
}

fn text_content(html: &str) -> String {
    let text = Document::from(html).root().text();
    text.split_ascii_whitespace().collect::<Vec<_>>().join(" ")
}

fn escape(text: &str) -> String {
    let mut out = String::with_capacity(text.len());
    for c in text.chars() {
        match c {
            '&' => out.push_str("&amp;"),
            '<' => out.push_str("&lt;"),
            '>' => out.push_str("&gt;"),
            '"' => out.push_str("&quot;"),
            '\'' => out.push_str("&#39;"),
            c => out.push(c),
        }
    }
    out
}

fn report_path() -> PathBuf {
    let target = std::env::var_os("CARGO_TARGET_DIR")
        .map(PathBuf::from)
        .unwrap_or_else(|| PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/../target")));
    let target = std::fs::canonicalize(&target).unwrap_or(target);
    target.join("fixture-review")
}

async fn collect_entries() -> Vec<Entry> {
    let mut entries = Vec::new();

    for (name, _) in FTR_FIXTURES {
        let outcome = ftr_fixture_output(name)
            .await
            .and_then(|article| article.html.ok_or_else(|| "no html extracted".into()));
        entries.push(Entry {
            kind: "ftr",
            name: name.to_string(),
            expected: std::fs::read_to_string(ftr_path(name, "expected.html")).ok(),
            outcome,
        });
    }

    let mut readability_names = std::fs::read_dir("./resources/tests/readability")
        .expect("Failed to list readability fixtures")
        .filter_map(|entry| entry.ok())
        .filter(|entry| entry.path().join("source.html").exists())
        .map(|entry| entry.file_name().to_string_lossy().into_owned())
        .collect::<Vec<_>>();
    readability_names.sort();

    for name in readability_names {
        let outcome = std::panic::catch_unwind(|| readability_fixture_output(&name))
            .unwrap_or_else(|_| Err("panicked".into()));
        entries.push(Entry {
            kind: "readability",
            expected: std::fs::read_to_string(readability_path(&name, "expected.html")).ok(),
            name,
            outcome,
        });
    }

    entries
}

fn render(entries: &[Entry]) -> (String, usize) {
    let mut rows = entries
        .iter()
        .map(|entry| (entry, entry.status()))
        .filter(|(_, status)| !matches!(status, Status::Unchanged))
        .collect::<Vec<_>>();
    let rank = |status: &Status| match status {
        Status::Failed(_) => -2.0,
        Status::New => -1.0,
        Status::Changed { similarity, .. } => *similarity,
        Status::Unchanged => 2.0,
    };
    rows.sort_by(|(_, a), (_, b)| rank(a).total_cmp(&rank(b)));

    let mut html = String::new();
    _ = write!(
        html,
        r#"<!DOCTYPE html>
<html><head><meta charset="utf-8"><title>Fixture review</title>
<style>
body {{ font-family: sans-serif; margin: 1em; }}
section {{ border-top: 2px solid #888; margin-top: 2em; padding-top: .5em; }}
.panes {{ display: flex; gap: 1em; }}
.pane {{ flex: 1; min-width: 0; }}
iframe {{ width: 100%; height: 600px; border: 1px solid #ccc; resize: vertical; }}
pre {{ background: #f6f6f6; padding: .5em; overflow-x: auto; max-height: 600px; }}
.add {{ background: #dfd; }} .del {{ background: #fdd; }} .hunk {{ color: #888; }}
.error {{ color: #b00; }}
</style></head><body>
<h1>Fixture review</h1>
<p>{total} fixtures, {changed} changed or new or failed, {unchanged} unchanged. Sorted by text similarity, lowest first.</p>
<p>Accept a fixture with <code>UPDATE_FIXTURES=1 cargo test -p article_scraper --all-features &lt;test name&gt;</code>.</p>
<ul>"#,
        total = entries.len(),
        changed = rows.len(),
        unchanged = entries.len() - rows.len(),
    );

    for (entry, status) in &rows {
        let label = match status {
            Status::Failed(_) => "failed".to_string(),
            Status::New => "new".to_string(),
            Status::Changed { similarity, .. } => format!("{:.1}% similar", similarity * 100.0),
            Status::Unchanged => unreachable!(),
        };
        _ = write!(
            html,
            "<li><a href=\"#{kind}-{name}\">{kind}/{name}</a>: {label}</li>",
            kind = entry.kind,
            name = escape(&entry.name),
        );
    }
    html.push_str("</ul>");

    for (entry, status) in &rows {
        _ = write!(
            html,
            "<section id=\"{kind}-{name}\"><h2>{kind}/{name}</h2>",
            kind = entry.kind,
            name = escape(&entry.name),
        );

        match status {
            Status::Failed(error) => {
                _ = write!(html, "<p class=\"error\">Failed: {}</p>", escape(error));
            }
            Status::New => html.push_str("<p>No expected.html yet.</p>"),
            Status::Changed { similarity, .. } => {
                _ = write!(html, "<p>Text similarity: {:.1}%</p>", similarity * 100.0);
            }
            Status::Unchanged => unreachable!(),
        }

        html.push_str("<div class=\"panes\">");
        for (title, content) in [
            ("expected", entry.expected.as_deref()),
            ("actual", entry.outcome.as_deref().ok()),
        ] {
            _ = write!(html, "<div class=\"pane\"><h3>{title}</h3>");
            match content {
                Some(content) => {
                    // An empty sandbox attribute disables scripts, forms, plugins and navigation.
                    _ = write!(
                        html,
                        "<iframe sandbox=\"\" loading=\"lazy\" srcdoc=\"{}\"></iframe>",
                        escape(content)
                    );
                }
                None => html.push_str("<p>none</p>"),
            }
            html.push_str("</div>");
        }
        html.push_str("</div>");

        if let Status::Changed { diff, .. } = status {
            html.push_str("<details open><summary>Canonical diff</summary><pre>");
            for line in diff.lines() {
                let class = if line.starts_with("@@") {
                    "hunk"
                } else if line.starts_with('+') {
                    "add"
                } else if line.starts_with('-') {
                    "del"
                } else {
                    ""
                };
                _ = writeln!(html, "<span class=\"{class}\">{}</span>", escape(line));
            }
            html.push_str("</pre></details>");
        }

        html.push_str("</section>");
    }

    html.push_str("</body></html>\n");
    (html, rows.len())
}

#[tokio::test]
#[ignore = "writes the fixture review report; run on request"]
async fn fixture_report() {
    let entries = collect_entries().await;
    let (html, changed) = render(&entries);

    let dir = report_path();
    std::fs::create_dir_all(&dir).expect("Failed to create report directory");
    let path = dir.join("index.html");
    std::fs::write(&path, html).expect("Failed to write report");

    println!(
        "{changed} of {} fixtures changed. Report: {}",
        entries.len(),
        path.display()
    );
}