wikrs 0.4.0

Fast, honest wikitext extraction and parsing in Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

//! Offline integration smoke test for the layer-2 differential pipeline
//! (`docs/TESTING.md` layer 2). It runs real wikitext through the same path
//! `xtask diff-report` uses — `parser::parse` -> `render::plain` ->
//! `diff::classify` — against hand-written ground-truth prose, so CI verifies the
//! end-to-end bucketing WITHOUT network. The full N-page run is
//! `cargo xtask diff-report` (manual / scheduled, per TESTING.md).

use wikrs::diag::Severity;
use wikrs::diff::{self, Bucket};

/// Render a page the way `diff-report` does, and report whether wikrs flagged
/// anything `Unsupported` and whether it rendered a table (the license for the
/// word-precision fallback).
fn run(wikitext: &str) -> (String, bool, bool) {
    let parsed = wikrs::parser::parse(wikitext);
    let text = wikrs::render::plain(&parsed.nodes);
    let has_unsupported = parsed
        .diagnostics
        .iter()
        .any(|d| d.severity == Severity::Unsupported);
    let has_table = parsed
        .nodes
        .iter()
        .any(|n| matches!(n, wikrs::ast::Node::Table { .. }));
    (text, has_unsupported, has_table)
}

#[test]
fn clean_prose_is_faithful_against_superset_truth() {
    // wikrs emits clean prose (bold stripped, links flattened); the ground truth
    // is a superset with an extra, template-expanded sentence. The page is
    // Faithful — never penalized for the content wikrs omits by design.
    let (text, unsupported, has_table) = run("The '''quick''' brown [[fox]] jumps over the lazy \
         dog. It was a [[bright]] cold day in April.");
    assert!(!unsupported, "clean prose should not be flagged");
    let truth = "The quick brown fox jumps over the lazy dog. It was a bright cold \
                 day in April. Foxes are small carnivorous mammals.";
    assert_eq!(
        diff::classify(&text, truth, unsupported, has_table),
        Bucket::Faithful
    );
}

#[test]
fn unsupported_construct_lands_in_reported() {
    // An HTML table is out of declared range -> U-HTML (Unsupported) -> Reported,
    // regardless of how the fallback text compares. Honesty precedence.
    let (text, unsupported, has_table) = run("<table><tr><td>cell</td></tr></table>");
    assert!(unsupported, "an HTML table should be flagged Unsupported");
    assert_eq!(
        diff::classify(&text, "cell", unsupported, has_table),
        Bucket::Reported
    );
}

#[test]
fn silent_disagreement_is_divergent() {
    // Clean parse, no diagnostic, but the emitted prose is absent from the
    // ground truth: the silent-error bucket the project exists to drive to zero.
    let (text, unsupported, has_table) = run("Paris is the capital of France and a major city.");
    assert!(!unsupported);
    let truth = "Berlin is the largest city and capital of Germany.";
    assert_eq!(
        diff::classify(&text, truth, unsupported, has_table),
        Bucket::Divergent
    );
}