fp-conformance 0.4.0

Differential conformance harness for frankenpandas against a live pandas oracle — packet fixtures + fuzz seed tests + live-oracle parity gates.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

use std::{fs, path::PathBuf};

fn repo_root() -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..")
}

#[test]
fn parse_live_oracle_report_counts_runs_skips_and_failures() {
    let output = "\
running 5 tests
test a ... ok
live pandas unavailable; skipping dataframe foo oracle test: missing pandas
test b ... ok
test c ... FAILED
test d ... ok
test e ... ok

test result: FAILED. 4 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s
";

    let report = super::parse_live_oracle_report_output(output, 5, false, Some(101));
    assert_eq!(report.expected_tests, 5);
    assert_eq!(report.total_tests, 5);
    assert_eq!(report.passed, 4);
    assert_eq!(report.ran, 3);
    assert_eq!(report.skipped, 1);
    assert_eq!(report.failed, 1);
    assert!(!report.command_ok);
    assert_eq!(report.exit_code, Some(101));
}

#[test]
fn write_live_oracle_report_json_machine_readable_artifact() {
    let tmp = tempfile::tempdir().expect("tmp");
    let mut cfg = super::HarnessConfig::default_paths();
    cfg.repo_root = tmp.path().to_path_buf();

    let report = super::LiveOracleReport {
        expected_tests: 89,
        total_tests: 89,
        passed: 89,
        ran: 89,
        skipped: 0,
        failed: 0,
        command_ok: true,
        exit_code: Some(0),
    };

    let path = super::write_live_oracle_report(&cfg, &report).expect("write live report");
    assert!(
        path.ends_with("artifacts/ci/live_oracle_report.json"),
        "unexpected live oracle report path: {}",
        path.display()
    );

    let written: super::LiveOracleReport =
        serde_json::from_str(&fs::read_to_string(path).expect("read")).expect("json");
    assert_eq!(written, report);
    assert!(written.is_green(true));
}

#[test]
fn ci_workflow_uploads_live_oracle_report_artifact() {
    let root = repo_root();
    let ci = fs::read_to_string(root.join(".github/workflows/ci.yml")).expect("read ci");
    assert!(
        ci.contains("artifacts/ci/live_oracle_report.json"),
        "expected CI artifact upload for live_oracle_report.json"
    );
}

#[test]
fn ci_workflow_requires_live_oracle_system_pandas_fallback() {
    let root = repo_root();
    let ci = fs::read_to_string(root.join(".github/workflows/ci.yml")).expect("read ci");
    let require_count = ci.matches("FP_REQUIRE_LIVE_ORACLE: \"1\"").count();
    let fallback_count = ci.matches("FP_ALLOW_SYSTEM_PANDAS_FALLBACK: \"1\"").count();

    assert!(
        require_count >= 3,
        "expected all CI test/conformance/gate jobs to require live oracle, found {require_count}"
    );
    assert!(
        fallback_count >= 3,
        "expected all CI test/conformance/gate jobs to allow system pandas fallback, found {fallback_count}"
    );
    assert!(
        ci.contains("--allow-system-pandas-fallback"),
        "expected fp-ci-gates to opt into system pandas fallback"
    );
}