assay-cli 3.9.2

CLI for Assay
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

use std::path::Path;
use std::process::Command;

#[test]
fn test_golden_harness() {
    let manifest_dir = env!("CARGO_MANIFEST_DIR");
    let golden_dir = Path::new(manifest_dir).join("../../tests/fixtures/golden"); // Go up to workspace root

    // Build binary first usually, but for test we can use cargo run?
    // Better to use the binary if available. But cargo run is easiest for dev/CI.

    let output = Command::new("cargo")
        .args([
            "run",
            "--quiet",
            "--bin",
            "assay",
            "--",
            "run",
            "--config",
            golden_dir.join("eval.yaml").to_str().unwrap(),
            "--trace-file",
            golden_dir.join("trace.jsonl").to_str().unwrap(),
            "--db",
            ":memory:",
            "--strict",
            "--replay-strict",
        ])
        .output()
        .expect("Failed to execute assay command");

    assert!(
        output.status.success(),
        "Assay run failed: {}",
        String::from_utf8_lossy(&output.stderr)
    );

    let stderr = String::from_utf8(output.stderr).expect("Stderr not UTF-8");

    let golden_stderr = std::fs::read_to_string(golden_dir.join("stderr.golden"))
        .expect("Failed to read golden stderr");

    // Normalize: Remove variable timing info "(0.0s)" or similar if it changes
    // The golden file has "(0.0s)". If it changes to "(0.1s)", test fails.
    // Strip E7 seed line ("Info: No seed provided. Using generated seed: NNNN") so golden is stable.
    // E7.2: Normalize Seeds footer (order_seed/judge_seed vary per run) to fixed placeholder.
    let ansi_re = regex::Regex::new(r"\x1b\[[0-9;]*m").unwrap();
    let duration_re = regex::Regex::new(r"\(\d+\.\d+s\)").unwrap();
    let seed_re =
        regex::Regex::new(r"Info: No seed provided\. Using generated seed: \d+\n\n").unwrap();
    // Match "Seeds:" or "Seeds (replay):"; order_seed is number, judge_seed number or "—" (null until E9)
    let seeds_footer_re = regex::Regex::new(
        r"Seeds( \(replay\))?: seed_version=\d+ order_seed=\d+ judge_seed=(\d+|—)",
    )
    .unwrap();

    let stripped_actual = ansi_re.replace_all(&stderr, "").to_string();
    let without_seed = seed_re.replace_all(&stripped_actual, "").to_string();
    let without_seeds_footer = seeds_footer_re.replace_all(
        &without_seed,
        "Seeds: seed_version=1 order_seed=<SEED> judge_seed=<SEED>",
    );

    let normalized_actual = duration_re
        .replace_all(&without_seeds_footer, "(0.0s)")
        .replace("\r\n", "\n")
        .trim()
        .to_string();
    let normalized_expected = golden_stderr.replace("\r\n", "\n").trim().to_string();

    assert_eq!(
        normalized_actual, normalized_expected,
        "Golden output mismatch"
    );
}