quietset-cli 0.11.0

CLI for quietset — filter datasets by label stability
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192

use std::process::Output;

use assert_cmd::Command;

fn run(args: &[&str]) -> Output {
    Command::cargo_bin("quietset")
        .unwrap()
        .args(args)
        .output()
        .unwrap()
}

fn stdout(out: &Output) -> String {
    String::from_utf8_lossy(&out.stdout).into_owned()
}

fn stderr(out: &Output) -> String {
    String::from_utf8_lossy(&out.stderr).into_owned()
}

fn write_temp(name: &str, contents: &str) -> std::path::PathBuf {
    let path =
        std::env::temp_dir().join(format!("quietset_cli_test_{name}_{}", std::process::id()));
    std::fs::write(&path, contents).unwrap();
    path
}

#[test]
fn score_jsonl_succeeds_on_valid_fixture() {
    let out = run(&["score", "../../tests/fixtures/simple.jsonl"]);
    assert!(out.status.success());
    assert!(stdout(&out).contains("\"sample_id\":\"a\""));
}

#[test]
fn score_invalid_jsonl_without_skip_invalid_fails() {
    let path = write_temp(
        "bad.jsonl",
        "{\"sample_id\":\"a\",\"score\":0.5}\nNOT JSON\n",
    );
    let out = run(&["score", path.to_str().unwrap()]);
    assert!(!out.status.success());
}

#[test]
fn score_invalid_jsonl_with_skip_invalid_succeeds_and_warns() {
    let path = write_temp(
        "bad_skip.jsonl",
        "{\"sample_id\":\"a\",\"score\":0.5}\nNOT JSON\n",
    );
    let out = run(&["score", path.to_str().unwrap(), "--skip-invalid"]);
    assert!(out.status.success());
    assert!(stderr(&out).contains("warning: skipping"));
}

// Regression test for the bug fixed in this session: `--skip-invalid` used to be
// silently ignored for `--format csv`, so a single bad row failed the whole run
// regardless of the flag.
#[test]
fn score_csv_skip_invalid_skips_bad_row() {
    let path = write_temp(
        "bad.csv",
        "sample_id,label,score\na,win,0.9\nb,win,not_a_number\nc,win,0.8\n",
    );

    let failing = run(&["score", path.to_str().unwrap(), "--format", "csv"]);
    assert!(!failing.status.success());

    let ok = run(&[
        "score",
        path.to_str().unwrap(),
        "--format",
        "csv",
        "--skip-invalid",
    ]);
    assert!(ok.status.success());
    assert!(stderr(&ok).contains("warning: skipping row 2"));
}

// Regression test for the bug fixed in this session: `explain` used to swallow
// JSONL parse errors with `.ok()`, so a malformed line anywhere in the file made
// unrelated, later sample_ids report a misleading "not found" instead of the
// real parse error.
#[test]
fn explain_reports_parse_error_not_misleading_not_found() {
    let scored = run(&["score", "../../tests/fixtures/simple.jsonl"]);
    assert!(scored.status.success());
    let scored_text = stdout(&scored);
    let lines: Vec<&str> = scored_text.lines().collect();
    assert!(lines.len() >= 2, "fixture should score at least 2 samples");

    let corrupted = format!("{}\nNOT JSON\n{}\n", lines[0], lines[1]);
    let path = write_temp("explain_corrupted.jsonl", &corrupted);

    // sample_id on line 1 is found before the corrupted line is ever read.
    let found_before_corruption = run(&["explain", path.to_str().unwrap(), "--sample-id", "a"]);
    assert!(found_before_corruption.status.success());

    // sample_id on line 3 requires scanning past the corrupted line 2 — this
    // must surface the parse error, not "sample_id 'b' not found".
    let must_scan_past_corruption = run(&["explain", path.to_str().unwrap(), "--sample-id", "b"]);
    assert!(!must_scan_past_corruption.status.success());
    assert!(stderr(&must_scan_past_corruption).contains("parsing line 2"));
}

#[test]
fn explain_missing_sample_id_without_corruption_reports_not_found() {
    let scored = run(&["score", "../../tests/fixtures/simple.jsonl"]);
    assert!(scored.status.success());
    let path = write_temp("scored_ok.jsonl", &stdout(&scored));

    let out = run(&["explain", path.to_str().unwrap(), "--sample-id", "zzz"]);
    assert!(!out.status.success());
    assert!(stderr(&out).contains("not found"));
}

#[test]
fn score_then_filter_pipeline_via_files() {
    let scored = run(&["score", "../../tests/fixtures/simple.jsonl"]);
    assert!(scored.status.success());
    let path = write_temp("pipeline.jsonl", &stdout(&scored));

    let out = run(&["filter", path.to_str().unwrap(), "--decision", "keep"]);
    assert!(out.status.success());
}

// Regression test for the bug fixed in this session: `filter` had no empty-result
// guard, so if every input row was invalid and dropped via `--skip-invalid`, it
// silently produced 0 lines of output with exit code 0 instead of erroring like
// `summary`/`audit`/`select`/`recommend` already do.
#[test]
fn filter_all_invalid_with_skip_invalid_fails_instead_of_silently_succeeding() {
    let path = write_temp("filter_all_bad.jsonl", "NOT JSON\nALSO NOT JSON\n");
    let out = run(&["filter", path.to_str().unwrap(), "--skip-invalid"]);
    assert!(!out.status.success());
    assert!(stderr(&out).contains("no records found"));
}

fn gold_labeled_fixture() -> std::path::PathBuf {
    write_temp(
        "gold_labeled.jsonl",
        "{\"sample_id\":\"a\",\"label\":\"win\",\"score\":0.9,\"gold_label\":\"win\"}\n\
         {\"sample_id\":\"a\",\"label\":\"win\",\"score\":0.9,\"gold_label\":\"win\"}\n\
         {\"sample_id\":\"a\",\"label\":\"win\",\"score\":0.9,\"gold_label\":\"win\"}\n\
         {\"sample_id\":\"b\",\"label\":\"win\",\"score\":0.9,\"gold_label\":\"win\"}\n\
         {\"sample_id\":\"b\",\"label\":\"win\",\"score\":0.9,\"gold_label\":\"win\"}\n\
         {\"sample_id\":\"b\",\"label\":\"win\",\"score\":0.9,\"gold_label\":\"win\"}\n",
    )
}

#[test]
fn calibrate_csv_output_has_header_and_one_data_row() {
    let path = gold_labeled_fixture();
    let out = run(&[
        "calibrate",
        path.to_str().unwrap(),
        "--target-precision",
        "0.9",
        "--output-format",
        "csv",
    ]);
    assert!(out.status.success());
    let text = stdout(&out);
    let mut lines = text.lines();
    let header = lines.next().expect("header row");
    assert_eq!(
        header,
        "decision_score,keep_threshold,drop_threshold,achieved_precision,precision_ci_low,precision_ci_high,coverage,n_keep,n_total"
    );
    let data = lines.next().expect("one data row");
    assert_eq!(data.split(',').count(), 9);
    assert!(lines.next().is_none(), "should be exactly one data row");
}

#[test]
fn policy_csv_output_has_header_and_fixed_columns() {
    let path = gold_labeled_fixture();
    let out = run(&["policy", path.to_str().unwrap(), "--output-format", "csv"]);
    assert!(out.status.success());
    let text = stdout(&out);
    let mut lines = text.lines();
    let header = lines.next().expect("header row");
    assert_eq!(
        header,
        "threshold,n_keep,coverage,precision,stable_wrong_rate,best"
    );
    let data_rows: Vec<&str> = lines.collect();
    assert_eq!(data_rows.len(), 50, "one row per swept threshold");
    for row in &data_rows {
        assert_eq!(row.split(',').count(), 6);
    }
}