logprobe 0.4.0

Detect normalization errors, entropy bias, and truncation artifacts in LLM logprob data
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

//! End-to-end tests that drive the compiled `logprobe` binary.
//!
//! Cargo sets `CARGO_BIN_EXE_logprobe` for integration tests, so no extra
//! dependency is needed to locate and run the binary.

use std::io::Write;
use std::path::PathBuf;
use std::process::{Command, Stdio};

fn bin() -> Command {
    Command::new(env!("CARGO_BIN_EXE_logprobe"))
}

fn fixture(rel: &str) -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(rel)
}

#[test]
fn summary_from_file_argument() {
    let out = bin()
        .arg("summary")
        .arg(fixture("demo/gpt4o_mini_creative.json"))
        .output()
        .expect("run logprobe");
    assert!(out.status.success(), "summary should exit 0");
    let stdout = String::from_utf8(out.stdout).unwrap();
    assert!(stdout.contains("=== Summary ==="));
    assert!(stdout.contains("Tokens:"));
}

#[test]
fn summary_from_stdin() {
    let mut child = bin()
        .arg("summary")
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .spawn()
        .expect("spawn logprobe");
    child
        .stdin
        .take()
        .unwrap()
        .write_all(br#"[{"token":"Hi","logprob":-0.5},{"token":"!","logprob":-1.0}]"#)
        .unwrap();
    let out = child.wait_with_output().unwrap();
    assert!(out.status.success());
    let stdout = String::from_utf8(out.stdout).unwrap();
    assert!(stdout.contains("Tokens:           2"));
}

#[test]
fn compare_preserves_argument_order_in_json() {
    let a = fixture("demo/gpt4o_mini_creative.json");
    let b = fixture("demo/gpt4o_mini_code.json");
    let out = bin()
        .arg("compare")
        .arg(&a)
        .arg(&b)
        .arg("--json")
        .output()
        .expect("run logprobe compare");
    assert!(out.status.success(), "compare should exit 0");

    let report: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap();
    // First positional becomes file_a, second becomes file_b.
    assert_eq!(report["file_a"]["label"], a.to_string_lossy().as_ref());
    assert_eq!(report["file_b"]["label"], b.to_string_lossy().as_ref());
}

#[test]
fn bpb_without_bytes_exits_nonzero_in_text_mode() {
    // The JSONL fixture has no byte arrays, so strict BPB must refuse.
    let out = bin()
        .arg("bpb")
        .arg(fixture("tests/fixtures/stream.jsonl"))
        .output()
        .expect("run logprobe bpb");
    assert!(
        !out.status.success(),
        "BPB without bytes should exit non-zero in text mode"
    );
}

#[test]
fn bpb_without_bytes_json_mode_exits_zero_with_error_status() {
    let out = bin()
        .arg("bpb")
        .arg(fixture("tests/fixtures/stream.jsonl"))
        .arg("--json")
        .output()
        .expect("run logprobe bpb --json");
    assert!(out.status.success(), "JSON mode should still exit 0");
    let parsed: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap();
    assert_eq!(parsed["status"], "error");
}