1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
use serde::{Deserialize, Serialize};
/// A single evaluation run for one sample.
///
/// All fields except `sample_id` are optional — provide only what your
/// pipeline produces. The library uses whichever fields are present to
/// compute the relevant stability sub-scores.
#[derive(Debug, Clone, Default, Deserialize, Serialize)]
pub struct Observation {
/// Identifies the sample across all its repeated evaluations.
#[serde(default)]
pub sample_id: String,
/// Categorical label assigned in this evaluation run.
pub label: Option<String>,
/// Numeric evaluation score for this run.
pub score: Option<f64>,
/// Identifies the evaluator (model, human annotator, or tool).
pub evaluator_id: Option<String>,
/// Compute budget consumed by this evaluation (e.g. search depth, token count).
pub budget: Option<f64>,
/// Random seed used for this run.
pub seed: Option<u64>,
/// Model checkpoint or version that produced this evaluation.
pub model_id: Option<String>,
/// Unique identifier for this evaluation run.
pub run_id: Option<String>,
/// Known correct label for this sample. When provided, `quietset reliability` uses it
/// instead of majority label to compute evaluator reliability.
pub gold_label: Option<String>,
/// Second seed axis (data-order/shuffle seed), distinct from `seed` (init seed).
pub shuffle_seed: Option<u64>,
/// Loss-function or training-recipe variant used for this run. Same role as `model_id`/
/// `evaluator_id`: a categorical condition axis.
pub loss_recipe: Option<String>,
/// Groups this sample into a training block (e.g. a 32-position block) for block-level
/// stability analysis, distinct from `sample_id`.
///
/// Must be unique within one input dataset and within one `trajectory-audit` before/after
/// pair — quietset has no run-level field to disambiguate an accidental ID collision from
/// an intentional cross-checkpoint match. A producer combining independent training runs
/// into one file must namespace the identifier itself, e.g. `"{run_id}:{block_id}"`.
pub block_id: Option<String>,
/// Generic layer identifier for `dead_unit_count`/`saturated_unit_count`. quietset assigns
/// no meaning to specific values — the caller's training harness decides what "ft", "l2",
/// or any other layer name means.
pub layer_id: Option<String>,
/// Signed per-run scalar whose sign indicates gradient direction for this sample. Only the
/// sign is used (mirrors `score`'s role in `score_sign_agreement`).
pub gradient_sign: Option<f64>,
/// Per-run cosine similarity between this sample's parameter-update contribution and a
/// reference direction (e.g. the mean update direction across seeds), computed upstream.
pub update_cosine: Option<f64>,
/// Per-run signed scalar capturing disagreement with a teacher/reference model (e.g.
/// model output minus teacher output).
pub teacher_residual: Option<f64>,
/// Signed per-run scalar driving growth/shrink classification (e.g. a radial-norm delta).
/// Positive = growth, negative = shrink.
pub trajectory_effect: Option<f64>,
/// Count of newly-dead units in `layer_id` observed for this (sample, run).
pub dead_unit_count: Option<f64>,
/// Count of newly-saturated units in `layer_id` observed for this (sample, run).
pub saturated_unit_count: Option<f64>,
/// Identifies the root artifact this sample was derived from — one game record/kifu, one
/// source document, one conversation. Samples sharing a `source_root_id` are not
/// independent: nearby positions from the same game record correlate strongly, so a
/// calibration/held-out split that puts some on each side leaks information and makes the
/// held-out precision optimistic. `calibrate --group-by source-root-id` reports that
/// leakage; quietset never builds or repairs the split itself.
pub source_root_id: Option<String>,
/// Coarser grouping above `source_root_id` (e.g. an opening family for game records, a
/// topic cluster for judge data). Same independence caveat one level up: two different
/// game records from the same opening family still share structure.
pub opening_family: Option<String>,
}
impl Observation {
/// Validate required and numeric fields. `line` is reported in error messages.
pub fn validate(&self, line: usize) -> crate::error::Result<()> {
if self.sample_id.trim().is_empty() {
return Err(crate::error::Error::MissingField("sample_id"));
}
if self.score.is_some_and(|s| !s.is_finite()) {
return Err(crate::error::Error::InvalidScore { line });
}
if self.budget.is_some_and(|b| !b.is_finite()) {
return Err(crate::error::Error::InvalidBudget { line });
}
for (field, value) in [
("gradient_sign", self.gradient_sign),
("update_cosine", self.update_cosine),
("teacher_residual", self.teacher_residual),
("trajectory_effect", self.trajectory_effect),
("dead_unit_count", self.dead_unit_count),
("saturated_unit_count", self.saturated_unit_count),
] {
if value.is_some_and(|v| !v.is_finite()) {
return Err(crate::error::Error::InvalidField { field, line });
}
}
Ok(())
}
}
/// Parse observations from a JSONL string, returning typed errors with line numbers.
pub fn parse_jsonl(input: &str) -> crate::error::Result<Vec<Observation>> {
let mut out = Vec::new();
for (i, line) in input.lines().enumerate() {
let line = line.trim();
if line.is_empty() {
continue;
}
let obs: Observation =
serde_json::from_str(line).map_err(|source| crate::error::Error::ParseError {
line: i + 1,
source,
})?;
obs.validate(i + 1)?;
out.push(obs);
}
Ok(out)
}
/// Parse observations from CSV bytes.
pub fn parse_csv(input: &[u8]) -> crate::error::Result<Vec<Observation>> {
let mut rdr = csv::Reader::from_reader(input);
let mut out = Vec::new();
for (i, record) in rdr.deserialize::<Observation>().enumerate() {
let obs = record?;
obs.validate(i + 1)?;
out.push(obs);
}
Ok(out)
}