1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
use indexmap::IndexMap;
use serde::{Deserialize, Serialize};
/// The filtering decision for a sample based on its stability score.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Decision {
/// Stable sample — include in the filtered dataset.
Keep,
/// Borderline sample — manual review recommended.
Review,
/// Unstable sample — exclude from the filtered dataset.
Drop,
}
impl std::fmt::Display for Decision {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Decision::Keep => write!(f, "keep"),
Decision::Review => write!(f, "review"),
Decision::Drop => write!(f, "drop"),
}
}
}
/// Per-dimension sub-scores that contributed to `stability_score`.
///
/// Each value is in `[0.0, 1.0]` where `1.0` is fully stable.
/// Fields are omitted when the dimension was not computable (e.g. no labels, no budgets).
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct StabilityComponents {
/// Label agreement across observations.
#[serde(skip_serializing_if = "Option::is_none")]
pub label: Option<f64>,
/// Score consistency (`1 - normalized_score_std`).
#[serde(skip_serializing_if = "Option::is_none")]
pub score_consistency: Option<f64>,
/// Fraction of numeric scores sharing the majority sign. Only contributes to
/// `stability_score` when `ScoreWeights::score_sign` is set above its default `0.0`.
#[serde(skip_serializing_if = "Option::is_none")]
pub score_sign_agreement: Option<f64>,
/// Budget robustness (`1 - budget_sensitivity`).
#[serde(skip_serializing_if = "Option::is_none")]
pub budget_robustness: Option<f64>,
/// Seed robustness (`1 - seed_sensitivity`).
#[serde(skip_serializing_if = "Option::is_none")]
pub seed_robustness: Option<f64>,
/// Label agreement across models.
#[serde(skip_serializing_if = "Option::is_none")]
pub model_agreement: Option<f64>,
/// Label agreement across evaluators.
#[serde(skip_serializing_if = "Option::is_none")]
pub evaluator_agreement: Option<f64>,
}
impl StabilityComponents {
/// Returns the name and value of the weakest (lowest) component, if any exist.
///
/// Ties are resolved by the fixed component declaration order:
/// `label` → `score_consistency` → `budget_robustness` → `seed_robustness`
/// → `model_agreement` → `evaluator_agreement`.
pub fn weakest(&self) -> Option<(&'static str, f64)> {
let candidates = [
("label", self.label),
("score_consistency", self.score_consistency),
("score_sign_agreement", self.score_sign_agreement),
("budget_robustness", self.budget_robustness),
("seed_robustness", self.seed_robustness),
("model_agreement", self.model_agreement),
("evaluator_agreement", self.evaluator_agreement),
];
candidates
.into_iter()
.filter_map(|(name, val)| val.map(|v| (name, v)))
.min_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal))
}
}
/// Stability report for one sample, aggregated across all its observations.
///
/// All `Option` fields are omitted from JSON output when absent so that
/// downstream tools only see metrics that were actually computable.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StabilityReport {
/// The sample this report describes.
pub sample_id: String,
/// Number of observations used to compute this report.
pub n_observations: usize,
/// The label that appeared most often across observations.
#[serde(skip_serializing_if = "Option::is_none")]
pub majority_label: Option<String>,
/// Fraction of observations that carry the majority label (`[0.0, 1.0]`).
#[serde(skip_serializing_if = "Option::is_none")]
pub label_agreement: Option<f64>,
/// Wilson confidence interval lower bound of label_agreement at `confidence_level`.
#[serde(skip_serializing_if = "Option::is_none")]
pub label_agreement_lcb: Option<f64>,
/// (majority_count - runner_up_count) / total_labels. 1.0 if only one label type. None if no labels.
#[serde(skip_serializing_if = "Option::is_none")]
pub label_margin: Option<f64>,
/// Normalized Shannon entropy of label distribution [0, 1]. 0.0 = unanimous, 1.0 = uniform. None if no labels.
#[serde(skip_serializing_if = "Option::is_none")]
pub label_entropy: Option<f64>,
/// Fraction of observations per label, sorted by frequency descending. None if no labels.
#[serde(skip_serializing_if = "Option::is_none")]
pub label_distribution: Option<IndexMap<String, f64>>,
/// Reliability-weighted majority label. None if no labels or no evaluator_ids present.
#[serde(skip_serializing_if = "Option::is_none")]
pub weighted_majority_label: Option<String>,
/// Confidence of weighted_majority_label: max weighted fraction in [0, 1]. None if not computed.
#[serde(skip_serializing_if = "Option::is_none")]
pub weighted_label_confidence: Option<f64>,
/// Reliability-weighted label distribution, sorted by weight descending. None if not computed.
#[serde(skip_serializing_if = "Option::is_none")]
pub weighted_label_distribution: Option<IndexMap<String, f64>>,
/// True when weighted_majority_label differs from majority_label. None if not computed.
#[serde(skip_serializing_if = "Option::is_none")]
pub majority_weighted_conflict: Option<bool>,
/// EM-estimated most likely true label (Dawid-Skene posterior argmax), inferred from
/// disagreement patterns alone — no `gold_label` needed. None if latent-truth EM did not
/// run for this sample (see `DecisionScore::LatentTruth`'s fallback conditions).
#[serde(skip_serializing_if = "Option::is_none")]
pub latent_truth_label: Option<String>,
/// Posterior probability of `latent_truth_label`, in `[0.0, 1.0]`. None if not computed.
#[serde(skip_serializing_if = "Option::is_none")]
pub latent_truth_confidence: Option<f64>,
/// Full EM posterior distribution over labels for this sample, sorted by probability
/// descending. None if not computed.
#[serde(skip_serializing_if = "Option::is_none")]
pub latent_truth_label_distribution: Option<IndexMap<String, f64>>,
/// True when latent_truth_label differs from majority_label. None if not computed.
#[serde(skip_serializing_if = "Option::is_none")]
pub majority_latent_conflict: Option<bool>,
/// Effective number of independent evaluators behind `latent_truth_label`, correcting for
/// pairwise-correlated evaluator errors (Kish design effect). Equals the nominal
/// qualifying evaluator count when no correlation is detected; always in `(0, n]`.
/// Diagnostic of self-consistency only — correlation is measured against EM's own
/// converged label, not an external gold label, so it can only detect correlated
/// evaluators when other evaluators elsewhere reveal their deviation; it is blind to
/// correlated bias that has captured the EM consensus outright. None if not computed.
#[serde(skip_serializing_if = "Option::is_none")]
pub evaluator_effective_n: Option<f64>,
/// True when `evaluator_effective_n` is below 70% of the nominal qualifying evaluator
/// count for this sample — a "N evaluators agree" reading is really closer to a smaller
/// number of independent confirmations. None if not computed.
#[serde(skip_serializing_if = "Option::is_none")]
pub correlated_evaluator_warning: Option<bool>,
/// True if the latent-truth EM loop converged (its `delta` dropped below its epsilon)
/// before exhausting its iteration cap. False means the posterior is less certain — the
/// loop ran the full cap without settling. Same value for every sample scored by the same
/// batch's EM run. None if latent-truth EM did not run.
#[serde(skip_serializing_if = "Option::is_none")]
pub latent_truth_converged: Option<bool>,
/// Number of EM iterations actually run for this sample's batch. Same value for every
/// sample scored by the same batch's EM run. None if latent-truth EM did not run.
#[serde(skip_serializing_if = "Option::is_none")]
pub latent_truth_iterations: Option<usize>,
/// The EM loop's final iteration `delta` (summed absolute change in posteriors). Useful
/// alongside `latent_truth_converged` to see how close a non-converged run got to
/// settling. Same value for every sample scored by the same batch's EM run. None if
/// latent-truth EM did not run.
#[serde(skip_serializing_if = "Option::is_none")]
pub latent_truth_convergence_delta: Option<f64>,
/// Which safety-demotion condition (if any) downgraded this sample from `Keep` to
/// `Review`: `"correlated_evaluator_warning"`, `"evaluator_effective_n_below_minimum"`, or
/// `"latent_truth_not_converged"`. `None` when: `ScoreConfig::latent_truth_safety`'s flags
/// are all off/unset (the default), latent-truth EM did not run for this sample, or the
/// sample's decision was never `Keep` to begin with (already-`Review`/`Drop` samples are
/// never touched). When more than one condition fires at once, this reports a single
/// reason via a fixed priority order — see the (crate-private)
/// `latent_truth::latent_truth_demotion_reason`.
#[serde(skip_serializing_if = "Option::is_none")]
pub latent_truth_demotion_reason: Option<String>,
/// Mean of all numeric scores.
#[serde(skip_serializing_if = "Option::is_none")]
pub score_mean: Option<f64>,
/// Population standard deviation of numeric scores.
#[serde(skip_serializing_if = "Option::is_none")]
pub score_std: Option<f64>,
/// Difference between the maximum and minimum score (`max - min`).
#[serde(skip_serializing_if = "Option::is_none")]
pub score_range: Option<f64>,
/// Median absolute deviation of numeric scores. More robust to outliers than score_std.
#[serde(skip_serializing_if = "Option::is_none")]
pub score_mad: Option<f64>,
/// Interquartile range of numeric scores (Q3 - Q1).
#[serde(skip_serializing_if = "Option::is_none")]
pub score_iqr: Option<f64>,
/// Fraction of numeric scores sharing the majority sign (negative / zero / positive).
/// Domain-aware complement to score_std/score_mad/score_iqr: a tightly-clustered pair like
/// `[+0.01, -0.01]` reads as near-perfect score consistency by magnitude alone but is a
/// coin-flip on sign — this field catches sign flips that dispersion metrics mask.
/// `None` if there are no numeric scores.
#[serde(skip_serializing_if = "Option::is_none")]
pub score_sign_agreement: Option<f64>,
/// Normalized range of per-budget mean scores; `None` if fewer than two budget levels. (`[0.0, 1.0]`)
#[serde(skip_serializing_if = "Option::is_none")]
pub budget_sensitivity: Option<f64>,
/// Slope of (budget, mean_score) trend. None if < 2 budget levels.
#[serde(skip_serializing_if = "Option::is_none")]
pub budget_slope: Option<f64>,
/// Normalized range of per-seed mean scores; `None` if fewer than two seeds. (`[0.0, 1.0]`)
#[serde(skip_serializing_if = "Option::is_none")]
pub seed_sensitivity: Option<f64>,
/// Fraction of models whose majority label matches the overall majority; `None` if fewer than two models. (`[0.0, 1.0]`)
#[serde(skip_serializing_if = "Option::is_none")]
pub model_agreement: Option<f64>,
/// Fraction of evaluators whose majority label matches the overall majority; `None` if fewer than two evaluators. (`[0.0, 1.0]`)
#[serde(skip_serializing_if = "Option::is_none")]
pub evaluator_agreement: Option<f64>,
/// Reliability confidence based on observation count: n / (n + confidence_k). In [0, 1].
pub confidence: f64,
/// stability_score adjusted for confidence: stability_score * confidence + 0.5 * (1 - confidence)
pub adjusted_stability_score: f64,
/// `1.0 - stability_score`.
pub disagreement_score: f64,
/// Overall stability score in `[0.0, 1.0]`. Higher is more stable.
pub stability_score: f64,
/// Filtering decision derived from `stability_score` and the configured thresholds.
pub decision: Decision,
/// Per-dimension sub-scores that contributed to `stability_score`.
pub components: StabilityComponents,
}