use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use tracing::{debug, info, warn};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IsotonicCalibrationConfig {
pub min_slope: f32,
pub max_slope: f32,
pub min_samples: usize,
pub max_ece: f32,
pub num_bins: usize,
pub apply_smoothing: bool,
}
impl Default for IsotonicCalibrationConfig {
fn default() -> Self {
Self {
min_slope: 0.9, max_slope: 1.1, min_samples: 50,
max_ece: 0.02, num_bins: 15,
apply_smoothing: true,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct IntentLanguageKey {
pub intent: String, pub language: String, }
impl IntentLanguageKey {
pub fn new(intent: impl Into<String>, language: impl Into<String>) -> Self {
Self {
intent: intent.into(),
language: language.into(),
}
}
pub fn key_string(&self) -> String {
format!("{}×{}", self.intent, self.language)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IsotonicRegressor {
pub key: IntentLanguageKey,
pub x_values: Vec<f32>,
pub y_values: Vec<f32>,
pub min_slope: f32,
pub max_slope: f32,
pub n_samples: usize,
pub ece_before: f32,
pub ece_after: f32,
pub created_at: chrono::DateTime<chrono::Utc>,
pub model_hash: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IsotonicCalibrationSystem {
pub config: IsotonicCalibrationConfig,
pub regressors: HashMap<IntentLanguageKey, IsotonicRegressor>,
pub global_regressor: Option<IsotonicRegressor>,
pub created_at: chrono::DateTime<chrono::Utc>,
pub version: String,
}
#[derive(Debug, Clone)]
pub struct CalibrationTrainingSample {
pub intent: String,
pub language: String,
pub raw_score: f32,
pub actual_relevance: f32, }
#[derive(Debug, Clone)]
pub struct CalibratedPrediction {
pub raw_score: f32,
pub calibrated_score: f32,
pub intent: String,
pub language: String,
pub model_used: String, }
impl IsotonicCalibrationSystem {
pub fn new(config: IsotonicCalibrationConfig) -> Self {
Self {
config,
regressors: HashMap::new(),
global_regressor: None,
created_at: chrono::Utc::now(),
version: "1.0.0".to_string(),
}
}
pub async fn train(&mut self, training_samples: &[CalibrationTrainingSample]) -> Result<()> {
info!("Training isotonic calibration on {} samples", training_samples.len());
let mut grouped_samples: HashMap<IntentLanguageKey, Vec<&CalibrationTrainingSample>> = HashMap::new();
for sample in training_samples {
let key = IntentLanguageKey::new(&sample.intent, &sample.language);
grouped_samples.entry(key).or_default().push(sample);
}
info!("Found {} intent×language combinations", grouped_samples.len());
let mut trained_regressors = 0;
for (key, samples) in grouped_samples {
if samples.len() >= self.config.min_samples {
debug!("Training regressor for {}: {} samples", key.key_string(), samples.len());
let regressor = self.train_regressor(&key, &samples).await
.with_context(|| format!("Failed to train regressor for {}", key.key_string()))?;
if regressor.ece_after <= self.config.max_ece {
self.regressors.insert(key.clone(), regressor);
trained_regressors += 1;
} else {
warn!("Regressor for {} failed ECE requirement: {:.4} > {:.4}",
key.key_string(), regressor.ece_after, self.config.max_ece);
}
} else {
debug!("Insufficient samples for {}: {} < {}",
key.key_string(), samples.len(), self.config.min_samples);
}
}
if training_samples.len() >= self.config.min_samples {
let global_key = IntentLanguageKey::new("global", "all");
let global_samples: Vec<&CalibrationTrainingSample> = training_samples.iter().collect();
let global_regressor = self.train_regressor(&global_key, &global_samples).await?;
if global_regressor.ece_after <= self.config.max_ece {
self.global_regressor = Some(global_regressor);
}
}
info!("Isotonic calibration training complete: {} specific regressors + {} global",
trained_regressors, if self.global_regressor.is_some() { 1 } else { 0 });
Ok(())
}
pub fn calibrate_prediction(
&self,
raw_score: f32,
intent: &str,
language: &str
) -> CalibratedPrediction {
let key = IntentLanguageKey::new(intent, language);
if let Some(regressor) = self.regressors.get(&key) {
let calibrated = regressor.predict(raw_score);
return CalibratedPrediction {
raw_score,
calibrated_score: calibrated,
intent: intent.to_string(),
language: language.to_string(),
model_used: format!("specific:{}", key.key_string()),
};
}
let intent_key = IntentLanguageKey::new(intent, "all");
if let Some(regressor) = self.regressors.get(&intent_key) {
let calibrated = regressor.predict(raw_score);
return CalibratedPrediction {
raw_score,
calibrated_score: calibrated,
intent: intent.to_string(),
language: language.to_string(),
model_used: format!("intent:{}", intent),
};
}
let language_key = IntentLanguageKey::new("all", language);
if let Some(regressor) = self.regressors.get(&language_key) {
let calibrated = regressor.predict(raw_score);
return CalibratedPrediction {
raw_score,
calibrated_score: calibrated,
intent: intent.to_string(),
language: language.to_string(),
model_used: format!("language:{}", language),
};
}
if let Some(regressor) = &self.global_regressor {
let calibrated = regressor.predict(raw_score);
return CalibratedPrediction {
raw_score,
calibrated_score: calibrated,
intent: intent.to_string(),
language: language.to_string(),
model_used: "global".to_string(),
};
}
warn!("No calibration model available for {}×{}, returning raw score", intent, language);
CalibratedPrediction {
raw_score,
calibrated_score: raw_score.clamp(0.001, 0.999),
intent: intent.to_string(),
language: language.to_string(),
model_used: "none".to_string(),
}
}
pub async fn evaluate(&self, test_samples: &[CalibrationTrainingSample]) -> Result<CalibrationEvaluationResult> {
info!("Evaluating isotonic calibration on {} test samples", test_samples.len());
let mut predictions = Vec::new();
let mut model_usage_counts = HashMap::new();
for sample in test_samples {
let pred = self.calibrate_prediction(sample.raw_score, &sample.intent, &sample.language);
*model_usage_counts.entry(pred.model_used.clone()).or_insert(0) += 1;
predictions.push((pred.calibrated_score, sample.actual_relevance));
}
let overall_ece = self.calculate_ece(&predictions);
let mut by_intent_language = HashMap::new();
let mut grouped_preds: HashMap<IntentLanguageKey, Vec<(f32, f32)>> = HashMap::new();
for (sample, (pred_score, actual)) in test_samples.iter().zip(predictions.iter()) {
let key = IntentLanguageKey::new(&sample.intent, &sample.language);
grouped_preds.entry(key).or_default().push((*pred_score, *actual));
}
for (key, preds) in grouped_preds {
if preds.len() >= 10 { let ece = self.calculate_ece(&preds);
by_intent_language.insert(key, ece);
}
}
Ok(CalibrationEvaluationResult {
overall_ece,
by_intent_language,
model_usage_counts,
total_samples: test_samples.len(),
passes_ece_requirement: overall_ece <= self.config.max_ece,
})
}
pub async fn save_to_artifact(&self, base_path: &str) -> Result<String> {
let timestamp = chrono::Utc::now().format("%Y%m%dT%H%M%SZ");
let filename = format!("iso_{}.json", timestamp);
let file_path = format!("{}/calib/{}", base_path, filename);
if let Some(parent) = std::path::Path::new(&file_path).parent() {
tokio::fs::create_dir_all(parent).await
.with_context(|| format!("Failed to create directory: {}", parent.display()))?;
}
let json_content = serde_json::to_string_pretty(self)
.context("Failed to serialize calibration system")?;
tokio::fs::write(&file_path, json_content).await
.with_context(|| format!("Failed to write calibration file: {}", file_path))?;
info!("Isotonic calibration saved to: {}", file_path);
Ok(file_path)
}
pub async fn load_from_artifact(file_path: &str) -> Result<Self> {
let json_content = tokio::fs::read_to_string(file_path).await
.with_context(|| format!("Failed to read calibration file: {}", file_path))?;
let system: IsotonicCalibrationSystem = serde_json::from_str(&json_content)
.with_context(|| format!("Failed to deserialize calibration file: {}", file_path))?;
info!("Isotonic calibration loaded from: {} ({} regressors)",
file_path, system.regressors.len());
Ok(system)
}
pub fn get_statistics(&self) -> CalibrationStatistics {
let regressor_stats = self.regressors.iter()
.map(|(key, regressor)| (key.key_string(), regressor.ece_after))
.collect();
let global_ece = self.global_regressor.as_ref().map(|r| r.ece_after);
CalibrationStatistics {
num_specific_regressors: self.regressors.len(),
has_global_fallback: self.global_regressor.is_some(),
regressor_eces: regressor_stats,
global_ece,
mean_ece: self.regressors.values().map(|r| r.ece_after).sum::<f32>() / self.regressors.len().max(1) as f32,
created_at: self.created_at,
}
}
async fn train_regressor(
&self,
key: &IntentLanguageKey,
samples: &[&CalibrationTrainingSample]
) -> Result<IsotonicRegressor> {
let mut data_points: Vec<(f32, f32)> = samples.iter()
.map(|sample| (sample.raw_score, sample.actual_relevance))
.collect();
data_points.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal));
let ece_before = self.calculate_ece(&data_points);
let (x_values, y_values) = self.fit_isotonic_with_slope_clamping(&data_points)?;
let calibrated_points: Vec<(f32, f32)> = data_points.iter()
.map(|(score, label)| {
let calibrated = self.interpolate_prediction(*score, &x_values, &y_values);
(calibrated, *label)
})
.collect();
let ece_after = self.calculate_ece(&calibrated_points);
let model_hash = self.calculate_model_hash(&x_values, &y_values);
Ok(IsotonicRegressor {
key: key.clone(),
x_values,
y_values,
min_slope: self.config.min_slope,
max_slope: self.config.max_slope,
n_samples: samples.len(),
ece_before,
ece_after,
created_at: chrono::Utc::now(),
model_hash,
})
}
fn fit_isotonic_with_slope_clamping(
&self,
data_points: &[(f32, f32)]
) -> Result<(Vec<f32>, Vec<f32>)> {
if data_points.is_empty() {
return Ok((Vec::new(), Vec::new()));
}
let mut x_values = Vec::new();
let mut y_values = Vec::new();
let mut weights = Vec::new();
x_values.push(data_points[0].0);
y_values.push(data_points[0].1);
weights.push(1.0);
for &(x, y) in data_points.iter().skip(1) {
x_values.push(x);
y_values.push(y);
weights.push(1.0);
let mut i = x_values.len() - 1;
while i > 0 && y_values[i] < y_values[i - 1] {
let total_weight = weights[i] + weights[i - 1];
let pooled_y = (y_values[i] * weights[i] + y_values[i - 1] * weights[i - 1]) / total_weight;
y_values[i - 1] = pooled_y;
weights[i - 1] = total_weight;
x_values.remove(i);
y_values.remove(i);
weights.remove(i);
i -= 1;
}
}
self.apply_slope_clamping(&mut x_values, &mut y_values)?;
if self.config.apply_smoothing {
self.apply_smoothing(&mut y_values);
}
Ok((x_values, y_values))
}
fn apply_slope_clamping(&self, x_values: &mut [f32], y_values: &mut [f32]) -> Result<()> {
if x_values.len() < 2 {
return Ok(());
}
for i in 1..x_values.len() {
let dx = x_values[i] - x_values[i - 1];
if dx <= 0.0 {
continue; }
let current_slope = (y_values[i] - y_values[i - 1]) / dx;
let clamped_slope = current_slope.clamp(self.config.min_slope, self.config.max_slope);
if (current_slope - clamped_slope).abs() > 1e-6 {
y_values[i] = y_values[i - 1] + clamped_slope * dx;
debug!("Clamped slope from {:.3} to {:.3} at interval {}",
current_slope, clamped_slope, i);
}
}
Ok(())
}
fn apply_smoothing(&self, y_values: &mut [f32]) {
if y_values.len() < 3 {
return;
}
let mut smoothed = y_values.to_vec();
for i in 1..y_values.len() - 1 {
smoothed[i] = (y_values[i - 1] + 2.0 * y_values[i] + y_values[i + 1]) / 4.0;
}
for (original, smoothed_val) in y_values.iter_mut().zip(smoothed.iter()) {
*original = *smoothed_val;
}
}
fn calculate_ece(&self, predictions: &[(f32, f32)]) -> f32 {
if predictions.is_empty() {
return 0.0;
}
let mut bins = vec![Vec::new(); self.config.num_bins];
for &(pred, actual) in predictions {
let bin_idx = ((pred * self.config.num_bins as f32) as usize).min(self.config.num_bins - 1);
bins[bin_idx].push((pred, actual));
}
let mut total_ece = 0.0;
let total_samples = predictions.len() as f32;
for bin in bins.iter() {
if bin.is_empty() {
continue;
}
let bin_size = bin.len() as f32;
let avg_confidence = bin.iter().map(|(pred, _)| *pred).sum::<f32>() / bin_size;
let avg_accuracy = bin.iter().map(|(_, actual)| *actual).sum::<f32>() / bin_size;
let bin_ece = (avg_confidence - avg_accuracy).abs() * (bin_size / total_samples);
total_ece += bin_ece;
}
total_ece
}
fn calculate_model_hash(&self, x_values: &[f32], y_values: &[f32]) -> String {
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
let mut hasher = DefaultHasher::new();
for &x in x_values {
x.to_bits().hash(&mut hasher);
}
for &y in y_values {
y.to_bits().hash(&mut hasher);
}
self.config.min_slope.to_bits().hash(&mut hasher);
self.config.max_slope.to_bits().hash(&mut hasher);
format!("{:x}", hasher.finish())
}
fn interpolate_prediction(&self, score: f32, x_values: &[f32], y_values: &[f32]) -> f32 {
if x_values.is_empty() {
return score.clamp(0.001, 0.999);
}
if x_values.len() == 1 {
return y_values[0];
}
for i in 0..x_values.len() - 1 {
if score >= x_values[i] && score <= x_values[i + 1] {
let t = (score - x_values[i]) / (x_values[i + 1] - x_values[i]);
return y_values[i] + t * (y_values[i + 1] - y_values[i]);
}
}
if score < x_values[0] {
y_values[0]
} else {
y_values[y_values.len() - 1]
}
}
}
impl IsotonicRegressor {
pub fn predict(&self, raw_score: f32) -> f32 {
if self.x_values.is_empty() {
return raw_score.clamp(0.001, 0.999);
}
if self.x_values.len() == 1 {
return self.y_values[0];
}
for i in 0..self.x_values.len() - 1 {
if raw_score >= self.x_values[i] && raw_score <= self.x_values[i + 1] {
let dx = self.x_values[i + 1] - self.x_values[i];
if dx <= 0.0 {
return self.y_values[i];
}
let t = (raw_score - self.x_values[i]) / dx;
return self.y_values[i] + t * (self.y_values[i + 1] - self.y_values[i]);
}
}
if raw_score < self.x_values[0] {
self.y_values[0]
} else {
self.y_values[self.y_values.len() - 1]
}
}
pub fn validate_slope_constraints(&self) -> bool {
for i in 1..self.x_values.len() {
let dx = self.x_values[i] - self.x_values[i - 1];
if dx <= 0.0 {
continue;
}
let slope = (self.y_values[i] - self.y_values[i - 1]) / dx;
if slope < self.min_slope - 1e-6 || slope > self.max_slope + 1e-6 {
return false;
}
}
true
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CalibrationEvaluationResult {
pub overall_ece: f32,
pub by_intent_language: HashMap<IntentLanguageKey, f32>,
pub model_usage_counts: HashMap<String, usize>,
pub total_samples: usize,
pub passes_ece_requirement: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CalibrationStatistics {
pub num_specific_regressors: usize,
pub has_global_fallback: bool,
pub regressor_eces: Vec<(String, f32)>,
pub global_ece: Option<f32>,
pub mean_ece: f32,
pub created_at: chrono::DateTime<chrono::Utc>,
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_isotonic_calibration_training() {
let mut config = IsotonicCalibrationConfig::default();
config.max_ece = 0.1;
let mut system = IsotonicCalibrationSystem::new(config);
let mut training_samples = Vec::new();
for i in 0..25 {
let score = 0.1 + (i as f32 / 24.0) * 0.3; training_samples.push(CalibrationTrainingSample {
intent: "NL".to_string(),
language: "python".to_string(),
raw_score: score,
actual_relevance: 0.0
});
}
for i in 0..25 {
let score = 0.6 + (i as f32 / 24.0) * 0.3; training_samples.push(CalibrationTrainingSample {
intent: "NL".to_string(),
language: "python".to_string(),
raw_score: score,
actual_relevance: 1.0
});
}
for i in 0..10 {
let score = 0.45 + (i as f32 / 9.0) * 0.1; training_samples.push(CalibrationTrainingSample {
intent: "NL".to_string(),
language: "python".to_string(),
raw_score: score,
actual_relevance: if i % 2 == 0 { 0.0 } else { 1.0 }
});
}
system.train(&training_samples).await.unwrap();
let stats = system.get_statistics();
assert!(training_samples.len() >= 50); }
#[test]
fn test_isotonic_regressor_prediction() {
let regressor = IsotonicRegressor {
key: IntentLanguageKey::new("test", "test"),
x_values: vec![0.0, 0.5, 1.0],
y_values: vec![0.0, 0.5, 1.0],
min_slope: 0.9,
max_slope: 1.1,
n_samples: 100,
ece_before: 0.1,
ece_after: 0.01,
created_at: chrono::Utc::now(),
model_hash: "test".to_string(),
};
assert!((regressor.predict(0.25) - 0.25).abs() < 0.01);
assert!((regressor.predict(0.75) - 0.75).abs() < 0.01);
assert_eq!(regressor.predict(-0.1), 0.0);
assert_eq!(regressor.predict(1.1), 1.0);
}
#[test]
fn test_slope_constraint_validation() {
let valid_regressor = IsotonicRegressor {
key: IntentLanguageKey::new("test", "test"),
x_values: vec![0.0, 0.5, 1.0],
y_values: vec![0.0, 0.5, 1.0], min_slope: 0.9,
max_slope: 1.1,
n_samples: 100,
ece_before: 0.1,
ece_after: 0.01,
created_at: chrono::Utc::now(),
model_hash: "test".to_string(),
};
let invalid_regressor = IsotonicRegressor {
key: IntentLanguageKey::new("test", "test"),
x_values: vec![0.0, 0.5, 1.0],
y_values: vec![0.0, 0.8, 1.0], min_slope: 0.9,
max_slope: 1.1,
n_samples: 100,
ece_before: 0.1,
ece_after: 0.01,
created_at: chrono::Utc::now(),
model_hash: "test".to_string(),
};
assert!(valid_regressor.validate_slope_constraints());
assert!(!invalid_regressor.validate_slope_constraints());
}
#[test]
fn test_intent_language_key() {
let key1 = IntentLanguageKey::new("NL", "python");
let key2 = IntentLanguageKey::new("NL", "python");
let key3 = IntentLanguageKey::new("identifier", "rust");
assert_eq!(key1, key2);
assert_ne!(key1, key3);
assert_eq!(key1.key_string(), "NL×python");
assert_eq!(key3.key_string(), "identifier×rust");
}
}