use std::collections::HashMap;
use flodl::nn::Module;
use flodl::{Tensor, Variable};
use crate::engine::core::CoreEngine;
use crate::utils::tabular_data::Dataset;
const EXAM_CHUNK: usize = 4096;
pub(crate) struct ExamReport {
pub rows: usize,
pub scores: HashMap<String, Option<f32>>,
pub correlation: Option<f32>,
}
impl CoreEngine {
pub(crate) fn exam_report(&self, hashes: &[String], fits: &[f32]) -> Option<ExamReport> {
let exam = self.exam.as_ref()?;
if !self.fitness.is_computed() || exam.len() == 0 {
return None;
}
let dir = self.run_dir.join("nets");
let mut scores: HashMap<String, Option<f32>> = HashMap::new();
for hash in hashes {
scores.insert(hash.clone(), self.exam_score_one(hash, exam, &dir));
}
let pa: Vec<f32> = hashes
.iter()
.zip(fits)
.filter_map(|(h, f)| scores.get(h).copied().flatten().map(|e| (*f, e)))
.map(|(f, _)| f)
.collect();
let pb: Vec<f32> = hashes
.iter()
.zip(fits)
.filter_map(|(h, f)| scores.get(h).copied().flatten().map(|e| (*f, e)))
.map(|(_, e)| e)
.collect();
Some(ExamReport {
rows: exam.len(),
scores,
correlation: pearson(&pa, &pb),
})
}
fn exam_score_one(
&self,
hash: &str,
exam: &Dataset,
nets_dir: &std::path::Path,
) -> Option<f32> {
let mut from_disk: Option<crate::graph::network::Network> = None;
if !self.networks.contains_key(hash) {
let state = self.state.net(hash)?;
let topo = state.topology().ok()?;
let mut net =
crate::graph::network::Network::build(&topo, self.config.device()).ok()?;
let weights = nets_dir.join(format!("{hash}.safetensors"));
crate::utils::safetensors::load_safetensors(&mut net, &weights).ok()?;
from_disk = Some(net);
}
let net_ref: &crate::graph::network::Network = match self.networks.get(hash) {
Some(net) => net,
None => from_disk.as_ref()?,
};
let rows = exam.len();
let mut weighted = 0.0f64;
let mut seen = 0usize;
for start in (0..rows).step_by(EXAM_CHUNK) {
let len = EXAM_CHUNK.min(rows - start);
let (x, y) = slice_rows(exam, start, len).ok()?;
let pred = net_ref.forward(&Variable::new(x, false)).ok()?;
let score = self.fitness.score(&pred, &Variable::new(y, false)).ok()?;
if !score.is_finite() {
return None;
}
weighted += score as f64 * len as f64;
seen += len;
}
(seen > 0).then(|| (weighted / seen as f64) as f32)
}
pub(crate) fn write_exam_json(
&self,
report: &ExamReport,
fits: &[(String, f32)],
) -> Option<()> {
let mut rows: Vec<serde_json::Value> = Vec::new();
for (hash, fit) in fits {
rows.push(serde_json::json!({
"hash": hash,
"fit": fit,
"exam": report.scores.get(hash).copied().flatten(),
}));
}
let doc = serde_json::json!({
"exam_rows": report.rows,
"reported_nets": fits.len(),
"correlation_fit_vs_exam": report.correlation,
"note": "Report only: these rows were never in the run tensor and nothing ranks, culls or seats on them.",
"nets": rows,
});
let path = self.run_dir.join("exam.json");
match serde_json::to_string_pretty(&doc).map_err(|e| e.to_string()) {
Ok(text) => match std::fs::write(&path, text) {
Ok(()) => Some(()),
Err(e) => {
tracing::warn!("exam.json write failed: {e}");
None
}
},
Err(e) => {
tracing::warn!("exam.json encode failed: {e}");
None
}
}
}
}
fn slice_rows(exam: &Dataset, start: usize, len: usize) -> flodl::tensor::Result<(Tensor, Tensor)> {
let x: Tensor = exam.inputs.narrow(0, start as i64, len as i64)?;
let y: Tensor = exam.targets.narrow(0, start as i64, len as i64)?;
Ok((x, y))
}
pub(crate) fn pearson(xs: &[f32], ys: &[f32]) -> Option<f32> {
if xs.len() < 3 || xs.len() != ys.len() {
return None;
}
let n = xs.len() as f64;
let mx = xs.iter().map(|v| *v as f64).sum::<f64>() / n;
let my = ys.iter().map(|v| *v as f64).sum::<f64>() / n;
let mut sxy = 0.0;
let mut sxx = 0.0;
let mut syy = 0.0;
for (x, y) in xs.iter().zip(ys) {
let dx = *x as f64 - mx;
let dy = *y as f64 - my;
sxy += dx * dy;
sxx += dx * dx;
syy += dy * dy;
}
if sxx <= f64::EPSILON || syy <= f64::EPSILON {
return None;
}
Some((sxy / (sxx.sqrt() * syy.sqrt())) as f32)
}
#[cfg(test)]
mod tests {
use super::pearson;
#[test]
fn pearson_reads_agreement_and_refuses_degenerate_input() {
let r = pearson(&[1.0, 2.0, 3.0, 4.0], &[2.0, 4.0, 6.0, 8.0]).unwrap();
assert!((r - 1.0).abs() < 1e-6, "a straight line is r = 1, got {r}");
let r = pearson(&[1.0, 2.0, 3.0, 4.0], &[8.0, 6.0, 4.0, 2.0]).unwrap();
assert!((r + 1.0).abs() < 1e-6, "inverted rank is r = -1, got {r}");
assert!(pearson(&[1.0, 2.0], &[1.0, 2.0]).is_none(), "n < 3");
assert!(
pearson(&[1.0, 1.0, 1.0], &[1.0, 2.0, 3.0]).is_none(),
"no spread on one side"
);
}
}