rsinter 0.4.0

Experimental parallel sampling and decoder benchmark harness for RustQEC
Documentation
use std::collections::BTreeMap;
use std::io::{BufRead, BufReader, Read, Write};

use serde::ser::{SerializeStruct, Serializer};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sha2::{Digest, Sha256};

use crate::failure::{FailureKind, classify_error};

pub type ArtifactMap = BTreeMap<String, String>;
pub type CaseSummary = BTreeMap<String, Value>;
pub type ParamMap = BTreeMap<String, Value>;
pub type MetricMap = BTreeMap<String, f64>;

pub trait PairMapExt<K, V> {
    fn from_pairs<const N: usize>(pairs: [(K, V); N]) -> Self;
}

impl PairMapExt<&str, Value> for BTreeMap<String, Value> {
    fn from_pairs<const N: usize>(pairs: [(&str, Value); N]) -> Self {
        pairs.into_iter().map(|(k, v)| (k.to_string(), v)).collect()
    }
}

impl PairMapExt<&str, f64> for MetricMap {
    fn from_pairs<const N: usize>(pairs: [(&str, f64); N]) -> Self {
        pairs.into_iter().map(|(k, v)| (k.to_string(), v)).collect()
    }
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct RunManifest {
    pub benchmark: String,
    pub benchmark_version: u64,
    pub runner: String,
    pub language: String,
    pub output_dir: String,
}

impl RunManifest {
    pub fn new(
        benchmark: String,
        benchmark_version: u64,
        runner: String,
        language: String,
        output_dir: String,
    ) -> Self {
        Self {
            benchmark,
            benchmark_version,
            runner,
            language,
            output_dir,
        }
    }
}

#[derive(Debug, Clone, PartialEq)]
pub struct BenchmarkResultRow {
    pub benchmark: String,
    pub runner: String,
    pub language: String,
    pub status: String,
    pub failure_kind: FailureKind,
    pub params: ParamMap,
    pub case_summary: CaseSummary,
    pub metrics: MetricMap,
    pub artifacts: ArtifactMap,
    pub error: Option<String>,
}

const ROW_IDENTITY_SCHEMA: &str = "rsinter.benchmark_result_row.v1";
const CASE_SUMMARY_ADDITIVE_KEYS: [&str; 1] = ["num_shots_generated"];

#[derive(Serialize)]
struct BenchmarkResultRowIdentityInput<'a> {
    schema: &'static str,
    benchmark: &'a str,
    runner: &'a str,
    language: &'a str,
    params: &'a ParamMap,
    case_summary: CaseSummary,
}

impl BenchmarkResultRow {
    pub fn identity(&self) -> Result<String, String> {
        let input = BenchmarkResultRowIdentityInput {
            schema: ROW_IDENTITY_SCHEMA,
            benchmark: &self.benchmark,
            runner: &self.runner,
            language: &self.language,
            params: &self.params,
            case_summary: stable_case_summary(&self.case_summary),
        };
        let bytes = serde_json::to_vec(&input).map_err(|error| error.to_string())?;
        let digest = Sha256::digest(bytes);
        Ok(format!("sha256:{}", lower_hex(&digest)))
    }
}

pub(crate) fn stable_case_summary(case_summary: &CaseSummary) -> CaseSummary {
    case_summary
        .iter()
        .filter(|(key, _)| !case_summary_additive_keys().contains(&key.as_str()))
        .map(|(key, value)| (key.clone(), value.clone()))
        .collect()
}

pub(crate) fn case_summary_additive_keys() -> &'static [&'static str] {
    &CASE_SUMMARY_ADDITIVE_KEYS
}

fn lower_hex(bytes: &[u8]) -> String {
    const HEX: &[u8; 16] = b"0123456789abcdef";
    let mut out = String::with_capacity(bytes.len() * 2);
    for byte in bytes {
        out.push(HEX[(byte >> 4) as usize] as char);
        out.push(HEX[(byte & 0x0f) as usize] as char);
    }
    out
}

impl Serialize for BenchmarkResultRow {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        let identity = self.identity().map_err(serde::ser::Error::custom)?;
        let mut row = serializer.serialize_struct("BenchmarkResultRow", 11)?;
        row.serialize_field("identity", &identity)?;
        row.serialize_field("benchmark", &self.benchmark)?;
        row.serialize_field("runner", &self.runner)?;
        row.serialize_field("language", &self.language)?;
        row.serialize_field("status", &self.status)?;
        row.serialize_field("failure_kind", &self.failure_kind)?;
        row.serialize_field("params", &self.params)?;
        row.serialize_field("case_summary", &self.case_summary)?;
        row.serialize_field("metrics", &self.metrics)?;
        row.serialize_field("artifacts", &self.artifacts)?;
        row.serialize_field("error", &self.error)?;
        row.end()
    }
}

#[derive(Deserialize)]
struct RawBenchmarkResultRow {
    benchmark: String,
    runner: String,
    language: String,
    status: String,
    #[serde(default)]
    failure_kind: Option<FailureKind>,
    params: ParamMap,
    case_summary: CaseSummary,
    metrics: MetricMap,
    artifacts: ArtifactMap,
    error: Option<String>,
}

impl<'de> Deserialize<'de> for BenchmarkResultRow {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: serde::Deserializer<'de>,
    {
        let raw = RawBenchmarkResultRow::deserialize(deserializer)?;
        let failure_kind = raw.failure_kind.unwrap_or_else(|| {
            infer_legacy_failure_kind(&raw.status, raw.error.as_deref(), &raw.params, &raw.metrics)
        });
        Ok(Self {
            benchmark: raw.benchmark,
            runner: raw.runner,
            language: raw.language,
            status: raw.status,
            failure_kind,
            params: raw.params,
            case_summary: raw.case_summary,
            metrics: raw.metrics,
            artifacts: raw.artifacts,
            error: raw.error,
        })
    }
}

fn infer_legacy_failure_kind(
    status: &str,
    error: Option<&str>,
    params: &ParamMap,
    metrics: &MetricMap,
) -> FailureKind {
    if status == "error" {
        return error
            .map(|message| classify_error(message, FailureKind::SolverFailure))
            .unwrap_or(FailureKind::SolverFailure);
    }
    if legacy_timed_out(params, metrics) {
        FailureKind::Timeout
    } else if metrics.get("logical_errors").copied().unwrap_or(0.0) > 0.0 {
        FailureKind::LogicalFailure
    } else {
        FailureKind::Ok
    }
}

fn legacy_timed_out(params: &ParamMap, metrics: &MetricMap) -> bool {
    let Some(max_wall_seconds) = params.get("max_wall_seconds").and_then(Value::as_f64) else {
        return false;
    };
    let Some(wall_seconds) = metrics.get("wall_seconds") else {
        return false;
    };
    wall_seconds.is_finite()
        && *wall_seconds >= max_wall_seconds
        && !legacy_reached_cap(params, metrics, "max_shots", "shots_used")
        && !legacy_reached_cap(params, metrics, "max_errors", "logical_errors")
}

fn legacy_reached_cap(
    params: &ParamMap,
    metrics: &MetricMap,
    param_key: &str,
    metric_key: &str,
) -> bool {
    match (
        params.get(param_key).and_then(Value::as_f64),
        metrics.get(metric_key).copied(),
    ) {
        (Some(cap), Some(value)) => value.is_finite() && value >= cap,
        _ => false,
    }
}

pub fn write_results_jsonl(rows: &[BenchmarkResultRow], out: &mut dyn Write) -> Result<(), String> {
    for row in rows {
        serde_json::to_writer(&mut *out, row).map_err(|e| e.to_string())?;
        out.write_all(b"\n").map_err(|e| e.to_string())?;
    }
    Ok(())
}

pub fn read_results_jsonl(input: impl Read) -> Result<Vec<BenchmarkResultRow>, String> {
    let reader = BufReader::new(input);
    let mut rows = Vec::new();
    for line in reader.lines() {
        let line = line.map_err(|e| e.to_string())?;
        if line.trim().is_empty() {
            continue;
        }
        let row = serde_json::from_str::<BenchmarkResultRow>(&line).map_err(|e| e.to_string())?;
        rows.push(row);
    }
    Ok(rows)
}