rsinter 0.4.0

Experimental parallel sampling and decoder benchmark harness for RustQEC
Documentation
use std::cmp::Ordering;
use std::collections::btree_map::Entry;
use std::collections::BTreeMap;

use serde_json::{Number, Value};

use crate::bench::result::{
    case_summary_additive_keys, stable_case_summary, BenchmarkResultRow, MetricMap,
};
use crate::failure::{combine_failure_kind, FailureKind};

const ADDITIVE_METRICS: [&str; 5] = [
    "shots_used",
    "logical_errors",
    "compile_us",
    "total_decode_us",
    "wall_seconds",
];
const DERIVED_METRICS: [&str; 2] = ["logical_error_rate", "decode_us_per_shot"];

pub fn merge_result_rows(
    row_sets: Vec<Vec<BenchmarkResultRow>>,
) -> Result<Vec<BenchmarkResultRow>, String> {
    let mut rows_by_identity = BTreeMap::new();
    for row in row_sets.into_iter().flatten() {
        let identity = row.identity()?;
        match rows_by_identity.entry(identity) {
            Entry::Vacant(entry) => {
                entry.insert(row);
            }
            Entry::Occupied(mut entry) => {
                let identity = entry.key().clone();
                merge_row_into(&identity, entry.get_mut(), row)?;
            }
        }
    }

    let mut rows: Vec<BenchmarkResultRow> = rows_by_identity.into_values().collect();
    rows.sort_by(compare_rows);
    Ok(rows)
}

fn compare_rows(a: &BenchmarkResultRow, b: &BenchmarkResultRow) -> Ordering {
    let a_distance = a
        .params
        .get("distance")
        .and_then(|value| value.as_i64())
        .unwrap_or_default();
    let b_distance = b
        .params
        .get("distance")
        .and_then(|value| value.as_i64())
        .unwrap_or_default();
    let a_p = a
        .params
        .get("p")
        .and_then(|value| value.as_f64().or_else(|| value.as_i64().map(|n| n as f64)))
        .unwrap_or_default();
    let b_p = b
        .params
        .get("p")
        .and_then(|value| value.as_f64().or_else(|| value.as_i64().map(|n| n as f64)))
        .unwrap_or_default();
    a.runner
        .cmp(&b.runner)
        .then(a_distance.cmp(&b_distance))
        .then(a_p.partial_cmp(&b_p).unwrap_or(Ordering::Equal))
}

fn merge_row_into(
    identity: &str,
    base: &mut BenchmarkResultRow,
    incoming: BenchmarkResultRow,
) -> Result<(), String> {
    ensure_same(identity, "benchmark", base.benchmark == incoming.benchmark)?;
    ensure_same(identity, "runner", base.runner == incoming.runner)?;
    ensure_same(identity, "language", base.language == incoming.language)?;
    ensure_same(identity, "status", base.status == incoming.status)?;
    let merged_failure_kind = merge_failure_kind(
        identity,
        &base.status,
        base.failure_kind,
        incoming.failure_kind,
    )?;
    ensure_same(identity, "params", base.params == incoming.params)?;
    ensure_same(identity, "error", base.error == incoming.error)?;
    ensure_same(identity, "artifacts", base.artifacts == incoming.artifacts)?;
    ensure_same(
        identity,
        "case_summary",
        stable_case_summary(&base.case_summary) == stable_case_summary(&incoming.case_summary),
    )?;

    merge_case_summary(identity, &mut base.case_summary, incoming.case_summary)?;
    merge_metrics(identity, &mut base.metrics, incoming.metrics)?;
    recompute_derived_metrics(&mut base.metrics);
    base.failure_kind = merged_failure_kind;
    Ok(())
}

fn merge_failure_kind(
    identity: &str,
    status: &str,
    base: FailureKind,
    incoming: FailureKind,
) -> Result<FailureKind, String> {
    if status == "ok" {
        ensure_same(
            identity,
            "failure_kind",
            base.status() == "ok" && incoming.status() == "ok",
        )?;
        Ok(combine_failure_kind(base, incoming))
    } else {
        ensure_same(identity, "failure_kind", base == incoming)?;
        Ok(base)
    }
}

fn ensure_same(identity: &str, field: &str, same: bool) -> Result<(), String> {
    if same {
        Ok(())
    } else {
        Err(format!(
            "cannot merge benchmark rows with identity {identity}: conflicting {field}"
        ))
    }
}

fn merge_case_summary(
    identity: &str,
    base: &mut BTreeMap<String, Value>,
    incoming: BTreeMap<String, Value>,
) -> Result<(), String> {
    for key in case_summary_additive_keys() {
        let Some(incoming_value) = incoming.get(*key) else {
            continue;
        };
        match base.entry((*key).to_string()) {
            Entry::Vacant(entry) => {
                entry.insert(incoming_value.clone());
            }
            Entry::Occupied(mut entry) => {
                let merged = sum_json_numbers(
                    entry.get(),
                    incoming_value,
                    identity,
                    &format!("case_summary.{key}"),
                )?;
                entry.insert(merged);
            }
        }
    }
    Ok(())
}

fn sum_json_numbers(
    left: &Value,
    right: &Value,
    identity: &str,
    field: &str,
) -> Result<Value, String> {
    if let (Some(left), Some(right)) = (left.as_u64(), right.as_u64()) {
        return left
            .checked_add(right)
            .map(Value::from)
            .ok_or_else(|| conflict(identity, field));
    }
    if let (Some(left), Some(right)) = (left.as_i64(), right.as_i64()) {
        return left
            .checked_add(right)
            .map(Value::from)
            .ok_or_else(|| conflict(identity, field));
    }
    match (left.as_f64(), right.as_f64()) {
        (Some(left), Some(right)) => Number::from_f64(left + right)
            .map(Value::Number)
            .ok_or_else(|| conflict(identity, field)),
        _ => Err(conflict(identity, field)),
    }
}

fn merge_metrics(identity: &str, base: &mut MetricMap, incoming: MetricMap) -> Result<(), String> {
    for key in base.keys().chain(incoming.keys()) {
        if !is_merge_metric_allowed(key) {
            return Err(conflict(identity, &format!("metrics.{key}")));
        }
    }

    for (key, value) in incoming {
        if ADDITIVE_METRICS.contains(&key.as_str()) {
            *base.entry(key).or_insert(0.0) += value;
        } else if DERIVED_METRICS.contains(&key.as_str()) {
            continue;
        }
    }
    Ok(())
}

fn is_merge_metric_allowed(key: &str) -> bool {
    ADDITIVE_METRICS.contains(&key) || DERIVED_METRICS.contains(&key)
}

fn recompute_derived_metrics(metrics: &mut MetricMap) {
    let shots_used = metrics.get("shots_used").copied();
    let logical_errors = metrics.get("logical_errors").copied();
    let total_decode_us = metrics.get("total_decode_us").copied();

    if let (Some(logical_errors), Some(shots_used)) = (logical_errors, shots_used) {
        if shots_used != 0.0 {
            metrics.insert("logical_error_rate".into(), logical_errors / shots_used);
        } else {
            metrics.remove("logical_error_rate");
        }
    } else {
        metrics.remove("logical_error_rate");
    }

    if let (Some(total_decode_us), Some(shots_used)) = (total_decode_us, shots_used) {
        if shots_used != 0.0 {
            metrics.insert("decode_us_per_shot".into(), total_decode_us / shots_used);
        } else {
            metrics.remove("decode_us_per_shot");
        }
    } else {
        metrics.remove("decode_us_per_shot");
    }
}

fn conflict(identity: &str, field: &str) -> String {
    format!("cannot merge benchmark rows with identity {identity}: conflicting {field}")
}