rsinter 0.4.0

Experimental parallel sampling and decoder benchmark harness for RustQEC
Documentation
use std::collections::{BTreeMap, BTreeSet};
use std::fs::{self, File};
use std::path::{Path, PathBuf};

use crate::bench::merge::merge_result_rows;
use crate::bench::registry::{
    expand_runner_points_for_runner, BenchCasePoint, BenchRunContext, RustBenchRunner,
    RustRunnerRegistry,
};
use crate::bench::result::{
    read_results_jsonl, write_results_jsonl, BenchmarkResultRow, RunManifest,
};
use crate::bench::spec::{BenchmarkSpec, RunnerSpec};

struct PlannedRustRun<'a> {
    runner: &'a RunnerSpec,
    runner_impl: &'a dyn RustBenchRunner,
    points: Vec<BenchCasePoint>,
}

#[derive(Debug, Clone, Copy, Default)]
pub struct BenchRunOptions {
    pub resume: bool,
}

pub fn run_rust_benchmark(
    spec: &BenchmarkSpec,
    language: &str,
    out_root: &Path,
    registry: &RustRunnerRegistry,
    spec_dir: &Path,
) -> Result<PathBuf, String> {
    run_rust_benchmark_with_options(
        spec,
        language,
        out_root,
        registry,
        spec_dir,
        BenchRunOptions::default(),
    )
}

pub fn run_rust_benchmark_with_options(
    spec: &BenchmarkSpec,
    language: &str,
    out_root: &Path,
    registry: &RustRunnerRegistry,
    spec_dir: &Path,
    options: BenchRunOptions,
) -> Result<PathBuf, String> {
    spec.validate()?;
    fs::create_dir_all(out_root).map_err(|e| e.to_string())?;
    if !options.resume {
        clear_rust_run_artifacts(spec, language, out_root)?;
    }
    let planned_runs = plan_rust_runs(spec, language, registry)?;
    let resume_rows = if options.resume {
        load_resume_rows(&planned_runs, out_root)?
    } else {
        BTreeMap::new()
    };

    for PlannedRustRun {
        runner,
        runner_impl,
        points,
    } in planned_runs
    {
        let artifact_dir = out_root.join(&runner.name).join("test-run");
        let staging_dir = out_root.join(&runner.name).join("test-run.tmp");
        if staging_dir.exists() {
            fs::remove_dir_all(&staging_dir).map_err(|e| e.to_string())?;
        }

        let ctx = BenchRunContext {
            benchmark_name: spec.name.clone(),
            runner_name: runner.name.clone(),
            language: language.to_string(),
            seed: 12_345,
            spec_dir: spec_dir.to_path_buf(),
        };
        let runner_resume_rows = resume_rows.get(&runner.name).cloned().unwrap_or_default();
        let existing_rows = prune_completed_identity_failures(runner_resume_rows)?;
        let completed = completed_identities(&existing_rows)?;
        let mut fresh_rows = Vec::new();
        for point in &points {
            match runner_impl.plan_point_identity(point, &ctx) {
                Ok(identity) if completed.contains(&identity) => continue,
                Ok(_) | Err(_) => {
                    fresh_rows.push(runner_impl.run_point(point, &ctx)?);
                }
            }
        }
        let rows = if options.resume {
            let retained_rows = drop_replaced_incomplete_rows(existing_rows, &fresh_rows)?;
            merge_result_rows(vec![retained_rows, fresh_rows])?
        } else {
            fresh_rows
        };

        fs::create_dir_all(&staging_dir).map_err(|e| e.to_string())?;
        let manifest = RunManifest::new(
            spec.name.clone(),
            spec.version,
            runner.name.clone(),
            language.to_string(),
            artifact_dir.display().to_string(),
        );
        serde_json::to_writer_pretty(
            File::create(staging_dir.join("run_manifest.json")).map_err(|e| e.to_string())?,
            &manifest,
        )
        .map_err(|e| e.to_string())?;
        let mut file =
            File::create(staging_dir.join("results.jsonl")).map_err(|e| e.to_string())?;
        write_results_jsonl(&rows, &mut file)?;
        if artifact_dir.exists() {
            fs::remove_dir_all(&artifact_dir).map_err(|e| e.to_string())?;
        }
        fs::rename(&staging_dir, &artifact_dir).map_err(|e| e.to_string())?;
    }

    Ok(out_root.to_path_buf())
}

fn clear_rust_run_artifacts(
    spec: &BenchmarkSpec,
    language: &str,
    out_root: &Path,
) -> Result<(), String> {
    for runner in spec
        .runners
        .iter()
        .filter(|runner| runner.language == language)
    {
        let artifact_dir = out_root.join(&runner.name).join("test-run");
        let staging_dir = out_root.join(&runner.name).join("test-run.tmp");
        if artifact_dir.exists() {
            fs::remove_dir_all(&artifact_dir).map_err(|e| e.to_string())?;
        }
        if staging_dir.exists() {
            fs::remove_dir_all(&staging_dir).map_err(|e| e.to_string())?;
        }
    }
    Ok(())
}

fn plan_rust_runs<'a>(
    spec: &'a BenchmarkSpec,
    language: &str,
    registry: &'a RustRunnerRegistry,
) -> Result<Vec<PlannedRustRun<'a>>, String> {
    let mut planned_runs = Vec::new();
    for runner in spec
        .runners
        .iter()
        .filter(|runner| runner.language == language)
    {
        let runner_impl = registry
            .get(&runner.impl_key)
            .ok_or_else(|| format!("unknown rust runner: {}", runner.impl_key))?;
        let points = expand_runner_points_for_runner(runner_impl.name(), &runner.params)?;
        for point in &points {
            runner_impl.preflight_point(point)?;
        }
        planned_runs.push(PlannedRustRun {
            runner,
            runner_impl: runner_impl.as_ref(),
            points,
        });
    }
    Ok(planned_runs)
}

fn load_resume_rows(
    planned_runs: &[PlannedRustRun<'_>],
    out_root: &Path,
) -> Result<BTreeMap<String, Vec<BenchmarkResultRow>>, String> {
    let mut rows_by_runner = BTreeMap::new();
    for planned in planned_runs {
        let path = out_root
            .join(&planned.runner.name)
            .join("test-run")
            .join("results.jsonl");
        if !path.exists() {
            continue;
        }
        let data = fs::read(&path).map_err(|error| {
            format!("failed to read resume results {}: {error}", path.display())
        })?;
        let rows = read_results_jsonl(&data[..]).map_err(|error| {
            format!("failed to read resume results {}: {error}", path.display())
        })?;
        rows_by_runner.insert(planned.runner.name.clone(), rows);
    }
    Ok(rows_by_runner)
}

fn completed_identities(rows: &[BenchmarkResultRow]) -> Result<BTreeSet<String>, String> {
    let mut completed = BTreeSet::new();
    for row in rows {
        if row.status == "ok" {
            completed.insert(row.identity()?);
        }
    }
    Ok(completed)
}

fn prune_completed_identity_failures(
    existing_rows: Vec<BenchmarkResultRow>,
) -> Result<Vec<BenchmarkResultRow>, String> {
    let completed = completed_identities(&existing_rows)?;
    let mut kept_rows = Vec::with_capacity(existing_rows.len());
    for row in existing_rows {
        if row.status == "ok" {
            kept_rows.push(row);
            continue;
        }
        let identity = row.identity()?;
        if !completed.contains(&identity) {
            kept_rows.push(row);
        }
    }
    Ok(kept_rows)
}

fn drop_replaced_incomplete_rows(
    existing_rows: Vec<BenchmarkResultRow>,
    fresh_rows: &[BenchmarkResultRow],
) -> Result<Vec<BenchmarkResultRow>, String> {
    let rerun_identities = fresh_rows
        .iter()
        .map(BenchmarkResultRow::identity)
        .collect::<Result<BTreeSet<_>, _>>()?;
    let mut kept_rows = Vec::with_capacity(existing_rows.len());
    for row in existing_rows {
        if row.status == "ok" {
            kept_rows.push(row);
            continue;
        }
        let identity = row.identity()?;
        if !rerun_identities.contains(&identity) {
            kept_rows.push(row);
        }
    }
    Ok(kept_rows)
}