deep_delta_learning/

benchmark.rs

use std::fmt::{Display, Formatter};
use std::hint::black_box;
use std::mem::size_of;
use std::time::Instant;

use burn::prelude::*;
use burn::tensor::{Int, Shape, TensorData};
use serde::{Deserialize, Serialize};

use crate::compressor::{ChannelConvCompressor, TokenConvCompressor};
use crate::config::{CompressionVariant, DdlConfig};
use crate::delta_operator::DeltaOperator;
use crate::delta_res_block::DeltaResBlock;
use crate::rms_norm::RmsNormConfig;
use crate::variant::ModelVariant;

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum BenchmarkSuite {
    DeltaOperator,
    DeltaResBlock,
    Compressor,
    Normalization,
    Model,
}

impl BenchmarkSuite {
    pub const ALL: [Self; 5] = [
        Self::DeltaOperator,
        Self::DeltaResBlock,
        Self::Compressor,
        Self::Normalization,
        Self::Model,
    ];

    pub fn all() -> &'static [Self] {
        &Self::ALL
    }

    pub fn slug(&self) -> &'static str {
        match self {
            Self::DeltaOperator => "operator",
            Self::DeltaResBlock => "block",
            Self::Compressor => "compressor",
            Self::Normalization => "normalization",
            Self::Model => "model",
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct BenchmarkConfig {
    pub iterations: usize,
    pub warmup_iterations: usize,
    pub batch_size: usize,
    pub seq_len: usize,
}

impl BenchmarkConfig {
    pub fn new(iterations: usize) -> Self {
        Self {
            iterations,
            ..Default::default()
        }
    }

    pub fn with_warmup_iterations(mut self, warmup_iterations: usize) -> Self {
        self.warmup_iterations = warmup_iterations;
        self
    }

    pub fn with_batch_size(mut self, batch_size: usize) -> Self {
        self.batch_size = batch_size;
        self
    }

    pub fn with_seq_len(mut self, seq_len: usize) -> Self {
        self.seq_len = seq_len;
        self
    }

    pub fn validate(&self) -> Result<(), BenchmarkError> {
        if self.iterations == 0 {
            return Err(BenchmarkError::InvalidConfig(
                "iterations must be greater than zero".to_string(),
            ));
        }
        if self.batch_size == 0 {
            return Err(BenchmarkError::InvalidConfig(
                "batch_size must be greater than zero".to_string(),
            ));
        }
        if self.seq_len == 0 {
            return Err(BenchmarkError::InvalidConfig(
                "seq_len must be greater than zero".to_string(),
            ));
        }

        Ok(())
    }
}

impl Default for BenchmarkConfig {
    fn default() -> Self {
        Self {
            iterations: 10,
            warmup_iterations: 2,
            batch_size: 2,
            seq_len: 16,
        }
    }
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum BenchmarkError {
    InvalidConfig(String),
    ComparisonMismatch(String),
}

impl Display for BenchmarkError {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
        match self {
            Self::InvalidConfig(message) => f.write_str(message),
            Self::ComparisonMismatch(message) => f.write_str(message),
        }
    }
}

impl std::error::Error for BenchmarkError {}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
pub struct BenchmarkRegressionThresholds {
    pub min_throughput_ratio: Option<f64>,
    pub max_avg_latency_ratio: Option<f64>,
    pub max_p95_latency_ratio: Option<f64>,
    pub max_peak_memory_ratio: Option<f64>,
}

impl BenchmarkRegressionThresholds {
    pub fn new() -> Self {
        Self::default()
    }

    pub fn with_min_throughput_ratio(mut self, min_throughput_ratio: f64) -> Self {
        self.min_throughput_ratio = Some(min_throughput_ratio);
        self
    }

    pub fn with_max_avg_latency_ratio(mut self, max_avg_latency_ratio: f64) -> Self {
        self.max_avg_latency_ratio = Some(max_avg_latency_ratio);
        self
    }

    pub fn with_max_p95_latency_ratio(mut self, max_p95_latency_ratio: f64) -> Self {
        self.max_p95_latency_ratio = Some(max_p95_latency_ratio);
        self
    }

    pub fn with_max_peak_memory_ratio(mut self, max_peak_memory_ratio: f64) -> Self {
        self.max_peak_memory_ratio = Some(max_peak_memory_ratio);
        self
    }

    pub fn is_configured(&self) -> bool {
        self.min_throughput_ratio.is_some()
            || self.max_avg_latency_ratio.is_some()
            || self.max_p95_latency_ratio.is_some()
            || self.max_peak_memory_ratio.is_some()
    }

    pub fn validate(&self) -> Result<(), BenchmarkError> {
        validate_positive_threshold(self.min_throughput_ratio, "min_throughput_ratio")?;
        validate_positive_threshold(self.max_avg_latency_ratio, "max_avg_latency_ratio")?;
        validate_positive_threshold(self.max_p95_latency_ratio, "max_p95_latency_ratio")?;
        validate_positive_threshold(self.max_peak_memory_ratio, "max_peak_memory_ratio")?;
        Ok(())
    }
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum BenchmarkTarget {
    Case {
        suite: BenchmarkSuite,
        label: String,
    },
    Model {
        variant: ModelVariant,
    },
}

impl BenchmarkTarget {
    pub fn label(&self) -> String {
        match self {
            Self::Case { suite, label } => format!("{}:{label}", suite.slug()),
            Self::Model { variant } => format!("model:{}", variant.slug()),
        }
    }
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct BenchmarkDelta {
    pub target: BenchmarkTarget,
    pub input_shape: Vec<usize>,
    pub output_shape: Vec<usize>,
    pub avg_latency_ratio: f64,
    pub p95_latency_ratio: f64,
    pub throughput_ratio: f64,
    pub peak_memory_ratio: f64,
    pub parameter_ratio: Option<f64>,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct BenchmarkGateOutcome {
    pub passed: bool,
    pub failures: Vec<String>,
}

impl BenchmarkGateOutcome {
    fn from_failures(failures: Vec<String>) -> Self {
        Self {
            passed: failures.is_empty(),
            failures,
        }
    }
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct BenchmarkComparisonReport {
    pub current_benchmark: BenchmarkConfig,
    pub baseline_benchmark: BenchmarkConfig,
    pub current_suites: Vec<BenchmarkSuite>,
    pub baseline_suites: Vec<BenchmarkSuite>,
    pub thresholds: BenchmarkRegressionThresholds,
    pub deltas: Vec<BenchmarkDelta>,
    pub gate: BenchmarkGateOutcome,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BenchmarkTiming {
    pub iterations: usize,
    pub warmup_iterations: usize,
    pub total_duration_ms: f64,
    pub avg_duration_ms: f64,
    pub min_duration_ms: f64,
    pub median_duration_ms: f64,
    pub p95_duration_ms: f64,
    pub max_duration_ms: f64,
    pub iterations_per_second: f64,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct BenchmarkMemory {
    pub input_bytes: usize,
    pub output_bytes: usize,
    pub parameter_bytes: usize,
    pub working_set_bytes: usize,
    pub peak_live_bytes: usize,
}

impl BenchmarkMemory {
    fn new(
        input_bytes: usize,
        output_bytes: usize,
        parameter_bytes: usize,
        working_set_bytes: usize,
    ) -> Self {
        Self {
            input_bytes,
            output_bytes,
            parameter_bytes,
            working_set_bytes,
            peak_live_bytes: input_bytes
                .saturating_add(output_bytes)
                .saturating_add(parameter_bytes)
                .saturating_add(working_set_bytes),
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BenchmarkCase {
    pub label: String,
    pub input_shape: Vec<usize>,
    pub output_shape: Vec<usize>,
    pub timing: BenchmarkTiming,
    pub memory: BenchmarkMemory,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ModelMemoryBreakdown {
    pub input_ids_bytes: usize,
    pub parameter_bytes: usize,
    pub embedding_bytes: usize,
    pub state_bytes: usize,
    pub residual_stream_bytes: usize,
    pub mask_bytes: usize,
    pub attention_scores_bytes: usize,
    pub compressor_workspace_bytes: usize,
    pub delta_branch_bytes: usize,
    pub mlp_activation_bytes: usize,
    pub logits_bytes: usize,
    pub peak_activation_bytes: usize,
    pub peak_total_bytes: usize,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DeltaOperatorBenchmarks {
    pub vector: BenchmarkCase,
    pub matrix: BenchmarkCase,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DeltaResBlockBenchmarks {
    pub vector: BenchmarkCase,
    pub matrix: BenchmarkCase,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CompressorBenchmarks {
    pub token_conv: BenchmarkCase,
    pub channel_conv: BenchmarkCase,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NormalizationBenchmarks {
    pub precision_friendly: BenchmarkCase,
    pub explicit_l2: BenchmarkCase,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelBenchmark {
    pub variant: ModelVariant,
    pub resolved_config: DdlConfig,
    pub num_params: usize,
    pub input_shape: [usize; 2],
    pub output_shape: [usize; 3],
    pub timing: BenchmarkTiming,
    pub tokens_per_second: f64,
    pub estimated_hidden_state_bytes: usize,
    pub estimated_logit_bytes: usize,
    pub estimated_parameter_bytes: usize,
    pub memory: ModelMemoryBreakdown,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ModelBenchmarkComparison {
    pub variant: ModelVariant,
    pub tokens_per_second: f64,
    pub throughput_ratio_vs_baseline: Option<f64>,
    pub parameter_ratio_vs_baseline: Option<f64>,
    pub peak_activation_ratio_vs_baseline: Option<f64>,
    pub peak_total_ratio_vs_baseline: Option<f64>,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ModelBenchmarkSummary {
    pub fastest_variant: ModelVariant,
    pub lowest_peak_total_variant: ModelVariant,
    pub comparisons: Vec<ModelBenchmarkComparison>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BenchmarkReport {
    pub benchmark: BenchmarkConfig,
    pub base_config: DdlConfig,
    pub suites: Vec<BenchmarkSuite>,
    pub delta_operator: Option<DeltaOperatorBenchmarks>,
    pub delta_res_block: Option<DeltaResBlockBenchmarks>,
    pub compressors: Option<CompressorBenchmarks>,
    pub normalization: Option<NormalizationBenchmarks>,
    pub models: Vec<ModelBenchmark>,
    pub model_summary: Option<ModelBenchmarkSummary>,
}

pub fn run_benchmarks<B: Backend>(
    base_config: &DdlConfig,
    benchmark: &BenchmarkConfig,
    suites: &[BenchmarkSuite],
    variants: &[ModelVariant],
    device: &B::Device,
) -> Result<BenchmarkReport, BenchmarkError> {
    benchmark.validate()?;
    if benchmark.seq_len > base_config.max_seq_len {
        return Err(BenchmarkError::InvalidConfig(format!(
            "benchmark seq_len {} exceeds model max_seq_len {}",
            benchmark.seq_len, base_config.max_seq_len
        )));
    }

    let suites = normalize_suites(suites);
    let variants = normalize_variants(variants);
    let delta_operator = suites
        .contains(&BenchmarkSuite::DeltaOperator)
        .then(|| benchmark_delta_operator::<B>(base_config, benchmark, device));
    let delta_res_block = suites
        .contains(&BenchmarkSuite::DeltaResBlock)
        .then(|| benchmark_delta_res_block::<B>(base_config, benchmark, device));
    let compressors = suites
        .contains(&BenchmarkSuite::Compressor)
        .then(|| benchmark_compressors::<B>(base_config, benchmark, device));
    let normalization = suites
        .contains(&BenchmarkSuite::Normalization)
        .then(|| benchmark_normalization::<B>(base_config, benchmark, device));
    let models = if suites.contains(&BenchmarkSuite::Model) {
        benchmark_models::<B>(base_config, benchmark, &variants, device)
    } else {
        Vec::new()
    };
    let model_summary = (!models.is_empty()).then(|| summarize_model_benchmarks(&models));

    Ok(BenchmarkReport {
        benchmark: *benchmark,
        base_config: base_config.clone(),
        suites,
        delta_operator,
        delta_res_block,
        compressors,
        normalization,
        models,
        model_summary,
    })
}

pub fn compare_benchmark_reports(
    current: &BenchmarkReport,
    baseline: &BenchmarkReport,
    thresholds: &BenchmarkRegressionThresholds,
) -> Result<BenchmarkComparisonReport, BenchmarkError> {
    thresholds.validate()?;

    let mut deltas = Vec::new();
    for suite in &current.suites {
        match suite {
            BenchmarkSuite::DeltaOperator => {
                let current_suite = current.delta_operator.as_ref().ok_or_else(|| {
                    BenchmarkError::ComparisonMismatch(
                        "current report is missing the delta operator suite".to_string(),
                    )
                })?;
                let baseline_suite = baseline.delta_operator.as_ref().ok_or_else(|| {
                    BenchmarkError::ComparisonMismatch(
                        "baseline report is missing the delta operator suite".to_string(),
                    )
                })?;
                deltas.push(compare_case(
                    *suite,
                    &current_suite.vector,
                    &baseline_suite.vector,
                )?);
                deltas.push(compare_case(
                    *suite,
                    &current_suite.matrix,
                    &baseline_suite.matrix,
                )?);
            }
            BenchmarkSuite::DeltaResBlock => {
                let current_suite = current.delta_res_block.as_ref().ok_or_else(|| {
                    BenchmarkError::ComparisonMismatch(
                        "current report is missing the delta-res block suite".to_string(),
                    )
                })?;
                let baseline_suite = baseline.delta_res_block.as_ref().ok_or_else(|| {
                    BenchmarkError::ComparisonMismatch(
                        "baseline report is missing the delta-res block suite".to_string(),
                    )
                })?;
                deltas.push(compare_case(
                    *suite,
                    &current_suite.vector,
                    &baseline_suite.vector,
                )?);
                deltas.push(compare_case(
                    *suite,
                    &current_suite.matrix,
                    &baseline_suite.matrix,
                )?);
            }
            BenchmarkSuite::Compressor => {
                let current_suite = current.compressors.as_ref().ok_or_else(|| {
                    BenchmarkError::ComparisonMismatch(
                        "current report is missing the compressor suite".to_string(),
                    )
                })?;
                let baseline_suite = baseline.compressors.as_ref().ok_or_else(|| {
                    BenchmarkError::ComparisonMismatch(
                        "baseline report is missing the compressor suite".to_string(),
                    )
                })?;
                deltas.push(compare_case(
                    *suite,
                    &current_suite.token_conv,
                    &baseline_suite.token_conv,
                )?);
                deltas.push(compare_case(
                    *suite,
                    &current_suite.channel_conv,
                    &baseline_suite.channel_conv,
                )?);
            }
            BenchmarkSuite::Normalization => {
                let current_suite = current.normalization.as_ref().ok_or_else(|| {
                    BenchmarkError::ComparisonMismatch(
                        "current report is missing the normalization suite".to_string(),
                    )
                })?;
                let baseline_suite = baseline.normalization.as_ref().ok_or_else(|| {
                    BenchmarkError::ComparisonMismatch(
                        "baseline report is missing the normalization suite".to_string(),
                    )
                })?;
                deltas.push(compare_case(
                    *suite,
                    &current_suite.precision_friendly,
                    &baseline_suite.precision_friendly,
                )?);
                deltas.push(compare_case(
                    *suite,
                    &current_suite.explicit_l2,
                    &baseline_suite.explicit_l2,
                )?);
            }
            BenchmarkSuite::Model => {
                if current.models.is_empty() {
                    return Err(BenchmarkError::ComparisonMismatch(
                        "current report is missing the model suite".to_string(),
                    ));
                }
                if baseline.models.is_empty() {
                    return Err(BenchmarkError::ComparisonMismatch(
                        "baseline report is missing the model suite".to_string(),
                    ));
                }
                for current_model in &current.models {
                    let baseline_model = baseline
                        .models
                        .iter()
                        .find(|model| model.variant == current_model.variant)
                        .ok_or_else(|| {
                            BenchmarkError::ComparisonMismatch(format!(
                                "baseline report is missing model variant {}",
                                current_model.variant.slug()
                            ))
                        })?;
                    deltas.push(compare_model(current_model, baseline_model)?);
                }
            }
        }
    }

    let failures = evaluate_gate(&deltas, thresholds);
    Ok(BenchmarkComparisonReport {
        current_benchmark: current.benchmark,
        baseline_benchmark: baseline.benchmark,
        current_suites: current.suites.clone(),
        baseline_suites: baseline.suites.clone(),
        thresholds: thresholds.clone(),
        deltas,
        gate: BenchmarkGateOutcome::from_failures(failures),
    })
}

fn benchmark_delta_operator<B: Backend>(
    base_config: &DdlConfig,
    benchmark: &BenchmarkConfig,
    device: &B::Device,
) -> DeltaOperatorBenchmarks {
    let batch_tokens = benchmark.batch_size * benchmark.seq_len;
    let d_model = base_config.d_model;
    let d_value = matrix_d_value(base_config);
    let k = direction_tensor::<B>(batch_tokens, d_model, device);
    let beta = beta_tensor::<B>(batch_tokens, 0.75, device);
    let x_vector = float_tensor::<B, 2>([batch_tokens, d_model], 0.125, device);
    let x_matrix = float_tensor::<B, 3>([batch_tokens, d_model, d_value], 0.375, device);

    let vector_operator = DeltaOperator::new(k.clone(), beta.clone());
    let vector_output = vector_operator.apply_vector(&x_vector);
    let vector = BenchmarkCase {
        label: "delta-operator-vector".to_string(),
        input_shape: vec![batch_tokens, d_model],
        output_shape: vector_output.dims().to_vec(),
        timing: measure(benchmark, || {
            let output = vector_operator.apply_vector(&x_vector);
            black_box(output);
        }),
        memory: BenchmarkMemory::new(
            tensor_bytes(&[batch_tokens, d_model]),
            tensor_bytes(&[batch_tokens, d_model]),
            0,
            tensor_bytes(&[batch_tokens, d_model])
                + tensor_bytes(&[batch_tokens])
                + tensor_bytes(&[batch_tokens])
                + tensor_bytes(&[batch_tokens, d_model]),
        ),
    };

    let matrix_operator = DeltaOperator::new(k, beta);
    let matrix_output = matrix_operator.apply(&x_matrix);
    let matrix = BenchmarkCase {
        label: "delta-operator-matrix".to_string(),
        input_shape: vec![batch_tokens, d_model, d_value],
        output_shape: matrix_output.dims().to_vec(),
        timing: measure(benchmark, || {
            let output = matrix_operator.apply(&x_matrix);
            black_box(output);
        }),
        memory: BenchmarkMemory::new(
            tensor_bytes(&[batch_tokens, d_model, d_value]),
            tensor_bytes(&[batch_tokens, d_model, d_value]),
            0,
            tensor_bytes(&[batch_tokens, d_model])
                + tensor_bytes(&[batch_tokens])
                + tensor_bytes(&[batch_tokens, d_value])
                + tensor_bytes(&[batch_tokens, d_model, d_value]),
        ),
    };

    DeltaOperatorBenchmarks { vector, matrix }
}

fn benchmark_delta_res_block<B: Backend>(
    base_config: &DdlConfig,
    benchmark: &BenchmarkConfig,
    device: &B::Device,
) -> DeltaResBlockBenchmarks {
    let vector_config = base_config.clone().with_d_value(1);
    let matrix_config = base_config
        .clone()
        .with_d_value(matrix_d_value(base_config));
    let [batch_size, seq_len, d_model] =
        [benchmark.batch_size, benchmark.seq_len, base_config.d_model];

    let vector_block = DeltaResBlock::<B>::new(&vector_config, device);
    let x_vector = float_tensor::<B, 3>([batch_size, seq_len, d_model], 0.125, device);
    let vector_backbone = float_tensor::<B, 3>([batch_size, seq_len, d_model], 0.25, device);
    let vector_ctx = float_tensor::<B, 3>([batch_size, seq_len, d_model], 0.5, device);
    let vector_residual = float_tensor::<B, 3>([batch_size, seq_len, d_model], 0.75, device);
    let vector_output = vector_block.forward_vector_hidden(
        x_vector.clone(),
        vector_backbone.clone(),
        vector_ctx.clone(),
        vector_residual.clone(),
    );
    let vector = BenchmarkCase {
        label: "delta-res-block-vector".to_string(),
        input_shape: vec![batch_size, seq_len, d_model],
        output_shape: vector_output.dims().to_vec(),
        timing: measure(benchmark, || {
            let output = vector_block.forward_vector_hidden(
                x_vector.clone(),
                vector_backbone.clone(),
                vector_ctx.clone(),
                vector_residual.clone(),
            );
            black_box(output);
        }),
        memory: BenchmarkMemory::new(
            tensor_bytes(&[batch_size, seq_len, d_model]) * 4,
            tensor_bytes(&[batch_size, seq_len, d_model]),
            f32_bytes(vector_block.num_params()),
            tensor_bytes(&[batch_size, seq_len, d_model])
                + tensor_bytes(&[batch_size, seq_len]) * 4
                + tensor_bytes(&[batch_size, seq_len, 1]),
        ),
    };

    let matrix_block = DeltaResBlock::<B>::new(&matrix_config, device);
    let d_value = matrix_config.d_value;
    let x_state = float_tensor::<B, 4>([batch_size, seq_len, d_model, d_value], 0.125, device);
    let matrix_backbone = float_tensor::<B, 3>([batch_size, seq_len, d_model], 0.25, device);
    let matrix_ctx = float_tensor::<B, 3>([batch_size, seq_len, d_model], 0.5, device);
    let matrix_residual = float_tensor::<B, 3>([batch_size, seq_len, d_model], 0.75, device);
    let matrix_output = matrix_block.forward_matrix_hidden(
        x_state.clone(),
        matrix_backbone.clone(),
        matrix_ctx.clone(),
        matrix_residual.clone(),
    );
    let matrix = BenchmarkCase {
        label: "delta-res-block-matrix".to_string(),
        input_shape: vec![batch_size, seq_len, d_model, d_value],
        output_shape: matrix_output.dims().to_vec(),
        timing: measure(benchmark, || {
            let output = matrix_block.forward_matrix_hidden(
                x_state.clone(),
                matrix_backbone.clone(),
                matrix_ctx.clone(),
                matrix_residual.clone(),
            );
            black_box(output);
        }),
        memory: BenchmarkMemory::new(
            tensor_bytes(&[batch_size, seq_len, d_model, d_value])
                + tensor_bytes(&[batch_size, seq_len, d_model]) * 3,
            tensor_bytes(&[batch_size, seq_len, d_model, d_value]),
            f32_bytes(matrix_block.num_params()),
            tensor_bytes(&[batch_size, seq_len, d_model])
                + tensor_bytes(&[batch_size, seq_len, d_value]) * 3
                + tensor_bytes(&[batch_size, seq_len])
                + tensor_bytes(&[batch_size, seq_len, d_model, d_value]),
        ),
    };

    DeltaResBlockBenchmarks { vector, matrix }
}

fn benchmark_compressors<B: Backend>(
    base_config: &DdlConfig,
    benchmark: &BenchmarkConfig,
    device: &B::Device,
) -> CompressorBenchmarks {
    let d_model = base_config.d_model;
    let d_value = matrix_d_value(base_config);
    let batch_size = benchmark.batch_size;
    let seq_len = benchmark.seq_len;
    let state = float_tensor::<B, 4>([batch_size, seq_len, d_model, d_value], 0.5, device);

    let token =
        TokenConvCompressor::<B>::new(d_model, d_value, base_config.shortconv_kernel_size, device);
    let token_output = token.forward(state.clone());
    let token_conv = BenchmarkCase {
        label: "token-conv-compressor".to_string(),
        input_shape: vec![batch_size, seq_len, d_model, d_value],
        output_shape: token_output.dims().to_vec(),
        timing: measure(benchmark, || {
            let output = token.forward(state.clone());
            black_box(output);
        }),
        memory: BenchmarkMemory::new(
            tensor_bytes(&[batch_size, seq_len, d_model, d_value]),
            tensor_bytes(&[batch_size, seq_len, d_model]),
            f32_bytes(token.num_params()),
            tensor_bytes(&[batch_size, seq_len, d_model, d_value]),
        ),
    };

    let channel = ChannelConvCompressor::<B>::new(d_model, d_value, device);
    let channel_output = channel.forward(state.clone());
    let channel_conv = BenchmarkCase {
        label: "channel-conv-compressor".to_string(),
        input_shape: vec![batch_size, seq_len, d_model, d_value],
        output_shape: channel_output.dims().to_vec(),
        timing: measure(benchmark, || {
            let output = channel.forward(state.clone());
            black_box(output);
        }),
        memory: BenchmarkMemory::new(
            tensor_bytes(&[batch_size, seq_len, d_model, d_value]),
            tensor_bytes(&[batch_size, seq_len, d_model]),
            f32_bytes(channel.num_params()),
            tensor_bytes(&[batch_size * seq_len, d_model, 1]),
        ),
    };

    CompressorBenchmarks {
        token_conv,
        channel_conv,
    }
}

fn benchmark_normalization<B: Backend>(
    base_config: &DdlConfig,
    benchmark: &BenchmarkConfig,
    device: &B::Device,
) -> NormalizationBenchmarks {
    let d_model = base_config.d_model;
    let input = float_tensor::<B, 3>(
        [benchmark.batch_size, benchmark.seq_len, d_model],
        0.875,
        device,
    );
    let epsilon = base_config.k_eps * base_config.k_eps / d_model as f64;
    let scale = 1.0 / (d_model as f32).sqrt();
    let precision_friendly_norm = RmsNormConfig::new(d_model)
        .with_epsilon(epsilon)
        .init(device);
    let precision_output = precision_friendly_norm
        .forward(input.clone())
        .mul_scalar(scale);
    let precision_friendly = BenchmarkCase {
        label: "k-normalization-rms".to_string(),
        input_shape: vec![benchmark.batch_size, benchmark.seq_len, d_model],
        output_shape: precision_output.dims().to_vec(),
        timing: measure(benchmark, || {
            let output = precision_friendly_norm
                .forward(input.clone())
                .mul_scalar(scale);
            black_box(output);
        }),
        memory: BenchmarkMemory::new(
            tensor_bytes(&[benchmark.batch_size, benchmark.seq_len, d_model]),
            tensor_bytes(&[benchmark.batch_size, benchmark.seq_len, d_model]),
            f32_bytes(precision_friendly_norm.num_params()),
            tensor_bytes(&[benchmark.batch_size, benchmark.seq_len, d_model]),
        ),
    };

    let explicit_output = explicit_l2_normalize(input.clone());
    let explicit_l2 = BenchmarkCase {
        label: "k-normalization-l2".to_string(),
        input_shape: vec![benchmark.batch_size, benchmark.seq_len, d_model],
        output_shape: explicit_output.dims().to_vec(),
        timing: measure(benchmark, || {
            let output = explicit_l2_normalize(input.clone());
            black_box(output);
        }),
        memory: BenchmarkMemory::new(
            tensor_bytes(&[benchmark.batch_size, benchmark.seq_len, d_model]),
            tensor_bytes(&[benchmark.batch_size, benchmark.seq_len, d_model]),
            0,
            tensor_bytes(&[benchmark.batch_size, benchmark.seq_len]),
        ),
    };

    NormalizationBenchmarks {
        precision_friendly,
        explicit_l2,
    }
}

fn benchmark_models<B: Backend>(
    base_config: &DdlConfig,
    benchmark: &BenchmarkConfig,
    variants: &[ModelVariant],
    device: &B::Device,
) -> Vec<ModelBenchmark> {
    let input_shape = [benchmark.batch_size, benchmark.seq_len];
    let input_ids = int_tensor::<B>(input_shape, base_config.vocab_size, device);

    variants
        .iter()
        .copied()
        .map(|variant| {
            let (resolved_config, model) = variant.build::<B>(base_config, device);
            let output = model.forward_logits(input_ids.clone(), None);
            let output_shape = output.dims();
            let timing = measure(benchmark, || {
                let logits = model.forward_logits(input_ids.clone(), None);
                black_box(logits);
            });
            let total_tokens = benchmark.batch_size * benchmark.seq_len * benchmark.iterations;
            let estimated_hidden_state_bytes = f32_bytes(
                benchmark.batch_size
                    * benchmark.seq_len
                    * resolved_config.d_model
                    * resolved_config.d_value,
            );
            let estimated_logit_bytes =
                f32_bytes(benchmark.batch_size * benchmark.seq_len * resolved_config.vocab_size);
            let memory =
                estimate_model_memory(variant, &resolved_config, model.num_params(), benchmark);

            ModelBenchmark {
                variant,
                resolved_config,
                num_params: model.num_params(),
                input_shape,
                output_shape,
                tokens_per_second: if timing.total_duration_ms == 0.0 {
                    0.0
                } else {
                    total_tokens as f64 / (timing.total_duration_ms / 1_000.0)
                },
                estimated_hidden_state_bytes,
                estimated_logit_bytes,
                estimated_parameter_bytes: f32_bytes(model.num_params()),
                memory,
                timing,
            }
        })
        .collect()
}

fn estimate_model_memory(
    variant: ModelVariant,
    config: &DdlConfig,
    num_params: usize,
    benchmark: &BenchmarkConfig,
) -> ModelMemoryBreakdown {
    let batch_size = benchmark.batch_size;
    let seq_len = benchmark.seq_len;
    let d_model = config.d_model;
    let d_value = config.d_value;
    let hidden_bytes = tensor_bytes(&[batch_size, seq_len, d_model]);
    let state_bytes = if config.uses_matrix_state() {
        tensor_bytes(&[batch_size, seq_len, d_model, d_value])
    } else {
        hidden_bytes
    };
    let input_ids_bytes = int_bytes(&[batch_size, seq_len]);
    let parameter_bytes = f32_bytes(num_params);
    let mask_bytes = tensor_bytes(&[batch_size, seq_len, seq_len]);
    let attention_scores_bytes = tensor_bytes(&[batch_size, config.num_heads, seq_len, seq_len]);
    let compressor_workspace_bytes = compressor_workspace_bytes(config, batch_size, seq_len);
    let delta_branch_bytes = if variant.uses_ddl() {
        ddl_branch_bytes(config, batch_size, seq_len)
    } else {
        0
    };
    let mlp_activation_bytes =
        hidden_bytes + tensor_bytes(&[batch_size, seq_len, config.effective_mlp_hidden()]) * 2;
    let logits_bytes = tensor_bytes(&[batch_size, seq_len, config.vocab_size]);
    let qkv_bytes = hidden_bytes * 3;
    let attn_stage_peak = state_bytes
        + compressor_workspace_bytes
        + hidden_bytes
        + qkv_bytes
        + attention_scores_bytes
        + delta_branch_bytes;
    let mlp_stage_peak = state_bytes
        + compressor_workspace_bytes
        + hidden_bytes
        + mlp_activation_bytes
        + delta_branch_bytes;
    let embedding_stage_peak = input_ids_bytes
        + hidden_bytes
        + state_bytes
        + if config.uses_matrix_state() && config.embed_conv {
            state_bytes
        } else {
            0
        };
    let final_stage_peak = state_bytes + compressor_workspace_bytes + hidden_bytes + logits_bytes;
    let peak_activation_bytes = embedding_stage_peak
        .max(attn_stage_peak.max(mlp_stage_peak))
        .max(final_stage_peak);

    ModelMemoryBreakdown {
        input_ids_bytes,
        parameter_bytes,
        embedding_bytes: hidden_bytes,
        state_bytes,
        residual_stream_bytes: hidden_bytes,
        mask_bytes,
        attention_scores_bytes,
        compressor_workspace_bytes,
        delta_branch_bytes,
        mlp_activation_bytes,
        logits_bytes,
        peak_activation_bytes,
        peak_total_bytes: parameter_bytes
            .saturating_add(mask_bytes)
            .saturating_add(peak_activation_bytes),
    }
}

fn summarize_model_benchmarks(models: &[ModelBenchmark]) -> ModelBenchmarkSummary {
    let baseline = models
        .iter()
        .find(|model| model.variant == ModelVariant::Baseline);
    let fastest_variant = models
        .iter()
        .max_by(|left, right| left.tokens_per_second.total_cmp(&right.tokens_per_second))
        .map(|model| model.variant)
        .unwrap_or(ModelVariant::Baseline);
    let lowest_peak_total_variant = models
        .iter()
        .min_by_key(|model| model.memory.peak_total_bytes)
        .map(|model| model.variant)
        .unwrap_or(ModelVariant::Baseline);
    let comparisons = models
        .iter()
        .map(|model| ModelBenchmarkComparison {
            variant: model.variant,
            tokens_per_second: model.tokens_per_second,
            throughput_ratio_vs_baseline: baseline.and_then(|baseline| {
                ratio_f64(model.tokens_per_second, baseline.tokens_per_second)
            }),
            parameter_ratio_vs_baseline: baseline
                .map(|baseline| ratio_usize(model.num_params, baseline.num_params)),
            peak_activation_ratio_vs_baseline: baseline.map(|baseline| {
                ratio_usize(
                    model.memory.peak_activation_bytes,
                    baseline.memory.peak_activation_bytes,
                )
            }),
            peak_total_ratio_vs_baseline: baseline.map(|baseline| {
                ratio_usize(
                    model.memory.peak_total_bytes,
                    baseline.memory.peak_total_bytes,
                )
            }),
        })
        .collect();

    ModelBenchmarkSummary {
        fastest_variant,
        lowest_peak_total_variant,
        comparisons,
    }
}

fn explicit_l2_normalize<B: Backend>(input: Tensor<B, 3>) -> Tensor<B, 3> {
    let norms = (input.clone() * input.clone()).sum_dim(2).sqrt();
    input / norms
}

fn measure<F>(benchmark: &BenchmarkConfig, mut bench_fn: F) -> BenchmarkTiming
where
    F: FnMut(),
{
    for _ in 0..benchmark.warmup_iterations {
        bench_fn();
    }

    let mut samples_ms = Vec::with_capacity(benchmark.iterations);
    let total_start = Instant::now();
    for _ in 0..benchmark.iterations {
        let sample_start = Instant::now();
        bench_fn();
        samples_ms.push(sample_start.elapsed().as_secs_f64() * 1_000.0);
    }
    let total_duration_ms = total_start.elapsed().as_secs_f64() * 1_000.0;
    let avg_duration_ms = samples_ms.iter().copied().sum::<f64>() / samples_ms.len() as f64;
    let min_duration_ms = samples_ms.iter().copied().fold(f64::INFINITY, f64::min);
    let max_duration_ms = samples_ms.iter().copied().fold(0.0, f64::max);

    BenchmarkTiming {
        iterations: benchmark.iterations,
        warmup_iterations: benchmark.warmup_iterations,
        total_duration_ms,
        avg_duration_ms,
        min_duration_ms,
        median_duration_ms: percentile(&samples_ms, 0.5),
        p95_duration_ms: percentile(&samples_ms, 0.95),
        max_duration_ms,
        iterations_per_second: if total_duration_ms == 0.0 {
            0.0
        } else {
            benchmark.iterations as f64 / (total_duration_ms / 1_000.0)
        },
    }
}

fn percentile(samples: &[f64], percentile: f64) -> f64 {
    let mut ordered = samples.to_vec();
    ordered.sort_by(|left, right| left.total_cmp(right));
    let index = ((ordered.len() - 1) as f64 * percentile).round() as usize;
    ordered[index]
}

fn normalize_suites(suites: &[BenchmarkSuite]) -> Vec<BenchmarkSuite> {
    let suites = if suites.is_empty() {
        BenchmarkSuite::all().to_vec()
    } else {
        suites.to_vec()
    };

    dedup_preserve_order(suites)
}

fn normalize_variants(variants: &[ModelVariant]) -> Vec<ModelVariant> {
    let variants = if variants.is_empty() {
        ModelVariant::all().to_vec()
    } else {
        variants.to_vec()
    };

    dedup_preserve_order(variants)
}

fn dedup_preserve_order<T>(items: Vec<T>) -> Vec<T>
where
    T: Copy + PartialEq,
{
    let mut deduped = Vec::with_capacity(items.len());
    for item in items {
        if !deduped.contains(&item) {
            deduped.push(item);
        }
    }
    deduped
}

fn matrix_d_value(base_config: &DdlConfig) -> usize {
    base_config.d_value.max(4)
}

fn direction_tensor<B: Backend>(rows: usize, d_model: usize, device: &B::Device) -> Tensor<B, 2> {
    let row = normalized_row(d_model);
    let mut data = Vec::with_capacity(rows * d_model);
    for _ in 0..rows {
        data.extend(row.iter().copied());
    }
    Tensor::<B, 2>::from_data(TensorData::new(data, Shape::new([rows, d_model])), device)
}

fn beta_tensor<B: Backend>(rows: usize, value: f32, device: &B::Device) -> Tensor<B, 1> {
    let data = vec![value; rows];
    Tensor::<B, 1>::from_data(TensorData::new(data, Shape::new([rows])), device)
}

fn float_tensor<B: Backend, const D: usize>(
    shape: [usize; D],
    offset: f32,
    device: &B::Device,
) -> Tensor<B, D> {
    let numel = shape.iter().product::<usize>();
    let data = (0..numel)
        .map(|idx| ((idx % 97) as f32 + 1.0) / 97.0 + offset)
        .collect::<Vec<_>>();
    Tensor::<B, D>::from_data(TensorData::new(data, Shape::new(shape)), device)
}

fn int_tensor<B: Backend>(
    shape: [usize; 2],
    vocab_size: usize,
    device: &B::Device,
) -> Tensor<B, 2, Int> {
    let numel = shape.iter().product::<usize>();
    let data = (0..numel)
        .map(|idx| ((idx * 7 + 3) % vocab_size.max(2)) as i64)
        .collect::<Vec<_>>();
    Tensor::<B, 2, Int>::from_data(TensorData::new(data, Shape::new(shape)), device)
}

fn normalized_row(d_model: usize) -> Vec<f32> {
    let mut row = (0..d_model).map(|idx| idx as f32 + 1.0).collect::<Vec<_>>();
    let norm = row.iter().map(|value| value * value).sum::<f32>().sqrt();
    for value in &mut row {
        *value /= norm;
    }
    row
}

fn f32_bytes(elements: usize) -> usize {
    elements * size_of::<f32>()
}

fn int_bytes(shape: &[usize]) -> usize {
    shape.iter().product::<usize>() * size_of::<i64>()
}

fn tensor_bytes(shape: &[usize]) -> usize {
    f32_bytes(shape.iter().product())
}

fn compressor_workspace_bytes(config: &DdlConfig, batch_size: usize, seq_len: usize) -> usize {
    if !config.uses_matrix_state() {
        return 0;
    }

    match config.compression {
        CompressionVariant::TokenConv => {
            tensor_bytes(&[batch_size, seq_len, config.d_model, config.d_value])
        }
        CompressionVariant::ChannelConv => tensor_bytes(&[batch_size * seq_len, config.d_model, 1]),
    }
}

fn ddl_branch_bytes(config: &DdlConfig, batch_size: usize, seq_len: usize) -> usize {
    let direction_bytes = tensor_bytes(&[batch_size, seq_len, config.d_model]);
    let beta_bytes = tensor_bytes(&[batch_size, seq_len]);

    if config.d_value == 1 {
        let scalar_bytes = tensor_bytes(&[batch_size, seq_len]);
        direction_bytes + beta_bytes + scalar_bytes * 4 + tensor_bytes(&[batch_size, seq_len, 1])
    } else {
        let value_bytes = tensor_bytes(&[batch_size, seq_len, config.d_value]);
        direction_bytes
            + beta_bytes
            + value_bytes * 3
            + tensor_bytes(&[batch_size, seq_len, config.d_model, config.d_value])
    }
}

fn ratio_usize(value: usize, baseline: usize) -> f64 {
    if baseline == 0 {
        0.0
    } else {
        value as f64 / baseline as f64
    }
}

fn ratio_f64(value: f64, baseline: f64) -> Option<f64> {
    (baseline > 0.0).then_some(value / baseline)
}

fn validate_positive_threshold(value: Option<f64>, label: &str) -> Result<(), BenchmarkError> {
    if let Some(value) = value
        && !(value.is_finite() && value > 0.0)
    {
        return Err(BenchmarkError::InvalidConfig(format!(
            "{label} must be finite and positive"
        )));
    }

    Ok(())
}

fn compare_case(
    suite: BenchmarkSuite,
    current: &BenchmarkCase,
    baseline: &BenchmarkCase,
) -> Result<BenchmarkDelta, BenchmarkError> {
    if current.label != baseline.label {
        return Err(BenchmarkError::ComparisonMismatch(format!(
            "suite {} case label mismatch: current `{}` vs baseline `{}`",
            suite.slug(),
            current.label,
            baseline.label
        )));
    }
    ensure_shapes_match(
        &BenchmarkTarget::Case {
            suite,
            label: current.label.clone(),
        },
        &current.input_shape,
        &baseline.input_shape,
        &current.output_shape,
        &baseline.output_shape,
    )?;

    Ok(BenchmarkDelta {
        target: BenchmarkTarget::Case {
            suite,
            label: current.label.clone(),
        },
        input_shape: current.input_shape.clone(),
        output_shape: current.output_shape.clone(),
        avg_latency_ratio: compare_ratio(
            current.timing.avg_duration_ms,
            baseline.timing.avg_duration_ms,
            "avg latency",
            &BenchmarkTarget::Case {
                suite,
                label: current.label.clone(),
            },
        )?,
        p95_latency_ratio: compare_ratio(
            current.timing.p95_duration_ms,
            baseline.timing.p95_duration_ms,
            "p95 latency",
            &BenchmarkTarget::Case {
                suite,
                label: current.label.clone(),
            },
        )?,
        throughput_ratio: compare_ratio(
            current.timing.iterations_per_second,
            baseline.timing.iterations_per_second,
            "iterations per second",
            &BenchmarkTarget::Case {
                suite,
                label: current.label.clone(),
            },
        )?,
        peak_memory_ratio: compare_usize_ratio(
            current.memory.peak_live_bytes,
            baseline.memory.peak_live_bytes,
            "peak live bytes",
            &BenchmarkTarget::Case {
                suite,
                label: current.label.clone(),
            },
        )?,
        parameter_ratio: compare_optional_usize_ratio(
            current.memory.parameter_bytes,
            baseline.memory.parameter_bytes,
        ),
    })
}

fn compare_model(
    current: &ModelBenchmark,
    baseline: &ModelBenchmark,
) -> Result<BenchmarkDelta, BenchmarkError> {
    let target = BenchmarkTarget::Model {
        variant: current.variant,
    };
    if current.variant != baseline.variant {
        return Err(BenchmarkError::ComparisonMismatch(format!(
            "model variant mismatch: current {} vs baseline {}",
            current.variant.slug(),
            baseline.variant.slug()
        )));
    }
    if current.resolved_config != baseline.resolved_config {
        return Err(BenchmarkError::ComparisonMismatch(format!(
            "model variant {} uses a different resolved config in the baseline report",
            current.variant.slug()
        )));
    }

    ensure_shapes_match(
        &target,
        &current.input_shape,
        &baseline.input_shape,
        &current.output_shape,
        &baseline.output_shape,
    )?;

    Ok(BenchmarkDelta {
        target,
        input_shape: current.input_shape.to_vec(),
        output_shape: current.output_shape.to_vec(),
        avg_latency_ratio: compare_ratio(
            current.timing.avg_duration_ms,
            baseline.timing.avg_duration_ms,
            "avg latency",
            &BenchmarkTarget::Model {
                variant: current.variant,
            },
        )?,
        p95_latency_ratio: compare_ratio(
            current.timing.p95_duration_ms,
            baseline.timing.p95_duration_ms,
            "p95 latency",
            &BenchmarkTarget::Model {
                variant: current.variant,
            },
        )?,
        throughput_ratio: compare_ratio(
            current.tokens_per_second,
            baseline.tokens_per_second,
            "tokens per second",
            &BenchmarkTarget::Model {
                variant: current.variant,
            },
        )?,
        peak_memory_ratio: compare_usize_ratio(
            current.memory.peak_total_bytes,
            baseline.memory.peak_total_bytes,
            "peak total bytes",
            &BenchmarkTarget::Model {
                variant: current.variant,
            },
        )?,
        parameter_ratio: compare_optional_usize_ratio(current.num_params, baseline.num_params),
    })
}

fn ensure_shapes_match(
    target: &BenchmarkTarget,
    current_input_shape: &[usize],
    baseline_input_shape: &[usize],
    current_output_shape: &[usize],
    baseline_output_shape: &[usize],
) -> Result<(), BenchmarkError> {
    if current_input_shape != baseline_input_shape {
        return Err(BenchmarkError::ComparisonMismatch(format!(
            "{} input shape mismatch: current {:?} vs baseline {:?}",
            target.label(),
            current_input_shape,
            baseline_input_shape
        )));
    }
    if current_output_shape != baseline_output_shape {
        return Err(BenchmarkError::ComparisonMismatch(format!(
            "{} output shape mismatch: current {:?} vs baseline {:?}",
            target.label(),
            current_output_shape,
            baseline_output_shape
        )));
    }
    Ok(())
}

fn compare_ratio(
    value: f64,
    baseline: f64,
    metric: &str,
    target: &BenchmarkTarget,
) -> Result<f64, BenchmarkError> {
    if !(baseline.is_finite() && baseline > 0.0) {
        return Err(BenchmarkError::ComparisonMismatch(format!(
            "{} has a non-positive baseline {} value: {baseline}",
            target.label(),
            metric
        )));
    }
    if !value.is_finite() {
        return Err(BenchmarkError::ComparisonMismatch(format!(
            "{} has a non-finite current {} value: {value}",
            target.label(),
            metric
        )));
    }

    Ok(value / baseline)
}

fn compare_usize_ratio(
    value: usize,
    baseline: usize,
    metric: &str,
    target: &BenchmarkTarget,
) -> Result<f64, BenchmarkError> {
    if baseline == 0 {
        return Err(BenchmarkError::ComparisonMismatch(format!(
            "{} has a zero baseline {metric} value",
            target.label()
        )));
    }

    Ok(value as f64 / baseline as f64)
}

fn compare_optional_usize_ratio(value: usize, baseline: usize) -> Option<f64> {
    match (value, baseline) {
        (0, 0) => Some(1.0),
        (_, 0) => None,
        _ => Some(value as f64 / baseline as f64),
    }
}

fn evaluate_gate(
    deltas: &[BenchmarkDelta],
    thresholds: &BenchmarkRegressionThresholds,
) -> Vec<String> {
    let mut failures = Vec::new();
    for delta in deltas {
        let target = delta.target.label();
        if let Some(min_throughput_ratio) = thresholds.min_throughput_ratio
            && delta.throughput_ratio < min_throughput_ratio
        {
            failures.push(format!(
                "{target} throughput ratio {:.3} fell below {:.3}",
                delta.throughput_ratio, min_throughput_ratio
            ));
        }
        if let Some(max_avg_latency_ratio) = thresholds.max_avg_latency_ratio
            && delta.avg_latency_ratio > max_avg_latency_ratio
        {
            failures.push(format!(
                "{target} avg latency ratio {:.3} exceeded {:.3}",
                delta.avg_latency_ratio, max_avg_latency_ratio
            ));
        }
        if let Some(max_p95_latency_ratio) = thresholds.max_p95_latency_ratio
            && delta.p95_latency_ratio > max_p95_latency_ratio
        {
            failures.push(format!(
                "{target} p95 latency ratio {:.3} exceeded {:.3}",
                delta.p95_latency_ratio, max_p95_latency_ratio
            ));
        }
        if let Some(max_peak_memory_ratio) = thresholds.max_peak_memory_ratio
            && delta.peak_memory_ratio > max_peak_memory_ratio
        {
            failures.push(format!(
                "{target} peak memory ratio {:.3} exceeded {:.3}",
                delta.peak_memory_ratio, max_peak_memory_ratio
            ));
        }
    }

    failures
}