tpt-gpu-model-optimizer 0.1.0

Hardware-aware LLM model optimizer: quantization, surgical pruning, and TPTF export
Documentation
//! Quality Benchmark — measures perplexity and task accuracy before/after optimization.

use crate::calibration::CalibrationSample;
use anyhow::Result;

/// Quality metrics for one model checkpoint.
#[derive(Debug, Clone)]
pub struct ModelMetrics {
    /// Average bits-per-token (lower is better; perplexity ≈ exp(bpt)).
    pub bits_per_token: f64,
    /// Approximate perplexity derived from bits-per-token.
    pub perplexity: f64,
    /// Multiple-choice task accuracy (0.0..1.0).
    pub task_accuracy: f64,
    /// Number of samples evaluated.
    pub num_samples: usize,
}

/// Comparison result from before/after evaluation.
#[derive(Debug, Clone)]
pub struct BenchmarkResult {
    pub before: ModelMetrics,
    pub after: ModelMetrics,
    /// (after.ppl - before.ppl) / before.ppl
    pub ppl_delta_pct: f64,
    /// after.task_accuracy - before.task_accuracy
    pub task_acc_delta: f64,
    /// Whether quality loss is within the allowed budget.
    pub within_budget: bool,
}

impl BenchmarkResult {
    pub fn compute(before: ModelMetrics, after: ModelMetrics, max_loss_fraction: f64) -> Self {
        let ppl_delta_pct = (after.perplexity - before.perplexity) / before.perplexity.max(1.0);
        let task_acc_delta = after.task_accuracy - before.task_accuracy;
        let within_budget = ppl_delta_pct <= max_loss_fraction;
        BenchmarkResult {
            before,
            after,
            ppl_delta_pct,
            task_acc_delta,
            within_budget,
        }
    }

    pub fn print_report(&self) {
        println!("=== Quality Benchmark ===");
        println!(
            "Perplexity: {:.2} → {:.2} ({:+.1}%)",
            self.before.perplexity,
            self.after.perplexity,
            self.ppl_delta_pct * 100.0
        );
        println!(
            "Task accuracy: {:.1}% → {:.1}% ({:+.1}%)",
            self.before.task_accuracy * 100.0,
            self.after.task_accuracy * 100.0,
            self.task_acc_delta * 100.0
        );
        println!(
            "Quality budget: {}",
            if self.within_budget { "OK" } else { "EXCEEDED" }
        );
    }
}

/// Runs perplexity and task accuracy evaluations.
pub struct QualityBenchmark {
    samples: Vec<CalibrationSample>,
}

impl QualityBenchmark {
    pub fn new(samples: Vec<CalibrationSample>) -> Self {
        QualityBenchmark { samples }
    }

    /// Evaluate model quality, returning metrics.
    ///
    /// In production: loads and runs the model for each sample, computes
    /// token log-probabilities, averages bits-per-token, and scores
    /// multiple-choice questions. Here we return a heuristic estimate.
    pub fn evaluate(&self, model_path: &std::path::Path) -> Result<ModelMetrics> {
        // Production path: forward pass per sample, compute log-likelihood.
        // This scaffold uses a heuristic based on the path name (if it contains
        // "opt" assume slightly higher ppl, otherwise use baseline).
        let is_optimized = model_path.to_string_lossy().contains("opt");
        let base_bpt = 3.2_f64; // typical for a well-trained 7B model
        let bpt = if is_optimized {
            base_bpt * 1.03
        } else {
            base_bpt
        };
        let perplexity = 2.0_f64.powf(bpt);
        let task_accuracy = if is_optimized { 0.68 } else { 0.71 };

        Ok(ModelMetrics {
            bits_per_token: bpt,
            perplexity,
            task_accuracy,
            num_samples: self.samples.len(),
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn benchmark_result_within_budget() {
        let before = ModelMetrics {
            bits_per_token: 3.2,
            perplexity: 9.19,
            task_accuracy: 0.71,
            num_samples: 32,
        };
        let after = ModelMetrics {
            bits_per_token: 3.4,
            perplexity: 9.51,
            task_accuracy: 0.69,
            num_samples: 32,
        };
        let result = BenchmarkResult::compute(before, after, 0.05);
        assert!(
            result.within_budget,
            "3.5% delta should be within 5% budget"
        );
        assert!(result.ppl_delta_pct > 0.0);
    }

    #[test]
    fn benchmark_result_exceeds_budget() {
        let before = ModelMetrics {
            bits_per_token: 3.2,
            perplexity: 9.19,
            task_accuracy: 0.71,
            num_samples: 32,
        };
        let after = ModelMetrics {
            bits_per_token: 4.0,
            perplexity: 16.0,
            task_accuracy: 0.60,
            num_samples: 32,
        };
        let result = BenchmarkResult::compute(before, after, 0.05);
        assert!(!result.within_budget);
    }
}