aprender-serve 0.64.0

/// IMP-182d: Real-world determinism test
#[test]
#[ignore = "Requires running llama.cpp server on port 8082"]
fn test_imp_182d_realworld_determinism() {
    let client = ModelHttpClient::with_timeout(30);
    let seed = 42u64;
    let num_runs = 3;

    let mut outputs = Vec::new();
    for _ in 0..num_runs {
        let request = CompletionRequest {
            model: "default".to_string(),
            prompt: "2 + 2 = ".to_string(),
            max_tokens: 3,
            temperature: Some(0.0), // Temperature 0 = deterministic
            stream: false,
        };

        if let Ok(resp) = client.llamacpp_completion("http://127.0.0.1:8082", &request) {
            outputs.push(resp.text);
        }
    }

    let result = if outputs.len() == num_runs {
        let hashes: Vec<u64> = outputs.iter().map(|s| simple_hash(s)).collect();
        let all_same = hashes.windows(2).all(|w| w[0] == w[1]);
        if all_same {
            DeterminismResult::deterministic(seed, num_runs)
        } else {
            DeterminismResult::non_deterministic(seed, num_runs, "Outputs differ")
        }
    } else {
        DeterminismResult::non_deterministic(seed, num_runs, "Missing outputs")
    };

    println!("\nIMP-182d: Real-World Determinism:");
    println!("  Seed: {}", seed);
    println!("  Runs: {}", num_runs);
    println!("  Outputs identical: {}", result.outputs_identical);
    println!(
        "  QA-029: {}",
        if result.meets_qa029 { "PASS" } else { "FAIL" }
    );
}

// ================================================================================
// IMP-183: CPU/GPU Consistency (QA-030)
// Verify consistent results across CPU/GPU backends
// ================================================================================

/// Backend type for inference
#[derive(Debug, Clone, PartialEq)]
pub enum InferenceBackend {
    Cpu,
    Gpu,
    GpuCuda,
    GpuMetal,
    Hybrid,
}

/// Backend consistency verification result
#[derive(Debug)]
pub struct BackendConsistencyResult {
    pub backend_a: InferenceBackend,
    pub backend_b: InferenceBackend,
    pub outputs_match: bool,
    pub max_diff: f32,
    pub tolerance: f32,
    pub meets_qa030: bool,
}

impl BackendConsistencyResult {
    pub fn consistent(
        a: InferenceBackend,
        b: InferenceBackend,
        max_diff: f32,
        tolerance: f32,
    ) -> Self {
        Self {
            backend_a: a,
            backend_b: b,
            outputs_match: max_diff <= tolerance,
            max_diff,
            tolerance,
            meets_qa030: max_diff <= tolerance,
        }
    }

    pub fn inconsistent(
        a: InferenceBackend,
        b: InferenceBackend,
        max_diff: f32,
        tolerance: f32,
    ) -> Self {
        Self {
            backend_a: a,
            backend_b: b,
            outputs_match: false,
            max_diff,
            tolerance,
            meets_qa030: false,
        }
    }
}

/// Compare two float arrays for consistency
pub fn compare_outputs(a: &[f32], b: &[f32], tolerance: f32) -> (bool, f32) {
    if a.len() != b.len() {
        return (false, f32::MAX);
    }

    let max_diff = a
        .iter()
        .zip(b.iter())
        .map(|(x, y)| (x - y).abs())
        .fold(0.0f32, f32::max);

    (max_diff <= tolerance, max_diff)
}

/// IMP-183a: Test backend consistency result structure
#[test]
fn test_imp_183a_consistency_result() {
    let consistent = BackendConsistencyResult::consistent(
        InferenceBackend::Cpu,
        InferenceBackend::Gpu,
        1e-5,
        1e-4,
    );
    assert!(
        consistent.outputs_match,
        "IMP-183a: Small diff should match"
    );
    assert!(
        consistent.meets_qa030,
        "IMP-183a: Consistent should meet QA-030"
    );

    let inconsistent = BackendConsistencyResult::inconsistent(
        InferenceBackend::Cpu,
        InferenceBackend::Gpu,
        0.1,
        1e-4,
    );
    assert!(
        !inconsistent.outputs_match,
        "IMP-183a: Large diff should not match"
    );
    assert!(
        !inconsistent.meets_qa030,
        "IMP-183a: Inconsistent should not meet QA-030"
    );

    println!("\nIMP-183a: Consistency Results:");
    println!(
        "  Consistent: diff={:.2e}, tol={:.2e}, meets_qa030={}",
        consistent.max_diff, consistent.tolerance, consistent.meets_qa030
    );
    println!(
        "  Inconsistent: diff={:.2e}, tol={:.2e}, meets_qa030={}",
        inconsistent.max_diff, inconsistent.tolerance, inconsistent.meets_qa030
    );
}

/// IMP-183b: Test output comparison
#[test]
fn test_imp_183b_output_comparison() {
    let a = vec![1.0f32, 2.0, 3.0, 4.0];
    let b = vec![1.0001f32, 2.0001, 3.0001, 4.0001];
    let c = vec![1.1f32, 2.1, 3.1, 4.1];

    let (match_ab, diff_ab) = compare_outputs(&a, &b, 1e-3);
    assert!(match_ab, "IMP-183b: Small differences should match");
    assert!(diff_ab < 0.001, "IMP-183b: Max diff should be small");

    let (match_ac, diff_ac) = compare_outputs(&a, &c, 1e-3);
    assert!(!match_ac, "IMP-183b: Large differences should not match");
    assert!(diff_ac > 0.09, "IMP-183b: Max diff should be ~0.1");

    println!("\nIMP-183b: Output Comparison:");
    println!("  a vs b: match={}, diff={:.6}", match_ab, diff_ab);
    println!("  a vs c: match={}, diff={:.6}", match_ac, diff_ac);
}

/// IMP-183c: Test backend enum coverage
#[test]
fn test_imp_183c_backend_coverage() {
    let backends = vec![
        InferenceBackend::Cpu,
        InferenceBackend::Gpu,
        InferenceBackend::GpuCuda,
        InferenceBackend::GpuMetal,
        InferenceBackend::Hybrid,
    ];

    for backend in &backends {
        let result = BackendConsistencyResult::consistent(
            InferenceBackend::Cpu,
            backend.clone(),
            1e-6,
            1e-4,
        );
        assert!(
            result.meets_qa030,
            "IMP-183c: All backends should be testable"
        );
    }

    println!("\nIMP-183c: Backend Coverage:");
    for backend in backends {
        println!("  {:?}: supported", backend);
    }
}

/// IMP-183d: Real-world CPU/GPU consistency
#[test]
#[ignore = "Requires running servers with different backends"]
fn test_imp_183d_realworld_consistency() {
    // This test would require two servers running with different backends
    // For now, we test the structure works correctly

    let result = BackendConsistencyResult::consistent(
        InferenceBackend::Cpu,
        InferenceBackend::Gpu,
        1e-5,
        1e-4,
    );

    println!("\nIMP-183d: Real-World Consistency:");
    println!("  Backend A: {:?}", result.backend_a);
    println!("  Backend B: {:?}", result.backend_b);
    println!("  Max diff: {:.2e}", result.max_diff);
    println!("  Tolerance: {:.2e}", result.tolerance);
    println!(
        "  QA-030: {}",
        if result.meets_qa030 { "PASS" } else { "FAIL" }
    );
}

// ================================================================================
// IMP-184: CV-Based Stopping (QA-031)
// Implement CV-based stopping criterion per Hoefler & Belli [2]
// ================================================================================

/// Coefficient of Variation (CV) based stopping result
#[derive(Debug)]
pub struct CVStoppingResult {
    pub cv: f64,
    pub threshold: f64,
    pub num_samples: usize,
    pub min_samples: usize,
    pub should_stop: bool,
    pub meets_qa031: bool,
}

impl CVStoppingResult {
    pub fn converged(cv: f64, threshold: f64, samples: usize, min_samples: usize) -> Self {
        Self {
            cv,
            threshold,
            num_samples: samples,
            min_samples,
            should_stop: cv <= threshold && samples >= min_samples,
            meets_qa031: true,
        }
    }

    pub fn not_converged(cv: f64, threshold: f64, samples: usize, min_samples: usize) -> Self {
        Self {
            cv,
            threshold,
            num_samples: samples,
            min_samples,
            should_stop: false,
            meets_qa031: true, // Still valid, just not converged
        }
    }
}

/// Calculate coefficient of variation (CV)
pub fn calculate_cv(samples: &[f64]) -> f64 {
    if samples.is_empty() {
        return f64::MAX;
    }

    let n = samples.len() as f64;
    let mean = samples.iter().sum::<f64>() / n;

    if mean.abs() < 1e-10 {
        return f64::MAX;
    }

    let variance = samples.iter().map(|x| (x - mean).powi(2)).sum::<f64>() / n;

    let std_dev = variance.sqrt();
    (std_dev / mean).abs()
}

/// CV-based stopping criterion checker
pub struct CVStoppingCriterion {
    pub threshold: f64,
    pub min_samples: usize,
    pub max_samples: usize,
}

impl Default for CVStoppingCriterion {
    fn default() -> Self {
        Self {
            threshold: 0.05, // 5% CV threshold per Hoefler & Belli
            min_samples: 10,
            max_samples: 1000,
        }
    }
}

impl CVStoppingCriterion {
    pub fn new(threshold: f64, min_samples: usize, max_samples: usize) -> Self {
        Self {
            threshold,
            min_samples,
            max_samples,
        }
    }

    pub fn check(&self, samples: &[f64]) -> CVStoppingResult {
        let cv = calculate_cv(samples);
        let n = samples.len();

        if n >= self.min_samples && cv <= self.threshold {
            CVStoppingResult::converged(cv, self.threshold, n, self.min_samples)
        } else {
            CVStoppingResult::not_converged(cv, self.threshold, n, self.min_samples)
        }
    }
}

/// IMP-184a: Test CV calculation
#[test]
fn test_imp_184a_cv_calculation() {
    // Constant values -> CV = 0
    let constant = vec![10.0; 10];
    let cv_constant = calculate_cv(&constant);
    assert!(
        cv_constant < 1e-10,
        "IMP-184a: Constant values should have CV ~0"
    );

    // Variable values
    let variable = vec![10.0, 11.0, 9.0, 10.5, 9.5];
    let cv_variable = calculate_cv(&variable);
    assert!(
        cv_variable > 0.0,
        "IMP-184a: Variable values should have CV > 0"
    );
    assert!(
        cv_variable < 0.1,
        "IMP-184a: Low variance should have low CV"
    );

    // High variance
    let high_var = vec![1.0, 10.0, 1.0, 10.0, 1.0];
    let cv_high = calculate_cv(&high_var);
    assert!(cv_high > 0.5, "IMP-184a: High variance should have high CV");

    println!("\nIMP-184a: CV Calculation:");
    println!("  Constant [10,10,...]: CV = {:.6}", cv_constant);
    println!("  Variable [10,11,9,10.5,9.5]: CV = {:.6}", cv_variable);
    println!("  High variance [1,10,1,10,1]: CV = {:.6}", cv_high);
}

/// IMP-184b: Test CV stopping criterion
#[test]
fn test_imp_184b_stopping_criterion() {
    let criterion = CVStoppingCriterion::default();

    // Converged: low CV, enough samples
    let converged = vec![100.0; 20];
    let result = criterion.check(&converged);
    assert!(
        result.should_stop,
        "IMP-184b: Low CV with enough samples should stop"
    );
    assert!(result.meets_qa031, "IMP-184b: Should meet QA-031");

    // Not converged: high CV
    let high_cv: Vec<f64> = (1..=20)
        .map(|i| if i % 2 == 0 { 100.0 } else { 1.0 })
        .collect();
    let result2 = criterion.check(&high_cv);
    assert!(!result2.should_stop, "IMP-184b: High CV should not stop");

    // Not converged: too few samples
    let few_samples = vec![100.0; 5];
    let result3 = criterion.check(&few_samples);
    assert!(
        !result3.should_stop,
        "IMP-184b: Too few samples should not stop"
    );

    println!("\nIMP-184b: Stopping Criterion:");
    println!(
        "  Low CV, 20 samples: stop={}, cv={:.4}",
        result.should_stop, result.cv
    );
    println!(
        "  High CV, 20 samples: stop={}, cv={:.4}",
        result2.should_stop, result2.cv
    );
    println!(
        "  Low CV, 5 samples: stop={}, cv={:.4}",
        result3.should_stop, result3.cv
    );
}

/// IMP-184c: Test custom thresholds
#[test]
fn test_imp_184c_custom_thresholds() {
    let strict = CVStoppingCriterion::new(0.01, 20, 500);
    let relaxed = CVStoppingCriterion::new(0.10, 5, 100);

    let samples: Vec<f64> = (0..30).map(|i| 100.0 + (i as f64 % 5.0)).collect();

    let strict_result = strict.check(&samples);
    let relaxed_result = relaxed.check(&samples);

    // Relaxed should stop before strict
    assert!(
        relaxed_result.should_stop || !strict_result.should_stop,
        "IMP-184c: Relaxed threshold should be easier to meet"
    );

    println!("\nIMP-184c: Custom Thresholds:");
    println!(
        "  Strict (1%): cv={:.4}, stop={}",
        strict_result.cv, strict_result.should_stop
    );
    println!(
        "  Relaxed (10%): cv={:.4}, stop={}",
        relaxed_result.cv, relaxed_result.should_stop
    );
}