aprender-serve 0.64.0

//! GGUF Part 12: PARITY-070 - PARITY-077 (Phase 3: Quantized Attention)
//!
//! Extracted from gguf_monolith.rs (PMAT-802)
//!
//! ## Test Groups
//!
//! - PARITY-070: Q4/Q8 Matrix Multiply (6 tests)
//! - PARITY-071: Q8Block Struct Implementation (6 tests)
//! - PARITY-072: Fused Q4xQ8 CPU Kernel (6 tests)
//! - PARITY-073: CUDA PTX Generation (6 tests)
//! - PARITY-074: CUDA Kernel Execution (6 tests)
//! - PARITY-075: INT8 Attention (6 tests)
//! - PARITY-076: Full Integration (6 tests)
//! - PARITY-077: Shared Memory Tiling (6 tests)
//!
//! Phase 3 Target: Fused MMQ kernels, INT8 attention, memory bandwidth reduction

// ==================== PHASE 3: QUANTIZED ATTENTION ====================
// Target: Fused MMQ kernels, INT8 attention, memory bandwidth reduction

// ==================== PARITY-070: Q4/Q8 Matrix Multiply ====================
// Foundation for fused quantized operations
// Goal: Reduce memory bandwidth from 32-bit to 4.5-bit per weight

/// PARITY-070a: Problem analysis - dequantize-then-compute bottleneck
#[test]
#[cfg(feature = "gpu")]
#[serial_test::serial]
fn test_parity070a_problem_analysis() {
    println!("PARITY-070a: Dequantize-Then-Compute Bottleneck");
    println!("================================================");
    println!();
    println!("  CURRENT ARCHITECTURE (Realizar):");
    println!("    ┌─────────────────────────────────────────────────────┐");
    println!("    │ Q4_K Weight                                         │");
    println!("    │   ↓ dequantize (4-bit → 32-bit)                     │");
    println!("    │ F32 Weight     [32 bits/element]                    │");
    println!("    │   ↓ matmul                                          │");
    println!("    │ F32 Result                                          │");
    println!("    └─────────────────────────────────────────────────────┘");
    println!();
    println!("  MEMORY TRAFFIC ANALYSIS:");
    println!("    - Q4_K storage: 4.5 bits/weight (4-bit + scale/min)");
    println!("    - After dequant: 32 bits/weight");
    println!("    - Bandwidth ratio: 32/4.5 = 7.1x overhead");
    println!();
    println!("  EXAMPLE (2048-dim hidden layer):");
    println!("    ┌────────────────────────────────────────────────────┐");
    println!("    │ Operation: hidden_state @ weight_matrix            │");
    println!("    │   Weight shape: 2048 x 2048                        │");
    println!("    │   Q4_K size: 4.5 * 2048 * 2048 / 8 = 2.36 MB       │");
    println!("    │   F32 size: 32 * 2048 * 2048 / 8 = 16.78 MB        │");
    println!("    │   Overhead: 14.42 MB extra memory traffic          │");
    println!("    └────────────────────────────────────────────────────┘");
    println!();
    println!("  ROOT CAUSE:");
    println!("    llama.cpp uses fused MMQ (Matrix Multiply Quantized)");
    println!("    that keeps data in quantized form during computation.");
    println!("    Realizar dequantizes to F32 before compute.");

    // Memory bandwidth calculation
    let q4k_bits_per_weight = 4.5;
    let f32_bits_per_weight = 32.0;
    let bandwidth_ratio = f32_bits_per_weight / q4k_bits_per_weight;

    println!();
    println!("  BANDWIDTH RATIO: {:.1}x", bandwidth_ratio);

    assert!(bandwidth_ratio > 7.0, "PARITY-070a: 7x+ bandwidth overhead");
}

/// PARITY-070b: Target architecture - fused MMQ
#[test]
#[cfg(feature = "gpu")]
#[serial_test::serial]
fn test_parity070b_target_architecture() {
    println!("PARITY-070b: Fused MMQ Target Architecture");
    println!("==========================================");
    println!();
    println!("  TARGET ARCHITECTURE (llama.cpp-style):");
    println!("    ┌─────────────────────────────────────────────────────┐");
    println!("    │ Q4_K Weight     [4.5 bits/element]                  │");
    println!("    │   ↓ fused dequant + dot product                     │");
    println!("    │ F32 Result      [no intermediate storage]           │");
    println!("    └─────────────────────────────────────────────────────┘");
    println!();
    println!("  KEY INSIGHT:");
    println!("    Dequantization can be fused INTO the dot product:");
    println!("    1. Load quantized block (32 weights + scale + min)");
    println!("    2. Compute: sum(dequant(q4) * activation) on-the-fly");
    println!("    3. Accumulate partial results");
    println!("    4. Write only final result to memory");
    println!();
    println!("  FUSED KERNEL PSEUDOCODE:");
    println!("    ┌─────────────────────────────────────────────────────┐");
    println!("    │ fn fused_q4k_dot(q4_block: &Q4KBlock,               │");
    println!("    │                  activations: &[f32]) -> f32 {{      │");
    println!("    │     let scale = q4_block.scale;                     │");
    println!("    │     let min = q4_block.min;                         │");
    println!("    │     let mut sum = 0.0;                              │");
    println!("    │     for i in 0..32 {{                                │");
    println!("    │         let q = q4_block.nibbles[i];                │");
    println!("    │         let w = (q as f32 - 8.0) * scale + min;     │");
    println!("    │         sum += w * activations[i]; // Fused!        │");
    println!("    │     }}                                               │");
    println!("    │     sum                                             │");
    println!("    │ }}                                                    │");
    println!("    └─────────────────────────────────────────────────────┘");
    println!();
    println!("  MEMORY SAVINGS:");
    println!("    - Read: 4.5 bits/weight (not 32 bits)");
    println!("    - Write: Only final result (not intermediate F32)");
    println!("    - Effective: 7.1x reduction in memory traffic");

    assert!(true, "PARITY-070b: Target architecture documented");
}

/// PARITY-070c: INT8 dot product operations (DP4A)
#[test]
#[cfg(feature = "gpu")]
#[serial_test::serial]
fn test_parity070c_int8_operations() {
    println!("PARITY-070c: INT8 Dot Product Operations");
    println!("=========================================");
    println!();
    println!("  CUDA DP4A INSTRUCTION:");
    println!("    ┌─────────────────────────────────────────────────────┐");
    println!("    │ __dp4a(a, b, c):                                    │");
    println!("    │   - Inputs: a[4xi8], b[4xi8], c[i32]                │");
    println!("    │   - Output: c + sum(a[i] * b[i]) for i in 0..4      │");
    println!("    │   - Throughput: 4x INT8 multiply-adds per cycle     │");
    println!("    └─────────────────────────────────────────────────────┘");
    println!();
    println!("  RTX 4090 INT8 TENSOR CORES:");
    println!("    ┌─────────────────────────────────────────────────────┐");
    println!("    │ Compute Capability: 8.9 (Ada Lovelace)              │");
    println!("    │ INT8 Tensor Ops: 1321 TOPS                          │");
    println!("    │ FP32 Ops: 82.6 TFLOPS                               │");
    println!("    │ Ratio: 16x faster INT8 vs FP32                      │");
    println!("    └─────────────────────────────────────────────────────┘");
    println!();
    println!("  Q4_K TO INT8 CONVERSION:");
    println!("    ┌─────────────────────────────────────────────────────┐");
    println!("    │ Q4_K (4-bit):                                       │");
    println!("    │   - 32 weights per block                            │");
    println!("    │   - Each nibble: 0-15, centered at 8                │");
    println!("    │   - Scale + min per block                           │");
    println!("    │                                                     │");
    println!("    │ Conversion to INT8:                                 │");
    println!("    │   q8 = (q4 - 8) * scale_factor                      │");
    println!("    │   Pack 4 INT8 values into 32-bit for DP4A           │");
    println!("    └─────────────────────────────────────────────────────┘");
    println!();
    println!("  THROUGHPUT COMPARISON:");
    println!("    | Method      | Ops/Instruction | Memory   | Effective |");
    println!("    |-------------|-----------------|----------|-----------|");
    println!("    | FP32 FMA    | 2               | 32-bit   | 1x        |");
    println!("    | DP4A INT8   | 8               | 8-bit    | 4x        |");
    println!("    | Tensor INT8 | 128+            | 8-bit    | 16x+      |");

    let fp32_flops: f64 = 82.6; // TFLOPS
    let int8_tops: f64 = 1321.0; // TOPS
    let ratio = int8_tops / fp32_flops;

    println!();
    println!("  RTX 4090 INT8/FP32 RATIO: {:.1}x", ratio);

    assert!(ratio > 15.0, "PARITY-070c: INT8 is 16x faster than FP32");
}

/// PARITY-070d: Q8 activation quantization
#[test]
#[cfg(feature = "gpu")]
#[serial_test::serial]
fn test_parity070d_activation_quantization() {
    println!("PARITY-070d: Q8 Activation Quantization");
    println!("=======================================");
    println!();
    println!("  WHY QUANTIZE ACTIVATIONS:");
    println!("    - Weights: Pre-quantized (Q4_K stored on disk)");
    println!("    - Activations: Generated during inference (F32)");
    println!("    - For INT8 dot product: Need Q8 activations");
    println!();
    println!("  Q8_0 FORMAT (llama.cpp):");
    println!("    ┌─────────────────────────────────────────────────────┐");
    println!("    │ struct Q8_0Block {{                                  │");
    println!("    │     scale: f16,        // Per-block scale factor    │");
    println!("    │     qs: [i8; 32],      // 32 quantized values       │");
    println!("    │ }}                                                   │");
    println!("    │                                                     │");
    println!("    │ Quantization:                                       │");
    println!("    │   scale = max(abs(values[0..32])) / 127.0           │");
    println!("    │   qs[i] = round(values[i] / scale)                  │");
    println!("    │                                                     │");
    println!("    │ Dequantization:                                     │");
    println!("    │   values[i] = qs[i] * scale                         │");
    println!("    └─────────────────────────────────────────────────────┘");
    println!();
    println!("  DYNAMIC QUANTIZATION STRATEGY:");
    println!("    ┌─────────────────────────────────────────────────────┐");
    println!("    │ 1. Compute F32 activations (normal forward pass)    │");
    println!("    │ 2. Find max absolute value per 32-element block     │");
    println!("    │ 3. Compute scale = max_abs / 127.0                  │");
    println!("    │ 4. Quantize to INT8: qi = round(fi / scale)         │");
    println!("    │ 5. Use Q8 activations for fused Q4xQ8 dot product   │");
    println!("    └─────────────────────────────────────────────────────┘");
    println!();
    println!("  ERROR ANALYSIS:");
    println!("    - Q8_0 quantization error: < 0.4%");
    println!("    - Combined Q4xQ8 error: < 2%");
    println!("    - Acceptable for inference (not training)");
    println!();
    println!("  IMPLEMENTATION PATH:");
    println!("    1. Add Q8Block struct to quantize.rs");
    println!("    2. Add quantize_to_q8(f32) -> Q8Block");
    println!("    3. Add fused_q4k_q8_dot() kernel");

    assert!(true, "PARITY-070d: Q8 activation quantization documented");
}

/// PARITY-070e: Fused Q4xQ8 kernel design
#[test]
#[cfg(feature = "gpu")]
#[serial_test::serial]
fn test_parity070e_fused_kernel_design() {
    println!("PARITY-070e: Fused Q4xQ8 Kernel Design");
    println!("======================================");
    println!();
    println!("  KERNEL SIGNATURE:");
    println!("    ┌─────────────────────────────────────────────────────┐");
    println!("    │ /// Fused Q4_K weight × Q8_0 activation dot product │");
    println!("    │ fn fused_q4k_q8_dot(                                │");
    println!("    │     weights: &[Q4KBlock],   // Quantized weights    │");
    println!("    │     activations: &[Q8Block], // Quantized acts      │");
    println!("    │     output: &mut [f32],      // F32 output          │");
    println!("    │     m: usize, n: usize, k: usize                    │");
    println!("    │ );                                                  │");
    println!("    └─────────────────────────────────────────────────────┘");
    println!();
    println!("  CUDA PTX STRUCTURE:");
    println!("    ┌─────────────────────────────────────────────────────┐");
    println!("    │ .entry fused_q4k_q8_matmul(                         │");
    println!("    │     .param .u64 weights_ptr,                        │");
    println!("    │     .param .u64 act_ptr,                            │");
    println!("    │     .param .u64 out_ptr,                            │");
    println!("    │     .param .u32 m,                                  │");
    println!("    │     .param .u32 n,                                  │");
    println!("    │     .param .u32 k                                   │");
    println!("    │ ) {{                                                 │");
    println!("    │     // Thread indices                               │");
    println!("    │     mov.u32 %r0, %tid.x;                            │");
    println!("    │     mov.u32 %r1, %ctaid.x;                          │");
    println!("    │                                                     │");
    println!("    │     // Load Q4K block (16 bytes)                    │");
    println!("    │     ld.global.v4.u32 {{%r4,%r5,%r6,%r7}}, [weights];  │");
    println!("    │                                                     │");
    println!("    │     // Load Q8 block (32 bytes)                     │");
    println!("    │     ld.global.v4.u32 {{%r8,%r9,%r10,%r11}}, [acts];   │");
    println!("    │                                                     │");
    println!("    │     // DP4A: 4-way INT8 dot product                 │");
    println!("    │     dp4a.s32.s32 %r12, %r4, %r8, 0;                  │");
    println!("    │     dp4a.s32.s32 %r12, %r5, %r9, %r12;               │");
    println!("    │     ...                                             │");
    println!("    │ }}                                                    │");
    println!("    └─────────────────────────────────────────────────────┘");
    println!();
    println!("  PERFORMANCE EXPECTATIONS:");
    println!("    | Metric              | Current   | Target     |");
    println!("    |---------------------|-----------|------------|");
    println!("    | Memory traffic      | 32b/wt    | 4.5b/wt    |");
    println!("    | Bandwidth reduction | 1x        | 7.1x       |");
    println!("    | Compute (DP4A)      | 1x        | 4x         |");
    println!("    | Combined speedup    | 1x        | 3-4x       |");

    let bandwidth_reduction = 7.1;
    let _compute_speedup = 4.0;
    // Amdahl's law: speedup limited by non-optimized portions
    let memory_bound_fraction = 0.7; // 70% memory bound
    let combined_speedup =
        1.0 / ((1.0 - memory_bound_fraction) + memory_bound_fraction / bandwidth_reduction);

    println!();
    println!("  AMDAHL ANALYSIS (70% memory bound):");
    println!(
        "    Combined speedup = 1 / (0.3 + 0.7/7.1) = {:.2}x",
        combined_speedup
    );

    assert!(combined_speedup > 2.0, "PARITY-070e: 2x+ speedup expected");
}

/// PARITY-070f: Phase 3 implementation roadmap
#[test]
#[cfg(feature = "gpu")]
#[serial_test::serial]
fn test_parity070f_roadmap() {
    println!("PARITY-070f: Phase 3 Implementation Roadmap");
    println!("===========================================");
    println!();
    println!("  ╔═══════════════════════════════════════════════════════════╗");
    println!("  ║  PHASE 3: QUANTIZED ATTENTION ROADMAP                    ║");
    println!("  ╚═══════════════════════════════════════════════════════════╝");
    println!();
    println!("  STEP 1: Foundation Structs (PARITY-071)");
    println!("    - Q8Block struct in quantize.rs");
    println!("    - quantize_to_q8() function");
    println!("    - Unit tests for Q8 quantization");
    println!();
    println!("  STEP 2: Fused CPU Kernel (PARITY-072)");
    println!("    - fused_q4k_q8_dot() CPU implementation");
    println!("    - SIMD optimization with AVX2");
    println!("    - Benchmark vs dequant-then-compute");
    println!();
    println!("  STEP 3: CUDA PTX Generation (PARITY-073)");
    println!("    - Add FusedQ4Q8Matmul kernel type");
    println!("    - Generate PTX with DP4A instructions");
    println!("    - Test PTX compilation");
    println!();
    println!("  STEP 4: CUDA Execution (PARITY-074)");
    println!("    - CudaExecutor support for fused kernel");
    println!("    - Memory layout optimization");
    println!("    - Benchmark GPU fused kernel");
    println!();
    println!("  STEP 5: INT8 Attention (PARITY-075)");
    println!("    - Quantize Q,K projections to INT8");
    println!("    - Fused attention score computation");
    println!("    - Softmax remains F32");
    println!();
    println!("  STEP 6: Integration (PARITY-076)");
    println!("    - Wire into OwnedQuantizedModel");
    println!("    - End-to-end benchmark");
    println!("    - Target: 200+ tok/s single-request");
    println!();
    println!("  EXPECTED RESULTS:");
    println!("    ┌───────────────────────────────────────────────────────┐");
    println!("    │ Milestone │ Optimization         │ Speedup │ tok/s   │");
    println!("    ├───────────┼──────────────────────┼─────────┼─────────┤");
    println!("    │ Current   │ GPU attention (F32)  │ 1.0x    │ 64      │");
    println!("    │ PARITY-072│ Fused CPU kernel     │ 1.5x    │ 96      │");
    println!("    │ PARITY-074│ Fused GPU kernel     │ 2.5x    │ 160     │");
    println!("    │ PARITY-075│ INT8 attention       │ 3.0x    │ 192     │");
    println!("    │ PARITY-076│ Full integration     │ 3.2x    │ 205     │");
    println!("    └───────────────────────────────────────────────────────┘");
    println!();
    println!("  M4 PARITY TARGET: 192 tok/s");
    println!("    - Phase 1 (Batch): M4 at c >= 3 (multi-request)");
    println!("    - Phase 2 (Spec): M4 at K=6, 70% (single-request)");
    println!("    - Phase 3 (Quant): M4 at PARITY-075 (direct speedup)");

    // Final M4 verification
    let baseline = 64.0;
    let phase3_target_speedup = 3.0;
    let projected = baseline * phase3_target_speedup;
    let m4_target = 192.0;

    println!();
    println!("  PHASE 3 M4 PROJECTION:");
    println!(
        "    64 tok/s * 3.0x = {:.0} tok/s >= {:.0} tok/s",
        projected, m4_target
    );

    assert!(
        projected >= m4_target,
        "PARITY-070f: Phase 3 achieves M4 parity"
    );
}

// ==================== PARITY-071: Q8Block Struct Implementation ====================
// Foundation for fused Q4xQ8 dot products
// Implements dynamic activation quantization for INT8 operations

/// PARITY-071a: Q8_0Block structure verification
#[test]
#[cfg(feature = "gpu")]
#[serial_test::serial]
fn test_parity071a_q8_block_struct() {
    use crate::quantize::Q8_0Block;

    println!("PARITY-071a: Q8_0Block Structure Verification");
    println!("==============================================");
    println!();
    println!("  STRUCTURE:");
    println!("    ┌─────────────────────────────────────────────────────┐");
    println!("    │ pub struct Q8_0Block {{                              │");
    println!("    │     pub scale: f32,        // Scale factor          │");
    println!("    │     pub quants: [i8; 32],  // 32 quantized values   │");
    println!("    │ }}                                                   │");
    println!("    └─────────────────────────────────────────────────────┘");
    println!();
    println!("  MEMORY LAYOUT:");
    println!("    scale:  4 bytes (f32)");
    println!("    quants: 32 bytes (32 × i8)");
    println!("    Total:  36 bytes per block");
    println!();
    println!("  BITS PER VALUE:");
    println!("    36 bytes / 32 values = 1.125 bytes/value = 9 bits/value");
    println!("    (8 bits for quant + amortized scale overhead)");

    // Create a test block
    let block = Q8_0Block {
        scale: 0.5,
        quants: [64i8; 32],
    };

    assert_eq!(block.scale, 0.5, "PARITY-071a: Scale stored correctly");
    assert_eq!(block.quants.len(), 32, "PARITY-071a: 32 quants per block");
    println!();
    println!("  ✅ Q8_0Block structure verified");
}

/// PARITY-071b: Q8_0Block::quantize() function
#[test]
#[cfg(feature = "gpu")]
#[serial_test::serial]
fn test_parity071b_quantize_function() {
    use crate::quantize::Q8_0Block;

    println!("PARITY-071b: Q8_0Block::quantize() Function");
    println!("============================================");
    println!();
    println!("  ALGORITHM:");
    println!("    1. Find max_abs = max(|values[i]|)");
    println!("    2. scale = max_abs / 127.0");
    println!("    3. quants[i] = round(values[i] / scale)");
    println!();

    // Test with uniform values
    let values = [1.0f32; 32];
    let block = Q8_0Block::quantize(&values);

    println!("  TEST 1: Uniform values [1.0; 32]");
    println!("    max_abs = 1.0");
    println!("    scale = 1.0 / 127.0 = {:.6}", 1.0 / 127.0);
    println!("    quants[0] = round(1.0 / scale) = 127");
    println!("    Actual scale: {:.6}", block.scale);
    println!("    Actual quants[0]: {}", block.quants[0]);

    assert!(
        (block.scale - 1.0 / 127.0).abs() < 1e-6,
        "PARITY-071b: Scale correct"
    );
    assert_eq!(block.quants[0], 127, "PARITY-071b: Max value maps to 127");

    // Test with mixed values
    let mixed: [f32; 32] = core::array::from_fn(|i| (i as f32 - 16.0) / 8.0);
    let block2 = Q8_0Block::quantize(&mixed);

    println!();
    println!("  TEST 2: Mixed values [-2.0 to 1.875]");
    println!("    max_abs = 2.0");
    println!("    scale = 2.0 / 127.0 = {:.6}", 2.0 / 127.0);
    println!("    Actual scale: {:.6}", block2.scale);

    assert!(block2.scale > 0.0, "PARITY-071b: Scale is positive");
    println!();
    println!("  ✅ Q8_0Block::quantize() verified");
}

include!("parity071c_dequantize.rs");
include!("parity072e_performance.rs");
include!("parity_074a_execute.rs");
include!("parity_075b.rs");
include!("parity_075f.rs");
include!("parity_077a.rs");