Skip to main content

tpt_gpu_model_optimizer/
lib.rs

1//! TPT Model Optimizer — hardware-aware LLM compression pipeline.
2//!
3//! # Pipeline
4//! ```text
5//! GGUF source
6//!   → HardwareProfiler   (benchmark GPU; cache to ~/.tpt/hardware_profile.json)
7//!   → LayerSensitivityMap (rank layers by quantization sensitivity)
8//!   → DomainMapper        (identify neurons by domain via activation analysis)
9//!   → SurgicalPruner      (zero unwanted domain neurons)
10//!   → MixedPrecisionAllocator (5% loss frontier per layer)
11//!   → KvCacheCalculator   (recommend context window for remaining VRAM)
12//!   → TptfFormat writer   (output .tptf — self-contained: weights + tokenizer + chat template)
13//!
14//! Export: GgufExporter, Exl2Exporter
15//! Quality: QualityBenchmark (perplexity + task accuracy)
16//! Large models: StreamingLoader (70B+ via layer-by-layer mmap)
17//! ```
18
19pub mod activation_capture;
20pub mod benchmark;
21pub mod calibration;
22pub mod domain_mapper;
23pub mod export;
24pub mod import;
25pub mod kv_calculator;
26pub mod profiler;
27pub mod pruner;
28pub mod quant_allocator;
29pub mod sensitivity;
30pub mod streaming;
31pub mod tptf_format;
32
33pub use activation_capture::{
34    load_domain_maps_from_dir, ActivationCallback, ActivationCapture, ActivationMap,
35    LayerActivations,
36};
37pub use benchmark::QualityBenchmark;
38pub use calibration::CalibrationGenerator;
39pub use domain_mapper::DomainMapper;
40pub use export::detect::ModelFormat;
41pub use import::gguf::{GgufImporter, GgufModel};
42pub use kv_calculator::KvCacheCalculator;
43pub use profiler::HardwareProfiler;
44pub use pruner::SurgicalPruner;
45pub use quant_allocator::MixedPrecisionAllocator;
46pub use sensitivity::{LayerSensitivityMap, SensitivityConfig};
47pub use streaming::StreamingLoader;
48pub use tptf_format::{TensorBlock, TptfHeader, TptfWriter};
49
50use std::path::PathBuf;
51
52/// Configuration for a full optimization run.
53#[derive(Debug, Clone)]
54pub struct OptimizerConfig {
55    /// Source model path (.gguf or .tptf).
56    pub model_path: PathBuf,
57    /// Output path (defaults to `<stem>.tptf`).
58    pub output_path: PathBuf,
59    /// Maximum acceptable quality loss as a fraction (0.05 = 5%).
60    pub max_loss_fraction: f32,
61    /// Domains to remove via surgical pruning (e.g. `["sql", "typescript"]`).
62    pub domains_to_drop: Vec<String>,
63    /// Force streaming mode (auto-enabled when model_size > 80% vram_free).
64    pub force_streaming: bool,
65}
66
67impl OptimizerConfig {
68    pub fn new(model_path: impl Into<PathBuf>, output_path: impl Into<PathBuf>) -> Self {
69        OptimizerConfig {
70            model_path: model_path.into(),
71            output_path: output_path.into(),
72            max_loss_fraction: 0.05,
73            domains_to_drop: Vec::new(),
74            force_streaming: false,
75        }
76    }
77}
78
79/// Summary produced after a full optimization run.
80#[derive(Debug, Clone)]
81pub struct OptimizationResult {
82    pub output_path: PathBuf,
83    pub per_layer_bits: Vec<u8>,
84    pub pruned_domains: Vec<String>,
85    pub size_before_gb: f64,
86    pub size_after_gb: f64,
87    pub compression_ratio: f32,
88    pub ppl_delta_pct: f32,
89}