1use crate::{
2 catalog::{BACKENDS, CatalogModel, PerfClass, PerfDataset, require},
3 sizing::{
4 Architecture, GpuVendor, Hardware, Platform, Quantization, SizingRequest, ValidationError,
5 evaluate, parse_param_count, quant_bits,
6 },
7};
8use serde::Serialize;
9use serde_json::{Value, json};
10
11const GIB: f64 = 1073741824.0;
12pub fn unified(hardware: &Hardware) -> bool {
13 hardware.is_unified()
14}
15pub fn largest_vram(hardware: &Hardware) -> f64 {
16 hardware
17 .gpu
18 .iter()
19 .map(|gpu| gpu.vram_bytes)
20 .fold(0.0_f64, f64::max)
21}
22pub fn hardware_score(hardware: &Hardware) -> Value {
23 let vram = largest_vram(hardware);
24 let pool = if vram > 0.0 {
25 vram
26 } else if unified(hardware) {
27 hardware.total_ram_bytes
28 } else {
29 0.0
30 };
31 let sub = [
32 (pool / (24.0 * GIB)).clamp(0.0, 1.0),
33 (hardware.total_ram_bytes / (64.0 * GIB)).clamp(0.0, 1.0),
34 if vram > 0.0 {
35 0.5 + 0.5 * (vram / (24.0 * GIB)).clamp(0.0, 1.0)
36 } else if unified(hardware) {
37 0.65
38 } else {
39 0.25
40 },
41 (hardware.free_disk_bytes / (200.0 * GIB)).clamp(0.0, 1.0),
42 ];
43 let weights = [0.4, 0.25, 0.25, 0.1];
44 let mut best = 0;
45 let mut deficit = -1.0;
46 let mut total = 0.0;
47 for index in 0..4 {
48 total += weights[index] * sub[index];
49 let current = weights[index] * (1.0 - sub[index]);
50 if current > deficit {
51 best = index;
52 deficit = current;
53 }
54 }
55 json!({"total":(total * 100.0).round() as u32,"sub":{"vram":sub[0],"ram":sub[1],"compute":sub[2],"storage":sub[3]},"bottleneck":(["vram","ram","compute","storage"][best])})
56}
57
58pub fn match_perf<'a>(hardware: &Hardware, dataset: &'a PerfDataset) -> Option<&'a PerfClass> {
59 let discrete = hardware
60 .gpu
61 .iter()
62 .filter(|gpu| {
63 matches!(
64 gpu.vendor,
65 GpuVendor::Nvidia | GpuVendor::Amd | GpuVendor::Intel
66 ) && gpu.vram_bytes > 0.0
67 })
68 .fold(None, |best: Option<&crate::sizing::Gpu>, gpu| {
69 if best.is_none_or(|prior| gpu.vram_bytes > prior.vram_bytes) {
70 Some(gpu)
71 } else {
72 best
73 }
74 });
75 let (vendor, kind, bytes) = if unified(hardware) && hardware.platform != Platform::Darwin {
76 ("shared", "unified", hardware.total_ram_bytes)
78 } else if let Some(gpu) = discrete {
79 (
80 match gpu.vendor {
81 GpuVendor::Nvidia => "nvidia",
82 GpuVendor::Amd => "amd",
83 _ => "intel",
84 },
85 "discrete",
86 gpu.vram_bytes,
87 )
88 } else if unified(hardware) {
89 ("apple", "unified", hardware.total_ram_bytes)
90 } else {
91 ("none", "cpu", hardware.total_ram_bytes)
92 };
93 dataset.classes.iter().find(|class| {
94 class.vendor == vendor
95 && class.kind == kind
96 && bytes >= class.min_bytes
97 && bytes < class.max_bytes
98 })
99}
100
101#[derive(Debug, Clone, Serialize)]
102#[serde(rename_all = "camelCase")]
103pub struct Throughput {
104 pub low_tok_per_sec: f64,
105 pub high_tok_per_sec: f64,
106 pub known: bool,
107}
108impl Throughput {
109 pub fn unknown() -> Self {
110 Self {
111 low_tok_per_sec: 0.0,
112 high_tok_per_sec: 0.0,
113 known: false,
114 }
115 }
116}
117
118pub fn throughput(
119 model: &CatalogModel,
120 quant: &Quantization,
121 hardware: &Hardware,
122 dataset: &PerfDataset,
123 backend: &str,
124) -> Result<Throughput, ValidationError> {
125 require(BACKENDS.contains(&backend), "unknown backend")?;
126 let Some(class) = match_perf(hardware, dataset) else {
127 return Ok(Throughput::unknown());
128 };
129 let efficiency = class
130 .efficiency_by_backend
131 .as_ref()
132 .and_then(|values| values.get(backend))
133 .copied()
134 .or_else(|| {
135 ["ollama", "llamacpp"]
136 .contains(&backend)
137 .then_some(class.efficiency)
138 });
139 let Some(efficiency) = efficiency else {
140 return Ok(Throughput::unknown());
141 };
142 let label = if matches!(model.architecture, Architecture::Moe) {
143 model.active_params.as_deref()
144 } else {
145 Some(model.params.as_str())
146 };
147 let Some(label) = label else {
148 return Ok(Throughput::unknown());
149 };
150 let bytes = if let Some(bits) = quant_bits(&quant.name) {
151 (parse_param_count(label)? * bits / 8.0).ceil()
152 } else if matches!(model.architecture, Architecture::Moe) {
153 return Ok(Throughput::unknown());
154 } else {
155 quant.disk_bytes
156 };
157 if bytes <= 0.0 {
158 return Ok(Throughput::unknown());
159 }
160 let point = class.mem_bandwidth_gbps * 1e9 * efficiency / bytes;
161 Ok(Throughput {
162 low_tok_per_sec: (point * 0.7 * 10.0).round() / 10.0,
163 high_tok_per_sec: (point * 1.3 * 10.0).round() / 10.0,
164 known: true,
165 })
166}
167
168pub fn verdict(
169 model: &CatalogModel,
170 hardware: &Hardware,
171 dataset: &PerfDataset,
172 context: Option<f64>,
173 backend: &str,
174) -> Result<Value, ValidationError> {
175 let sized = evaluate(&SizingRequest {
176 model: model.sizing(),
177 hardware: hardware.clone(),
178 context,
179 })?;
180 let fit = sized.fit;
181 if !fit.fits {
182 return Ok(
183 json!({"runnable":"no","throughput":Throughput::unknown(),"reason":fit.reason,"requiredBytes":fit.required_bytes,"usableBytes":fit.usable_bytes}),
184 );
185 }
186 let quant = fit
187 .quant
188 .ok_or_else(|| ValidationError("fitting quantization missing".into()))?;
189 let estimate = throughput(model, &quant, hardware, dataset, backend)?;
190 let runnable =
191 if estimate.known && (estimate.low_tok_per_sec + estimate.high_tok_per_sec) / 2.0 >= 10.0 {
192 "yes"
193 } else {
194 "slow"
195 };
196 Ok(
197 json!({"runnable":runnable,"throughput":estimate,"quant":quant,"requiredBytes":fit.required_bytes,"usableBytes":fit.usable_bytes}),
198 )
199}