rigspark_core/
advice.rs

1use crate::{
2    catalog::{BACKENDS, CatalogModel, PerfClass, PerfDataset, require},
3    sizing::{
4        Architecture, GpuVendor, Hardware, Platform, Quantization, SizingRequest, ValidationError,
5        evaluate, parse_param_count, quant_bits,
6    },
7};
8use serde::Serialize;
9use serde_json::{Value, json};
10
11const GIB: f64 = 1073741824.0;
12pub fn unified(hardware: &Hardware) -> bool {
13    hardware.is_unified()
14}
15pub fn largest_vram(hardware: &Hardware) -> f64 {
16    hardware
17        .gpu
18        .iter()
19        .map(|gpu| gpu.vram_bytes)
20        .fold(0.0_f64, f64::max)
21}
22pub fn hardware_score(hardware: &Hardware) -> Value {
23    let vram = largest_vram(hardware);
24    let pool = if vram > 0.0 {
25        vram
26    } else if unified(hardware) {
27        hardware.total_ram_bytes
28    } else {
29        0.0
30    };
31    let sub = [
32        (pool / (24.0 * GIB)).clamp(0.0, 1.0),
33        (hardware.total_ram_bytes / (64.0 * GIB)).clamp(0.0, 1.0),
34        if vram > 0.0 {
35            0.5 + 0.5 * (vram / (24.0 * GIB)).clamp(0.0, 1.0)
36        } else if unified(hardware) {
37            0.65
38        } else {
39            0.25
40        },
41        (hardware.free_disk_bytes / (200.0 * GIB)).clamp(0.0, 1.0),
42    ];
43    let weights = [0.4, 0.25, 0.25, 0.1];
44    let mut best = 0;
45    let mut deficit = -1.0;
46    let mut total = 0.0;
47    for index in 0..4 {
48        total += weights[index] * sub[index];
49        let current = weights[index] * (1.0 - sub[index]);
50        if current > deficit {
51            best = index;
52            deficit = current;
53        }
54    }
55    json!({"total":(total * 100.0).round() as u32,"sub":{"vram":sub[0],"ram":sub[1],"compute":sub[2],"storage":sub[3]},"bottleneck":(["vram","ram","compute","storage"][best])})
56}
57
58pub fn match_perf<'a>(hardware: &Hardware, dataset: &'a PerfDataset) -> Option<&'a PerfClass> {
59    let discrete = hardware
60        .gpu
61        .iter()
62        .filter(|gpu| {
63            matches!(
64                gpu.vendor,
65                GpuVendor::Nvidia | GpuVendor::Amd | GpuVendor::Intel
66            ) && gpu.vram_bytes > 0.0
67        })
68        .fold(None, |best: Option<&crate::sizing::Gpu>, gpu| {
69            if best.is_none_or(|prior| gpu.vram_bytes > prior.vram_bytes) {
70                Some(gpu)
71            } else {
72                best
73            }
74        });
75    let (vendor, kind, bytes) = if unified(hardware) && hardware.platform != Platform::Darwin {
76        // Only Apple unified classes are sourced; other unified SoCs stay honest `unknown`.
77        ("shared", "unified", hardware.total_ram_bytes)
78    } else if let Some(gpu) = discrete {
79        (
80            match gpu.vendor {
81                GpuVendor::Nvidia => "nvidia",
82                GpuVendor::Amd => "amd",
83                _ => "intel",
84            },
85            "discrete",
86            gpu.vram_bytes,
87        )
88    } else if unified(hardware) {
89        ("apple", "unified", hardware.total_ram_bytes)
90    } else {
91        ("none", "cpu", hardware.total_ram_bytes)
92    };
93    dataset.classes.iter().find(|class| {
94        class.vendor == vendor
95            && class.kind == kind
96            && bytes >= class.min_bytes
97            && bytes < class.max_bytes
98    })
99}
100
101#[derive(Debug, Clone, Serialize)]
102#[serde(rename_all = "camelCase")]
103pub struct Throughput {
104    pub low_tok_per_sec: f64,
105    pub high_tok_per_sec: f64,
106    pub known: bool,
107}
108impl Throughput {
109    pub fn unknown() -> Self {
110        Self {
111            low_tok_per_sec: 0.0,
112            high_tok_per_sec: 0.0,
113            known: false,
114        }
115    }
116}
117
118pub fn throughput(
119    model: &CatalogModel,
120    quant: &Quantization,
121    hardware: &Hardware,
122    dataset: &PerfDataset,
123    backend: &str,
124) -> Result<Throughput, ValidationError> {
125    require(BACKENDS.contains(&backend), "unknown backend")?;
126    if quant.name.eq_ignore_ascii_case("Q1_0") || quant.is_bitnet() || backend == "bitnet" {
127        return Ok(Throughput::unknown());
128    }
129    let Some(class) = match_perf(hardware, dataset) else {
130        return Ok(Throughput::unknown());
131    };
132    let efficiency = class
133        .efficiency_by_backend
134        .as_ref()
135        .and_then(|values| values.get(backend))
136        .copied()
137        .or_else(|| {
138            ["ollama", "llamacpp"]
139                .contains(&backend)
140                .then_some(class.efficiency)
141        });
142    let Some(efficiency) = efficiency else {
143        return Ok(Throughput::unknown());
144    };
145    let Some(bytes) = decode_bytes(model, quant)? else {
146        return Ok(Throughput::unknown());
147    };
148    let point = class.mem_bandwidth_gbps * 1e9 * efficiency / bytes;
149    Ok(Throughput {
150        low_tok_per_sec: (point * 0.7 * 10.0).round() / 10.0,
151        high_tok_per_sec: (point * 1.3 * 10.0).round() / 10.0,
152        known: true,
153    })
154}
155
156/// Bytes read per decoded token (active weights at the quantization's bit width), or `None`
157/// when no sourced estimate exists (1-bit quants, MoE without active params or known bits).
158pub fn decode_bytes(
159    model: &CatalogModel,
160    quant: &Quantization,
161) -> Result<Option<f64>, ValidationError> {
162    if quant.name.eq_ignore_ascii_case("Q1_0") || quant.is_bitnet() {
163        return Ok(None);
164    }
165    let label = if matches!(model.architecture, Architecture::Moe) {
166        model.active_params.as_deref()
167    } else {
168        Some(model.params.as_str())
169    };
170    let Some(label) = label else {
171        return Ok(None);
172    };
173    let bytes = if let Some(bits) = quant_bits(&quant.name) {
174        (parse_param_count(label)? * bits / 8.0).ceil()
175    } else if matches!(model.architecture, Architecture::Moe) {
176        return Ok(None);
177    } else {
178        quant.disk_bytes
179    };
180    Ok((bytes > 0.0).then_some(bytes))
181}
182
183pub fn verdict(
184    model: &CatalogModel,
185    hardware: &Hardware,
186    dataset: &PerfDataset,
187    context: Option<f64>,
188    backend: &str,
189) -> Result<Value, ValidationError> {
190    let sized = evaluate(&SizingRequest {
191        model: model.sizing(),
192        hardware: hardware.clone(),
193        context,
194    })?;
195    let fit = sized.fit;
196    if !fit.fits {
197        return Ok(
198            json!({"runnable":"no","throughput":Throughput::unknown(),"reason":fit.reason,"requiredBytes":fit.required_bytes,"usableBytes":fit.usable_bytes}),
199        );
200    }
201    let quant = fit
202        .quant
203        .ok_or_else(|| ValidationError("fitting quantization missing".into()))?;
204    let estimate = throughput(model, &quant, hardware, dataset, backend)?;
205    let runnable =
206        if estimate.known && (estimate.low_tok_per_sec + estimate.high_tok_per_sec) / 2.0 >= 10.0 {
207            "yes"
208        } else {
209            "slow"
210        };
211    Ok(
212        json!({"runnable":runnable,"throughput":estimate,"quant":quant,"requiredBytes":fit.required_bytes,"usableBytes":fit.usable_bytes}),
213    )
214}
rigspark_core/advice.rs

rigspark_core/
advice.rs
