use anyhow::{anyhow, Result};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[allow(non_camel_case_types)]
pub enum QuantType {
Q8_0,
Q6_K,
Q4_K_M,
Q3_K_M,
}
impl QuantType {
pub fn as_str(self) -> &'static str {
match self {
Self::Q8_0 => "Q8_0",
Self::Q6_K => "Q6_K",
Self::Q4_K_M => "Q4_K_M",
Self::Q3_K_M => "Q3_K_M",
}
}
pub fn from_canonical_str(name: &str) -> std::result::Result<Self, String> {
match name.to_ascii_uppercase().as_str() {
"Q8_0" => Ok(Self::Q8_0),
"Q6_K" => Ok(Self::Q6_K),
"Q4_K_M" => Ok(Self::Q4_K_M),
"Q3_K_M" => Ok(Self::Q3_K_M),
other => Err(format!(
"unknown quant type {other:?}: supported = Q8_0, Q6_K, Q4_K_M, Q3_K_M"
)),
}
}
}
impl std::fmt::Display for QuantType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct GpuInfo {
pub memory_bytes: u64,
}
impl GpuInfo {
pub fn from_gib(gib: u64) -> Self {
Self {
memory_bytes: gib.saturating_mul(1u64 << 30),
}
}
pub fn from_bytes(memory_bytes: u64) -> Self {
Self { memory_bytes }
}
pub fn from_hardware_profile(profile: &crate::core::hardware::HardwareProfile) -> Self {
Self::from_bytes(profile.available_memory_bytes)
}
pub fn memory_gib_f64(&self) -> f64 {
self.memory_bytes as f64 / (1u64 << 30) as f64
}
pub fn memory_gib_floor(&self) -> u64 {
self.memory_bytes / (1u64 << 30)
}
}
const THRESHOLDS_GIB: &[(u64, QuantType)] = &[
(64, QuantType::Q8_0),
(32, QuantType::Q6_K),
(16, QuantType::Q4_K_M),
(8, QuantType::Q3_K_M),
];
pub const MIN_SUPPORTED_GIB: u64 = 8;
pub fn select_quant(info: &GpuInfo) -> Result<QuantType> {
let gib_floor = info.memory_gib_floor();
if gib_floor < MIN_SUPPORTED_GIB {
return Err(anyhow!(
"hf2q requires at least {min} GiB of GPU/unified memory; \
detected {detected} GiB ({bytes} bytes). \
Minimum supported configuration: {min} GiB → Q3_K_M.",
min = MIN_SUPPORTED_GIB,
detected = gib_floor,
bytes = info.memory_bytes,
));
}
for &(threshold, quant) in THRESHOLDS_GIB {
if gib_floor >= threshold {
return Ok(quant);
}
}
unreachable!(
"quant selection fell through table at {gib_floor} GiB — \
THRESHOLDS_GIB lower bound and MIN_SUPPORTED_GIB are out of sync"
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn select_quant_64_gib_exact_q8() {
let info = GpuInfo::from_gib(64);
assert_eq!(select_quant(&info).unwrap(), QuantType::Q8_0);
}
#[test]
fn select_quant_just_above_64_gib_q8() {
let info = GpuInfo::from_bytes((64u64 << 30) + 1);
assert_eq!(select_quant(&info).unwrap(), QuantType::Q8_0);
}
#[test]
fn select_quant_huge_machine_q8() {
let info = GpuInfo::from_gib(128);
assert_eq!(select_quant(&info).unwrap(), QuantType::Q8_0);
}
#[test]
fn select_quant_63_gib_q6() {
let info = GpuInfo::from_gib(63);
assert_eq!(select_quant(&info).unwrap(), QuantType::Q6_K);
}
#[test]
fn select_quant_just_below_64_gib_q6() {
let info = GpuInfo::from_bytes((64u64 << 30) - 1);
assert_eq!(select_quant(&info).unwrap(), QuantType::Q6_K);
}
#[test]
fn select_quant_32_gib_exact_q6() {
let info = GpuInfo::from_gib(32);
assert_eq!(select_quant(&info).unwrap(), QuantType::Q6_K);
}
#[test]
fn select_quant_31_gib_q4() {
let info = GpuInfo::from_gib(31);
assert_eq!(select_quant(&info).unwrap(), QuantType::Q4_K_M);
}
#[test]
fn select_quant_just_below_32_gib_q4() {
let info = GpuInfo::from_bytes((32u64 << 30) - 1);
assert_eq!(select_quant(&info).unwrap(), QuantType::Q4_K_M);
}
#[test]
fn select_quant_16_gib_exact_q4() {
let info = GpuInfo::from_gib(16);
assert_eq!(select_quant(&info).unwrap(), QuantType::Q4_K_M);
}
#[test]
fn select_quant_15_gib_q3() {
let info = GpuInfo::from_gib(15);
assert_eq!(select_quant(&info).unwrap(), QuantType::Q3_K_M);
}
#[test]
fn select_quant_just_below_16_gib_q3() {
let info = GpuInfo::from_bytes((16u64 << 30) - 1);
assert_eq!(select_quant(&info).unwrap(), QuantType::Q3_K_M);
}
#[test]
fn select_quant_8_gib_exact_q3() {
let info = GpuInfo::from_gib(8);
assert_eq!(select_quant(&info).unwrap(), QuantType::Q3_K_M);
}
#[test]
fn select_quant_7_gib_refuse() {
let info = GpuInfo::from_gib(7);
let err = select_quant(&info).unwrap_err();
let msg = format!("{err}");
assert!(
msg.contains("8 GiB"),
"error must name the 8 GiB minimum, got: {msg}"
);
}
#[test]
fn select_quant_just_below_8_gib_refuse() {
let info = GpuInfo::from_bytes((8u64 << 30) - 1);
assert!(select_quant(&info).is_err());
}
#[test]
fn select_quant_zero_refuse() {
let info = GpuInfo::from_bytes(0);
assert!(select_quant(&info).is_err());
}
#[test]
fn select_quant_error_message_names_min() {
let info = GpuInfo::from_gib(4);
let err = select_quant(&info).unwrap_err();
let msg = format!("{err}");
assert!(msg.contains("8 GiB"), "missing '8 GiB' in: {msg}");
assert!(msg.contains("Q3_K_M"), "missing 'Q3_K_M' in: {msg}");
assert!(
msg.contains("4 GiB"),
"missing detected size '4 GiB' in: {msg}"
);
}
#[test]
fn quant_type_as_str_matches_ggml_names() {
assert_eq!(QuantType::Q8_0.as_str(), "Q8_0");
assert_eq!(QuantType::Q6_K.as_str(), "Q6_K");
assert_eq!(QuantType::Q4_K_M.as_str(), "Q4_K_M");
assert_eq!(QuantType::Q3_K_M.as_str(), "Q3_K_M");
}
#[test]
fn quant_type_display_matches_as_str() {
assert_eq!(format!("{}", QuantType::Q4_K_M), "Q4_K_M");
}
#[test]
fn gpu_info_gib_helpers_roundtrip() {
let info = GpuInfo::from_gib(48);
assert_eq!(info.memory_gib_floor(), 48);
assert!((info.memory_gib_f64() - 48.0).abs() < 1e-9);
assert_eq!(info.memory_bytes, 48u64 << 30);
}
}