use crate::format::ArchInfo;
use crate::io::parse_quant_format;
use crate::kernels::quants::QuantFormat;
#[derive(Debug, Clone, Copy)]
pub(crate) struct AllowedQuant {
pub(crate) desc: &'static str,
pub(crate) formats: &'static [QuantFormat],
}
impl AllowedQuant {
pub(crate) fn match_dtype(&self, dtype: &str) -> Option<QuantFormat> {
let format = parse_quant_format(dtype)?;
self.formats.contains(&format).then_some(format)
}
}
pub(crate) const Q4K: AllowedQuant = AllowedQuant {
desc: "q4_k",
formats: &[QuantFormat::Q4_K],
};
pub(crate) const Q5K: AllowedQuant = AllowedQuant {
desc: "q5_k",
formats: &[QuantFormat::Q5_K],
};
pub(crate) const Q6K: AllowedQuant = AllowedQuant {
desc: "q6_k",
formats: &[QuantFormat::Q6_K],
};
pub(crate) const Q8_0: AllowedQuant = AllowedQuant {
desc: "q8_0",
formats: &[QuantFormat::Q8_0],
};
pub(crate) const Q4K_OR_Q6K: AllowedQuant = AllowedQuant {
desc: "q4_k or q6_k",
formats: &[QuantFormat::Q4_K, QuantFormat::Q6_K],
};
#[derive(Debug, Clone, Copy)]
pub(crate) struct Dims {
pub(crate) hidden: usize,
pub(crate) q_dim: usize,
pub(crate) kv_dim: usize,
pub(crate) head_dim: usize,
pub(crate) vocab: usize,
pub(crate) n_experts: usize,
pub(crate) moe_intermediate: usize,
}
impl Dims {
pub(crate) fn from_arch(arch: &ArchInfo) -> Self {
Self {
hidden: arch.hidden as usize,
q_dim: arch.n_heads as usize * arch.head_dim as usize,
kv_dim: arch.n_kv_heads as usize * arch.head_dim as usize,
head_dim: arch.head_dim as usize,
vocab: arch.vocab as usize,
n_experts: arch.n_experts as usize,
moe_intermediate: arch.moe_intermediate as usize,
}
}
}