use std::marker::PhantomData;
use std::sync::Arc;
use candle_core::quantized::QuantizedType;
use candle_core::quantized::{gguf_file, k_quants, GgmlDType, GgmlType, QStorage, QTensor};
use candle_core::{CpuStorage, Result};
use half::{bf16, f16};
use memmap2::Mmap;
pub struct MmapBlocks<T: GgmlType> {
_mmap: Arc<Mmap>,
ptr: *const T,
len: usize,
_marker: PhantomData<T>,
}
pub trait MmapPrefetch: Send + Sync + 'static {
fn prefetch(&self);
}
impl<T: GgmlType + 'static> MmapPrefetch for MmapBlocks<T> {
fn prefetch(&self) {
let base = self._mmap.as_ptr() as usize;
let off = self.ptr as usize - base;
let len = self.len * std::mem::size_of::<T>();
let _ = self._mmap.advise_range(memmap2::Advice::WillNeed, off, len);
}
}
unsafe impl<T: GgmlType> Send for MmapBlocks<T> {}
unsafe impl<T: GgmlType> Sync for MmapBlocks<T> {}
impl<T: GgmlType> MmapBlocks<T> {
fn blocks(&self) -> &[T] {
unsafe { std::slice::from_raw_parts(self.ptr, self.len) }
}
fn borrow(mmap: &Arc<Mmap>, byte_offset: usize, n_blocks: usize) -> Option<Self> {
let size = n_blocks.checked_mul(std::mem::size_of::<T>())?;
let end = byte_offset.checked_add(size)?;
if end > mmap.len() {
return None;
}
let ptr = unsafe { mmap.as_ptr().add(byte_offset) };
if !(ptr as usize).is_multiple_of(std::mem::align_of::<T>()) {
return None;
}
Some(Self {
_mmap: Arc::clone(mmap),
ptr: ptr as *const T,
len: n_blocks,
_marker: PhantomData,
})
}
}
impl<T: GgmlType + Send + Sync> QuantizedType for MmapBlocks<T> {
fn dtype(&self) -> GgmlDType {
T::DTYPE
}
fn matmul_t(&self, mkn: (usize, usize, usize), lhs: &[f32], dst: &mut [f32]) -> Result<()> {
crate::quant_matmul::matmul_kquant(mkn, lhs, self.blocks(), dst)
}
fn matmul_t_f16(&self, mkn: (usize, usize, usize), lhs: &[f16], dst: &mut [f16]) -> Result<()> {
k_quants::matmul_f16(mkn, lhs, self.blocks(), dst)
}
fn embedding(&self, ids: &[u32], rows: usize, hidden: usize) -> Result<CpuStorage> {
if !hidden.is_multiple_of(T::BLCK_SIZE) {
candle_core::bail!(
"quantized embedding hidden size {hidden} is not divisible by block size {}",
T::BLCK_SIZE
)
}
let blocks = self.blocks();
let row_blocks = hidden / T::BLCK_SIZE;
if blocks.len() != rows * row_blocks {
candle_core::bail!(
"quantized tensor has {} blocks, expected {}",
blocks.len(),
rows * row_blocks
)
}
let mut out = vec![0f32; ids.len() * hidden];
for (out_row, &row_id) in ids.iter().enumerate() {
let row = row_id as usize;
if row >= rows {
candle_core::bail!("embedding id {row} is out of range for {rows} rows")
}
let src = &blocks[row * row_blocks..(row + 1) * row_blocks];
let dst = &mut out[out_row * hidden..(out_row + 1) * hidden];
T::to_float(src, dst);
}
Ok(CpuStorage::F32(out))
}
fn dequantize(&self, elem_count: usize) -> Result<CpuStorage> {
let mut ys = vec![0.0f32; elem_count];
T::to_float(self.blocks(), &mut ys);
Ok(CpuStorage::F32(ys))
}
fn storage_size_in_bytes(&self) -> usize {
self.len * std::mem::size_of::<T>()
}
fn as_ptr(&self) -> *const u8 {
self.ptr as *const u8
}
fn block_size(&self) -> usize {
T::BLCK_SIZE
}
fn size(&self) -> usize {
self.len * std::mem::size_of::<T>()
}
fn from_float(&mut self, _xs: &[f32]) {
panic!("cannot quantize into a read-only memory-mapped tensor")
}
fn from_float_imatrix(&mut self, _xs: &[f32], _imatrix_weights: &[f32], _n_per_row: usize) {
panic!("cannot quantize into a read-only memory-mapped tensor")
}
}
pub struct MmapBlocksIq2Xxs {
_mmap: Arc<Mmap>,
ptr: *const crate::iq2xxs::BlockIq2Xxs,
len: usize,
}
unsafe impl Send for MmapBlocksIq2Xxs {}
unsafe impl Sync for MmapBlocksIq2Xxs {}
impl MmapPrefetch for MmapBlocksIq2Xxs {
fn prefetch(&self) {
let base = self._mmap.as_ptr() as usize;
let off = self.ptr as usize - base;
let len = self.len * crate::iq2xxs::BLOCK_BYTES;
let _ = self._mmap.advise_range(memmap2::Advice::WillNeed, off, len);
}
}
impl MmapBlocksIq2Xxs {
fn blocks(&self) -> &[crate::iq2xxs::BlockIq2Xxs] {
unsafe { std::slice::from_raw_parts(self.ptr, self.len) }
}
fn borrow(mmap: &Arc<Mmap>, byte_offset: usize, n_blocks: usize) -> Option<Self> {
let size = n_blocks.checked_mul(crate::iq2xxs::BLOCK_BYTES)?;
let end = byte_offset.checked_add(size)?;
if end > mmap.len() {
return None;
}
let ptr = unsafe { mmap.as_ptr().add(byte_offset) };
Some(Self {
_mmap: Arc::clone(mmap),
ptr: ptr as *const crate::iq2xxs::BlockIq2Xxs,
len: n_blocks,
})
}
}
pub fn prefetch_handle(
mmap: &Arc<Mmap>,
dtype: GgmlDType,
byte_offset: usize,
n_blocks: usize,
) -> Option<Arc<dyn MmapPrefetch>> {
macro_rules! handle {
($ty:ty) => {
MmapBlocks::<$ty>::borrow(mmap, byte_offset, n_blocks)
.map(|b| Arc::new(b) as Arc<dyn MmapPrefetch>)
};
}
match dtype {
GgmlDType::F32 => handle!(f32),
GgmlDType::F16 => handle!(f16),
GgmlDType::BF16 => handle!(bf16),
GgmlDType::Q4_0 => handle!(k_quants::BlockQ4_0),
GgmlDType::Q4_1 => handle!(k_quants::BlockQ4_1),
GgmlDType::Q5_0 => handle!(k_quants::BlockQ5_0),
GgmlDType::Q5_1 => handle!(k_quants::BlockQ5_1),
GgmlDType::Q8_0 => handle!(k_quants::BlockQ8_0),
GgmlDType::Q8_1 => handle!(k_quants::BlockQ8_1),
GgmlDType::Q2K => handle!(k_quants::BlockQ2K),
GgmlDType::Q3K => handle!(k_quants::BlockQ3K),
GgmlDType::Q4K => handle!(k_quants::BlockQ4K),
GgmlDType::Q5K => handle!(k_quants::BlockQ5K),
GgmlDType::Q6K => handle!(k_quants::BlockQ6K),
GgmlDType::Q8K => handle!(k_quants::BlockQ8K),
}
}
pub fn prefetch_handle_iq2xxs(
mmap: &Arc<Mmap>,
byte_offset: usize,
n_blocks: usize,
) -> Option<Arc<dyn MmapPrefetch>> {
MmapBlocksIq2Xxs::borrow(mmap, byte_offset, n_blocks)
.map(|b| Arc::new(b) as Arc<dyn MmapPrefetch>)
}
impl QuantizedType for MmapBlocksIq2Xxs {
fn dtype(&self) -> GgmlDType {
GgmlDType::Q2K }
fn matmul_t(&self, mkn: (usize, usize, usize), lhs: &[f32], dst: &mut [f32]) -> Result<()> {
crate::iq2xxs::matmul_t(mkn, lhs, self.blocks(), dst)
}
fn matmul_t_f16(&self, mkn: (usize, usize, usize), lhs: &[f16], dst: &mut [f16]) -> Result<()> {
crate::iq2xxs::matmul_t_f16(mkn, lhs, self.blocks(), dst)
}
fn embedding(&self, ids: &[u32], rows: usize, hidden: usize) -> Result<CpuStorage> {
if !hidden.is_multiple_of(crate::iq2xxs::QK_IQ2_XXS) {
candle_core::bail!(
"quantized embedding hidden size {hidden} is not divisible by block size {}",
crate::iq2xxs::QK_IQ2_XXS
)
}
let blocks = self.blocks();
let row_blocks = hidden / crate::iq2xxs::QK_IQ2_XXS;
if blocks.len() != rows * row_blocks {
candle_core::bail!(
"quantized tensor has {} blocks, expected {}",
blocks.len(),
rows * row_blocks
)
}
let mut out = vec![0f32; ids.len() * hidden];
for (out_row, &row_id) in ids.iter().enumerate() {
let row = row_id as usize;
if row >= rows {
candle_core::bail!("embedding id {row} is out of range for {rows} rows")
}
let src = &blocks[row * row_blocks..(row + 1) * row_blocks];
let dst = &mut out[out_row * hidden..(out_row + 1) * hidden];
crate::iq2xxs::dequantize(src, dst)?;
}
Ok(CpuStorage::F32(out))
}
fn dequantize(&self, elem_count: usize) -> Result<CpuStorage> {
let mut ys = vec![0.0f32; elem_count];
crate::iq2xxs::dequantize(self.blocks(), &mut ys)?;
Ok(CpuStorage::F32(ys))
}
fn storage_size_in_bytes(&self) -> usize {
self.len * crate::iq2xxs::BLOCK_BYTES
}
fn as_ptr(&self) -> *const u8 {
self.ptr as *const u8
}
fn block_size(&self) -> usize {
crate::iq2xxs::QK_IQ2_XXS
}
fn size(&self) -> usize {
self.len * crate::iq2xxs::BLOCK_BYTES
}
fn from_float(&mut self, _xs: &[f32]) {
panic!("cannot quantize into a read-only memory-mapped tensor")
}
fn from_float_imatrix(&mut self, _xs: &[f32], _imatrix_weights: &[f32], _n_per_row: usize) {
panic!("cannot quantize into a read-only memory-mapped tensor")
}
}
pub fn borrowed_qtensor_raw(
mmap: &Arc<Mmap>,
dtype: u32,
offset: u64,
tensor_data_offset: u64,
shape: candle_core::Shape,
) -> Result<Option<QTensor>> {
if dtype == crate::iq2xxs::GGML_TYPE_IQ2_XXS {
let Ok(off) = usize::try_from(tensor_data_offset.saturating_add(offset)) else {
return Ok(None);
};
return borrowed_range_iq2xxs(mmap, off, shape);
}
let Some(ggml_dtype) = crate::gguf_ext::ggml_dtype_from_id(dtype) else {
return Ok(None);
};
let info = gguf_file::TensorInfo {
ggml_dtype,
shape,
offset,
};
borrowed_qtensor(mmap, &info, tensor_data_offset)
}
pub fn borrowed_range_iq2xxs(
mmap: &Arc<Mmap>,
offset: usize,
shape: candle_core::Shape,
) -> Result<Option<QTensor>> {
let elem_count = shape.elem_count();
if !elem_count.is_multiple_of(crate::iq2xxs::QK_IQ2_XXS) {
return Ok(None);
}
let n_blocks = elem_count / crate::iq2xxs::QK_IQ2_XXS;
let Some(blocks) = MmapBlocksIq2Xxs::borrow(mmap, offset, n_blocks) else {
return Ok(None);
};
let storage: Box<dyn QuantizedType> = Box::new(blocks);
QTensor::new(QStorage::Cpu(storage), shape).map(Some)
}
pub fn borrowed_qtensor(
mmap: &Arc<Mmap>,
info: &gguf_file::TensorInfo,
tensor_data_offset: u64,
) -> Result<Option<QTensor>> {
let Ok(offset) = usize::try_from(tensor_data_offset.saturating_add(info.offset)) else {
return Ok(None);
};
borrowed_range(mmap, info.ggml_dtype, offset, info.shape.clone())
}
pub fn borrowed_range(
mmap: &Arc<Mmap>,
dtype: GgmlDType,
offset: usize,
shape: candle_core::Shape,
) -> Result<Option<QTensor>> {
let elem_count = shape.elem_count();
let block_size = dtype.block_size();
if block_size == 0 || !elem_count.is_multiple_of(block_size) {
return Ok(None);
}
let n_blocks = elem_count / block_size;
macro_rules! borrow {
($ty:ty) => {
match MmapBlocks::<$ty>::borrow(mmap, offset, n_blocks) {
Some(blocks) => blocks,
None => return Ok(None),
}
};
}
let storage: Box<dyn QuantizedType> = match dtype {
GgmlDType::F32 => Box::new(borrow!(f32)),
GgmlDType::F16 => Box::new(borrow!(f16)),
GgmlDType::BF16 => Box::new(borrow!(bf16)),
GgmlDType::Q4_0 => Box::new(borrow!(k_quants::BlockQ4_0)),
GgmlDType::Q4_1 => Box::new(borrow!(k_quants::BlockQ4_1)),
GgmlDType::Q5_0 => Box::new(borrow!(k_quants::BlockQ5_0)),
GgmlDType::Q5_1 => Box::new(borrow!(k_quants::BlockQ5_1)),
GgmlDType::Q8_0 => Box::new(borrow!(k_quants::BlockQ8_0)),
GgmlDType::Q8_1 => Box::new(borrow!(k_quants::BlockQ8_1)),
GgmlDType::Q2K => Box::new(borrow!(k_quants::BlockQ2K)),
GgmlDType::Q3K => Box::new(borrow!(k_quants::BlockQ3K)),
GgmlDType::Q4K => Box::new(borrow!(k_quants::BlockQ4K)),
GgmlDType::Q5K => Box::new(borrow!(k_quants::BlockQ5K)),
GgmlDType::Q6K => Box::new(borrow!(k_quants::BlockQ6K)),
GgmlDType::Q8K => Box::new(borrow!(k_quants::BlockQ8K)),
};
QTensor::new(QStorage::Cpu(storage), shape).map(Some)
}
pub fn qtensor_from_mmap<R: std::io::Read + std::io::Seek>(
content: &gguf_file::Content,
mmap: &Arc<Mmap>,
reader: &mut R,
name: &str,
device: &candle_core::Device,
) -> Result<QTensor> {
if device.is_cpu() {
if let Some(info) = content.tensor_infos.get(name) {
if let Some(t) = borrowed_qtensor(mmap, info, content.tensor_data_offset)? {
return Ok(t);
}
tracing::debug!(
tensor = name,
"tensor not borrowable from the mapping, copying instead"
);
}
}
content.tensor(reader, name, device)
}
#[cfg(test)]
mod tests {
use super::*;
use candle_core::quantized::QTensor as QT;
use candle_core::{Device, Tensor};
fn gguf_with_tensor(dir: &std::path::Path, data: &[f32], shape: &[usize]) -> Arc<Mmap> {
let path = dir.join("t.gguf");
let t = Tensor::from_vec(data.to_vec(), shape, &Device::Cpu).unwrap();
let q = QT::quantize(&t, GgmlDType::F32).unwrap();
let mut f = std::fs::File::create(&path).unwrap();
gguf_file::write(&mut f, &[], &[("w", &q)]).unwrap();
drop(f);
let f = std::fs::File::open(&path).unwrap();
Arc::new(unsafe { Mmap::map(&f) }.unwrap())
}
#[test]
fn borrowed_tensor_matches_copied_tensor_and_shares_the_mapping() {
let dir = std::env::temp_dir().join(format!("joshua-mmapt-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let data: Vec<f32> = (0..64).map(|i| i as f32 * 0.25 - 8.0).collect();
let mmap = gguf_with_tensor(&dir, &data, &[8, 8]);
let mut cursor = std::io::Cursor::new(&mmap[..]);
let content = gguf_file::Content::read(&mut cursor).unwrap();
let info = content.tensor_infos.get("w").unwrap();
let borrowed = borrowed_qtensor(&mmap, info, content.tensor_data_offset)
.unwrap()
.expect("F32 tensor should be borrowable");
let copied = content.tensor(&mut cursor, "w", &Device::Cpu).unwrap();
let a: Vec<f32> = borrowed
.dequantize(&Device::Cpu)
.unwrap()
.flatten_all()
.unwrap()
.to_vec1()
.unwrap();
let b: Vec<f32> = copied
.dequantize(&Device::Cpu)
.unwrap()
.flatten_all()
.unwrap()
.to_vec1()
.unwrap();
assert_eq!(a, b, "borrowed and copied tensors must agree");
assert_eq!(a, data);
let base = mmap.as_ptr() as usize;
let ptr = borrowed.data().unwrap().as_ptr() as usize;
assert!(
ptr >= base && ptr < base + mmap.len(),
"borrowed tensor must point inside the mapping"
);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn out_of_bounds_tensor_declines_the_borrow() {
let dir = std::env::temp_dir().join(format!("joshua-mmapt-oob-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let data: Vec<f32> = vec![1.0; 32];
let mmap = gguf_with_tensor(&dir, &data, &[4, 8]);
let mut cursor = std::io::Cursor::new(&mmap[..]);
let content = gguf_file::Content::read(&mut cursor).unwrap();
let info = content.tensor_infos.get("w").unwrap();
let bogus = gguf_file::TensorInfo {
ggml_dtype: info.ggml_dtype,
shape: info.shape.clone(),
offset: mmap.len() as u64,
};
assert!(borrowed_qtensor(&mmap, &bogus, content.tensor_data_offset)
.unwrap()
.is_none());
std::fs::remove_dir_all(&dir).ok();
}
}