#[cfg(test)]
mod tests;
use crate::error::WhisperResult;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum MmapMode {
#[default]
ReadOnly,
ReadWrite,
CopyOnWrite,
}
impl MmapMode {
#[must_use]
pub fn description(&self) -> &str {
match self {
Self::ReadOnly => "read-only",
Self::ReadWrite => "read-write",
Self::CopyOnWrite => "copy-on-write",
}
}
#[must_use]
pub fn is_writable(&self) -> bool {
matches!(self, Self::ReadWrite | Self::CopyOnWrite)
}
}
#[derive(Debug, Clone)]
pub struct MmapConfig {
pub mode: MmapMode,
pub prefetch: bool,
pub page_size_hint: usize,
pub lock_pages: bool,
pub sequential_access: bool,
}
impl Default for MmapConfig {
fn default() -> Self {
Self {
mode: MmapMode::default(),
prefetch: false,
page_size_hint: 0,
lock_pages: false,
sequential_access: true,
}
}
}
impl MmapConfig {
#[must_use]
pub fn for_inference() -> Self {
Self {
mode: MmapMode::ReadOnly,
prefetch: true,
sequential_access: true,
..Default::default()
}
}
#[must_use]
pub fn random_access() -> Self {
Self {
sequential_access: false,
..Default::default()
}
}
#[must_use]
pub fn with_mode(mut self, mode: MmapMode) -> Self {
self.mode = mode;
self
}
#[must_use]
pub fn with_prefetch(mut self) -> Self {
self.prefetch = true;
self
}
#[must_use]
pub fn with_locked_pages(mut self) -> Self {
self.lock_pages = true;
self
}
#[must_use]
pub fn with_page_size(mut self, size: usize) -> Self {
self.page_size_hint = size;
self
}
}
#[derive(Debug, Clone)]
pub struct MemoryRegion {
pub offset: u64,
pub size: u64,
pub alignment: u64,
pub label: Option<String>,
}
impl MemoryRegion {
#[must_use]
pub fn new(offset: u64, size: u64) -> Self {
Self {
offset,
size,
alignment: 1,
label: None,
}
}
#[must_use]
pub fn entire_file(size: u64) -> Self {
Self::new(0, size)
}
#[must_use]
pub fn with_alignment(mut self, alignment: u64) -> Self {
self.alignment = alignment;
self
}
#[must_use]
pub fn with_label(mut self, label: impl Into<String>) -> Self {
self.label = Some(label.into());
self
}
#[must_use]
pub fn end(&self) -> u64 {
self.offset + self.size
}
#[must_use]
pub fn contains(&self, offset: u64) -> bool {
offset >= self.offset && offset < self.end()
}
#[must_use]
pub fn aligned_offset(&self) -> u64 {
(self.offset / self.alignment) * self.alignment
}
#[must_use]
pub fn aligned_size(&self) -> u64 {
let aligned_start = self.aligned_offset();
let end = self.offset + self.size;
let aligned_end = end.div_ceil(self.alignment) * self.alignment;
aligned_end - aligned_start
}
}
#[derive(Debug)]
pub struct MmapHandle {
id: u64,
size: u64,
config: MmapConfig,
regions: Vec<MemoryRegion>,
valid: bool,
}
impl MmapHandle {
pub fn new(size: u64, config: MmapConfig) -> WhisperResult<Self> {
static HANDLE_ID: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
Ok(Self {
id: HANDLE_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
size,
config,
regions: Vec::new(),
valid: true,
})
}
pub fn for_inference(size: u64) -> WhisperResult<Self> {
Self::new(size, MmapConfig::for_inference())
}
#[must_use]
pub fn id(&self) -> u64 {
self.id
}
#[must_use]
pub fn size(&self) -> u64 {
self.size
}
#[must_use]
pub fn config(&self) -> &MmapConfig {
&self.config
}
#[must_use]
pub fn is_valid(&self) -> bool {
self.valid
}
pub fn add_region(&mut self, region: MemoryRegion) {
self.regions.push(region);
}
#[must_use]
pub fn region_count(&self) -> usize {
self.regions.len()
}
#[must_use]
pub fn tracked_bytes(&self) -> u64 {
self.regions.iter().map(|r| r.size).sum()
}
pub fn invalidate(&mut self) {
self.valid = false;
}
pub fn read_at(&self, offset: u64, size: usize) -> WhisperResult<Vec<u8>> {
if !self.valid {
return Err(crate::error::WhisperError::Model(
"Memory map handle is invalid".to_string(),
));
}
if offset + size as u64 > self.size {
return Err(crate::error::WhisperError::Model(
"Read extends beyond mapped region".to_string(),
));
}
Ok(vec![0u8; size])
}
pub fn write_at(&mut self, offset: u64, data: &[u8]) -> WhisperResult<()> {
if !self.valid {
return Err(crate::error::WhisperError::Model(
"Memory map handle is invalid".to_string(),
));
}
if !self.config.mode.is_writable() {
return Err(crate::error::WhisperError::Model(
"Memory map is read-only".to_string(),
));
}
if offset + data.len() as u64 > self.size {
return Err(crate::error::WhisperError::Model(
"Write extends beyond mapped region".to_string(),
));
}
Ok(())
}
}
#[derive(Debug, Clone)]
pub struct WeightRegion {
pub name: String,
pub param_type: WeightType,
pub region: MemoryRegion,
pub dtype: WeightDtype,
pub shape: Vec<usize>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WeightType {
Weight,
Bias,
Scale,
Offset,
Embedding,
QueryProj,
KeyProj,
ValueProj,
OutProj,
}
impl WeightType {
#[must_use]
pub fn name(&self) -> &str {
match self {
Self::Weight => "weight",
Self::Bias => "bias",
Self::Scale => "scale",
Self::Offset => "offset",
Self::Embedding => "embedding",
Self::QueryProj => "query_proj",
Self::KeyProj => "key_proj",
Self::ValueProj => "value_proj",
Self::OutProj => "out_proj",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WeightDtype {
F32,
F16,
Bf16,
Int8,
Int4,
}
impl WeightDtype {
#[must_use]
pub fn bytes_per_element(&self) -> usize {
match self {
Self::F32 => 4,
Self::F16 | Self::Bf16 => 2,
Self::Int8 | Self::Int4 => 1, }
}
#[must_use]
pub fn name(&self) -> &str {
match self {
Self::F32 => "float32",
Self::F16 => "float16",
Self::Bf16 => "bfloat16",
Self::Int8 => "int8",
Self::Int4 => "int4",
}
}
}
impl WeightRegion {
#[must_use]
pub fn new(
name: impl Into<String>,
param_type: WeightType,
region: MemoryRegion,
dtype: WeightDtype,
shape: Vec<usize>,
) -> Self {
Self {
name: name.into(),
param_type,
region,
dtype,
shape,
}
}
#[must_use]
pub fn num_elements(&self) -> usize {
self.shape.iter().product()
}
#[must_use]
pub fn expected_bytes(&self) -> usize {
self.num_elements() * self.dtype.bytes_per_element()
}
#[must_use]
pub fn size_matches(&self) -> bool {
self.region.size as usize >= self.expected_bytes()
}
}