use std::fs::{File, OpenOptions};
use std::io::{ErrorKind, Read};
use std::os::unix::fs::{FileExt, OpenOptionsExt};
use std::path::Path;
use memmap2::{Mmap, MmapOptions};
use crate::format::{COMMON_FILE, CommonTensor, FormatError, Manifest, sha256_bytes};
use super::direct::{DIO_ALIGN, O_DIRECT};
use super::{CommonMemory, IoError, LoadOptions, to_usize, verify_named_file};
const RESIDENT_CHUNK: usize = 16 << 20;
#[derive(Debug)]
pub struct MappedCommon {
map: Mmap,
len: usize,
memory: CommonMemory,
}
impl MappedCommon {
pub fn open(dir: &Path, manifest: &Manifest, options: LoadOptions) -> Result<Self, IoError> {
let resident = options.common_memory == CommonMemory::Resident;
let size_only = LoadOptions {
skip_hashes: true,
..options
};
let verify = if resident { size_only } else { options };
verify_named_file(dir, manifest, COMMON_FILE, verify)?;
let path = dir.join(COMMON_FILE);
let file = File::open(&path).map_err(|e| IoError::io(&path, e))?;
let len = file.metadata().map_err(|e| IoError::io(&path, e))?.len();
let len = to_usize(len, "common.bin length")?;
let map = match options.common_memory {
CommonMemory::Mapped => unsafe { Mmap::map(&file) },
CommonMemory::Resident => read_resident(&path, &file, len),
}
.map_err(|e| IoError::io(&path, e))?;
if resident && !options.skip_hashes {
let entry = &manifest.files[COMMON_FILE];
let actual = sha256_bytes(&map[..len]);
if !actual.eq_ignore_ascii_case(&entry.sha256) {
return Err(FormatError::HashMismatch {
name: COMMON_FILE.to_owned(),
expected: entry.sha256.clone(),
actual,
}
.into());
}
}
tracing::debug!(
path = %path.display(),
bytes = len,
memory = ?options.common_memory,
"loaded common weights"
);
Ok(Self {
map,
len,
memory: options.common_memory,
})
}
pub fn bytes(&self) -> &[u8] {
&self.map[..self.len]
}
pub fn len(&self) -> usize {
self.len
}
pub fn is_empty(&self) -> bool {
self.len == 0
}
pub fn memory(&self) -> CommonMemory {
self.memory
}
pub fn tensor(&self, name: &str, entry: &CommonTensor) -> Result<&[u8], IoError> {
let offset = to_usize(entry.offset, "common tensor offset")?;
let len = to_usize(entry.len, "common tensor len")?;
self.bytes()
.get(offset..offset.saturating_add(len))
.ok_or_else(|| IoError::RangeOutOfBounds {
what: format!("common tensor {name:?}"),
offset: entry.offset,
len: entry.len,
available: self.len as u64,
})
}
}
fn read_resident(path: &Path, buffered: &File, len: usize) -> std::io::Result<Mmap> {
let align = DIO_ALIGN as usize;
let rounded = len.div_ceil(align).max(1) * align;
let mut region = MmapOptions::new().len(rounded).map_anon()?;
let direct = O_DIRECT.and_then(|flag| {
OpenOptions::new()
.read(true)
.custom_flags(flag)
.open(path)
.ok()
});
let mut filled = 0usize;
if let Some(file) = direct {
while filled < len {
let end = (filled + RESIDENT_CHUNK).min(rounded);
match file.read_at(&mut region[filled..end], filled as u64) {
Ok(0) => break,
Ok(n) => {
drop_cached(buffered, filled, n);
filled += n;
}
Err(e) if e.kind() == ErrorKind::Interrupted => {}
Err(e) if e.raw_os_error() == Some(libc::EINVAL) => break,
Err(e) => return Err(e),
}
}
}
while filled < len {
let end = (filled + RESIDENT_CHUNK).min(len);
let mut file = buffered;
std::io::Seek::seek(&mut file, std::io::SeekFrom::Start(filled as u64))?;
file.read_exact(&mut region[filled..end])?;
drop_cached(buffered, filled, end - filled);
filled = end;
}
region.make_read_only()
}
fn drop_cached(file: &File, offset: usize, len: usize) {
unsafe {
libc::posix_fadvise(
std::os::fd::AsRawFd::as_raw_fd(file),
offset as libc::off_t,
len as libc::off_t,
libc::POSIX_FADV_DONTNEED,
);
}
}
#[cfg(test)]
mod tests {
use super::super::testutil::build_install;
use super::*;
use crate::format::FormatError;
#[test]
fn maps_and_slices_common() {
let fx = build_install("mapped-common");
let common = MappedCommon::open(&fx.root, &fx.manifest, LoadOptions::default()).unwrap();
assert_eq!(common.len() as u64, fx.manifest.files[COMMON_FILE].size);
assert!(!common.is_empty());
let entry = &fx.manifest.common_tensors["output_norm.weight"];
let bytes = common.tensor("output_norm.weight", entry).unwrap();
assert_eq!(bytes.len() as u64, entry.len);
let whole = common.bytes();
let offset = entry.offset as usize;
assert_eq!(&whole[offset..offset + bytes.len()], bytes);
}
#[test]
fn resident_and_mapped_hold_the_same_bytes() {
let fx = build_install("resident-common");
let mapped = MappedCommon::open(
&fx.root,
&fx.manifest,
LoadOptions {
common_memory: CommonMemory::Mapped,
..LoadOptions::default()
},
)
.unwrap();
let resident = MappedCommon::open(&fx.root, &fx.manifest, LoadOptions::default()).unwrap();
assert_eq!(resident.memory(), CommonMemory::Resident);
assert_eq!(mapped.memory(), CommonMemory::Mapped);
assert_eq!(resident.len(), mapped.len());
assert_eq!(resident.bytes(), mapped.bytes());
for (name, entry) in &fx.manifest.common_tensors {
assert_eq!(
resident.tensor(name, entry).unwrap(),
mapped.tensor(name, entry).unwrap(),
"{name}"
);
}
}
#[test]
fn a_resident_load_still_rejects_a_corrupt_file() {
let fx = build_install("resident-corrupt");
let path = fx.root.join(COMMON_FILE);
let mut data = std::fs::read(&path).unwrap();
data[10] ^= 0xff;
std::fs::write(&path, data).unwrap();
let resident = LoadOptions {
common_memory: CommonMemory::Resident,
..LoadOptions::default()
};
let err = MappedCommon::open(&fx.root, &fx.manifest, resident).unwrap_err();
assert!(matches!(
err,
IoError::Format(FormatError::HashMismatch { ref name, .. }) if name == COMMON_FILE
));
MappedCommon::open(
&fx.root,
&fx.manifest,
LoadOptions {
skip_hashes: true,
..resident
},
)
.unwrap();
}
#[test]
fn read_resident_matches_a_plain_read_at_awkward_lengths() {
let dir = std::env::current_dir()
.unwrap()
.join("target")
.join("resident-read-test");
std::fs::create_dir_all(&dir).unwrap();
for len in [1usize, 4095, 4096, 4097, (16 << 20) + 513] {
let path = dir.join(format!("blob-{len}"));
let data: Vec<u8> = (0..len).map(|i| (i * 31 % 251) as u8).collect();
std::fs::write(&path, &data).unwrap();
let file = File::open(&path).unwrap();
let region = read_resident(&path, &file, len).unwrap();
assert_eq!(®ion[..len], &data[..], "len {len}");
std::fs::remove_file(&path).unwrap();
}
}
#[test]
fn rejects_corrupt_common_unless_hashes_skipped() {
let fx = build_install("mapped-corrupt");
let path = fx.root.join(COMMON_FILE);
let mut data = std::fs::read(&path).unwrap();
data[10] ^= 0xff;
std::fs::write(&path, data).unwrap();
let err = MappedCommon::open(&fx.root, &fx.manifest, LoadOptions::default()).unwrap_err();
assert!(matches!(
err,
IoError::Format(FormatError::HashMismatch { name, .. }) if name == COMMON_FILE
));
MappedCommon::open(
&fx.root,
&fx.manifest,
LoadOptions {
skip_hashes: true,
..LoadOptions::default()
},
)
.unwrap();
}
#[test]
fn size_check_survives_skip_hashes() {
let fx = build_install("mapped-truncated");
let path = fx.root.join(COMMON_FILE);
let data = std::fs::read(&path).unwrap();
std::fs::write(&path, &data[..data.len() - 64]).unwrap();
let err = MappedCommon::open(
&fx.root,
&fx.manifest,
LoadOptions {
skip_hashes: true,
..LoadOptions::default()
},
)
.unwrap_err();
assert!(matches!(
err,
IoError::Format(FormatError::SizeMismatch { name, .. }) if name == COMMON_FILE
));
}
#[test]
fn tensor_range_is_bounds_checked() {
let fx = build_install("mapped-bounds");
let common = MappedCommon::open(&fx.root, &fx.manifest, LoadOptions::default()).unwrap();
let bogus = CommonTensor {
offset: common.len() as u64,
len: 64,
dtype: "f32".to_owned(),
};
assert!(matches!(
common.tensor("bogus", &bogus).unwrap_err(),
IoError::RangeOutOfBounds { .. }
));
}
}