use crate::matrix::Matrix;
use crate::network::NetworkError;
use std::collections::BTreeMap;
use std::fs::File;
use std::io::{BufReader, Read, Seek, SeekFrom};
use std::path::{Path, PathBuf};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Dtype {
F64,
F32,
F16,
Bf16,
F8E4M3,
F8E5M2,
I64,
I32,
I16,
I8,
U8,
Bool,
}
impl Dtype {
fn parse(name: &str) -> Result<Self, NetworkError> {
Ok(match name {
"F64" => Self::F64,
"F32" => Self::F32,
"F16" => Self::F16,
"BF16" => Self::Bf16,
"F8_E4M3" => Self::F8E4M3,
"F8_E5M2" => Self::F8E5M2,
"I64" => Self::I64,
"I32" => Self::I32,
"I16" => Self::I16,
"I8" => Self::I8,
"U8" => Self::U8,
"BOOL" => Self::Bool,
other => {
return Err(NetworkError::InvalidDataset(format!(
"unknown safetensors dtype {other}"
)));
}
})
}
pub fn size(self) -> usize {
match self {
Self::F64 | Self::I64 => 8,
Self::F32 | Self::I32 => 4,
Self::F16 | Self::Bf16 | Self::I16 => 2,
Self::F8E4M3 | Self::F8E5M2 | Self::I8 | Self::U8 | Self::Bool => 1,
}
}
fn read(self, bytes: &[u8]) -> f32 {
match self {
Self::F64 => f64::from_le_bytes(bytes[..8].try_into().expect("8 bytes")) as f32,
Self::F32 => f32::from_le_bytes(bytes[..4].try_into().expect("4 bytes")),
Self::F16 => f16_to_f32(u16::from_le_bytes(bytes[..2].try_into().expect("2 bytes"))),
Self::Bf16 => f32::from_bits(
(u16::from_le_bytes(bytes[..2].try_into().expect("2 bytes")) as u32) << 16,
),
Self::F8E4M3 => f8_to_f32(bytes[0], 4, 3),
Self::F8E5M2 => f8_to_f32(bytes[0], 5, 2),
Self::I64 => i64::from_le_bytes(bytes[..8].try_into().expect("8 bytes")) as f32,
Self::I32 => i32::from_le_bytes(bytes[..4].try_into().expect("4 bytes")) as f32,
Self::I16 => i16::from_le_bytes(bytes[..2].try_into().expect("2 bytes")) as f32,
Self::I8 => bytes[0] as i8 as f32,
Self::U8 | Self::Bool => bytes[0] as f32,
}
}
}
fn f16_to_f32(bits: u16) -> f32 {
let sign = (bits as u32 & 0x8000) << 16;
let exponent = (bits >> 10) & 0x1f;
let mantissa = bits as u32 & 0x3ff;
let rest = match exponent {
0 => {
if mantissa == 0 {
0
} else {
let shift = mantissa.leading_zeros() - 21;
let exponent = 127 - 15 - shift;
(exponent << 23) | ((mantissa << (shift + 1)) & 0x7f_ffff)
}
}
0x1f => 0x7f80_0000 | (mantissa << 13),
_ => ((exponent as u32 + 127 - 15) << 23) | (mantissa << 13),
};
f32::from_bits(sign | rest)
}
fn f8_to_f32(bits: u8, exponent_bits: u32, mantissa_bits: u32) -> f32 {
let sign = ((bits as u32) & 0x80) << 24;
let exponent = ((bits as u32) >> mantissa_bits) & ((1 << exponent_bits) - 1);
let mantissa = (bits as u32) & ((1 << mantissa_bits) - 1);
let bias = (1 << (exponent_bits - 1)) - 1;
let all_ones = (1 << exponent_bits) - 1;
if exponent == 0 {
if mantissa == 0 {
return f32::from_bits(sign);
}
let scale = (1.0f32 / (1 << mantissa_bits) as f32) * 2.0f32.powi(1 - bias as i32);
let magnitude = mantissa as f32 * scale;
return if sign == 0 { magnitude } else { -magnitude };
}
if exponent == all_ones && (exponent_bits == 5 || mantissa == (1 << mantissa_bits) - 1) {
return if mantissa == 0 && exponent_bits == 5 {
f32::from_bits(sign | 0x7f80_0000)
} else {
f32::NAN
};
}
let shifted_exponent = exponent + 127 - bias;
f32::from_bits(sign | (shifted_exponent << 23) | (mantissa << (23 - mantissa_bits)))
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TensorInfo {
pub dtype: Dtype,
pub shape: Vec<usize>,
pub start: u64,
pub end: u64,
}
impl TensorInfo {
pub fn len(&self) -> usize {
self.shape.iter().product()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
#[derive(Debug)]
pub struct SafeTensors {
file: BufReader<File>,
path: PathBuf,
tensors: BTreeMap<String, TensorInfo>,
data_offset: u64,
metadata: BTreeMap<String, String>,
}
impl SafeTensors {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, NetworkError> {
let path = path.as_ref().to_path_buf();
let mut file = BufReader::new(File::open(&path)?);
let mut length = [0u8; 8];
file.read_exact(&mut length)?;
let length = u64::from_le_bytes(length);
let size = file.get_ref().metadata()?.len();
if length == 0 || length > size.saturating_sub(8) {
return Err(NetworkError::InvalidDataset(format!(
"{}: a {length}-byte safetensors header does not fit in a {size}-byte file",
path.display()
)));
}
let mut header = vec![0u8; length as usize];
file.read_exact(&mut header)?;
let header: serde_json::Value = serde_json::from_slice(&header).map_err(|error| {
NetworkError::InvalidDataset(format!("{}: {error}", path.display()))
})?;
let header = header.as_object().ok_or_else(|| {
NetworkError::InvalidDataset(format!("{}: the header is not an object", path.display()))
})?;
let mut tensors = BTreeMap::new();
let mut metadata = BTreeMap::new();
for (name, value) in header {
if name == "__metadata__" {
for (key, value) in value.as_object().into_iter().flatten() {
metadata.insert(key.clone(), value.as_str().unwrap_or_default().to_string());
}
continue;
}
tensors.insert(name.clone(), Self::parse_tensor(&path, name, value)?);
}
Ok(Self {
file,
path,
tensors,
data_offset: 8 + length,
metadata,
})
}
fn parse_tensor(
path: &Path,
name: &str,
value: &serde_json::Value,
) -> Result<TensorInfo, NetworkError> {
let bad = |what: &str| {
NetworkError::InvalidDataset(format!("{}: {name} has {what}", path.display()))
};
let dtype = Dtype::parse(value["dtype"].as_str().ok_or_else(|| bad("no dtype"))?)?;
let shape: Vec<usize> = value["shape"]
.as_array()
.ok_or_else(|| bad("no shape"))?
.iter()
.map(|dimension| dimension.as_u64().map(|size| size as usize))
.collect::<Option<_>>()
.ok_or_else(|| bad("a shape that is not a list of sizes"))?;
let offsets = value["data_offsets"]
.as_array()
.ok_or_else(|| bad("no data_offsets"))?;
let (start, end) = match offsets.as_slice() {
[start, end] => (
start.as_u64().ok_or_else(|| bad("a bad start offset"))?,
end.as_u64().ok_or_else(|| bad("a bad end offset"))?,
),
_ => return Err(bad("data_offsets that are not a pair")),
};
let info = TensorInfo {
dtype,
shape,
start,
end,
};
if end < start || end - start != (info.len() * dtype.size()) as u64 {
return Err(bad("a byte range that does not match its shape and dtype"));
}
Ok(info)
}
pub fn names(&self) -> impl Iterator<Item = &str> {
self.tensors.keys().map(String::as_str)
}
pub fn info(&self, name: &str) -> Option<&TensorInfo> {
self.tensors.get(name)
}
pub fn metadata(&self) -> &BTreeMap<String, String> {
&self.metadata
}
pub fn tensor(&mut self, name: &str) -> Result<(Vec<f32>, Vec<usize>), NetworkError> {
let info = self.tensors.get(name).cloned().ok_or_else(|| {
NetworkError::InvalidDataset(format!("{}: no tensor named {name}", self.path.display()))
})?;
let mut bytes = vec![0u8; (info.end - info.start) as usize];
self.file
.seek(SeekFrom::Start(self.data_offset + info.start))?;
self.file.read_exact(&mut bytes)?;
let size = info.dtype.size();
let count = bytes.len() / size;
let lanes = match count >= 1 << 20 {
true => std::thread::available_parallelism().map_or(1, |count| count.get()),
false => 1,
};
if lanes == 1 {
let values = bytes
.chunks_exact(size)
.map(|element| info.dtype.read(element))
.collect();
return Ok((values, info.shape));
}
let mut values = vec![0f32; count];
let stride = count.div_ceil(lanes);
let dtype = info.dtype;
std::thread::scope(|scope| {
for (slot, source) in values.chunks_mut(stride).zip(bytes.chunks(stride * size)) {
scope.spawn(move || {
for (target, element) in slot.iter_mut().zip(source.chunks_exact(size)) {
*target = dtype.read(element);
}
});
}
});
Ok((values, info.shape))
}
pub fn matrix(&mut self, name: &str) -> Result<Matrix, NetworkError> {
let (values, shape) = self.tensor(name)?;
match shape.as_slice() {
[cols] => Ok(Matrix::from_vec(1, *cols, values)),
[rows, cols] => Ok(Matrix::from_vec(*rows, *cols, values)),
_ => Err(NetworkError::InvalidDataset(format!(
"{}: {name} has shape {shape:?}, which is not a matrix",
self.path.display()
))),
}
}
}
#[derive(Debug)]
pub struct ShardedSafeTensors {
directory: PathBuf,
map: BTreeMap<String, String>,
shards: BTreeMap<String, SafeTensors>,
}
impl ShardedSafeTensors {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, NetworkError> {
let path = path.as_ref();
let directory = path
.parent()
.unwrap_or_else(|| Path::new("."))
.to_path_buf();
if path
.extension()
.is_some_and(|extension| extension == "safetensors")
{
let name = path
.file_name()
.and_then(|name| name.to_str())
.unwrap_or_default()
.to_string();
let shard = SafeTensors::open(path)?;
let map = shard
.names()
.map(|tensor| (tensor.to_string(), name.clone()))
.collect();
return Ok(Self {
directory,
map,
shards: BTreeMap::from([(name, shard)]),
});
}
let index: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(path)?)
.map_err(|error| {
NetworkError::InvalidDataset(format!("{}: {error}", path.display()))
})?;
let weight_map = index["weight_map"].as_object().ok_or_else(|| {
NetworkError::InvalidDataset(format!(
"{}: an index needs a weight_map of tensor names to files",
path.display()
))
})?;
let map = weight_map
.iter()
.map(|(tensor, file)| {
file.as_str()
.map(|file| (tensor.clone(), file.to_string()))
.ok_or_else(|| {
NetworkError::InvalidDataset(format!(
"{}: {tensor} does not name a file",
path.display()
))
})
})
.collect::<Result<_, NetworkError>>()?;
Ok(Self {
directory,
map,
shards: BTreeMap::new(),
})
}
pub fn names(&self) -> impl Iterator<Item = &str> {
self.map.keys().map(String::as_str)
}
pub fn tensor(&mut self, name: &str) -> Result<(Vec<f32>, Vec<usize>), NetworkError> {
let file = self
.map
.get(name)
.ok_or_else(|| {
NetworkError::InvalidDataset(format!("no tensor named {name} in the index"))
})?
.clone();
if !self.shards.contains_key(&file) {
let shard = SafeTensors::open(self.directory.join(&file))?;
self.shards.insert(file.clone(), shard);
}
self.shards
.get_mut(&file)
.expect("the shard was just opened")
.tensor(name)
}
pub fn matrix(&mut self, name: &str) -> Result<Matrix, NetworkError> {
let file = self
.map
.get(name)
.ok_or_else(|| {
NetworkError::InvalidDataset(format!("no tensor named {name} in the index"))
})?
.clone();
if !self.shards.contains_key(&file) {
let shard = SafeTensors::open(self.directory.join(&file))?;
self.shards.insert(file.clone(), shard);
}
self.shards
.get_mut(&file)
.expect("the shard was just opened")
.matrix(name)
}
}
#[cfg(test)]
pub(crate) fn write_checkpoint(
path: &std::path::Path,
tensors: &BTreeMap<String, (Vec<usize>, Vec<f32>)>,
) {
let mut header = String::from("{");
let mut body = Vec::new();
for (name, (shape, values)) in tensors {
let start = body.len();
for value in values {
body.extend_from_slice(&value.to_le_bytes());
}
if header.len() > 1 {
header.push(',');
}
header.push_str(&format!(
"\"{name}\":{{\"dtype\":\"F32\",\"shape\":{shape:?},\"data_offsets\":[{start},{}]}}",
body.len()
));
}
header.push('}');
let mut file = (header.len() as u64).to_le_bytes().to_vec();
file.extend_from_slice(header.as_bytes());
file.extend_from_slice(&body);
std::fs::write(path, file).unwrap();
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
fn write(path: &Path, header: &str, data: &[u8]) {
let mut file = File::create(path).unwrap();
file.write_all(&(header.len() as u64).to_le_bytes())
.unwrap();
file.write_all(header.as_bytes()).unwrap();
file.write_all(data).unwrap();
}
fn scratch(name: &str) -> PathBuf {
std::env::temp_dir().join(format!("rb_{name}_{}", std::process::id()))
}
#[test]
fn a_float_tensor_reads_back_with_its_shape() {
let path = scratch("st_f32.safetensors");
let values: Vec<f32> = vec![1.0, -2.0, 0.5, 4.25, -0.125, 8.0];
let data: Vec<u8> = values.iter().flat_map(|v| v.to_le_bytes()).collect();
write(
&path,
r#"{"__metadata__":{"format":"pt"},"weight":{"dtype":"F32","shape":[2,3],"data_offsets":[0,24]}}"#,
&data,
);
let mut file = SafeTensors::open(&path).unwrap();
assert_eq!(file.names().collect::<Vec<_>>(), ["weight"]);
assert_eq!(file.metadata()["format"], "pt");
assert_eq!(file.info("weight").unwrap().shape, vec![2, 3]);
let matrix = file.matrix("weight").unwrap();
std::fs::remove_file(&path).ok();
assert_eq!((matrix.rows, matrix.cols), (2, 3));
assert_eq!(matrix.data, values);
}
#[test]
fn half_precision_dtypes_arrive_as_f32() {
let path = scratch("st_half.safetensors");
let halves: [u16; 3] = [0x3c00, 0xc000, 0x0400];
let bf16: [u16; 3] = [0x3f80, 0xc000, 0x3f00];
let data: Vec<u8> = halves
.iter()
.chain(&bf16)
.flat_map(|bits| bits.to_le_bytes())
.collect();
write(
&path,
r#"{"half":{"dtype":"F16","shape":[3],"data_offsets":[0,6]},"brain":{"dtype":"BF16","shape":[3],"data_offsets":[6,12]}}"#,
&data,
);
let mut file = SafeTensors::open(&path).unwrap();
let (half, _) = file.tensor("half").unwrap();
let (brain, _) = file.tensor("brain").unwrap();
std::fs::remove_file(&path).ok();
assert_eq!(half, vec![1.0, -2.0, 2.0f32.powi(-14)]);
assert_eq!(brain, vec![1.0, -2.0, 0.5]);
}
#[test]
fn eight_bit_floats_and_integers_arrive_as_f32() {
let path = scratch("st_f8.safetensors");
let data = vec![0x38u8, 0x40, 0xb8, 0x3c, 0x40, 0xbc, 0xff, 0x7f, 0x01];
write(
&path,
r#"{"e4m3":{"dtype":"F8_E4M3","shape":[3],"data_offsets":[0,3]},"e5m2":{"dtype":"F8_E5M2","shape":[3],"data_offsets":[3,6]},"ints":{"dtype":"I8","shape":[3],"data_offsets":[6,9]}}"#,
&data,
);
let mut file = SafeTensors::open(&path).unwrap();
let (e4m3, _) = file.tensor("e4m3").unwrap();
let (e5m2, _) = file.tensor("e5m2").unwrap();
let (ints, _) = file.tensor("ints").unwrap();
std::fs::remove_file(&path).ok();
assert_eq!(e4m3, vec![1.0, 2.0, -1.0]);
assert_eq!(f8_to_f32(0x01, 4, 3), 2.0f32.powi(-9));
assert_eq!(f8_to_f32(0x81, 4, 3), -2.0f32.powi(-9));
assert_eq!(e5m2, vec![1.0, 2.0, -1.0]);
assert_eq!(ints, vec![-1.0, 127.0, 1.0]);
}
#[test]
fn a_header_that_lies_about_a_tensor_is_refused() {
let path = scratch("st_bad.safetensors");
write(
&path,
r#"{"weight":{"dtype":"F32","shape":[2,3],"data_offsets":[0,8]}}"#,
&[0u8; 8],
);
let error = SafeTensors::open(&path).unwrap_err().to_string();
assert!(error.contains("byte range"), "{error}");
let mut file = File::create(&path).unwrap();
file.write_all(&u64::MAX.to_le_bytes()).unwrap();
file.write_all(b"{}").unwrap();
drop(file);
assert!(SafeTensors::open(&path).is_err());
std::fs::remove_file(&path).ok();
}
#[test]
fn a_three_dimensional_tensor_is_not_a_matrix_but_still_reads() {
let path = scratch("st_conv.safetensors");
let data: Vec<u8> = (0..8).flat_map(|v| (v as f32).to_le_bytes()).collect();
write(
&path,
r#"{"conv":{"dtype":"F32","shape":[2,2,2],"data_offsets":[0,32]}}"#,
&data,
);
let mut file = SafeTensors::open(&path).unwrap();
let (values, shape) = file.tensor("conv").unwrap();
let error = file.matrix("conv").unwrap_err().to_string();
std::fs::remove_file(&path).ok();
assert_eq!(shape, vec![2, 2, 2]);
assert_eq!(values.len(), 8);
assert!(error.contains("not a matrix"), "{error}");
}
#[test]
fn a_sharded_model_reads_through_its_index() {
let directory = scratch("st_shards");
std::fs::create_dir_all(&directory).unwrap();
let first: Vec<u8> = [1.0f32, 2.0].iter().flat_map(|v| v.to_le_bytes()).collect();
let second: Vec<u8> = [3.0f32, 4.0].iter().flat_map(|v| v.to_le_bytes()).collect();
write(
&directory.join("model-00001-of-00002.safetensors"),
r#"{"a.weight":{"dtype":"F32","shape":[2],"data_offsets":[0,8]}}"#,
&first,
);
write(
&directory.join("model-00002-of-00002.safetensors"),
r#"{"b.weight":{"dtype":"F32","shape":[1,2],"data_offsets":[0,8]}}"#,
&second,
);
std::fs::write(
directory.join("model.safetensors.index.json"),
r#"{"metadata":{"total_size":16},"weight_map":{"a.weight":"model-00001-of-00002.safetensors","b.weight":"model-00002-of-00002.safetensors"}}"#,
)
.unwrap();
let mut model =
ShardedSafeTensors::open(directory.join("model.safetensors.index.json")).unwrap();
assert_eq!(model.names().collect::<Vec<_>>(), ["a.weight", "b.weight"]);
assert_eq!(model.tensor("a.weight").unwrap().0, vec![1.0, 2.0]);
assert_eq!(model.matrix("b.weight").unwrap().data, vec![3.0, 4.0]);
let mut single =
ShardedSafeTensors::open(directory.join("model-00001-of-00002.safetensors")).unwrap();
assert_eq!(single.tensor("a.weight").unwrap().0, vec![1.0, 2.0]);
std::fs::remove_dir_all(&directory).ok();
}
}