use crate::network::NetworkError;
use std::io::{BufReader, Read};
use std::path::Path;
const BLOCK: usize = 512;
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Sample {
pub key: String,
pub files: Vec<(String, Vec<u8>)>,
}
impl Sample {
pub fn get(&self, extension: &str) -> Option<&[u8]> {
self.files
.iter()
.find(|(ext, _)| ext == extension)
.map(|(_, bytes)| bytes.as_slice())
}
}
pub struct WebDataset<R: Read> {
reader: R,
pending: Option<(String, String, Vec<u8>)>,
done: bool,
}
impl WebDataset<BufReader<std::fs::File>> {
pub fn open(path: impl AsRef<Path>) -> Result<Self, NetworkError> {
Ok(Self::new(BufReader::new(std::fs::File::open(path)?)))
}
}
impl<R: Read> WebDataset<R> {
pub fn new(reader: R) -> Self {
Self {
reader,
pending: None,
done: false,
}
}
fn next_member(&mut self) -> Result<Option<(String, String, Vec<u8>)>, NetworkError> {
let mut long_name: Option<String> = None;
loop {
let mut header = [0u8; BLOCK];
if !read_block(&mut self.reader, &mut header)? || header.iter().all(|&b| b == 0) {
return Ok(None);
}
check_sum(&header)?;
let size = octal(&header[124..136])?;
let data = read_data(&mut self.reader, size)?;
let name = match long_name.take() {
Some(name) => name,
None => header_name(&header),
};
match header[156] {
b'0' | 0 | b'7' => {}
b'L' => {
long_name = Some(c_string(&data));
continue;
}
b'x' => {
long_name = pax_path(&data)?;
continue;
}
b'S' => return Err(invalid(format!("{name} is a sparse file"))),
_ => continue,
}
let (directory, file) = name.rsplit_once('/').unwrap_or(("", &name));
let Some((stem, extension)) = file.split_once('.') else {
continue;
};
let key = if directory.is_empty() {
stem.to_string()
} else {
format!("{directory}/{stem}")
};
return Ok(Some((key, extension.to_string(), data)));
}
}
fn next_sample(&mut self) -> Result<Option<Sample>, NetworkError> {
let first = match self.pending.take() {
Some(member) => Some(member),
None => self.next_member()?,
};
let Some((key, extension, data)) = first else {
return Ok(None);
};
let mut sample = Sample {
key,
files: vec![(extension, data)],
};
while let Some(member) = self.next_member()? {
if member.0 != sample.key {
self.pending = Some(member);
break;
}
sample.files.push((member.1, member.2));
}
Ok(Some(sample))
}
}
impl<R: Read> Iterator for WebDataset<R> {
type Item = Result<Sample, NetworkError>;
fn next(&mut self) -> Option<Self::Item> {
if self.done {
return None;
}
let next = self.next_sample();
self.done = !matches!(next, Ok(Some(_)));
next.transpose()
}
}
fn invalid(message: String) -> NetworkError {
NetworkError::InvalidDataset(format!("tar: {message}"))
}
fn read_block(reader: &mut impl Read, block: &mut [u8; BLOCK]) -> Result<bool, NetworkError> {
let mut filled = 0;
while filled < BLOCK {
match reader.read(&mut block[filled..])? {
0 if filled == 0 => return Ok(false),
0 => return Err(invalid("the file ends inside a header".into())),
n => filled += n,
}
}
Ok(true)
}
fn read_data(reader: &mut impl Read, size: u64) -> Result<Vec<u8>, NetworkError> {
let padded = size.div_ceil(BLOCK as u64) * BLOCK as u64;
let mut data = Vec::new();
reader.take(padded).read_to_end(&mut data)?;
if (data.len() as u64) < padded {
return Err(invalid("the file ends inside a member".into()));
}
data.truncate(size as usize);
Ok(data)
}
fn check_sum(header: &[u8; BLOCK]) -> Result<(), NetworkError> {
let stored = octal(&header[148..156])?;
let sum: u64 = header
.iter()
.enumerate()
.map(|(i, &b)| {
if (148..156).contains(&i) {
32
} else {
b as u64
}
})
.sum();
if stored != sum {
return Err(invalid(format!(
"header checksum {stored} does not match {sum}"
)));
}
Ok(())
}
fn octal(field: &[u8]) -> Result<u64, NetworkError> {
if field[0] & 0x80 != 0 {
return Err(invalid("base-256 numbers are not supported".into()));
}
let digits = c_string(field);
let digits = digits.trim_matches(' ');
if digits.is_empty() {
return Ok(0);
}
u64::from_str_radix(digits, 8).map_err(|_| invalid(format!("bad octal field {digits:?}")))
}
fn c_string(bytes: &[u8]) -> String {
let end = bytes.iter().position(|&b| b == 0).unwrap_or(bytes.len());
String::from_utf8_lossy(&bytes[..end]).into_owned()
}
fn header_name(header: &[u8; BLOCK]) -> String {
let name = c_string(&header[..100]);
let prefix = if &header[257..262] == b"ustar" {
c_string(&header[345..500])
} else {
String::new()
};
if prefix.is_empty() {
name
} else {
format!("{prefix}/{name}")
}
}
fn pax_path(data: &[u8]) -> Result<Option<String>, NetworkError> {
let mut rest = data;
let mut path = None;
while !rest.is_empty() {
let space = rest
.iter()
.position(|&b| b == b' ')
.ok_or_else(|| invalid("bad pax record".into()))?;
let length: usize = std::str::from_utf8(&rest[..space])
.ok()
.and_then(|digits| digits.parse().ok())
.filter(|&length| length > space && length <= rest.len())
.ok_or_else(|| invalid("bad pax record length".into()))?;
let record = &rest[space + 1..length];
if let Some(value) = record.strip_prefix(b"path=") {
let value = value.strip_suffix(b"\n").unwrap_or(value);
path = Some(String::from_utf8_lossy(value).into_owned());
}
rest = &rest[length..];
}
Ok(path)
}
#[cfg(test)]
pub(crate) mod tests {
use super::*;
fn member(out: &mut Vec<u8>, name: &str, kind: u8, data: &[u8]) {
let mut header = [0u8; BLOCK];
header[..name.len()].copy_from_slice(name.as_bytes());
header[100..108].copy_from_slice(b"0000644\0");
header[124..136].copy_from_slice(format!("{:011o}\0", data.len()).as_bytes());
header[156] = kind;
header[257..263].copy_from_slice(b"ustar\0");
header[148..156].fill(b' ');
let sum: u32 = header.iter().map(|&b| b as u32).sum();
header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes());
out.extend_from_slice(&header);
out.extend_from_slice(data);
out.resize(out.len().div_ceil(BLOCK) * BLOCK, 0);
}
pub(crate) fn shard() -> Vec<u8> {
let mut tar = Vec::new();
member(&mut tar, "train/", b'5', b"");
member(&mut tar, "train/000.jpg", b'0', &[0xFF, 0xD8, 0xFF]);
member(&mut tar, "train/000.txt", b'0', b"a red cube");
member(&mut tar, "README", b'0', b"not a sample");
let long = format!("train/{}.txt", "x".repeat(120));
member(
&mut tar,
"././@LongLink",
b'L',
format!("{long}\0").as_bytes(),
);
member(&mut tar, "truncated", b'0', b"long name");
let record = "path=train/001.cls\n";
let pax = format!("{} {record}", record.len() + 3);
member(&mut tar, "PaxHeader", b'x', pax.as_bytes());
member(&mut tar, "short", b'0', b"7");
member(&mut tar, "train/001.json", b'0', b"{}");
tar.resize(tar.len() + 2 * BLOCK, 0);
tar
}
#[test]
fn members_group_into_samples_by_key() {
let samples: Vec<Sample> = WebDataset::new(shard().as_slice())
.collect::<Result<_, _>>()
.unwrap();
let keys: Vec<&str> = samples.iter().map(|s| s.key.as_str()).collect();
let long = format!("train/{}", "x".repeat(120));
assert_eq!(keys, ["train/000", long.as_str(), "train/001"]);
assert_eq!(samples[0].get("jpg"), Some(&[0xFF, 0xD8, 0xFF][..]));
assert_eq!(samples[0].get("txt"), Some(&b"a red cube"[..]));
assert_eq!(samples[1].get("txt"), Some(&b"long name"[..]));
assert_eq!(samples[2].get("cls"), Some(&b"7"[..]));
assert_eq!(samples[2].get("json"), Some(&b"{}"[..]));
}
#[test]
fn a_truncated_or_corrupt_shard_is_an_error() {
let tar = shard();
let cut: Vec<_> = WebDataset::new(&tar[..BLOCK * 3 + 100]).collect();
assert!(cut.last().unwrap().is_err());
let mut bad = tar.clone();
bad[BLOCK + 130] = b'7';
let read: Vec<_> = WebDataset::new(bad.as_slice()).collect();
assert!(read.last().unwrap().is_err());
let mut huge = tar;
huge[BLOCK + 124..BLOCK + 136].copy_from_slice(b"77777777777\0");
huge[BLOCK + 148..BLOCK + 156].fill(b' ');
let sum: u32 = huge[BLOCK..2 * BLOCK].iter().map(|&b| b as u32).sum();
huge[BLOCK + 148..BLOCK + 156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes());
let read: Vec<_> = WebDataset::new(huge.as_slice()).collect();
assert!(read.last().unwrap().is_err());
}
}