use std::io::Read;
use std::path::{Path, PathBuf};
use base64::Engine;
use serde_json::{json, Value};
const MIB: u64 = 1024 * 1024;
pub const MULTIPART_THRESHOLD: u64 = 64 * MIB;
pub const DEFAULT_PART_SIZE: u64 = 32 * MIB;
const MAX_PARTS: u64 = 10_000;
const HEADER_LIMIT: usize = 1 << 20;
#[derive(Debug, PartialEq, Eq)]
pub enum DatasetArg {
Ref(String),
File(PathBuf),
}
pub fn classify_dataset(value: &str) -> DatasetArg {
if value.contains("://") {
return DatasetArg::Ref(value.to_string());
}
let path = Path::new(value);
if path.is_file() {
DatasetArg::File(path.to_path_buf())
} else {
DatasetArg::Ref(value.to_string())
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FilePlan {
pub name: String,
pub path: String,
pub size: u64,
pub sha256: String,
pub part_size: Option<u64>,
pub part_sha256s: Vec<String>,
pub tabular: Option<(u64, Vec<String>)>,
}
pub fn part_size_for(size: u64, threshold: u64, default_part: u64) -> Option<u64> {
if size <= threshold {
return None;
}
if size.div_ceil(default_part) <= MAX_PARTS {
return Some(default_part);
}
Some(size.div_ceil(MAX_PARTS).div_ceil(MIB) * MIB)
}
pub fn part_ranges(size: u64, part_size: u64) -> Vec<(u64, u64, u64)> {
let mut out = Vec::new();
let mut offset = 0;
while offset < size {
let length = part_size.min(size - offset);
out.push((out.len() as u64 + 1, offset, length));
offset += length;
}
out
}
pub fn dataset_name(file_name: &str, sha256: &str) -> String {
let stem = Path::new(file_name)
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("");
let mut slug = String::new();
for c in stem.chars() {
if c.is_ascii_alphanumeric() || matches!(c, '_' | '.') {
slug.push(c);
} else if !slug.ends_with('-') {
slug.push('-');
}
}
let slug: String = slug.trim_matches(['-', '.']).chars().take(96).collect();
let slug = if slug.is_empty() { "dataset" } else { &slug };
format!("{slug}-{}", &sha256[..sha256.len().min(12)])
}
pub fn checksum_header(sha256_hex: &str) -> Result<String, String> {
let raw = hex::decode(sha256_hex).map_err(|_| "a digest is not hex".to_string())?;
Ok(base64::engine::general_purpose::STANDARD.encode(raw))
}
#[derive(Default)]
pub struct TabularScanner {
first: Option<Vec<u8>>,
line: Vec<u8>,
too_long: bool,
content: bool,
lines: u64,
binary: bool,
any: bool,
}
impl TabularScanner {
pub fn update(&mut self, chunk: &[u8]) {
if self.binary {
return;
}
for &b in chunk {
self.any = true;
match b {
0 => {
self.binary = true;
return;
}
b'\n' => self.end_line(),
_ => {
if self.first.is_none() && !self.too_long {
if self.line.len() < HEADER_LIMIT {
self.line.push(b);
} else {
self.too_long = true;
}
}
if !matches!(b, b'\r' | b' ' | b'\t') {
self.content = true;
}
}
}
}
}
fn end_line(&mut self) {
if self.first.is_none() {
self.first = Some(std::mem::take(&mut self.line));
}
if self.content {
self.lines += 1;
}
self.content = false;
}
pub fn finish(mut self) -> Option<(u64, Vec<String>)> {
if self.binary || !self.any {
return None;
}
if self.content {
self.lines += 1;
}
let first = self.first.take().unwrap_or(self.line);
if self.too_long {
return None;
}
let jsonl = first.iter().find(|b| !matches!(b, b' ' | b'\t')) == Some(&b'{');
let columns = header_columns(&first)?;
let records = self.lines.saturating_sub(u64::from(!jsonl));
Some((records, columns))
}
}
struct KeysInOrder(Vec<String>);
impl<'de> serde::Deserialize<'de> for KeysInOrder {
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
struct V;
impl<'de> serde::de::Visitor<'de> for V {
type Value = KeysInOrder;
fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
f.write_str("a JSON object")
}
fn visit_map<A: serde::de::MapAccess<'de>>(
self,
mut map: A,
) -> Result<KeysInOrder, A::Error> {
let mut keys: Vec<String> = Vec::new();
while let Some(k) = map.next_key::<String>()? {
map.next_value::<serde::de::IgnoredAny>()?;
if !keys.contains(&k) {
keys.push(k);
}
}
Ok(KeysInOrder(keys))
}
}
d.deserialize_map(V)
}
}
pub fn header_columns(first_line: &[u8]) -> Option<Vec<String>> {
let line = std::str::from_utf8(first_line).ok()?;
let line = line.trim_start_matches('\u{feff}');
let line = line.strip_suffix('\r').unwrap_or(line);
if line.trim().is_empty() {
return None;
}
let lead = line.trim_start();
if lead.starts_with('{') {
let keys = serde_json::from_str::<KeysInOrder>(line).ok()?.0;
return (!keys.is_empty()).then_some(keys);
}
if lead.starts_with('[') {
return None;
}
split_csv(line)
}
fn split_csv(line: &str) -> Option<Vec<String>> {
let mut out = Vec::new();
let mut field = String::new();
let mut quoted = false;
let mut chars = line.chars().peekable();
while let Some(c) = chars.next() {
if quoted {
if c == '"' {
if chars.peek() == Some(&'"') {
field.push('"');
chars.next();
} else {
quoted = false;
}
} else {
field.push(c);
}
} else if c == '"' {
quoted = true;
} else if c == ',' {
out.push(field.trim().to_string());
field.clear();
} else {
field.push(c);
}
}
out.push(field.trim().to_string());
out.iter().any(|c| !c.is_empty()).then_some(out)
}
pub fn plan_file(path: &Path) -> Result<FilePlan, String> {
plan_file_with(path, MULTIPART_THRESHOLD, DEFAULT_PART_SIZE)
}
pub fn plan_file_with(path: &Path, threshold: u64, default_part: u64) -> Result<FilePlan, String> {
let shown = path.display();
let file_name = path
.file_name()
.and_then(|n| n.to_str())
.ok_or_else(|| format!("{shown}: the file name is not valid UTF-8"))?
.to_string();
if file_name.contains('\\') || file_name.len() > 512 {
return Err(format!(
"{shown}: the hub does not take this file name (a backslash, or over 512 bytes)"
));
}
let mut file = std::fs::File::open(path).map_err(|e| format!("{shown}: {e}"))?;
let size = file.metadata().map_err(|e| format!("{shown}: {e}"))?.len();
if size == 0 {
return Err(format!("{shown}: the file is empty"));
}
let part_size = part_size_for(size, threshold, default_part);
let mut whole = ring::digest::Context::new(&ring::digest::SHA256);
let mut part = ring::digest::Context::new(&ring::digest::SHA256);
let mut in_part = 0u64;
let mut part_sha256s = Vec::new();
let mut scanner = TabularScanner::default();
let mut buf = vec![0u8; 1 << 16];
let mut total = 0u64;
loop {
let n = file.read(&mut buf).map_err(|e| format!("{shown}: {e}"))?;
if n == 0 {
break;
}
total += n as u64;
whole.update(&buf[..n]);
scanner.update(&buf[..n]);
let mut rest = &buf[..n];
while let (Some(ps), false) = (part_size, rest.is_empty()) {
let take = ((ps - in_part) as usize).min(rest.len());
part.update(&rest[..take]);
in_part += take as u64;
rest = &rest[take..];
if in_part == ps {
let done =
std::mem::replace(&mut part, ring::digest::Context::new(&ring::digest::SHA256));
part_sha256s.push(hex::encode(done.finish()));
in_part = 0;
}
}
}
if total != size {
return Err(format!("{shown}: the file changed while it was read"));
}
if part_size.is_some() && in_part > 0 {
part_sha256s.push(hex::encode(part.finish()));
}
let sha256 = hex::encode(whole.finish());
Ok(FilePlan {
name: dataset_name(&file_name, &sha256),
path: file_name,
size,
sha256,
part_size,
part_sha256s,
tabular: scanner.finish(),
})
}
pub fn open_body(plan: &FilePlan) -> Value {
let mut file = json!({
"path": plan.path,
"sha256": plan.sha256,
"size_bytes": plan.size,
});
if let Some(part_size) = plan.part_size {
file["part_size"] = json!(part_size);
file["parts"] = plan
.part_sha256s
.iter()
.enumerate()
.map(|(i, sha)| json!({"part_number": i + 1, "sha256": sha}))
.collect();
}
if let Some((records, columns)) = &plan.tabular {
file["meta"] = json!({"records": records, "columns": columns});
}
json!({"name": plan.name, "visibility": "private", "files": [file]})
}
#[derive(Debug, PartialEq, Eq)]
pub enum Target {
Landed,
Single(String),
Parts(Vec<(u64, String)>),
}
#[derive(Debug, PartialEq, Eq)]
pub struct Opened {
pub upload_id: String,
pub dataset_id: String,
pub target: Target,
}
fn plain_id(v: &Value, key: &str) -> Result<String, String> {
let id = v.get(key).and_then(Value::as_str).unwrap_or("");
if id.is_empty() || !id.chars().all(|c| c.is_ascii_alphanumeric() || c == '-') {
return Err(format!("the hub's answer has no usable {key}"));
}
Ok(id.to_string())
}
pub fn parse_open(answer: &Value, plan: &FilePlan) -> Result<Opened, String> {
let upload_id = plain_id(answer, "upload_id")?;
let dataset_id = plain_id(answer, "dataset_id")?;
let entry = answer
.get("files")
.and_then(Value::as_array)
.and_then(|files| {
files
.iter()
.find(|f| f.get("path").and_then(Value::as_str) == Some(plan.path.as_str()))
});
let target = match entry {
None => Target::Landed,
Some(f) => {
if let Some(url) = f.get("url").and_then(Value::as_str) {
Target::Single(url.to_string())
} else {
let parts: Vec<(u64, String)> = f
.get("parts")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(|p| {
Some((
p.get("part_number")?.as_u64()?,
p.get("url")?.as_str()?.to_string(),
))
})
.collect();
let wanted: Vec<u64> = (1..=plan.part_sha256s.len() as u64).collect();
let offered: Vec<u64> = parts.iter().map(|(n, _)| *n).collect();
if plan.part_size.is_none()
|| f.get("part_size").and_then(Value::as_u64) != plan.part_size
|| offered != wanted
{
return Err(format!(
"the hub holds an unfinished upload of these bytes that this command \
cannot resume (parts offered: {} of {}); finish or remove it on the \
hub's Datasets page, or retry after it expires (24 h)",
offered.len(),
wanted.len()
));
}
Target::Parts(parts)
}
}
};
Ok(Opened {
upload_id,
dataset_id,
target,
})
}
pub fn dataset_uri(dataset_id: &str, digest: &str) -> String {
format!("zc://dataset/{dataset_id}@sha256:{digest}")
}
pub fn set_data_uri(spec: &str, uri: &str) -> Option<String> {
let indent = |l: &str| l.len() - l.trim_start_matches([' ', '\t']).len();
let key_only = |l: &str, key: &str| {
l.strip_prefix(key)
.is_some_and(|rest| rest.trim().is_empty() || rest.trim_start().starts_with('#'))
};
let mut out = String::with_capacity(spec.len() + uri.len());
let mut in_data = false;
let mut child: Option<usize> = None;
let mut done = false;
for line in spec.split_inclusive('\n') {
let body = line.trim_end_matches(['\n', '\r']);
let blank = body.trim().is_empty() || body.trim_start().starts_with('#');
if !done && !blank {
if indent(body) == 0 {
in_data = key_only(body, "data:");
child = None;
} else if in_data {
let depth = *child.get_or_insert(indent(body));
if indent(body) == depth && body[depth..].starts_with("uri:") {
out.push_str(&body[..depth]);
out.push_str(&format!("uri: \"{uri}\""));
out.push_str(&line[body.len()..]);
done = true;
continue;
}
}
}
out.push_str(line);
}
done.then_some(out)
}
#[cfg(test)]
mod tests {
use super::*;
fn temp(tag: &str, bytes: &[u8]) -> PathBuf {
let dir = std::env::temp_dir().join(format!(
"zc-atelier-upload-{tag}-{}-{:?}",
std::process::id(),
std::thread::current().id()
));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("train set.csv");
std::fs::write(&path, bytes).unwrap();
path
}
fn sha(bytes: &[u8]) -> String {
hex::encode(ring::digest::digest(&ring::digest::SHA256, bytes))
}
#[test]
fn an_existing_file_is_uploaded_and_everything_else_is_a_reference() {
let file = temp("classify", b"a,b\n1,2\n");
let s = file.to_str().unwrap();
assert_eq!(classify_dataset(s), DatasetArg::File(file.clone()));
for r in [
"e629e662-1d32-4e52-88e9-b0e83416c852",
"zc://alice/mine",
"hf://datasets/org/name",
"https://example.test/data.csv",
"s3://bucket/key.csv",
] {
assert_eq!(classify_dataset(r), DatasetArg::Ref(r.to_string()), "{r}");
}
let url = format!("file://{s}");
assert_eq!(classify_dataset(&url), DatasetArg::Ref(url.clone()));
let dir = file.parent().unwrap().to_str().unwrap();
assert_eq!(classify_dataset(dir), DatasetArg::Ref(dir.to_string()));
let missing = format!("{dir}/nope.csv");
assert_eq!(classify_dataset(&missing), DatasetArg::Ref(missing.clone()));
}
#[test]
fn a_small_file_is_one_put_and_declares_its_rows_and_header() {
let bytes = b"id,\"last, first\",score\r\n1,\"a, b\",0.5\r\n2,\"c, d\",0.7\r\n\r\n";
let path = temp("small", bytes);
let plan = plan_file(&path).unwrap();
assert_eq!(plan.path, "train set.csv");
assert_eq!(plan.size, bytes.len() as u64);
assert_eq!(plan.sha256, sha(bytes));
assert_eq!(plan.name, format!("train-set-{}", &sha(bytes)[..12]));
assert_eq!((plan.part_size, plan.part_sha256s.len()), (None, 0));
assert_eq!(
open_body(&plan),
json!({
"name": plan.name,
"visibility": "private",
"files": [{
"path": "train set.csv",
"sha256": sha(bytes),
"size_bytes": bytes.len(),
"meta": {"records": 2, "columns": ["id", "last, first", "score"]},
}],
})
);
}
#[test]
fn a_file_over_the_threshold_declares_each_parts_own_hash() {
let bytes: Vec<u8> = (0..2500u32).map(|i| (i % 251) as u8).collect();
let path = temp("parts", &bytes);
let plan = plan_file_with(&path, 1000, 1000).unwrap();
assert_eq!(plan.part_size, Some(1000));
assert_eq!(
plan.part_sha256s,
vec![
sha(&bytes[..1000]),
sha(&bytes[1000..2000]),
sha(&bytes[2000..])
]
);
assert_eq!(plan.sha256, sha(&bytes));
assert_eq!(plan.tabular, None);
let body = open_body(&plan);
let file = &body["files"][0];
assert_eq!(file["part_size"], 1000);
assert_eq!(
file["parts"][2],
json!({"part_number": 3, "sha256": sha(&bytes[2000..])})
);
assert!(file.get("meta").is_none());
assert_eq!(plan_file_with(&path, 2500, 1000).unwrap().part_size, None);
assert_eq!(
plan_file_with(&path, 100, 500).unwrap().part_sha256s.len(),
5
);
}
#[test]
fn part_sizes_follow_the_hubs_rules() {
assert_eq!(
part_size_for(64 * MIB, MULTIPART_THRESHOLD, DEFAULT_PART_SIZE),
None
);
assert_eq!(
part_size_for(64 * MIB + 1, MULTIPART_THRESHOLD, DEFAULT_PART_SIZE),
Some(32 * MIB)
);
let big = 400_000 * MIB;
let grown = part_size_for(big, MULTIPART_THRESHOLD, DEFAULT_PART_SIZE).unwrap();
assert_eq!(grown % MIB, 0);
assert!(big.div_ceil(grown) <= 10_000 && grown > 32 * MIB);
assert_eq!(
part_ranges(2500, 1000),
vec![(1, 0, 1000), (2, 1000, 1000), (3, 2000, 500)]
);
assert_eq!(part_ranges(2000, 1000).len(), 2);
}
#[test]
fn rows_and_header_are_read_as_the_hub_reads_them() {
let scan = |bytes: &[u8], chunk: usize| {
let mut s = TabularScanner::default();
for c in bytes.chunks(chunk) {
s.update(c);
}
s.finish()
};
let cols = |v: &[&str]| v.iter().map(|s| s.to_string()).collect::<Vec<_>>();
for chunk in [1, 3, 1000] {
assert_eq!(
scan(b"\xef\xbb\xbfa, b ,c\n1,2,3\n\n4,5,6", chunk),
Some((2, cols(&["a", "b", "c"])))
);
}
assert_eq!(
scan(b"{\"z\": 1, \"a\": {\"k\": 2}}\n{\"z\": 3, \"a\": 4}\n", 5),
Some((2, cols(&["z", "a"])))
);
assert_eq!(
scan(b"only,a,header", 4),
Some((0, cols(&["only", "a", "header"])))
);
assert_eq!(scan(b"PK\x03\x04\x00\x00data\n", 3), None);
assert_eq!(scan(b"", 3), None);
assert_eq!(scan(b"[1, 2]\n[3, 4]\n", 3), None);
assert_eq!(scan(b"\n a,b\n", 3), None);
assert_eq!(scan(b"{not json\n", 3), None);
assert_eq!(scan(b"\xff\xfe,a\n1,2\n", 3), None);
}
#[test]
fn a_dataset_is_named_after_the_file_and_its_bytes() {
let h = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
assert_eq!(dataset_name("train.csv", h), "train-0123456789ab");
assert_eq!(
dataset_name("My Data (v2).tar.gz", h),
"My-Data-v2-.tar-0123456789ab"
);
assert_eq!(dataset_name("données.csv", h), "donn-es-0123456789ab");
assert_eq!(dataset_name(".csv", h), "csv-0123456789ab");
assert_eq!(dataset_name("---", h), "dataset-0123456789ab");
assert!(dataset_name(&"x".repeat(400), h).len() <= 128);
}
#[test]
fn the_checksum_header_is_base64_of_the_raw_digest() {
assert_eq!(
checksum_header("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
.unwrap(),
"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
);
assert!(checksum_header("zz").is_err());
}
#[test]
fn the_specs_data_uri_is_pointed_at_the_uploaded_version_and_nothing_else_moves() {
let uri = dataset_uri("d-1", &"a".repeat(64));
assert_eq!(uri, format!("zc://dataset/d-1@sha256:{}", "a".repeat(64)));
let spec = "# my job\ntask: tabular\ndata:\n # where\n format: csv\n uri: ./train.csv # local\r\n target:\n uri: keep\nmodel:\n uri: keep-too\n";
let want = format!("# my job\ntask: tabular\ndata:\n # where\n format: csv\n uri: \"{uri}\"\r\n target:\n uri: keep\nmodel:\n uri: keep-too\n");
assert_eq!(set_data_uri(spec, &uri).as_deref(), Some(want.as_str()));
assert_eq!(
set_data_uri("data: # in\n uri: x", "U").as_deref(),
Some("data: # in\n uri: \"U\"")
);
for spec in [
"task: llm\nmodel_ref: m\n",
"data:\n format: csv\n",
"data: {uri: x}\n",
"model:\n uri: x\n",
"data:\n source:\n uri: x\n",
"metadata:\n data:\n uri: x\n",
"",
] {
assert_eq!(set_data_uri(spec, "U"), None, "{spec:?}");
}
}
fn plan(parts: usize) -> FilePlan {
FilePlan {
name: "d-abc".into(),
path: "d.csv".into(),
size: 10,
sha256: "a".repeat(64),
part_size: (parts > 0).then_some(1000),
part_sha256s: vec!["b".repeat(64); parts],
tabular: None,
}
}
#[test]
fn the_hubs_answer_says_where_the_bytes_go() {
let ids = json!({"upload_id": "u-1", "dataset_id": "d-1", "digest": "x"});
let with = |files: Value| {
let mut v = ids.clone();
v["files"] = files;
v
};
let single = with(json!([{"path": "d.csv", "url": "https://s3.test/put?sig=1"}]));
assert_eq!(
parse_open(&single, &plan(0)).unwrap(),
Opened {
upload_id: "u-1".into(),
dataset_id: "d-1".into(),
target: Target::Single("https://s3.test/put?sig=1".into()),
}
);
assert_eq!(
parse_open(&with(json!([])), &plan(0)).unwrap().target,
Target::Landed
);
let parts = with(json!([{"path": "d.csv", "part_size": 1000, "parts": [
{"part_number": 1, "url": "https://s3.test/p1"},
{"part_number": 2, "url": "https://s3.test/p2"}]}]));
assert_eq!(
parse_open(&parts, &plan(2)).unwrap().target,
Target::Parts(vec![
(1, "https://s3.test/p1".into()),
(2, "https://s3.test/p2".into())
])
);
let err = parse_open(&parts, &plan(3)).unwrap_err();
assert!(
err.contains("cannot resume") && err.contains("2 of 3"),
"{err}"
);
let mut bad = single.clone();
bad["upload_id"] = json!("../x");
assert!(parse_open(&bad, &plan(0))
.unwrap_err()
.contains("upload_id"));
assert!(parse_open(&json!({}), &plan(0)).is_err());
}
}