use async_trait::async_trait;
use futures_util::TryStreamExt;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::{
collections::HashSet,
path::{Path, PathBuf},
pin::Pin,
time::Duration,
};
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt};
use tokio_util::{io::StreamReader, sync::CancellationToken};
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct Artifact {
pub backend: String,
pub repo: String,
pub revision: String,
pub file: String,
pub sha256: String,
pub bytes: u64,
}
pub struct DownloadResponse {
pub status: u16,
pub commit: Option<String>,
pub length: Option<u64>,
pub body: Pin<Box<dyn AsyncRead + Send>>,
}
#[async_trait]
pub trait DownloadTransport: Send + Sync {
async fn get(&self, url: &str) -> Result<DownloadResponse, String>;
async fn get_from(&self, url: &str, offset: u64) -> Result<DownloadResponse, String> {
if offset == 0 {
self.get(url).await
} else {
Err("download resume unsupported".into())
}
}
}
const MAX_RESUMES: u32 = 8;
pub struct HfTransport {
client: reqwest::Client,
}
fn retain_commit(evidence: &mut Option<String>, observed: Option<&str>) -> Result<(), String> {
if let Some(observed) = observed {
if observed.len() != 40 || !observed.bytes().all(|byte| byte.is_ascii_hexdigit()) {
return Err("invalid repository commit evidence".into());
}
if evidence
.as_ref()
.is_some_and(|previous| !previous.eq_ignore_ascii_case(observed))
{
return Err("conflicting repository commit evidence".into());
}
*evidence = Some(observed.to_owned());
}
Ok(())
}
#[test]
fn commit_evidence_survives_cdn_redirects_and_rejects_conflicts() {
let revision = "a".repeat(40);
let mut evidence = None;
retain_commit(&mut evidence, Some(&revision)).unwrap();
retain_commit(&mut evidence, None).unwrap();
assert_eq!(evidence.as_deref(), Some(revision.as_str()));
assert!(retain_commit(&mut evidence, Some(&"b".repeat(40))).is_err());
assert!(retain_commit(&mut None, Some("not-a-commit")).is_err());
let mut absent = None;
retain_commit(&mut absent, None).unwrap();
assert!(absent.is_none());
}
impl HfTransport {
pub fn new() -> Result<Self, String> {
Ok(Self {
client: reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.no_proxy()
.dns_resolver(std::sync::Arc::new(PublicResolver))
.connect_timeout(Duration::from_secs(10))
.build()
.map_err(|error| error.to_string())?,
})
}
}
fn safe_url(raw: &str) -> Result<url::Url, String> {
let url = url::Url::parse(raw).map_err(|error| error.to_string())?;
let host = url.host_str().ok_or("missing download host")?;
if url.scheme() != "https"
|| !url.username().is_empty()
|| url.password().is_some()
|| url.port().is_some()
|| !["huggingface.co", "hf.co"]
.iter()
.any(|allowed| host == *allowed || host.ends_with(&format!(".{allowed}")))
{
return Err("download URL is not approved HTTPS source".into());
}
Ok(url)
}
fn content_range_starts_at(value: Option<&str>, offset: u64) -> bool {
value
.and_then(|value| value.strip_prefix("bytes "))
.and_then(|range| range.split_once('-'))
.and_then(|(start, _)| start.parse::<u64>().ok())
== Some(offset)
}
#[test]
fn resume_requires_matching_content_range() {
assert!(content_range_starts_at(Some("bytes 40-99/100"), 40));
assert!(!content_range_starts_at(Some("bytes 0-99/100"), 40));
assert!(!content_range_starts_at(Some("items 40-99/100"), 40));
assert!(!content_range_starts_at(None, 40));
}
#[async_trait]
impl DownloadTransport for HfTransport {
async fn get(&self, raw: &str) -> Result<DownloadResponse, String> {
self.get_from(raw, 0).await
}
async fn get_from(&self, raw: &str, offset: u64) -> Result<DownloadResponse, String> {
let mut url = safe_url(raw)?;
let mut commit = None;
for redirect in 0..=5 {
let mut request = self.client.get(url.clone());
if offset > 0 {
request = request.header("range", format!("bytes={offset}-"));
}
let response = request
.send()
.await
.map_err(|_| "download transport failed".to_owned())?;
retain_commit(
&mut commit,
response
.headers()
.get("x-repo-commit")
.map(|value| {
value
.to_str()
.map_err(|_| "invalid repository commit header".to_owned())
})
.transpose()?,
)?;
if response.status().is_redirection() {
if redirect == 5 {
return Err("too many download redirects".into());
}
let location = response
.headers()
.get("location")
.and_then(|value| value.to_str().ok())
.ok_or("redirect has no location")?;
url = safe_url(
url.join(location)
.map_err(|error| error.to_string())?
.as_str(),
)?;
continue;
}
let length = response.content_length();
let status = response.status().as_u16();
if offset > 0
&& (status != 206
|| !content_range_starts_at(
response
.headers()
.get("content-range")
.and_then(|value| value.to_str().ok()),
offset,
))
{
return Err("download resume was not honored".into());
}
let stream = response.bytes_stream().map_err(std::io::Error::other);
return Ok(DownloadResponse {
status,
commit,
length,
body: Box::pin(StreamReader::new(stream)),
});
}
Err("download redirect limit".into())
}
}
pub fn public_addresses(
addresses: Vec<std::net::SocketAddr>,
) -> Result<Vec<std::net::SocketAddr>, String> {
fn public(address: std::net::IpAddr) -> bool {
match address {
std::net::IpAddr::V4(address) => {
let octets = address.octets();
!address.is_private()
&& !address.is_loopback()
&& !address.is_link_local()
&& !address.is_multicast()
&& !address.is_unspecified()
&& !address.is_broadcast()
&& !address.is_documentation()
&& octets[0] != 0
&& octets[0] < 224
&& !(octets[0] == 100 && (64..=127).contains(&octets[1]))
&& !(octets[0] == 198 && (18..=19).contains(&octets[1]))
&& !(octets[0] == 192 && octets[1] == 0 && octets[2] == 0)
}
std::net::IpAddr::V6(address) => {
if let Some(mapped) = address.to_ipv4_mapped() {
return public(mapped.into());
}
let segments = address.segments();
segments[0] & 0xe000 == 0x2000
&& segments[0] != 0x2002
&& !(segments[0] == 0x2001 && (segments[1] < 0x200 || segments[1] == 0xdb8))
}
}
}
if addresses.is_empty()
|| addresses.len() > 64
|| addresses.iter().any(|address| !public(address.ip()))
{
return Err("download DNS contains private or unsupported addresses".into());
}
Ok(addresses)
}
struct PublicResolver;
impl reqwest::dns::Resolve for PublicResolver {
fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving {
Box::pin(async move {
let addresses = tokio::time::timeout(
Duration::from_secs(5),
tokio::net::lookup_host((name.as_str(), 0)),
)
.await
.map_err(|_| std::io::Error::other("download DNS timed out"))??;
let addresses =
public_addresses(addresses.take(65).collect()).map_err(std::io::Error::other)?;
Ok(Box::new(addresses.into_iter()) as reqwest::dns::Addrs)
})
}
}
#[derive(Debug)]
pub struct Acquired {
pub path: PathBuf,
pub bytes: u64,
pub cached: bool,
pub rehashed: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SizePolicy {
Exact,
Ceiling,
}
impl SizePolicy {
fn accepts(self, actual: u64, expected: u64) -> bool {
actual > 0
&& match self {
Self::Exact => actual == expected,
Self::Ceiling => actual <= expected,
}
}
}
pub struct Acquisition {
root: PathBuf,
progress: Option<std::sync::Arc<Progress>>,
timeout: Duration,
}
type Progress = dyn Fn(u64, u64, &str) + Send + Sync;
struct ArtifactLock {
guard: crate::state::LockGuard,
}
impl ArtifactLock {
fn acquire(path: PathBuf) -> Result<Self, String> {
let mut config =
crate::state::Config::from_home(path.parent().ok_or("missing lock parent")?)
.map_err(|error| error.to_string())?;
config.lock = path;
let guard = crate::state::StateStore::new(config)
.lock(Duration::ZERO)
.map_err(|error| error.to_string())?;
Ok(Self { guard })
}
fn check(&self) -> Result<(), String> {
self.guard.assert_owned().map_err(|error| error.to_string())
}
}
fn suffix(path: &Path, suffix: &str) -> PathBuf {
let mut value = path.as_os_str().to_owned();
value.push(suffix);
PathBuf::from(value)
}
pub fn cleanup_partials(parent: &Path, alive: impl Fn(u32) -> bool) -> Result<(), String> {
for (index, entry) in std::fs::read_dir(parent)
.map_err(|error| error.to_string())?
.enumerate()
{
if index >= 10000 {
return Err("cache directory exceeds cleanup entry limit".into());
}
let entry = entry.map_err(|error| error.to_string())?;
let name = entry.file_name();
let Some(name) = name.to_str() else {
continue;
};
let Some(rest) = name
.strip_prefix(".llmup-download.")
.and_then(|value| value.strip_suffix(".part"))
else {
continue;
};
let Some((pid, unique)) = rest.split_once('.') else {
continue;
};
if unique.is_empty()
|| unique.contains('.')
|| !pid.bytes().all(|byte| byte.is_ascii_digit())
|| pid.starts_with('0')
{
continue;
}
let Some(pid) = pid
.parse::<u32>()
.ok()
.filter(|pid| *pid > 0 && *pid <= i32::MAX as u32)
else {
continue;
};
if entry
.file_type()
.map_err(|error| error.to_string())?
.is_file()
&& !alive(pid)
{
std::fs::remove_file(entry.path()).map_err(|error| error.to_string())?;
}
}
Ok(())
}
fn segment(value: &str) -> bool {
!value.is_empty()
&& portable_component(value)
&& value.as_bytes()[0].is_ascii_alphanumeric()
&& value
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte))
}
fn portable_component(value: &str) -> bool {
let base = value.split('.').next().unwrap_or("").to_ascii_uppercase();
!value.is_empty()
&& value != "."
&& value != ".."
&& !value.ends_with('.')
&& value.trim() == value
&& !["CON", "PRN", "AUX", "NUL", "CONIN$", "CONOUT$"].contains(&base.as_str())
&& !(base.starts_with("COM") || base.starts_with("LPT"))
.then(|| &base[3..])
.is_some_and(|suffix| {
[
"1", "2", "3", "4", "5", "6", "7", "8", "9", "\u{b9}", "\u{b2}", "\u{b3}",
]
.contains(&suffix)
})
}
pub fn safe_file(value: &str) -> bool {
!value.is_empty()
&& value.len() <= 512
&& !value
.chars()
.any(|ch| ch.is_control() || "\\%*?[]{}:".contains(ch))
&& value.split('/').all(portable_component)
}
fn digest(value: &str, length: usize) -> bool {
value.len() == length && value.bytes().all(|byte| byte.is_ascii_hexdigit())
}
impl Artifact {
pub fn validate(&self) -> Result<(), String> {
let coordinates: Vec<_> = self.repo.split('/').collect();
if !["llamacpp", "mlx", "comfyui", "bitnet", "whispercpp"].contains(&self.backend.as_str())
|| coordinates.len() != 2
|| !coordinates.iter().all(|part| segment(part))
|| !digest(&self.revision, 40)
|| !digest(&self.sha256, 64)
|| !safe_file(&self.file)
|| self.bytes == 0
|| self.bytes > 9007199254740991
{
return Err("invalid pinned artifact".into());
}
Ok(())
}
}
fn private_directory(path: &Path) -> Result<(), String> {
match std::fs::symlink_metadata(path) {
Ok(metadata) => {
if !metadata.is_dir() || metadata.file_type().is_symlink() {
return Err("unsafe cache directory".into());
}
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
if let Some(parent) = path.parent() {
private_directory(parent)?;
}
let builder = std::fs::DirBuilder::new();
#[cfg(unix)]
let mut builder = builder;
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
builder.mode(0o700);
}
match builder.create(path) {
Ok(()) => (),
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
return private_directory(path);
}
Err(error) => return Err(error.to_string()),
}
}
Err(error) => return Err(error.to_string()),
}
Ok(())
}
const VERIFIED_DIRECTORY: &str = ".verified";
const MAX_STAMP_BYTES: u64 = 4096;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct FileIdentity {
device: u64,
inode: u64,
modified: i64,
modified_nanos: i64,
changed: i64,
changed_nanos: i64,
}
#[cfg(unix)]
fn file_identity(metadata: &std::fs::Metadata) -> Option<FileIdentity> {
use std::os::unix::fs::MetadataExt;
if metadata.mtime_nsec() == 0 && metadata.ctime_nsec() == 0 {
return None;
}
Some(FileIdentity {
device: metadata.dev(),
inode: metadata.ino(),
modified: metadata.mtime(),
modified_nanos: metadata.mtime_nsec(),
changed: metadata.ctime(),
changed_nanos: metadata.ctime_nsec(),
})
}
#[cfg(not(unix))]
fn file_identity(_metadata: &std::fs::Metadata) -> Option<FileIdentity> {
None
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct VerifiedStamp {
file: String,
sha256: String,
bytes: u64,
identity: FileIdentity,
}
fn open_no_follow(path: &Path) -> std::io::Result<std::fs::File> {
let mut options = std::fs::OpenOptions::new();
options.read(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.custom_flags(
(rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::NONBLOCK).bits() as i32,
);
}
options.open(path)
}
pub async fn hash_file(path: &Path, cancel: &CancellationToken) -> Result<(String, u64), String> {
let file = open_no_follow(path).map_err(|error| error.to_string())?;
if !file
.metadata()
.map_err(|error| error.to_string())?
.is_file()
|| std::fs::symlink_metadata(path)
.map_err(|error| error.to_string())?
.file_type()
.is_symlink()
{
return Err("artifact is not a regular file".into());
}
let mut file = tokio::fs::File::from_std(file);
let mut hash = Sha256::new();
let mut total = 0u64;
let mut buffer = vec![0u8; 65536];
loop {
let count = tokio::select! { _=cancel.cancelled()=>return Err("cancelled".into()), result=file.read(&mut buffer)=>result.map_err(|error|error.to_string())? };
if count == 0 {
break;
}
total = total
.checked_add(count as u64)
.ok_or("artifact size overflow")?;
hash.update(&buffer[..count]);
}
Ok((format!("{:x}", hash.finalize()), total))
}
impl Acquisition {
fn stamp(
&self,
target: &Path,
artifact: &Artifact,
metadata: &std::fs::Metadata,
) -> Option<(PathBuf, VerifiedStamp)> {
let file = target.strip_prefix(&self.root).ok()?.to_str()?.to_owned();
let key = format!("{:x}", Sha256::digest(file.as_bytes()));
Some((
self.root
.join(VERIFIED_DIRECTORY)
.join(format!("{key}.json")),
VerifiedStamp {
file,
sha256: artifact.sha256.to_ascii_lowercase(),
bytes: metadata.len(),
identity: file_identity(metadata)?,
},
))
}
fn already_verified(
&self,
target: &Path,
artifact: &Artifact,
metadata: &std::fs::Metadata,
) -> bool {
let Some((path, expected)) = self.stamp(target, artifact, metadata) else {
return false;
};
let Ok(file) = open_no_follow(&path) else {
return false;
};
if !file.metadata().is_ok_and(|stamp| stamp.is_file()) {
return false;
}
let mut raw = Vec::new();
use std::io::Read;
if file
.take(MAX_STAMP_BYTES + 1)
.read_to_end(&mut raw)
.is_err()
|| raw.len() as u64 > MAX_STAMP_BYTES
{
return false;
}
serde_json::from_slice::<VerifiedStamp>(&raw).is_ok_and(|stamp| stamp == expected)
}
fn record_verified(&self, target: &Path, artifact: &Artifact, verified: Option<FileIdentity>) {
let _ = (|| -> Result<(), String> {
let metadata = std::fs::symlink_metadata(target).map_err(|error| error.to_string())?;
if !metadata.is_file() || metadata.file_type().is_symlink() {
return Err("unsafe cached artifact".into());
}
let (path, stamp) = self
.stamp(target, artifact, &metadata)
.ok_or("no stable file identity")?;
if verified.is_some_and(|identity| identity != stamp.identity) {
return Err("artifact changed while it was being verified".into());
}
let directory = path.parent().ok_or("missing stamp directory")?;
private_directory(directory)?;
let mut temporary = tempfile::Builder::new()
.prefix(".stamp.")
.tempfile_in(directory)
.map_err(|error| error.to_string())?;
use std::io::Write;
temporary
.write_all(&serde_json::to_vec(&stamp).map_err(|error| error.to_string())?)
.map_err(|error| error.to_string())?;
temporary
.persist(&path)
.map_err(|error| error.to_string())?;
Ok(())
})();
}
fn cache_directory(&self, path: &Path) -> Result<(), String> {
let relative = path
.strip_prefix(&self.root)
.map_err(|_| "cache path escaped root")?;
let mut current = self.root.clone();
private_directory(¤t)?;
for component in relative.components() {
let std::path::Component::Normal(name) = component else {
return Err("invalid cache path component".into());
};
current.push(name);
private_directory(¤t)?;
if !current
.canonicalize()
.map_err(|error| error.to_string())?
.starts_with(&self.root)
{
return Err("cache path escaped root".into());
}
}
Ok(())
}
pub fn new(root: impl AsRef<Path>) -> Result<Self, String> {
let root = std::path::absolute(root).map_err(|error| error.to_string())?;
private_directory(&root)?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700))
.map_err(|error| error.to_string())?;
}
Ok(Self {
root: root.canonicalize().map_err(|error| error.to_string())?,
progress: None,
timeout: Duration::from_secs(1800),
})
}
pub fn with_progress(
mut self,
progress: impl Fn(u64, u64, &str) + Send + Sync + 'static,
) -> Self {
self.progress = Some(std::sync::Arc::new(progress));
self
}
pub fn with_timeout(mut self, timeout: Duration) -> Self {
self.timeout = timeout;
self
}
fn emit(&self, completed: u64, total: u64, artifact: &Artifact) {
if let Some(progress) = &self.progress {
progress(completed, total.max(completed), &artifact.file);
}
}
pub async fn acquire(
&self,
artifact: &Artifact,
transport: &dyn DownloadTransport,
cancel: &CancellationToken,
) -> Result<Acquired, String> {
self.acquire_sized(artifact, SizePolicy::Exact, transport, cancel)
.await
}
pub async fn acquire_sized(
&self,
artifact: &Artifact,
size: SizePolicy,
transport: &dyn DownloadTransport,
cancel: &CancellationToken,
) -> Result<Acquired, String> {
let operation = async {
artifact.validate()?;
let coordinates: Vec<_> = artifact.repo.split('/').collect();
let owner = self.root.join(&artifact.backend).join(coordinates[0]);
self.cache_directory(&owner)?;
let repository = owner.join(format!("{}@{}", coordinates[1], artifact.revision));
let lock = ArtifactLock::acquire(suffix(&repository, ".repository.lock"))?;
let acquired = self
.acquire_inner(artifact, size, transport, cancel)
.await?;
lock.check()?;
Ok(acquired)
};
tokio::select! {biased;_=cancel.cancelled()=>Err("cancelled".into()),result=tokio::time::timeout(self.timeout,operation)=>result.map_err(|_|"acquisition timed out".to_owned())?}
}
async fn acquire_inner(
&self,
artifact: &Artifact,
size: SizePolicy,
transport: &dyn DownloadTransport,
cancel: &CancellationToken,
) -> Result<Acquired, String> {
artifact.validate()?;
if cancel.is_cancelled() {
return Err("cancelled".into());
}
let coordinates: Vec<_> = artifact.repo.split('/').collect();
let directory = self
.root
.join(&artifact.backend)
.join(coordinates[0])
.join(format!("{}@{}", coordinates[1], artifact.revision));
let target = directory.join(&artifact.file);
let parent = target.parent().ok_or("missing artifact parent")?;
self.cache_directory(parent)?;
if !parent
.canonicalize()
.map_err(|error| error.to_string())?
.starts_with(&self.root)
{
return Err("cache escaped root".into());
}
let lock = ArtifactLock::acquire(suffix(&target, ".lock"))?;
cleanup_partials(parent, crate::state::process_alive)?;
if size == SizePolicy::Exact {
self.emit(0, artifact.bytes, artifact);
}
if let Ok(metadata) = std::fs::symlink_metadata(&target) {
if !metadata.is_file() || metadata.file_type().is_symlink() {
return Err("unsafe cached artifact".into());
}
if size.accepts(metadata.len(), artifact.bytes)
&& self.already_verified(&target, artifact, &metadata)
{
lock.check()?;
self.emit(metadata.len(), metadata.len(), artifact);
return Ok(Acquired {
path: target,
bytes: metadata.len(),
cached: true,
rehashed: false,
});
}
let (sha, bytes) = hash_file(&target, cancel).await?;
if sha.eq_ignore_ascii_case(&artifact.sha256) && size.accepts(bytes, artifact.bytes) {
lock.check()?;
self.record_verified(&target, artifact, file_identity(&metadata));
self.emit(bytes, bytes, artifact);
return Ok(Acquired {
path: target,
bytes,
cached: true,
rehashed: true,
});
}
}
let mut url =
url::Url::parse("https://huggingface.co").map_err(|error| error.to_string())?;
{
let mut path = url.path_segments_mut().map_err(|_| "invalid URL base")?;
path.extend(coordinates)
.push("resolve")
.push(&artifact.revision)
.extend(artifact.file.split('/'));
}
url.set_query(Some("download=true"));
let operation = async {
let mut response = transport.get(url.as_str()).await?;
if !(200..300).contains(&response.status) {
return Err(format!("download HTTP {}", response.status));
}
if response
.commit
.as_ref()
.is_none_or(|commit| !commit.eq_ignore_ascii_case(&artifact.revision))
{
return Err("resolved commit mismatch".into());
}
if response
.length
.is_some_and(|length| length > artifact.bytes)
{
return Err("download exceeds byte limit".into());
}
let expected = match size {
SizePolicy::Exact => artifact.bytes,
SizePolicy::Ceiling => response.length.unwrap_or(0),
};
if expected > 0 {
self.emit(0, expected, artifact);
}
let temporary = tempfile::Builder::new()
.prefix(&format!(".llmup-download.{}.", std::process::id()))
.suffix(".part")
.tempfile_in(parent)
.map_err(|error| error.to_string())?;
let mut output =
tokio::fs::File::from_std(temporary.reopen().map_err(|error| error.to_string())?);
let mut hash = Sha256::new();
let mut total = 0u64;
let mut reported = 0u64;
let mut buffer = vec![0u8; 65536];
let mut resumes = 0u32;
loop {
let count = match response.body.read(&mut buffer).await {
Ok(count) => count,
Err(error) => {
let mut failure = error.to_string();
let resumed = loop {
resumes += 1;
if resumes > MAX_RESUMES {
return Err(failure);
}
tokio::time::sleep(Duration::from_secs(u64::from(resumes.min(5))))
.await;
match transport.get_from(url.as_str(), total).await {
Ok(resumed) => break resumed,
Err(resume) => failure = format!("{error}; {resume}"),
}
};
response = resumed;
if response.status != 206
|| response.commit.as_ref().is_none_or(|commit| {
!commit.eq_ignore_ascii_case(&artifact.revision)
})
|| response
.length
.is_some_and(|length| length > artifact.bytes - total)
{
return Err("download resume response mismatch".into());
}
continue;
}
};
if count == 0 {
break;
}
total = total
.checked_add(count as u64)
.ok_or("download size overflow")?;
if total > artifact.bytes {
return Err("download exceeds byte limit".into());
}
hash.update(&buffer[..count]);
output
.write_all(&buffer[..count])
.await
.map_err(|error| error.to_string())?;
if total - reported >= 8 * 1024 * 1024 {
self.emit(total, expected, artifact);
reported = total;
}
}
if !size.accepts(total, artifact.bytes)
|| !format!("{:x}", hash.finalize()).eq_ignore_ascii_case(&artifact.sha256)
{
return Err("artifact integrity mismatch".into());
}
output.sync_all().await.map_err(|error| error.to_string())?;
drop(output);
lock.check()?;
if cancel.is_cancelled() {
return Err("cancelled".into());
}
temporary
.persist(&target)
.map_err(|error| error.to_string())?;
self.record_verified(&target, artifact, None);
self.emit(total, total, artifact);
Ok(Acquired {
path: target.clone(),
bytes: total,
cached: false,
rehashed: false,
})
};
tokio::select! { _=cancel.cancelled()=>Err("cancelled".into()), result=tokio::time::timeout(self.timeout,operation)=>result.map_err(|_|"download timed out".to_owned())? }
}
pub async fn repository(
&self,
artifacts: &[Artifact],
transport: &dyn DownloadTransport,
cancel: &CancellationToken,
) -> Result<PathBuf, String> {
tokio::select! {biased;_=cancel.cancelled()=>Err("cancelled".into()),result=tokio::time::timeout(Duration::from_secs(1800),self.repository_inner(artifacts,transport,cancel))=>result.map_err(|_|"repository acquisition timed out".to_owned())?}
}
async fn repository_inner(
&self,
artifacts: &[Artifact],
transport: &dyn DownloadTransport,
cancel: &CancellationToken,
) -> Result<PathBuf, String> {
let first = artifacts.first().ok_or("empty repository manifest")?;
if artifacts.len() > 256 {
return Err("repository file count exceeds limit".into());
}
let mut names = HashSet::new();
let mut root = None;
for artifact in artifacts {
artifact.validate()?;
if ["py", "pyc", "pyo", "so", "dylib", "dll", "bundle"]
.iter()
.any(|extension| {
artifact
.file
.to_ascii_lowercase()
.ends_with(&format!(".{extension}"))
})
{
return Err("executable repository artifact forbidden".into());
}
if artifact.repo != first.repo
|| artifact.revision != first.revision
|| artifact.backend != first.backend
|| !names.insert(artifact.file.clone())
{
return Err("inconsistent repository manifest".into());
}
}
let folded: HashSet<_> = names.iter().map(|name| name.to_lowercase()).collect();
if folded.len() != names.len()
|| folded.iter().any(|name| {
name.match_indices('/')
.any(|(index, _)| folded.contains(&name[..index]))
})
{
return Err("repository contains path aliases or file/directory collisions".into());
}
if first.backend == "mlx"
&& (artifacts.len() < 3
|| !names.contains("config.json")
|| !names.contains("tokenizer_config.json")
|| !names.iter().any(|name| name.ends_with(".safetensors")))
{
return Err("incomplete MLX repository manifest".into());
}
let total = artifacts
.iter()
.try_fold(0u64, |total, artifact| total.checked_add(artifact.bytes))
.ok_or("repository byte total overflow")?;
if total > 9007199254740991 {
return Err("repository byte total exceeds safe integer".into());
}
let coordinates: Vec<_> = first.repo.split('/').collect();
let owner = self.root.join(&first.backend).join(coordinates[0]);
self.cache_directory(&owner)?;
let repository = owner.join(format!("{}@{}", coordinates[1], first.revision));
let lock = ArtifactLock::acquire(suffix(&repository, ".repository.lock"))?;
for artifact in artifacts {
let acquired = self
.acquire_inner(artifact, SizePolicy::Exact, transport, cancel)
.await?;
let mut directory = acquired.path;
for _ in artifact.file.split('/') {
directory.pop();
}
if root.as_ref().is_some_and(|prior| prior != &directory) {
return Err("repository root mismatch".into());
}
root = Some(directory);
}
let root = root.ok_or("missing repository root")?;
fn list(root: &Path, path: &Path, files: &mut HashSet<String>) -> Result<(), String> {
for entry in std::fs::read_dir(path).map_err(|error| error.to_string())? {
let entry = entry.map_err(|error| error.to_string())?;
let kind = entry.file_type().map_err(|error| error.to_string())?;
if kind.is_symlink() {
return Err("symlink in repository".into());
}
if kind.is_dir() {
list(root, &entry.path(), files)?;
} else if kind.is_file() {
files.insert(
entry
.path()
.strip_prefix(root)
.map_err(|error| error.to_string())?
.to_string_lossy()
.replace('\\', "/"),
);
} else {
return Err("special file in repository".into());
}
}
Ok(())
}
let mut actual = HashSet::new();
list(&root, &root, &mut actual)?;
if actual != names {
return Err("repository content differs from manifest".into());
}
lock.check()?;
Ok(root)
}
}
#[cfg(test)]
mod boundary_tests {
use super::*;
struct Plain;
#[async_trait]
impl DownloadTransport for Plain {
async fn get(&self, _url: &str) -> Result<DownloadResponse, String> {
Ok(DownloadResponse {
status: 200,
commit: None,
length: Some(0),
body: Box::pin(std::io::Cursor::new(Vec::new())),
})
}
}
#[tokio::test]
async fn transports_without_ranges_only_start_from_zero() {
assert_eq!(
Plain.get_from("https://hf.co/x", 0).await.unwrap().status,
200
);
assert!(Plain.get_from("https://hf.co/x", 5).await.is_err());
}
#[test]
fn commit_evidence_is_exactly_forty_hex_digits() {
assert!(retain_commit(&mut None, Some(&"a".repeat(39))).is_err());
assert!(retain_commit(&mut None, Some(&"g".repeat(40))).is_err());
assert!(retain_commit(&mut None, Some(&"A".repeat(40))).is_ok());
}
#[test]
fn downloads_come_only_from_hugging_face_over_plain_https() {
for good in [
"https://huggingface.co/a/b",
"https://hf.co/a/b",
"https://cdn-lfs.huggingface.co/x",
"https://cas-bridge.xethub.hf.co/x",
] {
assert!(safe_url(good).is_ok(), "{good}");
}
for bad in [
"http://huggingface.co/a/b",
"https://user@huggingface.co/a/b",
"https://user:pass@huggingface.co/a/b",
"https://:pass@huggingface.co/a/b",
"https://huggingface.co:8443/a/b",
"https://evilhuggingface.co/a/b",
"https://huggingface.co.evil.com/a/b",
"https://example.com/a/b",
] {
assert!(safe_url(bad).is_err(), "{bad}");
}
}
#[test]
fn artifact_lock_check_fails_once_the_lock_file_is_replaced() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("model.gguf.lock");
let lock = ArtifactLock::acquire(path.clone()).unwrap();
assert!(lock.check().is_ok());
std::fs::remove_file(&path).unwrap();
std::fs::write(&path, b"someone else").unwrap();
assert!(lock.check().is_err());
}
fn addresses(ips: &[&str]) -> Vec<std::net::SocketAddr> {
ips.iter()
.map(|ip| std::net::SocketAddr::new(ip.parse().unwrap(), 443))
.collect()
}
#[test]
fn download_dns_must_resolve_only_to_public_addresses() {
for good in [
"8.8.8.8",
"100.63.255.255",
"100.128.0.1",
"198.17.255.255",
"198.20.0.1",
"192.0.1.1",
"223.255.255.255",
"2606:4700::1",
"2001:200::1",
"::ffff:8.8.8.8",
] {
assert!(public_addresses(addresses(&[good])).is_ok(), "{good}");
}
for bad in [
"10.0.0.1",
"127.0.0.1",
"169.254.1.1",
"0.1.2.3",
"224.0.0.1",
"240.0.0.1",
"255.255.255.255",
"192.0.2.1",
"100.64.0.1",
"100.127.255.255",
"198.18.0.1",
"198.19.255.255",
"192.0.0.8",
"::1",
"fe80::1",
"fc00::1",
"2002::1",
"2001:1ff::1",
"2001:db8::1",
"::ffff:10.0.0.1",
] {
assert!(public_addresses(addresses(&[bad])).is_err(), "{bad}");
}
assert!(public_addresses(Vec::new()).is_err());
assert!(public_addresses(addresses(&["8.8.8.8", "10.0.0.1"])).is_err());
assert!(public_addresses(addresses(&["8.8.8.8"; 64])).is_ok());
assert!(public_addresses(addresses(&["8.8.8.8"; 65])).is_err());
}
#[test]
fn size_policies_never_accept_an_empty_file() {
assert!(SizePolicy::Exact.accepts(5, 5));
assert!(!SizePolicy::Exact.accepts(4, 5));
assert!(SizePolicy::Ceiling.accepts(1, 5));
assert!(SizePolicy::Ceiling.accepts(5, 5));
assert!(!SizePolicy::Ceiling.accepts(6, 5));
assert!(!SizePolicy::Exact.accepts(0, 0));
assert!(!SizePolicy::Ceiling.accepts(0, 5));
}
#[test]
fn only_dead_processes_partial_downloads_are_removed() {
let root = tempfile::tempdir().unwrap();
let keep = [
".llmup-download.123.abc.part.extra",
".llmup-download.123..part",
".llmup-download.123.a.b.part",
".llmup-download.12x.abc.part",
".llmup-download.0123.abc.part",
".llmup-download.0.abc.part",
".llmup-download.2147483648.abc.part",
".llmup-download.123abc.part",
"weights.gguf",
];
for name in keep {
std::fs::write(root.path().join(name), b"x").unwrap();
}
std::fs::write(root.path().join(".llmup-download.123.abc.part"), b"x").unwrap();
std::fs::write(
root.path().join(".llmup-download.2147483647.xyz.part"),
b"x",
)
.unwrap();
std::fs::write(root.path().join(".llmup-download.7.live.part"), b"x").unwrap();
std::fs::create_dir(root.path().join(".llmup-download.124.dir.part")).unwrap();
cleanup_partials(root.path(), |pid| pid == 7).unwrap();
let mut left: Vec<String> = std::fs::read_dir(root.path())
.unwrap()
.map(|entry| entry.unwrap().file_name().into_string().unwrap())
.collect();
left.sort();
let mut expected: Vec<String> = keep.iter().map(|name| (*name).to_owned()).collect();
expected.extend([
".llmup-download.7.live.part".to_owned(),
".llmup-download.124.dir.part".to_owned(),
]);
expected.sort();
assert_eq!(left, expected);
}
fn artifact() -> Artifact {
Artifact {
backend: "llamacpp".into(),
repo: "owner/repo".into(),
revision: "a".repeat(40),
file: "models/weights.gguf".into(),
sha256: "b".repeat(64),
bytes: 1,
}
}
#[test]
fn pinned_artifacts_name_one_safe_repository_file_and_digest() {
assert!(artifact().validate().is_ok());
let mut max = artifact();
max.bytes = 9007199254740991;
assert!(max.validate().is_ok());
for backend in ["mlx", "comfyui", "bitnet"] {
let mut ok = artifact();
ok.backend = backend.into();
assert!(ok.validate().is_ok(), "{backend}");
}
type Edit = Box<dyn Fn(&mut Artifact)>;
let cases: Vec<Edit> = vec![
Box::new(|a| a.backend = "ollama".into()),
Box::new(|a| a.repo = "owner".into()),
Box::new(|a| a.repo = "owner/repo/extra".into()),
Box::new(|a| a.repo = "owner/.repo".into()),
Box::new(|a| a.repo = "-owner/repo".into()),
Box::new(|a| a.repo = "owner/re po".into()),
Box::new(|a| a.repo = "owner/con".into()),
Box::new(|a| a.repo = "/repo".into()),
Box::new(|a| a.revision = "a".repeat(39)),
Box::new(|a| a.revision = "g".repeat(40)),
Box::new(|a| a.sha256 = "b".repeat(63)),
Box::new(|a| a.sha256 = "z".repeat(64)),
Box::new(|a| a.file = "../weights.gguf".into()),
Box::new(|a| a.file = "a\\b.gguf".into()),
Box::new(|a| a.file = "weights*.gguf".into()),
Box::new(|a| a.file = "w\u{7}.gguf".into()),
Box::new(|a| a.file = String::new()),
Box::new(|a| a.file = "a".repeat(513)),
Box::new(|a| a.bytes = 0),
Box::new(|a| a.bytes = 9007199254740992),
];
for (index, mutate) in cases.iter().enumerate() {
let mut value = artifact();
mutate(&mut value);
assert!(value.validate().is_err(), "case {index}: {value:?}");
}
assert!(safe_file(&"a".repeat(512)));
assert!(!safe_file("a/b:c"));
}
#[test]
fn digests_are_exact_length_hex() {
assert!(digest("abc", 3));
assert!(!digest("abc", 4));
assert!(!digest("abz", 3));
}
#[cfg(unix)]
#[test]
fn cache_directories_are_created_private_and_never_followed() {
use std::os::unix::fs::PermissionsExt;
let root = tempfile::tempdir().unwrap();
let nested = root.path().join("a/b/c");
private_directory(&nested).unwrap();
assert_eq!(
std::fs::metadata(&nested).unwrap().permissions().mode() & 0o777,
0o700
);
private_directory(&nested).unwrap();
let file = root.path().join("file");
std::fs::write(&file, b"x").unwrap();
assert!(private_directory(&file).is_err());
assert!(private_directory(&file.join("below")).is_err());
let link = root.path().join("link");
std::os::unix::fs::symlink(&nested, &link).unwrap();
assert!(private_directory(&link).is_err());
}
#[cfg(unix)]
#[tokio::test]
async fn hashing_refuses_symlinks_and_non_files() {
let root = tempfile::tempdir().unwrap();
let file = root.path().join("weights");
std::fs::write(&file, b"weights").unwrap();
let (sha, bytes) = hash_file(&file, &CancellationToken::new()).await.unwrap();
assert_eq!(bytes, 7);
assert_eq!(sha, format!("{:x}", Sha256::digest(b"weights")));
let link = root.path().join("link");
std::os::unix::fs::symlink(&file, &link).unwrap();
assert!(hash_file(&link, &CancellationToken::new()).await.is_err());
assert!(
hash_file(root.path(), &CancellationToken::new())
.await
.is_err()
);
let fifo = root.path().join("fifo");
assert!(
std::process::Command::new("mkfifo")
.arg(&fifo)
.status()
.unwrap()
.success()
);
let hashed = tokio::time::timeout(
Duration::from_secs(5),
hash_file(&fifo, &CancellationToken::new()),
)
.await
.expect("opening a FIFO must not block");
assert!(hashed.is_err());
}
#[cfg(unix)]
#[test]
fn file_identity_tracks_device_inode_and_change_times() {
let root = tempfile::tempdir().unwrap();
let file = root.path().join("weights");
std::fs::write(&file, b"a").unwrap();
let first = file_identity(&std::fs::metadata(&file).unwrap());
if let Some(first) = first {
use std::os::unix::fs::MetadataExt;
let metadata = std::fs::metadata(&file).unwrap();
assert_eq!(
(first.device, first.inode, first.modified, first.changed),
(
metadata.dev(),
metadata.ino(),
metadata.mtime(),
metadata.ctime()
)
);
std::thread::sleep(Duration::from_millis(20));
std::fs::write(&file, b"b").unwrap();
let second = file_identity(&std::fs::metadata(&file).unwrap()).unwrap();
assert_ne!(first, second);
}
}
}