use std::collections::BTreeMap;
use std::io::Write;
use std::path::Path;
use anyhow::{Context, Result};
use futures::stream::{self, StreamExt, TryStreamExt};
use super::api::{self, HfFile};
use super::download::{self, FetchRequest};
use super::oci::{self, Descriptor, ModelMeta};
use super::progress;
use crate::fsutil::HashingWriter;
use crate::xet_fetch::{self, XetFileRef};
pub async fn pull(reference: &str, layout_dir: &Path, progress_key: &str) -> Result<()> {
pull_from(reference, layout_dir, progress_key, super::hf_endpoint).await
}
async fn pull_from(
reference: &str,
layout_dir: &Path,
progress_key: &str,
endpoint: impl FnOnce(&str) -> String,
) -> Result<()> {
let _guard = progress::DoneGuard(progress_key);
let (host, owner, repo, tag) = api::parse_hf_ref(reference)?;
oci::ensure_layout(layout_dir)?;
if oci::report_cached(layout_dir, reference) {
return Ok(());
}
let endpoint = endpoint(&host);
let token = super::token();
let api_client = super::api_client()?;
let dl_client = super::download_client()?;
let head_client = super::head_client()?;
progress::set_status(progress_key, "pulling");
let info =
api::fetch_model_info(&api_client, &endpoint, &owner, &repo, token.as_deref()).await?;
let commit = info.commit().to_string();
let revision = info.sha().to_string();
let mut meta = ModelMeta {
licenses: info.license().into_iter().collect(),
..Default::default()
};
let files = api::fetch_files(
&api_client,
&endpoint,
&owner,
&repo,
&commit,
token.as_deref(),
)
.await?;
let diffusion = api::is_diffusion_repo(&files);
let selected = if diffusion {
api::select_diffusion_gguf(&files, &tag)
} else {
api::select_gguf(&files, &tag)
};
let manifest_desc = match selected {
Ok(shards) if diffusion && shards.len() != 1 => {
anyhow::bail!("{owner}/{repo}: split diffusion transformers are not supported");
}
Ok(shards) => {
meta.format = "gguf".to_string();
let filepath_annotation = if shards.len() == 1 {
basename(&shards[0].path)
} else {
String::new()
};
let plan = diffusion.then(|| api::diffusion_plan(&files, &shards[0].path));
let mmproj = if diffusion {
None
} else {
api::select_mmproj(&files)
};
let license = api::select_license_file(&files);
let wanted: Vec<&HfFile> = shards
.iter()
.chain(plan.iter().flat_map(|p| p.sidecars.iter().map(|(_, f)| f)))
.chain(mmproj.iter())
.chain(license.iter())
.collect();
ensure_room(reference, layout_dir, &wanted)?;
let mut layers = Vec::new();
for f in &shards {
layers.push(
download_layer(
&dl_client,
&head_client,
&endpoint,
&owner,
&repo,
&commit,
f,
token.as_deref(),
layout_dir,
progress_key,
)
.await?,
);
}
if let Some(plan) = plan {
meta.diffusion_outputs = plan.outputs;
for (role, file) in &plan.sidecars {
let d = download_layer(
&dl_client,
&head_client,
&endpoint,
&owner,
&repo,
&commit,
file,
token.as_deref(),
layout_dir,
progress_key,
)
.await?;
layers.push(api::sidecar_layer(d, file, role));
}
if let Some(te_ref) = plan.text_encoder {
let mut d = pull_text_encoder(
&api_client,
&dl_client,
&head_client,
&host,
te_ref,
token.as_deref(),
layout_dir,
progress_key,
)
.await?;
d.annotations
.get_or_insert_with(BTreeMap::new)
.insert(oci::ANNOTATION_ROLE.to_string(), "text_encoder".to_string());
layers.push(d);
}
} else if let Some(mmproj) = mmproj {
layers.push(
download_layer(
&dl_client,
&head_client,
&endpoint,
&owner,
&repo,
&commit,
&mmproj,
token.as_deref(),
layout_dir,
progress_key,
)
.await?,
);
meta.vision = true;
}
if let Some(lic) = license {
let mut d = download_layer(
&dl_client,
&head_client,
&endpoint,
&owner,
&repo,
&commit,
&lic,
token.as_deref(),
layout_dir,
progress_key,
)
.await?;
d.media_type = oci::MEDIA_TYPE_MODEL_DOC_RAW.to_string();
layers.push(d);
}
oci::build_cncf_manifest(
layout_dir,
&meta,
&format!("{owner}/{repo}"),
&filepath_annotation,
&revision,
layers,
)?
}
Err(e) if !tag.is_empty() && tag != "latest" => return Err(e),
Err(_) => {
meta.format = "safetensors".to_string();
let to_download = api::select_downloadable_hf_files(&files);
if to_download.is_empty() {
anyhow::bail!("no model files found in repository {owner}/{repo}");
}
ensure_room(
reference,
layout_dir,
&to_download.iter().collect::<Vec<_>>(),
)?;
let max_streams = super::max_transfer_streams();
let mut indexed_layers: Vec<(usize, Descriptor)> =
stream::iter(to_download.iter().enumerate())
.map(|(i, f)| {
let dl_client = &dl_client;
let head_client = &head_client;
let endpoint = &endpoint;
let owner = &owner;
let repo = &repo;
let commit = &commit;
let token = token.as_deref();
async move {
let mut d = download_layer(
dl_client,
head_client,
endpoint,
owner,
repo,
commit,
f,
token,
layout_dir,
progress_key,
)
.await?;
d.media_type = api::safetensors_media_type(&f.path).to_string();
d.annotations = Some(BTreeMap::from([(
oci::ANNOTATION_FILEPATH.to_string(),
f.path.clone(),
)]));
Ok::<(usize, Descriptor), anyhow::Error>((i, d))
}
})
.buffer_unordered(max_streams)
.try_collect()
.await?;
indexed_layers.sort_by_key(|(i, _)| *i);
let layers: Vec<Descriptor> = indexed_layers.into_iter().map(|(_, d)| d).collect();
if api::is_diffusers_repo(&files) {
let class = api::diffusers_pipeline_class(layout_dir, &layers);
meta.diffusion_outputs = api::diffusers_outputs(class.as_deref());
}
oci::build_cncf_manifest(
layout_dir,
&meta,
&format!("{owner}/{repo}"),
"",
&revision,
layers,
)?
}
};
oci::write_manifest_ref(layout_dir, reference, manifest_desc)
}
fn ensure_room(reference: &str, layout_dir: &Path, files: &[&HfFile]) -> Result<()> {
let needed = bytes_missing(layout_dir, files);
crate::fsutil::ensure_room(
reference,
layout_dir,
needed,
crate::fsutil::available_space(layout_dir),
)
}
fn bytes_missing(layout_dir: &Path, files: &[&HfFile]) -> u64 {
let blobs = layout_dir.join("blobs").join("sha256");
let mut seen = std::collections::HashSet::new();
let mut needed = 0u64;
for f in files {
let stored = f.lfs.as_ref().is_some_and(|lfs| {
!seen.insert(lfs.oid.as_str())
|| std::fs::metadata(blobs.join(&lfs.oid))
.is_ok_and(|m| m.is_file() && m.len() as i64 == f.size)
});
if !stored {
needed = needed.saturating_add(f.size.max(0) as u64);
}
}
needed
}
fn basename(path: &str) -> String {
path.rsplit('/').next().unwrap_or(path).to_string()
}
#[allow(clippy::too_many_arguments)]
async fn pull_text_encoder(
api_client: &reqwest::Client,
dl_client: &reqwest::Client,
head_client: &reqwest::Client,
host: &str,
reference: &str,
token: Option<&str>,
layout_dir: &Path,
progress_key: &str,
) -> Result<Descriptor> {
let (_, owner, repo, tag) = api::parse_hf_ref(&format!("{host}/{reference}"))?;
let endpoint = super::hf_endpoint(host);
let info = api::fetch_model_info(api_client, &endpoint, &owner, &repo, token)
.await
.with_context(|| format!("text encoder {reference}"))?;
let commit = info.commit().to_string();
let files = api::fetch_files(api_client, &endpoint, &owner, &repo, &commit, token)
.await
.with_context(|| format!("text encoder {reference}"))?;
let shards = api::select_gguf(&files, &tag)?;
if shards.len() != 1 {
anyhow::bail!("text encoder {reference}: split GGUFs are not supported");
}
ensure_room(reference, layout_dir, &[&shards[0]])?;
let mut d = download_layer(
dl_client,
head_client,
&endpoint,
&owner,
&repo,
&commit,
&shards[0],
token,
layout_dir,
progress_key,
)
.await?;
d.annotations = Some(BTreeMap::from([(
oci::ANNOTATION_FILEPATH.to_string(),
basename(&shards[0].path),
)]));
Ok(d)
}
struct TmpFileGuard<'a>(&'a Path, bool);
impl TmpFileGuard<'_> {
fn disarm(&mut self) {
self.1 = false;
}
}
impl Drop for TmpFileGuard<'_> {
fn drop(&mut self) {
if self.1 {
let _ = std::fs::remove_file(self.0);
}
}
}
fn move_into_place(tmp_path: &Path, dest: &Path) -> Result<bool> {
if dest.exists() {
return Ok(false);
}
match std::fs::rename(tmp_path, dest) {
Ok(()) => Ok(true),
Err(_) if dest.exists() => Ok(false),
Err(e) => Err(e).with_context(|| format!("move downloaded blob to {}", dest.display())),
}
}
#[allow(clippy::too_many_arguments)]
async fn download_layer(
client: &reqwest::Client,
head_client: &reqwest::Client,
endpoint: &str,
owner: &str,
repo: &str,
commit: &str,
file: &HfFile,
token: Option<&str>,
layout_dir: &Path,
progress_key: &str,
) -> Result<Descriptor> {
let url = format!("{endpoint}{owner}/{repo}/resolve/{commit}/{}", file.path);
let label = format!("Pulling {}", basename(&file.path));
let meta = super::client::probe(&format!("HEAD {}", file.path), || {
download::head_metadata(head_client, url.clone(), token)
})
.await
.unwrap_or_default();
let size = if meta.size > 0 { meta.size } else { file.size };
let xet = meta.xet_hash.map(|hash| XetFileRef {
endpoint: endpoint.to_string(),
repo_type: "models".to_string(),
owner_repo: format!("{owner}/{repo}"),
revision: commit.to_string(),
hash,
size: size as u64,
sha256: meta.digest.clone(),
hf_token: token.map(str::to_string),
});
progress::add_total(progress_key, size);
if let Some(want) = &meta.digest {
let dest = layout_dir.join("blobs").join("sha256").join(want);
if std::fs::metadata(&dest)
.map(|m| m.is_file() && m.len() as i64 == size)
.unwrap_or(false)
{
progress::add_completed(progress_key, size);
return Ok(Descriptor {
media_type: oci::MEDIA_TYPE_MODEL_WEIGHT_RAW.to_string(),
digest: format!("sha256:{want}"),
size,
annotations: Some(BTreeMap::from([(
oci::ANNOTATION_FILEPATH.to_string(),
basename(&file.path),
)])),
..Default::default()
});
}
}
let tmp_dir = layout_dir.join("blobs").join("tmp");
std::fs::create_dir_all(&tmp_dir)?;
static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let unique = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let tmp_path = tmp_dir.join(format!(
"hf-{}-{unique}-{}.part",
std::process::id(),
sanitize(&file.path)
));
let mut tmp_guard = TmpFileGuard(&tmp_path, true);
let req = FetchRequest {
url,
token,
xet,
label: label.clone(),
};
let mut last_err: Option<anyhow::Error> = None;
let mut next_delay = None;
let digest = 'attempts: {
for attempt in 0..super::client::MAX_ATTEMPTS {
if let Some(delay) = next_delay.take() {
eprintln!(
"\n[llmman] retrying {label} (attempt {}/{}, wait {delay:?})",
attempt + 1,
super::client::MAX_ATTEMPTS
);
tokio::time::sleep(delay).await;
}
let got = match &req.xet {
Some(xet) => {
match xet_fetch::download_to_path(xet, &tmp_path, |n| {
progress::add_completed(progress_key, n as i64)
})
.await
.with_context(|| label.clone())
{
Ok(()) => sha256_of_file(&tmp_path).await,
Err(e) => Err(e),
}
}
None => {
let f = std::fs::File::create(&tmp_path)?;
let mut writer = HashingWriter::with_progress(f, |n| {
progress::add_completed(progress_key, n as i64)
});
download::fetch_once(client, &req, &mut writer)
.await
.and_then(|_| {
writer.flush()?;
Ok(format!("sha256:{}", writer.finish().1))
})
}
};
let result = got.and_then(|got| match &meta.digest {
Some(want) if format!("sha256:{want}") != got => {
anyhow::bail!("digest mismatch: expected sha256:{want}, got {got}")
}
_ => Ok(got),
});
match result {
Ok(digest) => break 'attempts digest,
Err(e) => {
eprintln!("[llmman] {label} error: {e:#}");
match download::should_retry(&e, attempt + 1) {
Some(delay) => next_delay = Some(delay),
None => {
last_err = Some(e);
break;
}
}
last_err = Some(e);
}
}
}
return Err(last_err.unwrap()).with_context(|| {
format!(
"download {} failed after {} attempts",
file.path,
super::client::MAX_ATTEMPTS
)
});
};
let dest_dir = layout_dir.join("blobs").join("sha256");
std::fs::create_dir_all(&dest_dir)?;
let dest = dest_dir.join(digest.trim_start_matches("sha256:"));
let total = std::fs::metadata(&tmp_path)?.len() as i64;
if move_into_place(&tmp_path, &dest)? {
tmp_guard.disarm(); }
Ok(Descriptor {
media_type: oci::MEDIA_TYPE_MODEL_WEIGHT_RAW.to_string(),
digest,
size: total,
annotations: Some(BTreeMap::from([(
oci::ANNOTATION_FILEPATH.to_string(),
basename(&file.path),
)])),
..Default::default()
})
}
fn sanitize(s: &str) -> String {
s.chars()
.map(|c| if c.is_ascii_alphanumeric() { c } else { '_' })
.collect()
}
async fn sha256_of_file(path: &Path) -> Result<String> {
let path = path.to_path_buf();
let hex = tokio::task::spawn_blocking(move || crate::fsutil::sha256_of_file(&path))
.await
.context("sha256 hashing task panicked")??;
Ok(format!("sha256:{hex}"))
}
#[cfg(test)]
mod tests {
use super::*;
fn lfs_file(path: &str, size: i64, oid: Option<&str>) -> HfFile {
HfFile {
path: path.to_string(),
size,
kind: "file".to_string(),
lfs: oid.map(|oid| api::HfLfs {
oid: oid.to_string(),
}),
}
}
#[test]
fn bytes_missing_skips_stored_blobs_and_repeated_content() {
let dir = tempfile::tempdir().unwrap();
let blobs = dir.path().join("blobs").join("sha256");
std::fs::create_dir_all(&blobs).unwrap();
std::fs::write(blobs.join("aa"), [0u8; 100]).unwrap();
std::fs::write(blobs.join("bb"), [0u8; 5]).unwrap();
let stored = lfs_file("stored.gguf", 100, Some("aa"));
let truncated = lfs_file("truncated.gguf", 200, Some("bb"));
let absent = lfs_file("absent.gguf", 300, Some("cc"));
let again = lfs_file("copy/absent.gguf", 300, Some("cc"));
let config = lfs_file("config.json", 7, None);
assert_eq!(
bytes_missing(dir.path(), &[&stored, &truncated, &absent, &again, &config]),
200 + 300 + 7
);
}
#[test]
fn bytes_missing_saturates_on_overflow() {
let dir = tempfile::tempdir().unwrap();
let files: Vec<HfFile> = ["a", "b", "c"]
.iter()
.map(|oid| lfs_file(oid, i64::MAX, Some(oid)))
.collect();
let files: Vec<&HfFile> = files.iter().collect();
assert_eq!(bytes_missing(dir.path(), &files), u64::MAX);
}
#[test]
fn tmp_file_guard_removes_the_file_on_drop_by_default() {
let dir = std::env::temp_dir().join(format!("llmman-tmpguard-test-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("armed.part");
std::fs::write(&path, b"partial").unwrap();
{
let _guard = TmpFileGuard(&path, true);
}
assert!(
!path.exists(),
"an armed guard must remove its file on drop"
);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn tmp_file_guard_leaves_the_file_alone_once_disarmed() {
let dir = std::env::temp_dir().join(format!(
"llmman-tmpguard-test-disarm-{}",
std::process::id()
));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("disarmed.part");
std::fs::write(&path, b"done").unwrap();
{
let mut guard = TmpFileGuard(&path, true);
guard.disarm();
}
assert!(path.exists(), "a disarmed guard must not remove its file");
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn move_into_place_moves_the_file_when_dest_is_absent() {
let dir = std::env::temp_dir().join(format!("llmman-move-test-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let tmp = dir.join("a.part");
let dest = dir.join("dest");
std::fs::write(&tmp, b"content").unwrap();
assert!(move_into_place(&tmp, &dest).unwrap());
assert!(!tmp.exists());
assert!(dest.exists());
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn move_into_place_tolerates_a_destination_that_already_exists() {
let dir =
std::env::temp_dir().join(format!("llmman-move-test-existing-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let tmp = dir.join("a.part");
let dest = dir.join("dest");
std::fs::write(&tmp, b"content").unwrap();
std::fs::write(&dest, b"content").unwrap();
assert!(!move_into_place(&tmp, &dest).unwrap());
assert!(
tmp.exists(),
"move_into_place itself must leave the redundant tmp file for the caller's guard to clean up"
);
std::fs::remove_dir_all(&dir).ok();
}
const SHA: &str = "7ae557604adf67be50417f59c2c2f167def9a775";
const WEIGHTS: &[u8] = b"not really a gguf";
async fn mock_hub() -> String {
mock_hub_at(format!("{{\"sha\":\"{SHA}\"}}"), SHA).await
}
async fn mock_hub_at(model: String, revision: &str) -> String {
let tree = || async {
format!(
"[{{\"type\":\"file\",\"path\":\"model.gguf\",\"size\":{}}}]",
WEIGHTS.len()
)
};
let app = axum::Router::new()
.route(
"/api/models/o/r",
axum::routing::get(move || async move { model }),
)
.route(
&format!("/api/models/o/r/tree/{revision}"),
axum::routing::get(tree),
)
.route(
&format!("/o/r/resolve/{revision}/model.gguf"),
axum::routing::get(|| async { WEIGHTS }),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let endpoint = format!("http://{}/", listener.local_addr().unwrap());
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
endpoint
}
#[tokio::test]
async fn a_plain_pull_records_the_hub_sha_as_the_revision() {
let endpoint = mock_hub().await;
let dir = tempfile::tempdir().unwrap();
pull_from(
"hf.co/o/r:latest",
dir.path(),
"hf-pull-plain-revision",
|_| endpoint.clone(),
)
.await
.unwrap();
let (manifest, config) = oci::read_manifest_and_config(dir.path(), "hf.co/o/r:latest");
assert_eq!(
manifest.annotations.as_ref().unwrap()[oci::ANNOTATION_REVISION],
SHA
);
assert_eq!(config["descriptor"]["revision"], SHA);
}
#[tokio::test]
async fn a_pull_from_a_hub_without_a_sha_records_no_revision() {
let endpoint = mock_hub_at("{}".to_string(), "main").await;
let dir = tempfile::tempdir().unwrap();
pull_from(
"hf.co/o/r:latest",
dir.path(),
"hf-pull-no-sha-revision",
|_| endpoint.clone(),
)
.await
.unwrap();
let (manifest, config) = oci::read_manifest_and_config(dir.path(), "hf.co/o/r:latest");
assert!(manifest
.annotations
.as_ref()
.is_none_or(|a| !a.contains_key(oci::ANNOTATION_REVISION)));
assert!(config["descriptor"].get("revision").is_none());
}
#[tokio::test]
async fn a_file_served_without_xet_is_hashed_and_counted_as_it_downloads() {
use sha2::Digest as _;
let body: Vec<u8> = (0..1_000_000u32).map(|i| (i % 251) as u8).collect();
let hex = hex::encode(sha2::Sha256::digest(&body));
let headers = [
("x-linked-etag", format!("\"{hex}\"")),
("x-linked-size", body.len().to_string()),
];
let served = body.clone();
let app = axum::Router::new().route(
"/o/r/resolve/c/model.gguf",
axum::routing::get(move || {
let (headers, served) = (headers.clone(), served.clone());
async move { (headers, served) }
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let endpoint = format!("http://{}/", listener.local_addr().unwrap());
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let dir = tempfile::tempdir().unwrap();
oci::ensure_layout(dir.path()).unwrap();
let progress_key = "hf-pull-plain-http-download";
progress::reset(progress_key, "pulling");
let descriptor = download_layer(
&crate::hf::download_client().unwrap(),
&crate::hf::head_client().unwrap(),
&endpoint,
"o",
"r",
"c",
&lfs_file("model.gguf", body.len() as i64, None),
None,
dir.path(),
progress_key,
)
.await
.unwrap();
assert_eq!(descriptor.digest, format!("sha256:{hex}"));
let blob = dir.path().join("blobs").join("sha256").join(&hex);
assert_eq!(std::fs::read(blob).unwrap(), body);
let progress = progress::poll(progress_key);
let size = body.len() as i64;
assert_eq!((progress.completed, progress.total), (size, size));
}
}