pub mod gcs;
pub mod local;
pub mod modelscope;
pub mod ngc;
pub mod s3;
use std::io::Write as _;
use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
use bytes::Bytes;
use futures::stream::{Stream, StreamExt as _};
use crate::hf::oci::{self, ModelMeta};
use crate::hf::progress;
const SCHEMES: &[&str] = &["ms://", "modelscope://", "ngc://", "s3://", "gs://"];
pub fn handles(reference: &str) -> bool {
reference.starts_with('/') || SCHEMES.iter().any(|s| reference.starts_with(s))
}
pub async fn pull(reference: &str, layout_dir: &Path, progress_key: &str) -> Result<()> {
let _guard = progress::DoneGuard(progress_key);
pull_into(
reference,
&Target {
layout_dir,
progress_key,
store_as: None,
},
)
.await
}
pub async fn transfer(reference: &str, destination: &str) -> Result<crate::ffi::TransferOutcome> {
let tmp = std::env::temp_dir().join(format!(
"llmman-source-transfer-{}-{}",
std::process::id(),
unique()
));
let result = pull_into(
reference,
&Target {
layout_dir: &tmp,
progress_key: "",
store_as: Some(destination),
},
)
.await
.and_then(|()| {
crate::ffi::push(
tmp.to_str()
.context("temp layout path is not valid UTF-8")?,
destination,
)
})
.and_then(|()| crate::hf::oci::read_manifest_ref(&tmp, destination))
.map(|desc| crate::ffi::TransferOutcome::new(true, desc.digest));
let _ = std::fs::remove_dir_all(&tmp);
result
}
pub(crate) struct Target<'a> {
pub layout_dir: &'a Path,
pub progress_key: &'a str,
pub store_as: Option<&'a str>,
}
impl Target<'_> {
fn store_ref<'a>(&'a self, canonical: &'a str) -> &'a str {
self.store_as.unwrap_or(canonical)
}
fn report_cached(&self, canonical: &str, label: &str) -> bool {
if self.store_as.is_some() {
return false;
}
if oci::cached_layer_name(self.layout_dir, canonical).is_none() {
return false;
}
eprintln!("Cached {label}");
true
}
}
async fn pull_into(reference: &str, target: &Target<'_>) -> Result<()> {
oci::ensure_layout(target.layout_dir)?;
progress::set_status(target.progress_key, "pulling");
if let Some(rest) = strip_any_prefix(reference, &["ms://", "modelscope://"]) {
return modelscope::pull(reference, rest, target).await;
}
if let Some(rest) = reference.strip_prefix("ngc://") {
return ngc::pull(reference, rest, target).await;
}
if let Some(rest) = reference.strip_prefix("s3://") {
return s3::pull(reference, rest, target).await;
}
if let Some(rest) = reference.strip_prefix("gs://") {
return gcs::pull(reference, rest, target).await;
}
if reference.starts_with('/') {
return local::pull(reference, target);
}
anyhow::bail!("{reference}: not a source reference this module handles")
}
fn ensure_room(
reference: &str,
target: &Target<'_>,
sizes: impl IntoIterator<Item = i64>,
) -> Result<()> {
let needed = sizes
.into_iter()
.fold(0u64, |total, size| total.saturating_add(size.max(0) as u64));
crate::fsutil::ensure_room(
reference,
target.layout_dir,
needed,
crate::fsutil::available_space(target.layout_dir),
)
}
fn strip_any_prefix<'a>(s: &'a str, prefixes: &[&str]) -> Option<&'a str> {
prefixes.iter().find_map(|p| s.strip_prefix(p))
}
pub(crate) fn split_bucket_prefix<'a>(
rest: &'a str,
reference: &str,
) -> Result<(&'a str, &'a str)> {
rest.split_once('/').with_context(|| {
let scheme = reference.split_once("://").map(|(s, _)| s).unwrap_or("");
format!("invalid {scheme} reference {reference:?}: expected {scheme}://bucket/prefix")
})
}
pub(crate) fn relative_to_prefix<'a>(key: &'a str, prefix: &str) -> Option<&'a str> {
let rest = key.strip_prefix(prefix)?;
if !(rest.starts_with('/') || prefix.ends_with('/') || prefix.is_empty()) {
return None;
}
let rel = rest.trim_start_matches('/');
(!rel.is_empty()).then_some(rel)
}
pub fn classify_file(name: &str) -> &'static str {
let lower = name.rsplit('/').next().unwrap_or(name).to_lowercase();
let ext = lower.rsplit_once('.').map(|(_, e)| e).unwrap_or("");
match ext {
"safetensors" | "bin" | "pt" | "pth" | "gguf" | "ggml" | "gguf_v2" | "ot" | "engine"
| "trt" | "onnx" => oci::MEDIA_TYPE_MODEL_WEIGHT_RAW,
"json" | "yaml" | "yml" | "toml" | "ini" | "cfg" | "conf" | "model" | "tiktoken"
| "vocab" | "merges" | "spm" | "jinja" => oci::MEDIA_TYPE_MODEL_WEIGHT_CONFIG_RAW,
"txt" if lower.contains("vocab") || lower.contains("merges") => {
oci::MEDIA_TYPE_MODEL_WEIGHT_CONFIG_RAW
}
"py" | "sh" | "js" | "ts" => oci::MEDIA_TYPE_MODEL_CODE_RAW,
_ => oci::MEDIA_TYPE_MODEL_DOC_RAW,
}
}
pub fn is_safe_relative_path(rel_path: &str) -> bool {
!rel_path.is_empty()
&& !rel_path.starts_with('/')
&& !rel_path.contains('\\')
&& !rel_path.contains(':')
&& rel_path
.split('/')
.all(|c| !matches!(c, ".." | ".") && !c.is_empty())
}
pub(crate) fn should_pack(path: &str) -> bool {
let base = path.rsplit('/').next().unwrap_or(path).to_lowercase();
is_safe_relative_path(path)
&& !base.starts_with('.')
&& (classify_file(path) != oci::MEDIA_TYPE_MODEL_DOC_RAW
|| base.starts_with("readme")
|| base.starts_with("licen")
|| base.ends_with(".txt")
|| base.ends_with(".md"))
}
pub(crate) struct PackFile {
pub(crate) local_path: PathBuf,
pub(crate) relative_path: String,
pub(crate) owned: bool,
}
impl Drop for PackFile {
fn drop(&mut self) {
if self.owned {
let _ = std::fs::remove_file(&self.local_path);
}
}
}
pub(crate) async fn download_to_pack_file<E>(
target: &Target<'_>,
tmp_prefix: &str,
kind: &str,
rel_path: &str,
size: i64,
body: impl Stream<Item = std::result::Result<Bytes, E>>,
) -> Result<PackFile>
where
E: std::error::Error + Send + Sync + 'static,
{
let base = rel_path.rsplit('/').next().unwrap_or(rel_path);
eprintln!("Pulling {base}");
progress::add_total(target.progress_key, size);
let tmp_dir = target.layout_dir.join("blobs").join("tmp");
std::fs::create_dir_all(&tmp_dir)?;
let tmp_path = tmp_dir.join(format!(
"{tmp_prefix}-{}-{}-{}.part",
std::process::id(),
unique(),
sanitize(rel_path)
));
let pack = PackFile {
local_path: tmp_path.clone(),
relative_path: rel_path.to_string(),
owned: true,
};
let mut file = std::fs::File::create(&tmp_path)
.with_context(|| format!("{kind}: create staging file for {rel_path}"))?;
let mut body = std::pin::pin!(body);
while let Some(chunk) = body.next().await {
let chunk = chunk.with_context(|| format!("{kind} download {rel_path}"))?;
file.write_all(&chunk)
.with_context(|| format!("{kind} write {rel_path}"))?;
progress::add_completed(target.progress_key, chunk.len() as i64);
}
file.flush()
.with_context(|| format!("{kind} write {rel_path}"))?;
drop(file);
eprintln!("Pulled {base}");
Ok(pack)
}
pub(crate) fn pack_as_model_pack(
target: &Target<'_>,
canonical_ref: &str,
model_repo: &str,
files: Vec<PackFile>,
no_files_err: String,
) -> Result<()> {
if files.is_empty() {
anyhow::bail!(no_files_err);
}
let mut layers = Vec::with_capacity(files.len());
for f in &files {
let mut desc = oci::write_blob_from_file(
target.layout_dir,
classify_file(&f.relative_path),
&f.local_path,
f.owned,
)
.with_context(|| format!("store {}", f.relative_path))?;
desc.annotations = Some(std::collections::BTreeMap::from([(
oci::ANNOTATION_FILEPATH.to_string(),
f.relative_path.clone(),
)]));
layers.push(desc);
}
let meta = ModelMeta {
format: weight_format(&files).unwrap_or_default().to_string(),
..Default::default()
};
let manifest_desc =
oci::build_cncf_manifest(target.layout_dir, &meta, model_repo, "", "", layers)?;
oci::write_manifest_ref(
target.layout_dir,
target.store_ref(canonical_ref),
manifest_desc,
)
}
fn weight_format(files: &[PackFile]) -> Option<&'static str> {
let mut format = None;
for f in files {
let lower = f.relative_path.to_lowercase();
if lower.ends_with(".gguf") || lower.ends_with(".ggml") {
return Some("gguf");
}
if lower.ends_with(".safetensors") && format.is_none() {
format = Some("safetensors");
}
}
format
}
fn sanitize(s: &str) -> String {
s.chars()
.map(|c| if c.is_ascii_alphanumeric() { c } else { '_' })
.collect()
}
fn unique() -> u64 {
static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn handles_matches_every_documented_scheme_and_local_paths() {
for r in [
"ms://owner/repo",
"modelscope://owner/repo",
"ngc://org/team/model",
"s3://bucket/prefix",
"gs://bucket/prefix",
"/abs/path/to/model",
] {
assert!(handles(r), "{r} should be handled by crate::sources");
}
}
#[test]
fn handles_leaves_hf_and_registry_references_alone() {
for r in [
"hf://owner/repo",
"huggingface://owner/repo",
"hf.co/owner/repo",
"docker.io/library/alpine:latest",
"registry.example.com:5000/owner/repo:v1",
] {
assert!(!handles(r), "{r} must not be claimed by crate::sources");
}
}
#[test]
fn classify_file_maps_weights_config_code_and_docs() {
assert_eq!(
classify_file("model-00001-of-00002.safetensors"),
oci::MEDIA_TYPE_MODEL_WEIGHT_RAW
);
assert_eq!(
classify_file("model.onnx"),
oci::MEDIA_TYPE_MODEL_WEIGHT_RAW
);
assert_eq!(
classify_file("config.json"),
oci::MEDIA_TYPE_MODEL_WEIGHT_CONFIG_RAW
);
assert_eq!(
classify_file("chat_template.jinja"),
oci::MEDIA_TYPE_MODEL_WEIGHT_CONFIG_RAW
);
assert_eq!(classify_file("handler.py"), oci::MEDIA_TYPE_MODEL_CODE_RAW);
assert_eq!(classify_file("README.md"), oci::MEDIA_TYPE_MODEL_DOC_RAW);
}
#[test]
fn classify_file_treats_vocab_and_merges_text_files_as_config() {
assert_eq!(
classify_file("vocab.txt"),
oci::MEDIA_TYPE_MODEL_WEIGHT_CONFIG_RAW
);
assert_eq!(
classify_file("tokenizer/merges.txt"),
oci::MEDIA_TYPE_MODEL_WEIGHT_CONFIG_RAW
);
assert_eq!(classify_file("notes.txt"), oci::MEDIA_TYPE_MODEL_DOC_RAW);
}
#[test]
fn should_pack_accepts_every_weight_format_classify_file_knows() {
for f in [
"model.gguf",
"model.ggml",
"model.onnx",
"model.engine",
"model.trt",
"model.safetensors",
"config.json",
"tokenizer.model",
"chat_template.jinja",
"handler.py",
"README.md",
"LICENSE",
"vocab.txt",
] {
assert!(should_pack(f), "{f} should be packed");
}
}
#[test]
fn should_pack_skips_dotfiles_and_unknown_binaries() {
for f in [".gitattributes", ".DS_Store", "weights.tflite", "cache.db"] {
assert!(!should_pack(f), "{f} should not be packed");
}
}
#[test]
fn is_safe_relative_path_rejects_traversal_and_absolute_paths() {
for p in [
"../../id_rsa",
"a/../../b",
"/etc/passwd",
"..",
"a//b",
"..\\windows",
"C:/weights.safetensors",
"",
] {
assert!(!is_safe_relative_path(p), "{p:?} must be rejected");
assert!(!should_pack(p), "{p:?} must not be packed");
}
for p in ["config.json", "sub/dir/model.safetensors"] {
assert!(is_safe_relative_path(p), "{p:?} must be accepted");
}
}
#[test]
fn weight_format_prefers_gguf_and_is_none_without_weights() {
let f = |p: &str| PackFile {
local_path: PathBuf::from(p),
relative_path: p.to_string(),
owned: false,
};
assert_eq!(
weight_format(&[f("model.safetensors"), f("model.gguf")]),
Some("gguf")
);
assert_eq!(
weight_format(&[f("config.json"), f("model.safetensors")]),
Some("safetensors")
);
assert_eq!(weight_format(&[f("model.onnx")]), None);
}
#[test]
fn split_bucket_prefix_requires_a_prefix() {
assert_eq!(
split_bucket_prefix("bucket/some/prefix", "s3://bucket/some/prefix").unwrap(),
("bucket", "some/prefix")
);
assert!(split_bucket_prefix("bucket", "s3://bucket").is_err());
}
#[test]
fn relative_to_prefix_strips_the_prefix_and_skips_the_prefix_entry_itself() {
assert_eq!(
relative_to_prefix("models/llama/config.json", "models/llama"),
Some("config.json")
);
assert_eq!(
relative_to_prefix("models/llama/config.json", "models/llama/"),
Some("config.json")
);
assert_eq!(
relative_to_prefix("models/llama/", "models/llama"),
None,
"the prefix \"directory\" entry has nothing left over"
);
}
#[test]
fn relative_to_prefix_rejects_a_key_that_only_shares_a_textual_prefix() {
assert_eq!(
relative_to_prefix("models/llama-2/config.json", "models/llama"),
None
);
assert_eq!(relative_to_prefix("other/config.json", "models"), None);
}
}