use std::collections::HashSet;
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use anyhow::{anyhow, Context};
use serde::{Deserialize, Serialize};
use sha2::{Digest as Sha2Digest, Sha256};
static WRITE_REF_COUNTER: AtomicU64 = AtomicU64::new(0);
const MANIFESTS_V2_DIR: &str = "manifests-v2";
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Descriptor {
pub media_type: String,
pub digest: String,
pub size: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub annotations: Option<std::collections::HashMap<String, String>>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Manifest {
pub schema_version: u32,
pub media_type: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub artifact_type: Option<String>,
pub config: Descriptor,
pub layers: Vec<Descriptor>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub annotations: Option<std::collections::HashMap<String, String>>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct CncfConfigDescriptor {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub created_at: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct CncfConfigConfig {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub format: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CncfModelFs {
#[serde(rename = "type")]
pub fs_type: String,
pub diff_ids: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CncfModelConfig {
pub descriptor: CncfConfigDescriptor,
pub config: CncfConfigConfig,
pub modelfs: CncfModelFs,
}
#[derive(Debug)]
pub struct NotFound(pub String);
impl std::fmt::Display for NotFound {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "image not found: {}", self.0)
}
}
impl std::error::Error for NotFound {}
#[derive(Debug)]
pub struct SourceNotFound {
cause: anyhow::Error,
}
impl std::fmt::Display for SourceNotFound {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(&self.cause, f)
}
}
impl std::error::Error for SourceNotFound {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
self.cause.source()
}
}
pub fn source_not_found(cause: anyhow::Error) -> anyhow::Error {
anyhow::Error::new(SourceNotFound { cause })
}
pub fn is_source_not_found(err: &anyhow::Error) -> bool {
err.chain().any(|e| e.is::<SourceNotFound>())
}
#[derive(Debug)]
pub struct ImageSummary {
pub reference: String,
pub digest: String,
#[allow(dead_code)]
pub media_type: String,
pub size: u64,
pub modified_at: Option<std::time::SystemTime>,
}
pub struct OciStore {
root: PathBuf,
}
impl OciStore {
pub fn open(root: impl AsRef<Path>) -> anyhow::Result<Self> {
let root = root.as_ref().to_path_buf();
fs::create_dir_all(&root)?;
fs::create_dir_all(root.join("blobs").join("sha256"))?;
fs::create_dir_all(root.join(MANIFESTS_V2_DIR))?;
let marker = root.join("oci-layout");
if !marker.exists() {
fs::write(&marker, r#"{"imageLayoutVersion":"1.0.0"}"#)?;
}
Ok(Self { root })
}
pub fn root(&self) -> &Path {
&self.root
}
fn ref_path(&self, reference: &str) -> PathBuf {
manifest_ref_path(&self.root, reference)
}
fn legacy_ref_path(&self, reference: &str) -> PathBuf {
legacy_manifest_ref_path(&self.root, reference)
}
fn read_ref_at(&self, path: &Path, reference: &str) -> anyhow::Result<Descriptor> {
let data = fs::read(path).with_context(|| format!("read manifest ref {reference}"))?;
serde_json::from_slice(&data).context("parse manifest ref")
}
fn read_ref_if_present(
&self,
path: &Path,
reference: &str,
) -> anyhow::Result<Option<Descriptor>> {
match self.read_ref_at(path, reference) {
Ok(desc) => Ok(Some(desc)),
Err(err)
if err.chain().any(|cause| {
cause
.downcast_ref::<std::io::Error>()
.is_some_and(|io| io.kind() == std::io::ErrorKind::NotFound)
}) =>
{
Ok(None)
}
Err(err) => Err(err),
}
}
fn read_legacy_ref(&self, reference: &str) -> anyhow::Result<Option<Descriptor>> {
let path = self.legacy_ref_path(reference);
match fs::symlink_metadata(&path) {
Ok(meta) if !meta.is_file() => return Ok(None),
Ok(_) => {}
Err(err) if missing_legacy_ref(&err) => return Ok(None),
Err(err) => {
return Err(err).with_context(|| format!("inspect manifest ref {reference}"))
}
}
match self.read_ref_if_present(&path, reference) {
Err(err) if err.chain().any(|cause| cause.is::<serde_json::Error>()) => Ok(None),
result => result,
}
}
fn write_ref(&self, reference: &str, desc: &Descriptor) -> anyhow::Result<()> {
let path = self.ref_path(reference);
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)?;
}
let n = WRITE_REF_COUNTER.fetch_add(1, Ordering::Relaxed);
let mut tmp = path.clone().into_os_string();
tmp.push(format!(".{}.{n}.tmp", std::process::id()));
let tmp = PathBuf::from(tmp);
if let Err(e) = fs::write(&tmp, serde_json::to_string_pretty(desc)?) {
let _ = fs::remove_file(&tmp);
return Err(e.into());
}
if let Err(e) = fs::rename(&tmp, &path) {
let _ = fs::remove_file(&tmp);
return Err(e.into());
}
Ok(())
}
fn remove_ref(&self, reference: &str) -> anyhow::Result<bool> {
let path = self.ref_path(reference);
let legacy = self.legacy_ref_path(reference);
let remove_new = match self.read_ref_if_present(&path, reference) {
Ok(desc) => desc
.as_ref()
.and_then(stored_ref_name)
.is_some_and(|stored| same_ref_name(stored, reference)),
Err(err) if err.chain().any(|cause| cause.is::<serde_json::Error>()) => true,
Err(err) => return Err(err),
};
let remove_legacy = self
.read_legacy_ref(reference)?
.as_ref()
.and_then(stored_ref_name)
.is_some_and(|stored| same_ref_name(stored, reference));
for candidate in [
remove_new.then_some(&path),
remove_legacy.then_some(&legacy),
]
.into_iter()
.flatten()
{
fs::remove_file(candidate)
.with_context(|| format!("remove manifest ref {reference}"))?;
}
Ok(remove_new || remove_legacy)
}
pub(crate) fn list_refs(&self) -> Vec<Descriptor> {
let mut out = Vec::new();
collect_refs(&self.root.join(MANIFESTS_V2_DIR), &mut out);
collect_refs(&self.root.join("manifests"), &mut out);
dedup_refs(&mut out);
out
}
pub(crate) fn list_refs_strict(&self) -> anyhow::Result<Vec<Descriptor>> {
let mut out = Vec::new();
collect_refs_strict(&self.root.join(MANIFESTS_V2_DIR), &mut out)?;
collect_refs_strict(&self.root.join("manifests"), &mut out)?;
Ok(out)
}
pub(crate) fn blob_path(&self, digest: &str) -> anyhow::Result<PathBuf> {
let (algo, hex) = split_digest(digest)?;
Ok(self.root.join("blobs").join(algo).join(hex))
}
pub fn write_blob(&self, media_type: &str, data: &[u8]) -> anyhow::Result<Descriptor> {
let hex = hex::encode(Sha256::digest(data));
let digest = format!("sha256:{}", hex);
let path = self.root.join("blobs").join("sha256").join(&hex);
if path.exists() {
let _ = fs::OpenOptions::new()
.write(true)
.open(&path)
.and_then(|f| f.set_modified(std::time::SystemTime::now()));
} else {
let tmp = path.with_extension("tmp");
fs::write(&tmp, data)?;
fs::rename(tmp, &path)?;
}
Ok(Descriptor {
media_type: media_type.into(),
digest,
size: data.len() as u64,
annotations: None,
})
}
pub fn write_blob_file(
&self,
media_type: &str,
path: impl AsRef<Path>,
) -> anyhow::Result<Descriptor> {
let desc =
crate::hf::oci::write_blob_from_file(&self.root, media_type, path.as_ref(), false)?;
Ok(Descriptor {
media_type: desc.media_type,
digest: desc.digest,
size: u64::try_from(desc.size).context("blob size is negative")?,
annotations: None,
})
}
pub fn read_blob(&self, digest: &str) -> anyhow::Result<Vec<u8>> {
fs::read(self.blob_path(digest)?).with_context(|| format!("read blob {}", digest))
}
pub fn write_manifest(&self, manifest: &Manifest) -> anyhow::Result<Descriptor> {
let data = serde_json::to_vec(manifest)?;
self.write_blob("application/vnd.oci.image.manifest.v1+json", &data)
}
pub fn read_manifest(&self, digest: &str) -> anyhow::Result<Manifest> {
let data = self.read_blob(digest)?;
serde_json::from_slice(&data).context("parse manifest")
}
pub fn tag(&self, mut desc: Descriptor, reference: &str) -> anyhow::Result<()> {
let mut ann = desc.annotations.take().unwrap_or_default();
ann.insert(
"org.opencontainers.image.ref.name".into(),
reference.to_string(),
);
desc.annotations = Some(ann);
self.write_ref(reference, &desc)
}
pub fn find(&self, reference: &str) -> anyhow::Result<Descriptor> {
let path = self.ref_path(reference);
for current in [true, false] {
let found = if current {
self.read_ref_if_present(&path, reference)?
} else {
self.read_legacy_ref(reference)?
};
let Some(desc) = found else { continue };
let same_name =
stored_ref_name(&desc).is_some_and(|stored| same_ref_name(stored, reference));
match split_ref_digest(reference).1 {
Some(digest) if !desc.digest.eq_ignore_ascii_case(digest) => {}
Some(_) | None if same_name => return Ok(desc),
_ => {}
}
}
let refs = self.list_refs();
matching_index(&refs, reference)
.map(|i| refs[i].clone())
.ok_or_else(|| NotFound(reference.to_string()).into())
}
pub fn total_size(&self, desc: &Descriptor) -> u64 {
self.read_manifest(&desc.digest)
.map(|manifest| manifest.layers.iter().map(|l| l.size).sum())
.unwrap_or(desc.size)
}
pub fn list(&self) -> anyhow::Result<Vec<ImageSummary>> {
let refs = self.list_refs();
Ok(refs
.into_iter()
.map(|m| {
let reference = m
.annotations
.as_ref()
.and_then(|a| a.get("org.opencontainers.image.ref.name"))
.cloned()
.unwrap_or_else(|| m.digest.clone());
let modified_at = self
.blob_path(&m.digest)
.ok()
.and_then(|p| fs::metadata(p).ok())
.and_then(|meta| meta.modified().ok());
let size = self.total_size(&m);
ImageSummary {
reference,
digest: m.digest,
media_type: m.media_type,
size,
modified_at,
}
})
.collect())
}
pub fn remove(&self, reference: &str) -> anyhow::Result<String> {
if self.remove_ref(reference)? {
return Ok(reference.to_owned());
}
let refs = self.list_refs();
let (base, digest) = split_ref_digest(reference);
if digest.is_some() {
let spelled = (default_tag(base) == base).then_some(base);
let held_by: Vec<&str> = refs
.iter()
.filter(|m| ref_matches_precise(m, reference))
.filter_map(stored_ref_name)
.collect();
if held_by.len() > 1
&& !held_by
.iter()
.any(|s| spelled.is_some_and(|t| default_tag(s) == t))
{
return Err(anyhow!(
"{reference} is held by more than one tag: {}; name the tag to remove",
held_by.join(", ")
));
}
}
let Some(i) = matching_index(&refs, reference) else {
return Err(anyhow!("image not found: {}", reference));
};
let Some(name) = stored_ref_name(&refs[i]) else {
return Err(anyhow!("image not found: {}", reference));
};
anyhow::ensure!(self.remove_ref(name)?, "image not found: {reference}");
Ok(name.to_owned())
}
pub fn build(
&self,
src_dir: impl AsRef<Path>,
reference: &str,
labels: &std::collections::HashMap<String, String>,
raw: bool,
) -> anyhow::Result<Descriptor> {
use walkdir::WalkDir;
let src_dir = src_dir.as_ref();
let canonical_src = dunce::canonicalize(src_dir)
.with_context(|| format!("canonicalize source dir {}", src_dir.display()))?;
let mut layers: Vec<Descriptor> = Vec::new();
let mut format: Option<&'static str> = None;
for entry in WalkDir::new(&canonical_src).follow_links(true) {
let entry = entry?;
let canonical_entry = dunce::canonicalize(entry.path())
.with_context(|| format!("canonicalize {}", entry.path().display()))?;
if !canonical_entry.starts_with(&canonical_src) {
return Err(anyhow!(
"path {} resolves to {} which is outside source dir {}",
entry.path().display(),
canonical_entry.display(),
canonical_src.display()
));
}
if !entry.file_type().is_file() {
continue;
}
let rel = entry
.path()
.strip_prefix(&canonical_src)
.with_context(|| {
format!(
"strip prefix {:?} from entry {:?}",
canonical_src,
entry.path()
)
})?
.to_string_lossy()
.replace('\\', "/");
let (media_type, weight_type) = if raw {
(
crate::sources::classify_file(&rel),
crate::hf::oci::MEDIA_TYPE_MODEL_WEIGHT_RAW,
)
} else {
(classify_model_layer(&rel), WEIGHT_TAR_MEDIA_TYPE)
};
if media_type == weight_type {
let lower = rel.to_lowercase();
if lower.ends_with(".gguf") || lower.ends_with(".ggml") {
format = Some("gguf");
} else if lower.ends_with(".safetensors") && format.is_none() {
format = Some("safetensors");
}
}
let mut desc = if raw {
self.write_blob_file(media_type, &canonical_entry)?
} else {
let tar_data = make_single_file_tar(&canonical_entry, &rel)?;
self.write_blob(media_type, &tar_data)?
};
desc.annotations = Some({
let mut m = std::collections::HashMap::new();
m.insert("org.cncf.model.filepath".into(), rel.clone());
m.insert("org.opencontainers.image.title".into(), rel);
m
});
layers.push(desc);
}
if layers.is_empty() {
return Err(anyhow!("no files found in {}", src_dir.display()));
}
let cncf_config = CncfModelConfig {
descriptor: CncfConfigDescriptor {
created_at: Some(chrono::Utc::now().to_rfc3339()),
},
config: CncfConfigConfig {
format: format.map(str::to_string),
},
modelfs: CncfModelFs {
fs_type: "layers".into(),
diff_ids: layers.iter().map(|l| l.digest.clone()).collect(),
},
};
let config_data = serde_json::to_vec(&cncf_config)?;
let config_desc =
self.write_blob("application/vnd.cncf.model.config.v1+json", &config_data)?;
let manifest_annotations = if labels.is_empty() {
None
} else {
Some(labels.clone())
};
let manifest = Manifest {
schema_version: 2,
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
artifact_type: Some("application/vnd.cncf.model.manifest.v1+json".into()),
config: config_desc,
layers,
annotations: manifest_annotations,
};
let manifest_desc = self.write_manifest(&manifest)?;
self.tag(manifest_desc.clone(), reference)?;
Ok(manifest_desc)
}
}
const WEIGHT_TAR_MEDIA_TYPE: &str = "application/vnd.cncf.model.weight.v1.tar";
const WEIGHT_CONFIG_TAR_MEDIA_TYPE: &str = "application/vnd.cncf.model.weight.config.v1.tar";
const DOC_TAR_MEDIA_TYPE: &str = "application/vnd.cncf.model.doc.v1.tar";
const CODE_TAR_MEDIA_TYPE: &str = "application/vnd.cncf.model.code.v1.tar";
fn classify_model_layer(rel_path: &str) -> &'static str {
let lower = rel_path.to_lowercase();
let base = Path::new(&lower)
.file_name()
.and_then(|f| f.to_str())
.unwrap_or(lower.as_str());
let ext = Path::new(base)
.extension()
.and_then(|e| e.to_str())
.unwrap_or("");
match ext {
"safetensors" | "bin" | "pt" | "pth" | "gguf" | "ggml" | "ot" | "engine" | "trt"
| "onnx" => WEIGHT_TAR_MEDIA_TYPE,
"json" | "yaml" | "yml" | "toml" | "ini" | "cfg" | "conf" | "model" | "tiktoken"
| "vocab" | "merges" | "spm" | "jinja" => WEIGHT_CONFIG_TAR_MEDIA_TYPE,
"txt" => {
if base.contains("vocab") || base.contains("merges") {
WEIGHT_CONFIG_TAR_MEDIA_TYPE
} else {
DOC_TAR_MEDIA_TYPE
}
}
"py" | "sh" | "js" | "ts" => CODE_TAR_MEDIA_TYPE,
"md" | "rst" | "pdf" => DOC_TAR_MEDIA_TYPE,
_ => {
DOC_TAR_MEDIA_TYPE
}
}
}
fn ref_matches_precise(desc: &Descriptor, reference: &str) -> bool {
let Some(stored) = stored_ref_name(desc) else {
return false;
};
if let (base, Some(digest)) = split_ref_digest(reference) {
return desc.digest.eq_ignore_ascii_case(digest) && repo_name(stored) == repo_name(base);
}
if stored == reference || tag_from_ref(stored) == reference {
return true;
}
if stored == default_tag(reference) {
return true;
}
false
}
pub fn split_ref_digest(reference: &str) -> (&str, Option<&str>) {
if reference.starts_with('/') || reference.contains("://") {
return (reference, None);
}
let last_slash = reference.rfind('/').map_or(0, |i| i + 1);
match reference[last_slash..].find('@') {
Some(offset) => {
let at = last_slash + offset;
(&reference[..at], Some(&reference[at + 1..]))
}
None => (reference, None),
}
}
pub fn repo_name(reference: &str) -> &str {
let (base, _) = split_ref_digest(reference);
let last_slash = base.rfind('/').map_or(0, |i| i + 1);
match base[last_slash..].find(':') {
Some(offset) => &base[..last_slash + offset],
None => base,
}
}
fn matching_index(manifests: &[Descriptor], reference: &str) -> Option<usize> {
let (base, digest) = split_ref_digest(reference);
if digest.is_some() {
let spelled = default_tag(base);
let own_tag = manifests.iter().position(|m| {
ref_matches_precise(m, reference)
&& stored_ref_name(m).is_some_and(|stored| default_tag(stored) == spelled)
});
if own_tag.is_some() {
return own_tag;
}
}
manifests
.iter()
.position(|m| ref_matches_precise(m, reference))
}
fn stored_ref_name(desc: &Descriptor) -> Option<&str> {
desc.annotations
.as_ref()
.and_then(|a| a.get("org.opencontainers.image.ref.name"))
.map(|s| s.as_str())
}
fn dedup_refs(refs: &mut Vec<Descriptor>) {
let mut seen = HashSet::new();
refs.retain(|desc| {
stored_ref_name(desc)
.map(|name| seen.insert(default_tag(name)))
.unwrap_or(true)
});
}
fn same_ref_name(stored: &str, requested: &str) -> bool {
default_tag(stored) == default_tag(requested)
}
pub(crate) fn manifest_ref_path(root: &Path, reference: &str) -> PathBuf {
let key = default_tag(reference);
let opaque = reference.starts_with('/') || reference.contains("://");
let mut path = root.join(MANIFESTS_V2_DIR);
if opaque {
path.push("~");
}
let (name, tag) = if opaque {
key.rsplit_once('/')
.map_or(("", key.as_str()), |(name, tag)| (name, tag))
} else {
let pos = key
.rfind(':')
.expect("canonical registry reference has a tag");
(&key[..pos], &key[pos + 1..])
};
if !name.is_empty() || opaque {
for segment in name.split('/') {
push_ref_segment(&mut path, segment, true);
}
}
push_ref_segment(&mut path, tag, false);
path
}
fn push_ref_segment(path: &mut PathBuf, segment: &str, directory: bool) {
let encoded = escape_ref_segment(segment, directory);
if encoded.len() <= 200 {
path.push(if directory {
encoded
} else {
format!("{encoded}.json")
});
} else {
for chunk in encoded.as_bytes().chunks(180) {
let chunk = std::str::from_utf8(chunk).expect("escaped ASCII");
path.push(format!("~{}", escape_chunk_trailing_dot(chunk)));
}
path.push(if directory { "~~" } else { "~~.json" });
}
}
fn escape_chunk_trailing_dot(chunk: &str) -> String {
chunk
.strip_suffix('.')
.map_or_else(|| chunk.to_owned(), |rest| format!("{rest}%2E"))
}
fn escape_ref_segment(segment: &str, directory: bool) -> String {
if segment.is_empty() {
return "~".to_owned();
}
let reserved = matches!(
segment
.split('.')
.next()
.unwrap_or("")
.to_ascii_lowercase()
.as_str(),
"con"
| "prn"
| "aux"
| "nul"
| "com1"
| "com2"
| "com3"
| "com4"
| "com5"
| "com6"
| "com7"
| "com8"
| "com9"
| "lpt1"
| "lpt2"
| "lpt3"
| "lpt4"
| "lpt5"
| "lpt6"
| "lpt7"
| "lpt8"
| "lpt9"
);
let mut out = String::new();
for (i, byte) in segment.bytes().enumerate() {
let safe = byte.is_ascii_lowercase()
|| byte.is_ascii_digit()
|| matches!(byte, b'-' | b'_' | b'.');
if safe && !(reserved && i == 0) && !(matches!(segment, "." | "..") && byte == b'.') {
out.push(byte as char);
} else {
out.push_str(&format!("%{byte:02X}"));
}
}
if directory && (out.ends_with(".json") || out.ends_with('.')) {
let dot = out.rfind('.').expect("suffix contains a dot");
out.replace_range(dot..=dot, "%2E");
}
out
}
pub(crate) fn legacy_manifest_ref_path(root: &Path, reference: &str) -> PathBuf {
let mut path = root.join("manifests");
for segment in legacy_ref_path_segments(reference) {
path.push(segment);
}
path
}
pub(crate) fn missing_legacy_ref(err: &std::io::Error) -> bool {
if matches!(
err.kind(),
std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory
) {
return true;
}
#[cfg(unix)]
if err.raw_os_error() == Some(libc::ENAMETOOLONG) {
return true;
}
#[cfg(windows)]
if err.raw_os_error() == Some(206) {
return true; }
false
}
fn legacy_ref_path_segments(reference: &str) -> Vec<String> {
let tagged = default_tag(reference);
let (name, tag): (&str, &str) = match tagged.rfind(':') {
Some(pos) if pos > tagged.rfind('/').unwrap_or(0) => (&tagged[..pos], &tagged[pos + 1..]),
_ => (tagged.as_str(), "latest"),
};
name.split('/')
.filter(|s| !s.is_empty())
.map(legacy_sanitize_ref_segment)
.chain(std::iter::once(legacy_sanitize_ref_segment(tag)))
.collect()
}
fn legacy_sanitize_ref_segment(s: &str) -> String {
if s.is_empty() || s == "." || s == ".." {
return "__".to_string();
}
s.replace([':', '\\'], "_")
}
fn collect_refs(dir: &Path, out: &mut Vec<Descriptor>) {
let Ok(entries) = fs::read_dir(dir) else {
return; };
for entry in entries {
let Ok(entry) = entry else { continue };
let path = entry.path();
let Ok(file_type) = entry.file_type() else {
continue;
};
if file_type.is_dir() {
collect_refs(&path, out);
continue;
}
if path.extension().and_then(|e| e.to_str()) == Some("tmp") {
continue; }
let Ok(data) = fs::read(&path) else { continue };
if let Ok(desc) = serde_json::from_slice::<Descriptor>(&data) {
out.push(desc);
}
}
}
fn collect_refs_strict(dir: &Path, out: &mut Vec<Descriptor>) -> anyhow::Result<()> {
let entries = match fs::read_dir(dir) {
Ok(entries) => entries,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(e) => return Err(e).with_context(|| format!("read {}", dir.display())),
};
for entry in entries {
let entry = entry.with_context(|| format!("read entry in {}", dir.display()))?;
let path = entry.path();
let file_type = entry
.file_type()
.with_context(|| format!("stat {}", path.display()))?;
if file_type.is_dir() {
collect_refs_strict(&path, out)?;
continue;
}
if path.extension().and_then(|e| e.to_str()) == Some("tmp") {
continue; }
let data = fs::read(&path).with_context(|| format!("read {}", path.display()))?;
let desc = serde_json::from_slice::<Descriptor>(&data)
.with_context(|| format!("parse manifest ref {}", path.display()))?;
out.push(desc);
}
Ok(())
}
pub fn default_tag(reference: &str) -> String {
if reference.starts_with('/') || reference.contains("://") {
return reference.to_owned();
}
let after_slash = &reference[reference.rfind('/').unwrap_or(0)..];
if after_slash.contains(':') {
reference.to_owned()
} else {
format!("{reference}:latest")
}
}
pub(crate) fn split_digest(digest: &str) -> anyhow::Result<(&str, &str)> {
match digest.split_once(':') {
Some((algo @ "sha256", hex))
if hex.len() == 64 && hex.bytes().all(|b| b.is_ascii_hexdigit()) =>
{
Ok((algo, hex))
}
_ => Err(anyhow!("invalid digest: {}", digest)),
}
}
pub fn tag_from_ref(reference: &str) -> &str {
if let Some(pos) = reference.rfind(':') {
if pos > reference.rfind('/').unwrap_or(0) {
return &reference[pos + 1..];
}
}
"latest"
}
fn make_single_file_tar(path: &Path, name: &str) -> anyhow::Result<Vec<u8>> {
let file_data = fs::read(path).with_context(|| format!("read {}", path.display()))?;
let mut buf = Vec::new();
{
let mut archive = tar::Builder::new(&mut buf);
let mut header = tar::Header::new_gnu();
header.set_path(name)?;
header.set_size(file_data.len() as u64);
header.set_mode(0o644);
header.set_cksum();
archive.append(&header, file_data.as_slice())?;
archive.finish()?;
}
Ok(buf)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn source_not_found_is_transparent() {
use crate::hf::client::HttpStatusError;
use anyhow::Context as _;
let plain = || {
Err::<(), _>(HttpStatusError::new(
"GET https://huggingface.co/api/models/x/y",
401,
&reqwest::header::HeaderMap::new(),
))
.context("HF model info")
.unwrap_err()
};
let marked = source_not_found(plain());
assert!(is_source_not_found(&marked));
assert!(!is_source_not_found(&plain()));
assert_eq!(marked.to_string(), plain().to_string());
assert_eq!(format!("{marked:#}"), format!("{:#}", plain()));
assert_eq!(
marked.chain().count(),
plain().chain().count(),
"the marker takes the place of its cause's own link"
);
assert!(crate::hf::client::is_permanent(&marked));
let wrapped = marked.context("pull failed");
assert!(is_source_not_found(&wrapped));
assert_eq!(
format!("{wrapped:#}"),
"pull failed: HF model info: GET https://huggingface.co/api/models/x/y: HTTP 401"
);
}
fn desc_with_ref(ref_name: &str) -> Descriptor {
let mut ann = std::collections::HashMap::new();
ann.insert(
"org.opencontainers.image.ref.name".to_string(),
ref_name.to_string(),
);
Descriptor {
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
digest: "sha256:deadbeef".into(),
size: 123,
annotations: Some(ann),
}
}
#[test]
fn ref_matches_precise_a_full_reference_stored_verbatim() {
let d = desc_with_ref("docker.io/ai/qwen3.5:0.8b");
assert!(ref_matches_precise(&d, "docker.io/ai/qwen3.5:0.8b"));
assert!(!ref_matches_precise(&d, "docker.io/ai/qwen3.5:1.5b"));
assert!(!ref_matches_precise(&d, "docker.io/ai/other:0.8b"));
}
#[test]
fn ref_matches_precise_resolves_a_digest_reference_by_content() {
let mut d = desc_with_ref("docker.io/ai/m:v9");
d.digest = "sha256:aaaa".into();
assert!(ref_matches_precise(&d, "docker.io/ai/m@sha256:aaaa"));
assert!(ref_matches_precise(&d, "docker.io/ai/m:latest@sha256:aaaa"));
assert!(!ref_matches_precise(&d, "docker.io/ai/m@sha256:bbbb"));
assert!(!ref_matches_precise(&d, "docker.io/ai/other@sha256:aaaa"));
assert!(ref_matches_precise(&d, "docker.io/ai/m:other@sha256:aaaa"));
assert!(ref_matches_precise(&d, "docker.io/ai/m@sha256:AAAA"));
}
#[test]
fn matching_index_prefers_the_tag_a_digest_reference_spells() {
let mut latest = desc_with_ref("docker.io/ai/m:latest");
latest.digest = "sha256:aaaa".into();
let mut v9 = desc_with_ref("docker.io/ai/m:v9");
v9.digest = "sha256:aaaa".into();
let manifests = [latest, v9];
assert_eq!(
matching_index(&manifests, "docker.io/ai/m:v9@sha256:aaaa"),
Some(1)
);
assert_eq!(
matching_index(&manifests, "docker.io/ai/m@sha256:aaaa"),
Some(0)
);
assert_eq!(
matching_index(&manifests, "docker.io/ai/m:v8@sha256:aaaa"),
Some(0)
);
assert_eq!(
matching_index(&manifests, "docker.io/ai/m@sha256:bbbb"),
None
);
let mut tagless = desc_with_ref("docker.io/ai/m");
tagless.digest = "sha256:aaaa".into();
let mut v9 = desc_with_ref("docker.io/ai/m:v9");
v9.digest = "sha256:aaaa".into();
assert_eq!(
matching_index(&[v9, tagless], "docker.io/ai/m@sha256:aaaa"),
Some(1)
);
}
#[test]
fn split_ref_digest_only_splits_after_the_last_slash() {
assert_eq!(
split_ref_digest("docker.io/ai/m@sha256:aa"),
("docker.io/ai/m", Some("sha256:aa"))
);
assert_eq!(
split_ref_digest("docker.io/ai/m:v9@sha256:aa"),
("docker.io/ai/m:v9", Some("sha256:aa"))
);
assert_eq!(
split_ref_digest("docker.io/ai/m:v9"),
("docker.io/ai/m:v9", None)
);
assert_eq!(
split_ref_digest("s3://bucket/model@sha256:aa"),
("s3://bucket/model@sha256:aa", None)
);
assert_eq!(
split_ref_digest("/models/m@sha256:aa"),
("/models/m@sha256:aa", None)
);
assert_eq!(repo_name("docker.io/ai/m:v9@sha256:aa"), "docker.io/ai/m");
assert_eq!(repo_name("localhost:5000/m"), "localhost:5000/m");
assert_eq!(repo_name("localhost:5000/m:v1"), "localhost:5000/m");
}
#[test]
fn find_passes_over_a_digest_named_pointer_holding_other_content() {
let dir = temp_store_dir("digest-pointer");
let store = OciStore::open(&dir).unwrap();
let mut lying = desc_with_ref("unused");
lying.digest = "sha256:aaaa".into();
store.tag(lying, "docker.io/ai/m@sha256:bbbb").unwrap();
assert!(store.find("docker.io/ai/m@sha256:bbbb").is_err());
assert_eq!(
store.find("docker.io/ai/m@sha256:aaaa").unwrap().digest,
"sha256:aaaa",
"by content, the pointer is the entry for the digest it holds"
);
let mut real = desc_with_ref("unused");
real.digest = "sha256:bbbb".into();
store.tag(real, "docker.io/ai/m:v9").unwrap();
assert_eq!(
store
.find("docker.io/ai/m@sha256:bbbb")
.unwrap()
.annotations
.unwrap()["org.opencontainers.image.ref.name"],
"docker.io/ai/m:v9"
);
store.remove("docker.io/ai/m@sha256:bbbb").unwrap();
assert!(store.find("docker.io/ai/m@sha256:aaaa").is_err());
assert!(store.find("docker.io/ai/m:v9").is_ok());
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn remove_by_digest_names_what_went_and_refuses_an_ambiguous_one() {
let dir = temp_store_dir("digest-remove");
let store = OciStore::open(&dir).unwrap();
let desc = |digest: &str| {
let mut d = desc_with_ref("unused");
d.digest = digest.into();
d
};
store.tag(desc("sha256:aaaa"), "docker.io/ai/m:v8").unwrap();
store.tag(desc("sha256:aaaa"), "docker.io/ai/m:v9").unwrap();
store
.tag(desc("sha256:aaaa"), "docker.io/ai/m:latest")
.unwrap();
let err = store.remove("docker.io/ai/m@sha256:aaaa").unwrap_err();
let msg = err.to_string();
assert!(
[
"docker.io/ai/m:v8",
"docker.io/ai/m:v9",
"docker.io/ai/m:latest"
]
.iter()
.all(|t| msg.contains(t)),
"{msg}"
);
assert!(store.find("docker.io/ai/m:latest").is_ok());
assert_eq!(
store.remove("docker.io/ai/m:latest@sha256:aaaa").unwrap(),
"docker.io/ai/m:latest"
);
assert_eq!(
store.remove("docker.io/ai/m:v9@sha256:aaaa").unwrap(),
"docker.io/ai/m:v9"
);
assert_eq!(
store.remove("docker.io/ai/m@sha256:aaaa").unwrap(),
"docker.io/ai/m:v8",
"one holder left is not ambiguous, and the name that went is reported"
);
assert!(store.remove("docker.io/ai/m@sha256:aaaa").is_err());
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn find_resolves_a_digest_reference_to_the_tagged_entry() {
let dir = temp_store_dir("digest-find");
let store = OciStore::open(&dir).unwrap();
let mut d = desc_with_ref("unused");
d.digest = "sha256:cafe".into();
store.tag(d, "docker.io/ai/m:latest").unwrap();
let found = store.find("docker.io/ai/m@sha256:cafe").unwrap();
assert_eq!(
found.annotations.unwrap()["org.opencontainers.image.ref.name"],
"docker.io/ai/m:latest"
);
assert!(store.find("docker.io/ai/m@sha256:dead").is_err());
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn ref_matches_precise_defaults_a_tagless_reference_to_latest() {
let d = desc_with_ref("docker.io/ai/qwen3.5:latest");
assert!(ref_matches_precise(&d, "docker.io/ai/qwen3.5"));
}
#[test]
fn default_tag_appends_latest_only_when_no_tag_is_present() {
assert_eq!(
default_tag("docker.io/ai/gemma4"),
"docker.io/ai/gemma4:latest"
);
assert_eq!(
default_tag("docker.io/ai/gemma4:latest"),
"docker.io/ai/gemma4:latest"
);
assert_eq!(
default_tag("docker.io/ai/gemma4:e4b"),
"docker.io/ai/gemma4:e4b"
);
assert_eq!(
default_tag("localhost:5000/gemma4"),
"localhost:5000/gemma4:latest"
);
}
#[test]
fn default_tag_leaves_non_registry_sources_untouched() {
assert_eq!(default_tag("ngc://org/model"), "ngc://org/model");
assert_eq!(default_tag("s3://bucket/key"), "s3://bucket/key");
assert_eq!(default_tag("gs://bucket/key"), "gs://bucket/key");
assert_eq!(default_tag("ms://owner/repo"), "ms://owner/repo");
assert_eq!(default_tag("/abs/path/model.gguf"), "/abs/path/model.gguf");
}
#[test]
fn ref_matches_precise_returns_false_without_a_ref_name_annotation() {
let d = Descriptor {
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
digest: "sha256:deadbeef".into(),
size: 123,
annotations: None,
};
assert!(!ref_matches_precise(&d, "docker.io/ai/qwen3.5:0.8b"));
}
#[test]
fn matching_index_prefers_a_precise_match_over_an_unrelated_bare_latest() {
let unrelated_bare_latest = desc_with_ref("latest");
let inferact = desc_with_ref("hf.co/Inferact/Qwen3.8-27B-NVFP4:latest");
let manifests = vec![unrelated_bare_latest, inferact.clone()];
let i = matching_index(&manifests, "hf.co/Inferact/Qwen3.8-27B-NVFP4")
.expect("must find a match");
assert_eq!(manifests[i].digest, inferact.digest);
let i = matching_index(&manifests, "hf.co/Inferact/Qwen3.8-27B-NVFP4:latest")
.expect("must find a match");
assert_eq!(manifests[i].digest, inferact.digest);
}
#[test]
fn tag_from_ref_extracts_the_tag_after_the_last_slash() {
assert_eq!(tag_from_ref("docker.io/ai/qwen3.5:0.8b"), "0.8b");
assert_eq!(tag_from_ref("docker.io/ai/qwen3.5"), "latest");
assert_eq!(tag_from_ref("0.8b"), "latest");
}
#[test]
fn tag_from_ref_ignores_a_port_in_the_host_component() {
assert_eq!(tag_from_ref("example.com:5000/ns/model"), "latest");
assert_eq!(tag_from_ref("example.com:5000/ns/model:tag"), "tag");
assert_eq!(tag_from_ref("localhost:11434/library/mistral:7b"), "7b");
}
fn temp_store_dir(name: &str) -> std::path::PathBuf {
std::env::temp_dir().join(format!(
"llmman-oci-test-{name}-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
))
}
#[test]
fn rewriting_an_existing_blob_refreshes_its_age() {
let dir = temp_store_dir("blob-reuse");
let store = OciStore::open(&dir).unwrap();
let desc = store
.write_blob("application/octet-stream", b"layer")
.unwrap();
let path = store.blob_path(&desc.digest).unwrap();
let old = std::time::SystemTime::now() - std::time::Duration::from_secs(2 * 60 * 60);
let touch = |t| {
fs::OpenOptions::new()
.write(true)
.open(&path)
.unwrap()
.set_modified(t)
.unwrap()
};
touch(old);
store
.write_blob("application/octet-stream", b"layer")
.unwrap();
let age = fs::metadata(&path)
.unwrap()
.modified()
.unwrap()
.elapsed()
.unwrap();
assert!(age < std::time::Duration::from_secs(60), "{age:?}");
fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn rewriting_an_existing_blob_file_refreshes_its_age() {
let dir = temp_store_dir("blob-file-reuse");
let store = OciStore::open(&dir).unwrap();
let src = dir.join("layer.bin");
fs::write(&src, b"layer file").unwrap();
let first = store
.write_blob_file("application/octet-stream", &src)
.unwrap();
assert_eq!(first.size, 10);
assert_eq!(
first.digest,
format!("sha256:{}", hex::encode(Sha256::digest(b"layer file")))
);
let path = store.blob_path(&first.digest).unwrap();
assert_eq!(fs::read(&path).unwrap(), b"layer file");
let old = std::time::SystemTime::now() - std::time::Duration::from_secs(2 * 60 * 60);
fs::OpenOptions::new()
.write(true)
.open(&path)
.unwrap()
.set_modified(old)
.unwrap();
let second = store
.write_blob_file("application/octet-stream", &src)
.unwrap();
assert_eq!(second.digest, first.digest);
assert_eq!(second.size, first.size);
let age = fs::metadata(&path)
.unwrap()
.modified()
.unwrap()
.elapsed()
.unwrap();
assert!(age < std::time::Duration::from_secs(60), "{age:?}");
let leftovers: Vec<_> = fs::read_dir(path.parent().unwrap())
.unwrap()
.map(|e| e.unwrap().file_name())
.filter(|n| n.to_string_lossy().ends_with(".tmp"))
.collect();
assert!(leftovers.is_empty(), "{leftovers:?}");
fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn build_wraps_each_file_in_its_own_tar_layer() {
let dir = temp_store_dir("build-tar");
let store = OciStore::open(&dir).unwrap();
let context = dir.join("context");
fs::create_dir_all(context.join("sub")).unwrap();
fs::write(
context.join("sub").join("model.Q4.gguf"),
b"GGUF\x03\0\0\0body",
)
.unwrap();
let labels: std::collections::HashMap<String, String> =
[("org.example.team".to_string(), "canary".to_string())].into();
let desc = store
.build(&context, "example.invalid/tar:v1", &labels, false)
.unwrap();
let manifest = store.read_manifest(&desc.digest).unwrap();
assert_eq!(manifest.layers.len(), 1);
let layer = &manifest.layers[0];
assert_eq!(layer.media_type, WEIGHT_TAR_MEDIA_TYPE);
assert!(layer.media_type.ends_with(".tar"), "{}", layer.media_type);
let ann = layer.annotations.as_ref().unwrap();
assert_eq!(ann["org.cncf.model.filepath"], "sub/model.Q4.gguf");
assert_eq!(ann["org.opencontainers.image.title"], "sub/model.Q4.gguf");
let blob = store.read_blob(&layer.digest).unwrap();
let mut archive = tar::Archive::new(blob.as_slice());
let mut entries = archive.entries().unwrap();
let mut entry = entries.next().unwrap().unwrap();
assert_eq!(
entry.path().unwrap().to_string_lossy().replace('\\', "/"),
"sub/model.Q4.gguf"
);
let mut body = Vec::new();
std::io::Read::read_to_end(&mut entry, &mut body).unwrap();
assert_eq!(body, b"GGUF\x03\0\0\0body");
assert!(entries.next().is_none(), "more than one tar entry");
let config: CncfModelConfig =
serde_json::from_slice(&store.read_blob(&manifest.config.digest).unwrap()).unwrap();
assert_eq!(config.modelfs.fs_type, "layers");
let digests: Vec<String> = manifest.layers.iter().map(|l| l.digest.clone()).collect();
assert_eq!(config.modelfs.diff_ids, digests);
assert_eq!(config.config.format.as_deref(), Some("gguf"));
assert_eq!(manifest.annotations.as_ref(), Some(&labels));
assert_eq!(
store.find("example.invalid/tar:v1").unwrap().digest,
desc.digest
);
fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn ref_path_segments_mirrors_ollama_host_namespace_model_tag() {
assert_eq!(
legacy_ref_path_segments("docker.io/ai/qwen3.5:0.8b"),
vec!["docker.io", "ai", "qwen3.5", "0.8b"]
);
assert_eq!(
legacy_ref_path_segments("docker.io/ai/qwen3.5"),
vec!["docker.io", "ai", "qwen3.5", "latest"]
);
}
#[test]
fn current_paths_are_portable_and_distinct() {
let root = Path::new("store");
let path = manifest_ref_path(root, "s3://bucket/key");
assert_eq!(
path,
root.join(MANIFESTS_V2_DIR)
.join("~")
.join("s3%3A")
.join("~")
.join("bucket")
.join("key.json")
);
assert_eq!(
manifest_ref_path(root, "reg.example:5000/ns/model:t"),
root.join(MANIFESTS_V2_DIR)
.join("reg.example%3A5000/ns/model/t.json")
);
assert_ne!(path, manifest_ref_path(root, "../../etc/passwd"));
assert_ne!(path, manifest_ref_path(root, "S3://bucket/key"));
assert_ne!(path, manifest_ref_path(root, "s3://bucket/key:latest"));
assert_ne!(
manifest_ref_path(root, "reg.example/ns/model:Prod"),
manifest_ref_path(root, "reg.example/ns/model:prod")
);
assert_ne!(
manifest_ref_path(root, "reg.example/ns/model:t"),
manifest_ref_path(root, "reg.example/ns/model/t.json:x")
);
}
#[test]
fn manifest_ref_paths_match_shared_golden_vectors() {
#[derive(serde::Deserialize)]
struct Vector {
reference: String,
path: String,
}
let vectors: Vec<Vector> =
serde_json::from_str(include_str!("../../tests/fixtures/manifest-ref-paths.json"))
.unwrap();
for vector in vectors {
assert_eq!(
manifest_ref_path(Path::new("store"), &vector.reference),
Path::new("store").join(vector.path),
"{}",
vector.reference
);
}
}
#[test]
fn new_store_does_not_create_legacy_tree() {
let dir = temp_store_dir("new-v2-only");
let store = OciStore::open(&dir).unwrap();
assert!(store.root().join(MANIFESTS_V2_DIR).is_dir());
assert!(!store.root().join("manifests").exists());
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn long_escaped_tags_stay_within_component_limits() {
let root = Path::new("store");
let ref_name = format!("reg.example/ns/model:{}", "A".repeat(128));
let path = manifest_ref_path(root, &ref_name);
for part in path.components() {
assert!(part.as_os_str().len() <= 255);
}
assert!(path.ends_with("~~.json"));
}
#[cfg(unix)]
#[test]
fn long_opaque_key_ignores_unrepresentable_legacy_pointer() {
let dir = temp_store_dir("long-opaque-key");
let store = OciStore::open(&dir).unwrap();
let reference = format!("s3://bucket/{}", "a".repeat(256));
fs::create_dir_all(
store
.legacy_ref_path(&reference)
.parent()
.unwrap()
.parent()
.unwrap(),
)
.unwrap();
store.tag(desc_with_ref(&reference), &reference).unwrap();
assert_eq!(store.find(&reference).unwrap().digest, "sha256:deadbeef");
store.remove(&reference).unwrap();
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn opaque_reference_and_literal_latest_suffix_remain_distinct() {
let dir = temp_store_dir("opaque-latest");
let store = OciStore::open(&dir).unwrap();
let first = "s3://bucket/key";
let second = "s3://bucket/key:latest";
let mut first_desc = desc_with_ref(first);
first_desc.digest = "sha256:first".into();
let mut second_desc = desc_with_ref(second);
second_desc.digest = "sha256:second".into();
store.tag(first_desc, first).unwrap();
store.tag(second_desc, second).unwrap();
assert_eq!(store.find(first).unwrap().digest, "sha256:first");
assert_eq!(store.find(second).unwrap().digest, "sha256:second");
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn distinct_references_never_share_a_manifest_pointer() {
let dir = temp_store_dir("distinct-refs");
let store = OciStore::open(&dir).unwrap();
let first = "reg.example:5000/ns/model:t";
let second = "reg.example_5000/ns/model:t";
assert!(crate::shortnames::validate_reference(first).is_ok());
assert!(crate::shortnames::validate_reference(second).is_ok());
assert_eq!(store.legacy_ref_path(first), store.legacy_ref_path(second));
assert_ne!(store.ref_path(first), store.ref_path(second));
assert_ne!(
store.ref_path("docker.io/ai/model:Prod"),
store.ref_path("docker.io/ai/model:prod")
);
let mut first_desc = desc_with_ref(first);
first_desc.digest = "sha256:first".into();
let mut second_desc = desc_with_ref(second);
second_desc.digest = "sha256:second".into();
store.tag(first_desc, first).unwrap();
store.tag(second_desc, second).unwrap();
assert_eq!(store.find(first).unwrap().digest, "sha256:first");
assert_eq!(store.find(second).unwrap().digest, "sha256:second");
store.remove(first).unwrap();
assert!(store.find(first).is_err());
assert_eq!(store.find(second).unwrap().digest, "sha256:second");
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn legacy_pointer_is_read_and_preserved_without_hiding_a_v2_reference() {
let dir = temp_store_dir("legacy-collision");
let store = OciStore::open(&dir).unwrap();
let first = "reg.example:5000/ns/model:t";
let second = "reg.example_5000/ns/model:t";
let legacy = store.legacy_ref_path(first);
fs::create_dir_all(legacy.parent().unwrap()).unwrap();
fs::write(&legacy, serde_json::to_vec(&desc_with_ref(first)).unwrap()).unwrap();
assert_eq!(store.find(first).unwrap().digest, "sha256:deadbeef");
let mut second_desc = desc_with_ref(second);
second_desc.digest = "sha256:second".into();
store.tag(second_desc, second).unwrap();
assert!(
legacy.exists(),
"the other reference's old pointer must survive"
);
assert_eq!(store.find(first).unwrap().digest, "sha256:deadbeef");
assert_eq!(store.find(second).unwrap().digest, "sha256:second");
assert_eq!(store.list_refs().len(), 2);
assert_eq!(store.list_refs_strict().unwrap().len(), 2);
let mut updated = desc_with_ref(first);
updated.digest = "sha256:updated".into();
store.tag(updated, first).unwrap();
assert!(legacy.exists(), "the old tree remains a read-only fallback");
assert_eq!(store.find(first).unwrap().digest, "sha256:updated");
assert_eq!(store.find(second).unwrap().digest, "sha256:second");
assert_eq!(store.list_refs_strict().unwrap().len(), 3);
store.remove(first).unwrap();
assert!(store.find(first).is_err());
assert!(!legacy.exists());
assert_eq!(store.find(second).unwrap().digest, "sha256:second");
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn malformed_legacy_pointer_does_not_block_tag_or_remove() {
let dir = temp_store_dir("malformed-legacy");
let store = OciStore::open(&dir).unwrap();
let reference = "docker.io/ai/model:latest";
let legacy = store.legacy_ref_path(reference);
fs::create_dir_all(legacy.parent().unwrap()).unwrap();
fs::write(&legacy, b"not JSON").unwrap();
store.tag(desc_with_ref(reference), reference).unwrap();
assert!(store.ref_path(reference).is_file());
assert_eq!(fs::read(&legacy).unwrap(), b"not JSON");
assert_eq!(store.remove(reference).unwrap(), reference);
assert!(!store.ref_path(reference).exists());
assert_eq!(fs::read(&legacy).unwrap(), b"not JSON");
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn legacy_directory_does_not_block_writing_or_removing() {
let dir = temp_store_dir("legacy-directory");
let store = OciStore::open(&dir).unwrap();
let reference = "docker.io/ai/model:latest";
let legacy = store.legacy_ref_path(reference);
fs::create_dir_all(&legacy).unwrap();
let nested = legacy.join("another-tag");
fs::write(&nested, b"another model's pointer").unwrap();
store.tag(desc_with_ref(reference), reference).unwrap();
assert!(store.ref_path(reference).exists());
assert_eq!(store.remove(reference).unwrap(), reference);
assert!(!store.ref_path(reference).exists());
assert!(legacy.is_dir());
assert!(nested.exists());
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn legacy_ancestor_file_does_not_block_tag_or_remove() {
let dir = temp_store_dir("legacy-ancestor-file");
let store = OciStore::open(&dir).unwrap();
let reference = "reg.example/ns/model/sub:t";
let ancestor = store.legacy_ref_path("reg.example/ns/model:sub");
fs::create_dir_all(ancestor.parent().unwrap()).unwrap();
fs::write(&ancestor, b"another pointer").unwrap();
store.tag(desc_with_ref(reference), reference).unwrap();
assert!(store.ref_path(reference).is_file());
assert_eq!(store.remove(reference).unwrap(), reference);
assert!(!store.ref_path(reference).exists());
assert_eq!(fs::read(&ancestor).unwrap(), b"another pointer");
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn tag_and_find_round_trip_one_file_per_model() {
let dir = temp_store_dir("round-trip");
let store = OciStore::open(&dir).unwrap();
store
.tag(desc_with_ref("unused"), "docker.io/ai/qwen3.5:0.8b")
.unwrap();
store
.tag(desc_with_ref("unused"), "docker.io/ai/gemma4:latest")
.unwrap();
let a = store.ref_path("docker.io/ai/qwen3.5:0.8b");
let b = store.ref_path("docker.io/ai/gemma4:latest");
assert_ne!(a, b, "distinct references must live at distinct paths");
assert!(a.is_file());
assert!(b.is_file());
let found = store.find("docker.io/ai/qwen3.5:0.8b").unwrap();
assert_eq!(
found.annotations.unwrap()["org.opencontainers.image.ref.name"],
"docker.io/ai/qwen3.5:0.8b"
);
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn write_ref_is_atomic_and_leaves_no_tmp_file() {
let dir = temp_store_dir("atomic");
let store = OciStore::open(&dir).unwrap();
store
.tag(desc_with_ref("unused"), "docker.io/ai/gemma4:latest")
.unwrap();
let path = store.ref_path("docker.io/ai/gemma4:latest");
assert!(path.is_file());
let leftover_tmp = std::fs::read_dir(path.parent().unwrap())
.unwrap()
.filter_map(|e| e.ok())
.any(|e| e.path().extension().is_some_and(|ext| ext == "tmp"));
assert!(
!leftover_tmp,
"no .tmp file should survive a successful write"
);
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn list_and_remove_work_across_many_models() {
let dir = temp_store_dir("list-remove");
let store = OciStore::open(&dir).unwrap();
for i in 0..5 {
store
.tag(
desc_with_ref("unused"),
&format!("docker.io/ai/model-{i}:latest"),
)
.unwrap();
}
let images = store.list().unwrap();
assert_eq!(images.len(), 5);
store.remove("docker.io/ai/model-2:latest").unwrap();
let images = store.list().unwrap();
assert_eq!(images.len(), 4);
assert!(!images
.iter()
.any(|i| i.reference == "docker.io/ai/model-2:latest"));
assert!(store.remove("docker.io/ai/model-2:latest").is_err());
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
#[cfg(unix)]
fn list_skips_an_unreadable_subtree() {
use std::os::unix::fs::PermissionsExt;
let dir = temp_store_dir("unreadable-subtree");
let store = OciStore::open(&dir).unwrap();
store
.tag(desc_with_ref("unused"), "docker.io/ai/ok:latest")
.unwrap();
let blocked = dir.join("manifests").join("unreadable");
std::fs::create_dir_all(&blocked).unwrap();
std::fs::set_permissions(&blocked, std::fs::Permissions::from_mode(0o0)).unwrap();
let images = store.list().unwrap();
assert_eq!(images.len(), 1);
assert_eq!(images[0].reference, "docker.io/ai/ok:latest");
std::fs::set_permissions(&blocked, std::fs::Permissions::from_mode(0o755)).unwrap();
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn concurrent_tag_calls_across_distinct_refs_lose_nothing() {
let dir = temp_store_dir("concurrent");
let store = std::sync::Arc::new(OciStore::open(&dir).unwrap());
let handles: Vec<_> = (0..25)
.map(|i| {
let store = store.clone();
std::thread::spawn(move || {
store
.tag(
desc_with_ref("unused"),
&format!("docker.io/ai/model-{i}:latest"),
)
.unwrap();
})
})
.collect();
for h in handles {
h.join().unwrap();
}
let images = store.list().unwrap();
assert_eq!(images.len(), 25);
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn concurrent_writes_to_the_same_ref_never_corrupt_it() {
let dir = temp_store_dir("concurrent-same-ref");
let store = std::sync::Arc::new(OciStore::open(&dir).unwrap());
let handles: Vec<_> = (0..25)
.map(|i| {
let store = store.clone();
std::thread::spawn(move || {
let mut desc = desc_with_ref("unused");
desc.digest = format!("sha256:{i:064x}");
store.tag(desc, "docker.io/ai/same:latest").unwrap();
})
})
.collect();
for h in handles {
h.join().unwrap();
}
assert!(store.find("docker.io/ai/same:latest").is_ok());
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn tag_with_a_dot_does_not_collide_with_a_different_tag() {
let dir = temp_store_dir("dotted-tag");
let store = OciStore::open(&dir).unwrap();
store
.tag(desc_with_ref("unused"), "docker.io/ai/x:0.8b")
.unwrap();
store
.tag(desc_with_ref("unused"), "docker.io/ai/x:0.5b")
.unwrap();
let images = store.list().unwrap();
assert_eq!(images.len(), 2, "distinct dotted tags must not collide");
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn tag_with_an_empty_tag_does_not_break_other_tags() {
let dir = temp_store_dir("empty-tag");
let store = OciStore::open(&dir).unwrap();
store
.tag(desc_with_ref("unused"), "docker.io/ai/x:")
.unwrap();
store
.tag(desc_with_ref("unused"), "docker.io/ai/x:latest")
.unwrap();
assert!(store.find("docker.io/ai/x:latest").is_ok());
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn find_distinguishes_a_corrupt_ref_from_a_missing_one() {
let dir = temp_store_dir("corrupt-ref");
let store = OciStore::open(&dir).unwrap();
let path = store.ref_path("docker.io/ai/broken:latest");
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, b"not json").unwrap();
let err = store.find("docker.io/ai/broken:latest").unwrap_err();
assert!(!err.to_string().contains("not found"), "got: {err}");
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn corrupt_current_pointer_can_be_removed() {
let dir = temp_store_dir("corrupt-remove");
let store = OciStore::open(&dir).unwrap();
let reference = "docker.io/ai/broken:latest";
let path = store.ref_path(reference);
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, b"not json").unwrap();
assert_eq!(store.remove(reference).unwrap(), reference);
assert!(!path.exists());
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn find_ignores_a_legacy_directory_of_another_reference() {
let dir = temp_store_dir("find-legacy-dir");
let store = OciStore::open(&dir).unwrap();
let nested = "reg.example/ns/model/sub:t";
let legacy = store.legacy_ref_path(nested);
std::fs::create_dir_all(legacy.parent().unwrap()).unwrap();
std::fs::write(&legacy, serde_json::to_vec(&desc_with_ref(nested)).unwrap()).unwrap();
let err = store.find("reg.example/ns/model:sub").unwrap_err();
assert!(err.downcast_ref::<NotFound>().is_some(), "got: {err}");
std::fs::remove_dir_all(&dir).unwrap();
}
}