use crate::catalog_proposals::allowed_registry_redirect;
use futures_util::StreamExt;
use rigspark_core::{
admission::{
AdmissionInput, Gguf, GgufError, LibraryEntry, MAX_CONFIG_BYTES, MAX_GGUF_HEADER_BYTES,
MAX_HTML_BYTES, MAX_LICENSE_HEAD_BYTES, RegistryConfig, build_entry, catalog_capabilities,
kv_bytes_per_token, license_id, observation, parse_gguf, parse_library, parse_tags,
select_variants,
},
catalog::{Catalog, CatalogModel, EntryProvenance, SCHEMA_VERSION},
registry_collector::{
MANIFEST_ACCEPT, MAX_MANIFEST_BYTES, parse_layer, parse_reference, pulled_quantization,
},
sizing::Architecture,
};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use std::{
collections::{BTreeMap, BTreeSet},
io,
time::Duration,
};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use tokio_util::sync::CancellationToken;
use url::Url;
pub const LIBRARY_URL: &str = "https://ollama.com/library?sort=newest";
pub const GENERATION_LISTING_URL: &str =
"https://huggingface.co/api/models?author=Comfy-Org&sort=createdAt&direction=-1&limit=500";
pub const SCOPE_NAME: &str = "ollama-local-variants";
const GGUF_STEPS: [usize; 3] = [8 * 1024 * 1024, 32 * 1024 * 1024, MAX_GGUF_HEADER_BYTES];
fn name(part: &str) -> bool {
!part.is_empty()
&& part.len() <= 128
&& part.as_bytes()[0].is_ascii_alphanumeric()
&& part
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte))
}
pub fn allowed_request(url: &Url) -> bool {
if url.scheme() != "https"
|| url.port().is_some()
|| !url.username().is_empty()
|| url.password().is_some()
|| url.fragment().is_some()
{
return false;
}
let segments: Vec<&str> = url.path().trim_start_matches('/').split('/').collect();
match (url.host_str(), url.query(), segments.as_slice()) {
(Some("ollama.com"), Some("sort=newest"), ["library"]) => true,
(Some("ollama.com"), None, ["library", repo, "tags"]) => name(repo),
(Some("registry.ollama.ai"), None, ["v2", "library", repo, "manifests", tag]) => {
name(repo) && name(tag)
}
(Some("registry.ollama.ai"), None, ["v2", "library", repo, "blobs", digest]) => {
name(repo)
&& digest.strip_prefix("sha256:").is_some_and(|hex| {
hex.len() == 64 && hex.bytes().all(|byte| byte.is_ascii_hexdigit())
})
}
(Some("huggingface.co"), Some(query), ["api", "models"]) => {
Some(query)
== GENERATION_LISTING_URL
.split_once('?')
.map(|(_, query)| query)
}
(Some("huggingface.co"), None | Some("blobs=true"), ["api", "models", owner, repo]) => {
name(owner) && name(repo)
}
(
Some("huggingface.co"),
Some("blobs=true"),
["api", "models", owner, repo, "revision", revision],
) => {
name(owner)
&& name(repo)
&& revision.len() == 40
&& revision.bytes().all(|byte| byte.is_ascii_hexdigit())
}
(Some("huggingface.co"), None, [owner, repo, "resolve", revision, path @ ..]) => {
name(owner)
&& name(repo)
&& revision.len() == 40
&& revision.bytes().all(|byte| byte.is_ascii_hexdigit())
&& !path.is_empty()
&& path.len() <= 8
&& path.iter().all(|segment| name(segment))
&& path.last().is_some_and(|file| {
file.ends_with(".gguf")
|| (*owner == "Comfy-Org" && file.ends_with(".safetensors"))
})
}
_ => false,
}
}
pub fn allowed_hub_redirect(url: &Url) -> bool {
url.scheme() == "https"
&& url.port().is_none()
&& url.username().is_empty()
&& url.password().is_none()
&& url
.host_str()
.is_some_and(|host| host == "huggingface.co" || host.ends_with(".hf.co"))
}
pub struct Fetched {
pub status: u16,
pub body: Vec<u8>,
}
#[async_trait::async_trait]
pub trait AdmissionTransport: Send + Sync {
async fn fetch(
&self,
url: &Url,
accept: Option<&'static str>,
range: Option<usize>,
limit: usize,
) -> io::Result<Fetched>;
}
pub struct NativeAdmissionTransport {
client: reqwest::Client,
}
impl NativeAdmissionTransport {
pub fn new() -> io::Result<Self> {
let client = reqwest::Client::builder()
.https_only(true)
.no_proxy()
.user_agent(concat!(
"rigspark-catalog-admission/",
env!("CARGO_PKG_VERSION")
))
.redirect(reqwest::redirect::Policy::custom(|attempt| {
let previous = attempt.previous().last();
let from_blob = previous.is_some_and(|url| {
url.host_str() == Some("registry.ollama.ai")
&& url.path().contains("/blobs/sha256:")
});
let from_resolve = previous.is_some_and(|url| {
url.host_str() == Some("huggingface.co") && url.path().contains("/resolve/")
});
let allowed = (from_blob && allowed_registry_redirect(attempt.url().as_str()))
|| (from_resolve && allowed_hub_redirect(attempt.url()));
if attempt.previous().len() > 2 || !allowed {
attempt.error("registry redirect refused")
} else {
attempt.follow()
}
}))
.connect_timeout(Duration::from_secs(10))
.timeout(Duration::from_secs(120))
.build()
.map_err(|_| io::Error::other("admission client initialization failed"))?;
Ok(Self { client })
}
}
#[async_trait::async_trait]
impl AdmissionTransport for NativeAdmissionTransport {
async fn fetch(
&self,
url: &Url,
accept: Option<&'static str>,
range: Option<usize>,
limit: usize,
) -> io::Result<Fetched> {
if !allowed_request(url) {
return Err(io::Error::other("admission URL is not allow-listed"));
}
let mut request = self.client.get(url.clone());
if let Some(accept) = accept {
request = request.header(reqwest::header::ACCEPT, accept);
}
if let Some(range) = range {
request = request.header(reqwest::header::RANGE, format!("bytes=0-{}", range - 1));
}
let response = request
.send()
.await
.map_err(|_| io::Error::other("admission request failed"))?;
let status = response.status().as_u16();
let mut stream = response.bytes_stream();
let mut body = Vec::new();
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(|_| io::Error::other("admission body read failed"))?;
body.extend_from_slice(&chunk);
if body.len() >= limit {
if range.is_none() && body.len() > limit {
return Err(io::Error::other("admission response exceeds its size cap"));
}
body.truncate(limit);
break;
}
}
Ok(Fetched { status, body })
}
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct RecordedBody {
status: u16,
#[serde(default)]
text: Option<String>,
#[serde(default)]
json: Option<Value>,
#[serde(default)]
hex: Option<String>,
}
pub struct RecordedAdmissionTransport {
responses: BTreeMap<String, (u16, Vec<u8>)>,
}
impl RecordedAdmissionTransport {
pub fn new(responses: BTreeMap<String, (u16, Vec<u8>)>) -> io::Result<Self> {
for key in responses.keys() {
let url = Url::parse(key).map_err(io::Error::other)?;
if !allowed_request(&url) {
return Err(io::Error::other("recorded URL is not allow-listed"));
}
}
Ok(Self { responses })
}
pub fn parse(raw: &str) -> io::Result<Self> {
let recorded: BTreeMap<String, RecordedBody> =
serde_json::from_str(raw).map_err(io::Error::other)?;
let mut responses = BTreeMap::new();
for (url, body) in recorded {
let bytes = match (body.text, body.json, body.hex) {
(Some(text), None, None) => text.into_bytes(),
(None, Some(value), None) => {
serde_json::to_vec(&value).map_err(io::Error::other)?
}
(None, None, Some(hex)) => decode_hex(&hex)?,
(None, None, None) => Vec::new(),
_ => return Err(io::Error::other("recorded body must have one encoding")),
};
responses.insert(url, (body.status, bytes));
}
Self::new(responses)
}
}
fn decode_hex(hex: &str) -> io::Result<Vec<u8>> {
if !hex.len().is_multiple_of(2) {
return Err(io::Error::other("invalid recorded hex body"));
}
(0..hex.len())
.step_by(2)
.map(|index| {
u8::from_str_radix(&hex[index..index + 2], 16)
.map_err(|_| io::Error::other("invalid recorded hex body"))
})
.collect()
}
#[async_trait::async_trait]
impl AdmissionTransport for RecordedAdmissionTransport {
async fn fetch(
&self,
url: &Url,
_accept: Option<&'static str>,
range: Option<usize>,
limit: usize,
) -> io::Result<Fetched> {
if !allowed_request(url) {
return Err(io::Error::other("admission URL is not allow-listed"));
}
let (status, body) = self
.responses
.get(url.as_str())
.cloned()
.unwrap_or((404, Vec::new()));
let cap = range.unwrap_or(usize::MAX).min(limit);
if range.is_none() && body.len() > limit {
return Err(io::Error::other("admission response exceeds its size cap"));
}
Ok(Fetched {
status,
body: body[..body.len().min(cap)].to_vec(),
})
}
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct ManifestLayers {
config: Descriptor,
layers: Vec<Descriptor>,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct Descriptor {
media_type: String,
digest: String,
}
fn sha256(digest: &str) -> Option<&str> {
digest
.strip_prefix("sha256:")
.filter(|hex| hex.len() == 64 && hex.bytes().all(|byte| byte.is_ascii_hexdigit()))
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct AdmissionState {
#[serde(default)]
pub rejections: BTreeMap<String, String>,
#[serde(default)]
pub facts: BTreeMap<String, SourcedFacts>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SourcedFacts {
pub context_length: u64,
pub architecture: Architecture,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub license: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub kv_bytes_per_token: Option<f64>,
}
pub struct AdmissionOptions {
pub now: String,
pub max_new_variants: usize,
pub state: AdmissionState,
pub correct_curated: bool,
}
#[derive(Clone, Debug, Serialize)]
pub struct Correction {
pub id: String,
pub field: String,
pub from: Value,
pub to: Value,
pub sources: Vec<String>,
}
#[derive(Default, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct AdmissionOutcome {
#[serde(skip)]
pub catalog: Option<Catalog>,
#[serde(skip)]
pub observations: Vec<Value>,
#[serde(skip)]
pub curated_observations: Vec<Value>,
#[serde(skip)]
pub scope: Value,
#[serde(skip)]
pub state: AdmissionState,
pub added: Vec<String>,
pub updated: Vec<String>,
pub reverified: Vec<String>,
pub curated_observed: Vec<String>,
pub corrections: Vec<Correction>,
pub removed: Vec<String>,
pub rejected: BTreeMap<String, String>,
pub deferred: Vec<String>,
pub failures: Vec<String>,
pub complete: bool,
}
struct Run<'a> {
transport: &'a dyn AdmissionTransport,
cancel: &'a CancellationToken,
}
impl Run<'_> {
async fn get(
&self,
url: &str,
accept: Option<&'static str>,
range: Option<usize>,
limit: usize,
) -> io::Result<Vec<u8>> {
if self.cancel.is_cancelled() {
return Err(io::Error::new(
io::ErrorKind::Interrupted,
"catalog admission cancelled",
));
}
let url = Url::parse(url).map_err(io::Error::other)?;
let fetched = tokio::select! {
biased;
_ = self.cancel.cancelled() => {
return Err(io::Error::new(io::ErrorKind::Interrupted, "catalog admission cancelled"));
}
result = tokio::time::timeout(Duration::from_secs(180), self.transport.fetch(&url, accept, range, limit)) => {
result.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "admission request timed out"))??
}
};
let ok = if range.is_some() {
matches!(fetched.status, 200 | 206)
} else {
(200..300).contains(&fetched.status)
};
if !ok {
return Err(io::Error::new(
io::ErrorKind::NotFound,
format!("upstream status {}", fetched.status),
));
}
Ok(fetched.body)
}
async fn text(&self, url: &str, limit: usize) -> io::Result<String> {
String::from_utf8(self.get(url, None, None, limit).await?)
.map_err(|_| io::Error::other("upstream text is not UTF-8"))
}
async fn gguf(&self, blob: &str) -> io::Result<Result<Gguf, &'static str>> {
for step in GGUF_STEPS {
let bytes = self.get(blob, None, Some(step), step).await?;
match parse_gguf(&bytes) {
Ok(parsed) => return Ok(Ok(parsed)),
Err(GgufError::Truncated) if bytes.len() >= step => continue,
Err(GgufError::Truncated) => return Ok(Err("GGUF header truncated")),
Err(GgufError::Invalid(reason)) => return Ok(Err(reason)),
}
}
Ok(Err("GGUF header exceeds 64 MiB"))
}
}
fn interrupted(error: &io::Error) -> bool {
error.kind() == io::ErrorKind::Interrupted
}
pub async fn admit(
catalog: &Catalog,
transport: &dyn AdmissionTransport,
options: &AdmissionOptions,
cancel: &CancellationToken,
) -> io::Result<AdmissionOutcome> {
let now = OffsetDateTime::parse(&options.now, &Rfc3339)
.map_err(|_| io::Error::other("invalid admission clock"))?
.to_offset(time::UtcOffset::UTC);
let stamp = now
.format(time::macros::format_description!(
"[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
))
.map_err(io::Error::other)?;
let today = stamp[..10].to_string();
let run = Run { transport, cancel };
let library =
parse_library(&run.text(LIBRARY_URL, MAX_HTML_BYTES).await?).map_err(io::Error::other)?;
let mut outcome = AdmissionOutcome {
complete: true,
state: options.state.clone(),
..Default::default()
};
let curated: BTreeMap<&str, &CatalogModel> = catalog
.models
.iter()
.filter(|model| model.provenance == EntryProvenance::Curated)
.map(|model| (model.id.as_str(), model))
.collect();
let curated_digests: BTreeSet<&str> = curated
.values()
.flat_map(|model| model.quantizations.iter())
.filter_map(|quant| quant.sha256.as_deref())
.collect();
let mut auto: BTreeMap<String, CatalogModel> = catalog
.models
.iter()
.filter(|model| model.provenance == EntryProvenance::Auto)
.map(|model| (model.id.clone(), model.clone()))
.collect();
let listed: BTreeSet<&str> = library
.iter()
.map(|entry| entry.repository.as_str())
.collect();
for id in auto.keys().cloned().collect::<Vec<_>>() {
if !listed.contains(parse_reference(&id).0) {
auto.remove(&id);
outcome.removed.push(id);
}
}
let mut eligible: BTreeSet<String> = BTreeSet::new();
let mut seen_digests: BTreeSet<String> = BTreeSet::new();
let mut new_variants = 0;
for entry in &library {
let repo = entry.repository.as_str();
let tags_url = format!("https://ollama.com/library/{repo}/tags");
let selected = match run.text(&tags_url, MAX_HTML_BYTES).await {
Ok(html) => match parse_tags(&html, repo) {
Ok(tags) => select_variants(&tags),
Err(_) => {
outcome
.failures
.push(format!("{repo}: tags page unparseable"));
outcome.complete = false;
continue;
}
},
Err(error) if interrupted(&error) => return Err(error),
Err(_) => {
outcome
.failures
.push(format!("{repo}: tags page unavailable"));
outcome.complete = false;
continue;
}
};
let selected_ids: BTreeSet<String> =
selected.iter().map(|tag| format!("{repo}:{tag}")).collect();
for id in auto.keys().cloned().collect::<Vec<_>>() {
if parse_reference(&id).0 == repo && !selected_ids.contains(&id) {
auto.remove(&id);
outcome.removed.push(id);
}
}
for tag in &selected {
let id = format!("{repo}:{tag}");
if curated.contains_key(id.as_str()) {
continue;
}
match admit_variant(
&run,
entry,
tag,
&id,
&auto,
&curated_digests,
&mut seen_digests,
&today,
&mut new_variants,
options,
&mut outcome,
)
.await
{
Ok(Some(model)) => {
eligible.insert(id.clone());
auto.insert(id, model);
}
Ok(None) => {}
Err(error) if interrupted(&error) => return Err(error),
Err(error) => {
outcome.failures.push(format!("{id}: {error}"));
outcome.complete = false;
if auto.contains_key(&id) {
eligible.insert(id);
}
}
}
}
}
for id in &outcome.deferred {
eligible.insert(id.clone());
}
let mut corrected: BTreeMap<String, CatalogModel> = BTreeMap::new();
for model in curated.values() {
match reobserve(&run, model, &stamp, &mut outcome.state).await {
Ok(Some(observed)) => {
if options.correct_curated
&& let Some((fixed, changes)) = correct(model, &observed)?
{
corrected.insert(model.id.clone(), fixed);
outcome.corrections.extend(changes);
}
outcome.curated_observed.push(model.id.clone());
outcome.curated_observations.push(observed);
}
Ok(None) => {}
Err(error) if interrupted(&error) => return Err(error),
Err(error) => outcome
.failures
.push(format!("{} (curated): {error}", model.id)),
}
}
let mut models: Vec<CatalogModel> = catalog
.models
.iter()
.filter(|model| model.provenance == EntryProvenance::Curated)
.map(|model| corrected.remove(&model.id).unwrap_or_else(|| model.clone()))
.chain(auto.into_values())
.collect();
models.sort_by(|left, right| {
right
.recency()
.map(|(day, _)| day)
.cmp(&left.recency().map(|(day, _)| day))
.then_with(|| left.id.cmp(&right.id))
});
outcome.observations = models
.iter()
.filter(|model| model.provenance == EntryProvenance::Auto)
.map(|model| observation(model, &stamp))
.collect();
let mut variants: BTreeSet<String> = models.iter().map(|model| model.id.clone()).collect();
variants.extend(eligible);
outcome.scope = json!({
"name": SCOPE_NAME,
"checkedAt": stamp,
"source": "https://ollama.com/library",
"complete": outcome.complete,
"variants": variants,
});
let changed =
!outcome.added.is_empty() || !outcome.updated.is_empty() || !outcome.removed.is_empty();
let encoded = json!({
"schemaVersion": SCHEMA_VERSION,
"generatedAt": if changed { stamp.clone() } else { catalog.generated_at.clone() },
"models": models,
});
outcome.catalog = Some(Catalog::parse(&encoded.to_string()).map_err(io::Error::other)?);
outcome.removed.sort();
Ok(outcome)
}
#[allow(clippy::too_many_arguments)]
async fn admit_variant(
run: &Run<'_>,
entry: &LibraryEntry,
tag: &str,
id: &str,
auto: &BTreeMap<String, CatalogModel>,
curated_digests: &BTreeSet<&str>,
seen_digests: &mut BTreeSet<String>,
today: &str,
new_variants: &mut usize,
options: &AdmissionOptions,
outcome: &mut AdmissionOutcome,
) -> io::Result<Option<CatalogModel>> {
let repo = entry.repository.as_str();
let blob =
|digest: &str| format!("https://registry.ollama.ai/v2/library/{repo}/blobs/{digest}");
let manifest_raw = run
.get(
&format!("https://registry.ollama.ai/v2/library/{repo}/manifests/{tag}"),
Some(MANIFEST_ACCEPT),
None,
MAX_MANIFEST_BYTES,
)
.await?;
let manifest_text = std::str::from_utf8(&manifest_raw)
.map_err(|_| io::Error::other("manifest is not UTF-8"))?;
let Some(layer) = parse_layer(manifest_text).map_err(io::Error::other)? else {
outcome
.rejected
.insert(id.into(), "no GGUF model layer".into());
return Ok(None);
};
let descriptors: ManifestLayers =
serde_json::from_str(manifest_text).map_err(io::Error::other)?;
if curated_digests.contains(layer.sha256.as_str()) || !seen_digests.insert(layer.sha256.clone())
{
return Ok(None);
}
let license_digest = descriptors
.layers
.iter()
.find(|layer| layer.media_type == "application/vnd.ollama.image.license")
.and_then(|layer| sha256(&layer.digest))
.map(str::to_string);
let cache_key = format!(
"{}:{}",
layer.sha256,
license_digest.as_deref().unwrap_or("none")
);
let prior = auto.get(id);
if let Some(prior) = prior
&& prior.quantizations[0].sha256.as_deref() == Some(layer.sha256.as_str())
&& prior.quantizations[0].disk_bytes == layer.disk_bytes
&& !outcome.state.rejections.contains_key(&cache_key)
{
let mut refreshed = prior.clone();
let capabilities = catalog_capabilities(&entry.capabilities);
if capabilities != prior.capabilities {
refreshed.capabilities = capabilities;
outcome.updated.push(id.into());
} else {
outcome.reverified.push(id.into());
}
return Ok(Some(refreshed));
}
if let Some(reason) = outcome.state.rejections.get(&cache_key) {
outcome.rejected.insert(id.into(), reason.clone());
return Ok(None);
}
if prior.is_none() {
if *new_variants >= options.max_new_variants {
outcome.deferred.push(id.into());
outcome.complete = false;
return Ok(None);
}
*new_variants += 1;
}
let config_digest = sha256(&descriptors.config.digest)
.ok_or_else(|| io::Error::other("invalid config digest"))?;
let config_raw = run
.get(
&blob(&format!("sha256:{config_digest}")),
None,
None,
MAX_CONFIG_BYTES,
)
.await?;
let config = RegistryConfig::parse(
std::str::from_utf8(&config_raw).map_err(|_| io::Error::other("config is not UTF-8"))?,
);
let license_head = match &license_digest {
Some(digest) => Some(
String::from_utf8_lossy(
&run.get(
&blob(&format!("sha256:{digest}")),
None,
Some(MAX_LICENSE_HEAD_BYTES),
MAX_LICENSE_HEAD_BYTES,
)
.await?,
)
.into_owned(),
),
None => None,
};
let reject = |outcome: &mut AdmissionOutcome, reason: &str| {
outcome.rejected.insert(id.into(), reason.into());
outcome
.state
.rejections
.insert(cache_key.clone(), reason.into());
Ok(None)
};
let Ok(config) = config else {
return reject(outcome, "model config lacks family, size or file type");
};
let gguf = match run.gguf(&blob(&format!("sha256:{}", layer.sha256))).await? {
Ok(gguf) => gguf,
Err(reason) => return reject(outcome, reason),
};
let built = build_entry(&AdmissionInput {
repository: repo,
tag,
capabilities: &entry.capabilities,
layer: &layer,
config: &config,
gguf: &gguf,
license_head: license_head.as_deref(),
today: prior
.and_then(|model| model.added_at.as_deref())
.unwrap_or(today),
});
match built {
Ok(model) => {
outcome.state.rejections.remove(&cache_key);
if prior.is_some() {
outcome.updated.push(id.into());
} else {
outcome.added.push(id.into());
}
Ok(Some(model))
}
Err(rejection) => reject(outcome, rejection.reason()),
}
}
async fn reobserve(
run: &Run<'_>,
model: &CatalogModel,
stamp: &str,
state: &mut AdmissionState,
) -> io::Result<Option<Value>> {
let Some(reference) = model.source.ollama.as_deref() else {
return Ok(None);
};
let (path, tag) = parse_reference(reference);
let repo = path.strip_prefix("library/").unwrap_or(path);
if repo.contains('/') || !name(repo) || !name(tag) {
return Ok(None);
}
let Some(index) = pulled_quantization(model) else {
return Ok(None);
};
let manifest_url = format!("https://registry.ollama.ai/v2/library/{repo}/manifests/{tag}");
let manifest_raw = run
.get(
&manifest_url,
Some(MANIFEST_ACCEPT),
None,
MAX_MANIFEST_BYTES,
)
.await?;
let manifest_text = std::str::from_utf8(&manifest_raw)
.map_err(|_| io::Error::other("manifest is not UTF-8"))?;
let Some(layer) = parse_layer(manifest_text).map_err(io::Error::other)? else {
return Ok(None);
};
let descriptors: ManifestLayers =
serde_json::from_str(manifest_text).map_err(io::Error::other)?;
let blob = |digest: &str| {
format!("https://registry.ollama.ai/v2/library/{repo}/blobs/sha256:{digest}")
};
let config_digest = sha256(&descriptors.config.digest)
.ok_or_else(|| io::Error::other("invalid config digest"))?;
let config_raw = run
.get(&blob(config_digest), None, None, MAX_CONFIG_BYTES)
.await?;
let Ok(config) = RegistryConfig::parse(
std::str::from_utf8(&config_raw).map_err(|_| io::Error::other("config is not UTF-8"))?,
) else {
return Ok(None);
};
let license_digest = descriptors
.layers
.iter()
.find(|layer| layer.media_type == "application/vnd.ollama.image.license")
.and_then(|layer| sha256(&layer.digest))
.map(str::to_string);
let key = format!(
"{}:{}",
layer.sha256,
license_digest.as_deref().unwrap_or("none")
);
let sourced = match state.facts.get(&key) {
Some(sourced) => sourced.clone(),
None => {
let Ok(gguf) = run.gguf(&blob(&layer.sha256)).await? else {
return Ok(None);
};
let Some(arch) = gguf.architecture().map(str::to_string) else {
return Ok(None);
};
let Some(context_length) = gguf.u64(&format!("{arch}.context_length")) else {
return Ok(None);
};
let head = match &license_digest {
Some(digest) => Some(
String::from_utf8_lossy(
&run.get(
&blob(digest),
None,
Some(MAX_LICENSE_HEAD_BYTES),
MAX_LICENSE_HEAD_BYTES,
)
.await?,
)
.into_owned(),
),
None => None,
};
let sourced = SourcedFacts {
context_length,
architecture: if gguf
.u64(&format!("{arch}.expert_count"))
.is_some_and(|count| count > 0)
{
Architecture::Moe
} else {
Architecture::Dense
},
license: license_id(
gguf.text("general.license"),
gguf.text("general.license.name"),
head.as_deref(),
)
.ok()
.map(str::to_string),
kv_bytes_per_token: kv_bytes_per_token(&gguf),
};
state.facts.insert(key, sourced.clone());
sourced
}
};
let mut quant = serde_json::to_value(&model.quantizations[index]).map_err(io::Error::other)?;
let object = quant
.as_object_mut()
.ok_or_else(|| io::Error::other("invalid quantization"))?;
for field in ["minRamBytes", "minVramBytes", "digestVerified"] {
object.remove(field);
}
object.insert("name".into(), json!(config.file_type.to_ascii_uppercase()));
object.insert("diskBytes".into(), json!(layer.disk_bytes));
object.insert("sha256".into(), json!(layer.sha256));
if layer.projectors.is_empty() {
object.remove("projectors");
} else {
object.insert(
"projectors".into(),
serde_json::to_value(&layer.projectors).map_err(io::Error::other)?,
);
}
let mut facts = serde_json::Map::new();
facts.insert("defaultQuantization".into(), quant);
facts.insert("contextLength".into(), json!(sourced.context_length as f64));
facts.insert(
"architecture".into(),
serde_json::to_value(&sourced.architecture).map_err(io::Error::other)?,
);
if let Some(license) = sourced.license {
facts.insert("license".into(), json!(license));
}
if let (Some(rate), Some(_)) = (sourced.kv_bytes_per_token, model.kv_bytes_per_token) {
facts.insert("kvBytesPerToken".into(), json!(rate));
}
Ok(Some(json!({
"id": model.id,
"checkedAt": stamp,
"sources": [manifest_url, format!("https://ollama.com/library/{repo}:{tag}")],
"facts": facts,
})))
}
fn catalog_value(model: &CatalogModel, field: &str) -> io::Result<Value> {
if field == "defaultQuantization" {
let index = pulled_quantization(model).unwrap_or(0);
let mut quant =
serde_json::to_value(&model.quantizations[index]).map_err(io::Error::other)?;
if let Some(object) = quant.as_object_mut() {
for key in ["minRamBytes", "minVramBytes", "digestVerified"] {
object.remove(key);
}
}
return Ok(quant);
}
Ok(serde_json::to_value(model)
.map_err(io::Error::other)?
.get(field)
.cloned()
.unwrap_or(Value::Null))
}
fn correct(
model: &CatalogModel,
observed: &Value,
) -> io::Result<Option<(CatalogModel, Vec<Correction>)>> {
let Some(facts) = observed["facts"].as_object() else {
return Ok(None);
};
let sources: Vec<String> = observed["sources"]
.as_array()
.map(|items| {
items
.iter()
.filter_map(|item| item.as_str().map(str::to_string))
.collect()
})
.unwrap_or_default();
let mut fixed = model.clone();
let mut changes = Vec::new();
for (field, to) in facts {
let from = catalog_value(model, field)?;
if from == *to || field == "architecture" {
continue;
}
match field.as_str() {
"contextLength" => {
fixed.context_length = to
.as_f64()
.ok_or_else(|| io::Error::other("invalid context"))?
}
"license" => {
fixed.license = to
.as_str()
.ok_or_else(|| io::Error::other("invalid license"))?
.into()
}
"kvBytesPerToken" => fixed.kv_bytes_per_token = to.as_f64(),
"defaultQuantization" => {
let index = pulled_quantization(model).unwrap_or(0);
let name = to["name"]
.as_str()
.ok_or_else(|| io::Error::other("invalid quant"))?;
let projectors: Vec<rigspark_core::sizing::ProjectorArtifact> =
serde_json::from_value(to.get("projectors").cloned().unwrap_or(json!([])))
.map_err(io::Error::other)?;
let quant = rigspark_core::enrich::sized_quantization(
rigspark_core::sizing::parse_param_count(&model.params)
.map_err(io::Error::other)?,
&model.architecture,
name,
to["diskBytes"]
.as_f64()
.ok_or_else(|| io::Error::other("invalid size"))?,
to["sha256"].as_str().map(str::to_string),
projectors,
)
.map_err(io::Error::other)?;
fixed.quantizations[index] = quant;
let mut position = 0;
fixed.quantizations.retain(|candidate| {
let keep = position == index || candidate.name != name;
position += 1;
keep
});
}
_ => continue,
}
changes.push(Correction {
id: model.id.clone(),
field: field.clone(),
from,
to: to.clone(),
sources: sources.clone(),
});
}
if changes.is_empty() {
return Ok(None);
}
let check = json!({"schemaVersion": SCHEMA_VERSION, "generatedAt": "2026-01-01T00:00:00.000Z", "models": [fixed]});
Catalog::parse(&check.to_string()).map_err(io::Error::other)?;
Ok(Some((fixed, changes)))
}