Skip to main content

rigspark_runtime/
admission.rs

1//! Network side of automatic catalog admission: allow-listed fetches and the weekly run.
2
3use crate::catalog_proposals::allowed_registry_redirect;
4use futures_util::StreamExt;
5use rigspark_core::{
6    admission::{
7        AdmissionInput, Gguf, GgufError, LibraryEntry, MAX_CONFIG_BYTES, MAX_GGUF_HEADER_BYTES,
8        MAX_HTML_BYTES, MAX_LICENSE_HEAD_BYTES, RegistryConfig, build_entry, catalog_capabilities,
9        kv_bytes_per_token, license_id, observation, parse_gguf, parse_library, parse_tags,
10        select_variants,
11    },
12    catalog::{Catalog, CatalogModel, EntryProvenance, SCHEMA_VERSION},
13    registry_collector::{
14        MANIFEST_ACCEPT, MAX_MANIFEST_BYTES, parse_layer, parse_reference, pulled_quantization,
15    },
16    sizing::Architecture,
17};
18use serde::{Deserialize, Serialize};
19use serde_json::{Value, json};
20use std::{
21    collections::{BTreeMap, BTreeSet},
22    io,
23    time::Duration,
24};
25use time::{OffsetDateTime, format_description::well_known::Rfc3339};
26use tokio_util::sync::CancellationToken;
27use url::Url;
28
29pub const LIBRARY_URL: &str = "https://ollama.com/library?sort=newest";
30pub const GENERATION_LISTING_URL: &str =
31    "https://huggingface.co/api/models?author=Comfy-Org&sort=createdAt&direction=-1&limit=500";
32pub const SCOPE_NAME: &str = "ollama-local-variants";
33const GGUF_STEPS: [usize; 3] = [8 * 1024 * 1024, 32 * 1024 * 1024, MAX_GGUF_HEADER_BYTES];
34
35fn name(part: &str) -> bool {
36    !part.is_empty()
37        && part.len() <= 128
38        && part.as_bytes()[0].is_ascii_alphanumeric()
39        && part
40            .bytes()
41            .all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte))
42}
43
44/// Only the listing, tags pages, manifests and blobs of official `library/` models.
45pub fn allowed_request(url: &Url) -> bool {
46    if url.scheme() != "https"
47        || url.port().is_some()
48        || !url.username().is_empty()
49        || url.password().is_some()
50        || url.fragment().is_some()
51    {
52        return false;
53    }
54    let segments: Vec<&str> = url.path().trim_start_matches('/').split('/').collect();
55    match (url.host_str(), url.query(), segments.as_slice()) {
56        (Some("ollama.com"), Some("sort=newest"), ["library"]) => true,
57        (Some("ollama.com"), None, ["library", repo, "tags"]) => name(repo),
58        (Some("registry.ollama.ai"), None, ["v2", "library", repo, "manifests", tag]) => {
59            name(repo) && name(tag)
60        }
61        (Some("registry.ollama.ai"), None, ["v2", "library", repo, "blobs", digest]) => {
62            name(repo)
63                && digest.strip_prefix("sha256:").is_some_and(|hex| {
64                    hex.len() == 64 && hex.bytes().all(|byte| byte.is_ascii_hexdigit())
65                })
66        }
67        (Some("huggingface.co"), Some(query), ["api", "models"]) => {
68            Some(query)
69                == GENERATION_LISTING_URL
70                    .split_once('?')
71                    .map(|(_, query)| query)
72        }
73        (Some("huggingface.co"), None | Some("blobs=true"), ["api", "models", owner, repo]) => {
74            name(owner) && name(repo)
75        }
76        (Some("huggingface.co"), None, ["Comfy-Org", repo, "resolve", revision, path @ ..]) => {
77            name(repo)
78                && revision.len() == 40
79                && revision.bytes().all(|byte| byte.is_ascii_hexdigit())
80                && !path.is_empty()
81                && path.len() <= 8
82                && path.iter().all(|segment| name(segment))
83                && path
84                    .last()
85                    .is_some_and(|file| file.ends_with(".safetensors"))
86        }
87        _ => false,
88    }
89}
90
91/// Hugging Face serves `resolve` downloads from its own CDN under `hf.co`.
92pub fn allowed_hub_redirect(url: &Url) -> bool {
93    url.scheme() == "https"
94        && url.port().is_none()
95        && url.username().is_empty()
96        && url.password().is_none()
97        && url
98            .host_str()
99            .is_some_and(|host| host == "huggingface.co" || host.ends_with(".hf.co"))
100}
101
102pub struct Fetched {
103    pub status: u16,
104    pub body: Vec<u8>,
105}
106
107#[async_trait::async_trait]
108pub trait AdmissionTransport: Send + Sync {
109    /// GETs an allow-listed URL. `range` asks for the first `range` bytes; `limit` caps the body.
110    async fn fetch(
111        &self,
112        url: &Url,
113        accept: Option<&'static str>,
114        range: Option<usize>,
115        limit: usize,
116    ) -> io::Result<Fetched>;
117}
118
119pub struct NativeAdmissionTransport {
120    client: reqwest::Client,
121}
122impl NativeAdmissionTransport {
123    pub fn new() -> io::Result<Self> {
124        let client = reqwest::Client::builder()
125            .https_only(true)
126            .no_proxy()
127            .user_agent(concat!(
128                "rigspark-catalog-admission/",
129                env!("CARGO_PKG_VERSION")
130            ))
131            .redirect(reqwest::redirect::Policy::custom(|attempt| {
132                let previous = attempt.previous().last();
133                let from_blob = previous.is_some_and(|url| {
134                    url.host_str() == Some("registry.ollama.ai")
135                        && url.path().contains("/blobs/sha256:")
136                });
137                let from_resolve = previous.is_some_and(|url| {
138                    url.host_str() == Some("huggingface.co") && url.path().contains("/resolve/")
139                });
140                let allowed = (from_blob && allowed_registry_redirect(attempt.url().as_str()))
141                    || (from_resolve && allowed_hub_redirect(attempt.url()));
142                if attempt.previous().len() > 2 || !allowed {
143                    attempt.error("registry redirect refused")
144                } else {
145                    attempt.follow()
146                }
147            }))
148            .connect_timeout(Duration::from_secs(10))
149            .timeout(Duration::from_secs(120))
150            .build()
151            .map_err(|_| io::Error::other("admission client initialization failed"))?;
152        Ok(Self { client })
153    }
154}
155#[async_trait::async_trait]
156impl AdmissionTransport for NativeAdmissionTransport {
157    async fn fetch(
158        &self,
159        url: &Url,
160        accept: Option<&'static str>,
161        range: Option<usize>,
162        limit: usize,
163    ) -> io::Result<Fetched> {
164        if !allowed_request(url) {
165            return Err(io::Error::other("admission URL is not allow-listed"));
166        }
167        let mut request = self.client.get(url.clone());
168        if let Some(accept) = accept {
169            request = request.header(reqwest::header::ACCEPT, accept);
170        }
171        if let Some(range) = range {
172            request = request.header(reqwest::header::RANGE, format!("bytes=0-{}", range - 1));
173        }
174        let response = request
175            .send()
176            .await
177            .map_err(|_| io::Error::other("admission request failed"))?;
178        let status = response.status().as_u16();
179        let mut stream = response.bytes_stream();
180        let mut body = Vec::new();
181        while let Some(chunk) = stream.next().await {
182            let chunk = chunk.map_err(|_| io::Error::other("admission body read failed"))?;
183            body.extend_from_slice(&chunk);
184            if body.len() >= limit {
185                // A range read may stop at the cap; anything else larger than the cap is refused.
186                if range.is_none() && body.len() > limit {
187                    return Err(io::Error::other("admission response exceeds its size cap"));
188                }
189                body.truncate(limit);
190                break;
191            }
192        }
193        Ok(Fetched { status, body })
194    }
195}
196
197#[derive(Deserialize)]
198#[serde(deny_unknown_fields)]
199struct RecordedBody {
200    status: u16,
201    #[serde(default)]
202    text: Option<String>,
203    #[serde(default)]
204    json: Option<Value>,
205    #[serde(default)]
206    hex: Option<String>,
207}
208
209/// Replays recorded responses keyed by URL; unknown URLs answer 404.
210pub struct RecordedAdmissionTransport {
211    responses: BTreeMap<String, (u16, Vec<u8>)>,
212}
213impl RecordedAdmissionTransport {
214    pub fn new(responses: BTreeMap<String, (u16, Vec<u8>)>) -> io::Result<Self> {
215        for key in responses.keys() {
216            let url = Url::parse(key).map_err(io::Error::other)?;
217            if !allowed_request(&url) {
218                return Err(io::Error::other("recorded URL is not allow-listed"));
219            }
220        }
221        Ok(Self { responses })
222    }
223    pub fn parse(raw: &str) -> io::Result<Self> {
224        let recorded: BTreeMap<String, RecordedBody> =
225            serde_json::from_str(raw).map_err(io::Error::other)?;
226        let mut responses = BTreeMap::new();
227        for (url, body) in recorded {
228            let bytes = match (body.text, body.json, body.hex) {
229                (Some(text), None, None) => text.into_bytes(),
230                (None, Some(value), None) => {
231                    serde_json::to_vec(&value).map_err(io::Error::other)?
232                }
233                (None, None, Some(hex)) => decode_hex(&hex)?,
234                (None, None, None) => Vec::new(),
235                _ => return Err(io::Error::other("recorded body must have one encoding")),
236            };
237            responses.insert(url, (body.status, bytes));
238        }
239        Self::new(responses)
240    }
241}
242fn decode_hex(hex: &str) -> io::Result<Vec<u8>> {
243    if !hex.len().is_multiple_of(2) {
244        return Err(io::Error::other("invalid recorded hex body"));
245    }
246    (0..hex.len())
247        .step_by(2)
248        .map(|index| {
249            u8::from_str_radix(&hex[index..index + 2], 16)
250                .map_err(|_| io::Error::other("invalid recorded hex body"))
251        })
252        .collect()
253}
254#[async_trait::async_trait]
255impl AdmissionTransport for RecordedAdmissionTransport {
256    async fn fetch(
257        &self,
258        url: &Url,
259        _accept: Option<&'static str>,
260        range: Option<usize>,
261        limit: usize,
262    ) -> io::Result<Fetched> {
263        if !allowed_request(url) {
264            return Err(io::Error::other("admission URL is not allow-listed"));
265        }
266        let (status, body) = self
267            .responses
268            .get(url.as_str())
269            .cloned()
270            .unwrap_or((404, Vec::new()));
271        let cap = range.unwrap_or(usize::MAX).min(limit);
272        if range.is_none() && body.len() > limit {
273            return Err(io::Error::other("admission response exceeds its size cap"));
274        }
275        Ok(Fetched {
276            status,
277            body: body[..body.len().min(cap)].to_vec(),
278        })
279    }
280}
281
282#[derive(Deserialize)]
283#[serde(rename_all = "camelCase")]
284struct ManifestLayers {
285    config: Descriptor,
286    layers: Vec<Descriptor>,
287}
288#[derive(Deserialize)]
289#[serde(rename_all = "camelCase")]
290struct Descriptor {
291    media_type: String,
292    digest: String,
293}
294fn sha256(digest: &str) -> Option<&str> {
295    digest
296        .strip_prefix("sha256:")
297        .filter(|hex| hex.len() == 64 && hex.bytes().all(|byte| byte.is_ascii_hexdigit()))
298}
299
300#[derive(Clone, Debug, Default, Serialize, Deserialize)]
301#[serde(rename_all = "camelCase", deny_unknown_fields)]
302pub struct AdmissionState {
303    /// `"<model sha256>:<license sha256 or none>"` → rejection reason, so ineligible weights
304    /// are not re-downloaded every week.
305    #[serde(default)]
306    pub rejections: BTreeMap<String, String>,
307    /// Same key → facts read from immutable GGUF headers of curated entries' weights.
308    #[serde(default)]
309    pub facts: BTreeMap<String, SourcedFacts>,
310}
311
312#[derive(Clone, Debug, Serialize, Deserialize)]
313#[serde(rename_all = "camelCase", deny_unknown_fields)]
314pub struct SourcedFacts {
315    pub context_length: u64,
316    pub architecture: Architecture,
317    #[serde(default, skip_serializing_if = "Option::is_none")]
318    pub license: Option<String>,
319    #[serde(default, skip_serializing_if = "Option::is_none")]
320    pub kv_bytes_per_token: Option<f64>,
321}
322
323pub struct AdmissionOptions {
324    pub now: String,
325    pub max_new_variants: usize,
326    pub state: AdmissionState,
327    /// Apply source-backed corrections to curated facts that upstream contradicts.
328    pub correct_curated: bool,
329}
330
331/// One cited change to a curated fact, made because the shipped artifact says otherwise.
332#[derive(Clone, Debug, Serialize)]
333pub struct Correction {
334    pub id: String,
335    pub field: String,
336    pub from: Value,
337    pub to: Value,
338    pub sources: Vec<String>,
339}
340
341#[derive(Default, Serialize)]
342#[serde(rename_all = "camelCase")]
343pub struct AdmissionOutcome {
344    #[serde(skip)]
345    pub catalog: Option<Catalog>,
346    #[serde(skip)]
347    pub observations: Vec<Value>,
348    /// Partial observations of curated entries (facts the collector could source).
349    #[serde(skip)]
350    pub curated_observations: Vec<Value>,
351    #[serde(skip)]
352    pub scope: Value,
353    #[serde(skip)]
354    pub state: AdmissionState,
355    pub added: Vec<String>,
356    pub updated: Vec<String>,
357    pub reverified: Vec<String>,
358    pub curated_observed: Vec<String>,
359    pub corrections: Vec<Correction>,
360    pub removed: Vec<String>,
361    pub rejected: BTreeMap<String, String>,
362    pub deferred: Vec<String>,
363    pub failures: Vec<String>,
364    pub complete: bool,
365}
366
367struct Run<'a> {
368    transport: &'a dyn AdmissionTransport,
369    cancel: &'a CancellationToken,
370}
371impl Run<'_> {
372    async fn get(
373        &self,
374        url: &str,
375        accept: Option<&'static str>,
376        range: Option<usize>,
377        limit: usize,
378    ) -> io::Result<Vec<u8>> {
379        if self.cancel.is_cancelled() {
380            return Err(io::Error::new(
381                io::ErrorKind::Interrupted,
382                "catalog admission cancelled",
383            ));
384        }
385        let url = Url::parse(url).map_err(io::Error::other)?;
386        let fetched = tokio::select! {
387            biased;
388            _ = self.cancel.cancelled() => {
389                return Err(io::Error::new(io::ErrorKind::Interrupted, "catalog admission cancelled"));
390            }
391            result = tokio::time::timeout(Duration::from_secs(180), self.transport.fetch(&url, accept, range, limit)) => {
392                result.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "admission request timed out"))??
393            }
394        };
395        let ok = if range.is_some() {
396            matches!(fetched.status, 200 | 206)
397        } else {
398            (200..300).contains(&fetched.status)
399        };
400        if !ok {
401            return Err(io::Error::new(
402                io::ErrorKind::NotFound,
403                format!("upstream status {}", fetched.status),
404            ));
405        }
406        Ok(fetched.body)
407    }
408    async fn text(&self, url: &str, limit: usize) -> io::Result<String> {
409        String::from_utf8(self.get(url, None, None, limit).await?)
410            .map_err(|_| io::Error::other("upstream text is not UTF-8"))
411    }
412    /// `Ok(Err(reason))` is a content rejection (cacheable); `Err` is a transient failure.
413    async fn gguf(&self, blob: &str) -> io::Result<Result<Gguf, &'static str>> {
414        for step in GGUF_STEPS {
415            let bytes = self.get(blob, None, Some(step), step).await?;
416            match parse_gguf(&bytes) {
417                Ok(parsed) => return Ok(Ok(parsed)),
418                Err(GgufError::Truncated) if bytes.len() >= step => continue,
419                Err(GgufError::Truncated) => return Ok(Err("GGUF header truncated")),
420                Err(GgufError::Invalid(reason)) => return Ok(Err(reason)),
421            }
422        }
423        Ok(Err("GGUF header exceeds 64 MiB"))
424    }
425}
426
427fn interrupted(error: &io::Error) -> bool {
428    error.kind() == io::ErrorKind::Interrupted
429}
430
431/// One admission pass: discover, source, merge and produce evidence. Curated entries are
432/// never modified; a failed listing aborts without changes.
433pub async fn admit(
434    catalog: &Catalog,
435    transport: &dyn AdmissionTransport,
436    options: &AdmissionOptions,
437    cancel: &CancellationToken,
438) -> io::Result<AdmissionOutcome> {
439    let now = OffsetDateTime::parse(&options.now, &Rfc3339)
440        .map_err(|_| io::Error::other("invalid admission clock"))?
441        .to_offset(time::UtcOffset::UTC);
442    let stamp = now
443        .format(time::macros::format_description!(
444            "[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
445        ))
446        .map_err(io::Error::other)?;
447    let today = stamp[..10].to_string();
448    let run = Run { transport, cancel };
449    let library =
450        parse_library(&run.text(LIBRARY_URL, MAX_HTML_BYTES).await?).map_err(io::Error::other)?;
451
452    let mut outcome = AdmissionOutcome {
453        complete: true,
454        state: options.state.clone(),
455        ..Default::default()
456    };
457    let curated: BTreeMap<&str, &CatalogModel> = catalog
458        .models
459        .iter()
460        .filter(|model| model.provenance == EntryProvenance::Curated)
461        .map(|model| (model.id.as_str(), model))
462        .collect();
463    let curated_digests: BTreeSet<&str> = curated
464        .values()
465        .flat_map(|model| model.quantizations.iter())
466        .filter_map(|quant| quant.sha256.as_deref())
467        .collect();
468    let mut auto: BTreeMap<String, CatalogModel> = catalog
469        .models
470        .iter()
471        .filter(|model| model.provenance == EntryProvenance::Auto)
472        .map(|model| (model.id.clone(), model.clone()))
473        .collect();
474    let listed: BTreeSet<&str> = library
475        .iter()
476        .map(|entry| entry.repository.as_str())
477        .collect();
478    for id in auto.keys().cloned().collect::<Vec<_>>() {
479        if !listed.contains(parse_reference(&id).0) {
480            auto.remove(&id);
481            outcome.removed.push(id);
482        }
483    }
484    let mut eligible: BTreeSet<String> = BTreeSet::new();
485    let mut seen_digests: BTreeSet<String> = BTreeSet::new();
486    let mut new_variants = 0;
487
488    for entry in &library {
489        let repo = entry.repository.as_str();
490        let tags_url = format!("https://ollama.com/library/{repo}/tags");
491        let selected = match run.text(&tags_url, MAX_HTML_BYTES).await {
492            Ok(html) => match parse_tags(&html, repo) {
493                Ok(tags) => select_variants(&tags),
494                Err(_) => {
495                    outcome
496                        .failures
497                        .push(format!("{repo}: tags page unparseable"));
498                    outcome.complete = false;
499                    continue;
500                }
501            },
502            Err(error) if interrupted(&error) => return Err(error),
503            Err(_) => {
504                outcome
505                    .failures
506                    .push(format!("{repo}: tags page unavailable"));
507                outcome.complete = false;
508                continue;
509            }
510        };
511        let selected_ids: BTreeSet<String> =
512            selected.iter().map(|tag| format!("{repo}:{tag}")).collect();
513        for id in auto.keys().cloned().collect::<Vec<_>>() {
514            if parse_reference(&id).0 == repo && !selected_ids.contains(&id) {
515                auto.remove(&id);
516                outcome.removed.push(id);
517            }
518        }
519        for tag in &selected {
520            let id = format!("{repo}:{tag}");
521            if curated.contains_key(id.as_str()) {
522                continue;
523            }
524            match admit_variant(
525                &run,
526                entry,
527                tag,
528                &id,
529                &auto,
530                &curated_digests,
531                &mut seen_digests,
532                &today,
533                &mut new_variants,
534                options,
535                &mut outcome,
536            )
537            .await
538            {
539                Ok(Some(model)) => {
540                    eligible.insert(id.clone());
541                    auto.insert(id, model);
542                }
543                Ok(None) => {}
544                Err(error) if interrupted(&error) => return Err(error),
545                Err(error) => {
546                    outcome.failures.push(format!("{id}: {error}"));
547                    outcome.complete = false;
548                    if auto.contains_key(&id) {
549                        eligible.insert(id);
550                    }
551                }
552            }
553        }
554    }
555    for id in &outcome.deferred {
556        eligible.insert(id.clone());
557    }
558    let mut corrected: BTreeMap<String, CatalogModel> = BTreeMap::new();
559    for model in curated.values() {
560        match reobserve(&run, model, &stamp, &mut outcome.state).await {
561            Ok(Some(observed)) => {
562                if options.correct_curated
563                    && let Some((fixed, changes)) = correct(model, &observed)?
564                {
565                    corrected.insert(model.id.clone(), fixed);
566                    outcome.corrections.extend(changes);
567                }
568                outcome.curated_observed.push(model.id.clone());
569                outcome.curated_observations.push(observed);
570            }
571            Ok(None) => {}
572            Err(error) if interrupted(&error) => return Err(error),
573            Err(error) => outcome
574                .failures
575                .push(format!("{} (curated): {error}", model.id)),
576        }
577    }
578
579    let mut models: Vec<CatalogModel> = catalog
580        .models
581        .iter()
582        .filter(|model| model.provenance == EntryProvenance::Curated)
583        .map(|model| corrected.remove(&model.id).unwrap_or_else(|| model.clone()))
584        .chain(auto.into_values())
585        .collect();
586    models.sort_by(|left, right| {
587        right
588            .recency()
589            .map(|(day, _)| day)
590            .cmp(&left.recency().map(|(day, _)| day))
591            .then_with(|| left.id.cmp(&right.id))
592    });
593    outcome.observations = models
594        .iter()
595        .filter(|model| model.provenance == EntryProvenance::Auto)
596        .map(|model| observation(model, &stamp))
597        .collect();
598    let mut variants: BTreeSet<String> = models.iter().map(|model| model.id.clone()).collect();
599    variants.extend(eligible);
600    outcome.scope = json!({
601        "name": SCOPE_NAME,
602        "checkedAt": stamp,
603        "source": "https://ollama.com/library",
604        "complete": outcome.complete,
605        "variants": variants,
606    });
607    let changed =
608        !outcome.added.is_empty() || !outcome.updated.is_empty() || !outcome.removed.is_empty();
609    let encoded = json!({
610        "schemaVersion": SCHEMA_VERSION,
611        "generatedAt": if changed { stamp.clone() } else { catalog.generated_at.clone() },
612        "models": models,
613    });
614    outcome.catalog = Some(Catalog::parse(&encoded.to_string()).map_err(io::Error::other)?);
615    outcome.removed.sort();
616    Ok(outcome)
617}
618
619#[allow(clippy::too_many_arguments)]
620async fn admit_variant(
621    run: &Run<'_>,
622    entry: &LibraryEntry,
623    tag: &str,
624    id: &str,
625    auto: &BTreeMap<String, CatalogModel>,
626    curated_digests: &BTreeSet<&str>,
627    seen_digests: &mut BTreeSet<String>,
628    today: &str,
629    new_variants: &mut usize,
630    options: &AdmissionOptions,
631    outcome: &mut AdmissionOutcome,
632) -> io::Result<Option<CatalogModel>> {
633    let repo = entry.repository.as_str();
634    let blob =
635        |digest: &str| format!("https://registry.ollama.ai/v2/library/{repo}/blobs/{digest}");
636    let manifest_raw = run
637        .get(
638            &format!("https://registry.ollama.ai/v2/library/{repo}/manifests/{tag}"),
639            Some(MANIFEST_ACCEPT),
640            None,
641            MAX_MANIFEST_BYTES,
642        )
643        .await?;
644    let manifest_text = std::str::from_utf8(&manifest_raw)
645        .map_err(|_| io::Error::other("manifest is not UTF-8"))?;
646    let Some(layer) = parse_layer(manifest_text).map_err(io::Error::other)? else {
647        outcome
648            .rejected
649            .insert(id.into(), "no GGUF model layer".into());
650        return Ok(None);
651    };
652    let descriptors: ManifestLayers =
653        serde_json::from_str(manifest_text).map_err(io::Error::other)?;
654    if curated_digests.contains(layer.sha256.as_str()) || !seen_digests.insert(layer.sha256.clone())
655    {
656        return Ok(None);
657    }
658    let license_digest = descriptors
659        .layers
660        .iter()
661        .find(|layer| layer.media_type == "application/vnd.ollama.image.license")
662        .and_then(|layer| sha256(&layer.digest))
663        .map(str::to_string);
664    let cache_key = format!(
665        "{}:{}",
666        layer.sha256,
667        license_digest.as_deref().unwrap_or("none")
668    );
669    let prior = auto.get(id);
670    if let Some(prior) = prior
671        && prior.quantizations[0].sha256.as_deref() == Some(layer.sha256.as_str())
672        && prior.quantizations[0].disk_bytes == layer.disk_bytes
673        && !outcome.state.rejections.contains_key(&cache_key)
674    {
675        let mut refreshed = prior.clone();
676        let capabilities = catalog_capabilities(&entry.capabilities);
677        if capabilities != prior.capabilities {
678            refreshed.capabilities = capabilities;
679            outcome.updated.push(id.into());
680        } else {
681            outcome.reverified.push(id.into());
682        }
683        return Ok(Some(refreshed));
684    }
685    if let Some(reason) = outcome.state.rejections.get(&cache_key) {
686        outcome.rejected.insert(id.into(), reason.clone());
687        return Ok(None);
688    }
689    if prior.is_none() {
690        if *new_variants >= options.max_new_variants {
691            outcome.deferred.push(id.into());
692            outcome.complete = false;
693            return Ok(None);
694        }
695        *new_variants += 1;
696    }
697    let config_digest = sha256(&descriptors.config.digest)
698        .ok_or_else(|| io::Error::other("invalid config digest"))?;
699    let config_raw = run
700        .get(
701            &blob(&format!("sha256:{config_digest}")),
702            None,
703            None,
704            MAX_CONFIG_BYTES,
705        )
706        .await?;
707    let config = RegistryConfig::parse(
708        std::str::from_utf8(&config_raw).map_err(|_| io::Error::other("config is not UTF-8"))?,
709    );
710    let license_head = match &license_digest {
711        Some(digest) => Some(
712            String::from_utf8_lossy(
713                &run.get(
714                    &blob(&format!("sha256:{digest}")),
715                    None,
716                    Some(MAX_LICENSE_HEAD_BYTES),
717                    MAX_LICENSE_HEAD_BYTES,
718                )
719                .await?,
720            )
721            .into_owned(),
722        ),
723        None => None,
724    };
725    let reject = |outcome: &mut AdmissionOutcome, reason: &str| {
726        outcome.rejected.insert(id.into(), reason.into());
727        outcome
728            .state
729            .rejections
730            .insert(cache_key.clone(), reason.into());
731        Ok(None)
732    };
733    let Ok(config) = config else {
734        return reject(outcome, "model config lacks family, size or file type");
735    };
736    let gguf = match run.gguf(&blob(&format!("sha256:{}", layer.sha256))).await? {
737        Ok(gguf) => gguf,
738        Err(reason) => return reject(outcome, reason),
739    };
740    let built = build_entry(&AdmissionInput {
741        repository: repo,
742        tag,
743        capabilities: &entry.capabilities,
744        layer: &layer,
745        config: &config,
746        gguf: &gguf,
747        license_head: license_head.as_deref(),
748        today: prior
749            .and_then(|model| model.added_at.as_deref())
750            .unwrap_or(today),
751    });
752    match built {
753        Ok(model) => {
754            outcome.state.rejections.remove(&cache_key);
755            if prior.is_some() {
756                outcome.updated.push(id.into());
757            } else {
758                outcome.added.push(id.into());
759            }
760            Ok(Some(model))
761        }
762        Err(rejection) => reject(outcome, rejection.reason()),
763    }
764}
765
766/// A partial observation of a curated entry: the pulled artifact plus GGUF-sourced facts,
767/// citing only the Ollama reads that produced them. Curated entries are never modified.
768async fn reobserve(
769    run: &Run<'_>,
770    model: &CatalogModel,
771    stamp: &str,
772    state: &mut AdmissionState,
773) -> io::Result<Option<Value>> {
774    let Some(reference) = model.source.ollama.as_deref() else {
775        return Ok(None);
776    };
777    let (path, tag) = parse_reference(reference);
778    let repo = path.strip_prefix("library/").unwrap_or(path);
779    if repo.contains('/') || !name(repo) || !name(tag) {
780        return Ok(None);
781    }
782    let Some(index) = pulled_quantization(model) else {
783        return Ok(None);
784    };
785    let manifest_url = format!("https://registry.ollama.ai/v2/library/{repo}/manifests/{tag}");
786    let manifest_raw = run
787        .get(
788            &manifest_url,
789            Some(MANIFEST_ACCEPT),
790            None,
791            MAX_MANIFEST_BYTES,
792        )
793        .await?;
794    let manifest_text = std::str::from_utf8(&manifest_raw)
795        .map_err(|_| io::Error::other("manifest is not UTF-8"))?;
796    let Some(layer) = parse_layer(manifest_text).map_err(io::Error::other)? else {
797        return Ok(None);
798    };
799    let descriptors: ManifestLayers =
800        serde_json::from_str(manifest_text).map_err(io::Error::other)?;
801    let blob = |digest: &str| {
802        format!("https://registry.ollama.ai/v2/library/{repo}/blobs/sha256:{digest}")
803    };
804    let config_digest = sha256(&descriptors.config.digest)
805        .ok_or_else(|| io::Error::other("invalid config digest"))?;
806    let config_raw = run
807        .get(&blob(config_digest), None, None, MAX_CONFIG_BYTES)
808        .await?;
809    let Ok(config) = RegistryConfig::parse(
810        std::str::from_utf8(&config_raw).map_err(|_| io::Error::other("config is not UTF-8"))?,
811    ) else {
812        return Ok(None);
813    };
814    let license_digest = descriptors
815        .layers
816        .iter()
817        .find(|layer| layer.media_type == "application/vnd.ollama.image.license")
818        .and_then(|layer| sha256(&layer.digest))
819        .map(str::to_string);
820    let key = format!(
821        "{}:{}",
822        layer.sha256,
823        license_digest.as_deref().unwrap_or("none")
824    );
825    let sourced = match state.facts.get(&key) {
826        Some(sourced) => sourced.clone(),
827        None => {
828            let Ok(gguf) = run.gguf(&blob(&layer.sha256)).await? else {
829                return Ok(None);
830            };
831            let Some(arch) = gguf.architecture().map(str::to_string) else {
832                return Ok(None);
833            };
834            let Some(context_length) = gguf.u64(&format!("{arch}.context_length")) else {
835                return Ok(None);
836            };
837            let head = match &license_digest {
838                Some(digest) => Some(
839                    String::from_utf8_lossy(
840                        &run.get(
841                            &blob(digest),
842                            None,
843                            Some(MAX_LICENSE_HEAD_BYTES),
844                            MAX_LICENSE_HEAD_BYTES,
845                        )
846                        .await?,
847                    )
848                    .into_owned(),
849                ),
850                None => None,
851            };
852            let sourced = SourcedFacts {
853                context_length,
854                architecture: if gguf
855                    .u64(&format!("{arch}.expert_count"))
856                    .is_some_and(|count| count > 0)
857                {
858                    Architecture::Moe
859                } else {
860                    Architecture::Dense
861                },
862                license: license_id(
863                    gguf.text("general.license"),
864                    gguf.text("general.license.name"),
865                    head.as_deref(),
866                )
867                .ok()
868                .map(str::to_string),
869                kv_bytes_per_token: kv_bytes_per_token(&gguf),
870            };
871            state.facts.insert(key, sourced.clone());
872            sourced
873        }
874    };
875    let mut quant = serde_json::to_value(&model.quantizations[index]).map_err(io::Error::other)?;
876    let object = quant
877        .as_object_mut()
878        .ok_or_else(|| io::Error::other("invalid quantization"))?;
879    for field in ["minRamBytes", "minVramBytes", "digestVerified"] {
880        object.remove(field);
881    }
882    object.insert("name".into(), json!(config.file_type.to_ascii_uppercase()));
883    object.insert("diskBytes".into(), json!(layer.disk_bytes));
884    object.insert("sha256".into(), json!(layer.sha256));
885    if layer.projectors.is_empty() {
886        object.remove("projectors");
887    } else {
888        object.insert(
889            "projectors".into(),
890            serde_json::to_value(&layer.projectors).map_err(io::Error::other)?,
891        );
892    }
893    let mut facts = serde_json::Map::new();
894    facts.insert("defaultQuantization".into(), quant);
895    facts.insert("contextLength".into(), json!(sourced.context_length as f64));
896    facts.insert(
897        "architecture".into(),
898        serde_json::to_value(&sourced.architecture).map_err(io::Error::other)?,
899    );
900    if let Some(license) = sourced.license {
901        facts.insert("license".into(), json!(license));
902    }
903    // Asserted only when the catalog has a value to compare; unknown is not a contradiction.
904    if let (Some(rate), Some(_)) = (sourced.kv_bytes_per_token, model.kv_bytes_per_token) {
905        facts.insert("kvBytesPerToken".into(), json!(rate));
906    }
907    Ok(Some(json!({
908        "id": model.id,
909        "checkedAt": stamp,
910        "sources": [manifest_url, format!("https://ollama.com/library/{repo}:{tag}")],
911        "facts": facts,
912    })))
913}
914
915/// The catalog's own representation of a partially observed fact.
916fn catalog_value(model: &CatalogModel, field: &str) -> io::Result<Value> {
917    if field == "defaultQuantization" {
918        let index = pulled_quantization(model).unwrap_or(0);
919        let mut quant =
920            serde_json::to_value(&model.quantizations[index]).map_err(io::Error::other)?;
921        if let Some(object) = quant.as_object_mut() {
922            for key in ["minRamBytes", "minVramBytes", "digestVerified"] {
923                object.remove(key);
924            }
925        }
926        return Ok(quant);
927    }
928    Ok(serde_json::to_value(model)
929        .map_err(io::Error::other)?
930        .get(field)
931        .cloned()
932        .unwrap_or(Value::Null))
933}
934
935/// Rewrites contradicted, correctable curated facts to what the shipped artifact says.
936/// Architecture contradictions are left for review: they change how the model is sized.
937fn correct(
938    model: &CatalogModel,
939    observed: &Value,
940) -> io::Result<Option<(CatalogModel, Vec<Correction>)>> {
941    let Some(facts) = observed["facts"].as_object() else {
942        return Ok(None);
943    };
944    let sources: Vec<String> = observed["sources"]
945        .as_array()
946        .map(|items| {
947            items
948                .iter()
949                .filter_map(|item| item.as_str().map(str::to_string))
950                .collect()
951        })
952        .unwrap_or_default();
953    let mut fixed = model.clone();
954    let mut changes = Vec::new();
955    for (field, to) in facts {
956        let from = catalog_value(model, field)?;
957        if from == *to || field == "architecture" {
958            continue;
959        }
960        match field.as_str() {
961            "contextLength" => {
962                fixed.context_length = to
963                    .as_f64()
964                    .ok_or_else(|| io::Error::other("invalid context"))?
965            }
966            "license" => {
967                fixed.license = to
968                    .as_str()
969                    .ok_or_else(|| io::Error::other("invalid license"))?
970                    .into()
971            }
972            "kvBytesPerToken" => fixed.kv_bytes_per_token = to.as_f64(),
973            "defaultQuantization" => {
974                let index = pulled_quantization(model).unwrap_or(0);
975                let name = to["name"]
976                    .as_str()
977                    .ok_or_else(|| io::Error::other("invalid quant"))?;
978                let projectors: Vec<rigspark_core::sizing::ProjectorArtifact> =
979                    serde_json::from_value(to.get("projectors").cloned().unwrap_or(json!([])))
980                        .map_err(io::Error::other)?;
981                let quant = rigspark_core::enrich::sized_quantization(
982                    rigspark_core::sizing::parse_param_count(&model.params)
983                        .map_err(io::Error::other)?,
984                    &model.architecture,
985                    name,
986                    to["diskBytes"]
987                        .as_f64()
988                        .ok_or_else(|| io::Error::other("invalid size"))?,
989                    to["sha256"].as_str().map(str::to_string),
990                    projectors,
991                )
992                .map_err(io::Error::other)?;
993                fixed.quantizations[index] = quant;
994                // Facts for other quantizations named like the new one are no longer sourced.
995                let mut position = 0;
996                fixed.quantizations.retain(|candidate| {
997                    let keep = position == index || candidate.name != name;
998                    position += 1;
999                    keep
1000                });
1001            }
1002            _ => continue,
1003        }
1004        changes.push(Correction {
1005            id: model.id.clone(),
1006            field: field.clone(),
1007            from,
1008            to: to.clone(),
1009            sources: sources.clone(),
1010        });
1011    }
1012    if changes.is_empty() {
1013        return Ok(None);
1014    }
1015    let check = json!({"schemaVersion": SCHEMA_VERSION, "generatedAt": "2026-01-01T00:00:00.000Z", "models": [fixed]});
1016    Catalog::parse(&check.to_string()).map_err(io::Error::other)?;
1017    Ok(Some((fixed, changes)))
1018}