Skip to main content

rigspark_runtime/
admission.rs

1//! Network side of automatic catalog admission: allow-listed fetches and the weekly run.
2
3use crate::catalog_proposals::allowed_registry_redirect;
4use futures_util::StreamExt;
5use rigspark_core::{
6    admission::{
7        AdmissionInput, Gguf, GgufError, LibraryEntry, MAX_CONFIG_BYTES, MAX_GGUF_HEADER_BYTES,
8        MAX_HTML_BYTES, MAX_LICENSE_HEAD_BYTES, RegistryConfig, build_entry, catalog_capabilities,
9        kv_bytes_per_token, license_id, observation, parse_gguf, parse_library, parse_tags,
10        select_variants,
11    },
12    catalog::{Catalog, CatalogModel, EntryProvenance, SCHEMA_VERSION},
13    registry_collector::{
14        MANIFEST_ACCEPT, MAX_MANIFEST_BYTES, parse_layer, parse_reference, pulled_quantization,
15    },
16    sizing::Architecture,
17};
18use serde::{Deserialize, Serialize};
19use serde_json::{Value, json};
20use std::{
21    collections::{BTreeMap, BTreeSet},
22    io,
23    time::Duration,
24};
25use time::{OffsetDateTime, format_description::well_known::Rfc3339};
26use tokio_util::sync::CancellationToken;
27use url::Url;
28
29pub const LIBRARY_URL: &str = "https://ollama.com/library?sort=newest";
30pub const GENERATION_LISTING_URL: &str =
31    "https://huggingface.co/api/models?author=Comfy-Org&sort=createdAt&direction=-1&limit=500";
32pub const SCOPE_NAME: &str = "ollama-local-variants";
33const GGUF_STEPS: [usize; 3] = [8 * 1024 * 1024, 32 * 1024 * 1024, MAX_GGUF_HEADER_BYTES];
34
35fn name(part: &str) -> bool {
36    !part.is_empty()
37        && part.len() <= 128
38        && part.as_bytes()[0].is_ascii_alphanumeric()
39        && part
40            .bytes()
41            .all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte))
42}
43
44/// Only the listing, tags pages, manifests and blobs of official `library/` models.
45pub fn allowed_request(url: &Url) -> bool {
46    if url.scheme() != "https"
47        || url.port().is_some()
48        || !url.username().is_empty()
49        || url.password().is_some()
50        || url.fragment().is_some()
51    {
52        return false;
53    }
54    let segments: Vec<&str> = url.path().trim_start_matches('/').split('/').collect();
55    match (url.host_str(), url.query(), segments.as_slice()) {
56        (Some("ollama.com"), Some("sort=newest"), ["library"]) => true,
57        (Some("ollama.com"), None, ["library", repo, "tags"]) => name(repo),
58        (Some("registry.ollama.ai"), None, ["v2", "library", repo, "manifests", tag]) => {
59            name(repo) && name(tag)
60        }
61        (Some("registry.ollama.ai"), None, ["v2", "library", repo, "blobs", digest]) => {
62            name(repo)
63                && digest.strip_prefix("sha256:").is_some_and(|hex| {
64                    hex.len() == 64 && hex.bytes().all(|byte| byte.is_ascii_hexdigit())
65                })
66        }
67        (Some("huggingface.co"), Some(query), ["api", "models"]) => {
68            Some(query)
69                == GENERATION_LISTING_URL
70                    .split_once('?')
71                    .map(|(_, query)| query)
72        }
73        (Some("huggingface.co"), None | Some("blobs=true"), ["api", "models", owner, repo]) => {
74            name(owner) && name(repo)
75        }
76        (
77            Some("huggingface.co"),
78            Some("blobs=true"),
79            ["api", "models", owner, repo, "revision", revision],
80        ) => {
81            name(owner)
82                && name(repo)
83                && revision.len() == 40
84                && revision.bytes().all(|byte| byte.is_ascii_hexdigit())
85        }
86        (Some("huggingface.co"), None, [owner, repo, "resolve", revision, path @ ..]) => {
87            name(owner)
88                && name(repo)
89                && revision.len() == 40
90                && revision.bytes().all(|byte| byte.is_ascii_hexdigit())
91                && !path.is_empty()
92                && path.len() <= 8
93                && path.iter().all(|segment| name(segment))
94                && path.last().is_some_and(|file| {
95                    file.ends_with(".gguf")
96                        || (*owner == "Comfy-Org" && file.ends_with(".safetensors"))
97                })
98        }
99        _ => false,
100    }
101}
102
103/// Hugging Face serves `resolve` downloads from its own CDN under `hf.co`.
104pub fn allowed_hub_redirect(url: &Url) -> bool {
105    url.scheme() == "https"
106        && url.port().is_none()
107        && url.username().is_empty()
108        && url.password().is_none()
109        && url
110            .host_str()
111            .is_some_and(|host| host == "huggingface.co" || host.ends_with(".hf.co"))
112}
113
114pub struct Fetched {
115    pub status: u16,
116    pub body: Vec<u8>,
117}
118
119#[async_trait::async_trait]
120pub trait AdmissionTransport: Send + Sync {
121    /// GETs an allow-listed URL. `range` asks for the first `range` bytes; `limit` caps the body.
122    async fn fetch(
123        &self,
124        url: &Url,
125        accept: Option<&'static str>,
126        range: Option<usize>,
127        limit: usize,
128    ) -> io::Result<Fetched>;
129}
130
131pub struct NativeAdmissionTransport {
132    client: reqwest::Client,
133}
134impl NativeAdmissionTransport {
135    pub fn new() -> io::Result<Self> {
136        let client = reqwest::Client::builder()
137            .https_only(true)
138            .no_proxy()
139            .user_agent(concat!(
140                "rigspark-catalog-admission/",
141                env!("CARGO_PKG_VERSION")
142            ))
143            .redirect(reqwest::redirect::Policy::custom(|attempt| {
144                let previous = attempt.previous().last();
145                let from_blob = previous.is_some_and(|url| {
146                    url.host_str() == Some("registry.ollama.ai")
147                        && url.path().contains("/blobs/sha256:")
148                });
149                let from_resolve = previous.is_some_and(|url| {
150                    url.host_str() == Some("huggingface.co") && url.path().contains("/resolve/")
151                });
152                let allowed = (from_blob && allowed_registry_redirect(attempt.url().as_str()))
153                    || (from_resolve && allowed_hub_redirect(attempt.url()));
154                if attempt.previous().len() > 2 || !allowed {
155                    attempt.error("registry redirect refused")
156                } else {
157                    attempt.follow()
158                }
159            }))
160            .connect_timeout(Duration::from_secs(10))
161            .timeout(Duration::from_secs(120))
162            .build()
163            .map_err(|_| io::Error::other("admission client initialization failed"))?;
164        Ok(Self { client })
165    }
166}
167#[async_trait::async_trait]
168impl AdmissionTransport for NativeAdmissionTransport {
169    async fn fetch(
170        &self,
171        url: &Url,
172        accept: Option<&'static str>,
173        range: Option<usize>,
174        limit: usize,
175    ) -> io::Result<Fetched> {
176        if !allowed_request(url) {
177            return Err(io::Error::other("admission URL is not allow-listed"));
178        }
179        let mut request = self.client.get(url.clone());
180        if let Some(accept) = accept {
181            request = request.header(reqwest::header::ACCEPT, accept);
182        }
183        if let Some(range) = range {
184            request = request.header(reqwest::header::RANGE, format!("bytes=0-{}", range - 1));
185        }
186        let response = request
187            .send()
188            .await
189            .map_err(|_| io::Error::other("admission request failed"))?;
190        let status = response.status().as_u16();
191        let mut stream = response.bytes_stream();
192        let mut body = Vec::new();
193        while let Some(chunk) = stream.next().await {
194            let chunk = chunk.map_err(|_| io::Error::other("admission body read failed"))?;
195            body.extend_from_slice(&chunk);
196            if body.len() >= limit {
197                // A range read may stop at the cap; anything else larger than the cap is refused.
198                if range.is_none() && body.len() > limit {
199                    return Err(io::Error::other("admission response exceeds its size cap"));
200                }
201                body.truncate(limit);
202                break;
203            }
204        }
205        Ok(Fetched { status, body })
206    }
207}
208
209#[derive(Deserialize)]
210#[serde(deny_unknown_fields)]
211struct RecordedBody {
212    status: u16,
213    #[serde(default)]
214    text: Option<String>,
215    #[serde(default)]
216    json: Option<Value>,
217    #[serde(default)]
218    hex: Option<String>,
219}
220
221/// Replays recorded responses keyed by URL; unknown URLs answer 404.
222pub struct RecordedAdmissionTransport {
223    responses: BTreeMap<String, (u16, Vec<u8>)>,
224}
225impl RecordedAdmissionTransport {
226    pub fn new(responses: BTreeMap<String, (u16, Vec<u8>)>) -> io::Result<Self> {
227        for key in responses.keys() {
228            let url = Url::parse(key).map_err(io::Error::other)?;
229            if !allowed_request(&url) {
230                return Err(io::Error::other("recorded URL is not allow-listed"));
231            }
232        }
233        Ok(Self { responses })
234    }
235    pub fn parse(raw: &str) -> io::Result<Self> {
236        let recorded: BTreeMap<String, RecordedBody> =
237            serde_json::from_str(raw).map_err(io::Error::other)?;
238        let mut responses = BTreeMap::new();
239        for (url, body) in recorded {
240            let bytes = match (body.text, body.json, body.hex) {
241                (Some(text), None, None) => text.into_bytes(),
242                (None, Some(value), None) => {
243                    serde_json::to_vec(&value).map_err(io::Error::other)?
244                }
245                (None, None, Some(hex)) => decode_hex(&hex)?,
246                (None, None, None) => Vec::new(),
247                _ => return Err(io::Error::other("recorded body must have one encoding")),
248            };
249            responses.insert(url, (body.status, bytes));
250        }
251        Self::new(responses)
252    }
253}
254fn decode_hex(hex: &str) -> io::Result<Vec<u8>> {
255    if !hex.len().is_multiple_of(2) {
256        return Err(io::Error::other("invalid recorded hex body"));
257    }
258    (0..hex.len())
259        .step_by(2)
260        .map(|index| {
261            u8::from_str_radix(&hex[index..index + 2], 16)
262                .map_err(|_| io::Error::other("invalid recorded hex body"))
263        })
264        .collect()
265}
266#[async_trait::async_trait]
267impl AdmissionTransport for RecordedAdmissionTransport {
268    async fn fetch(
269        &self,
270        url: &Url,
271        _accept: Option<&'static str>,
272        range: Option<usize>,
273        limit: usize,
274    ) -> io::Result<Fetched> {
275        if !allowed_request(url) {
276            return Err(io::Error::other("admission URL is not allow-listed"));
277        }
278        let (status, body) = self
279            .responses
280            .get(url.as_str())
281            .cloned()
282            .unwrap_or((404, Vec::new()));
283        let cap = range.unwrap_or(usize::MAX).min(limit);
284        if range.is_none() && body.len() > limit {
285            return Err(io::Error::other("admission response exceeds its size cap"));
286        }
287        Ok(Fetched {
288            status,
289            body: body[..body.len().min(cap)].to_vec(),
290        })
291    }
292}
293
294#[derive(Deserialize)]
295#[serde(rename_all = "camelCase")]
296struct ManifestLayers {
297    config: Descriptor,
298    layers: Vec<Descriptor>,
299}
300#[derive(Deserialize)]
301#[serde(rename_all = "camelCase")]
302struct Descriptor {
303    media_type: String,
304    digest: String,
305}
306fn sha256(digest: &str) -> Option<&str> {
307    digest
308        .strip_prefix("sha256:")
309        .filter(|hex| hex.len() == 64 && hex.bytes().all(|byte| byte.is_ascii_hexdigit()))
310}
311
312#[derive(Clone, Debug, Default, Serialize, Deserialize)]
313#[serde(rename_all = "camelCase", deny_unknown_fields)]
314pub struct AdmissionState {
315    /// `"<model sha256>:<license sha256 or none>"` → rejection reason, so ineligible weights
316    /// are not re-downloaded every week.
317    #[serde(default)]
318    pub rejections: BTreeMap<String, String>,
319    /// Same key → facts read from immutable GGUF headers of curated entries' weights.
320    #[serde(default)]
321    pub facts: BTreeMap<String, SourcedFacts>,
322}
323
324#[derive(Clone, Debug, Serialize, Deserialize)]
325#[serde(rename_all = "camelCase", deny_unknown_fields)]
326pub struct SourcedFacts {
327    pub context_length: u64,
328    pub architecture: Architecture,
329    #[serde(default, skip_serializing_if = "Option::is_none")]
330    pub license: Option<String>,
331    #[serde(default, skip_serializing_if = "Option::is_none")]
332    pub kv_bytes_per_token: Option<f64>,
333}
334
335pub struct AdmissionOptions {
336    pub now: String,
337    pub max_new_variants: usize,
338    pub state: AdmissionState,
339    /// Apply source-backed corrections to curated facts that upstream contradicts.
340    pub correct_curated: bool,
341}
342
343/// One cited change to a curated fact, made because the shipped artifact says otherwise.
344#[derive(Clone, Debug, Serialize)]
345pub struct Correction {
346    pub id: String,
347    pub field: String,
348    pub from: Value,
349    pub to: Value,
350    pub sources: Vec<String>,
351}
352
353#[derive(Default, Serialize)]
354#[serde(rename_all = "camelCase")]
355pub struct AdmissionOutcome {
356    #[serde(skip)]
357    pub catalog: Option<Catalog>,
358    #[serde(skip)]
359    pub observations: Vec<Value>,
360    /// Partial observations of curated entries (facts the collector could source).
361    #[serde(skip)]
362    pub curated_observations: Vec<Value>,
363    #[serde(skip)]
364    pub scope: Value,
365    #[serde(skip)]
366    pub state: AdmissionState,
367    pub added: Vec<String>,
368    pub updated: Vec<String>,
369    pub reverified: Vec<String>,
370    pub curated_observed: Vec<String>,
371    pub corrections: Vec<Correction>,
372    pub removed: Vec<String>,
373    pub rejected: BTreeMap<String, String>,
374    pub deferred: Vec<String>,
375    pub failures: Vec<String>,
376    pub complete: bool,
377}
378
379struct Run<'a> {
380    transport: &'a dyn AdmissionTransport,
381    cancel: &'a CancellationToken,
382}
383impl Run<'_> {
384    async fn get(
385        &self,
386        url: &str,
387        accept: Option<&'static str>,
388        range: Option<usize>,
389        limit: usize,
390    ) -> io::Result<Vec<u8>> {
391        if self.cancel.is_cancelled() {
392            return Err(io::Error::new(
393                io::ErrorKind::Interrupted,
394                "catalog admission cancelled",
395            ));
396        }
397        let url = Url::parse(url).map_err(io::Error::other)?;
398        let fetched = tokio::select! {
399            biased;
400            _ = self.cancel.cancelled() => {
401                return Err(io::Error::new(io::ErrorKind::Interrupted, "catalog admission cancelled"));
402            }
403            result = tokio::time::timeout(Duration::from_secs(180), self.transport.fetch(&url, accept, range, limit)) => {
404                result.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "admission request timed out"))??
405            }
406        };
407        let ok = if range.is_some() {
408            matches!(fetched.status, 200 | 206)
409        } else {
410            (200..300).contains(&fetched.status)
411        };
412        if !ok {
413            return Err(io::Error::new(
414                io::ErrorKind::NotFound,
415                format!("upstream status {}", fetched.status),
416            ));
417        }
418        Ok(fetched.body)
419    }
420    async fn text(&self, url: &str, limit: usize) -> io::Result<String> {
421        String::from_utf8(self.get(url, None, None, limit).await?)
422            .map_err(|_| io::Error::other("upstream text is not UTF-8"))
423    }
424    /// `Ok(Err(reason))` is a content rejection (cacheable); `Err` is a transient failure.
425    async fn gguf(&self, blob: &str) -> io::Result<Result<Gguf, &'static str>> {
426        for step in GGUF_STEPS {
427            let bytes = self.get(blob, None, Some(step), step).await?;
428            match parse_gguf(&bytes) {
429                Ok(parsed) => return Ok(Ok(parsed)),
430                Err(GgufError::Truncated) if bytes.len() >= step => continue,
431                Err(GgufError::Truncated) => return Ok(Err("GGUF header truncated")),
432                Err(GgufError::Invalid(reason)) => return Ok(Err(reason)),
433            }
434        }
435        Ok(Err("GGUF header exceeds 64 MiB"))
436    }
437}
438
439fn interrupted(error: &io::Error) -> bool {
440    error.kind() == io::ErrorKind::Interrupted
441}
442
443/// One admission pass: discover, source, merge and produce evidence. Curated entries are
444/// never modified; a failed listing aborts without changes.
445pub async fn admit(
446    catalog: &Catalog,
447    transport: &dyn AdmissionTransport,
448    options: &AdmissionOptions,
449    cancel: &CancellationToken,
450) -> io::Result<AdmissionOutcome> {
451    let now = OffsetDateTime::parse(&options.now, &Rfc3339)
452        .map_err(|_| io::Error::other("invalid admission clock"))?
453        .to_offset(time::UtcOffset::UTC);
454    let stamp = now
455        .format(time::macros::format_description!(
456            "[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
457        ))
458        .map_err(io::Error::other)?;
459    let today = stamp[..10].to_string();
460    let run = Run { transport, cancel };
461    let library =
462        parse_library(&run.text(LIBRARY_URL, MAX_HTML_BYTES).await?).map_err(io::Error::other)?;
463
464    let mut outcome = AdmissionOutcome {
465        complete: true,
466        state: options.state.clone(),
467        ..Default::default()
468    };
469    let curated: BTreeMap<&str, &CatalogModel> = catalog
470        .models
471        .iter()
472        .filter(|model| model.provenance == EntryProvenance::Curated)
473        .map(|model| (model.id.as_str(), model))
474        .collect();
475    let curated_digests: BTreeSet<&str> = curated
476        .values()
477        .flat_map(|model| model.quantizations.iter())
478        .filter_map(|quant| quant.sha256.as_deref())
479        .collect();
480    let mut auto: BTreeMap<String, CatalogModel> = catalog
481        .models
482        .iter()
483        .filter(|model| model.provenance == EntryProvenance::Auto)
484        .map(|model| (model.id.clone(), model.clone()))
485        .collect();
486    let listed: BTreeSet<&str> = library
487        .iter()
488        .map(|entry| entry.repository.as_str())
489        .collect();
490    for id in auto.keys().cloned().collect::<Vec<_>>() {
491        if !listed.contains(parse_reference(&id).0) {
492            auto.remove(&id);
493            outcome.removed.push(id);
494        }
495    }
496    let mut eligible: BTreeSet<String> = BTreeSet::new();
497    let mut seen_digests: BTreeSet<String> = BTreeSet::new();
498    let mut new_variants = 0;
499
500    for entry in &library {
501        let repo = entry.repository.as_str();
502        let tags_url = format!("https://ollama.com/library/{repo}/tags");
503        let selected = match run.text(&tags_url, MAX_HTML_BYTES).await {
504            Ok(html) => match parse_tags(&html, repo) {
505                Ok(tags) => select_variants(&tags),
506                Err(_) => {
507                    outcome
508                        .failures
509                        .push(format!("{repo}: tags page unparseable"));
510                    outcome.complete = false;
511                    continue;
512                }
513            },
514            Err(error) if interrupted(&error) => return Err(error),
515            Err(_) => {
516                outcome
517                    .failures
518                    .push(format!("{repo}: tags page unavailable"));
519                outcome.complete = false;
520                continue;
521            }
522        };
523        let selected_ids: BTreeSet<String> =
524            selected.iter().map(|tag| format!("{repo}:{tag}")).collect();
525        for id in auto.keys().cloned().collect::<Vec<_>>() {
526            if parse_reference(&id).0 == repo && !selected_ids.contains(&id) {
527                auto.remove(&id);
528                outcome.removed.push(id);
529            }
530        }
531        for tag in &selected {
532            let id = format!("{repo}:{tag}");
533            if curated.contains_key(id.as_str()) {
534                continue;
535            }
536            match admit_variant(
537                &run,
538                entry,
539                tag,
540                &id,
541                &auto,
542                &curated_digests,
543                &mut seen_digests,
544                &today,
545                &mut new_variants,
546                options,
547                &mut outcome,
548            )
549            .await
550            {
551                Ok(Some(model)) => {
552                    eligible.insert(id.clone());
553                    auto.insert(id, model);
554                }
555                Ok(None) => {}
556                Err(error) if interrupted(&error) => return Err(error),
557                Err(error) => {
558                    outcome.failures.push(format!("{id}: {error}"));
559                    outcome.complete = false;
560                    if auto.contains_key(&id) {
561                        eligible.insert(id);
562                    }
563                }
564            }
565        }
566    }
567    for id in &outcome.deferred {
568        eligible.insert(id.clone());
569    }
570    let mut corrected: BTreeMap<String, CatalogModel> = BTreeMap::new();
571    for model in curated.values() {
572        match reobserve(&run, model, &stamp, &mut outcome.state).await {
573            Ok(Some(observed)) => {
574                if options.correct_curated
575                    && let Some((fixed, changes)) = correct(model, &observed)?
576                {
577                    corrected.insert(model.id.clone(), fixed);
578                    outcome.corrections.extend(changes);
579                }
580                outcome.curated_observed.push(model.id.clone());
581                outcome.curated_observations.push(observed);
582            }
583            Ok(None) => {}
584            Err(error) if interrupted(&error) => return Err(error),
585            Err(error) => outcome
586                .failures
587                .push(format!("{} (curated): {error}", model.id)),
588        }
589    }
590
591    let mut models: Vec<CatalogModel> = catalog
592        .models
593        .iter()
594        .filter(|model| model.provenance == EntryProvenance::Curated)
595        .map(|model| corrected.remove(&model.id).unwrap_or_else(|| model.clone()))
596        .chain(auto.into_values())
597        .collect();
598    models.sort_by(|left, right| {
599        right
600            .recency()
601            .map(|(day, _)| day)
602            .cmp(&left.recency().map(|(day, _)| day))
603            .then_with(|| left.id.cmp(&right.id))
604    });
605    outcome.observations = models
606        .iter()
607        .filter(|model| model.provenance == EntryProvenance::Auto)
608        .map(|model| observation(model, &stamp))
609        .collect();
610    let mut variants: BTreeSet<String> = models.iter().map(|model| model.id.clone()).collect();
611    variants.extend(eligible);
612    outcome.scope = json!({
613        "name": SCOPE_NAME,
614        "checkedAt": stamp,
615        "source": "https://ollama.com/library",
616        "complete": outcome.complete,
617        "variants": variants,
618    });
619    let changed =
620        !outcome.added.is_empty() || !outcome.updated.is_empty() || !outcome.removed.is_empty();
621    let encoded = json!({
622        "schemaVersion": SCHEMA_VERSION,
623        "generatedAt": if changed { stamp.clone() } else { catalog.generated_at.clone() },
624        "models": models,
625    });
626    outcome.catalog = Some(Catalog::parse(&encoded.to_string()).map_err(io::Error::other)?);
627    outcome.removed.sort();
628    Ok(outcome)
629}
630
631#[allow(clippy::too_many_arguments)]
632async fn admit_variant(
633    run: &Run<'_>,
634    entry: &LibraryEntry,
635    tag: &str,
636    id: &str,
637    auto: &BTreeMap<String, CatalogModel>,
638    curated_digests: &BTreeSet<&str>,
639    seen_digests: &mut BTreeSet<String>,
640    today: &str,
641    new_variants: &mut usize,
642    options: &AdmissionOptions,
643    outcome: &mut AdmissionOutcome,
644) -> io::Result<Option<CatalogModel>> {
645    let repo = entry.repository.as_str();
646    let blob =
647        |digest: &str| format!("https://registry.ollama.ai/v2/library/{repo}/blobs/{digest}");
648    let manifest_raw = run
649        .get(
650            &format!("https://registry.ollama.ai/v2/library/{repo}/manifests/{tag}"),
651            Some(MANIFEST_ACCEPT),
652            None,
653            MAX_MANIFEST_BYTES,
654        )
655        .await?;
656    let manifest_text = std::str::from_utf8(&manifest_raw)
657        .map_err(|_| io::Error::other("manifest is not UTF-8"))?;
658    let Some(layer) = parse_layer(manifest_text).map_err(io::Error::other)? else {
659        outcome
660            .rejected
661            .insert(id.into(), "no GGUF model layer".into());
662        return Ok(None);
663    };
664    let descriptors: ManifestLayers =
665        serde_json::from_str(manifest_text).map_err(io::Error::other)?;
666    if curated_digests.contains(layer.sha256.as_str()) || !seen_digests.insert(layer.sha256.clone())
667    {
668        return Ok(None);
669    }
670    let license_digest = descriptors
671        .layers
672        .iter()
673        .find(|layer| layer.media_type == "application/vnd.ollama.image.license")
674        .and_then(|layer| sha256(&layer.digest))
675        .map(str::to_string);
676    let cache_key = format!(
677        "{}:{}",
678        layer.sha256,
679        license_digest.as_deref().unwrap_or("none")
680    );
681    let prior = auto.get(id);
682    if let Some(prior) = prior
683        && prior.quantizations[0].sha256.as_deref() == Some(layer.sha256.as_str())
684        && prior.quantizations[0].disk_bytes == layer.disk_bytes
685        && !outcome.state.rejections.contains_key(&cache_key)
686    {
687        let mut refreshed = prior.clone();
688        let capabilities = catalog_capabilities(&entry.capabilities);
689        if capabilities != prior.capabilities {
690            refreshed.capabilities = capabilities;
691            outcome.updated.push(id.into());
692        } else {
693            outcome.reverified.push(id.into());
694        }
695        return Ok(Some(refreshed));
696    }
697    if let Some(reason) = outcome.state.rejections.get(&cache_key) {
698        outcome.rejected.insert(id.into(), reason.clone());
699        return Ok(None);
700    }
701    if prior.is_none() {
702        if *new_variants >= options.max_new_variants {
703            outcome.deferred.push(id.into());
704            outcome.complete = false;
705            return Ok(None);
706        }
707        *new_variants += 1;
708    }
709    let config_digest = sha256(&descriptors.config.digest)
710        .ok_or_else(|| io::Error::other("invalid config digest"))?;
711    let config_raw = run
712        .get(
713            &blob(&format!("sha256:{config_digest}")),
714            None,
715            None,
716            MAX_CONFIG_BYTES,
717        )
718        .await?;
719    let config = RegistryConfig::parse(
720        std::str::from_utf8(&config_raw).map_err(|_| io::Error::other("config is not UTF-8"))?,
721    );
722    let license_head = match &license_digest {
723        Some(digest) => Some(
724            String::from_utf8_lossy(
725                &run.get(
726                    &blob(&format!("sha256:{digest}")),
727                    None,
728                    Some(MAX_LICENSE_HEAD_BYTES),
729                    MAX_LICENSE_HEAD_BYTES,
730                )
731                .await?,
732            )
733            .into_owned(),
734        ),
735        None => None,
736    };
737    let reject = |outcome: &mut AdmissionOutcome, reason: &str| {
738        outcome.rejected.insert(id.into(), reason.into());
739        outcome
740            .state
741            .rejections
742            .insert(cache_key.clone(), reason.into());
743        Ok(None)
744    };
745    let Ok(config) = config else {
746        return reject(outcome, "model config lacks family, size or file type");
747    };
748    let gguf = match run.gguf(&blob(&format!("sha256:{}", layer.sha256))).await? {
749        Ok(gguf) => gguf,
750        Err(reason) => return reject(outcome, reason),
751    };
752    let built = build_entry(&AdmissionInput {
753        repository: repo,
754        tag,
755        capabilities: &entry.capabilities,
756        layer: &layer,
757        config: &config,
758        gguf: &gguf,
759        license_head: license_head.as_deref(),
760        today: prior
761            .and_then(|model| model.added_at.as_deref())
762            .unwrap_or(today),
763    });
764    match built {
765        Ok(model) => {
766            outcome.state.rejections.remove(&cache_key);
767            if prior.is_some() {
768                outcome.updated.push(id.into());
769            } else {
770                outcome.added.push(id.into());
771            }
772            Ok(Some(model))
773        }
774        Err(rejection) => reject(outcome, rejection.reason()),
775    }
776}
777
778/// A partial observation of a curated entry: the pulled artifact plus GGUF-sourced facts,
779/// citing only the Ollama reads that produced them. Curated entries are never modified.
780async fn reobserve(
781    run: &Run<'_>,
782    model: &CatalogModel,
783    stamp: &str,
784    state: &mut AdmissionState,
785) -> io::Result<Option<Value>> {
786    let Some(reference) = model.source.ollama.as_deref() else {
787        return Ok(None);
788    };
789    let (path, tag) = parse_reference(reference);
790    let repo = path.strip_prefix("library/").unwrap_or(path);
791    if repo.contains('/') || !name(repo) || !name(tag) {
792        return Ok(None);
793    }
794    let Some(index) = pulled_quantization(model) else {
795        return Ok(None);
796    };
797    let manifest_url = format!("https://registry.ollama.ai/v2/library/{repo}/manifests/{tag}");
798    let manifest_raw = run
799        .get(
800            &manifest_url,
801            Some(MANIFEST_ACCEPT),
802            None,
803            MAX_MANIFEST_BYTES,
804        )
805        .await?;
806    let manifest_text = std::str::from_utf8(&manifest_raw)
807        .map_err(|_| io::Error::other("manifest is not UTF-8"))?;
808    let Some(layer) = parse_layer(manifest_text).map_err(io::Error::other)? else {
809        return Ok(None);
810    };
811    let descriptors: ManifestLayers =
812        serde_json::from_str(manifest_text).map_err(io::Error::other)?;
813    let blob = |digest: &str| {
814        format!("https://registry.ollama.ai/v2/library/{repo}/blobs/sha256:{digest}")
815    };
816    let config_digest = sha256(&descriptors.config.digest)
817        .ok_or_else(|| io::Error::other("invalid config digest"))?;
818    let config_raw = run
819        .get(&blob(config_digest), None, None, MAX_CONFIG_BYTES)
820        .await?;
821    let Ok(config) = RegistryConfig::parse(
822        std::str::from_utf8(&config_raw).map_err(|_| io::Error::other("config is not UTF-8"))?,
823    ) else {
824        return Ok(None);
825    };
826    let license_digest = descriptors
827        .layers
828        .iter()
829        .find(|layer| layer.media_type == "application/vnd.ollama.image.license")
830        .and_then(|layer| sha256(&layer.digest))
831        .map(str::to_string);
832    let key = format!(
833        "{}:{}",
834        layer.sha256,
835        license_digest.as_deref().unwrap_or("none")
836    );
837    let sourced = match state.facts.get(&key) {
838        Some(sourced) => sourced.clone(),
839        None => {
840            let Ok(gguf) = run.gguf(&blob(&layer.sha256)).await? else {
841                return Ok(None);
842            };
843            let Some(arch) = gguf.architecture().map(str::to_string) else {
844                return Ok(None);
845            };
846            let Some(context_length) = gguf.u64(&format!("{arch}.context_length")) else {
847                return Ok(None);
848            };
849            let head = match &license_digest {
850                Some(digest) => Some(
851                    String::from_utf8_lossy(
852                        &run.get(
853                            &blob(digest),
854                            None,
855                            Some(MAX_LICENSE_HEAD_BYTES),
856                            MAX_LICENSE_HEAD_BYTES,
857                        )
858                        .await?,
859                    )
860                    .into_owned(),
861                ),
862                None => None,
863            };
864            let sourced = SourcedFacts {
865                context_length,
866                architecture: if gguf
867                    .u64(&format!("{arch}.expert_count"))
868                    .is_some_and(|count| count > 0)
869                {
870                    Architecture::Moe
871                } else {
872                    Architecture::Dense
873                },
874                license: license_id(
875                    gguf.text("general.license"),
876                    gguf.text("general.license.name"),
877                    head.as_deref(),
878                )
879                .ok()
880                .map(str::to_string),
881                kv_bytes_per_token: kv_bytes_per_token(&gguf),
882            };
883            state.facts.insert(key, sourced.clone());
884            sourced
885        }
886    };
887    let mut quant = serde_json::to_value(&model.quantizations[index]).map_err(io::Error::other)?;
888    let object = quant
889        .as_object_mut()
890        .ok_or_else(|| io::Error::other("invalid quantization"))?;
891    for field in ["minRamBytes", "minVramBytes", "digestVerified"] {
892        object.remove(field);
893    }
894    object.insert("name".into(), json!(config.file_type.to_ascii_uppercase()));
895    object.insert("diskBytes".into(), json!(layer.disk_bytes));
896    object.insert("sha256".into(), json!(layer.sha256));
897    if layer.projectors.is_empty() {
898        object.remove("projectors");
899    } else {
900        object.insert(
901            "projectors".into(),
902            serde_json::to_value(&layer.projectors).map_err(io::Error::other)?,
903        );
904    }
905    let mut facts = serde_json::Map::new();
906    facts.insert("defaultQuantization".into(), quant);
907    facts.insert("contextLength".into(), json!(sourced.context_length as f64));
908    facts.insert(
909        "architecture".into(),
910        serde_json::to_value(&sourced.architecture).map_err(io::Error::other)?,
911    );
912    if let Some(license) = sourced.license {
913        facts.insert("license".into(), json!(license));
914    }
915    // Asserted only when the catalog has a value to compare; unknown is not a contradiction.
916    if let (Some(rate), Some(_)) = (sourced.kv_bytes_per_token, model.kv_bytes_per_token) {
917        facts.insert("kvBytesPerToken".into(), json!(rate));
918    }
919    Ok(Some(json!({
920        "id": model.id,
921        "checkedAt": stamp,
922        "sources": [manifest_url, format!("https://ollama.com/library/{repo}:{tag}")],
923        "facts": facts,
924    })))
925}
926
927/// The catalog's own representation of a partially observed fact.
928fn catalog_value(model: &CatalogModel, field: &str) -> io::Result<Value> {
929    if field == "defaultQuantization" {
930        let index = pulled_quantization(model).unwrap_or(0);
931        let mut quant =
932            serde_json::to_value(&model.quantizations[index]).map_err(io::Error::other)?;
933        if let Some(object) = quant.as_object_mut() {
934            for key in ["minRamBytes", "minVramBytes", "digestVerified"] {
935                object.remove(key);
936            }
937        }
938        return Ok(quant);
939    }
940    Ok(serde_json::to_value(model)
941        .map_err(io::Error::other)?
942        .get(field)
943        .cloned()
944        .unwrap_or(Value::Null))
945}
946
947/// Rewrites contradicted, correctable curated facts to what the shipped artifact says.
948/// Architecture contradictions are left for review: they change how the model is sized.
949fn correct(
950    model: &CatalogModel,
951    observed: &Value,
952) -> io::Result<Option<(CatalogModel, Vec<Correction>)>> {
953    let Some(facts) = observed["facts"].as_object() else {
954        return Ok(None);
955    };
956    let sources: Vec<String> = observed["sources"]
957        .as_array()
958        .map(|items| {
959            items
960                .iter()
961                .filter_map(|item| item.as_str().map(str::to_string))
962                .collect()
963        })
964        .unwrap_or_default();
965    let mut fixed = model.clone();
966    let mut changes = Vec::new();
967    for (field, to) in facts {
968        let from = catalog_value(model, field)?;
969        if from == *to || field == "architecture" {
970            continue;
971        }
972        match field.as_str() {
973            "contextLength" => {
974                fixed.context_length = to
975                    .as_f64()
976                    .ok_or_else(|| io::Error::other("invalid context"))?
977            }
978            "license" => {
979                fixed.license = to
980                    .as_str()
981                    .ok_or_else(|| io::Error::other("invalid license"))?
982                    .into()
983            }
984            "kvBytesPerToken" => fixed.kv_bytes_per_token = to.as_f64(),
985            "defaultQuantization" => {
986                let index = pulled_quantization(model).unwrap_or(0);
987                let name = to["name"]
988                    .as_str()
989                    .ok_or_else(|| io::Error::other("invalid quant"))?;
990                let projectors: Vec<rigspark_core::sizing::ProjectorArtifact> =
991                    serde_json::from_value(to.get("projectors").cloned().unwrap_or(json!([])))
992                        .map_err(io::Error::other)?;
993                let quant = rigspark_core::enrich::sized_quantization(
994                    rigspark_core::sizing::parse_param_count(&model.params)
995                        .map_err(io::Error::other)?,
996                    &model.architecture,
997                    name,
998                    to["diskBytes"]
999                        .as_f64()
1000                        .ok_or_else(|| io::Error::other("invalid size"))?,
1001                    to["sha256"].as_str().map(str::to_string),
1002                    projectors,
1003                )
1004                .map_err(io::Error::other)?;
1005                fixed.quantizations[index] = quant;
1006                // Facts for other quantizations named like the new one are no longer sourced.
1007                let mut position = 0;
1008                fixed.quantizations.retain(|candidate| {
1009                    let keep = position == index || candidate.name != name;
1010                    position += 1;
1011                    keep
1012                });
1013            }
1014            _ => continue,
1015        }
1016        changes.push(Correction {
1017            id: model.id.clone(),
1018            field: field.clone(),
1019            from,
1020            to: to.clone(),
1021            sources: sources.clone(),
1022        });
1023    }
1024    if changes.is_empty() {
1025        return Ok(None);
1026    }
1027    let check = json!({"schemaVersion": SCHEMA_VERSION, "generatedAt": "2026-01-01T00:00:00.000Z", "models": [fixed]});
1028    Catalog::parse(&check.to_string()).map_err(io::Error::other)?;
1029    Ok(Some((fixed, changes)))
1030}