Skip to main content

rigspark_runtime/
admission.rs

1//! Network side of automatic catalog admission: allow-listed fetches and the weekly run.
2
3use crate::catalog_proposals::allowed_registry_redirect;
4use futures_util::StreamExt;
5use rigspark_core::{
6    admission::{
7        AdmissionInput, Gguf, GgufError, LibraryEntry, MAX_CONFIG_BYTES, MAX_GGUF_HEADER_BYTES,
8        MAX_HTML_BYTES, MAX_LICENSE_HEAD_BYTES, RegistryConfig, build_entry, catalog_capabilities,
9        kv_bytes_per_token, license_id, observation, parse_gguf, parse_library, parse_tags,
10        select_variants,
11    },
12    catalog::{Catalog, CatalogModel, EntryProvenance, SCHEMA_VERSION},
13    registry_collector::{
14        MANIFEST_ACCEPT, MAX_MANIFEST_BYTES, parse_layer, parse_reference, pulled_quantization,
15    },
16    sizing::Architecture,
17};
18use serde::{Deserialize, Serialize};
19use serde_json::{Value, json};
20use std::{
21    collections::{BTreeMap, BTreeSet},
22    io,
23    time::Duration,
24};
25use time::{OffsetDateTime, format_description::well_known::Rfc3339};
26use tokio_util::sync::CancellationToken;
27use url::Url;
28
29pub const LIBRARY_URL: &str = "https://ollama.com/library?sort=newest";
30pub const GENERATION_LISTING_URL: &str =
31    "https://huggingface.co/api/models?author=Comfy-Org&sort=createdAt&direction=-1&limit=500";
32pub const SCOPE_NAME: &str = "ollama-local-variants";
33const GGUF_STEPS: [usize; 3] = [8 * 1024 * 1024, 32 * 1024 * 1024, MAX_GGUF_HEADER_BYTES];
34
35fn name(part: &str) -> bool {
36    !part.is_empty()
37        && part.len() <= 128
38        && part.as_bytes()[0].is_ascii_alphanumeric()
39        && part
40            .bytes()
41            .all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte))
42}
43
44/// Only the listing, tags pages, manifests and blobs of official `library/` models.
45pub fn allowed_request(url: &Url) -> bool {
46    if url.scheme() != "https"
47        || url.port().is_some()
48        || !url.username().is_empty()
49        || url.password().is_some()
50        || url.fragment().is_some()
51    {
52        return false;
53    }
54    let segments: Vec<&str> = url.path().trim_start_matches('/').split('/').collect();
55    match (url.host_str(), url.query(), segments.as_slice()) {
56        (Some("ollama.com"), Some("sort=newest"), ["library"]) => true,
57        (Some("ollama.com"), None, ["library", repo, "tags"]) => name(repo),
58        (Some("registry.ollama.ai"), None, ["v2", "library", repo, "manifests", tag]) => {
59            name(repo) && name(tag)
60        }
61        (Some("registry.ollama.ai"), None, ["v2", "library", repo, "blobs", digest]) => {
62            name(repo)
63                && digest.strip_prefix("sha256:").is_some_and(|hex| {
64                    hex.len() == 64 && hex.bytes().all(|byte| byte.is_ascii_hexdigit())
65                })
66        }
67        (Some("huggingface.co"), Some(query), ["api", "models"]) => {
68            Some(query)
69                == GENERATION_LISTING_URL
70                    .split_once('?')
71                    .map(|(_, query)| query)
72                || crate::hf_add::is_search_url(url)
73        }
74        (Some("huggingface.co"), None | Some("blobs=true"), ["api", "models", owner, repo]) => {
75            name(owner) && name(repo)
76        }
77        (
78            Some("huggingface.co"),
79            Some("blobs=true"),
80            ["api", "models", owner, repo, "revision", revision],
81        ) => {
82            name(owner)
83                && name(repo)
84                && revision.len() == 40
85                && revision.bytes().all(|byte| byte.is_ascii_hexdigit())
86        }
87        (Some("huggingface.co"), None, [owner, repo, "resolve", revision, path @ ..]) => {
88            name(owner)
89                && name(repo)
90                && revision.len() == 40
91                && revision.bytes().all(|byte| byte.is_ascii_hexdigit())
92                && !path.is_empty()
93                && path.len() <= 8
94                && path.iter().all(|segment| name(segment))
95                && path.last().is_some_and(|file| {
96                    file.ends_with(".gguf")
97                        || (*owner == "Comfy-Org" && file.ends_with(".safetensors"))
98                })
99        }
100        _ => false,
101    }
102}
103
104/// Hugging Face serves `resolve` downloads from its own CDN under `hf.co`.
105pub fn allowed_hub_redirect(url: &Url) -> bool {
106    url.scheme() == "https"
107        && url.port().is_none()
108        && url.username().is_empty()
109        && url.password().is_none()
110        && url
111            .host_str()
112            .is_some_and(|host| host == "huggingface.co" || host.ends_with(".hf.co"))
113}
114
115pub struct Fetched {
116    pub status: u16,
117    pub body: Vec<u8>,
118}
119
120#[async_trait::async_trait]
121pub trait AdmissionTransport: Send + Sync {
122    /// GETs an allow-listed URL. `range` asks for the first `range` bytes; `limit` caps the body.
123    async fn fetch(
124        &self,
125        url: &Url,
126        accept: Option<&'static str>,
127        range: Option<usize>,
128        limit: usize,
129    ) -> io::Result<Fetched>;
130}
131
132pub struct NativeAdmissionTransport {
133    client: reqwest::Client,
134}
135impl NativeAdmissionTransport {
136    pub fn new() -> io::Result<Self> {
137        let client = reqwest::Client::builder()
138            .https_only(true)
139            .no_proxy()
140            .user_agent(concat!(
141                "rigspark-catalog-admission/",
142                env!("CARGO_PKG_VERSION")
143            ))
144            .redirect(reqwest::redirect::Policy::custom(|attempt| {
145                let previous = attempt.previous().last();
146                let from_blob = previous.is_some_and(|url| {
147                    url.host_str() == Some("registry.ollama.ai")
148                        && url.path().contains("/blobs/sha256:")
149                });
150                let from_resolve = previous.is_some_and(|url| {
151                    url.host_str() == Some("huggingface.co") && url.path().contains("/resolve/")
152                });
153                let allowed = (from_blob && allowed_registry_redirect(attempt.url().as_str()))
154                    || (from_resolve && allowed_hub_redirect(attempt.url()));
155                if attempt.previous().len() > 2 || !allowed {
156                    attempt.error("registry redirect refused")
157                } else {
158                    attempt.follow()
159                }
160            }))
161            .connect_timeout(Duration::from_secs(10))
162            .timeout(Duration::from_secs(120))
163            .build()
164            .map_err(|_| io::Error::other("admission client initialization failed"))?;
165        Ok(Self { client })
166    }
167}
168#[async_trait::async_trait]
169impl AdmissionTransport for NativeAdmissionTransport {
170    async fn fetch(
171        &self,
172        url: &Url,
173        accept: Option<&'static str>,
174        range: Option<usize>,
175        limit: usize,
176    ) -> io::Result<Fetched> {
177        if !allowed_request(url) {
178            return Err(io::Error::other("admission URL is not allow-listed"));
179        }
180        let mut request = self.client.get(url.clone());
181        if let Some(accept) = accept {
182            request = request.header(reqwest::header::ACCEPT, accept);
183        }
184        if let Some(range) = range {
185            request = request.header(reqwest::header::RANGE, format!("bytes=0-{}", range - 1));
186        }
187        let response = request
188            .send()
189            .await
190            .map_err(|_| io::Error::other("admission request failed"))?;
191        let status = response.status().as_u16();
192        let mut stream = response.bytes_stream();
193        let mut body = Vec::new();
194        while let Some(chunk) = stream.next().await {
195            let chunk = chunk.map_err(|_| io::Error::other("admission body read failed"))?;
196            body.extend_from_slice(&chunk);
197            if body.len() >= limit {
198                // A range read may stop at the cap; anything else larger than the cap is refused.
199                if range.is_none() && body.len() > limit {
200                    return Err(io::Error::other("admission response exceeds its size cap"));
201                }
202                body.truncate(limit);
203                break;
204            }
205        }
206        Ok(Fetched { status, body })
207    }
208}
209
210#[derive(Deserialize)]
211#[serde(deny_unknown_fields)]
212struct RecordedBody {
213    status: u16,
214    #[serde(default)]
215    text: Option<String>,
216    #[serde(default)]
217    json: Option<Value>,
218    #[serde(default)]
219    hex: Option<String>,
220}
221
222/// Replays recorded responses keyed by URL; unknown URLs answer 404.
223pub struct RecordedAdmissionTransport {
224    responses: BTreeMap<String, (u16, Vec<u8>)>,
225}
226impl RecordedAdmissionTransport {
227    pub fn new(responses: BTreeMap<String, (u16, Vec<u8>)>) -> io::Result<Self> {
228        for key in responses.keys() {
229            let url = Url::parse(key).map_err(io::Error::other)?;
230            if !allowed_request(&url) {
231                return Err(io::Error::other("recorded URL is not allow-listed"));
232            }
233        }
234        Ok(Self { responses })
235    }
236    pub fn parse(raw: &str) -> io::Result<Self> {
237        let recorded: BTreeMap<String, RecordedBody> =
238            serde_json::from_str(raw).map_err(io::Error::other)?;
239        let mut responses = BTreeMap::new();
240        for (url, body) in recorded {
241            let bytes = match (body.text, body.json, body.hex) {
242                (Some(text), None, None) => text.into_bytes(),
243                (None, Some(value), None) => {
244                    serde_json::to_vec(&value).map_err(io::Error::other)?
245                }
246                (None, None, Some(hex)) => decode_hex(&hex)?,
247                (None, None, None) => Vec::new(),
248                _ => return Err(io::Error::other("recorded body must have one encoding")),
249            };
250            responses.insert(url, (body.status, bytes));
251        }
252        Self::new(responses)
253    }
254}
255fn decode_hex(hex: &str) -> io::Result<Vec<u8>> {
256    if !hex.len().is_multiple_of(2) {
257        return Err(io::Error::other("invalid recorded hex body"));
258    }
259    (0..hex.len())
260        .step_by(2)
261        .map(|index| {
262            u8::from_str_radix(&hex[index..index + 2], 16)
263                .map_err(|_| io::Error::other("invalid recorded hex body"))
264        })
265        .collect()
266}
267#[async_trait::async_trait]
268impl AdmissionTransport for RecordedAdmissionTransport {
269    async fn fetch(
270        &self,
271        url: &Url,
272        _accept: Option<&'static str>,
273        range: Option<usize>,
274        limit: usize,
275    ) -> io::Result<Fetched> {
276        if !allowed_request(url) {
277            return Err(io::Error::other("admission URL is not allow-listed"));
278        }
279        let (status, body) = self
280            .responses
281            .get(url.as_str())
282            .cloned()
283            .unwrap_or((404, Vec::new()));
284        let cap = range.unwrap_or(usize::MAX).min(limit);
285        if range.is_none() && body.len() > limit {
286            return Err(io::Error::other("admission response exceeds its size cap"));
287        }
288        Ok(Fetched {
289            status,
290            body: body[..body.len().min(cap)].to_vec(),
291        })
292    }
293}
294
295#[derive(Deserialize)]
296#[serde(rename_all = "camelCase")]
297struct ManifestLayers {
298    config: Descriptor,
299    layers: Vec<Descriptor>,
300}
301#[derive(Deserialize)]
302#[serde(rename_all = "camelCase")]
303struct Descriptor {
304    media_type: String,
305    digest: String,
306}
307fn sha256(digest: &str) -> Option<&str> {
308    digest
309        .strip_prefix("sha256:")
310        .filter(|hex| hex.len() == 64 && hex.bytes().all(|byte| byte.is_ascii_hexdigit()))
311}
312
313#[derive(Clone, Debug, Default, Serialize, Deserialize)]
314#[serde(rename_all = "camelCase", deny_unknown_fields)]
315pub struct AdmissionState {
316    /// `"<model sha256>:<license sha256 or none>"` → rejection reason, so ineligible weights
317    /// are not re-downloaded every week.
318    #[serde(default)]
319    pub rejections: BTreeMap<String, String>,
320    /// Same key → facts read from immutable GGUF headers of curated entries' weights.
321    #[serde(default)]
322    pub facts: BTreeMap<String, SourcedFacts>,
323}
324
325#[derive(Clone, Debug, Serialize, Deserialize)]
326#[serde(rename_all = "camelCase", deny_unknown_fields)]
327pub struct SourcedFacts {
328    pub context_length: u64,
329    pub architecture: Architecture,
330    #[serde(default, skip_serializing_if = "Option::is_none")]
331    pub license: Option<String>,
332    #[serde(default, skip_serializing_if = "Option::is_none")]
333    pub kv_bytes_per_token: Option<f64>,
334}
335
336pub struct AdmissionOptions {
337    pub now: String,
338    pub max_new_variants: usize,
339    pub state: AdmissionState,
340    /// Apply source-backed corrections to curated facts that upstream contradicts.
341    pub correct_curated: bool,
342}
343
344/// One cited change to a curated fact, made because the shipped artifact says otherwise.
345#[derive(Clone, Debug, Serialize)]
346pub struct Correction {
347    pub id: String,
348    pub field: String,
349    pub from: Value,
350    pub to: Value,
351    pub sources: Vec<String>,
352}
353
354#[derive(Default, Serialize)]
355#[serde(rename_all = "camelCase")]
356pub struct AdmissionOutcome {
357    #[serde(skip)]
358    pub catalog: Option<Catalog>,
359    #[serde(skip)]
360    pub observations: Vec<Value>,
361    /// Partial observations of curated entries (facts the collector could source).
362    #[serde(skip)]
363    pub curated_observations: Vec<Value>,
364    #[serde(skip)]
365    pub scope: Value,
366    #[serde(skip)]
367    pub state: AdmissionState,
368    pub added: Vec<String>,
369    pub updated: Vec<String>,
370    pub reverified: Vec<String>,
371    pub curated_observed: Vec<String>,
372    pub corrections: Vec<Correction>,
373    pub removed: Vec<String>,
374    pub rejected: BTreeMap<String, String>,
375    pub deferred: Vec<String>,
376    pub failures: Vec<String>,
377    pub complete: bool,
378}
379
380struct Run<'a> {
381    transport: &'a dyn AdmissionTransport,
382    cancel: &'a CancellationToken,
383}
384impl Run<'_> {
385    async fn get(
386        &self,
387        url: &str,
388        accept: Option<&'static str>,
389        range: Option<usize>,
390        limit: usize,
391    ) -> io::Result<Vec<u8>> {
392        if self.cancel.is_cancelled() {
393            return Err(io::Error::new(
394                io::ErrorKind::Interrupted,
395                "catalog admission cancelled",
396            ));
397        }
398        let url = Url::parse(url).map_err(io::Error::other)?;
399        let fetched = tokio::select! {
400            biased;
401            _ = self.cancel.cancelled() => {
402                return Err(io::Error::new(io::ErrorKind::Interrupted, "catalog admission cancelled"));
403            }
404            result = tokio::time::timeout(Duration::from_secs(180), self.transport.fetch(&url, accept, range, limit)) => {
405                result.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "admission request timed out"))??
406            }
407        };
408        let ok = if range.is_some() {
409            matches!(fetched.status, 200 | 206)
410        } else {
411            (200..300).contains(&fetched.status)
412        };
413        if !ok {
414            return Err(io::Error::new(
415                io::ErrorKind::NotFound,
416                format!("upstream status {}", fetched.status),
417            ));
418        }
419        Ok(fetched.body)
420    }
421    async fn text(&self, url: &str, limit: usize) -> io::Result<String> {
422        String::from_utf8(self.get(url, None, None, limit).await?)
423            .map_err(|_| io::Error::other("upstream text is not UTF-8"))
424    }
425    /// `Ok(Err(reason))` is a content rejection (cacheable); `Err` is a transient failure.
426    async fn gguf(&self, blob: &str) -> io::Result<Result<Gguf, &'static str>> {
427        for step in GGUF_STEPS {
428            let bytes = self.get(blob, None, Some(step), step).await?;
429            match parse_gguf(&bytes) {
430                Ok(parsed) => return Ok(Ok(parsed)),
431                Err(GgufError::Truncated) if bytes.len() >= step => continue,
432                Err(GgufError::Truncated) => return Ok(Err("GGUF header truncated")),
433                Err(GgufError::Invalid(reason)) => return Ok(Err(reason)),
434            }
435        }
436        Ok(Err("GGUF header exceeds 64 MiB"))
437    }
438}
439
440fn interrupted(error: &io::Error) -> bool {
441    error.kind() == io::ErrorKind::Interrupted
442}
443
444/// One admission pass: discover, source, merge and produce evidence. Curated entries are
445/// never modified; a failed listing aborts without changes.
446pub async fn admit(
447    catalog: &Catalog,
448    transport: &dyn AdmissionTransport,
449    options: &AdmissionOptions,
450    cancel: &CancellationToken,
451) -> io::Result<AdmissionOutcome> {
452    let now = OffsetDateTime::parse(&options.now, &Rfc3339)
453        .map_err(|_| io::Error::other("invalid admission clock"))?
454        .to_offset(time::UtcOffset::UTC);
455    let stamp = now
456        .format(time::macros::format_description!(
457            "[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
458        ))
459        .map_err(io::Error::other)?;
460    let today = stamp[..10].to_string();
461    let run = Run { transport, cancel };
462    let library =
463        parse_library(&run.text(LIBRARY_URL, MAX_HTML_BYTES).await?).map_err(io::Error::other)?;
464
465    let mut outcome = AdmissionOutcome {
466        complete: true,
467        state: options.state.clone(),
468        ..Default::default()
469    };
470    let curated: BTreeMap<&str, &CatalogModel> = catalog
471        .models
472        .iter()
473        .filter(|model| model.provenance == EntryProvenance::Curated)
474        .map(|model| (model.id.as_str(), model))
475        .collect();
476    let curated_digests: BTreeSet<&str> = curated
477        .values()
478        .flat_map(|model| model.quantizations.iter())
479        .filter_map(|quant| quant.sha256.as_deref())
480        .collect();
481    let mut auto: BTreeMap<String, CatalogModel> = catalog
482        .models
483        .iter()
484        .filter(|model| model.provenance == EntryProvenance::Auto)
485        .map(|model| (model.id.clone(), model.clone()))
486        .collect();
487    let listed: BTreeSet<&str> = library
488        .iter()
489        .map(|entry| entry.repository.as_str())
490        .collect();
491    for id in auto.keys().cloned().collect::<Vec<_>>() {
492        if !listed.contains(parse_reference(&id).0) {
493            auto.remove(&id);
494            outcome.removed.push(id);
495        }
496    }
497    let mut eligible: BTreeSet<String> = BTreeSet::new();
498    let mut seen_digests: BTreeSet<String> = BTreeSet::new();
499    let mut new_variants = 0;
500
501    for entry in &library {
502        let repo = entry.repository.as_str();
503        let tags_url = format!("https://ollama.com/library/{repo}/tags");
504        let selected = match run.text(&tags_url, MAX_HTML_BYTES).await {
505            Ok(html) => match parse_tags(&html, repo) {
506                Ok(tags) => select_variants(&tags),
507                Err(_) => {
508                    outcome
509                        .failures
510                        .push(format!("{repo}: tags page unparseable"));
511                    outcome.complete = false;
512                    continue;
513                }
514            },
515            Err(error) if interrupted(&error) => return Err(error),
516            Err(_) => {
517                outcome
518                    .failures
519                    .push(format!("{repo}: tags page unavailable"));
520                outcome.complete = false;
521                continue;
522            }
523        };
524        let selected_ids: BTreeSet<String> =
525            selected.iter().map(|tag| format!("{repo}:{tag}")).collect();
526        for id in auto.keys().cloned().collect::<Vec<_>>() {
527            if parse_reference(&id).0 == repo && !selected_ids.contains(&id) {
528                auto.remove(&id);
529                outcome.removed.push(id);
530            }
531        }
532        for tag in &selected {
533            let id = format!("{repo}:{tag}");
534            if curated.contains_key(id.as_str()) {
535                continue;
536            }
537            match admit_variant(
538                &run,
539                entry,
540                tag,
541                &id,
542                &auto,
543                &curated_digests,
544                &mut seen_digests,
545                &today,
546                &mut new_variants,
547                options,
548                &mut outcome,
549            )
550            .await
551            {
552                Ok(Some(model)) => {
553                    eligible.insert(id.clone());
554                    auto.insert(id, model);
555                }
556                Ok(None) => {}
557                Err(error) if interrupted(&error) => return Err(error),
558                Err(error) => {
559                    outcome.failures.push(format!("{id}: {error}"));
560                    outcome.complete = false;
561                    if auto.contains_key(&id) {
562                        eligible.insert(id);
563                    }
564                }
565            }
566        }
567    }
568    for id in &outcome.deferred {
569        eligible.insert(id.clone());
570    }
571    let mut corrected: BTreeMap<String, CatalogModel> = BTreeMap::new();
572    for model in curated.values() {
573        match reobserve(&run, model, &stamp, &mut outcome.state).await {
574            Ok(Some(observed)) => {
575                if options.correct_curated
576                    && let Some((fixed, changes)) = correct(model, &observed)?
577                {
578                    corrected.insert(model.id.clone(), fixed);
579                    outcome.corrections.extend(changes);
580                }
581                outcome.curated_observed.push(model.id.clone());
582                outcome.curated_observations.push(observed);
583            }
584            Ok(None) => {}
585            Err(error) if interrupted(&error) => return Err(error),
586            Err(error) => outcome
587                .failures
588                .push(format!("{} (curated): {error}", model.id)),
589        }
590    }
591
592    let mut models: Vec<CatalogModel> = catalog
593        .models
594        .iter()
595        .filter(|model| model.provenance == EntryProvenance::Curated)
596        .map(|model| corrected.remove(&model.id).unwrap_or_else(|| model.clone()))
597        .chain(auto.into_values())
598        .collect();
599    models.sort_by(|left, right| {
600        right
601            .recency()
602            .map(|(day, _)| day)
603            .cmp(&left.recency().map(|(day, _)| day))
604            .then_with(|| left.id.cmp(&right.id))
605    });
606    outcome.observations = models
607        .iter()
608        .filter(|model| model.provenance == EntryProvenance::Auto)
609        .map(|model| observation(model, &stamp))
610        .collect();
611    let mut variants: BTreeSet<String> = models.iter().map(|model| model.id.clone()).collect();
612    variants.extend(eligible);
613    outcome.scope = json!({
614        "name": SCOPE_NAME,
615        "checkedAt": stamp,
616        "source": "https://ollama.com/library",
617        "complete": outcome.complete,
618        "variants": variants,
619    });
620    let changed =
621        !outcome.added.is_empty() || !outcome.updated.is_empty() || !outcome.removed.is_empty();
622    let encoded = json!({
623        "schemaVersion": rigspark_core::catalog::required_schema_version(&models),
624        "generatedAt": if changed { stamp.clone() } else { catalog.generated_at.clone() },
625        "models": models,
626    });
627    outcome.catalog = Some(Catalog::parse(&encoded.to_string()).map_err(io::Error::other)?);
628    outcome.removed.sort();
629    Ok(outcome)
630}
631
632#[allow(clippy::too_many_arguments)]
633async fn admit_variant(
634    run: &Run<'_>,
635    entry: &LibraryEntry,
636    tag: &str,
637    id: &str,
638    auto: &BTreeMap<String, CatalogModel>,
639    curated_digests: &BTreeSet<&str>,
640    seen_digests: &mut BTreeSet<String>,
641    today: &str,
642    new_variants: &mut usize,
643    options: &AdmissionOptions,
644    outcome: &mut AdmissionOutcome,
645) -> io::Result<Option<CatalogModel>> {
646    let repo = entry.repository.as_str();
647    let blob =
648        |digest: &str| format!("https://registry.ollama.ai/v2/library/{repo}/blobs/{digest}");
649    let manifest_raw = run
650        .get(
651            &format!("https://registry.ollama.ai/v2/library/{repo}/manifests/{tag}"),
652            Some(MANIFEST_ACCEPT),
653            None,
654            MAX_MANIFEST_BYTES,
655        )
656        .await?;
657    let manifest_text = std::str::from_utf8(&manifest_raw)
658        .map_err(|_| io::Error::other("manifest is not UTF-8"))?;
659    let Some(layer) = parse_layer(manifest_text).map_err(io::Error::other)? else {
660        outcome
661            .rejected
662            .insert(id.into(), "no GGUF model layer".into());
663        return Ok(None);
664    };
665    let descriptors: ManifestLayers =
666        serde_json::from_str(manifest_text).map_err(io::Error::other)?;
667    if curated_digests.contains(layer.sha256.as_str()) || !seen_digests.insert(layer.sha256.clone())
668    {
669        return Ok(None);
670    }
671    let license_digest = descriptors
672        .layers
673        .iter()
674        .find(|layer| layer.media_type == "application/vnd.ollama.image.license")
675        .and_then(|layer| sha256(&layer.digest))
676        .map(str::to_string);
677    let cache_key = format!(
678        "{}:{}",
679        layer.sha256,
680        license_digest.as_deref().unwrap_or("none")
681    );
682    let prior = auto.get(id);
683    if let Some(prior) = prior
684        && prior.quantizations[0].sha256.as_deref() == Some(layer.sha256.as_str())
685        && prior.quantizations[0].disk_bytes == layer.disk_bytes
686        && !outcome.state.rejections.contains_key(&cache_key)
687    {
688        let mut refreshed = prior.clone();
689        let capabilities = catalog_capabilities(&entry.capabilities);
690        if capabilities != prior.capabilities {
691            refreshed.capabilities = capabilities;
692            outcome.updated.push(id.into());
693        } else {
694            outcome.reverified.push(id.into());
695        }
696        return Ok(Some(refreshed));
697    }
698    if let Some(reason) = outcome.state.rejections.get(&cache_key) {
699        outcome.rejected.insert(id.into(), reason.clone());
700        return Ok(None);
701    }
702    if prior.is_none() {
703        if *new_variants >= options.max_new_variants {
704            outcome.deferred.push(id.into());
705            outcome.complete = false;
706            return Ok(None);
707        }
708        *new_variants += 1;
709    }
710    let config_digest = sha256(&descriptors.config.digest)
711        .ok_or_else(|| io::Error::other("invalid config digest"))?;
712    let config_raw = run
713        .get(
714            &blob(&format!("sha256:{config_digest}")),
715            None,
716            None,
717            MAX_CONFIG_BYTES,
718        )
719        .await?;
720    let config = RegistryConfig::parse(
721        std::str::from_utf8(&config_raw).map_err(|_| io::Error::other("config is not UTF-8"))?,
722    );
723    let license_head = match &license_digest {
724        Some(digest) => Some(
725            String::from_utf8_lossy(
726                &run.get(
727                    &blob(&format!("sha256:{digest}")),
728                    None,
729                    Some(MAX_LICENSE_HEAD_BYTES),
730                    MAX_LICENSE_HEAD_BYTES,
731                )
732                .await?,
733            )
734            .into_owned(),
735        ),
736        None => None,
737    };
738    let reject = |outcome: &mut AdmissionOutcome, reason: &str| {
739        outcome.rejected.insert(id.into(), reason.into());
740        outcome
741            .state
742            .rejections
743            .insert(cache_key.clone(), reason.into());
744        Ok(None)
745    };
746    let Ok(config) = config else {
747        return reject(outcome, "model config lacks family, size or file type");
748    };
749    let gguf = match run.gguf(&blob(&format!("sha256:{}", layer.sha256))).await? {
750        Ok(gguf) => gguf,
751        Err(reason) => return reject(outcome, reason),
752    };
753    let built = build_entry(&AdmissionInput {
754        repository: repo,
755        tag,
756        capabilities: &entry.capabilities,
757        layer: &layer,
758        config: &config,
759        gguf: &gguf,
760        license_head: license_head.as_deref(),
761        today: prior
762            .and_then(|model| model.added_at.as_deref())
763            .unwrap_or(today),
764    });
765    match built {
766        Ok(model) => {
767            outcome.state.rejections.remove(&cache_key);
768            if prior.is_some() {
769                outcome.updated.push(id.into());
770            } else {
771                outcome.added.push(id.into());
772            }
773            Ok(Some(model))
774        }
775        Err(rejection) => reject(outcome, rejection.reason()),
776    }
777}
778
779/// A partial observation of a curated entry: the pulled artifact plus GGUF-sourced facts,
780/// citing only the Ollama reads that produced them. Curated entries are never modified.
781async fn reobserve(
782    run: &Run<'_>,
783    model: &CatalogModel,
784    stamp: &str,
785    state: &mut AdmissionState,
786) -> io::Result<Option<Value>> {
787    let Some(reference) = model.source.ollama.as_deref() else {
788        return Ok(None);
789    };
790    let (path, tag) = parse_reference(reference);
791    let repo = path.strip_prefix("library/").unwrap_or(path);
792    if repo.contains('/') || !name(repo) || !name(tag) {
793        return Ok(None);
794    }
795    let Some(index) = pulled_quantization(model) else {
796        return Ok(None);
797    };
798    let manifest_url = format!("https://registry.ollama.ai/v2/library/{repo}/manifests/{tag}");
799    let manifest_raw = run
800        .get(
801            &manifest_url,
802            Some(MANIFEST_ACCEPT),
803            None,
804            MAX_MANIFEST_BYTES,
805        )
806        .await?;
807    let manifest_text = std::str::from_utf8(&manifest_raw)
808        .map_err(|_| io::Error::other("manifest is not UTF-8"))?;
809    let Some(layer) = parse_layer(manifest_text).map_err(io::Error::other)? else {
810        return Ok(None);
811    };
812    let descriptors: ManifestLayers =
813        serde_json::from_str(manifest_text).map_err(io::Error::other)?;
814    let blob = |digest: &str| {
815        format!("https://registry.ollama.ai/v2/library/{repo}/blobs/sha256:{digest}")
816    };
817    let config_digest = sha256(&descriptors.config.digest)
818        .ok_or_else(|| io::Error::other("invalid config digest"))?;
819    let config_raw = run
820        .get(&blob(config_digest), None, None, MAX_CONFIG_BYTES)
821        .await?;
822    let Ok(config) = RegistryConfig::parse(
823        std::str::from_utf8(&config_raw).map_err(|_| io::Error::other("config is not UTF-8"))?,
824    ) else {
825        return Ok(None);
826    };
827    let license_digest = descriptors
828        .layers
829        .iter()
830        .find(|layer| layer.media_type == "application/vnd.ollama.image.license")
831        .and_then(|layer| sha256(&layer.digest))
832        .map(str::to_string);
833    let key = format!(
834        "{}:{}",
835        layer.sha256,
836        license_digest.as_deref().unwrap_or("none")
837    );
838    let sourced = match state.facts.get(&key) {
839        Some(sourced) => sourced.clone(),
840        None => {
841            let Ok(gguf) = run.gguf(&blob(&layer.sha256)).await? else {
842                return Ok(None);
843            };
844            let Some(arch) = gguf.architecture().map(str::to_string) else {
845                return Ok(None);
846            };
847            let Some(context_length) = gguf.u64(&format!("{arch}.context_length")) else {
848                return Ok(None);
849            };
850            let head = match &license_digest {
851                Some(digest) => Some(
852                    String::from_utf8_lossy(
853                        &run.get(
854                            &blob(digest),
855                            None,
856                            Some(MAX_LICENSE_HEAD_BYTES),
857                            MAX_LICENSE_HEAD_BYTES,
858                        )
859                        .await?,
860                    )
861                    .into_owned(),
862                ),
863                None => None,
864            };
865            let sourced = SourcedFacts {
866                context_length,
867                architecture: if gguf
868                    .u64(&format!("{arch}.expert_count"))
869                    .is_some_and(|count| count > 0)
870                {
871                    Architecture::Moe
872                } else {
873                    Architecture::Dense
874                },
875                license: license_id(
876                    gguf.text("general.license"),
877                    gguf.text("general.license.name"),
878                    head.as_deref(),
879                )
880                .ok()
881                .map(str::to_string),
882                kv_bytes_per_token: kv_bytes_per_token(&gguf),
883            };
884            state.facts.insert(key, sourced.clone());
885            sourced
886        }
887    };
888    let mut quant = serde_json::to_value(&model.quantizations[index]).map_err(io::Error::other)?;
889    let object = quant
890        .as_object_mut()
891        .ok_or_else(|| io::Error::other("invalid quantization"))?;
892    for field in ["minRamBytes", "minVramBytes", "digestVerified"] {
893        object.remove(field);
894    }
895    object.insert("name".into(), json!(config.file_type.to_ascii_uppercase()));
896    object.insert("diskBytes".into(), json!(layer.disk_bytes));
897    object.insert("sha256".into(), json!(layer.sha256));
898    if layer.projectors.is_empty() {
899        object.remove("projectors");
900    } else {
901        object.insert(
902            "projectors".into(),
903            serde_json::to_value(&layer.projectors).map_err(io::Error::other)?,
904        );
905    }
906    let mut facts = serde_json::Map::new();
907    facts.insert("defaultQuantization".into(), quant);
908    facts.insert("contextLength".into(), json!(sourced.context_length as f64));
909    facts.insert(
910        "architecture".into(),
911        serde_json::to_value(&sourced.architecture).map_err(io::Error::other)?,
912    );
913    if let Some(license) = sourced.license {
914        facts.insert("license".into(), json!(license));
915    }
916    // Asserted only when the catalog has a value to compare; unknown is not a contradiction.
917    if let (Some(rate), Some(_)) = (sourced.kv_bytes_per_token, model.kv_bytes_per_token) {
918        facts.insert("kvBytesPerToken".into(), json!(rate));
919    }
920    Ok(Some(json!({
921        "id": model.id,
922        "checkedAt": stamp,
923        "sources": [manifest_url, format!("https://ollama.com/library/{repo}:{tag}")],
924        "facts": facts,
925    })))
926}
927
928/// The catalog's own representation of a partially observed fact.
929fn catalog_value(model: &CatalogModel, field: &str) -> io::Result<Value> {
930    if field == "defaultQuantization" {
931        let index = pulled_quantization(model).unwrap_or(0);
932        let mut quant =
933            serde_json::to_value(&model.quantizations[index]).map_err(io::Error::other)?;
934        if let Some(object) = quant.as_object_mut() {
935            for key in ["minRamBytes", "minVramBytes", "digestVerified"] {
936                object.remove(key);
937            }
938        }
939        return Ok(quant);
940    }
941    Ok(serde_json::to_value(model)
942        .map_err(io::Error::other)?
943        .get(field)
944        .cloned()
945        .unwrap_or(Value::Null))
946}
947
948/// Rewrites contradicted, correctable curated facts to what the shipped artifact says.
949/// Architecture contradictions are left for review: they change how the model is sized.
950fn correct(
951    model: &CatalogModel,
952    observed: &Value,
953) -> io::Result<Option<(CatalogModel, Vec<Correction>)>> {
954    let Some(facts) = observed["facts"].as_object() else {
955        return Ok(None);
956    };
957    let sources: Vec<String> = observed["sources"]
958        .as_array()
959        .map(|items| {
960            items
961                .iter()
962                .filter_map(|item| item.as_str().map(str::to_string))
963                .collect()
964        })
965        .unwrap_or_default();
966    let mut fixed = model.clone();
967    let mut changes = Vec::new();
968    for (field, to) in facts {
969        let from = catalog_value(model, field)?;
970        if from == *to || field == "architecture" {
971            continue;
972        }
973        match field.as_str() {
974            "contextLength" => {
975                fixed.context_length = to
976                    .as_f64()
977                    .ok_or_else(|| io::Error::other("invalid context"))?
978            }
979            "license" => {
980                fixed.license = to
981                    .as_str()
982                    .ok_or_else(|| io::Error::other("invalid license"))?
983                    .into()
984            }
985            "kvBytesPerToken" => fixed.kv_bytes_per_token = to.as_f64(),
986            "defaultQuantization" => {
987                let index = pulled_quantization(model).unwrap_or(0);
988                let name = to["name"]
989                    .as_str()
990                    .ok_or_else(|| io::Error::other("invalid quant"))?;
991                let projectors: Vec<rigspark_core::sizing::ProjectorArtifact> =
992                    serde_json::from_value(to.get("projectors").cloned().unwrap_or(json!([])))
993                        .map_err(io::Error::other)?;
994                let quant = rigspark_core::enrich::sized_quantization(
995                    rigspark_core::sizing::parse_param_count(&model.params)
996                        .map_err(io::Error::other)?,
997                    &model.architecture,
998                    name,
999                    to["diskBytes"]
1000                        .as_f64()
1001                        .ok_or_else(|| io::Error::other("invalid size"))?,
1002                    to["sha256"].as_str().map(str::to_string),
1003                    projectors,
1004                )
1005                .map_err(io::Error::other)?;
1006                fixed.quantizations[index] = quant;
1007                // Facts for other quantizations named like the new one are no longer sourced.
1008                let mut position = 0;
1009                fixed.quantizations.retain(|candidate| {
1010                    let keep = position == index || candidate.name != name;
1011                    position += 1;
1012                    keep
1013                });
1014            }
1015            _ => continue,
1016        }
1017        changes.push(Correction {
1018            id: model.id.clone(),
1019            field: field.clone(),
1020            from,
1021            to: to.clone(),
1022            sources: sources.clone(),
1023        });
1024    }
1025    if changes.is_empty() {
1026        return Ok(None);
1027    }
1028    let check = json!({"schemaVersion": SCHEMA_VERSION, "generatedAt": "2026-01-01T00:00:00.000Z", "models": [fixed]});
1029    Catalog::parse(&check.to_string()).map_err(io::Error::other)?;
1030    Ok(Some((fixed, changes)))
1031}