1use crate::catalog_proposals::allowed_registry_redirect;
4use futures_util::StreamExt;
5use rigspark_core::{
6 admission::{
7 AdmissionInput, Gguf, GgufError, LibraryEntry, MAX_CONFIG_BYTES, MAX_GGUF_HEADER_BYTES,
8 MAX_HTML_BYTES, MAX_LICENSE_HEAD_BYTES, RegistryConfig, build_entry, catalog_capabilities,
9 kv_bytes_per_token, license_id, observation, parse_gguf, parse_library, parse_tags,
10 select_variants,
11 },
12 catalog::{Catalog, CatalogModel, EntryProvenance, SCHEMA_VERSION},
13 registry_collector::{
14 MANIFEST_ACCEPT, MAX_MANIFEST_BYTES, parse_layer, parse_reference, pulled_quantization,
15 },
16 sizing::Architecture,
17};
18use serde::{Deserialize, Serialize};
19use serde_json::{Value, json};
20use std::{
21 collections::{BTreeMap, BTreeSet},
22 io,
23 time::Duration,
24};
25use time::{OffsetDateTime, format_description::well_known::Rfc3339};
26use tokio_util::sync::CancellationToken;
27use url::Url;
28
29pub const LIBRARY_URL: &str = "https://ollama.com/library?sort=newest";
30pub const GENERATION_LISTING_URL: &str =
31 "https://huggingface.co/api/models?author=Comfy-Org&sort=createdAt&direction=-1&limit=500";
32pub const SCOPE_NAME: &str = "ollama-local-variants";
33const GGUF_STEPS: [usize; 3] = [8 * 1024 * 1024, 32 * 1024 * 1024, MAX_GGUF_HEADER_BYTES];
34
35fn name(part: &str) -> bool {
36 !part.is_empty()
37 && part.len() <= 128
38 && part.as_bytes()[0].is_ascii_alphanumeric()
39 && part
40 .bytes()
41 .all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte))
42}
43
44pub fn allowed_request(url: &Url) -> bool {
46 if url.scheme() != "https"
47 || url.port().is_some()
48 || !url.username().is_empty()
49 || url.password().is_some()
50 || url.fragment().is_some()
51 {
52 return false;
53 }
54 let segments: Vec<&str> = url.path().trim_start_matches('/').split('/').collect();
55 match (url.host_str(), url.query(), segments.as_slice()) {
56 (Some("ollama.com"), Some("sort=newest"), ["library"]) => true,
57 (Some("ollama.com"), None, ["library", repo, "tags"]) => name(repo),
58 (Some("registry.ollama.ai"), None, ["v2", "library", repo, "manifests", tag]) => {
59 name(repo) && name(tag)
60 }
61 (Some("registry.ollama.ai"), None, ["v2", "library", repo, "blobs", digest]) => {
62 name(repo)
63 && digest.strip_prefix("sha256:").is_some_and(|hex| {
64 hex.len() == 64 && hex.bytes().all(|byte| byte.is_ascii_hexdigit())
65 })
66 }
67 (Some("huggingface.co"), Some(query), ["api", "models"]) => {
68 Some(query)
69 == GENERATION_LISTING_URL
70 .split_once('?')
71 .map(|(_, query)| query)
72 }
73 (Some("huggingface.co"), None | Some("blobs=true"), ["api", "models", owner, repo]) => {
74 name(owner) && name(repo)
75 }
76 (
77 Some("huggingface.co"),
78 Some("blobs=true"),
79 ["api", "models", owner, repo, "revision", revision],
80 ) => {
81 name(owner)
82 && name(repo)
83 && revision.len() == 40
84 && revision.bytes().all(|byte| byte.is_ascii_hexdigit())
85 }
86 (Some("huggingface.co"), None, [owner, repo, "resolve", revision, path @ ..]) => {
87 name(owner)
88 && name(repo)
89 && revision.len() == 40
90 && revision.bytes().all(|byte| byte.is_ascii_hexdigit())
91 && !path.is_empty()
92 && path.len() <= 8
93 && path.iter().all(|segment| name(segment))
94 && path.last().is_some_and(|file| {
95 file.ends_with(".gguf")
96 || (*owner == "Comfy-Org" && file.ends_with(".safetensors"))
97 })
98 }
99 _ => false,
100 }
101}
102
103pub fn allowed_hub_redirect(url: &Url) -> bool {
105 url.scheme() == "https"
106 && url.port().is_none()
107 && url.username().is_empty()
108 && url.password().is_none()
109 && url
110 .host_str()
111 .is_some_and(|host| host == "huggingface.co" || host.ends_with(".hf.co"))
112}
113
114pub struct Fetched {
115 pub status: u16,
116 pub body: Vec<u8>,
117}
118
119#[async_trait::async_trait]
120pub trait AdmissionTransport: Send + Sync {
121 async fn fetch(
123 &self,
124 url: &Url,
125 accept: Option<&'static str>,
126 range: Option<usize>,
127 limit: usize,
128 ) -> io::Result<Fetched>;
129}
130
131pub struct NativeAdmissionTransport {
132 client: reqwest::Client,
133}
134impl NativeAdmissionTransport {
135 pub fn new() -> io::Result<Self> {
136 let client = reqwest::Client::builder()
137 .https_only(true)
138 .no_proxy()
139 .user_agent(concat!(
140 "rigspark-catalog-admission/",
141 env!("CARGO_PKG_VERSION")
142 ))
143 .redirect(reqwest::redirect::Policy::custom(|attempt| {
144 let previous = attempt.previous().last();
145 let from_blob = previous.is_some_and(|url| {
146 url.host_str() == Some("registry.ollama.ai")
147 && url.path().contains("/blobs/sha256:")
148 });
149 let from_resolve = previous.is_some_and(|url| {
150 url.host_str() == Some("huggingface.co") && url.path().contains("/resolve/")
151 });
152 let allowed = (from_blob && allowed_registry_redirect(attempt.url().as_str()))
153 || (from_resolve && allowed_hub_redirect(attempt.url()));
154 if attempt.previous().len() > 2 || !allowed {
155 attempt.error("registry redirect refused")
156 } else {
157 attempt.follow()
158 }
159 }))
160 .connect_timeout(Duration::from_secs(10))
161 .timeout(Duration::from_secs(120))
162 .build()
163 .map_err(|_| io::Error::other("admission client initialization failed"))?;
164 Ok(Self { client })
165 }
166}
167#[async_trait::async_trait]
168impl AdmissionTransport for NativeAdmissionTransport {
169 async fn fetch(
170 &self,
171 url: &Url,
172 accept: Option<&'static str>,
173 range: Option<usize>,
174 limit: usize,
175 ) -> io::Result<Fetched> {
176 if !allowed_request(url) {
177 return Err(io::Error::other("admission URL is not allow-listed"));
178 }
179 let mut request = self.client.get(url.clone());
180 if let Some(accept) = accept {
181 request = request.header(reqwest::header::ACCEPT, accept);
182 }
183 if let Some(range) = range {
184 request = request.header(reqwest::header::RANGE, format!("bytes=0-{}", range - 1));
185 }
186 let response = request
187 .send()
188 .await
189 .map_err(|_| io::Error::other("admission request failed"))?;
190 let status = response.status().as_u16();
191 let mut stream = response.bytes_stream();
192 let mut body = Vec::new();
193 while let Some(chunk) = stream.next().await {
194 let chunk = chunk.map_err(|_| io::Error::other("admission body read failed"))?;
195 body.extend_from_slice(&chunk);
196 if body.len() >= limit {
197 if range.is_none() && body.len() > limit {
199 return Err(io::Error::other("admission response exceeds its size cap"));
200 }
201 body.truncate(limit);
202 break;
203 }
204 }
205 Ok(Fetched { status, body })
206 }
207}
208
209#[derive(Deserialize)]
210#[serde(deny_unknown_fields)]
211struct RecordedBody {
212 status: u16,
213 #[serde(default)]
214 text: Option<String>,
215 #[serde(default)]
216 json: Option<Value>,
217 #[serde(default)]
218 hex: Option<String>,
219}
220
221pub struct RecordedAdmissionTransport {
223 responses: BTreeMap<String, (u16, Vec<u8>)>,
224}
225impl RecordedAdmissionTransport {
226 pub fn new(responses: BTreeMap<String, (u16, Vec<u8>)>) -> io::Result<Self> {
227 for key in responses.keys() {
228 let url = Url::parse(key).map_err(io::Error::other)?;
229 if !allowed_request(&url) {
230 return Err(io::Error::other("recorded URL is not allow-listed"));
231 }
232 }
233 Ok(Self { responses })
234 }
235 pub fn parse(raw: &str) -> io::Result<Self> {
236 let recorded: BTreeMap<String, RecordedBody> =
237 serde_json::from_str(raw).map_err(io::Error::other)?;
238 let mut responses = BTreeMap::new();
239 for (url, body) in recorded {
240 let bytes = match (body.text, body.json, body.hex) {
241 (Some(text), None, None) => text.into_bytes(),
242 (None, Some(value), None) => {
243 serde_json::to_vec(&value).map_err(io::Error::other)?
244 }
245 (None, None, Some(hex)) => decode_hex(&hex)?,
246 (None, None, None) => Vec::new(),
247 _ => return Err(io::Error::other("recorded body must have one encoding")),
248 };
249 responses.insert(url, (body.status, bytes));
250 }
251 Self::new(responses)
252 }
253}
254fn decode_hex(hex: &str) -> io::Result<Vec<u8>> {
255 if !hex.len().is_multiple_of(2) {
256 return Err(io::Error::other("invalid recorded hex body"));
257 }
258 (0..hex.len())
259 .step_by(2)
260 .map(|index| {
261 u8::from_str_radix(&hex[index..index + 2], 16)
262 .map_err(|_| io::Error::other("invalid recorded hex body"))
263 })
264 .collect()
265}
266#[async_trait::async_trait]
267impl AdmissionTransport for RecordedAdmissionTransport {
268 async fn fetch(
269 &self,
270 url: &Url,
271 _accept: Option<&'static str>,
272 range: Option<usize>,
273 limit: usize,
274 ) -> io::Result<Fetched> {
275 if !allowed_request(url) {
276 return Err(io::Error::other("admission URL is not allow-listed"));
277 }
278 let (status, body) = self
279 .responses
280 .get(url.as_str())
281 .cloned()
282 .unwrap_or((404, Vec::new()));
283 let cap = range.unwrap_or(usize::MAX).min(limit);
284 if range.is_none() && body.len() > limit {
285 return Err(io::Error::other("admission response exceeds its size cap"));
286 }
287 Ok(Fetched {
288 status,
289 body: body[..body.len().min(cap)].to_vec(),
290 })
291 }
292}
293
294#[derive(Deserialize)]
295#[serde(rename_all = "camelCase")]
296struct ManifestLayers {
297 config: Descriptor,
298 layers: Vec<Descriptor>,
299}
300#[derive(Deserialize)]
301#[serde(rename_all = "camelCase")]
302struct Descriptor {
303 media_type: String,
304 digest: String,
305}
306fn sha256(digest: &str) -> Option<&str> {
307 digest
308 .strip_prefix("sha256:")
309 .filter(|hex| hex.len() == 64 && hex.bytes().all(|byte| byte.is_ascii_hexdigit()))
310}
311
312#[derive(Clone, Debug, Default, Serialize, Deserialize)]
313#[serde(rename_all = "camelCase", deny_unknown_fields)]
314pub struct AdmissionState {
315 #[serde(default)]
318 pub rejections: BTreeMap<String, String>,
319 #[serde(default)]
321 pub facts: BTreeMap<String, SourcedFacts>,
322}
323
324#[derive(Clone, Debug, Serialize, Deserialize)]
325#[serde(rename_all = "camelCase", deny_unknown_fields)]
326pub struct SourcedFacts {
327 pub context_length: u64,
328 pub architecture: Architecture,
329 #[serde(default, skip_serializing_if = "Option::is_none")]
330 pub license: Option<String>,
331 #[serde(default, skip_serializing_if = "Option::is_none")]
332 pub kv_bytes_per_token: Option<f64>,
333}
334
335pub struct AdmissionOptions {
336 pub now: String,
337 pub max_new_variants: usize,
338 pub state: AdmissionState,
339 pub correct_curated: bool,
341}
342
343#[derive(Clone, Debug, Serialize)]
345pub struct Correction {
346 pub id: String,
347 pub field: String,
348 pub from: Value,
349 pub to: Value,
350 pub sources: Vec<String>,
351}
352
353#[derive(Default, Serialize)]
354#[serde(rename_all = "camelCase")]
355pub struct AdmissionOutcome {
356 #[serde(skip)]
357 pub catalog: Option<Catalog>,
358 #[serde(skip)]
359 pub observations: Vec<Value>,
360 #[serde(skip)]
362 pub curated_observations: Vec<Value>,
363 #[serde(skip)]
364 pub scope: Value,
365 #[serde(skip)]
366 pub state: AdmissionState,
367 pub added: Vec<String>,
368 pub updated: Vec<String>,
369 pub reverified: Vec<String>,
370 pub curated_observed: Vec<String>,
371 pub corrections: Vec<Correction>,
372 pub removed: Vec<String>,
373 pub rejected: BTreeMap<String, String>,
374 pub deferred: Vec<String>,
375 pub failures: Vec<String>,
376 pub complete: bool,
377}
378
379struct Run<'a> {
380 transport: &'a dyn AdmissionTransport,
381 cancel: &'a CancellationToken,
382}
383impl Run<'_> {
384 async fn get(
385 &self,
386 url: &str,
387 accept: Option<&'static str>,
388 range: Option<usize>,
389 limit: usize,
390 ) -> io::Result<Vec<u8>> {
391 if self.cancel.is_cancelled() {
392 return Err(io::Error::new(
393 io::ErrorKind::Interrupted,
394 "catalog admission cancelled",
395 ));
396 }
397 let url = Url::parse(url).map_err(io::Error::other)?;
398 let fetched = tokio::select! {
399 biased;
400 _ = self.cancel.cancelled() => {
401 return Err(io::Error::new(io::ErrorKind::Interrupted, "catalog admission cancelled"));
402 }
403 result = tokio::time::timeout(Duration::from_secs(180), self.transport.fetch(&url, accept, range, limit)) => {
404 result.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "admission request timed out"))??
405 }
406 };
407 let ok = if range.is_some() {
408 matches!(fetched.status, 200 | 206)
409 } else {
410 (200..300).contains(&fetched.status)
411 };
412 if !ok {
413 return Err(io::Error::new(
414 io::ErrorKind::NotFound,
415 format!("upstream status {}", fetched.status),
416 ));
417 }
418 Ok(fetched.body)
419 }
420 async fn text(&self, url: &str, limit: usize) -> io::Result<String> {
421 String::from_utf8(self.get(url, None, None, limit).await?)
422 .map_err(|_| io::Error::other("upstream text is not UTF-8"))
423 }
424 async fn gguf(&self, blob: &str) -> io::Result<Result<Gguf, &'static str>> {
426 for step in GGUF_STEPS {
427 let bytes = self.get(blob, None, Some(step), step).await?;
428 match parse_gguf(&bytes) {
429 Ok(parsed) => return Ok(Ok(parsed)),
430 Err(GgufError::Truncated) if bytes.len() >= step => continue,
431 Err(GgufError::Truncated) => return Ok(Err("GGUF header truncated")),
432 Err(GgufError::Invalid(reason)) => return Ok(Err(reason)),
433 }
434 }
435 Ok(Err("GGUF header exceeds 64 MiB"))
436 }
437}
438
439fn interrupted(error: &io::Error) -> bool {
440 error.kind() == io::ErrorKind::Interrupted
441}
442
443pub async fn admit(
446 catalog: &Catalog,
447 transport: &dyn AdmissionTransport,
448 options: &AdmissionOptions,
449 cancel: &CancellationToken,
450) -> io::Result<AdmissionOutcome> {
451 let now = OffsetDateTime::parse(&options.now, &Rfc3339)
452 .map_err(|_| io::Error::other("invalid admission clock"))?
453 .to_offset(time::UtcOffset::UTC);
454 let stamp = now
455 .format(time::macros::format_description!(
456 "[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
457 ))
458 .map_err(io::Error::other)?;
459 let today = stamp[..10].to_string();
460 let run = Run { transport, cancel };
461 let library =
462 parse_library(&run.text(LIBRARY_URL, MAX_HTML_BYTES).await?).map_err(io::Error::other)?;
463
464 let mut outcome = AdmissionOutcome {
465 complete: true,
466 state: options.state.clone(),
467 ..Default::default()
468 };
469 let curated: BTreeMap<&str, &CatalogModel> = catalog
470 .models
471 .iter()
472 .filter(|model| model.provenance == EntryProvenance::Curated)
473 .map(|model| (model.id.as_str(), model))
474 .collect();
475 let curated_digests: BTreeSet<&str> = curated
476 .values()
477 .flat_map(|model| model.quantizations.iter())
478 .filter_map(|quant| quant.sha256.as_deref())
479 .collect();
480 let mut auto: BTreeMap<String, CatalogModel> = catalog
481 .models
482 .iter()
483 .filter(|model| model.provenance == EntryProvenance::Auto)
484 .map(|model| (model.id.clone(), model.clone()))
485 .collect();
486 let listed: BTreeSet<&str> = library
487 .iter()
488 .map(|entry| entry.repository.as_str())
489 .collect();
490 for id in auto.keys().cloned().collect::<Vec<_>>() {
491 if !listed.contains(parse_reference(&id).0) {
492 auto.remove(&id);
493 outcome.removed.push(id);
494 }
495 }
496 let mut eligible: BTreeSet<String> = BTreeSet::new();
497 let mut seen_digests: BTreeSet<String> = BTreeSet::new();
498 let mut new_variants = 0;
499
500 for entry in &library {
501 let repo = entry.repository.as_str();
502 let tags_url = format!("https://ollama.com/library/{repo}/tags");
503 let selected = match run.text(&tags_url, MAX_HTML_BYTES).await {
504 Ok(html) => match parse_tags(&html, repo) {
505 Ok(tags) => select_variants(&tags),
506 Err(_) => {
507 outcome
508 .failures
509 .push(format!("{repo}: tags page unparseable"));
510 outcome.complete = false;
511 continue;
512 }
513 },
514 Err(error) if interrupted(&error) => return Err(error),
515 Err(_) => {
516 outcome
517 .failures
518 .push(format!("{repo}: tags page unavailable"));
519 outcome.complete = false;
520 continue;
521 }
522 };
523 let selected_ids: BTreeSet<String> =
524 selected.iter().map(|tag| format!("{repo}:{tag}")).collect();
525 for id in auto.keys().cloned().collect::<Vec<_>>() {
526 if parse_reference(&id).0 == repo && !selected_ids.contains(&id) {
527 auto.remove(&id);
528 outcome.removed.push(id);
529 }
530 }
531 for tag in &selected {
532 let id = format!("{repo}:{tag}");
533 if curated.contains_key(id.as_str()) {
534 continue;
535 }
536 match admit_variant(
537 &run,
538 entry,
539 tag,
540 &id,
541 &auto,
542 &curated_digests,
543 &mut seen_digests,
544 &today,
545 &mut new_variants,
546 options,
547 &mut outcome,
548 )
549 .await
550 {
551 Ok(Some(model)) => {
552 eligible.insert(id.clone());
553 auto.insert(id, model);
554 }
555 Ok(None) => {}
556 Err(error) if interrupted(&error) => return Err(error),
557 Err(error) => {
558 outcome.failures.push(format!("{id}: {error}"));
559 outcome.complete = false;
560 if auto.contains_key(&id) {
561 eligible.insert(id);
562 }
563 }
564 }
565 }
566 }
567 for id in &outcome.deferred {
568 eligible.insert(id.clone());
569 }
570 let mut corrected: BTreeMap<String, CatalogModel> = BTreeMap::new();
571 for model in curated.values() {
572 match reobserve(&run, model, &stamp, &mut outcome.state).await {
573 Ok(Some(observed)) => {
574 if options.correct_curated
575 && let Some((fixed, changes)) = correct(model, &observed)?
576 {
577 corrected.insert(model.id.clone(), fixed);
578 outcome.corrections.extend(changes);
579 }
580 outcome.curated_observed.push(model.id.clone());
581 outcome.curated_observations.push(observed);
582 }
583 Ok(None) => {}
584 Err(error) if interrupted(&error) => return Err(error),
585 Err(error) => outcome
586 .failures
587 .push(format!("{} (curated): {error}", model.id)),
588 }
589 }
590
591 let mut models: Vec<CatalogModel> = catalog
592 .models
593 .iter()
594 .filter(|model| model.provenance == EntryProvenance::Curated)
595 .map(|model| corrected.remove(&model.id).unwrap_or_else(|| model.clone()))
596 .chain(auto.into_values())
597 .collect();
598 models.sort_by(|left, right| {
599 right
600 .recency()
601 .map(|(day, _)| day)
602 .cmp(&left.recency().map(|(day, _)| day))
603 .then_with(|| left.id.cmp(&right.id))
604 });
605 outcome.observations = models
606 .iter()
607 .filter(|model| model.provenance == EntryProvenance::Auto)
608 .map(|model| observation(model, &stamp))
609 .collect();
610 let mut variants: BTreeSet<String> = models.iter().map(|model| model.id.clone()).collect();
611 variants.extend(eligible);
612 outcome.scope = json!({
613 "name": SCOPE_NAME,
614 "checkedAt": stamp,
615 "source": "https://ollama.com/library",
616 "complete": outcome.complete,
617 "variants": variants,
618 });
619 let changed =
620 !outcome.added.is_empty() || !outcome.updated.is_empty() || !outcome.removed.is_empty();
621 let encoded = json!({
622 "schemaVersion": SCHEMA_VERSION,
623 "generatedAt": if changed { stamp.clone() } else { catalog.generated_at.clone() },
624 "models": models,
625 });
626 outcome.catalog = Some(Catalog::parse(&encoded.to_string()).map_err(io::Error::other)?);
627 outcome.removed.sort();
628 Ok(outcome)
629}
630
631#[allow(clippy::too_many_arguments)]
632async fn admit_variant(
633 run: &Run<'_>,
634 entry: &LibraryEntry,
635 tag: &str,
636 id: &str,
637 auto: &BTreeMap<String, CatalogModel>,
638 curated_digests: &BTreeSet<&str>,
639 seen_digests: &mut BTreeSet<String>,
640 today: &str,
641 new_variants: &mut usize,
642 options: &AdmissionOptions,
643 outcome: &mut AdmissionOutcome,
644) -> io::Result<Option<CatalogModel>> {
645 let repo = entry.repository.as_str();
646 let blob =
647 |digest: &str| format!("https://registry.ollama.ai/v2/library/{repo}/blobs/{digest}");
648 let manifest_raw = run
649 .get(
650 &format!("https://registry.ollama.ai/v2/library/{repo}/manifests/{tag}"),
651 Some(MANIFEST_ACCEPT),
652 None,
653 MAX_MANIFEST_BYTES,
654 )
655 .await?;
656 let manifest_text = std::str::from_utf8(&manifest_raw)
657 .map_err(|_| io::Error::other("manifest is not UTF-8"))?;
658 let Some(layer) = parse_layer(manifest_text).map_err(io::Error::other)? else {
659 outcome
660 .rejected
661 .insert(id.into(), "no GGUF model layer".into());
662 return Ok(None);
663 };
664 let descriptors: ManifestLayers =
665 serde_json::from_str(manifest_text).map_err(io::Error::other)?;
666 if curated_digests.contains(layer.sha256.as_str()) || !seen_digests.insert(layer.sha256.clone())
667 {
668 return Ok(None);
669 }
670 let license_digest = descriptors
671 .layers
672 .iter()
673 .find(|layer| layer.media_type == "application/vnd.ollama.image.license")
674 .and_then(|layer| sha256(&layer.digest))
675 .map(str::to_string);
676 let cache_key = format!(
677 "{}:{}",
678 layer.sha256,
679 license_digest.as_deref().unwrap_or("none")
680 );
681 let prior = auto.get(id);
682 if let Some(prior) = prior
683 && prior.quantizations[0].sha256.as_deref() == Some(layer.sha256.as_str())
684 && prior.quantizations[0].disk_bytes == layer.disk_bytes
685 && !outcome.state.rejections.contains_key(&cache_key)
686 {
687 let mut refreshed = prior.clone();
688 let capabilities = catalog_capabilities(&entry.capabilities);
689 if capabilities != prior.capabilities {
690 refreshed.capabilities = capabilities;
691 outcome.updated.push(id.into());
692 } else {
693 outcome.reverified.push(id.into());
694 }
695 return Ok(Some(refreshed));
696 }
697 if let Some(reason) = outcome.state.rejections.get(&cache_key) {
698 outcome.rejected.insert(id.into(), reason.clone());
699 return Ok(None);
700 }
701 if prior.is_none() {
702 if *new_variants >= options.max_new_variants {
703 outcome.deferred.push(id.into());
704 outcome.complete = false;
705 return Ok(None);
706 }
707 *new_variants += 1;
708 }
709 let config_digest = sha256(&descriptors.config.digest)
710 .ok_or_else(|| io::Error::other("invalid config digest"))?;
711 let config_raw = run
712 .get(
713 &blob(&format!("sha256:{config_digest}")),
714 None,
715 None,
716 MAX_CONFIG_BYTES,
717 )
718 .await?;
719 let config = RegistryConfig::parse(
720 std::str::from_utf8(&config_raw).map_err(|_| io::Error::other("config is not UTF-8"))?,
721 );
722 let license_head = match &license_digest {
723 Some(digest) => Some(
724 String::from_utf8_lossy(
725 &run.get(
726 &blob(&format!("sha256:{digest}")),
727 None,
728 Some(MAX_LICENSE_HEAD_BYTES),
729 MAX_LICENSE_HEAD_BYTES,
730 )
731 .await?,
732 )
733 .into_owned(),
734 ),
735 None => None,
736 };
737 let reject = |outcome: &mut AdmissionOutcome, reason: &str| {
738 outcome.rejected.insert(id.into(), reason.into());
739 outcome
740 .state
741 .rejections
742 .insert(cache_key.clone(), reason.into());
743 Ok(None)
744 };
745 let Ok(config) = config else {
746 return reject(outcome, "model config lacks family, size or file type");
747 };
748 let gguf = match run.gguf(&blob(&format!("sha256:{}", layer.sha256))).await? {
749 Ok(gguf) => gguf,
750 Err(reason) => return reject(outcome, reason),
751 };
752 let built = build_entry(&AdmissionInput {
753 repository: repo,
754 tag,
755 capabilities: &entry.capabilities,
756 layer: &layer,
757 config: &config,
758 gguf: &gguf,
759 license_head: license_head.as_deref(),
760 today: prior
761 .and_then(|model| model.added_at.as_deref())
762 .unwrap_or(today),
763 });
764 match built {
765 Ok(model) => {
766 outcome.state.rejections.remove(&cache_key);
767 if prior.is_some() {
768 outcome.updated.push(id.into());
769 } else {
770 outcome.added.push(id.into());
771 }
772 Ok(Some(model))
773 }
774 Err(rejection) => reject(outcome, rejection.reason()),
775 }
776}
777
778async fn reobserve(
781 run: &Run<'_>,
782 model: &CatalogModel,
783 stamp: &str,
784 state: &mut AdmissionState,
785) -> io::Result<Option<Value>> {
786 let Some(reference) = model.source.ollama.as_deref() else {
787 return Ok(None);
788 };
789 let (path, tag) = parse_reference(reference);
790 let repo = path.strip_prefix("library/").unwrap_or(path);
791 if repo.contains('/') || !name(repo) || !name(tag) {
792 return Ok(None);
793 }
794 let Some(index) = pulled_quantization(model) else {
795 return Ok(None);
796 };
797 let manifest_url = format!("https://registry.ollama.ai/v2/library/{repo}/manifests/{tag}");
798 let manifest_raw = run
799 .get(
800 &manifest_url,
801 Some(MANIFEST_ACCEPT),
802 None,
803 MAX_MANIFEST_BYTES,
804 )
805 .await?;
806 let manifest_text = std::str::from_utf8(&manifest_raw)
807 .map_err(|_| io::Error::other("manifest is not UTF-8"))?;
808 let Some(layer) = parse_layer(manifest_text).map_err(io::Error::other)? else {
809 return Ok(None);
810 };
811 let descriptors: ManifestLayers =
812 serde_json::from_str(manifest_text).map_err(io::Error::other)?;
813 let blob = |digest: &str| {
814 format!("https://registry.ollama.ai/v2/library/{repo}/blobs/sha256:{digest}")
815 };
816 let config_digest = sha256(&descriptors.config.digest)
817 .ok_or_else(|| io::Error::other("invalid config digest"))?;
818 let config_raw = run
819 .get(&blob(config_digest), None, None, MAX_CONFIG_BYTES)
820 .await?;
821 let Ok(config) = RegistryConfig::parse(
822 std::str::from_utf8(&config_raw).map_err(|_| io::Error::other("config is not UTF-8"))?,
823 ) else {
824 return Ok(None);
825 };
826 let license_digest = descriptors
827 .layers
828 .iter()
829 .find(|layer| layer.media_type == "application/vnd.ollama.image.license")
830 .and_then(|layer| sha256(&layer.digest))
831 .map(str::to_string);
832 let key = format!(
833 "{}:{}",
834 layer.sha256,
835 license_digest.as_deref().unwrap_or("none")
836 );
837 let sourced = match state.facts.get(&key) {
838 Some(sourced) => sourced.clone(),
839 None => {
840 let Ok(gguf) = run.gguf(&blob(&layer.sha256)).await? else {
841 return Ok(None);
842 };
843 let Some(arch) = gguf.architecture().map(str::to_string) else {
844 return Ok(None);
845 };
846 let Some(context_length) = gguf.u64(&format!("{arch}.context_length")) else {
847 return Ok(None);
848 };
849 let head = match &license_digest {
850 Some(digest) => Some(
851 String::from_utf8_lossy(
852 &run.get(
853 &blob(digest),
854 None,
855 Some(MAX_LICENSE_HEAD_BYTES),
856 MAX_LICENSE_HEAD_BYTES,
857 )
858 .await?,
859 )
860 .into_owned(),
861 ),
862 None => None,
863 };
864 let sourced = SourcedFacts {
865 context_length,
866 architecture: if gguf
867 .u64(&format!("{arch}.expert_count"))
868 .is_some_and(|count| count > 0)
869 {
870 Architecture::Moe
871 } else {
872 Architecture::Dense
873 },
874 license: license_id(
875 gguf.text("general.license"),
876 gguf.text("general.license.name"),
877 head.as_deref(),
878 )
879 .ok()
880 .map(str::to_string),
881 kv_bytes_per_token: kv_bytes_per_token(&gguf),
882 };
883 state.facts.insert(key, sourced.clone());
884 sourced
885 }
886 };
887 let mut quant = serde_json::to_value(&model.quantizations[index]).map_err(io::Error::other)?;
888 let object = quant
889 .as_object_mut()
890 .ok_or_else(|| io::Error::other("invalid quantization"))?;
891 for field in ["minRamBytes", "minVramBytes", "digestVerified"] {
892 object.remove(field);
893 }
894 object.insert("name".into(), json!(config.file_type.to_ascii_uppercase()));
895 object.insert("diskBytes".into(), json!(layer.disk_bytes));
896 object.insert("sha256".into(), json!(layer.sha256));
897 if layer.projectors.is_empty() {
898 object.remove("projectors");
899 } else {
900 object.insert(
901 "projectors".into(),
902 serde_json::to_value(&layer.projectors).map_err(io::Error::other)?,
903 );
904 }
905 let mut facts = serde_json::Map::new();
906 facts.insert("defaultQuantization".into(), quant);
907 facts.insert("contextLength".into(), json!(sourced.context_length as f64));
908 facts.insert(
909 "architecture".into(),
910 serde_json::to_value(&sourced.architecture).map_err(io::Error::other)?,
911 );
912 if let Some(license) = sourced.license {
913 facts.insert("license".into(), json!(license));
914 }
915 if let (Some(rate), Some(_)) = (sourced.kv_bytes_per_token, model.kv_bytes_per_token) {
917 facts.insert("kvBytesPerToken".into(), json!(rate));
918 }
919 Ok(Some(json!({
920 "id": model.id,
921 "checkedAt": stamp,
922 "sources": [manifest_url, format!("https://ollama.com/library/{repo}:{tag}")],
923 "facts": facts,
924 })))
925}
926
927fn catalog_value(model: &CatalogModel, field: &str) -> io::Result<Value> {
929 if field == "defaultQuantization" {
930 let index = pulled_quantization(model).unwrap_or(0);
931 let mut quant =
932 serde_json::to_value(&model.quantizations[index]).map_err(io::Error::other)?;
933 if let Some(object) = quant.as_object_mut() {
934 for key in ["minRamBytes", "minVramBytes", "digestVerified"] {
935 object.remove(key);
936 }
937 }
938 return Ok(quant);
939 }
940 Ok(serde_json::to_value(model)
941 .map_err(io::Error::other)?
942 .get(field)
943 .cloned()
944 .unwrap_or(Value::Null))
945}
946
947fn correct(
950 model: &CatalogModel,
951 observed: &Value,
952) -> io::Result<Option<(CatalogModel, Vec<Correction>)>> {
953 let Some(facts) = observed["facts"].as_object() else {
954 return Ok(None);
955 };
956 let sources: Vec<String> = observed["sources"]
957 .as_array()
958 .map(|items| {
959 items
960 .iter()
961 .filter_map(|item| item.as_str().map(str::to_string))
962 .collect()
963 })
964 .unwrap_or_default();
965 let mut fixed = model.clone();
966 let mut changes = Vec::new();
967 for (field, to) in facts {
968 let from = catalog_value(model, field)?;
969 if from == *to || field == "architecture" {
970 continue;
971 }
972 match field.as_str() {
973 "contextLength" => {
974 fixed.context_length = to
975 .as_f64()
976 .ok_or_else(|| io::Error::other("invalid context"))?
977 }
978 "license" => {
979 fixed.license = to
980 .as_str()
981 .ok_or_else(|| io::Error::other("invalid license"))?
982 .into()
983 }
984 "kvBytesPerToken" => fixed.kv_bytes_per_token = to.as_f64(),
985 "defaultQuantization" => {
986 let index = pulled_quantization(model).unwrap_or(0);
987 let name = to["name"]
988 .as_str()
989 .ok_or_else(|| io::Error::other("invalid quant"))?;
990 let projectors: Vec<rigspark_core::sizing::ProjectorArtifact> =
991 serde_json::from_value(to.get("projectors").cloned().unwrap_or(json!([])))
992 .map_err(io::Error::other)?;
993 let quant = rigspark_core::enrich::sized_quantization(
994 rigspark_core::sizing::parse_param_count(&model.params)
995 .map_err(io::Error::other)?,
996 &model.architecture,
997 name,
998 to["diskBytes"]
999 .as_f64()
1000 .ok_or_else(|| io::Error::other("invalid size"))?,
1001 to["sha256"].as_str().map(str::to_string),
1002 projectors,
1003 )
1004 .map_err(io::Error::other)?;
1005 fixed.quantizations[index] = quant;
1006 let mut position = 0;
1008 fixed.quantizations.retain(|candidate| {
1009 let keep = position == index || candidate.name != name;
1010 position += 1;
1011 keep
1012 });
1013 }
1014 _ => continue,
1015 }
1016 changes.push(Correction {
1017 id: model.id.clone(),
1018 field: field.clone(),
1019 from,
1020 to: to.clone(),
1021 sources: sources.clone(),
1022 });
1023 }
1024 if changes.is_empty() {
1025 return Ok(None);
1026 }
1027 let check = json!({"schemaVersion": SCHEMA_VERSION, "generatedAt": "2026-01-01T00:00:00.000Z", "models": [fixed]});
1028 Catalog::parse(&check.to_string()).map_err(io::Error::other)?;
1029 Ok(Some((fixed, changes)))
1030}