1use crate::catalog_proposals::allowed_registry_redirect;
4use futures_util::StreamExt;
5use rigspark_core::{
6 admission::{
7 AdmissionInput, Gguf, GgufError, LibraryEntry, MAX_CONFIG_BYTES, MAX_GGUF_HEADER_BYTES,
8 MAX_HTML_BYTES, MAX_LICENSE_HEAD_BYTES, RegistryConfig, build_entry, catalog_capabilities,
9 kv_bytes_per_token, license_id, observation, parse_gguf, parse_library, parse_tags,
10 select_variants,
11 },
12 catalog::{Catalog, CatalogModel, EntryProvenance, SCHEMA_VERSION},
13 registry_collector::{
14 MANIFEST_ACCEPT, MAX_MANIFEST_BYTES, parse_layer, parse_reference, pulled_quantization,
15 },
16 sizing::Architecture,
17};
18use serde::{Deserialize, Serialize};
19use serde_json::{Value, json};
20use std::{
21 collections::{BTreeMap, BTreeSet},
22 io,
23 time::Duration,
24};
25use time::{OffsetDateTime, format_description::well_known::Rfc3339};
26use tokio_util::sync::CancellationToken;
27use url::Url;
28
29pub const LIBRARY_URL: &str = "https://ollama.com/library?sort=newest";
30pub const GENERATION_LISTING_URL: &str =
31 "https://huggingface.co/api/models?author=Comfy-Org&sort=createdAt&direction=-1&limit=500";
32pub const SCOPE_NAME: &str = "ollama-local-variants";
33const GGUF_STEPS: [usize; 3] = [8 * 1024 * 1024, 32 * 1024 * 1024, MAX_GGUF_HEADER_BYTES];
34
35fn name(part: &str) -> bool {
36 !part.is_empty()
37 && part.len() <= 128
38 && part.as_bytes()[0].is_ascii_alphanumeric()
39 && part
40 .bytes()
41 .all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte))
42}
43
44pub fn allowed_request(url: &Url) -> bool {
46 if url.scheme() != "https"
47 || url.port().is_some()
48 || !url.username().is_empty()
49 || url.password().is_some()
50 || url.fragment().is_some()
51 {
52 return false;
53 }
54 let segments: Vec<&str> = url.path().trim_start_matches('/').split('/').collect();
55 match (url.host_str(), url.query(), segments.as_slice()) {
56 (Some("ollama.com"), Some("sort=newest"), ["library"]) => true,
57 (Some("ollama.com"), None, ["library", repo, "tags"]) => name(repo),
58 (Some("registry.ollama.ai"), None, ["v2", "library", repo, "manifests", tag]) => {
59 name(repo) && name(tag)
60 }
61 (Some("registry.ollama.ai"), None, ["v2", "library", repo, "blobs", digest]) => {
62 name(repo)
63 && digest.strip_prefix("sha256:").is_some_and(|hex| {
64 hex.len() == 64 && hex.bytes().all(|byte| byte.is_ascii_hexdigit())
65 })
66 }
67 (Some("huggingface.co"), Some(query), ["api", "models"]) => {
68 Some(query)
69 == GENERATION_LISTING_URL
70 .split_once('?')
71 .map(|(_, query)| query)
72 }
73 (Some("huggingface.co"), None | Some("blobs=true"), ["api", "models", owner, repo]) => {
74 name(owner) && name(repo)
75 }
76 (Some("huggingface.co"), None, ["Comfy-Org", repo, "resolve", revision, path @ ..]) => {
77 name(repo)
78 && revision.len() == 40
79 && revision.bytes().all(|byte| byte.is_ascii_hexdigit())
80 && !path.is_empty()
81 && path.len() <= 8
82 && path.iter().all(|segment| name(segment))
83 && path
84 .last()
85 .is_some_and(|file| file.ends_with(".safetensors"))
86 }
87 _ => false,
88 }
89}
90
91pub fn allowed_hub_redirect(url: &Url) -> bool {
93 url.scheme() == "https"
94 && url.port().is_none()
95 && url.username().is_empty()
96 && url.password().is_none()
97 && url
98 .host_str()
99 .is_some_and(|host| host == "huggingface.co" || host.ends_with(".hf.co"))
100}
101
102pub struct Fetched {
103 pub status: u16,
104 pub body: Vec<u8>,
105}
106
107#[async_trait::async_trait]
108pub trait AdmissionTransport: Send + Sync {
109 async fn fetch(
111 &self,
112 url: &Url,
113 accept: Option<&'static str>,
114 range: Option<usize>,
115 limit: usize,
116 ) -> io::Result<Fetched>;
117}
118
119pub struct NativeAdmissionTransport {
120 client: reqwest::Client,
121}
122impl NativeAdmissionTransport {
123 pub fn new() -> io::Result<Self> {
124 let client = reqwest::Client::builder()
125 .https_only(true)
126 .no_proxy()
127 .user_agent(concat!(
128 "rigspark-catalog-admission/",
129 env!("CARGO_PKG_VERSION")
130 ))
131 .redirect(reqwest::redirect::Policy::custom(|attempt| {
132 let previous = attempt.previous().last();
133 let from_blob = previous.is_some_and(|url| {
134 url.host_str() == Some("registry.ollama.ai")
135 && url.path().contains("/blobs/sha256:")
136 });
137 let from_resolve = previous.is_some_and(|url| {
138 url.host_str() == Some("huggingface.co") && url.path().contains("/resolve/")
139 });
140 let allowed = (from_blob && allowed_registry_redirect(attempt.url().as_str()))
141 || (from_resolve && allowed_hub_redirect(attempt.url()));
142 if attempt.previous().len() > 2 || !allowed {
143 attempt.error("registry redirect refused")
144 } else {
145 attempt.follow()
146 }
147 }))
148 .connect_timeout(Duration::from_secs(10))
149 .timeout(Duration::from_secs(120))
150 .build()
151 .map_err(|_| io::Error::other("admission client initialization failed"))?;
152 Ok(Self { client })
153 }
154}
155#[async_trait::async_trait]
156impl AdmissionTransport for NativeAdmissionTransport {
157 async fn fetch(
158 &self,
159 url: &Url,
160 accept: Option<&'static str>,
161 range: Option<usize>,
162 limit: usize,
163 ) -> io::Result<Fetched> {
164 if !allowed_request(url) {
165 return Err(io::Error::other("admission URL is not allow-listed"));
166 }
167 let mut request = self.client.get(url.clone());
168 if let Some(accept) = accept {
169 request = request.header(reqwest::header::ACCEPT, accept);
170 }
171 if let Some(range) = range {
172 request = request.header(reqwest::header::RANGE, format!("bytes=0-{}", range - 1));
173 }
174 let response = request
175 .send()
176 .await
177 .map_err(|_| io::Error::other("admission request failed"))?;
178 let status = response.status().as_u16();
179 let mut stream = response.bytes_stream();
180 let mut body = Vec::new();
181 while let Some(chunk) = stream.next().await {
182 let chunk = chunk.map_err(|_| io::Error::other("admission body read failed"))?;
183 body.extend_from_slice(&chunk);
184 if body.len() >= limit {
185 if range.is_none() && body.len() > limit {
187 return Err(io::Error::other("admission response exceeds its size cap"));
188 }
189 body.truncate(limit);
190 break;
191 }
192 }
193 Ok(Fetched { status, body })
194 }
195}
196
197#[derive(Deserialize)]
198#[serde(deny_unknown_fields)]
199struct RecordedBody {
200 status: u16,
201 #[serde(default)]
202 text: Option<String>,
203 #[serde(default)]
204 json: Option<Value>,
205 #[serde(default)]
206 hex: Option<String>,
207}
208
209pub struct RecordedAdmissionTransport {
211 responses: BTreeMap<String, (u16, Vec<u8>)>,
212}
213impl RecordedAdmissionTransport {
214 pub fn new(responses: BTreeMap<String, (u16, Vec<u8>)>) -> io::Result<Self> {
215 for key in responses.keys() {
216 let url = Url::parse(key).map_err(io::Error::other)?;
217 if !allowed_request(&url) {
218 return Err(io::Error::other("recorded URL is not allow-listed"));
219 }
220 }
221 Ok(Self { responses })
222 }
223 pub fn parse(raw: &str) -> io::Result<Self> {
224 let recorded: BTreeMap<String, RecordedBody> =
225 serde_json::from_str(raw).map_err(io::Error::other)?;
226 let mut responses = BTreeMap::new();
227 for (url, body) in recorded {
228 let bytes = match (body.text, body.json, body.hex) {
229 (Some(text), None, None) => text.into_bytes(),
230 (None, Some(value), None) => {
231 serde_json::to_vec(&value).map_err(io::Error::other)?
232 }
233 (None, None, Some(hex)) => decode_hex(&hex)?,
234 (None, None, None) => Vec::new(),
235 _ => return Err(io::Error::other("recorded body must have one encoding")),
236 };
237 responses.insert(url, (body.status, bytes));
238 }
239 Self::new(responses)
240 }
241}
242fn decode_hex(hex: &str) -> io::Result<Vec<u8>> {
243 if !hex.len().is_multiple_of(2) {
244 return Err(io::Error::other("invalid recorded hex body"));
245 }
246 (0..hex.len())
247 .step_by(2)
248 .map(|index| {
249 u8::from_str_radix(&hex[index..index + 2], 16)
250 .map_err(|_| io::Error::other("invalid recorded hex body"))
251 })
252 .collect()
253}
254#[async_trait::async_trait]
255impl AdmissionTransport for RecordedAdmissionTransport {
256 async fn fetch(
257 &self,
258 url: &Url,
259 _accept: Option<&'static str>,
260 range: Option<usize>,
261 limit: usize,
262 ) -> io::Result<Fetched> {
263 if !allowed_request(url) {
264 return Err(io::Error::other("admission URL is not allow-listed"));
265 }
266 let (status, body) = self
267 .responses
268 .get(url.as_str())
269 .cloned()
270 .unwrap_or((404, Vec::new()));
271 let cap = range.unwrap_or(usize::MAX).min(limit);
272 if range.is_none() && body.len() > limit {
273 return Err(io::Error::other("admission response exceeds its size cap"));
274 }
275 Ok(Fetched {
276 status,
277 body: body[..body.len().min(cap)].to_vec(),
278 })
279 }
280}
281
282#[derive(Deserialize)]
283#[serde(rename_all = "camelCase")]
284struct ManifestLayers {
285 config: Descriptor,
286 layers: Vec<Descriptor>,
287}
288#[derive(Deserialize)]
289#[serde(rename_all = "camelCase")]
290struct Descriptor {
291 media_type: String,
292 digest: String,
293}
294fn sha256(digest: &str) -> Option<&str> {
295 digest
296 .strip_prefix("sha256:")
297 .filter(|hex| hex.len() == 64 && hex.bytes().all(|byte| byte.is_ascii_hexdigit()))
298}
299
300#[derive(Clone, Debug, Default, Serialize, Deserialize)]
301#[serde(rename_all = "camelCase", deny_unknown_fields)]
302pub struct AdmissionState {
303 #[serde(default)]
306 pub rejections: BTreeMap<String, String>,
307 #[serde(default)]
309 pub facts: BTreeMap<String, SourcedFacts>,
310}
311
312#[derive(Clone, Debug, Serialize, Deserialize)]
313#[serde(rename_all = "camelCase", deny_unknown_fields)]
314pub struct SourcedFacts {
315 pub context_length: u64,
316 pub architecture: Architecture,
317 #[serde(default, skip_serializing_if = "Option::is_none")]
318 pub license: Option<String>,
319 #[serde(default, skip_serializing_if = "Option::is_none")]
320 pub kv_bytes_per_token: Option<f64>,
321}
322
323pub struct AdmissionOptions {
324 pub now: String,
325 pub max_new_variants: usize,
326 pub state: AdmissionState,
327 pub correct_curated: bool,
329}
330
331#[derive(Clone, Debug, Serialize)]
333pub struct Correction {
334 pub id: String,
335 pub field: String,
336 pub from: Value,
337 pub to: Value,
338 pub sources: Vec<String>,
339}
340
341#[derive(Default, Serialize)]
342#[serde(rename_all = "camelCase")]
343pub struct AdmissionOutcome {
344 #[serde(skip)]
345 pub catalog: Option<Catalog>,
346 #[serde(skip)]
347 pub observations: Vec<Value>,
348 #[serde(skip)]
350 pub curated_observations: Vec<Value>,
351 #[serde(skip)]
352 pub scope: Value,
353 #[serde(skip)]
354 pub state: AdmissionState,
355 pub added: Vec<String>,
356 pub updated: Vec<String>,
357 pub reverified: Vec<String>,
358 pub curated_observed: Vec<String>,
359 pub corrections: Vec<Correction>,
360 pub removed: Vec<String>,
361 pub rejected: BTreeMap<String, String>,
362 pub deferred: Vec<String>,
363 pub failures: Vec<String>,
364 pub complete: bool,
365}
366
367struct Run<'a> {
368 transport: &'a dyn AdmissionTransport,
369 cancel: &'a CancellationToken,
370}
371impl Run<'_> {
372 async fn get(
373 &self,
374 url: &str,
375 accept: Option<&'static str>,
376 range: Option<usize>,
377 limit: usize,
378 ) -> io::Result<Vec<u8>> {
379 if self.cancel.is_cancelled() {
380 return Err(io::Error::new(
381 io::ErrorKind::Interrupted,
382 "catalog admission cancelled",
383 ));
384 }
385 let url = Url::parse(url).map_err(io::Error::other)?;
386 let fetched = tokio::select! {
387 biased;
388 _ = self.cancel.cancelled() => {
389 return Err(io::Error::new(io::ErrorKind::Interrupted, "catalog admission cancelled"));
390 }
391 result = tokio::time::timeout(Duration::from_secs(180), self.transport.fetch(&url, accept, range, limit)) => {
392 result.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "admission request timed out"))??
393 }
394 };
395 let ok = if range.is_some() {
396 matches!(fetched.status, 200 | 206)
397 } else {
398 (200..300).contains(&fetched.status)
399 };
400 if !ok {
401 return Err(io::Error::new(
402 io::ErrorKind::NotFound,
403 format!("upstream status {}", fetched.status),
404 ));
405 }
406 Ok(fetched.body)
407 }
408 async fn text(&self, url: &str, limit: usize) -> io::Result<String> {
409 String::from_utf8(self.get(url, None, None, limit).await?)
410 .map_err(|_| io::Error::other("upstream text is not UTF-8"))
411 }
412 async fn gguf(&self, blob: &str) -> io::Result<Result<Gguf, &'static str>> {
414 for step in GGUF_STEPS {
415 let bytes = self.get(blob, None, Some(step), step).await?;
416 match parse_gguf(&bytes) {
417 Ok(parsed) => return Ok(Ok(parsed)),
418 Err(GgufError::Truncated) if bytes.len() >= step => continue,
419 Err(GgufError::Truncated) => return Ok(Err("GGUF header truncated")),
420 Err(GgufError::Invalid(reason)) => return Ok(Err(reason)),
421 }
422 }
423 Ok(Err("GGUF header exceeds 64 MiB"))
424 }
425}
426
427fn interrupted(error: &io::Error) -> bool {
428 error.kind() == io::ErrorKind::Interrupted
429}
430
431pub async fn admit(
434 catalog: &Catalog,
435 transport: &dyn AdmissionTransport,
436 options: &AdmissionOptions,
437 cancel: &CancellationToken,
438) -> io::Result<AdmissionOutcome> {
439 let now = OffsetDateTime::parse(&options.now, &Rfc3339)
440 .map_err(|_| io::Error::other("invalid admission clock"))?
441 .to_offset(time::UtcOffset::UTC);
442 let stamp = now
443 .format(time::macros::format_description!(
444 "[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
445 ))
446 .map_err(io::Error::other)?;
447 let today = stamp[..10].to_string();
448 let run = Run { transport, cancel };
449 let library =
450 parse_library(&run.text(LIBRARY_URL, MAX_HTML_BYTES).await?).map_err(io::Error::other)?;
451
452 let mut outcome = AdmissionOutcome {
453 complete: true,
454 state: options.state.clone(),
455 ..Default::default()
456 };
457 let curated: BTreeMap<&str, &CatalogModel> = catalog
458 .models
459 .iter()
460 .filter(|model| model.provenance == EntryProvenance::Curated)
461 .map(|model| (model.id.as_str(), model))
462 .collect();
463 let curated_digests: BTreeSet<&str> = curated
464 .values()
465 .flat_map(|model| model.quantizations.iter())
466 .filter_map(|quant| quant.sha256.as_deref())
467 .collect();
468 let mut auto: BTreeMap<String, CatalogModel> = catalog
469 .models
470 .iter()
471 .filter(|model| model.provenance == EntryProvenance::Auto)
472 .map(|model| (model.id.clone(), model.clone()))
473 .collect();
474 let listed: BTreeSet<&str> = library
475 .iter()
476 .map(|entry| entry.repository.as_str())
477 .collect();
478 for id in auto.keys().cloned().collect::<Vec<_>>() {
479 if !listed.contains(parse_reference(&id).0) {
480 auto.remove(&id);
481 outcome.removed.push(id);
482 }
483 }
484 let mut eligible: BTreeSet<String> = BTreeSet::new();
485 let mut seen_digests: BTreeSet<String> = BTreeSet::new();
486 let mut new_variants = 0;
487
488 for entry in &library {
489 let repo = entry.repository.as_str();
490 let tags_url = format!("https://ollama.com/library/{repo}/tags");
491 let selected = match run.text(&tags_url, MAX_HTML_BYTES).await {
492 Ok(html) => match parse_tags(&html, repo) {
493 Ok(tags) => select_variants(&tags),
494 Err(_) => {
495 outcome
496 .failures
497 .push(format!("{repo}: tags page unparseable"));
498 outcome.complete = false;
499 continue;
500 }
501 },
502 Err(error) if interrupted(&error) => return Err(error),
503 Err(_) => {
504 outcome
505 .failures
506 .push(format!("{repo}: tags page unavailable"));
507 outcome.complete = false;
508 continue;
509 }
510 };
511 let selected_ids: BTreeSet<String> =
512 selected.iter().map(|tag| format!("{repo}:{tag}")).collect();
513 for id in auto.keys().cloned().collect::<Vec<_>>() {
514 if parse_reference(&id).0 == repo && !selected_ids.contains(&id) {
515 auto.remove(&id);
516 outcome.removed.push(id);
517 }
518 }
519 for tag in &selected {
520 let id = format!("{repo}:{tag}");
521 if curated.contains_key(id.as_str()) {
522 continue;
523 }
524 match admit_variant(
525 &run,
526 entry,
527 tag,
528 &id,
529 &auto,
530 &curated_digests,
531 &mut seen_digests,
532 &today,
533 &mut new_variants,
534 options,
535 &mut outcome,
536 )
537 .await
538 {
539 Ok(Some(model)) => {
540 eligible.insert(id.clone());
541 auto.insert(id, model);
542 }
543 Ok(None) => {}
544 Err(error) if interrupted(&error) => return Err(error),
545 Err(error) => {
546 outcome.failures.push(format!("{id}: {error}"));
547 outcome.complete = false;
548 if auto.contains_key(&id) {
549 eligible.insert(id);
550 }
551 }
552 }
553 }
554 }
555 for id in &outcome.deferred {
556 eligible.insert(id.clone());
557 }
558 let mut corrected: BTreeMap<String, CatalogModel> = BTreeMap::new();
559 for model in curated.values() {
560 match reobserve(&run, model, &stamp, &mut outcome.state).await {
561 Ok(Some(observed)) => {
562 if options.correct_curated
563 && let Some((fixed, changes)) = correct(model, &observed)?
564 {
565 corrected.insert(model.id.clone(), fixed);
566 outcome.corrections.extend(changes);
567 }
568 outcome.curated_observed.push(model.id.clone());
569 outcome.curated_observations.push(observed);
570 }
571 Ok(None) => {}
572 Err(error) if interrupted(&error) => return Err(error),
573 Err(error) => outcome
574 .failures
575 .push(format!("{} (curated): {error}", model.id)),
576 }
577 }
578
579 let mut models: Vec<CatalogModel> = catalog
580 .models
581 .iter()
582 .filter(|model| model.provenance == EntryProvenance::Curated)
583 .map(|model| corrected.remove(&model.id).unwrap_or_else(|| model.clone()))
584 .chain(auto.into_values())
585 .collect();
586 models.sort_by(|left, right| {
587 right
588 .recency()
589 .map(|(day, _)| day)
590 .cmp(&left.recency().map(|(day, _)| day))
591 .then_with(|| left.id.cmp(&right.id))
592 });
593 outcome.observations = models
594 .iter()
595 .filter(|model| model.provenance == EntryProvenance::Auto)
596 .map(|model| observation(model, &stamp))
597 .collect();
598 let mut variants: BTreeSet<String> = models.iter().map(|model| model.id.clone()).collect();
599 variants.extend(eligible);
600 outcome.scope = json!({
601 "name": SCOPE_NAME,
602 "checkedAt": stamp,
603 "source": "https://ollama.com/library",
604 "complete": outcome.complete,
605 "variants": variants,
606 });
607 let changed =
608 !outcome.added.is_empty() || !outcome.updated.is_empty() || !outcome.removed.is_empty();
609 let encoded = json!({
610 "schemaVersion": SCHEMA_VERSION,
611 "generatedAt": if changed { stamp.clone() } else { catalog.generated_at.clone() },
612 "models": models,
613 });
614 outcome.catalog = Some(Catalog::parse(&encoded.to_string()).map_err(io::Error::other)?);
615 outcome.removed.sort();
616 Ok(outcome)
617}
618
619#[allow(clippy::too_many_arguments)]
620async fn admit_variant(
621 run: &Run<'_>,
622 entry: &LibraryEntry,
623 tag: &str,
624 id: &str,
625 auto: &BTreeMap<String, CatalogModel>,
626 curated_digests: &BTreeSet<&str>,
627 seen_digests: &mut BTreeSet<String>,
628 today: &str,
629 new_variants: &mut usize,
630 options: &AdmissionOptions,
631 outcome: &mut AdmissionOutcome,
632) -> io::Result<Option<CatalogModel>> {
633 let repo = entry.repository.as_str();
634 let blob =
635 |digest: &str| format!("https://registry.ollama.ai/v2/library/{repo}/blobs/{digest}");
636 let manifest_raw = run
637 .get(
638 &format!("https://registry.ollama.ai/v2/library/{repo}/manifests/{tag}"),
639 Some(MANIFEST_ACCEPT),
640 None,
641 MAX_MANIFEST_BYTES,
642 )
643 .await?;
644 let manifest_text = std::str::from_utf8(&manifest_raw)
645 .map_err(|_| io::Error::other("manifest is not UTF-8"))?;
646 let Some(layer) = parse_layer(manifest_text).map_err(io::Error::other)? else {
647 outcome
648 .rejected
649 .insert(id.into(), "no GGUF model layer".into());
650 return Ok(None);
651 };
652 let descriptors: ManifestLayers =
653 serde_json::from_str(manifest_text).map_err(io::Error::other)?;
654 if curated_digests.contains(layer.sha256.as_str()) || !seen_digests.insert(layer.sha256.clone())
655 {
656 return Ok(None);
657 }
658 let license_digest = descriptors
659 .layers
660 .iter()
661 .find(|layer| layer.media_type == "application/vnd.ollama.image.license")
662 .and_then(|layer| sha256(&layer.digest))
663 .map(str::to_string);
664 let cache_key = format!(
665 "{}:{}",
666 layer.sha256,
667 license_digest.as_deref().unwrap_or("none")
668 );
669 let prior = auto.get(id);
670 if let Some(prior) = prior
671 && prior.quantizations[0].sha256.as_deref() == Some(layer.sha256.as_str())
672 && prior.quantizations[0].disk_bytes == layer.disk_bytes
673 && !outcome.state.rejections.contains_key(&cache_key)
674 {
675 let mut refreshed = prior.clone();
676 let capabilities = catalog_capabilities(&entry.capabilities);
677 if capabilities != prior.capabilities {
678 refreshed.capabilities = capabilities;
679 outcome.updated.push(id.into());
680 } else {
681 outcome.reverified.push(id.into());
682 }
683 return Ok(Some(refreshed));
684 }
685 if let Some(reason) = outcome.state.rejections.get(&cache_key) {
686 outcome.rejected.insert(id.into(), reason.clone());
687 return Ok(None);
688 }
689 if prior.is_none() {
690 if *new_variants >= options.max_new_variants {
691 outcome.deferred.push(id.into());
692 outcome.complete = false;
693 return Ok(None);
694 }
695 *new_variants += 1;
696 }
697 let config_digest = sha256(&descriptors.config.digest)
698 .ok_or_else(|| io::Error::other("invalid config digest"))?;
699 let config_raw = run
700 .get(
701 &blob(&format!("sha256:{config_digest}")),
702 None,
703 None,
704 MAX_CONFIG_BYTES,
705 )
706 .await?;
707 let config = RegistryConfig::parse(
708 std::str::from_utf8(&config_raw).map_err(|_| io::Error::other("config is not UTF-8"))?,
709 );
710 let license_head = match &license_digest {
711 Some(digest) => Some(
712 String::from_utf8_lossy(
713 &run.get(
714 &blob(&format!("sha256:{digest}")),
715 None,
716 Some(MAX_LICENSE_HEAD_BYTES),
717 MAX_LICENSE_HEAD_BYTES,
718 )
719 .await?,
720 )
721 .into_owned(),
722 ),
723 None => None,
724 };
725 let reject = |outcome: &mut AdmissionOutcome, reason: &str| {
726 outcome.rejected.insert(id.into(), reason.into());
727 outcome
728 .state
729 .rejections
730 .insert(cache_key.clone(), reason.into());
731 Ok(None)
732 };
733 let Ok(config) = config else {
734 return reject(outcome, "model config lacks family, size or file type");
735 };
736 let gguf = match run.gguf(&blob(&format!("sha256:{}", layer.sha256))).await? {
737 Ok(gguf) => gguf,
738 Err(reason) => return reject(outcome, reason),
739 };
740 let built = build_entry(&AdmissionInput {
741 repository: repo,
742 tag,
743 capabilities: &entry.capabilities,
744 layer: &layer,
745 config: &config,
746 gguf: &gguf,
747 license_head: license_head.as_deref(),
748 today: prior
749 .and_then(|model| model.added_at.as_deref())
750 .unwrap_or(today),
751 });
752 match built {
753 Ok(model) => {
754 outcome.state.rejections.remove(&cache_key);
755 if prior.is_some() {
756 outcome.updated.push(id.into());
757 } else {
758 outcome.added.push(id.into());
759 }
760 Ok(Some(model))
761 }
762 Err(rejection) => reject(outcome, rejection.reason()),
763 }
764}
765
766async fn reobserve(
769 run: &Run<'_>,
770 model: &CatalogModel,
771 stamp: &str,
772 state: &mut AdmissionState,
773) -> io::Result<Option<Value>> {
774 let Some(reference) = model.source.ollama.as_deref() else {
775 return Ok(None);
776 };
777 let (path, tag) = parse_reference(reference);
778 let repo = path.strip_prefix("library/").unwrap_or(path);
779 if repo.contains('/') || !name(repo) || !name(tag) {
780 return Ok(None);
781 }
782 let Some(index) = pulled_quantization(model) else {
783 return Ok(None);
784 };
785 let manifest_url = format!("https://registry.ollama.ai/v2/library/{repo}/manifests/{tag}");
786 let manifest_raw = run
787 .get(
788 &manifest_url,
789 Some(MANIFEST_ACCEPT),
790 None,
791 MAX_MANIFEST_BYTES,
792 )
793 .await?;
794 let manifest_text = std::str::from_utf8(&manifest_raw)
795 .map_err(|_| io::Error::other("manifest is not UTF-8"))?;
796 let Some(layer) = parse_layer(manifest_text).map_err(io::Error::other)? else {
797 return Ok(None);
798 };
799 let descriptors: ManifestLayers =
800 serde_json::from_str(manifest_text).map_err(io::Error::other)?;
801 let blob = |digest: &str| {
802 format!("https://registry.ollama.ai/v2/library/{repo}/blobs/sha256:{digest}")
803 };
804 let config_digest = sha256(&descriptors.config.digest)
805 .ok_or_else(|| io::Error::other("invalid config digest"))?;
806 let config_raw = run
807 .get(&blob(config_digest), None, None, MAX_CONFIG_BYTES)
808 .await?;
809 let Ok(config) = RegistryConfig::parse(
810 std::str::from_utf8(&config_raw).map_err(|_| io::Error::other("config is not UTF-8"))?,
811 ) else {
812 return Ok(None);
813 };
814 let license_digest = descriptors
815 .layers
816 .iter()
817 .find(|layer| layer.media_type == "application/vnd.ollama.image.license")
818 .and_then(|layer| sha256(&layer.digest))
819 .map(str::to_string);
820 let key = format!(
821 "{}:{}",
822 layer.sha256,
823 license_digest.as_deref().unwrap_or("none")
824 );
825 let sourced = match state.facts.get(&key) {
826 Some(sourced) => sourced.clone(),
827 None => {
828 let Ok(gguf) = run.gguf(&blob(&layer.sha256)).await? else {
829 return Ok(None);
830 };
831 let Some(arch) = gguf.architecture().map(str::to_string) else {
832 return Ok(None);
833 };
834 let Some(context_length) = gguf.u64(&format!("{arch}.context_length")) else {
835 return Ok(None);
836 };
837 let head = match &license_digest {
838 Some(digest) => Some(
839 String::from_utf8_lossy(
840 &run.get(
841 &blob(digest),
842 None,
843 Some(MAX_LICENSE_HEAD_BYTES),
844 MAX_LICENSE_HEAD_BYTES,
845 )
846 .await?,
847 )
848 .into_owned(),
849 ),
850 None => None,
851 };
852 let sourced = SourcedFacts {
853 context_length,
854 architecture: if gguf
855 .u64(&format!("{arch}.expert_count"))
856 .is_some_and(|count| count > 0)
857 {
858 Architecture::Moe
859 } else {
860 Architecture::Dense
861 },
862 license: license_id(
863 gguf.text("general.license"),
864 gguf.text("general.license.name"),
865 head.as_deref(),
866 )
867 .ok()
868 .map(str::to_string),
869 kv_bytes_per_token: kv_bytes_per_token(&gguf),
870 };
871 state.facts.insert(key, sourced.clone());
872 sourced
873 }
874 };
875 let mut quant = serde_json::to_value(&model.quantizations[index]).map_err(io::Error::other)?;
876 let object = quant
877 .as_object_mut()
878 .ok_or_else(|| io::Error::other("invalid quantization"))?;
879 for field in ["minRamBytes", "minVramBytes", "digestVerified"] {
880 object.remove(field);
881 }
882 object.insert("name".into(), json!(config.file_type.to_ascii_uppercase()));
883 object.insert("diskBytes".into(), json!(layer.disk_bytes));
884 object.insert("sha256".into(), json!(layer.sha256));
885 if layer.projectors.is_empty() {
886 object.remove("projectors");
887 } else {
888 object.insert(
889 "projectors".into(),
890 serde_json::to_value(&layer.projectors).map_err(io::Error::other)?,
891 );
892 }
893 let mut facts = serde_json::Map::new();
894 facts.insert("defaultQuantization".into(), quant);
895 facts.insert("contextLength".into(), json!(sourced.context_length as f64));
896 facts.insert(
897 "architecture".into(),
898 serde_json::to_value(&sourced.architecture).map_err(io::Error::other)?,
899 );
900 if let Some(license) = sourced.license {
901 facts.insert("license".into(), json!(license));
902 }
903 if let (Some(rate), Some(_)) = (sourced.kv_bytes_per_token, model.kv_bytes_per_token) {
905 facts.insert("kvBytesPerToken".into(), json!(rate));
906 }
907 Ok(Some(json!({
908 "id": model.id,
909 "checkedAt": stamp,
910 "sources": [manifest_url, format!("https://ollama.com/library/{repo}:{tag}")],
911 "facts": facts,
912 })))
913}
914
915fn catalog_value(model: &CatalogModel, field: &str) -> io::Result<Value> {
917 if field == "defaultQuantization" {
918 let index = pulled_quantization(model).unwrap_or(0);
919 let mut quant =
920 serde_json::to_value(&model.quantizations[index]).map_err(io::Error::other)?;
921 if let Some(object) = quant.as_object_mut() {
922 for key in ["minRamBytes", "minVramBytes", "digestVerified"] {
923 object.remove(key);
924 }
925 }
926 return Ok(quant);
927 }
928 Ok(serde_json::to_value(model)
929 .map_err(io::Error::other)?
930 .get(field)
931 .cloned()
932 .unwrap_or(Value::Null))
933}
934
935fn correct(
938 model: &CatalogModel,
939 observed: &Value,
940) -> io::Result<Option<(CatalogModel, Vec<Correction>)>> {
941 let Some(facts) = observed["facts"].as_object() else {
942 return Ok(None);
943 };
944 let sources: Vec<String> = observed["sources"]
945 .as_array()
946 .map(|items| {
947 items
948 .iter()
949 .filter_map(|item| item.as_str().map(str::to_string))
950 .collect()
951 })
952 .unwrap_or_default();
953 let mut fixed = model.clone();
954 let mut changes = Vec::new();
955 for (field, to) in facts {
956 let from = catalog_value(model, field)?;
957 if from == *to || field == "architecture" {
958 continue;
959 }
960 match field.as_str() {
961 "contextLength" => {
962 fixed.context_length = to
963 .as_f64()
964 .ok_or_else(|| io::Error::other("invalid context"))?
965 }
966 "license" => {
967 fixed.license = to
968 .as_str()
969 .ok_or_else(|| io::Error::other("invalid license"))?
970 .into()
971 }
972 "kvBytesPerToken" => fixed.kv_bytes_per_token = to.as_f64(),
973 "defaultQuantization" => {
974 let index = pulled_quantization(model).unwrap_or(0);
975 let name = to["name"]
976 .as_str()
977 .ok_or_else(|| io::Error::other("invalid quant"))?;
978 let projectors: Vec<rigspark_core::sizing::ProjectorArtifact> =
979 serde_json::from_value(to.get("projectors").cloned().unwrap_or(json!([])))
980 .map_err(io::Error::other)?;
981 let quant = rigspark_core::enrich::sized_quantization(
982 rigspark_core::sizing::parse_param_count(&model.params)
983 .map_err(io::Error::other)?,
984 &model.architecture,
985 name,
986 to["diskBytes"]
987 .as_f64()
988 .ok_or_else(|| io::Error::other("invalid size"))?,
989 to["sha256"].as_str().map(str::to_string),
990 projectors,
991 )
992 .map_err(io::Error::other)?;
993 fixed.quantizations[index] = quant;
994 let mut position = 0;
996 fixed.quantizations.retain(|candidate| {
997 let keep = position == index || candidate.name != name;
998 position += 1;
999 keep
1000 });
1001 }
1002 _ => continue,
1003 }
1004 changes.push(Correction {
1005 id: model.id.clone(),
1006 field: field.clone(),
1007 from,
1008 to: to.clone(),
1009 sources: sources.clone(),
1010 });
1011 }
1012 if changes.is_empty() {
1013 return Ok(None);
1014 }
1015 let check = json!({"schemaVersion": SCHEMA_VERSION, "generatedAt": "2026-01-01T00:00:00.000Z", "models": [fixed]});
1016 Catalog::parse(&check.to_string()).map_err(io::Error::other)?;
1017 Ok(Some((fixed, changes)))
1018}