1use crate::catalog_proposals::allowed_registry_redirect;
4use futures_util::StreamExt;
5use rigspark_core::{
6 admission::{
7 AdmissionInput, Gguf, GgufError, LibraryEntry, MAX_CONFIG_BYTES, MAX_GGUF_HEADER_BYTES,
8 MAX_HTML_BYTES, MAX_LICENSE_HEAD_BYTES, RegistryConfig, build_entry, catalog_capabilities,
9 kv_bytes_per_token, license_id, observation, parse_gguf, parse_library, parse_tags,
10 select_variants,
11 },
12 catalog::{Catalog, CatalogModel, EntryProvenance, SCHEMA_VERSION},
13 registry_collector::{
14 MANIFEST_ACCEPT, MAX_MANIFEST_BYTES, parse_layer, parse_reference, pulled_quantization,
15 },
16 sizing::Architecture,
17};
18use serde::{Deserialize, Serialize};
19use serde_json::{Value, json};
20use std::{
21 collections::{BTreeMap, BTreeSet},
22 io,
23 time::Duration,
24};
25use time::{OffsetDateTime, format_description::well_known::Rfc3339};
26use tokio_util::sync::CancellationToken;
27use url::Url;
28
29pub const LIBRARY_URL: &str = "https://ollama.com/library?sort=newest";
30pub const GENERATION_LISTING_URL: &str =
31 "https://huggingface.co/api/models?author=Comfy-Org&sort=createdAt&direction=-1&limit=500";
32pub const SCOPE_NAME: &str = "ollama-local-variants";
33const GGUF_STEPS: [usize; 3] = [8 * 1024 * 1024, 32 * 1024 * 1024, MAX_GGUF_HEADER_BYTES];
34
35fn name(part: &str) -> bool {
36 !part.is_empty()
37 && part.len() <= 128
38 && part.as_bytes()[0].is_ascii_alphanumeric()
39 && part
40 .bytes()
41 .all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte))
42}
43
44pub fn allowed_request(url: &Url) -> bool {
46 if url.scheme() != "https"
47 || url.port().is_some()
48 || !url.username().is_empty()
49 || url.password().is_some()
50 || url.fragment().is_some()
51 {
52 return false;
53 }
54 let segments: Vec<&str> = url.path().trim_start_matches('/').split('/').collect();
55 match (url.host_str(), url.query(), segments.as_slice()) {
56 (Some("ollama.com"), Some("sort=newest"), ["library"]) => true,
57 (Some("ollama.com"), None, ["library", repo, "tags"]) => name(repo),
58 (Some("registry.ollama.ai"), None, ["v2", "library", repo, "manifests", tag]) => {
59 name(repo) && name(tag)
60 }
61 (Some("registry.ollama.ai"), None, ["v2", "library", repo, "blobs", digest]) => {
62 name(repo)
63 && digest.strip_prefix("sha256:").is_some_and(|hex| {
64 hex.len() == 64 && hex.bytes().all(|byte| byte.is_ascii_hexdigit())
65 })
66 }
67 (Some("huggingface.co"), Some(query), ["api", "models"]) => {
68 Some(query)
69 == GENERATION_LISTING_URL
70 .split_once('?')
71 .map(|(_, query)| query)
72 || crate::hf_add::is_search_url(url)
73 }
74 (Some("huggingface.co"), None | Some("blobs=true"), ["api", "models", owner, repo]) => {
75 name(owner) && name(repo)
76 }
77 (
78 Some("huggingface.co"),
79 Some("blobs=true"),
80 ["api", "models", owner, repo, "revision", revision],
81 ) => {
82 name(owner)
83 && name(repo)
84 && revision.len() == 40
85 && revision.bytes().all(|byte| byte.is_ascii_hexdigit())
86 }
87 (Some("huggingface.co"), None, [owner, repo, "resolve", revision, path @ ..]) => {
88 name(owner)
89 && name(repo)
90 && revision.len() == 40
91 && revision.bytes().all(|byte| byte.is_ascii_hexdigit())
92 && !path.is_empty()
93 && path.len() <= 8
94 && path.iter().all(|segment| name(segment))
95 && path.last().is_some_and(|file| {
96 file.ends_with(".gguf")
97 || (*owner == "Comfy-Org" && file.ends_with(".safetensors"))
98 })
99 }
100 _ => false,
101 }
102}
103
104pub fn allowed_hub_redirect(url: &Url) -> bool {
106 url.scheme() == "https"
107 && url.port().is_none()
108 && url.username().is_empty()
109 && url.password().is_none()
110 && url
111 .host_str()
112 .is_some_and(|host| host == "huggingface.co" || host.ends_with(".hf.co"))
113}
114
115pub struct Fetched {
116 pub status: u16,
117 pub body: Vec<u8>,
118}
119
120#[async_trait::async_trait]
121pub trait AdmissionTransport: Send + Sync {
122 async fn fetch(
124 &self,
125 url: &Url,
126 accept: Option<&'static str>,
127 range: Option<usize>,
128 limit: usize,
129 ) -> io::Result<Fetched>;
130}
131
132pub struct NativeAdmissionTransport {
133 client: reqwest::Client,
134}
135impl NativeAdmissionTransport {
136 pub fn new() -> io::Result<Self> {
137 let client = reqwest::Client::builder()
138 .https_only(true)
139 .no_proxy()
140 .user_agent(concat!(
141 "rigspark-catalog-admission/",
142 env!("CARGO_PKG_VERSION")
143 ))
144 .redirect(reqwest::redirect::Policy::custom(|attempt| {
145 let previous = attempt.previous().last();
146 let from_blob = previous.is_some_and(|url| {
147 url.host_str() == Some("registry.ollama.ai")
148 && url.path().contains("/blobs/sha256:")
149 });
150 let from_resolve = previous.is_some_and(|url| {
151 url.host_str() == Some("huggingface.co") && url.path().contains("/resolve/")
152 });
153 let allowed = (from_blob && allowed_registry_redirect(attempt.url().as_str()))
154 || (from_resolve && allowed_hub_redirect(attempt.url()));
155 if attempt.previous().len() > 2 || !allowed {
156 attempt.error("registry redirect refused")
157 } else {
158 attempt.follow()
159 }
160 }))
161 .connect_timeout(Duration::from_secs(10))
162 .timeout(Duration::from_secs(120))
163 .build()
164 .map_err(|_| io::Error::other("admission client initialization failed"))?;
165 Ok(Self { client })
166 }
167}
168#[async_trait::async_trait]
169impl AdmissionTransport for NativeAdmissionTransport {
170 async fn fetch(
171 &self,
172 url: &Url,
173 accept: Option<&'static str>,
174 range: Option<usize>,
175 limit: usize,
176 ) -> io::Result<Fetched> {
177 if !allowed_request(url) {
178 return Err(io::Error::other("admission URL is not allow-listed"));
179 }
180 let mut request = self.client.get(url.clone());
181 if let Some(accept) = accept {
182 request = request.header(reqwest::header::ACCEPT, accept);
183 }
184 if let Some(range) = range {
185 request = request.header(reqwest::header::RANGE, format!("bytes=0-{}", range - 1));
186 }
187 let response = request
188 .send()
189 .await
190 .map_err(|_| io::Error::other("admission request failed"))?;
191 let status = response.status().as_u16();
192 let mut stream = response.bytes_stream();
193 let mut body = Vec::new();
194 while let Some(chunk) = stream.next().await {
195 let chunk = chunk.map_err(|_| io::Error::other("admission body read failed"))?;
196 body.extend_from_slice(&chunk);
197 if body.len() >= limit {
198 if range.is_none() && body.len() > limit {
200 return Err(io::Error::other("admission response exceeds its size cap"));
201 }
202 body.truncate(limit);
203 break;
204 }
205 }
206 Ok(Fetched { status, body })
207 }
208}
209
210#[derive(Deserialize)]
211#[serde(deny_unknown_fields)]
212struct RecordedBody {
213 status: u16,
214 #[serde(default)]
215 text: Option<String>,
216 #[serde(default)]
217 json: Option<Value>,
218 #[serde(default)]
219 hex: Option<String>,
220}
221
222pub struct RecordedAdmissionTransport {
224 responses: BTreeMap<String, (u16, Vec<u8>)>,
225}
226impl RecordedAdmissionTransport {
227 pub fn new(responses: BTreeMap<String, (u16, Vec<u8>)>) -> io::Result<Self> {
228 for key in responses.keys() {
229 let url = Url::parse(key).map_err(io::Error::other)?;
230 if !allowed_request(&url) {
231 return Err(io::Error::other("recorded URL is not allow-listed"));
232 }
233 }
234 Ok(Self { responses })
235 }
236 pub fn parse(raw: &str) -> io::Result<Self> {
237 let recorded: BTreeMap<String, RecordedBody> =
238 serde_json::from_str(raw).map_err(io::Error::other)?;
239 let mut responses = BTreeMap::new();
240 for (url, body) in recorded {
241 let bytes = match (body.text, body.json, body.hex) {
242 (Some(text), None, None) => text.into_bytes(),
243 (None, Some(value), None) => {
244 serde_json::to_vec(&value).map_err(io::Error::other)?
245 }
246 (None, None, Some(hex)) => decode_hex(&hex)?,
247 (None, None, None) => Vec::new(),
248 _ => return Err(io::Error::other("recorded body must have one encoding")),
249 };
250 responses.insert(url, (body.status, bytes));
251 }
252 Self::new(responses)
253 }
254}
255fn decode_hex(hex: &str) -> io::Result<Vec<u8>> {
256 if !hex.len().is_multiple_of(2) {
257 return Err(io::Error::other("invalid recorded hex body"));
258 }
259 (0..hex.len())
260 .step_by(2)
261 .map(|index| {
262 u8::from_str_radix(&hex[index..index + 2], 16)
263 .map_err(|_| io::Error::other("invalid recorded hex body"))
264 })
265 .collect()
266}
267#[async_trait::async_trait]
268impl AdmissionTransport for RecordedAdmissionTransport {
269 async fn fetch(
270 &self,
271 url: &Url,
272 _accept: Option<&'static str>,
273 range: Option<usize>,
274 limit: usize,
275 ) -> io::Result<Fetched> {
276 if !allowed_request(url) {
277 return Err(io::Error::other("admission URL is not allow-listed"));
278 }
279 let (status, body) = self
280 .responses
281 .get(url.as_str())
282 .cloned()
283 .unwrap_or((404, Vec::new()));
284 let cap = range.unwrap_or(usize::MAX).min(limit);
285 if range.is_none() && body.len() > limit {
286 return Err(io::Error::other("admission response exceeds its size cap"));
287 }
288 Ok(Fetched {
289 status,
290 body: body[..body.len().min(cap)].to_vec(),
291 })
292 }
293}
294
295#[derive(Deserialize)]
296#[serde(rename_all = "camelCase")]
297struct ManifestLayers {
298 config: Descriptor,
299 layers: Vec<Descriptor>,
300}
301#[derive(Deserialize)]
302#[serde(rename_all = "camelCase")]
303struct Descriptor {
304 media_type: String,
305 digest: String,
306}
307fn sha256(digest: &str) -> Option<&str> {
308 digest
309 .strip_prefix("sha256:")
310 .filter(|hex| hex.len() == 64 && hex.bytes().all(|byte| byte.is_ascii_hexdigit()))
311}
312
313#[derive(Clone, Debug, Default, Serialize, Deserialize)]
314#[serde(rename_all = "camelCase", deny_unknown_fields)]
315pub struct AdmissionState {
316 #[serde(default)]
319 pub rejections: BTreeMap<String, String>,
320 #[serde(default)]
322 pub facts: BTreeMap<String, SourcedFacts>,
323}
324
325#[derive(Clone, Debug, Serialize, Deserialize)]
326#[serde(rename_all = "camelCase", deny_unknown_fields)]
327pub struct SourcedFacts {
328 pub context_length: u64,
329 pub architecture: Architecture,
330 #[serde(default, skip_serializing_if = "Option::is_none")]
331 pub license: Option<String>,
332 #[serde(default, skip_serializing_if = "Option::is_none")]
333 pub kv_bytes_per_token: Option<f64>,
334}
335
336pub struct AdmissionOptions {
337 pub now: String,
338 pub max_new_variants: usize,
339 pub state: AdmissionState,
340 pub correct_curated: bool,
342}
343
344#[derive(Clone, Debug, Serialize)]
346pub struct Correction {
347 pub id: String,
348 pub field: String,
349 pub from: Value,
350 pub to: Value,
351 pub sources: Vec<String>,
352}
353
354#[derive(Default, Serialize)]
355#[serde(rename_all = "camelCase")]
356pub struct AdmissionOutcome {
357 #[serde(skip)]
358 pub catalog: Option<Catalog>,
359 #[serde(skip)]
360 pub observations: Vec<Value>,
361 #[serde(skip)]
363 pub curated_observations: Vec<Value>,
364 #[serde(skip)]
365 pub scope: Value,
366 #[serde(skip)]
367 pub state: AdmissionState,
368 pub added: Vec<String>,
369 pub updated: Vec<String>,
370 pub reverified: Vec<String>,
371 pub curated_observed: Vec<String>,
372 pub corrections: Vec<Correction>,
373 pub removed: Vec<String>,
374 pub rejected: BTreeMap<String, String>,
375 pub deferred: Vec<String>,
376 pub failures: Vec<String>,
377 pub complete: bool,
378}
379
380struct Run<'a> {
381 transport: &'a dyn AdmissionTransport,
382 cancel: &'a CancellationToken,
383}
384impl Run<'_> {
385 async fn get(
386 &self,
387 url: &str,
388 accept: Option<&'static str>,
389 range: Option<usize>,
390 limit: usize,
391 ) -> io::Result<Vec<u8>> {
392 if self.cancel.is_cancelled() {
393 return Err(io::Error::new(
394 io::ErrorKind::Interrupted,
395 "catalog admission cancelled",
396 ));
397 }
398 let url = Url::parse(url).map_err(io::Error::other)?;
399 let fetched = tokio::select! {
400 biased;
401 _ = self.cancel.cancelled() => {
402 return Err(io::Error::new(io::ErrorKind::Interrupted, "catalog admission cancelled"));
403 }
404 result = tokio::time::timeout(Duration::from_secs(180), self.transport.fetch(&url, accept, range, limit)) => {
405 result.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "admission request timed out"))??
406 }
407 };
408 let ok = if range.is_some() {
409 matches!(fetched.status, 200 | 206)
410 } else {
411 (200..300).contains(&fetched.status)
412 };
413 if !ok {
414 return Err(io::Error::new(
415 io::ErrorKind::NotFound,
416 format!("upstream status {}", fetched.status),
417 ));
418 }
419 Ok(fetched.body)
420 }
421 async fn text(&self, url: &str, limit: usize) -> io::Result<String> {
422 String::from_utf8(self.get(url, None, None, limit).await?)
423 .map_err(|_| io::Error::other("upstream text is not UTF-8"))
424 }
425 async fn gguf(&self, blob: &str) -> io::Result<Result<Gguf, &'static str>> {
427 for step in GGUF_STEPS {
428 let bytes = self.get(blob, None, Some(step), step).await?;
429 match parse_gguf(&bytes) {
430 Ok(parsed) => return Ok(Ok(parsed)),
431 Err(GgufError::Truncated) if bytes.len() >= step => continue,
432 Err(GgufError::Truncated) => return Ok(Err("GGUF header truncated")),
433 Err(GgufError::Invalid(reason)) => return Ok(Err(reason)),
434 }
435 }
436 Ok(Err("GGUF header exceeds 64 MiB"))
437 }
438}
439
440fn interrupted(error: &io::Error) -> bool {
441 error.kind() == io::ErrorKind::Interrupted
442}
443
444pub async fn admit(
447 catalog: &Catalog,
448 transport: &dyn AdmissionTransport,
449 options: &AdmissionOptions,
450 cancel: &CancellationToken,
451) -> io::Result<AdmissionOutcome> {
452 let now = OffsetDateTime::parse(&options.now, &Rfc3339)
453 .map_err(|_| io::Error::other("invalid admission clock"))?
454 .to_offset(time::UtcOffset::UTC);
455 let stamp = now
456 .format(time::macros::format_description!(
457 "[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
458 ))
459 .map_err(io::Error::other)?;
460 let today = stamp[..10].to_string();
461 let run = Run { transport, cancel };
462 let library =
463 parse_library(&run.text(LIBRARY_URL, MAX_HTML_BYTES).await?).map_err(io::Error::other)?;
464
465 let mut outcome = AdmissionOutcome {
466 complete: true,
467 state: options.state.clone(),
468 ..Default::default()
469 };
470 let curated: BTreeMap<&str, &CatalogModel> = catalog
471 .models
472 .iter()
473 .filter(|model| model.provenance == EntryProvenance::Curated)
474 .map(|model| (model.id.as_str(), model))
475 .collect();
476 let curated_digests: BTreeSet<&str> = curated
477 .values()
478 .flat_map(|model| model.quantizations.iter())
479 .filter_map(|quant| quant.sha256.as_deref())
480 .collect();
481 let mut auto: BTreeMap<String, CatalogModel> = catalog
482 .models
483 .iter()
484 .filter(|model| model.provenance == EntryProvenance::Auto)
485 .map(|model| (model.id.clone(), model.clone()))
486 .collect();
487 let listed: BTreeSet<&str> = library
488 .iter()
489 .map(|entry| entry.repository.as_str())
490 .collect();
491 for id in auto.keys().cloned().collect::<Vec<_>>() {
492 if !listed.contains(parse_reference(&id).0) {
493 auto.remove(&id);
494 outcome.removed.push(id);
495 }
496 }
497 let mut eligible: BTreeSet<String> = BTreeSet::new();
498 let mut seen_digests: BTreeSet<String> = BTreeSet::new();
499 let mut new_variants = 0;
500
501 for entry in &library {
502 let repo = entry.repository.as_str();
503 let tags_url = format!("https://ollama.com/library/{repo}/tags");
504 let selected = match run.text(&tags_url, MAX_HTML_BYTES).await {
505 Ok(html) => match parse_tags(&html, repo) {
506 Ok(tags) => select_variants(&tags),
507 Err(_) => {
508 outcome
509 .failures
510 .push(format!("{repo}: tags page unparseable"));
511 outcome.complete = false;
512 continue;
513 }
514 },
515 Err(error) if interrupted(&error) => return Err(error),
516 Err(_) => {
517 outcome
518 .failures
519 .push(format!("{repo}: tags page unavailable"));
520 outcome.complete = false;
521 continue;
522 }
523 };
524 let selected_ids: BTreeSet<String> =
525 selected.iter().map(|tag| format!("{repo}:{tag}")).collect();
526 for id in auto.keys().cloned().collect::<Vec<_>>() {
527 if parse_reference(&id).0 == repo && !selected_ids.contains(&id) {
528 auto.remove(&id);
529 outcome.removed.push(id);
530 }
531 }
532 for tag in &selected {
533 let id = format!("{repo}:{tag}");
534 if curated.contains_key(id.as_str()) {
535 continue;
536 }
537 match admit_variant(
538 &run,
539 entry,
540 tag,
541 &id,
542 &auto,
543 &curated_digests,
544 &mut seen_digests,
545 &today,
546 &mut new_variants,
547 options,
548 &mut outcome,
549 )
550 .await
551 {
552 Ok(Some(model)) => {
553 eligible.insert(id.clone());
554 auto.insert(id, model);
555 }
556 Ok(None) => {}
557 Err(error) if interrupted(&error) => return Err(error),
558 Err(error) => {
559 outcome.failures.push(format!("{id}: {error}"));
560 outcome.complete = false;
561 if auto.contains_key(&id) {
562 eligible.insert(id);
563 }
564 }
565 }
566 }
567 }
568 for id in &outcome.deferred {
569 eligible.insert(id.clone());
570 }
571 let mut corrected: BTreeMap<String, CatalogModel> = BTreeMap::new();
572 for model in curated.values() {
573 match reobserve(&run, model, &stamp, &mut outcome.state).await {
574 Ok(Some(observed)) => {
575 if options.correct_curated
576 && let Some((fixed, changes)) = correct(model, &observed)?
577 {
578 corrected.insert(model.id.clone(), fixed);
579 outcome.corrections.extend(changes);
580 }
581 outcome.curated_observed.push(model.id.clone());
582 outcome.curated_observations.push(observed);
583 }
584 Ok(None) => {}
585 Err(error) if interrupted(&error) => return Err(error),
586 Err(error) => outcome
587 .failures
588 .push(format!("{} (curated): {error}", model.id)),
589 }
590 }
591
592 let mut models: Vec<CatalogModel> = catalog
593 .models
594 .iter()
595 .filter(|model| model.provenance == EntryProvenance::Curated)
596 .map(|model| corrected.remove(&model.id).unwrap_or_else(|| model.clone()))
597 .chain(auto.into_values())
598 .collect();
599 models.sort_by(|left, right| {
600 right
601 .recency()
602 .map(|(day, _)| day)
603 .cmp(&left.recency().map(|(day, _)| day))
604 .then_with(|| left.id.cmp(&right.id))
605 });
606 outcome.observations = models
607 .iter()
608 .filter(|model| model.provenance == EntryProvenance::Auto)
609 .map(|model| observation(model, &stamp))
610 .collect();
611 let mut variants: BTreeSet<String> = models.iter().map(|model| model.id.clone()).collect();
612 variants.extend(eligible);
613 outcome.scope = json!({
614 "name": SCOPE_NAME,
615 "checkedAt": stamp,
616 "source": "https://ollama.com/library",
617 "complete": outcome.complete,
618 "variants": variants,
619 });
620 let changed =
621 !outcome.added.is_empty() || !outcome.updated.is_empty() || !outcome.removed.is_empty();
622 let encoded = json!({
623 "schemaVersion": rigspark_core::catalog::required_schema_version(&models),
624 "generatedAt": if changed { stamp.clone() } else { catalog.generated_at.clone() },
625 "models": models,
626 });
627 outcome.catalog = Some(Catalog::parse(&encoded.to_string()).map_err(io::Error::other)?);
628 outcome.removed.sort();
629 Ok(outcome)
630}
631
632#[allow(clippy::too_many_arguments)]
633async fn admit_variant(
634 run: &Run<'_>,
635 entry: &LibraryEntry,
636 tag: &str,
637 id: &str,
638 auto: &BTreeMap<String, CatalogModel>,
639 curated_digests: &BTreeSet<&str>,
640 seen_digests: &mut BTreeSet<String>,
641 today: &str,
642 new_variants: &mut usize,
643 options: &AdmissionOptions,
644 outcome: &mut AdmissionOutcome,
645) -> io::Result<Option<CatalogModel>> {
646 let repo = entry.repository.as_str();
647 let blob =
648 |digest: &str| format!("https://registry.ollama.ai/v2/library/{repo}/blobs/{digest}");
649 let manifest_raw = run
650 .get(
651 &format!("https://registry.ollama.ai/v2/library/{repo}/manifests/{tag}"),
652 Some(MANIFEST_ACCEPT),
653 None,
654 MAX_MANIFEST_BYTES,
655 )
656 .await?;
657 let manifest_text = std::str::from_utf8(&manifest_raw)
658 .map_err(|_| io::Error::other("manifest is not UTF-8"))?;
659 let Some(layer) = parse_layer(manifest_text).map_err(io::Error::other)? else {
660 outcome
661 .rejected
662 .insert(id.into(), "no GGUF model layer".into());
663 return Ok(None);
664 };
665 let descriptors: ManifestLayers =
666 serde_json::from_str(manifest_text).map_err(io::Error::other)?;
667 if curated_digests.contains(layer.sha256.as_str()) || !seen_digests.insert(layer.sha256.clone())
668 {
669 return Ok(None);
670 }
671 let license_digest = descriptors
672 .layers
673 .iter()
674 .find(|layer| layer.media_type == "application/vnd.ollama.image.license")
675 .and_then(|layer| sha256(&layer.digest))
676 .map(str::to_string);
677 let cache_key = format!(
678 "{}:{}",
679 layer.sha256,
680 license_digest.as_deref().unwrap_or("none")
681 );
682 let prior = auto.get(id);
683 if let Some(prior) = prior
684 && prior.quantizations[0].sha256.as_deref() == Some(layer.sha256.as_str())
685 && prior.quantizations[0].disk_bytes == layer.disk_bytes
686 && !outcome.state.rejections.contains_key(&cache_key)
687 {
688 let mut refreshed = prior.clone();
689 let capabilities = catalog_capabilities(&entry.capabilities);
690 if capabilities != prior.capabilities {
691 refreshed.capabilities = capabilities;
692 outcome.updated.push(id.into());
693 } else {
694 outcome.reverified.push(id.into());
695 }
696 return Ok(Some(refreshed));
697 }
698 if let Some(reason) = outcome.state.rejections.get(&cache_key) {
699 outcome.rejected.insert(id.into(), reason.clone());
700 return Ok(None);
701 }
702 if prior.is_none() {
703 if *new_variants >= options.max_new_variants {
704 outcome.deferred.push(id.into());
705 outcome.complete = false;
706 return Ok(None);
707 }
708 *new_variants += 1;
709 }
710 let config_digest = sha256(&descriptors.config.digest)
711 .ok_or_else(|| io::Error::other("invalid config digest"))?;
712 let config_raw = run
713 .get(
714 &blob(&format!("sha256:{config_digest}")),
715 None,
716 None,
717 MAX_CONFIG_BYTES,
718 )
719 .await?;
720 let config = RegistryConfig::parse(
721 std::str::from_utf8(&config_raw).map_err(|_| io::Error::other("config is not UTF-8"))?,
722 );
723 let license_head = match &license_digest {
724 Some(digest) => Some(
725 String::from_utf8_lossy(
726 &run.get(
727 &blob(&format!("sha256:{digest}")),
728 None,
729 Some(MAX_LICENSE_HEAD_BYTES),
730 MAX_LICENSE_HEAD_BYTES,
731 )
732 .await?,
733 )
734 .into_owned(),
735 ),
736 None => None,
737 };
738 let reject = |outcome: &mut AdmissionOutcome, reason: &str| {
739 outcome.rejected.insert(id.into(), reason.into());
740 outcome
741 .state
742 .rejections
743 .insert(cache_key.clone(), reason.into());
744 Ok(None)
745 };
746 let Ok(config) = config else {
747 return reject(outcome, "model config lacks family, size or file type");
748 };
749 let gguf = match run.gguf(&blob(&format!("sha256:{}", layer.sha256))).await? {
750 Ok(gguf) => gguf,
751 Err(reason) => return reject(outcome, reason),
752 };
753 let built = build_entry(&AdmissionInput {
754 repository: repo,
755 tag,
756 capabilities: &entry.capabilities,
757 layer: &layer,
758 config: &config,
759 gguf: &gguf,
760 license_head: license_head.as_deref(),
761 today: prior
762 .and_then(|model| model.added_at.as_deref())
763 .unwrap_or(today),
764 });
765 match built {
766 Ok(model) => {
767 outcome.state.rejections.remove(&cache_key);
768 if prior.is_some() {
769 outcome.updated.push(id.into());
770 } else {
771 outcome.added.push(id.into());
772 }
773 Ok(Some(model))
774 }
775 Err(rejection) => reject(outcome, rejection.reason()),
776 }
777}
778
779async fn reobserve(
782 run: &Run<'_>,
783 model: &CatalogModel,
784 stamp: &str,
785 state: &mut AdmissionState,
786) -> io::Result<Option<Value>> {
787 let Some(reference) = model.source.ollama.as_deref() else {
788 return Ok(None);
789 };
790 let (path, tag) = parse_reference(reference);
791 let repo = path.strip_prefix("library/").unwrap_or(path);
792 if repo.contains('/') || !name(repo) || !name(tag) {
793 return Ok(None);
794 }
795 let Some(index) = pulled_quantization(model) else {
796 return Ok(None);
797 };
798 let manifest_url = format!("https://registry.ollama.ai/v2/library/{repo}/manifests/{tag}");
799 let manifest_raw = run
800 .get(
801 &manifest_url,
802 Some(MANIFEST_ACCEPT),
803 None,
804 MAX_MANIFEST_BYTES,
805 )
806 .await?;
807 let manifest_text = std::str::from_utf8(&manifest_raw)
808 .map_err(|_| io::Error::other("manifest is not UTF-8"))?;
809 let Some(layer) = parse_layer(manifest_text).map_err(io::Error::other)? else {
810 return Ok(None);
811 };
812 let descriptors: ManifestLayers =
813 serde_json::from_str(manifest_text).map_err(io::Error::other)?;
814 let blob = |digest: &str| {
815 format!("https://registry.ollama.ai/v2/library/{repo}/blobs/sha256:{digest}")
816 };
817 let config_digest = sha256(&descriptors.config.digest)
818 .ok_or_else(|| io::Error::other("invalid config digest"))?;
819 let config_raw = run
820 .get(&blob(config_digest), None, None, MAX_CONFIG_BYTES)
821 .await?;
822 let Ok(config) = RegistryConfig::parse(
823 std::str::from_utf8(&config_raw).map_err(|_| io::Error::other("config is not UTF-8"))?,
824 ) else {
825 return Ok(None);
826 };
827 let license_digest = descriptors
828 .layers
829 .iter()
830 .find(|layer| layer.media_type == "application/vnd.ollama.image.license")
831 .and_then(|layer| sha256(&layer.digest))
832 .map(str::to_string);
833 let key = format!(
834 "{}:{}",
835 layer.sha256,
836 license_digest.as_deref().unwrap_or("none")
837 );
838 let sourced = match state.facts.get(&key) {
839 Some(sourced) => sourced.clone(),
840 None => {
841 let Ok(gguf) = run.gguf(&blob(&layer.sha256)).await? else {
842 return Ok(None);
843 };
844 let Some(arch) = gguf.architecture().map(str::to_string) else {
845 return Ok(None);
846 };
847 let Some(context_length) = gguf.u64(&format!("{arch}.context_length")) else {
848 return Ok(None);
849 };
850 let head = match &license_digest {
851 Some(digest) => Some(
852 String::from_utf8_lossy(
853 &run.get(
854 &blob(digest),
855 None,
856 Some(MAX_LICENSE_HEAD_BYTES),
857 MAX_LICENSE_HEAD_BYTES,
858 )
859 .await?,
860 )
861 .into_owned(),
862 ),
863 None => None,
864 };
865 let sourced = SourcedFacts {
866 context_length,
867 architecture: if gguf
868 .u64(&format!("{arch}.expert_count"))
869 .is_some_and(|count| count > 0)
870 {
871 Architecture::Moe
872 } else {
873 Architecture::Dense
874 },
875 license: license_id(
876 gguf.text("general.license"),
877 gguf.text("general.license.name"),
878 head.as_deref(),
879 )
880 .ok()
881 .map(str::to_string),
882 kv_bytes_per_token: kv_bytes_per_token(&gguf),
883 };
884 state.facts.insert(key, sourced.clone());
885 sourced
886 }
887 };
888 let mut quant = serde_json::to_value(&model.quantizations[index]).map_err(io::Error::other)?;
889 let object = quant
890 .as_object_mut()
891 .ok_or_else(|| io::Error::other("invalid quantization"))?;
892 for field in ["minRamBytes", "minVramBytes", "digestVerified"] {
893 object.remove(field);
894 }
895 object.insert("name".into(), json!(config.file_type.to_ascii_uppercase()));
896 object.insert("diskBytes".into(), json!(layer.disk_bytes));
897 object.insert("sha256".into(), json!(layer.sha256));
898 if layer.projectors.is_empty() {
899 object.remove("projectors");
900 } else {
901 object.insert(
902 "projectors".into(),
903 serde_json::to_value(&layer.projectors).map_err(io::Error::other)?,
904 );
905 }
906 let mut facts = serde_json::Map::new();
907 facts.insert("defaultQuantization".into(), quant);
908 facts.insert("contextLength".into(), json!(sourced.context_length as f64));
909 facts.insert(
910 "architecture".into(),
911 serde_json::to_value(&sourced.architecture).map_err(io::Error::other)?,
912 );
913 if let Some(license) = sourced.license {
914 facts.insert("license".into(), json!(license));
915 }
916 if let (Some(rate), Some(_)) = (sourced.kv_bytes_per_token, model.kv_bytes_per_token) {
918 facts.insert("kvBytesPerToken".into(), json!(rate));
919 }
920 Ok(Some(json!({
921 "id": model.id,
922 "checkedAt": stamp,
923 "sources": [manifest_url, format!("https://ollama.com/library/{repo}:{tag}")],
924 "facts": facts,
925 })))
926}
927
928fn catalog_value(model: &CatalogModel, field: &str) -> io::Result<Value> {
930 if field == "defaultQuantization" {
931 let index = pulled_quantization(model).unwrap_or(0);
932 let mut quant =
933 serde_json::to_value(&model.quantizations[index]).map_err(io::Error::other)?;
934 if let Some(object) = quant.as_object_mut() {
935 for key in ["minRamBytes", "minVramBytes", "digestVerified"] {
936 object.remove(key);
937 }
938 }
939 return Ok(quant);
940 }
941 Ok(serde_json::to_value(model)
942 .map_err(io::Error::other)?
943 .get(field)
944 .cloned()
945 .unwrap_or(Value::Null))
946}
947
948fn correct(
951 model: &CatalogModel,
952 observed: &Value,
953) -> io::Result<Option<(CatalogModel, Vec<Correction>)>> {
954 let Some(facts) = observed["facts"].as_object() else {
955 return Ok(None);
956 };
957 let sources: Vec<String> = observed["sources"]
958 .as_array()
959 .map(|items| {
960 items
961 .iter()
962 .filter_map(|item| item.as_str().map(str::to_string))
963 .collect()
964 })
965 .unwrap_or_default();
966 let mut fixed = model.clone();
967 let mut changes = Vec::new();
968 for (field, to) in facts {
969 let from = catalog_value(model, field)?;
970 if from == *to || field == "architecture" {
971 continue;
972 }
973 match field.as_str() {
974 "contextLength" => {
975 fixed.context_length = to
976 .as_f64()
977 .ok_or_else(|| io::Error::other("invalid context"))?
978 }
979 "license" => {
980 fixed.license = to
981 .as_str()
982 .ok_or_else(|| io::Error::other("invalid license"))?
983 .into()
984 }
985 "kvBytesPerToken" => fixed.kv_bytes_per_token = to.as_f64(),
986 "defaultQuantization" => {
987 let index = pulled_quantization(model).unwrap_or(0);
988 let name = to["name"]
989 .as_str()
990 .ok_or_else(|| io::Error::other("invalid quant"))?;
991 let projectors: Vec<rigspark_core::sizing::ProjectorArtifact> =
992 serde_json::from_value(to.get("projectors").cloned().unwrap_or(json!([])))
993 .map_err(io::Error::other)?;
994 let quant = rigspark_core::enrich::sized_quantization(
995 rigspark_core::sizing::parse_param_count(&model.params)
996 .map_err(io::Error::other)?,
997 &model.architecture,
998 name,
999 to["diskBytes"]
1000 .as_f64()
1001 .ok_or_else(|| io::Error::other("invalid size"))?,
1002 to["sha256"].as_str().map(str::to_string),
1003 projectors,
1004 )
1005 .map_err(io::Error::other)?;
1006 fixed.quantizations[index] = quant;
1007 let mut position = 0;
1009 fixed.quantizations.retain(|candidate| {
1010 let keep = position == index || candidate.name != name;
1011 position += 1;
1012 keep
1013 });
1014 }
1015 _ => continue,
1016 }
1017 changes.push(Correction {
1018 id: model.id.clone(),
1019 field: field.clone(),
1020 from,
1021 to: to.clone(),
1022 sources: sources.clone(),
1023 });
1024 }
1025 if changes.is_empty() {
1026 return Ok(None);
1027 }
1028 let check = json!({"schemaVersion": SCHEMA_VERSION, "generatedAt": "2026-01-01T00:00:00.000Z", "models": [fixed]});
1029 Catalog::parse(&check.to_string()).map_err(io::Error::other)?;
1030 Ok(Some((fixed, changes)))
1031}