1use crate::catalog_update::MAX_ARTIFACT_BYTES;
2use rigspark_core::catalog::{Catalog, CatalogModel};
3use serde::{Deserialize, Serialize};
4use sha2::{Digest, Sha256};
5use std::collections::{BTreeMap, BTreeSet};
6use thiserror::Error;
7use time::{OffsetDateTime, format_description::well_known::Rfc3339};
8
9pub const MINIMUM_PERCENT: usize = 90;
10pub const EVIDENCE_MAX_AGE_DAYS: i64 = 7;
11
12#[derive(Debug, Error)]
13#[error("invalid catalog quality evidence: {0}")]
14pub struct QualityError(pub &'static str);
15
16#[derive(Deserialize)]
17#[serde(rename_all = "camelCase", deny_unknown_fields)]
18struct Evidence {
19 policy_version: u8,
20 scopes: Vec<Scope>,
21 observations: Vec<Observation>,
22}
23
24#[derive(Deserialize, Serialize)]
25#[serde(rename_all = "camelCase", deny_unknown_fields)]
26pub struct Scope {
27 name: String,
28 checked_at: String,
29 source: String,
30 complete: bool,
31 variants: Vec<String>,
32}
33
34#[derive(Deserialize)]
35#[serde(rename_all = "camelCase", deny_unknown_fields)]
36struct Observation {
37 id: String,
38 checked_at: String,
39 sources: Vec<String>,
40 #[serde(default)]
42 model: Option<CatalogModel>,
43 #[serde(default)]
44 facts: Option<BTreeMap<String, serde_json::Value>>,
45}
46
47pub const PARTIAL_FACTS: [&str; 6] = [
49 "activeParams",
50 "architecture",
51 "contextLength",
52 "defaultQuantization",
53 "kvBytesPerToken",
54 "license",
55];
56
57fn catalog_fact(model: &CatalogModel, field: &str) -> Result<serde_json::Value, QualityError> {
58 let value = serde_json::to_value(model).map_err(|_| QualityError("invalid model facts"))?;
59 Ok(if field == "defaultQuantization" {
60 let index = rigspark_core::registry_collector::pulled_quantization(model).unwrap_or(0);
61 facts(model)?["quantizations"][index].clone()
62 } else {
63 value.get(field).cloned().unwrap_or(serde_json::Value::Null)
64 })
65}
66
67#[derive(Serialize)]
68#[serde(rename_all = "camelCase")]
69pub struct EntryResult {
70 id: String,
71 checked_at: Option<String>,
72 sources: Vec<String>,
73 fresh: bool,
74 verified: bool,
75 reason: &'static str,
76 #[serde(skip_serializing_if = "Option::is_none")]
78 checked_fields: Option<Vec<String>>,
79}
80
81#[derive(Serialize)]
82#[serde(rename_all = "camelCase")]
83pub struct QualityReport {
84 pub policy_version: u8,
85 pub evaluated_at: String,
86 pub catalog_sha256: String,
87 pub evidence_sha256: String,
88 pub minimum_percent: usize,
89 pub evidence_max_age_days: i64,
90 pub catalog_entries: usize,
91 pub fresh_entries: usize,
92 pub verified_entries: usize,
93 pub eligible_variants: usize,
94 pub covered_variants: usize,
95 pub coverage_percent: Option<f64>,
96 pub recency_percent: f64,
97 pub freshness_percent: Option<f64>,
98 pub correctness_percent: f64,
99 pub passed: bool,
100 pub blockers: Vec<String>,
101 pub missing_variants: Vec<String>,
102 pub scopes: Vec<Scope>,
103 pub entries: Vec<EntryResult>,
104}
105
106fn recent(raw: &str, now: OffsetDateTime) -> Result<bool, QualityError> {
107 let checked =
108 OffsetDateTime::parse(raw, &Rfc3339).map_err(|_| QualityError("invalid timestamp"))?;
109 if checked > now {
110 return Err(QualityError("future verification date"));
111 }
112 Ok(now - checked <= time::Duration::days(EVIDENCE_MAX_AGE_DAYS))
113}
114
115pub fn evidence_fresh(checked_at: &str, now: &str) -> bool {
117 OffsetDateTime::parse(now, &Rfc3339).is_ok_and(|now| recent(checked_at, now).unwrap_or(false))
118}
119
120fn source_url(raw: &str) -> bool {
121 url::Url::parse(raw).is_ok_and(|url| {
122 url.scheme() == "https"
123 && url.host_str().is_some()
124 && url.username().is_empty()
125 && url.password().is_none()
126 && url.port_or_known_default() == Some(443)
127 && url.query().is_none()
128 && url.fragment().is_none()
129 })
130}
131
132fn facts(model: &CatalogModel) -> Result<serde_json::Value, QualityError> {
133 let mut value = serde_json::to_value(model).map_err(|_| QualityError("invalid model facts"))?;
134 for quant in value["quantizations"].as_array_mut().unwrap() {
135 for field in ["minRamBytes", "minVramBytes", "digestVerified"] {
136 quant.as_object_mut().unwrap().remove(field);
137 }
138 }
139 Ok(value)
140}
141
142fn citations_match(observation: &Observation, model: &CatalogModel) -> bool {
143 if observation.model.is_none() {
144 return partial_citations_match(observation, model);
146 }
147 let source = &observation.model.as_ref().unwrap_or(model).source;
148 let cites_repo = |repo: &str| {
149 let root = format!("https://huggingface.co/{repo}");
150 observation
151 .sources
152 .iter()
153 .any(|url| url == &root || url.starts_with(&format!("{root}/")))
154 };
155 if source.hf.as_deref().is_some_and(|repo| !cites_repo(repo))
156 || source
157 .gguf
158 .as_ref()
159 .is_some_and(|gguf| !cites_repo(&gguf.repo))
160 || source
161 .mlx
162 .as_ref()
163 .is_some_and(|mlx| !cites_repo(&mlx.repo))
164 {
165 return false;
166 }
167 if let Some(reference) = &source.ollama {
168 let (repo, tag) = rigspark_core::registry_collector::parse_reference(reference);
169 let repo = if repo.contains('/') {
170 repo.to_string()
171 } else {
172 format!("library/{repo}")
173 };
174 let manifest = format!("https://registry.ollama.ai/v2/{repo}/manifests/{tag}");
175 if !observation.sources.contains(&manifest) {
176 return false;
177 }
178 if source.hf.is_none() {
179 let page = format!("https://ollama.com/{repo}:{tag}");
180 if !observation.sources.contains(&page) {
181 return false;
182 }
183 }
184 }
185 source.hf.is_some() || source.ollama.is_some() || source.gguf.is_some() || source.mlx.is_some()
186}
187
188fn partial_citations_match(observation: &Observation, model: &CatalogModel) -> bool {
189 let Some(reference) = &model.source.ollama else {
190 return false;
191 };
192 let (repo, tag) = rigspark_core::registry_collector::parse_reference(reference);
193 let repo = if repo.contains('/') {
194 repo.to_string()
195 } else {
196 format!("library/{repo}")
197 };
198 observation.sources.contains(&format!(
199 "https://registry.ollama.ai/v2/{repo}/manifests/{tag}"
200 )) && observation
201 .sources
202 .contains(&format!("https://ollama.com/{repo}:{tag}"))
203}
204
205fn matches_facts(model: &CatalogModel, observed: &CatalogModel) -> Result<bool, QualityError> {
206 let mut observed = observed.clone();
207 if model.kv_bytes_per_token.is_none() {
208 observed.kv_bytes_per_token = None;
209 }
210 if model.benchmark_proxy.is_none() {
211 observed.benchmark_proxy = None;
212 }
213 Ok(facts(model)? == facts(&observed)?)
214}
215
216fn pinned(model: &CatalogModel) -> bool {
217 if model.source.ollama.is_some() {
218 model
219 .quantizations
220 .iter()
221 .all(|quant| quant.sha256.is_some())
222 } else {
223 model.source.gguf.is_some() || model.source.mlx.is_some()
224 }
225}
226
227fn percent(numerator: usize, denominator: usize) -> f64 {
228 if denominator == 0 {
229 0.0
230 } else {
231 numerator as f64 * 100.0 / denominator as f64
232 }
233}
234
235pub fn evaluate(
236 catalog_raw: &str,
237 evidence_raw: &str,
238 now: &str,
239) -> Result<QualityReport, QualityError> {
240 if catalog_raw.len() > MAX_ARTIFACT_BYTES || evidence_raw.len() > MAX_ARTIFACT_BYTES {
241 return Err(QualityError("input exceeds 16 MiB"));
242 }
243 let catalog = Catalog::parse(catalog_raw).map_err(|_| QualityError("catalog schema"))?;
244 if catalog.models.is_empty() {
245 return Err(QualityError("empty catalog"));
246 }
247 let evidence: Evidence = rigspark_core::catalog::parse_document(evidence_raw)
248 .map_err(|_| QualityError("evidence schema"))?;
249 if evidence.policy_version != 1
250 || evidence.scopes.len() > 32
251 || evidence.observations.len() > 10000
252 {
253 return Err(QualityError("unsupported policy or collection limit"));
254 }
255 let clock =
256 OffsetDateTime::parse(now, &Rfc3339).map_err(|_| QualityError("evaluation date"))?;
257 recent(&catalog.generated_at, clock)?;
258 let mut blockers = Vec::new();
259 let mut inventory = BTreeSet::new();
260 let mut names = BTreeSet::new();
261 let mut inventory_known = !evidence.scopes.is_empty();
262 for scope in &evidence.scopes {
263 if scope.name.is_empty()
264 || !names.insert(scope.name.clone())
265 || !source_url(&scope.source)
266 || scope.variants.len() > 10000
267 {
268 return Err(QualityError("invalid coverage scope"));
269 }
270 if !recent(&scope.checked_at, clock)? || !scope.complete || scope.variants.is_empty() {
271 inventory_known = false;
272 }
273 for variant in &scope.variants {
274 if variant.is_empty()
275 || variant.len() > 256
276 || variant.chars().any(char::is_control)
277 || !inventory.insert(variant.clone())
278 {
279 return Err(QualityError("duplicate or invalid inventory variant"));
280 }
281 }
282 }
283 if !names.contains("ollama-local-variants") {
284 inventory_known = false;
285 }
286 if !inventory_known {
287 blockers.push(
288 "Upstream variant inventory is missing, incomplete, empty, or older than 7 days."
289 .into(),
290 );
291 }
292 let models: BTreeMap<_, _> = catalog
293 .models
294 .iter()
295 .map(|model| (model.id.as_str(), model))
296 .collect();
297 let mut observations = BTreeMap::new();
298 for observation in &evidence.observations {
299 let Some(model) = models.get(observation.id.as_str()) else {
300 return Err(QualityError("duplicate, unbound, or uncited observation"));
301 };
302 match (&observation.model, &observation.facts) {
303 (Some(observed), None) => {
304 if observed.id != observation.id {
305 return Err(QualityError("duplicate, unbound, or uncited observation"));
306 }
307 let observed_catalog = serde_json::json!({"schemaVersion":rigspark_core::catalog::SCHEMA_VERSION,"generatedAt":observation.checked_at,"models":[observed]});
308 Catalog::parse(&observed_catalog.to_string())
309 .map_err(|_| QualityError("observed model facts"))?;
310 }
311 (None, Some(facts)) => {
312 if !facts.contains_key("defaultQuantization")
314 || !facts
315 .keys()
316 .all(|key| PARTIAL_FACTS.contains(&key.as_str()))
317 {
318 return Err(QualityError(
319 "partial observation asserts unsupported facts",
320 ));
321 }
322 }
323 _ => {
324 return Err(QualityError(
325 "observation needs exactly one of model or facts",
326 ));
327 }
328 }
329 if observations
330 .insert(observation.id.as_str(), observation)
331 .is_some()
332 || observation.sources.is_empty()
333 || observation.sources.len() > 16
334 || !observation.sources.iter().all(|source| source_url(source))
335 || !citations_match(observation, model)
336 {
337 return Err(QualityError("duplicate, unbound, or uncited observation"));
338 }
339 recent(&observation.checked_at, clock)?;
340 }
341 let mut entries = Vec::new();
342 let mut fresh_entries = 0;
343 let mut verified_entries = 0;
344 for model in &catalog.models {
345 let mut entry = EntryResult {
346 id: model.id.clone(),
347 checked_at: None,
348 sources: Vec::new(),
349 fresh: false,
350 verified: false,
351 reason: "missing evidence",
352 checked_fields: None,
353 };
354 if inventory_known && !inventory.contains(&model.id) {
355 blockers.push(format!(
356 "{}: catalog entry excluded from declared coverage scope",
357 model.id
358 ));
359 }
360 if let Some(observation) = observations.get(model.id.as_str()) {
361 entry.checked_at = Some(observation.checked_at.clone());
362 entry.sources = observation.sources.clone();
363 entry.fresh = recent(&observation.checked_at, clock)?;
364 let agrees = match (&observation.model, &observation.facts) {
365 (Some(observed), _) => matches_facts(model, observed)?,
366 (None, Some(facts)) => {
367 entry.checked_fields = Some(facts.keys().cloned().collect());
368 let mut agrees = true;
369 for (field, value) in facts {
370 agrees &= catalog_fact(model, field)? == *value;
371 }
372 agrees
373 }
374 (None, None) => false,
375 };
376 if !agrees {
377 entry.reason = "authoritative facts contradict catalog";
378 blockers.push(format!(
379 "{}: authoritative facts contradict catalog",
380 model.id
381 ));
382 } else if !pinned(model) {
383 entry.reason = "required artifact integrity evidence missing";
384 } else if !entry.fresh {
385 entry.reason = "verification evidence older than 7 days";
386 } else {
387 entry.verified = true;
388 entry.reason = "verified";
389 }
390 }
391 fresh_entries += usize::from(entry.fresh);
392 verified_entries += usize::from(entry.verified);
393 entries.push(entry);
394 }
395 let missing_variants: Vec<_> = inventory
396 .iter()
397 .filter(|id| !models.contains_key(id.as_str()))
398 .cloned()
399 .collect();
400 let covered_variants = inventory.len() - missing_variants.len();
401 let coverage_percent = inventory_known.then(|| percent(covered_variants, inventory.len()));
402 let recency_percent = percent(fresh_entries, catalog.models.len());
403 let freshness_percent = coverage_percent.map(|coverage| coverage.min(recency_percent));
404 let correctness_percent = percent(verified_entries, catalog.models.len());
405 let passed = blockers.is_empty()
406 && inventory_known
407 && covered_variants * 100 >= inventory.len() * MINIMUM_PERCENT
408 && fresh_entries * 100 >= catalog.models.len() * MINIMUM_PERCENT
409 && verified_entries * 100 >= catalog.models.len() * MINIMUM_PERCENT;
410 Ok(QualityReport {
411 policy_version: 1,
412 evaluated_at: now.into(),
413 catalog_sha256: format!("{:x}", Sha256::digest(catalog_raw.as_bytes())),
414 evidence_sha256: format!("{:x}", Sha256::digest(evidence_raw.as_bytes())),
415 minimum_percent: MINIMUM_PERCENT,
416 evidence_max_age_days: EVIDENCE_MAX_AGE_DAYS,
417 catalog_entries: catalog.models.len(),
418 fresh_entries,
419 verified_entries,
420 eligible_variants: inventory.len(),
421 covered_variants,
422 coverage_percent,
423 recency_percent,
424 freshness_percent,
425 correctness_percent,
426 passed,
427 blockers,
428 missing_variants,
429 scopes: evidence.scopes,
430 entries,
431 })
432}
433
434pub fn sign_reviewed_catalog(
435 payload: crate::catalog_update::CatalogPayload,
436 evidence: &str,
437 seed: &[u8; 32],
438 public_key: &[u8; 32],
439) -> Result<(Vec<u8>, Vec<u8>), QualityError> {
440 use crate::catalog_update::{SignedCatalog, sign_catalog};
441 use ed25519_dalek::{Signer, SigningKey};
442 let report = evaluate(&payload.catalog, evidence, &payload.published_at)?;
443 if !report.passed {
444 return Err(QualityError(
445 "both scores must be at least 90% with no hard blockers",
446 ));
447 }
448 let revision = payload.revision;
449 let artifact = sign_catalog(payload, seed, public_key)
450 .map_err(|_| QualityError("catalog signing failed"))?;
451 let report_payload = serde_json::json!({
452 "reportType":"rigspark-catalog-quality", "reportVersion":1,
453 "revision":revision, "catalogArtifactSha256":format!("{:x}", Sha256::digest(&artifact)),
454 "quality":report, "evidence":evidence
455 })
456 .to_string();
457 let signature = SigningKey::from_bytes(seed)
458 .sign(report_payload.as_bytes())
459 .to_bytes()
460 .iter()
461 .map(|byte| format!("{byte:02x}"))
462 .collect();
463 let quality = serde_json::to_vec(&SignedCatalog {
464 payload: report_payload,
465 signature,
466 })
467 .map_err(|_| QualityError("report encoding"))?;
468 if quality.len() > MAX_ARTIFACT_BYTES {
469 return Err(QualityError("signed quality report exceeds 16 MiB"));
470 }
471 Ok((artifact, quality))
472}