Skip to main content

rigspark_runtime/
catalog_quality.rs

1use crate::catalog_update::MAX_ARTIFACT_BYTES;
2use rigspark_core::catalog::{Catalog, CatalogModel};
3use serde::{Deserialize, Serialize};
4use sha2::{Digest, Sha256};
5use std::collections::{BTreeMap, BTreeSet};
6use thiserror::Error;
7use time::{OffsetDateTime, format_description::well_known::Rfc3339};
8
9pub const MINIMUM_PERCENT: usize = 90;
10pub const EVIDENCE_MAX_AGE_DAYS: i64 = 7;
11
12#[derive(Debug, Error)]
13#[error("invalid catalog quality evidence: {0}")]
14pub struct QualityError(pub &'static str);
15
16#[derive(Deserialize)]
17#[serde(rename_all = "camelCase", deny_unknown_fields)]
18struct Evidence {
19    policy_version: u8,
20    scopes: Vec<Scope>,
21    observations: Vec<Observation>,
22}
23
24#[derive(Deserialize, Serialize)]
25#[serde(rename_all = "camelCase", deny_unknown_fields)]
26pub struct Scope {
27    name: String,
28    checked_at: String,
29    source: String,
30    complete: bool,
31    variants: Vec<String>,
32}
33
34#[derive(Deserialize)]
35#[serde(rename_all = "camelCase", deny_unknown_fields)]
36struct Observation {
37    id: String,
38    checked_at: String,
39    sources: Vec<String>,
40    /// A full observed model, or `facts` for the subset the collector could source.
41    #[serde(default)]
42    model: Option<CatalogModel>,
43    #[serde(default)]
44    facts: Option<BTreeMap<String, serde_json::Value>>,
45}
46
47/// Facts a partial observation may assert, each compared in the catalog's own representation.
48pub const PARTIAL_FACTS: [&str; 6] = [
49    "activeParams",
50    "architecture",
51    "contextLength",
52    "defaultQuantization",
53    "kvBytesPerToken",
54    "license",
55];
56
57fn catalog_fact(model: &CatalogModel, field: &str) -> Result<serde_json::Value, QualityError> {
58    let value = serde_json::to_value(model).map_err(|_| QualityError("invalid model facts"))?;
59    Ok(if field == "defaultQuantization" {
60        let index = rigspark_core::registry_collector::pulled_quantization(model).unwrap_or(0);
61        facts(model)?["quantizations"][index].clone()
62    } else {
63        value.get(field).cloned().unwrap_or(serde_json::Value::Null)
64    })
65}
66
67#[derive(Serialize)]
68#[serde(rename_all = "camelCase")]
69pub struct EntryResult {
70    id: String,
71    checked_at: Option<String>,
72    sources: Vec<String>,
73    fresh: bool,
74    verified: bool,
75    reason: &'static str,
76    /// Facts a partial observation verified; absent when the whole entry was observed.
77    #[serde(skip_serializing_if = "Option::is_none")]
78    checked_fields: Option<Vec<String>>,
79}
80
81#[derive(Serialize)]
82#[serde(rename_all = "camelCase")]
83pub struct QualityReport {
84    pub policy_version: u8,
85    pub evaluated_at: String,
86    pub catalog_sha256: String,
87    pub evidence_sha256: String,
88    pub minimum_percent: usize,
89    pub evidence_max_age_days: i64,
90    pub catalog_entries: usize,
91    pub fresh_entries: usize,
92    pub verified_entries: usize,
93    pub eligible_variants: usize,
94    pub covered_variants: usize,
95    pub coverage_percent: Option<f64>,
96    pub recency_percent: f64,
97    pub freshness_percent: Option<f64>,
98    pub correctness_percent: f64,
99    pub passed: bool,
100    pub blockers: Vec<String>,
101    pub missing_variants: Vec<String>,
102    pub scopes: Vec<Scope>,
103    pub entries: Vec<EntryResult>,
104}
105
106fn recent(raw: &str, now: OffsetDateTime) -> Result<bool, QualityError> {
107    let checked =
108        OffsetDateTime::parse(raw, &Rfc3339).map_err(|_| QualityError("invalid timestamp"))?;
109    if checked > now {
110        return Err(QualityError("future verification date"));
111    }
112    Ok(now - checked <= time::Duration::days(EVIDENCE_MAX_AGE_DAYS))
113}
114
115/// Whether evidence checked at `checked_at` is still within the gate's freshness window at `now`.
116pub fn evidence_fresh(checked_at: &str, now: &str) -> bool {
117    OffsetDateTime::parse(now, &Rfc3339).is_ok_and(|now| recent(checked_at, now).unwrap_or(false))
118}
119
120fn source_url(raw: &str) -> bool {
121    url::Url::parse(raw).is_ok_and(|url| {
122        url.scheme() == "https"
123            && url.host_str().is_some()
124            && url.username().is_empty()
125            && url.password().is_none()
126            && url.port_or_known_default() == Some(443)
127            && url.query().is_none()
128            && url.fragment().is_none()
129    })
130}
131
132fn facts(model: &CatalogModel) -> Result<serde_json::Value, QualityError> {
133    let mut value = serde_json::to_value(model).map_err(|_| QualityError("invalid model facts"))?;
134    for quant in value["quantizations"].as_array_mut().unwrap() {
135        for field in ["minRamBytes", "minVramBytes", "digestVerified"] {
136            quant.as_object_mut().unwrap().remove(field);
137        }
138    }
139    Ok(value)
140}
141
142fn citations_match(observation: &Observation, model: &CatalogModel) -> bool {
143    if observation.model.is_none() {
144        // Partial observations read only Ollama; they must cite exactly those reads.
145        return partial_citations_match(observation, model);
146    }
147    let source = &observation.model.as_ref().unwrap_or(model).source;
148    let cites_repo = |repo: &str| {
149        let root = format!("https://huggingface.co/{repo}");
150        observation
151            .sources
152            .iter()
153            .any(|url| url == &root || url.starts_with(&format!("{root}/")))
154    };
155    if source.hf.as_deref().is_some_and(|repo| !cites_repo(repo))
156        || source
157            .gguf
158            .as_ref()
159            .is_some_and(|gguf| !cites_repo(&gguf.repo))
160        || source
161            .mlx
162            .as_ref()
163            .is_some_and(|mlx| !cites_repo(&mlx.repo))
164    {
165        return false;
166    }
167    if let Some(reference) = &source.ollama {
168        let (repo, tag) = rigspark_core::registry_collector::parse_reference(reference);
169        let repo = if repo.contains('/') {
170            repo.to_string()
171        } else {
172            format!("library/{repo}")
173        };
174        let manifest = format!("https://registry.ollama.ai/v2/{repo}/manifests/{tag}");
175        if !observation.sources.contains(&manifest) {
176            return false;
177        }
178        if source.hf.is_none() {
179            let page = format!("https://ollama.com/{repo}:{tag}");
180            if !observation.sources.contains(&page) {
181                return false;
182            }
183        }
184    }
185    source.hf.is_some() || source.ollama.is_some() || source.gguf.is_some() || source.mlx.is_some()
186}
187
188fn partial_citations_match(observation: &Observation, model: &CatalogModel) -> bool {
189    let Some(reference) = &model.source.ollama else {
190        return false;
191    };
192    let (repo, tag) = rigspark_core::registry_collector::parse_reference(reference);
193    let repo = if repo.contains('/') {
194        repo.to_string()
195    } else {
196        format!("library/{repo}")
197    };
198    observation.sources.contains(&format!(
199        "https://registry.ollama.ai/v2/{repo}/manifests/{tag}"
200    )) && observation
201        .sources
202        .contains(&format!("https://ollama.com/{repo}:{tag}"))
203}
204
205fn matches_facts(model: &CatalogModel, observed: &CatalogModel) -> Result<bool, QualityError> {
206    let mut observed = observed.clone();
207    if model.kv_bytes_per_token.is_none() {
208        observed.kv_bytes_per_token = None;
209    }
210    if model.benchmark_proxy.is_none() {
211        observed.benchmark_proxy = None;
212    }
213    Ok(facts(model)? == facts(&observed)?)
214}
215
216fn pinned(model: &CatalogModel) -> bool {
217    if model.source.ollama.is_some() {
218        model
219            .quantizations
220            .iter()
221            .all(|quant| quant.sha256.is_some())
222    } else {
223        model.source.gguf.is_some() || model.source.mlx.is_some()
224    }
225}
226
227fn percent(numerator: usize, denominator: usize) -> f64 {
228    if denominator == 0 {
229        0.0
230    } else {
231        numerator as f64 * 100.0 / denominator as f64
232    }
233}
234
235pub fn evaluate(
236    catalog_raw: &str,
237    evidence_raw: &str,
238    now: &str,
239) -> Result<QualityReport, QualityError> {
240    if catalog_raw.len() > MAX_ARTIFACT_BYTES || evidence_raw.len() > MAX_ARTIFACT_BYTES {
241        return Err(QualityError("input exceeds 16 MiB"));
242    }
243    let catalog = Catalog::parse(catalog_raw).map_err(|_| QualityError("catalog schema"))?;
244    if catalog.models.is_empty() {
245        return Err(QualityError("empty catalog"));
246    }
247    let evidence: Evidence = rigspark_core::catalog::parse_document(evidence_raw)
248        .map_err(|_| QualityError("evidence schema"))?;
249    if evidence.policy_version != 1
250        || evidence.scopes.len() > 32
251        || evidence.observations.len() > 10000
252    {
253        return Err(QualityError("unsupported policy or collection limit"));
254    }
255    let clock =
256        OffsetDateTime::parse(now, &Rfc3339).map_err(|_| QualityError("evaluation date"))?;
257    recent(&catalog.generated_at, clock)?;
258    let mut blockers = Vec::new();
259    let mut inventory = BTreeSet::new();
260    let mut names = BTreeSet::new();
261    let mut inventory_known = !evidence.scopes.is_empty();
262    for scope in &evidence.scopes {
263        if scope.name.is_empty()
264            || !names.insert(scope.name.clone())
265            || !source_url(&scope.source)
266            || scope.variants.len() > 10000
267        {
268            return Err(QualityError("invalid coverage scope"));
269        }
270        if !recent(&scope.checked_at, clock)? || !scope.complete || scope.variants.is_empty() {
271            inventory_known = false;
272        }
273        for variant in &scope.variants {
274            if variant.is_empty()
275                || variant.len() > 256
276                || variant.chars().any(char::is_control)
277                || !inventory.insert(variant.clone())
278            {
279                return Err(QualityError("duplicate or invalid inventory variant"));
280            }
281        }
282    }
283    if !names.contains("ollama-local-variants") {
284        inventory_known = false;
285    }
286    if !inventory_known {
287        blockers.push(
288            "Upstream variant inventory is missing, incomplete, empty, or older than 7 days."
289                .into(),
290        );
291    }
292    let models: BTreeMap<_, _> = catalog
293        .models
294        .iter()
295        .map(|model| (model.id.as_str(), model))
296        .collect();
297    let mut observations = BTreeMap::new();
298    for observation in &evidence.observations {
299        let Some(model) = models.get(observation.id.as_str()) else {
300            return Err(QualityError("duplicate, unbound, or uncited observation"));
301        };
302        match (&observation.model, &observation.facts) {
303            (Some(observed), None) => {
304                if observed.id != observation.id {
305                    return Err(QualityError("duplicate, unbound, or uncited observation"));
306                }
307                let observed_catalog = serde_json::json!({"schemaVersion":rigspark_core::catalog::SCHEMA_VERSION,"generatedAt":observation.checked_at,"models":[observed]});
308                Catalog::parse(&observed_catalog.to_string())
309                    .map_err(|_| QualityError("observed model facts"))?;
310            }
311            (None, Some(facts)) => {
312                // Partial verification always includes artifact integrity.
313                if !facts.contains_key("defaultQuantization")
314                    || !facts
315                        .keys()
316                        .all(|key| PARTIAL_FACTS.contains(&key.as_str()))
317                {
318                    return Err(QualityError(
319                        "partial observation asserts unsupported facts",
320                    ));
321                }
322            }
323            _ => {
324                return Err(QualityError(
325                    "observation needs exactly one of model or facts",
326                ));
327            }
328        }
329        if observations
330            .insert(observation.id.as_str(), observation)
331            .is_some()
332            || observation.sources.is_empty()
333            || observation.sources.len() > 16
334            || !observation.sources.iter().all(|source| source_url(source))
335            || !citations_match(observation, model)
336        {
337            return Err(QualityError("duplicate, unbound, or uncited observation"));
338        }
339        recent(&observation.checked_at, clock)?;
340    }
341    let mut entries = Vec::new();
342    let mut fresh_entries = 0;
343    let mut verified_entries = 0;
344    for model in &catalog.models {
345        let mut entry = EntryResult {
346            id: model.id.clone(),
347            checked_at: None,
348            sources: Vec::new(),
349            fresh: false,
350            verified: false,
351            reason: "missing evidence",
352            checked_fields: None,
353        };
354        if inventory_known && !inventory.contains(&model.id) {
355            blockers.push(format!(
356                "{}: catalog entry excluded from declared coverage scope",
357                model.id
358            ));
359        }
360        if let Some(observation) = observations.get(model.id.as_str()) {
361            entry.checked_at = Some(observation.checked_at.clone());
362            entry.sources = observation.sources.clone();
363            entry.fresh = recent(&observation.checked_at, clock)?;
364            let agrees = match (&observation.model, &observation.facts) {
365                (Some(observed), _) => matches_facts(model, observed)?,
366                (None, Some(facts)) => {
367                    entry.checked_fields = Some(facts.keys().cloned().collect());
368                    let mut agrees = true;
369                    for (field, value) in facts {
370                        agrees &= catalog_fact(model, field)? == *value;
371                    }
372                    agrees
373                }
374                (None, None) => false,
375            };
376            if !agrees {
377                entry.reason = "authoritative facts contradict catalog";
378                blockers.push(format!(
379                    "{}: authoritative facts contradict catalog",
380                    model.id
381                ));
382            } else if !pinned(model) {
383                entry.reason = "required artifact integrity evidence missing";
384            } else if !entry.fresh {
385                entry.reason = "verification evidence older than 7 days";
386            } else {
387                entry.verified = true;
388                entry.reason = "verified";
389            }
390        }
391        fresh_entries += usize::from(entry.fresh);
392        verified_entries += usize::from(entry.verified);
393        entries.push(entry);
394    }
395    let missing_variants: Vec<_> = inventory
396        .iter()
397        .filter(|id| !models.contains_key(id.as_str()))
398        .cloned()
399        .collect();
400    let covered_variants = inventory.len() - missing_variants.len();
401    let coverage_percent = inventory_known.then(|| percent(covered_variants, inventory.len()));
402    let recency_percent = percent(fresh_entries, catalog.models.len());
403    let freshness_percent = coverage_percent.map(|coverage| coverage.min(recency_percent));
404    let correctness_percent = percent(verified_entries, catalog.models.len());
405    let passed = blockers.is_empty()
406        && inventory_known
407        && covered_variants * 100 >= inventory.len() * MINIMUM_PERCENT
408        && fresh_entries * 100 >= catalog.models.len() * MINIMUM_PERCENT
409        && verified_entries * 100 >= catalog.models.len() * MINIMUM_PERCENT;
410    Ok(QualityReport {
411        policy_version: 1,
412        evaluated_at: now.into(),
413        catalog_sha256: format!("{:x}", Sha256::digest(catalog_raw.as_bytes())),
414        evidence_sha256: format!("{:x}", Sha256::digest(evidence_raw.as_bytes())),
415        minimum_percent: MINIMUM_PERCENT,
416        evidence_max_age_days: EVIDENCE_MAX_AGE_DAYS,
417        catalog_entries: catalog.models.len(),
418        fresh_entries,
419        verified_entries,
420        eligible_variants: inventory.len(),
421        covered_variants,
422        coverage_percent,
423        recency_percent,
424        freshness_percent,
425        correctness_percent,
426        passed,
427        blockers,
428        missing_variants,
429        scopes: evidence.scopes,
430        entries,
431    })
432}
433
434pub fn sign_reviewed_catalog(
435    payload: crate::catalog_update::CatalogPayload,
436    evidence: &str,
437    seed: &[u8; 32],
438    public_key: &[u8; 32],
439) -> Result<(Vec<u8>, Vec<u8>), QualityError> {
440    use crate::catalog_update::{SignedCatalog, sign_catalog};
441    use ed25519_dalek::{Signer, SigningKey};
442    let report = evaluate(&payload.catalog, evidence, &payload.published_at)?;
443    if !report.passed {
444        return Err(QualityError(
445            "both scores must be at least 90% with no hard blockers",
446        ));
447    }
448    let revision = payload.revision;
449    let artifact = sign_catalog(payload, seed, public_key)
450        .map_err(|_| QualityError("catalog signing failed"))?;
451    let report_payload = serde_json::json!({
452        "reportType":"rigspark-catalog-quality", "reportVersion":1,
453        "revision":revision, "catalogArtifactSha256":format!("{:x}", Sha256::digest(&artifact)),
454        "quality":report, "evidence":evidence
455    })
456    .to_string();
457    let signature = SigningKey::from_bytes(seed)
458        .sign(report_payload.as_bytes())
459        .to_bytes()
460        .iter()
461        .map(|byte| format!("{byte:02x}"))
462        .collect();
463    let quality = serde_json::to_vec(&SignedCatalog {
464        payload: report_payload,
465        signature,
466    })
467    .map_err(|_| QualityError("report encoding"))?;
468    if quality.len() > MAX_ARTIFACT_BYTES {
469        return Err(QualityError("signed quality report exceeds 16 MiB"));
470    }
471    Ok((artifact, quality))
472}