Skip to main content

rigspark_runtime/
catalog_quality.rs

1use crate::catalog_update::MAX_ARTIFACT_BYTES;
2use rigspark_core::catalog::{Catalog, CatalogModel};
3use serde::{Deserialize, Serialize};
4use sha2::{Digest, Sha256};
5use std::collections::{BTreeMap, BTreeSet};
6use thiserror::Error;
7use time::{OffsetDateTime, format_description::well_known::Rfc3339};
8
9pub const MINIMUM_PERCENT: usize = 90;
10pub const EVIDENCE_MAX_AGE_DAYS: i64 = 7;
11
12#[derive(Debug, Error)]
13#[error("invalid catalog quality evidence: {0}")]
14pub struct QualityError(pub &'static str);
15
16#[derive(Deserialize)]
17#[serde(rename_all = "camelCase", deny_unknown_fields)]
18struct Evidence {
19    policy_version: u8,
20    scopes: Vec<Scope>,
21    observations: Vec<Observation>,
22}
23
24#[derive(Deserialize, Serialize)]
25#[serde(rename_all = "camelCase", deny_unknown_fields)]
26pub struct Scope {
27    name: String,
28    checked_at: String,
29    source: String,
30    complete: bool,
31    variants: Vec<String>,
32}
33
34#[derive(Deserialize)]
35#[serde(rename_all = "camelCase", deny_unknown_fields)]
36struct Observation {
37    id: String,
38    checked_at: String,
39    sources: Vec<String>,
40    model: CatalogModel,
41}
42
43#[derive(Serialize)]
44#[serde(rename_all = "camelCase")]
45pub struct EntryResult {
46    id: String,
47    checked_at: Option<String>,
48    sources: Vec<String>,
49    fresh: bool,
50    verified: bool,
51    reason: &'static str,
52}
53
54#[derive(Serialize)]
55#[serde(rename_all = "camelCase")]
56pub struct QualityReport {
57    pub policy_version: u8,
58    pub evaluated_at: String,
59    pub catalog_sha256: String,
60    pub evidence_sha256: String,
61    pub minimum_percent: usize,
62    pub evidence_max_age_days: i64,
63    pub catalog_entries: usize,
64    pub fresh_entries: usize,
65    pub verified_entries: usize,
66    pub eligible_variants: usize,
67    pub covered_variants: usize,
68    pub coverage_percent: Option<f64>,
69    pub recency_percent: f64,
70    pub freshness_percent: Option<f64>,
71    pub correctness_percent: f64,
72    pub passed: bool,
73    pub blockers: Vec<String>,
74    pub missing_variants: Vec<String>,
75    pub scopes: Vec<Scope>,
76    pub entries: Vec<EntryResult>,
77}
78
79fn recent(raw: &str, now: OffsetDateTime) -> Result<bool, QualityError> {
80    let checked =
81        OffsetDateTime::parse(raw, &Rfc3339).map_err(|_| QualityError("invalid timestamp"))?;
82    if checked > now {
83        return Err(QualityError("future verification date"));
84    }
85    Ok(now - checked <= time::Duration::days(EVIDENCE_MAX_AGE_DAYS))
86}
87
88fn source_url(raw: &str) -> bool {
89    url::Url::parse(raw).is_ok_and(|url| {
90        url.scheme() == "https"
91            && url.host_str().is_some()
92            && url.username().is_empty()
93            && url.password().is_none()
94            && url.port_or_known_default() == Some(443)
95            && url.query().is_none()
96            && url.fragment().is_none()
97    })
98}
99
100fn facts(model: &CatalogModel) -> Result<serde_json::Value, QualityError> {
101    let mut value = serde_json::to_value(model).map_err(|_| QualityError("invalid model facts"))?;
102    for quant in value["quantizations"].as_array_mut().unwrap() {
103        for field in ["minRamBytes", "minVramBytes", "digestVerified"] {
104            quant.as_object_mut().unwrap().remove(field);
105        }
106    }
107    Ok(value)
108}
109
110fn citations_match(observation: &Observation) -> bool {
111    let source = &observation.model.source;
112    let cites_repo = |repo: &str| {
113        let root = format!("https://huggingface.co/{repo}");
114        observation
115            .sources
116            .iter()
117            .any(|url| url == &root || url.starts_with(&format!("{root}/")))
118    };
119    if source.hf.as_deref().is_some_and(|repo| !cites_repo(repo))
120        || source
121            .gguf
122            .as_ref()
123            .is_some_and(|gguf| !cites_repo(&gguf.repo))
124        || source
125            .mlx
126            .as_ref()
127            .is_some_and(|mlx| !cites_repo(&mlx.repo))
128    {
129        return false;
130    }
131    if let Some(reference) = &source.ollama {
132        let (repo, tag) = rigspark_core::registry_collector::parse_reference(reference);
133        let repo = if repo.contains('/') {
134            repo.to_string()
135        } else {
136            format!("library/{repo}")
137        };
138        let manifest = format!("https://registry.ollama.ai/v2/{repo}/manifests/{tag}");
139        if !observation.sources.contains(&manifest) {
140            return false;
141        }
142        if source.hf.is_none() {
143            let page = format!("https://ollama.com/{repo}:{tag}");
144            if !observation.sources.contains(&page) {
145                return false;
146            }
147        }
148    }
149    source.hf.is_some() || source.ollama.is_some() || source.gguf.is_some() || source.mlx.is_some()
150}
151
152fn matches_facts(model: &CatalogModel, observed: &CatalogModel) -> Result<bool, QualityError> {
153    let mut observed = observed.clone();
154    if model.kv_bytes_per_token.is_none() {
155        observed.kv_bytes_per_token = None;
156    }
157    if model.benchmark_proxy.is_none() {
158        observed.benchmark_proxy = None;
159    }
160    Ok(facts(model)? == facts(&observed)?)
161}
162
163fn pinned(model: &CatalogModel) -> bool {
164    if model.source.ollama.is_some() {
165        model
166            .quantizations
167            .iter()
168            .all(|quant| quant.sha256.is_some())
169    } else {
170        model.source.gguf.is_some() || model.source.mlx.is_some()
171    }
172}
173
174fn percent(numerator: usize, denominator: usize) -> f64 {
175    if denominator == 0 {
176        0.0
177    } else {
178        numerator as f64 * 100.0 / denominator as f64
179    }
180}
181
182pub fn evaluate(
183    catalog_raw: &str,
184    evidence_raw: &str,
185    now: &str,
186) -> Result<QualityReport, QualityError> {
187    if catalog_raw.len() > MAX_ARTIFACT_BYTES || evidence_raw.len() > MAX_ARTIFACT_BYTES {
188        return Err(QualityError("input exceeds 16 MiB"));
189    }
190    let catalog = Catalog::parse(catalog_raw).map_err(|_| QualityError("catalog schema"))?;
191    if catalog.models.is_empty() {
192        return Err(QualityError("empty catalog"));
193    }
194    let evidence: Evidence = rigspark_core::catalog::parse_document(evidence_raw)
195        .map_err(|_| QualityError("evidence schema"))?;
196    if evidence.policy_version != 1
197        || evidence.scopes.len() > 32
198        || evidence.observations.len() > 10000
199    {
200        return Err(QualityError("unsupported policy or collection limit"));
201    }
202    let clock =
203        OffsetDateTime::parse(now, &Rfc3339).map_err(|_| QualityError("evaluation date"))?;
204    recent(&catalog.generated_at, clock)?;
205    let mut blockers = Vec::new();
206    let mut inventory = BTreeSet::new();
207    let mut names = BTreeSet::new();
208    let mut inventory_known = !evidence.scopes.is_empty();
209    for scope in &evidence.scopes {
210        if scope.name.is_empty()
211            || !names.insert(scope.name.clone())
212            || !source_url(&scope.source)
213            || scope.variants.len() > 10000
214        {
215            return Err(QualityError("invalid coverage scope"));
216        }
217        if !recent(&scope.checked_at, clock)? || !scope.complete || scope.variants.is_empty() {
218            inventory_known = false;
219        }
220        for variant in &scope.variants {
221            if variant.is_empty()
222                || variant.len() > 256
223                || variant.chars().any(char::is_control)
224                || !inventory.insert(variant.clone())
225            {
226                return Err(QualityError("duplicate or invalid inventory variant"));
227            }
228        }
229    }
230    if !names.contains("ollama-local-variants") {
231        inventory_known = false;
232    }
233    if !inventory_known {
234        blockers.push(
235            "Upstream variant inventory is missing, incomplete, empty, or older than 7 days."
236                .into(),
237        );
238    }
239    let models: BTreeMap<_, _> = catalog
240        .models
241        .iter()
242        .map(|model| (model.id.as_str(), model))
243        .collect();
244    let mut observations = BTreeMap::new();
245    for observation in &evidence.observations {
246        if observation.id != observation.model.id
247            || !models.contains_key(observation.id.as_str())
248            || observations
249                .insert(observation.id.as_str(), observation)
250                .is_some()
251            || observation.sources.is_empty()
252            || observation.sources.len() > 16
253            || !observation.sources.iter().all(|source| source_url(source))
254            || !citations_match(observation)
255        {
256            return Err(QualityError("duplicate, unbound, or uncited observation"));
257        }
258        let observed_catalog = serde_json::json!({"schemaVersion":2,"generatedAt":observation.checked_at,"models":[observation.model]});
259        Catalog::parse(&observed_catalog.to_string())
260            .map_err(|_| QualityError("observed model facts"))?;
261        recent(&observation.checked_at, clock)?;
262    }
263    let mut entries = Vec::new();
264    let mut fresh_entries = 0;
265    let mut verified_entries = 0;
266    for model in &catalog.models {
267        let mut entry = EntryResult {
268            id: model.id.clone(),
269            checked_at: None,
270            sources: Vec::new(),
271            fresh: false,
272            verified: false,
273            reason: "missing evidence",
274        };
275        if inventory_known && !inventory.contains(&model.id) {
276            blockers.push(format!(
277                "{}: catalog entry excluded from declared coverage scope",
278                model.id
279            ));
280        }
281        if let Some(observation) = observations.get(model.id.as_str()) {
282            entry.checked_at = Some(observation.checked_at.clone());
283            entry.sources = observation.sources.clone();
284            entry.fresh = recent(&observation.checked_at, clock)?;
285            if !matches_facts(model, &observation.model)? {
286                entry.reason = "authoritative facts contradict catalog";
287                blockers.push(format!(
288                    "{}: authoritative facts contradict catalog",
289                    model.id
290                ));
291            } else if !pinned(model) {
292                entry.reason = "required artifact integrity evidence missing";
293            } else if !entry.fresh {
294                entry.reason = "verification evidence older than 7 days";
295            } else {
296                entry.verified = true;
297                entry.reason = "verified";
298            }
299        }
300        fresh_entries += usize::from(entry.fresh);
301        verified_entries += usize::from(entry.verified);
302        entries.push(entry);
303    }
304    let missing_variants: Vec<_> = inventory
305        .iter()
306        .filter(|id| !models.contains_key(id.as_str()))
307        .cloned()
308        .collect();
309    let covered_variants = inventory.len() - missing_variants.len();
310    let coverage_percent = inventory_known.then(|| percent(covered_variants, inventory.len()));
311    let recency_percent = percent(fresh_entries, catalog.models.len());
312    let freshness_percent = coverage_percent.map(|coverage| coverage.min(recency_percent));
313    let correctness_percent = percent(verified_entries, catalog.models.len());
314    let passed = blockers.is_empty()
315        && inventory_known
316        && covered_variants * 100 >= inventory.len() * MINIMUM_PERCENT
317        && fresh_entries * 100 >= catalog.models.len() * MINIMUM_PERCENT
318        && verified_entries * 100 >= catalog.models.len() * MINIMUM_PERCENT;
319    Ok(QualityReport {
320        policy_version: 1,
321        evaluated_at: now.into(),
322        catalog_sha256: format!("{:x}", Sha256::digest(catalog_raw.as_bytes())),
323        evidence_sha256: format!("{:x}", Sha256::digest(evidence_raw.as_bytes())),
324        minimum_percent: MINIMUM_PERCENT,
325        evidence_max_age_days: EVIDENCE_MAX_AGE_DAYS,
326        catalog_entries: catalog.models.len(),
327        fresh_entries,
328        verified_entries,
329        eligible_variants: inventory.len(),
330        covered_variants,
331        coverage_percent,
332        recency_percent,
333        freshness_percent,
334        correctness_percent,
335        passed,
336        blockers,
337        missing_variants,
338        scopes: evidence.scopes,
339        entries,
340    })
341}
342
343pub fn sign_reviewed_catalog(
344    payload: crate::catalog_update::CatalogPayload,
345    evidence: &str,
346    seed: &[u8; 32],
347    public_key: &[u8; 32],
348) -> Result<(Vec<u8>, Vec<u8>), QualityError> {
349    use crate::catalog_update::{SignedCatalog, sign_catalog};
350    use ed25519_dalek::{Signer, SigningKey};
351    let report = evaluate(&payload.catalog, evidence, &payload.published_at)?;
352    if !report.passed {
353        return Err(QualityError(
354            "both scores must be at least 90% with no hard blockers",
355        ));
356    }
357    let revision = payload.revision;
358    let artifact = sign_catalog(payload, seed, public_key)
359        .map_err(|_| QualityError("catalog signing failed"))?;
360    let report_payload = serde_json::json!({
361        "reportType":"rigspark-catalog-quality", "reportVersion":1,
362        "revision":revision, "catalogArtifactSha256":format!("{:x}", Sha256::digest(&artifact)),
363        "quality":report, "evidence":evidence
364    })
365    .to_string();
366    let signature = SigningKey::from_bytes(seed)
367        .sign(report_payload.as_bytes())
368        .to_bytes()
369        .iter()
370        .map(|byte| format!("{byte:02x}"))
371        .collect();
372    let quality = serde_json::to_vec(&SignedCatalog {
373        payload: report_payload,
374        signature,
375    })
376    .map_err(|_| QualityError("report encoding"))?;
377    if quality.len() > MAX_ARTIFACT_BYTES {
378        return Err(QualityError("signed quality report exceeds 16 MiB"));
379    }
380    Ok((artifact, quality))
381}