1use crate::admission::{AdmissionTransport, Fetched};
2use rigspark_core::{
3 admission::{Gguf, GgufError, active_params, kv_bytes_per_token, license_id, parse_gguf},
4 catalog::{Catalog, CatalogModel, EntryProvenance, GgufSource, SCHEMA_VERSION, Source},
5 catalog_request::Repository,
6 enrich::sized_quantization,
7 generation::GenerationCatalog,
8 generation_admission::{HfModel, MAX_METADATA_BYTES, kind_from_pipeline, license_from_tags},
9 sizing::Architecture,
10};
11use serde::Serialize;
12use serde_json::{Value, json};
13use std::{io, time::Duration};
14use time::{OffsetDateTime, format_description::well_known::Rfc3339};
15use tokio_util::sync::CancellationToken;
16use url::Url;
17
18mod generation;
19
20const GGUF_STEPS: [usize; 3] = [8 * 1024 * 1024, 32 * 1024 * 1024, 64 * 1024 * 1024];
21
22#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
23#[serde(rename_all = "camelCase")]
24pub enum RequestDisposition {
25 Eligible,
26 NeedsRunnableArtifact,
27 NeedsWorkflow,
28 NeedsMaintainer,
29 Rejected,
30 AlreadyAvailable,
31}
32
33#[derive(Serialize)]
34#[serde(rename_all = "camelCase")]
35pub struct RequestOutcome {
36 pub schema_version: u8,
37 pub repository: String,
38 pub disposition: RequestDisposition,
39 pub reason_code: &'static str,
40 pub summary: String,
41 pub model_ids: Vec<String>,
42 pub evidence_urls: Vec<String>,
43 pub changed_files: Vec<&'static str>,
44 #[serde(skip)]
45 pub catalog: Option<Catalog>,
46 #[serde(skip)]
47 pub observation: Option<Value>,
48}
49
50impl RequestOutcome {
51 fn no_change(
52 repository: &Repository,
53 disposition: RequestDisposition,
54 reason_code: &'static str,
55 summary: impl Into<String>,
56 ) -> Self {
57 Self {
58 schema_version: 1,
59 repository: repository.as_str().into(),
60 disposition,
61 reason_code,
62 summary: summary.into(),
63 model_ids: Vec::new(),
64 evidence_urls: vec![repository.url()],
65 changed_files: Vec::new(),
66 catalog: None,
67 observation: None,
68 }
69 }
70}
71
72async fn get(
73 transport: &dyn AdmissionTransport,
74 cancel: &CancellationToken,
75 raw: &str,
76 range: Option<usize>,
77 limit: usize,
78) -> io::Result<Vec<u8>> {
79 if cancel.is_cancelled() {
80 return Err(io::Error::new(
81 io::ErrorKind::Interrupted,
82 "catalog request cancelled",
83 ));
84 }
85 let url = Url::parse(raw).map_err(io::Error::other)?;
86 let Fetched { status, body } = tokio::select! {
87 biased;
88 _ = cancel.cancelled() => {
89 return Err(io::Error::new(io::ErrorKind::Interrupted, "catalog request cancelled"));
90 }
91 result = tokio::time::timeout(
92 Duration::from_secs(180),
93 transport.fetch(&url, None, range, limit),
94 ) => result.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "catalog request timed out"))??,
95 };
96 let accepted = if range.is_some() {
97 matches!(status, 200 | 206)
98 } else {
99 (200..300).contains(&status)
100 };
101 if !accepted {
102 return Err(io::Error::new(
103 io::ErrorKind::NotFound,
104 format!("upstream status {status}"),
105 ));
106 }
107 Ok(body)
108}
109
110async fn gguf(
111 transport: &dyn AdmissionTransport,
112 cancel: &CancellationToken,
113 url: &str,
114) -> io::Result<Result<Gguf, &'static str>> {
115 for step in GGUF_STEPS {
116 let bytes = get(transport, cancel, url, Some(step), step).await?;
117 match parse_gguf(&bytes) {
118 Ok(parsed) => return Ok(Ok(parsed)),
119 Err(GgufError::Truncated) if bytes.len() >= step => {}
120 Err(GgufError::Truncated) => return Ok(Err("GGUF header truncated")),
121 Err(GgufError::Invalid(reason)) => return Ok(Err(reason)),
122 }
123 }
124 Ok(Err("GGUF header exceeds 64 MiB"))
125}
126
127pub async fn evaluate_request(
128 catalog: &Catalog,
129 generation_catalog: &GenerationCatalog,
130 repository: &Repository,
131 transport: &dyn AdmissionTransport,
132 now: &str,
133 cancel: &CancellationToken,
134) -> io::Result<RequestOutcome> {
135 if let Some(outcome) = generation::direct_match(generation_catalog, repository) {
136 return Ok(outcome);
137 }
138 if catalog.models.iter().any(|model| {
139 model.source.hf.as_deref() == Some(repository.as_str())
140 || model
141 .source
142 .gguf
143 .as_ref()
144 .is_some_and(|source| source.repo == repository.as_str())
145 || model
146 .source
147 .mlx
148 .as_ref()
149 .is_some_and(|source| source.repo == repository.as_str())
150 }) {
151 return Ok(RequestOutcome::no_change(
152 repository,
153 RequestDisposition::AlreadyAvailable,
154 "ALREADY_AVAILABLE",
155 "The requested repository is already represented in the catalog.",
156 ));
157 }
158
159 let metadata_url = format!(
160 "https://huggingface.co/api/models/{}?blobs=true",
161 repository.as_str()
162 );
163 let metadata = get(transport, cancel, &metadata_url, None, MAX_METADATA_BYTES).await?;
164 let model = HfModel::parse(
165 std::str::from_utf8(&metadata)
166 .map_err(|_| io::Error::other("Hugging Face metadata is not UTF-8"))?,
167 )
168 .map_err(io::Error::other)?;
169 if model.id != repository.as_str() || !model.pinned() {
170 return Ok(RequestOutcome::no_change(
171 repository,
172 RequestDisposition::Rejected,
173 "PUBLIC_PINNED_REVISION_REQUIRED",
174 "The repository must be public, ungated, and pinned by Hugging Face.",
175 ));
176 }
177 if let Some(kind) = kind_from_pipeline(model.pipeline_tag.as_deref()) {
178 return generation::evaluate(generation_catalog, repository, kind, transport, cancel).await;
179 }
180 if model.pipeline_tag.as_deref() != Some("text-generation") {
181 return Ok(RequestOutcome::no_change(
182 repository,
183 RequestDisposition::Rejected,
184 "UNSUPPORTED_PIPELINE",
185 "The repository is not identified as text generation, text-to-image, or text-to-video.",
186 ));
187 }
188 let Some(tag_license) = license_from_tags(&model.tags) else {
189 return Ok(RequestOutcome::no_change(
190 repository,
191 RequestDisposition::Rejected,
192 "OPEN_LICENSE_REQUIRED",
193 "The repository does not declare one unambiguous supported open-weight license.",
194 ));
195 };
196 let Some((file, artifact)) = model
197 .siblings
198 .iter()
199 .filter_map(|sibling| {
200 let lower = sibling.rfilename.to_ascii_lowercase();
201 (lower.ends_with("q4_k_m.gguf")).then_some((sibling, sibling.lfs.as_ref()?))
202 })
203 .min_by_key(|(sibling, artifact)| (artifact.size, sibling.rfilename.as_str()))
204 else {
205 return Ok(RequestOutcome::no_change(
206 repository,
207 RequestDisposition::NeedsRunnableArtifact,
208 "RUNNABLE_ARTIFACT_MISSING",
209 "No pinned Q4_K_M GGUF artifact was published in the requested repository.",
210 ));
211 };
212 if artifact.size == 0
213 || artifact.size > 9_007_199_254_740_991
214 || artifact.sha256.len() != 64
215 || !artifact.sha256.bytes().all(|byte| byte.is_ascii_hexdigit())
216 {
217 return Ok(RequestOutcome::no_change(
218 repository,
219 RequestDisposition::Rejected,
220 "ARTIFACT_INTEGRITY_INVALID",
221 "The selected GGUF artifact lacks valid size or SHA-256 evidence.",
222 ));
223 }
224 let artifact_url = format!(
225 "https://huggingface.co/{}/resolve/{}/{}",
226 repository.as_str(),
227 model.sha,
228 file.rfilename
229 );
230 let parsed = match gguf(transport, cancel, &artifact_url).await? {
231 Ok(parsed) => parsed,
232 Err(reason) => {
233 return Ok(RequestOutcome::no_change(
234 repository,
235 RequestDisposition::Rejected,
236 "GGUF_METADATA_INVALID",
237 format!("The selected GGUF cannot be admitted: {reason}."),
238 ));
239 }
240 };
241 let today = today(now)?;
242 let selected = SelectedGguf {
243 file: &file.rfilename,
244 sha256: &artifact.sha256,
245 bytes: artifact.size,
246 };
247 let entry = match build_entry(repository, &model, &parsed, &selected, tag_license, &today) {
248 Ok(entry) => entry,
249 Err(reason) => {
250 return Ok(RequestOutcome::no_change(
251 repository,
252 RequestDisposition::Rejected,
253 "MODEL_FACTS_INCOMPLETE",
254 reason,
255 ));
256 }
257 };
258 if catalog.models.iter().any(|model| model.id == entry.id) {
259 return Ok(RequestOutcome::no_change(
260 repository,
261 RequestDisposition::Rejected,
262 "MODEL_ID_CONFLICT",
263 format!("The generated model id {} already exists.", entry.id),
264 ));
265 }
266
267 let mut models = catalog.models.clone();
268 let model_id = entry.id.clone();
269 models.push(entry.clone());
270 models.sort_by(|left, right| left.id.cmp(&right.id));
271 let stamp = stamp(now)?;
272 let encoded = json!({
273 "schemaVersion": SCHEMA_VERSION,
274 "generatedAt": stamp,
275 "models": models,
276 });
277 let updated = Catalog::parse(&encoded.to_string()).map_err(io::Error::other)?;
278 let mut sources = vec![repository.url(), artifact_url];
279 if let Some(base) = &entry.source.hf {
280 let base_url = format!("https://huggingface.co/{base}");
281 if !sources.contains(&base_url) {
282 sources.push(base_url);
283 }
284 }
285 let observation = json!({
286 "id": entry.id,
287 "checkedAt": stamp,
288 "sources": sources,
289 "model": entry,
290 });
291 Ok(RequestOutcome {
292 schema_version: 1,
293 repository: repository.as_str().into(),
294 disposition: RequestDisposition::Eligible,
295 reason_code: "ELIGIBLE",
296 summary: format!("Validated source-backed catalog entry {model_id}."),
297 model_ids: vec![model_id],
298 evidence_urls: sources,
299 changed_files: vec![
300 "crates/rigspark-core/data/models.json",
301 "docs/references/catalog-quality-evidence.json",
302 ],
303 catalog: Some(updated),
304 observation: Some(observation),
305 })
306}
307
308struct SelectedGguf<'a> {
309 file: &'a str,
310 sha256: &'a str,
311 bytes: u64,
312}
313
314fn build_entry(
315 repository: &Repository,
316 model: &HfModel,
317 gguf: &Gguf,
318 artifact: &SelectedGguf<'_>,
319 tag_license: &str,
320 today: &str,
321) -> Result<CatalogModel, String> {
322 let architecture = gguf
323 .architecture()
324 .ok_or_else(|| "GGUF architecture is missing.".to_string())?;
325 if let Some(raw) = gguf.text("general.license") {
326 let gguf_license = license_id(Some(raw), gguf.text("general.license.name"), None)
327 .map_err(|_| "GGUF license is not supported.".to_string())?;
328 if gguf_license != tag_license {
329 return Err("Hugging Face and GGUF license evidence disagree.".into());
330 }
331 }
332 let context = gguf
333 .u64(&format!("{architecture}.context_length"))
334 .filter(|tokens| *tokens > 0)
335 .ok_or_else(|| "GGUF context length is missing.".to_string())?;
336 let total = gguf.total_elements();
337 if total == 0 {
338 return Err("GGUF tensor count is missing.".into());
339 }
340 let (architecture_kind, active_params) = if gguf
341 .u64(&format!("{architecture}.expert_count"))
342 .is_some_and(|count| count > 0)
343 {
344 let active = active_params(gguf)
345 .ok_or_else(|| "MoE active parameter count is not sourced.".to_string())?;
346 (Architecture::Moe, Some(parameter_label(active)))
347 } else {
348 (Architecture::Dense, None)
349 };
350 let quantization = sized_quantization(
351 total as f64,
352 &architecture_kind,
353 "Q4_K_M",
354 artifact.bytes as f64,
355 Some(artifact.sha256.to_owned()),
356 Vec::new(),
357 )
358 .map_err(|error| error.to_string())?;
359 let source_hf = model
360 .base_model()
361 .unwrap_or_else(|| repository.as_str().to_owned());
362 let id = format!("{}:q4_k_m", source_hf.to_ascii_lowercase());
363 Ok(CatalogModel {
364 id,
365 family: architecture.into(),
366 params: parameter_label(total as f64),
367 architecture: architecture_kind,
368 active_params,
369 license: tag_license.into(),
370 open_weight: true,
371 context_length: context as f64,
372 capabilities: vec!["chat".into()],
373 release_date: None,
374 added_at: Some(today.into()),
375 provenance: EntryProvenance::Auto,
376 source: Source {
377 hf: Some(source_hf),
378 gguf: Some(GgufSource {
379 repo: repository.as_str().into(),
380 revision: model.sha.clone(),
381 file: artifact.file.into(),
382 sha256: artifact.sha256.into(),
383 }),
384 ..Default::default()
385 },
386 quantizations: vec![quantization],
387 kv_bytes_per_token: kv_bytes_per_token(gguf),
388 benchmark_proxy: None,
389 })
390}
391
392fn parameter_label(count: f64) -> String {
393 let (value, unit) = if count >= 1e12 {
394 (count / 1e12, "T")
395 } else if count >= 1e9 {
396 (count / 1e9, "B")
397 } else {
398 (count / 1e6, "M")
399 };
400 let text = format!("{value:.1}");
401 format!("{}{unit}", text.strip_suffix(".0").unwrap_or(&text))
402}
403
404fn clock(raw: &str) -> io::Result<OffsetDateTime> {
405 OffsetDateTime::parse(raw, &Rfc3339)
406 .map(|clock| clock.to_offset(time::UtcOffset::UTC))
407 .map_err(|_| io::Error::other("invalid request clock"))
408}
409
410fn stamp(raw: &str) -> io::Result<String> {
411 clock(raw)?
412 .format(time::macros::format_description!(
413 "[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
414 ))
415 .map_err(io::Error::other)
416}
417
418fn today(raw: &str) -> io::Result<String> {
419 Ok(stamp(raw)?[..10].into())
420}