1use crate::admission::{AdmissionTransport, Fetched};
2use rigspark_core::{
3 admission::{Gguf, GgufError, active_params, kv_bytes_per_token, license_id, parse_gguf},
4 catalog::{Catalog, CatalogModel, EntryProvenance, GgufSource, Source},
5 catalog_request::Repository,
6 enrich::sized_quantization,
7 generation::GenerationCatalog,
8 generation_admission::{HfModel, MAX_METADATA_BYTES, kind_from_pipeline, license_from_tags},
9 sizing::Architecture,
10};
11use serde::Serialize;
12use serde_json::{Value, json};
13use std::{io, time::Duration};
14use time::{OffsetDateTime, format_description::well_known::Rfc3339};
15use tokio_util::sync::CancellationToken;
16use url::Url;
17
18mod generation;
19
20const GGUF_STEPS: [usize; 3] = [8 * 1024 * 1024, 32 * 1024 * 1024, 64 * 1024 * 1024];
21
22#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
23#[serde(rename_all = "camelCase")]
24pub enum RequestDisposition {
25 Eligible,
26 NeedsRunnableArtifact,
27 NeedsWorkflow,
28 NeedsMaintainer,
29 Rejected,
30 AlreadyAvailable,
31}
32
33#[derive(Serialize)]
34#[serde(rename_all = "camelCase")]
35pub struct RequestOutcome {
36 pub schema_version: u8,
37 pub repository: String,
38 pub disposition: RequestDisposition,
39 pub reason_code: &'static str,
40 pub summary: String,
41 pub model_ids: Vec<String>,
42 pub evidence_urls: Vec<String>,
43 pub changed_files: Vec<&'static str>,
44 #[serde(skip)]
45 pub catalog: Option<Catalog>,
46 #[serde(skip)]
47 pub observation: Option<Value>,
48}
49
50impl RequestOutcome {
51 fn no_change(
52 repository: &Repository,
53 disposition: RequestDisposition,
54 reason_code: &'static str,
55 summary: impl Into<String>,
56 ) -> Self {
57 Self {
58 schema_version: 1,
59 repository: repository.as_str().into(),
60 disposition,
61 reason_code,
62 summary: summary.into(),
63 model_ids: Vec::new(),
64 evidence_urls: vec![repository.url()],
65 changed_files: Vec::new(),
66 catalog: None,
67 observation: None,
68 }
69 }
70}
71
72pub(crate) async fn get(
73 transport: &dyn AdmissionTransport,
74 cancel: &CancellationToken,
75 raw: &str,
76 range: Option<usize>,
77 limit: usize,
78) -> io::Result<Vec<u8>> {
79 if cancel.is_cancelled() {
80 return Err(io::Error::new(
81 io::ErrorKind::Interrupted,
82 "catalog request cancelled",
83 ));
84 }
85 let url = Url::parse(raw).map_err(io::Error::other)?;
86 let Fetched { status, body } = tokio::select! {
87 biased;
88 _ = cancel.cancelled() => {
89 return Err(io::Error::new(io::ErrorKind::Interrupted, "catalog request cancelled"));
90 }
91 result = tokio::time::timeout(
92 Duration::from_secs(180),
93 transport.fetch(&url, None, range, limit),
94 ) => result.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "catalog request timed out"))??,
95 };
96 let accepted = if range.is_some() {
97 matches!(status, 200 | 206)
98 } else {
99 (200..300).contains(&status)
100 };
101 if !accepted {
102 return Err(io::Error::new(
103 io::ErrorKind::NotFound,
104 format!("upstream status {status}"),
105 ));
106 }
107 Ok(body)
108}
109
110pub(crate) async fn gguf(
111 transport: &dyn AdmissionTransport,
112 cancel: &CancellationToken,
113 url: &str,
114) -> io::Result<Result<Gguf, &'static str>> {
115 for step in GGUF_STEPS {
116 let bytes = get(transport, cancel, url, Some(step), step).await?;
117 match parse_gguf(&bytes) {
118 Ok(parsed) => return Ok(Ok(parsed)),
119 Err(GgufError::Truncated) if bytes.len() >= step => {}
120 Err(GgufError::Truncated) => return Ok(Err("GGUF header truncated")),
121 Err(GgufError::Invalid(reason)) => return Ok(Err(reason)),
122 }
123 }
124 Ok(Err("GGUF header exceeds 64 MiB"))
125}
126
127pub async fn evaluate_request(
128 catalog: &Catalog,
129 generation_catalog: &GenerationCatalog,
130 repository: &Repository,
131 transport: &dyn AdmissionTransport,
132 now: &str,
133 cancel: &CancellationToken,
134) -> io::Result<RequestOutcome> {
135 if let Some(outcome) = generation::direct_match(generation_catalog, repository) {
136 return Ok(outcome);
137 }
138 if catalog.models.iter().any(|model| {
139 model.source.hf.as_deref() == Some(repository.as_str())
140 || model
141 .source
142 .gguf
143 .as_ref()
144 .is_some_and(|source| source.repo == repository.as_str())
145 || model
146 .source
147 .mlx
148 .as_ref()
149 .is_some_and(|source| source.repo == repository.as_str())
150 }) {
151 return Ok(RequestOutcome::no_change(
152 repository,
153 RequestDisposition::AlreadyAvailable,
154 "ALREADY_AVAILABLE",
155 "The requested repository is already represented in the catalog.",
156 ));
157 }
158
159 let metadata_url = format!(
160 "https://huggingface.co/api/models/{}?blobs=true",
161 repository.as_str()
162 );
163 let metadata = get(transport, cancel, &metadata_url, None, MAX_METADATA_BYTES).await?;
164 let model = HfModel::parse(
165 std::str::from_utf8(&metadata)
166 .map_err(|_| io::Error::other("Hugging Face metadata is not UTF-8"))?,
167 )
168 .map_err(io::Error::other)?;
169 if model.id != repository.as_str() || !model.pinned() {
170 return Ok(RequestOutcome::no_change(
171 repository,
172 RequestDisposition::Rejected,
173 "PUBLIC_PINNED_REVISION_REQUIRED",
174 "The repository must be public, ungated, and pinned by Hugging Face.",
175 ));
176 }
177 if let Some(kind) = kind_from_pipeline(model.pipeline_tag.as_deref()) {
178 return generation::evaluate(generation_catalog, repository, kind, transport, cancel).await;
179 }
180 if model.pipeline_tag.as_deref() != Some("text-generation") {
181 return Ok(RequestOutcome::no_change(
182 repository,
183 RequestDisposition::Rejected,
184 "UNSUPPORTED_PIPELINE",
185 "The repository is not identified as text generation, text-to-image, or text-to-video.",
186 ));
187 }
188 let Some(tag_license) = license_from_tags(&model.tags) else {
189 return Ok(RequestOutcome::no_change(
190 repository,
191 RequestDisposition::Rejected,
192 "OPEN_LICENSE_REQUIRED",
193 "The repository does not declare one unambiguous supported open-weight license.",
194 ));
195 };
196 let Some((file, artifact)) = model
197 .siblings
198 .iter()
199 .filter_map(|sibling| {
200 let lower = sibling.rfilename.to_ascii_lowercase();
201 (lower.ends_with("q4_k_m.gguf")).then_some((sibling, sibling.lfs.as_ref()?))
202 })
203 .min_by_key(|(sibling, artifact)| (artifact.size, sibling.rfilename.as_str()))
204 else {
205 return Ok(RequestOutcome::no_change(
206 repository,
207 RequestDisposition::NeedsRunnableArtifact,
208 "RUNNABLE_ARTIFACT_MISSING",
209 "No pinned Q4_K_M GGUF artifact was published in the requested repository.",
210 ));
211 };
212 if artifact.size == 0
213 || artifact.size > 9_007_199_254_740_991
214 || artifact.sha256.len() != 64
215 || !artifact.sha256.bytes().all(|byte| byte.is_ascii_hexdigit())
216 {
217 return Ok(RequestOutcome::no_change(
218 repository,
219 RequestDisposition::Rejected,
220 "ARTIFACT_INTEGRITY_INVALID",
221 "The selected GGUF artifact lacks valid size or SHA-256 evidence.",
222 ));
223 }
224 let artifact_url = format!(
225 "https://huggingface.co/{}/resolve/{}/{}",
226 repository.as_str(),
227 model.sha,
228 file.rfilename
229 );
230 let parsed = match gguf(transport, cancel, &artifact_url).await? {
231 Ok(parsed) => parsed,
232 Err(reason) => {
233 return Ok(RequestOutcome::no_change(
234 repository,
235 RequestDisposition::Rejected,
236 "GGUF_METADATA_INVALID",
237 format!("The selected GGUF cannot be admitted: {reason}."),
238 ));
239 }
240 };
241 let today = today(now)?;
242 let selected = SelectedGguf {
243 file: &file.rfilename,
244 sha256: &artifact.sha256,
245 bytes: artifact.size,
246 quant: "Q4_K_M",
247 };
248 let entry = match build_entry(repository, &model, &parsed, &selected, tag_license, &today) {
249 Ok(entry) => entry,
250 Err(reason) => {
251 return Ok(RequestOutcome::no_change(
252 repository,
253 RequestDisposition::Rejected,
254 "MODEL_FACTS_INCOMPLETE",
255 reason,
256 ));
257 }
258 };
259 if catalog.models.iter().any(|model| model.id == entry.id) {
260 return Ok(RequestOutcome::no_change(
261 repository,
262 RequestDisposition::Rejected,
263 "MODEL_ID_CONFLICT",
264 format!("The generated model id {} already exists.", entry.id),
265 ));
266 }
267
268 let mut models = catalog.models.clone();
269 let model_id = entry.id.clone();
270 models.push(entry.clone());
271 models.sort_by(|left, right| left.id.cmp(&right.id));
272 let stamp = stamp(now)?;
273 let encoded = json!({
274 "schemaVersion": rigspark_core::catalog::required_schema_version(&models),
275 "generatedAt": stamp,
276 "models": models,
277 });
278 let updated = Catalog::parse(&encoded.to_string()).map_err(io::Error::other)?;
279 let mut sources = vec![repository.url(), artifact_url];
280 if let Some(base) = &entry.source.hf {
281 let base_url = format!("https://huggingface.co/{base}");
282 if !sources.contains(&base_url) {
283 sources.push(base_url);
284 }
285 }
286 let observation = json!({
287 "id": entry.id,
288 "checkedAt": stamp,
289 "sources": sources,
290 "model": entry,
291 });
292 Ok(RequestOutcome {
293 schema_version: 1,
294 repository: repository.as_str().into(),
295 disposition: RequestDisposition::Eligible,
296 reason_code: "ELIGIBLE",
297 summary: format!("Validated source-backed catalog entry {model_id}."),
298 model_ids: vec![model_id],
299 evidence_urls: sources,
300 changed_files: vec![
301 "crates/rigspark-core/data/models.json",
302 "docs/references/catalog-quality-evidence.json",
303 ],
304 catalog: Some(updated),
305 observation: Some(observation),
306 })
307}
308
309pub(crate) struct SelectedGguf<'a> {
310 pub(crate) file: &'a str,
311 pub(crate) sha256: &'a str,
312 pub(crate) bytes: u64,
313 pub(crate) quant: &'a str,
314}
315
316pub(crate) fn build_entry(
317 repository: &Repository,
318 model: &HfModel,
319 gguf: &Gguf,
320 artifact: &SelectedGguf<'_>,
321 tag_license: &str,
322 today: &str,
323) -> Result<CatalogModel, String> {
324 let architecture = gguf
325 .architecture()
326 .ok_or_else(|| "GGUF architecture is missing.".to_string())?;
327 if let Some(raw) = gguf.text("general.license") {
328 let gguf_license = license_id(Some(raw), gguf.text("general.license.name"), None)
329 .map_err(|_| "GGUF license is not supported.".to_string())?;
330 if gguf_license != tag_license {
331 return Err("Hugging Face and GGUF license evidence disagree.".into());
332 }
333 }
334 let context = gguf
335 .u64(&format!("{architecture}.context_length"))
336 .filter(|tokens| *tokens > 0)
337 .ok_or_else(|| "GGUF context length is missing.".to_string())?;
338 let total = gguf.total_elements();
339 if total == 0 {
340 return Err("GGUF tensor count is missing.".into());
341 }
342 let (architecture_kind, active_params) = if gguf
343 .u64(&format!("{architecture}.expert_count"))
344 .is_some_and(|count| count > 0)
345 {
346 let active = active_params(gguf)
347 .ok_or_else(|| "MoE active parameter count is not sourced.".to_string())?;
348 (Architecture::Moe, Some(parameter_label(active)))
349 } else {
350 (Architecture::Dense, None)
351 };
352 let quantization = sized_quantization(
353 total as f64,
354 &architecture_kind,
355 artifact.quant,
356 artifact.bytes as f64,
357 Some(artifact.sha256.to_owned()),
358 Vec::new(),
359 )
360 .map_err(|error| error.to_string())?;
361 let source_hf = model
362 .base_model()
363 .unwrap_or_else(|| repository.as_str().to_owned());
364 let id = format!(
365 "{}:{}",
366 source_hf.to_ascii_lowercase(),
367 artifact.quant.to_ascii_lowercase()
368 );
369 Ok(CatalogModel {
370 id,
371 family: architecture.into(),
372 params: parameter_label(total as f64),
373 architecture: architecture_kind,
374 active_params,
375 license: tag_license.into(),
376 open_weight: true,
377 context_length: context as f64,
378 capabilities: vec!["chat".into()],
379 release_date: None,
380 added_at: Some(today.into()),
381 provenance: EntryProvenance::Auto,
382 source: Source {
383 hf: Some(source_hf),
384 gguf: Some(GgufSource {
385 repo: repository.as_str().into(),
386 revision: model.sha.clone(),
387 file: artifact.file.into(),
388 sha256: artifact.sha256.into(),
389 }),
390 ..Default::default()
391 },
392 quantizations: vec![quantization],
393 kv_bytes_per_token: kv_bytes_per_token(gguf),
394 benchmark_proxy: None,
395 drafter: None,
396 })
397}
398
399fn parameter_label(count: f64) -> String {
400 let (value, unit) = if count >= 1e12 {
401 (count / 1e12, "T")
402 } else if count >= 1e9 {
403 (count / 1e9, "B")
404 } else {
405 (count / 1e6, "M")
406 };
407 let text = format!("{value:.1}");
408 format!("{}{unit}", text.strip_suffix(".0").unwrap_or(&text))
409}
410
411fn clock(raw: &str) -> io::Result<OffsetDateTime> {
412 OffsetDateTime::parse(raw, &Rfc3339)
413 .map(|clock| clock.to_offset(time::UtcOffset::UTC))
414 .map_err(|_| io::Error::other("invalid request clock"))
415}
416
417fn stamp(raw: &str) -> io::Result<String> {
418 clock(raw)?
419 .format(time::macros::format_description!(
420 "[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
421 ))
422 .map_err(io::Error::other)
423}
424
425pub(crate) fn today(raw: &str) -> io::Result<String> {
426 Ok(stamp(raw)?[..10].into())
427}