Skip to main content

rigspark_runtime/
generation_admission.rs

1//! Weekly admission of Comfy-Org image and video releases as fit-only generation entries.
2
3use crate::admission::{AdmissionTransport, GENERATION_LISTING_URL};
4use rigspark_core::{
5    catalog::EntryProvenance,
6    generation::{GENERATION_SCHEMA_VERSION, GenerationCatalog, GenerationModel},
7    generation_admission::{
8        FitOnlyInput, HfModel, MAX_METADATA_BYTES, MAX_SAFETENSORS_HEADER_BYTES, SafetensorsError,
9        diffusion_files, fit_only_entries, kind_from_pipeline, license_from_tags, parse_listing,
10        recipe, safetensors_params,
11    },
12};
13use serde::{Deserialize, Serialize};
14use serde_json::json;
15use std::{
16    collections::{BTreeMap, BTreeSet},
17    io,
18    time::Duration,
19};
20use time::{OffsetDateTime, format_description::well_known::Rfc3339};
21use tokio_util::sync::CancellationToken;
22use url::Url;
23
24#[derive(Clone, Debug, Default, Serialize, Deserialize)]
25#[serde(rename_all = "camelCase", deny_unknown_fields)]
26pub struct GenerationAdmissionState {
27    /// LFS sha256 of a diffusion file → exact parameter count from its safetensors header.
28    #[serde(default)]
29    pub params: BTreeMap<String, u64>,
30}
31
32#[derive(Default, Serialize)]
33#[serde(rename_all = "camelCase")]
34pub struct GenerationOutcome {
35    #[serde(skip)]
36    pub catalog: Option<GenerationCatalog>,
37    #[serde(skip)]
38    pub state: GenerationAdmissionState,
39    pub added: Vec<String>,
40    pub removed: Vec<String>,
41    pub rejected: BTreeMap<String, String>,
42    pub failures: Vec<String>,
43    pub complete: bool,
44}
45
46async fn get(
47    transport: &dyn AdmissionTransport,
48    cancel: &CancellationToken,
49    url: &str,
50    range: Option<usize>,
51    limit: usize,
52) -> io::Result<Vec<u8>> {
53    if cancel.is_cancelled() {
54        return Err(io::Error::new(
55            io::ErrorKind::Interrupted,
56            "generation admission cancelled",
57        ));
58    }
59    let url = Url::parse(url).map_err(io::Error::other)?;
60    let fetched = tokio::select! {
61        biased;
62        _ = cancel.cancelled() => {
63            return Err(io::Error::new(io::ErrorKind::Interrupted, "generation admission cancelled"));
64        }
65        result = tokio::time::timeout(Duration::from_secs(120), transport.fetch(&url, None, range, limit)) => {
66            result.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "admission request timed out"))??
67        }
68    };
69    let ok = match range {
70        Some(_) => matches!(fetched.status, 200 | 206),
71        None => (200..300).contains(&fetched.status),
72    };
73    if !ok {
74        return Err(io::Error::new(
75            io::ErrorKind::NotFound,
76            format!("upstream status {}", fetched.status),
77        ));
78    }
79    Ok(fetched.body)
80}
81
82async fn header_params(
83    transport: &dyn AdmissionTransport,
84    cancel: &CancellationToken,
85    url: &str,
86) -> io::Result<Result<u64, &'static str>> {
87    let mut want = 64 * 1024;
88    loop {
89        let bytes = get(transport, cancel, url, Some(want), want).await?;
90        match safetensors_params(&bytes) {
91            Ok(params) => return Ok(Ok(params)),
92            Err(SafetensorsError::Truncated(needed))
93                if bytes.len() >= want && needed <= MAX_SAFETENSORS_HEADER_BYTES + 8 =>
94            {
95                want = needed as usize;
96            }
97            Err(SafetensorsError::Truncated(_)) => return Ok(Err("safetensors header truncated")),
98            Err(SafetensorsError::Invalid(reason)) => return Ok(Err(reason)),
99        }
100    }
101}
102
103fn interrupted(error: &io::Error) -> bool {
104    error.kind() == io::ErrorKind::Interrupted
105}
106
107/// Admits fit-only entries for new Comfy-Org text-to-image and text-to-video releases.
108/// Curated entries are never modified; a failed listing aborts without changes.
109pub async fn admit_generation(
110    catalog: &GenerationCatalog,
111    transport: &dyn AdmissionTransport,
112    now: &str,
113    state: GenerationAdmissionState,
114    cancel: &CancellationToken,
115) -> io::Result<GenerationOutcome> {
116    let now = OffsetDateTime::parse(now, &Rfc3339)
117        .map_err(|_| io::Error::other("invalid admission clock"))?
118        .to_offset(time::UtcOffset::UTC);
119    let stamp = now
120        .format(time::macros::format_description!(
121            "[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
122        ))
123        .map_err(io::Error::other)?;
124    let today = &stamp[..10];
125    let listing = get(
126        transport,
127        cancel,
128        GENERATION_LISTING_URL,
129        None,
130        MAX_METADATA_BYTES,
131    )
132    .await?;
133    let repos = parse_listing(
134        std::str::from_utf8(&listing).map_err(|_| io::Error::other("listing is not UTF-8"))?,
135    )
136    .map_err(io::Error::other)?;
137
138    let mut outcome = GenerationOutcome {
139        complete: true,
140        state,
141        ..Default::default()
142    };
143    let curated: Vec<&GenerationModel> = catalog
144        .models
145        .iter()
146        .filter(|model| model.provenance == EntryProvenance::Curated)
147        .collect();
148    let curated_ids: BTreeSet<&str> = curated.iter().map(|model| model.id.as_str()).collect();
149    let curated_digests: BTreeSet<&str> = curated
150        .iter()
151        .flat_map(|model| model.files.iter())
152        .map(|file| file.sha256.as_str())
153        .collect();
154    let prior: BTreeMap<&str, &GenerationModel> = catalog
155        .models
156        .iter()
157        .filter(|model| model.provenance == EntryProvenance::Auto)
158        .map(|model| (model.id.as_str(), model))
159        .collect();
160    let mut admitted: BTreeMap<String, GenerationModel> = BTreeMap::new();
161    let mut unknown_sources: BTreeSet<String> = BTreeSet::new();
162
163    for repo in &repos {
164        let model = match get(
165            transport,
166            cancel,
167            &format!("https://huggingface.co/api/models/{repo}?blobs=true"),
168            None,
169            MAX_METADATA_BYTES,
170        )
171        .await
172        {
173            Ok(raw) => match std::str::from_utf8(&raw).ok().map(HfModel::parse) {
174                Some(Ok(model)) => model,
175                _ => {
176                    outcome
177                        .failures
178                        .push(format!("{repo}: metadata unparseable"));
179                    outcome.complete = false;
180                    unknown_sources.insert(format!("https://huggingface.co/{repo}"));
181                    continue;
182                }
183            },
184            Err(error) if interrupted(&error) => return Err(error),
185            Err(error) => {
186                outcome.failures.push(format!("{repo}: {error}"));
187                outcome.complete = false;
188                unknown_sources.insert(format!("https://huggingface.co/{repo}"));
189                continue;
190            }
191        };
192        // Official recipes define their kind; otherwise any base model's pipeline decides.
193        let mut kind = recipe(&model.id).map(|recipe| recipe.kind);
194        if kind.is_none() {
195            kind = kind_from_pipeline(model.pipeline_tag.as_deref());
196        }
197        for base in model.base_models() {
198            if kind.is_some() {
199                break;
200            }
201            match get(
202                transport,
203                cancel,
204                &format!("https://huggingface.co/api/models/{base}"),
205                None,
206                MAX_METADATA_BYTES,
207            )
208            .await
209            {
210                Ok(raw) => {
211                    kind = serde_json::from_slice::<serde_json::Value>(&raw)
212                        .ok()
213                        .and_then(|base| kind_from_pipeline(base["pipeline_tag"].as_str()));
214                }
215                Err(error) if interrupted(&error) => return Err(error),
216                Err(_) => {}
217            }
218        }
219        let Some(kind) = kind else {
220            continue;
221        };
222        let Some(license) = license_from_tags(&model.tags) else {
223            outcome
224                .rejected
225                .insert(repo.clone(), "license not recognised as open".into());
226            continue;
227        };
228        if !model.pinned() {
229            outcome
230                .rejected
231                .insert(repo.clone(), "not a public pinned revision".into());
232            continue;
233        }
234        let mut params: BTreeMap<String, u64> = BTreeMap::new();
235        let mut failed = false;
236        for sibling in diffusion_files(&model) {
237            let Some(lfs) = &sibling.lfs else { continue };
238            if curated_digests.contains(lfs.sha256.as_str()) {
239                continue;
240            }
241            if let Some(count) = outcome.state.params.get(&lfs.sha256) {
242                params.insert(sibling.rfilename.clone(), *count);
243                continue;
244            }
245            let url = format!(
246                "https://huggingface.co/{repo}/resolve/{}/{}",
247                model.sha, sibling.rfilename
248            );
249            match header_params(transport, cancel, &url).await {
250                Ok(Ok(count)) => {
251                    outcome.state.params.insert(lfs.sha256.clone(), count);
252                    params.insert(sibling.rfilename.clone(), count);
253                }
254                Ok(Err(reason)) => {
255                    outcome
256                        .rejected
257                        .insert(format!("{repo}/{}", sibling.rfilename), reason.into());
258                }
259                Err(error) if interrupted(&error) => return Err(error),
260                Err(error) => {
261                    outcome
262                        .failures
263                        .push(format!("{repo}/{}: {error}", sibling.rfilename));
264                    failed = true;
265                }
266            }
267        }
268        if failed {
269            outcome.complete = false;
270            unknown_sources.insert(format!("https://huggingface.co/{repo}"));
271        }
272        let entries = fit_only_entries(&FitOnlyInput {
273            model: &model,
274            kind,
275            license,
276            params: &|path| params.get(path).copied(),
277            today,
278        })
279        .map_err(io::Error::other)?;
280        for mut entry in entries {
281            if curated_ids.contains(entry.id.as_str()) {
282                continue;
283            }
284            if let Some(previous) = prior.get(entry.id.as_str()) {
285                entry.added_at = previous.added_at.clone();
286            }
287            admitted.insert(entry.id.clone(), entry);
288        }
289    }
290
291    // Keep prior auto entries whose repository could not be read this run.
292    for (id, model) in &prior {
293        if !admitted.contains_key(*id) && unknown_sources.contains(&model.source) {
294            admitted.insert(id.to_string(), (*model).clone());
295        }
296    }
297    for id in admitted.keys() {
298        if !prior.contains_key(id.as_str()) {
299            outcome.added.push(id.clone());
300        }
301    }
302    for id in prior.keys() {
303        if !admitted.contains_key(*id) {
304            outcome.removed.push(id.to_string());
305        }
306    }
307    let models: Vec<GenerationModel> = catalog
308        .models
309        .iter()
310        .filter(|model| model.provenance == EntryProvenance::Curated)
311        .cloned()
312        .chain(admitted.into_values())
313        .collect();
314    let changed = models != catalog.models;
315    let encoded = json!({
316        "schemaVersion": GENERATION_SCHEMA_VERSION,
317        "generatedAt": if changed { stamp.clone() } else { catalog.generated_at.clone() },
318        "models": models,
319    });
320    outcome.catalog =
321        Some(GenerationCatalog::parse(&encoded.to_string()).map_err(io::Error::other)?);
322    Ok(outcome)
323}