1use crate::admission::{AdmissionTransport, GENERATION_LISTING_URL};
4use rigspark_core::{
5 catalog::EntryProvenance,
6 generation::{GENERATION_SCHEMA_VERSION, GenerationCatalog, GenerationModel},
7 generation_admission::{
8 FitOnlyInput, HfModel, MAX_METADATA_BYTES, MAX_SAFETENSORS_HEADER_BYTES, SafetensorsError,
9 diffusion_files, fit_only_entries, kind_from_pipeline, license_from_tags, parse_listing,
10 recipe, safetensors_params,
11 },
12};
13use serde::{Deserialize, Serialize};
14use serde_json::json;
15use std::{
16 collections::{BTreeMap, BTreeSet},
17 io,
18 time::Duration,
19};
20use time::{OffsetDateTime, format_description::well_known::Rfc3339};
21use tokio_util::sync::CancellationToken;
22use url::Url;
23
24#[derive(Clone, Debug, Default, Serialize, Deserialize)]
25#[serde(rename_all = "camelCase", deny_unknown_fields)]
26pub struct GenerationAdmissionState {
27 #[serde(default)]
29 pub params: BTreeMap<String, u64>,
30}
31
32#[derive(Default, Serialize)]
33#[serde(rename_all = "camelCase")]
34pub struct GenerationOutcome {
35 #[serde(skip)]
36 pub catalog: Option<GenerationCatalog>,
37 #[serde(skip)]
38 pub state: GenerationAdmissionState,
39 pub added: Vec<String>,
40 pub removed: Vec<String>,
41 pub rejected: BTreeMap<String, String>,
42 pub failures: Vec<String>,
43 pub complete: bool,
44}
45
46async fn get(
47 transport: &dyn AdmissionTransport,
48 cancel: &CancellationToken,
49 url: &str,
50 range: Option<usize>,
51 limit: usize,
52) -> io::Result<Vec<u8>> {
53 if cancel.is_cancelled() {
54 return Err(io::Error::new(
55 io::ErrorKind::Interrupted,
56 "generation admission cancelled",
57 ));
58 }
59 let url = Url::parse(url).map_err(io::Error::other)?;
60 let fetched = tokio::select! {
61 biased;
62 _ = cancel.cancelled() => {
63 return Err(io::Error::new(io::ErrorKind::Interrupted, "generation admission cancelled"));
64 }
65 result = tokio::time::timeout(Duration::from_secs(120), transport.fetch(&url, None, range, limit)) => {
66 result.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "admission request timed out"))??
67 }
68 };
69 let ok = match range {
70 Some(_) => matches!(fetched.status, 200 | 206),
71 None => (200..300).contains(&fetched.status),
72 };
73 if !ok {
74 return Err(io::Error::new(
75 io::ErrorKind::NotFound,
76 format!("upstream status {}", fetched.status),
77 ));
78 }
79 Ok(fetched.body)
80}
81
82async fn header_params(
83 transport: &dyn AdmissionTransport,
84 cancel: &CancellationToken,
85 url: &str,
86) -> io::Result<Result<u64, &'static str>> {
87 let mut want = 64 * 1024;
88 loop {
89 let bytes = get(transport, cancel, url, Some(want), want).await?;
90 match safetensors_params(&bytes) {
91 Ok(params) => return Ok(Ok(params)),
92 Err(SafetensorsError::Truncated(needed))
93 if bytes.len() >= want && needed <= MAX_SAFETENSORS_HEADER_BYTES + 8 =>
94 {
95 want = needed as usize;
96 }
97 Err(SafetensorsError::Truncated(_)) => return Ok(Err("safetensors header truncated")),
98 Err(SafetensorsError::Invalid(reason)) => return Ok(Err(reason)),
99 }
100 }
101}
102
103fn interrupted(error: &io::Error) -> bool {
104 error.kind() == io::ErrorKind::Interrupted
105}
106
107pub async fn admit_generation(
110 catalog: &GenerationCatalog,
111 transport: &dyn AdmissionTransport,
112 now: &str,
113 state: GenerationAdmissionState,
114 cancel: &CancellationToken,
115) -> io::Result<GenerationOutcome> {
116 let now = OffsetDateTime::parse(now, &Rfc3339)
117 .map_err(|_| io::Error::other("invalid admission clock"))?
118 .to_offset(time::UtcOffset::UTC);
119 let stamp = now
120 .format(time::macros::format_description!(
121 "[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
122 ))
123 .map_err(io::Error::other)?;
124 let today = &stamp[..10];
125 let listing = get(
126 transport,
127 cancel,
128 GENERATION_LISTING_URL,
129 None,
130 MAX_METADATA_BYTES,
131 )
132 .await?;
133 let repos = parse_listing(
134 std::str::from_utf8(&listing).map_err(|_| io::Error::other("listing is not UTF-8"))?,
135 )
136 .map_err(io::Error::other)?;
137
138 let mut outcome = GenerationOutcome {
139 complete: true,
140 state,
141 ..Default::default()
142 };
143 let curated: Vec<&GenerationModel> = catalog
144 .models
145 .iter()
146 .filter(|model| model.provenance == EntryProvenance::Curated)
147 .collect();
148 let curated_ids: BTreeSet<&str> = curated.iter().map(|model| model.id.as_str()).collect();
149 let curated_digests: BTreeSet<&str> = curated
150 .iter()
151 .flat_map(|model| model.files.iter())
152 .map(|file| file.sha256.as_str())
153 .collect();
154 let prior: BTreeMap<&str, &GenerationModel> = catalog
155 .models
156 .iter()
157 .filter(|model| model.provenance == EntryProvenance::Auto)
158 .map(|model| (model.id.as_str(), model))
159 .collect();
160 let mut admitted: BTreeMap<String, GenerationModel> = BTreeMap::new();
161 let mut unknown_sources: BTreeSet<String> = BTreeSet::new();
162
163 for repo in &repos {
164 let model = match get(
165 transport,
166 cancel,
167 &format!("https://huggingface.co/api/models/{repo}?blobs=true"),
168 None,
169 MAX_METADATA_BYTES,
170 )
171 .await
172 {
173 Ok(raw) => match std::str::from_utf8(&raw).ok().map(HfModel::parse) {
174 Some(Ok(model)) => model,
175 _ => {
176 outcome
177 .failures
178 .push(format!("{repo}: metadata unparseable"));
179 outcome.complete = false;
180 unknown_sources.insert(format!("https://huggingface.co/{repo}"));
181 continue;
182 }
183 },
184 Err(error) if interrupted(&error) => return Err(error),
185 Err(error) => {
186 outcome.failures.push(format!("{repo}: {error}"));
187 outcome.complete = false;
188 unknown_sources.insert(format!("https://huggingface.co/{repo}"));
189 continue;
190 }
191 };
192 let mut kind = recipe(&model.id).map(|recipe| recipe.kind);
194 if kind.is_none() {
195 kind = kind_from_pipeline(model.pipeline_tag.as_deref());
196 }
197 for base in model.base_models() {
198 if kind.is_some() {
199 break;
200 }
201 match get(
202 transport,
203 cancel,
204 &format!("https://huggingface.co/api/models/{base}"),
205 None,
206 MAX_METADATA_BYTES,
207 )
208 .await
209 {
210 Ok(raw) => {
211 kind = serde_json::from_slice::<serde_json::Value>(&raw)
212 .ok()
213 .and_then(|base| kind_from_pipeline(base["pipeline_tag"].as_str()));
214 }
215 Err(error) if interrupted(&error) => return Err(error),
216 Err(_) => {}
217 }
218 }
219 let Some(kind) = kind else {
220 continue;
221 };
222 let Some(license) = license_from_tags(&model.tags) else {
223 outcome
224 .rejected
225 .insert(repo.clone(), "license not recognised as open".into());
226 continue;
227 };
228 if !model.pinned() {
229 outcome
230 .rejected
231 .insert(repo.clone(), "not a public pinned revision".into());
232 continue;
233 }
234 let mut params: BTreeMap<String, u64> = BTreeMap::new();
235 let mut failed = false;
236 for sibling in diffusion_files(&model) {
237 let Some(lfs) = &sibling.lfs else { continue };
238 if curated_digests.contains(lfs.sha256.as_str()) {
239 continue;
240 }
241 if let Some(count) = outcome.state.params.get(&lfs.sha256) {
242 params.insert(sibling.rfilename.clone(), *count);
243 continue;
244 }
245 let url = format!(
246 "https://huggingface.co/{repo}/resolve/{}/{}",
247 model.sha, sibling.rfilename
248 );
249 match header_params(transport, cancel, &url).await {
250 Ok(Ok(count)) => {
251 outcome.state.params.insert(lfs.sha256.clone(), count);
252 params.insert(sibling.rfilename.clone(), count);
253 }
254 Ok(Err(reason)) => {
255 outcome
256 .rejected
257 .insert(format!("{repo}/{}", sibling.rfilename), reason.into());
258 }
259 Err(error) if interrupted(&error) => return Err(error),
260 Err(error) => {
261 outcome
262 .failures
263 .push(format!("{repo}/{}: {error}", sibling.rfilename));
264 failed = true;
265 }
266 }
267 }
268 if failed {
269 outcome.complete = false;
270 unknown_sources.insert(format!("https://huggingface.co/{repo}"));
271 }
272 let entries = fit_only_entries(&FitOnlyInput {
273 model: &model,
274 kind,
275 license,
276 params: &|path| params.get(path).copied(),
277 today,
278 })
279 .map_err(io::Error::other)?;
280 for mut entry in entries {
281 if curated_ids.contains(entry.id.as_str()) {
282 continue;
283 }
284 if let Some(previous) = prior.get(entry.id.as_str()) {
285 entry.added_at = previous.added_at.clone();
286 }
287 admitted.insert(entry.id.clone(), entry);
288 }
289 }
290
291 for (id, model) in &prior {
293 if !admitted.contains_key(*id) && unknown_sources.contains(&model.source) {
294 admitted.insert(id.to_string(), (*model).clone());
295 }
296 }
297 for id in admitted.keys() {
298 if !prior.contains_key(id.as_str()) {
299 outcome.added.push(id.clone());
300 }
301 }
302 for id in prior.keys() {
303 if !admitted.contains_key(*id) {
304 outcome.removed.push(id.to_string());
305 }
306 }
307 let models: Vec<GenerationModel> = catalog
308 .models
309 .iter()
310 .filter(|model| model.provenance == EntryProvenance::Curated)
311 .cloned()
312 .chain(admitted.into_values())
313 .collect();
314 let changed = models != catalog.models;
315 let encoded = json!({
316 "schemaVersion": GENERATION_SCHEMA_VERSION,
317 "generatedAt": if changed { stamp.clone() } else { catalog.generated_at.clone() },
318 "models": models,
319 });
320 outcome.catalog =
321 Some(GenerationCatalog::parse(&encoded.to_string()).map_err(io::Error::other)?);
322 Ok(outcome)
323}