Skip to main content

rigspark_core/
generation.rs

1//! Curated, offline catalog of local image/video generation models served by ComfyUI.
2//!
3//! Kept separate from the LLM catalog so LLM ranking, signing, and refresh are unaffected.
4use crate::{
5    catalog::{
6        EntryProvenance, LICENSES, RecencyBasis, coordinates, date, digest, model_file,
7        parameter_label, parse_document, recency, recency_label, require, timestamp,
8    },
9    reports::{strip_control, table},
10    sizing::{HEADROOM, Hardware, ValidationError, memory_capacity},
11};
12use serde::{Deserialize, Serialize};
13use std::collections::HashSet;
14
15const MAX_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
16
17#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize)]
18#[serde(rename_all = "lowercase")]
19pub enum GenerationKind {
20    Image,
21    Video,
22}
23impl GenerationKind {
24    pub fn name(self) -> &'static str {
25        match self {
26            Self::Image => "image",
27            Self::Video => "video",
28        }
29    }
30    pub fn parse(value: &str) -> Option<Self> {
31        match value {
32            "image" => Some(Self::Image),
33            "video" => Some(Self::Video),
34            _ => None,
35        }
36    }
37}
38
39/// Built-in ComfyUI workflow graphs implemented by `rigspark-runtime`.
40#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
41#[serde(rename_all = "kebab-case")]
42pub enum Workflow {
43    FluxCheckpoint,
44    FluxSplit,
45    WanT2v,
46    /// Wan 2.2 TI2V-5B text-to-video (https://comfyanonymous.github.io/ComfyUI_examples/wan22/).
47    #[serde(rename = "wan22-ti2v")]
48    Wan22Ti2v,
49    /// Qwen-Image text-to-image (https://comfyanonymous.github.io/ComfyUI_examples/qwen_image/).
50    QwenImage,
51    /// Candidate only; bundled entries stay fit-only until live generation is verified.
52    MingDesign,
53}
54impl Workflow {
55    pub fn kind(self) -> GenerationKind {
56        match self {
57            Self::FluxCheckpoint | Self::FluxSplit | Self::QwenImage | Self::MingDesign => {
58                GenerationKind::Image
59            }
60            Self::WanT2v | Self::Wan22Ti2v => GenerationKind::Video,
61        }
62    }
63    /// Exact file roles the workflow loads, in a stable order.
64    pub fn roles(self) -> &'static [FileRole] {
65        match self {
66            Self::FluxCheckpoint => &[FileRole::Checkpoint],
67            Self::FluxSplit => &[
68                FileRole::Diffusion,
69                FileRole::TextEncoder,
70                FileRole::Clip,
71                FileRole::Vae,
72            ],
73            Self::WanT2v | Self::Wan22Ti2v | Self::QwenImage | Self::MingDesign => {
74                &[FileRole::Diffusion, FileRole::TextEncoder, FileRole::Vae]
75            }
76        }
77    }
78}
79
80#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize)]
81#[serde(rename_all = "kebab-case")]
82pub enum FileRole {
83    Checkpoint,
84    Diffusion,
85    TextEncoder,
86    Clip,
87    Vae,
88}
89impl FileRole {
90    /// ComfyUI `models/` subfolder the loader node reads this role from.
91    pub fn folder(self) -> &'static str {
92        match self {
93            Self::Checkpoint => "checkpoints",
94            Self::Diffusion => "diffusion_models",
95            Self::TextEncoder => "text_encoders",
96            Self::Clip => "text_encoders",
97            Self::Vae => "vae",
98        }
99    }
100}
101
102#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
103#[serde(rename_all = "camelCase", deny_unknown_fields)]
104pub struct GenerationFile {
105    pub role: FileRole,
106    pub folder: String,
107    pub repo: String,
108    pub revision: String,
109    pub file: String,
110    pub sha256: String,
111    pub bytes: u64,
112}
113
114#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
115#[serde(rename_all = "camelCase", deny_unknown_fields)]
116pub struct GenerationModel {
117    pub id: String,
118    pub family: String,
119    pub kind: GenerationKind,
120    pub params: String,
121    pub license: String,
122    pub open_weight: bool,
123    #[serde(default, skip_serializing_if = "Option::is_none")]
124    pub release_date: Option<String>,
125    #[serde(default, skip_serializing_if = "Option::is_none")]
126    pub added_at: Option<String>,
127    #[serde(default, skip_serializing_if = "EntryProvenance::is_curated")]
128    pub provenance: EntryProvenance,
129    #[serde(default)]
130    pub default: bool,
131    pub source: String,
132    /// Absent for fit-only entries: sourced weights without a built-in workflow yet.
133    #[serde(default, skip_serializing_if = "Option::is_none")]
134    pub workflow: Option<Workflow>,
135    #[serde(default, skip_serializing_if = "Option::is_none")]
136    pub workflow_source: Option<String>,
137    pub files: Vec<GenerationFile>,
138}
139
140pub const GENERATION_SCHEMA_VERSION: u8 = 2;
141
142#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
143#[serde(rename_all = "camelCase", deny_unknown_fields)]
144pub struct GenerationCatalog {
145    pub schema_version: u8,
146    pub generated_at: String,
147    pub models: Vec<GenerationModel>,
148}
149
150fn https(value: &str) -> Result<(), ValidationError> {
151    require(
152        value.len() <= 2048
153            && value.starts_with("https://")
154            && value.len() > "https://".len()
155            && !value
156                .chars()
157                .any(|ch| ch.is_control() || ch.is_whitespace()),
158        "source links must be HTTPS",
159    )
160}
161fn model_id(value: &str) -> bool {
162    !value.is_empty()
163        && value.len() <= 128
164        && value.as_bytes()[0].is_ascii_alphanumeric()
165        && value.bytes().all(|byte| {
166            byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._-:".contains(&byte)
167        })
168        && GenerationKind::parse(value).is_none()
169}
170
171impl GenerationModel {
172    pub fn recency(&self) -> Option<(&str, RecencyBasis)> {
173        recency(self.release_date.as_deref(), self.added_at.as_deref())
174    }
175    pub fn recency_label(&self) -> String {
176        recency_label(self.recency())
177    }
178    /// The built-in workflow that runs this model, or why it cannot run yet.
179    pub fn runnable(&self) -> Result<Workflow, ValidationError> {
180        self.workflow.ok_or_else(|| {
181            ValidationError(format!(
182                "{} is fit-only (workflow coming): RigSpark has no built-in ComfyUI workflow for this architecture yet",
183                self.id
184            ))
185        })
186    }
187    pub fn total_bytes(&self) -> u64 {
188        self.files.iter().map(|file| file.bytes).sum()
189    }
190    pub fn largest_bytes(&self) -> u64 {
191        self.files.iter().map(|file| file.bytes).max().unwrap_or(0)
192    }
193    pub fn file(&self, role: FileRole) -> Option<&GenerationFile> {
194        self.files.iter().find(|file| file.role == role)
195    }
196    fn validate(&self) -> Result<(), ValidationError> {
197        require(model_id(&self.id), "invalid generation model id")?;
198        require(
199            !self.family.is_empty() && self.family.len() <= 128,
200            "empty text field",
201        )?;
202        require(parameter_label(&self.params), "invalid parameter label")?;
203        require(
204            self.open_weight && LICENSES.contains(&self.license.as_str()),
205            "generation catalog requires an allowlisted open-weight license",
206        )?;
207        if let Some(released) = &self.release_date {
208            date(released)?;
209        }
210        if let Some(added) = &self.added_at {
211            date(added)?;
212        }
213        match self.provenance {
214            EntryProvenance::Curated => require(
215                self.release_date.is_some() && self.workflow.is_some(),
216                "curated generation entries need releaseDate and a workflow",
217            )?,
218            EntryProvenance::Auto => require(
219                self.added_at.is_some(),
220                "auto generation entries need addedAt",
221            )?,
222            EntryProvenance::Local => require(false, "generation catalog has no local entries")?,
223        }
224        https(&self.source)?;
225        require(
226            self.workflow.is_some() == self.workflow_source.is_some(),
227            "workflowSource must accompany a workflow",
228        )?;
229        let roles: Vec<_> = self.files.iter().map(|file| file.role).collect();
230        match (self.workflow, &self.workflow_source) {
231            (Some(workflow), Some(source)) => {
232                https(source)?;
233                require(
234                    roles.len() == workflow.roles().len()
235                        && workflow.roles().iter().all(|role| roles.contains(role)),
236                    "files must match the workflow roles exactly",
237                )?;
238                require(
239                    workflow.kind() == self.kind,
240                    "workflow does not produce this model kind",
241                )?;
242            }
243            _ => require(
244                !roles.is_empty()
245                    && roles.len() <= 8
246                    && roles.iter().collect::<HashSet<_>>().len() == roles.len(),
247                "fit-only entries need sourced files with distinct roles",
248            )?,
249        }
250        let mut total = 0u64;
251        for file in &self.files {
252            require(
253                file.folder == file.role.folder(),
254                "file folder does not match its role",
255            )?;
256            coordinates(&file.repo, &file.revision)?;
257            model_file(&file.file, 512)?;
258            require(
259                file.file.ends_with(".safetensors"),
260                "generation weights must be .safetensors files",
261            )?;
262            digest(&file.sha256)?;
263            require(
264                file.bytes > 0 && file.bytes <= MAX_SAFE_INTEGER,
265                "invalid weight file size",
266            )?;
267            total = total
268                .checked_add(file.bytes)
269                .filter(|total| *total <= MAX_SAFE_INTEGER)
270                .ok_or_else(|| ValidationError("invalid weight file size".into()))?;
271        }
272        Ok(())
273    }
274}
275
276impl GenerationCatalog {
277    pub fn parse(raw: &str) -> Result<Self, ValidationError> {
278        let mut result: Self = parse_document(raw)?;
279        require(
280            (1..=GENERATION_SCHEMA_VERSION).contains(&result.schema_version)
281                && !result.models.is_empty()
282                && result.models.len() <= 1024,
283            "unsupported or empty generation catalog",
284        )?;
285        timestamp(&result.generated_at)?;
286        let mut ids = HashSet::new();
287        let mut defaults = HashSet::new();
288        let mut kinds = HashSet::new();
289        for model in &result.models {
290            require(
291                result.schema_version >= 2
292                    || (model.provenance.is_curated() && model.added_at.is_none()),
293                "schema v1 generation catalogs cannot carry v2 fields",
294            )?;
295            model.validate()?;
296            require(
297                !model.default || model.workflow.is_some(),
298                "a default model must be runnable",
299            )?;
300            require(ids.insert(&model.id), "duplicate generation model id")?;
301            kinds.insert(model.kind);
302            if model.default {
303                require(
304                    defaults.insert(model.kind),
305                    "each kind needs exactly one default model",
306                )?;
307            }
308        }
309        require(
310            kinds == defaults,
311            "each kind needs exactly one default model",
312        )?;
313        for model in &mut result.models {
314            model.family = strip_control(&model.family);
315        }
316        Ok(result)
317    }
318    pub fn bundled() -> Result<Self, ValidationError> {
319        Self::parse(crate::GENERATION_JSON)
320    }
321    /// Resolves `image`/`video` to that kind's default model, otherwise an exact id.
322    pub fn resolve(&self, query: &str) -> Result<&GenerationModel, ValidationError> {
323        let found = match GenerationKind::parse(query) {
324            Some(kind) => self
325                .models
326                .iter()
327                .find(|model| model.kind == kind && model.default),
328            None => self.models.iter().find(|model| model.id == query),
329        };
330        found.ok_or_else(|| {
331            ValidationError(
332                "unknown generation model; use image, video, or an id from `rigspark catalog --generation`"
333                    .into(),
334            )
335        })
336    }
337}
338
339#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
340#[serde(rename_all = "camelCase")]
341pub struct RejectedGenerationCandidate {
342    pub id: String,
343    pub reason: String,
344}
345
346#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
347#[serde(rename_all = "camelCase")]
348pub struct GenerationEnrichment {
349    pub catalog: GenerationCatalog,
350    pub updated: Vec<String>,
351    pub rejected: Vec<RejectedGenerationCandidate>,
352}
353
354fn built_in_family(model: &GenerationModel) -> bool {
355    matches!(
356        (model.kind, model.workflow, model.family.as_str()),
357        (
358            GenerationKind::Image,
359            Some(Workflow::FluxCheckpoint | Workflow::FluxSplit),
360            "flux1-schnell"
361        ) | (GenerationKind::Video, Some(Workflow::WanT2v), "wan2.1-t2v")
362    )
363}
364
365/// Applies already-collected, pinned metadata to the bundled generation catalog.
366///
367/// Collection is intentionally outside `rigspark-core`; this function remains pure and
368/// accepts only complete models for workflow families implemented by the runtime.
369pub fn enrich_generation_catalog(
370    catalog: &GenerationCatalog,
371    candidates: Vec<GenerationModel>,
372    kind: GenerationKind,
373    now: &str,
374) -> Result<GenerationEnrichment, ValidationError> {
375    timestamp(now)?;
376    let mut next = catalog.clone();
377    let mut updated = Vec::new();
378    let mut rejected = Vec::new();
379
380    for mut candidate in candidates {
381        let reason = if candidate.kind != kind {
382            Some("candidate does not match the requested kind".into())
383        } else if !built_in_family(&candidate) {
384            Some("candidate has no built-in workflow family".into())
385        } else {
386            candidate.validate().err().map(|error| error.to_string())
387        };
388        if let Some(reason) = reason {
389            rejected.push(RejectedGenerationCandidate {
390                id: strip_control(&candidate.id),
391                reason,
392            });
393            continue;
394        }
395
396        if let Some(existing) = next
397            .models
398            .iter_mut()
399            .find(|model| model.id == candidate.id)
400        {
401            candidate.default = existing.default;
402            if *existing != candidate {
403                updated.push(candidate.id.clone());
404                *existing = candidate;
405            }
406        } else {
407            updated.push(candidate.id.clone());
408            next.models.push(candidate);
409        }
410    }
411
412    updated.sort();
413    updated.dedup();
414    rejected.sort_by(|left, right| left.id.cmp(&right.id));
415    if !updated.is_empty() {
416        next.generated_at = now.into();
417        next.models.sort_by(|left, right| left.id.cmp(&right.id));
418        next = GenerationCatalog::parse(
419            &serde_json::to_string(&next)
420                .map_err(|_| ValidationError("invalid generation catalog".into()))?,
421        )?;
422    }
423    Ok(GenerationEnrichment {
424        catalog: next,
425        updated,
426        rejected,
427    })
428}
429
430#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
431#[serde(rename_all = "lowercase")]
432pub enum FitVerdict {
433    Yes,
434    Slow,
435    No,
436}
437impl FitVerdict {
438    pub fn name(self) -> &'static str {
439        match self {
440            Self::Yes => "yes",
441            Self::Slow => "slow",
442            Self::No => "no",
443        }
444    }
445}
446
447/// Memory-only fit of a model's weights; generation speed is never estimated.
448#[derive(Debug, Clone, Serialize)]
449#[serde(rename_all = "camelCase")]
450pub struct GenerationFit {
451    pub verdict: FitVerdict,
452    pub reason: &'static str,
453    pub memory_kind: &'static str,
454    pub usable_bytes: f64,
455    pub budget_bytes: f64,
456    pub total_bytes: u64,
457    pub largest_bytes: u64,
458    /// Always `"unknown"`: no sourced generation throughput dataset exists.
459    pub speed: &'static str,
460}
461
462fn system_ram_budget(hardware: &Hardware) -> f64 {
463    let mut cpu = hardware.clone();
464    cpu.gpu.clear();
465    cpu.unified_memory = Some(false);
466    memory_capacity(&cpu).1 * (1.0 - HEADROOM)
467}
468
469pub fn fit(model: &GenerationModel, hardware: &Hardware) -> GenerationFit {
470    let (memory_kind, usable_bytes) = memory_capacity(hardware);
471    let budget_bytes = usable_bytes * (1.0 - HEADROOM);
472    let total_bytes = model.total_bytes();
473    let largest_bytes = model.largest_bytes();
474    let (verdict, reason) = if total_bytes as f64 <= budget_bytes {
475        (FitVerdict::Yes, "all weights fit in memory together")
476    } else if largest_bytes as f64 <= budget_bytes {
477        (
478            FitVerdict::Slow,
479            "weights fit one stage at a time; ComfyUI swaps models between stages",
480        )
481    } else if memory_kind == "vram" && largest_bytes as f64 <= system_ram_budget(hardware) {
482        (
483            FitVerdict::Slow,
484            "largest weight file exceeds VRAM; ComfyUI offloads to system RAM",
485        )
486    } else {
487        (FitVerdict::No, "largest weight file exceeds usable memory")
488    };
489    GenerationFit {
490        verdict,
491        reason,
492        memory_kind,
493        usable_bytes,
494        budget_bytes,
495        total_bytes,
496        largest_bytes,
497        speed: "unknown",
498    }
499}
500
501pub fn catalog_text(catalog: &GenerationCatalog, hardware: &Hardware) -> String {
502    let mut models: Vec<_> = catalog.models.iter().collect();
503    models.sort_by(|left, right| {
504        left.kind
505            .name()
506            .cmp(right.kind.name())
507            .then_with(|| right.release_date.cmp(&left.release_date))
508            .then_with(|| left.id.cmp(&right.id))
509    });
510    let rows = models
511        .into_iter()
512        .map(|model| {
513            let fit = fit(model, hardware);
514            vec![
515                model.id.clone(),
516                model.kind.name().into(),
517                model.params.clone(),
518                format!("{:.1}", model.total_bytes() as f64 / 1073741824.0),
519                fit.verdict.name().into(),
520                fit.speed.into(),
521                model.license.clone(),
522                model.recency_label(),
523                if model.workflow.is_some() {
524                    "runs".into()
525                } else {
526                    "workflow coming".into()
527                },
528            ]
529        })
530        .collect();
531    format!(
532        "Generation catalog (runtime: ComfyUI, local only; models: {})\n{}\n\
533         Fit covers weight memory only; generation speed is unknown (no sourced data).\n\
534         Run: rigspark generate image --prompt \"...\" --comfyui-dir <ComfyUI directory>\n",
535        catalog.models.len(),
536        table(
537            &[
538                ("Model", false),
539                ("Kind", false),
540                ("Params", false),
541                ("Weights GiB", true),
542                ("Fit", false),
543                ("Speed", false),
544                ("License", false),
545                ("Release", false),
546                ("Generate", false),
547            ],
548            rows,
549        )
550    )
551}