Skip to main content

rigspark_runtime/
generation.rs

1//! Local image/video generation through an attached, loopback-only ComfyUI server.
2//!
3//! Only built-in workflows composed of allowlisted core nodes are ever submitted, so paid
4//! ComfyUI Partner (API) nodes and cloud services can never be reached through rigspark.
5use crate::{
6    acquire::{Acquisition, Artifact, DownloadTransport},
7    http::{HttpError, Request, Transport, read_json},
8};
9use rigspark_core::generation::{
10    FileRole, FitVerdict, GenerationCatalog, GenerationFit, GenerationKind, GenerationModel,
11    Workflow, fit,
12};
13use serde::Serialize;
14use serde_json::{Value, json};
15use std::{
16    collections::HashMap,
17    path::{Path, PathBuf},
18    sync::Arc,
19    time::Duration,
20};
21use tokio::io::{AsyncReadExt, AsyncWriteExt};
22use tokio_util::sync::CancellationToken;
23
24pub const DEFAULT_PORT: u16 = 8188;
25pub const MAX_PROMPT_BYTES: usize = 16 * 1024;
26mod preflight;
27pub const MAX_SEED: u64 = 9_007_199_254_740_991;
28const MAX_OUTPUT_BYTES: u64 = 1024 * 1024 * 1024;
29const WEIGHT_TIMEOUT: Duration = Duration::from_secs(6 * 60 * 60);
30/// Core ComfyUI nodes used by the built-in workflows; anything else is refused.
31pub const LOCAL_NODES: [&str; 22] = [
32    "BasicGuider",
33    "BasicScheduler",
34    "CheckpointLoaderSimple",
35    "CLIPLoader",
36    "CLIPTextEncode",
37    "DualCLIPLoader",
38    "EmptyHunyuanLatentVideo",
39    "EmptyLatentImage",
40    "EmptySD3LatentImage",
41    "KSampler",
42    "KSamplerSelect",
43    "ModelSamplingAuraFlow",
44    "ModelSamplingFlux",
45    "ModelSamplingSD3",
46    "RandomNoise",
47    "SamplerCustomAdvanced",
48    "SaveAnimatedWEBP",
49    "SaveImage",
50    "UNETLoader",
51    "VAEDecode",
52    "VAELoader",
53    "Wan22ImageToVideoLatent",
54];
55/// Wan 2.1's published default negative prompt (Apache-2.0), as used by the official
56/// ComfyUI example: https://comfyanonymous.github.io/ComfyUI_examples/wan/
57const WAN_NEGATIVE_PROMPT: &str = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走";
58
59#[derive(Debug, thiserror::Error)]
60pub enum GenerationError {
61    #[error("invalid generation request: {0}")]
62    Invalid(&'static str),
63    #[error("{0}")]
64    Catalog(String),
65    #[error("{0}")]
66    NoFit(String),
67    #[error("hardware detection failed: {0}")]
68    Hardware(String),
69    #[error(
70        "ComfyUI is not reachable at {0}; start ComfyUI on loopback (default port 8188) and retry"
71    )]
72    Unreachable(String),
73    #[error("weight installation failed: {0}")]
74    Install(String),
75    #[error(
76        "ComfyUI does not list {0}; make sure --comfyui-dir points at the running ComfyUI installation"
77    )]
78    NotVisible(String),
79    #[error("workflow node `{0}` is not an allowlisted local node; refusing to submit")]
80    NonLocalNode(String),
81    #[error("{0}")]
82    Incompatible(String),
83    #[error("ComfyUI rejected the workflow: {0}")]
84    Rejected(String),
85    #[error("ComfyUI generation failed: {0}")]
86    Failed(String),
87    #[error("invalid ComfyUI response: {0}")]
88    Response(&'static str),
89    #[error("output already exists: {0}")]
90    Exists(PathBuf),
91    #[error("could not write output: {0}")]
92    Output(String),
93    #[error("generation timed out")]
94    Timeout,
95    #[error("operation cancelled")]
96    Cancelled,
97    #[error(transparent)]
98    Http(HttpError),
99}
100impl From<HttpError> for GenerationError {
101    fn from(error: HttpError) -> Self {
102        match error {
103            HttpError::Cancelled => Self::Cancelled,
104            other => Self::Http(other),
105        }
106    }
107}
108
109pub fn output_extension(kind: GenerationKind) -> &'static str {
110    match kind {
111        GenerationKind::Image => "png",
112        GenerationKind::Video => "webp",
113    }
114}
115fn save_node(workflow: Workflow) -> &'static str {
116    match workflow {
117        Workflow::FluxCheckpoint | Workflow::FluxSplit | Workflow::MingDesign => "9",
118        Workflow::WanT2v | Workflow::Wan22Ti2v => "28",
119        Workflow::QwenImage => "60",
120    }
121}
122
123/// Random seed in the JSON-safe integer range, reported back so runs are reproducible.
124pub fn random_seed() -> u64 {
125    uuid::Uuid::new_v4().as_u64_pair().0 & MAX_SEED
126}
127
128pub fn validate_prompt(prompt: &str) -> Result<(), GenerationError> {
129    if prompt.trim().is_empty()
130        || prompt.len() > MAX_PROMPT_BYTES
131        || prompt
132            .chars()
133            .any(|ch| ch.is_control() && ch != '\n' && ch != '\t')
134    {
135        return Err(GenerationError::Invalid(
136            "prompt must be 1..=16384 bytes of text without control characters",
137        ));
138    }
139    Ok(())
140}
141
142/// ComfyUI reports its compute device per `/system_stats`; Apple GPUs report `mps`.
143pub fn apple_mps(devices: &[String]) -> bool {
144    devices.iter().any(|device| device == "mps")
145}
146
147/// Wan's official `uni_pc` sampler diverged into noise on Apple MPS in live testing (ComfyUI
148/// 0.38, 30 steps), while `euler` at the official size and length stayed coherent.
149fn wan_sampler(devices: &[String]) -> &'static str {
150    if apple_mps(devices) {
151        "euler"
152    } else {
153        "uni_pc"
154    }
155}
156
157/// Builds the API-format graph published in the official ComfyUI examples.
158pub fn workflow(
159    model: &GenerationModel,
160    names: &HashMap<FileRole, String>,
161    prompt: &str,
162    seed: u64,
163    devices: &[String],
164) -> Result<Value, GenerationError> {
165    let name = |role| {
166        names
167            .get(&role)
168            .cloned()
169            .ok_or(GenerationError::Invalid("missing installed weight name"))
170    };
171    let workflow = model
172        .runnable()
173        .map_err(|error| GenerationError::Catalog(error.to_string()))?;
174    Ok(match workflow {
175        // https://comfyanonymous.github.io/ComfyUI_examples/flux/ (schnell fp8 checkpoint)
176        Workflow::FluxCheckpoint => json!({
177            "6": {"class_type": "CLIPTextEncode", "inputs": {"text": prompt, "clip": ["30", 1]}},
178            "8": {"class_type": "VAEDecode", "inputs": {"samples": ["31", 0], "vae": ["30", 2]}},
179            "9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "rigspark", "images": ["8", 0]}},
180            "27": {"class_type": "EmptySD3LatentImage", "inputs": {"width": 1024, "height": 1024, "batch_size": 1}},
181            "30": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": name(FileRole::Checkpoint)?}},
182            "31": {"class_type": "KSampler", "inputs": {
183                "seed": seed, "steps": 4, "cfg": 1.0, "sampler_name": "euler", "scheduler": "simple",
184                "denoise": 1.0, "model": ["30", 0], "positive": ["6", 0], "negative": ["33", 0],
185                "latent_image": ["27", 0]}},
186            "33": {"class_type": "CLIPTextEncode", "inputs": {"text": "", "clip": ["30", 1]}},
187        }),
188        // https://comfyanonymous.github.io/ComfyUI_examples/flux/ (regular full version)
189        Workflow::FluxSplit => json!({
190            "5": {"class_type": "EmptyLatentImage", "inputs": {"width": 1024, "height": 1024, "batch_size": 1}},
191            "6": {"class_type": "CLIPTextEncode", "inputs": {"text": prompt, "clip": ["11", 0]}},
192            "8": {"class_type": "VAEDecode", "inputs": {"samples": ["13", 0], "vae": ["10", 0]}},
193            "9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "rigspark", "images": ["8", 0]}},
194            "10": {"class_type": "VAELoader", "inputs": {"vae_name": name(FileRole::Vae)?}},
195            "11": {"class_type": "DualCLIPLoader", "inputs": {
196                "clip_name1": name(FileRole::TextEncoder)?,
197                "clip_name2": name(FileRole::Clip)?,
198                "type": "flux"}},
199            "12": {"class_type": "UNETLoader", "inputs": {
200                "unet_name": name(FileRole::Diffusion)?, "weight_dtype": "default"}},
201            "13": {"class_type": "SamplerCustomAdvanced", "inputs": {
202                "noise": ["25", 0], "guider": ["22", 0], "sampler": ["16", 0],
203                "sigmas": ["17", 0], "latent_image": ["5", 0]}},
204            "16": {"class_type": "KSamplerSelect", "inputs": {"sampler_name": "euler"}},
205            "17": {"class_type": "BasicScheduler", "inputs": {
206                "scheduler": "simple", "steps": 4, "denoise": 1.0, "model": ["12", 0]}},
207            "22": {"class_type": "BasicGuider", "inputs": {
208                "model": ["12", 0], "conditioning": ["6", 0]}},
209            "25": {"class_type": "RandomNoise", "inputs": {"noise_seed": seed}},
210        }),
211        // https://comfyanonymous.github.io/ComfyUI_examples/wan/ (text to video, 1.3B)
212        Workflow::WanT2v => json!({
213            "3": {"class_type": "KSampler", "inputs": {
214                "seed": seed, "steps": 30, "cfg": 6.0, "sampler_name": wan_sampler(devices), "scheduler": "simple",
215                "denoise": 1.0, "model": ["48", 0], "positive": ["6", 0], "negative": ["7", 0],
216                "latent_image": ["40", 0]}},
217            "6": {"class_type": "CLIPTextEncode", "inputs": {"text": prompt, "clip": ["38", 0]}},
218            "7": {"class_type": "CLIPTextEncode", "inputs": {"text": WAN_NEGATIVE_PROMPT, "clip": ["38", 0]}},
219            "8": {"class_type": "VAEDecode", "inputs": {"samples": ["3", 0], "vae": ["39", 0]}},
220            "28": {"class_type": "SaveAnimatedWEBP", "inputs": {
221                "filename_prefix": "rigspark", "fps": 16.0, "lossless": false, "quality": 90,
222                "method": "default", "images": ["8", 0]}},
223            "37": {"class_type": "UNETLoader", "inputs": {"unet_name": name(FileRole::Diffusion)?, "weight_dtype": "default"}},
224            "38": {"class_type": "CLIPLoader", "inputs": {"clip_name": name(FileRole::TextEncoder)?, "type": "wan", "device": "default"}},
225            "39": {"class_type": "VAELoader", "inputs": {"vae_name": name(FileRole::Vae)?}},
226            "40": {"class_type": "EmptyHunyuanLatentVideo", "inputs": {"width": 832, "height": 480, "length": 49, "batch_size": 1}},
227            "48": {"class_type": "ModelSamplingSD3", "inputs": {"shift": 8.0, "model": ["37", 0]}},
228        }),
229        // https://comfyanonymous.github.io/ComfyUI_examples/wan22/ (5B text to video)
230        Workflow::Wan22Ti2v => json!({
231            "3": {"class_type": "KSampler", "inputs": {
232                "seed": seed, "steps": 30, "cfg": 5.0, "sampler_name": wan_sampler(devices), "scheduler": "simple",
233                "denoise": 1.0, "model": ["48", 0], "positive": ["6", 0], "negative": ["7", 0],
234                "latent_image": ["55", 0]}},
235            "6": {"class_type": "CLIPTextEncode", "inputs": {"text": prompt, "clip": ["38", 0]}},
236            "7": {"class_type": "CLIPTextEncode", "inputs": {"text": WAN_NEGATIVE_PROMPT, "clip": ["38", 0]}},
237            "8": {"class_type": "VAEDecode", "inputs": {"samples": ["3", 0], "vae": ["39", 0]}},
238            "28": {"class_type": "SaveAnimatedWEBP", "inputs": {
239                "filename_prefix": "rigspark", "fps": 24.0, "lossless": false, "quality": 90,
240                "method": "default", "images": ["8", 0]}},
241            "37": {"class_type": "UNETLoader", "inputs": {"unet_name": name(FileRole::Diffusion)?, "weight_dtype": "default"}},
242            "38": {"class_type": "CLIPLoader", "inputs": {"clip_name": name(FileRole::TextEncoder)?, "type": "wan", "device": "default"}},
243            "39": {"class_type": "VAELoader", "inputs": {"vae_name": name(FileRole::Vae)?}},
244            "48": {"class_type": "ModelSamplingSD3", "inputs": {"shift": 8.0, "model": ["37", 0]}},
245            "55": {"class_type": "Wan22ImageToVideoLatent", "inputs": {
246                "width": 1280, "height": 704, "length": 41, "batch_size": 1, "vae": ["39", 0]}},
247        }),
248        // Pinned Ming contract and deliberate template reductions: docs/references/ming-design-workflow.md.
249        Workflow::MingDesign => json!({
250            "5": {"class_type": "EmptyLatentImage", "inputs": {"width": 1024, "height": 1024, "batch_size": 1}},
251            "6": {"class_type": "CLIPTextEncode", "inputs": {"text": prompt, "clip": ["11", 0]}},
252            "8": {"class_type": "VAEDecode", "inputs": {"samples": ["13", 0], "vae": ["10", 0]}},
253            "9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "rigspark", "images": ["8", 0]}},
254            "10": {"class_type": "VAELoader", "inputs": {"vae_name": name(FileRole::Vae)?}},
255            "11": {"class_type": "CLIPLoader", "inputs": {
256                "clip_name": name(FileRole::TextEncoder)?, "type": "qwen_image", "device": "default"}},
257            "12": {"class_type": "UNETLoader", "inputs": {
258                "unet_name": name(FileRole::Diffusion)?, "weight_dtype": "default"}},
259            "13": {"class_type": "SamplerCustomAdvanced", "inputs": {
260                "noise": ["25", 0], "guider": ["22", 0], "sampler": ["16", 0],
261                "sigmas": ["17", 0], "latent_image": ["5", 0]}},
262            "16": {"class_type": "KSamplerSelect", "inputs": {"sampler_name": "euler"}},
263            "17": {"class_type": "BasicScheduler", "inputs": {
264                "scheduler": "simple", "steps": 12, "denoise": 1.0, "model": ["26", 0]}},
265            "22": {"class_type": "BasicGuider", "inputs": {
266                "model": ["26", 0], "conditioning": ["6", 0]}},
267            "25": {"class_type": "RandomNoise", "inputs": {"noise_seed": seed}},
268            "26": {"class_type": "ModelSamplingFlux", "inputs": {
269                "model": ["12", 0], "max_shift": 1.15, "base_shift": 0.5, "width": 1024, "height": 1024}},
270        }),
271        // https://comfyanonymous.github.io/ComfyUI_examples/qwen_image/ (basic workflow)
272        Workflow::QwenImage => json!({
273            "3": {"class_type": "KSampler", "inputs": {
274                "seed": seed, "steps": 20, "cfg": 2.5, "sampler_name": "euler", "scheduler": "simple",
275                "denoise": 1.0, "model": ["66", 0], "positive": ["6", 0], "negative": ["7", 0],
276                "latent_image": ["58", 0]}},
277            "6": {"class_type": "CLIPTextEncode", "inputs": {"text": prompt, "clip": ["38", 0]}},
278            "7": {"class_type": "CLIPTextEncode", "inputs": {"text": " ", "clip": ["38", 0]}},
279            "8": {"class_type": "VAEDecode", "inputs": {"samples": ["3", 0], "vae": ["39", 0]}},
280            "37": {"class_type": "UNETLoader", "inputs": {"unet_name": name(FileRole::Diffusion)?, "weight_dtype": "default"}},
281            "38": {"class_type": "CLIPLoader", "inputs": {"clip_name": name(FileRole::TextEncoder)?, "type": "qwen_image", "device": "default"}},
282            "39": {"class_type": "VAELoader", "inputs": {"vae_name": name(FileRole::Vae)?}},
283            "58": {"class_type": "EmptySD3LatentImage", "inputs": {"width": 1328, "height": 1328, "batch_size": 1}},
284            "60": {"class_type": "SaveImage", "inputs": {"filename_prefix": "rigspark", "images": ["8", 0]}},
285            "66": {"class_type": "ModelSamplingAuraFlow", "inputs": {"shift": 3.1, "model": ["37", 0]}},
286        }),
287    })
288}
289
290/// Fails closed unless every node is an allowlisted core local node.
291pub fn ensure_local_only(workflow: &Value) -> Result<(), GenerationError> {
292    let nodes = workflow
293        .as_object()
294        .filter(|nodes| !nodes.is_empty() && nodes.len() <= 64)
295        .ok_or(GenerationError::Invalid(
296            "workflow must be a non-empty node map",
297        ))?;
298    for node in nodes.values() {
299        let class = node
300            .get("class_type")
301            .and_then(Value::as_str)
302            .ok_or(GenerationError::Invalid("workflow node has no class_type"))?;
303        if !LOCAL_NODES.contains(&class) || !node.get("inputs").is_some_and(Value::is_object) {
304            return Err(GenerationError::NonLocalNode(
305                class.chars().take(128).collect(),
306            ));
307        }
308    }
309    Ok(())
310}
311
312pub struct GenerationRequest<'request> {
313    pub model: &'request GenerationModel,
314    pub prompt: &'request str,
315    pub seed: u64,
316    pub endpoint: &'request str,
317    pub comfyui_dir: &'request Path,
318    /// Exact output file; when absent a name is derived inside `output_dir`.
319    pub output: Option<&'request Path>,
320    pub output_dir: &'request Path,
321}
322
323#[derive(Debug, Clone, Serialize)]
324#[serde(rename_all = "camelCase")]
325pub struct InstalledWeight {
326    pub role: FileRole,
327    pub name: String,
328    pub bytes: u64,
329    pub cached: bool,
330}
331
332#[derive(Debug, Clone, Serialize)]
333#[serde(rename_all = "camelCase")]
334pub struct GenerationOutcome {
335    pub model: String,
336    pub kind: GenerationKind,
337    pub path: PathBuf,
338    pub bytes: u64,
339    pub prompt_id: String,
340    pub seed: u64,
341    pub weights: Vec<InstalledWeight>,
342}
343
344pub type Events = Arc<dyn Fn(String) + Send + Sync>;
345
346/// What ComfyUI reports about itself before a run.
347#[derive(Debug, Clone, Default)]
348pub struct ComfyStatus {
349    pub devices: Vec<String>,
350    /// Smallest free device memory, falling back to free system RAM; `None` when unreported.
351    pub free_bytes: Option<u64>,
352}
353
354/// Weight file names a workflow graph loads, with `\` normalised to `/`.
355fn model_files(graph: &Value) -> std::collections::BTreeSet<String> {
356    graph
357        .as_object()
358        .into_iter()
359        .flat_map(|nodes| nodes.values())
360        .filter_map(|node| node.get("inputs").and_then(Value::as_object))
361        .flat_map(|inputs| inputs.values())
362        .filter_map(Value::as_str)
363        .filter(|value| value.ends_with(".safetensors"))
364        .map(|value| value.replace('\\', "/"))
365        .collect()
366}
367
368pub struct ComfyUi<'runtime> {
369    pub http: &'runtime dyn Transport,
370    pub download: &'runtime dyn DownloadTransport,
371    pub poll_interval: Duration,
372    pub deadline: Duration,
373    pub events: Option<Events>,
374}
375
376fn prompt_id(value: &Value) -> Result<String, GenerationError> {
377    value
378        .get("prompt_id")
379        .and_then(Value::as_str)
380        .filter(|id| {
381            !id.is_empty()
382                && id.len() <= 128
383                && id
384                    .bytes()
385                    .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
386        })
387        .map(str::to_owned)
388        .ok_or(GenerationError::Response("missing or invalid prompt_id"))
389}
390fn summary(value: &Value) -> String {
391    let mut parts = Vec::new();
392    if let Some(message) = value.pointer("/error/message").and_then(Value::as_str) {
393        parts.push(message.to_owned());
394    }
395    if let Some(nodes) = value.get("node_errors").and_then(Value::as_object) {
396        for (node, detail) in nodes.iter().take(8) {
397            let class = detail
398                .get("class_type")
399                .and_then(Value::as_str)
400                .unwrap_or("?");
401            for error in detail
402                .get("errors")
403                .and_then(Value::as_array)
404                .into_iter()
405                .flatten()
406                .take(4)
407            {
408                let text = error
409                    .get("details")
410                    .or_else(|| error.get("message"))
411                    .and_then(Value::as_str)
412                    .unwrap_or("error");
413                parts.push(format!("node {node} ({class}): {text}"));
414            }
415        }
416    }
417    if let Some(message) = value.get("exception_message").and_then(Value::as_str) {
418        let node = value
419            .get("node_type")
420            .and_then(Value::as_str)
421            .unwrap_or("?");
422        parts.push(format!("{node}: {message}"));
423    }
424    let text = if parts.is_empty() {
425        "no details reported".to_owned()
426    } else {
427        parts.join("; ")
428    };
429    text.chars()
430        .filter(|ch| !ch.is_control())
431        .take(1000)
432        .collect()
433}
434fn safe_output_name(value: &str, extension: &str) -> bool {
435    !value.is_empty()
436        && value.len() <= 255
437        && !value.starts_with('.')
438        && !value.contains(['/', '\\', ':'])
439        && !value.chars().any(char::is_control)
440        && value
441            .rsplit_once('.')
442            .is_some_and(|(_, ext)| ext.eq_ignore_ascii_case(extension))
443}
444fn safe_subfolder(value: &str) -> bool {
445    value.is_empty()
446        || (value.len() <= 255
447            && !value.starts_with('/')
448            && !value.contains(['\\', ':'])
449            && !value.chars().any(char::is_control)
450            && value
451                .split('/')
452                .all(|part| !part.is_empty() && part != "." && part != ".."))
453}
454fn has_magic(kind: GenerationKind, head: &[u8]) -> bool {
455    match kind {
456        GenerationKind::Image => head.starts_with(b"\x89PNG\r\n\x1a\n"),
457        GenerationKind::Video => {
458            head.len() >= 12 && &head[..4] == b"RIFF" && &head[8..12] == b"WEBP"
459        }
460    }
461}
462
463impl ComfyUi<'_> {
464    fn emit(&self, message: String) {
465        if let Some(events) = &self.events {
466            events(message);
467        }
468    }
469
470    async fn get_json(
471        &self,
472        endpoint: &str,
473        path: &str,
474        cancel: &CancellationToken,
475        limit: usize,
476    ) -> Result<Value, GenerationError> {
477        let request = Request::new(endpoint, path, None, None)?;
478        Ok(read_json(self.http, request, cancel, limit).await?)
479    }
480
481    /// Reads a JSON body regardless of status so ComfyUI validation errors can be shown.
482    async fn post_json(
483        &self,
484        endpoint: &str,
485        path: &str,
486        body: Value,
487        cancel: &CancellationToken,
488    ) -> Result<(u16, Value), GenerationError> {
489        let request = Request::new(endpoint, path, Some(body), None)?;
490        let operation = async {
491            let response = self.http.send(request).await?;
492            let mut bytes = Vec::new();
493            response
494                .body
495                .take(4 * 1024 * 1024 + 1)
496                .read_to_end(&mut bytes)
497                .await
498                .map_err(|_| HttpError::Transport)?;
499            if bytes.len() > 4 * 1024 * 1024 {
500                return Err(HttpError::Limit);
501            }
502            let value = if bytes.is_empty() {
503                Value::Null
504            } else {
505                serde_json::from_slice(&bytes).map_err(|_| HttpError::Response)?
506            };
507            Ok((response.status, value))
508        };
509        tokio::select! {
510            biased;
511            _ = cancel.cancelled() => Err(GenerationError::Cancelled),
512            result = tokio::time::timeout(Duration::from_secs(120), operation) =>
513                Ok(result.map_err(|_| HttpError::Timeout)??),
514        }
515    }
516
517    /// Confirms the listener is ComfyUI and returns its reported devices and free memory.
518    pub async fn ready(
519        &self,
520        endpoint: &str,
521        cancel: &CancellationToken,
522    ) -> Result<ComfyStatus, GenerationError> {
523        match self
524            .get_json(endpoint, "/system_stats", cancel, 1024 * 1024)
525            .await
526        {
527            Ok(stats) if stats.get("system").is_some_and(Value::is_object) => {
528                let devices: Vec<&Value> = stats
529                    .get("devices")
530                    .and_then(Value::as_array)
531                    .into_iter()
532                    .flatten()
533                    .take(16)
534                    .collect();
535                let free_bytes = devices
536                    .iter()
537                    .filter_map(|device| device.get("vram_free").and_then(Value::as_u64))
538                    .min()
539                    .or_else(|| stats.pointer("/system/ram_free").and_then(Value::as_u64));
540                Ok(ComfyStatus {
541                    devices: devices
542                        .iter()
543                        .filter_map(|device| device.get("type").and_then(Value::as_str))
544                        .filter(|kind| {
545                            kind.len() <= 32
546                                && kind.bytes().all(|byte| byte.is_ascii_alphanumeric())
547                        })
548                        .map(str::to_owned)
549                        .collect(),
550                    free_bytes,
551                })
552            }
553            Ok(_) => Err(GenerationError::Response("listener is not ComfyUI")),
554            Err(GenerationError::Cancelled) => Err(GenerationError::Cancelled),
555            Err(GenerationError::Http(HttpError::Invalid)) => Err(GenerationError::Invalid(
556                "ComfyUI endpoint must be loopback HTTP",
557            )),
558            Err(_) => Err(GenerationError::Unreachable(endpoint.to_owned())),
559        }
560    }
561
562    /// Installs (or re-verifies) every pinned weight under `<comfyui>/models/<folder>/rigspark`.
563    pub async fn install(
564        &self,
565        model: &GenerationModel,
566        comfyui_dir: &Path,
567        cancel: &CancellationToken,
568    ) -> Result<Vec<InstalledWeight>, GenerationError> {
569        let models = comfyui_dir.join("models");
570        if !std::fs::metadata(&models).is_ok_and(|metadata| metadata.is_dir()) {
571            return Err(GenerationError::Invalid(
572                "ComfyUI directory must contain a models/ folder",
573            ));
574        }
575        let workflow = model
576            .runnable()
577            .map_err(|error| GenerationError::Catalog(error.to_string()))?;
578        let mut installed = Vec::new();
579        for role in workflow.roles() {
580            let file = model
581                .file(*role)
582                .ok_or(GenerationError::Invalid("model is missing a workflow file"))?;
583            let artifact = Artifact {
584                backend: "comfyui".into(),
585                repo: file.repo.clone(),
586                revision: file.revision.clone(),
587                file: file.file.clone(),
588                sha256: file.sha256.clone(),
589                bytes: file.bytes,
590            };
591            let (owner, repository) = file
592                .repo
593                .split_once('/')
594                .ok_or(GenerationError::Invalid("invalid weight repository"))?;
595            let name = format!(
596                "rigspark/comfyui/{owner}/{repository}@{}/{}",
597                file.revision, file.file
598            );
599            let mut acquisition = Acquisition::new(models.join(role.folder()).join("rigspark"))
600                .map_err(GenerationError::Install)?
601                .with_timeout(WEIGHT_TIMEOUT);
602            if let Some(events) = self.events.clone() {
603                let last = Arc::new(std::sync::atomic::AtomicU64::new(u64::MAX));
604                acquisition = acquisition.with_progress(move |done, total, file| {
605                    let percent = done.saturating_mul(100).checked_div(total).unwrap_or(100);
606                    if last.swap(percent / 5, std::sync::atomic::Ordering::Relaxed) != percent / 5 {
607                        events(format!(
608                            "Weights {file}: {percent}% of {:.1} GiB",
609                            total as f64 / 1073741824.0
610                        ));
611                    }
612                });
613            }
614            let acquired = acquisition
615                .acquire(&artifact, self.download, cancel)
616                .await
617                .map_err(|error| {
618                    if cancel.is_cancelled() {
619                        GenerationError::Cancelled
620                    } else {
621                        GenerationError::Install(error)
622                    }
623                })?;
624            if !acquired
625                .path
626                .ends_with(name.trim_start_matches("rigspark/"))
627            {
628                return Err(GenerationError::Install(
629                    "unexpected weight location".into(),
630                ));
631            }
632            installed.push(InstalledWeight {
633                role: *role,
634                name,
635                bytes: acquired.bytes,
636                cached: acquired.cached,
637            });
638        }
639        Ok(installed)
640    }
641
642    /// Replaces each expected name with ComfyUI's own spelling (e.g. `\` on Windows).
643    async fn resolve_names(
644        &self,
645        endpoint: &str,
646        weights: &mut [InstalledWeight],
647        cancel: &CancellationToken,
648    ) -> Result<(), GenerationError> {
649        for weight in weights {
650            let listing = self
651                .get_json(
652                    endpoint,
653                    &format!("/models/{}", weight.role.folder()),
654                    cancel,
655                    4 * 1024 * 1024,
656                )
657                .await?;
658            let listed = listing
659                .as_array()
660                .ok_or(GenerationError::Response("model listing is not an array"))?
661                .iter()
662                .filter_map(Value::as_str)
663                .find(|entry| entry.replace('\\', "/") == weight.name)
664                .ok_or_else(|| GenerationError::NotVisible(weight.name.clone()))?;
665            weight.name = listed.to_owned();
666        }
667        Ok(())
668    }
669
670    /// Weight files of the most recent ComfyUI prompt, best effort.
671    async fn previous_files(
672        &self,
673        endpoint: &str,
674        cancel: &CancellationToken,
675    ) -> Option<std::collections::BTreeSet<String>> {
676        let request = Request::new(endpoint, "/history", None, None)
677            .ok()?
678            .with_query(&[("max_items", "1")])
679            .ok()?;
680        let history = read_json(self.http, request, cancel, 16 * 1024 * 1024)
681            .await
682            .ok()?;
683        let graph = history
684            .as_object()?
685            .values()
686            .next_back()?
687            .get("prompt")?
688            .get(2)?;
689        Some(model_files(graph)).filter(|files| !files.is_empty())
690    }
691
692    /// Unloads another workflow's weights before switching models when memory is shared (Apple
693    /// MPS) or free device memory is below this model's weights. Live testing on a 36 GB M4 Max:
694    /// Wan swapped and sampled ~4x slower beside a resident FLUX even with more free memory than
695    /// Wan's weights, because activations and upcast encoders need far more than the files.
696    async fn make_room(
697        &self,
698        endpoint: &str,
699        status: &ComfyStatus,
700        needed: u64,
701        graph: &Value,
702        cancel: &CancellationToken,
703    ) -> Result<(), GenerationError> {
704        let shared = apple_mps(&status.devices);
705        let short = status.free_bytes.is_some_and(|free| free < needed);
706        if !shared && !short {
707            return Ok(());
708        }
709        let Some(previous) = self.previous_files(endpoint, cancel).await else {
710            return Ok(());
711        };
712        if previous == model_files(graph) {
713            return Ok(());
714        }
715        let free = status.free_bytes.map_or_else(
716            || "unknown".to_owned(),
717            |free| format!("{:.1} GiB", free as f64 / 1073741824.0),
718        );
719        self.emit(format!(
720            "Freeing ComfyUI memory before switching models ({free} free, {:.1} GiB of weights needed{})",
721            needed as f64 / 1073741824.0,
722            if shared { ", shared Apple memory" } else { "" }
723        ));
724        let (code, _) = self
725            .post_json(
726                endpoint,
727                "/free",
728                json!({"unload_models": true, "free_memory": true}),
729                cancel,
730            )
731            .await?;
732        if !(200..300).contains(&code) {
733            self.emit(format!(
734                "ComfyUI declined to free memory (HTTP {code}); continuing"
735            ));
736        }
737        Ok(())
738    }
739
740    async fn cancel_remote(&self, endpoint: &str, id: &str) {
741        let fresh = CancellationToken::new();
742        for (path, body) in [
743            ("/queue", json!({"delete": [id]})),
744            ("/interrupt", json!({"prompt_id": id})),
745        ] {
746            let _ = tokio::time::timeout(
747                Duration::from_secs(5),
748                self.post_json(endpoint, path, body, &fresh),
749            )
750            .await;
751        }
752    }
753
754    async fn wait(
755        &self,
756        endpoint: &str,
757        id: &str,
758        cancel: &CancellationToken,
759    ) -> Result<Value, GenerationError> {
760        let started = tokio::time::Instant::now();
761        loop {
762            if started.elapsed() > self.deadline {
763                return Err(GenerationError::Timeout);
764            }
765            tokio::select! {
766                biased;
767                _ = cancel.cancelled() => return Err(GenerationError::Cancelled),
768                _ = tokio::time::sleep(self.poll_interval) => {}
769            }
770            let history = self
771                .get_json(
772                    endpoint,
773                    &format!("/history/{id}"),
774                    cancel,
775                    16 * 1024 * 1024,
776                )
777                .await?;
778            let Some(entry) = history.get(id) else {
779                continue;
780            };
781            let status = entry.get("status");
782            if status
783                .and_then(|status| status.get("status_str"))
784                .and_then(Value::as_str)
785                == Some("error")
786            {
787                let detail = status
788                    .and_then(|status| status.get("messages"))
789                    .and_then(Value::as_array)
790                    .into_iter()
791                    .flatten()
792                    .find(|message| {
793                        message.get(0).and_then(Value::as_str) == Some("execution_error")
794                    })
795                    .and_then(|message| message.get(1))
796                    .map(summary)
797                    .unwrap_or_else(|| "execution error".into());
798                return Err(GenerationError::Failed(detail));
799            }
800            if status
801                .and_then(|status| status.get("completed"))
802                .and_then(Value::as_bool)
803                == Some(true)
804            {
805                return entry
806                    .get("outputs")
807                    .cloned()
808                    .ok_or(GenerationError::Response("completed prompt has no outputs"));
809            }
810        }
811    }
812
813    async fn fetch(
814        &self,
815        endpoint: &str,
816        kind: GenerationKind,
817        descriptor: &Value,
818        target: &Path,
819        cancel: &CancellationToken,
820    ) -> Result<u64, GenerationError> {
821        let extension = output_extension(kind);
822        let field = |key| descriptor.get(key).and_then(Value::as_str);
823        let filename = field("filename")
824            .filter(|name| safe_output_name(name, extension))
825            .ok_or(GenerationError::Response("unsafe output filename"))?;
826        let subfolder = field("subfolder")
827            .filter(|folder| safe_subfolder(folder))
828            .ok_or(GenerationError::Response("unsafe output subfolder"))?;
829        if field("type") != Some("output") {
830            return Err(GenerationError::Response("output is not a saved file"));
831        }
832        let request = Request::new(endpoint, "/view", None, None)?.with_query(&[
833            ("filename", filename),
834            ("subfolder", subfolder),
835            ("type", "output"),
836        ])?;
837        let parent = target
838            .parent()
839            .filter(|parent| !parent.as_os_str().is_empty())
840            .unwrap_or(Path::new("."));
841        let operation = async {
842            let response = self.http.send(request).await?;
843            if !(200..300).contains(&response.status) {
844                return Err(GenerationError::Http(HttpError::Status(response.status)));
845            }
846            let temporary = tempfile::Builder::new()
847                .prefix(".rigspark-output.")
848                .suffix(".part")
849                .tempfile_in(parent)
850                .map_err(|error| GenerationError::Output(error.to_string()))?;
851            let mut output = tokio::fs::File::from_std(
852                temporary
853                    .reopen()
854                    .map_err(|error| GenerationError::Output(error.to_string()))?,
855            );
856            let mut body = response.body.take(MAX_OUTPUT_BYTES + 1);
857            let mut buffer = vec![0u8; 65536];
858            let mut head = Vec::with_capacity(16);
859            let mut total = 0u64;
860            loop {
861                let count = body
862                    .read(&mut buffer)
863                    .await
864                    .map_err(|_| HttpError::Transport)?;
865                if count == 0 {
866                    break;
867                }
868                total += count as u64;
869                if total > MAX_OUTPUT_BYTES {
870                    return Err(GenerationError::Response("output exceeds 1 GiB"));
871                }
872                if head.len() < 16 {
873                    let needed = (16 - head.len()).min(count);
874                    head.extend_from_slice(&buffer[..needed]);
875                }
876                output
877                    .write_all(&buffer[..count])
878                    .await
879                    .map_err(|error| GenerationError::Output(error.to_string()))?;
880            }
881            if !has_magic(kind, &head) {
882                return Err(GenerationError::Response(
883                    "output is not the expected media type",
884                ));
885            }
886            output
887                .sync_all()
888                .await
889                .map_err(|error| GenerationError::Output(error.to_string()))?;
890            drop(output);
891            temporary.persist_noclobber(target).map_err(|error| {
892                if error.error.kind() == std::io::ErrorKind::AlreadyExists {
893                    GenerationError::Exists(target.to_path_buf())
894                } else {
895                    GenerationError::Output(error.error.to_string())
896                }
897            })?;
898            Ok(total)
899        };
900        tokio::select! {
901            biased;
902            _ = cancel.cancelled() => Err(GenerationError::Cancelled),
903            result = tokio::time::timeout(Duration::from_secs(600), operation) =>
904                result.map_err(|_| GenerationError::Timeout)?,
905        }
906    }
907
908    pub async fn generate(
909        &self,
910        request: &GenerationRequest<'_>,
911        cancel: &CancellationToken,
912    ) -> Result<GenerationOutcome, GenerationError> {
913        let model = request.model;
914        validate_prompt(request.prompt)?;
915        if request.seed > MAX_SEED {
916            return Err(GenerationError::Invalid(
917                "seed must be in 0..=9007199254740991",
918            ));
919        }
920        crate::state::loopback(request.endpoint)
921            .map_err(|_| GenerationError::Invalid("ComfyUI endpoint must be loopback HTTP"))?;
922        let extension = output_extension(model.kind);
923        if let Some(output) = request.output {
924            if !output
925                .extension()
926                .and_then(|ext| ext.to_str())
927                .is_some_and(|ext| ext.eq_ignore_ascii_case(extension))
928            {
929                return Err(GenerationError::Invalid(match model.kind {
930                    GenerationKind::Image => "image output must use the .png extension",
931                    GenerationKind::Video => "video output must use the .webp extension",
932                }));
933            }
934            if std::fs::symlink_metadata(output).is_ok() {
935                return Err(GenerationError::Exists(output.to_path_buf()));
936            }
937        }
938        let built_in = model
939            .runnable()
940            .map_err(|error| GenerationError::Catalog(error.to_string()))?;
941        let status = self.ready(request.endpoint, cancel).await?;
942        let devices = &status.devices;
943        if matches!(built_in, Workflow::WanT2v | Workflow::Wan22Ti2v) && apple_mps(devices) {
944            self.emit(
945                "ComfyUI reports Apple MPS: using the euler sampler (uni_pc diverges on MPS)"
946                    .into(),
947            );
948        }
949        if built_in == Workflow::MingDesign {
950            let names = model
951                .files
952                .iter()
953                .map(|file| (file.role, file.file.clone()))
954                .collect();
955            let graph = workflow(model, &names, request.prompt, request.seed, devices)?;
956            self.emit("Checking native Ming Design capabilities before downloading weights".into());
957            preflight::ming(self, request.endpoint, &graph, cancel).await?;
958        }
959        self.emit(format!("Verifying weights for {}", model.id));
960        let mut weights = self.install(model, request.comfyui_dir, cancel).await?;
961        self.resolve_names(request.endpoint, &mut weights, cancel)
962            .await?;
963        let names = weights
964            .iter()
965            .map(|weight| (weight.role, weight.name.clone()))
966            .collect();
967        let graph = workflow(model, &names, request.prompt, request.seed, devices)?;
968        ensure_local_only(&graph)?;
969        self.make_room(
970            request.endpoint,
971            &status,
972            model.total_bytes(),
973            &graph,
974            cancel,
975        )
976        .await?;
977        let (status, response) = self
978            .post_json(
979                request.endpoint,
980                "/prompt",
981                json!({"prompt": graph}),
982                cancel,
983            )
984            .await?;
985        if !(200..300).contains(&status) {
986            return Err(GenerationError::Rejected(summary(&response)));
987        }
988        if response
989            .get("node_errors")
990            .and_then(Value::as_object)
991            .is_some_and(|errors| !errors.is_empty())
992        {
993            return Err(GenerationError::Rejected(summary(&response)));
994        }
995        let id = prompt_id(&response)?;
996        self.emit(format!(
997            "Queued ComfyUI prompt {id}; generating {}",
998            model.kind.name()
999        ));
1000        let outputs = match self.wait(request.endpoint, &id, cancel).await {
1001            Ok(outputs) => outputs,
1002            Err(error) => {
1003                if matches!(error, GenerationError::Cancelled | GenerationError::Timeout) {
1004                    self.cancel_remote(request.endpoint, &id).await;
1005                }
1006                return Err(error);
1007            }
1008        };
1009        let descriptor = outputs
1010            .get(save_node(built_in))
1011            .and_then(|node| node.get("images"))
1012            .and_then(Value::as_array)
1013            .and_then(|images| images.first())
1014            .ok_or(GenerationError::Response("save node produced no output"))?;
1015        let target = match request.output {
1016            Some(path) => path.to_path_buf(),
1017            None => request.output_dir.join(format!(
1018                "rigspark-{}-{}.{extension}",
1019                model.kind.name(),
1020                &id[..id.len().min(8)]
1021            )),
1022        };
1023        let bytes = self
1024            .fetch(request.endpoint, model.kind, descriptor, &target, cancel)
1025            .await?;
1026        Ok(GenerationOutcome {
1027            model: model.id.clone(),
1028            kind: model.kind,
1029            path: target,
1030            bytes,
1031            prompt_id: id,
1032            seed: request.seed,
1033            weights,
1034        })
1035    }
1036}
1037
1038/// Inputs shared by the CLI, terminal UI, and browser GUI.
1039#[derive(Debug, Clone)]
1040pub struct NativeOptions {
1041    pub model: String,
1042    pub prompt: String,
1043    pub seed: Option<u64>,
1044    pub comfyui_dir: PathBuf,
1045    pub port: u16,
1046    pub bypass: bool,
1047    pub output: Option<PathBuf>,
1048    pub output_dir: PathBuf,
1049}
1050
1051#[derive(Debug, Clone, Serialize)]
1052pub struct NativeOutcome {
1053    pub result: GenerationOutcome,
1054    pub fit: GenerationFit,
1055}
1056
1057/// `RIGSPARK_COMFYUI_DIR`, else `~/ComfyUI` when it looks like a ComfyUI install.
1058pub fn default_comfyui_dir() -> Option<PathBuf> {
1059    if let Some(dir) = std::env::var_os("RIGSPARK_COMFYUI_DIR").filter(|dir| !dir.is_empty()) {
1060        return Some(PathBuf::from(dir));
1061    }
1062    let home = std::env::var_os("HOME")
1063        .or_else(|| std::env::var_os("USERPROFILE"))
1064        .filter(|home| !home.is_empty())?;
1065    let candidate = PathBuf::from(home).join("ComfyUI");
1066    candidate.join("models").is_dir().then_some(candidate)
1067}
1068
1069/// Validates everything that needs no hardware, network, or ComfyUI access.
1070pub fn prepare(
1071    options: &NativeOptions,
1072) -> Result<(GenerationModel, u64, PathBuf), GenerationError> {
1073    validate_prompt(&options.prompt)?;
1074    let seed = match options.seed {
1075        Some(seed) if seed <= MAX_SEED => seed,
1076        Some(_) => {
1077            return Err(GenerationError::Invalid(
1078                "seed must be in 0..=9007199254740991",
1079            ));
1080        }
1081        None => random_seed(),
1082    };
1083    if options.port == 0 {
1084        return Err(GenerationError::Invalid(
1085            "ComfyUI port must be in 1..=65535",
1086        ));
1087    }
1088    let catalog = GenerationCatalog::bundled()
1089        .map_err(|error| GenerationError::Catalog(error.to_string()))?;
1090    let model = catalog
1091        .resolve(&options.model)
1092        .map_err(|error| GenerationError::Catalog(error.to_string()))?
1093        .clone();
1094    model
1095        .runnable()
1096        .map_err(|error| GenerationError::Catalog(error.to_string()))?;
1097    if options.comfyui_dir.as_os_str().is_empty() {
1098        return Err(GenerationError::Invalid(
1099            "a ComfyUI directory is required (--comfyui-dir or RIGSPARK_COMFYUI_DIR)",
1100        ));
1101    }
1102    let dir = std::path::absolute(&options.comfyui_dir)
1103        .map_err(|_| GenerationError::Invalid("invalid ComfyUI directory"))?;
1104    if !dir.join("models").is_dir() {
1105        return Err(GenerationError::Invalid(
1106            "ComfyUI directory must contain a models/ folder",
1107        ));
1108    }
1109    Ok((model, seed, dir))
1110}
1111
1112/// Full local run: validate, check memory fit, then drive the loopback ComfyUI server.
1113pub async fn run_native(
1114    options: &NativeOptions,
1115    events: Option<Events>,
1116    cancel: &CancellationToken,
1117) -> Result<NativeOutcome, GenerationError> {
1118    let (model, seed, comfyui_dir) = prepare(options)?;
1119    let emit = |message: String| {
1120        if let Some(events) = &events {
1121            events(message);
1122        }
1123    };
1124    let (hardware, warnings) = crate::hardware::detect()
1125        .await
1126        .map_err(|error| GenerationError::Hardware(error.to_string()))?;
1127    for warning in warnings {
1128        emit(warning);
1129    }
1130    let memory = fit(&model, &hardware);
1131    if memory.verdict == FitVerdict::No && !options.bypass {
1132        return Err(GenerationError::NoFit(format!(
1133            "{} will not fit this machine ({}); bypass to try anyway (integrity checks still apply)",
1134            model.id, memory.reason
1135        )));
1136    }
1137    emit(format!(
1138        "Model {} ({}): fit {} - {}; speed unknown",
1139        model.id,
1140        model.kind.name(),
1141        memory.verdict.name(),
1142        memory.reason
1143    ));
1144    let http = crate::http::NativeTransport::new()?;
1145    let download = crate::acquire::HfTransport::new().map_err(GenerationError::Install)?;
1146    let client = ComfyUi {
1147        http: &http,
1148        download: &download,
1149        poll_interval: Duration::from_secs(1),
1150        deadline: Duration::from_secs(2 * 60 * 60),
1151        events: events.clone(),
1152    };
1153    let endpoint = format!("http://127.0.0.1:{}", options.port);
1154    let result = client
1155        .generate(
1156            &GenerationRequest {
1157                model: &model,
1158                prompt: &options.prompt,
1159                seed,
1160                endpoint: &endpoint,
1161                comfyui_dir: &comfyui_dir,
1162                output: options.output.as_deref(),
1163                output_dir: &options.output_dir,
1164            },
1165            cancel,
1166        )
1167        .await?;
1168    Ok(NativeOutcome {
1169        result,
1170        fit: memory,
1171    })
1172}
1173
1174pub type GenerationFuture<'call> = std::pin::Pin<
1175    Box<dyn std::future::Future<Output = Result<NativeOutcome, GenerationError>> + Send + 'call>,
1176>;
1177
1178/// Injectable generation boundary so the TUI and GUI can be tested without ComfyUI.
1179pub trait Generator: Send + Sync {
1180    fn generate(
1181        &self,
1182        options: NativeOptions,
1183        events: Events,
1184        cancel: CancellationToken,
1185    ) -> GenerationFuture<'_>;
1186}
1187
1188pub struct NativeGenerator;
1189impl Generator for NativeGenerator {
1190    fn generate(
1191        &self,
1192        options: NativeOptions,
1193        events: Events,
1194        cancel: CancellationToken,
1195    ) -> GenerationFuture<'_> {
1196        Box::pin(async move { run_native(&options, Some(events), &cancel).await })
1197    }
1198}