Skip to main content

rigspark_runtime/
generation.rs

1//! Local image/video generation through an attached, loopback-only ComfyUI server.
2//!
3//! Only built-in workflows composed of allowlisted core nodes are ever submitted, so paid
4//! ComfyUI Partner (API) nodes and cloud services can never be reached through rigspark.
5use crate::{
6    acquire::{Acquisition, Artifact, DownloadTransport},
7    http::{HttpError, Request, Transport, read_json},
8};
9use rigspark_core::generation::{
10    FileRole, FitVerdict, GenerationCatalog, GenerationFit, GenerationKind, GenerationModel,
11    Workflow, fit,
12};
13use serde::Serialize;
14use serde_json::{Value, json};
15use std::{
16    collections::HashMap,
17    path::{Path, PathBuf},
18    sync::Arc,
19    time::Duration,
20};
21use tokio::io::{AsyncReadExt, AsyncWriteExt};
22use tokio_util::sync::CancellationToken;
23
24pub const DEFAULT_PORT: u16 = 8188;
25pub const MAX_PROMPT_BYTES: usize = 16 * 1024;
26pub const MAX_SEED: u64 = 9_007_199_254_740_991;
27const MAX_OUTPUT_BYTES: u64 = 1024 * 1024 * 1024;
28const WEIGHT_TIMEOUT: Duration = Duration::from_secs(6 * 60 * 60);
29/// Core ComfyUI nodes used by the built-in workflows; anything else is refused.
30pub const LOCAL_NODES: [&str; 21] = [
31    "BasicGuider",
32    "BasicScheduler",
33    "CheckpointLoaderSimple",
34    "CLIPLoader",
35    "CLIPTextEncode",
36    "DualCLIPLoader",
37    "EmptyHunyuanLatentVideo",
38    "EmptyLatentImage",
39    "EmptySD3LatentImage",
40    "KSampler",
41    "KSamplerSelect",
42    "ModelSamplingAuraFlow",
43    "ModelSamplingSD3",
44    "RandomNoise",
45    "SamplerCustomAdvanced",
46    "SaveAnimatedWEBP",
47    "SaveImage",
48    "UNETLoader",
49    "VAEDecode",
50    "VAELoader",
51    "Wan22ImageToVideoLatent",
52];
53/// Wan 2.1's published default negative prompt (Apache-2.0), as used by the official
54/// ComfyUI example: https://comfyanonymous.github.io/ComfyUI_examples/wan/
55const WAN_NEGATIVE_PROMPT: &str = "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走";
56
57#[derive(Debug, thiserror::Error)]
58pub enum GenerationError {
59    #[error("invalid generation request: {0}")]
60    Invalid(&'static str),
61    #[error("{0}")]
62    Catalog(String),
63    #[error("{0}")]
64    NoFit(String),
65    #[error("hardware detection failed: {0}")]
66    Hardware(String),
67    #[error(
68        "ComfyUI is not reachable at {0}; start ComfyUI on loopback (default port 8188) and retry"
69    )]
70    Unreachable(String),
71    #[error("weight installation failed: {0}")]
72    Install(String),
73    #[error(
74        "ComfyUI does not list {0}; make sure --comfyui-dir points at the running ComfyUI installation"
75    )]
76    NotVisible(String),
77    #[error("workflow node `{0}` is not an allowlisted local node; refusing to submit")]
78    NonLocalNode(String),
79    #[error("ComfyUI rejected the workflow: {0}")]
80    Rejected(String),
81    #[error("ComfyUI generation failed: {0}")]
82    Failed(String),
83    #[error("invalid ComfyUI response: {0}")]
84    Response(&'static str),
85    #[error("output already exists: {0}")]
86    Exists(PathBuf),
87    #[error("could not write output: {0}")]
88    Output(String),
89    #[error("generation timed out")]
90    Timeout,
91    #[error("operation cancelled")]
92    Cancelled,
93    #[error(transparent)]
94    Http(HttpError),
95}
96impl From<HttpError> for GenerationError {
97    fn from(error: HttpError) -> Self {
98        match error {
99            HttpError::Cancelled => Self::Cancelled,
100            other => Self::Http(other),
101        }
102    }
103}
104
105pub fn output_extension(kind: GenerationKind) -> &'static str {
106    match kind {
107        GenerationKind::Image => "png",
108        GenerationKind::Video => "webp",
109    }
110}
111fn save_node(workflow: Workflow) -> &'static str {
112    match workflow {
113        Workflow::FluxCheckpoint | Workflow::FluxSplit => "9",
114        Workflow::WanT2v | Workflow::Wan22Ti2v => "28",
115        Workflow::QwenImage => "60",
116    }
117}
118
119/// Random seed in the JSON-safe integer range, reported back so runs are reproducible.
120pub fn random_seed() -> u64 {
121    uuid::Uuid::new_v4().as_u64_pair().0 & MAX_SEED
122}
123
124pub fn validate_prompt(prompt: &str) -> Result<(), GenerationError> {
125    if prompt.trim().is_empty()
126        || prompt.len() > MAX_PROMPT_BYTES
127        || prompt
128            .chars()
129            .any(|ch| ch.is_control() && ch != '\n' && ch != '\t')
130    {
131        return Err(GenerationError::Invalid(
132            "prompt must be 1..=16384 bytes of text without control characters",
133        ));
134    }
135    Ok(())
136}
137
138/// ComfyUI reports its compute device per `/system_stats`; Apple GPUs report `mps`.
139pub fn apple_mps(devices: &[String]) -> bool {
140    devices.iter().any(|device| device == "mps")
141}
142
143/// Wan's official `uni_pc` sampler diverged into noise on Apple MPS in live testing (ComfyUI
144/// 0.38, 30 steps), while `euler` at the official size and length stayed coherent.
145fn wan_sampler(devices: &[String]) -> &'static str {
146    if apple_mps(devices) {
147        "euler"
148    } else {
149        "uni_pc"
150    }
151}
152
153/// Builds the API-format graph published in the official ComfyUI examples.
154pub fn workflow(
155    model: &GenerationModel,
156    names: &HashMap<FileRole, String>,
157    prompt: &str,
158    seed: u64,
159    devices: &[String],
160) -> Result<Value, GenerationError> {
161    let name = |role| {
162        names
163            .get(&role)
164            .cloned()
165            .ok_or(GenerationError::Invalid("missing installed weight name"))
166    };
167    let workflow = model
168        .runnable()
169        .map_err(|error| GenerationError::Catalog(error.to_string()))?;
170    Ok(match workflow {
171        // https://comfyanonymous.github.io/ComfyUI_examples/flux/ (schnell fp8 checkpoint)
172        Workflow::FluxCheckpoint => json!({
173            "6": {"class_type": "CLIPTextEncode", "inputs": {"text": prompt, "clip": ["30", 1]}},
174            "8": {"class_type": "VAEDecode", "inputs": {"samples": ["31", 0], "vae": ["30", 2]}},
175            "9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "rigspark", "images": ["8", 0]}},
176            "27": {"class_type": "EmptySD3LatentImage", "inputs": {"width": 1024, "height": 1024, "batch_size": 1}},
177            "30": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": name(FileRole::Checkpoint)?}},
178            "31": {"class_type": "KSampler", "inputs": {
179                "seed": seed, "steps": 4, "cfg": 1.0, "sampler_name": "euler", "scheduler": "simple",
180                "denoise": 1.0, "model": ["30", 0], "positive": ["6", 0], "negative": ["33", 0],
181                "latent_image": ["27", 0]}},
182            "33": {"class_type": "CLIPTextEncode", "inputs": {"text": "", "clip": ["30", 1]}},
183        }),
184        // https://comfyanonymous.github.io/ComfyUI_examples/flux/ (regular full version)
185        Workflow::FluxSplit => json!({
186            "5": {"class_type": "EmptyLatentImage", "inputs": {"width": 1024, "height": 1024, "batch_size": 1}},
187            "6": {"class_type": "CLIPTextEncode", "inputs": {"text": prompt, "clip": ["11", 0]}},
188            "8": {"class_type": "VAEDecode", "inputs": {"samples": ["13", 0], "vae": ["10", 0]}},
189            "9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "rigspark", "images": ["8", 0]}},
190            "10": {"class_type": "VAELoader", "inputs": {"vae_name": name(FileRole::Vae)?}},
191            "11": {"class_type": "DualCLIPLoader", "inputs": {
192                "clip_name1": name(FileRole::TextEncoder)?,
193                "clip_name2": name(FileRole::Clip)?,
194                "type": "flux"}},
195            "12": {"class_type": "UNETLoader", "inputs": {
196                "unet_name": name(FileRole::Diffusion)?, "weight_dtype": "default"}},
197            "13": {"class_type": "SamplerCustomAdvanced", "inputs": {
198                "noise": ["25", 0], "guider": ["22", 0], "sampler": ["16", 0],
199                "sigmas": ["17", 0], "latent_image": ["5", 0]}},
200            "16": {"class_type": "KSamplerSelect", "inputs": {"sampler_name": "euler"}},
201            "17": {"class_type": "BasicScheduler", "inputs": {
202                "scheduler": "simple", "steps": 4, "denoise": 1.0, "model": ["12", 0]}},
203            "22": {"class_type": "BasicGuider", "inputs": {
204                "model": ["12", 0], "conditioning": ["6", 0]}},
205            "25": {"class_type": "RandomNoise", "inputs": {"noise_seed": seed}},
206        }),
207        // https://comfyanonymous.github.io/ComfyUI_examples/wan/ (text to video, 1.3B)
208        Workflow::WanT2v => json!({
209            "3": {"class_type": "KSampler", "inputs": {
210                "seed": seed, "steps": 30, "cfg": 6.0, "sampler_name": wan_sampler(devices), "scheduler": "simple",
211                "denoise": 1.0, "model": ["48", 0], "positive": ["6", 0], "negative": ["7", 0],
212                "latent_image": ["40", 0]}},
213            "6": {"class_type": "CLIPTextEncode", "inputs": {"text": prompt, "clip": ["38", 0]}},
214            "7": {"class_type": "CLIPTextEncode", "inputs": {"text": WAN_NEGATIVE_PROMPT, "clip": ["38", 0]}},
215            "8": {"class_type": "VAEDecode", "inputs": {"samples": ["3", 0], "vae": ["39", 0]}},
216            "28": {"class_type": "SaveAnimatedWEBP", "inputs": {
217                "filename_prefix": "rigspark", "fps": 16.0, "lossless": false, "quality": 90,
218                "method": "default", "images": ["8", 0]}},
219            "37": {"class_type": "UNETLoader", "inputs": {"unet_name": name(FileRole::Diffusion)?, "weight_dtype": "default"}},
220            "38": {"class_type": "CLIPLoader", "inputs": {"clip_name": name(FileRole::TextEncoder)?, "type": "wan", "device": "default"}},
221            "39": {"class_type": "VAELoader", "inputs": {"vae_name": name(FileRole::Vae)?}},
222            "40": {"class_type": "EmptyHunyuanLatentVideo", "inputs": {"width": 832, "height": 480, "length": 49, "batch_size": 1}},
223            "48": {"class_type": "ModelSamplingSD3", "inputs": {"shift": 8.0, "model": ["37", 0]}},
224        }),
225        // https://comfyanonymous.github.io/ComfyUI_examples/wan22/ (5B text to video)
226        Workflow::Wan22Ti2v => json!({
227            "3": {"class_type": "KSampler", "inputs": {
228                "seed": seed, "steps": 30, "cfg": 5.0, "sampler_name": wan_sampler(devices), "scheduler": "simple",
229                "denoise": 1.0, "model": ["48", 0], "positive": ["6", 0], "negative": ["7", 0],
230                "latent_image": ["55", 0]}},
231            "6": {"class_type": "CLIPTextEncode", "inputs": {"text": prompt, "clip": ["38", 0]}},
232            "7": {"class_type": "CLIPTextEncode", "inputs": {"text": WAN_NEGATIVE_PROMPT, "clip": ["38", 0]}},
233            "8": {"class_type": "VAEDecode", "inputs": {"samples": ["3", 0], "vae": ["39", 0]}},
234            "28": {"class_type": "SaveAnimatedWEBP", "inputs": {
235                "filename_prefix": "rigspark", "fps": 24.0, "lossless": false, "quality": 90,
236                "method": "default", "images": ["8", 0]}},
237            "37": {"class_type": "UNETLoader", "inputs": {"unet_name": name(FileRole::Diffusion)?, "weight_dtype": "default"}},
238            "38": {"class_type": "CLIPLoader", "inputs": {"clip_name": name(FileRole::TextEncoder)?, "type": "wan", "device": "default"}},
239            "39": {"class_type": "VAELoader", "inputs": {"vae_name": name(FileRole::Vae)?}},
240            "48": {"class_type": "ModelSamplingSD3", "inputs": {"shift": 8.0, "model": ["37", 0]}},
241            "55": {"class_type": "Wan22ImageToVideoLatent", "inputs": {
242                "width": 1280, "height": 704, "length": 41, "batch_size": 1, "vae": ["39", 0]}},
243        }),
244        // https://comfyanonymous.github.io/ComfyUI_examples/qwen_image/ (basic workflow)
245        Workflow::QwenImage => json!({
246            "3": {"class_type": "KSampler", "inputs": {
247                "seed": seed, "steps": 20, "cfg": 2.5, "sampler_name": "euler", "scheduler": "simple",
248                "denoise": 1.0, "model": ["66", 0], "positive": ["6", 0], "negative": ["7", 0],
249                "latent_image": ["58", 0]}},
250            "6": {"class_type": "CLIPTextEncode", "inputs": {"text": prompt, "clip": ["38", 0]}},
251            "7": {"class_type": "CLIPTextEncode", "inputs": {"text": " ", "clip": ["38", 0]}},
252            "8": {"class_type": "VAEDecode", "inputs": {"samples": ["3", 0], "vae": ["39", 0]}},
253            "37": {"class_type": "UNETLoader", "inputs": {"unet_name": name(FileRole::Diffusion)?, "weight_dtype": "default"}},
254            "38": {"class_type": "CLIPLoader", "inputs": {"clip_name": name(FileRole::TextEncoder)?, "type": "qwen_image", "device": "default"}},
255            "39": {"class_type": "VAELoader", "inputs": {"vae_name": name(FileRole::Vae)?}},
256            "58": {"class_type": "EmptySD3LatentImage", "inputs": {"width": 1328, "height": 1328, "batch_size": 1}},
257            "60": {"class_type": "SaveImage", "inputs": {"filename_prefix": "rigspark", "images": ["8", 0]}},
258            "66": {"class_type": "ModelSamplingAuraFlow", "inputs": {"shift": 3.1, "model": ["37", 0]}},
259        }),
260    })
261}
262
263/// Fails closed unless every node is an allowlisted core local node.
264pub fn ensure_local_only(workflow: &Value) -> Result<(), GenerationError> {
265    let nodes = workflow
266        .as_object()
267        .filter(|nodes| !nodes.is_empty() && nodes.len() <= 64)
268        .ok_or(GenerationError::Invalid(
269            "workflow must be a non-empty node map",
270        ))?;
271    for node in nodes.values() {
272        let class = node
273            .get("class_type")
274            .and_then(Value::as_str)
275            .ok_or(GenerationError::Invalid("workflow node has no class_type"))?;
276        if !LOCAL_NODES.contains(&class) || !node.get("inputs").is_some_and(Value::is_object) {
277            return Err(GenerationError::NonLocalNode(
278                class.chars().take(128).collect(),
279            ));
280        }
281    }
282    Ok(())
283}
284
285pub struct GenerationRequest<'request> {
286    pub model: &'request GenerationModel,
287    pub prompt: &'request str,
288    pub seed: u64,
289    pub endpoint: &'request str,
290    pub comfyui_dir: &'request Path,
291    /// Exact output file; when absent a name is derived inside `output_dir`.
292    pub output: Option<&'request Path>,
293    pub output_dir: &'request Path,
294}
295
296#[derive(Debug, Clone, Serialize)]
297#[serde(rename_all = "camelCase")]
298pub struct InstalledWeight {
299    pub role: FileRole,
300    pub name: String,
301    pub bytes: u64,
302    pub cached: bool,
303}
304
305#[derive(Debug, Clone, Serialize)]
306#[serde(rename_all = "camelCase")]
307pub struct GenerationOutcome {
308    pub model: String,
309    pub kind: GenerationKind,
310    pub path: PathBuf,
311    pub bytes: u64,
312    pub prompt_id: String,
313    pub seed: u64,
314    pub weights: Vec<InstalledWeight>,
315}
316
317pub type Events = Arc<dyn Fn(String) + Send + Sync>;
318
319/// What ComfyUI reports about itself before a run.
320#[derive(Debug, Clone, Default)]
321pub struct ComfyStatus {
322    pub devices: Vec<String>,
323    /// Smallest free device memory, falling back to free system RAM; `None` when unreported.
324    pub free_bytes: Option<u64>,
325}
326
327/// Weight file names a workflow graph loads, with `\` normalised to `/`.
328fn model_files(graph: &Value) -> std::collections::BTreeSet<String> {
329    graph
330        .as_object()
331        .into_iter()
332        .flat_map(|nodes| nodes.values())
333        .filter_map(|node| node.get("inputs").and_then(Value::as_object))
334        .flat_map(|inputs| inputs.values())
335        .filter_map(Value::as_str)
336        .filter(|value| value.ends_with(".safetensors"))
337        .map(|value| value.replace('\\', "/"))
338        .collect()
339}
340
341pub struct ComfyUi<'runtime> {
342    pub http: &'runtime dyn Transport,
343    pub download: &'runtime dyn DownloadTransport,
344    pub poll_interval: Duration,
345    pub deadline: Duration,
346    pub events: Option<Events>,
347}
348
349fn prompt_id(value: &Value) -> Result<String, GenerationError> {
350    value
351        .get("prompt_id")
352        .and_then(Value::as_str)
353        .filter(|id| {
354            !id.is_empty()
355                && id.len() <= 128
356                && id
357                    .bytes()
358                    .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
359        })
360        .map(str::to_owned)
361        .ok_or(GenerationError::Response("missing or invalid prompt_id"))
362}
363fn summary(value: &Value) -> String {
364    let mut parts = Vec::new();
365    if let Some(message) = value.pointer("/error/message").and_then(Value::as_str) {
366        parts.push(message.to_owned());
367    }
368    if let Some(nodes) = value.get("node_errors").and_then(Value::as_object) {
369        for (node, detail) in nodes.iter().take(8) {
370            let class = detail
371                .get("class_type")
372                .and_then(Value::as_str)
373                .unwrap_or("?");
374            for error in detail
375                .get("errors")
376                .and_then(Value::as_array)
377                .into_iter()
378                .flatten()
379                .take(4)
380            {
381                let text = error
382                    .get("details")
383                    .or_else(|| error.get("message"))
384                    .and_then(Value::as_str)
385                    .unwrap_or("error");
386                parts.push(format!("node {node} ({class}): {text}"));
387            }
388        }
389    }
390    if let Some(message) = value.get("exception_message").and_then(Value::as_str) {
391        let node = value
392            .get("node_type")
393            .and_then(Value::as_str)
394            .unwrap_or("?");
395        parts.push(format!("{node}: {message}"));
396    }
397    let text = if parts.is_empty() {
398        "no details reported".to_owned()
399    } else {
400        parts.join("; ")
401    };
402    text.chars()
403        .filter(|ch| !ch.is_control())
404        .take(1000)
405        .collect()
406}
407fn safe_output_name(value: &str, extension: &str) -> bool {
408    !value.is_empty()
409        && value.len() <= 255
410        && !value.starts_with('.')
411        && !value.contains(['/', '\\', ':'])
412        && !value.chars().any(char::is_control)
413        && value
414            .rsplit_once('.')
415            .is_some_and(|(_, ext)| ext.eq_ignore_ascii_case(extension))
416}
417fn safe_subfolder(value: &str) -> bool {
418    value.is_empty()
419        || (value.len() <= 255
420            && !value.starts_with('/')
421            && !value.contains(['\\', ':'])
422            && !value.chars().any(char::is_control)
423            && value
424                .split('/')
425                .all(|part| !part.is_empty() && part != "." && part != ".."))
426}
427fn has_magic(kind: GenerationKind, head: &[u8]) -> bool {
428    match kind {
429        GenerationKind::Image => head.starts_with(b"\x89PNG\r\n\x1a\n"),
430        GenerationKind::Video => {
431            head.len() >= 12 && &head[..4] == b"RIFF" && &head[8..12] == b"WEBP"
432        }
433    }
434}
435
436impl ComfyUi<'_> {
437    fn emit(&self, message: String) {
438        if let Some(events) = &self.events {
439            events(message);
440        }
441    }
442
443    async fn get_json(
444        &self,
445        endpoint: &str,
446        path: &str,
447        cancel: &CancellationToken,
448        limit: usize,
449    ) -> Result<Value, GenerationError> {
450        let request = Request::new(endpoint, path, None, None)?;
451        Ok(read_json(self.http, request, cancel, limit).await?)
452    }
453
454    /// Reads a JSON body regardless of status so ComfyUI validation errors can be shown.
455    async fn post_json(
456        &self,
457        endpoint: &str,
458        path: &str,
459        body: Value,
460        cancel: &CancellationToken,
461    ) -> Result<(u16, Value), GenerationError> {
462        let request = Request::new(endpoint, path, Some(body), None)?;
463        let operation = async {
464            let response = self.http.send(request).await?;
465            let mut bytes = Vec::new();
466            response
467                .body
468                .take(4 * 1024 * 1024 + 1)
469                .read_to_end(&mut bytes)
470                .await
471                .map_err(|_| HttpError::Transport)?;
472            if bytes.len() > 4 * 1024 * 1024 {
473                return Err(HttpError::Limit);
474            }
475            let value = if bytes.is_empty() {
476                Value::Null
477            } else {
478                serde_json::from_slice(&bytes).map_err(|_| HttpError::Response)?
479            };
480            Ok((response.status, value))
481        };
482        tokio::select! {
483            biased;
484            _ = cancel.cancelled() => Err(GenerationError::Cancelled),
485            result = tokio::time::timeout(Duration::from_secs(120), operation) =>
486                Ok(result.map_err(|_| HttpError::Timeout)??),
487        }
488    }
489
490    /// Confirms the listener is ComfyUI and returns its reported devices and free memory.
491    pub async fn ready(
492        &self,
493        endpoint: &str,
494        cancel: &CancellationToken,
495    ) -> Result<ComfyStatus, GenerationError> {
496        match self
497            .get_json(endpoint, "/system_stats", cancel, 1024 * 1024)
498            .await
499        {
500            Ok(stats) if stats.get("system").is_some_and(Value::is_object) => {
501                let devices: Vec<&Value> = stats
502                    .get("devices")
503                    .and_then(Value::as_array)
504                    .into_iter()
505                    .flatten()
506                    .take(16)
507                    .collect();
508                let free_bytes = devices
509                    .iter()
510                    .filter_map(|device| device.get("vram_free").and_then(Value::as_u64))
511                    .min()
512                    .or_else(|| stats.pointer("/system/ram_free").and_then(Value::as_u64));
513                Ok(ComfyStatus {
514                    devices: devices
515                        .iter()
516                        .filter_map(|device| device.get("type").and_then(Value::as_str))
517                        .filter(|kind| {
518                            kind.len() <= 32
519                                && kind.bytes().all(|byte| byte.is_ascii_alphanumeric())
520                        })
521                        .map(str::to_owned)
522                        .collect(),
523                    free_bytes,
524                })
525            }
526            Ok(_) => Err(GenerationError::Response("listener is not ComfyUI")),
527            Err(GenerationError::Cancelled) => Err(GenerationError::Cancelled),
528            Err(GenerationError::Http(HttpError::Invalid)) => Err(GenerationError::Invalid(
529                "ComfyUI endpoint must be loopback HTTP",
530            )),
531            Err(_) => Err(GenerationError::Unreachable(endpoint.to_owned())),
532        }
533    }
534
535    /// Installs (or re-verifies) every pinned weight under `<comfyui>/models/<folder>/rigspark`.
536    pub async fn install(
537        &self,
538        model: &GenerationModel,
539        comfyui_dir: &Path,
540        cancel: &CancellationToken,
541    ) -> Result<Vec<InstalledWeight>, GenerationError> {
542        let models = comfyui_dir.join("models");
543        if !std::fs::metadata(&models).is_ok_and(|metadata| metadata.is_dir()) {
544            return Err(GenerationError::Invalid(
545                "ComfyUI directory must contain a models/ folder",
546            ));
547        }
548        let workflow = model
549            .runnable()
550            .map_err(|error| GenerationError::Catalog(error.to_string()))?;
551        let mut installed = Vec::new();
552        for role in workflow.roles() {
553            let file = model
554                .file(*role)
555                .ok_or(GenerationError::Invalid("model is missing a workflow file"))?;
556            let artifact = Artifact {
557                backend: "comfyui".into(),
558                repo: file.repo.clone(),
559                revision: file.revision.clone(),
560                file: file.file.clone(),
561                sha256: file.sha256.clone(),
562                bytes: file.bytes,
563            };
564            let (owner, repository) = file
565                .repo
566                .split_once('/')
567                .ok_or(GenerationError::Invalid("invalid weight repository"))?;
568            let name = format!(
569                "rigspark/comfyui/{owner}/{repository}@{}/{}",
570                file.revision, file.file
571            );
572            let mut acquisition = Acquisition::new(models.join(role.folder()).join("rigspark"))
573                .map_err(GenerationError::Install)?
574                .with_timeout(WEIGHT_TIMEOUT);
575            if let Some(events) = self.events.clone() {
576                let last = Arc::new(std::sync::atomic::AtomicU64::new(u64::MAX));
577                acquisition = acquisition.with_progress(move |done, total, file| {
578                    let percent = done.saturating_mul(100).checked_div(total).unwrap_or(100);
579                    if last.swap(percent / 5, std::sync::atomic::Ordering::Relaxed) != percent / 5 {
580                        events(format!(
581                            "Weights {file}: {percent}% of {:.1} GiB",
582                            total as f64 / 1073741824.0
583                        ));
584                    }
585                });
586            }
587            let acquired = acquisition
588                .acquire(&artifact, self.download, cancel)
589                .await
590                .map_err(|error| {
591                    if cancel.is_cancelled() {
592                        GenerationError::Cancelled
593                    } else {
594                        GenerationError::Install(error)
595                    }
596                })?;
597            if !acquired
598                .path
599                .ends_with(name.trim_start_matches("rigspark/"))
600            {
601                return Err(GenerationError::Install(
602                    "unexpected weight location".into(),
603                ));
604            }
605            installed.push(InstalledWeight {
606                role: *role,
607                name,
608                bytes: acquired.bytes,
609                cached: acquired.cached,
610            });
611        }
612        Ok(installed)
613    }
614
615    /// Replaces each expected name with ComfyUI's own spelling (e.g. `\` on Windows).
616    async fn resolve_names(
617        &self,
618        endpoint: &str,
619        weights: &mut [InstalledWeight],
620        cancel: &CancellationToken,
621    ) -> Result<(), GenerationError> {
622        for weight in weights {
623            let listing = self
624                .get_json(
625                    endpoint,
626                    &format!("/models/{}", weight.role.folder()),
627                    cancel,
628                    4 * 1024 * 1024,
629                )
630                .await?;
631            let listed = listing
632                .as_array()
633                .ok_or(GenerationError::Response("model listing is not an array"))?
634                .iter()
635                .filter_map(Value::as_str)
636                .find(|entry| entry.replace('\\', "/") == weight.name)
637                .ok_or_else(|| GenerationError::NotVisible(weight.name.clone()))?;
638            weight.name = listed.to_owned();
639        }
640        Ok(())
641    }
642
643    /// Weight files of the most recent ComfyUI prompt, best effort.
644    async fn previous_files(
645        &self,
646        endpoint: &str,
647        cancel: &CancellationToken,
648    ) -> Option<std::collections::BTreeSet<String>> {
649        let request = Request::new(endpoint, "/history", None, None)
650            .ok()?
651            .with_query(&[("max_items", "1")])
652            .ok()?;
653        let history = read_json(self.http, request, cancel, 16 * 1024 * 1024)
654            .await
655            .ok()?;
656        let graph = history
657            .as_object()?
658            .values()
659            .next_back()?
660            .get("prompt")?
661            .get(2)?;
662        Some(model_files(graph)).filter(|files| !files.is_empty())
663    }
664
665    /// Unloads another workflow's weights before switching models when memory is shared (Apple
666    /// MPS) or free device memory is below this model's weights. Live testing on a 36 GB M4 Max:
667    /// Wan swapped and sampled ~4x slower beside a resident FLUX even with more free memory than
668    /// Wan's weights, because activations and upcast encoders need far more than the files.
669    async fn make_room(
670        &self,
671        endpoint: &str,
672        status: &ComfyStatus,
673        needed: u64,
674        graph: &Value,
675        cancel: &CancellationToken,
676    ) -> Result<(), GenerationError> {
677        let shared = apple_mps(&status.devices);
678        let short = status.free_bytes.is_some_and(|free| free < needed);
679        if !shared && !short {
680            return Ok(());
681        }
682        let Some(previous) = self.previous_files(endpoint, cancel).await else {
683            return Ok(());
684        };
685        if previous == model_files(graph) {
686            return Ok(());
687        }
688        let free = status.free_bytes.map_or_else(
689            || "unknown".to_owned(),
690            |free| format!("{:.1} GiB", free as f64 / 1073741824.0),
691        );
692        self.emit(format!(
693            "Freeing ComfyUI memory before switching models ({free} free, {:.1} GiB of weights needed{})",
694            needed as f64 / 1073741824.0,
695            if shared { ", shared Apple memory" } else { "" }
696        ));
697        let (code, _) = self
698            .post_json(
699                endpoint,
700                "/free",
701                json!({"unload_models": true, "free_memory": true}),
702                cancel,
703            )
704            .await?;
705        if !(200..300).contains(&code) {
706            self.emit(format!(
707                "ComfyUI declined to free memory (HTTP {code}); continuing"
708            ));
709        }
710        Ok(())
711    }
712
713    async fn cancel_remote(&self, endpoint: &str, id: &str) {
714        let fresh = CancellationToken::new();
715        for (path, body) in [
716            ("/queue", json!({"delete": [id]})),
717            ("/interrupt", json!({"prompt_id": id})),
718        ] {
719            let _ = tokio::time::timeout(
720                Duration::from_secs(5),
721                self.post_json(endpoint, path, body, &fresh),
722            )
723            .await;
724        }
725    }
726
727    async fn wait(
728        &self,
729        endpoint: &str,
730        id: &str,
731        cancel: &CancellationToken,
732    ) -> Result<Value, GenerationError> {
733        let started = tokio::time::Instant::now();
734        loop {
735            if started.elapsed() > self.deadline {
736                return Err(GenerationError::Timeout);
737            }
738            tokio::select! {
739                biased;
740                _ = cancel.cancelled() => return Err(GenerationError::Cancelled),
741                _ = tokio::time::sleep(self.poll_interval) => {}
742            }
743            let history = self
744                .get_json(
745                    endpoint,
746                    &format!("/history/{id}"),
747                    cancel,
748                    16 * 1024 * 1024,
749                )
750                .await?;
751            let Some(entry) = history.get(id) else {
752                continue;
753            };
754            let status = entry.get("status");
755            if status
756                .and_then(|status| status.get("status_str"))
757                .and_then(Value::as_str)
758                == Some("error")
759            {
760                let detail = status
761                    .and_then(|status| status.get("messages"))
762                    .and_then(Value::as_array)
763                    .into_iter()
764                    .flatten()
765                    .find(|message| {
766                        message.get(0).and_then(Value::as_str) == Some("execution_error")
767                    })
768                    .and_then(|message| message.get(1))
769                    .map(summary)
770                    .unwrap_or_else(|| "execution error".into());
771                return Err(GenerationError::Failed(detail));
772            }
773            if status
774                .and_then(|status| status.get("completed"))
775                .and_then(Value::as_bool)
776                == Some(true)
777            {
778                return entry
779                    .get("outputs")
780                    .cloned()
781                    .ok_or(GenerationError::Response("completed prompt has no outputs"));
782            }
783        }
784    }
785
786    async fn fetch(
787        &self,
788        endpoint: &str,
789        kind: GenerationKind,
790        descriptor: &Value,
791        target: &Path,
792        cancel: &CancellationToken,
793    ) -> Result<u64, GenerationError> {
794        let extension = output_extension(kind);
795        let field = |key| descriptor.get(key).and_then(Value::as_str);
796        let filename = field("filename")
797            .filter(|name| safe_output_name(name, extension))
798            .ok_or(GenerationError::Response("unsafe output filename"))?;
799        let subfolder = field("subfolder")
800            .filter(|folder| safe_subfolder(folder))
801            .ok_or(GenerationError::Response("unsafe output subfolder"))?;
802        if field("type") != Some("output") {
803            return Err(GenerationError::Response("output is not a saved file"));
804        }
805        let request = Request::new(endpoint, "/view", None, None)?.with_query(&[
806            ("filename", filename),
807            ("subfolder", subfolder),
808            ("type", "output"),
809        ])?;
810        let parent = target
811            .parent()
812            .filter(|parent| !parent.as_os_str().is_empty())
813            .unwrap_or(Path::new("."));
814        let operation = async {
815            let response = self.http.send(request).await?;
816            if !(200..300).contains(&response.status) {
817                return Err(GenerationError::Http(HttpError::Status(response.status)));
818            }
819            let temporary = tempfile::Builder::new()
820                .prefix(".rigspark-output.")
821                .suffix(".part")
822                .tempfile_in(parent)
823                .map_err(|error| GenerationError::Output(error.to_string()))?;
824            let mut output = tokio::fs::File::from_std(
825                temporary
826                    .reopen()
827                    .map_err(|error| GenerationError::Output(error.to_string()))?,
828            );
829            let mut body = response.body.take(MAX_OUTPUT_BYTES + 1);
830            let mut buffer = vec![0u8; 65536];
831            let mut head = Vec::with_capacity(16);
832            let mut total = 0u64;
833            loop {
834                let count = body
835                    .read(&mut buffer)
836                    .await
837                    .map_err(|_| HttpError::Transport)?;
838                if count == 0 {
839                    break;
840                }
841                total += count as u64;
842                if total > MAX_OUTPUT_BYTES {
843                    return Err(GenerationError::Response("output exceeds 1 GiB"));
844                }
845                if head.len() < 16 {
846                    let needed = (16 - head.len()).min(count);
847                    head.extend_from_slice(&buffer[..needed]);
848                }
849                output
850                    .write_all(&buffer[..count])
851                    .await
852                    .map_err(|error| GenerationError::Output(error.to_string()))?;
853            }
854            if !has_magic(kind, &head) {
855                return Err(GenerationError::Response(
856                    "output is not the expected media type",
857                ));
858            }
859            output
860                .sync_all()
861                .await
862                .map_err(|error| GenerationError::Output(error.to_string()))?;
863            drop(output);
864            temporary.persist_noclobber(target).map_err(|error| {
865                if error.error.kind() == std::io::ErrorKind::AlreadyExists {
866                    GenerationError::Exists(target.to_path_buf())
867                } else {
868                    GenerationError::Output(error.error.to_string())
869                }
870            })?;
871            Ok(total)
872        };
873        tokio::select! {
874            biased;
875            _ = cancel.cancelled() => Err(GenerationError::Cancelled),
876            result = tokio::time::timeout(Duration::from_secs(600), operation) =>
877                result.map_err(|_| GenerationError::Timeout)?,
878        }
879    }
880
881    pub async fn generate(
882        &self,
883        request: &GenerationRequest<'_>,
884        cancel: &CancellationToken,
885    ) -> Result<GenerationOutcome, GenerationError> {
886        let model = request.model;
887        validate_prompt(request.prompt)?;
888        if request.seed > MAX_SEED {
889            return Err(GenerationError::Invalid(
890                "seed must be in 0..=9007199254740991",
891            ));
892        }
893        crate::state::loopback(request.endpoint)
894            .map_err(|_| GenerationError::Invalid("ComfyUI endpoint must be loopback HTTP"))?;
895        let extension = output_extension(model.kind);
896        if let Some(output) = request.output {
897            if !output
898                .extension()
899                .and_then(|ext| ext.to_str())
900                .is_some_and(|ext| ext.eq_ignore_ascii_case(extension))
901            {
902                return Err(GenerationError::Invalid(match model.kind {
903                    GenerationKind::Image => "image output must use the .png extension",
904                    GenerationKind::Video => "video output must use the .webp extension",
905                }));
906            }
907            if std::fs::symlink_metadata(output).is_ok() {
908                return Err(GenerationError::Exists(output.to_path_buf()));
909            }
910        }
911        let status = self.ready(request.endpoint, cancel).await?;
912        let devices = &status.devices;
913        let built_in = model
914            .runnable()
915            .map_err(|error| GenerationError::Catalog(error.to_string()))?;
916        if matches!(built_in, Workflow::WanT2v | Workflow::Wan22Ti2v) && apple_mps(devices) {
917            self.emit(
918                "ComfyUI reports Apple MPS: using the euler sampler (uni_pc diverges on MPS)"
919                    .into(),
920            );
921        }
922        self.emit(format!("Verifying weights for {}", model.id));
923        let mut weights = self.install(model, request.comfyui_dir, cancel).await?;
924        self.resolve_names(request.endpoint, &mut weights, cancel)
925            .await?;
926        let names = weights
927            .iter()
928            .map(|weight| (weight.role, weight.name.clone()))
929            .collect();
930        let graph = workflow(model, &names, request.prompt, request.seed, devices)?;
931        ensure_local_only(&graph)?;
932        self.make_room(
933            request.endpoint,
934            &status,
935            model.total_bytes(),
936            &graph,
937            cancel,
938        )
939        .await?;
940        let (status, response) = self
941            .post_json(
942                request.endpoint,
943                "/prompt",
944                json!({"prompt": graph}),
945                cancel,
946            )
947            .await?;
948        if !(200..300).contains(&status) {
949            return Err(GenerationError::Rejected(summary(&response)));
950        }
951        if response
952            .get("node_errors")
953            .and_then(Value::as_object)
954            .is_some_and(|errors| !errors.is_empty())
955        {
956            return Err(GenerationError::Rejected(summary(&response)));
957        }
958        let id = prompt_id(&response)?;
959        self.emit(format!(
960            "Queued ComfyUI prompt {id}; generating {}",
961            model.kind.name()
962        ));
963        let outputs = match self.wait(request.endpoint, &id, cancel).await {
964            Ok(outputs) => outputs,
965            Err(error) => {
966                if matches!(error, GenerationError::Cancelled | GenerationError::Timeout) {
967                    self.cancel_remote(request.endpoint, &id).await;
968                }
969                return Err(error);
970            }
971        };
972        let descriptor = outputs
973            .get(save_node(built_in))
974            .and_then(|node| node.get("images"))
975            .and_then(Value::as_array)
976            .and_then(|images| images.first())
977            .ok_or(GenerationError::Response("save node produced no output"))?;
978        let target = match request.output {
979            Some(path) => path.to_path_buf(),
980            None => request.output_dir.join(format!(
981                "rigspark-{}-{}.{extension}",
982                model.kind.name(),
983                &id[..id.len().min(8)]
984            )),
985        };
986        let bytes = self
987            .fetch(request.endpoint, model.kind, descriptor, &target, cancel)
988            .await?;
989        Ok(GenerationOutcome {
990            model: model.id.clone(),
991            kind: model.kind,
992            path: target,
993            bytes,
994            prompt_id: id,
995            seed: request.seed,
996            weights,
997        })
998    }
999}
1000
1001/// Inputs shared by the CLI, terminal UI, and browser GUI.
1002#[derive(Debug, Clone)]
1003pub struct NativeOptions {
1004    pub model: String,
1005    pub prompt: String,
1006    pub seed: Option<u64>,
1007    pub comfyui_dir: PathBuf,
1008    pub port: u16,
1009    pub bypass: bool,
1010    pub output: Option<PathBuf>,
1011    pub output_dir: PathBuf,
1012}
1013
1014#[derive(Debug, Clone, Serialize)]
1015pub struct NativeOutcome {
1016    pub result: GenerationOutcome,
1017    pub fit: GenerationFit,
1018}
1019
1020/// `RIGSPARK_COMFYUI_DIR`, else `~/ComfyUI` when it looks like a ComfyUI install.
1021pub fn default_comfyui_dir() -> Option<PathBuf> {
1022    if let Some(dir) = std::env::var_os("RIGSPARK_COMFYUI_DIR").filter(|dir| !dir.is_empty()) {
1023        return Some(PathBuf::from(dir));
1024    }
1025    let home = std::env::var_os("HOME")
1026        .or_else(|| std::env::var_os("USERPROFILE"))
1027        .filter(|home| !home.is_empty())?;
1028    let candidate = PathBuf::from(home).join("ComfyUI");
1029    candidate.join("models").is_dir().then_some(candidate)
1030}
1031
1032/// Validates everything that needs no hardware, network, or ComfyUI access.
1033pub fn prepare(
1034    options: &NativeOptions,
1035) -> Result<(GenerationModel, u64, PathBuf), GenerationError> {
1036    validate_prompt(&options.prompt)?;
1037    let seed = match options.seed {
1038        Some(seed) if seed <= MAX_SEED => seed,
1039        Some(_) => {
1040            return Err(GenerationError::Invalid(
1041                "seed must be in 0..=9007199254740991",
1042            ));
1043        }
1044        None => random_seed(),
1045    };
1046    if options.port == 0 {
1047        return Err(GenerationError::Invalid(
1048            "ComfyUI port must be in 1..=65535",
1049        ));
1050    }
1051    let catalog = GenerationCatalog::bundled()
1052        .map_err(|error| GenerationError::Catalog(error.to_string()))?;
1053    let model = catalog
1054        .resolve(&options.model)
1055        .map_err(|error| GenerationError::Catalog(error.to_string()))?
1056        .clone();
1057    if options.comfyui_dir.as_os_str().is_empty() {
1058        return Err(GenerationError::Invalid(
1059            "a ComfyUI directory is required (--comfyui-dir or RIGSPARK_COMFYUI_DIR)",
1060        ));
1061    }
1062    let dir = std::path::absolute(&options.comfyui_dir)
1063        .map_err(|_| GenerationError::Invalid("invalid ComfyUI directory"))?;
1064    if !dir.join("models").is_dir() {
1065        return Err(GenerationError::Invalid(
1066            "ComfyUI directory must contain a models/ folder",
1067        ));
1068    }
1069    Ok((model, seed, dir))
1070}
1071
1072/// Full local run: validate, check memory fit, then drive the loopback ComfyUI server.
1073pub async fn run_native(
1074    options: &NativeOptions,
1075    events: Option<Events>,
1076    cancel: &CancellationToken,
1077) -> Result<NativeOutcome, GenerationError> {
1078    let (model, seed, comfyui_dir) = prepare(options)?;
1079    let emit = |message: String| {
1080        if let Some(events) = &events {
1081            events(message);
1082        }
1083    };
1084    let (hardware, warnings) = crate::hardware::detect()
1085        .await
1086        .map_err(|error| GenerationError::Hardware(error.to_string()))?;
1087    for warning in warnings {
1088        emit(warning);
1089    }
1090    let memory = fit(&model, &hardware);
1091    if memory.verdict == FitVerdict::No && !options.bypass {
1092        return Err(GenerationError::NoFit(format!(
1093            "{} will not fit this machine ({}); bypass to try anyway (integrity checks still apply)",
1094            model.id, memory.reason
1095        )));
1096    }
1097    emit(format!(
1098        "Model {} ({}): fit {} - {}; speed unknown",
1099        model.id,
1100        model.kind.name(),
1101        memory.verdict.name(),
1102        memory.reason
1103    ));
1104    let http = crate::http::NativeTransport::new()?;
1105    let download = crate::acquire::HfTransport::new().map_err(GenerationError::Install)?;
1106    let client = ComfyUi {
1107        http: &http,
1108        download: &download,
1109        poll_interval: Duration::from_secs(1),
1110        deadline: Duration::from_secs(2 * 60 * 60),
1111        events: events.clone(),
1112    };
1113    let endpoint = format!("http://127.0.0.1:{}", options.port);
1114    let result = client
1115        .generate(
1116            &GenerationRequest {
1117                model: &model,
1118                prompt: &options.prompt,
1119                seed,
1120                endpoint: &endpoint,
1121                comfyui_dir: &comfyui_dir,
1122                output: options.output.as_deref(),
1123                output_dir: &options.output_dir,
1124            },
1125            cancel,
1126        )
1127        .await?;
1128    Ok(NativeOutcome {
1129        result,
1130        fit: memory,
1131    })
1132}
1133
1134pub type GenerationFuture<'call> = std::pin::Pin<
1135    Box<dyn std::future::Future<Output = Result<NativeOutcome, GenerationError>> + Send + 'call>,
1136>;
1137
1138/// Injectable generation boundary so the TUI and GUI can be tested without ComfyUI.
1139pub trait Generator: Send + Sync {
1140    fn generate(
1141        &self,
1142        options: NativeOptions,
1143        events: Events,
1144        cancel: CancellationToken,
1145    ) -> GenerationFuture<'_>;
1146}
1147
1148pub struct NativeGenerator;
1149impl Generator for NativeGenerator {
1150    fn generate(
1151        &self,
1152        options: NativeOptions,
1153        events: Events,
1154        cancel: CancellationToken,
1155    ) -> GenerationFuture<'_> {
1156        Box::pin(async move { run_native(&options, Some(events), &cancel).await })
1157    }
1158}