Skip to main content

mold_server/
routes.rs

1use axum::{
2    extract::{Path, Request, State},
3    http::{header, HeaderMap, HeaderValue, StatusCode},
4    response::{
5        sse::{Event as SseEvent, KeepAlive, Sse},
6        IntoResponse,
7    },
8    routing::{delete, get, patch, post},
9    Json, Router,
10};
11use base64::Engine as _;
12use mold_core::{
13    types::GpuSelection, ActiveGenerationStatus, GenerateRequest, GpuInfo, GpuWorkerState,
14    ModelInfoExtended, ResourceSnapshot, ServerStatus, SseErrorEvent, SseProgressEvent,
15};
16use serde::{Deserialize, Serialize};
17use std::cmp::Reverse;
18use std::convert::Infallible;
19use std::sync::atomic::Ordering;
20use tokio_stream::StreamExt as _;
21use utoipa::OpenApi;
22
23use crate::model_manager;
24use crate::state::{AppState, GenerationJob, SseMessage, SubmitError};
25
26fn submit_error_to_api(e: SubmitError) -> ApiError {
27    match e {
28        SubmitError::Full { pending, capacity } => {
29            ApiError::queue_full(format!("generation queue is full ({pending}/{capacity})"))
30        }
31        SubmitError::Shutdown => ApiError::internal("generation queue shut down"),
32    }
33}
34
35// ── ApiError — structured JSON error response ────────────────────────────────
36
37#[derive(Debug, Serialize)]
38pub struct ApiError {
39    pub error: String,
40    pub code: String,
41    #[serde(skip)]
42    status: StatusCode,
43}
44
45impl ApiError {
46    pub fn validation(msg: impl Into<String>) -> Self {
47        Self {
48            error: msg.into(),
49            code: "VALIDATION_ERROR".to_string(),
50            status: StatusCode::UNPROCESSABLE_ENTITY,
51        }
52    }
53
54    pub fn not_found(msg: impl Into<String>) -> Self {
55        Self {
56            error: msg.into(),
57            code: "MODEL_NOT_FOUND".to_string(),
58            status: StatusCode::NOT_FOUND,
59        }
60    }
61
62    pub fn unknown_model(msg: impl Into<String>) -> Self {
63        Self {
64            error: msg.into(),
65            code: "UNKNOWN_MODEL".to_string(),
66            status: StatusCode::BAD_REQUEST,
67        }
68    }
69
70    pub fn inference(msg: impl Into<String>) -> Self {
71        Self {
72            error: msg.into(),
73            code: "INFERENCE_ERROR".to_string(),
74            status: StatusCode::INTERNAL_SERVER_ERROR,
75        }
76    }
77
78    pub fn internal(msg: impl Into<String>) -> Self {
79        Self {
80            error: msg.into(),
81            code: "INTERNAL_ERROR".to_string(),
82            status: StatusCode::INTERNAL_SERVER_ERROR,
83        }
84    }
85
86    pub fn internal_with_status(msg: impl Into<String>, status: StatusCode) -> Self {
87        Self {
88            error: msg.into(),
89            code: "INTERNAL_ERROR".to_string(),
90            status,
91        }
92    }
93
94    pub fn queue_job_not_found(msg: impl Into<String>) -> Self {
95        Self {
96            error: msg.into(),
97            code: "QUEUE_JOB_NOT_FOUND".to_string(),
98            status: StatusCode::NOT_FOUND,
99        }
100    }
101
102    pub fn queue_job_running(msg: impl Into<String>) -> Self {
103        Self {
104            error: msg.into(),
105            code: "QUEUE_JOB_RUNNING".to_string(),
106            status: StatusCode::CONFLICT,
107        }
108    }
109
110    pub fn insufficient_memory(msg: impl Into<String>) -> Self {
111        Self {
112            error: msg.into(),
113            code: "INSUFFICIENT_MEMORY".to_string(),
114            status: StatusCode::SERVICE_UNAVAILABLE,
115        }
116    }
117
118    pub fn forbidden(msg: impl Into<String>) -> Self {
119        Self {
120            error: msg.into(),
121            code: "FORBIDDEN".to_string(),
122            status: StatusCode::FORBIDDEN,
123        }
124    }
125
126    pub fn queue_full(msg: impl Into<String>) -> Self {
127        Self {
128            error: msg.into(),
129            code: "QUEUE_FULL".to_string(),
130            status: StatusCode::SERVICE_UNAVAILABLE,
131        }
132    }
133}
134
135impl IntoResponse for ApiError {
136    fn into_response(self) -> axum::response::Response {
137        let status = self.status;
138        // On queue-full (503), hint clients to retry with a short delay.
139        if self.code == "QUEUE_FULL" {
140            let mut headers = HeaderMap::new();
141            headers.insert(header::RETRY_AFTER, HeaderValue::from_static("1"));
142            return (status, headers, Json(self)).into_response();
143        }
144        (status, Json(self)).into_response()
145    }
146}
147
148// Re-export for tests — the canonical implementation lives in queue.rs.
149#[cfg(test)]
150use crate::queue::clean_error_message;
151
152#[derive(OpenApi)]
153#[openapi(
154    paths(
155        generate,
156        generate_stream,
157        expand_prompt,
158        list_models,
159        crate::catalog_api::list_loras,
160        load_model,
161        pull_model_endpoint,
162        unload_model,
163        server_status,
164        list_queue,
165        patch_queue_job,
166        health,
167        capabilities_chain_limits,
168        crate::routes_chain::generate_chain,
169        crate::routes_chain::generate_chain_stream,
170    ),
171    components(schemas(
172        mold_core::GenerateRequest,
173        mold_core::GenerateResponse,
174        mold_core::ExpandRequest,
175        mold_core::ExpandResponse,
176        mold_core::ImageData,
177        mold_core::OutputFormat,
178        mold_core::ModelInfo,
179        mold_core::LoraInfo,
180        mold_core::ServerStatus,
181        mold_core::ActiveGenerationStatus,
182        mold_core::GpuInfo,
183        mold_core::SseProgressEvent,
184        mold_core::SseCompleteEvent,
185        mold_core::SseErrorEvent,
186        mold_core::ChainRequest,
187        mold_core::ChainResponse,
188        mold_core::ChainStage,
189        mold_core::ChainProgressEvent,
190        mold_core::SseChainCompleteEvent,
191        ModelInfoExtended,
192        LoadModelBody,
193        UnloadRequest,
194        QueuePatchRequest,
195        crate::job_registry::JobEntry,
196        crate::job_registry::QueueListing,
197        crate::chain_limits::ChainLimits,
198    )),
199    tags(
200        (name = "generation", description = "Image generation"),
201        (name = "models", description = "Model management"),
202        (name = "server", description = "Server status and health"),
203    ),
204    info(
205        title = "mold",
206        description = "Local AI image generation server — FLUX, SD3.5, SD1.5, SDXL, Z-Image, Flux.2, Qwen-Image",
207        version = env!("CARGO_PKG_VERSION"),
208    )
209)]
210pub struct ApiDoc;
211
212pub fn create_router(state: AppState) -> Router {
213    // Stateful routes (need AppState) are added first, then .with_state() converts
214    // Router<AppState> → Router<()>. Stateless routes (OpenAPI, docs) are merged after.
215    Router::new()
216        .route("/api/generate", post(generate))
217        .route("/api/generate/estimate", post(generate_estimate))
218        .route("/api/generate/stream", post(generate_stream))
219        .route(
220            "/api/generate/chain",
221            post(crate::routes_chain::generate_chain),
222        )
223        .route(
224            "/api/generate/chain/stream",
225            post(crate::routes_chain::generate_chain_stream),
226        )
227        .route("/api/expand", post(expand_prompt))
228        .route("/api/models", get(list_models))
229        .route("/api/models/:model/components", get(model_components))
230        .route("/api/loras", get(crate::catalog_api::list_loras))
231        .route("/api/models/load", post(load_model))
232        .route("/api/models/pull", post(pull_model_endpoint))
233        .route("/api/models/unload", delete(unload_model))
234        .route("/api/gallery", get(list_gallery))
235        .route(
236            "/api/gallery/image/:filename",
237            get(get_gallery_image).delete(delete_gallery_image),
238        )
239        .route(
240            "/api/gallery/thumbnail/:filename",
241            get(get_gallery_thumbnail),
242        )
243        .route("/api/gallery/preview/:filename", get(get_gallery_preview))
244        // ─── Downloads UI (Agent A) ────────────────────────────────────────
245        .route("/api/downloads", get(list_downloads).post(create_download))
246        .route("/api/downloads/:id", delete(delete_download))
247        .route("/api/downloads/stream", get(stream_downloads))
248        // ─── Catalog (live HF + Civitai proxy) ──────────────────────────
249        .route(
250            "/api/catalog/families",
251            get(crate::catalog_api::list_families),
252        )
253        .route(
254            "/api/catalog/search",
255            get(crate::catalog_api::live_search_catalog),
256        )
257        .route(
258            "/api/catalog/installed",
259            get(crate::catalog_api::list_installed_catalog),
260        )
261        .route(
262            "/api/catalog/*id",
263            get(crate::catalog_api::get_catalog_entry)
264                .post(crate::catalog_api::post_catalog_dispatch),
265        )
266        .route("/api/upscale", post(upscale))
267        .route("/api/upscale/stream", post(upscale_stream))
268        .route("/api/resources", get(get_resources))
269        .route("/api/resources/stream", get(get_resources_stream))
270        .route("/api/status", get(server_status))
271        .route("/api/queue", get(list_queue))
272        .route("/api/queue/:id", patch(patch_queue_job))
273        .route("/api/capabilities", get(server_capabilities))
274        .route(
275            "/api/capabilities/chain-limits",
276            get(capabilities_chain_limits),
277        )
278        .route("/api/shutdown", post(shutdown_server))
279        // Agent C (model-ui-overhaul §3): placement persistence.
280        .route(
281            "/api/config/model/:name/placement",
282            axum::routing::put(put_model_placement).delete(delete_model_placement),
283        )
284        .route("/health", get(health))
285        .with_state(state)
286        .route("/api/openapi.json", get(openapi_json))
287        .route("/api/docs", get(scalar_docs))
288}
289
290// ── Model readiness ──────────────────────────────────────────────────────────
291
292fn sse_message_to_event(msg: SseMessage) -> SseEvent {
293    fn serialize_event<T: Serialize>(event_name: &str, payload: &T) -> SseEvent {
294        match serde_json::to_string(payload) {
295            Ok(data) => SseEvent::default().event(event_name).data(data),
296            Err(err) => SseEvent::default().event("error").data(
297                serde_json::json!({
298                    "message": format!("failed to serialize SSE payload: {err}")
299                })
300                .to_string(),
301            ),
302        }
303    }
304
305    match msg {
306        SseMessage::Progress(payload) => serialize_event("progress", &payload),
307        SseMessage::Complete(payload) => serialize_event("complete", &payload),
308        SseMessage::UpscaleComplete(payload) => serialize_event("complete", &payload),
309        SseMessage::Error(payload) => serialize_event("error", &payload),
310    }
311}
312
313#[cfg(test)]
314fn save_image_to_dir(
315    dir: &std::path::Path,
316    img: &mold_core::ImageData,
317    model: &str,
318    batch_size: u32,
319) {
320    if let Err(e) = std::fs::create_dir_all(dir) {
321        tracing::warn!("failed to create output dir {}: {e}", dir.display());
322        return;
323    }
324    // Use milliseconds for server-side filenames to avoid overwrites when
325    // concurrent requests finish in the same second.
326    let timestamp_ms = std::time::SystemTime::now()
327        .duration_since(std::time::UNIX_EPOCH)
328        .unwrap_or_default()
329        .as_millis() as u64;
330    let ext = img.format.to_string();
331    let filename =
332        mold_core::default_output_filename(model, timestamp_ms, &ext, batch_size, img.index);
333    let path = dir.join(&filename);
334    match std::fs::write(&path, &img.data) {
335        Ok(()) => tracing::info!("saved image to {}", path.display()),
336        Err(e) => tracing::warn!("failed to save image to {}: {e}", path.display()),
337    }
338}
339
340// ── Shared pre-queue validation ───────────────────────────────────────────────
341
342/// Validate a generate request and resolve server-side defaults.
343///
344/// Performs the identical pre-queue checks used by both `generate` and
345/// `generate_stream`: applies the default metadata setting, validates the
346/// request, checks model availability, and resolves the output directory.
347async fn prepare_generation(
348    state: &AppState,
349    request: &mut mold_core::GenerateRequest,
350) -> Result<(Option<std::path::PathBuf>, Option<String>, Option<usize>), ApiError> {
351    // NOTE: the capacity check is enforced inside `state.queue.submit(...)` so
352    // that a burst of concurrent callers can't all slip past an open check
353    // (classic TOCTOU).  The submit call in `generate`/`generate_stream` will
354    // return `SubmitError::Full`, which is mapped to `ApiError::queue_full()`.
355    apply_default_metadata_setting(state, request).await;
356
357    let preferred_gpu = validate_multi_gpu_placement(state, request.placement.as_ref())?;
358
359    // Expand prompt if requested (before validation, so the expanded prompt gets validated)
360    maybe_expand_prompt(state, request, preferred_gpu).await?;
361
362    // Catalog (`cv:*` / `hf:*`) IDs aren't in the static manifest, so the
363    // pure-mold-core family lookup returns `None` for them. Run the live
364    // single-id install first so the intent cache has the entry; then
365    // feed its family string through as a hint so audio / keyframes /
366    // pipeline gates work for installed Civitai LTX-2 checkpoints.
367    //
368    // A `Network` error here means Civitai/HF is unreachable — surface it
369    // immediately as 502 rather than letting the user fall through to
370    // a "not installed" 404 they can't act on.
371    if let Err(e) = model_manager::install_catalog_model(state, &request.model).await {
372        return Err(model_manager::install_error_to_api_error(&e));
373    }
374    let family_hint = model_manager::catalog_family_for(state, &request.model).await;
375
376    // Resolve the model family for normalisation. `family_for_model` checks the
377    // static manifest first (covers all built-in models), then falls back to the
378    // catalog DB entry (covers `cv:*` / `hf:*` models installed above).
379    let resolved_family = model_manager::family_for_model(state, &request.model).await;
380    // Fill in a family-aware output format default when the caller omitted the
381    // field. This must happen before validation so the validator sees a concrete
382    // format and can gate on it correctly.
383    request.normalise_output_format(resolved_family.as_deref());
384
385    if let Err(e) = validate_generate_request(request, family_hint.as_deref()) {
386        return Err(ApiError::validation(e));
387    }
388
389    resolve_server_local_media_paths(state, request).await?;
390
391    let _ = model_manager::check_model_available(state, &request.model).await?;
392
393    let (output_dir, dim_warning) = {
394        let config = state.config.read().await;
395        let output_dir = if config.is_output_disabled() {
396            None
397        } else {
398            Some(config.effective_output_dir())
399        };
400        let family = config.resolved_model_config(&request.model).family;
401        let dim_warning = family
402            .as_deref()
403            .and_then(|f| mold_core::dimension_warning(request.width, request.height, f));
404        (output_dir, dim_warning)
405    };
406
407    Ok((output_dir, dim_warning, preferred_gpu))
408}
409
410pub(crate) async fn resolve_server_local_media_paths(
411    state: &AppState,
412    request: &mut mold_core::GenerateRequest,
413) -> Result<(), ApiError> {
414    if request.audio_file_path.is_none() && request.source_video_path.is_none() {
415        return Ok(());
416    }
417
418    let roots = state.config.read().await.resolved_media_roots();
419    if let Some(path) = request.audio_file_path.as_deref() {
420        let resolved = mold_core::resolve_server_media_path(path, &roots)
421            .map_err(|e| ApiError::validation(format!("audio_file_path: {e}")))?;
422        request.audio_file_path = Some(resolved.to_string_lossy().to_string());
423    }
424    if let Some(path) = request.source_video_path.as_deref() {
425        let resolved = mold_core::resolve_server_media_path(path, &roots)
426            .map_err(|e| ApiError::validation(format!("source_video_path: {e}")))?;
427        request.source_video_path = Some(resolved.to_string_lossy().to_string());
428    }
429
430    Ok(())
431}
432
433fn active_gpu_selection(state: &AppState) -> GpuSelection {
434    let ordinals: Vec<usize> = state
435        .gpu_pool
436        .workers
437        .iter()
438        .map(|w| w.gpu.ordinal)
439        .collect();
440    if ordinals.is_empty() {
441        GpuSelection::All
442    } else {
443        GpuSelection::Specific(ordinals)
444    }
445}
446
447fn validate_multi_gpu_placement(
448    state: &AppState,
449    placement: Option<&mold_core::types::DevicePlacement>,
450) -> Result<Option<usize>, ApiError> {
451    state
452        .gpu_pool
453        .resolve_explicit_placement_gpu(placement)
454        .map_err(ApiError::validation)
455}
456
457fn select_aux_worker(
458    state: &AppState,
459) -> Result<std::sync::Arc<crate::gpu_pool::GpuWorker>, ApiError> {
460    let mut workers: Vec<_> = state
461        .gpu_pool
462        .workers
463        .iter()
464        .filter(|w| !w.is_degraded())
465        .cloned()
466        .collect();
467    workers.sort_by_key(|w| {
468        (
469            w.in_flight.load(Ordering::SeqCst),
470            Reverse(w.gpu.total_vram_bytes),
471        )
472    });
473    workers
474        .into_iter()
475        .next()
476        .ok_or_else(|| ApiError::internal("no GPU worker available for auxiliary workload"))
477}
478
479fn clear_global_upscaler_cache(state: &AppState) {
480    if let Ok(mut cache) = state.upscaler_cache.try_lock() {
481        if cache.is_some() {
482            *cache = None;
483            tracing::info!("upscaler cache cleared");
484        }
485    }
486}
487
488// ── /api/generate ─────────────────────────────────────────────────────────────
489
490#[utoipa::path(
491    post,
492    path = "/api/generate",
493    tag = "generation",
494    request_body = mold_core::GenerateRequest,
495    responses(
496        (status = 200, description = "Generated image bytes", content_type = "image/png"),
497        (status = 404, description = "Model not downloaded"),
498        (status = 422, description = "Invalid request parameters"),
499        (status = 500, description = "Inference error"),
500        (status = 503, description = "Generation queue full"),
501    )
502)]
503// The server always produces 1 image per request; batch looping (--batch N)
504// is handled client-side by the CLI, which sends N requests with incrementing seeds.
505async fn generate(
506    State(state): State<AppState>,
507    Json(mut req): Json<mold_core::GenerateRequest>,
508) -> Result<impl IntoResponse, ApiError> {
509    let (output_dir, dim_warning, preferred_gpu) = prepare_generation(&state, &mut req).await?;
510
511    tracing::info!(
512        model = %req.model,
513        prompt = %req.prompt,
514        width = req.width,
515        height = req.height,
516        steps = req.steps,
517        guidance = req.guidance,
518        seed = ?req.seed,
519        format = %req.resolved_output_format(),
520        lora = ?req.lora.as_ref().map(|l| &l.path),
521        lora_scale = ?req.lora.as_ref().map(|l| l.scale),
522        loras = ?req.loras.as_ref().map(|v| v.iter().map(|l| &l.path).collect::<Vec<_>>()),
523        "generate request"
524    );
525
526    // Submit to generation queue
527    let (result_tx, result_rx) = tokio::sync::oneshot::channel();
528    // Non-streaming path still gets a registry entry so an HTTP client
529    // hanging on the response is visible via GET /api/queue alongside
530    // SSE clients. Cleanup happens unconditionally on every terminal
531    // path (drop guard in `gpu_worker::process_job`).
532    let job_id = uuid::Uuid::new_v4().to_string();
533    state
534        .job_registry
535        .register_with_target_gpu(&job_id, &req.model, preferred_gpu);
536    let job = GenerationJob {
537        id: job_id.clone(),
538        request: req,
539        progress_tx: None,
540        result_tx,
541        output_dir,
542    };
543
544    let _position = state
545        .queue
546        .submit(job, state.queue_capacity)
547        .await
548        .inspect_err(|_| state.job_registry.remove(&job_id))
549        .map_err(submit_error_to_api)?;
550
551    // Wait for the queue worker to process the job
552    let result = result_rx
553        .await
554        .map_err(|_| ApiError::internal("generation queue worker dropped the job"))?;
555
556    match result {
557        Ok(job_result) => {
558            let img = job_result.image;
559            let response = job_result.response;
560            let content_type = HeaderValue::from_static(img.format.content_type());
561            let mut headers = HeaderMap::new();
562            headers.insert(header::CONTENT_TYPE, content_type);
563            headers.insert(
564                "x-mold-seed-used",
565                HeaderValue::from_str(&response.seed_used.to_string()).map_err(|e| {
566                    ApiError::internal(format!("failed to serialize seed header: {e}"))
567                })?,
568            );
569            if let Some(ordinal) = response.gpu {
570                headers.insert(
571                    "x-mold-gpu",
572                    HeaderValue::from_str(&ordinal.to_string()).map_err(|e| {
573                        ApiError::internal(format!("failed to serialize gpu header: {e}"))
574                    })?,
575                );
576            }
577            if let Some(warning) = dim_warning {
578                match HeaderValue::from_str(&warning.replace('\n', " ")) {
579                    Ok(val) => {
580                        headers.insert("x-mold-dimension-warning", val);
581                    }
582                    Err(e) => {
583                        tracing::warn!("dimension warning could not be encoded as header: {e}");
584                    }
585                }
586            }
587            // For video responses, return the actual video data (not the thumbnail)
588            // and send video metadata in headers so the client can reconstruct VideoData.
589            let output_data = if let Some(ref video) = response.video {
590                let ct = HeaderValue::from_static(video.format.content_type());
591                headers.insert(header::CONTENT_TYPE, ct);
592                if let Ok(v) = HeaderValue::from_str(&video.frames.to_string()) {
593                    headers.insert("x-mold-video-frames", v);
594                }
595                if let Ok(v) = HeaderValue::from_str(&video.fps.to_string()) {
596                    headers.insert("x-mold-video-fps", v);
597                }
598                if let Ok(v) = HeaderValue::from_str(&video.width.to_string()) {
599                    headers.insert("x-mold-video-width", v);
600                }
601                if let Ok(v) = HeaderValue::from_str(&video.height.to_string()) {
602                    headers.insert("x-mold-video-height", v);
603                }
604                if video.has_audio {
605                    headers.insert("x-mold-video-has-audio", HeaderValue::from_static("1"));
606                }
607                if let Some(dur) = video.duration_ms {
608                    if let Ok(v) = HeaderValue::from_str(&dur.to_string()) {
609                        headers.insert("x-mold-video-duration-ms", v);
610                    }
611                }
612                if let Some(sr) = video.audio_sample_rate {
613                    if let Ok(v) = HeaderValue::from_str(&sr.to_string()) {
614                        headers.insert("x-mold-video-audio-sample-rate", v);
615                    }
616                }
617                if let Some(ch) = video.audio_channels {
618                    if let Ok(v) = HeaderValue::from_str(&ch.to_string()) {
619                        headers.insert("x-mold-video-audio-channels", v);
620                    }
621                }
622                video.data.clone()
623            } else {
624                img.data
625            };
626            Ok((headers, output_data))
627        }
628        Err(err_msg) => {
629            // The multi-GPU dispatcher sends a queue-full error through result_tx
630            // when a per-worker channel is saturated; surface that as a proper 503
631            // instead of the generic INFERENCE_ERROR 500.
632            if err_msg.contains("queue is full") {
633                Err(ApiError::queue_full(err_msg))
634            } else {
635                Err(ApiError::inference(err_msg))
636            }
637        }
638    }
639}
640
641fn validate_generate_request(
642    req: &mold_core::GenerateRequest,
643    family_hint: Option<&str>,
644) -> Result<(), String> {
645    mold_core::validate_generate_request_with_family(req, family_hint)
646}
647
648async fn apply_default_metadata_setting(state: &AppState, req: &mut mold_core::GenerateRequest) {
649    if req.embed_metadata.is_some() {
650        return;
651    }
652
653    let config = state.config.read().await;
654    req.embed_metadata = Some(config.effective_embed_metadata(None));
655}
656
657/// Apply prompt expansion if `expand: true` is set on a generate request.
658async fn maybe_expand_prompt(
659    state: &AppState,
660    req: &mut mold_core::GenerateRequest,
661    preferred_gpu: Option<usize>,
662) -> Result<(), ApiError> {
663    if req.expand != Some(true) {
664        return Ok(());
665    }
666
667    let config = state.config.read().await;
668    let config_snapshot = config.clone();
669    let expand_settings = config.expand.clone().with_env_overrides();
670
671    // Resolve model family for prompt style
672    let model_family = config
673        .resolved_model_config(&req.model)
674        .family
675        .or_else(|| mold_core::manifest::find_manifest(&req.model).map(|m| m.family.clone()))
676        .unwrap_or_else(|| {
677            tracing::warn!(
678                model = %req.model,
679                "could not resolve model family for prompt expansion, defaulting to \"flux\""
680            );
681            "flux".to_string()
682        });
683
684    let expand_config = expand_settings.to_expand_config(&model_family, 1);
685    let original_prompt = req.prompt.clone();
686
687    // Drop config lock before blocking
688    drop(config);
689
690    let expander = create_server_expander(
691        &config_snapshot,
692        &expand_settings,
693        active_gpu_selection(state),
694        preferred_gpu,
695    )?;
696    let result =
697        tokio::task::spawn_blocking(move || expander.expand(&original_prompt, &expand_config))
698            .await
699            .map_err(|e| ApiError::internal(format!("expand task failed: {e}")))?
700            .map_err(|e| ApiError::internal(format!("prompt expansion failed: {e}")))?;
701
702    if let Some(expanded) = result.expanded.first() {
703        req.original_prompt = Some(req.prompt.clone());
704        req.prompt = expanded.clone();
705    }
706
707    Ok(())
708}
709
710/// Create the appropriate expander for server-side use.
711fn create_server_expander(
712    _config: &mold_core::Config,
713    settings: &mold_core::ExpandSettings,
714    _gpu_selection: GpuSelection,
715    _preferred_gpu: Option<usize>,
716) -> Result<Box<dyn mold_core::PromptExpander>, ApiError> {
717    if let Some(api_expander) = settings.create_api_expander() {
718        return Ok(Box::new(api_expander));
719    }
720
721    #[cfg(feature = "expand")]
722    {
723        if let Some(local) =
724            mold_inference::expand::LocalExpander::from_config(_config, Some(&settings.model))
725        {
726            return Ok(Box::new(
727                local
728                    .with_gpu_selection(_gpu_selection)
729                    .with_preferred_gpu(_preferred_gpu),
730            ));
731        }
732        return Err(ApiError::validation(
733            "local expand model not found — run: mold pull qwen3-expand".to_string(),
734        ));
735    }
736
737    #[cfg(not(feature = "expand"))]
738    {
739        Err(ApiError::validation(
740            "local prompt expansion not available — built without expand feature. \
741             Configure an API backend in [expand] settings."
742                .to_string(),
743        ))
744    }
745}
746
747// ── /api/expand ──────────────────────────────────────────────────────────────
748
749#[utoipa::path(
750    post,
751    path = "/api/expand",
752    tag = "generation",
753    request_body = mold_core::ExpandRequest,
754    responses(
755        (status = 200, description = "Expanded prompt(s)", body = mold_core::ExpandResponse),
756        (status = 422, description = "Invalid request parameters"),
757        (status = 500, description = "Expansion failed"),
758    )
759)]
760async fn expand_prompt(
761    State(state): State<AppState>,
762    Json(req): Json<mold_core::ExpandRequest>,
763) -> Result<Json<mold_core::ExpandResponse>, ApiError> {
764    if req.variations == 0 || req.variations > mold_core::expand::MAX_VARIATIONS {
765        return Err(ApiError::validation(format!(
766            "variations must be between 1 and {}",
767            mold_core::expand::MAX_VARIATIONS,
768        )));
769    }
770
771    let config = state.config.read().await;
772    let expand_settings = config.expand.clone().with_env_overrides();
773    let expand_config = expand_settings.to_expand_config(&req.model_family, req.variations);
774    let prompt = req.prompt.clone();
775    let config_snapshot = config.clone();
776    drop(config);
777
778    let expander = create_server_expander(
779        &config_snapshot,
780        &expand_settings,
781        active_gpu_selection(&state),
782        None,
783    )?;
784    let result = tokio::task::spawn_blocking(move || expander.expand(&prompt, &expand_config))
785        .await
786        .map_err(|e| ApiError::internal(format!("expand task failed: {e}")))?
787        .map_err(|e| ApiError::internal(format!("prompt expansion failed: {e}")))?;
788
789    Ok(Json(mold_core::ExpandResponse {
790        original: req.prompt,
791        expanded: result.expanded,
792    }))
793}
794
795// ── /api/upscale ────────────────────────────────────────────────────────────
796
797async fn upscale(
798    State(state): State<AppState>,
799    Json(req): Json<mold_core::UpscaleRequest>,
800) -> Result<Json<mold_core::UpscaleResponse>, ApiError> {
801    if let Err(msg) = mold_core::validate_upscale_request(&req) {
802        return Err(ApiError::validation(msg));
803    }
804
805    let model_name = mold_core::manifest::resolve_model_name(&req.model);
806
807    // Auto-pull upscaler model if not downloaded
808    let needs_pull = {
809        let config = state.config.read().await;
810        config
811            .models
812            .get(&model_name)
813            .and_then(|c| c.transformer.as_ref())
814            .is_none()
815    };
816    if needs_pull {
817        if mold_core::manifest::find_manifest(&model_name).is_none() {
818            return Err(ApiError::not_found(format!(
819                "unknown upscaler model '{}'. Run 'mold list' to see available models.",
820                model_name
821            )));
822        }
823        model_manager::pull_model(&state, &model_name, None).await?;
824    }
825
826    let config = state.config.read().await;
827    let weights_path = config
828        .models
829        .get(&model_name)
830        .and_then(|c| c.transformer.as_ref())
831        .ok_or_else(|| {
832            ApiError::not_found(format!(
833                "upscaler model '{}' not configured after pull",
834                model_name
835            ))
836        })?;
837    let weights_path = std::path::PathBuf::from(weights_path);
838    let model_name_owned = model_name.clone();
839    drop(config);
840
841    let resp = if state.gpu_pool.worker_count() > 0 {
842        let worker = select_aux_worker(&state)?;
843        worker.in_flight.fetch_add(1, Ordering::SeqCst);
844        let worker_clone = worker.clone();
845        let result =
846            tokio::task::spawn_blocking(move || -> anyhow::Result<mold_core::UpscaleResponse> {
847                struct ThreadGpuGuard;
848                impl Drop for ThreadGpuGuard {
849                    fn drop(&mut self) {
850                        mold_inference::device::clear_thread_gpu_ordinal();
851                    }
852                }
853
854                mold_inference::device::init_thread_gpu_ordinal(worker_clone.gpu.ordinal);
855                let _thread_gpu = ThreadGpuGuard;
856                let _load_lock = worker_clone.model_load_lock.lock().unwrap();
857                let mut engine = mold_inference::create_upscale_engine(
858                    model_name_owned,
859                    weights_path,
860                    mold_inference::LoadStrategy::Eager,
861                    worker_clone.gpu.ordinal,
862                )?;
863                engine.upscale(&req)
864            })
865            .await
866            .map_err(|e| ApiError::internal(format!("upscale task panicked: {e}")));
867        worker.in_flight.fetch_sub(1, Ordering::SeqCst);
868        result?.map_err(|e| ApiError::internal(format!("upscale failed: {e}")))?
869    } else {
870        let upscaler_cache = state.upscaler_cache.clone();
871        tokio::task::spawn_blocking(move || -> anyhow::Result<mold_core::UpscaleResponse> {
872            let mut cache = upscaler_cache.lock().unwrap_or_else(|e| e.into_inner());
873
874            // Reuse cached engine if same model.
875            let needs_new = cache
876                .as_ref()
877                .is_none_or(|e| e.model_name() != model_name_owned);
878            if needs_new {
879                let new_engine = mold_inference::create_upscale_engine(
880                    model_name_owned,
881                    weights_path,
882                    mold_inference::LoadStrategy::Eager,
883                    0,
884                )?;
885                *cache = Some(new_engine);
886            }
887
888            cache.as_mut().unwrap().upscale(&req)
889        })
890        .await
891        .map_err(|e| ApiError::internal(format!("upscale task panicked: {e}")))?
892        .map_err(|e| ApiError::internal(format!("upscale failed: {e}")))?
893    };
894
895    Ok(Json(resp))
896}
897
898// ── /api/upscale/stream (SSE) ──────────────────────────────────────────────
899
900async fn upscale_stream(
901    State(state): State<AppState>,
902    Json(req): Json<mold_core::UpscaleRequest>,
903) -> Result<Sse<impl futures_core::Stream<Item = Result<SseEvent, Infallible>>>, ApiError> {
904    if let Err(msg) = mold_core::validate_upscale_request(&req) {
905        return Err(ApiError::validation(msg));
906    }
907
908    let model_name = mold_core::manifest::resolve_model_name(&req.model);
909
910    // Check if model needs pulling before spawning the SSE stream
911    let needs_pull = {
912        let config = state.config.read().await;
913        config
914            .models
915            .get(&model_name)
916            .and_then(|c| c.transformer.as_ref())
917            .is_none()
918    };
919
920    // Validate the model exists in the manifest if we need to pull
921    if needs_pull && mold_core::manifest::find_manifest(&model_name).is_none() {
922        return Err(ApiError::not_found(format!(
923            "unknown upscaler model '{}'. Run 'mold list' to see available models.",
924            model_name
925        )));
926    }
927
928    let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<SseMessage>();
929    let model_name_owned = model_name.clone();
930    let state_clone = state.clone();
931    let upscaler_cache = state.upscaler_cache.clone();
932
933    tokio::spawn(async move {
934        // Auto-pull the upscaler model if not downloaded
935        if needs_pull {
936            let progress_tx = tx.clone();
937            let callback =
938                std::sync::Arc::new(move |event: mold_core::download::DownloadProgressEvent| {
939                    let sse_event = match event {
940                        mold_core::download::DownloadProgressEvent::Status { message } => {
941                            SseProgressEvent::Info { message }
942                        }
943                        mold_core::download::DownloadProgressEvent::FileStart {
944                            filename,
945                            file_index,
946                            total_files,
947                            size_bytes,
948                            batch_bytes_downloaded,
949                            batch_bytes_total,
950                            batch_elapsed_ms,
951                        } => SseProgressEvent::DownloadProgress {
952                            filename,
953                            file_index,
954                            total_files,
955                            bytes_downloaded: 0,
956                            bytes_total: size_bytes,
957                            batch_bytes_downloaded,
958                            batch_bytes_total,
959                            batch_elapsed_ms,
960                        },
961                        mold_core::download::DownloadProgressEvent::FileProgress {
962                            filename,
963                            file_index,
964                            bytes_downloaded,
965                            bytes_total,
966                            batch_bytes_downloaded,
967                            batch_bytes_total,
968                            batch_elapsed_ms,
969                        } => SseProgressEvent::DownloadProgress {
970                            filename,
971                            file_index,
972                            total_files: 0,
973                            bytes_downloaded,
974                            bytes_total,
975                            batch_bytes_downloaded,
976                            batch_bytes_total,
977                            batch_elapsed_ms,
978                        },
979                        mold_core::download::DownloadProgressEvent::FileDone {
980                            filename,
981                            file_index,
982                            total_files,
983                            batch_bytes_downloaded,
984                            batch_bytes_total,
985                            batch_elapsed_ms,
986                        } => SseProgressEvent::DownloadDone {
987                            filename,
988                            file_index,
989                            total_files,
990                            batch_bytes_downloaded,
991                            batch_bytes_total,
992                            batch_elapsed_ms,
993                        },
994                    };
995                    let _ = progress_tx.send(SseMessage::Progress(sse_event));
996                });
997
998            match model_manager::pull_model(&state_clone, &model_name_owned, Some(callback)).await {
999                Ok(_) => {
1000                    let _ = tx.send(SseMessage::Progress(SseProgressEvent::PullComplete {
1001                        model: model_name_owned.clone(),
1002                    }));
1003                }
1004                Err(e) => {
1005                    let _ = tx.send(SseMessage::Error(mold_core::SseErrorEvent {
1006                        message: format!("failed to pull upscaler model: {}", e.error),
1007                    }));
1008                    return;
1009                }
1010            }
1011        }
1012
1013        // Read weights path after potential pull
1014        let weights_path = {
1015            let config = state_clone.config.read().await;
1016            config
1017                .models
1018                .get(&model_name_owned)
1019                .and_then(|c| c.transformer.as_ref())
1020                .map(std::path::PathBuf::from)
1021        };
1022
1023        let Some(weights_path) = weights_path else {
1024            let _ = tx.send(SseMessage::Error(mold_core::SseErrorEvent {
1025                message: format!(
1026                    "upscaler model '{}' not configured after pull",
1027                    model_name_owned
1028                ),
1029            }));
1030            return;
1031        };
1032
1033        let result = if state_clone.gpu_pool.worker_count() > 0 {
1034            match select_aux_worker(&state_clone) {
1035                Ok(worker) => {
1036                    worker.in_flight.fetch_add(1, Ordering::SeqCst);
1037                    let worker_clone = worker.clone();
1038                    let tx_for_worker = tx.clone();
1039                    let model_name_for_worker = model_name_owned.clone();
1040                    let weights_path_for_worker = weights_path.clone();
1041                    let req_for_worker = req.clone();
1042                    let result = tokio::task::spawn_blocking(move || {
1043                        struct ThreadGpuGuard;
1044                        impl Drop for ThreadGpuGuard {
1045                            fn drop(&mut self) {
1046                                mold_inference::device::clear_thread_gpu_ordinal();
1047                            }
1048                        }
1049
1050                        mold_inference::device::init_thread_gpu_ordinal(worker_clone.gpu.ordinal);
1051                        let _thread_gpu = ThreadGpuGuard;
1052                        let _load_lock = worker_clone.model_load_lock.lock().unwrap();
1053                        let _ = tx_for_worker.send(SseMessage::Progress(
1054                            mold_core::SseProgressEvent::StageStart {
1055                                name: format!(
1056                                    "Loading upscaler model on GPU {}",
1057                                    worker_clone.gpu.ordinal
1058                                ),
1059                            },
1060                        ));
1061                        let mut engine = match mold_inference::create_upscale_engine(
1062                            model_name_for_worker,
1063                            weights_path_for_worker,
1064                            mold_inference::LoadStrategy::Eager,
1065                            worker_clone.gpu.ordinal,
1066                        ) {
1067                            Ok(engine) => engine,
1068                            Err(e) => {
1069                                let _ = tx_for_worker.send(SseMessage::Error(
1070                                    mold_core::SseErrorEvent {
1071                                        message: format!("failed to load upscaler: {e}"),
1072                                    },
1073                                ));
1074                                return;
1075                            }
1076                        };
1077
1078                        let tx_progress = tx_for_worker.clone();
1079                        engine.set_on_progress(Box::new(move |event| {
1080                            let sse_event: mold_core::SseProgressEvent = event.into();
1081                            let _ = tx_progress.send(SseMessage::Progress(sse_event));
1082                        }));
1083
1084                        match engine.upscale(&req_for_worker) {
1085                            Ok(resp) => {
1086                                let image_b64 = base64::engine::general_purpose::STANDARD
1087                                    .encode(&resp.image.data);
1088                                let _ = tx_for_worker.send(SseMessage::UpscaleComplete(
1089                                    mold_core::SseUpscaleCompleteEvent {
1090                                        image: image_b64,
1091                                        format: resp.image.format,
1092                                        model: resp.model,
1093                                        scale_factor: resp.scale_factor,
1094                                        original_width: resp.original_width,
1095                                        original_height: resp.original_height,
1096                                        upscale_time_ms: resp.upscale_time_ms,
1097                                    },
1098                                ));
1099                            }
1100                            Err(e) => {
1101                                let _ = tx_for_worker.send(SseMessage::Error(
1102                                    mold_core::SseErrorEvent {
1103                                        message: format!("upscale failed: {e}"),
1104                                    },
1105                                ));
1106                            }
1107                        }
1108
1109                        engine.clear_on_progress();
1110                    })
1111                    .await;
1112                    worker.in_flight.fetch_sub(1, Ordering::SeqCst);
1113                    result
1114                }
1115                Err(e) => {
1116                    let _ = tx.send(SseMessage::Error(mold_core::SseErrorEvent {
1117                        message: e.error,
1118                    }));
1119                    return;
1120                }
1121            }
1122        } else {
1123            let model_name_for_cache = model_name_owned.clone();
1124            let weights_path_for_cache = weights_path.clone();
1125            let req_for_cache = req.clone();
1126            tokio::task::spawn_blocking(move || {
1127                let mut cache = upscaler_cache.lock().unwrap();
1128
1129                let needs_new = cache
1130                    .as_ref()
1131                    .is_none_or(|e| e.model_name() != model_name_for_cache);
1132                if needs_new {
1133                    let _ = tx.send(SseMessage::Progress(
1134                        mold_core::SseProgressEvent::StageStart {
1135                            name: "Loading upscaler model".to_string(),
1136                        },
1137                    ));
1138                    match mold_inference::create_upscale_engine(
1139                        model_name_for_cache,
1140                        weights_path_for_cache,
1141                        mold_inference::LoadStrategy::Eager,
1142                        0,
1143                    ) {
1144                        Ok(new_engine) => {
1145                            *cache = Some(new_engine);
1146                        }
1147                        Err(e) => {
1148                            let _ = tx.send(SseMessage::Error(mold_core::SseErrorEvent {
1149                                message: format!("failed to load upscaler: {e}"),
1150                            }));
1151                            return;
1152                        }
1153                    }
1154                }
1155
1156                let engine = cache.as_mut().unwrap();
1157                let tx_progress = tx.clone();
1158                engine.set_on_progress(Box::new(move |event| {
1159                    let sse_event: mold_core::SseProgressEvent = event.into();
1160                    let _ = tx_progress.send(SseMessage::Progress(sse_event));
1161                }));
1162
1163                match engine.upscale(&req_for_cache) {
1164                    Ok(resp) => {
1165                        let image_b64 =
1166                            base64::engine::general_purpose::STANDARD.encode(&resp.image.data);
1167                        let _ = tx.send(SseMessage::UpscaleComplete(
1168                            mold_core::SseUpscaleCompleteEvent {
1169                                image: image_b64,
1170                                format: resp.image.format,
1171                                model: resp.model,
1172                                scale_factor: resp.scale_factor,
1173                                original_width: resp.original_width,
1174                                original_height: resp.original_height,
1175                                upscale_time_ms: resp.upscale_time_ms,
1176                            },
1177                        ));
1178                    }
1179                    Err(e) => {
1180                        let _ = tx.send(SseMessage::Error(mold_core::SseErrorEvent {
1181                            message: format!("upscale failed: {e}"),
1182                        }));
1183                    }
1184                }
1185
1186                engine.clear_on_progress();
1187            })
1188            .await
1189        };
1190
1191        if let Err(e) = result {
1192            tracing::error!("upscale task panicked: {e}");
1193        }
1194    });
1195
1196    let stream = tokio_stream::wrappers::UnboundedReceiverStream::new(rx)
1197        .map(|msg| Ok::<_, Infallible>(sse_message_to_event(msg)));
1198
1199    Ok(Sse::new(stream).keep_alive(
1200        KeepAlive::new()
1201            .interval(std::time::Duration::from_secs(15))
1202            .text("ping"),
1203    ))
1204}
1205
1206// ── /api/generate/stream (SSE) ───────────────────────────────────────────────
1207
1208#[utoipa::path(
1209    post,
1210    path = "/api/generate/stream",
1211    tag = "generation",
1212    request_body = mold_core::GenerateRequest,
1213    responses(
1214        (status = 200, description = "SSE event stream with progress and result"),
1215        (status = 404, description = "Model not downloaded"),
1216        (status = 422, description = "Invalid request parameters"),
1217        (status = 500, description = "Inference error"),
1218        (status = 503, description = "Generation queue full"),
1219    )
1220)]
1221async fn generate_stream(
1222    State(state): State<AppState>,
1223    Json(mut req): Json<mold_core::GenerateRequest>,
1224) -> Result<Sse<impl futures_core::Stream<Item = Result<SseEvent, Infallible>>>, ApiError> {
1225    let (output_dir, dim_warning, preferred_gpu) = prepare_generation(&state, &mut req).await?;
1226
1227    tracing::info!(
1228        model = %req.model,
1229        prompt = %req.prompt,
1230        "generate/stream request"
1231    );
1232
1233    // Create SSE channel
1234    let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<SseMessage>();
1235
1236    // Send dimension warning before queuing so the client sees it early
1237    if let Some(warning) = dim_warning {
1238        let _ = tx.send(SseMessage::Progress(SseProgressEvent::Info {
1239            message: warning,
1240        }));
1241    }
1242
1243    let (result_tx, result_rx) = tokio::sync::oneshot::channel();
1244    // Assign a server-side ID and register before submit so the entry is
1245    // visible to /api/queue from the moment we accept the request.
1246    let job_id = uuid::Uuid::new_v4().to_string();
1247    state
1248        .job_registry
1249        .register_with_target_gpu(&job_id, &req.model, preferred_gpu);
1250    let job = GenerationJob {
1251        id: job_id.clone(),
1252        request: req,
1253        progress_tx: Some(tx.clone()),
1254        result_tx,
1255        output_dir,
1256    };
1257
1258    let position = state
1259        .queue
1260        .submit(job, state.queue_capacity)
1261        .await
1262        .inspect_err(|_| state.job_registry.remove(&job_id))
1263        .map_err(submit_error_to_api)?;
1264
1265    // First event the client sees — carries the position AND the server
1266    // ID so the SPA can later reconcile this job against /api/queue.
1267    let _ = tx.send(SseMessage::Progress(SseProgressEvent::Queued {
1268        position,
1269        id: job_id,
1270    }));
1271
1272    // Hold `tx` alive in a background task until the job completes, so the SSE
1273    // stream never closes prematurely even if the queue worker hasn't received
1274    // the job yet.
1275    tokio::spawn(async move {
1276        let _ = result_rx.await;
1277        drop(tx); // closes the SSE stream
1278    });
1279
1280    // Build SSE stream from the channel receiver.
1281    let stream = tokio_stream::wrappers::UnboundedReceiverStream::new(rx)
1282        .map(|msg| Ok::<_, Infallible>(sse_message_to_event(msg)));
1283
1284    Ok(Sse::new(stream).keep_alive(
1285        KeepAlive::new()
1286            .interval(std::time::Duration::from_secs(15))
1287            .text("ping"),
1288    ))
1289}
1290
1291// ── /api/models ───────────────────────────────────────────────────────────────
1292
1293#[utoipa::path(
1294    get,
1295    path = "/api/models",
1296    tag = "models",
1297    responses(
1298        (status = 200, description = "List of available models", body = Vec<ModelInfoExtended>),
1299    )
1300)]
1301async fn list_models(State(state): State<AppState>) -> Json<Vec<ModelInfoExtended>> {
1302    Json(model_manager::list_models(&state).await)
1303}
1304
1305async fn generate_estimate(
1306    State(state): State<AppState>,
1307    Json(req): Json<GenerateRequest>,
1308) -> Result<Json<mold_core::GenerationMemoryEstimate>, ApiError> {
1309    Ok(Json(
1310        model_manager::estimate_generation_memory(&state, &req).await?,
1311    ))
1312}
1313
1314async fn model_components(
1315    State(state): State<AppState>,
1316    Path(model): Path<String>,
1317) -> Result<Json<mold_core::ModelComponentsResponse>, ApiError> {
1318    Ok(Json(
1319        model_manager::model_component_status(&state, &model).await?,
1320    ))
1321}
1322
1323// ── /api/models/load ──────────────────────────────────────────────────────────
1324
1325#[derive(Debug, Deserialize, utoipa::ToSchema)]
1326pub struct LoadModelBody {
1327    #[schema(example = "flux-schnell:q8")]
1328    pub model: String,
1329    /// Target GPU ordinal (multi-GPU only). If omitted, the server uses its
1330    /// default placement strategy.
1331    #[serde(default, skip_serializing_if = "Option::is_none")]
1332    pub gpu: Option<usize>,
1333}
1334
1335#[utoipa::path(
1336    post,
1337    path = "/api/models/load",
1338    tag = "models",
1339    request_body = LoadModelBody,
1340    responses(
1341        (status = 200, description = "Model loaded successfully"),
1342        (status = 404, description = "Model not downloaded"),
1343        (status = 400, description = "Unknown model"),
1344        (status = 500, description = "Failed to load model"),
1345    )
1346)]
1347async fn load_model(
1348    State(state): State<AppState>,
1349    Json(body): Json<LoadModelBody>,
1350) -> Result<impl IntoResponse, ApiError> {
1351    if let Err(e) = model_manager::install_catalog_model(&state, &body.model).await {
1352        return Err(model_manager::install_error_to_api_error(&e));
1353    }
1354    let _ = model_manager::check_model_available(&state, &body.model).await?;
1355
1356    // Multi-GPU path: route through the pool.
1357    if state.gpu_pool.worker_count() > 0 {
1358        let worker = match body.gpu {
1359            Some(ordinal) => state
1360                .gpu_pool
1361                .workers
1362                .iter()
1363                .find(|w| w.gpu.ordinal == ordinal)
1364                .cloned()
1365                .ok_or_else(|| {
1366                    ApiError::not_found(format!("no GPU worker with ordinal {ordinal}"))
1367                })?,
1368            None => {
1369                let est = crate::queue::estimate_model_vram(&body.model);
1370                state
1371                    .gpu_pool
1372                    .select_worker(&body.model, est)
1373                    .ok_or_else(|| {
1374                        ApiError::internal(format!(
1375                            "no GPU available to load model '{}'",
1376                            body.model
1377                        ))
1378                    })?
1379            }
1380        };
1381        let config_snapshot = state.config.read().await.clone();
1382        let model_name = body.model.clone();
1383        let worker_clone = worker.clone();
1384        tokio::task::spawn_blocking(move || {
1385            crate::gpu_worker::load_blocking(&worker_clone, &model_name, &config_snapshot)
1386        })
1387        .await
1388        .map_err(|e| ApiError::internal(format!("model load task failed: {e}")))?
1389        .map_err(|e| ApiError::internal(format!("model load error: {e}")))?;
1390
1391        tracing::info!(
1392            model = %body.model,
1393            gpu = worker.gpu.ordinal,
1394            "model loaded via API"
1395        );
1396        return Ok(StatusCode::OK);
1397    }
1398
1399    // Legacy single-GPU path (no workers discovered). No resolution context
1400    // here — admin load uses size-only peak via the `None` hint.
1401    model_manager::ensure_model_ready(&state, &body.model, None, None, false).await?;
1402    tracing::info!(model = %body.model, gpu = ?body.gpu, "model loaded via API (legacy)");
1403    Ok(StatusCode::OK)
1404}
1405
1406// ── /api/models/pull ──────────────────────────────────────────────────────────
1407
1408#[utoipa::path(
1409    post,
1410    path = "/api/models/pull",
1411    tag = "models",
1412    request_body = LoadModelBody,
1413    responses(
1414        (status = 200, description = "Model pulled (SSE stream or plain text)"),
1415        (status = 400, description = "Unknown model"),
1416        (status = 500, description = "Download failed"),
1417    )
1418)]
1419async fn pull_model_endpoint(
1420    State(state): State<AppState>,
1421    headers: HeaderMap,
1422    Json(body): Json<LoadModelBody>,
1423) -> Result<impl IntoResponse, ApiError> {
1424    let wants_sse = headers
1425        .get(header::ACCEPT)
1426        .and_then(|v| v.to_str().ok())
1427        .is_some_and(|v| v.contains("text/event-stream"));
1428
1429    if body.model.starts_with("cv:") || body.model.starts_with("hf:") {
1430        if let Err(e) = model_manager::install_catalog_model(&state, &body.model).await {
1431            return Err(model_manager::install_error_to_api_error(&e));
1432        }
1433        if model_manager::check_model_available(&state, &body.model)
1434            .await
1435            .is_ok()
1436        {
1437            return Ok(PullResponse::Text(format!(
1438                "model '{}' is already present",
1439                body.model
1440            )));
1441        }
1442        let companion_names = {
1443            let intents = state.catalog_intents.read().await;
1444            intents
1445                .get(&body.model)
1446                .map(|intent| {
1447                    intent
1448                        .companions
1449                        .iter()
1450                        .map(|companion| companion.name.clone())
1451                        .collect::<Vec<_>>()
1452                })
1453                .unwrap_or_default()
1454        };
1455        let models_dir = state.config.read().await.resolved_models_dir();
1456        let companion_jobs = crate::catalog_api::enqueue_missing_companions(
1457            &companion_names,
1458            &models_dir,
1459            &state.downloads,
1460            Some(&body.model),
1461        )
1462        .await;
1463        let primary_job = crate::catalog_api::enqueue_catalog_primary_repair(&state, &body.model)
1464            .await
1465            .map_err(|(status, msg)| ApiError::internal_with_status(msg, status))?;
1466        if !companion_jobs.is_empty() || primary_job.is_some() {
1467            let primary_count = usize::from(primary_job.is_some());
1468            return Ok(PullResponse::Text(format!(
1469                "queued repair for model '{}' ({} primary job(s), {} companion job(s))",
1470                body.model,
1471                primary_count,
1472                companion_jobs.len()
1473            )));
1474        }
1475        model_manager::check_model_available(&state, &body.model).await?;
1476        return Ok(PullResponse::Text(format!(
1477            "model '{}' is already present",
1478            body.model
1479        )));
1480    }
1481
1482    // Enqueue via the queue. Treat idempotent AlreadyPresent as success.
1483    let (job_id, _position) = match state.downloads.enqueue(body.model.clone()).await {
1484        Ok((id, pos, _)) => (id, pos),
1485        Err(crate::downloads::EnqueueError::UnknownModel(_)) => {
1486            return Err(ApiError::unknown_model(format!(
1487                "unknown model '{}'. Run 'mold list' to see available models.",
1488                body.model
1489            )));
1490        }
1491        Err(crate::downloads::EnqueueError::LockPoisoned) => {
1492            return Err(ApiError::internal("download queue state is corrupt"));
1493        }
1494    };
1495
1496    if !wants_sse {
1497        // Await terminal event for this job, return plain text.
1498        let mut rx = state.downloads.subscribe();
1499        loop {
1500            match rx.recv().await {
1501                Ok(mold_core::types::DownloadEvent::JobDone { id, model }) if id == job_id => {
1502                    return Ok(PullResponse::Text(format!(
1503                        "model '{model}' pulled successfully"
1504                    )));
1505                }
1506                Ok(mold_core::types::DownloadEvent::JobFailed { id, error }) if id == job_id => {
1507                    return Err(ApiError::internal(format!(
1508                        "failed to pull model '{}': {error}",
1509                        body.model
1510                    )));
1511                }
1512                Ok(mold_core::types::DownloadEvent::JobCancelled { id }) if id == job_id => {
1513                    return Err(ApiError::internal(format!(
1514                        "pull of '{}' was cancelled",
1515                        body.model
1516                    )));
1517                }
1518                Ok(_) => continue,
1519                Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
1520                Err(tokio::sync::broadcast::error::RecvError::Closed) => {
1521                    return Err(ApiError::internal("download queue channel closed"));
1522                }
1523            }
1524        }
1525    }
1526
1527    // SSE: re-emit queue events shaped like the legacy SseProgressEvent::DownloadProgress
1528    // so the TUI's existing consumer continues to work unchanged.
1529    let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<SseMessage>();
1530    let mut events = state.downloads.subscribe();
1531    let model_for_cb = body.model.clone();
1532    tokio::spawn(async move {
1533        loop {
1534            match events.recv().await {
1535                Ok(mold_core::types::DownloadEvent::Started {
1536                    id,
1537                    files_total,
1538                    bytes_total,
1539                }) if id == job_id => {
1540                    let _ = tx.send(SseMessage::Progress(SseProgressEvent::DownloadProgress {
1541                        filename: String::new(),
1542                        file_index: 0,
1543                        total_files: files_total,
1544                        bytes_downloaded: 0,
1545                        bytes_total,
1546                        batch_bytes_downloaded: 0,
1547                        batch_bytes_total: bytes_total,
1548                        batch_elapsed_ms: 0,
1549                    }));
1550                }
1551                Ok(mold_core::types::DownloadEvent::Progress {
1552                    id,
1553                    files_done,
1554                    bytes_done,
1555                    current_file,
1556                }) if id == job_id => {
1557                    let _ = tx.send(SseMessage::Progress(SseProgressEvent::DownloadProgress {
1558                        filename: current_file.unwrap_or_default(),
1559                        file_index: files_done,
1560                        total_files: 0,
1561                        bytes_downloaded: bytes_done,
1562                        bytes_total: 0,
1563                        batch_bytes_downloaded: bytes_done,
1564                        batch_bytes_total: 0,
1565                        batch_elapsed_ms: 0,
1566                    }));
1567                }
1568                Ok(mold_core::types::DownloadEvent::JobDone { id, .. }) if id == job_id => {
1569                    let _ = tx.send(SseMessage::Progress(SseProgressEvent::PullComplete {
1570                        model: model_for_cb.clone(),
1571                    }));
1572                    break;
1573                }
1574                Ok(mold_core::types::DownloadEvent::JobFailed { id, error }) if id == job_id => {
1575                    let _ = tx.send(SseMessage::Error(SseErrorEvent { message: error }));
1576                    break;
1577                }
1578                Ok(mold_core::types::DownloadEvent::JobCancelled { id }) if id == job_id => {
1579                    let _ = tx.send(SseMessage::Error(SseErrorEvent {
1580                        message: "pull cancelled".into(),
1581                    }));
1582                    break;
1583                }
1584                Ok(_) => continue,
1585                Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
1586                Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
1587            }
1588        }
1589    });
1590
1591    let stream = tokio_stream::wrappers::UnboundedReceiverStream::new(rx)
1592        .map(|msg| Ok::<_, Infallible>(sse_message_to_event(msg)));
1593
1594    Ok(PullResponse::Sse(
1595        Sse::new(stream)
1596            .keep_alive(
1597                KeepAlive::new()
1598                    .interval(std::time::Duration::from_secs(15))
1599                    .text("ping"),
1600            )
1601            .into_response(),
1602    ))
1603}
1604
1605/// Response type that can be either SSE stream or plain text.
1606enum PullResponse {
1607    Sse(axum::response::Response),
1608    Text(String),
1609}
1610
1611impl IntoResponse for PullResponse {
1612    fn into_response(self) -> axum::response::Response {
1613        match self {
1614            PullResponse::Sse(resp) => resp,
1615            PullResponse::Text(text) => text.into_response(),
1616        }
1617    }
1618}
1619
1620// ── /api/models/unload ────────────────────────────────────────────────────────
1621
1622/// Optional request body for unload — clients may specify a model or GPU target.
1623/// An empty body (or no body) unloads the active model on the legacy path.
1624#[derive(Debug, Default, Deserialize, utoipa::ToSchema)]
1625pub struct UnloadRequest {
1626    /// Specific model to unload. If omitted, the active model is unloaded.
1627    #[serde(default, skip_serializing_if = "Option::is_none")]
1628    pub model: Option<String>,
1629    /// Target GPU ordinal (multi-GPU only).
1630    #[serde(default, skip_serializing_if = "Option::is_none")]
1631    pub gpu: Option<usize>,
1632}
1633
1634#[utoipa::path(
1635    delete,
1636    path = "/api/models/unload",
1637    tag = "models",
1638    request_body(content = Option<UnloadRequest>, content_type = "application/json"),
1639    responses(
1640        (status = 200, description = "Model unloaded or no model was loaded", body = String),
1641    )
1642)]
1643async fn unload_model(
1644    State(state): State<AppState>,
1645    body: Option<Json<UnloadRequest>>,
1646) -> Result<impl IntoResponse, ApiError> {
1647    let req = body.map(|b| b.0).unwrap_or_default();
1648    tracing::debug!(model = ?req.model, gpu = ?req.gpu, "unload request");
1649    clear_global_upscaler_cache(&state);
1650
1651    // Multi-GPU path: target specific GPU or model across the pool.
1652    if state.gpu_pool.worker_count() > 0 {
1653        // Select the workers to unload from.
1654        let targets: Vec<_> = match (req.gpu, req.model.as_deref()) {
1655            (Some(ordinal), _) => state
1656                .gpu_pool
1657                .workers
1658                .iter()
1659                .filter(|w| w.gpu.ordinal == ordinal)
1660                .cloned()
1661                .collect(),
1662            (None, Some(model)) => state
1663                .gpu_pool
1664                .workers
1665                .iter()
1666                .filter(|w| {
1667                    let cache = w.model_cache.lock().unwrap();
1668                    cache
1669                        .get(model)
1670                        .map(|e| e.residency == crate::model_cache::ModelResidency::Gpu)
1671                        .unwrap_or(false)
1672                })
1673                .cloned()
1674                .collect(),
1675            (None, None) => state.gpu_pool.workers.clone(),
1676        };
1677
1678        if targets.is_empty() {
1679            return Ok((StatusCode::OK, "no model loaded".to_string()));
1680        }
1681
1682        let mut unloaded_pairs: Vec<(usize, String)> = Vec::new();
1683        for worker in targets {
1684            let worker_clone = worker.clone();
1685            let result = tokio::task::spawn_blocking(move || {
1686                crate::gpu_worker::unload_blocking(&worker_clone)
1687            })
1688            .await
1689            .map_err(|e| ApiError::internal(format!("unload task failed: {e}")))?;
1690            if let Some(name) = result {
1691                unloaded_pairs.push((worker.gpu.ordinal, name));
1692            }
1693        }
1694
1695        let msg = if unloaded_pairs.is_empty() {
1696            "no model loaded".to_string()
1697        } else {
1698            let joined: Vec<String> = unloaded_pairs
1699                .iter()
1700                .map(|(o, m)| format!("gpu{o}:{m}"))
1701                .collect();
1702            format!("unloaded {}", joined.join(", "))
1703        };
1704        return Ok((StatusCode::OK, msg));
1705    }
1706
1707    // Legacy single-GPU path.
1708    Ok((StatusCode::OK, model_manager::unload_model(&state).await))
1709}
1710
1711// ── /api/status ───────────────────────────────────────────────────────────────
1712
1713#[utoipa::path(
1714    get,
1715    path = "/api/status",
1716    tag = "server",
1717    responses(
1718        (status = 200, description = "Server status", body = ServerStatus),
1719    )
1720)]
1721async fn server_status(State(state): State<AppState>) -> Json<ServerStatus> {
1722    // Aggregate GPU status from the pool.
1723    let gpu_statuses = state.gpu_pool.gpu_status();
1724    let has_gpus = !gpu_statuses.is_empty();
1725
1726    // Collect loaded models from GPU workers.
1727    let gpu_models_loaded: Vec<String> = gpu_statuses
1728        .iter()
1729        .filter_map(|g| g.loaded_model.clone())
1730        .collect();
1731    let gpu_busy = gpu_statuses
1732        .iter()
1733        .any(|g| g.state == GpuWorkerState::Generating);
1734
1735    // Pull current_generation from the first busy worker (multi-GPU) or
1736    // from the legacy snapshot.
1737    let multi_gpu_current_gen = if has_gpus {
1738        state.gpu_pool.workers.iter().find_map(|w| {
1739            let gen = w.active_generation.read().ok()?;
1740            gen.as_ref().map(|g| ActiveGenerationStatus {
1741                model: g.model.clone(),
1742                prompt_sha256: g.prompt_sha256.clone(),
1743                started_at_unix_ms: g.started_at_unix_ms,
1744                elapsed_ms: g.started_at.elapsed().as_millis() as u64,
1745            })
1746        })
1747    } else {
1748        None
1749    };
1750
1751    // Fall back to legacy single-GPU snapshot for backwards compat.
1752    let (models_loaded, busy, current_generation) = if has_gpus {
1753        (gpu_models_loaded, gpu_busy, multi_gpu_current_gen)
1754    } else {
1755        let snapshot = state.model_cache.lock().await.snapshot();
1756        let models = match (snapshot.model_name, snapshot.is_loaded) {
1757            (Some(model_name), true) => vec![model_name],
1758            _ => vec![],
1759        };
1760        let gen = state
1761            .active_generation
1762            .read()
1763            .unwrap_or_else(|e| e.into_inner())
1764            .as_ref()
1765            .map(|active| ActiveGenerationStatus {
1766                model: active.model.clone(),
1767                prompt_sha256: active.prompt_sha256.clone(),
1768                started_at_unix_ms: active.started_at_unix_ms,
1769                elapsed_ms: active.started_at.elapsed().as_millis() as u64,
1770            });
1771        let is_busy = gen.is_some();
1772        (models, is_busy, gen)
1773    };
1774
1775    Json(ServerStatus {
1776        version: env!("CARGO_PKG_VERSION").to_string(),
1777        git_sha: if mold_core::build_info::GIT_SHA == "unknown" {
1778            None
1779        } else {
1780            Some(mold_core::build_info::GIT_SHA.to_string())
1781        },
1782        build_date: if mold_core::build_info::BUILD_DATE == "unknown" {
1783            None
1784        } else {
1785            Some(mold_core::build_info::BUILD_DATE.to_string())
1786        },
1787        models_loaded,
1788        busy,
1789        current_generation,
1790        gpu_info: query_gpu_info(),
1791        uptime_secs: state.start_time.elapsed().as_secs(),
1792        hostname: hostname::get().ok().and_then(|h| h.into_string().ok()),
1793        memory_status: mold_inference::device::memory_status_string(),
1794        gpus: if has_gpus { Some(gpu_statuses) } else { None },
1795        queue_depth: Some(state.queue.pending()),
1796        queue_capacity: Some(state.queue_capacity),
1797    })
1798}
1799
1800// ── /health ───────────────────────────────────────────────────────────────────
1801
1802#[utoipa::path(
1803    get,
1804    path = "/health",
1805    tag = "server",
1806    responses(
1807        (status = 200, description = "Server is healthy"),
1808    )
1809)]
1810async fn health() -> impl IntoResponse {
1811    StatusCode::OK
1812}
1813
1814// ── /api/queue ───────────────────────────────────────────────────────────────
1815
1816/// Snapshot of every job currently queued or running on the server. Clients
1817/// (notably the web SPA) poll this to reconcile their local card list — any
1818/// "running" card whose server id isn't here is a zombie left over from a
1819/// dropped SSE stream and should be dead-lettered.
1820#[utoipa::path(
1821    get,
1822    path = "/api/queue",
1823    tag = "queue",
1824    responses(
1825        (status = 200, description = "Queue snapshot", body = crate::job_registry::QueueListing),
1826    )
1827)]
1828async fn list_queue(State(state): State<AppState>) -> Json<crate::job_registry::QueueListing> {
1829    Json(state.job_registry.snapshot())
1830}
1831
1832#[derive(Debug, Deserialize, utoipa::ToSchema)]
1833struct QueuePatchRequest {
1834    target_gpu: Option<usize>,
1835}
1836
1837#[utoipa::path(
1838    patch,
1839    path = "/api/queue/{id}",
1840    tag = "queue",
1841    request_body = QueuePatchRequest,
1842    responses(
1843        (status = 200, description = "Updated queue entry", body = crate::job_registry::JobEntry),
1844        (status = 404, description = "Queue job not found"),
1845        (status = 409, description = "Queue job is already running"),
1846        (status = 422, description = "Invalid GPU target"),
1847    )
1848)]
1849async fn patch_queue_job(
1850    State(state): State<AppState>,
1851    Path(id): Path<String>,
1852    Json(req): Json<QueuePatchRequest>,
1853) -> Result<Json<crate::job_registry::JobEntry>, ApiError> {
1854    if let Some(target) = req.target_gpu {
1855        let available = state
1856            .gpu_pool
1857            .workers
1858            .iter()
1859            .any(|w| w.gpu.ordinal == target);
1860        if !available {
1861            return Err(ApiError::validation(format!(
1862                "gpu:{target} is not available in this server's worker pool"
1863            )));
1864        }
1865    }
1866
1867    state
1868        .job_registry
1869        .set_target_gpu(&id, req.target_gpu)
1870        .map_err(|e| match e {
1871            crate::job_registry::TargetGpuUpdateError::NotFound => {
1872                ApiError::queue_job_not_found(format!("queue job {id} not found"))
1873            }
1874            crate::job_registry::TargetGpuUpdateError::AlreadyRunning => {
1875                ApiError::queue_job_running(format!(
1876                    "queue job {id} is already running; lane changes only apply to queued jobs"
1877                ))
1878            }
1879        })?;
1880
1881    let entry = state
1882        .job_registry
1883        .entry(&id)
1884        .ok_or_else(|| ApiError::queue_job_not_found(format!("queue job {id} not found")))?;
1885    Ok(Json(entry))
1886}
1887
1888// ── /api/capabilities ────────────────────────────────────────────────────────
1889
1890/// Report the feature toggles a client needs to render correctly (hide the
1891/// delete button when delete isn't allowed, etc.). No auth required — this
1892/// is a read-only introspection endpoint.
1893async fn server_capabilities() -> Json<mold_core::ServerCapabilities> {
1894    let catalog_available = std::env::var("MOLD_CATALOG_DISABLE")
1895        .map(|v| v != "1" && !v.eq_ignore_ascii_case("true"))
1896        .unwrap_or(true);
1897
1898    Json(mold_core::ServerCapabilities {
1899        gallery: mold_core::GalleryCapabilities { can_delete: true },
1900        catalog: mold_core::CatalogCapabilities {
1901            available: catalog_available,
1902            families: mold_catalog::families::ALL_FAMILIES
1903                .iter()
1904                .map(|f| f.as_str().to_string())
1905                .collect::<Vec<_>>(),
1906        },
1907    })
1908}
1909
1910// ── /api/capabilities/chain-limits ───────────────────────────────────────────
1911
1912#[utoipa::path(
1913    get,
1914    path = "/api/capabilities/chain-limits",
1915    tag = "server",
1916    params(
1917        ("model" = String, Query, description = "Model name (e.g. ltx-2-19b-distilled:fp8)")
1918    ),
1919    responses(
1920        (status = 200, description = "Chain limits for the requested model",
1921         body = crate::chain_limits::ChainLimits),
1922        (status = 400, description = "Missing required 'model' query parameter"),
1923        (status = 404, description = "Unknown or unsupported model"),
1924    )
1925)]
1926async fn capabilities_chain_limits(
1927    axum::extract::Query(params): axum::extract::Query<std::collections::HashMap<String, String>>,
1928) -> axum::response::Response {
1929    let raw_model = match params.get("model") {
1930        Some(m) => m.clone(),
1931        None => {
1932            return (
1933                StatusCode::BAD_REQUEST,
1934                "missing required 'model' query parameter\n",
1935            )
1936                .into_response();
1937        }
1938    };
1939
1940    let resolved = mold_core::manifest::resolve_model_name(&raw_model);
1941    let Some(manifest) = mold_core::manifest::find_manifest(&resolved) else {
1942        return (StatusCode::NOT_FOUND, "unknown model\n").into_response();
1943    };
1944    let family = manifest.family.clone();
1945
1946    if crate::chain_limits::family_cap(&family).is_none() {
1947        return (StatusCode::NOT_FOUND, "model is not chain-capable\n").into_response();
1948    }
1949
1950    let quant = resolved
1951        .split_once(':')
1952        .map(|(_, tag)| tag.to_string())
1953        .unwrap_or_default();
1954
1955    // TODO(sub-project D): pass live free VRAM from AppState.
1956    let limits = crate::chain_limits::compute_limits(&resolved, &family, &quant, 0);
1957    Json(limits).into_response()
1958}
1959
1960// ── /api/shutdown ─────────────────────────────────────────────────────────────
1961
1962/// Trigger graceful server shutdown.
1963///
1964/// When API key auth is enabled, the auth middleware protects this endpoint.
1965/// When auth is disabled, only requests from loopback addresses (127.0.0.1, ::1)
1966/// are accepted to prevent remote shutdown.
1967#[utoipa::path(
1968    post,
1969    path = "/api/shutdown",
1970    tag = "server",
1971    responses(
1972        (status = 200, description = "Shutdown initiated"),
1973        (status = 403, description = "Forbidden — remote shutdown requires API key auth"),
1974    )
1975)]
1976async fn shutdown_server(State(state): State<AppState>, request: Request) -> impl IntoResponse {
1977    // When auth is disabled (no AuthState extension or AuthState is None),
1978    // restrict shutdown to loopback addresses only.
1979    let auth_enabled = request
1980        .extensions()
1981        .get::<crate::auth::AuthState>()
1982        .is_some_and(|s| s.is_some());
1983
1984    if !auth_enabled {
1985        let is_loopback = request
1986            .extensions()
1987            .get::<axum::extract::ConnectInfo<std::net::SocketAddr>>()
1988            .map(|ci| ci.0.ip().is_loopback())
1989            .unwrap_or(false);
1990        if !is_loopback {
1991            return (
1992                StatusCode::FORBIDDEN,
1993                "shutdown requires API key auth or localhost access\n",
1994            );
1995        }
1996    }
1997
1998    tracing::info!("shutdown requested via API");
1999    if let Some(tx) = state.shutdown_tx.lock().await.take() {
2000        let _ = tx.send(());
2001    }
2002    (StatusCode::OK, "shutdown initiated\n")
2003}
2004
2005// ── /api/gallery ──────────────────────────────────────────────────────────────
2006
2007/// List gallery images from the server's output directory.
2008///
2009/// Prefers the SQLite metadata DB when available so listings stay fast on
2010/// large galleries (no per-request directory walk). Falls back to the
2011/// filesystem scan when the DB is disabled, can't be opened, or — as a
2012/// safety net — has no rows for this directory yet (e.g. the reconciliation
2013/// background task has not finished on first startup).
2014async fn list_gallery(
2015    State(state): State<AppState>,
2016) -> Result<Json<Vec<mold_core::GalleryImage>>, ApiError> {
2017    let config = state.config.read().await;
2018    if config.is_output_disabled() {
2019        return Ok(Json(Vec::new()));
2020    }
2021    let output_dir = config.effective_output_dir();
2022    drop(config);
2023
2024    if !output_dir.is_dir() {
2025        return Ok(Json(Vec::new()));
2026    }
2027
2028    if state.metadata_db.is_some() {
2029        let db_arc = state.metadata_db.clone();
2030        let dir = output_dir.clone();
2031        let listed = tokio::task::spawn_blocking(move || {
2032            db_arc
2033                .as_ref()
2034                .as_ref()
2035                .map(|db| db.list(Some(&dir)))
2036                .transpose()
2037        })
2038        .await
2039        .map_err(|e| ApiError::internal(format!("gallery DB query failed: {e}")))?
2040        .map_err(|e| ApiError::internal(format!("gallery DB query failed: {e:#}")))?;
2041        if let Some(rows) = listed {
2042            if !rows.is_empty() {
2043                let images = rows.iter().map(|r| r.to_gallery_image()).collect();
2044                return Ok(Json(images));
2045            }
2046        }
2047    }
2048
2049    let images = tokio::task::spawn_blocking(move || scan_gallery_dir(&output_dir))
2050        .await
2051        .map_err(|e| ApiError::internal(format!("gallery scan failed: {e}")))?;
2052
2053    Ok(Json(images))
2054}
2055
2056/// Serve a gallery file by filename.
2057///
2058/// Supports HTTP `Range` requests so `<video>` elements can scrub MP4
2059/// outputs without downloading the whole clip up front. Partial responses
2060/// stream straight from disk via `tokio_util::io::ReaderStream` — nothing
2061/// buffers the full file in server RAM, which matters once a gallery
2062/// contains multi-GB LTX-2 outputs. Non-range requests still return the
2063/// whole file (streamed) with `Accept-Ranges: bytes` so the client knows
2064/// it can seek on subsequent requests.
2065async fn get_gallery_image(
2066    State(state): State<AppState>,
2067    headers: HeaderMap,
2068    axum::extract::Path(filename): axum::extract::Path<String>,
2069) -> Result<axum::response::Response, ApiError> {
2070    let config = state.config.read().await;
2071    if config.is_output_disabled() {
2072        return Err(ApiError::not_found("image output is disabled"));
2073    }
2074    let output_dir = config.effective_output_dir();
2075    drop(config);
2076
2077    // Sanitize: prevent directory traversal
2078    let clean_name = std::path::Path::new(&filename)
2079        .file_name()
2080        .map(|f| f.to_string_lossy().to_string())
2081        .unwrap_or_default();
2082
2083    if clean_name.is_empty() || clean_name != filename {
2084        return Err(ApiError::validation("invalid filename"));
2085    }
2086
2087    let path = output_dir.join(&clean_name);
2088    let meta = match tokio::fs::metadata(&path).await {
2089        Ok(m) if m.is_file() => m,
2090        _ => {
2091            return Err(ApiError::not_found(format!(
2092                "image not found: {clean_name}"
2093            )));
2094        }
2095    };
2096    let total_len = meta.len();
2097    let content_type = content_type_for_filename(&clean_name);
2098
2099    let range_header = headers
2100        .get(header::RANGE)
2101        .and_then(|v| v.to_str().ok())
2102        .map(|s| s.to_string());
2103
2104    let file = tokio::fs::File::open(&path)
2105        .await
2106        .map_err(|e| ApiError::internal(format!("failed to open file: {e}")))?;
2107
2108    if let Some(raw) = range_header {
2109        if let Some((start, end)) = parse_byte_range(&raw, total_len) {
2110            return serve_range(file, start, end, total_len, content_type).await;
2111        } else {
2112            // A `Range` header we can't satisfy ⇒ 416 per RFC 9110 §15.6.2.
2113            return Ok(axum::response::Response::builder()
2114                .status(StatusCode::RANGE_NOT_SATISFIABLE)
2115                .header(header::CONTENT_RANGE, format!("bytes */{total_len}"))
2116                .body(axum::body::Body::empty())
2117                .unwrap());
2118        }
2119    }
2120
2121    // Full response: stream the file rather than buffer it in RAM.
2122    let stream = tokio_util::io::ReaderStream::new(file);
2123    let body = axum::body::Body::from_stream(stream);
2124    Ok(axum::response::Response::builder()
2125        .status(StatusCode::OK)
2126        .header(header::CONTENT_TYPE, content_type)
2127        .header(header::ACCEPT_RANGES, "bytes")
2128        .header(header::CONTENT_LENGTH, total_len)
2129        .header(header::CACHE_CONTROL, "public, max-age=3600")
2130        .body(body)
2131        .unwrap())
2132}
2133
2134/// Parse a `Range: bytes=start-end` header into a concrete (start, end)
2135/// byte range inclusive on both ends. Returns `None` for unsatisfiable or
2136/// malformed ranges — the caller translates that into a 416 response.
2137///
2138/// Only the single-range form is supported (multipart ranges are vanishingly
2139/// rare in practice and substantially more complex to implement correctly;
2140/// browsers for `<video>` always send single ranges).
2141fn parse_byte_range(header: &str, total_len: u64) -> Option<(u64, u64)> {
2142    let spec = header.strip_prefix("bytes=")?;
2143    if spec.contains(',') {
2144        return None;
2145    }
2146    let (start_s, end_s) = spec.split_once('-')?;
2147    let start_s = start_s.trim();
2148    let end_s = end_s.trim();
2149
2150    if total_len == 0 {
2151        return None;
2152    }
2153
2154    if start_s.is_empty() {
2155        // Suffix range: `bytes=-N` means "the last N bytes".
2156        let suffix: u64 = end_s.parse().ok()?;
2157        if suffix == 0 {
2158            return None;
2159        }
2160        let start = total_len.saturating_sub(suffix);
2161        return Some((start, total_len - 1));
2162    }
2163
2164    let start: u64 = start_s.parse().ok()?;
2165    if start >= total_len {
2166        return None;
2167    }
2168    let end: u64 = if end_s.is_empty() {
2169        total_len - 1
2170    } else {
2171        end_s.parse().ok()?
2172    };
2173    let end = end.min(total_len - 1);
2174    if end < start {
2175        return None;
2176    }
2177    Some((start, end))
2178}
2179
2180/// Emit a `206 Partial Content` response streaming `[start, end]` inclusive
2181/// from the already-open file handle. `take(len)` bounds the reader so the
2182/// body terminates exactly at `end + 1` instead of reading the tail.
2183async fn serve_range(
2184    mut file: tokio::fs::File,
2185    start: u64,
2186    end: u64,
2187    total_len: u64,
2188    content_type: &'static str,
2189) -> Result<axum::response::Response, ApiError> {
2190    use tokio::io::{AsyncReadExt, AsyncSeekExt};
2191    file.seek(std::io::SeekFrom::Start(start))
2192        .await
2193        .map_err(|e| ApiError::internal(format!("seek failed: {e}")))?;
2194    let len = end - start + 1;
2195    let stream = tokio_util::io::ReaderStream::new(file.take(len));
2196    let body = axum::body::Body::from_stream(stream);
2197    Ok(axum::response::Response::builder()
2198        .status(StatusCode::PARTIAL_CONTENT)
2199        .header(header::CONTENT_TYPE, content_type)
2200        .header(header::ACCEPT_RANGES, "bytes")
2201        .header(header::CONTENT_LENGTH, len)
2202        .header(
2203            header::CONTENT_RANGE,
2204            format!("bytes {start}-{end}/{total_len}"),
2205        )
2206        // Partial content is less cacheable at intermediaries than a plain
2207        // 200; keep a short TTL so the client's own cache still helps.
2208        .header(header::CACHE_CONTROL, "public, max-age=300")
2209        .body(body)
2210        .unwrap())
2211}
2212
2213/// Pick an HTTP Content-Type for a gallery filename. Covers every format
2214/// `OutputFormat` can emit plus a safe default.
2215fn content_type_for_filename(name: &str) -> &'static str {
2216    let lower = name.to_ascii_lowercase();
2217    if lower.ends_with(".png") {
2218        "image/png"
2219    } else if lower.ends_with(".jpg") || lower.ends_with(".jpeg") {
2220        "image/jpeg"
2221    } else if lower.ends_with(".gif") {
2222        "image/gif"
2223    } else if lower.ends_with(".webp") {
2224        "image/webp"
2225    } else if lower.ends_with(".apng") {
2226        "image/apng"
2227    } else if lower.ends_with(".mp4") {
2228        "video/mp4"
2229    } else {
2230        "application/octet-stream"
2231    }
2232}
2233
2234/// Delete a gallery image and its server-side thumbnail.
2235///
2236/// Destructive, but always enabled — pair with the `MOLD_API_KEY` middleware
2237/// when the server is exposed beyond localhost.
2238async fn delete_gallery_image(
2239    State(state): State<AppState>,
2240    axum::extract::Path(filename): axum::extract::Path<String>,
2241) -> Result<impl IntoResponse, ApiError> {
2242    let config = state.config.read().await;
2243    if config.is_output_disabled() {
2244        return Err(ApiError::not_found("image output is disabled"));
2245    }
2246    let output_dir = config.effective_output_dir();
2247    drop(config);
2248
2249    let clean_name = std::path::Path::new(&filename)
2250        .file_name()
2251        .map(|f| f.to_string_lossy().to_string())
2252        .unwrap_or_default();
2253
2254    if clean_name.is_empty() || clean_name != filename {
2255        return Err(ApiError::validation("invalid filename"));
2256    }
2257
2258    let path = output_dir.join(&clean_name);
2259    if path.is_file() {
2260        std::fs::remove_file(&path)
2261            .map_err(|e| ApiError::internal(format!("failed to delete image: {e}")))?;
2262    }
2263
2264    // Also remove server-side thumbnail (both legacy no-suffix and current
2265    // `.png`-suffixed cache layouts) and the animated preview sidecar so
2266    // `/api/gallery/preview/:filename` doesn't keep serving the GIF after
2267    // the source MP4 is gone.
2268    let thumb_dir = server_thumbnail_dir();
2269    let _ = std::fs::remove_file(thumb_dir.join(&clean_name));
2270    let _ = std::fs::remove_file(thumb_dir.join(format!("{clean_name}.png")));
2271    let _ =
2272        std::fs::remove_file(server_preview_gif_dir().join(format!("{clean_name}.preview.gif")));
2273
2274    // Drop the matching metadata row if the DB is enabled. Errors here are
2275    // logged — they don't roll back the disk delete since the file is the
2276    // source of truth and reconciliation will re-sync on the next restart.
2277    if let Some(db) = state.metadata_db.as_ref().as_ref() {
2278        match db.delete(&output_dir, &clean_name) {
2279            Ok(true) => {}
2280            Ok(false) => tracing::debug!(
2281                "delete: no metadata row for {}",
2282                output_dir.join(&clean_name).display()
2283            ),
2284            Err(e) => tracing::warn!(
2285                "metadata DB delete failed for {}: {e:#}",
2286                output_dir.join(&clean_name).display()
2287            ),
2288        }
2289    }
2290
2291    Ok(StatusCode::NO_CONTENT)
2292}
2293
2294/// Serve a thumbnail for a gallery image. Generated on-demand and cached
2295/// at ~/.mold/cache/thumbnails/ on the server side.
2296async fn get_gallery_thumbnail(
2297    State(state): State<AppState>,
2298    axum::extract::Path(filename): axum::extract::Path<String>,
2299) -> Result<impl IntoResponse, ApiError> {
2300    let config = state.config.read().await;
2301    if config.is_output_disabled() {
2302        return Err(ApiError::not_found("image output is disabled"));
2303    }
2304    let output_dir = config.effective_output_dir();
2305    drop(config);
2306
2307    let clean_name = std::path::Path::new(&filename)
2308        .file_name()
2309        .map(|f| f.to_string_lossy().to_string())
2310        .unwrap_or_default();
2311
2312    if clean_name.is_empty() || clean_name != filename {
2313        return Err(ApiError::validation("invalid filename"));
2314    }
2315
2316    let source_path = output_dir.join(&clean_name);
2317    if !source_path.is_file() {
2318        return Err(ApiError::not_found(format!(
2319            "image not found: {clean_name}"
2320        )));
2321    }
2322
2323    // Thumbnail cache path: always `.png` regardless of the source extension,
2324    // so mp4 / gif / apng / webp / jpg all coexist cleanly in the same cache
2325    // dir and `image.save()` doesn't pick the wrong format from the path.
2326    let thumb_dir = server_thumbnail_dir();
2327    let thumb_path = thumb_dir.join(format!("{clean_name}.png"));
2328    let lower = clean_name.to_ascii_lowercase();
2329    let is_video = lower.ends_with(".mp4");
2330
2331    if !thumb_path.is_file() {
2332        // Generate thumbnail on-demand. Videos go through openh264 for a real
2333        // first-frame extract; everything else decodes via the `image` crate.
2334        // If either path fails, we fall back to serving the source bytes
2335        // directly — browsers are more lenient about partial / checksum-
2336        // mismatched images than either decoder, and the SPA would rather
2337        // show something than a 500.
2338        let source = source_path.clone();
2339        let dest = thumb_path.clone();
2340        let gen_result = tokio::task::spawn_blocking(move || {
2341            if is_video {
2342                generate_video_thumbnail(&source, &dest)
2343            } else {
2344                generate_server_thumbnail(&source, &dest)
2345            }
2346        })
2347        .await
2348        .map_err(|e| ApiError::internal(format!("thumbnail generation failed: {e}")))?;
2349
2350        if let Err(err) = gen_result {
2351            tracing::warn!(
2352                file = %clean_name,
2353                error = %err,
2354                "thumbnail decode failed; falling back to source bytes"
2355            );
2356            // For videos, the browser can't render the raw mp4 as an <img>
2357            // either, so serving the source doesn't help — fall back to the
2358            // SVG play-icon placeholder instead.
2359            if is_video {
2360                let mut headers = HeaderMap::new();
2361                headers.insert(
2362                    header::CONTENT_TYPE,
2363                    HeaderValue::from_static("image/svg+xml"),
2364                );
2365                headers.insert(
2366                    header::CACHE_CONTROL,
2367                    HeaderValue::from_static("public, max-age=300"),
2368                );
2369                return Ok((headers, VIDEO_PLACEHOLDER_SVG.as_bytes().to_vec()));
2370            }
2371            let raw = tokio::fs::read(&source_path)
2372                .await
2373                .map_err(|e| ApiError::internal(format!("failed to read source: {e}")))?;
2374            let mut headers = HeaderMap::new();
2375            headers.insert(
2376                header::CONTENT_TYPE,
2377                HeaderValue::from_static(content_type_for_filename(&clean_name)),
2378            );
2379            headers.insert(
2380                header::CACHE_CONTROL,
2381                HeaderValue::from_static("public, max-age=300"),
2382            );
2383            return Ok((headers, raw));
2384        }
2385    }
2386
2387    let data = tokio::fs::read(&thumb_path)
2388        .await
2389        .map_err(|e| ApiError::internal(format!("failed to read thumbnail: {e}")))?;
2390
2391    let mut headers = HeaderMap::new();
2392    headers.insert(header::CONTENT_TYPE, HeaderValue::from_static("image/png"));
2393    headers.insert(
2394        header::CACHE_CONTROL,
2395        HeaderValue::from_static("public, max-age=3600"),
2396    );
2397
2398    Ok((headers, data))
2399}
2400
2401const VIDEO_PLACEHOLDER_SVG: &str = r##"<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 256 256" width="256" height="256"><defs><linearGradient id="g" x1="0" y1="0" x2="1" y2="1"><stop offset="0" stop-color="#1e293b"/><stop offset="1" stop-color="#0f172a"/></linearGradient></defs><rect width="256" height="256" fill="url(#g)"/><circle cx="128" cy="128" r="52" fill="rgba(255,255,255,0.08)"/><polygon points="112,100 112,156 160,128" fill="rgba(226,232,240,0.85)"/></svg>"##;
2402
2403/// Serve a cached animated GIF preview for a gallery video output.
2404///
2405/// Looks up `<preview_dir>/<filename>.preview.gif` (default:
2406/// `~/.mold/cache/previews/`). When present, streams the file back as
2407/// `image/gif`; otherwise returns 404. This exists so the TUI's remote
2408/// gallery detail pane can animate video entries the same way it animates
2409/// local ones — previously it fell through to fetching the raw MP4 over
2410/// `/api/gallery/image/:filename`, which `image::open` couldn't decode,
2411/// leaving the panel on `Loading…` forever.
2412async fn get_gallery_preview(
2413    State(state): State<AppState>,
2414    axum::extract::Path(filename): axum::extract::Path<String>,
2415) -> Result<axum::response::Response, ApiError> {
2416    let config = state.config.read().await;
2417    if config.is_output_disabled() {
2418        return Err(ApiError::not_found("image output is disabled"));
2419    }
2420    let output_dir = config.effective_output_dir();
2421    drop(config);
2422
2423    // Sanitize: prevent directory traversal — the filename must be a bare
2424    // basename with no separators.
2425    let clean_name = std::path::Path::new(&filename)
2426        .file_name()
2427        .map(|f| f.to_string_lossy().to_string())
2428        .unwrap_or_default();
2429    if clean_name.is_empty() || clean_name != filename {
2430        return Err(ApiError::validation("invalid filename"));
2431    }
2432
2433    // The preview cache lifecycle is tied to the underlying gallery file:
2434    // if the MP4 has been deleted (via `DELETE /api/gallery/image/:filename`
2435    // or an out-of-band `rm`), the sidecar may still be on disk but is
2436    // orphaned and must not be served.
2437    // Check the source file first and 404 before touching the cache so a
2438    // stale `.preview.gif` never leaks deleted content.
2439    let source_path = output_dir.join(&clean_name);
2440    if !tokio::fs::metadata(&source_path)
2441        .await
2442        .map(|m| m.is_file())
2443        .unwrap_or(false)
2444    {
2445        return Err(ApiError::not_found(format!(
2446            "image not found: {clean_name}"
2447        )));
2448    }
2449
2450    let preview_path = server_preview_gif_dir().join(format!("{clean_name}.preview.gif"));
2451    let meta = match tokio::fs::metadata(&preview_path).await {
2452        Ok(m) if m.is_file() => m,
2453        _ => {
2454            return Err(ApiError::not_found(format!(
2455                "preview not found: {clean_name}"
2456            )));
2457        }
2458    };
2459    let total_len = meta.len();
2460
2461    let file = tokio::fs::File::open(&preview_path)
2462        .await
2463        .map_err(|e| ApiError::internal(format!("failed to open preview: {e}")))?;
2464    let stream = tokio_util::io::ReaderStream::new(file);
2465    let body = axum::body::Body::from_stream(stream);
2466    Ok(axum::response::Response::builder()
2467        .status(StatusCode::OK)
2468        .header(header::CONTENT_TYPE, "image/gif")
2469        .header(header::CONTENT_LENGTH, total_len)
2470        .header(header::CACHE_CONTROL, "public, max-age=3600")
2471        .body(body)
2472        .unwrap())
2473}
2474
2475/// Server-side GIF preview cache directory. Mirrors the layout the TUI
2476/// writes to (`crates/mold-tui/src/thumbnails.rs::preview_dir`) so a
2477/// single preview.gif authored on either side is reachable via this
2478/// endpoint.
2479fn server_preview_gif_dir() -> std::path::PathBuf {
2480    mold_core::Config::mold_dir()
2481        .unwrap_or_else(|| std::path::PathBuf::from(".mold"))
2482        .join("cache")
2483        .join("previews")
2484}
2485
2486/// Server-side thumbnail cache directory.
2487fn server_thumbnail_dir() -> std::path::PathBuf {
2488    mold_core::Config::mold_dir()
2489        .unwrap_or_else(|| std::path::PathBuf::from(".mold"))
2490        .join("cache")
2491        .join("thumbnails")
2492}
2493
2494/// Generate a 256x256 max thumbnail from source image. The result is always
2495/// written as a PNG regardless of the source format, so callers should pass
2496/// a `.png`-suffixed `dest` to keep the on-disk cache unambiguous.
2497fn generate_server_thumbnail(
2498    source: &std::path::Path,
2499    dest: &std::path::Path,
2500) -> anyhow::Result<()> {
2501    let img = image::open(source)?;
2502    let thumb = img.thumbnail(256, 256);
2503    if let Some(parent) = dest.parent() {
2504        std::fs::create_dir_all(parent)?;
2505    }
2506    thumb.save_with_format(dest, image::ImageFormat::Png)?;
2507    Ok(())
2508}
2509
2510/// Extract the first frame of an MP4 as a PNG thumbnail and downscale to
2511/// 256px max via the `image` crate. Uses the openh264 pipeline that
2512/// `mold_inference::ltx2::media` already ships for video probes.
2513///
2514/// The full-frame PNG is written to a sibling temp path first, then decoded
2515/// and resized — this keeps `mold_inference`'s existing helper surface stable
2516/// while still producing a compact thumbnail.
2517fn generate_video_thumbnail(
2518    source: &std::path::Path,
2519    dest: &std::path::Path,
2520) -> anyhow::Result<()> {
2521    if let Some(parent) = dest.parent() {
2522        std::fs::create_dir_all(parent)?;
2523    }
2524    // Decode the first frame to a temporary full-resolution PNG, then
2525    // thumbnail-resize via the `image` crate. We stage through a temp file
2526    // rather than through memory to reuse `extract_thumbnail`'s existing
2527    // I/O-based API.
2528    let tmp = dest.with_extension("firstframe.png");
2529    mold_inference::ltx2::media::extract_thumbnail(source, &tmp)?;
2530    let decode_result = (|| -> anyhow::Result<()> {
2531        let img = image::open(&tmp)?;
2532        let thumb = img.thumbnail(256, 256);
2533        thumb.save_with_format(dest, image::ImageFormat::Png)?;
2534        Ok(())
2535    })();
2536    let _ = std::fs::remove_file(&tmp);
2537    decode_result
2538}
2539
2540/// Pre-generate thumbnails for all gallery images on server startup.
2541pub fn spawn_thumbnail_warmup(config: &mold_core::Config) {
2542    if !thumbnail_warmup_enabled() {
2543        tracing::info!("thumbnail warmup disabled; thumbnails will be generated on demand");
2544        return;
2545    }
2546
2547    let output_dir = config.effective_output_dir();
2548    std::thread::spawn(move || {
2549        if !output_dir.is_dir() {
2550            return;
2551        }
2552        let thumb_dir = server_thumbnail_dir();
2553        let walker = walkdir::WalkDir::new(&output_dir).max_depth(1).into_iter();
2554        for entry in walker.filter_map(|e| e.ok()) {
2555            let path = entry.path();
2556            if !path.is_file() {
2557                continue;
2558            }
2559            let ext = path
2560                .extension()
2561                .and_then(|e| e.to_str())
2562                .map(|e| e.to_lowercase());
2563            let is_raster = matches!(
2564                ext.as_deref(),
2565                Some("png" | "jpg" | "jpeg" | "gif" | "apng" | "webp")
2566            );
2567            let is_video = matches!(ext.as_deref(), Some("mp4"));
2568            if !is_raster && !is_video {
2569                continue;
2570            }
2571            let filename = path
2572                .file_name()
2573                .map(|f| f.to_string_lossy().to_string())
2574                .unwrap_or_default();
2575            let thumb_path = thumb_dir.join(format!("{filename}.png"));
2576            if thumb_path.is_file() {
2577                continue;
2578            }
2579            let result = if is_video {
2580                generate_video_thumbnail(path, &thumb_path)
2581            } else {
2582                generate_server_thumbnail(path, &thumb_path)
2583            };
2584            if let Err(e) = result {
2585                tracing::warn!("failed to generate thumbnail for {}: {e}", path.display());
2586            }
2587        }
2588        tracing::info!("thumbnail warmup complete");
2589    });
2590}
2591
2592fn thumbnail_warmup_enabled() -> bool {
2593    std::env::var("MOLD_THUMBNAIL_WARMUP")
2594        .map(|v| matches!(v.as_str(), "1" | "true" | "TRUE" | "yes" | "YES"))
2595        .unwrap_or(false)
2596}
2597
2598/// Scan a directory for gallery outputs (images + videos).
2599///
2600/// Picks up every format `OutputFormat` can emit: png / jpg / jpeg / gif /
2601/// apng / webp / mp4. For files with no embedded `mold:parameters` chunk
2602/// (notably gif / webp / mp4), we synthesize a stub `OutputMetadata` from
2603/// the filename so the UI can still display them alongside annotated items.
2604///
2605/// Invalid files are filtered out at scan time rather than surfaced as
2606/// broken tiles in the UI. "Invalid" here means any of:
2607/// - below a format-specific size floor (tiny stubs left by abandoned
2608///   writes, aborted generations, or test harnesses)
2609/// - no decodable image header (raster formats)
2610/// - no `ftyp` box at the start of the file (mp4)
2611///
2612/// This is a header-only validation, not a full pixel decode, so a file
2613/// that passes the check can still be corrupt mid-stream (e.g. broken
2614/// IDAT CRC). Those fall through to the thumbnail endpoint which serves
2615/// the raw bytes as a last resort.
2616fn scan_gallery_dir(dir: &std::path::Path) -> Vec<mold_core::GalleryImage> {
2617    let mut images = Vec::new();
2618
2619    let walker = walkdir::WalkDir::new(dir).max_depth(1).into_iter();
2620    for entry in walker.filter_map(|e| e.ok()) {
2621        let path = entry.path();
2622        if !path.is_file() {
2623            continue;
2624        }
2625
2626        let ext = path
2627            .extension()
2628            .and_then(|e| e.to_str())
2629            .map(|e| e.to_lowercase());
2630        let format = match ext.as_deref() {
2631            Some("png") => Some(mold_core::OutputFormat::Png),
2632            Some("jpg") | Some("jpeg") => Some(mold_core::OutputFormat::Jpeg),
2633            Some("gif") => Some(mold_core::OutputFormat::Gif),
2634            Some("apng") => Some(mold_core::OutputFormat::Apng),
2635            Some("webp") => Some(mold_core::OutputFormat::Webp),
2636            Some("mp4") => Some(mold_core::OutputFormat::Mp4),
2637            _ => None,
2638        };
2639        let Some(format) = format else { continue };
2640
2641        let fs_meta = entry.metadata().ok();
2642        let timestamp = fs_meta
2643            .as_ref()
2644            .and_then(|m| m.modified().ok())
2645            .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
2646            .map(|d| d.as_secs())
2647            .unwrap_or(0);
2648        let size_bytes = fs_meta.as_ref().map(|m| m.len()).unwrap_or(0);
2649
2650        // Size floor: anything below this is guaranteed not a real output.
2651        if size_bytes < min_valid_size(format) {
2652            continue;
2653        }
2654
2655        // Header-level validation (fast, O(1) bytes per file).
2656        let header_ok = match format {
2657            mold_core::OutputFormat::Mp4 => has_ftyp_box(path),
2658            _ => image_header_dims(path).is_some(),
2659        };
2660        if !header_ok {
2661            continue;
2662        }
2663
2664        // Solid-black detection. Only inspect small files where a solid-color
2665        // image is plausible (real renderings at any meaningful resolution
2666        // weigh tens of KB or more). For those, we decode and sample a 16×16
2667        // thumbnail so a failed / empty generation doesn't pollute the feed.
2668        if !matches!(format, mold_core::OutputFormat::Mp4)
2669            && is_probably_solid_black(path, format, size_bytes)
2670        {
2671            continue;
2672        }
2673
2674        let filename = path
2675            .file_name()
2676            .map(|f| f.to_string_lossy().to_string())
2677            .unwrap_or_default();
2678
2679        // Try embedded metadata first — PNG text chunks (also covers APNG
2680        // since APNG files are valid PNGs) and JPEG COM markers.
2681        let embedded = match ext.as_deref() {
2682            Some("png") | Some("apng") => read_png_metadata(path),
2683            Some("jpg") | Some("jpeg") => read_jpeg_metadata(path),
2684            _ => None,
2685        };
2686
2687        let (metadata, synthetic) = match embedded {
2688            Some(m) => (m, false),
2689            None => {
2690                // Synthesize. If the file is a raster whose header decodes,
2691                // use its real dimensions so the UI can render the card at
2692                // the correct aspect ratio even without mold metadata.
2693                let mut meta = synthesize_metadata_from_filename(&filename, timestamp);
2694                if !matches!(format, mold_core::OutputFormat::Mp4) {
2695                    if let Some((w, h)) = image_header_dims(path) {
2696                        meta.width = w;
2697                        meta.height = h;
2698                    }
2699                }
2700                (meta, true)
2701            }
2702        };
2703
2704        images.push(mold_core::GalleryImage {
2705            filename,
2706            metadata,
2707            timestamp,
2708            format: Some(format),
2709            size_bytes: Some(size_bytes),
2710            metadata_synthetic: synthetic,
2711        });
2712    }
2713
2714    images.sort_by_key(|img| std::cmp::Reverse(img.timestamp));
2715    images
2716}
2717
2718/// Minimum on-disk size (in bytes) below which a file is treated as a
2719/// corrupt / aborted output and hidden from the gallery listing. The
2720/// thresholds are well below any real mold-generated output but above any
2721/// parseable-but-empty stub — a 1×1 pixel PNG is ~67 bytes, a real 512×512
2722/// PNG is at least tens of KB.
2723fn min_valid_size(format: mold_core::OutputFormat) -> u64 {
2724    match format {
2725        // Raster images: any real mold output is multi-KB. The 256-byte
2726        // floor comfortably filters truncated PNG stubs (signature + IHDR
2727        // only, ~45 bytes) and similar degenerate cases without touching
2728        // legitimate tiny gifs (e.g. a few hundred bytes for a 1-frame GIF).
2729        mold_core::OutputFormat::Png
2730        | mold_core::OutputFormat::Apng
2731        | mold_core::OutputFormat::Jpeg
2732        | mold_core::OutputFormat::Webp => 256,
2733        mold_core::OutputFormat::Gif => 128,
2734        // An mp4 with a single frame and no audio is still many KB.
2735        mold_core::OutputFormat::Mp4 => 4096,
2736    }
2737}
2738
2739/// Fast "does this decode as an image?" check. Returns the image's
2740/// pixel dimensions (width, height) on success. Only reads the header —
2741/// typically under 1 KB — so it's safe to call for every file on every
2742/// `/api/gallery` request.
2743fn image_header_dims(path: &std::path::Path) -> Option<(u32, u32)> {
2744    image::ImageReader::open(path)
2745        .ok()?
2746        .with_guessed_format()
2747        .ok()?
2748        .into_dimensions()
2749        .ok()
2750}
2751
2752/// Heuristic detector for "solid black" (or near-black) raster images —
2753/// typically the artefact of an aborted / NaN-poisoned generation that
2754/// wrote an all-zero image tensor. We only even consider images below a
2755/// format-specific suspect size (any real content at meaningful resolution
2756/// compresses to tens of KB at minimum; a solid-color PNG fits in a few
2757/// hundred bytes), then decode and sample a 16×16 thumbnail to check
2758/// whether any pixel's max channel exceeds a small threshold.
2759fn is_probably_solid_black(
2760    path: &std::path::Path,
2761    format: mold_core::OutputFormat,
2762    size_bytes: u64,
2763) -> bool {
2764    const SAMPLE_DIM: u32 = 16;
2765    // Allow any single channel up to this intensity before we conclude the
2766    // file is "real" content. 16 out of 255 is ~6%: enough to accept dark
2767    // images that aren't literal black, but tight enough to reject the
2768    // artefacts we actually want to filter.
2769    const CHANNEL_CEILING: u8 = 16;
2770
2771    let suspect_threshold: u64 = match format {
2772        // PNG / APNG: zlib-compressed raw pixels; 8 KB is comfortably above
2773        // any solid-color encoding at 1k-ish resolution.
2774        mold_core::OutputFormat::Png | mold_core::OutputFormat::Apng => 8 * 1024,
2775        // JPEG compresses solid color to a few hundred bytes; generous ceiling.
2776        mold_core::OutputFormat::Jpeg => 4 * 1024,
2777        mold_core::OutputFormat::Gif | mold_core::OutputFormat::Webp => 4 * 1024,
2778        mold_core::OutputFormat::Mp4 => return false,
2779    };
2780    if size_bytes > suspect_threshold {
2781        return false;
2782    }
2783
2784    let Ok(img) = image::open(path) else {
2785        return false;
2786    };
2787    let thumb = img.thumbnail(SAMPLE_DIM, SAMPLE_DIM).to_rgb8();
2788    let mut max_channel: u8 = 0;
2789    for pixel in thumb.pixels() {
2790        let m = pixel.0[0].max(pixel.0[1]).max(pixel.0[2]);
2791        if m > max_channel {
2792            max_channel = m;
2793        }
2794        if max_channel > CHANNEL_CEILING {
2795            return false;
2796        }
2797    }
2798    max_channel <= CHANNEL_CEILING
2799}
2800
2801/// Check for the ISO BMFF `ftyp` box at offset 4 of the file. A real mp4
2802/// always starts with a top-level `ftyp` box; files that fail this check
2803/// are typically truncated writes or wrong-extension text files.
2804fn has_ftyp_box(path: &std::path::Path) -> bool {
2805    use std::io::Read;
2806    let Ok(mut f) = std::fs::File::open(path) else {
2807        return false;
2808    };
2809    let mut buf = [0u8; 12];
2810    if f.read_exact(&mut buf).is_err() {
2811        return false;
2812    }
2813    &buf[4..8] == b"ftyp"
2814}
2815
2816/// Build a best-effort `OutputMetadata` from a filename like
2817/// `mold-<model>-<unix>[-<idx>].<ext>`. Fields we can't recover (seed, steps,
2818/// guidance, resolution, prompt) are left at zero / empty so the UI can
2819/// render them as "unknown". The client reads `metadata_synthetic=true`
2820/// from the enclosing `GalleryImage` to treat these as placeholders.
2821fn synthesize_metadata_from_filename(filename: &str, timestamp: u64) -> mold_core::OutputMetadata {
2822    let stem = std::path::Path::new(filename)
2823        .file_stem()
2824        .and_then(|s| s.to_str())
2825        .unwrap_or("");
2826
2827    let model = stem
2828        .strip_prefix("mold-")
2829        .and_then(|rest| {
2830            // Trim trailing `-<unix>` and optional `-<idx>` suffixes by
2831            // walking back across numeric segments.
2832            let mut parts: Vec<&str> = rest.split('-').collect();
2833            while parts
2834                .last()
2835                .map(|p| p.chars().all(|c| c.is_ascii_digit()))
2836                .unwrap_or(false)
2837                && parts.len() > 1
2838            {
2839                parts.pop();
2840            }
2841            if parts.is_empty() {
2842                None
2843            } else {
2844                Some(parts.join("-"))
2845            }
2846        })
2847        .unwrap_or_else(|| "unknown".to_string());
2848
2849    mold_core::OutputMetadata {
2850        prompt: String::new(),
2851        negative_prompt: None,
2852        original_prompt: None,
2853        model,
2854        seed: 0,
2855        steps: 0,
2856        guidance: 0.0,
2857        width: 0,
2858        height: 0,
2859        strength: None,
2860        scheduler: None,
2861        output_format: None,
2862        cfg_plus: None,
2863        lora: None,
2864        lora_scale: None,
2865        loras: None,
2866        control_model: None,
2867        control_scale: None,
2868        upscale_model: None,
2869        gif_preview: None,
2870        enable_audio: None,
2871        audio_file_path: None,
2872        source_video_path: None,
2873        pipeline: None,
2874        retake_range: None,
2875        spatial_upscale: None,
2876        temporal_upscale: None,
2877        frames: None,
2878        fps: None,
2879        version: format!("synthesized@{timestamp}"),
2880    }
2881}
2882
2883/// Read OutputMetadata from a PNG file's text chunks.
2884fn read_png_metadata(path: &std::path::Path) -> Option<mold_core::OutputMetadata> {
2885    let file = std::fs::File::open(path).ok()?;
2886    let decoder = png::Decoder::new(std::io::BufReader::new(file));
2887    let reader = decoder.read_info().ok()?;
2888    let info = reader.info();
2889
2890    for chunk in &info.uncompressed_latin1_text {
2891        if chunk.keyword == "mold:parameters" {
2892            if let Ok(meta) = serde_json::from_str::<mold_core::OutputMetadata>(&chunk.text) {
2893                return Some(meta);
2894            }
2895        }
2896    }
2897    for chunk in &info.utf8_text {
2898        if chunk.keyword == "mold:parameters" {
2899            if let Ok(text) = chunk.get_text() {
2900                if let Ok(meta) = serde_json::from_str::<mold_core::OutputMetadata>(&text) {
2901                    return Some(meta);
2902                }
2903            }
2904        }
2905    }
2906    None
2907}
2908
2909/// Read OutputMetadata from a JPEG file's COM marker.
2910fn read_jpeg_metadata(path: &std::path::Path) -> Option<mold_core::OutputMetadata> {
2911    let data = std::fs::read(path).ok()?;
2912    let mut i = 0;
2913    while i + 1 < data.len() {
2914        if data[i] != 0xFF {
2915            i += 1;
2916            continue;
2917        }
2918        let marker = data[i + 1];
2919        match marker {
2920            // Standalone markers (no length field): SOI, EOI, RST0-7, TEM
2921            0xD8 | 0x01 => {
2922                i += 2;
2923            }
2924            0xD9 => break, // EOI — end of image
2925            0xD0..=0xD7 => {
2926                i += 2; // RST markers
2927            }
2928            // COM marker — check for mold:parameters
2929            0xFE => {
2930                if i + 3 >= data.len() {
2931                    break;
2932                }
2933                let len = u16::from_be_bytes([data[i + 2], data[i + 3]]) as usize;
2934                if len < 2 || i + 2 + len > data.len() {
2935                    break;
2936                }
2937                let comment = &data[i + 4..i + 2 + len];
2938                if let Ok(text) = std::str::from_utf8(comment) {
2939                    if let Some(json) = text.strip_prefix("mold:parameters ") {
2940                        if let Ok(meta) = serde_json::from_str::<mold_core::OutputMetadata>(json) {
2941                            return Some(meta);
2942                        }
2943                    }
2944                }
2945                i += 2 + len;
2946            }
2947            // All other markers have a 2-byte length field
2948            _ => {
2949                if i + 3 >= data.len() {
2950                    break;
2951                }
2952                let len = u16::from_be_bytes([data[i + 2], data[i + 3]]) as usize;
2953                if len < 2 || i + 2 + len > data.len() {
2954                    break;
2955                }
2956                i += 2 + len;
2957            }
2958        }
2959    }
2960    None
2961}
2962
2963// ── /api/config/model/:name/placement (Agent C, model-ui-overhaul §3) ────────
2964
2965async fn put_model_placement(
2966    State(state): State<AppState>,
2967    axum::extract::Path(name): axum::extract::Path<String>,
2968    Json(placement): Json<mold_core::types::DevicePlacement>,
2969) -> Result<Json<serde_json::Value>, ApiError> {
2970    validate_multi_gpu_placement(&state, Some(&placement))?;
2971    {
2972        let mut cfg = state.config.write().await;
2973        cfg.set_model_placement(&name, Some(placement.clone()));
2974        cfg.save().map_err(|e| {
2975            tracing::warn!("failed to persist placement to config.toml: {e}");
2976            ApiError::internal(format!("failed to persist placement to config.toml: {e}"))
2977        })?;
2978    }
2979    Ok(Json(serde_json::json!({
2980        "ok": true,
2981        "model": name,
2982    })))
2983}
2984
2985async fn delete_model_placement(
2986    State(state): State<AppState>,
2987    axum::extract::Path(name): axum::extract::Path<String>,
2988) -> Result<Json<serde_json::Value>, ApiError> {
2989    let mut cfg = state.config.write().await;
2990    cfg.set_model_placement(&name, None);
2991    cfg.save().map_err(|e| {
2992        tracing::warn!("failed to persist placement removal to config.toml: {e}");
2993        ApiError::internal(format!(
2994            "failed to persist placement removal to config.toml: {e}"
2995        ))
2996    })?;
2997    Ok(Json(serde_json::json!({ "ok": true })))
2998}
2999
3000// ── /api/openapi.json ─────────────────────────────────────────────────────────
3001
3002async fn openapi_json() -> impl IntoResponse {
3003    Json(ApiDoc::openapi())
3004}
3005
3006// ── /api/docs ─────────────────────────────────────────────────────────────────
3007
3008async fn scalar_docs() -> impl IntoResponse {
3009    (
3010        [(header::CONTENT_TYPE, "text/html")],
3011        r#"<!DOCTYPE html>
3012<html>
3013<head>
3014  <title>mold API</title>
3015  <meta charset="utf-8" />
3016  <meta name="viewport" content="width=device-width, initial-scale=1" />
3017</head>
3018<body>
3019  <script id="api-reference" data-url="/api/openapi.json"></script>
3020  <script src="https://cdn.jsdelivr.net/npm/@scalar/api-reference"></script>
3021</body>
3022</html>"#,
3023    )
3024}
3025
3026// ── GPU info ──────────────────────────────────────────────────────────────────
3027
3028fn query_gpu_info() -> Option<GpuInfo> {
3029    let nvidia_smi = if std::path::Path::new("/run/current-system/sw/bin/nvidia-smi").exists() {
3030        "/run/current-system/sw/bin/nvidia-smi"
3031    } else {
3032        "nvidia-smi"
3033    };
3034
3035    let output = std::process::Command::new(nvidia_smi)
3036        .args([
3037            "--query-gpu=name,memory.total,memory.used",
3038            "--format=csv,noheader,nounits",
3039        ])
3040        .output()
3041        .ok()?;
3042
3043    if !output.status.success() {
3044        return None;
3045    }
3046
3047    let text = String::from_utf8(output.stdout).ok()?;
3048    let line = text.lines().next()?;
3049    let parts: Vec<&str> = line.split(',').map(str::trim).collect();
3050    if parts.len() < 3 {
3051        return None;
3052    }
3053
3054    Some(GpuInfo {
3055        name: parts[0].to_string(),
3056        vram_total_mb: parts[1].parse().ok()?,
3057        vram_used_mb: parts[2].parse().ok()?,
3058    })
3059}
3060
3061// ─── Downloads UI (Agent A) ──────────────────────────────────────────────────
3062
3063#[derive(serde::Deserialize, utoipa::ToSchema)]
3064pub struct CreateDownloadBody {
3065    pub model: String,
3066}
3067
3068#[derive(serde::Serialize, utoipa::ToSchema)]
3069pub struct CreateDownloadResponse {
3070    pub id: String,
3071    pub position: usize,
3072}
3073
3074#[utoipa::path(
3075    post,
3076    path = "/api/downloads",
3077    tag = "downloads",
3078    request_body = CreateDownloadBody,
3079    responses(
3080        (status = 200, description = "Enqueued; position 0 = will start immediately", body = CreateDownloadResponse),
3081        (status = 400, description = "Unknown model"),
3082        (status = 409, description = "Already active or queued; body contains existing id", body = CreateDownloadResponse),
3083    )
3084)]
3085pub async fn create_download(
3086    State(state): State<AppState>,
3087    Json(body): Json<CreateDownloadBody>,
3088) -> axum::response::Response {
3089    use crate::downloads::{EnqueueError, EnqueueOutcome};
3090    match state.downloads.enqueue(body.model.clone()).await {
3091        Ok((id, position, EnqueueOutcome::Created)) => (
3092            StatusCode::OK,
3093            Json(CreateDownloadResponse { id, position }),
3094        )
3095            .into_response(),
3096        Ok((id, position, EnqueueOutcome::AlreadyPresent)) => (
3097            StatusCode::CONFLICT,
3098            Json(CreateDownloadResponse { id, position }),
3099        )
3100            .into_response(),
3101        Err(EnqueueError::UnknownModel(_)) => (
3102            StatusCode::BAD_REQUEST,
3103            Json(serde_json::json!({
3104                "error": format!("unknown model '{}'. Run 'mold list' to see available models.", body.model)
3105            })),
3106        )
3107            .into_response(),
3108        Err(EnqueueError::LockPoisoned) => (
3109            StatusCode::INTERNAL_SERVER_ERROR,
3110            Json(serde_json::json!({ "error": "download queue state is corrupt" })),
3111        )
3112            .into_response(),
3113    }
3114}
3115
3116#[utoipa::path(
3117    delete,
3118    path = "/api/downloads/{id}",
3119    tag = "downloads",
3120    params(("id" = String, Path, description = "Job id")),
3121    responses(
3122        (status = 204, description = "Cancelled"),
3123        (status = 404, description = "Unknown id"),
3124    )
3125)]
3126pub async fn delete_download(
3127    State(state): State<AppState>,
3128    axum::extract::Path(id): axum::extract::Path<String>,
3129) -> axum::response::Response {
3130    if state.downloads.cancel(&id).await {
3131        StatusCode::NO_CONTENT.into_response()
3132    } else {
3133        (
3134            StatusCode::NOT_FOUND,
3135            Json(serde_json::json!({ "error": format!("unknown download id '{id}'") })),
3136        )
3137            .into_response()
3138    }
3139}
3140
3141#[utoipa::path(
3142    get,
3143    path = "/api/downloads",
3144    tag = "downloads",
3145    responses((status = 200, description = "Current queue state"))
3146)]
3147pub async fn list_downloads(State(state): State<AppState>) -> axum::response::Response {
3148    Json(state.downloads.listing().await).into_response()
3149}
3150
3151#[utoipa::path(
3152    get,
3153    path = "/api/downloads/stream",
3154    tag = "downloads",
3155    responses((status = 200, description = "SSE stream of DownloadEvent JSON")),
3156)]
3157pub async fn stream_downloads(
3158    State(state): State<AppState>,
3159) -> Sse<
3160    impl futures_core::Stream<Item = Result<axum::response::sse::Event, std::convert::Infallible>>,
3161> {
3162    use axum::response::sse::Event;
3163    use tokio_stream::wrappers::BroadcastStream;
3164    use tokio_stream::StreamExt as _;
3165
3166    // Subscribe BEFORE snapshotting so any event arriving during the
3167    // snapshot read is queued in the broadcast channel instead of being
3168    // missed. The first frame we yield is `Snapshot { listing }` —
3169    // mirrors `/api/resources/stream`'s initial-snapshot pattern so a
3170    // freshly-mounted SPA paints current state without waiting for the
3171    // next delta.
3172    let rx = state.downloads.subscribe();
3173    let initial = state.downloads.listing().await;
3174    let snapshot_event = mold_core::types::DownloadEvent::Snapshot { listing: initial };
3175
3176    let stream = async_stream::stream! {
3177        let data = serde_json::to_string(&snapshot_event).unwrap_or_else(|_| "{}".to_string());
3178        yield Ok::<_, std::convert::Infallible>(Event::default().event("download").data(data));
3179
3180        let mut bs = BroadcastStream::new(rx);
3181        while let Some(item) = bs.next().await {
3182            match item {
3183                Ok(event) => {
3184                    let data = serde_json::to_string(&event)
3185                        .unwrap_or_else(|_| "{}".to_string());
3186                    yield Ok(Event::default().event("download").data(data));
3187                }
3188                // Slow subscribers see lag silently; the snapshot above
3189                // already carries the full state so we don't need to
3190                // resync on every drop.
3191                Err(_lagged) => continue,
3192            }
3193        }
3194    };
3195
3196    Sse::new(stream).keep_alive(
3197        KeepAlive::new()
3198            .interval(std::time::Duration::from_secs(15))
3199            .text("ping"),
3200    )
3201}
3202
3203// ── Resource telemetry (Agent B scope) ───────────────────────────────────────
3204
3205/// `GET /api/resources` — one-shot JSON snapshot from the aggregator cache.
3206/// Returns 503 if the aggregator has not yet fired (first 1 s after startup
3207/// and before `spawn_aggregator` has run).
3208async fn get_resources(State(state): State<AppState>) -> Result<Json<ResourceSnapshot>, ApiError> {
3209    match state.resources.latest() {
3210        Some(snap) => Ok(Json(snap)),
3211        None => Err(ApiError::internal_with_status(
3212            "resource telemetry not ready",
3213            StatusCode::SERVICE_UNAVAILABLE,
3214        )),
3215    }
3216}
3217
3218/// `GET /api/resources/stream` — SSE stream of `ResourceSnapshot` frames.
3219/// Event name: `snapshot`. Matches the keepalive cadence of `/api/generate/stream`.
3220async fn get_resources_stream(
3221    State(state): State<AppState>,
3222) -> Sse<impl futures_core::Stream<Item = Result<SseEvent, Infallible>>> {
3223    use tokio_stream::wrappers::BroadcastStream;
3224
3225    let rx = state.resources.subscribe();
3226    // Attach the cached `latest` snapshot as the first frame so clients
3227    // don't wait up to one full tick for their initial value.
3228    let initial = state.resources.latest();
3229
3230    let stream = async_stream::stream! {
3231        if let Some(snap) = initial {
3232            yield Ok::<_, Infallible>(snapshot_to_sse(&snap));
3233        }
3234        let mut bs = BroadcastStream::new(rx);
3235        while let Some(item) = bs.next().await {
3236            match item {
3237                Ok(snap) => yield Ok(snapshot_to_sse(&snap)),
3238                // Lag is normal for slow clients — skip dropped frames
3239                // silently; the next one will catch them up.
3240                Err(_lagged) => continue,
3241            }
3242        }
3243    };
3244
3245    Sse::new(stream).keep_alive(
3246        KeepAlive::new()
3247            .interval(std::time::Duration::from_secs(15))
3248            .text("ping"),
3249    )
3250}
3251
3252fn snapshot_to_sse(snap: &ResourceSnapshot) -> SseEvent {
3253    match serde_json::to_string(snap) {
3254        Ok(data) => SseEvent::default().event("snapshot").data(data),
3255        Err(e) => SseEvent::default()
3256            .event("error")
3257            .data(format!("{{\"message\":\"serialize failed: {e}\"}}")),
3258    }
3259}
3260
3261#[cfg(test)]
3262mod tests {
3263    use super::*;
3264
3265    fn env_lock() -> &'static std::sync::Mutex<()> {
3266        static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
3267        &ENV_LOCK
3268    }
3269
3270    #[test]
3271    fn clean_error_message_strips_backtrace() {
3272        let err = anyhow::anyhow!(
3273            "DriverError(CUDA_ERROR_OUT_OF_MEMORY, \"out of memory\")\n\
3274             \x20  0: candle_core::error::Error::bt\n\
3275             \x20           at /home/user/.cargo/git/candle/src/error.rs:264:25\n\
3276             \x20  1: <core::result::Result<O,E> as candle_core::cuda_backend::error::WrapErr<O>>::w\n\
3277             \x20           at /home/user/.cargo/git/candle/src/cuda_backend/error.rs:60:65"
3278        );
3279        let msg = clean_error_message(&err);
3280        assert_eq!(
3281            msg,
3282            "DriverError(CUDA_ERROR_OUT_OF_MEMORY, \"out of memory\")"
3283        );
3284    }
3285
3286    #[test]
3287    fn clean_error_message_preserves_simple_error() {
3288        let err = anyhow::anyhow!("model not found: flux-dev:q4");
3289        let msg = clean_error_message(&err);
3290        assert_eq!(msg, "model not found: flux-dev:q4");
3291    }
3292
3293    #[test]
3294    fn clean_error_message_preserves_multiline_without_backtrace() {
3295        let err = anyhow::anyhow!("validation failed\nprompt is empty\nsteps must be > 0");
3296        let msg = clean_error_message(&err);
3297        assert_eq!(msg, "validation failed\nprompt is empty\nsteps must be > 0");
3298    }
3299
3300    #[test]
3301    fn clean_error_message_strips_high_numbered_frames() {
3302        let err = anyhow::anyhow!(
3303            "some error\n\
3304             \x20 10: tokio::runtime::task::core::Core<T,S>::poll at /home/user/.cargo/tokio/src/core.rs:375\n\
3305             \x20 11: std::panicking::catch_unwind at /nix/store/rust/src/panicking.rs:544"
3306        );
3307        let msg = clean_error_message(&err);
3308        assert_eq!(msg, "some error");
3309    }
3310
3311    #[test]
3312    fn clean_error_message_empty_fallback() {
3313        // An error whose Display starts immediately with a backtrace-like line
3314        let err = anyhow::anyhow!("0: candle_core::error::Error::bt at /some/path.rs:10:5");
3315        let msg = clean_error_message(&err);
3316        // Should fall back to root_cause since all lines look like backtrace
3317        assert!(!msg.is_empty());
3318    }
3319
3320    #[test]
3321    fn clean_error_message_renders_full_anyhow_chain() {
3322        // Wrapped errors must surface the root cause; previously the outer
3323        // `with_context` swallowed everything below it (cv:2739091 truncated
3324        // checkpoint surfaced as "mmap single-file checkpoint at …" with no
3325        // hint that the safetensors data was short).
3326        let root = std::io::Error::new(std::io::ErrorKind::InvalidData, "bytes past end");
3327        let err: anyhow::Error = anyhow::Error::new(root)
3328            .context("validate single-file checkpoint at /tmp/foo.safetensors");
3329        let msg = clean_error_message(&err);
3330        assert!(
3331            msg.contains("validate single-file checkpoint") && msg.contains("bytes past end"),
3332            "expected both context layers in the rendered chain, got: {msg}",
3333        );
3334    }
3335
3336    #[test]
3337    fn save_image_to_dir_creates_directory_and_writes_file() {
3338        let dir = std::env::temp_dir().join(format!(
3339            "mold-save-test-{}",
3340            std::time::SystemTime::now()
3341                .duration_since(std::time::UNIX_EPOCH)
3342                .unwrap()
3343                .as_nanos()
3344        ));
3345        assert!(!dir.exists());
3346
3347        let img = mold_core::ImageData {
3348            data: vec![0x89, 0x50, 0x4E, 0x47], // PNG magic bytes
3349            format: mold_core::OutputFormat::Png,
3350            width: 64,
3351            height: 64,
3352            index: 0,
3353        };
3354
3355        save_image_to_dir(&dir, &img, "test-model:q8", 1);
3356
3357        assert!(dir.exists(), "directory should be created");
3358        let files: Vec<_> = std::fs::read_dir(&dir).unwrap().collect();
3359        assert_eq!(files.len(), 1, "should have exactly one file");
3360        let file = files[0].as_ref().unwrap();
3361        let filename = file.file_name().to_str().unwrap().to_string();
3362        assert!(filename.starts_with("mold-test-model-q8-"), "{filename}");
3363        assert!(filename.ends_with(".png"), "{filename}");
3364        let contents = std::fs::read(file.path()).unwrap();
3365        assert_eq!(contents, vec![0x89, 0x50, 0x4E, 0x47]);
3366
3367        std::fs::remove_dir_all(&dir).ok();
3368    }
3369
3370    #[test]
3371    fn save_image_to_dir_batch_includes_index() {
3372        let dir = std::env::temp_dir().join(format!(
3373            "mold-save-batch-{}",
3374            std::time::SystemTime::now()
3375                .duration_since(std::time::UNIX_EPOCH)
3376                .unwrap()
3377                .as_nanos()
3378        ));
3379
3380        let img = mold_core::ImageData {
3381            data: vec![0xFF, 0xD8], // JPEG magic
3382            format: mold_core::OutputFormat::Jpeg,
3383            width: 64,
3384            height: 64,
3385            index: 2,
3386        };
3387
3388        save_image_to_dir(&dir, &img, "flux-dev", 4);
3389
3390        let files: Vec<_> = std::fs::read_dir(&dir).unwrap().collect();
3391        assert_eq!(files.len(), 1);
3392        let filename = files[0]
3393            .as_ref()
3394            .unwrap()
3395            .file_name()
3396            .to_str()
3397            .unwrap()
3398            .to_string();
3399        assert!(
3400            filename.contains("-2.jpeg"),
3401            "batch index in name: {filename}"
3402        );
3403
3404        std::fs::remove_dir_all(&dir).ok();
3405    }
3406
3407    #[test]
3408    fn save_image_to_dir_invalid_path_logs_warning_no_panic() {
3409        // Saving to a path that can't be created should not panic
3410        let img = mold_core::ImageData {
3411            data: vec![0x00],
3412            format: mold_core::OutputFormat::Png,
3413            width: 1,
3414            height: 1,
3415            index: 0,
3416        };
3417        // /dev/null/impossible can't be created as a directory
3418        save_image_to_dir(
3419            std::path::Path::new("/dev/null/impossible"),
3420            &img,
3421            "test",
3422            1,
3423        );
3424        // Test passes if no panic occurred
3425    }
3426
3427    #[test]
3428    fn thumbnail_warmup_is_disabled_by_default() {
3429        let _guard = env_lock().lock().unwrap();
3430        unsafe {
3431            std::env::remove_var("MOLD_THUMBNAIL_WARMUP");
3432        }
3433        assert!(!thumbnail_warmup_enabled());
3434    }
3435
3436    #[test]
3437    fn thumbnail_warmup_accepts_truthy_env_values() {
3438        let _guard = env_lock().lock().unwrap();
3439        unsafe {
3440            std::env::set_var("MOLD_THUMBNAIL_WARMUP", "1");
3441        }
3442        assert!(thumbnail_warmup_enabled());
3443        unsafe {
3444            std::env::set_var("MOLD_THUMBNAIL_WARMUP", "true");
3445        }
3446        assert!(thumbnail_warmup_enabled());
3447        unsafe {
3448            std::env::set_var("MOLD_THUMBNAIL_WARMUP", "YES");
3449        }
3450        assert!(thumbnail_warmup_enabled());
3451        unsafe {
3452            std::env::remove_var("MOLD_THUMBNAIL_WARMUP");
3453        }
3454    }
3455
3456    #[test]
3457    fn thumbnail_warmup_rejects_falsey_env_values() {
3458        let _guard = env_lock().lock().unwrap();
3459        unsafe {
3460            std::env::set_var("MOLD_THUMBNAIL_WARMUP", "0");
3461        }
3462        assert!(!thumbnail_warmup_enabled());
3463        unsafe {
3464            std::env::set_var("MOLD_THUMBNAIL_WARMUP", "false");
3465        }
3466        assert!(!thumbnail_warmup_enabled());
3467        unsafe {
3468            std::env::remove_var("MOLD_THUMBNAIL_WARMUP");
3469        }
3470    }
3471
3472    #[test]
3473    fn content_type_covers_every_output_format() {
3474        assert_eq!(content_type_for_filename("a.png"), "image/png");
3475        assert_eq!(content_type_for_filename("a.PNG"), "image/png");
3476        assert_eq!(content_type_for_filename("a.jpg"), "image/jpeg");
3477        assert_eq!(content_type_for_filename("a.jpeg"), "image/jpeg");
3478        assert_eq!(content_type_for_filename("a.gif"), "image/gif");
3479        assert_eq!(content_type_for_filename("a.webp"), "image/webp");
3480        assert_eq!(content_type_for_filename("a.apng"), "image/apng");
3481        assert_eq!(content_type_for_filename("a.mp4"), "video/mp4");
3482        assert_eq!(
3483            content_type_for_filename("a.unknown"),
3484            "application/octet-stream"
3485        );
3486    }
3487
3488    #[test]
3489    fn synthesized_metadata_parses_model_from_filename() {
3490        let meta = synthesize_metadata_from_filename("mold-flux-dev-q8-1710000000.mp4", 1710000000);
3491        // Trailing unix timestamp should be stripped; model tag preserved.
3492        assert_eq!(meta.model, "flux-dev-q8");
3493        assert_eq!(meta.prompt, "");
3494        assert_eq!(meta.seed, 0);
3495        assert!(meta.version.starts_with("synthesized@"));
3496
3497        // Batch suffix (trailing `-<idx>`) also stripped along with timestamp.
3498        let meta =
3499            synthesize_metadata_from_filename("mold-ltx-video-bf16-1710000030-2.gif", 1710000030);
3500        assert_eq!(meta.model, "ltx-video-bf16");
3501
3502        // Non-mold filename falls back to "unknown".
3503        let meta = synthesize_metadata_from_filename("unrelated.png", 0);
3504        assert_eq!(meta.model, "unknown");
3505    }
3506
3507    // ── Gallery validation ───────────────────────────────────────────────
3508
3509    /// Create a scratch directory unique to this test and delete it on drop.
3510    /// Using `std::env::temp_dir()` rather than pulling in a `tempfile`
3511    /// dev-dep for three tests' worth of fixtures.
3512    struct TempDir(std::path::PathBuf);
3513    impl TempDir {
3514        fn new(tag: &str) -> Self {
3515            let mut p = std::env::temp_dir();
3516            p.push(format!("mold-gallery-test-{tag}-{}", uuid::Uuid::new_v4()));
3517            std::fs::create_dir_all(&p).expect("create tempdir");
3518            Self(p)
3519        }
3520        fn path(&self) -> &std::path::Path {
3521            &self.0
3522        }
3523    }
3524    impl Drop for TempDir {
3525        fn drop(&mut self) {
3526            let _ = std::fs::remove_dir_all(&self.0);
3527        }
3528    }
3529
3530    /// Encode a noisy PNG in-memory via the `image` crate. The checkerboard
3531    /// pattern resists zlib compression so the encoded bytes exceed the
3532    /// gallery size floor — a solid-color PNG of the same dimensions would
3533    /// compress to ~80 bytes and be filtered out by `min_valid_size`.
3534    fn make_png_bytes(width: u32, height: u32) -> Vec<u8> {
3535        let img = image::RgbImage::from_fn(width, height, |x, y| {
3536            let n = (x.wrapping_mul(37) ^ y.wrapping_mul(131)) as u8;
3537            image::Rgb([n, n.wrapping_add(85), n.wrapping_sub(17)])
3538        });
3539        let mut buf = Vec::new();
3540        image::DynamicImage::ImageRgb8(img)
3541            .write_to(&mut std::io::Cursor::new(&mut buf), image::ImageFormat::Png)
3542            .expect("encode png");
3543        buf
3544    }
3545
3546    #[test]
3547    fn min_valid_size_thresholds_are_sensible() {
3548        // Raster formats: high enough to catch every truncated / stub file
3549        // we've seen in dev harnesses (largest known bad fixture was ~200 B).
3550        assert!(min_valid_size(mold_core::OutputFormat::Png) >= 128);
3551        assert!(min_valid_size(mold_core::OutputFormat::Jpeg) >= 128);
3552        assert!(min_valid_size(mold_core::OutputFormat::Apng) >= 128);
3553        assert!(min_valid_size(mold_core::OutputFormat::Webp) >= 128);
3554        // But not so high we filter out legitimate tiny GIFs.
3555        assert!(min_valid_size(mold_core::OutputFormat::Gif) <= 512);
3556        // MP4: no real rendering is < 1 KB; 4 KB is a comfortable floor.
3557        assert!(min_valid_size(mold_core::OutputFormat::Mp4) >= 1024);
3558    }
3559
3560    #[test]
3561    fn has_ftyp_box_accepts_real_header_and_rejects_garbage() {
3562        let td = TempDir::new("ftyp");
3563
3564        // Real-ish mp4 header: `\0\0\0\x20 ftypisom ...`
3565        let mut real = Vec::new();
3566        real.extend_from_slice(&[0x00, 0x00, 0x00, 0x20]);
3567        real.extend_from_slice(b"ftyp");
3568        real.extend_from_slice(b"isom\x00\x00\x02\x00isomiso2mp41");
3569        let real_path = td.path().join("real.mp4");
3570        std::fs::write(&real_path, &real).unwrap();
3571        assert!(has_ftyp_box(&real_path));
3572
3573        // Wrong magic — random text with an mp4 extension.
3574        let fake_path = td.path().join("fake.mp4");
3575        std::fs::write(&fake_path, b"this is not an mp4 file at all").unwrap();
3576        assert!(!has_ftyp_box(&fake_path));
3577
3578        // Too short (fewer than 12 bytes) — can't contain an ftyp box.
3579        let trunc_path = td.path().join("truncated.mp4");
3580        std::fs::write(&trunc_path, b"\x00\x00\x00\x20").unwrap();
3581        assert!(!has_ftyp_box(&trunc_path));
3582
3583        // Missing entirely.
3584        assert!(!has_ftyp_box(&td.path().join("nope.mp4")));
3585    }
3586
3587    #[test]
3588    fn image_header_dims_returns_real_dimensions() {
3589        let td = TempDir::new("header");
3590        let p = td.path().join("valid.png");
3591        std::fs::write(&p, make_png_bytes(42, 24)).unwrap();
3592        assert_eq!(image_header_dims(&p), Some((42, 24)));
3593
3594        // Truncated: PNG signature only, no IHDR.
3595        let stub = td.path().join("stub.png");
3596        std::fs::write(&stub, b"\x89PNG\r\n\x1a\n").unwrap();
3597        assert!(image_header_dims(&stub).is_none());
3598
3599        // Non-image bytes entirely.
3600        let text = td.path().join("text.png");
3601        std::fs::write(&text, b"hello world, not a png").unwrap();
3602        assert!(image_header_dims(&text).is_none());
3603    }
3604
3605    #[test]
3606    fn scan_gallery_dir_filters_invalid_and_keeps_valid() {
3607        let td = TempDir::new("scan");
3608        let dir = td.path();
3609
3610        // A valid PNG large enough to exceed the 256-byte raster size floor.
3611        std::fs::write(dir.join("mold-model-1000.png"), make_png_bytes(32, 32)).unwrap();
3612
3613        // Truncated raster that passes size floor but has no valid header.
3614        let mut junk = vec![0u8; 512];
3615        junk[..4].copy_from_slice(b"JUNK");
3616        std::fs::write(dir.join("mold-broken-2000.png"), &junk).unwrap();
3617
3618        // Tiny raster under the size floor (sub-IHDR).
3619        std::fs::write(
3620            dir.join("mold-tiny-3000.png"),
3621            b"\x89PNG\r\n\x1a\n", // 8 bytes: signature only
3622        )
3623        .unwrap();
3624
3625        // Valid-enough mp4 (ftyp at offset 4) — should survive.
3626        let mut mp4 = Vec::new();
3627        mp4.extend_from_slice(&[0x00, 0x00, 0x00, 0x20]);
3628        mp4.extend_from_slice(b"ftyp");
3629        mp4.extend_from_slice(b"isom\x00\x00\x02\x00");
3630        // Pad above the 4096-byte mp4 size floor so it isn't filtered on
3631        // size alone — the scan still checks ftyp either way.
3632        mp4.resize(8192, 0);
3633        std::fs::write(dir.join("mold-ltx-4000.mp4"), &mp4).unwrap();
3634
3635        // Mp4 extension but no ftyp.
3636        let bad_mp4 = vec![0u8; 8192];
3637        std::fs::write(dir.join("mold-no-ftyp-5000.mp4"), &bad_mp4).unwrap();
3638
3639        // Unsupported extension — ignored entirely.
3640        std::fs::write(dir.join("random.txt"), b"not an output").unwrap();
3641
3642        let results = scan_gallery_dir(dir);
3643        let names: Vec<&str> = results.iter().map(|i| i.filename.as_str()).collect();
3644        assert!(
3645            names.contains(&"mold-model-1000.png"),
3646            "valid PNG should survive: {names:?}"
3647        );
3648        assert!(
3649            names.contains(&"mold-ltx-4000.mp4"),
3650            "valid MP4 with ftyp should survive: {names:?}"
3651        );
3652        assert!(
3653            !names.contains(&"mold-broken-2000.png"),
3654            "PNG with no valid header should be filtered: {names:?}"
3655        );
3656        assert!(
3657            !names.contains(&"mold-tiny-3000.png"),
3658            "under-size PNG stub should be filtered: {names:?}"
3659        );
3660        assert!(
3661            !names.contains(&"mold-no-ftyp-5000.mp4"),
3662            "MP4 without ftyp should be filtered: {names:?}"
3663        );
3664        assert_eq!(names.len(), 2, "only the 2 valid fixtures remain");
3665    }
3666
3667    #[test]
3668    fn solid_black_png_is_filtered_at_scan_time() {
3669        let td = TempDir::new("black");
3670        let dir = td.path();
3671
3672        // A 256×256 solid-black PNG — definitely below the suspect-size
3673        // threshold (compresses to a few hundred bytes) and every pixel is
3674        // below the channel ceiling.
3675        let black = image::RgbImage::from_pixel(256, 256, image::Rgb([0, 0, 0]));
3676        let mut buf = Vec::new();
3677        image::DynamicImage::ImageRgb8(black)
3678            .write_to(&mut std::io::Cursor::new(&mut buf), image::ImageFormat::Png)
3679            .unwrap();
3680        std::fs::write(dir.join("mold-noisy-1000.png"), &buf).unwrap();
3681
3682        // A normal noisy PNG with the same dimensions — should survive.
3683        std::fs::write(dir.join("mold-valid-2000.png"), make_png_bytes(256, 256)).unwrap();
3684
3685        let results = scan_gallery_dir(dir);
3686        let names: Vec<&str> = results.iter().map(|i| i.filename.as_str()).collect();
3687        assert!(
3688            !names.contains(&"mold-noisy-1000.png"),
3689            "solid-black PNG should be filtered: {names:?}"
3690        );
3691        assert!(
3692            names.contains(&"mold-valid-2000.png"),
3693            "noisy PNG should survive: {names:?}"
3694        );
3695    }
3696
3697    #[test]
3698    fn probably_solid_black_ignores_large_files() {
3699        // Files above the per-format suspect size are trusted without a full
3700        // decode (the check is purely a cheap heuristic to catch NaN /
3701        // abort-flavored dev outputs). Verify we bail out on size alone.
3702        let td = TempDir::new("bigblack");
3703        let big_path = td.path().join("big.png");
3704        // Write arbitrary bytes — we never decode because of the size guard.
3705        std::fs::write(&big_path, vec![0u8; 20 * 1024]).unwrap();
3706        assert!(!is_probably_solid_black(
3707            &big_path,
3708            mold_core::OutputFormat::Png,
3709            20 * 1024,
3710        ));
3711    }
3712
3713    #[test]
3714    fn parse_byte_range_handles_common_forms() {
3715        // `bytes=0-499` — first 500 bytes
3716        assert_eq!(parse_byte_range("bytes=0-499", 2000), Some((0, 499)));
3717        // open-ended `bytes=100-` — from byte 100 to EOF
3718        assert_eq!(parse_byte_range("bytes=100-", 2000), Some((100, 1999)));
3719        // suffix `bytes=-500` — last 500 bytes
3720        assert_eq!(parse_byte_range("bytes=-500", 2000), Some((1500, 1999)));
3721        // end past EOF — clamped to last byte
3722        assert_eq!(parse_byte_range("bytes=0-9999", 2000), Some((0, 1999)));
3723        // whole file
3724        assert_eq!(parse_byte_range("bytes=0-1999", 2000), Some((0, 1999)));
3725    }
3726
3727    #[test]
3728    fn parse_byte_range_rejects_malformed_and_unsatisfiable() {
3729        assert_eq!(parse_byte_range("bytes=", 1000), None);
3730        assert_eq!(parse_byte_range("bytes=abc-100", 1000), None);
3731        // start past EOF
3732        assert_eq!(parse_byte_range("bytes=2000-", 1000), None);
3733        // end before start
3734        assert_eq!(parse_byte_range("bytes=500-100", 1000), None);
3735        // multi-range not supported
3736        assert_eq!(parse_byte_range("bytes=0-10,20-30", 1000), None);
3737        // suffix of 0 bytes is meaningless
3738        assert_eq!(parse_byte_range("bytes=-0", 1000), None);
3739        // empty file can't satisfy any range
3740        assert_eq!(parse_byte_range("bytes=0-10", 0), None);
3741        // wrong unit prefix
3742        assert_eq!(parse_byte_range("items=0-10", 1000), None);
3743    }
3744
3745    #[test]
3746    fn scan_populates_real_dimensions_for_synthesized_metadata() {
3747        // Files without an embedded mold:parameters chunk still get their
3748        // actual width/height filled in from the header decode — useful for
3749        // the SPA's aspect-ratio-preserving layout.
3750        let td = TempDir::new("dims");
3751        let dir = td.path();
3752        std::fs::write(dir.join("mold-nometa-1000.png"), make_png_bytes(128, 96)).unwrap();
3753
3754        let results = scan_gallery_dir(dir);
3755        assert_eq!(results.len(), 1);
3756        let entry = &results[0];
3757        assert!(entry.metadata_synthetic);
3758        assert_eq!(entry.metadata.width, 128);
3759        assert_eq!(entry.metadata.height, 96);
3760    }
3761}