1use super::onnx_provision;
23use crate::engine::{download, Engine, EngineCapabilities};
24use crate::types::*;
25use anyhow::{anyhow, bail, Context, Result};
26use image::{imageops::FilterType, DynamicImage, GrayImage, RgbImage};
27use ort::session::Session;
28use ort::value::Tensor;
29use parking_lot::Mutex;
30use std::collections::BTreeMap;
31use std::io::Cursor;
32use std::path::{Path, PathBuf};
33use std::time::Instant;
34use tracing::{debug, info, warn};
35
36const TRACE_TARGET: &str = "studio_worker::engine::onnx";
38
39pub const ENGINE_NAME: &str = "onnx";
41
42const LAMA_SIZE: u32 = 512;
44
45const FEATHER_SIGMA: f32 = 4.0;
49
50pub struct OnnxImageEngine {
54 models_root: PathBuf,
55 cached: Mutex<Option<(PathBuf, Session)>>,
56}
57
58impl OnnxImageEngine {
59 pub fn new(models_root: PathBuf) -> Self {
60 debug!(
61 target: TRACE_TARGET,
62 op = "new",
63 models_root = %models_root.display(),
64 "onnx image engine constructed"
65 );
66 Self {
67 models_root,
68 cached: Mutex::new(None),
69 }
70 }
71
72 #[cfg_attr(coverage_nightly, coverage(off))]
75 fn ensure_model(&self, source: &ModelSource) -> Result<PathBuf> {
76 let file = source
77 .files
78 .iter()
79 .find(|f| f.role == ModelFileRole::Model)
80 .ok_or_else(|| anyhow!("onnx modelSource has no `model` file (the .onnx weights)"))?;
81 download::ensure_file(&self.models_root, file)
82 .with_context(|| format!("downloading onnx model {}", file.url))
83 }
84
85 #[cfg_attr(coverage_nightly, coverage(off))]
91 fn run_session(&self, model_path: &Path, image: Vec<f32>, mask: Vec<f32>) -> Result<Vec<f32>> {
92 self.ensure_ort_runtime()?;
93 let mut guard = self.cached.lock();
94 if guard.as_ref().map(|(p, _)| p.as_path()) != Some(model_path) {
95 let session = Session::builder()
96 .context("ort Session::builder")?
97 .commit_from_file(model_path)
98 .with_context(|| format!("loading onnx model {}", model_path.display()))?;
99 info!(
100 target: TRACE_TARGET,
101 op = "load",
102 model = %model_path.display(),
103 "onnx session loaded"
104 );
105 *guard = Some((model_path.to_path_buf(), session));
106 }
107 let session = &mut guard.as_mut().expect("session just set").1;
108
109 let image_t =
110 Tensor::from_array(([1_usize, 3, LAMA_SIZE as usize, LAMA_SIZE as usize], image))
111 .context("building image tensor")?;
112 let mask_t =
113 Tensor::from_array(([1_usize, 1, LAMA_SIZE as usize, LAMA_SIZE as usize], mask))
114 .context("building mask tensor")?;
115
116 let outputs = session
117 .run(ort::inputs!["image" => image_t, "mask" => mask_t])
118 .context("onnx session.run")?;
119 let (_, data) = outputs["output"]
120 .try_extract_tensor::<f32>()
121 .context("extracting onnx output tensor")?;
122 Ok(data.to_vec())
123 }
124
125 #[cfg_attr(coverage_nightly, coverage(off))]
129 fn ensure_ort_runtime(&self) -> Result<()> {
130 if std::env::var_os("ORT_DYLIB_PATH").is_some() {
131 return Ok(());
132 }
133 let lib = onnx_provision::provision(&self.models_root)?;
134 std::env::set_var("ORT_DYLIB_PATH", &lib);
135 info!(
136 target: TRACE_TARGET,
137 op = "runtime",
138 dylib = %lib.display(),
139 version = onnx_provision::ORT_VERSION,
140 "onnx runtime provisioned"
141 );
142 Ok(())
143 }
144
145 #[cfg_attr(coverage_nightly, coverage(off))]
147 fn dispatch_removal(&self, params: &ImageParams, source: &ModelSource) -> Result<TaskResult> {
148 let init_url = params
149 .init_image_url
150 .as_deref()
151 .filter(|s| !s.is_empty())
152 .ok_or_else(|| {
153 anyhow!("onnx/LaMa removal requires `initImageUrl` (the original image)")
154 })?;
155 let mask_url = params
156 .mask_url
157 .as_deref()
158 .filter(|s| !s.is_empty())
159 .ok_or_else(|| {
160 anyhow!("onnx/LaMa removal requires `maskUrl` (the region to remove)")
161 })?;
162
163 let model_path = self.ensure_model(source)?;
164
165 let work_dir = std::env::temp_dir().join("studio-worker-onnx");
166 std::fs::create_dir_all(&work_dir)
167 .with_context(|| format!("creating onnx work dir {}", work_dir.display()))?;
168 let stem = format!(
169 "onnx-{}-{}",
170 std::process::id(),
171 chrono::Utc::now().timestamp_nanos_opt().unwrap_or_default()
172 );
173 let init_path = work_dir.join(format!("{stem}-init"));
174 let mask_path = work_dir.join(format!("{stem}-mask"));
175 let mut scratch = download::TempFileGuard::new();
182 scratch.push(init_path.clone());
183 scratch.push(mask_path.clone());
184 download::download_file(init_url, &init_path)
185 .with_context(|| format!("downloading init image {init_url}"))?;
186 download::download_file(mask_url, &mask_path)
187 .with_context(|| format!("downloading mask {mask_url}"))?;
188
189 let started = Instant::now();
190 let bytes = self.remove(&model_path, &init_path, &mask_path, params)?;
191
192 debug!(
193 target: TRACE_TARGET,
194 op = "dispatch",
195 model = %model_path.display(),
196 width = params.width,
197 height = params.height,
198 elapsed_ms = started.elapsed().as_millis() as u64,
199 "lama removal complete"
200 );
201 Ok(TaskResult::Image {
202 bytes,
203 ext: params.ext.clone(),
204 })
205 }
206
207 #[cfg_attr(coverage_nightly, coverage(off))]
212 fn run_lama_region(
213 &self,
214 model_path: &Path,
215 rgb: &RgbImage,
216 mask: &GrayImage,
217 ) -> Result<RgbImage> {
218 let (rw, rh) = rgb.dimensions();
219 let lama_rgb = DynamicImage::ImageRgb8(rgb.clone())
220 .resize_exact(LAMA_SIZE, LAMA_SIZE, FilterType::Triangle)
221 .to_rgb8();
222 let lama_mask = DynamicImage::ImageLuma8(mask.clone())
223 .resize_exact(LAMA_SIZE, LAMA_SIZE, FilterType::Triangle)
224 .to_luma8();
225 let out_raw = self.run_session(
226 model_path,
227 image_to_chw(&lama_rgb),
228 mask_to_binary(&lama_mask),
229 )?;
230 let scale = detect_scale(&out_raw);
231 let lama_512 = chw_to_rgb(&out_raw, LAMA_SIZE, scale)?;
232 Ok(DynamicImage::ImageRgb8(lama_512)
233 .resize_exact(rw.max(1), rh.max(1), FilterType::Triangle)
234 .to_rgb8())
235 }
236
237 #[cfg_attr(coverage_nightly, coverage(off))]
239 fn remove(
240 &self,
241 model_path: &Path,
242 init_path: &Path,
243 mask_path: &Path,
244 params: &ImageParams,
245 ) -> Result<Vec<u8>> {
246 let (w, h) = (params.width.max(1), params.height.max(1));
247 let init_bytes = std::fs::read(init_path)
251 .with_context(|| format!("reading init image {}", init_path.display()))?;
252 let mask_bytes = std::fs::read(mask_path)
253 .with_context(|| format!("reading mask {}", mask_path.display()))?;
254 let original = image::load_from_memory(&init_bytes)
255 .context("decoding init image")?
256 .resize_exact(w, h, FilterType::Triangle)
257 .to_rgb8();
258 let mask_full = image::load_from_memory(&mask_bytes)
259 .context("decoding mask")?
260 .resize_exact(w, h, FilterType::Triangle)
261 .to_luma8();
262
263 let fill = match hd_crop_box(&mask_full, w, h) {
272 Some((cx, cy, cw, ch)) => {
273 let crop_rgb = image::imageops::crop_imm(&original, cx, cy, cw, ch).to_image();
274 let crop_mask = image::imageops::crop_imm(&mask_full, cx, cy, cw, ch).to_image();
275 let inpainted = self.run_lama_region(model_path, &crop_rgb, &crop_mask)?;
276 let mut f = original.clone();
277 image::imageops::replace(&mut f, &inpainted, cx as i64, cy as i64);
278 debug!(
279 target: TRACE_TARGET, op = "remove", crop_w = cw, crop_h = ch,
280 full_w = w, full_h = h, "lama HD crop inpaint"
281 );
282 f
283 }
284 None => self.run_lama_region(model_path, &original, &mask_full)?,
286 };
287 let alpha = image::imageops::blur(&mask_full, FEATHER_SIGMA);
290 let composited = alpha_composite(&original, &fill, &alpha);
291
292 let mut out = Cursor::new(Vec::<u8>::new());
293 let dyn_img = DynamicImage::ImageRgb8(composited);
294 match params.ext.as_str() {
295 "png" => dyn_img.write_to(&mut out, image::ImageFormat::Png)?,
296 _ => dyn_img.write_to(&mut out, image::ImageFormat::WebP)?,
297 }
298 Ok(out.into_inner())
299 }
300}
301
302impl Engine for OnnxImageEngine {
303 fn name(&self) -> &'static str {
304 ENGINE_NAME
305 }
306
307 fn capabilities(&self) -> EngineCapabilities {
308 let mut map: BTreeMap<TaskKind, Vec<String>> = BTreeMap::new();
309 map.insert(TaskKind::Image, vec!["onnx:*".to_string()]);
313 EngineCapabilities {
314 supported_models_per_kind: map,
315 }
316 }
317
318 fn dispatch(&self, _model: &str, _task: Task) -> Result<TaskResult> {
319 bail!("onnx engine requires a ModelSource; use dispatch_with_source")
320 }
321
322 fn dispatch_with_source(
323 &self,
324 model: &str,
325 task: Task,
326 source: &ModelSource,
327 ) -> Result<TaskResult> {
328 match task {
329 Task::Image(p) => {
330 let started = Instant::now();
337 self.dispatch_removal(&p, source).inspect_err(|e| {
338 warn!(
339 target: TRACE_TARGET,
340 op = "dispatch",
341 model,
342 error = %e,
343 elapsed_ms = started.elapsed().as_millis() as u64,
344 "lama removal failed"
345 );
346 })
347 }
348 other => {
349 warn!(
350 target: TRACE_TARGET,
351 op = "dispatch",
352 model,
353 kind = other.kind().as_str(),
354 "onnx engine only serves image removal jobs"
355 );
356 bail!(
357 "onnx engine only serves image tasks, got {}",
358 other.kind().as_str()
359 )
360 }
361 }
362 }
363}
364
365fn image_to_chw(rgb: &RgbImage) -> Vec<f32> {
372 let n = (LAMA_SIZE * LAMA_SIZE) as usize;
373 let mut out = vec![0.0_f32; 3 * n];
374 for (i, px) in rgb.pixels().enumerate() {
375 out[i] = px.0[0] as f32 / 255.0;
376 out[n + i] = px.0[1] as f32 / 255.0;
377 out[2 * n + i] = px.0[2] as f32 / 255.0;
378 }
379 out
380}
381
382fn mask_to_binary(mask: &GrayImage) -> Vec<f32> {
385 mask.pixels()
386 .map(|p| if p.0[0] > 128 { 1.0_f32 } else { 0.0 })
387 .collect()
388}
389
390fn detect_scale(out: &[f32]) -> f32 {
393 let max = out.iter().copied().fold(0.0_f32, f32::max);
394 if max > 2.0 {
395 1.0
396 } else {
397 255.0
398 }
399}
400
401fn chw_to_rgb(out: &[f32], size: u32, scale: f32) -> Result<RgbImage> {
404 let n = (size * size) as usize;
405 if out.len() < 3 * n {
406 bail!("onnx output too small: {} < {}", out.len(), 3 * n);
407 }
408 let mut img = RgbImage::new(size, size);
409 for (i, px) in img.pixels_mut().enumerate() {
410 let r = (out[i] * scale).clamp(0.0, 255.0) as u8;
411 let g = (out[n + i] * scale).clamp(0.0, 255.0) as u8;
412 let b = (out[2 * n + i] * scale).clamp(0.0, 255.0) as u8;
413 *px = image::Rgb([r, g, b]);
414 }
415 Ok(img)
416}
417
418fn hd_crop_box(mask: &GrayImage, w: u32, h: u32) -> Option<(u32, u32, u32, u32)> {
425 let (mut x0, mut y0, mut x1, mut y1) = (w, h, 0_u32, 0_u32);
426 let mut any = false;
427 for (x, y, p) in mask.enumerate_pixels() {
428 if p.0[0] > 128 {
429 any = true;
430 x0 = x0.min(x);
431 y0 = y0.min(y);
432 x1 = x1.max(x);
433 y1 = y1.max(y);
434 }
435 }
436 if !any {
437 return None;
438 }
439 let bw = x1 - x0 + 1;
440 let bh = y1 - y0 + 1;
441 let max_dim = bw.max(bh);
442 let margin = (max_dim as f32 * 0.25).round() as u32 + 32;
445 let limit = w.min(h);
446 let side = (max_dim + 2 * margin).clamp(256, limit);
447 if side >= w && side >= h {
449 return None;
450 }
451 let centre_x = x0 + bw / 2;
452 let centre_y = y0 + bh / 2;
453 let half = side / 2;
454 let x = centre_x.saturating_sub(half).min(w - side);
455 let y = centre_y.saturating_sub(half).min(h - side);
456 Some((x, y, side, side))
457}
458
459fn alpha_composite(base: &RgbImage, fill: &RgbImage, alpha: &GrayImage) -> RgbImage {
462 let (w, h) = base.dimensions();
463 let mut out = RgbImage::new(w, h);
464 for (x, y, px) in out.enumerate_pixels_mut() {
465 let a = alpha.get_pixel(x, y).0[0] as f32 / 255.0;
466 let b = base.get_pixel(x, y).0;
467 let f = fill.get_pixel(x, y).0;
468 *px = image::Rgb([
469 (b[0] as f32 * (1.0 - a) + f[0] as f32 * a).round() as u8,
470 (b[1] as f32 * (1.0 - a) + f[1] as f32 * a).round() as u8,
471 (b[2] as f32 * (1.0 - a) + f[2] as f32 * a).round() as u8,
472 ]);
473 }
474 out
475}
476
477#[cfg(test)]
478mod tests {
479 use super::*;
480
481 #[test]
488 fn build_decodes_jpeg_init_images() {
489 let jpeg = include_bytes!("../../tests/fixtures/sample.jpg");
490 let decoded = image::load_from_memory(jpeg)
491 .expect("this build must decode JPEG init images")
492 .to_rgb8();
493 assert_eq!((decoded.width(), decoded.height()), (16, 12));
494 }
495
496 #[test]
497 fn image_to_chw_packs_planar_normalised() {
498 let mut img = RgbImage::new(LAMA_SIZE, LAMA_SIZE);
499 img.put_pixel(0, 0, image::Rgb([255, 0, 0]));
500 img.put_pixel(1, 0, image::Rgb([0, 255, 0]));
501 let chw = image_to_chw(&img);
502 let n = (LAMA_SIZE * LAMA_SIZE) as usize;
503 assert_eq!(chw.len(), 3 * n);
504 assert_eq!(chw[0], 1.0); assert_eq!(chw[n], 0.0); assert_eq!(chw[2 * n], 0.0); assert_eq!(chw[1], 0.0);
510 assert_eq!(chw[n + 1], 1.0);
511 }
512
513 #[test]
514 fn mask_to_binary_thresholds_at_128() {
515 let mut m = GrayImage::new(LAMA_SIZE, LAMA_SIZE);
516 m.put_pixel(0, 0, image::Luma([0]));
517 m.put_pixel(1, 0, image::Luma([128]));
518 m.put_pixel(2, 0, image::Luma([129]));
519 m.put_pixel(3, 0, image::Luma([255]));
520 let bin = mask_to_binary(&m);
521 assert_eq!(bin[0], 0.0);
522 assert_eq!(bin[1], 0.0); assert_eq!(bin[2], 1.0);
524 assert_eq!(bin[3], 1.0);
525 }
526
527 fn mask_with_rect(w: u32, h: u32, x0: u32, y0: u32, rw: u32, rh: u32) -> GrayImage {
528 let mut m = GrayImage::new(w, h);
529 for y in y0..(y0 + rh) {
530 for x in x0..(x0 + rw) {
531 m.put_pixel(x, y, image::Luma([255]));
532 }
533 }
534 m
535 }
536
537 #[test]
538 fn hd_crop_box_is_none_for_an_empty_mask() {
539 assert_eq!(hd_crop_box(&GrayImage::new(1024, 768), 1024, 768), None);
540 }
541
542 #[test]
543 fn hd_crop_box_brackets_a_small_object_with_a_square_in_bounds_crop() {
544 let mask = mask_with_rect(1024, 768, 412, 284, 200, 200);
546 let (x, y, cw, ch) = hd_crop_box(&mask, 1024, 768).expect("should crop");
547 assert_eq!(cw, ch);
550 assert!(
551 cw < 1024,
552 "crop {cw} should be smaller than the frame width"
553 );
554 assert!(cw >= 200, "crop must at least contain the object");
555 assert!(x + cw <= 1024 && y + ch <= 768);
557 }
558
559 #[test]
560 fn hd_crop_box_clamps_a_corner_object_inside_the_frame() {
561 let mask = mask_with_rect(1024, 768, 0, 0, 180, 180);
562 let (x, y, cw, ch) = hd_crop_box(&mask, 1024, 768).expect("should crop");
563 assert_eq!((x, y), (0, 0));
564 assert!(x + cw <= 1024 && y + ch <= 768);
565 }
566
567 #[test]
568 fn hd_crop_box_falls_back_to_none_when_the_mask_spans_the_frame() {
569 let mask = mask_with_rect(512, 512, 0, 0, 512, 512);
570 assert_eq!(hd_crop_box(&mask, 512, 512), None);
571 }
572
573 #[test]
574 fn detect_scale_distinguishes_unit_and_byte_ranges() {
575 assert_eq!(detect_scale(&[0.0, 0.5, 1.0]), 255.0);
576 assert_eq!(detect_scale(&[0.0, 128.0, 240.0]), 1.0);
577 assert_eq!(detect_scale(&[0.0, 0.0]), 255.0);
579 }
580
581 #[test]
582 fn chw_to_rgb_roundtrips_unit_scale() {
583 let n = (LAMA_SIZE * LAMA_SIZE) as usize;
584 let mut buf = vec![0.0_f32; 3 * n];
585 buf[0] = 1.0; buf[2 * n + 1] = 1.0; let img = chw_to_rgb(&buf, LAMA_SIZE, 255.0).unwrap();
588 assert_eq!(img.get_pixel(0, 0).0, [255, 0, 0]);
589 assert_eq!(img.get_pixel(1, 0).0, [0, 0, 255]);
590 }
591
592 #[test]
593 fn chw_to_rgb_rejects_short_buffer() {
594 assert!(chw_to_rgb(&[0.0; 10], LAMA_SIZE, 255.0).is_err());
595 }
596
597 #[test]
598 fn alpha_composite_blends_by_mask() {
599 let base = RgbImage::from_pixel(2, 1, image::Rgb([0, 0, 0]));
600 let fill = RgbImage::from_pixel(2, 1, image::Rgb([100, 100, 100]));
601 let mut alpha = GrayImage::new(2, 1);
602 alpha.put_pixel(0, 0, image::Luma([0])); alpha.put_pixel(1, 0, image::Luma([255])); let out = alpha_composite(&base, &fill, &alpha);
605 assert_eq!(out.get_pixel(0, 0).0, [0, 0, 0]);
606 assert_eq!(out.get_pixel(1, 0).0, [100, 100, 100]);
607 }
608
609 #[test]
610 fn alpha_composite_half_blends_midpoint() {
611 let base = RgbImage::from_pixel(1, 1, image::Rgb([0, 0, 0]));
612 let fill = RgbImage::from_pixel(1, 1, image::Rgb([200, 200, 200]));
613 let alpha = GrayImage::from_pixel(1, 1, image::Luma([128]));
614 let out = alpha_composite(&base, &fill, &alpha);
615 let v = out.get_pixel(0, 0).0[0];
617 assert!((99..=101).contains(&v), "got {v}");
618 }
619
620 #[test]
628 #[ignore = "needs the real LaMa onnx model + assets via env"]
629 fn lama_removal_end_to_end() {
630 let onnx = std::env::var("LAMA_ONNX").expect("LAMA_ONNX");
631 let init = std::env::var("LAMA_INIT").expect("LAMA_INIT");
632 let mask = std::env::var("LAMA_MASK").expect("LAMA_MASK");
633 let params = ImageParams {
634 width: 1024,
635 height: 768,
636 ext: "webp".into(),
637 ..Default::default()
638 };
639 let engine = OnnxImageEngine::new(std::env::temp_dir());
640 let bytes = engine
641 .remove(
642 std::path::Path::new(&onnx),
643 std::path::Path::new(&init),
644 std::path::Path::new(&mask),
645 ¶ms,
646 )
647 .expect("removal");
648 assert!(!bytes.is_empty(), "empty output");
649 let out = image::load_from_memory(&bytes)
650 .expect("decode output")
651 .to_rgb8();
652 assert_eq!(out.dimensions(), (1024, 768));
653 if let Ok(out_path) = std::env::var("LAMA_OUT") {
654 std::fs::write(&out_path, &bytes).expect("write out");
655 }
656 let original = image::load_from_memory(&std::fs::read(&init).unwrap())
660 .unwrap()
661 .resize_exact(1024, 768, FilterType::Triangle)
662 .to_rgb8();
663 let d_corner: i32 = (0..3)
664 .map(|i| {
665 (out.get_pixel(20, 20).0[i] as i32 - original.get_pixel(20, 20).0[i] as i32).abs()
666 })
667 .sum();
668 assert!(d_corner < 16, "outside-mask pixel drifted: {d_corner}");
669 }
670}