1use super::onnx_provision;
23use crate::engine::{download, Engine, EngineCapabilities};
24use crate::types::*;
25use anyhow::{anyhow, bail, Context, Result};
26use image::{imageops::FilterType, DynamicImage, GrayImage, RgbImage};
27use ort::session::Session;
28use ort::value::Tensor;
29use parking_lot::Mutex;
30use std::collections::BTreeMap;
31use std::io::Cursor;
32use std::path::{Path, PathBuf};
33use std::time::Instant;
34use tracing::{debug, info, warn};
35
36const TRACE_TARGET: &str = "studio_worker::engine::onnx";
38
39pub const ENGINE_NAME: &str = "onnx";
41
42const LAMA_SIZE: u32 = 512;
44
45const FEATHER_SIGMA: f32 = 4.0;
49
50pub struct OnnxImageEngine {
54 models_root: PathBuf,
55 cached: Mutex<Option<(PathBuf, Session)>>,
56}
57
58impl OnnxImageEngine {
59 pub fn new(models_root: PathBuf) -> Self {
60 debug!(
61 target: TRACE_TARGET,
62 op = "new",
63 models_root = %models_root.display(),
64 "onnx image engine constructed"
65 );
66 Self {
67 models_root,
68 cached: Mutex::new(None),
69 }
70 }
71
72 #[cfg_attr(coverage_nightly, coverage(off))]
75 fn ensure_model(&self, model: &str, source: &ModelSource) -> Result<PathBuf> {
76 let file = source
77 .files
78 .iter()
79 .find(|f| f.role == ModelFileRole::Model)
80 .ok_or_else(|| anyhow!("onnx modelSource has no `model` file (the .onnx weights)"))?;
81 download::ensure_file_for_model(&self.models_root, model, file)
82 .with_context(|| format!("downloading onnx model {}", file.url))
83 }
84
85 #[cfg_attr(coverage_nightly, coverage(off))]
91 fn run_session(&self, model_path: &Path, image: Vec<f32>, mask: Vec<f32>) -> Result<Vec<f32>> {
92 self.ensure_ort_runtime()?;
93 let mut guard = self.cached.lock();
94 if guard.as_ref().map(|(p, _)| p.as_path()) != Some(model_path) {
95 let session = Session::builder()
96 .context("ort Session::builder")?
97 .commit_from_file(model_path)
98 .with_context(|| format!("loading onnx model {}", model_path.display()))?;
99 info!(
100 target: TRACE_TARGET,
101 op = "load",
102 model = %model_path.display(),
103 "onnx session loaded"
104 );
105 *guard = Some((model_path.to_path_buf(), session));
106 }
107 let session = &mut guard.as_mut().expect("session just set").1;
108
109 let image_t =
110 Tensor::from_array(([1_usize, 3, LAMA_SIZE as usize, LAMA_SIZE as usize], image))
111 .context("building image tensor")?;
112 let mask_t =
113 Tensor::from_array(([1_usize, 1, LAMA_SIZE as usize, LAMA_SIZE as usize], mask))
114 .context("building mask tensor")?;
115
116 let outputs = session
117 .run(ort::inputs!["image" => image_t, "mask" => mask_t])
118 .context("onnx session.run")?;
119 let (_, data) = outputs["output"]
120 .try_extract_tensor::<f32>()
121 .context("extracting onnx output tensor")?;
122 Ok(data.to_vec())
123 }
124
125 #[cfg_attr(coverage_nightly, coverage(off))]
129 fn ensure_ort_runtime(&self) -> Result<()> {
130 let runtime = onnx_provision::ensure_runtime(&self.models_root)?;
131 info!(
132 target: TRACE_TARGET,
133 op = "runtime",
134 dylib = %runtime.lib.display(),
135 flavour = runtime.flavour.name(),
136 version = onnx_provision::ORT_VERSION,
137 "onnx runtime ready"
138 );
139 Ok(())
140 }
141
142 #[cfg_attr(coverage_nightly, coverage(off))]
144 fn dispatch_removal(
145 &self,
146 model: &str,
147 params: &ImageParams,
148 source: &ModelSource,
149 ) -> Result<TaskResult> {
150 let init_url = params
151 .init_image_url
152 .as_deref()
153 .filter(|s| !s.is_empty())
154 .ok_or_else(|| {
155 anyhow!("onnx/LaMa removal requires `initImageUrl` (the original image)")
156 })?;
157 let mask_url = params
158 .mask_url
159 .as_deref()
160 .filter(|s| !s.is_empty())
161 .ok_or_else(|| {
162 anyhow!("onnx/LaMa removal requires `maskUrl` (the region to remove)")
163 })?;
164
165 let model_path = self.ensure_model(model, source)?;
166
167 let work_dir = std::env::temp_dir().join("studio-worker-onnx");
168 std::fs::create_dir_all(&work_dir)
169 .with_context(|| format!("creating onnx work dir {}", work_dir.display()))?;
170 let stem = format!(
171 "onnx-{}-{}",
172 std::process::id(),
173 chrono::Utc::now().timestamp_nanos_opt().unwrap_or_default()
174 );
175 let init_path = work_dir.join(format!("{stem}-init"));
176 let mask_path = work_dir.join(format!("{stem}-mask"));
177 let mut scratch = download::TempFileGuard::new();
184 scratch.push(init_path.clone());
185 scratch.push(mask_path.clone());
186 download::download_file(init_url, &init_path)
187 .with_context(|| format!("downloading init image {init_url}"))?;
188 download::download_file(mask_url, &mask_path)
189 .with_context(|| format!("downloading mask {mask_url}"))?;
190
191 let started = Instant::now();
192 let bytes = self.remove(&model_path, &init_path, &mask_path, params)?;
193
194 debug!(
195 target: TRACE_TARGET,
196 op = "dispatch",
197 model = %model_path.display(),
198 width = params.width,
199 height = params.height,
200 elapsed_ms = started.elapsed().as_millis() as u64,
201 "lama removal complete"
202 );
203 Ok(TaskResult::Image {
204 bytes,
205 ext: params.ext.clone(),
206 })
207 }
208
209 #[cfg_attr(coverage_nightly, coverage(off))]
214 fn run_lama_region(
215 &self,
216 model_path: &Path,
217 rgb: &RgbImage,
218 mask: &GrayImage,
219 ) -> Result<RgbImage> {
220 let (rw, rh) = rgb.dimensions();
221 let lama_rgb = DynamicImage::ImageRgb8(rgb.clone())
222 .resize_exact(LAMA_SIZE, LAMA_SIZE, FilterType::Triangle)
223 .to_rgb8();
224 let lama_mask = DynamicImage::ImageLuma8(mask.clone())
225 .resize_exact(LAMA_SIZE, LAMA_SIZE, FilterType::Triangle)
226 .to_luma8();
227 let out_raw = self.run_session(
228 model_path,
229 image_to_chw(&lama_rgb),
230 mask_to_binary(&lama_mask),
231 )?;
232 let scale = detect_scale(&out_raw);
233 let lama_512 = chw_to_rgb(&out_raw, LAMA_SIZE, scale)?;
234 Ok(DynamicImage::ImageRgb8(lama_512)
235 .resize_exact(rw.max(1), rh.max(1), FilterType::Triangle)
236 .to_rgb8())
237 }
238
239 #[cfg_attr(coverage_nightly, coverage(off))]
241 fn remove(
242 &self,
243 model_path: &Path,
244 init_path: &Path,
245 mask_path: &Path,
246 params: &ImageParams,
247 ) -> Result<Vec<u8>> {
248 let (w, h) = (params.width.max(1), params.height.max(1));
249 let init_bytes = std::fs::read(init_path)
253 .with_context(|| format!("reading init image {}", init_path.display()))?;
254 let mask_bytes = std::fs::read(mask_path)
255 .with_context(|| format!("reading mask {}", mask_path.display()))?;
256 let original = image::load_from_memory(&init_bytes)
257 .context("decoding init image")?
258 .resize_exact(w, h, FilterType::Triangle)
259 .to_rgb8();
260 let mask_full = image::load_from_memory(&mask_bytes)
261 .context("decoding mask")?
262 .resize_exact(w, h, FilterType::Triangle)
263 .to_luma8();
264
265 let fill = match hd_crop_box(&mask_full, w, h) {
274 Some((cx, cy, cw, ch)) => {
275 let crop_rgb = image::imageops::crop_imm(&original, cx, cy, cw, ch).to_image();
276 let crop_mask = image::imageops::crop_imm(&mask_full, cx, cy, cw, ch).to_image();
277 let inpainted = self.run_lama_region(model_path, &crop_rgb, &crop_mask)?;
278 let mut f = original.clone();
279 image::imageops::replace(&mut f, &inpainted, cx as i64, cy as i64);
280 debug!(
281 target: TRACE_TARGET, op = "remove", crop_w = cw, crop_h = ch,
282 full_w = w, full_h = h, "lama HD crop inpaint"
283 );
284 f
285 }
286 None => self.run_lama_region(model_path, &original, &mask_full)?,
288 };
289 let alpha = image::imageops::blur(&mask_full, FEATHER_SIGMA);
292 let composited = alpha_composite(&original, &fill, &alpha);
293
294 let mut out = Cursor::new(Vec::<u8>::new());
295 let dyn_img = DynamicImage::ImageRgb8(composited);
296 match params.ext.as_str() {
297 "png" => dyn_img.write_to(&mut out, image::ImageFormat::Png)?,
298 _ => dyn_img.write_to(&mut out, image::ImageFormat::WebP)?,
299 }
300 Ok(out.into_inner())
301 }
302}
303
304impl Engine for OnnxImageEngine {
305 fn name(&self) -> &'static str {
306 ENGINE_NAME
307 }
308
309 fn capabilities(&self) -> EngineCapabilities {
310 let mut map: BTreeMap<TaskKind, Vec<String>> = BTreeMap::new();
311 map.insert(TaskKind::Image, vec!["onnx:*".to_string()]);
315 EngineCapabilities {
316 supported_models_per_kind: map,
317 }
318 }
319
320 fn dispatch(&self, _model: &str, _task: Task) -> Result<TaskResult> {
321 bail!("onnx engine requires a ModelSource; use dispatch_with_source")
322 }
323
324 fn dispatch_with_source(
325 &self,
326 model: &str,
327 task: Task,
328 source: &ModelSource,
329 ) -> Result<TaskResult> {
330 match task {
331 Task::Image(p) => {
332 let started = Instant::now();
339 self.dispatch_removal(model, &p, source).inspect_err(|e| {
340 warn!(
341 target: TRACE_TARGET,
342 op = "dispatch",
343 model,
344 error = %e,
345 elapsed_ms = started.elapsed().as_millis() as u64,
346 "lama removal failed"
347 );
348 })
349 }
350 other => {
351 warn!(
352 target: TRACE_TARGET,
353 op = "dispatch",
354 model,
355 kind = other.kind().as_str(),
356 "onnx engine only serves image removal jobs"
357 );
358 bail!(
359 "onnx engine only serves image tasks, got {}",
360 other.kind().as_str()
361 )
362 }
363 }
364 }
365}
366
367fn image_to_chw(rgb: &RgbImage) -> Vec<f32> {
374 let n = (LAMA_SIZE * LAMA_SIZE) as usize;
375 let mut out = vec![0.0_f32; 3 * n];
376 for (i, px) in rgb.pixels().enumerate() {
377 out[i] = px.0[0] as f32 / 255.0;
378 out[n + i] = px.0[1] as f32 / 255.0;
379 out[2 * n + i] = px.0[2] as f32 / 255.0;
380 }
381 out
382}
383
384fn mask_to_binary(mask: &GrayImage) -> Vec<f32> {
387 mask.pixels()
388 .map(|p| if p.0[0] > 128 { 1.0_f32 } else { 0.0 })
389 .collect()
390}
391
392fn detect_scale(out: &[f32]) -> f32 {
395 let max = out.iter().copied().fold(0.0_f32, f32::max);
396 if max > 2.0 {
397 1.0
398 } else {
399 255.0
400 }
401}
402
403fn chw_to_rgb(out: &[f32], size: u32, scale: f32) -> Result<RgbImage> {
406 let n = (size * size) as usize;
407 if out.len() < 3 * n {
408 bail!("onnx output too small: {} < {}", out.len(), 3 * n);
409 }
410 let mut img = RgbImage::new(size, size);
411 for (i, px) in img.pixels_mut().enumerate() {
412 let r = (out[i] * scale).clamp(0.0, 255.0) as u8;
413 let g = (out[n + i] * scale).clamp(0.0, 255.0) as u8;
414 let b = (out[2 * n + i] * scale).clamp(0.0, 255.0) as u8;
415 *px = image::Rgb([r, g, b]);
416 }
417 Ok(img)
418}
419
420fn hd_crop_box(mask: &GrayImage, w: u32, h: u32) -> Option<(u32, u32, u32, u32)> {
427 let (mut x0, mut y0, mut x1, mut y1) = (w, h, 0_u32, 0_u32);
428 let mut any = false;
429 for (x, y, p) in mask.enumerate_pixels() {
430 if p.0[0] > 128 {
431 any = true;
432 x0 = x0.min(x);
433 y0 = y0.min(y);
434 x1 = x1.max(x);
435 y1 = y1.max(y);
436 }
437 }
438 if !any {
439 return None;
440 }
441 let bw = x1 - x0 + 1;
442 let bh = y1 - y0 + 1;
443 let max_dim = bw.max(bh);
444 let margin = (max_dim as f32 * 0.25).round() as u32 + 32;
447 let limit = w.min(h);
448 let side = (max_dim + 2 * margin).clamp(256, limit);
449 if side >= w && side >= h {
451 return None;
452 }
453 let centre_x = x0 + bw / 2;
454 let centre_y = y0 + bh / 2;
455 let half = side / 2;
456 let x = centre_x.saturating_sub(half).min(w - side);
457 let y = centre_y.saturating_sub(half).min(h - side);
458 Some((x, y, side, side))
459}
460
461fn alpha_composite(base: &RgbImage, fill: &RgbImage, alpha: &GrayImage) -> RgbImage {
464 let (w, h) = base.dimensions();
465 let mut out = RgbImage::new(w, h);
466 for (x, y, px) in out.enumerate_pixels_mut() {
467 let a = alpha.get_pixel(x, y).0[0] as f32 / 255.0;
468 let b = base.get_pixel(x, y).0;
469 let f = fill.get_pixel(x, y).0;
470 *px = image::Rgb([
471 (b[0] as f32 * (1.0 - a) + f[0] as f32 * a).round() as u8,
472 (b[1] as f32 * (1.0 - a) + f[1] as f32 * a).round() as u8,
473 (b[2] as f32 * (1.0 - a) + f[2] as f32 * a).round() as u8,
474 ]);
475 }
476 out
477}
478
479#[cfg(test)]
480mod tests {
481 use super::*;
482
483 #[test]
490 fn build_decodes_jpeg_init_images() {
491 let jpeg = include_bytes!("../../tests/fixtures/sample.jpg");
492 let decoded = image::load_from_memory(jpeg)
493 .expect("this build must decode JPEG init images")
494 .to_rgb8();
495 assert_eq!((decoded.width(), decoded.height()), (16, 12));
496 }
497
498 #[test]
499 fn image_to_chw_packs_planar_normalised() {
500 let mut img = RgbImage::new(LAMA_SIZE, LAMA_SIZE);
501 img.put_pixel(0, 0, image::Rgb([255, 0, 0]));
502 img.put_pixel(1, 0, image::Rgb([0, 255, 0]));
503 let chw = image_to_chw(&img);
504 let n = (LAMA_SIZE * LAMA_SIZE) as usize;
505 assert_eq!(chw.len(), 3 * n);
506 assert_eq!(chw[0], 1.0); assert_eq!(chw[n], 0.0); assert_eq!(chw[2 * n], 0.0); assert_eq!(chw[1], 0.0);
512 assert_eq!(chw[n + 1], 1.0);
513 }
514
515 #[test]
516 fn mask_to_binary_thresholds_at_128() {
517 let mut m = GrayImage::new(LAMA_SIZE, LAMA_SIZE);
518 m.put_pixel(0, 0, image::Luma([0]));
519 m.put_pixel(1, 0, image::Luma([128]));
520 m.put_pixel(2, 0, image::Luma([129]));
521 m.put_pixel(3, 0, image::Luma([255]));
522 let bin = mask_to_binary(&m);
523 assert_eq!(bin[0], 0.0);
524 assert_eq!(bin[1], 0.0); assert_eq!(bin[2], 1.0);
526 assert_eq!(bin[3], 1.0);
527 }
528
529 fn mask_with_rect(w: u32, h: u32, x0: u32, y0: u32, rw: u32, rh: u32) -> GrayImage {
530 let mut m = GrayImage::new(w, h);
531 for y in y0..(y0 + rh) {
532 for x in x0..(x0 + rw) {
533 m.put_pixel(x, y, image::Luma([255]));
534 }
535 }
536 m
537 }
538
539 #[test]
540 fn hd_crop_box_is_none_for_an_empty_mask() {
541 assert_eq!(hd_crop_box(&GrayImage::new(1024, 768), 1024, 768), None);
542 }
543
544 #[test]
545 fn hd_crop_box_brackets_a_small_object_with_a_square_in_bounds_crop() {
546 let mask = mask_with_rect(1024, 768, 412, 284, 200, 200);
548 let (x, y, cw, ch) = hd_crop_box(&mask, 1024, 768).expect("should crop");
549 assert_eq!(cw, ch);
552 assert!(
553 cw < 1024,
554 "crop {cw} should be smaller than the frame width"
555 );
556 assert!(cw >= 200, "crop must at least contain the object");
557 assert!(x + cw <= 1024 && y + ch <= 768);
559 }
560
561 #[test]
562 fn hd_crop_box_clamps_a_corner_object_inside_the_frame() {
563 let mask = mask_with_rect(1024, 768, 0, 0, 180, 180);
564 let (x, y, cw, ch) = hd_crop_box(&mask, 1024, 768).expect("should crop");
565 assert_eq!((x, y), (0, 0));
566 assert!(x + cw <= 1024 && y + ch <= 768);
567 }
568
569 #[test]
570 fn hd_crop_box_falls_back_to_none_when_the_mask_spans_the_frame() {
571 let mask = mask_with_rect(512, 512, 0, 0, 512, 512);
572 assert_eq!(hd_crop_box(&mask, 512, 512), None);
573 }
574
575 #[test]
576 fn detect_scale_distinguishes_unit_and_byte_ranges() {
577 assert_eq!(detect_scale(&[0.0, 0.5, 1.0]), 255.0);
578 assert_eq!(detect_scale(&[0.0, 128.0, 240.0]), 1.0);
579 assert_eq!(detect_scale(&[0.0, 0.0]), 255.0);
581 }
582
583 #[test]
584 fn chw_to_rgb_roundtrips_unit_scale() {
585 let n = (LAMA_SIZE * LAMA_SIZE) as usize;
586 let mut buf = vec![0.0_f32; 3 * n];
587 buf[0] = 1.0; buf[2 * n + 1] = 1.0; let img = chw_to_rgb(&buf, LAMA_SIZE, 255.0).unwrap();
590 assert_eq!(img.get_pixel(0, 0).0, [255, 0, 0]);
591 assert_eq!(img.get_pixel(1, 0).0, [0, 0, 255]);
592 }
593
594 #[test]
595 fn chw_to_rgb_rejects_short_buffer() {
596 assert!(chw_to_rgb(&[0.0; 10], LAMA_SIZE, 255.0).is_err());
597 }
598
599 #[test]
600 fn alpha_composite_blends_by_mask() {
601 let base = RgbImage::from_pixel(2, 1, image::Rgb([0, 0, 0]));
602 let fill = RgbImage::from_pixel(2, 1, image::Rgb([100, 100, 100]));
603 let mut alpha = GrayImage::new(2, 1);
604 alpha.put_pixel(0, 0, image::Luma([0])); alpha.put_pixel(1, 0, image::Luma([255])); let out = alpha_composite(&base, &fill, &alpha);
607 assert_eq!(out.get_pixel(0, 0).0, [0, 0, 0]);
608 assert_eq!(out.get_pixel(1, 0).0, [100, 100, 100]);
609 }
610
611 #[test]
612 fn alpha_composite_half_blends_midpoint() {
613 let base = RgbImage::from_pixel(1, 1, image::Rgb([0, 0, 0]));
614 let fill = RgbImage::from_pixel(1, 1, image::Rgb([200, 200, 200]));
615 let alpha = GrayImage::from_pixel(1, 1, image::Luma([128]));
616 let out = alpha_composite(&base, &fill, &alpha);
617 let v = out.get_pixel(0, 0).0[0];
619 assert!((99..=101).contains(&v), "got {v}");
620 }
621
622 #[test]
630 #[ignore = "needs the real LaMa onnx model + assets via env"]
631 fn lama_removal_end_to_end() {
632 let onnx = std::env::var("LAMA_ONNX").expect("LAMA_ONNX");
633 let init = std::env::var("LAMA_INIT").expect("LAMA_INIT");
634 let mask = std::env::var("LAMA_MASK").expect("LAMA_MASK");
635 let params = ImageParams {
636 width: 1024,
637 height: 768,
638 ext: "webp".into(),
639 ..Default::default()
640 };
641 let engine = OnnxImageEngine::new(std::env::temp_dir());
642 let bytes = engine
643 .remove(
644 std::path::Path::new(&onnx),
645 std::path::Path::new(&init),
646 std::path::Path::new(&mask),
647 ¶ms,
648 )
649 .expect("removal");
650 assert!(!bytes.is_empty(), "empty output");
651 let out = image::load_from_memory(&bytes)
652 .expect("decode output")
653 .to_rgb8();
654 assert_eq!(out.dimensions(), (1024, 768));
655 if let Ok(out_path) = std::env::var("LAMA_OUT") {
656 std::fs::write(&out_path, &bytes).expect("write out");
657 }
658 let original = image::load_from_memory(&std::fs::read(&init).unwrap())
662 .unwrap()
663 .resize_exact(1024, 768, FilterType::Triangle)
664 .to_rgb8();
665 let d_corner: i32 = (0..3)
666 .map(|i| {
667 (out.get_pixel(20, 20).0[i] as i32 - original.get_pixel(20, 20).0[i] as i32).abs()
668 })
669 .sum();
670 assert!(d_corner < 16, "outside-mask pixel drifted: {d_corner}");
671 }
672}