1#![cfg(target_os = "linux")]
5
6use crate::{
7 CPUProcessor, Crop, Error, Flip, ImageProcessorTrait, Result, Rotation, TensorImage,
8 TensorImageRef, NV12, RGB, RGBA, YUYV,
9};
10use edgefirst_tensor::Tensor;
11use g2d_sys::{G2DFormat, G2DPhysical, G2DSurface, G2D};
12use std::{os::fd::AsRawFd, time::Instant};
13
14#[derive(Debug)]
17pub struct G2DProcessor {
18 g2d: G2D,
19}
20
21unsafe impl Send for G2DProcessor {}
22unsafe impl Sync for G2DProcessor {}
23
24impl G2DProcessor {
25 pub fn new() -> Result<Self> {
27 let mut g2d = G2D::new("libg2d.so.2")?;
28 g2d.set_bt709_colorspace()?;
29
30 log::debug!("G2DConverter created with version {:?}", g2d.version());
31 Ok(Self { g2d })
32 }
33
34 pub fn version(&self) -> g2d_sys::Version {
37 self.g2d.version()
38 }
39
40 fn convert_(
41 &mut self,
42 src: &TensorImage,
43 dst: &mut TensorImage,
44 rotation: Rotation,
45 flip: Flip,
46 crop: Crop,
47 ) -> Result<()> {
48 let mut src_surface: G2DSurface = src.try_into()?;
49 let mut dst_surface: G2DSurface = dst.try_into()?;
50
51 src_surface.rot = match flip {
52 Flip::None => g2d_sys::g2d_rotation_G2D_ROTATION_0,
53 Flip::Vertical => g2d_sys::g2d_rotation_G2D_FLIP_V,
54 Flip::Horizontal => g2d_sys::g2d_rotation_G2D_FLIP_H,
55 };
56
57 dst_surface.rot = match rotation {
58 Rotation::None => g2d_sys::g2d_rotation_G2D_ROTATION_0,
59 Rotation::Clockwise90 => g2d_sys::g2d_rotation_G2D_ROTATION_90,
60 Rotation::Rotate180 => g2d_sys::g2d_rotation_G2D_ROTATION_180,
61 Rotation::CounterClockwise90 => g2d_sys::g2d_rotation_G2D_ROTATION_270,
62 };
63
64 if let Some(crop_rect) = crop.src_rect {
65 src_surface.left = crop_rect.left as i32;
66 src_surface.top = crop_rect.top as i32;
67 src_surface.right = (crop_rect.left + crop_rect.width) as i32;
68 src_surface.bottom = (crop_rect.top + crop_rect.height) as i32;
69 }
70
71 let needs_clear = crop.dst_color.is_some()
73 && crop.dst_rect.is_some_and(|dst_rect| {
74 dst_rect.left != 0
75 || dst_rect.top != 0
76 || dst_rect.width != dst.width()
77 || dst_rect.height != dst.height()
78 });
79
80 let mut cleared = false;
81 if needs_clear && dst.fourcc != RGB {
82 if let (Some(dst_rect), Some(dst_color)) = (crop.dst_rect, crop.dst_color) {
83 if dst_rect.width * dst_rect.height < dst.width() * dst.height() / 2 {
84 let start = Instant::now();
85 self.g2d.clear(&mut dst_surface, dst_color)?;
86 log::trace!("clear takes {:?}", start.elapsed());
87 cleared = true;
88 }
89 }
90 }
91
92 if let Some(crop_rect) = crop.dst_rect {
93 dst_surface.planes[0] += ((crop_rect.top * dst_surface.width as usize + crop_rect.left)
94 * dst.channels()) as u64;
95
96 dst_surface.right = crop_rect.width as i32;
97 dst_surface.bottom = crop_rect.height as i32;
98 dst_surface.width = crop_rect.width as i32;
99 dst_surface.height = crop_rect.height as i32;
100
101 }
112
113 log::trace!("Blitting from {src_surface:?} to {dst_surface:?}");
114 self.g2d.blit(&src_surface, &dst_surface)?;
115
116 if needs_clear && !cleared {
117 if let (Some(dst_color), Some(dst_rect)) = (crop.dst_color, crop.dst_rect) {
118 let start = Instant::now();
119 CPUProcessor::fill_image_outside_crop(dst, dst_color, dst_rect)?;
120 log::trace!("clear takes {:?}", start.elapsed());
121 }
122 }
123
124 Ok(())
125 }
126}
127
128impl ImageProcessorTrait for G2DProcessor {
129 fn convert(
144 &mut self,
145 src: &TensorImage,
146 dst: &mut TensorImage,
147 rotation: Rotation,
148 flip: Flip,
149 crop: Crop,
150 ) -> Result<()> {
151 crop.check_crop(src, dst)?;
152 match (src.fourcc(), dst.fourcc()) {
153 (RGBA, RGBA) => {}
154 (RGBA, YUYV) => {}
155 (RGBA, RGB) => {}
156 (YUYV, RGBA) => {}
157 (YUYV, YUYV) => {}
158 (YUYV, RGB) => {}
159 (NV12, RGBA) => {}
160 (NV12, YUYV) => {}
161 (NV12, RGB) => {}
162 (s, d) => {
163 return Err(Error::NotSupported(format!(
164 "G2D does not support {} to {} conversion",
165 s.display(),
166 d.display()
167 )));
168 }
169 }
170 self.convert_(src, dst, rotation, flip, crop)
171 }
172
173 fn convert_ref(
174 &mut self,
175 src: &TensorImage,
176 dst: &mut TensorImageRef<'_>,
177 rotation: Rotation,
178 flip: Flip,
179 crop: Crop,
180 ) -> Result<()> {
181 let mut cpu = CPUProcessor::new();
183 cpu.convert_ref(src, dst, rotation, flip, crop)
184 }
185
186 #[cfg(feature = "decoder")]
187 fn render_to_image(
188 &mut self,
189 _dst: &mut TensorImage,
190 _detect: &[crate::DetectBox],
191 _segmentation: &[crate::Segmentation],
192 ) -> Result<()> {
193 Err(Error::NotImplemented(
194 "G2D does not support rendering detection or segmentation overlays".to_string(),
195 ))
196 }
197
198 #[cfg(feature = "decoder")]
199 fn set_class_colors(&mut self, _: &[[u8; 4]]) -> Result<()> {
200 Err(Error::NotImplemented(
201 "G2D does not support setting colors for rendering detection or segmentation overlays"
202 .to_string(),
203 ))
204 }
205}
206
207impl TryFrom<&TensorImage> for G2DSurface {
208 type Error = Error;
209
210 fn try_from(img: &TensorImage) -> Result<Self, Self::Error> {
211 let phys: G2DPhysical = match img.tensor() {
212 Tensor::Dma(t) => t.as_raw_fd(),
213 _ => {
214 return Err(Error::NotImplemented(
215 "g2d only supports Dma memory".to_string(),
216 ));
217 }
218 }
219 .try_into()?;
220
221 let base_addr = phys.address();
225 let planes = if img.fourcc() == NV12 {
226 let uv_offset = (img.width() * img.height()) as u64;
227 [base_addr, base_addr + uv_offset, 0]
228 } else {
229 [base_addr, 0, 0]
230 };
231
232 Ok(Self {
233 planes,
234 format: G2DFormat::try_from(img.fourcc())?.format(),
235 left: 0,
236 top: 0,
237 right: img.width() as i32,
238 bottom: img.height() as i32,
239 stride: img.width() as i32,
240 width: img.width() as i32,
241 height: img.height() as i32,
242 blendfunc: 0,
243 clrcolor: 0,
244 rot: 0,
245 global_alpha: 0,
246 })
247 }
248}
249
250impl TryFrom<&mut TensorImage> for G2DSurface {
251 type Error = Error;
252
253 fn try_from(img: &mut TensorImage) -> Result<Self, Self::Error> {
254 let phys: G2DPhysical = match img.tensor() {
255 Tensor::Dma(t) => t.as_raw_fd(),
256 _ => {
257 return Err(Error::NotImplemented(
258 "g2d only supports Dma memory".to_string(),
259 ));
260 }
261 }
262 .try_into()?;
263
264 let base_addr = phys.address();
266 let planes = if img.fourcc() == NV12 {
267 let uv_offset = (img.width() * img.height()) as u64;
268 [base_addr, base_addr + uv_offset, 0]
269 } else {
270 [base_addr, 0, 0]
271 };
272
273 Ok(Self {
274 planes,
275 format: G2DFormat::try_from(img.fourcc())?.format(),
276 left: 0,
277 top: 0,
278 right: img.width() as i32,
279 bottom: img.height() as i32,
280 stride: img.width() as i32,
281 width: img.width() as i32,
282 height: img.height() as i32,
283 blendfunc: 0,
284 clrcolor: 0,
285 rot: 0,
286 global_alpha: 0,
287 })
288 }
289}
290
291#[cfg(feature = "g2d_test_formats")]
292#[cfg(test)]
293mod g2d_tests {
294 use super::*;
295 use crate::{
296 CPUProcessor, Flip, G2DProcessor, ImageProcessorTrait, Rect, Rotation, TensorImage, GREY,
297 NV12, RGB, RGBA, YUYV,
298 };
299 use edgefirst_tensor::{TensorMapTrait, TensorMemory, TensorTrait};
300 use four_char_code::FourCharCode;
301 use image::buffer::ConvertBuffer;
302
303 #[test]
304 #[cfg(target_os = "linux")]
305 fn test_g2d_formats_no_resize() {
306 for i in [RGBA, YUYV, RGB, GREY, NV12] {
307 for o in [RGBA, YUYV, RGB, GREY] {
308 let res = test_g2d_format_no_resize_(i, o);
309 if let Err(e) = res {
310 println!("{} to {} failed: {e:?}", i.display(), o.display());
311 } else {
312 println!("{} to {} success", i.display(), o.display());
313 }
314 }
315 }
316 }
317
318 fn test_g2d_format_no_resize_(
319 g2d_in_fmt: FourCharCode,
320 g2d_out_fmt: FourCharCode,
321 ) -> Result<(), crate::Error> {
322 let dst_width = 1280;
323 let dst_height = 720;
324 let file = include_bytes!("../../../testdata/zidane.jpg").to_vec();
325 let src = TensorImage::load_jpeg(&file, Some(RGB), None)?;
326
327 let mut src2 = TensorImage::new(1280, 720, g2d_in_fmt, Some(TensorMemory::Dma))?;
329
330 let mut cpu_converter = CPUProcessor::new();
331
332 if g2d_in_fmt == NV12 {
334 let nv12_bytes = include_bytes!("../../../testdata/zidane.nv12");
335 src2.tensor()
336 .map()?
337 .as_mut_slice()
338 .copy_from_slice(nv12_bytes);
339 } else {
340 cpu_converter.convert(&src, &mut src2, Rotation::None, Flip::None, Crop::no_crop())?;
341 }
342
343 let mut g2d_dst =
344 TensorImage::new(dst_width, dst_height, g2d_out_fmt, Some(TensorMemory::Dma))?;
345 let mut g2d_converter = G2DProcessor::new()?;
346 g2d_converter.convert_(
347 &src2,
348 &mut g2d_dst,
349 Rotation::None,
350 Flip::None,
351 Crop::no_crop(),
352 )?;
353
354 let mut cpu_dst = TensorImage::new(dst_width, dst_height, RGB, None)?;
355 cpu_converter.convert(
356 &g2d_dst,
357 &mut cpu_dst,
358 Rotation::None,
359 Flip::None,
360 Crop::no_crop(),
361 )?;
362
363 compare_images(
364 &src,
365 &cpu_dst,
366 0.98,
367 &format!("{}_to_{}", g2d_in_fmt.display(), g2d_out_fmt.display()),
368 )
369 }
370
371 #[test]
372 #[cfg(target_os = "linux")]
373 fn test_g2d_formats_with_resize() {
374 for i in [RGBA, YUYV, RGB, GREY, NV12] {
375 for o in [RGBA, YUYV, RGB, GREY] {
376 let res = test_g2d_format_with_resize_(i, o);
377 if let Err(e) = res {
378 println!("{} to {} failed: {e:?}", i.display(), o.display());
379 } else {
380 println!("{} to {} success", i.display(), o.display());
381 }
382 }
383 }
384 }
385
386 #[test]
387 #[cfg(target_os = "linux")]
388 fn test_g2d_formats_with_resize_dst_crop() {
389 for i in [RGBA, YUYV, RGB, GREY, NV12] {
390 for o in [RGBA, YUYV, RGB, GREY] {
391 let res = test_g2d_format_with_resize_dst_crop(i, o);
392 if let Err(e) = res {
393 println!("{} to {} failed: {e:?}", i.display(), o.display());
394 } else {
395 println!("{} to {} success", i.display(), o.display());
396 }
397 }
398 }
399 }
400
401 fn test_g2d_format_with_resize_(
402 g2d_in_fmt: FourCharCode,
403 g2d_out_fmt: FourCharCode,
404 ) -> Result<(), crate::Error> {
405 let dst_width = 600;
406 let dst_height = 400;
407 let file = include_bytes!("../../../testdata/zidane.jpg").to_vec();
408 let src = TensorImage::load_jpeg(&file, Some(RGB), None)?;
409
410 let mut cpu_converter = CPUProcessor::new();
411
412 let mut reference = TensorImage::new(dst_width, dst_height, RGB, Some(TensorMemory::Dma))?;
413 cpu_converter.convert(
414 &src,
415 &mut reference,
416 Rotation::None,
417 Flip::None,
418 Crop::no_crop(),
419 )?;
420
421 let mut src2 = TensorImage::new(1280, 720, g2d_in_fmt, Some(TensorMemory::Dma))?;
423
424 if g2d_in_fmt == NV12 {
426 let nv12_bytes = include_bytes!("../../../testdata/zidane.nv12");
427 src2.tensor()
428 .map()?
429 .as_mut_slice()
430 .copy_from_slice(nv12_bytes);
431 } else {
432 cpu_converter.convert(&src, &mut src2, Rotation::None, Flip::None, Crop::no_crop())?;
433 }
434
435 let mut g2d_dst =
436 TensorImage::new(dst_width, dst_height, g2d_out_fmt, Some(TensorMemory::Dma))?;
437 let mut g2d_converter = G2DProcessor::new()?;
438 g2d_converter.convert_(
439 &src2,
440 &mut g2d_dst,
441 Rotation::None,
442 Flip::None,
443 Crop::no_crop(),
444 )?;
445
446 let mut cpu_dst = TensorImage::new(dst_width, dst_height, RGB, None)?;
447 cpu_converter.convert(
448 &g2d_dst,
449 &mut cpu_dst,
450 Rotation::None,
451 Flip::None,
452 Crop::no_crop(),
453 )?;
454
455 compare_images(
456 &reference,
457 &cpu_dst,
458 0.98,
459 &format!(
460 "{}_to_{}_resized",
461 g2d_in_fmt.display(),
462 g2d_out_fmt.display()
463 ),
464 )
465 }
466
467 fn test_g2d_format_with_resize_dst_crop(
468 g2d_in_fmt: FourCharCode,
469 g2d_out_fmt: FourCharCode,
470 ) -> Result<(), crate::Error> {
471 let dst_width = 600;
472 let dst_height = 400;
473 let crop = Crop {
474 src_rect: None,
475 dst_rect: Some(Rect {
476 top: 100,
477 left: 100,
478 height: 100,
479 width: 200,
480 }),
481 dst_color: None,
482 };
483 let file = include_bytes!("../../../testdata/zidane.jpg").to_vec();
484 let src = TensorImage::load_jpeg(&file, Some(RGB), None)?;
485
486 let mut cpu_converter = CPUProcessor::new();
487
488 let mut reference = TensorImage::new(dst_width, dst_height, RGB, Some(TensorMemory::Dma))?;
489 reference.tensor.map().unwrap().as_mut_slice().fill(128);
490 cpu_converter.convert(&src, &mut reference, Rotation::None, Flip::None, crop)?;
491
492 let mut src2 = TensorImage::new(1280, 720, g2d_in_fmt, Some(TensorMemory::Dma))?;
494
495 if g2d_in_fmt == NV12 {
497 let nv12_bytes = include_bytes!("../../../testdata/zidane.nv12");
498 src2.tensor()
499 .map()?
500 .as_mut_slice()
501 .copy_from_slice(nv12_bytes);
502 } else {
503 cpu_converter.convert(&src, &mut src2, Rotation::None, Flip::None, Crop::no_crop())?;
504 }
505
506 let mut g2d_dst =
507 TensorImage::new(dst_width, dst_height, g2d_out_fmt, Some(TensorMemory::Dma))?;
508 g2d_dst.tensor.map().unwrap().as_mut_slice().fill(128);
509 let mut g2d_converter = G2DProcessor::new()?;
510 g2d_converter.convert_(&src2, &mut g2d_dst, Rotation::None, Flip::None, crop)?;
511
512 let mut cpu_dst = TensorImage::new(dst_width, dst_height, RGB, None)?;
513 cpu_converter.convert(
514 &g2d_dst,
515 &mut cpu_dst,
516 Rotation::None,
517 Flip::None,
518 Crop::no_crop(),
519 )?;
520
521 compare_images(
522 &reference,
523 &cpu_dst,
524 0.98,
525 &format!(
526 "{}_to_{}_resized_dst_crop",
527 g2d_in_fmt.display(),
528 g2d_out_fmt.display()
529 ),
530 )
531 }
532
533 fn compare_images(
534 img1: &TensorImage,
535 img2: &TensorImage,
536 threshold: f64,
537 name: &str,
538 ) -> Result<(), crate::Error> {
539 assert_eq!(img1.height(), img2.height(), "Heights differ");
540 assert_eq!(img1.width(), img2.width(), "Widths differ");
541 assert_eq!(img1.fourcc(), img2.fourcc(), "FourCC differ");
542 assert!(
543 matches!(img1.fourcc(), RGB | RGBA),
544 "FourCC must be RGB or RGBA for comparison"
545 );
546 let image1 = match img1.fourcc() {
547 RGB => image::RgbImage::from_vec(
548 img1.width() as u32,
549 img1.height() as u32,
550 img1.tensor().map().unwrap().to_vec(),
551 )
552 .unwrap(),
553 RGBA => image::RgbaImage::from_vec(
554 img1.width() as u32,
555 img1.height() as u32,
556 img1.tensor().map().unwrap().to_vec(),
557 )
558 .unwrap()
559 .convert(),
560
561 _ => unreachable!(),
562 };
563
564 let image2 = match img2.fourcc() {
565 RGB => image::RgbImage::from_vec(
566 img2.width() as u32,
567 img2.height() as u32,
568 img2.tensor().map().unwrap().to_vec(),
569 )
570 .unwrap(),
571 RGBA => image::RgbaImage::from_vec(
572 img2.width() as u32,
573 img2.height() as u32,
574 img2.tensor().map().unwrap().to_vec(),
575 )
576 .unwrap()
577 .convert(),
578
579 _ => unreachable!(),
580 };
581
582 let similarity = image_compare::rgb_similarity_structure(
583 &image_compare::Algorithm::RootMeanSquared,
584 &image1,
585 &image2,
586 )
587 .expect("Image Comparison failed");
588
589 if similarity.score < threshold {
590 image1.save(format!("{name}_1.png")).unwrap();
591 image2.save(format!("{name}_2.png")).unwrap();
592 return Err(Error::Internal(format!(
598 "{name}: converted image and target image have similarity score too low: {} < {}",
599 similarity.score, threshold
600 )));
601 }
602 Ok(())
603 }
604
605 fn load_raw_image(
611 width: usize,
612 height: usize,
613 fourcc: FourCharCode,
614 memory: Option<TensorMemory>,
615 bytes: &[u8],
616 ) -> Result<TensorImage, crate::Error> {
617 let img = TensorImage::new(width, height, fourcc, memory)?;
618 let mut map = img.tensor().map()?;
619 map.as_mut_slice()[..bytes.len()].copy_from_slice(bytes);
620 Ok(img)
621 }
622
623 #[test]
625 #[cfg(target_os = "linux")]
626 fn test_g2d_nv12_to_rgba_reference() -> Result<(), crate::Error> {
627 let src = load_raw_image(
629 1280,
630 720,
631 NV12,
632 Some(TensorMemory::Dma),
633 include_bytes!("../../../testdata/camera720p.nv12"),
634 )?;
635
636 let reference = load_raw_image(
638 1280,
639 720,
640 RGBA,
641 None,
642 include_bytes!("../../../testdata/camera720p.rgba"),
643 )?;
644
645 let mut dst = TensorImage::new(1280, 720, RGBA, Some(TensorMemory::Dma))?;
647 let mut g2d = G2DProcessor::new()?;
648 g2d.convert_(&src, &mut dst, Rotation::None, Flip::None, Crop::no_crop())?;
649
650 let cpu_dst = TensorImage::new(1280, 720, RGBA, None)?;
652 cpu_dst
653 .tensor()
654 .map()?
655 .as_mut_slice()
656 .copy_from_slice(dst.tensor().map()?.as_slice());
657
658 compare_images(&reference, &cpu_dst, 0.98, "g2d_nv12_to_rgba_reference")
659 }
660
661 #[test]
663 #[cfg(target_os = "linux")]
664 fn test_g2d_nv12_to_rgb_reference() -> Result<(), crate::Error> {
665 let src = load_raw_image(
667 1280,
668 720,
669 NV12,
670 Some(TensorMemory::Dma),
671 include_bytes!("../../../testdata/camera720p.nv12"),
672 )?;
673
674 let reference = load_raw_image(
676 1280,
677 720,
678 RGB,
679 None,
680 include_bytes!("../../../testdata/camera720p.rgb"),
681 )?;
682
683 let mut dst = TensorImage::new(1280, 720, RGB, Some(TensorMemory::Dma))?;
685 let mut g2d = G2DProcessor::new()?;
686 g2d.convert_(&src, &mut dst, Rotation::None, Flip::None, Crop::no_crop())?;
687
688 let cpu_dst = TensorImage::new(1280, 720, RGB, None)?;
690 cpu_dst
691 .tensor()
692 .map()?
693 .as_mut_slice()
694 .copy_from_slice(dst.tensor().map()?.as_slice());
695
696 compare_images(&reference, &cpu_dst, 0.98, "g2d_nv12_to_rgb_reference")
697 }
698
699 #[test]
701 #[cfg(target_os = "linux")]
702 fn test_g2d_yuyv_to_rgba_reference() -> Result<(), crate::Error> {
703 let src = load_raw_image(
705 1280,
706 720,
707 YUYV,
708 Some(TensorMemory::Dma),
709 include_bytes!("../../../testdata/camera720p.yuyv"),
710 )?;
711
712 let reference = load_raw_image(
714 1280,
715 720,
716 RGBA,
717 None,
718 include_bytes!("../../../testdata/camera720p.rgba"),
719 )?;
720
721 let mut dst = TensorImage::new(1280, 720, RGBA, Some(TensorMemory::Dma))?;
723 let mut g2d = G2DProcessor::new()?;
724 g2d.convert_(&src, &mut dst, Rotation::None, Flip::None, Crop::no_crop())?;
725
726 let cpu_dst = TensorImage::new(1280, 720, RGBA, None)?;
728 cpu_dst
729 .tensor()
730 .map()?
731 .as_mut_slice()
732 .copy_from_slice(dst.tensor().map()?.as_slice());
733
734 compare_images(&reference, &cpu_dst, 0.98, "g2d_yuyv_to_rgba_reference")
735 }
736
737 #[test]
739 #[cfg(target_os = "linux")]
740 fn test_g2d_yuyv_to_rgb_reference() -> Result<(), crate::Error> {
741 let src = load_raw_image(
743 1280,
744 720,
745 YUYV,
746 Some(TensorMemory::Dma),
747 include_bytes!("../../../testdata/camera720p.yuyv"),
748 )?;
749
750 let reference = load_raw_image(
752 1280,
753 720,
754 RGB,
755 None,
756 include_bytes!("../../../testdata/camera720p.rgb"),
757 )?;
758
759 let mut dst = TensorImage::new(1280, 720, RGB, Some(TensorMemory::Dma))?;
761 let mut g2d = G2DProcessor::new()?;
762 g2d.convert_(&src, &mut dst, Rotation::None, Flip::None, Crop::no_crop())?;
763
764 let cpu_dst = TensorImage::new(1280, 720, RGB, None)?;
766 cpu_dst
767 .tensor()
768 .map()?
769 .as_mut_slice()
770 .copy_from_slice(dst.tensor().map()?.as_slice());
771
772 compare_images(&reference, &cpu_dst, 0.98, "g2d_yuyv_to_rgb_reference")
773 }
774}