1#![cfg(target_os = "linux")]
5
6use crate::{
7 CPUProcessor, Crop, Error, Flip, ImageProcessorTrait, Result, Rotation, TensorImage,
8 TensorImageRef, NV12, RGB, RGBA, YUYV,
9};
10use edgefirst_tensor::Tensor;
11use g2d_sys::{G2DFormat, G2DPhysical, G2DSurface, G2D};
12use std::{os::fd::AsRawFd, time::Instant};
13
14#[derive(Debug)]
17pub struct G2DProcessor {
18 g2d: G2D,
19}
20
21unsafe impl Send for G2DProcessor {}
22unsafe impl Sync for G2DProcessor {}
23
24impl G2DProcessor {
25 pub fn new() -> Result<Self> {
27 let mut g2d = G2D::new("libg2d.so.2")?;
28 g2d.set_bt709_colorspace()?;
29
30 log::debug!("G2DConverter created with version {:?}", g2d.version());
31 Ok(Self { g2d })
32 }
33
34 pub fn version(&self) -> g2d_sys::Version {
37 self.g2d.version()
38 }
39
40 fn convert_(
41 &mut self,
42 src: &TensorImage,
43 dst: &mut TensorImage,
44 rotation: Rotation,
45 flip: Flip,
46 crop: Crop,
47 ) -> Result<()> {
48 let mut src_surface: G2DSurface = src.try_into()?;
49 let mut dst_surface: G2DSurface = dst.try_into()?;
50
51 src_surface.rot = match flip {
52 Flip::None => g2d_sys::g2d_rotation_G2D_ROTATION_0,
53 Flip::Vertical => g2d_sys::g2d_rotation_G2D_FLIP_V,
54 Flip::Horizontal => g2d_sys::g2d_rotation_G2D_FLIP_H,
55 };
56
57 dst_surface.rot = match rotation {
58 Rotation::None => g2d_sys::g2d_rotation_G2D_ROTATION_0,
59 Rotation::Clockwise90 => g2d_sys::g2d_rotation_G2D_ROTATION_90,
60 Rotation::Rotate180 => g2d_sys::g2d_rotation_G2D_ROTATION_180,
61 Rotation::CounterClockwise90 => g2d_sys::g2d_rotation_G2D_ROTATION_270,
62 };
63
64 if let Some(crop_rect) = crop.src_rect {
65 src_surface.left = crop_rect.left as i32;
66 src_surface.top = crop_rect.top as i32;
67 src_surface.right = (crop_rect.left + crop_rect.width) as i32;
68 src_surface.bottom = (crop_rect.top + crop_rect.height) as i32;
69 }
70
71 let needs_clear = crop.dst_color.is_some()
77 && crop.dst_rect.is_some_and(|dst_rect| {
78 dst_rect.left != 0
79 || dst_rect.top != 0
80 || dst_rect.width != dst.width()
81 || dst_rect.height != dst.height()
82 });
83
84 if needs_clear && dst.fourcc != RGB {
85 if let Some(dst_color) = crop.dst_color {
86 let start = Instant::now();
87 self.g2d.clear(&mut dst_surface, dst_color)?;
88 log::trace!("g2d clear takes {:?}", start.elapsed());
89 }
90 }
91
92 if let Some(crop_rect) = crop.dst_rect {
93 dst_surface.planes[0] += ((crop_rect.top * dst_surface.width as usize + crop_rect.left)
94 * dst.channels()) as u64;
95
96 dst_surface.right = crop_rect.width as i32;
97 dst_surface.bottom = crop_rect.height as i32;
98 dst_surface.width = crop_rect.width as i32;
99 dst_surface.height = crop_rect.height as i32;
100 }
101
102 log::trace!("Blitting from {src_surface:?} to {dst_surface:?}");
103 self.g2d.blit(&src_surface, &dst_surface)?;
104 self.g2d.finish()?;
105
106 if needs_clear && dst.fourcc == RGB {
108 if let (Some(dst_color), Some(dst_rect)) = (crop.dst_color, crop.dst_rect) {
109 let start = Instant::now();
110 CPUProcessor::fill_image_outside_crop(dst, dst_color, dst_rect)?;
111 log::trace!("cpu fill takes {:?}", start.elapsed());
112 }
113 }
114
115 Ok(())
116 }
117}
118
119impl ImageProcessorTrait for G2DProcessor {
120 fn convert(
135 &mut self,
136 src: &TensorImage,
137 dst: &mut TensorImage,
138 rotation: Rotation,
139 flip: Flip,
140 crop: Crop,
141 ) -> Result<()> {
142 crop.check_crop(src, dst)?;
143 match (src.fourcc(), dst.fourcc()) {
144 (RGBA, RGBA) => {}
145 (RGBA, YUYV) => {}
146 (RGBA, RGB) => {}
147 (YUYV, RGBA) => {}
148 (YUYV, YUYV) => {}
149 (YUYV, RGB) => {}
150 (NV12, RGBA) => {}
151 (NV12, YUYV) => {}
152 (NV12, RGB) => {}
153 (s, d) => {
154 return Err(Error::NotSupported(format!(
155 "G2D does not support {} to {} conversion",
156 s.display(),
157 d.display()
158 )));
159 }
160 }
161 self.convert_(src, dst, rotation, flip, crop)
162 }
163
164 fn convert_ref(
165 &mut self,
166 src: &TensorImage,
167 dst: &mut TensorImageRef<'_>,
168 rotation: Rotation,
169 flip: Flip,
170 crop: Crop,
171 ) -> Result<()> {
172 let mut cpu = CPUProcessor::new();
174 cpu.convert_ref(src, dst, rotation, flip, crop)
175 }
176
177 #[cfg(feature = "decoder")]
178 fn render_to_image(
179 &mut self,
180 _dst: &mut TensorImage,
181 _detect: &[crate::DetectBox],
182 _segmentation: &[crate::Segmentation],
183 ) -> Result<()> {
184 Err(Error::NotImplemented(
185 "G2D does not support rendering detection or segmentation overlays".to_string(),
186 ))
187 }
188
189 #[cfg(feature = "decoder")]
190 fn set_class_colors(&mut self, _: &[[u8; 4]]) -> Result<()> {
191 Err(Error::NotImplemented(
192 "G2D does not support setting colors for rendering detection or segmentation overlays"
193 .to_string(),
194 ))
195 }
196}
197
198impl TryFrom<&TensorImage> for G2DSurface {
199 type Error = Error;
200
201 fn try_from(img: &TensorImage) -> Result<Self, Self::Error> {
202 let phys: G2DPhysical = match img.tensor() {
203 Tensor::Dma(t) => t.as_raw_fd(),
204 _ => {
205 return Err(Error::NotImplemented(
206 "g2d only supports Dma memory".to_string(),
207 ));
208 }
209 }
210 .try_into()?;
211
212 let base_addr = phys.address();
216 let planes = if img.fourcc() == NV12 {
217 let uv_offset = (img.width() * img.height()) as u64;
218 [base_addr, base_addr + uv_offset, 0]
219 } else {
220 [base_addr, 0, 0]
221 };
222
223 Ok(Self {
224 planes,
225 format: G2DFormat::try_from(img.fourcc())?.format(),
226 left: 0,
227 top: 0,
228 right: img.width() as i32,
229 bottom: img.height() as i32,
230 stride: img.width() as i32,
231 width: img.width() as i32,
232 height: img.height() as i32,
233 blendfunc: 0,
234 clrcolor: 0,
235 rot: 0,
236 global_alpha: 0,
237 })
238 }
239}
240
241impl TryFrom<&mut TensorImage> for G2DSurface {
242 type Error = Error;
243
244 fn try_from(img: &mut TensorImage) -> Result<Self, Self::Error> {
245 let phys: G2DPhysical = match img.tensor() {
246 Tensor::Dma(t) => t.as_raw_fd(),
247 _ => {
248 return Err(Error::NotImplemented(
249 "g2d only supports Dma memory".to_string(),
250 ));
251 }
252 }
253 .try_into()?;
254
255 let base_addr = phys.address();
257 let planes = if img.fourcc() == NV12 {
258 let uv_offset = (img.width() * img.height()) as u64;
259 [base_addr, base_addr + uv_offset, 0]
260 } else {
261 [base_addr, 0, 0]
262 };
263
264 Ok(Self {
265 planes,
266 format: G2DFormat::try_from(img.fourcc())?.format(),
267 left: 0,
268 top: 0,
269 right: img.width() as i32,
270 bottom: img.height() as i32,
271 stride: img.width() as i32,
272 width: img.width() as i32,
273 height: img.height() as i32,
274 blendfunc: 0,
275 clrcolor: 0,
276 rot: 0,
277 global_alpha: 0,
278 })
279 }
280}
281
282#[cfg(feature = "g2d_test_formats")]
283#[cfg(test)]
284mod g2d_tests {
285 use super::*;
286 use crate::{
287 CPUProcessor, Flip, G2DProcessor, ImageProcessorTrait, Rect, Rotation, TensorImage, GREY,
288 NV12, RGB, RGBA, YUYV,
289 };
290 use edgefirst_tensor::{TensorMapTrait, TensorMemory, TensorTrait};
291 use four_char_code::FourCharCode;
292 use image::buffer::ConvertBuffer;
293
294 #[test]
295 #[cfg(target_os = "linux")]
296 fn test_g2d_formats_no_resize() {
297 for i in [RGBA, YUYV, RGB, GREY, NV12] {
298 for o in [RGBA, YUYV, RGB, GREY] {
299 let res = test_g2d_format_no_resize_(i, o);
300 if let Err(e) = res {
301 println!("{} to {} failed: {e:?}", i.display(), o.display());
302 } else {
303 println!("{} to {} success", i.display(), o.display());
304 }
305 }
306 }
307 }
308
309 fn test_g2d_format_no_resize_(
310 g2d_in_fmt: FourCharCode,
311 g2d_out_fmt: FourCharCode,
312 ) -> Result<(), crate::Error> {
313 let dst_width = 1280;
314 let dst_height = 720;
315 let file = include_bytes!("../../../testdata/zidane.jpg").to_vec();
316 let src = TensorImage::load_jpeg(&file, Some(RGB), None)?;
317
318 let mut src2 = TensorImage::new(1280, 720, g2d_in_fmt, Some(TensorMemory::Dma))?;
320
321 let mut cpu_converter = CPUProcessor::new();
322
323 if g2d_in_fmt == NV12 {
325 let nv12_bytes = include_bytes!("../../../testdata/zidane.nv12");
326 src2.tensor()
327 .map()?
328 .as_mut_slice()
329 .copy_from_slice(nv12_bytes);
330 } else {
331 cpu_converter.convert(&src, &mut src2, Rotation::None, Flip::None, Crop::no_crop())?;
332 }
333
334 let mut g2d_dst =
335 TensorImage::new(dst_width, dst_height, g2d_out_fmt, Some(TensorMemory::Dma))?;
336 let mut g2d_converter = G2DProcessor::new()?;
337 g2d_converter.convert_(
338 &src2,
339 &mut g2d_dst,
340 Rotation::None,
341 Flip::None,
342 Crop::no_crop(),
343 )?;
344
345 let mut cpu_dst = TensorImage::new(dst_width, dst_height, RGB, None)?;
346 cpu_converter.convert(
347 &g2d_dst,
348 &mut cpu_dst,
349 Rotation::None,
350 Flip::None,
351 Crop::no_crop(),
352 )?;
353
354 compare_images(
355 &src,
356 &cpu_dst,
357 0.98,
358 &format!("{}_to_{}", g2d_in_fmt.display(), g2d_out_fmt.display()),
359 )
360 }
361
362 #[test]
363 #[cfg(target_os = "linux")]
364 fn test_g2d_formats_with_resize() {
365 for i in [RGBA, YUYV, RGB, GREY, NV12] {
366 for o in [RGBA, YUYV, RGB, GREY] {
367 let res = test_g2d_format_with_resize_(i, o);
368 if let Err(e) = res {
369 println!("{} to {} failed: {e:?}", i.display(), o.display());
370 } else {
371 println!("{} to {} success", i.display(), o.display());
372 }
373 }
374 }
375 }
376
377 #[test]
378 #[cfg(target_os = "linux")]
379 fn test_g2d_formats_with_resize_dst_crop() {
380 for i in [RGBA, YUYV, RGB, GREY, NV12] {
381 for o in [RGBA, YUYV, RGB, GREY] {
382 let res = test_g2d_format_with_resize_dst_crop(i, o);
383 if let Err(e) = res {
384 println!("{} to {} failed: {e:?}", i.display(), o.display());
385 } else {
386 println!("{} to {} success", i.display(), o.display());
387 }
388 }
389 }
390 }
391
392 fn test_g2d_format_with_resize_(
393 g2d_in_fmt: FourCharCode,
394 g2d_out_fmt: FourCharCode,
395 ) -> Result<(), crate::Error> {
396 let dst_width = 600;
397 let dst_height = 400;
398 let file = include_bytes!("../../../testdata/zidane.jpg").to_vec();
399 let src = TensorImage::load_jpeg(&file, Some(RGB), None)?;
400
401 let mut cpu_converter = CPUProcessor::new();
402
403 let mut reference = TensorImage::new(dst_width, dst_height, RGB, Some(TensorMemory::Dma))?;
404 cpu_converter.convert(
405 &src,
406 &mut reference,
407 Rotation::None,
408 Flip::None,
409 Crop::no_crop(),
410 )?;
411
412 let mut src2 = TensorImage::new(1280, 720, g2d_in_fmt, Some(TensorMemory::Dma))?;
414
415 if g2d_in_fmt == NV12 {
417 let nv12_bytes = include_bytes!("../../../testdata/zidane.nv12");
418 src2.tensor()
419 .map()?
420 .as_mut_slice()
421 .copy_from_slice(nv12_bytes);
422 } else {
423 cpu_converter.convert(&src, &mut src2, Rotation::None, Flip::None, Crop::no_crop())?;
424 }
425
426 let mut g2d_dst =
427 TensorImage::new(dst_width, dst_height, g2d_out_fmt, Some(TensorMemory::Dma))?;
428 let mut g2d_converter = G2DProcessor::new()?;
429 g2d_converter.convert_(
430 &src2,
431 &mut g2d_dst,
432 Rotation::None,
433 Flip::None,
434 Crop::no_crop(),
435 )?;
436
437 let mut cpu_dst = TensorImage::new(dst_width, dst_height, RGB, None)?;
438 cpu_converter.convert(
439 &g2d_dst,
440 &mut cpu_dst,
441 Rotation::None,
442 Flip::None,
443 Crop::no_crop(),
444 )?;
445
446 compare_images(
447 &reference,
448 &cpu_dst,
449 0.98,
450 &format!(
451 "{}_to_{}_resized",
452 g2d_in_fmt.display(),
453 g2d_out_fmt.display()
454 ),
455 )
456 }
457
458 fn test_g2d_format_with_resize_dst_crop(
459 g2d_in_fmt: FourCharCode,
460 g2d_out_fmt: FourCharCode,
461 ) -> Result<(), crate::Error> {
462 let dst_width = 600;
463 let dst_height = 400;
464 let crop = Crop {
465 src_rect: None,
466 dst_rect: Some(Rect {
467 top: 100,
468 left: 100,
469 height: 100,
470 width: 200,
471 }),
472 dst_color: None,
473 };
474 let file = include_bytes!("../../../testdata/zidane.jpg").to_vec();
475 let src = TensorImage::load_jpeg(&file, Some(RGB), None)?;
476
477 let mut cpu_converter = CPUProcessor::new();
478
479 let mut reference = TensorImage::new(dst_width, dst_height, RGB, Some(TensorMemory::Dma))?;
480 reference.tensor.map().unwrap().as_mut_slice().fill(128);
481 cpu_converter.convert(&src, &mut reference, Rotation::None, Flip::None, crop)?;
482
483 let mut src2 = TensorImage::new(1280, 720, g2d_in_fmt, Some(TensorMemory::Dma))?;
485
486 if g2d_in_fmt == NV12 {
488 let nv12_bytes = include_bytes!("../../../testdata/zidane.nv12");
489 src2.tensor()
490 .map()?
491 .as_mut_slice()
492 .copy_from_slice(nv12_bytes);
493 } else {
494 cpu_converter.convert(&src, &mut src2, Rotation::None, Flip::None, Crop::no_crop())?;
495 }
496
497 let mut g2d_dst =
498 TensorImage::new(dst_width, dst_height, g2d_out_fmt, Some(TensorMemory::Dma))?;
499 g2d_dst.tensor.map().unwrap().as_mut_slice().fill(128);
500 let mut g2d_converter = G2DProcessor::new()?;
501 g2d_converter.convert_(&src2, &mut g2d_dst, Rotation::None, Flip::None, crop)?;
502
503 let mut cpu_dst = TensorImage::new(dst_width, dst_height, RGB, None)?;
504 cpu_converter.convert(
505 &g2d_dst,
506 &mut cpu_dst,
507 Rotation::None,
508 Flip::None,
509 Crop::no_crop(),
510 )?;
511
512 compare_images(
513 &reference,
514 &cpu_dst,
515 0.98,
516 &format!(
517 "{}_to_{}_resized_dst_crop",
518 g2d_in_fmt.display(),
519 g2d_out_fmt.display()
520 ),
521 )
522 }
523
524 fn compare_images(
525 img1: &TensorImage,
526 img2: &TensorImage,
527 threshold: f64,
528 name: &str,
529 ) -> Result<(), crate::Error> {
530 assert_eq!(img1.height(), img2.height(), "Heights differ");
531 assert_eq!(img1.width(), img2.width(), "Widths differ");
532 assert_eq!(img1.fourcc(), img2.fourcc(), "FourCC differ");
533 assert!(
534 matches!(img1.fourcc(), RGB | RGBA),
535 "FourCC must be RGB or RGBA for comparison"
536 );
537 let image1 = match img1.fourcc() {
538 RGB => image::RgbImage::from_vec(
539 img1.width() as u32,
540 img1.height() as u32,
541 img1.tensor().map().unwrap().to_vec(),
542 )
543 .unwrap(),
544 RGBA => image::RgbaImage::from_vec(
545 img1.width() as u32,
546 img1.height() as u32,
547 img1.tensor().map().unwrap().to_vec(),
548 )
549 .unwrap()
550 .convert(),
551
552 _ => unreachable!(),
553 };
554
555 let image2 = match img2.fourcc() {
556 RGB => image::RgbImage::from_vec(
557 img2.width() as u32,
558 img2.height() as u32,
559 img2.tensor().map().unwrap().to_vec(),
560 )
561 .unwrap(),
562 RGBA => image::RgbaImage::from_vec(
563 img2.width() as u32,
564 img2.height() as u32,
565 img2.tensor().map().unwrap().to_vec(),
566 )
567 .unwrap()
568 .convert(),
569
570 _ => unreachable!(),
571 };
572
573 let similarity = image_compare::rgb_similarity_structure(
574 &image_compare::Algorithm::RootMeanSquared,
575 &image1,
576 &image2,
577 )
578 .expect("Image Comparison failed");
579
580 if similarity.score < threshold {
581 image1.save(format!("{name}_1.png")).unwrap();
582 image2.save(format!("{name}_2.png")).unwrap();
583 return Err(Error::Internal(format!(
589 "{name}: converted image and target image have similarity score too low: {} < {}",
590 similarity.score, threshold
591 )));
592 }
593 Ok(())
594 }
595
596 fn load_raw_image(
602 width: usize,
603 height: usize,
604 fourcc: FourCharCode,
605 memory: Option<TensorMemory>,
606 bytes: &[u8],
607 ) -> Result<TensorImage, crate::Error> {
608 let img = TensorImage::new(width, height, fourcc, memory)?;
609 let mut map = img.tensor().map()?;
610 map.as_mut_slice()[..bytes.len()].copy_from_slice(bytes);
611 Ok(img)
612 }
613
614 #[test]
616 #[cfg(target_os = "linux")]
617 fn test_g2d_nv12_to_rgba_reference() -> Result<(), crate::Error> {
618 let src = load_raw_image(
620 1280,
621 720,
622 NV12,
623 Some(TensorMemory::Dma),
624 include_bytes!("../../../testdata/camera720p.nv12"),
625 )?;
626
627 let reference = load_raw_image(
629 1280,
630 720,
631 RGBA,
632 None,
633 include_bytes!("../../../testdata/camera720p.rgba"),
634 )?;
635
636 let mut dst = TensorImage::new(1280, 720, RGBA, Some(TensorMemory::Dma))?;
638 let mut g2d = G2DProcessor::new()?;
639 g2d.convert_(&src, &mut dst, Rotation::None, Flip::None, Crop::no_crop())?;
640
641 let cpu_dst = TensorImage::new(1280, 720, RGBA, None)?;
643 cpu_dst
644 .tensor()
645 .map()?
646 .as_mut_slice()
647 .copy_from_slice(dst.tensor().map()?.as_slice());
648
649 compare_images(&reference, &cpu_dst, 0.98, "g2d_nv12_to_rgba_reference")
650 }
651
652 #[test]
654 #[cfg(target_os = "linux")]
655 fn test_g2d_nv12_to_rgb_reference() -> Result<(), crate::Error> {
656 let src = load_raw_image(
658 1280,
659 720,
660 NV12,
661 Some(TensorMemory::Dma),
662 include_bytes!("../../../testdata/camera720p.nv12"),
663 )?;
664
665 let reference = load_raw_image(
667 1280,
668 720,
669 RGB,
670 None,
671 include_bytes!("../../../testdata/camera720p.rgb"),
672 )?;
673
674 let mut dst = TensorImage::new(1280, 720, RGB, Some(TensorMemory::Dma))?;
676 let mut g2d = G2DProcessor::new()?;
677 g2d.convert_(&src, &mut dst, Rotation::None, Flip::None, Crop::no_crop())?;
678
679 let cpu_dst = TensorImage::new(1280, 720, RGB, None)?;
681 cpu_dst
682 .tensor()
683 .map()?
684 .as_mut_slice()
685 .copy_from_slice(dst.tensor().map()?.as_slice());
686
687 compare_images(&reference, &cpu_dst, 0.98, "g2d_nv12_to_rgb_reference")
688 }
689
690 #[test]
692 #[cfg(target_os = "linux")]
693 fn test_g2d_yuyv_to_rgba_reference() -> Result<(), crate::Error> {
694 let src = load_raw_image(
696 1280,
697 720,
698 YUYV,
699 Some(TensorMemory::Dma),
700 include_bytes!("../../../testdata/camera720p.yuyv"),
701 )?;
702
703 let reference = load_raw_image(
705 1280,
706 720,
707 RGBA,
708 None,
709 include_bytes!("../../../testdata/camera720p.rgba"),
710 )?;
711
712 let mut dst = TensorImage::new(1280, 720, RGBA, Some(TensorMemory::Dma))?;
714 let mut g2d = G2DProcessor::new()?;
715 g2d.convert_(&src, &mut dst, Rotation::None, Flip::None, Crop::no_crop())?;
716
717 let cpu_dst = TensorImage::new(1280, 720, RGBA, None)?;
719 cpu_dst
720 .tensor()
721 .map()?
722 .as_mut_slice()
723 .copy_from_slice(dst.tensor().map()?.as_slice());
724
725 compare_images(&reference, &cpu_dst, 0.98, "g2d_yuyv_to_rgba_reference")
726 }
727
728 #[test]
730 #[cfg(target_os = "linux")]
731 fn test_g2d_yuyv_to_rgb_reference() -> Result<(), crate::Error> {
732 let src = load_raw_image(
734 1280,
735 720,
736 YUYV,
737 Some(TensorMemory::Dma),
738 include_bytes!("../../../testdata/camera720p.yuyv"),
739 )?;
740
741 let reference = load_raw_image(
743 1280,
744 720,
745 RGB,
746 None,
747 include_bytes!("../../../testdata/camera720p.rgb"),
748 )?;
749
750 let mut dst = TensorImage::new(1280, 720, RGB, Some(TensorMemory::Dma))?;
752 let mut g2d = G2DProcessor::new()?;
753 g2d.convert_(&src, &mut dst, Rotation::None, Flip::None, Crop::no_crop())?;
754
755 let cpu_dst = TensorImage::new(1280, 720, RGB, None)?;
757 cpu_dst
758 .tensor()
759 .map()?
760 .as_mut_slice()
761 .copy_from_slice(dst.tensor().map()?.as_slice());
762
763 compare_images(&reference, &cpu_dst, 0.98, "g2d_yuyv_to_rgb_reference")
764 }
765}