1#![cfg(target_os = "linux")]
5#![cfg(feature = "opengl")]
6
7use edgefirst_decoder::DetectBox;
8#[cfg(feature = "decoder")]
9use edgefirst_decoder::Segmentation;
10use edgefirst_tensor::{TensorMemory, TensorTrait};
11use four_char_code::FourCharCode;
12use gbm::{
13 drm::{buffer::DrmFourcc, control::Device as DrmControlDevice, Device as DrmDevice},
14 AsRaw, Device,
15};
16use khronos_egl::{self as egl, Attrib, Display, Dynamic, Instance, EGL1_4};
17use log::{debug, error};
18use std::{
19 collections::BTreeSet,
20 ffi::{c_char, c_void, CStr, CString},
21 os::fd::AsRawFd,
22 ptr::{null, null_mut, NonNull},
23 rc::Rc,
24 str::FromStr,
25 sync::OnceLock,
26 thread::JoinHandle,
27 time::Instant,
28};
29use tokio::sync::mpsc::Sender;
30
31macro_rules! function {
32 () => {{
33 fn f() {}
34 fn type_name_of<T>(_: T) -> &'static str {
35 std::any::type_name::<T>()
36 }
37 let name = type_name_of(f);
38
39 match &name[..name.len() - 3].rfind(':') {
41 Some(pos) => &name[pos + 1..name.len() - 3],
42 None => &name[..name.len() - 3],
43 }
44 }};
45}
46
47#[cfg(feature = "decoder")]
48use crate::DEFAULT_COLORS;
49use crate::{
50 CPUProcessor, Crop, Error, Flip, ImageProcessorTrait, Rect, Rotation, TensorImage,
51 TensorImageRef, GREY, NV12, PLANAR_RGB, PLANAR_RGBA, RGB, RGBA, YUYV,
52};
53
54static EGL_LIB: OnceLock<libloading::Library> = OnceLock::new();
55
56fn get_egl_lib() -> Result<&'static libloading::Library, crate::Error> {
57 if let Some(egl) = EGL_LIB.get() {
58 Ok(egl)
59 } else {
60 let egl = unsafe { libloading::Library::new("libEGL.so.1")? };
61 Ok(EGL_LIB.get_or_init(|| egl))
62 }
63}
64
65type Egl = Instance<Dynamic<&'static libloading::Library, EGL1_4>>;
66pub(crate) struct GlContext {
67 pub(crate) support_dma: bool,
68 pub(crate) surface: Option<egl::Surface>,
69 pub(crate) display: EglDisplayType,
70 pub(crate) ctx: egl::Context,
71 pub(crate) egl: Rc<Egl>,
72}
73
74pub(crate) enum EglDisplayType {
75 Default(egl::Display),
76 Gbm(egl::Display, #[allow(dead_code)] Device<Card>),
77 PlatformDisplay(egl::Display),
78}
79
80impl EglDisplayType {
81 fn as_display(&self) -> egl::Display {
82 match self {
83 EglDisplayType::Default(disp) => *disp,
84 EglDisplayType::Gbm(disp, _) => *disp,
85 EglDisplayType::PlatformDisplay(disp) => *disp,
86 }
87 }
88}
89
90impl GlContext {
91 pub(crate) fn new() -> Result<GlContext, crate::Error> {
92 let egl: Rc<Egl> =
94 Rc::new(unsafe { Instance::<Dynamic<_, EGL1_4>>::load_required_from(get_egl_lib()?)? });
95
96 if let Ok(headless) = Self::try_initialize_egl(egl.clone(), Self::egl_get_default_display) {
97 return Ok(headless);
98 } else {
99 log::debug!("Didn't initialize EGL with Default Display");
100 }
101
102 if let Ok(headless) = Self::try_initialize_egl(egl.clone(), Self::egl_get_gbm_display) {
103 return Ok(headless);
104 } else {
105 log::debug!("Didn't initialize EGL with GBM Display");
106 }
107
108 if let Ok(headless) =
109 Self::try_initialize_egl(egl.clone(), Self::egl_get_platform_display_from_device)
110 {
111 return Ok(headless);
112 } else {
113 log::debug!("Didn't initialize EGL with platform display from device enumeration");
114 }
115
116 Err(Error::OpenGl(
117 "Could not initialize EGL with any known method".to_string(),
118 ))
119 }
120
121 fn try_initialize_egl(
122 egl: Rc<Egl>,
123 display_fn: impl Fn(&Egl) -> Result<EglDisplayType, crate::Error>,
124 ) -> Result<GlContext, crate::Error> {
125 let display = display_fn(&egl)?;
126 log::debug!("egl initialize with display: {:x?}", display.as_display());
127 egl.initialize(display.as_display())?;
128 let attributes = [
129 egl::SURFACE_TYPE,
130 egl::PBUFFER_BIT,
131 egl::RENDERABLE_TYPE,
132 egl::OPENGL_ES3_BIT,
133 egl::RED_SIZE,
134 8,
135 egl::GREEN_SIZE,
136 8,
137 egl::BLUE_SIZE,
138 8,
139 egl::ALPHA_SIZE,
140 8,
141 egl::NONE,
142 ];
143
144 let config =
145 if let Some(config) = egl.choose_first_config(display.as_display(), &attributes)? {
146 config
147 } else {
148 return Err(crate::Error::NotImplemented(
149 "Did not find valid OpenGL ES config".to_string(),
150 ));
151 };
152
153 debug!("config: {config:?}");
154
155 let surface = Some(egl.create_pbuffer_surface(
156 display.as_display(),
157 config,
158 &[egl::WIDTH, 64, egl::HEIGHT, 64, egl::NONE],
159 )?);
160
161 egl.bind_api(egl::OPENGL_ES_API)?;
162 let context_attributes = [egl::CONTEXT_MAJOR_VERSION, 3, egl::NONE, egl::NONE];
163
164 let ctx = egl.create_context(display.as_display(), config, None, &context_attributes)?;
165 debug!("ctx: {ctx:?}");
166
167 egl.make_current(display.as_display(), surface, surface, Some(ctx))?;
168
169 let support_dma = Self::egl_check_support_dma(&egl).is_ok();
170 let headless = GlContext {
171 display,
172 ctx,
173 egl,
174 surface,
175 support_dma,
176 };
177 Ok(headless)
178 }
179
180 fn egl_get_default_display(egl: &Egl) -> Result<EglDisplayType, crate::Error> {
181 if let Some(display) = unsafe { egl.get_display(egl::DEFAULT_DISPLAY) } {
183 debug!("default display: {display:?}");
184 return Ok(EglDisplayType::Default(display));
185 }
186
187 Err(Error::OpenGl(
188 "Could not obtain EGL Default Display".to_string(),
189 ))
190 }
191
192 fn egl_get_gbm_display(egl: &Egl) -> Result<EglDisplayType, crate::Error> {
193 let gbm = Device::new(Card::open_global()?)?;
195
196 debug!("gbm: {gbm:?}");
197 let display = Self::egl_get_platform_display_with_fallback(
198 egl,
199 egl_ext::PLATFORM_GBM_KHR,
200 gbm.as_raw() as *mut c_void,
201 &[egl::ATTRIB_NONE],
202 )?;
203
204 Ok(EglDisplayType::Gbm(display, gbm))
205 }
206
207 fn egl_get_platform_display_from_device(egl: &Egl) -> Result<EglDisplayType, crate::Error> {
208 let extensions = egl.query_string(None, egl::EXTENSIONS)?;
209 let extensions = extensions.to_string_lossy();
210 log::debug!("EGL Extensions: {}", extensions);
211
212 if !extensions.contains("EGL_EXT_device_enumeration") {
213 return Err(Error::GLVersion(
214 "EGL doesn't supported EGL_EXT_device_enumeration extension".to_string(),
215 ));
216 }
217
218 type EGLDeviceEXT = *mut c_void;
219 let devices = if let Some(ext) = egl.get_proc_address("eglQueryDevicesEXT") {
220 let func: unsafe extern "system" fn(
221 max_devices: egl::Int,
222 devices: *mut EGLDeviceEXT,
223 num_devices: *mut egl::Int,
224 ) -> *const c_char = unsafe { std::mem::transmute(ext) };
225 let mut devices = [std::ptr::null_mut(); 10];
226 let mut num_devices = 0;
227 unsafe { func(devices.len() as i32, devices.as_mut_ptr(), &mut num_devices) };
228 for i in 0..num_devices {
229 log::debug!("EGL device: {:?}", devices[i as usize]);
230 }
231 devices[0..num_devices as usize].to_vec()
232 } else {
233 return Err(Error::GLVersion(
234 "EGL doesn't supported eglQueryDevicesEXT function".to_string(),
235 ));
236 };
237
238 if !extensions.contains("EGL_EXT_platform_device") {
239 return Err(Error::GLVersion(
240 "EGL doesn't supported EGL_EXT_platform_device extension".to_string(),
241 ));
242 }
243
244 let disp = Self::egl_get_platform_display_with_fallback(
246 egl,
247 egl_ext::PLATFORM_DEVICE_EXT,
248 devices[0],
249 &[egl::ATTRIB_NONE],
250 )?;
251 Ok(EglDisplayType::PlatformDisplay(disp))
252 }
253
254 fn egl_check_support_dma(egl: &Egl) -> Result<(), crate::Error> {
255 let extensions = egl.query_string(None, egl::EXTENSIONS)?;
256 let extensions = extensions.to_string_lossy();
257 log::debug!("EGL Extensions: {}", extensions);
258
259 if egl.upcast::<egl::EGL1_5>().is_some() {
260 return Ok(());
261 }
262
263 if !extensions.contains("EGL_EXT_image_dma_buf_import") {
264 return Err(crate::Error::GLVersion(
265 "EGL does not support EGL_EXT_image_dma_buf_import extension".to_string(),
266 ));
267 }
268
269 if egl.get_proc_address("eglCreateImageKHR").is_none() {
270 return Err(crate::Error::GLVersion(
271 "EGL does not support eglCreateImageKHR function".to_string(),
272 ));
273 }
274
275 if egl.get_proc_address("eglDestroyImageKHR").is_none() {
276 return Err(crate::Error::GLVersion(
277 "EGL does not support eglDestroyImageKHR function".to_string(),
278 ));
279 }
280 Ok(())
282 }
283
284 fn egl_get_platform_display_with_fallback(
285 egl: &Egl,
286 platform: egl::Enum,
287 native_display: *mut c_void,
288 attrib_list: &[Attrib],
289 ) -> Result<Display, Error> {
290 if let Some(egl) = egl.upcast::<egl::EGL1_5>() {
291 unsafe { egl.get_platform_display(platform, native_display, attrib_list) }
292 .map_err(|e| e.into())
293 } else if let Some(ext) = egl.get_proc_address("eglGetPlatformDisplayEXT") {
294 let func: unsafe extern "system" fn(
295 platform: egl::Enum,
296 native_display: *mut c_void,
297 attrib_list: *const Attrib,
298 ) -> egl::EGLDisplay = unsafe { std::mem::transmute(ext) };
299 let disp = unsafe { func(platform, native_display, attrib_list.as_ptr()) };
300 if disp != egl::NO_DISPLAY {
301 Ok(unsafe { Display::from_ptr(disp) })
302 } else {
303 Err(egl.get_error().map(|e| e.into()).unwrap_or(Error::Internal(
304 "EGL failed but no error was reported".to_owned(),
305 )))
306 }
307 } else {
308 Err(Error::EGLLoad(egl::LoadError::InvalidVersion {
309 provided: egl.version(),
310 required: khronos_egl::Version::EGL1_5,
311 }))
312 }
313 }
314
315 fn egl_create_image_with_fallback(
316 egl: &Egl,
317 display: Display,
318 ctx: egl::Context,
319 target: egl::Enum,
320 buffer: egl::ClientBuffer,
321 attrib_list: &[Attrib],
322 ) -> Result<egl::Image, Error> {
323 if let Some(egl) = egl.upcast::<egl::EGL1_5>() {
324 egl.create_image(display, ctx, target, buffer, attrib_list)
325 .map_err(|e| e.into())
326 } else if let Some(ext) = egl.get_proc_address("eglCreateImageKHR") {
327 log::trace!("eglCreateImageKHR addr: {:?}", ext);
328 let func: unsafe extern "system" fn(
329 display: egl::EGLDisplay,
330 ctx: egl::EGLContext,
331 target: egl::Enum,
332 buffer: egl::EGLClientBuffer,
333 attrib_list: *const egl::Int,
334 ) -> egl::EGLImage = unsafe { std::mem::transmute(ext) };
335 let new_attrib_list = attrib_list
336 .iter()
337 .map(|x| *x as egl::Int)
338 .collect::<Vec<_>>();
339
340 let image = unsafe {
341 func(
342 display.as_ptr(),
343 ctx.as_ptr(),
344 target,
345 buffer.as_ptr(),
346 new_attrib_list.as_ptr(),
347 )
348 };
349 if image != egl::NO_IMAGE {
350 Ok(unsafe { egl::Image::from_ptr(image) })
351 } else {
352 Err(egl.get_error().map(|e| e.into()).unwrap_or(Error::Internal(
353 "EGL failed but no error was reported".to_owned(),
354 )))
355 }
356 } else {
357 Err(Error::EGLLoad(egl::LoadError::InvalidVersion {
358 provided: egl.version(),
359 required: khronos_egl::Version::EGL1_5,
360 }))
361 }
362 }
363
364 fn egl_destory_image_with_fallback(
365 egl: &Egl,
366 display: Display,
367 image: egl::Image,
368 ) -> Result<(), Error> {
369 if let Some(egl) = egl.upcast::<egl::EGL1_5>() {
370 egl.destroy_image(display, image).map_err(|e| e.into())
371 } else if let Some(ext) = egl.get_proc_address("eglDestroyImageKHR") {
372 let func: unsafe extern "system" fn(
373 display: egl::EGLDisplay,
374 image: egl::EGLImage,
375 ) -> egl::Boolean = unsafe { std::mem::transmute(ext) };
376 let res = unsafe { func(display.as_ptr(), image.as_ptr()) };
377 if res == egl::TRUE {
378 Ok(())
379 } else {
380 Err(egl.get_error().map(|e| e.into()).unwrap_or(Error::Internal(
381 "EGL failed but no error was reported".to_owned(),
382 )))
383 }
384 } else {
385 Err(Error::EGLLoad(egl::LoadError::InvalidVersion {
386 provided: egl.version(),
387 required: khronos_egl::Version::EGL1_5,
388 }))
389 }
390 }
391}
392
393impl Drop for GlContext {
394 fn drop(&mut self) {
395 let _ = self
396 .egl
397 .make_current(self.display.as_display(), None, None, None);
398
399 let _ = self
400 .egl
401 .destroy_context(self.display.as_display(), self.ctx);
402
403 if let Some(surface) = self.surface.take() {
404 let _ = self.egl.destroy_surface(self.display.as_display(), surface);
405 }
406
407 let _ = self.egl.terminate(self.display.as_display());
408 }
409}
410
411#[derive(Debug)]
412pub(crate) struct Card(std::fs::File);
414
415impl std::os::unix::io::AsFd for Card {
418 fn as_fd(&self) -> std::os::unix::io::BorrowedFd<'_> {
419 self.0.as_fd()
420 }
421}
422
423impl DrmDevice for Card {}
425impl DrmControlDevice for Card {}
426
427impl Card {
429 pub fn open(path: &str) -> Result<Self, crate::Error> {
430 let mut options = std::fs::OpenOptions::new();
431 options.read(true);
432 options.write(true);
433 let c = options.open(path);
434 match c {
435 Ok(c) => Ok(Card(c)),
436 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
437 Err(Error::NotFound(format!("File not found: {path}")))
438 }
439 Err(e) => Err(e.into()),
440 }
441 }
442
443 pub fn open_global() -> Result<Self, crate::Error> {
444 let targets = ["/dev/dri/render128", "/dev/dri/card0", "/dev/dri/card1"];
445 let e = Self::open(targets[0]);
446 if let Ok(t) = e {
447 return Ok(t);
448 }
449 for t in &targets[1..] {
450 if let Ok(t) = Self::open(t) {
451 return Ok(t);
452 }
453 }
454 e
455 }
456}
457
458#[derive(Debug, Clone, Copy)]
459struct RegionOfInterest {
460 left: f32,
461 top: f32,
462 right: f32,
463 bottom: f32,
464}
465
466enum GLProcessorMessage {
467 ImageConvert(
468 SendablePtr<TensorImage>,
469 SendablePtr<TensorImage>,
470 Rotation,
471 Flip,
472 Crop,
473 tokio::sync::oneshot::Sender<Result<(), Error>>,
474 ),
475 SetColors(
476 Vec<[u8; 4]>,
477 tokio::sync::oneshot::Sender<Result<(), Error>>,
478 ),
479 ImageRender(
480 SendablePtr<TensorImage>,
481 SendablePtr<DetectBox>,
482 SendablePtr<Segmentation>,
483 tokio::sync::oneshot::Sender<Result<(), Error>>,
484 ),
485}
486
487#[derive(Debug)]
492pub struct GLProcessorThreaded {
493 handle: Option<JoinHandle<()>>,
495
496 sender: Option<Sender<GLProcessorMessage>>,
498 support_dma: bool,
499}
500
501unsafe impl Send for GLProcessorThreaded {}
502unsafe impl Sync for GLProcessorThreaded {}
503
504struct SendablePtr<T: Send> {
505 ptr: NonNull<T>,
506 len: usize,
507}
508
509unsafe impl<T> Send for SendablePtr<T> where T: Send {}
510
511impl GLProcessorThreaded {
512 pub fn new() -> Result<Self, Error> {
514 let (send, mut recv) = tokio::sync::mpsc::channel::<GLProcessorMessage>(1);
515
516 let (create_ctx_send, create_ctx_recv) = tokio::sync::oneshot::channel();
517
518 let func = move || {
519 let mut gl_converter = match GLProcessorST::new() {
520 Ok(gl) => gl,
521 Err(e) => {
522 let _ = create_ctx_send.send(Err(e));
523 return;
524 }
525 };
526 let _ = create_ctx_send.send(Ok(gl_converter.gl_context.support_dma));
527 while let Some(msg) = recv.blocking_recv() {
528 match msg {
529 GLProcessorMessage::ImageConvert(src, mut dst, rotation, flip, crop, resp) => {
530 let src = unsafe { src.ptr.as_ref() };
533 let dst = unsafe { dst.ptr.as_mut() };
534 let res = gl_converter.convert(src, dst, rotation, flip, crop);
535 let _ = resp.send(res);
536 }
537 GLProcessorMessage::ImageRender(mut dst, det, seg, resp) => {
538 let dst = unsafe { dst.ptr.as_mut() };
542 let det = unsafe { std::slice::from_raw_parts(det.ptr.as_ptr(), det.len) };
543 let seg = unsafe { std::slice::from_raw_parts(seg.ptr.as_ptr(), seg.len) };
544 let res = gl_converter.render_to_image(dst, det, seg);
545 let _ = resp.send(res);
546 }
547 GLProcessorMessage::SetColors(colors, resp) => {
548 let res = gl_converter.set_class_colors(&colors);
549 let _ = resp.send(res);
550 }
551 }
552 }
553 };
554
555 let handle = std::thread::spawn(func);
557
558 let support_dma = match create_ctx_recv.blocking_recv() {
559 Ok(Err(e)) => return Err(e),
560 Err(_) => {
561 return Err(Error::Internal(
562 "GL converter error messaging closed without update".to_string(),
563 ));
564 }
565 Ok(Ok(supports_dma)) => supports_dma,
566 };
567
568 Ok(Self {
569 handle: Some(handle),
570 sender: Some(send),
571 support_dma,
572 })
573 }
574}
575
576impl ImageProcessorTrait for GLProcessorThreaded {
577 fn convert(
578 &mut self,
579 src: &TensorImage,
580 dst: &mut TensorImage,
581 rotation: crate::Rotation,
582 flip: Flip,
583 crop: Crop,
584 ) -> crate::Result<()> {
585 crop.check_crop(src, dst)?;
586 if !GLProcessorST::check_src_format_supported(self.support_dma, src) {
587 return Err(crate::Error::NotSupported(format!(
588 "Opengl doesn't support {} source texture",
589 src.fourcc().display()
590 )));
591 }
592
593 if !GLProcessorST::check_dst_format_supported(self.support_dma, dst) {
594 return Err(crate::Error::NotSupported(format!(
595 "Opengl doesn't support {} destination texture",
596 dst.fourcc().display()
597 )));
598 }
599
600 let (err_send, err_recv) = tokio::sync::oneshot::channel();
601 self.sender
602 .as_ref()
603 .unwrap()
604 .blocking_send(GLProcessorMessage::ImageConvert(
605 SendablePtr {
606 ptr: src.into(),
607 len: 1,
608 },
609 SendablePtr {
610 ptr: dst.into(),
611 len: 1,
612 },
613 rotation,
614 flip,
615 crop,
616 err_send,
617 ))
618 .map_err(|_| Error::Internal("GL converter thread exited".to_string()))?;
619 err_recv.blocking_recv().map_err(|_| {
620 Error::Internal("GL converter error messaging closed without update".to_string())
621 })?
622 }
623
624 fn convert_ref(
625 &mut self,
626 src: &TensorImage,
627 dst: &mut TensorImageRef<'_>,
628 rotation: Rotation,
629 flip: Flip,
630 crop: Crop,
631 ) -> crate::Result<()> {
632 let mut cpu = CPUProcessor::new();
634 cpu.convert_ref(src, dst, rotation, flip, crop)
635 }
636
637 #[cfg(feature = "decoder")]
638 fn render_to_image(
639 &mut self,
640 dst: &mut TensorImage,
641 detect: &[crate::DetectBox],
642 segmentation: &[crate::Segmentation],
643 ) -> crate::Result<()> {
644 let (err_send, err_recv) = tokio::sync::oneshot::channel();
645 self.sender
646 .as_ref()
647 .unwrap()
648 .blocking_send(GLProcessorMessage::ImageRender(
649 SendablePtr {
650 ptr: dst.into(),
651 len: 1,
652 },
653 SendablePtr {
654 ptr: NonNull::new(detect.as_ptr() as *mut DetectBox).unwrap(),
655 len: detect.len(),
656 },
657 SendablePtr {
658 ptr: NonNull::new(segmentation.as_ptr() as *mut Segmentation).unwrap(),
659 len: segmentation.len(),
660 },
661 err_send,
662 ))
663 .map_err(|_| Error::Internal("GL converter thread exited".to_string()))?;
664 err_recv.blocking_recv().map_err(|_| {
665 Error::Internal("GL converter error messaging closed without update".to_string())
666 })?
667 }
668
669 #[cfg(feature = "decoder")]
670 fn set_class_colors(&mut self, colors: &[[u8; 4]]) -> Result<(), crate::Error> {
671 let (err_send, err_recv) = tokio::sync::oneshot::channel();
672 self.sender
673 .as_ref()
674 .unwrap()
675 .blocking_send(GLProcessorMessage::SetColors(colors.to_vec(), err_send))
676 .map_err(|_| Error::Internal("GL converter thread exited".to_string()))?;
677 err_recv.blocking_recv().map_err(|_| {
678 Error::Internal("GL converter error messaging closed without update".to_string())
679 })?
680 }
681}
682
683impl Drop for GLProcessorThreaded {
684 fn drop(&mut self) {
685 drop(self.sender.take());
686 let _ = self.handle.take().and_then(|h| h.join().ok());
687 }
688}
689
690pub struct GLProcessorST {
692 camera_eglimage_texture: Texture,
693 camera_normal_texture: Texture,
694 render_texture: Texture,
695 #[cfg(feature = "decoder")]
696 segmentation_texture: Texture,
697 #[cfg(feature = "decoder")]
698 segmentation_program: GlProgram,
699 #[cfg(feature = "decoder")]
700 instanced_segmentation_program: GlProgram,
701 #[cfg(feature = "decoder")]
702 color_program: GlProgram,
703 vertex_buffer: Buffer,
704 texture_buffer: Buffer,
705 texture_program: GlProgram,
706 texture_program_yuv: GlProgram,
707 texture_program_planar: GlProgram,
708 gl_context: GlContext,
709}
710
711impl ImageProcessorTrait for GLProcessorST {
712 fn convert(
713 &mut self,
714 src: &TensorImage,
715 dst: &mut TensorImage,
716 rotation: crate::Rotation,
717 flip: Flip,
718 crop: Crop,
719 ) -> crate::Result<()> {
720 crop.check_crop(src, dst)?;
721 if !Self::check_src_format_supported(self.gl_context.support_dma, src) {
722 return Err(crate::Error::NotSupported(format!(
723 "Opengl doesn't support {} source texture",
724 src.fourcc().display()
725 )));
726 }
727
728 if !Self::check_dst_format_supported(self.gl_context.support_dma, dst) {
729 return Err(crate::Error::NotSupported(format!(
730 "Opengl doesn't support {} destination texture",
731 dst.fourcc().display()
732 )));
733 }
734 log::debug!(
735 "dst tensor: {:?} src tensor :{:?}",
736 dst.tensor().memory(),
737 src.tensor().memory()
738 );
739 check_gl_error(function!(), line!())?;
740 if self.gl_context.support_dma
741 && dst.tensor().memory() == TensorMemory::Dma
742 && dst.fourcc() != RGB
743 {
745 let res = self.convert_dest_dma(dst, src, rotation, flip, crop);
746 return res;
747 }
748 let start = Instant::now();
749 let res = self.convert_dest_non_dma(dst, src, rotation, flip, crop);
750 log::debug!("convert_dest_non_dma takes {:?}", start.elapsed());
751 res
752 }
753
754 fn convert_ref(
755 &mut self,
756 src: &TensorImage,
757 dst: &mut TensorImageRef<'_>,
758 rotation: Rotation,
759 flip: Flip,
760 crop: Crop,
761 ) -> crate::Result<()> {
762 let mut cpu = CPUProcessor::new();
764 cpu.convert_ref(src, dst, rotation, flip, crop)
765 }
766
767 #[cfg(feature = "decoder")]
768 fn render_to_image(
769 &mut self,
770 dst: &mut TensorImage,
771 detect: &[DetectBox],
772 segmentation: &[Segmentation],
773 ) -> Result<(), crate::Error> {
774 use crate::FunctionTimer;
775
776 let _timer = FunctionTimer::new("GLProcessorST::render_to_image");
777 if !matches!(dst.fourcc(), RGBA | RGB) {
778 return Err(crate::Error::NotSupported(
779 "Opengl image rendering only supports RGBA or RGB images".to_string(),
780 ));
781 }
782
783 let (_render_buffer, is_dma) = match dst.tensor.memory() {
784 edgefirst_tensor::TensorMemory::Dma => {
785 if let Ok(render_buffer) = self.setup_renderbuffer_dma(dst) {
786 (render_buffer, true)
787 } else {
788 (
789 self.setup_renderbuffer_non_dma(
790 dst,
791 Crop::new().with_dst_rect(Some(Rect::new(0, 0, 0, 0))),
792 )?,
793 false,
794 )
795 }
796 }
797 _ => (
798 self.setup_renderbuffer_non_dma(
799 dst,
800 Crop::new().with_dst_rect(Some(Rect::new(0, 0, 0, 0))),
801 )?,
802 false,
803 ), };
805
806 gls::enable(gls::gl::BLEND);
807 gls::blend_func_separate(
808 gls::gl::SRC_ALPHA,
809 gls::gl::ONE_MINUS_SRC_ALPHA,
810 gls::gl::ZERO,
811 gls::gl::ONE,
812 );
813
814 self.render_box(dst, detect)?;
815 self.render_segmentation(detect, segmentation)?;
816
817 gls::finish();
818 if !is_dma {
819 let mut dst_map = dst.tensor().map()?;
820 let format = match dst.fourcc() {
821 RGB => gls::gl::RGB,
822 RGBA => gls::gl::RGBA,
823 _ => unreachable!(),
824 };
825 unsafe {
826 gls::gl::ReadBuffer(gls::gl::COLOR_ATTACHMENT0);
827 gls::gl::ReadnPixels(
828 0,
829 0,
830 dst.width() as i32,
831 dst.height() as i32,
832 format,
833 gls::gl::UNSIGNED_BYTE,
834 dst.tensor.len() as i32,
835 dst_map.as_mut_ptr() as *mut c_void,
836 );
837 }
838 }
839
840 Ok(())
841 }
842
843 #[cfg(feature = "decoder")]
844 fn set_class_colors(&mut self, colors: &[[u8; 4]]) -> crate::Result<()> {
845 if colors.is_empty() {
846 return Ok(());
847 }
848 let mut colors_f32 = colors
849 .iter()
850 .map(|c| {
851 [
852 c[0] as f32 / 255.0,
853 c[1] as f32 / 255.0,
854 c[2] as f32 / 255.0,
855 c[3] as f32 / 255.0,
856 ]
857 })
858 .take(20)
859 .collect::<Vec<[f32; 4]>>();
860
861 self.segmentation_program
862 .load_uniform_4fv(c"colors", &colors_f32)?;
863 self.instanced_segmentation_program
864 .load_uniform_4fv(c"colors", &colors_f32)?;
865
866 colors_f32.iter_mut().for_each(|c| {
867 c[3] = 1.0; });
869 self.color_program
870 .load_uniform_4fv(c"colors", &colors_f32)?;
871
872 Ok(())
873 }
874}
875
876impl GLProcessorST {
877 pub fn new() -> Result<GLProcessorST, crate::Error> {
878 let gl_context = GlContext::new()?;
879 gls::load_with(|s| {
880 gl_context
881 .egl
882 .get_proc_address(s)
883 .map_or(std::ptr::null(), |p| p as *const _)
884 });
885
886 Self::gl_check_support()?;
887
888 unsafe {
890 gls::gl::PixelStorei(gls::gl::PACK_ALIGNMENT, 1);
891 gls::gl::PixelStorei(gls::gl::UNPACK_ALIGNMENT, 1);
892 }
893
894 let texture_program_planar =
895 GlProgram::new(generate_vertex_shader(), generate_planar_rgb_shader())?;
896
897 let texture_program =
898 GlProgram::new(generate_vertex_shader(), generate_texture_fragment_shader())?;
899
900 let texture_program_yuv = GlProgram::new(
901 generate_vertex_shader(),
902 generate_texture_fragment_shader_yuv(),
903 )?;
904
905 #[cfg(feature = "decoder")]
906 let segmentation_program =
907 GlProgram::new(generate_vertex_shader(), generate_segmentation_shader())?;
908 #[cfg(feature = "decoder")]
909 segmentation_program.load_uniform_4fv(c"colors", &DEFAULT_COLORS)?;
910 #[cfg(feature = "decoder")]
911 let instanced_segmentation_program = GlProgram::new(
912 generate_vertex_shader(),
913 generate_instanced_segmentation_shader(),
914 )?;
915 #[cfg(feature = "decoder")]
916 instanced_segmentation_program.load_uniform_4fv(c"colors", &DEFAULT_COLORS)?;
917
918 #[cfg(feature = "decoder")]
919 let color_program = GlProgram::new(generate_vertex_shader(), generate_color_shader())?;
920 #[cfg(feature = "decoder")]
921 color_program.load_uniform_4fv(c"colors", &DEFAULT_COLORS)?;
922
923 let camera_eglimage_texture = Texture::new();
924 let camera_normal_texture = Texture::new();
925 let render_texture = Texture::new();
926 let segmentation_texture = Texture::new();
927 let vertex_buffer = Buffer::new(0, 3, 100);
928 let texture_buffer = Buffer::new(1, 2, 100);
929
930 let converter = GLProcessorST {
931 gl_context,
932 texture_program,
933 texture_program_yuv,
934 texture_program_planar,
935 camera_eglimage_texture,
936 camera_normal_texture,
937 #[cfg(feature = "decoder")]
938 segmentation_texture,
939 vertex_buffer,
940 texture_buffer,
941 render_texture,
942 #[cfg(feature = "decoder")]
943 segmentation_program,
944 #[cfg(feature = "decoder")]
945 instanced_segmentation_program,
946 #[cfg(feature = "decoder")]
947 color_program,
948 };
949 check_gl_error(function!(), line!())?;
950
951 log::debug!("GLConverter created");
952 Ok(converter)
953 }
954
955 fn check_src_format_supported(support_dma: bool, img: &TensorImage) -> bool {
956 if support_dma && img.tensor().memory() == TensorMemory::Dma {
957 matches!(img.fourcc(), RGBA | GREY | YUYV | NV12)
959 } else {
960 matches!(img.fourcc(), RGB | RGBA | GREY)
961 }
962 }
963
964 fn check_dst_format_supported(support_dma: bool, img: &TensorImage) -> bool {
965 if support_dma && img.tensor().memory() == TensorMemory::Dma {
966 matches!(img.fourcc(), RGBA | GREY | PLANAR_RGB)
968 } else {
969 matches!(img.fourcc(), RGB | RGBA | GREY)
970 }
971 }
972
973 fn gl_check_support() -> Result<(), crate::Error> {
974 if let Ok(version) = gls::get_string(gls::gl::SHADING_LANGUAGE_VERSION) {
975 log::debug!("GL Shading Language Version: {version:?}");
976 } else {
977 log::warn!("Could not get GL Shading Language Version");
978 }
979
980 let extensions = unsafe {
981 let str = gls::gl::GetString(gls::gl::EXTENSIONS);
982 if str.is_null() {
983 return Err(crate::Error::GLVersion(
984 "GL returned no supported extensions".to_string(),
985 ));
986 }
987 CStr::from_ptr(str as *const c_char)
988 .to_string_lossy()
989 .to_string()
990 };
991 log::debug!("GL Extensions: {extensions}");
992 let required_ext = [
993 "GL_OES_EGL_image_external_essl3",
994 "GL_OES_surfaceless_context",
995 ];
996 let extensions = extensions.split_ascii_whitespace().collect::<BTreeSet<_>>();
997 for required in required_ext {
998 if !extensions.contains(required) {
999 return Err(crate::Error::GLVersion(format!(
1000 "GL does not support {required} extension",
1001 )));
1002 }
1003 }
1004
1005 Ok(())
1006 }
1007
1008 fn setup_renderbuffer_dma(&mut self, dst: &TensorImage) -> crate::Result<FrameBuffer> {
1009 let frame_buffer = FrameBuffer::new();
1010 frame_buffer.bind();
1011
1012 let (width, height) = if matches!(dst.fourcc(), PLANAR_RGB) {
1013 let width = dst.width();
1014 let height = dst.height() * 3;
1015 (width as i32, height as i32)
1016 } else {
1017 (dst.width() as i32, dst.height() as i32)
1018 };
1019 let dest_img = self.create_image_from_dma2(dst)?;
1020 unsafe {
1021 gls::gl::UseProgram(self.texture_program_yuv.id);
1022 gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1023 gls::gl::BindTexture(gls::gl::TEXTURE_2D, self.render_texture.id);
1024 gls::gl::TexParameteri(
1025 gls::gl::TEXTURE_2D,
1026 gls::gl::TEXTURE_MIN_FILTER,
1027 gls::gl::LINEAR as i32,
1028 );
1029 gls::gl::TexParameteri(
1030 gls::gl::TEXTURE_2D,
1031 gls::gl::TEXTURE_MAG_FILTER,
1032 gls::gl::LINEAR as i32,
1033 );
1034 gls::gl::EGLImageTargetTexture2DOES(gls::gl::TEXTURE_2D, dest_img.egl_image.as_ptr());
1035 gls::gl::FramebufferTexture2D(
1036 gls::gl::FRAMEBUFFER,
1037 gls::gl::COLOR_ATTACHMENT0,
1038 gls::gl::TEXTURE_2D,
1039 self.render_texture.id,
1040 0,
1041 );
1042 check_gl_error(function!(), line!())?;
1043 gls::gl::Viewport(0, 0, width, height);
1044 }
1045 Ok(frame_buffer)
1046 }
1047
1048 fn convert_dest_dma(
1049 &mut self,
1050 dst: &mut TensorImage,
1051 src: &TensorImage,
1052 rotation: crate::Rotation,
1053 flip: Flip,
1054 crop: Crop,
1055 ) -> crate::Result<()> {
1056 assert!(self.gl_context.support_dma);
1057 let _framebuffer = self.setup_renderbuffer_dma(dst)?;
1058 if dst.is_planar() {
1059 self.convert_to_planar(src, dst, rotation, flip, crop)
1060 } else {
1061 self.convert_to(src, dst, rotation, flip, crop)
1062 }
1063 }
1064
1065 fn setup_renderbuffer_non_dma(
1066 &mut self,
1067 dst: &TensorImage,
1068 crop: Crop,
1069 ) -> crate::Result<FrameBuffer> {
1070 debug_assert!(matches!(dst.fourcc(), RGB | RGBA | GREY | PLANAR_RGB));
1071 let (width, height) = if dst.is_planar() {
1072 let width = dst.width() / 4;
1073 let height = match dst.fourcc() {
1074 RGBA => dst.height() * 4,
1075 RGB => dst.height() * 3,
1076 GREY => dst.height(),
1077 _ => unreachable!(),
1078 };
1079 (width as i32, height as i32)
1080 } else {
1081 (dst.width() as i32, dst.height() as i32)
1082 };
1083
1084 let format = if dst.is_planar() {
1085 gls::gl::RED
1086 } else {
1087 match dst.fourcc() {
1088 RGB => gls::gl::RGB,
1089 RGBA => gls::gl::RGBA,
1090 GREY => gls::gl::RED,
1091 _ => unreachable!(),
1092 }
1093 };
1094
1095 let start = Instant::now();
1096 let frame_buffer = FrameBuffer::new();
1097 frame_buffer.bind();
1098
1099 let map;
1100
1101 let pixels = if crop.dst_rect.is_none_or(|crop| {
1102 crop.top == 0
1103 && crop.left == 0
1104 && crop.height == dst.height()
1105 && crop.width == dst.width()
1106 }) {
1107 std::ptr::null()
1108 } else {
1109 map = dst.tensor().map()?;
1110 map.as_ptr() as *const c_void
1111 };
1112 unsafe {
1113 gls::gl::UseProgram(self.texture_program.id);
1114 gls::gl::BindTexture(gls::gl::TEXTURE_2D, self.render_texture.id);
1115 gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1116 gls::gl::TexParameteri(
1117 gls::gl::TEXTURE_2D,
1118 gls::gl::TEXTURE_MIN_FILTER,
1119 gls::gl::LINEAR as i32,
1120 );
1121 gls::gl::TexParameteri(
1122 gls::gl::TEXTURE_2D,
1123 gls::gl::TEXTURE_MAG_FILTER,
1124 gls::gl::LINEAR as i32,
1125 );
1126
1127 gls::gl::TexImage2D(
1128 gls::gl::TEXTURE_2D,
1129 0,
1130 format as i32,
1131 width,
1132 height,
1133 0,
1134 format,
1135 gls::gl::UNSIGNED_BYTE,
1136 pixels,
1137 );
1138 check_gl_error(function!(), line!())?;
1139 gls::gl::FramebufferTexture2D(
1140 gls::gl::FRAMEBUFFER,
1141 gls::gl::COLOR_ATTACHMENT0,
1142 gls::gl::TEXTURE_2D,
1143 self.render_texture.id,
1144 0,
1145 );
1146 check_gl_error(function!(), line!())?;
1147 gls::gl::Viewport(0, 0, width, height);
1148 }
1149 log::debug!("Set up framebuffer takes {:?}", start.elapsed());
1150 Ok(frame_buffer)
1151 }
1152
1153 fn convert_dest_non_dma(
1154 &mut self,
1155 dst: &mut TensorImage,
1156 src: &TensorImage,
1157 rotation: crate::Rotation,
1158 flip: Flip,
1159 crop: Crop,
1160 ) -> crate::Result<()> {
1161 let _framebuffer = self.setup_renderbuffer_non_dma(dst, crop)?;
1162 let start = Instant::now();
1163 if dst.is_planar() {
1164 self.convert_to_planar(src, dst, rotation, flip, crop)?;
1165 } else {
1166 self.convert_to(src, dst, rotation, flip, crop)?;
1167 }
1168 log::debug!("Draw to framebuffer takes {:?}", start.elapsed());
1169 let start = Instant::now();
1170 let dest_format = match dst.fourcc() {
1171 RGB => gls::gl::RGB,
1172 RGBA => gls::gl::RGBA,
1173 GREY => gls::gl::RED,
1174 _ => unreachable!(),
1175 };
1176
1177 unsafe {
1178 let mut dst_map = dst.tensor().map()?;
1179 gls::gl::ReadBuffer(gls::gl::COLOR_ATTACHMENT0);
1180 gls::gl::ReadnPixels(
1181 0,
1182 0,
1183 dst.width() as i32,
1184 dst.height() as i32,
1185 dest_format,
1186 gls::gl::UNSIGNED_BYTE,
1187 dst.tensor.len() as i32,
1188 dst_map.as_mut_ptr() as *mut c_void,
1189 );
1190 }
1191 log::debug!("Read from framebuffer takes {:?}", start.elapsed());
1192 Ok(())
1193 }
1194
1195 fn convert_to(
1196 &mut self,
1197 src: &TensorImage,
1198 dst: &TensorImage,
1199 rotation: crate::Rotation,
1200 flip: Flip,
1201 crop: Crop,
1202 ) -> Result<(), crate::Error> {
1203 check_gl_error(function!(), line!())?;
1204
1205 let has_crop = crop.dst_rect.is_some_and(|x| {
1206 x.left != 0 || x.top != 0 || x.width != dst.width() || x.height != dst.height()
1207 });
1208 if has_crop {
1209 if let Some(dst_color) = crop.dst_color {
1210 unsafe {
1211 gls::gl::ClearColor(
1212 dst_color[0] as f32 / 255.0,
1213 dst_color[1] as f32 / 255.0,
1214 dst_color[2] as f32 / 255.0,
1215 dst_color[3] as f32 / 255.0,
1216 );
1217 gls::gl::Clear(gls::gl::COLOR_BUFFER_BIT);
1218 };
1219 }
1220 }
1221
1222 let src_roi = if let Some(crop) = crop.src_rect {
1224 RegionOfInterest {
1225 left: crop.left as f32 / src.width() as f32,
1226 top: (crop.top + crop.height) as f32 / src.height() as f32,
1227 right: (crop.left + crop.width) as f32 / src.width() as f32,
1228 bottom: crop.top as f32 / src.height() as f32,
1229 }
1230 } else {
1231 RegionOfInterest {
1232 left: 0.,
1233 top: 1.,
1234 right: 1.,
1235 bottom: 0.,
1236 }
1237 };
1238
1239 let cvt_screen_coord = |normalized| normalized * 2.0 - 1.0;
1241 let dst_roi = if let Some(crop) = crop.dst_rect {
1242 RegionOfInterest {
1243 left: cvt_screen_coord(crop.left as f32 / dst.width() as f32),
1244 top: cvt_screen_coord((crop.top + crop.height) as f32 / dst.height() as f32),
1245 right: cvt_screen_coord((crop.left + crop.width) as f32 / dst.width() as f32),
1246 bottom: cvt_screen_coord(crop.top as f32 / dst.height() as f32),
1247 }
1248 } else {
1249 RegionOfInterest {
1250 left: -1.,
1251 top: 1.,
1252 right: 1.,
1253 bottom: -1.,
1254 }
1255 };
1256 let rotation_offset = match rotation {
1257 crate::Rotation::None => 0,
1258 crate::Rotation::Clockwise90 => 1,
1259 crate::Rotation::Rotate180 => 2,
1260 crate::Rotation::CounterClockwise90 => 3,
1261 };
1262 if self.gl_context.support_dma {
1263 match self.create_image_from_dma2(src) {
1264 Ok(new_egl_image) => self.draw_camera_texture_eglimage(
1265 src,
1266 &new_egl_image,
1267 src_roi,
1268 dst_roi,
1269 rotation_offset,
1270 flip,
1271 )?,
1272 Err(e) => {
1273 log::warn!("EGL image creation failed for {:?}: {:?}", src.fourcc(), e);
1274 let start = Instant::now();
1275 self.draw_src_texture(src, src_roi, dst_roi, rotation_offset, flip)?;
1276 log::debug!("draw_src_texture takes {:?}", start.elapsed());
1277 }
1278 }
1279 } else {
1280 let start = Instant::now();
1281 self.draw_src_texture(src, src_roi, dst_roi, rotation_offset, flip)?;
1282 log::debug!("draw_src_texture takes {:?}", start.elapsed());
1283 }
1284
1285 let start = Instant::now();
1286 unsafe { gls::gl::Finish() };
1287 log::debug!("gl_Finish takes {:?}", start.elapsed());
1288 check_gl_error(function!(), line!())?;
1289 Ok(())
1290 }
1291
1292 fn convert_to_planar(
1293 &self,
1294 src: &TensorImage,
1295 dst: &TensorImage,
1296 rotation: crate::Rotation,
1297 flip: Flip,
1298 crop: Crop,
1299 ) -> Result<(), crate::Error> {
1300 let alpha = match dst.fourcc() {
1323 PLANAR_RGB => false,
1324 PLANAR_RGBA => true,
1325 _ => {
1326 return Err(crate::Error::NotSupported(
1327 "Destination format must be PLANAR_RGB or PLANAR_RGBA".to_string(),
1328 ));
1329 }
1330 };
1331
1332 let src_roi = if let Some(crop) = crop.src_rect {
1334 RegionOfInterest {
1335 left: crop.left as f32 / src.width() as f32,
1336 top: (crop.top + crop.height) as f32 / src.height() as f32,
1337 right: (crop.left + crop.width) as f32 / src.width() as f32,
1338 bottom: crop.top as f32 / src.height() as f32,
1339 }
1340 } else {
1341 RegionOfInterest {
1342 left: 0.,
1343 top: 1.,
1344 right: 1.,
1345 bottom: 0.,
1346 }
1347 };
1348
1349 let cvt_screen_coord = |normalized| normalized * 2.0 - 1.0;
1351 let dst_roi = if let Some(crop) = crop.dst_rect {
1352 RegionOfInterest {
1353 left: cvt_screen_coord(crop.left as f32 / dst.width() as f32),
1354 top: cvt_screen_coord((crop.top + crop.height) as f32 / dst.height() as f32),
1355 right: cvt_screen_coord((crop.left + crop.width) as f32 / dst.width() as f32),
1356 bottom: cvt_screen_coord(crop.top as f32 / dst.height() as f32),
1357 }
1358 } else {
1359 RegionOfInterest {
1360 left: -1.,
1361 top: 1.,
1362 right: 1.,
1363 bottom: -1.,
1364 }
1365 };
1366 let rotation_offset = match rotation {
1367 crate::Rotation::None => 0,
1368 crate::Rotation::Clockwise90 => 1,
1369 crate::Rotation::Rotate180 => 2,
1370 crate::Rotation::CounterClockwise90 => 3,
1371 };
1372
1373 let has_crop = crop.dst_rect.is_some_and(|x| {
1374 x.left != 0 || x.top != 0 || x.width != dst.width() || x.height != dst.height()
1375 });
1376 if has_crop {
1377 if let Some(dst_color) = crop.dst_color {
1378 self.clear_rect_planar(
1379 dst.width(),
1380 dst.height(),
1381 dst_roi,
1382 [
1383 dst_color[0] as f32 / 255.0,
1384 dst_color[1] as f32 / 255.0,
1385 dst_color[2] as f32 / 255.0,
1386 dst_color[3] as f32 / 255.0,
1387 ],
1388 alpha,
1389 )?;
1390 }
1391 }
1392
1393 let new_egl_image = self.create_image_from_dma2(src)?;
1394
1395 self.draw_camera_texture_to_rgb_planar(
1396 &new_egl_image,
1397 src_roi,
1398 dst_roi,
1399 rotation_offset,
1400 flip,
1401 alpha,
1402 )?;
1403 unsafe { gls::gl::Finish() };
1404
1405 Ok(())
1406 }
1407
1408 fn clear_rect_planar(
1409 &self,
1410 width: usize,
1411 height: usize,
1412 dst_roi: RegionOfInterest,
1413 color: [f32; 4],
1414 alpha: bool,
1415 ) -> Result<(), Error> {
1416 if !alpha && color[0] == color[1] && color[1] == color[2] {
1417 unsafe {
1418 gls::gl::ClearColor(color[0], color[0], color[0], 1.0);
1419 gls::gl::Clear(gls::gl::COLOR_BUFFER_BIT);
1420 };
1421 }
1422
1423 let split = if alpha { 4 } else { 3 };
1424
1425 unsafe {
1426 gls::gl::Enable(gls::gl::SCISSOR_TEST);
1427 let x = (((dst_roi.left + 1.0) / 2.0) * width as f32).round() as i32;
1428 let y = (((dst_roi.bottom + 1.0) / 2.0) * height as f32).round() as i32;
1429 let width = (((dst_roi.right - dst_roi.left) / 2.0) * width as f32).round() as i32;
1430 let height = (((dst_roi.top - dst_roi.bottom) / 2.0) * height as f32 / split as f32)
1431 .round() as i32;
1432 for (i, c) in color.iter().enumerate().take(split) {
1433 gls::gl::Scissor(x, y + i as i32 * height, width, height);
1434 gls::gl::ClearColor(*c, *c, *c, 1.0);
1435 gls::gl::Clear(gls::gl::COLOR_BUFFER_BIT);
1436 }
1437 gls::gl::Disable(gls::gl::SCISSOR_TEST);
1438 }
1439 Ok(())
1440 }
1441
1442 #[allow(clippy::too_many_arguments)]
1443 fn draw_camera_texture_to_rgb_planar(
1444 &self,
1445 egl_img: &EglImage,
1446 src_roi: RegionOfInterest,
1447 mut dst_roi: RegionOfInterest,
1448 rotation_offset: usize,
1449 flip: Flip,
1450 alpha: bool,
1451 ) -> Result<(), Error> {
1452 let texture_target = gls::gl::TEXTURE_EXTERNAL_OES;
1453 match flip {
1454 Flip::None => {}
1455 Flip::Vertical => {
1456 std::mem::swap(&mut dst_roi.top, &mut dst_roi.bottom);
1457 }
1458 Flip::Horizontal => {
1459 std::mem::swap(&mut dst_roi.left, &mut dst_roi.right);
1460 }
1461 }
1462 unsafe {
1463 gls::gl::UseProgram(self.texture_program_planar.id);
1465 gls::gl::BindTexture(texture_target, self.camera_eglimage_texture.id);
1466 gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1467 gls::gl::TexParameteri(
1468 texture_target,
1469 gls::gl::TEXTURE_MIN_FILTER,
1470 gls::gl::LINEAR as i32,
1471 );
1472 gls::gl::TexParameteri(
1473 texture_target,
1474 gls::gl::TEXTURE_MAG_FILTER,
1475 gls::gl::LINEAR as i32,
1476 );
1477 gls::gl::TexParameteri(
1478 texture_target,
1479 gls::gl::TEXTURE_WRAP_S,
1480 gls::gl::CLAMP_TO_EDGE as i32,
1481 );
1482
1483 gls::gl::TexParameteri(
1484 texture_target,
1485 gls::gl::TEXTURE_WRAP_T,
1486 gls::gl::CLAMP_TO_EDGE as i32,
1487 );
1488
1489 gls::egl_image_target_texture_2d_oes(texture_target, egl_img.egl_image.as_ptr());
1490 check_gl_error(function!(), line!())?;
1491 let y_centers = if alpha {
1492 vec![-3.0 / 4.0, -1.0 / 4.0, 1.0 / 4.0, 3.0 / 4.0]
1493 } else {
1494 vec![-2.0 / 3.0, 0.0, 2.0 / 3.0]
1495 };
1496 let swizzles = [gls::gl::RED, gls::gl::GREEN, gls::gl::BLUE, gls::gl::ALPHA];
1497 for (i, y_center) in y_centers.iter().enumerate() {
1499 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
1500 gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
1501 let camera_vertices: [f32; 12] = [
1502 dst_roi.left,
1503 dst_roi.top / 3.0 + y_center,
1504 0., dst_roi.right,
1506 dst_roi.top / 3.0 + y_center,
1507 0., dst_roi.right,
1509 dst_roi.bottom / 3.0 + y_center,
1510 0., dst_roi.left,
1512 dst_roi.bottom / 3.0 + y_center,
1513 0., ];
1515 gls::gl::BufferData(
1516 gls::gl::ARRAY_BUFFER,
1517 (size_of::<f32>() * camera_vertices.len()) as isize,
1518 camera_vertices.as_ptr() as *const c_void,
1519 gls::gl::DYNAMIC_DRAW,
1520 );
1521
1522 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
1523 gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
1524 let texture_vertices: [f32; 16] = [
1525 src_roi.left,
1526 src_roi.top,
1527 src_roi.right,
1528 src_roi.top,
1529 src_roi.right,
1530 src_roi.bottom,
1531 src_roi.left,
1532 src_roi.bottom,
1533 src_roi.left,
1534 src_roi.top,
1535 src_roi.right,
1536 src_roi.top,
1537 src_roi.right,
1538 src_roi.bottom,
1539 src_roi.left,
1540 src_roi.bottom,
1541 ];
1542
1543 gls::gl::BufferData(
1544 gls::gl::ARRAY_BUFFER,
1545 (size_of::<f32>() * 8) as isize,
1546 (texture_vertices[(rotation_offset * 2)..]).as_ptr() as *const c_void,
1547 gls::gl::DYNAMIC_DRAW,
1548 );
1549 let vertices_index: [u32; 4] = [0, 1, 2, 3];
1550 gls::gl::TexParameteri(
1554 texture_target,
1555 gls::gl::TEXTURE_SWIZZLE_R,
1556 swizzles[i] as i32,
1557 );
1558
1559 gls::gl::DrawElements(
1560 gls::gl::TRIANGLE_FAN,
1561 vertices_index.len() as i32,
1562 gls::gl::UNSIGNED_INT,
1563 vertices_index.as_ptr() as *const c_void,
1564 );
1565 }
1566 check_gl_error(function!(), line!())?;
1567 }
1568 Ok(())
1569 }
1570
1571 fn draw_src_texture(
1572 &mut self,
1573 src: &TensorImage,
1574 src_roi: RegionOfInterest,
1575 mut dst_roi: RegionOfInterest,
1576 rotation_offset: usize,
1577 flip: Flip,
1578 ) -> Result<(), Error> {
1579 let texture_target = gls::gl::TEXTURE_2D;
1580 let texture_format = match src.fourcc() {
1581 RGB => gls::gl::RGB,
1582 RGBA => gls::gl::RGBA,
1583 GREY => gls::gl::RED,
1584 _ => unreachable!(),
1585 };
1586 unsafe {
1587 gls::gl::UseProgram(self.texture_program.id);
1588 gls::gl::BindTexture(texture_target, self.camera_normal_texture.id);
1589 gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1590 gls::gl::TexParameteri(
1591 texture_target,
1592 gls::gl::TEXTURE_MIN_FILTER,
1593 gls::gl::LINEAR as i32,
1594 );
1595 gls::gl::TexParameteri(
1596 texture_target,
1597 gls::gl::TEXTURE_MAG_FILTER,
1598 gls::gl::LINEAR as i32,
1599 );
1600 if src.fourcc() == GREY {
1601 for swizzle in [
1602 gls::gl::TEXTURE_SWIZZLE_R,
1603 gls::gl::TEXTURE_SWIZZLE_G,
1604 gls::gl::TEXTURE_SWIZZLE_B,
1605 ] {
1606 gls::gl::TexParameteri(gls::gl::TEXTURE_2D, swizzle, gls::gl::RED as i32);
1607 }
1608 } else {
1609 for (swizzle, src) in [
1610 (gls::gl::TEXTURE_SWIZZLE_R, gls::gl::RED),
1611 (gls::gl::TEXTURE_SWIZZLE_G, gls::gl::GREEN),
1612 (gls::gl::TEXTURE_SWIZZLE_B, gls::gl::BLUE),
1613 ] {
1614 gls::gl::TexParameteri(gls::gl::TEXTURE_2D, swizzle, src as i32);
1615 }
1616 }
1617 self.camera_normal_texture.update_texture(
1618 texture_target,
1619 src.width(),
1620 src.height(),
1621 texture_format,
1622 &src.tensor().map()?,
1623 );
1624
1625 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
1626 gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
1627
1628 match flip {
1629 Flip::None => {}
1630 Flip::Vertical => {
1631 std::mem::swap(&mut dst_roi.top, &mut dst_roi.bottom);
1632 }
1633 Flip::Horizontal => {
1634 std::mem::swap(&mut dst_roi.left, &mut dst_roi.right);
1635 }
1636 }
1637
1638 let camera_vertices: [f32; 12] = [
1639 dst_roi.left,
1640 dst_roi.top,
1641 0., dst_roi.right,
1643 dst_roi.top,
1644 0., dst_roi.right,
1646 dst_roi.bottom,
1647 0., dst_roi.left,
1649 dst_roi.bottom,
1650 0., ];
1652 gls::gl::BufferData(
1653 gls::gl::ARRAY_BUFFER,
1654 (size_of::<f32>() * camera_vertices.len()) as isize,
1655 camera_vertices.as_ptr() as *const c_void,
1656 gls::gl::DYNAMIC_DRAW,
1657 );
1658 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
1659 gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
1660 let texture_vertices: [f32; 16] = [
1661 src_roi.left,
1662 src_roi.top,
1663 src_roi.right,
1664 src_roi.top,
1665 src_roi.right,
1666 src_roi.bottom,
1667 src_roi.left,
1668 src_roi.bottom,
1669 src_roi.left,
1670 src_roi.top,
1671 src_roi.right,
1672 src_roi.top,
1673 src_roi.right,
1674 src_roi.bottom,
1675 src_roi.left,
1676 src_roi.bottom,
1677 ];
1678
1679 gls::gl::BufferData(
1680 gls::gl::ARRAY_BUFFER,
1681 (size_of::<f32>() * 8) as isize,
1682 (texture_vertices[(rotation_offset * 2)..]).as_ptr() as *const c_void,
1683 gls::gl::DYNAMIC_DRAW,
1684 );
1685 let vertices_index: [u32; 4] = [0, 1, 2, 3];
1686 gls::gl::DrawElements(
1687 gls::gl::TRIANGLE_FAN,
1688 vertices_index.len() as i32,
1689 gls::gl::UNSIGNED_INT,
1690 vertices_index.as_ptr() as *const c_void,
1691 );
1692 check_gl_error(function!(), line!())?;
1693
1694 Ok(())
1695 }
1696 }
1697
1698 fn draw_camera_texture_eglimage(
1699 &self,
1700 src: &TensorImage,
1701 egl_img: &EglImage,
1702 src_roi: RegionOfInterest,
1703 mut dst_roi: RegionOfInterest,
1704 rotation_offset: usize,
1705 flip: Flip,
1706 ) -> Result<(), Error> {
1707 let texture_target = gls::gl::TEXTURE_EXTERNAL_OES;
1709 unsafe {
1710 gls::gl::UseProgram(self.texture_program_yuv.id);
1711 gls::gl::BindTexture(texture_target, self.camera_eglimage_texture.id);
1712 gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1713 gls::gl::TexParameteri(
1714 texture_target,
1715 gls::gl::TEXTURE_MIN_FILTER,
1716 gls::gl::LINEAR as i32,
1717 );
1718 gls::gl::TexParameteri(
1719 texture_target,
1720 gls::gl::TEXTURE_MAG_FILTER,
1721 gls::gl::LINEAR as i32,
1722 );
1723
1724 if src.fourcc() == GREY {
1725 for swizzle in [
1726 gls::gl::TEXTURE_SWIZZLE_R,
1727 gls::gl::TEXTURE_SWIZZLE_G,
1728 gls::gl::TEXTURE_SWIZZLE_B,
1729 ] {
1730 gls::gl::TexParameteri(gls::gl::TEXTURE_2D, swizzle, gls::gl::RED as i32);
1731 }
1732 } else {
1733 for (swizzle, src) in [
1734 (gls::gl::TEXTURE_SWIZZLE_R, gls::gl::RED),
1735 (gls::gl::TEXTURE_SWIZZLE_G, gls::gl::GREEN),
1736 (gls::gl::TEXTURE_SWIZZLE_B, gls::gl::BLUE),
1737 ] {
1738 gls::gl::TexParameteri(gls::gl::TEXTURE_2D, swizzle, src as i32);
1739 }
1740 }
1741
1742 gls::egl_image_target_texture_2d_oes(texture_target, egl_img.egl_image.as_ptr());
1743 check_gl_error(function!(), line!())?;
1744 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
1745 gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
1746
1747 match flip {
1748 Flip::None => {}
1749 Flip::Vertical => {
1750 std::mem::swap(&mut dst_roi.top, &mut dst_roi.bottom);
1751 }
1752 Flip::Horizontal => {
1753 std::mem::swap(&mut dst_roi.left, &mut dst_roi.right);
1754 }
1755 }
1756
1757 let camera_vertices: [f32; 12] = [
1758 dst_roi.left,
1759 dst_roi.top,
1760 0., dst_roi.right,
1762 dst_roi.top,
1763 0., dst_roi.right,
1765 dst_roi.bottom,
1766 0., dst_roi.left,
1768 dst_roi.bottom,
1769 0., ];
1771 gls::gl::BufferSubData(
1772 gls::gl::ARRAY_BUFFER,
1773 0,
1774 (size_of::<f32>() * camera_vertices.len()) as isize,
1775 camera_vertices.as_ptr() as *const c_void,
1776 );
1777
1778 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
1779 gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
1780
1781 let texture_vertices: [f32; 16] = [
1782 src_roi.left,
1783 src_roi.top,
1784 src_roi.right,
1785 src_roi.top,
1786 src_roi.right,
1787 src_roi.bottom,
1788 src_roi.left,
1789 src_roi.bottom,
1790 src_roi.left,
1791 src_roi.top,
1792 src_roi.right,
1793 src_roi.top,
1794 src_roi.right,
1795 src_roi.bottom,
1796 src_roi.left,
1797 src_roi.bottom,
1798 ];
1799 gls::gl::BufferSubData(
1800 gls::gl::ARRAY_BUFFER,
1801 0,
1802 (size_of::<f32>() * 8) as isize,
1803 (texture_vertices[(rotation_offset * 2)..]).as_ptr() as *const c_void,
1804 );
1805
1806 let vertices_index: [u32; 4] = [0, 1, 2, 3];
1807 gls::gl::DrawElements(
1808 gls::gl::TRIANGLE_FAN,
1809 vertices_index.len() as i32,
1810 gls::gl::UNSIGNED_INT,
1811 vertices_index.as_ptr() as *const c_void,
1812 );
1813 }
1814 check_gl_error(function!(), line!())?;
1815 Ok(())
1816 }
1817
1818 fn create_image_from_dma2(&self, src: &TensorImage) -> Result<EglImage, crate::Error> {
1819 let width;
1820 let height;
1821 let format;
1822 let channels;
1823
1824 if src.fourcc() == NV12 {
1826 if !src.width().is_multiple_of(4) {
1827 return Err(Error::NotSupported(
1828 "OpenGL EGLImage doesn't support image widths which are not multiples of 4"
1829 .to_string(),
1830 ));
1831 }
1832 width = src.width();
1833 height = src.height();
1834 format = fourcc_to_drm(NV12);
1835 channels = 1; } else if src.is_planar() {
1837 if !src.width().is_multiple_of(16) {
1838 return Err(Error::NotSupported(
1839 "OpenGL Planar RGB EGLImage doesn't support image widths which are not multiples of 16"
1840 .to_string(),
1841 ));
1842 }
1843 match src.fourcc() {
1844 PLANAR_RGB => {
1845 format = DrmFourcc::R8;
1846 width = src.width();
1847 height = src.height() * 3;
1848 channels = 1;
1849 }
1850 fourcc => {
1851 return Err(crate::Error::NotSupported(format!(
1852 "Unsupported Planar FourCC {fourcc:?}"
1853 )));
1854 }
1855 };
1856 } else {
1857 if !src.width().is_multiple_of(4) {
1858 return Err(Error::NotSupported(
1859 "OpenGL EGLImage doesn't support image widths which are not multiples of 4"
1860 .to_string(),
1861 ));
1862 }
1863 width = src.width();
1864 height = src.height();
1865 format = fourcc_to_drm(src.fourcc());
1866 channels = src.channels();
1867 }
1868
1869 let fd = match &src.tensor {
1870 edgefirst_tensor::Tensor::Dma(dma_tensor) => dma_tensor.fd.as_raw_fd(),
1871 edgefirst_tensor::Tensor::Shm(_) => {
1872 return Err(Error::NotImplemented(
1873 "OpenGL EGLImage doesn't support SHM".to_string(),
1874 ));
1875 }
1876 edgefirst_tensor::Tensor::Mem(_) => {
1877 return Err(Error::NotImplemented(
1878 "OpenGL EGLImage doesn't support MEM".to_string(),
1879 ));
1880 }
1881 };
1882
1883 let plane0_pitch = if src.fourcc() == NV12 {
1886 width
1887 } else {
1888 width * channels
1889 };
1890
1891 let mut egl_img_attr = vec![
1892 egl_ext::LINUX_DRM_FOURCC as Attrib,
1893 format as Attrib,
1894 khronos_egl::WIDTH as Attrib,
1895 width as Attrib,
1896 khronos_egl::HEIGHT as Attrib,
1897 height as Attrib,
1898 egl_ext::DMA_BUF_PLANE0_PITCH as Attrib,
1899 plane0_pitch as Attrib,
1900 egl_ext::DMA_BUF_PLANE0_OFFSET as Attrib,
1901 0 as Attrib,
1902 egl_ext::DMA_BUF_PLANE0_FD as Attrib,
1903 fd as Attrib,
1904 egl::IMAGE_PRESERVED as Attrib,
1905 egl::TRUE as Attrib,
1906 ];
1907
1908 if src.fourcc() == NV12 {
1910 let uv_offset = width * height; egl_img_attr.append(&mut vec![
1912 egl_ext::DMA_BUF_PLANE1_FD as Attrib,
1913 fd as Attrib,
1914 egl_ext::DMA_BUF_PLANE1_OFFSET as Attrib,
1915 uv_offset as Attrib,
1916 egl_ext::DMA_BUF_PLANE1_PITCH as Attrib,
1917 width as Attrib, ]);
1919 }
1920
1921 if matches!(src.fourcc(), YUYV | NV12) {
1922 egl_img_attr.append(&mut vec![
1923 egl_ext::YUV_COLOR_SPACE_HINT as Attrib,
1924 egl_ext::ITU_REC709 as Attrib,
1925 egl_ext::SAMPLE_RANGE_HINT as Attrib,
1926 egl_ext::YUV_NARROW_RANGE as Attrib,
1927 ]);
1928 }
1929
1930 egl_img_attr.push(khronos_egl::NONE as Attrib);
1931
1932 match self.new_egl_image_owned(egl_ext::LINUX_DMA_BUF, &egl_img_attr) {
1933 Ok(v) => Ok(v),
1934 Err(e) => Err(e),
1935 }
1936 }
1937
1938 fn new_egl_image_owned(
1939 &'_ self,
1940 target: egl::Enum,
1941 attrib_list: &[Attrib],
1942 ) -> Result<EglImage, Error> {
1943 let image = GlContext::egl_create_image_with_fallback(
1944 &self.gl_context.egl,
1945 self.gl_context.display.as_display(),
1946 unsafe { egl::Context::from_ptr(egl::NO_CONTEXT) },
1947 target,
1948 unsafe { egl::ClientBuffer::from_ptr(null_mut()) },
1949 attrib_list,
1950 )?;
1951 Ok(EglImage {
1952 egl_image: image,
1953 display: self.gl_context.display.as_display(),
1954 egl: self.gl_context.egl.clone(),
1955 })
1956 }
1957
1958 fn reshape_segmentation_to_rgba(&self, segmentation: &[u8], shape: [usize; 3]) -> Vec<u8> {
1960 let [height, width, classes] = shape;
1961
1962 let n_layer_stride = height * width * 4;
1963 let n_row_stride = width * 4;
1964 let n_col_stride = 4;
1965 let row_stride = width * classes;
1966 let col_stride = classes;
1967
1968 let mut new_segmentation = vec![0u8; n_layer_stride * classes.div_ceil(4)];
1969
1970 for i in 0..height {
1971 for j in 0..width {
1972 for k in 0..classes.div_ceil(4) * 4 {
1973 if k >= classes {
1974 new_segmentation[n_layer_stride * (k / 4)
1975 + i * n_row_stride
1976 + j * n_col_stride
1977 + k % 4] = 0;
1978 } else {
1979 new_segmentation[n_layer_stride * (k / 4)
1980 + i * n_row_stride
1981 + j * n_col_stride
1982 + k % 4] = segmentation[i * row_stride + j * col_stride + k];
1983 }
1984 }
1985 }
1986 }
1987
1988 new_segmentation
1989 }
1990
1991 #[cfg(feature = "decoder")]
1992 fn render_modelpack_segmentation(
1993 &mut self,
1994 dst_roi: RegionOfInterest,
1995 segmentation: &[u8],
1996 shape: [usize; 3],
1997 ) -> Result<(), crate::Error> {
1998 log::debug!("start render_segmentation_to_image");
1999
2000 let new_segmentation = self.reshape_segmentation_to_rgba(segmentation, shape);
2003
2004 let [height, width, classes] = shape;
2005
2006 let format = gls::gl::RGBA;
2007 let texture_target = gls::gl::TEXTURE_2D_ARRAY;
2008 self.segmentation_program
2009 .load_uniform_1i(c"background_index", shape[2] as i32 - 1)?;
2010
2011 gls::use_program(self.segmentation_program.id);
2012
2013 gls::bind_texture(texture_target, self.segmentation_texture.id);
2014 gls::active_texture(gls::gl::TEXTURE0);
2015 gls::tex_parameteri(
2016 texture_target,
2017 gls::gl::TEXTURE_MIN_FILTER,
2018 gls::gl::LINEAR as i32,
2019 );
2020 gls::tex_parameteri(
2021 texture_target,
2022 gls::gl::TEXTURE_MAG_FILTER,
2023 gls::gl::LINEAR as i32,
2024 );
2025 gls::tex_parameteri(
2026 texture_target,
2027 gls::gl::TEXTURE_WRAP_S,
2028 gls::gl::CLAMP_TO_EDGE as i32,
2029 );
2030
2031 gls::tex_parameteri(
2032 texture_target,
2033 gls::gl::TEXTURE_WRAP_T,
2034 gls::gl::CLAMP_TO_EDGE as i32,
2035 );
2036
2037 gls::tex_image3d(
2038 texture_target,
2039 0,
2040 format as i32,
2041 width as i32,
2042 height as i32,
2043 classes.div_ceil(4) as i32,
2044 0,
2045 format,
2046 gls::gl::UNSIGNED_BYTE,
2047 Some(&new_segmentation),
2048 );
2049
2050 let src_roi = RegionOfInterest {
2051 left: 0.,
2052 top: 1.,
2053 right: 1.,
2054 bottom: 0.,
2055 };
2056
2057 unsafe {
2058 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
2059 gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
2060
2061 let camera_vertices: [f32; 12] = [
2062 dst_roi.left,
2063 dst_roi.top,
2064 0., dst_roi.right,
2066 dst_roi.top,
2067 0., dst_roi.right,
2069 dst_roi.bottom,
2070 0., dst_roi.left,
2072 dst_roi.bottom,
2073 0., ];
2075 gls::gl::BufferSubData(
2076 gls::gl::ARRAY_BUFFER,
2077 0,
2078 (size_of::<f32>() * camera_vertices.len()) as isize,
2079 camera_vertices.as_ptr() as *const c_void,
2080 );
2081
2082 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
2083 gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
2084
2085 let texture_vertices: [f32; 8] = [
2086 src_roi.left,
2087 src_roi.top,
2088 src_roi.right,
2089 src_roi.top,
2090 src_roi.right,
2091 src_roi.bottom,
2092 src_roi.left,
2093 src_roi.bottom,
2094 ];
2095 gls::gl::BufferSubData(
2096 gls::gl::ARRAY_BUFFER,
2097 0,
2098 (size_of::<f32>() * 8) as isize,
2099 (texture_vertices[0..]).as_ptr() as *const c_void,
2100 );
2101
2102 let vertices_index: [u32; 4] = [0, 1, 2, 3];
2103 gls::gl::DrawElements(
2104 gls::gl::TRIANGLE_FAN,
2105 vertices_index.len() as i32,
2106 gls::gl::UNSIGNED_INT,
2107 vertices_index.as_ptr() as *const c_void,
2108 );
2109 }
2110
2111 Ok(())
2112 }
2113
2114 #[cfg(feature = "decoder")]
2115 fn render_yolo_segmentation(
2116 &mut self,
2117 dst_roi: RegionOfInterest,
2118 segmentation: &[u8],
2119 shape: [usize; 2],
2120 class: usize,
2121 ) -> Result<(), crate::Error> {
2122 log::debug!("start render_yolo_segmentation");
2123
2124 let [height, width] = shape;
2125
2126 let format = gls::gl::RED;
2127 let texture_target = gls::gl::TEXTURE_2D;
2128 gls::use_program(self.instanced_segmentation_program.id);
2129 self.instanced_segmentation_program
2130 .load_uniform_1i(c"class_index", class as i32)?;
2131 gls::bind_texture(texture_target, self.segmentation_texture.id);
2132 gls::active_texture(gls::gl::TEXTURE0);
2133 gls::tex_parameteri(
2134 texture_target,
2135 gls::gl::TEXTURE_MIN_FILTER,
2136 gls::gl::LINEAR as i32,
2137 );
2138 gls::tex_parameteri(
2139 texture_target,
2140 gls::gl::TEXTURE_MAG_FILTER,
2141 gls::gl::LINEAR as i32,
2142 );
2143 gls::tex_parameteri(
2144 texture_target,
2145 gls::gl::TEXTURE_WRAP_S,
2146 gls::gl::CLAMP_TO_EDGE as i32,
2147 );
2148
2149 gls::tex_parameteri(
2150 texture_target,
2151 gls::gl::TEXTURE_WRAP_T,
2152 gls::gl::CLAMP_TO_EDGE as i32,
2153 );
2154
2155 gls::tex_image2d(
2156 texture_target,
2157 0,
2158 format as i32,
2159 width as i32,
2160 height as i32,
2161 0,
2162 format,
2163 gls::gl::UNSIGNED_BYTE,
2164 Some(segmentation),
2165 );
2166
2167 let src_roi = RegionOfInterest {
2168 left: 0.,
2169 top: 1.,
2170 right: 1.,
2171 bottom: 0.,
2172 };
2173
2174 unsafe {
2175 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
2176 gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
2177
2178 let camera_vertices: [f32; 12] = [
2179 dst_roi.left,
2180 dst_roi.top,
2181 0., dst_roi.right,
2183 dst_roi.top,
2184 0., dst_roi.right,
2186 dst_roi.bottom,
2187 0., dst_roi.left,
2189 dst_roi.bottom,
2190 0., ];
2192 gls::gl::BufferSubData(
2193 gls::gl::ARRAY_BUFFER,
2194 0,
2195 (size_of::<f32>() * camera_vertices.len()) as isize,
2196 camera_vertices.as_ptr() as *const c_void,
2197 );
2198
2199 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
2200 gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
2201
2202 let texture_vertices: [f32; 8] = [
2203 src_roi.left,
2204 src_roi.top,
2205 src_roi.right,
2206 src_roi.top,
2207 src_roi.right,
2208 src_roi.bottom,
2209 src_roi.left,
2210 src_roi.bottom,
2211 ];
2212 gls::gl::BufferSubData(
2213 gls::gl::ARRAY_BUFFER,
2214 0,
2215 (size_of::<f32>() * 8) as isize,
2216 (texture_vertices).as_ptr() as *const c_void,
2217 );
2218
2219 let vertices_index: [u32; 4] = [0, 1, 2, 3];
2220 gls::gl::DrawElements(
2221 gls::gl::TRIANGLE_FAN,
2222 vertices_index.len() as i32,
2223 gls::gl::UNSIGNED_INT,
2224 vertices_index.as_ptr() as *const c_void,
2225 );
2226 gls::gl::Finish();
2227 }
2228
2229 Ok(())
2230 }
2231
2232 fn render_segmentation(
2233 &mut self,
2234 detect: &[DetectBox],
2235 segmentation: &[Segmentation],
2236 ) -> crate::Result<()> {
2237 if segmentation.is_empty() {
2238 return Ok(());
2239 }
2240
2241 let is_modelpack = segmentation[0].segmentation.shape()[2] > 1;
2242 let cvt_screen_coord = |normalized| normalized * 2.0 - 1.0;
2244 if is_modelpack {
2245 let seg = &segmentation[0];
2246 let dst_roi = RegionOfInterest {
2247 left: cvt_screen_coord(seg.xmin),
2248 top: cvt_screen_coord(seg.ymax),
2249 right: cvt_screen_coord(seg.xmax),
2250 bottom: cvt_screen_coord(seg.ymin),
2251 };
2252 let segment = seg.segmentation.as_standard_layout();
2253 let slice = segment.as_slice().ok_or(Error::Internal(
2254 "Cannot get slice of segmentation".to_owned(),
2255 ))?;
2256
2257 self.render_modelpack_segmentation(
2258 dst_roi,
2259 slice,
2260 [
2261 seg.segmentation.shape()[0],
2262 seg.segmentation.shape()[1],
2263 seg.segmentation.shape()[2],
2264 ],
2265 )?;
2266 } else {
2267 for (seg, det) in segmentation.iter().zip(detect) {
2268 let dst_roi = RegionOfInterest {
2269 left: cvt_screen_coord(seg.xmin),
2270 top: cvt_screen_coord(seg.ymax),
2271 right: cvt_screen_coord(seg.xmax),
2272 bottom: cvt_screen_coord(seg.ymin),
2273 };
2274
2275 let segment = seg.segmentation.as_standard_layout();
2276 let slice = segment.as_slice().ok_or(Error::Internal(
2277 "Cannot get slice of segmentation".to_owned(),
2278 ))?;
2279
2280 self.render_yolo_segmentation(
2281 dst_roi,
2282 slice,
2283 [seg.segmentation.shape()[0], seg.segmentation.shape()[1]],
2284 det.label,
2285 )?;
2286 }
2287 }
2288
2289 gls::disable(gls::gl::BLEND);
2290 Ok(())
2291 }
2292
2293 fn render_box(&mut self, dst: &TensorImage, detect: &[DetectBox]) -> Result<(), Error> {
2294 unsafe {
2295 gls::gl::UseProgram(self.color_program.id);
2296 let rescale = |x: f32| x * 2.0 - 1.0;
2297 let thickness = 3.0;
2298 for d in detect {
2299 self.color_program
2300 .load_uniform_1i(c"class_index", d.label as i32)?;
2301 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
2302 gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
2303 let bbox: [f32; 4] = d.bbox.into();
2304 let outer_box = [
2305 bbox[0] - thickness / dst.width() as f32,
2306 bbox[1] - thickness / dst.height() as f32,
2307 bbox[2] + thickness / dst.width() as f32,
2308 bbox[3] + thickness / dst.height() as f32,
2309 ];
2310 let camera_vertices: [f32; 24] = [
2311 rescale(bbox[0]),
2312 rescale(bbox[3]),
2313 0., rescale(bbox[2]),
2315 rescale(bbox[3]),
2316 0., rescale(bbox[2]),
2318 rescale(bbox[1]),
2319 0., rescale(bbox[0]),
2321 rescale(bbox[1]),
2322 0., rescale(outer_box[0]),
2324 rescale(outer_box[3]),
2325 0., rescale(outer_box[2]),
2327 rescale(outer_box[3]),
2328 0., rescale(outer_box[2]),
2330 rescale(outer_box[1]),
2331 0., rescale(outer_box[0]),
2333 rescale(outer_box[1]),
2334 0., ];
2336 gls::gl::BufferData(
2337 gls::gl::ARRAY_BUFFER,
2338 (size_of::<f32>() * camera_vertices.len()) as isize,
2339 camera_vertices.as_ptr() as *const c_void,
2340 gls::gl::DYNAMIC_DRAW,
2341 );
2342
2343 let vertices_index: [u32; 10] = [0, 1, 5, 2, 6, 3, 7, 0, 4, 5];
2344 gls::gl::DrawElements(
2345 gls::gl::TRIANGLE_STRIP,
2346 vertices_index.len() as i32,
2347 gls::gl::UNSIGNED_INT,
2348 vertices_index.as_ptr() as *const c_void,
2349 );
2350 }
2351 }
2352 check_gl_error(function!(), line!())?;
2353 Ok(())
2354 }
2355}
2356struct EglImage {
2357 egl_image: egl::Image,
2358 egl: Rc<Egl>,
2359 display: egl::Display,
2360}
2361
2362impl Drop for EglImage {
2363 fn drop(&mut self) {
2364 if self.egl_image.as_ptr() == egl::NO_IMAGE {
2365 return;
2366 }
2367
2368 let e = GlContext::egl_destory_image_with_fallback(&self.egl, self.display, self.egl_image);
2369 if let Err(e) = e {
2370 error!("Could not destroy EGL image: {e:?}");
2371 }
2372 }
2373}
2374
2375struct Texture {
2376 id: u32,
2377 target: gls::gl::types::GLenum,
2378 width: usize,
2379 height: usize,
2380 format: gls::gl::types::GLenum,
2381}
2382
2383impl Default for Texture {
2384 fn default() -> Self {
2385 Self::new()
2386 }
2387}
2388
2389impl Texture {
2390 fn new() -> Self {
2391 let mut id = 0;
2392 unsafe { gls::gl::GenTextures(1, &raw mut id) };
2393 Self {
2394 id,
2395 target: 0,
2396 width: 0,
2397 height: 0,
2398 format: 0,
2399 }
2400 }
2401
2402 fn update_texture(
2403 &mut self,
2404 target: gls::gl::types::GLenum,
2405 width: usize,
2406 height: usize,
2407 format: gls::gl::types::GLenum,
2408 data: &[u8],
2409 ) {
2410 if target != self.target
2411 || width != self.width
2412 || height != self.height
2413 || format != self.format
2414 {
2415 unsafe {
2416 gls::gl::TexImage2D(
2417 target,
2418 0,
2419 format as i32,
2420 width as i32,
2421 height as i32,
2422 0,
2423 format,
2424 gls::gl::UNSIGNED_BYTE,
2425 data.as_ptr() as *const c_void,
2426 );
2427 }
2428 self.target = target;
2429 self.format = format;
2430 self.width = width;
2431 self.height = height;
2432 } else {
2433 unsafe {
2434 gls::gl::TexSubImage2D(
2435 target,
2436 0,
2437 0,
2438 0,
2439 width as i32,
2440 height as i32,
2441 format,
2442 gls::gl::UNSIGNED_BYTE,
2443 data.as_ptr() as *const c_void,
2444 );
2445 }
2446 }
2447 }
2448}
2449
2450impl Drop for Texture {
2451 fn drop(&mut self) {
2452 unsafe { gls::gl::DeleteTextures(1, &raw mut self.id) };
2453 }
2454}
2455
2456struct Buffer {
2457 id: u32,
2458 buffer_index: u32,
2459}
2460
2461impl Buffer {
2462 fn new(buffer_index: u32, size_per_point: usize, max_points: usize) -> Buffer {
2463 let mut id = 0;
2464 unsafe {
2465 gls::gl::EnableVertexAttribArray(buffer_index);
2466 gls::gl::GenBuffers(1, &raw mut id);
2467 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, id);
2468 gls::gl::VertexAttribPointer(
2469 buffer_index,
2470 size_per_point as i32,
2471 gls::gl::FLOAT,
2472 gls::gl::FALSE,
2473 0,
2474 null(),
2475 );
2476 gls::gl::BufferData(
2477 gls::gl::ARRAY_BUFFER,
2478 (size_of::<f32>() * size_per_point * max_points) as isize,
2479 null(),
2480 gls::gl::DYNAMIC_DRAW,
2481 );
2482 }
2483
2484 Buffer { id, buffer_index }
2485 }
2486}
2487
2488impl Drop for Buffer {
2489 fn drop(&mut self) {
2490 unsafe { gls::gl::DeleteBuffers(1, &raw mut self.id) };
2491 }
2492}
2493
2494struct FrameBuffer {
2495 id: u32,
2496}
2497
2498impl FrameBuffer {
2499 fn new() -> FrameBuffer {
2500 let mut id = 0;
2501 unsafe {
2502 gls::gl::GenFramebuffers(1, &raw mut id);
2503 }
2504
2505 FrameBuffer { id }
2506 }
2507
2508 fn bind(&self) {
2509 unsafe { gls::gl::BindFramebuffer(gls::gl::FRAMEBUFFER, self.id) };
2510 }
2511
2512 fn unbind(&self) {
2513 unsafe { gls::gl::BindFramebuffer(gls::gl::FRAMEBUFFER, 0) };
2514 }
2515}
2516
2517impl Drop for FrameBuffer {
2518 fn drop(&mut self) {
2519 self.unbind();
2520 unsafe {
2521 gls::gl::DeleteFramebuffers(1, &raw mut self.id);
2522 }
2523 }
2524}
2525
2526pub struct GlProgram {
2527 id: u32,
2528 vertex_id: u32,
2529 fragment_id: u32,
2530}
2531
2532impl GlProgram {
2533 fn new(vertex_shader: &str, fragment_shader: &str) -> Result<Self, crate::Error> {
2534 let id = unsafe { gls::gl::CreateProgram() };
2535 let vertex_id = unsafe { gls::gl::CreateShader(gls::gl::VERTEX_SHADER) };
2536 if compile_shader_from_str(vertex_id, vertex_shader, "shader_vert").is_err() {
2537 log::debug!("Vertex shader source:\n{}", vertex_shader);
2538 return Err(crate::Error::OpenGl(format!(
2539 "Shader compile error: {vertex_shader}"
2540 )));
2541 }
2542 unsafe {
2543 gls::gl::AttachShader(id, vertex_id);
2544 }
2545
2546 let fragment_id = unsafe { gls::gl::CreateShader(gls::gl::FRAGMENT_SHADER) };
2547 if compile_shader_from_str(fragment_id, fragment_shader, "shader_frag").is_err() {
2548 log::debug!("Fragment shader source:\n{}", fragment_shader);
2549 return Err(crate::Error::OpenGl(format!(
2550 "Shader compile error: {fragment_shader}"
2551 )));
2552 }
2553
2554 unsafe {
2555 gls::gl::AttachShader(id, fragment_id);
2556 gls::gl::LinkProgram(id);
2557 gls::gl::UseProgram(id);
2558 }
2559
2560 Ok(Self {
2561 id,
2562 vertex_id,
2563 fragment_id,
2564 })
2565 }
2566
2567 #[allow(dead_code)]
2568 fn load_uniform_1f(&self, name: &CStr, value: f32) -> Result<(), crate::Error> {
2569 unsafe {
2570 gls::gl::UseProgram(self.id);
2571 let location = gls::gl::GetUniformLocation(self.id, name.as_ptr());
2572 gls::gl::Uniform1f(location, value);
2573 }
2574 Ok(())
2575 }
2576
2577 #[allow(dead_code)]
2578 fn load_uniform_1i(&self, name: &CStr, value: i32) -> Result<(), crate::Error> {
2579 unsafe {
2580 gls::gl::UseProgram(self.id);
2581 let location = gls::gl::GetUniformLocation(self.id, name.as_ptr());
2582 gls::gl::Uniform1i(location, value);
2583 }
2584 Ok(())
2585 }
2586
2587 fn load_uniform_4fv(&self, name: &CStr, value: &[[f32; 4]]) -> Result<(), crate::Error> {
2588 unsafe {
2589 gls::gl::UseProgram(self.id);
2590 let location = gls::gl::GetUniformLocation(self.id, name.as_ptr());
2591 if location == -1 {
2592 return Err(crate::Error::OpenGl(format!(
2593 "Could not find uniform location for '{}'",
2594 name.to_string_lossy().into_owned()
2595 )));
2596 }
2597 gls::gl::Uniform4fv(location, value.len() as i32, value.as_flattened().as_ptr());
2598 }
2599 check_gl_error(function!(), line!())?;
2600 Ok(())
2601 }
2602}
2603
2604impl Drop for GlProgram {
2605 fn drop(&mut self) {
2606 unsafe {
2607 gls::gl::DeleteProgram(self.id);
2608 gls::gl::DeleteShader(self.fragment_id);
2609 gls::gl::DeleteShader(self.vertex_id);
2610 }
2611 }
2612}
2613
2614fn compile_shader_from_str(shader: u32, shader_source: &str, shader_name: &str) -> Result<(), ()> {
2615 let src = match CString::from_str(shader_source) {
2616 Ok(v) => v,
2617 Err(_) => return Err(()),
2618 };
2619 let src_ptr = src.as_ptr();
2620 unsafe {
2621 gls::gl::ShaderSource(shader, 1, &raw const src_ptr, null());
2622 gls::gl::CompileShader(shader);
2623 let mut is_compiled = 0;
2624 gls::gl::GetShaderiv(shader, gls::gl::COMPILE_STATUS, &raw mut is_compiled);
2625 if is_compiled == 0 {
2626 let mut max_length = 0;
2627 gls::gl::GetShaderiv(shader, gls::gl::INFO_LOG_LENGTH, &raw mut max_length);
2628 let mut error_log: Vec<u8> = vec![0; max_length as usize];
2629 gls::gl::GetShaderInfoLog(
2630 shader,
2631 max_length,
2632 &raw mut max_length,
2633 error_log.as_mut_ptr() as *mut c_char,
2634 );
2635 error!(
2636 "Shader '{}' failed: {:?}\n",
2637 shader_name,
2638 CString::from_vec_with_nul(error_log)
2639 .unwrap()
2640 .into_string()
2641 .unwrap()
2642 );
2643 gls::gl::DeleteShader(shader);
2644 return Err(());
2645 }
2646 Ok(())
2647 }
2648}
2649
2650fn check_gl_error(name: &str, line: u32) -> Result<(), Error> {
2651 unsafe {
2652 let err = gls::gl::GetError();
2653 if err != gls::gl::NO_ERROR {
2654 error!("GL Error: {name}:{line}: {err:#X}");
2655 return Err(Error::OpenGl(format!("{err:#X}")));
2657 }
2658 }
2659 Ok(())
2660}
2661
2662fn fourcc_to_drm(fourcc: FourCharCode) -> DrmFourcc {
2663 match fourcc {
2664 RGBA => DrmFourcc::Abgr8888,
2665 YUYV => DrmFourcc::Yuyv,
2666 RGB => DrmFourcc::Bgr888,
2667 GREY => DrmFourcc::R8,
2668 NV12 => DrmFourcc::Nv12,
2669 _ => todo!(),
2670 }
2671}
2672
2673mod egl_ext {
2674 #![allow(dead_code)]
2675 pub(crate) const LINUX_DMA_BUF: u32 = 0x3270;
2676 pub(crate) const LINUX_DRM_FOURCC: u32 = 0x3271;
2677 pub(crate) const DMA_BUF_PLANE0_FD: u32 = 0x3272;
2678 pub(crate) const DMA_BUF_PLANE0_OFFSET: u32 = 0x3273;
2679 pub(crate) const DMA_BUF_PLANE0_PITCH: u32 = 0x3274;
2680 pub(crate) const DMA_BUF_PLANE1_FD: u32 = 0x3275;
2681 pub(crate) const DMA_BUF_PLANE1_OFFSET: u32 = 0x3276;
2682 pub(crate) const DMA_BUF_PLANE1_PITCH: u32 = 0x3277;
2683 pub(crate) const DMA_BUF_PLANE2_FD: u32 = 0x3278;
2684 pub(crate) const DMA_BUF_PLANE2_OFFSET: u32 = 0x3279;
2685 pub(crate) const DMA_BUF_PLANE2_PITCH: u32 = 0x327A;
2686 pub(crate) const YUV_COLOR_SPACE_HINT: u32 = 0x327B;
2687 pub(crate) const SAMPLE_RANGE_HINT: u32 = 0x327C;
2688 pub(crate) const YUV_CHROMA_HORIZONTAL_SITING_HINT: u32 = 0x327D;
2689 pub(crate) const YUV_CHROMA_VERTICAL_SITING_HINT: u32 = 0x327E;
2690
2691 pub(crate) const ITU_REC601: u32 = 0x327F;
2692 pub(crate) const ITU_REC709: u32 = 0x3280;
2693 pub(crate) const ITU_REC2020: u32 = 0x3281;
2694
2695 pub(crate) const YUV_FULL_RANGE: u32 = 0x3282;
2696 pub(crate) const YUV_NARROW_RANGE: u32 = 0x3283;
2697
2698 pub(crate) const YUV_CHROMA_SITING_0: u32 = 0x3284;
2699 pub(crate) const YUV_CHROMA_SITING_0_5: u32 = 0x3285;
2700
2701 pub(crate) const PLATFORM_GBM_KHR: u32 = 0x31D7;
2702
2703 pub(crate) const PLATFORM_DEVICE_EXT: u32 = 0x313F;
2704}
2705
2706fn generate_vertex_shader() -> &'static str {
2707 "\
2708#version 300 es
2709precision mediump float;
2710layout(location = 0) in vec3 pos;
2711layout(location = 1) in vec2 texCoord;
2712
2713out vec3 fragPos;
2714out vec2 tc;
2715
2716void main() {
2717 fragPos = pos;
2718 tc = texCoord;
2719
2720 gl_Position = vec4(pos, 1.0);
2721}
2722"
2723}
2724
2725fn generate_texture_fragment_shader() -> &'static str {
2726 "\
2727#version 300 es
2728
2729precision mediump float;
2730uniform sampler2D tex;
2731in vec3 fragPos;
2732in vec2 tc;
2733
2734out vec4 color;
2735
2736void main(){
2737 color = texture(tex, tc);
2738}
2739"
2740}
2741
2742fn generate_texture_fragment_shader_yuv() -> &'static str {
2743 "\
2744#version 300 es
2745#extension GL_OES_EGL_image_external_essl3 : require
2746precision mediump float;
2747uniform samplerExternalOES tex;
2748in vec3 fragPos;
2749in vec2 tc;
2750
2751out vec4 color;
2752
2753void main(){
2754 color = texture(tex, tc);
2755}
2756"
2757}
2758
2759fn generate_planar_rgb_shader() -> &'static str {
2760 "\
2761#version 300 es
2762#extension GL_OES_EGL_image_external_essl3 : require
2763precision mediump float;
2764uniform samplerExternalOES tex;
2765in vec3 fragPos;
2766in vec2 tc;
2767
2768out vec4 color;
2769
2770void main(){
2771 color = texture(tex, tc);
2772}
2773"
2774}
2775
2776fn generate_segmentation_shader() -> &'static str {
2779 "\
2780#version 300 es
2781precision mediump float;
2782precision mediump sampler2DArray;
2783
2784uniform sampler2DArray tex;
2785uniform vec4 colors[20];
2786uniform int background_index;
2787
2788in vec3 fragPos;
2789in vec2 tc;
2790in vec4 fragColor;
2791
2792out vec4 color;
2793
2794float max_arg(const in vec4 args, out int argmax) {
2795 if (args[0] >= args[1] && args[0] >= args[2] && args[0] >= args[3]) {
2796 argmax = 0;
2797 return args[0];
2798 }
2799 if (args[1] >= args[0] && args[1] >= args[2] && args[1] >= args[3]) {
2800 argmax = 1;
2801 return args[1];
2802 }
2803 if (args[2] >= args[0] && args[2] >= args[1] && args[2] >= args[3]) {
2804 argmax = 2;
2805 return args[2];
2806 }
2807 argmax = 3;
2808 return args[3];
2809}
2810
2811void main() {
2812 mediump int layers = textureSize(tex, 0).z;
2813 float max_all = -4.0;
2814 int max_ind = 0;
2815 for (int i = 0; i < layers; i++) {
2816 vec4 d = texture(tex, vec3(tc, i));
2817 int max_ind_ = 0;
2818 float max_ = max_arg(d, max_ind_);
2819 if (max_ <= max_all) { continue; }
2820 max_all = max_;
2821 max_ind = i*4 + max_ind_;
2822 }
2823 if (max_ind == background_index) {
2824 discard;
2825 }
2826 max_ind = max_ind % 20;
2827 color = colors[max_ind];
2828}
2829"
2830}
2831
2832fn generate_instanced_segmentation_shader() -> &'static str {
2833 "\
2834#version 300 es
2835precision mediump float;
2836uniform sampler2D mask0;
2837uniform vec4 colors[20];
2838uniform int class_index;
2839in vec3 fragPos;
2840in vec2 tc;
2841in vec4 fragColor;
2842
2843out vec4 color;
2844void main() {
2845 float r0 = texture(mask0, tc).r;
2846 int arg = int(r0>=0.5);
2847 if (arg == 0) {
2848 discard;
2849 }
2850 color = colors[class_index % 20];
2851}
2852"
2853}
2854
2855fn generate_color_shader() -> &'static str {
2856 "\
2857#version 300 es
2858precision mediump float;
2859uniform vec4 colors[20];
2860uniform int class_index;
2861
2862out vec4 color;
2863void main() {
2864 int index = class_index % 20;
2865 color = colors[index];
2866}
2867"
2868}
2869
2870#[cfg(test)]
2871#[cfg(feature = "opengl")]
2872mod gl_tests {
2873 use super::*;
2874 use crate::{TensorImage, RGBA};
2875 #[cfg(feature = "dma_test_formats")]
2876 use crate::{NV12, YUYV};
2877 use edgefirst_tensor::TensorTrait;
2878 #[cfg(feature = "dma_test_formats")]
2879 use edgefirst_tensor::{TensorMapTrait, TensorMemory};
2880 use image::buffer::ConvertBuffer;
2881 use ndarray::Array3;
2882
2883 #[test]
2884 #[cfg(feature = "decoder")]
2885 fn test_segmentation() {
2886 use edgefirst_decoder::Segmentation;
2887
2888 if !is_opengl_available() {
2889 eprintln!("SKIPPED: {} - OpenGL not available", function!());
2890 return;
2891 }
2892
2893 let mut image = TensorImage::load(
2894 include_bytes!("../../../testdata/giraffe.jpg"),
2895 Some(RGBA),
2896 None,
2897 )
2898 .unwrap();
2899
2900 let mut segmentation = Array3::from_shape_vec(
2901 (2, 160, 160),
2902 include_bytes!("../../../testdata/modelpack_seg_2x160x160.bin").to_vec(),
2903 )
2904 .unwrap();
2905 segmentation.swap_axes(0, 1);
2906 segmentation.swap_axes(1, 2);
2907 let segmentation = segmentation.as_standard_layout().to_owned();
2908
2909 let seg = Segmentation {
2910 segmentation,
2911 xmin: 0.0,
2912 ymin: 0.0,
2913 xmax: 1.0,
2914 ymax: 1.0,
2915 };
2916
2917 let mut renderer = GLProcessorThreaded::new().unwrap();
2918 renderer.render_to_image(&mut image, &[], &[seg]).unwrap();
2919 }
2920
2921 #[test]
2922 #[cfg(feature = "decoder")]
2923 fn test_segmentation_mem() {
2924 use edgefirst_decoder::Segmentation;
2925
2926 if !is_opengl_available() {
2927 eprintln!("SKIPPED: {} - OpenGL not available", function!());
2928 return;
2929 }
2930
2931 let mut image = TensorImage::load(
2932 include_bytes!("../../../testdata/giraffe.jpg"),
2933 Some(RGBA),
2934 Some(edgefirst_tensor::TensorMemory::Mem),
2935 )
2936 .unwrap();
2937
2938 let mut segmentation = Array3::from_shape_vec(
2939 (2, 160, 160),
2940 include_bytes!("../../../testdata/modelpack_seg_2x160x160.bin").to_vec(),
2941 )
2942 .unwrap();
2943 segmentation.swap_axes(0, 1);
2944 segmentation.swap_axes(1, 2);
2945 let segmentation = segmentation.as_standard_layout().to_owned();
2946
2947 let seg = Segmentation {
2948 segmentation,
2949 xmin: 0.0,
2950 ymin: 0.0,
2951 xmax: 1.0,
2952 ymax: 1.0,
2953 };
2954
2955 let mut renderer = GLProcessorThreaded::new().unwrap();
2956 renderer.render_to_image(&mut image, &[], &[seg]).unwrap();
2957 }
2958
2959 #[test]
2960 #[cfg(feature = "decoder")]
2961 fn test_segmentation_yolo() {
2962 use edgefirst_decoder::Segmentation;
2963 use ndarray::Array3;
2964
2965 if !is_opengl_available() {
2966 eprintln!("SKIPPED: {} - OpenGL not available", function!());
2967 return;
2968 }
2969
2970 let mut image = TensorImage::load(
2971 include_bytes!("../../../testdata/giraffe.jpg"),
2972 Some(RGBA),
2973 None,
2974 )
2975 .unwrap();
2976
2977 let segmentation = Array3::from_shape_vec(
2978 (76, 55, 1),
2979 include_bytes!("../../../testdata/yolov8_seg_crop_76x55.bin").to_vec(),
2980 )
2981 .unwrap();
2982
2983 let detect = DetectBox {
2984 bbox: [0.59375, 0.25, 0.9375, 0.725].into(),
2985 score: 0.99,
2986 label: 1,
2987 };
2988
2989 let seg = Segmentation {
2990 segmentation,
2991 xmin: 0.59375,
2992 ymin: 0.25,
2993 xmax: 0.9375,
2994 ymax: 0.725,
2995 };
2996
2997 let mut renderer = GLProcessorThreaded::new().unwrap();
2998 renderer
2999 .set_class_colors(&[[255, 255, 0, 233], [128, 128, 255, 100]])
3000 .unwrap();
3001 renderer
3002 .render_to_image(&mut image, &[detect], &[seg])
3003 .unwrap();
3004
3005 let expected = TensorImage::load(
3006 include_bytes!("../../../testdata/output_render_gl.jpg"),
3007 Some(RGBA),
3008 None,
3009 )
3010 .unwrap();
3011
3012 compare_images(&image, &expected, 0.99, function!());
3013 }
3014
3015 #[test]
3016 #[cfg(feature = "decoder")]
3017 fn test_boxes() {
3018 use edgefirst_decoder::DetectBox;
3019
3020 if !is_opengl_available() {
3021 eprintln!("SKIPPED: {} - OpenGL not available", function!());
3022 return;
3023 }
3024
3025 let mut image = TensorImage::load(
3026 include_bytes!("../../../testdata/giraffe.jpg"),
3027 Some(RGBA),
3028 None,
3029 )
3030 .unwrap();
3031
3032 let detect = DetectBox {
3033 bbox: [0.59375, 0.25, 0.9375, 0.725].into(),
3034 score: 0.99,
3035 label: 0,
3036 };
3037 let mut renderer = GLProcessorThreaded::new().unwrap();
3038 renderer
3039 .set_class_colors(&[[255, 255, 0, 233], [128, 128, 255, 100]])
3040 .unwrap();
3041 renderer
3042 .render_to_image(&mut image, &[detect], &[])
3043 .unwrap();
3044 }
3045
3046 static GL_AVAILABLE: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
3047 fn is_opengl_available() -> bool {
3049 #[cfg(all(target_os = "linux", feature = "opengl"))]
3050 {
3051 *GL_AVAILABLE.get_or_init(|| GLProcessorThreaded::new().is_ok())
3052 }
3053
3054 #[cfg(not(all(target_os = "linux", feature = "opengl")))]
3055 {
3056 false
3057 }
3058 }
3059
3060 fn compare_images(img1: &TensorImage, img2: &TensorImage, threshold: f64, name: &str) {
3061 assert_eq!(img1.height(), img2.height(), "Heights differ");
3062 assert_eq!(img1.width(), img2.width(), "Widths differ");
3063 assert_eq!(img1.fourcc(), img2.fourcc(), "FourCC differ");
3064 assert!(
3065 matches!(img1.fourcc(), RGB | RGBA | GREY | PLANAR_RGB),
3066 "FourCC must be RGB or RGBA for comparison"
3067 );
3068
3069 let image1 = match img1.fourcc() {
3070 RGB => image::RgbImage::from_vec(
3071 img1.width() as u32,
3072 img1.height() as u32,
3073 img1.tensor().map().unwrap().to_vec(),
3074 )
3075 .unwrap(),
3076 RGBA => image::RgbaImage::from_vec(
3077 img1.width() as u32,
3078 img1.height() as u32,
3079 img1.tensor().map().unwrap().to_vec(),
3080 )
3081 .unwrap()
3082 .convert(),
3083 GREY => image::GrayImage::from_vec(
3084 img1.width() as u32,
3085 img1.height() as u32,
3086 img1.tensor().map().unwrap().to_vec(),
3087 )
3088 .unwrap()
3089 .convert(),
3090 PLANAR_RGB => image::GrayImage::from_vec(
3091 img1.width() as u32,
3092 (img1.height() * 3) as u32,
3093 img1.tensor().map().unwrap().to_vec(),
3094 )
3095 .unwrap()
3096 .convert(),
3097 _ => return,
3098 };
3099
3100 let image2 = match img2.fourcc() {
3101 RGB => image::RgbImage::from_vec(
3102 img2.width() as u32,
3103 img2.height() as u32,
3104 img2.tensor().map().unwrap().to_vec(),
3105 )
3106 .unwrap(),
3107 RGBA => image::RgbaImage::from_vec(
3108 img2.width() as u32,
3109 img2.height() as u32,
3110 img2.tensor().map().unwrap().to_vec(),
3111 )
3112 .unwrap()
3113 .convert(),
3114 GREY => image::GrayImage::from_vec(
3115 img2.width() as u32,
3116 img2.height() as u32,
3117 img2.tensor().map().unwrap().to_vec(),
3118 )
3119 .unwrap()
3120 .convert(),
3121 PLANAR_RGB => image::GrayImage::from_vec(
3122 img2.width() as u32,
3123 (img2.height() * 3) as u32,
3124 img2.tensor().map().unwrap().to_vec(),
3125 )
3126 .unwrap()
3127 .convert(),
3128 _ => return,
3129 };
3130
3131 let similarity = image_compare::rgb_similarity_structure(
3132 &image_compare::Algorithm::RootMeanSquared,
3133 &image1,
3134 &image2,
3135 )
3136 .expect("Image Comparison failed");
3137 if similarity.score < threshold {
3138 similarity
3141 .image
3142 .to_color_map()
3143 .save(format!("{name}.png"))
3144 .unwrap();
3145 panic!(
3146 "{name}: converted image and target image have similarity score too low: {} < {}",
3147 similarity.score, threshold
3148 )
3149 }
3150 }
3151
3152 #[cfg(feature = "dma_test_formats")]
3159 fn load_raw_image(
3160 width: usize,
3161 height: usize,
3162 fourcc: FourCharCode,
3163 memory: Option<TensorMemory>,
3164 bytes: &[u8],
3165 ) -> Result<TensorImage, crate::Error> {
3166 let img = TensorImage::new(width, height, fourcc, memory)?;
3167 let mut map = img.tensor().map()?;
3168 map.as_mut_slice()[..bytes.len()].copy_from_slice(bytes);
3169 Ok(img)
3170 }
3171
3172 #[test]
3174 #[cfg(all(target_os = "linux", feature = "dma_test_formats"))]
3175 fn test_opengl_nv12_to_rgba_reference() {
3176 let src = load_raw_image(
3178 1280,
3179 720,
3180 NV12,
3181 Some(TensorMemory::Dma),
3182 include_bytes!("../../../testdata/camera720p.nv12"),
3183 )
3184 .unwrap();
3185
3186 let reference = load_raw_image(
3188 1280,
3189 720,
3190 RGBA,
3191 None,
3192 include_bytes!("../../../testdata/camera720p.rgba"),
3193 )
3194 .unwrap();
3195
3196 let mut dst = TensorImage::new(1280, 720, RGBA, Some(TensorMemory::Dma)).unwrap();
3198 let mut gl = GLProcessorThreaded::new().unwrap();
3199 gl.convert(&src, &mut dst, Rotation::None, Flip::None, Crop::no_crop())
3200 .unwrap();
3201
3202 let cpu_dst = TensorImage::new(1280, 720, RGBA, None).unwrap();
3204 cpu_dst
3205 .tensor()
3206 .map()
3207 .unwrap()
3208 .as_mut_slice()
3209 .copy_from_slice(dst.tensor().map().unwrap().as_slice());
3210
3211 compare_images(&reference, &cpu_dst, 0.98, "opengl_nv12_to_rgba_reference");
3212 }
3213
3214 #[test]
3216 #[cfg(all(target_os = "linux", feature = "dma_test_formats"))]
3217 fn test_opengl_yuyv_to_rgba_reference() {
3218 let src = load_raw_image(
3220 1280,
3221 720,
3222 YUYV,
3223 Some(TensorMemory::Dma),
3224 include_bytes!("../../../testdata/camera720p.yuyv"),
3225 )
3226 .unwrap();
3227
3228 let reference = load_raw_image(
3230 1280,
3231 720,
3232 RGBA,
3233 None,
3234 include_bytes!("../../../testdata/camera720p.rgba"),
3235 )
3236 .unwrap();
3237
3238 let mut dst = TensorImage::new(1280, 720, RGBA, Some(TensorMemory::Dma)).unwrap();
3240 let mut gl = GLProcessorThreaded::new().unwrap();
3241 gl.convert(&src, &mut dst, Rotation::None, Flip::None, Crop::no_crop())
3242 .unwrap();
3243
3244 let cpu_dst = TensorImage::new(1280, 720, RGBA, None).unwrap();
3246 cpu_dst
3247 .tensor()
3248 .map()
3249 .unwrap()
3250 .as_mut_slice()
3251 .copy_from_slice(dst.tensor().map().unwrap().as_slice());
3252
3253 compare_images(&reference, &cpu_dst, 0.98, "opengl_yuyv_to_rgba_reference");
3254 }
3255}