1#![cfg(target_os = "linux")]
5#![cfg(feature = "opengl")]
6
7use edgefirst_decoder::DetectBox;
8#[cfg(feature = "decoder")]
9use edgefirst_decoder::Segmentation;
10use edgefirst_tensor::{TensorMemory, TensorTrait};
11use four_char_code::FourCharCode;
12use gbm::{
13 drm::{buffer::DrmFourcc, control::Device as DrmControlDevice, Device as DrmDevice},
14 AsRaw, Device,
15};
16use khronos_egl::{self as egl, Attrib, Display, Dynamic, Instance, EGL1_4};
17use log::{debug, error};
18use std::{
19 collections::BTreeSet,
20 ffi::{c_char, c_void, CStr, CString},
21 mem::ManuallyDrop,
22 os::fd::AsRawFd,
23 ptr::{null, null_mut, NonNull},
24 rc::Rc,
25 str::FromStr,
26 sync::OnceLock,
27 thread::JoinHandle,
28 time::Instant,
29};
30use tokio::sync::mpsc::Sender;
31
32macro_rules! function {
33 () => {{
34 fn f() {}
35 fn type_name_of<T>(_: T) -> &'static str {
36 std::any::type_name::<T>()
37 }
38 let name = type_name_of(f);
39
40 match &name[..name.len() - 3].rfind(':') {
42 Some(pos) => &name[pos + 1..name.len() - 3],
43 None => &name[..name.len() - 3],
44 }
45 }};
46}
47
48#[cfg(feature = "decoder")]
49use crate::DEFAULT_COLORS;
50use crate::{
51 CPUProcessor, Crop, Error, Flip, ImageProcessorTrait, Rect, Rotation, TensorImage,
52 TensorImageRef, GREY, NV12, PLANAR_RGB, PLANAR_RGBA, RGB, RGBA, YUYV,
53};
54
55static EGL_LIB: OnceLock<&'static libloading::Library> = OnceLock::new();
59
60fn get_egl_lib() -> Result<&'static libloading::Library, crate::Error> {
61 if let Some(egl) = EGL_LIB.get() {
62 Ok(egl)
63 } else {
64 let egl = unsafe { libloading::Library::new("libEGL.so.1")? };
65 let egl: &'static libloading::Library = Box::leak(Box::new(egl));
67 Ok(EGL_LIB.get_or_init(|| egl))
68 }
69}
70
71type Egl = Instance<Dynamic<&'static libloading::Library, EGL1_4>>;
72pub(crate) struct GlContext {
73 pub(crate) support_dma: bool,
74 pub(crate) surface: Option<egl::Surface>,
75 pub(crate) display: EglDisplayType,
76 pub(crate) ctx: egl::Context,
77 pub(crate) egl: ManuallyDrop<Rc<Egl>>,
82}
83
84pub(crate) enum EglDisplayType {
85 Default(egl::Display),
86 Gbm(egl::Display, #[allow(dead_code)] Device<Card>),
87 PlatformDisplay(egl::Display),
88}
89
90impl EglDisplayType {
91 fn as_display(&self) -> egl::Display {
92 match self {
93 EglDisplayType::Default(disp) => *disp,
94 EglDisplayType::Gbm(disp, _) => *disp,
95 EglDisplayType::PlatformDisplay(disp) => *disp,
96 }
97 }
98}
99
100impl GlContext {
101 pub(crate) fn new() -> Result<GlContext, crate::Error> {
102 let egl: Rc<Egl> =
104 Rc::new(unsafe { Instance::<Dynamic<_, EGL1_4>>::load_required_from(get_egl_lib()?)? });
105
106 if let Ok(headless) = Self::try_initialize_egl(egl.clone(), Self::egl_get_gbm_display) {
110 return Ok(headless);
111 } else {
112 log::debug!("Didn't initialize EGL with GBM Display");
113 }
114
115 if let Ok(headless) =
116 Self::try_initialize_egl(egl.clone(), Self::egl_get_platform_display_from_device)
117 {
118 return Ok(headless);
119 } else {
120 log::debug!("Didn't initialize EGL with platform display from device enumeration");
121 }
122
123 if let Ok(headless) = Self::try_initialize_egl(egl.clone(), Self::egl_get_default_display) {
124 return Ok(headless);
125 } else {
126 log::debug!("Didn't initialize EGL with Default Display");
127 }
128
129 Err(Error::OpenGl(
130 "Could not initialize EGL with any known method".to_string(),
131 ))
132 }
133
134 fn try_initialize_egl(
135 egl: Rc<Egl>,
136 display_fn: impl Fn(&Egl) -> Result<EglDisplayType, crate::Error>,
137 ) -> Result<GlContext, crate::Error> {
138 let display = display_fn(&egl)?;
139 log::debug!("egl initialize with display: {:x?}", display.as_display());
140 egl.initialize(display.as_display())?;
141 let attributes = [
142 egl::SURFACE_TYPE,
143 egl::PBUFFER_BIT,
144 egl::RENDERABLE_TYPE,
145 egl::OPENGL_ES3_BIT,
146 egl::RED_SIZE,
147 8,
148 egl::GREEN_SIZE,
149 8,
150 egl::BLUE_SIZE,
151 8,
152 egl::ALPHA_SIZE,
153 8,
154 egl::NONE,
155 ];
156
157 let config =
158 if let Some(config) = egl.choose_first_config(display.as_display(), &attributes)? {
159 config
160 } else {
161 return Err(crate::Error::NotImplemented(
162 "Did not find valid OpenGL ES config".to_string(),
163 ));
164 };
165
166 debug!("config: {config:?}");
167
168 let surface = Some(egl.create_pbuffer_surface(
169 display.as_display(),
170 config,
171 &[egl::WIDTH, 64, egl::HEIGHT, 64, egl::NONE],
172 )?);
173
174 egl.bind_api(egl::OPENGL_ES_API)?;
175 let context_attributes = [egl::CONTEXT_MAJOR_VERSION, 3, egl::NONE, egl::NONE];
176
177 let ctx = egl.create_context(display.as_display(), config, None, &context_attributes)?;
178 debug!("ctx: {ctx:?}");
179
180 egl.make_current(display.as_display(), surface, surface, Some(ctx))?;
181
182 let support_dma = Self::egl_check_support_dma(&egl).is_ok();
183 let headless = GlContext {
184 display,
185 ctx,
186 egl: ManuallyDrop::new(egl),
187 surface,
188 support_dma,
189 };
190 Ok(headless)
191 }
192
193 fn egl_get_default_display(egl: &Egl) -> Result<EglDisplayType, crate::Error> {
194 if let Some(display) = unsafe { egl.get_display(egl::DEFAULT_DISPLAY) } {
196 debug!("default display: {display:?}");
197 return Ok(EglDisplayType::Default(display));
198 }
199
200 Err(Error::OpenGl(
201 "Could not obtain EGL Default Display".to_string(),
202 ))
203 }
204
205 fn egl_get_gbm_display(egl: &Egl) -> Result<EglDisplayType, crate::Error> {
206 let gbm = Device::new(Card::open_global()?)?;
208
209 debug!("gbm: {gbm:?}");
210 let display = Self::egl_get_platform_display_with_fallback(
211 egl,
212 egl_ext::PLATFORM_GBM_KHR,
213 gbm.as_raw() as *mut c_void,
214 &[egl::ATTRIB_NONE],
215 )?;
216
217 Ok(EglDisplayType::Gbm(display, gbm))
218 }
219
220 fn egl_get_platform_display_from_device(egl: &Egl) -> Result<EglDisplayType, crate::Error> {
221 let extensions = egl.query_string(None, egl::EXTENSIONS)?;
222 let extensions = extensions.to_string_lossy();
223 log::debug!("EGL Extensions: {}", extensions);
224
225 if !extensions.contains("EGL_EXT_device_enumeration") {
226 return Err(Error::GLVersion(
227 "EGL doesn't supported EGL_EXT_device_enumeration extension".to_string(),
228 ));
229 }
230
231 type EGLDeviceEXT = *mut c_void;
232 let devices = if let Some(ext) = egl.get_proc_address("eglQueryDevicesEXT") {
233 let func: unsafe extern "system" fn(
234 max_devices: egl::Int,
235 devices: *mut EGLDeviceEXT,
236 num_devices: *mut egl::Int,
237 ) -> *const c_char = unsafe { std::mem::transmute(ext) };
238 let mut devices = [std::ptr::null_mut(); 10];
239 let mut num_devices = 0;
240 unsafe { func(devices.len() as i32, devices.as_mut_ptr(), &mut num_devices) };
241 for i in 0..num_devices {
242 log::debug!("EGL device: {:?}", devices[i as usize]);
243 }
244 devices[0..num_devices as usize].to_vec()
245 } else {
246 return Err(Error::GLVersion(
247 "EGL doesn't supported eglQueryDevicesEXT function".to_string(),
248 ));
249 };
250
251 if !extensions.contains("EGL_EXT_platform_device") {
252 return Err(Error::GLVersion(
253 "EGL doesn't supported EGL_EXT_platform_device extension".to_string(),
254 ));
255 }
256
257 let disp = Self::egl_get_platform_display_with_fallback(
259 egl,
260 egl_ext::PLATFORM_DEVICE_EXT,
261 devices[0],
262 &[egl::ATTRIB_NONE],
263 )?;
264 Ok(EglDisplayType::PlatformDisplay(disp))
265 }
266
267 fn egl_check_support_dma(egl: &Egl) -> Result<(), crate::Error> {
268 let extensions = egl.query_string(None, egl::EXTENSIONS)?;
269 let extensions = extensions.to_string_lossy();
270 log::debug!("EGL Extensions: {}", extensions);
271
272 if egl.upcast::<egl::EGL1_5>().is_some() {
273 return Ok(());
274 }
275
276 if !extensions.contains("EGL_EXT_image_dma_buf_import") {
277 return Err(crate::Error::GLVersion(
278 "EGL does not support EGL_EXT_image_dma_buf_import extension".to_string(),
279 ));
280 }
281
282 if egl.get_proc_address("eglCreateImageKHR").is_none() {
283 return Err(crate::Error::GLVersion(
284 "EGL does not support eglCreateImageKHR function".to_string(),
285 ));
286 }
287
288 if egl.get_proc_address("eglDestroyImageKHR").is_none() {
289 return Err(crate::Error::GLVersion(
290 "EGL does not support eglDestroyImageKHR function".to_string(),
291 ));
292 }
293 Ok(())
295 }
296
297 fn egl_get_platform_display_with_fallback(
298 egl: &Egl,
299 platform: egl::Enum,
300 native_display: *mut c_void,
301 attrib_list: &[Attrib],
302 ) -> Result<Display, Error> {
303 if let Some(egl) = egl.upcast::<egl::EGL1_5>() {
304 unsafe { egl.get_platform_display(platform, native_display, attrib_list) }
305 .map_err(|e| e.into())
306 } else if let Some(ext) = egl.get_proc_address("eglGetPlatformDisplayEXT") {
307 let func: unsafe extern "system" fn(
308 platform: egl::Enum,
309 native_display: *mut c_void,
310 attrib_list: *const Attrib,
311 ) -> egl::EGLDisplay = unsafe { std::mem::transmute(ext) };
312 let disp = unsafe { func(platform, native_display, attrib_list.as_ptr()) };
313 if disp != egl::NO_DISPLAY {
314 Ok(unsafe { Display::from_ptr(disp) })
315 } else {
316 Err(egl.get_error().map(|e| e.into()).unwrap_or(Error::Internal(
317 "EGL failed but no error was reported".to_owned(),
318 )))
319 }
320 } else {
321 Err(Error::EGLLoad(egl::LoadError::InvalidVersion {
322 provided: egl.version(),
323 required: khronos_egl::Version::EGL1_5,
324 }))
325 }
326 }
327
328 fn egl_create_image_with_fallback(
329 egl: &Egl,
330 display: Display,
331 ctx: egl::Context,
332 target: egl::Enum,
333 buffer: egl::ClientBuffer,
334 attrib_list: &[Attrib],
335 ) -> Result<egl::Image, Error> {
336 if let Some(egl) = egl.upcast::<egl::EGL1_5>() {
337 egl.create_image(display, ctx, target, buffer, attrib_list)
338 .map_err(|e| e.into())
339 } else if let Some(ext) = egl.get_proc_address("eglCreateImageKHR") {
340 log::trace!("eglCreateImageKHR addr: {:?}", ext);
341 let func: unsafe extern "system" fn(
342 display: egl::EGLDisplay,
343 ctx: egl::EGLContext,
344 target: egl::Enum,
345 buffer: egl::EGLClientBuffer,
346 attrib_list: *const egl::Int,
347 ) -> egl::EGLImage = unsafe { std::mem::transmute(ext) };
348 let new_attrib_list = attrib_list
349 .iter()
350 .map(|x| *x as egl::Int)
351 .collect::<Vec<_>>();
352
353 let image = unsafe {
354 func(
355 display.as_ptr(),
356 ctx.as_ptr(),
357 target,
358 buffer.as_ptr(),
359 new_attrib_list.as_ptr(),
360 )
361 };
362 if image != egl::NO_IMAGE {
363 Ok(unsafe { egl::Image::from_ptr(image) })
364 } else {
365 Err(egl.get_error().map(|e| e.into()).unwrap_or(Error::Internal(
366 "EGL failed but no error was reported".to_owned(),
367 )))
368 }
369 } else {
370 Err(Error::EGLLoad(egl::LoadError::InvalidVersion {
371 provided: egl.version(),
372 required: khronos_egl::Version::EGL1_5,
373 }))
374 }
375 }
376
377 fn egl_destory_image_with_fallback(
378 egl: &Egl,
379 display: Display,
380 image: egl::Image,
381 ) -> Result<(), Error> {
382 if let Some(egl) = egl.upcast::<egl::EGL1_5>() {
383 egl.destroy_image(display, image).map_err(|e| e.into())
384 } else if let Some(ext) = egl.get_proc_address("eglDestroyImageKHR") {
385 let func: unsafe extern "system" fn(
386 display: egl::EGLDisplay,
387 image: egl::EGLImage,
388 ) -> egl::Boolean = unsafe { std::mem::transmute(ext) };
389 let res = unsafe { func(display.as_ptr(), image.as_ptr()) };
390 if res == egl::TRUE {
391 Ok(())
392 } else {
393 Err(egl.get_error().map(|e| e.into()).unwrap_or(Error::Internal(
394 "EGL failed but no error was reported".to_owned(),
395 )))
396 }
397 } else {
398 Err(Error::EGLLoad(egl::LoadError::InvalidVersion {
399 provided: egl.version(),
400 required: khronos_egl::Version::EGL1_5,
401 }))
402 }
403 }
404}
405
406impl Drop for GlContext {
407 fn drop(&mut self) {
408 let prev_hook = std::panic::take_hook();
413 std::panic::set_hook(Box::new(|_| {}));
414 let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
415 let _ = self
416 .egl
417 .make_current(self.display.as_display(), None, None, None);
418
419 let _ = self
420 .egl
421 .destroy_context(self.display.as_display(), self.ctx);
422
423 if let Some(surface) = self.surface.take() {
424 let _ = self.egl.destroy_surface(self.display.as_display(), surface);
425 }
426
427 }));
433 std::panic::set_hook(prev_hook);
434
435 }
442}
443
444#[derive(Debug)]
445pub(crate) struct Card(std::fs::File);
447
448impl std::os::unix::io::AsFd for Card {
451 fn as_fd(&self) -> std::os::unix::io::BorrowedFd<'_> {
452 self.0.as_fd()
453 }
454}
455
456impl DrmDevice for Card {}
458impl DrmControlDevice for Card {}
459
460impl Card {
462 pub fn open(path: &str) -> Result<Self, crate::Error> {
463 let mut options = std::fs::OpenOptions::new();
464 options.read(true);
465 options.write(true);
466 let c = options.open(path);
467 match c {
468 Ok(c) => Ok(Card(c)),
469 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
470 Err(Error::NotFound(format!("File not found: {path}")))
471 }
472 Err(e) => Err(e.into()),
473 }
474 }
475
476 pub fn open_global() -> Result<Self, crate::Error> {
477 let targets = ["/dev/dri/renderD128", "/dev/dri/card0", "/dev/dri/card1"];
478 let e = Self::open(targets[0]);
479 if let Ok(t) = e {
480 return Ok(t);
481 }
482 for t in &targets[1..] {
483 if let Ok(t) = Self::open(t) {
484 return Ok(t);
485 }
486 }
487 e
488 }
489}
490
491#[derive(Debug, Clone, Copy)]
492struct RegionOfInterest {
493 left: f32,
494 top: f32,
495 right: f32,
496 bottom: f32,
497}
498
499enum GLProcessorMessage {
500 ImageConvert(
501 SendablePtr<TensorImage>,
502 SendablePtr<TensorImage>,
503 Rotation,
504 Flip,
505 Crop,
506 tokio::sync::oneshot::Sender<Result<(), Error>>,
507 ),
508 SetColors(
509 Vec<[u8; 4]>,
510 tokio::sync::oneshot::Sender<Result<(), Error>>,
511 ),
512 ImageRender(
513 SendablePtr<TensorImage>,
514 SendablePtr<DetectBox>,
515 SendablePtr<Segmentation>,
516 tokio::sync::oneshot::Sender<Result<(), Error>>,
517 ),
518}
519
520#[derive(Debug)]
525pub struct GLProcessorThreaded {
526 handle: Option<JoinHandle<()>>,
528
529 sender: Option<Sender<GLProcessorMessage>>,
531 support_dma: bool,
532}
533
534unsafe impl Send for GLProcessorThreaded {}
535unsafe impl Sync for GLProcessorThreaded {}
536
537struct SendablePtr<T: Send> {
538 ptr: NonNull<T>,
539 len: usize,
540}
541
542unsafe impl<T> Send for SendablePtr<T> where T: Send {}
543
544impl GLProcessorThreaded {
545 pub fn new() -> Result<Self, Error> {
547 let (send, mut recv) = tokio::sync::mpsc::channel::<GLProcessorMessage>(1);
548
549 let (create_ctx_send, create_ctx_recv) = tokio::sync::oneshot::channel();
550
551 let func = move || {
552 let mut gl_converter = match GLProcessorST::new() {
553 Ok(gl) => gl,
554 Err(e) => {
555 let _ = create_ctx_send.send(Err(e));
556 return;
557 }
558 };
559 let _ = create_ctx_send.send(Ok(gl_converter.gl_context.support_dma));
560 while let Some(msg) = recv.blocking_recv() {
561 match msg {
562 GLProcessorMessage::ImageConvert(src, mut dst, rotation, flip, crop, resp) => {
563 let src = unsafe { src.ptr.as_ref() };
566 let dst = unsafe { dst.ptr.as_mut() };
567 let res = gl_converter.convert(src, dst, rotation, flip, crop);
568 let _ = resp.send(res);
569 }
570 GLProcessorMessage::ImageRender(mut dst, det, seg, resp) => {
571 let dst = unsafe { dst.ptr.as_mut() };
575 let det = unsafe { std::slice::from_raw_parts(det.ptr.as_ptr(), det.len) };
576 let seg = unsafe { std::slice::from_raw_parts(seg.ptr.as_ptr(), seg.len) };
577 let res = gl_converter.render_to_image(dst, det, seg);
578 let _ = resp.send(res);
579 }
580 GLProcessorMessage::SetColors(colors, resp) => {
581 let res = gl_converter.set_class_colors(&colors);
582 let _ = resp.send(res);
583 }
584 }
585 }
586 };
587
588 let handle = std::thread::spawn(func);
590
591 let support_dma = match create_ctx_recv.blocking_recv() {
592 Ok(Err(e)) => return Err(e),
593 Err(_) => {
594 return Err(Error::Internal(
595 "GL converter error messaging closed without update".to_string(),
596 ));
597 }
598 Ok(Ok(supports_dma)) => supports_dma,
599 };
600
601 Ok(Self {
602 handle: Some(handle),
603 sender: Some(send),
604 support_dma,
605 })
606 }
607}
608
609impl ImageProcessorTrait for GLProcessorThreaded {
610 fn convert(
611 &mut self,
612 src: &TensorImage,
613 dst: &mut TensorImage,
614 rotation: crate::Rotation,
615 flip: Flip,
616 crop: Crop,
617 ) -> crate::Result<()> {
618 crop.check_crop(src, dst)?;
619 if !GLProcessorST::check_src_format_supported(self.support_dma, src) {
620 return Err(crate::Error::NotSupported(format!(
621 "Opengl doesn't support {} source texture",
622 src.fourcc().display()
623 )));
624 }
625
626 if !GLProcessorST::check_dst_format_supported(self.support_dma, dst) {
627 return Err(crate::Error::NotSupported(format!(
628 "Opengl doesn't support {} destination texture",
629 dst.fourcc().display()
630 )));
631 }
632
633 let (err_send, err_recv) = tokio::sync::oneshot::channel();
634 self.sender
635 .as_ref()
636 .unwrap()
637 .blocking_send(GLProcessorMessage::ImageConvert(
638 SendablePtr {
639 ptr: src.into(),
640 len: 1,
641 },
642 SendablePtr {
643 ptr: dst.into(),
644 len: 1,
645 },
646 rotation,
647 flip,
648 crop,
649 err_send,
650 ))
651 .map_err(|_| Error::Internal("GL converter thread exited".to_string()))?;
652 err_recv.blocking_recv().map_err(|_| {
653 Error::Internal("GL converter error messaging closed without update".to_string())
654 })?
655 }
656
657 fn convert_ref(
658 &mut self,
659 src: &TensorImage,
660 dst: &mut TensorImageRef<'_>,
661 rotation: Rotation,
662 flip: Flip,
663 crop: Crop,
664 ) -> crate::Result<()> {
665 let mut cpu = CPUProcessor::new();
667 cpu.convert_ref(src, dst, rotation, flip, crop)
668 }
669
670 #[cfg(feature = "decoder")]
671 fn render_to_image(
672 &mut self,
673 dst: &mut TensorImage,
674 detect: &[crate::DetectBox],
675 segmentation: &[crate::Segmentation],
676 ) -> crate::Result<()> {
677 let (err_send, err_recv) = tokio::sync::oneshot::channel();
678 self.sender
679 .as_ref()
680 .unwrap()
681 .blocking_send(GLProcessorMessage::ImageRender(
682 SendablePtr {
683 ptr: dst.into(),
684 len: 1,
685 },
686 SendablePtr {
687 ptr: NonNull::new(detect.as_ptr() as *mut DetectBox).unwrap(),
688 len: detect.len(),
689 },
690 SendablePtr {
691 ptr: NonNull::new(segmentation.as_ptr() as *mut Segmentation).unwrap(),
692 len: segmentation.len(),
693 },
694 err_send,
695 ))
696 .map_err(|_| Error::Internal("GL converter thread exited".to_string()))?;
697 err_recv.blocking_recv().map_err(|_| {
698 Error::Internal("GL converter error messaging closed without update".to_string())
699 })?
700 }
701
702 #[cfg(feature = "decoder")]
703 fn set_class_colors(&mut self, colors: &[[u8; 4]]) -> Result<(), crate::Error> {
704 let (err_send, err_recv) = tokio::sync::oneshot::channel();
705 self.sender
706 .as_ref()
707 .unwrap()
708 .blocking_send(GLProcessorMessage::SetColors(colors.to_vec(), err_send))
709 .map_err(|_| Error::Internal("GL converter thread exited".to_string()))?;
710 err_recv.blocking_recv().map_err(|_| {
711 Error::Internal("GL converter error messaging closed without update".to_string())
712 })?
713 }
714}
715
716impl Drop for GLProcessorThreaded {
717 fn drop(&mut self) {
718 drop(self.sender.take());
719 let _ = self.handle.take().and_then(|h| h.join().ok());
720 }
721}
722
723pub struct GLProcessorST {
725 camera_eglimage_texture: Texture,
726 camera_normal_texture: Texture,
727 render_texture: Texture,
728 #[cfg(feature = "decoder")]
729 segmentation_texture: Texture,
730 #[cfg(feature = "decoder")]
731 segmentation_program: GlProgram,
732 #[cfg(feature = "decoder")]
733 instanced_segmentation_program: GlProgram,
734 #[cfg(feature = "decoder")]
735 color_program: GlProgram,
736 vertex_buffer: Buffer,
737 texture_buffer: Buffer,
738 texture_program: GlProgram,
739 texture_program_yuv: GlProgram,
740 texture_program_planar: GlProgram,
741 gl_context: GlContext,
742}
743
744impl ImageProcessorTrait for GLProcessorST {
745 fn convert(
746 &mut self,
747 src: &TensorImage,
748 dst: &mut TensorImage,
749 rotation: crate::Rotation,
750 flip: Flip,
751 crop: Crop,
752 ) -> crate::Result<()> {
753 crop.check_crop(src, dst)?;
754 if !Self::check_src_format_supported(self.gl_context.support_dma, src) {
755 return Err(crate::Error::NotSupported(format!(
756 "Opengl doesn't support {} source texture",
757 src.fourcc().display()
758 )));
759 }
760
761 if !Self::check_dst_format_supported(self.gl_context.support_dma, dst) {
762 return Err(crate::Error::NotSupported(format!(
763 "Opengl doesn't support {} destination texture",
764 dst.fourcc().display()
765 )));
766 }
767 log::debug!(
768 "dst tensor: {:?} src tensor :{:?}",
769 dst.tensor().memory(),
770 src.tensor().memory()
771 );
772 check_gl_error(function!(), line!())?;
773 if self.gl_context.support_dma
774 && dst.tensor().memory() == TensorMemory::Dma
775 && dst.fourcc() != RGB
776 {
778 let res = self.convert_dest_dma(dst, src, rotation, flip, crop);
779 return res;
780 }
781 let start = Instant::now();
782 let res = self.convert_dest_non_dma(dst, src, rotation, flip, crop);
783 log::debug!("convert_dest_non_dma takes {:?}", start.elapsed());
784 res
785 }
786
787 fn convert_ref(
788 &mut self,
789 src: &TensorImage,
790 dst: &mut TensorImageRef<'_>,
791 rotation: Rotation,
792 flip: Flip,
793 crop: Crop,
794 ) -> crate::Result<()> {
795 let mut cpu = CPUProcessor::new();
797 cpu.convert_ref(src, dst, rotation, flip, crop)
798 }
799
800 #[cfg(feature = "decoder")]
801 fn render_to_image(
802 &mut self,
803 dst: &mut TensorImage,
804 detect: &[DetectBox],
805 segmentation: &[Segmentation],
806 ) -> Result<(), crate::Error> {
807 use crate::FunctionTimer;
808
809 let _timer = FunctionTimer::new("GLProcessorST::render_to_image");
810 if !matches!(dst.fourcc(), RGBA | RGB) {
811 return Err(crate::Error::NotSupported(
812 "Opengl image rendering only supports RGBA or RGB images".to_string(),
813 ));
814 }
815
816 let (_render_buffer, is_dma) = match dst.tensor.memory() {
817 edgefirst_tensor::TensorMemory::Dma => {
818 if let Ok(render_buffer) = self.setup_renderbuffer_dma(dst) {
819 (render_buffer, true)
820 } else {
821 (
822 self.setup_renderbuffer_non_dma(
823 dst,
824 Crop::new().with_dst_rect(Some(Rect::new(0, 0, 0, 0))),
825 )?,
826 false,
827 )
828 }
829 }
830 _ => (
831 self.setup_renderbuffer_non_dma(
832 dst,
833 Crop::new().with_dst_rect(Some(Rect::new(0, 0, 0, 0))),
834 )?,
835 false,
836 ), };
838
839 gls::enable(gls::gl::BLEND);
840 gls::blend_func_separate(
841 gls::gl::SRC_ALPHA,
842 gls::gl::ONE_MINUS_SRC_ALPHA,
843 gls::gl::ZERO,
844 gls::gl::ONE,
845 );
846
847 self.render_box(dst, detect)?;
848 self.render_segmentation(detect, segmentation)?;
849
850 gls::finish();
851 if !is_dma {
852 let mut dst_map = dst.tensor().map()?;
853 let format = match dst.fourcc() {
854 RGB => gls::gl::RGB,
855 RGBA => gls::gl::RGBA,
856 _ => unreachable!(),
857 };
858 unsafe {
859 gls::gl::ReadBuffer(gls::gl::COLOR_ATTACHMENT0);
860 gls::gl::ReadnPixels(
861 0,
862 0,
863 dst.width() as i32,
864 dst.height() as i32,
865 format,
866 gls::gl::UNSIGNED_BYTE,
867 dst.tensor.len() as i32,
868 dst_map.as_mut_ptr() as *mut c_void,
869 );
870 }
871 }
872
873 Ok(())
874 }
875
876 #[cfg(feature = "decoder")]
877 fn set_class_colors(&mut self, colors: &[[u8; 4]]) -> crate::Result<()> {
878 if colors.is_empty() {
879 return Ok(());
880 }
881 let mut colors_f32 = colors
882 .iter()
883 .map(|c| {
884 [
885 c[0] as f32 / 255.0,
886 c[1] as f32 / 255.0,
887 c[2] as f32 / 255.0,
888 c[3] as f32 / 255.0,
889 ]
890 })
891 .take(20)
892 .collect::<Vec<[f32; 4]>>();
893
894 self.segmentation_program
895 .load_uniform_4fv(c"colors", &colors_f32)?;
896 self.instanced_segmentation_program
897 .load_uniform_4fv(c"colors", &colors_f32)?;
898
899 colors_f32.iter_mut().for_each(|c| {
900 c[3] = 1.0; });
902 self.color_program
903 .load_uniform_4fv(c"colors", &colors_f32)?;
904
905 Ok(())
906 }
907}
908
909impl GLProcessorST {
910 pub fn new() -> Result<GLProcessorST, crate::Error> {
911 let gl_context = GlContext::new()?;
912 gls::load_with(|s| {
913 gl_context
914 .egl
915 .get_proc_address(s)
916 .map_or(std::ptr::null(), |p| p as *const _)
917 });
918
919 Self::gl_check_support()?;
920
921 unsafe {
923 gls::gl::PixelStorei(gls::gl::PACK_ALIGNMENT, 1);
924 gls::gl::PixelStorei(gls::gl::UNPACK_ALIGNMENT, 1);
925 }
926
927 let texture_program_planar =
928 GlProgram::new(generate_vertex_shader(), generate_planar_rgb_shader())?;
929
930 let texture_program =
931 GlProgram::new(generate_vertex_shader(), generate_texture_fragment_shader())?;
932
933 let texture_program_yuv = GlProgram::new(
934 generate_vertex_shader(),
935 generate_texture_fragment_shader_yuv(),
936 )?;
937
938 #[cfg(feature = "decoder")]
939 let segmentation_program =
940 GlProgram::new(generate_vertex_shader(), generate_segmentation_shader())?;
941 #[cfg(feature = "decoder")]
942 segmentation_program.load_uniform_4fv(c"colors", &DEFAULT_COLORS)?;
943 #[cfg(feature = "decoder")]
944 let instanced_segmentation_program = GlProgram::new(
945 generate_vertex_shader(),
946 generate_instanced_segmentation_shader(),
947 )?;
948 #[cfg(feature = "decoder")]
949 instanced_segmentation_program.load_uniform_4fv(c"colors", &DEFAULT_COLORS)?;
950
951 #[cfg(feature = "decoder")]
952 let color_program = GlProgram::new(generate_vertex_shader(), generate_color_shader())?;
953 #[cfg(feature = "decoder")]
954 color_program.load_uniform_4fv(c"colors", &DEFAULT_COLORS)?;
955
956 let camera_eglimage_texture = Texture::new();
957 let camera_normal_texture = Texture::new();
958 let render_texture = Texture::new();
959 let segmentation_texture = Texture::new();
960 let vertex_buffer = Buffer::new(0, 3, 100);
961 let texture_buffer = Buffer::new(1, 2, 100);
962
963 let converter = GLProcessorST {
964 gl_context,
965 texture_program,
966 texture_program_yuv,
967 texture_program_planar,
968 camera_eglimage_texture,
969 camera_normal_texture,
970 #[cfg(feature = "decoder")]
971 segmentation_texture,
972 vertex_buffer,
973 texture_buffer,
974 render_texture,
975 #[cfg(feature = "decoder")]
976 segmentation_program,
977 #[cfg(feature = "decoder")]
978 instanced_segmentation_program,
979 #[cfg(feature = "decoder")]
980 color_program,
981 };
982 check_gl_error(function!(), line!())?;
983
984 log::debug!("GLConverter created");
985 Ok(converter)
986 }
987
988 fn check_src_format_supported(support_dma: bool, img: &TensorImage) -> bool {
989 if support_dma && img.tensor().memory() == TensorMemory::Dma {
990 matches!(img.fourcc(), RGBA | GREY | YUYV | NV12)
992 } else {
993 matches!(img.fourcc(), RGB | RGBA | GREY)
994 }
995 }
996
997 fn check_dst_format_supported(support_dma: bool, img: &TensorImage) -> bool {
998 if support_dma && img.tensor().memory() == TensorMemory::Dma {
999 matches!(img.fourcc(), RGBA | GREY | PLANAR_RGB)
1001 } else {
1002 matches!(img.fourcc(), RGB | RGBA | GREY)
1003 }
1004 }
1005
1006 fn gl_check_support() -> Result<(), crate::Error> {
1007 if let Ok(version) = gls::get_string(gls::gl::SHADING_LANGUAGE_VERSION) {
1008 log::debug!("GL Shading Language Version: {version:?}");
1009 } else {
1010 log::warn!("Could not get GL Shading Language Version");
1011 }
1012
1013 let extensions = unsafe {
1014 let str = gls::gl::GetString(gls::gl::EXTENSIONS);
1015 if str.is_null() {
1016 return Err(crate::Error::GLVersion(
1017 "GL returned no supported extensions".to_string(),
1018 ));
1019 }
1020 CStr::from_ptr(str as *const c_char)
1021 .to_string_lossy()
1022 .to_string()
1023 };
1024 log::debug!("GL Extensions: {extensions}");
1025 let required_ext = [
1026 "GL_OES_EGL_image_external_essl3",
1027 "GL_OES_surfaceless_context",
1028 ];
1029 let extensions = extensions.split_ascii_whitespace().collect::<BTreeSet<_>>();
1030 for required in required_ext {
1031 if !extensions.contains(required) {
1032 return Err(crate::Error::GLVersion(format!(
1033 "GL does not support {required} extension",
1034 )));
1035 }
1036 }
1037
1038 Ok(())
1039 }
1040
1041 fn setup_renderbuffer_dma(&mut self, dst: &TensorImage) -> crate::Result<FrameBuffer> {
1042 let frame_buffer = FrameBuffer::new();
1043 frame_buffer.bind();
1044
1045 let (width, height) = if matches!(dst.fourcc(), PLANAR_RGB) {
1046 let width = dst.width();
1047 let height = dst.height() * 3;
1048 (width as i32, height as i32)
1049 } else {
1050 (dst.width() as i32, dst.height() as i32)
1051 };
1052 let dest_img = self.create_image_from_dma2(dst)?;
1053 unsafe {
1054 gls::gl::UseProgram(self.texture_program_yuv.id);
1055 gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1056 gls::gl::BindTexture(gls::gl::TEXTURE_2D, self.render_texture.id);
1057 gls::gl::TexParameteri(
1058 gls::gl::TEXTURE_2D,
1059 gls::gl::TEXTURE_MIN_FILTER,
1060 gls::gl::LINEAR as i32,
1061 );
1062 gls::gl::TexParameteri(
1063 gls::gl::TEXTURE_2D,
1064 gls::gl::TEXTURE_MAG_FILTER,
1065 gls::gl::LINEAR as i32,
1066 );
1067 gls::gl::EGLImageTargetTexture2DOES(gls::gl::TEXTURE_2D, dest_img.egl_image.as_ptr());
1068 gls::gl::FramebufferTexture2D(
1069 gls::gl::FRAMEBUFFER,
1070 gls::gl::COLOR_ATTACHMENT0,
1071 gls::gl::TEXTURE_2D,
1072 self.render_texture.id,
1073 0,
1074 );
1075 check_gl_error(function!(), line!())?;
1076 gls::gl::Viewport(0, 0, width, height);
1077 }
1078 Ok(frame_buffer)
1079 }
1080
1081 fn convert_dest_dma(
1082 &mut self,
1083 dst: &mut TensorImage,
1084 src: &TensorImage,
1085 rotation: crate::Rotation,
1086 flip: Flip,
1087 crop: Crop,
1088 ) -> crate::Result<()> {
1089 assert!(self.gl_context.support_dma);
1090 let _framebuffer = self.setup_renderbuffer_dma(dst)?;
1091 if dst.is_planar() {
1092 self.convert_to_planar(src, dst, rotation, flip, crop)
1093 } else {
1094 self.convert_to(src, dst, rotation, flip, crop)
1095 }
1096 }
1097
1098 fn setup_renderbuffer_non_dma(
1099 &mut self,
1100 dst: &TensorImage,
1101 crop: Crop,
1102 ) -> crate::Result<FrameBuffer> {
1103 debug_assert!(matches!(dst.fourcc(), RGB | RGBA | GREY | PLANAR_RGB));
1104 let (width, height) = if dst.is_planar() {
1105 let width = dst.width() / 4;
1106 let height = match dst.fourcc() {
1107 RGBA => dst.height() * 4,
1108 RGB => dst.height() * 3,
1109 GREY => dst.height(),
1110 _ => unreachable!(),
1111 };
1112 (width as i32, height as i32)
1113 } else {
1114 (dst.width() as i32, dst.height() as i32)
1115 };
1116
1117 let format = if dst.is_planar() {
1118 gls::gl::RED
1119 } else {
1120 match dst.fourcc() {
1121 RGB => gls::gl::RGB,
1122 RGBA => gls::gl::RGBA,
1123 GREY => gls::gl::RED,
1124 _ => unreachable!(),
1125 }
1126 };
1127
1128 let start = Instant::now();
1129 let frame_buffer = FrameBuffer::new();
1130 frame_buffer.bind();
1131
1132 let map;
1133
1134 let pixels = if crop.dst_rect.is_none_or(|crop| {
1135 crop.top == 0
1136 && crop.left == 0
1137 && crop.height == dst.height()
1138 && crop.width == dst.width()
1139 }) {
1140 std::ptr::null()
1141 } else {
1142 map = dst.tensor().map()?;
1143 map.as_ptr() as *const c_void
1144 };
1145 unsafe {
1146 gls::gl::UseProgram(self.texture_program.id);
1147 gls::gl::BindTexture(gls::gl::TEXTURE_2D, self.render_texture.id);
1148 gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1149 gls::gl::TexParameteri(
1150 gls::gl::TEXTURE_2D,
1151 gls::gl::TEXTURE_MIN_FILTER,
1152 gls::gl::LINEAR as i32,
1153 );
1154 gls::gl::TexParameteri(
1155 gls::gl::TEXTURE_2D,
1156 gls::gl::TEXTURE_MAG_FILTER,
1157 gls::gl::LINEAR as i32,
1158 );
1159
1160 gls::gl::TexImage2D(
1161 gls::gl::TEXTURE_2D,
1162 0,
1163 format as i32,
1164 width,
1165 height,
1166 0,
1167 format,
1168 gls::gl::UNSIGNED_BYTE,
1169 pixels,
1170 );
1171 check_gl_error(function!(), line!())?;
1172 gls::gl::FramebufferTexture2D(
1173 gls::gl::FRAMEBUFFER,
1174 gls::gl::COLOR_ATTACHMENT0,
1175 gls::gl::TEXTURE_2D,
1176 self.render_texture.id,
1177 0,
1178 );
1179 check_gl_error(function!(), line!())?;
1180 gls::gl::Viewport(0, 0, width, height);
1181 }
1182 log::debug!("Set up framebuffer takes {:?}", start.elapsed());
1183 Ok(frame_buffer)
1184 }
1185
1186 fn convert_dest_non_dma(
1187 &mut self,
1188 dst: &mut TensorImage,
1189 src: &TensorImage,
1190 rotation: crate::Rotation,
1191 flip: Flip,
1192 crop: Crop,
1193 ) -> crate::Result<()> {
1194 let _framebuffer = self.setup_renderbuffer_non_dma(dst, crop)?;
1195 let start = Instant::now();
1196 if dst.is_planar() {
1197 self.convert_to_planar(src, dst, rotation, flip, crop)?;
1198 } else {
1199 self.convert_to(src, dst, rotation, flip, crop)?;
1200 }
1201 log::debug!("Draw to framebuffer takes {:?}", start.elapsed());
1202 let start = Instant::now();
1203 let dest_format = match dst.fourcc() {
1204 RGB => gls::gl::RGB,
1205 RGBA => gls::gl::RGBA,
1206 GREY => gls::gl::RED,
1207 _ => unreachable!(),
1208 };
1209
1210 unsafe {
1211 let mut dst_map = dst.tensor().map()?;
1212 gls::gl::ReadBuffer(gls::gl::COLOR_ATTACHMENT0);
1213 gls::gl::ReadnPixels(
1214 0,
1215 0,
1216 dst.width() as i32,
1217 dst.height() as i32,
1218 dest_format,
1219 gls::gl::UNSIGNED_BYTE,
1220 dst.tensor.len() as i32,
1221 dst_map.as_mut_ptr() as *mut c_void,
1222 );
1223 }
1224 log::debug!("Read from framebuffer takes {:?}", start.elapsed());
1225 Ok(())
1226 }
1227
1228 fn convert_to(
1229 &mut self,
1230 src: &TensorImage,
1231 dst: &TensorImage,
1232 rotation: crate::Rotation,
1233 flip: Flip,
1234 crop: Crop,
1235 ) -> Result<(), crate::Error> {
1236 check_gl_error(function!(), line!())?;
1237
1238 let has_crop = crop.dst_rect.is_some_and(|x| {
1239 x.left != 0 || x.top != 0 || x.width != dst.width() || x.height != dst.height()
1240 });
1241 if has_crop {
1242 if let Some(dst_color) = crop.dst_color {
1243 unsafe {
1244 gls::gl::ClearColor(
1245 dst_color[0] as f32 / 255.0,
1246 dst_color[1] as f32 / 255.0,
1247 dst_color[2] as f32 / 255.0,
1248 dst_color[3] as f32 / 255.0,
1249 );
1250 gls::gl::Clear(gls::gl::COLOR_BUFFER_BIT);
1251 };
1252 }
1253 }
1254
1255 let src_roi = if let Some(crop) = crop.src_rect {
1257 RegionOfInterest {
1258 left: crop.left as f32 / src.width() as f32,
1259 top: (crop.top + crop.height) as f32 / src.height() as f32,
1260 right: (crop.left + crop.width) as f32 / src.width() as f32,
1261 bottom: crop.top as f32 / src.height() as f32,
1262 }
1263 } else {
1264 RegionOfInterest {
1265 left: 0.,
1266 top: 1.,
1267 right: 1.,
1268 bottom: 0.,
1269 }
1270 };
1271
1272 let cvt_screen_coord = |normalized| normalized * 2.0 - 1.0;
1274 let dst_roi = if let Some(crop) = crop.dst_rect {
1275 RegionOfInterest {
1276 left: cvt_screen_coord(crop.left as f32 / dst.width() as f32),
1277 top: cvt_screen_coord((crop.top + crop.height) as f32 / dst.height() as f32),
1278 right: cvt_screen_coord((crop.left + crop.width) as f32 / dst.width() as f32),
1279 bottom: cvt_screen_coord(crop.top as f32 / dst.height() as f32),
1280 }
1281 } else {
1282 RegionOfInterest {
1283 left: -1.,
1284 top: 1.,
1285 right: 1.,
1286 bottom: -1.,
1287 }
1288 };
1289 let rotation_offset = match rotation {
1290 crate::Rotation::None => 0,
1291 crate::Rotation::Clockwise90 => 1,
1292 crate::Rotation::Rotate180 => 2,
1293 crate::Rotation::CounterClockwise90 => 3,
1294 };
1295 if self.gl_context.support_dma && src.tensor().memory() == TensorMemory::Dma {
1296 match self.create_image_from_dma2(src) {
1297 Ok(new_egl_image) => self.draw_camera_texture_eglimage(
1298 src,
1299 &new_egl_image,
1300 src_roi,
1301 dst_roi,
1302 rotation_offset,
1303 flip,
1304 )?,
1305 Err(e) => {
1306 log::warn!("EGL image creation failed for {:?}: {:?}", src.fourcc(), e);
1307 let start = Instant::now();
1308 self.draw_src_texture(src, src_roi, dst_roi, rotation_offset, flip)?;
1309 log::debug!("draw_src_texture takes {:?}", start.elapsed());
1310 }
1311 }
1312 } else {
1313 let start = Instant::now();
1314 self.draw_src_texture(src, src_roi, dst_roi, rotation_offset, flip)?;
1315 log::debug!("draw_src_texture takes {:?}", start.elapsed());
1316 }
1317
1318 let start = Instant::now();
1319 unsafe { gls::gl::Finish() };
1320 log::debug!("gl_Finish takes {:?}", start.elapsed());
1321 check_gl_error(function!(), line!())?;
1322 Ok(())
1323 }
1324
1325 fn convert_to_planar(
1326 &self,
1327 src: &TensorImage,
1328 dst: &TensorImage,
1329 rotation: crate::Rotation,
1330 flip: Flip,
1331 crop: Crop,
1332 ) -> Result<(), crate::Error> {
1333 let alpha = match dst.fourcc() {
1356 PLANAR_RGB => false,
1357 PLANAR_RGBA => true,
1358 _ => {
1359 return Err(crate::Error::NotSupported(
1360 "Destination format must be PLANAR_RGB or PLANAR_RGBA".to_string(),
1361 ));
1362 }
1363 };
1364
1365 let src_roi = if let Some(crop) = crop.src_rect {
1367 RegionOfInterest {
1368 left: crop.left as f32 / src.width() as f32,
1369 top: (crop.top + crop.height) as f32 / src.height() as f32,
1370 right: (crop.left + crop.width) as f32 / src.width() as f32,
1371 bottom: crop.top as f32 / src.height() as f32,
1372 }
1373 } else {
1374 RegionOfInterest {
1375 left: 0.,
1376 top: 1.,
1377 right: 1.,
1378 bottom: 0.,
1379 }
1380 };
1381
1382 let cvt_screen_coord = |normalized| normalized * 2.0 - 1.0;
1384 let dst_roi = if let Some(crop) = crop.dst_rect {
1385 RegionOfInterest {
1386 left: cvt_screen_coord(crop.left as f32 / dst.width() as f32),
1387 top: cvt_screen_coord((crop.top + crop.height) as f32 / dst.height() as f32),
1388 right: cvt_screen_coord((crop.left + crop.width) as f32 / dst.width() as f32),
1389 bottom: cvt_screen_coord(crop.top as f32 / dst.height() as f32),
1390 }
1391 } else {
1392 RegionOfInterest {
1393 left: -1.,
1394 top: 1.,
1395 right: 1.,
1396 bottom: -1.,
1397 }
1398 };
1399 let rotation_offset = match rotation {
1400 crate::Rotation::None => 0,
1401 crate::Rotation::Clockwise90 => 1,
1402 crate::Rotation::Rotate180 => 2,
1403 crate::Rotation::CounterClockwise90 => 3,
1404 };
1405
1406 let has_crop = crop.dst_rect.is_some_and(|x| {
1407 x.left != 0 || x.top != 0 || x.width != dst.width() || x.height != dst.height()
1408 });
1409 if has_crop {
1410 if let Some(dst_color) = crop.dst_color {
1411 self.clear_rect_planar(
1412 dst.width(),
1413 dst.height(),
1414 dst_roi,
1415 [
1416 dst_color[0] as f32 / 255.0,
1417 dst_color[1] as f32 / 255.0,
1418 dst_color[2] as f32 / 255.0,
1419 dst_color[3] as f32 / 255.0,
1420 ],
1421 alpha,
1422 )?;
1423 }
1424 }
1425
1426 let new_egl_image = self.create_image_from_dma2(src)?;
1427
1428 self.draw_camera_texture_to_rgb_planar(
1429 &new_egl_image,
1430 src_roi,
1431 dst_roi,
1432 rotation_offset,
1433 flip,
1434 alpha,
1435 )?;
1436 unsafe { gls::gl::Finish() };
1437
1438 Ok(())
1439 }
1440
1441 fn clear_rect_planar(
1442 &self,
1443 width: usize,
1444 height: usize,
1445 dst_roi: RegionOfInterest,
1446 color: [f32; 4],
1447 alpha: bool,
1448 ) -> Result<(), Error> {
1449 if !alpha && color[0] == color[1] && color[1] == color[2] {
1450 unsafe {
1451 gls::gl::ClearColor(color[0], color[0], color[0], 1.0);
1452 gls::gl::Clear(gls::gl::COLOR_BUFFER_BIT);
1453 };
1454 }
1455
1456 let split = if alpha { 4 } else { 3 };
1457
1458 unsafe {
1459 gls::gl::Enable(gls::gl::SCISSOR_TEST);
1460 let x = (((dst_roi.left + 1.0) / 2.0) * width as f32).round() as i32;
1461 let y = (((dst_roi.bottom + 1.0) / 2.0) * height as f32).round() as i32;
1462 let width = (((dst_roi.right - dst_roi.left) / 2.0) * width as f32).round() as i32;
1463 let height = (((dst_roi.top - dst_roi.bottom) / 2.0) * height as f32 / split as f32)
1464 .round() as i32;
1465 for (i, c) in color.iter().enumerate().take(split) {
1466 gls::gl::Scissor(x, y + i as i32 * height, width, height);
1467 gls::gl::ClearColor(*c, *c, *c, 1.0);
1468 gls::gl::Clear(gls::gl::COLOR_BUFFER_BIT);
1469 }
1470 gls::gl::Disable(gls::gl::SCISSOR_TEST);
1471 }
1472 Ok(())
1473 }
1474
1475 #[allow(clippy::too_many_arguments)]
1476 fn draw_camera_texture_to_rgb_planar(
1477 &self,
1478 egl_img: &EglImage,
1479 src_roi: RegionOfInterest,
1480 mut dst_roi: RegionOfInterest,
1481 rotation_offset: usize,
1482 flip: Flip,
1483 alpha: bool,
1484 ) -> Result<(), Error> {
1485 let texture_target = gls::gl::TEXTURE_EXTERNAL_OES;
1486 match flip {
1487 Flip::None => {}
1488 Flip::Vertical => {
1489 std::mem::swap(&mut dst_roi.top, &mut dst_roi.bottom);
1490 }
1491 Flip::Horizontal => {
1492 std::mem::swap(&mut dst_roi.left, &mut dst_roi.right);
1493 }
1494 }
1495 unsafe {
1496 gls::gl::UseProgram(self.texture_program_planar.id);
1498 gls::gl::BindTexture(texture_target, self.camera_eglimage_texture.id);
1499 gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1500 gls::gl::TexParameteri(
1501 texture_target,
1502 gls::gl::TEXTURE_MIN_FILTER,
1503 gls::gl::LINEAR as i32,
1504 );
1505 gls::gl::TexParameteri(
1506 texture_target,
1507 gls::gl::TEXTURE_MAG_FILTER,
1508 gls::gl::LINEAR as i32,
1509 );
1510 gls::gl::TexParameteri(
1511 texture_target,
1512 gls::gl::TEXTURE_WRAP_S,
1513 gls::gl::CLAMP_TO_EDGE as i32,
1514 );
1515
1516 gls::gl::TexParameteri(
1517 texture_target,
1518 gls::gl::TEXTURE_WRAP_T,
1519 gls::gl::CLAMP_TO_EDGE as i32,
1520 );
1521
1522 gls::egl_image_target_texture_2d_oes(texture_target, egl_img.egl_image.as_ptr());
1523 check_gl_error(function!(), line!())?;
1524 let y_centers = if alpha {
1525 vec![-3.0 / 4.0, -1.0 / 4.0, 1.0 / 4.0, 3.0 / 4.0]
1526 } else {
1527 vec![-2.0 / 3.0, 0.0, 2.0 / 3.0]
1528 };
1529 let swizzles = [gls::gl::RED, gls::gl::GREEN, gls::gl::BLUE, gls::gl::ALPHA];
1530 for (i, y_center) in y_centers.iter().enumerate() {
1532 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
1533 gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
1534 let camera_vertices: [f32; 12] = [
1535 dst_roi.left,
1536 dst_roi.top / 3.0 + y_center,
1537 0., dst_roi.right,
1539 dst_roi.top / 3.0 + y_center,
1540 0., dst_roi.right,
1542 dst_roi.bottom / 3.0 + y_center,
1543 0., dst_roi.left,
1545 dst_roi.bottom / 3.0 + y_center,
1546 0., ];
1548 gls::gl::BufferData(
1549 gls::gl::ARRAY_BUFFER,
1550 (size_of::<f32>() * camera_vertices.len()) as isize,
1551 camera_vertices.as_ptr() as *const c_void,
1552 gls::gl::DYNAMIC_DRAW,
1553 );
1554
1555 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
1556 gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
1557 let texture_vertices: [f32; 16] = [
1558 src_roi.left,
1559 src_roi.top,
1560 src_roi.right,
1561 src_roi.top,
1562 src_roi.right,
1563 src_roi.bottom,
1564 src_roi.left,
1565 src_roi.bottom,
1566 src_roi.left,
1567 src_roi.top,
1568 src_roi.right,
1569 src_roi.top,
1570 src_roi.right,
1571 src_roi.bottom,
1572 src_roi.left,
1573 src_roi.bottom,
1574 ];
1575
1576 gls::gl::BufferData(
1577 gls::gl::ARRAY_BUFFER,
1578 (size_of::<f32>() * 8) as isize,
1579 (texture_vertices[(rotation_offset * 2)..]).as_ptr() as *const c_void,
1580 gls::gl::DYNAMIC_DRAW,
1581 );
1582 let vertices_index: [u32; 4] = [0, 1, 2, 3];
1583 gls::gl::TexParameteri(
1587 texture_target,
1588 gls::gl::TEXTURE_SWIZZLE_R,
1589 swizzles[i] as i32,
1590 );
1591
1592 gls::gl::DrawElements(
1593 gls::gl::TRIANGLE_FAN,
1594 vertices_index.len() as i32,
1595 gls::gl::UNSIGNED_INT,
1596 vertices_index.as_ptr() as *const c_void,
1597 );
1598 }
1599 check_gl_error(function!(), line!())?;
1600 }
1601 Ok(())
1602 }
1603
1604 fn draw_src_texture(
1605 &mut self,
1606 src: &TensorImage,
1607 src_roi: RegionOfInterest,
1608 mut dst_roi: RegionOfInterest,
1609 rotation_offset: usize,
1610 flip: Flip,
1611 ) -> Result<(), Error> {
1612 let texture_target = gls::gl::TEXTURE_2D;
1613 let texture_format = match src.fourcc() {
1614 RGB => gls::gl::RGB,
1615 RGBA => gls::gl::RGBA,
1616 GREY => gls::gl::RED,
1617 _ => {
1618 return Err(Error::NotSupported(format!(
1619 "draw_src_texture does not support {:?} (use DMA-BUF path for YUV)",
1620 src.fourcc()
1621 )));
1622 }
1623 };
1624 unsafe {
1625 gls::gl::UseProgram(self.texture_program.id);
1626 gls::gl::BindTexture(texture_target, self.camera_normal_texture.id);
1627 gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1628 gls::gl::TexParameteri(
1629 texture_target,
1630 gls::gl::TEXTURE_MIN_FILTER,
1631 gls::gl::LINEAR as i32,
1632 );
1633 gls::gl::TexParameteri(
1634 texture_target,
1635 gls::gl::TEXTURE_MAG_FILTER,
1636 gls::gl::LINEAR as i32,
1637 );
1638 if src.fourcc() == GREY {
1639 for swizzle in [
1640 gls::gl::TEXTURE_SWIZZLE_R,
1641 gls::gl::TEXTURE_SWIZZLE_G,
1642 gls::gl::TEXTURE_SWIZZLE_B,
1643 ] {
1644 gls::gl::TexParameteri(gls::gl::TEXTURE_2D, swizzle, gls::gl::RED as i32);
1645 }
1646 } else {
1647 for (swizzle, src) in [
1648 (gls::gl::TEXTURE_SWIZZLE_R, gls::gl::RED),
1649 (gls::gl::TEXTURE_SWIZZLE_G, gls::gl::GREEN),
1650 (gls::gl::TEXTURE_SWIZZLE_B, gls::gl::BLUE),
1651 ] {
1652 gls::gl::TexParameteri(gls::gl::TEXTURE_2D, swizzle, src as i32);
1653 }
1654 }
1655 self.camera_normal_texture.update_texture(
1656 texture_target,
1657 src.width(),
1658 src.height(),
1659 texture_format,
1660 &src.tensor().map()?,
1661 );
1662
1663 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
1664 gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
1665
1666 match flip {
1667 Flip::None => {}
1668 Flip::Vertical => {
1669 std::mem::swap(&mut dst_roi.top, &mut dst_roi.bottom);
1670 }
1671 Flip::Horizontal => {
1672 std::mem::swap(&mut dst_roi.left, &mut dst_roi.right);
1673 }
1674 }
1675
1676 let camera_vertices: [f32; 12] = [
1677 dst_roi.left,
1678 dst_roi.top,
1679 0., dst_roi.right,
1681 dst_roi.top,
1682 0., dst_roi.right,
1684 dst_roi.bottom,
1685 0., dst_roi.left,
1687 dst_roi.bottom,
1688 0., ];
1690 gls::gl::BufferData(
1691 gls::gl::ARRAY_BUFFER,
1692 (size_of::<f32>() * camera_vertices.len()) as isize,
1693 camera_vertices.as_ptr() as *const c_void,
1694 gls::gl::DYNAMIC_DRAW,
1695 );
1696 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
1697 gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
1698 let texture_vertices: [f32; 16] = [
1699 src_roi.left,
1700 src_roi.top,
1701 src_roi.right,
1702 src_roi.top,
1703 src_roi.right,
1704 src_roi.bottom,
1705 src_roi.left,
1706 src_roi.bottom,
1707 src_roi.left,
1708 src_roi.top,
1709 src_roi.right,
1710 src_roi.top,
1711 src_roi.right,
1712 src_roi.bottom,
1713 src_roi.left,
1714 src_roi.bottom,
1715 ];
1716
1717 gls::gl::BufferData(
1718 gls::gl::ARRAY_BUFFER,
1719 (size_of::<f32>() * 8) as isize,
1720 (texture_vertices[(rotation_offset * 2)..]).as_ptr() as *const c_void,
1721 gls::gl::DYNAMIC_DRAW,
1722 );
1723 let vertices_index: [u32; 4] = [0, 1, 2, 3];
1724 gls::gl::DrawElements(
1725 gls::gl::TRIANGLE_FAN,
1726 vertices_index.len() as i32,
1727 gls::gl::UNSIGNED_INT,
1728 vertices_index.as_ptr() as *const c_void,
1729 );
1730 check_gl_error(function!(), line!())?;
1731
1732 Ok(())
1733 }
1734 }
1735
1736 fn draw_camera_texture_eglimage(
1737 &self,
1738 src: &TensorImage,
1739 egl_img: &EglImage,
1740 src_roi: RegionOfInterest,
1741 mut dst_roi: RegionOfInterest,
1742 rotation_offset: usize,
1743 flip: Flip,
1744 ) -> Result<(), Error> {
1745 let texture_target = gls::gl::TEXTURE_EXTERNAL_OES;
1747 unsafe {
1748 gls::gl::UseProgram(self.texture_program_yuv.id);
1749 gls::gl::BindTexture(texture_target, self.camera_eglimage_texture.id);
1750 gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1751 gls::gl::TexParameteri(
1752 texture_target,
1753 gls::gl::TEXTURE_MIN_FILTER,
1754 gls::gl::LINEAR as i32,
1755 );
1756 gls::gl::TexParameteri(
1757 texture_target,
1758 gls::gl::TEXTURE_MAG_FILTER,
1759 gls::gl::LINEAR as i32,
1760 );
1761
1762 if src.fourcc() == GREY {
1763 for swizzle in [
1764 gls::gl::TEXTURE_SWIZZLE_R,
1765 gls::gl::TEXTURE_SWIZZLE_G,
1766 gls::gl::TEXTURE_SWIZZLE_B,
1767 ] {
1768 gls::gl::TexParameteri(gls::gl::TEXTURE_2D, swizzle, gls::gl::RED as i32);
1769 }
1770 } else {
1771 for (swizzle, src) in [
1772 (gls::gl::TEXTURE_SWIZZLE_R, gls::gl::RED),
1773 (gls::gl::TEXTURE_SWIZZLE_G, gls::gl::GREEN),
1774 (gls::gl::TEXTURE_SWIZZLE_B, gls::gl::BLUE),
1775 ] {
1776 gls::gl::TexParameteri(gls::gl::TEXTURE_2D, swizzle, src as i32);
1777 }
1778 }
1779
1780 gls::egl_image_target_texture_2d_oes(texture_target, egl_img.egl_image.as_ptr());
1781 check_gl_error(function!(), line!())?;
1782 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
1783 gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
1784
1785 match flip {
1786 Flip::None => {}
1787 Flip::Vertical => {
1788 std::mem::swap(&mut dst_roi.top, &mut dst_roi.bottom);
1789 }
1790 Flip::Horizontal => {
1791 std::mem::swap(&mut dst_roi.left, &mut dst_roi.right);
1792 }
1793 }
1794
1795 let camera_vertices: [f32; 12] = [
1796 dst_roi.left,
1797 dst_roi.top,
1798 0., dst_roi.right,
1800 dst_roi.top,
1801 0., dst_roi.right,
1803 dst_roi.bottom,
1804 0., dst_roi.left,
1806 dst_roi.bottom,
1807 0., ];
1809 gls::gl::BufferSubData(
1810 gls::gl::ARRAY_BUFFER,
1811 0,
1812 (size_of::<f32>() * camera_vertices.len()) as isize,
1813 camera_vertices.as_ptr() as *const c_void,
1814 );
1815
1816 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
1817 gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
1818
1819 let texture_vertices: [f32; 16] = [
1820 src_roi.left,
1821 src_roi.top,
1822 src_roi.right,
1823 src_roi.top,
1824 src_roi.right,
1825 src_roi.bottom,
1826 src_roi.left,
1827 src_roi.bottom,
1828 src_roi.left,
1829 src_roi.top,
1830 src_roi.right,
1831 src_roi.top,
1832 src_roi.right,
1833 src_roi.bottom,
1834 src_roi.left,
1835 src_roi.bottom,
1836 ];
1837 gls::gl::BufferSubData(
1838 gls::gl::ARRAY_BUFFER,
1839 0,
1840 (size_of::<f32>() * 8) as isize,
1841 (texture_vertices[(rotation_offset * 2)..]).as_ptr() as *const c_void,
1842 );
1843
1844 let vertices_index: [u32; 4] = [0, 1, 2, 3];
1845 gls::gl::DrawElements(
1846 gls::gl::TRIANGLE_FAN,
1847 vertices_index.len() as i32,
1848 gls::gl::UNSIGNED_INT,
1849 vertices_index.as_ptr() as *const c_void,
1850 );
1851 }
1852 check_gl_error(function!(), line!())?;
1853 Ok(())
1854 }
1855
1856 fn create_image_from_dma2(&self, src: &TensorImage) -> Result<EglImage, crate::Error> {
1857 let width;
1858 let height;
1859 let format;
1860 let channels;
1861
1862 if src.fourcc() == NV12 {
1864 if !src.width().is_multiple_of(4) {
1865 return Err(Error::NotSupported(
1866 "OpenGL EGLImage doesn't support image widths which are not multiples of 4"
1867 .to_string(),
1868 ));
1869 }
1870 width = src.width();
1871 height = src.height();
1872 format = fourcc_to_drm(NV12);
1873 channels = 1; } else if src.is_planar() {
1875 if !src.width().is_multiple_of(16) {
1876 return Err(Error::NotSupported(
1877 "OpenGL Planar RGB EGLImage doesn't support image widths which are not multiples of 16"
1878 .to_string(),
1879 ));
1880 }
1881 match src.fourcc() {
1882 PLANAR_RGB => {
1883 format = DrmFourcc::R8;
1884 width = src.width();
1885 height = src.height() * 3;
1886 channels = 1;
1887 }
1888 fourcc => {
1889 return Err(crate::Error::NotSupported(format!(
1890 "Unsupported Planar FourCC {fourcc:?}"
1891 )));
1892 }
1893 };
1894 } else {
1895 if !src.width().is_multiple_of(4) {
1896 return Err(Error::NotSupported(
1897 "OpenGL EGLImage doesn't support image widths which are not multiples of 4"
1898 .to_string(),
1899 ));
1900 }
1901 width = src.width();
1902 height = src.height();
1903 format = fourcc_to_drm(src.fourcc());
1904 channels = src.channels();
1905 }
1906
1907 let fd = match &src.tensor {
1908 edgefirst_tensor::Tensor::Dma(dma_tensor) => dma_tensor.fd.as_raw_fd(),
1909 edgefirst_tensor::Tensor::Shm(_) => {
1910 return Err(Error::NotImplemented(
1911 "OpenGL EGLImage doesn't support SHM".to_string(),
1912 ));
1913 }
1914 edgefirst_tensor::Tensor::Mem(_) => {
1915 return Err(Error::NotImplemented(
1916 "OpenGL EGLImage doesn't support MEM".to_string(),
1917 ));
1918 }
1919 };
1920
1921 let plane0_pitch = if src.fourcc() == NV12 {
1924 width
1925 } else {
1926 width * channels
1927 };
1928
1929 let mut egl_img_attr = vec![
1930 egl_ext::LINUX_DRM_FOURCC as Attrib,
1931 format as Attrib,
1932 khronos_egl::WIDTH as Attrib,
1933 width as Attrib,
1934 khronos_egl::HEIGHT as Attrib,
1935 height as Attrib,
1936 egl_ext::DMA_BUF_PLANE0_PITCH as Attrib,
1937 plane0_pitch as Attrib,
1938 egl_ext::DMA_BUF_PLANE0_OFFSET as Attrib,
1939 0 as Attrib,
1940 egl_ext::DMA_BUF_PLANE0_FD as Attrib,
1941 fd as Attrib,
1942 egl::IMAGE_PRESERVED as Attrib,
1943 egl::TRUE as Attrib,
1944 ];
1945
1946 if src.fourcc() == NV12 {
1948 let uv_offset = width * height; egl_img_attr.append(&mut vec![
1950 egl_ext::DMA_BUF_PLANE1_FD as Attrib,
1951 fd as Attrib,
1952 egl_ext::DMA_BUF_PLANE1_OFFSET as Attrib,
1953 uv_offset as Attrib,
1954 egl_ext::DMA_BUF_PLANE1_PITCH as Attrib,
1955 width as Attrib, ]);
1957 }
1958
1959 if matches!(src.fourcc(), YUYV | NV12) {
1960 egl_img_attr.append(&mut vec![
1961 egl_ext::YUV_COLOR_SPACE_HINT as Attrib,
1962 egl_ext::ITU_REC709 as Attrib,
1963 egl_ext::SAMPLE_RANGE_HINT as Attrib,
1964 egl_ext::YUV_NARROW_RANGE as Attrib,
1965 ]);
1966 }
1967
1968 egl_img_attr.push(khronos_egl::NONE as Attrib);
1969
1970 match self.new_egl_image_owned(egl_ext::LINUX_DMA_BUF, &egl_img_attr) {
1971 Ok(v) => Ok(v),
1972 Err(e) => Err(e),
1973 }
1974 }
1975
1976 fn new_egl_image_owned(
1977 &'_ self,
1978 target: egl::Enum,
1979 attrib_list: &[Attrib],
1980 ) -> Result<EglImage, Error> {
1981 let image = GlContext::egl_create_image_with_fallback(
1982 &self.gl_context.egl,
1983 self.gl_context.display.as_display(),
1984 unsafe { egl::Context::from_ptr(egl::NO_CONTEXT) },
1985 target,
1986 unsafe { egl::ClientBuffer::from_ptr(null_mut()) },
1987 attrib_list,
1988 )?;
1989 Ok(EglImage {
1990 egl_image: image,
1991 display: self.gl_context.display.as_display(),
1992 egl: Rc::clone(&self.gl_context.egl),
1993 })
1994 }
1995
1996 fn reshape_segmentation_to_rgba(&self, segmentation: &[u8], shape: [usize; 3]) -> Vec<u8> {
1998 let [height, width, classes] = shape;
1999
2000 let n_layer_stride = height * width * 4;
2001 let n_row_stride = width * 4;
2002 let n_col_stride = 4;
2003 let row_stride = width * classes;
2004 let col_stride = classes;
2005
2006 let mut new_segmentation = vec![0u8; n_layer_stride * classes.div_ceil(4)];
2007
2008 for i in 0..height {
2009 for j in 0..width {
2010 for k in 0..classes.div_ceil(4) * 4 {
2011 if k >= classes {
2012 new_segmentation[n_layer_stride * (k / 4)
2013 + i * n_row_stride
2014 + j * n_col_stride
2015 + k % 4] = 0;
2016 } else {
2017 new_segmentation[n_layer_stride * (k / 4)
2018 + i * n_row_stride
2019 + j * n_col_stride
2020 + k % 4] = segmentation[i * row_stride + j * col_stride + k];
2021 }
2022 }
2023 }
2024 }
2025
2026 new_segmentation
2027 }
2028
2029 #[cfg(feature = "decoder")]
2030 fn render_modelpack_segmentation(
2031 &mut self,
2032 dst_roi: RegionOfInterest,
2033 segmentation: &[u8],
2034 shape: [usize; 3],
2035 ) -> Result<(), crate::Error> {
2036 log::debug!("start render_segmentation_to_image");
2037
2038 let new_segmentation = self.reshape_segmentation_to_rgba(segmentation, shape);
2041
2042 let [height, width, classes] = shape;
2043
2044 let format = gls::gl::RGBA;
2045 let texture_target = gls::gl::TEXTURE_2D_ARRAY;
2046 self.segmentation_program
2047 .load_uniform_1i(c"background_index", shape[2] as i32 - 1)?;
2048
2049 gls::use_program(self.segmentation_program.id);
2050
2051 gls::bind_texture(texture_target, self.segmentation_texture.id);
2052 gls::active_texture(gls::gl::TEXTURE0);
2053 gls::tex_parameteri(
2054 texture_target,
2055 gls::gl::TEXTURE_MIN_FILTER,
2056 gls::gl::LINEAR as i32,
2057 );
2058 gls::tex_parameteri(
2059 texture_target,
2060 gls::gl::TEXTURE_MAG_FILTER,
2061 gls::gl::LINEAR as i32,
2062 );
2063 gls::tex_parameteri(
2064 texture_target,
2065 gls::gl::TEXTURE_WRAP_S,
2066 gls::gl::CLAMP_TO_EDGE as i32,
2067 );
2068
2069 gls::tex_parameteri(
2070 texture_target,
2071 gls::gl::TEXTURE_WRAP_T,
2072 gls::gl::CLAMP_TO_EDGE as i32,
2073 );
2074
2075 gls::tex_image3d(
2076 texture_target,
2077 0,
2078 format as i32,
2079 width as i32,
2080 height as i32,
2081 classes.div_ceil(4) as i32,
2082 0,
2083 format,
2084 gls::gl::UNSIGNED_BYTE,
2085 Some(&new_segmentation),
2086 );
2087
2088 let src_roi = RegionOfInterest {
2089 left: 0.,
2090 top: 1.,
2091 right: 1.,
2092 bottom: 0.,
2093 };
2094
2095 unsafe {
2096 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
2097 gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
2098
2099 let camera_vertices: [f32; 12] = [
2100 dst_roi.left,
2101 dst_roi.top,
2102 0., dst_roi.right,
2104 dst_roi.top,
2105 0., dst_roi.right,
2107 dst_roi.bottom,
2108 0., dst_roi.left,
2110 dst_roi.bottom,
2111 0., ];
2113 gls::gl::BufferSubData(
2114 gls::gl::ARRAY_BUFFER,
2115 0,
2116 (size_of::<f32>() * camera_vertices.len()) as isize,
2117 camera_vertices.as_ptr() as *const c_void,
2118 );
2119
2120 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
2121 gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
2122
2123 let texture_vertices: [f32; 8] = [
2124 src_roi.left,
2125 src_roi.top,
2126 src_roi.right,
2127 src_roi.top,
2128 src_roi.right,
2129 src_roi.bottom,
2130 src_roi.left,
2131 src_roi.bottom,
2132 ];
2133 gls::gl::BufferSubData(
2134 gls::gl::ARRAY_BUFFER,
2135 0,
2136 (size_of::<f32>() * 8) as isize,
2137 (texture_vertices[0..]).as_ptr() as *const c_void,
2138 );
2139
2140 let vertices_index: [u32; 4] = [0, 1, 2, 3];
2141 gls::gl::DrawElements(
2142 gls::gl::TRIANGLE_FAN,
2143 vertices_index.len() as i32,
2144 gls::gl::UNSIGNED_INT,
2145 vertices_index.as_ptr() as *const c_void,
2146 );
2147 }
2148
2149 Ok(())
2150 }
2151
2152 #[cfg(feature = "decoder")]
2153 fn render_yolo_segmentation(
2154 &mut self,
2155 dst_roi: RegionOfInterest,
2156 segmentation: &[u8],
2157 shape: [usize; 2],
2158 class: usize,
2159 ) -> Result<(), crate::Error> {
2160 log::debug!("start render_yolo_segmentation");
2161
2162 let [height, width] = shape;
2163
2164 let format = gls::gl::RED;
2165 let texture_target = gls::gl::TEXTURE_2D;
2166 gls::use_program(self.instanced_segmentation_program.id);
2167 self.instanced_segmentation_program
2168 .load_uniform_1i(c"class_index", class as i32)?;
2169 gls::bind_texture(texture_target, self.segmentation_texture.id);
2170 gls::active_texture(gls::gl::TEXTURE0);
2171 gls::tex_parameteri(
2172 texture_target,
2173 gls::gl::TEXTURE_MIN_FILTER,
2174 gls::gl::LINEAR as i32,
2175 );
2176 gls::tex_parameteri(
2177 texture_target,
2178 gls::gl::TEXTURE_MAG_FILTER,
2179 gls::gl::LINEAR as i32,
2180 );
2181 gls::tex_parameteri(
2182 texture_target,
2183 gls::gl::TEXTURE_WRAP_S,
2184 gls::gl::CLAMP_TO_EDGE as i32,
2185 );
2186
2187 gls::tex_parameteri(
2188 texture_target,
2189 gls::gl::TEXTURE_WRAP_T,
2190 gls::gl::CLAMP_TO_EDGE as i32,
2191 );
2192
2193 gls::tex_image2d(
2194 texture_target,
2195 0,
2196 format as i32,
2197 width as i32,
2198 height as i32,
2199 0,
2200 format,
2201 gls::gl::UNSIGNED_BYTE,
2202 Some(segmentation),
2203 );
2204
2205 let src_roi = RegionOfInterest {
2206 left: 0.,
2207 top: 1.,
2208 right: 1.,
2209 bottom: 0.,
2210 };
2211
2212 unsafe {
2213 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
2214 gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
2215
2216 let camera_vertices: [f32; 12] = [
2217 dst_roi.left,
2218 dst_roi.top,
2219 0., dst_roi.right,
2221 dst_roi.top,
2222 0., dst_roi.right,
2224 dst_roi.bottom,
2225 0., dst_roi.left,
2227 dst_roi.bottom,
2228 0., ];
2230 gls::gl::BufferSubData(
2231 gls::gl::ARRAY_BUFFER,
2232 0,
2233 (size_of::<f32>() * camera_vertices.len()) as isize,
2234 camera_vertices.as_ptr() as *const c_void,
2235 );
2236
2237 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
2238 gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
2239
2240 let texture_vertices: [f32; 8] = [
2241 src_roi.left,
2242 src_roi.top,
2243 src_roi.right,
2244 src_roi.top,
2245 src_roi.right,
2246 src_roi.bottom,
2247 src_roi.left,
2248 src_roi.bottom,
2249 ];
2250 gls::gl::BufferSubData(
2251 gls::gl::ARRAY_BUFFER,
2252 0,
2253 (size_of::<f32>() * 8) as isize,
2254 (texture_vertices).as_ptr() as *const c_void,
2255 );
2256
2257 let vertices_index: [u32; 4] = [0, 1, 2, 3];
2258 gls::gl::DrawElements(
2259 gls::gl::TRIANGLE_FAN,
2260 vertices_index.len() as i32,
2261 gls::gl::UNSIGNED_INT,
2262 vertices_index.as_ptr() as *const c_void,
2263 );
2264 gls::gl::Finish();
2265 }
2266
2267 Ok(())
2268 }
2269
2270 fn render_segmentation(
2271 &mut self,
2272 detect: &[DetectBox],
2273 segmentation: &[Segmentation],
2274 ) -> crate::Result<()> {
2275 if segmentation.is_empty() {
2276 return Ok(());
2277 }
2278
2279 let is_modelpack = segmentation[0].segmentation.shape()[2] > 1;
2280 let cvt_screen_coord = |normalized| normalized * 2.0 - 1.0;
2282 if is_modelpack {
2283 let seg = &segmentation[0];
2284 let dst_roi = RegionOfInterest {
2285 left: cvt_screen_coord(seg.xmin),
2286 top: cvt_screen_coord(seg.ymax),
2287 right: cvt_screen_coord(seg.xmax),
2288 bottom: cvt_screen_coord(seg.ymin),
2289 };
2290 let segment = seg.segmentation.as_standard_layout();
2291 let slice = segment.as_slice().ok_or(Error::Internal(
2292 "Cannot get slice of segmentation".to_owned(),
2293 ))?;
2294
2295 self.render_modelpack_segmentation(
2296 dst_roi,
2297 slice,
2298 [
2299 seg.segmentation.shape()[0],
2300 seg.segmentation.shape()[1],
2301 seg.segmentation.shape()[2],
2302 ],
2303 )?;
2304 } else {
2305 for (seg, det) in segmentation.iter().zip(detect) {
2306 let dst_roi = RegionOfInterest {
2307 left: cvt_screen_coord(seg.xmin),
2308 top: cvt_screen_coord(seg.ymax),
2309 right: cvt_screen_coord(seg.xmax),
2310 bottom: cvt_screen_coord(seg.ymin),
2311 };
2312
2313 let segment = seg.segmentation.as_standard_layout();
2314 let slice = segment.as_slice().ok_or(Error::Internal(
2315 "Cannot get slice of segmentation".to_owned(),
2316 ))?;
2317
2318 self.render_yolo_segmentation(
2319 dst_roi,
2320 slice,
2321 [seg.segmentation.shape()[0], seg.segmentation.shape()[1]],
2322 det.label,
2323 )?;
2324 }
2325 }
2326
2327 gls::disable(gls::gl::BLEND);
2328 Ok(())
2329 }
2330
2331 fn render_box(&mut self, dst: &TensorImage, detect: &[DetectBox]) -> Result<(), Error> {
2332 unsafe {
2333 gls::gl::UseProgram(self.color_program.id);
2334 let rescale = |x: f32| x * 2.0 - 1.0;
2335 let thickness = 3.0;
2336 for d in detect {
2337 self.color_program
2338 .load_uniform_1i(c"class_index", d.label as i32)?;
2339 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
2340 gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
2341 let bbox: [f32; 4] = d.bbox.into();
2342 let outer_box = [
2343 bbox[0] - thickness / dst.width() as f32,
2344 bbox[1] - thickness / dst.height() as f32,
2345 bbox[2] + thickness / dst.width() as f32,
2346 bbox[3] + thickness / dst.height() as f32,
2347 ];
2348 let camera_vertices: [f32; 24] = [
2349 rescale(bbox[0]),
2350 rescale(bbox[3]),
2351 0., rescale(bbox[2]),
2353 rescale(bbox[3]),
2354 0., rescale(bbox[2]),
2356 rescale(bbox[1]),
2357 0., rescale(bbox[0]),
2359 rescale(bbox[1]),
2360 0., rescale(outer_box[0]),
2362 rescale(outer_box[3]),
2363 0., rescale(outer_box[2]),
2365 rescale(outer_box[3]),
2366 0., rescale(outer_box[2]),
2368 rescale(outer_box[1]),
2369 0., rescale(outer_box[0]),
2371 rescale(outer_box[1]),
2372 0., ];
2374 gls::gl::BufferData(
2375 gls::gl::ARRAY_BUFFER,
2376 (size_of::<f32>() * camera_vertices.len()) as isize,
2377 camera_vertices.as_ptr() as *const c_void,
2378 gls::gl::DYNAMIC_DRAW,
2379 );
2380
2381 let vertices_index: [u32; 10] = [0, 1, 5, 2, 6, 3, 7, 0, 4, 5];
2382 gls::gl::DrawElements(
2383 gls::gl::TRIANGLE_STRIP,
2384 vertices_index.len() as i32,
2385 gls::gl::UNSIGNED_INT,
2386 vertices_index.as_ptr() as *const c_void,
2387 );
2388 }
2389 }
2390 check_gl_error(function!(), line!())?;
2391 Ok(())
2392 }
2393}
2394struct EglImage {
2395 egl_image: egl::Image,
2396 egl: Rc<Egl>,
2397 display: egl::Display,
2398}
2399
2400impl Drop for EglImage {
2401 fn drop(&mut self) {
2402 if self.egl_image.as_ptr() == egl::NO_IMAGE {
2403 return;
2404 }
2405
2406 let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
2407 let e =
2408 GlContext::egl_destory_image_with_fallback(&self.egl, self.display, self.egl_image);
2409 if let Err(e) = e {
2410 error!("Could not destroy EGL image: {e:?}");
2411 }
2412 }));
2413 }
2414}
2415
2416struct Texture {
2417 id: u32,
2418 target: gls::gl::types::GLenum,
2419 width: usize,
2420 height: usize,
2421 format: gls::gl::types::GLenum,
2422}
2423
2424impl Default for Texture {
2425 fn default() -> Self {
2426 Self::new()
2427 }
2428}
2429
2430impl Texture {
2431 fn new() -> Self {
2432 let mut id = 0;
2433 unsafe { gls::gl::GenTextures(1, &raw mut id) };
2434 Self {
2435 id,
2436 target: 0,
2437 width: 0,
2438 height: 0,
2439 format: 0,
2440 }
2441 }
2442
2443 fn update_texture(
2444 &mut self,
2445 target: gls::gl::types::GLenum,
2446 width: usize,
2447 height: usize,
2448 format: gls::gl::types::GLenum,
2449 data: &[u8],
2450 ) {
2451 if target != self.target
2452 || width != self.width
2453 || height != self.height
2454 || format != self.format
2455 {
2456 unsafe {
2457 gls::gl::TexImage2D(
2458 target,
2459 0,
2460 format as i32,
2461 width as i32,
2462 height as i32,
2463 0,
2464 format,
2465 gls::gl::UNSIGNED_BYTE,
2466 data.as_ptr() as *const c_void,
2467 );
2468 }
2469 self.target = target;
2470 self.format = format;
2471 self.width = width;
2472 self.height = height;
2473 } else {
2474 unsafe {
2475 gls::gl::TexSubImage2D(
2476 target,
2477 0,
2478 0,
2479 0,
2480 width as i32,
2481 height as i32,
2482 format,
2483 gls::gl::UNSIGNED_BYTE,
2484 data.as_ptr() as *const c_void,
2485 );
2486 }
2487 }
2488 }
2489}
2490
2491impl Drop for Texture {
2492 fn drop(&mut self) {
2493 let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| unsafe {
2494 gls::gl::DeleteTextures(1, &raw mut self.id)
2495 }));
2496 }
2497}
2498
2499struct Buffer {
2500 id: u32,
2501 buffer_index: u32,
2502}
2503
2504impl Buffer {
2505 fn new(buffer_index: u32, size_per_point: usize, max_points: usize) -> Buffer {
2506 let mut id = 0;
2507 unsafe {
2508 gls::gl::EnableVertexAttribArray(buffer_index);
2509 gls::gl::GenBuffers(1, &raw mut id);
2510 gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, id);
2511 gls::gl::VertexAttribPointer(
2512 buffer_index,
2513 size_per_point as i32,
2514 gls::gl::FLOAT,
2515 gls::gl::FALSE,
2516 0,
2517 null(),
2518 );
2519 gls::gl::BufferData(
2520 gls::gl::ARRAY_BUFFER,
2521 (size_of::<f32>() * size_per_point * max_points) as isize,
2522 null(),
2523 gls::gl::DYNAMIC_DRAW,
2524 );
2525 }
2526
2527 Buffer { id, buffer_index }
2528 }
2529}
2530
2531impl Drop for Buffer {
2532 fn drop(&mut self) {
2533 let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| unsafe {
2534 gls::gl::DeleteBuffers(1, &raw mut self.id)
2535 }));
2536 }
2537}
2538
2539struct FrameBuffer {
2540 id: u32,
2541}
2542
2543impl FrameBuffer {
2544 fn new() -> FrameBuffer {
2545 let mut id = 0;
2546 unsafe {
2547 gls::gl::GenFramebuffers(1, &raw mut id);
2548 }
2549
2550 FrameBuffer { id }
2551 }
2552
2553 fn bind(&self) {
2554 unsafe { gls::gl::BindFramebuffer(gls::gl::FRAMEBUFFER, self.id) };
2555 }
2556
2557 fn unbind(&self) {
2558 unsafe { gls::gl::BindFramebuffer(gls::gl::FRAMEBUFFER, 0) };
2559 }
2560}
2561
2562impl Drop for FrameBuffer {
2563 fn drop(&mut self) {
2564 let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
2565 self.unbind();
2566 unsafe {
2567 gls::gl::DeleteFramebuffers(1, &raw mut self.id);
2568 }
2569 }));
2570 }
2571}
2572
2573pub struct GlProgram {
2574 id: u32,
2575 vertex_id: u32,
2576 fragment_id: u32,
2577}
2578
2579impl GlProgram {
2580 fn new(vertex_shader: &str, fragment_shader: &str) -> Result<Self, crate::Error> {
2581 let id = unsafe { gls::gl::CreateProgram() };
2582 let vertex_id = unsafe { gls::gl::CreateShader(gls::gl::VERTEX_SHADER) };
2583 if compile_shader_from_str(vertex_id, vertex_shader, "shader_vert").is_err() {
2584 log::debug!("Vertex shader source:\n{}", vertex_shader);
2585 return Err(crate::Error::OpenGl(format!(
2586 "Shader compile error: {vertex_shader}"
2587 )));
2588 }
2589 unsafe {
2590 gls::gl::AttachShader(id, vertex_id);
2591 }
2592
2593 let fragment_id = unsafe { gls::gl::CreateShader(gls::gl::FRAGMENT_SHADER) };
2594 if compile_shader_from_str(fragment_id, fragment_shader, "shader_frag").is_err() {
2595 log::debug!("Fragment shader source:\n{}", fragment_shader);
2596 return Err(crate::Error::OpenGl(format!(
2597 "Shader compile error: {fragment_shader}"
2598 )));
2599 }
2600
2601 unsafe {
2602 gls::gl::AttachShader(id, fragment_id);
2603 gls::gl::LinkProgram(id);
2604 gls::gl::UseProgram(id);
2605 }
2606
2607 Ok(Self {
2608 id,
2609 vertex_id,
2610 fragment_id,
2611 })
2612 }
2613
2614 #[allow(dead_code)]
2615 fn load_uniform_1f(&self, name: &CStr, value: f32) -> Result<(), crate::Error> {
2616 unsafe {
2617 gls::gl::UseProgram(self.id);
2618 let location = gls::gl::GetUniformLocation(self.id, name.as_ptr());
2619 gls::gl::Uniform1f(location, value);
2620 }
2621 Ok(())
2622 }
2623
2624 #[allow(dead_code)]
2625 fn load_uniform_1i(&self, name: &CStr, value: i32) -> Result<(), crate::Error> {
2626 unsafe {
2627 gls::gl::UseProgram(self.id);
2628 let location = gls::gl::GetUniformLocation(self.id, name.as_ptr());
2629 gls::gl::Uniform1i(location, value);
2630 }
2631 Ok(())
2632 }
2633
2634 fn load_uniform_4fv(&self, name: &CStr, value: &[[f32; 4]]) -> Result<(), crate::Error> {
2635 unsafe {
2636 gls::gl::UseProgram(self.id);
2637 let location = gls::gl::GetUniformLocation(self.id, name.as_ptr());
2638 if location == -1 {
2639 return Err(crate::Error::OpenGl(format!(
2640 "Could not find uniform location for '{}'",
2641 name.to_string_lossy().into_owned()
2642 )));
2643 }
2644 gls::gl::Uniform4fv(location, value.len() as i32, value.as_flattened().as_ptr());
2645 }
2646 check_gl_error(function!(), line!())?;
2647 Ok(())
2648 }
2649}
2650
2651impl Drop for GlProgram {
2652 fn drop(&mut self) {
2653 let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| unsafe {
2654 gls::gl::DeleteProgram(self.id);
2655 gls::gl::DeleteShader(self.fragment_id);
2656 gls::gl::DeleteShader(self.vertex_id);
2657 }));
2658 }
2659}
2660
2661fn compile_shader_from_str(shader: u32, shader_source: &str, shader_name: &str) -> Result<(), ()> {
2662 let src = match CString::from_str(shader_source) {
2663 Ok(v) => v,
2664 Err(_) => return Err(()),
2665 };
2666 let src_ptr = src.as_ptr();
2667 unsafe {
2668 gls::gl::ShaderSource(shader, 1, &raw const src_ptr, null());
2669 gls::gl::CompileShader(shader);
2670 let mut is_compiled = 0;
2671 gls::gl::GetShaderiv(shader, gls::gl::COMPILE_STATUS, &raw mut is_compiled);
2672 if is_compiled == 0 {
2673 let mut max_length = 0;
2674 gls::gl::GetShaderiv(shader, gls::gl::INFO_LOG_LENGTH, &raw mut max_length);
2675 let mut error_log: Vec<u8> = vec![0; max_length as usize];
2676 gls::gl::GetShaderInfoLog(
2677 shader,
2678 max_length,
2679 &raw mut max_length,
2680 error_log.as_mut_ptr() as *mut c_char,
2681 );
2682 error!(
2683 "Shader '{}' failed: {:?}\n",
2684 shader_name,
2685 CString::from_vec_with_nul(error_log)
2686 .unwrap()
2687 .into_string()
2688 .unwrap()
2689 );
2690 gls::gl::DeleteShader(shader);
2691 return Err(());
2692 }
2693 Ok(())
2694 }
2695}
2696
2697fn check_gl_error(name: &str, line: u32) -> Result<(), Error> {
2698 unsafe {
2699 let err = gls::gl::GetError();
2700 if err != gls::gl::NO_ERROR {
2701 error!("GL Error: {name}:{line}: {err:#X}");
2702 return Err(Error::OpenGl(format!("{err:#X}")));
2704 }
2705 }
2706 Ok(())
2707}
2708
2709fn fourcc_to_drm(fourcc: FourCharCode) -> DrmFourcc {
2710 match fourcc {
2711 RGBA => DrmFourcc::Abgr8888,
2712 YUYV => DrmFourcc::Yuyv,
2713 RGB => DrmFourcc::Bgr888,
2714 GREY => DrmFourcc::R8,
2715 NV12 => DrmFourcc::Nv12,
2716 _ => todo!(),
2717 }
2718}
2719
2720mod egl_ext {
2721 #![allow(dead_code)]
2722 pub(crate) const LINUX_DMA_BUF: u32 = 0x3270;
2723 pub(crate) const LINUX_DRM_FOURCC: u32 = 0x3271;
2724 pub(crate) const DMA_BUF_PLANE0_FD: u32 = 0x3272;
2725 pub(crate) const DMA_BUF_PLANE0_OFFSET: u32 = 0x3273;
2726 pub(crate) const DMA_BUF_PLANE0_PITCH: u32 = 0x3274;
2727 pub(crate) const DMA_BUF_PLANE1_FD: u32 = 0x3275;
2728 pub(crate) const DMA_BUF_PLANE1_OFFSET: u32 = 0x3276;
2729 pub(crate) const DMA_BUF_PLANE1_PITCH: u32 = 0x3277;
2730 pub(crate) const DMA_BUF_PLANE2_FD: u32 = 0x3278;
2731 pub(crate) const DMA_BUF_PLANE2_OFFSET: u32 = 0x3279;
2732 pub(crate) const DMA_BUF_PLANE2_PITCH: u32 = 0x327A;
2733 pub(crate) const YUV_COLOR_SPACE_HINT: u32 = 0x327B;
2734 pub(crate) const SAMPLE_RANGE_HINT: u32 = 0x327C;
2735 pub(crate) const YUV_CHROMA_HORIZONTAL_SITING_HINT: u32 = 0x327D;
2736 pub(crate) const YUV_CHROMA_VERTICAL_SITING_HINT: u32 = 0x327E;
2737
2738 pub(crate) const ITU_REC601: u32 = 0x327F;
2739 pub(crate) const ITU_REC709: u32 = 0x3280;
2740 pub(crate) const ITU_REC2020: u32 = 0x3281;
2741
2742 pub(crate) const YUV_FULL_RANGE: u32 = 0x3282;
2743 pub(crate) const YUV_NARROW_RANGE: u32 = 0x3283;
2744
2745 pub(crate) const YUV_CHROMA_SITING_0: u32 = 0x3284;
2746 pub(crate) const YUV_CHROMA_SITING_0_5: u32 = 0x3285;
2747
2748 pub(crate) const PLATFORM_GBM_KHR: u32 = 0x31D7;
2749
2750 pub(crate) const PLATFORM_DEVICE_EXT: u32 = 0x313F;
2751}
2752
2753fn generate_vertex_shader() -> &'static str {
2754 "\
2755#version 300 es
2756precision mediump float;
2757layout(location = 0) in vec3 pos;
2758layout(location = 1) in vec2 texCoord;
2759
2760out vec3 fragPos;
2761out vec2 tc;
2762
2763void main() {
2764 fragPos = pos;
2765 tc = texCoord;
2766
2767 gl_Position = vec4(pos, 1.0);
2768}
2769"
2770}
2771
2772fn generate_texture_fragment_shader() -> &'static str {
2773 "\
2774#version 300 es
2775
2776precision mediump float;
2777uniform sampler2D tex;
2778in vec3 fragPos;
2779in vec2 tc;
2780
2781out vec4 color;
2782
2783void main(){
2784 color = texture(tex, tc);
2785}
2786"
2787}
2788
2789fn generate_texture_fragment_shader_yuv() -> &'static str {
2790 "\
2791#version 300 es
2792#extension GL_OES_EGL_image_external_essl3 : require
2793precision mediump float;
2794uniform samplerExternalOES tex;
2795in vec3 fragPos;
2796in vec2 tc;
2797
2798out vec4 color;
2799
2800void main(){
2801 color = texture(tex, tc);
2802}
2803"
2804}
2805
2806fn generate_planar_rgb_shader() -> &'static str {
2807 "\
2808#version 300 es
2809#extension GL_OES_EGL_image_external_essl3 : require
2810precision mediump float;
2811uniform samplerExternalOES tex;
2812in vec3 fragPos;
2813in vec2 tc;
2814
2815out vec4 color;
2816
2817void main(){
2818 color = texture(tex, tc);
2819}
2820"
2821}
2822
2823fn generate_segmentation_shader() -> &'static str {
2826 "\
2827#version 300 es
2828precision mediump float;
2829precision mediump sampler2DArray;
2830
2831uniform sampler2DArray tex;
2832uniform vec4 colors[20];
2833uniform int background_index;
2834
2835in vec3 fragPos;
2836in vec2 tc;
2837in vec4 fragColor;
2838
2839out vec4 color;
2840
2841float max_arg(const in vec4 args, out int argmax) {
2842 if (args[0] >= args[1] && args[0] >= args[2] && args[0] >= args[3]) {
2843 argmax = 0;
2844 return args[0];
2845 }
2846 if (args[1] >= args[0] && args[1] >= args[2] && args[1] >= args[3]) {
2847 argmax = 1;
2848 return args[1];
2849 }
2850 if (args[2] >= args[0] && args[2] >= args[1] && args[2] >= args[3]) {
2851 argmax = 2;
2852 return args[2];
2853 }
2854 argmax = 3;
2855 return args[3];
2856}
2857
2858void main() {
2859 mediump int layers = textureSize(tex, 0).z;
2860 float max_all = -4.0;
2861 int max_ind = 0;
2862 for (int i = 0; i < layers; i++) {
2863 vec4 d = texture(tex, vec3(tc, i));
2864 int max_ind_ = 0;
2865 float max_ = max_arg(d, max_ind_);
2866 if (max_ <= max_all) { continue; }
2867 max_all = max_;
2868 max_ind = i*4 + max_ind_;
2869 }
2870 if (max_ind == background_index) {
2871 discard;
2872 }
2873 max_ind = max_ind % 20;
2874 color = colors[max_ind];
2875}
2876"
2877}
2878
2879fn generate_instanced_segmentation_shader() -> &'static str {
2880 "\
2881#version 300 es
2882precision mediump float;
2883uniform sampler2D mask0;
2884uniform vec4 colors[20];
2885uniform int class_index;
2886in vec3 fragPos;
2887in vec2 tc;
2888in vec4 fragColor;
2889
2890out vec4 color;
2891void main() {
2892 float r0 = texture(mask0, tc).r;
2893 int arg = int(r0>=0.5);
2894 if (arg == 0) {
2895 discard;
2896 }
2897 color = colors[class_index % 20];
2898}
2899"
2900}
2901
2902fn generate_color_shader() -> &'static str {
2903 "\
2904#version 300 es
2905precision mediump float;
2906uniform vec4 colors[20];
2907uniform int class_index;
2908
2909out vec4 color;
2910void main() {
2911 int index = class_index % 20;
2912 color = colors[index];
2913}
2914"
2915}
2916
2917#[cfg(test)]
2918#[cfg(feature = "opengl")]
2919mod gl_tests {
2920 use super::*;
2921 use crate::{TensorImage, RGBA};
2922 #[cfg(feature = "dma_test_formats")]
2923 use crate::{NV12, YUYV};
2924 use edgefirst_tensor::TensorTrait;
2925 #[cfg(feature = "dma_test_formats")]
2926 use edgefirst_tensor::{is_dma_available, TensorMapTrait, TensorMemory};
2927 use image::buffer::ConvertBuffer;
2928 use ndarray::Array3;
2929
2930 #[test]
2931 #[cfg(feature = "decoder")]
2932 fn test_segmentation() {
2933 use edgefirst_decoder::Segmentation;
2934
2935 if !is_opengl_available() {
2936 eprintln!("SKIPPED: {} - OpenGL not available", function!());
2937 return;
2938 }
2939
2940 let mut image = TensorImage::load(
2941 include_bytes!("../../../testdata/giraffe.jpg"),
2942 Some(RGBA),
2943 None,
2944 )
2945 .unwrap();
2946
2947 let mut segmentation = Array3::from_shape_vec(
2948 (2, 160, 160),
2949 include_bytes!("../../../testdata/modelpack_seg_2x160x160.bin").to_vec(),
2950 )
2951 .unwrap();
2952 segmentation.swap_axes(0, 1);
2953 segmentation.swap_axes(1, 2);
2954 let segmentation = segmentation.as_standard_layout().to_owned();
2955
2956 let seg = Segmentation {
2957 segmentation,
2958 xmin: 0.0,
2959 ymin: 0.0,
2960 xmax: 1.0,
2961 ymax: 1.0,
2962 };
2963
2964 let mut renderer = GLProcessorThreaded::new().unwrap();
2965 renderer.render_to_image(&mut image, &[], &[seg]).unwrap();
2966 }
2967
2968 #[test]
2969 #[cfg(feature = "decoder")]
2970 fn test_segmentation_mem() {
2971 use edgefirst_decoder::Segmentation;
2972
2973 if !is_opengl_available() {
2974 eprintln!("SKIPPED: {} - OpenGL not available", function!());
2975 return;
2976 }
2977
2978 let mut image = TensorImage::load(
2979 include_bytes!("../../../testdata/giraffe.jpg"),
2980 Some(RGBA),
2981 Some(edgefirst_tensor::TensorMemory::Mem),
2982 )
2983 .unwrap();
2984
2985 let mut segmentation = Array3::from_shape_vec(
2986 (2, 160, 160),
2987 include_bytes!("../../../testdata/modelpack_seg_2x160x160.bin").to_vec(),
2988 )
2989 .unwrap();
2990 segmentation.swap_axes(0, 1);
2991 segmentation.swap_axes(1, 2);
2992 let segmentation = segmentation.as_standard_layout().to_owned();
2993
2994 let seg = Segmentation {
2995 segmentation,
2996 xmin: 0.0,
2997 ymin: 0.0,
2998 xmax: 1.0,
2999 ymax: 1.0,
3000 };
3001
3002 let mut renderer = GLProcessorThreaded::new().unwrap();
3003 renderer.render_to_image(&mut image, &[], &[seg]).unwrap();
3004 }
3005
3006 #[test]
3007 #[cfg(feature = "decoder")]
3008 fn test_segmentation_yolo() {
3009 use edgefirst_decoder::Segmentation;
3010 use ndarray::Array3;
3011
3012 if !is_opengl_available() {
3013 eprintln!("SKIPPED: {} - OpenGL not available", function!());
3014 return;
3015 }
3016
3017 let mut image = TensorImage::load(
3018 include_bytes!("../../../testdata/giraffe.jpg"),
3019 Some(RGBA),
3020 None,
3021 )
3022 .unwrap();
3023
3024 let segmentation = Array3::from_shape_vec(
3025 (76, 55, 1),
3026 include_bytes!("../../../testdata/yolov8_seg_crop_76x55.bin").to_vec(),
3027 )
3028 .unwrap();
3029
3030 let detect = DetectBox {
3031 bbox: [0.59375, 0.25, 0.9375, 0.725].into(),
3032 score: 0.99,
3033 label: 1,
3034 };
3035
3036 let seg = Segmentation {
3037 segmentation,
3038 xmin: 0.59375,
3039 ymin: 0.25,
3040 xmax: 0.9375,
3041 ymax: 0.725,
3042 };
3043
3044 let mut renderer = GLProcessorThreaded::new().unwrap();
3045 renderer
3046 .set_class_colors(&[[255, 255, 0, 233], [128, 128, 255, 100]])
3047 .unwrap();
3048 renderer
3049 .render_to_image(&mut image, &[detect], &[seg])
3050 .unwrap();
3051
3052 let expected = TensorImage::load(
3053 include_bytes!("../../../testdata/output_render_gl.jpg"),
3054 Some(RGBA),
3055 None,
3056 )
3057 .unwrap();
3058
3059 compare_images(&image, &expected, 0.99, function!());
3060 }
3061
3062 #[test]
3063 #[cfg(feature = "decoder")]
3064 fn test_boxes() {
3065 use edgefirst_decoder::DetectBox;
3066
3067 if !is_opengl_available() {
3068 eprintln!("SKIPPED: {} - OpenGL not available", function!());
3069 return;
3070 }
3071
3072 let mut image = TensorImage::load(
3073 include_bytes!("../../../testdata/giraffe.jpg"),
3074 Some(RGBA),
3075 None,
3076 )
3077 .unwrap();
3078
3079 let detect = DetectBox {
3080 bbox: [0.59375, 0.25, 0.9375, 0.725].into(),
3081 score: 0.99,
3082 label: 0,
3083 };
3084 let mut renderer = GLProcessorThreaded::new().unwrap();
3085 renderer
3086 .set_class_colors(&[[255, 255, 0, 233], [128, 128, 255, 100]])
3087 .unwrap();
3088 renderer
3089 .render_to_image(&mut image, &[detect], &[])
3090 .unwrap();
3091 }
3092
3093 static GL_AVAILABLE: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
3094 fn is_opengl_available() -> bool {
3096 #[cfg(all(target_os = "linux", feature = "opengl"))]
3097 {
3098 *GL_AVAILABLE.get_or_init(|| GLProcessorThreaded::new().is_ok())
3099 }
3100
3101 #[cfg(not(all(target_os = "linux", feature = "opengl")))]
3102 {
3103 false
3104 }
3105 }
3106
3107 fn compare_images(img1: &TensorImage, img2: &TensorImage, threshold: f64, name: &str) {
3108 assert_eq!(img1.height(), img2.height(), "Heights differ");
3109 assert_eq!(img1.width(), img2.width(), "Widths differ");
3110 assert_eq!(img1.fourcc(), img2.fourcc(), "FourCC differ");
3111 assert!(
3112 matches!(img1.fourcc(), RGB | RGBA | GREY | PLANAR_RGB),
3113 "FourCC must be RGB or RGBA for comparison"
3114 );
3115
3116 let image1 = match img1.fourcc() {
3117 RGB => image::RgbImage::from_vec(
3118 img1.width() as u32,
3119 img1.height() as u32,
3120 img1.tensor().map().unwrap().to_vec(),
3121 )
3122 .unwrap(),
3123 RGBA => image::RgbaImage::from_vec(
3124 img1.width() as u32,
3125 img1.height() as u32,
3126 img1.tensor().map().unwrap().to_vec(),
3127 )
3128 .unwrap()
3129 .convert(),
3130 GREY => image::GrayImage::from_vec(
3131 img1.width() as u32,
3132 img1.height() as u32,
3133 img1.tensor().map().unwrap().to_vec(),
3134 )
3135 .unwrap()
3136 .convert(),
3137 PLANAR_RGB => image::GrayImage::from_vec(
3138 img1.width() as u32,
3139 (img1.height() * 3) as u32,
3140 img1.tensor().map().unwrap().to_vec(),
3141 )
3142 .unwrap()
3143 .convert(),
3144 _ => return,
3145 };
3146
3147 let image2 = match img2.fourcc() {
3148 RGB => image::RgbImage::from_vec(
3149 img2.width() as u32,
3150 img2.height() as u32,
3151 img2.tensor().map().unwrap().to_vec(),
3152 )
3153 .unwrap(),
3154 RGBA => image::RgbaImage::from_vec(
3155 img2.width() as u32,
3156 img2.height() as u32,
3157 img2.tensor().map().unwrap().to_vec(),
3158 )
3159 .unwrap()
3160 .convert(),
3161 GREY => image::GrayImage::from_vec(
3162 img2.width() as u32,
3163 img2.height() as u32,
3164 img2.tensor().map().unwrap().to_vec(),
3165 )
3166 .unwrap()
3167 .convert(),
3168 PLANAR_RGB => image::GrayImage::from_vec(
3169 img2.width() as u32,
3170 (img2.height() * 3) as u32,
3171 img2.tensor().map().unwrap().to_vec(),
3172 )
3173 .unwrap()
3174 .convert(),
3175 _ => return,
3176 };
3177
3178 let similarity = image_compare::rgb_similarity_structure(
3179 &image_compare::Algorithm::RootMeanSquared,
3180 &image1,
3181 &image2,
3182 )
3183 .expect("Image Comparison failed");
3184 if similarity.score < threshold {
3185 similarity
3188 .image
3189 .to_color_map()
3190 .save(format!("{name}.png"))
3191 .unwrap();
3192 panic!(
3193 "{name}: converted image and target image have similarity score too low: {} < {}",
3194 similarity.score, threshold
3195 )
3196 }
3197 }
3198
3199 #[cfg(feature = "dma_test_formats")]
3206 fn load_raw_image(
3207 width: usize,
3208 height: usize,
3209 fourcc: FourCharCode,
3210 memory: Option<TensorMemory>,
3211 bytes: &[u8],
3212 ) -> Result<TensorImage, crate::Error> {
3213 let img = TensorImage::new(width, height, fourcc, memory)?;
3214 let mut map = img.tensor().map()?;
3215 map.as_mut_slice()[..bytes.len()].copy_from_slice(bytes);
3216 Ok(img)
3217 }
3218
3219 #[test]
3221 #[cfg(all(target_os = "linux", feature = "dma_test_formats"))]
3222 fn test_opengl_nv12_to_rgba_reference() {
3223 if !is_dma_available() {
3224 return;
3225 }
3226 let src = load_raw_image(
3228 1280,
3229 720,
3230 NV12,
3231 Some(TensorMemory::Dma),
3232 include_bytes!("../../../testdata/camera720p.nv12"),
3233 )
3234 .unwrap();
3235
3236 let reference = load_raw_image(
3238 1280,
3239 720,
3240 RGBA,
3241 None,
3242 include_bytes!("../../../testdata/camera720p.rgba"),
3243 )
3244 .unwrap();
3245
3246 let mut dst = TensorImage::new(1280, 720, RGBA, Some(TensorMemory::Dma)).unwrap();
3248 let mut gl = GLProcessorThreaded::new().unwrap();
3249 gl.convert(&src, &mut dst, Rotation::None, Flip::None, Crop::no_crop())
3250 .unwrap();
3251
3252 let cpu_dst = TensorImage::new(1280, 720, RGBA, None).unwrap();
3254 cpu_dst
3255 .tensor()
3256 .map()
3257 .unwrap()
3258 .as_mut_slice()
3259 .copy_from_slice(dst.tensor().map().unwrap().as_slice());
3260
3261 compare_images(&reference, &cpu_dst, 0.98, "opengl_nv12_to_rgba_reference");
3262 }
3263
3264 #[test]
3266 #[cfg(all(target_os = "linux", feature = "dma_test_formats"))]
3267 fn test_opengl_yuyv_to_rgba_reference() {
3268 if !is_dma_available() {
3269 return;
3270 }
3271 let src = load_raw_image(
3273 1280,
3274 720,
3275 YUYV,
3276 Some(TensorMemory::Dma),
3277 include_bytes!("../../../testdata/camera720p.yuyv"),
3278 )
3279 .unwrap();
3280
3281 let reference = load_raw_image(
3283 1280,
3284 720,
3285 RGBA,
3286 None,
3287 include_bytes!("../../../testdata/camera720p.rgba"),
3288 )
3289 .unwrap();
3290
3291 let mut dst = TensorImage::new(1280, 720, RGBA, Some(TensorMemory::Dma)).unwrap();
3293 let mut gl = GLProcessorThreaded::new().unwrap();
3294 gl.convert(&src, &mut dst, Rotation::None, Flip::None, Crop::no_crop())
3295 .unwrap();
3296
3297 let cpu_dst = TensorImage::new(1280, 720, RGBA, None).unwrap();
3299 cpu_dst
3300 .tensor()
3301 .map()
3302 .unwrap()
3303 .as_mut_slice()
3304 .copy_from_slice(dst.tensor().map().unwrap().as_slice());
3305
3306 compare_images(&reference, &cpu_dst, 0.98, "opengl_yuyv_to_rgba_reference");
3307 }
3308}