Skip to main content

edgefirst_image/
opengl_headless.rs

1// SPDX-FileCopyrightText: Copyright 2025 Au-Zone Technologies
2// SPDX-License-Identifier: Apache-2.0
3
4#![cfg(target_os = "linux")]
5#![cfg(feature = "opengl")]
6
7use edgefirst_decoder::DetectBox;
8#[cfg(feature = "decoder")]
9use edgefirst_decoder::Segmentation;
10use edgefirst_tensor::{TensorMemory, TensorTrait};
11use four_char_code::FourCharCode;
12use gbm::{
13    drm::{buffer::DrmFourcc, control::Device as DrmControlDevice, Device as DrmDevice},
14    AsRaw, Device,
15};
16use khronos_egl::{self as egl, Attrib, Display, Dynamic, Instance, EGL1_4};
17use log::{debug, error};
18use std::{
19    collections::BTreeSet,
20    ffi::{c_char, c_void, CStr, CString},
21    os::fd::AsRawFd,
22    ptr::{null, null_mut, NonNull},
23    rc::Rc,
24    str::FromStr,
25    sync::OnceLock,
26    thread::JoinHandle,
27    time::Instant,
28};
29use tokio::sync::mpsc::Sender;
30
31macro_rules! function {
32    () => {{
33        fn f() {}
34        fn type_name_of<T>(_: T) -> &'static str {
35            std::any::type_name::<T>()
36        }
37        let name = type_name_of(f);
38
39        // Find and cut the rest of the path
40        match &name[..name.len() - 3].rfind(':') {
41            Some(pos) => &name[pos + 1..name.len() - 3],
42            None => &name[..name.len() - 3],
43        }
44    }};
45}
46
47#[cfg(feature = "decoder")]
48use crate::DEFAULT_COLORS;
49use crate::{
50    CPUProcessor, Crop, Error, Flip, ImageProcessorTrait, Rect, Rotation, TensorImage,
51    TensorImageRef, GREY, NV12, PLANAR_RGB, PLANAR_RGBA, RGB, RGBA, YUYV,
52};
53
54static EGL_LIB: OnceLock<libloading::Library> = OnceLock::new();
55
56fn get_egl_lib() -> Result<&'static libloading::Library, crate::Error> {
57    if let Some(egl) = EGL_LIB.get() {
58        Ok(egl)
59    } else {
60        let egl = unsafe { libloading::Library::new("libEGL.so.1")? };
61        Ok(EGL_LIB.get_or_init(|| egl))
62    }
63}
64
65type Egl = Instance<Dynamic<&'static libloading::Library, EGL1_4>>;
66pub(crate) struct GlContext {
67    pub(crate) support_dma: bool,
68    pub(crate) surface: Option<egl::Surface>,
69    pub(crate) display: EglDisplayType,
70    pub(crate) ctx: egl::Context,
71    pub(crate) egl: Rc<Egl>,
72}
73
74pub(crate) enum EglDisplayType {
75    Default(egl::Display),
76    Gbm(egl::Display, #[allow(dead_code)] Device<Card>),
77    PlatformDisplay(egl::Display),
78}
79
80impl EglDisplayType {
81    fn as_display(&self) -> egl::Display {
82        match self {
83            EglDisplayType::Default(disp) => *disp,
84            EglDisplayType::Gbm(disp, _) => *disp,
85            EglDisplayType::PlatformDisplay(disp) => *disp,
86        }
87    }
88}
89
90impl GlContext {
91    pub(crate) fn new() -> Result<GlContext, crate::Error> {
92        // Create an EGL API instance.
93        let egl: Rc<Egl> =
94            Rc::new(unsafe { Instance::<Dynamic<_, EGL1_4>>::load_required_from(get_egl_lib()?)? });
95
96        if let Ok(headless) = Self::try_initialize_egl(egl.clone(), Self::egl_get_default_display) {
97            return Ok(headless);
98        } else {
99            log::debug!("Didn't initialize EGL with Default Display");
100        }
101
102        if let Ok(headless) = Self::try_initialize_egl(egl.clone(), Self::egl_get_gbm_display) {
103            return Ok(headless);
104        } else {
105            log::debug!("Didn't initialize EGL with GBM Display");
106        }
107
108        if let Ok(headless) =
109            Self::try_initialize_egl(egl.clone(), Self::egl_get_platform_display_from_device)
110        {
111            return Ok(headless);
112        } else {
113            log::debug!("Didn't initialize EGL with platform display from device enumeration");
114        }
115
116        Err(Error::OpenGl(
117            "Could not initialize EGL with any known method".to_string(),
118        ))
119    }
120
121    fn try_initialize_egl(
122        egl: Rc<Egl>,
123        display_fn: impl Fn(&Egl) -> Result<EglDisplayType, crate::Error>,
124    ) -> Result<GlContext, crate::Error> {
125        let display = display_fn(&egl)?;
126        log::debug!("egl initialize with display: {:x?}", display.as_display());
127        egl.initialize(display.as_display())?;
128        let attributes = [
129            egl::SURFACE_TYPE,
130            egl::PBUFFER_BIT,
131            egl::RENDERABLE_TYPE,
132            egl::OPENGL_ES3_BIT,
133            egl::RED_SIZE,
134            8,
135            egl::GREEN_SIZE,
136            8,
137            egl::BLUE_SIZE,
138            8,
139            egl::ALPHA_SIZE,
140            8,
141            egl::NONE,
142        ];
143
144        let config =
145            if let Some(config) = egl.choose_first_config(display.as_display(), &attributes)? {
146                config
147            } else {
148                return Err(crate::Error::NotImplemented(
149                    "Did not find valid OpenGL ES config".to_string(),
150                ));
151            };
152
153        debug!("config: {config:?}");
154
155        let surface = Some(egl.create_pbuffer_surface(
156            display.as_display(),
157            config,
158            &[egl::WIDTH, 64, egl::HEIGHT, 64, egl::NONE],
159        )?);
160
161        egl.bind_api(egl::OPENGL_ES_API)?;
162        let context_attributes = [egl::CONTEXT_MAJOR_VERSION, 3, egl::NONE, egl::NONE];
163
164        let ctx = egl.create_context(display.as_display(), config, None, &context_attributes)?;
165        debug!("ctx: {ctx:?}");
166
167        egl.make_current(display.as_display(), surface, surface, Some(ctx))?;
168
169        let support_dma = Self::egl_check_support_dma(&egl).is_ok();
170        let headless = GlContext {
171            display,
172            ctx,
173            egl,
174            surface,
175            support_dma,
176        };
177        Ok(headless)
178    }
179
180    fn egl_get_default_display(egl: &Egl) -> Result<EglDisplayType, crate::Error> {
181        // get the default display
182        if let Some(display) = unsafe { egl.get_display(egl::DEFAULT_DISPLAY) } {
183            debug!("default display: {display:?}");
184            return Ok(EglDisplayType::Default(display));
185        }
186
187        Err(Error::OpenGl(
188            "Could not obtain EGL Default Display".to_string(),
189        ))
190    }
191
192    fn egl_get_gbm_display(egl: &Egl) -> Result<EglDisplayType, crate::Error> {
193        // init a GBM device
194        let gbm = Device::new(Card::open_global()?)?;
195
196        debug!("gbm: {gbm:?}");
197        let display = Self::egl_get_platform_display_with_fallback(
198            egl,
199            egl_ext::PLATFORM_GBM_KHR,
200            gbm.as_raw() as *mut c_void,
201            &[egl::ATTRIB_NONE],
202        )?;
203
204        Ok(EglDisplayType::Gbm(display, gbm))
205    }
206
207    fn egl_get_platform_display_from_device(egl: &Egl) -> Result<EglDisplayType, crate::Error> {
208        let extensions = egl.query_string(None, egl::EXTENSIONS)?;
209        let extensions = extensions.to_string_lossy();
210        log::debug!("EGL Extensions: {}", extensions);
211
212        if !extensions.contains("EGL_EXT_device_enumeration") {
213            return Err(Error::GLVersion(
214                "EGL doesn't supported EGL_EXT_device_enumeration extension".to_string(),
215            ));
216        }
217
218        type EGLDeviceEXT = *mut c_void;
219        let devices = if let Some(ext) = egl.get_proc_address("eglQueryDevicesEXT") {
220            let func: unsafe extern "system" fn(
221                max_devices: egl::Int,
222                devices: *mut EGLDeviceEXT,
223                num_devices: *mut egl::Int,
224            ) -> *const c_char = unsafe { std::mem::transmute(ext) };
225            let mut devices = [std::ptr::null_mut(); 10];
226            let mut num_devices = 0;
227            unsafe { func(devices.len() as i32, devices.as_mut_ptr(), &mut num_devices) };
228            for i in 0..num_devices {
229                log::debug!("EGL device: {:?}", devices[i as usize]);
230            }
231            devices[0..num_devices as usize].to_vec()
232        } else {
233            return Err(Error::GLVersion(
234                "EGL doesn't supported eglQueryDevicesEXT function".to_string(),
235            ));
236        };
237
238        if !extensions.contains("EGL_EXT_platform_device") {
239            return Err(Error::GLVersion(
240                "EGL doesn't supported EGL_EXT_platform_device extension".to_string(),
241            ));
242        }
243
244        // just use the first device?
245        let disp = Self::egl_get_platform_display_with_fallback(
246            egl,
247            egl_ext::PLATFORM_DEVICE_EXT,
248            devices[0],
249            &[egl::ATTRIB_NONE],
250        )?;
251        Ok(EglDisplayType::PlatformDisplay(disp))
252    }
253
254    fn egl_check_support_dma(egl: &Egl) -> Result<(), crate::Error> {
255        let extensions = egl.query_string(None, egl::EXTENSIONS)?;
256        let extensions = extensions.to_string_lossy();
257        log::debug!("EGL Extensions: {}", extensions);
258
259        if egl.upcast::<egl::EGL1_5>().is_some() {
260            return Ok(());
261        }
262
263        if !extensions.contains("EGL_EXT_image_dma_buf_import") {
264            return Err(crate::Error::GLVersion(
265                "EGL does not support EGL_EXT_image_dma_buf_import extension".to_string(),
266            ));
267        }
268
269        if egl.get_proc_address("eglCreateImageKHR").is_none() {
270            return Err(crate::Error::GLVersion(
271                "EGL does not support eglCreateImageKHR function".to_string(),
272            ));
273        }
274
275        if egl.get_proc_address("eglDestroyImageKHR").is_none() {
276            return Err(crate::Error::GLVersion(
277                "EGL does not support eglDestroyImageKHR function".to_string(),
278            ));
279        }
280        // Err(crate::Error::GLVersion("EGL Version too low".to_string()))
281        Ok(())
282    }
283
284    fn egl_get_platform_display_with_fallback(
285        egl: &Egl,
286        platform: egl::Enum,
287        native_display: *mut c_void,
288        attrib_list: &[Attrib],
289    ) -> Result<Display, Error> {
290        if let Some(egl) = egl.upcast::<egl::EGL1_5>() {
291            unsafe { egl.get_platform_display(platform, native_display, attrib_list) }
292                .map_err(|e| e.into())
293        } else if let Some(ext) = egl.get_proc_address("eglGetPlatformDisplayEXT") {
294            let func: unsafe extern "system" fn(
295                platform: egl::Enum,
296                native_display: *mut c_void,
297                attrib_list: *const Attrib,
298            ) -> egl::EGLDisplay = unsafe { std::mem::transmute(ext) };
299            let disp = unsafe { func(platform, native_display, attrib_list.as_ptr()) };
300            if disp != egl::NO_DISPLAY {
301                Ok(unsafe { Display::from_ptr(disp) })
302            } else {
303                Err(egl.get_error().map(|e| e.into()).unwrap_or(Error::Internal(
304                    "EGL failed but no error was reported".to_owned(),
305                )))
306            }
307        } else {
308            Err(Error::EGLLoad(egl::LoadError::InvalidVersion {
309                provided: egl.version(),
310                required: khronos_egl::Version::EGL1_5,
311            }))
312        }
313    }
314
315    fn egl_create_image_with_fallback(
316        egl: &Egl,
317        display: Display,
318        ctx: egl::Context,
319        target: egl::Enum,
320        buffer: egl::ClientBuffer,
321        attrib_list: &[Attrib],
322    ) -> Result<egl::Image, Error> {
323        if let Some(egl) = egl.upcast::<egl::EGL1_5>() {
324            egl.create_image(display, ctx, target, buffer, attrib_list)
325                .map_err(|e| e.into())
326        } else if let Some(ext) = egl.get_proc_address("eglCreateImageKHR") {
327            log::trace!("eglCreateImageKHR addr: {:?}", ext);
328            let func: unsafe extern "system" fn(
329                display: egl::EGLDisplay,
330                ctx: egl::EGLContext,
331                target: egl::Enum,
332                buffer: egl::EGLClientBuffer,
333                attrib_list: *const egl::Int,
334            ) -> egl::EGLImage = unsafe { std::mem::transmute(ext) };
335            let new_attrib_list = attrib_list
336                .iter()
337                .map(|x| *x as egl::Int)
338                .collect::<Vec<_>>();
339
340            let image = unsafe {
341                func(
342                    display.as_ptr(),
343                    ctx.as_ptr(),
344                    target,
345                    buffer.as_ptr(),
346                    new_attrib_list.as_ptr(),
347                )
348            };
349            if image != egl::NO_IMAGE {
350                Ok(unsafe { egl::Image::from_ptr(image) })
351            } else {
352                Err(egl.get_error().map(|e| e.into()).unwrap_or(Error::Internal(
353                    "EGL failed but no error was reported".to_owned(),
354                )))
355            }
356        } else {
357            Err(Error::EGLLoad(egl::LoadError::InvalidVersion {
358                provided: egl.version(),
359                required: khronos_egl::Version::EGL1_5,
360            }))
361        }
362    }
363
364    fn egl_destory_image_with_fallback(
365        egl: &Egl,
366        display: Display,
367        image: egl::Image,
368    ) -> Result<(), Error> {
369        if let Some(egl) = egl.upcast::<egl::EGL1_5>() {
370            egl.destroy_image(display, image).map_err(|e| e.into())
371        } else if let Some(ext) = egl.get_proc_address("eglDestroyImageKHR") {
372            let func: unsafe extern "system" fn(
373                display: egl::EGLDisplay,
374                image: egl::EGLImage,
375            ) -> egl::Boolean = unsafe { std::mem::transmute(ext) };
376            let res = unsafe { func(display.as_ptr(), image.as_ptr()) };
377            if res == egl::TRUE {
378                Ok(())
379            } else {
380                Err(egl.get_error().map(|e| e.into()).unwrap_or(Error::Internal(
381                    "EGL failed but no error was reported".to_owned(),
382                )))
383            }
384        } else {
385            Err(Error::EGLLoad(egl::LoadError::InvalidVersion {
386                provided: egl.version(),
387                required: khronos_egl::Version::EGL1_5,
388            }))
389        }
390    }
391}
392
393impl Drop for GlContext {
394    fn drop(&mut self) {
395        let _ = self
396            .egl
397            .make_current(self.display.as_display(), None, None, None);
398
399        let _ = self
400            .egl
401            .destroy_context(self.display.as_display(), self.ctx);
402
403        if let Some(surface) = self.surface.take() {
404            let _ = self.egl.destroy_surface(self.display.as_display(), surface);
405        }
406
407        let _ = self.egl.terminate(self.display.as_display());
408    }
409}
410
411#[derive(Debug)]
412/// A simple wrapper for a device node.
413pub(crate) struct Card(std::fs::File);
414
415/// Implementing `AsFd` is a prerequisite to implementing the traits found
416/// in this crate. Here, we are just calling `as_fd()` on the inner File.
417impl std::os::unix::io::AsFd for Card {
418    fn as_fd(&self) -> std::os::unix::io::BorrowedFd<'_> {
419        self.0.as_fd()
420    }
421}
422
423/// With `AsFd` implemented, we can now implement `drm::Device`.
424impl DrmDevice for Card {}
425impl DrmControlDevice for Card {}
426
427/// Simple helper methods for opening a `Card`.
428impl Card {
429    pub fn open(path: &str) -> Result<Self, crate::Error> {
430        let mut options = std::fs::OpenOptions::new();
431        options.read(true);
432        options.write(true);
433        let c = options.open(path);
434        match c {
435            Ok(c) => Ok(Card(c)),
436            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
437                Err(Error::NotFound(format!("File not found: {path}")))
438            }
439            Err(e) => Err(e.into()),
440        }
441    }
442
443    pub fn open_global() -> Result<Self, crate::Error> {
444        let targets = ["/dev/dri/render128", "/dev/dri/card0", "/dev/dri/card1"];
445        let e = Self::open(targets[0]);
446        if let Ok(t) = e {
447            return Ok(t);
448        }
449        for t in &targets[1..] {
450            if let Ok(t) = Self::open(t) {
451                return Ok(t);
452            }
453        }
454        e
455    }
456}
457
458#[derive(Debug, Clone, Copy)]
459struct RegionOfInterest {
460    left: f32,
461    top: f32,
462    right: f32,
463    bottom: f32,
464}
465
466enum GLProcessorMessage {
467    ImageConvert(
468        SendablePtr<TensorImage>,
469        SendablePtr<TensorImage>,
470        Rotation,
471        Flip,
472        Crop,
473        tokio::sync::oneshot::Sender<Result<(), Error>>,
474    ),
475    SetColors(
476        Vec<[u8; 4]>,
477        tokio::sync::oneshot::Sender<Result<(), Error>>,
478    ),
479    ImageRender(
480        SendablePtr<TensorImage>,
481        SendablePtr<DetectBox>,
482        SendablePtr<Segmentation>,
483        tokio::sync::oneshot::Sender<Result<(), Error>>,
484    ),
485}
486
487/// OpenGL multi-threaded image converter. The actual conversion is done in a
488/// separate rendering thread, as OpenGL contexts are not thread-safe. This can
489/// be safely sent between threads. The `convert()` call sends the conversion
490/// request to the rendering thread and waits for the result.
491#[derive(Debug)]
492pub struct GLProcessorThreaded {
493    // This is only None when the converter is being dropped.
494    handle: Option<JoinHandle<()>>,
495
496    // This is only None when the converter is being dropped.
497    sender: Option<Sender<GLProcessorMessage>>,
498    support_dma: bool,
499}
500
501unsafe impl Send for GLProcessorThreaded {}
502unsafe impl Sync for GLProcessorThreaded {}
503
504struct SendablePtr<T: Send> {
505    ptr: NonNull<T>,
506    len: usize,
507}
508
509unsafe impl<T> Send for SendablePtr<T> where T: Send {}
510
511impl GLProcessorThreaded {
512    /// Creates a new OpenGL multi-threaded image converter.
513    pub fn new() -> Result<Self, Error> {
514        let (send, mut recv) = tokio::sync::mpsc::channel::<GLProcessorMessage>(1);
515
516        let (create_ctx_send, create_ctx_recv) = tokio::sync::oneshot::channel();
517
518        let func = move || {
519            let mut gl_converter = match GLProcessorST::new() {
520                Ok(gl) => gl,
521                Err(e) => {
522                    let _ = create_ctx_send.send(Err(e));
523                    return;
524                }
525            };
526            let _ = create_ctx_send.send(Ok(gl_converter.gl_context.support_dma));
527            while let Some(msg) = recv.blocking_recv() {
528                match msg {
529                    GLProcessorMessage::ImageConvert(src, mut dst, rotation, flip, crop, resp) => {
530                        // SAFETY: This is safe because the convert() function waits for the resp to
531                        // be sent before dropping the borrow for src and dst
532                        let src = unsafe { src.ptr.as_ref() };
533                        let dst = unsafe { dst.ptr.as_mut() };
534                        let res = gl_converter.convert(src, dst, rotation, flip, crop);
535                        let _ = resp.send(res);
536                    }
537                    GLProcessorMessage::ImageRender(mut dst, det, seg, resp) => {
538                        // SAFETY: This is safe because the render_to_image() function waits for the
539                        // resp to be sent before dropping the borrow for dst, detect, and
540                        // segmentation
541                        let dst = unsafe { dst.ptr.as_mut() };
542                        let det = unsafe { std::slice::from_raw_parts(det.ptr.as_ptr(), det.len) };
543                        let seg = unsafe { std::slice::from_raw_parts(seg.ptr.as_ptr(), seg.len) };
544                        let res = gl_converter.render_to_image(dst, det, seg);
545                        let _ = resp.send(res);
546                    }
547                    GLProcessorMessage::SetColors(colors, resp) => {
548                        let res = gl_converter.set_class_colors(&colors);
549                        let _ = resp.send(res);
550                    }
551                }
552            }
553        };
554
555        // let handle = tokio::task::spawn(func());
556        let handle = std::thread::spawn(func);
557
558        let support_dma = match create_ctx_recv.blocking_recv() {
559            Ok(Err(e)) => return Err(e),
560            Err(_) => {
561                return Err(Error::Internal(
562                    "GL converter error messaging closed without update".to_string(),
563                ));
564            }
565            Ok(Ok(supports_dma)) => supports_dma,
566        };
567
568        Ok(Self {
569            handle: Some(handle),
570            sender: Some(send),
571            support_dma,
572        })
573    }
574}
575
576impl ImageProcessorTrait for GLProcessorThreaded {
577    fn convert(
578        &mut self,
579        src: &TensorImage,
580        dst: &mut TensorImage,
581        rotation: crate::Rotation,
582        flip: Flip,
583        crop: Crop,
584    ) -> crate::Result<()> {
585        crop.check_crop(src, dst)?;
586        if !GLProcessorST::check_src_format_supported(self.support_dma, src) {
587            return Err(crate::Error::NotSupported(format!(
588                "Opengl doesn't support {} source texture",
589                src.fourcc().display()
590            )));
591        }
592
593        if !GLProcessorST::check_dst_format_supported(self.support_dma, dst) {
594            return Err(crate::Error::NotSupported(format!(
595                "Opengl doesn't support {} destination texture",
596                dst.fourcc().display()
597            )));
598        }
599
600        let (err_send, err_recv) = tokio::sync::oneshot::channel();
601        self.sender
602            .as_ref()
603            .unwrap()
604            .blocking_send(GLProcessorMessage::ImageConvert(
605                SendablePtr {
606                    ptr: src.into(),
607                    len: 1,
608                },
609                SendablePtr {
610                    ptr: dst.into(),
611                    len: 1,
612                },
613                rotation,
614                flip,
615                crop,
616                err_send,
617            ))
618            .map_err(|_| Error::Internal("GL converter thread exited".to_string()))?;
619        err_recv.blocking_recv().map_err(|_| {
620            Error::Internal("GL converter error messaging closed without update".to_string())
621        })?
622    }
623
624    fn convert_ref(
625        &mut self,
626        src: &TensorImage,
627        dst: &mut TensorImageRef<'_>,
628        rotation: Rotation,
629        flip: Flip,
630        crop: Crop,
631    ) -> crate::Result<()> {
632        // OpenGL doesn't support PLANAR_RGB output, delegate to CPU
633        let mut cpu = CPUProcessor::new();
634        cpu.convert_ref(src, dst, rotation, flip, crop)
635    }
636
637    #[cfg(feature = "decoder")]
638    fn render_to_image(
639        &mut self,
640        dst: &mut TensorImage,
641        detect: &[crate::DetectBox],
642        segmentation: &[crate::Segmentation],
643    ) -> crate::Result<()> {
644        let (err_send, err_recv) = tokio::sync::oneshot::channel();
645        self.sender
646            .as_ref()
647            .unwrap()
648            .blocking_send(GLProcessorMessage::ImageRender(
649                SendablePtr {
650                    ptr: dst.into(),
651                    len: 1,
652                },
653                SendablePtr {
654                    ptr: NonNull::new(detect.as_ptr() as *mut DetectBox).unwrap(),
655                    len: detect.len(),
656                },
657                SendablePtr {
658                    ptr: NonNull::new(segmentation.as_ptr() as *mut Segmentation).unwrap(),
659                    len: segmentation.len(),
660                },
661                err_send,
662            ))
663            .map_err(|_| Error::Internal("GL converter thread exited".to_string()))?;
664        err_recv.blocking_recv().map_err(|_| {
665            Error::Internal("GL converter error messaging closed without update".to_string())
666        })?
667    }
668
669    #[cfg(feature = "decoder")]
670    fn set_class_colors(&mut self, colors: &[[u8; 4]]) -> Result<(), crate::Error> {
671        let (err_send, err_recv) = tokio::sync::oneshot::channel();
672        self.sender
673            .as_ref()
674            .unwrap()
675            .blocking_send(GLProcessorMessage::SetColors(colors.to_vec(), err_send))
676            .map_err(|_| Error::Internal("GL converter thread exited".to_string()))?;
677        err_recv.blocking_recv().map_err(|_| {
678            Error::Internal("GL converter error messaging closed without update".to_string())
679        })?
680    }
681}
682
683impl Drop for GLProcessorThreaded {
684    fn drop(&mut self) {
685        drop(self.sender.take());
686        let _ = self.handle.take().and_then(|h| h.join().ok());
687    }
688}
689
690/// OpenGL single-threaded image converter.
691pub struct GLProcessorST {
692    camera_eglimage_texture: Texture,
693    camera_normal_texture: Texture,
694    render_texture: Texture,
695    #[cfg(feature = "decoder")]
696    segmentation_texture: Texture,
697    #[cfg(feature = "decoder")]
698    segmentation_program: GlProgram,
699    #[cfg(feature = "decoder")]
700    instanced_segmentation_program: GlProgram,
701    #[cfg(feature = "decoder")]
702    color_program: GlProgram,
703    vertex_buffer: Buffer,
704    texture_buffer: Buffer,
705    texture_program: GlProgram,
706    texture_program_yuv: GlProgram,
707    texture_program_planar: GlProgram,
708    gl_context: GlContext,
709}
710
711impl ImageProcessorTrait for GLProcessorST {
712    fn convert(
713        &mut self,
714        src: &TensorImage,
715        dst: &mut TensorImage,
716        rotation: crate::Rotation,
717        flip: Flip,
718        crop: Crop,
719    ) -> crate::Result<()> {
720        crop.check_crop(src, dst)?;
721        if !Self::check_src_format_supported(self.gl_context.support_dma, src) {
722            return Err(crate::Error::NotSupported(format!(
723                "Opengl doesn't support {} source texture",
724                src.fourcc().display()
725            )));
726        }
727
728        if !Self::check_dst_format_supported(self.gl_context.support_dma, dst) {
729            return Err(crate::Error::NotSupported(format!(
730                "Opengl doesn't support {} destination texture",
731                dst.fourcc().display()
732            )));
733        }
734        log::debug!(
735            "dst tensor: {:?} src tensor :{:?}",
736            dst.tensor().memory(),
737            src.tensor().memory()
738        );
739        check_gl_error(function!(), line!())?;
740        if self.gl_context.support_dma
741            && dst.tensor().memory() == TensorMemory::Dma
742            && dst.fourcc() != RGB
743        // DMA generally doesn't support RGB
744        {
745            let res = self.convert_dest_dma(dst, src, rotation, flip, crop);
746            return res;
747        }
748        let start = Instant::now();
749        let res = self.convert_dest_non_dma(dst, src, rotation, flip, crop);
750        log::debug!("convert_dest_non_dma takes {:?}", start.elapsed());
751        res
752    }
753
754    fn convert_ref(
755        &mut self,
756        src: &TensorImage,
757        dst: &mut TensorImageRef<'_>,
758        rotation: Rotation,
759        flip: Flip,
760        crop: Crop,
761    ) -> crate::Result<()> {
762        // OpenGL doesn't support PLANAR_RGB output, delegate to CPU
763        let mut cpu = CPUProcessor::new();
764        cpu.convert_ref(src, dst, rotation, flip, crop)
765    }
766
767    #[cfg(feature = "decoder")]
768    fn render_to_image(
769        &mut self,
770        dst: &mut TensorImage,
771        detect: &[DetectBox],
772        segmentation: &[Segmentation],
773    ) -> Result<(), crate::Error> {
774        use crate::FunctionTimer;
775
776        let _timer = FunctionTimer::new("GLProcessorST::render_to_image");
777        if !matches!(dst.fourcc(), RGBA | RGB) {
778            return Err(crate::Error::NotSupported(
779                "Opengl image rendering only supports RGBA or RGB images".to_string(),
780            ));
781        }
782
783        let (_render_buffer, is_dma) = match dst.tensor.memory() {
784            edgefirst_tensor::TensorMemory::Dma => {
785                if let Ok(render_buffer) = self.setup_renderbuffer_dma(dst) {
786                    (render_buffer, true)
787                } else {
788                    (
789                        self.setup_renderbuffer_non_dma(
790                            dst,
791                            Crop::new().with_dst_rect(Some(Rect::new(0, 0, 0, 0))),
792                        )?,
793                        false,
794                    )
795                }
796            }
797            _ => (
798                self.setup_renderbuffer_non_dma(
799                    dst,
800                    Crop::new().with_dst_rect(Some(Rect::new(0, 0, 0, 0))),
801                )?,
802                false,
803            ), // Add dest rect to make sure dst is rendered fully
804        };
805
806        gls::enable(gls::gl::BLEND);
807        gls::blend_func_separate(
808            gls::gl::SRC_ALPHA,
809            gls::gl::ONE_MINUS_SRC_ALPHA,
810            gls::gl::ZERO,
811            gls::gl::ONE,
812        );
813
814        self.render_box(dst, detect)?;
815        self.render_segmentation(detect, segmentation)?;
816
817        gls::finish();
818        if !is_dma {
819            let mut dst_map = dst.tensor().map()?;
820            let format = match dst.fourcc() {
821                RGB => gls::gl::RGB,
822                RGBA => gls::gl::RGBA,
823                _ => unreachable!(),
824            };
825            unsafe {
826                gls::gl::ReadBuffer(gls::gl::COLOR_ATTACHMENT0);
827                gls::gl::ReadnPixels(
828                    0,
829                    0,
830                    dst.width() as i32,
831                    dst.height() as i32,
832                    format,
833                    gls::gl::UNSIGNED_BYTE,
834                    dst.tensor.len() as i32,
835                    dst_map.as_mut_ptr() as *mut c_void,
836                );
837            }
838        }
839
840        Ok(())
841    }
842
843    #[cfg(feature = "decoder")]
844    fn set_class_colors(&mut self, colors: &[[u8; 4]]) -> crate::Result<()> {
845        if colors.is_empty() {
846            return Ok(());
847        }
848        let mut colors_f32 = colors
849            .iter()
850            .map(|c| {
851                [
852                    c[0] as f32 / 255.0,
853                    c[1] as f32 / 255.0,
854                    c[2] as f32 / 255.0,
855                    c[3] as f32 / 255.0,
856                ]
857            })
858            .take(20)
859            .collect::<Vec<[f32; 4]>>();
860
861        self.segmentation_program
862            .load_uniform_4fv(c"colors", &colors_f32)?;
863        self.instanced_segmentation_program
864            .load_uniform_4fv(c"colors", &colors_f32)?;
865
866        colors_f32.iter_mut().for_each(|c| {
867            c[3] = 1.0; // set alpha to 1.0 for color rendering
868        });
869        self.color_program
870            .load_uniform_4fv(c"colors", &colors_f32)?;
871
872        Ok(())
873    }
874}
875
876impl GLProcessorST {
877    pub fn new() -> Result<GLProcessorST, crate::Error> {
878        let gl_context = GlContext::new()?;
879        gls::load_with(|s| {
880            gl_context
881                .egl
882                .get_proc_address(s)
883                .map_or(std::ptr::null(), |p| p as *const _)
884        });
885
886        Self::gl_check_support()?;
887
888        // Uploads and downloads are all packed with no alignment requirements
889        unsafe {
890            gls::gl::PixelStorei(gls::gl::PACK_ALIGNMENT, 1);
891            gls::gl::PixelStorei(gls::gl::UNPACK_ALIGNMENT, 1);
892        }
893
894        let texture_program_planar =
895            GlProgram::new(generate_vertex_shader(), generate_planar_rgb_shader())?;
896
897        let texture_program =
898            GlProgram::new(generate_vertex_shader(), generate_texture_fragment_shader())?;
899
900        let texture_program_yuv = GlProgram::new(
901            generate_vertex_shader(),
902            generate_texture_fragment_shader_yuv(),
903        )?;
904
905        #[cfg(feature = "decoder")]
906        let segmentation_program =
907            GlProgram::new(generate_vertex_shader(), generate_segmentation_shader())?;
908        #[cfg(feature = "decoder")]
909        segmentation_program.load_uniform_4fv(c"colors", &DEFAULT_COLORS)?;
910        #[cfg(feature = "decoder")]
911        let instanced_segmentation_program = GlProgram::new(
912            generate_vertex_shader(),
913            generate_instanced_segmentation_shader(),
914        )?;
915        #[cfg(feature = "decoder")]
916        instanced_segmentation_program.load_uniform_4fv(c"colors", &DEFAULT_COLORS)?;
917
918        #[cfg(feature = "decoder")]
919        let color_program = GlProgram::new(generate_vertex_shader(), generate_color_shader())?;
920        #[cfg(feature = "decoder")]
921        color_program.load_uniform_4fv(c"colors", &DEFAULT_COLORS)?;
922
923        let camera_eglimage_texture = Texture::new();
924        let camera_normal_texture = Texture::new();
925        let render_texture = Texture::new();
926        let segmentation_texture = Texture::new();
927        let vertex_buffer = Buffer::new(0, 3, 100);
928        let texture_buffer = Buffer::new(1, 2, 100);
929
930        let converter = GLProcessorST {
931            gl_context,
932            texture_program,
933            texture_program_yuv,
934            texture_program_planar,
935            camera_eglimage_texture,
936            camera_normal_texture,
937            #[cfg(feature = "decoder")]
938            segmentation_texture,
939            vertex_buffer,
940            texture_buffer,
941            render_texture,
942            #[cfg(feature = "decoder")]
943            segmentation_program,
944            #[cfg(feature = "decoder")]
945            instanced_segmentation_program,
946            #[cfg(feature = "decoder")]
947            color_program,
948        };
949        check_gl_error(function!(), line!())?;
950
951        log::debug!("GLConverter created");
952        Ok(converter)
953    }
954
955    fn check_src_format_supported(support_dma: bool, img: &TensorImage) -> bool {
956        if support_dma && img.tensor().memory() == TensorMemory::Dma {
957            // EGLImage supports RGBA, GREY, YUYV, and NV12 for DMA buffers
958            matches!(img.fourcc(), RGBA | GREY | YUYV | NV12)
959        } else {
960            matches!(img.fourcc(), RGB | RGBA | GREY)
961        }
962    }
963
964    fn check_dst_format_supported(support_dma: bool, img: &TensorImage) -> bool {
965        if support_dma && img.tensor().memory() == TensorMemory::Dma {
966            // generally EGLImage doesn't support RGB
967            matches!(img.fourcc(), RGBA | GREY | PLANAR_RGB)
968        } else {
969            matches!(img.fourcc(), RGB | RGBA | GREY)
970        }
971    }
972
973    fn gl_check_support() -> Result<(), crate::Error> {
974        if let Ok(version) = gls::get_string(gls::gl::SHADING_LANGUAGE_VERSION) {
975            log::debug!("GL Shading Language Version: {version:?}");
976        } else {
977            log::warn!("Could not get GL Shading Language Version");
978        }
979
980        let extensions = unsafe {
981            let str = gls::gl::GetString(gls::gl::EXTENSIONS);
982            if str.is_null() {
983                return Err(crate::Error::GLVersion(
984                    "GL returned no supported extensions".to_string(),
985                ));
986            }
987            CStr::from_ptr(str as *const c_char)
988                .to_string_lossy()
989                .to_string()
990        };
991        log::debug!("GL Extensions: {extensions}");
992        let required_ext = [
993            "GL_OES_EGL_image_external_essl3",
994            "GL_OES_surfaceless_context",
995        ];
996        let extensions = extensions.split_ascii_whitespace().collect::<BTreeSet<_>>();
997        for required in required_ext {
998            if !extensions.contains(required) {
999                return Err(crate::Error::GLVersion(format!(
1000                    "GL does not support {required} extension",
1001                )));
1002            }
1003        }
1004
1005        Ok(())
1006    }
1007
1008    fn setup_renderbuffer_dma(&mut self, dst: &TensorImage) -> crate::Result<FrameBuffer> {
1009        let frame_buffer = FrameBuffer::new();
1010        frame_buffer.bind();
1011
1012        let (width, height) = if matches!(dst.fourcc(), PLANAR_RGB) {
1013            let width = dst.width();
1014            let height = dst.height() * 3;
1015            (width as i32, height as i32)
1016        } else {
1017            (dst.width() as i32, dst.height() as i32)
1018        };
1019        let dest_img = self.create_image_from_dma2(dst)?;
1020        unsafe {
1021            gls::gl::UseProgram(self.texture_program_yuv.id);
1022            gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1023            gls::gl::BindTexture(gls::gl::TEXTURE_2D, self.render_texture.id);
1024            gls::gl::TexParameteri(
1025                gls::gl::TEXTURE_2D,
1026                gls::gl::TEXTURE_MIN_FILTER,
1027                gls::gl::LINEAR as i32,
1028            );
1029            gls::gl::TexParameteri(
1030                gls::gl::TEXTURE_2D,
1031                gls::gl::TEXTURE_MAG_FILTER,
1032                gls::gl::LINEAR as i32,
1033            );
1034            gls::gl::EGLImageTargetTexture2DOES(gls::gl::TEXTURE_2D, dest_img.egl_image.as_ptr());
1035            gls::gl::FramebufferTexture2D(
1036                gls::gl::FRAMEBUFFER,
1037                gls::gl::COLOR_ATTACHMENT0,
1038                gls::gl::TEXTURE_2D,
1039                self.render_texture.id,
1040                0,
1041            );
1042            check_gl_error(function!(), line!())?;
1043            gls::gl::Viewport(0, 0, width, height);
1044        }
1045        Ok(frame_buffer)
1046    }
1047
1048    fn convert_dest_dma(
1049        &mut self,
1050        dst: &mut TensorImage,
1051        src: &TensorImage,
1052        rotation: crate::Rotation,
1053        flip: Flip,
1054        crop: Crop,
1055    ) -> crate::Result<()> {
1056        assert!(self.gl_context.support_dma);
1057        let _framebuffer = self.setup_renderbuffer_dma(dst)?;
1058        if dst.is_planar() {
1059            self.convert_to_planar(src, dst, rotation, flip, crop)
1060        } else {
1061            self.convert_to(src, dst, rotation, flip, crop)
1062        }
1063    }
1064
1065    fn setup_renderbuffer_non_dma(
1066        &mut self,
1067        dst: &TensorImage,
1068        crop: Crop,
1069    ) -> crate::Result<FrameBuffer> {
1070        debug_assert!(matches!(dst.fourcc(), RGB | RGBA | GREY | PLANAR_RGB));
1071        let (width, height) = if dst.is_planar() {
1072            let width = dst.width() / 4;
1073            let height = match dst.fourcc() {
1074                RGBA => dst.height() * 4,
1075                RGB => dst.height() * 3,
1076                GREY => dst.height(),
1077                _ => unreachable!(),
1078            };
1079            (width as i32, height as i32)
1080        } else {
1081            (dst.width() as i32, dst.height() as i32)
1082        };
1083
1084        let format = if dst.is_planar() {
1085            gls::gl::RED
1086        } else {
1087            match dst.fourcc() {
1088                RGB => gls::gl::RGB,
1089                RGBA => gls::gl::RGBA,
1090                GREY => gls::gl::RED,
1091                _ => unreachable!(),
1092            }
1093        };
1094
1095        let start = Instant::now();
1096        let frame_buffer = FrameBuffer::new();
1097        frame_buffer.bind();
1098
1099        let map;
1100
1101        let pixels = if crop.dst_rect.is_none_or(|crop| {
1102            crop.top == 0
1103                && crop.left == 0
1104                && crop.height == dst.height()
1105                && crop.width == dst.width()
1106        }) {
1107            std::ptr::null()
1108        } else {
1109            map = dst.tensor().map()?;
1110            map.as_ptr() as *const c_void
1111        };
1112        unsafe {
1113            gls::gl::UseProgram(self.texture_program.id);
1114            gls::gl::BindTexture(gls::gl::TEXTURE_2D, self.render_texture.id);
1115            gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1116            gls::gl::TexParameteri(
1117                gls::gl::TEXTURE_2D,
1118                gls::gl::TEXTURE_MIN_FILTER,
1119                gls::gl::LINEAR as i32,
1120            );
1121            gls::gl::TexParameteri(
1122                gls::gl::TEXTURE_2D,
1123                gls::gl::TEXTURE_MAG_FILTER,
1124                gls::gl::LINEAR as i32,
1125            );
1126
1127            gls::gl::TexImage2D(
1128                gls::gl::TEXTURE_2D,
1129                0,
1130                format as i32,
1131                width,
1132                height,
1133                0,
1134                format,
1135                gls::gl::UNSIGNED_BYTE,
1136                pixels,
1137            );
1138            check_gl_error(function!(), line!())?;
1139            gls::gl::FramebufferTexture2D(
1140                gls::gl::FRAMEBUFFER,
1141                gls::gl::COLOR_ATTACHMENT0,
1142                gls::gl::TEXTURE_2D,
1143                self.render_texture.id,
1144                0,
1145            );
1146            check_gl_error(function!(), line!())?;
1147            gls::gl::Viewport(0, 0, width, height);
1148        }
1149        log::debug!("Set up framebuffer takes {:?}", start.elapsed());
1150        Ok(frame_buffer)
1151    }
1152
1153    fn convert_dest_non_dma(
1154        &mut self,
1155        dst: &mut TensorImage,
1156        src: &TensorImage,
1157        rotation: crate::Rotation,
1158        flip: Flip,
1159        crop: Crop,
1160    ) -> crate::Result<()> {
1161        let _framebuffer = self.setup_renderbuffer_non_dma(dst, crop)?;
1162        let start = Instant::now();
1163        if dst.is_planar() {
1164            self.convert_to_planar(src, dst, rotation, flip, crop)?;
1165        } else {
1166            self.convert_to(src, dst, rotation, flip, crop)?;
1167        }
1168        log::debug!("Draw to framebuffer takes {:?}", start.elapsed());
1169        let start = Instant::now();
1170        let dest_format = match dst.fourcc() {
1171            RGB => gls::gl::RGB,
1172            RGBA => gls::gl::RGBA,
1173            GREY => gls::gl::RED,
1174            _ => unreachable!(),
1175        };
1176
1177        unsafe {
1178            let mut dst_map = dst.tensor().map()?;
1179            gls::gl::ReadBuffer(gls::gl::COLOR_ATTACHMENT0);
1180            gls::gl::ReadnPixels(
1181                0,
1182                0,
1183                dst.width() as i32,
1184                dst.height() as i32,
1185                dest_format,
1186                gls::gl::UNSIGNED_BYTE,
1187                dst.tensor.len() as i32,
1188                dst_map.as_mut_ptr() as *mut c_void,
1189            );
1190        }
1191        log::debug!("Read from framebuffer takes {:?}", start.elapsed());
1192        Ok(())
1193    }
1194
1195    fn convert_to(
1196        &mut self,
1197        src: &TensorImage,
1198        dst: &TensorImage,
1199        rotation: crate::Rotation,
1200        flip: Flip,
1201        crop: Crop,
1202    ) -> Result<(), crate::Error> {
1203        check_gl_error(function!(), line!())?;
1204
1205        let has_crop = crop.dst_rect.is_some_and(|x| {
1206            x.left != 0 || x.top != 0 || x.width != dst.width() || x.height != dst.height()
1207        });
1208        if has_crop {
1209            if let Some(dst_color) = crop.dst_color {
1210                unsafe {
1211                    gls::gl::ClearColor(
1212                        dst_color[0] as f32 / 255.0,
1213                        dst_color[1] as f32 / 255.0,
1214                        dst_color[2] as f32 / 255.0,
1215                        dst_color[3] as f32 / 255.0,
1216                    );
1217                    gls::gl::Clear(gls::gl::COLOR_BUFFER_BIT);
1218                };
1219            }
1220        }
1221
1222        // top and bottom are flipped because OpenGL uses 0,0 as bottom left
1223        let src_roi = if let Some(crop) = crop.src_rect {
1224            RegionOfInterest {
1225                left: crop.left as f32 / src.width() as f32,
1226                top: (crop.top + crop.height) as f32 / src.height() as f32,
1227                right: (crop.left + crop.width) as f32 / src.width() as f32,
1228                bottom: crop.top as f32 / src.height() as f32,
1229            }
1230        } else {
1231            RegionOfInterest {
1232                left: 0.,
1233                top: 1.,
1234                right: 1.,
1235                bottom: 0.,
1236            }
1237        };
1238
1239        // top and bottom are flipped because OpenGL uses 0,0 as bottom left
1240        let cvt_screen_coord = |normalized| normalized * 2.0 - 1.0;
1241        let dst_roi = if let Some(crop) = crop.dst_rect {
1242            RegionOfInterest {
1243                left: cvt_screen_coord(crop.left as f32 / dst.width() as f32),
1244                top: cvt_screen_coord((crop.top + crop.height) as f32 / dst.height() as f32),
1245                right: cvt_screen_coord((crop.left + crop.width) as f32 / dst.width() as f32),
1246                bottom: cvt_screen_coord(crop.top as f32 / dst.height() as f32),
1247            }
1248        } else {
1249            RegionOfInterest {
1250                left: -1.,
1251                top: 1.,
1252                right: 1.,
1253                bottom: -1.,
1254            }
1255        };
1256        let rotation_offset = match rotation {
1257            crate::Rotation::None => 0,
1258            crate::Rotation::Clockwise90 => 1,
1259            crate::Rotation::Rotate180 => 2,
1260            crate::Rotation::CounterClockwise90 => 3,
1261        };
1262        if self.gl_context.support_dma {
1263            match self.create_image_from_dma2(src) {
1264                Ok(new_egl_image) => self.draw_camera_texture_eglimage(
1265                    src,
1266                    &new_egl_image,
1267                    src_roi,
1268                    dst_roi,
1269                    rotation_offset,
1270                    flip,
1271                )?,
1272                Err(e) => {
1273                    log::warn!("EGL image creation failed for {:?}: {:?}", src.fourcc(), e);
1274                    let start = Instant::now();
1275                    self.draw_src_texture(src, src_roi, dst_roi, rotation_offset, flip)?;
1276                    log::debug!("draw_src_texture takes {:?}", start.elapsed());
1277                }
1278            }
1279        } else {
1280            let start = Instant::now();
1281            self.draw_src_texture(src, src_roi, dst_roi, rotation_offset, flip)?;
1282            log::debug!("draw_src_texture takes {:?}", start.elapsed());
1283        }
1284
1285        let start = Instant::now();
1286        unsafe { gls::gl::Finish() };
1287        log::debug!("gl_Finish takes {:?}", start.elapsed());
1288        check_gl_error(function!(), line!())?;
1289        Ok(())
1290    }
1291
1292    fn convert_to_planar(
1293        &self,
1294        src: &TensorImage,
1295        dst: &TensorImage,
1296        rotation: crate::Rotation,
1297        flip: Flip,
1298        crop: Crop,
1299    ) -> Result<(), crate::Error> {
1300        // if let Some(crop) = crop.src_rect
1301        //     && (crop.left > 0
1302        //         || crop.top > 0
1303        //         || crop.height < src.height()
1304        //         || crop.width < src.width())
1305        // {
1306        //     return Err(crate::Error::NotSupported(
1307        //         "Cropping in planar RGB mode is not supported".to_string(),
1308        //     ));
1309        // }
1310
1311        // if let Some(crop) = crop.dst_rect
1312        //     && (crop.left > 0
1313        //         || crop.top > 0
1314        //         || crop.height < src.height()
1315        //         || crop.width < src.width())
1316        // {
1317        //     return Err(crate::Error::NotSupported(
1318        //         "Cropping in planar RGB mode is not supported".to_string(),
1319        //     ));
1320        // }
1321
1322        let alpha = match dst.fourcc() {
1323            PLANAR_RGB => false,
1324            PLANAR_RGBA => true,
1325            _ => {
1326                return Err(crate::Error::NotSupported(
1327                    "Destination format must be PLANAR_RGB or PLANAR_RGBA".to_string(),
1328                ));
1329            }
1330        };
1331
1332        // top and bottom are flipped because OpenGL uses 0,0 as bottom left
1333        let src_roi = if let Some(crop) = crop.src_rect {
1334            RegionOfInterest {
1335                left: crop.left as f32 / src.width() as f32,
1336                top: (crop.top + crop.height) as f32 / src.height() as f32,
1337                right: (crop.left + crop.width) as f32 / src.width() as f32,
1338                bottom: crop.top as f32 / src.height() as f32,
1339            }
1340        } else {
1341            RegionOfInterest {
1342                left: 0.,
1343                top: 1.,
1344                right: 1.,
1345                bottom: 0.,
1346            }
1347        };
1348
1349        // top and bottom are flipped because OpenGL uses 0,0 as bottom left
1350        let cvt_screen_coord = |normalized| normalized * 2.0 - 1.0;
1351        let dst_roi = if let Some(crop) = crop.dst_rect {
1352            RegionOfInterest {
1353                left: cvt_screen_coord(crop.left as f32 / dst.width() as f32),
1354                top: cvt_screen_coord((crop.top + crop.height) as f32 / dst.height() as f32),
1355                right: cvt_screen_coord((crop.left + crop.width) as f32 / dst.width() as f32),
1356                bottom: cvt_screen_coord(crop.top as f32 / dst.height() as f32),
1357            }
1358        } else {
1359            RegionOfInterest {
1360                left: -1.,
1361                top: 1.,
1362                right: 1.,
1363                bottom: -1.,
1364            }
1365        };
1366        let rotation_offset = match rotation {
1367            crate::Rotation::None => 0,
1368            crate::Rotation::Clockwise90 => 1,
1369            crate::Rotation::Rotate180 => 2,
1370            crate::Rotation::CounterClockwise90 => 3,
1371        };
1372
1373        let has_crop = crop.dst_rect.is_some_and(|x| {
1374            x.left != 0 || x.top != 0 || x.width != dst.width() || x.height != dst.height()
1375        });
1376        if has_crop {
1377            if let Some(dst_color) = crop.dst_color {
1378                self.clear_rect_planar(
1379                    dst.width(),
1380                    dst.height(),
1381                    dst_roi,
1382                    [
1383                        dst_color[0] as f32 / 255.0,
1384                        dst_color[1] as f32 / 255.0,
1385                        dst_color[2] as f32 / 255.0,
1386                        dst_color[3] as f32 / 255.0,
1387                    ],
1388                    alpha,
1389                )?;
1390            }
1391        }
1392
1393        let new_egl_image = self.create_image_from_dma2(src)?;
1394
1395        self.draw_camera_texture_to_rgb_planar(
1396            &new_egl_image,
1397            src_roi,
1398            dst_roi,
1399            rotation_offset,
1400            flip,
1401            alpha,
1402        )?;
1403        unsafe { gls::gl::Finish() };
1404
1405        Ok(())
1406    }
1407
1408    fn clear_rect_planar(
1409        &self,
1410        width: usize,
1411        height: usize,
1412        dst_roi: RegionOfInterest,
1413        color: [f32; 4],
1414        alpha: bool,
1415    ) -> Result<(), Error> {
1416        if !alpha && color[0] == color[1] && color[1] == color[2] {
1417            unsafe {
1418                gls::gl::ClearColor(color[0], color[0], color[0], 1.0);
1419                gls::gl::Clear(gls::gl::COLOR_BUFFER_BIT);
1420            };
1421        }
1422
1423        let split = if alpha { 4 } else { 3 };
1424
1425        unsafe {
1426            gls::gl::Enable(gls::gl::SCISSOR_TEST);
1427            let x = (((dst_roi.left + 1.0) / 2.0) * width as f32).round() as i32;
1428            let y = (((dst_roi.bottom + 1.0) / 2.0) * height as f32).round() as i32;
1429            let width = (((dst_roi.right - dst_roi.left) / 2.0) * width as f32).round() as i32;
1430            let height = (((dst_roi.top - dst_roi.bottom) / 2.0) * height as f32 / split as f32)
1431                .round() as i32;
1432            for (i, c) in color.iter().enumerate().take(split) {
1433                gls::gl::Scissor(x, y + i as i32 * height, width, height);
1434                gls::gl::ClearColor(*c, *c, *c, 1.0);
1435                gls::gl::Clear(gls::gl::COLOR_BUFFER_BIT);
1436            }
1437            gls::gl::Disable(gls::gl::SCISSOR_TEST);
1438        }
1439        Ok(())
1440    }
1441
1442    #[allow(clippy::too_many_arguments)]
1443    fn draw_camera_texture_to_rgb_planar(
1444        &self,
1445        egl_img: &EglImage,
1446        src_roi: RegionOfInterest,
1447        mut dst_roi: RegionOfInterest,
1448        rotation_offset: usize,
1449        flip: Flip,
1450        alpha: bool,
1451    ) -> Result<(), Error> {
1452        let texture_target = gls::gl::TEXTURE_EXTERNAL_OES;
1453        match flip {
1454            Flip::None => {}
1455            Flip::Vertical => {
1456                std::mem::swap(&mut dst_roi.top, &mut dst_roi.bottom);
1457            }
1458            Flip::Horizontal => {
1459                std::mem::swap(&mut dst_roi.left, &mut dst_roi.right);
1460            }
1461        }
1462        unsafe {
1463            // self.texture_program.load_uniform_1f(c"width", width as f32);
1464            gls::gl::UseProgram(self.texture_program_planar.id);
1465            gls::gl::BindTexture(texture_target, self.camera_eglimage_texture.id);
1466            gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1467            gls::gl::TexParameteri(
1468                texture_target,
1469                gls::gl::TEXTURE_MIN_FILTER,
1470                gls::gl::LINEAR as i32,
1471            );
1472            gls::gl::TexParameteri(
1473                texture_target,
1474                gls::gl::TEXTURE_MAG_FILTER,
1475                gls::gl::LINEAR as i32,
1476            );
1477            gls::gl::TexParameteri(
1478                texture_target,
1479                gls::gl::TEXTURE_WRAP_S,
1480                gls::gl::CLAMP_TO_EDGE as i32,
1481            );
1482
1483            gls::gl::TexParameteri(
1484                texture_target,
1485                gls::gl::TEXTURE_WRAP_T,
1486                gls::gl::CLAMP_TO_EDGE as i32,
1487            );
1488
1489            gls::egl_image_target_texture_2d_oes(texture_target, egl_img.egl_image.as_ptr());
1490            check_gl_error(function!(), line!())?;
1491            let y_centers = if alpha {
1492                vec![-3.0 / 4.0, -1.0 / 4.0, 1.0 / 4.0, 3.0 / 4.0]
1493            } else {
1494                vec![-2.0 / 3.0, 0.0, 2.0 / 3.0]
1495            };
1496            let swizzles = [gls::gl::RED, gls::gl::GREEN, gls::gl::BLUE, gls::gl::ALPHA];
1497            // starts from bottom
1498            for (i, y_center) in y_centers.iter().enumerate() {
1499                gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
1500                gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
1501                let camera_vertices: [f32; 12] = [
1502                    dst_roi.left,
1503                    dst_roi.top / 3.0 + y_center,
1504                    0., // left top
1505                    dst_roi.right,
1506                    dst_roi.top / 3.0 + y_center,
1507                    0., // right top
1508                    dst_roi.right,
1509                    dst_roi.bottom / 3.0 + y_center,
1510                    0., // right bottom
1511                    dst_roi.left,
1512                    dst_roi.bottom / 3.0 + y_center,
1513                    0., // left bottom
1514                ];
1515                gls::gl::BufferData(
1516                    gls::gl::ARRAY_BUFFER,
1517                    (size_of::<f32>() * camera_vertices.len()) as isize,
1518                    camera_vertices.as_ptr() as *const c_void,
1519                    gls::gl::DYNAMIC_DRAW,
1520                );
1521
1522                gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
1523                gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
1524                let texture_vertices: [f32; 16] = [
1525                    src_roi.left,
1526                    src_roi.top,
1527                    src_roi.right,
1528                    src_roi.top,
1529                    src_roi.right,
1530                    src_roi.bottom,
1531                    src_roi.left,
1532                    src_roi.bottom,
1533                    src_roi.left,
1534                    src_roi.top,
1535                    src_roi.right,
1536                    src_roi.top,
1537                    src_roi.right,
1538                    src_roi.bottom,
1539                    src_roi.left,
1540                    src_roi.bottom,
1541                ];
1542
1543                gls::gl::BufferData(
1544                    gls::gl::ARRAY_BUFFER,
1545                    (size_of::<f32>() * 8) as isize,
1546                    (texture_vertices[(rotation_offset * 2)..]).as_ptr() as *const c_void,
1547                    gls::gl::DYNAMIC_DRAW,
1548                );
1549                let vertices_index: [u32; 4] = [0, 1, 2, 3];
1550                // self.texture_program_planar
1551                //     .load_uniform_1i(c"color_index", 2 - i as i32);
1552
1553                gls::gl::TexParameteri(
1554                    texture_target,
1555                    gls::gl::TEXTURE_SWIZZLE_R,
1556                    swizzles[i] as i32,
1557                );
1558
1559                gls::gl::DrawElements(
1560                    gls::gl::TRIANGLE_FAN,
1561                    vertices_index.len() as i32,
1562                    gls::gl::UNSIGNED_INT,
1563                    vertices_index.as_ptr() as *const c_void,
1564                );
1565            }
1566            check_gl_error(function!(), line!())?;
1567        }
1568        Ok(())
1569    }
1570
1571    fn draw_src_texture(
1572        &mut self,
1573        src: &TensorImage,
1574        src_roi: RegionOfInterest,
1575        mut dst_roi: RegionOfInterest,
1576        rotation_offset: usize,
1577        flip: Flip,
1578    ) -> Result<(), Error> {
1579        let texture_target = gls::gl::TEXTURE_2D;
1580        let texture_format = match src.fourcc() {
1581            RGB => gls::gl::RGB,
1582            RGBA => gls::gl::RGBA,
1583            GREY => gls::gl::RED,
1584            _ => unreachable!(),
1585        };
1586        unsafe {
1587            gls::gl::UseProgram(self.texture_program.id);
1588            gls::gl::BindTexture(texture_target, self.camera_normal_texture.id);
1589            gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1590            gls::gl::TexParameteri(
1591                texture_target,
1592                gls::gl::TEXTURE_MIN_FILTER,
1593                gls::gl::LINEAR as i32,
1594            );
1595            gls::gl::TexParameteri(
1596                texture_target,
1597                gls::gl::TEXTURE_MAG_FILTER,
1598                gls::gl::LINEAR as i32,
1599            );
1600            if src.fourcc() == GREY {
1601                for swizzle in [
1602                    gls::gl::TEXTURE_SWIZZLE_R,
1603                    gls::gl::TEXTURE_SWIZZLE_G,
1604                    gls::gl::TEXTURE_SWIZZLE_B,
1605                ] {
1606                    gls::gl::TexParameteri(gls::gl::TEXTURE_2D, swizzle, gls::gl::RED as i32);
1607                }
1608            } else {
1609                for (swizzle, src) in [
1610                    (gls::gl::TEXTURE_SWIZZLE_R, gls::gl::RED),
1611                    (gls::gl::TEXTURE_SWIZZLE_G, gls::gl::GREEN),
1612                    (gls::gl::TEXTURE_SWIZZLE_B, gls::gl::BLUE),
1613                ] {
1614                    gls::gl::TexParameteri(gls::gl::TEXTURE_2D, swizzle, src as i32);
1615                }
1616            }
1617            self.camera_normal_texture.update_texture(
1618                texture_target,
1619                src.width(),
1620                src.height(),
1621                texture_format,
1622                &src.tensor().map()?,
1623            );
1624
1625            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
1626            gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
1627
1628            match flip {
1629                Flip::None => {}
1630                Flip::Vertical => {
1631                    std::mem::swap(&mut dst_roi.top, &mut dst_roi.bottom);
1632                }
1633                Flip::Horizontal => {
1634                    std::mem::swap(&mut dst_roi.left, &mut dst_roi.right);
1635                }
1636            }
1637
1638            let camera_vertices: [f32; 12] = [
1639                dst_roi.left,
1640                dst_roi.top,
1641                0., // left top
1642                dst_roi.right,
1643                dst_roi.top,
1644                0., // right top
1645                dst_roi.right,
1646                dst_roi.bottom,
1647                0., // right bottom
1648                dst_roi.left,
1649                dst_roi.bottom,
1650                0., // left bottom
1651            ];
1652            gls::gl::BufferData(
1653                gls::gl::ARRAY_BUFFER,
1654                (size_of::<f32>() * camera_vertices.len()) as isize,
1655                camera_vertices.as_ptr() as *const c_void,
1656                gls::gl::DYNAMIC_DRAW,
1657            );
1658            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
1659            gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
1660            let texture_vertices: [f32; 16] = [
1661                src_roi.left,
1662                src_roi.top,
1663                src_roi.right,
1664                src_roi.top,
1665                src_roi.right,
1666                src_roi.bottom,
1667                src_roi.left,
1668                src_roi.bottom,
1669                src_roi.left,
1670                src_roi.top,
1671                src_roi.right,
1672                src_roi.top,
1673                src_roi.right,
1674                src_roi.bottom,
1675                src_roi.left,
1676                src_roi.bottom,
1677            ];
1678
1679            gls::gl::BufferData(
1680                gls::gl::ARRAY_BUFFER,
1681                (size_of::<f32>() * 8) as isize,
1682                (texture_vertices[(rotation_offset * 2)..]).as_ptr() as *const c_void,
1683                gls::gl::DYNAMIC_DRAW,
1684            );
1685            let vertices_index: [u32; 4] = [0, 1, 2, 3];
1686            gls::gl::DrawElements(
1687                gls::gl::TRIANGLE_FAN,
1688                vertices_index.len() as i32,
1689                gls::gl::UNSIGNED_INT,
1690                vertices_index.as_ptr() as *const c_void,
1691            );
1692            check_gl_error(function!(), line!())?;
1693
1694            Ok(())
1695        }
1696    }
1697
1698    fn draw_camera_texture_eglimage(
1699        &self,
1700        src: &TensorImage,
1701        egl_img: &EglImage,
1702        src_roi: RegionOfInterest,
1703        mut dst_roi: RegionOfInterest,
1704        rotation_offset: usize,
1705        flip: Flip,
1706    ) -> Result<(), Error> {
1707        // let texture_target = gls::gl::TEXTURE_2D;
1708        let texture_target = gls::gl::TEXTURE_EXTERNAL_OES;
1709        unsafe {
1710            gls::gl::UseProgram(self.texture_program_yuv.id);
1711            gls::gl::BindTexture(texture_target, self.camera_eglimage_texture.id);
1712            gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1713            gls::gl::TexParameteri(
1714                texture_target,
1715                gls::gl::TEXTURE_MIN_FILTER,
1716                gls::gl::LINEAR as i32,
1717            );
1718            gls::gl::TexParameteri(
1719                texture_target,
1720                gls::gl::TEXTURE_MAG_FILTER,
1721                gls::gl::LINEAR as i32,
1722            );
1723
1724            if src.fourcc() == GREY {
1725                for swizzle in [
1726                    gls::gl::TEXTURE_SWIZZLE_R,
1727                    gls::gl::TEXTURE_SWIZZLE_G,
1728                    gls::gl::TEXTURE_SWIZZLE_B,
1729                ] {
1730                    gls::gl::TexParameteri(gls::gl::TEXTURE_2D, swizzle, gls::gl::RED as i32);
1731                }
1732            } else {
1733                for (swizzle, src) in [
1734                    (gls::gl::TEXTURE_SWIZZLE_R, gls::gl::RED),
1735                    (gls::gl::TEXTURE_SWIZZLE_G, gls::gl::GREEN),
1736                    (gls::gl::TEXTURE_SWIZZLE_B, gls::gl::BLUE),
1737                ] {
1738                    gls::gl::TexParameteri(gls::gl::TEXTURE_2D, swizzle, src as i32);
1739                }
1740            }
1741
1742            gls::egl_image_target_texture_2d_oes(texture_target, egl_img.egl_image.as_ptr());
1743            check_gl_error(function!(), line!())?;
1744            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
1745            gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
1746
1747            match flip {
1748                Flip::None => {}
1749                Flip::Vertical => {
1750                    std::mem::swap(&mut dst_roi.top, &mut dst_roi.bottom);
1751                }
1752                Flip::Horizontal => {
1753                    std::mem::swap(&mut dst_roi.left, &mut dst_roi.right);
1754                }
1755            }
1756
1757            let camera_vertices: [f32; 12] = [
1758                dst_roi.left,
1759                dst_roi.top,
1760                0., // left top
1761                dst_roi.right,
1762                dst_roi.top,
1763                0., // right top
1764                dst_roi.right,
1765                dst_roi.bottom,
1766                0., // right bottom
1767                dst_roi.left,
1768                dst_roi.bottom,
1769                0., // left bottom
1770            ];
1771            gls::gl::BufferSubData(
1772                gls::gl::ARRAY_BUFFER,
1773                0,
1774                (size_of::<f32>() * camera_vertices.len()) as isize,
1775                camera_vertices.as_ptr() as *const c_void,
1776            );
1777
1778            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
1779            gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
1780
1781            let texture_vertices: [f32; 16] = [
1782                src_roi.left,
1783                src_roi.top,
1784                src_roi.right,
1785                src_roi.top,
1786                src_roi.right,
1787                src_roi.bottom,
1788                src_roi.left,
1789                src_roi.bottom,
1790                src_roi.left,
1791                src_roi.top,
1792                src_roi.right,
1793                src_roi.top,
1794                src_roi.right,
1795                src_roi.bottom,
1796                src_roi.left,
1797                src_roi.bottom,
1798            ];
1799            gls::gl::BufferSubData(
1800                gls::gl::ARRAY_BUFFER,
1801                0,
1802                (size_of::<f32>() * 8) as isize,
1803                (texture_vertices[(rotation_offset * 2)..]).as_ptr() as *const c_void,
1804            );
1805
1806            let vertices_index: [u32; 4] = [0, 1, 2, 3];
1807            gls::gl::DrawElements(
1808                gls::gl::TRIANGLE_FAN,
1809                vertices_index.len() as i32,
1810                gls::gl::UNSIGNED_INT,
1811                vertices_index.as_ptr() as *const c_void,
1812            );
1813        }
1814        check_gl_error(function!(), line!())?;
1815        Ok(())
1816    }
1817
1818    fn create_image_from_dma2(&self, src: &TensorImage) -> Result<EglImage, crate::Error> {
1819        let width;
1820        let height;
1821        let format;
1822        let channels;
1823
1824        // NV12 is semi-planar but handled specially via EGL multi-plane import
1825        if src.fourcc() == NV12 {
1826            if !src.width().is_multiple_of(4) {
1827                return Err(Error::NotSupported(
1828                    "OpenGL EGLImage doesn't support image widths which are not multiples of 4"
1829                        .to_string(),
1830                ));
1831            }
1832            width = src.width();
1833            height = src.height();
1834            format = fourcc_to_drm(NV12);
1835            channels = 1; // Y plane pitch is 1 byte per pixel
1836        } else if src.is_planar() {
1837            if !src.width().is_multiple_of(16) {
1838                return Err(Error::NotSupported(
1839                    "OpenGL Planar RGB EGLImage doesn't support image widths which are not multiples of 16"
1840                        .to_string(),
1841                ));
1842            }
1843            match src.fourcc() {
1844                PLANAR_RGB => {
1845                    format = DrmFourcc::R8;
1846                    width = src.width();
1847                    height = src.height() * 3;
1848                    channels = 1;
1849                }
1850                fourcc => {
1851                    return Err(crate::Error::NotSupported(format!(
1852                        "Unsupported Planar FourCC {fourcc:?}"
1853                    )));
1854                }
1855            };
1856        } else {
1857            if !src.width().is_multiple_of(4) {
1858                return Err(Error::NotSupported(
1859                    "OpenGL EGLImage doesn't support image widths which are not multiples of 4"
1860                        .to_string(),
1861                ));
1862            }
1863            width = src.width();
1864            height = src.height();
1865            format = fourcc_to_drm(src.fourcc());
1866            channels = src.channels();
1867        }
1868
1869        let fd = match &src.tensor {
1870            edgefirst_tensor::Tensor::Dma(dma_tensor) => dma_tensor.fd.as_raw_fd(),
1871            edgefirst_tensor::Tensor::Shm(_) => {
1872                return Err(Error::NotImplemented(
1873                    "OpenGL EGLImage doesn't support SHM".to_string(),
1874                ));
1875            }
1876            edgefirst_tensor::Tensor::Mem(_) => {
1877                return Err(Error::NotImplemented(
1878                    "OpenGL EGLImage doesn't support MEM".to_string(),
1879                ));
1880            }
1881        };
1882
1883        // For NV12, plane0 pitch is width (Y is 1 byte/pixel)
1884        // For other formats, pitch is width * channels
1885        let plane0_pitch = if src.fourcc() == NV12 {
1886            width
1887        } else {
1888            width * channels
1889        };
1890
1891        let mut egl_img_attr = vec![
1892            egl_ext::LINUX_DRM_FOURCC as Attrib,
1893            format as Attrib,
1894            khronos_egl::WIDTH as Attrib,
1895            width as Attrib,
1896            khronos_egl::HEIGHT as Attrib,
1897            height as Attrib,
1898            egl_ext::DMA_BUF_PLANE0_PITCH as Attrib,
1899            plane0_pitch as Attrib,
1900            egl_ext::DMA_BUF_PLANE0_OFFSET as Attrib,
1901            0 as Attrib,
1902            egl_ext::DMA_BUF_PLANE0_FD as Attrib,
1903            fd as Attrib,
1904            egl::IMAGE_PRESERVED as Attrib,
1905            egl::TRUE as Attrib,
1906        ];
1907
1908        // NV12 requires a second plane for UV data
1909        if src.fourcc() == NV12 {
1910            let uv_offset = width * height; // Y plane size
1911            egl_img_attr.append(&mut vec![
1912                egl_ext::DMA_BUF_PLANE1_FD as Attrib,
1913                fd as Attrib,
1914                egl_ext::DMA_BUF_PLANE1_OFFSET as Attrib,
1915                uv_offset as Attrib,
1916                egl_ext::DMA_BUF_PLANE1_PITCH as Attrib,
1917                width as Attrib, // UV plane has same width as Y plane
1918            ]);
1919        }
1920
1921        if matches!(src.fourcc(), YUYV | NV12) {
1922            egl_img_attr.append(&mut vec![
1923                egl_ext::YUV_COLOR_SPACE_HINT as Attrib,
1924                egl_ext::ITU_REC709 as Attrib,
1925                egl_ext::SAMPLE_RANGE_HINT as Attrib,
1926                egl_ext::YUV_NARROW_RANGE as Attrib,
1927            ]);
1928        }
1929
1930        egl_img_attr.push(khronos_egl::NONE as Attrib);
1931
1932        match self.new_egl_image_owned(egl_ext::LINUX_DMA_BUF, &egl_img_attr) {
1933            Ok(v) => Ok(v),
1934            Err(e) => Err(e),
1935        }
1936    }
1937
1938    fn new_egl_image_owned(
1939        &'_ self,
1940        target: egl::Enum,
1941        attrib_list: &[Attrib],
1942    ) -> Result<EglImage, Error> {
1943        let image = GlContext::egl_create_image_with_fallback(
1944            &self.gl_context.egl,
1945            self.gl_context.display.as_display(),
1946            unsafe { egl::Context::from_ptr(egl::NO_CONTEXT) },
1947            target,
1948            unsafe { egl::ClientBuffer::from_ptr(null_mut()) },
1949            attrib_list,
1950        )?;
1951        Ok(EglImage {
1952            egl_image: image,
1953            display: self.gl_context.display.as_display(),
1954            egl: self.gl_context.egl.clone(),
1955        })
1956    }
1957
1958    // Reshapes the segmentation to be compatible with RGBA texture array rendering.
1959    fn reshape_segmentation_to_rgba(&self, segmentation: &[u8], shape: [usize; 3]) -> Vec<u8> {
1960        let [height, width, classes] = shape;
1961
1962        let n_layer_stride = height * width * 4;
1963        let n_row_stride = width * 4;
1964        let n_col_stride = 4;
1965        let row_stride = width * classes;
1966        let col_stride = classes;
1967
1968        let mut new_segmentation = vec![0u8; n_layer_stride * classes.div_ceil(4)];
1969
1970        for i in 0..height {
1971            for j in 0..width {
1972                for k in 0..classes.div_ceil(4) * 4 {
1973                    if k >= classes {
1974                        new_segmentation[n_layer_stride * (k / 4)
1975                            + i * n_row_stride
1976                            + j * n_col_stride
1977                            + k % 4] = 0;
1978                    } else {
1979                        new_segmentation[n_layer_stride * (k / 4)
1980                            + i * n_row_stride
1981                            + j * n_col_stride
1982                            + k % 4] = segmentation[i * row_stride + j * col_stride + k];
1983                    }
1984                }
1985            }
1986        }
1987
1988        new_segmentation
1989    }
1990
1991    #[cfg(feature = "decoder")]
1992    fn render_modelpack_segmentation(
1993        &mut self,
1994        dst_roi: RegionOfInterest,
1995        segmentation: &[u8],
1996        shape: [usize; 3],
1997    ) -> Result<(), crate::Error> {
1998        log::debug!("start render_segmentation_to_image");
1999
2000        // TODO: Implement specialization for 2 classes and 4 classes which shouldn't
2001        // need rearranging the data
2002        let new_segmentation = self.reshape_segmentation_to_rgba(segmentation, shape);
2003
2004        let [height, width, classes] = shape;
2005
2006        let format = gls::gl::RGBA;
2007        let texture_target = gls::gl::TEXTURE_2D_ARRAY;
2008        self.segmentation_program
2009            .load_uniform_1i(c"background_index", shape[2] as i32 - 1)?;
2010
2011        gls::use_program(self.segmentation_program.id);
2012
2013        gls::bind_texture(texture_target, self.segmentation_texture.id);
2014        gls::active_texture(gls::gl::TEXTURE0);
2015        gls::tex_parameteri(
2016            texture_target,
2017            gls::gl::TEXTURE_MIN_FILTER,
2018            gls::gl::LINEAR as i32,
2019        );
2020        gls::tex_parameteri(
2021            texture_target,
2022            gls::gl::TEXTURE_MAG_FILTER,
2023            gls::gl::LINEAR as i32,
2024        );
2025        gls::tex_parameteri(
2026            texture_target,
2027            gls::gl::TEXTURE_WRAP_S,
2028            gls::gl::CLAMP_TO_EDGE as i32,
2029        );
2030
2031        gls::tex_parameteri(
2032            texture_target,
2033            gls::gl::TEXTURE_WRAP_T,
2034            gls::gl::CLAMP_TO_EDGE as i32,
2035        );
2036
2037        gls::tex_image3d(
2038            texture_target,
2039            0,
2040            format as i32,
2041            width as i32,
2042            height as i32,
2043            classes.div_ceil(4) as i32,
2044            0,
2045            format,
2046            gls::gl::UNSIGNED_BYTE,
2047            Some(&new_segmentation),
2048        );
2049
2050        let src_roi = RegionOfInterest {
2051            left: 0.,
2052            top: 1.,
2053            right: 1.,
2054            bottom: 0.,
2055        };
2056
2057        unsafe {
2058            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
2059            gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
2060
2061            let camera_vertices: [f32; 12] = [
2062                dst_roi.left,
2063                dst_roi.top,
2064                0., // left top
2065                dst_roi.right,
2066                dst_roi.top,
2067                0., // right top
2068                dst_roi.right,
2069                dst_roi.bottom,
2070                0., // right bottom
2071                dst_roi.left,
2072                dst_roi.bottom,
2073                0., // left bottom
2074            ];
2075            gls::gl::BufferSubData(
2076                gls::gl::ARRAY_BUFFER,
2077                0,
2078                (size_of::<f32>() * camera_vertices.len()) as isize,
2079                camera_vertices.as_ptr() as *const c_void,
2080            );
2081
2082            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
2083            gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
2084
2085            let texture_vertices: [f32; 8] = [
2086                src_roi.left,
2087                src_roi.top,
2088                src_roi.right,
2089                src_roi.top,
2090                src_roi.right,
2091                src_roi.bottom,
2092                src_roi.left,
2093                src_roi.bottom,
2094            ];
2095            gls::gl::BufferSubData(
2096                gls::gl::ARRAY_BUFFER,
2097                0,
2098                (size_of::<f32>() * 8) as isize,
2099                (texture_vertices[0..]).as_ptr() as *const c_void,
2100            );
2101
2102            let vertices_index: [u32; 4] = [0, 1, 2, 3];
2103            gls::gl::DrawElements(
2104                gls::gl::TRIANGLE_FAN,
2105                vertices_index.len() as i32,
2106                gls::gl::UNSIGNED_INT,
2107                vertices_index.as_ptr() as *const c_void,
2108            );
2109        }
2110
2111        Ok(())
2112    }
2113
2114    #[cfg(feature = "decoder")]
2115    fn render_yolo_segmentation(
2116        &mut self,
2117        dst_roi: RegionOfInterest,
2118        segmentation: &[u8],
2119        shape: [usize; 2],
2120        class: usize,
2121    ) -> Result<(), crate::Error> {
2122        log::debug!("start render_yolo_segmentation");
2123
2124        let [height, width] = shape;
2125
2126        let format = gls::gl::RED;
2127        let texture_target = gls::gl::TEXTURE_2D;
2128        gls::use_program(self.instanced_segmentation_program.id);
2129        self.instanced_segmentation_program
2130            .load_uniform_1i(c"class_index", class as i32)?;
2131        gls::bind_texture(texture_target, self.segmentation_texture.id);
2132        gls::active_texture(gls::gl::TEXTURE0);
2133        gls::tex_parameteri(
2134            texture_target,
2135            gls::gl::TEXTURE_MIN_FILTER,
2136            gls::gl::LINEAR as i32,
2137        );
2138        gls::tex_parameteri(
2139            texture_target,
2140            gls::gl::TEXTURE_MAG_FILTER,
2141            gls::gl::LINEAR as i32,
2142        );
2143        gls::tex_parameteri(
2144            texture_target,
2145            gls::gl::TEXTURE_WRAP_S,
2146            gls::gl::CLAMP_TO_EDGE as i32,
2147        );
2148
2149        gls::tex_parameteri(
2150            texture_target,
2151            gls::gl::TEXTURE_WRAP_T,
2152            gls::gl::CLAMP_TO_EDGE as i32,
2153        );
2154
2155        gls::tex_image2d(
2156            texture_target,
2157            0,
2158            format as i32,
2159            width as i32,
2160            height as i32,
2161            0,
2162            format,
2163            gls::gl::UNSIGNED_BYTE,
2164            Some(segmentation),
2165        );
2166
2167        let src_roi = RegionOfInterest {
2168            left: 0.,
2169            top: 1.,
2170            right: 1.,
2171            bottom: 0.,
2172        };
2173
2174        unsafe {
2175            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
2176            gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
2177
2178            let camera_vertices: [f32; 12] = [
2179                dst_roi.left,
2180                dst_roi.top,
2181                0., // left top
2182                dst_roi.right,
2183                dst_roi.top,
2184                0., // right top
2185                dst_roi.right,
2186                dst_roi.bottom,
2187                0., // right bottom
2188                dst_roi.left,
2189                dst_roi.bottom,
2190                0., // left bottom
2191            ];
2192            gls::gl::BufferSubData(
2193                gls::gl::ARRAY_BUFFER,
2194                0,
2195                (size_of::<f32>() * camera_vertices.len()) as isize,
2196                camera_vertices.as_ptr() as *const c_void,
2197            );
2198
2199            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
2200            gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
2201
2202            let texture_vertices: [f32; 8] = [
2203                src_roi.left,
2204                src_roi.top,
2205                src_roi.right,
2206                src_roi.top,
2207                src_roi.right,
2208                src_roi.bottom,
2209                src_roi.left,
2210                src_roi.bottom,
2211            ];
2212            gls::gl::BufferSubData(
2213                gls::gl::ARRAY_BUFFER,
2214                0,
2215                (size_of::<f32>() * 8) as isize,
2216                (texture_vertices).as_ptr() as *const c_void,
2217            );
2218
2219            let vertices_index: [u32; 4] = [0, 1, 2, 3];
2220            gls::gl::DrawElements(
2221                gls::gl::TRIANGLE_FAN,
2222                vertices_index.len() as i32,
2223                gls::gl::UNSIGNED_INT,
2224                vertices_index.as_ptr() as *const c_void,
2225            );
2226            gls::gl::Finish();
2227        }
2228
2229        Ok(())
2230    }
2231
2232    fn render_segmentation(
2233        &mut self,
2234        detect: &[DetectBox],
2235        segmentation: &[Segmentation],
2236    ) -> crate::Result<()> {
2237        if segmentation.is_empty() {
2238            return Ok(());
2239        }
2240
2241        let is_modelpack = segmentation[0].segmentation.shape()[2] > 1;
2242        // top and bottom are flipped because OpenGL uses 0,0 as bottom left
2243        let cvt_screen_coord = |normalized| normalized * 2.0 - 1.0;
2244        if is_modelpack {
2245            let seg = &segmentation[0];
2246            let dst_roi = RegionOfInterest {
2247                left: cvt_screen_coord(seg.xmin),
2248                top: cvt_screen_coord(seg.ymax),
2249                right: cvt_screen_coord(seg.xmax),
2250                bottom: cvt_screen_coord(seg.ymin),
2251            };
2252            let segment = seg.segmentation.as_standard_layout();
2253            let slice = segment.as_slice().ok_or(Error::Internal(
2254                "Cannot get slice of segmentation".to_owned(),
2255            ))?;
2256
2257            self.render_modelpack_segmentation(
2258                dst_roi,
2259                slice,
2260                [
2261                    seg.segmentation.shape()[0],
2262                    seg.segmentation.shape()[1],
2263                    seg.segmentation.shape()[2],
2264                ],
2265            )?;
2266        } else {
2267            for (seg, det) in segmentation.iter().zip(detect) {
2268                let dst_roi = RegionOfInterest {
2269                    left: cvt_screen_coord(seg.xmin),
2270                    top: cvt_screen_coord(seg.ymax),
2271                    right: cvt_screen_coord(seg.xmax),
2272                    bottom: cvt_screen_coord(seg.ymin),
2273                };
2274
2275                let segment = seg.segmentation.as_standard_layout();
2276                let slice = segment.as_slice().ok_or(Error::Internal(
2277                    "Cannot get slice of segmentation".to_owned(),
2278                ))?;
2279
2280                self.render_yolo_segmentation(
2281                    dst_roi,
2282                    slice,
2283                    [seg.segmentation.shape()[0], seg.segmentation.shape()[1]],
2284                    det.label,
2285                )?;
2286            }
2287        }
2288
2289        gls::disable(gls::gl::BLEND);
2290        Ok(())
2291    }
2292
2293    fn render_box(&mut self, dst: &TensorImage, detect: &[DetectBox]) -> Result<(), Error> {
2294        unsafe {
2295            gls::gl::UseProgram(self.color_program.id);
2296            let rescale = |x: f32| x * 2.0 - 1.0;
2297            let thickness = 3.0;
2298            for d in detect {
2299                self.color_program
2300                    .load_uniform_1i(c"class_index", d.label as i32)?;
2301                gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
2302                gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
2303                let bbox: [f32; 4] = d.bbox.into();
2304                let outer_box = [
2305                    bbox[0] - thickness / dst.width() as f32,
2306                    bbox[1] - thickness / dst.height() as f32,
2307                    bbox[2] + thickness / dst.width() as f32,
2308                    bbox[3] + thickness / dst.height() as f32,
2309                ];
2310                let camera_vertices: [f32; 24] = [
2311                    rescale(bbox[0]),
2312                    rescale(bbox[3]),
2313                    0., // bottom left
2314                    rescale(bbox[2]),
2315                    rescale(bbox[3]),
2316                    0., // bottom right
2317                    rescale(bbox[2]),
2318                    rescale(bbox[1]),
2319                    0., // top right
2320                    rescale(bbox[0]),
2321                    rescale(bbox[1]),
2322                    0., // top left
2323                    rescale(outer_box[0]),
2324                    rescale(outer_box[3]),
2325                    0., // bottom left
2326                    rescale(outer_box[2]),
2327                    rescale(outer_box[3]),
2328                    0., // bottom right
2329                    rescale(outer_box[2]),
2330                    rescale(outer_box[1]),
2331                    0., // top right
2332                    rescale(outer_box[0]),
2333                    rescale(outer_box[1]),
2334                    0., // top left
2335                ];
2336                gls::gl::BufferData(
2337                    gls::gl::ARRAY_BUFFER,
2338                    (size_of::<f32>() * camera_vertices.len()) as isize,
2339                    camera_vertices.as_ptr() as *const c_void,
2340                    gls::gl::DYNAMIC_DRAW,
2341                );
2342
2343                let vertices_index: [u32; 10] = [0, 1, 5, 2, 6, 3, 7, 0, 4, 5];
2344                gls::gl::DrawElements(
2345                    gls::gl::TRIANGLE_STRIP,
2346                    vertices_index.len() as i32,
2347                    gls::gl::UNSIGNED_INT,
2348                    vertices_index.as_ptr() as *const c_void,
2349                );
2350            }
2351        }
2352        check_gl_error(function!(), line!())?;
2353        Ok(())
2354    }
2355}
2356struct EglImage {
2357    egl_image: egl::Image,
2358    egl: Rc<Egl>,
2359    display: egl::Display,
2360}
2361
2362impl Drop for EglImage {
2363    fn drop(&mut self) {
2364        if self.egl_image.as_ptr() == egl::NO_IMAGE {
2365            return;
2366        }
2367
2368        let e = GlContext::egl_destory_image_with_fallback(&self.egl, self.display, self.egl_image);
2369        if let Err(e) = e {
2370            error!("Could not destroy EGL image: {e:?}");
2371        }
2372    }
2373}
2374
2375struct Texture {
2376    id: u32,
2377    target: gls::gl::types::GLenum,
2378    width: usize,
2379    height: usize,
2380    format: gls::gl::types::GLenum,
2381}
2382
2383impl Default for Texture {
2384    fn default() -> Self {
2385        Self::new()
2386    }
2387}
2388
2389impl Texture {
2390    fn new() -> Self {
2391        let mut id = 0;
2392        unsafe { gls::gl::GenTextures(1, &raw mut id) };
2393        Self {
2394            id,
2395            target: 0,
2396            width: 0,
2397            height: 0,
2398            format: 0,
2399        }
2400    }
2401
2402    fn update_texture(
2403        &mut self,
2404        target: gls::gl::types::GLenum,
2405        width: usize,
2406        height: usize,
2407        format: gls::gl::types::GLenum,
2408        data: &[u8],
2409    ) {
2410        if target != self.target
2411            || width != self.width
2412            || height != self.height
2413            || format != self.format
2414        {
2415            unsafe {
2416                gls::gl::TexImage2D(
2417                    target,
2418                    0,
2419                    format as i32,
2420                    width as i32,
2421                    height as i32,
2422                    0,
2423                    format,
2424                    gls::gl::UNSIGNED_BYTE,
2425                    data.as_ptr() as *const c_void,
2426                );
2427            }
2428            self.target = target;
2429            self.format = format;
2430            self.width = width;
2431            self.height = height;
2432        } else {
2433            unsafe {
2434                gls::gl::TexSubImage2D(
2435                    target,
2436                    0,
2437                    0,
2438                    0,
2439                    width as i32,
2440                    height as i32,
2441                    format,
2442                    gls::gl::UNSIGNED_BYTE,
2443                    data.as_ptr() as *const c_void,
2444                );
2445            }
2446        }
2447    }
2448}
2449
2450impl Drop for Texture {
2451    fn drop(&mut self) {
2452        unsafe { gls::gl::DeleteTextures(1, &raw mut self.id) };
2453    }
2454}
2455
2456struct Buffer {
2457    id: u32,
2458    buffer_index: u32,
2459}
2460
2461impl Buffer {
2462    fn new(buffer_index: u32, size_per_point: usize, max_points: usize) -> Buffer {
2463        let mut id = 0;
2464        unsafe {
2465            gls::gl::EnableVertexAttribArray(buffer_index);
2466            gls::gl::GenBuffers(1, &raw mut id);
2467            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, id);
2468            gls::gl::VertexAttribPointer(
2469                buffer_index,
2470                size_per_point as i32,
2471                gls::gl::FLOAT,
2472                gls::gl::FALSE,
2473                0,
2474                null(),
2475            );
2476            gls::gl::BufferData(
2477                gls::gl::ARRAY_BUFFER,
2478                (size_of::<f32>() * size_per_point * max_points) as isize,
2479                null(),
2480                gls::gl::DYNAMIC_DRAW,
2481            );
2482        }
2483
2484        Buffer { id, buffer_index }
2485    }
2486}
2487
2488impl Drop for Buffer {
2489    fn drop(&mut self) {
2490        unsafe { gls::gl::DeleteBuffers(1, &raw mut self.id) };
2491    }
2492}
2493
2494struct FrameBuffer {
2495    id: u32,
2496}
2497
2498impl FrameBuffer {
2499    fn new() -> FrameBuffer {
2500        let mut id = 0;
2501        unsafe {
2502            gls::gl::GenFramebuffers(1, &raw mut id);
2503        }
2504
2505        FrameBuffer { id }
2506    }
2507
2508    fn bind(&self) {
2509        unsafe { gls::gl::BindFramebuffer(gls::gl::FRAMEBUFFER, self.id) };
2510    }
2511
2512    fn unbind(&self) {
2513        unsafe { gls::gl::BindFramebuffer(gls::gl::FRAMEBUFFER, 0) };
2514    }
2515}
2516
2517impl Drop for FrameBuffer {
2518    fn drop(&mut self) {
2519        self.unbind();
2520        unsafe {
2521            gls::gl::DeleteFramebuffers(1, &raw mut self.id);
2522        }
2523    }
2524}
2525
2526pub struct GlProgram {
2527    id: u32,
2528    vertex_id: u32,
2529    fragment_id: u32,
2530}
2531
2532impl GlProgram {
2533    fn new(vertex_shader: &str, fragment_shader: &str) -> Result<Self, crate::Error> {
2534        let id = unsafe { gls::gl::CreateProgram() };
2535        let vertex_id = unsafe { gls::gl::CreateShader(gls::gl::VERTEX_SHADER) };
2536        if compile_shader_from_str(vertex_id, vertex_shader, "shader_vert").is_err() {
2537            log::debug!("Vertex shader source:\n{}", vertex_shader);
2538            return Err(crate::Error::OpenGl(format!(
2539                "Shader compile error: {vertex_shader}"
2540            )));
2541        }
2542        unsafe {
2543            gls::gl::AttachShader(id, vertex_id);
2544        }
2545
2546        let fragment_id = unsafe { gls::gl::CreateShader(gls::gl::FRAGMENT_SHADER) };
2547        if compile_shader_from_str(fragment_id, fragment_shader, "shader_frag").is_err() {
2548            log::debug!("Fragment shader source:\n{}", fragment_shader);
2549            return Err(crate::Error::OpenGl(format!(
2550                "Shader compile error: {fragment_shader}"
2551            )));
2552        }
2553
2554        unsafe {
2555            gls::gl::AttachShader(id, fragment_id);
2556            gls::gl::LinkProgram(id);
2557            gls::gl::UseProgram(id);
2558        }
2559
2560        Ok(Self {
2561            id,
2562            vertex_id,
2563            fragment_id,
2564        })
2565    }
2566
2567    #[allow(dead_code)]
2568    fn load_uniform_1f(&self, name: &CStr, value: f32) -> Result<(), crate::Error> {
2569        unsafe {
2570            gls::gl::UseProgram(self.id);
2571            let location = gls::gl::GetUniformLocation(self.id, name.as_ptr());
2572            gls::gl::Uniform1f(location, value);
2573        }
2574        Ok(())
2575    }
2576
2577    #[allow(dead_code)]
2578    fn load_uniform_1i(&self, name: &CStr, value: i32) -> Result<(), crate::Error> {
2579        unsafe {
2580            gls::gl::UseProgram(self.id);
2581            let location = gls::gl::GetUniformLocation(self.id, name.as_ptr());
2582            gls::gl::Uniform1i(location, value);
2583        }
2584        Ok(())
2585    }
2586
2587    fn load_uniform_4fv(&self, name: &CStr, value: &[[f32; 4]]) -> Result<(), crate::Error> {
2588        unsafe {
2589            gls::gl::UseProgram(self.id);
2590            let location = gls::gl::GetUniformLocation(self.id, name.as_ptr());
2591            if location == -1 {
2592                return Err(crate::Error::OpenGl(format!(
2593                    "Could not find uniform location for '{}'",
2594                    name.to_string_lossy().into_owned()
2595                )));
2596            }
2597            gls::gl::Uniform4fv(location, value.len() as i32, value.as_flattened().as_ptr());
2598        }
2599        check_gl_error(function!(), line!())?;
2600        Ok(())
2601    }
2602}
2603
2604impl Drop for GlProgram {
2605    fn drop(&mut self) {
2606        unsafe {
2607            gls::gl::DeleteProgram(self.id);
2608            gls::gl::DeleteShader(self.fragment_id);
2609            gls::gl::DeleteShader(self.vertex_id);
2610        }
2611    }
2612}
2613
2614fn compile_shader_from_str(shader: u32, shader_source: &str, shader_name: &str) -> Result<(), ()> {
2615    let src = match CString::from_str(shader_source) {
2616        Ok(v) => v,
2617        Err(_) => return Err(()),
2618    };
2619    let src_ptr = src.as_ptr();
2620    unsafe {
2621        gls::gl::ShaderSource(shader, 1, &raw const src_ptr, null());
2622        gls::gl::CompileShader(shader);
2623        let mut is_compiled = 0;
2624        gls::gl::GetShaderiv(shader, gls::gl::COMPILE_STATUS, &raw mut is_compiled);
2625        if is_compiled == 0 {
2626            let mut max_length = 0;
2627            gls::gl::GetShaderiv(shader, gls::gl::INFO_LOG_LENGTH, &raw mut max_length);
2628            let mut error_log: Vec<u8> = vec![0; max_length as usize];
2629            gls::gl::GetShaderInfoLog(
2630                shader,
2631                max_length,
2632                &raw mut max_length,
2633                error_log.as_mut_ptr() as *mut c_char,
2634            );
2635            error!(
2636                "Shader '{}' failed: {:?}\n",
2637                shader_name,
2638                CString::from_vec_with_nul(error_log)
2639                    .unwrap()
2640                    .into_string()
2641                    .unwrap()
2642            );
2643            gls::gl::DeleteShader(shader);
2644            return Err(());
2645        }
2646        Ok(())
2647    }
2648}
2649
2650fn check_gl_error(name: &str, line: u32) -> Result<(), Error> {
2651    unsafe {
2652        let err = gls::gl::GetError();
2653        if err != gls::gl::NO_ERROR {
2654            error!("GL Error: {name}:{line}: {err:#X}");
2655            // panic!("GL Error: {err}");
2656            return Err(Error::OpenGl(format!("{err:#X}")));
2657        }
2658    }
2659    Ok(())
2660}
2661
2662fn fourcc_to_drm(fourcc: FourCharCode) -> DrmFourcc {
2663    match fourcc {
2664        RGBA => DrmFourcc::Abgr8888,
2665        YUYV => DrmFourcc::Yuyv,
2666        RGB => DrmFourcc::Bgr888,
2667        GREY => DrmFourcc::R8,
2668        NV12 => DrmFourcc::Nv12,
2669        _ => todo!(),
2670    }
2671}
2672
2673mod egl_ext {
2674    #![allow(dead_code)]
2675    pub(crate) const LINUX_DMA_BUF: u32 = 0x3270;
2676    pub(crate) const LINUX_DRM_FOURCC: u32 = 0x3271;
2677    pub(crate) const DMA_BUF_PLANE0_FD: u32 = 0x3272;
2678    pub(crate) const DMA_BUF_PLANE0_OFFSET: u32 = 0x3273;
2679    pub(crate) const DMA_BUF_PLANE0_PITCH: u32 = 0x3274;
2680    pub(crate) const DMA_BUF_PLANE1_FD: u32 = 0x3275;
2681    pub(crate) const DMA_BUF_PLANE1_OFFSET: u32 = 0x3276;
2682    pub(crate) const DMA_BUF_PLANE1_PITCH: u32 = 0x3277;
2683    pub(crate) const DMA_BUF_PLANE2_FD: u32 = 0x3278;
2684    pub(crate) const DMA_BUF_PLANE2_OFFSET: u32 = 0x3279;
2685    pub(crate) const DMA_BUF_PLANE2_PITCH: u32 = 0x327A;
2686    pub(crate) const YUV_COLOR_SPACE_HINT: u32 = 0x327B;
2687    pub(crate) const SAMPLE_RANGE_HINT: u32 = 0x327C;
2688    pub(crate) const YUV_CHROMA_HORIZONTAL_SITING_HINT: u32 = 0x327D;
2689    pub(crate) const YUV_CHROMA_VERTICAL_SITING_HINT: u32 = 0x327E;
2690
2691    pub(crate) const ITU_REC601: u32 = 0x327F;
2692    pub(crate) const ITU_REC709: u32 = 0x3280;
2693    pub(crate) const ITU_REC2020: u32 = 0x3281;
2694
2695    pub(crate) const YUV_FULL_RANGE: u32 = 0x3282;
2696    pub(crate) const YUV_NARROW_RANGE: u32 = 0x3283;
2697
2698    pub(crate) const YUV_CHROMA_SITING_0: u32 = 0x3284;
2699    pub(crate) const YUV_CHROMA_SITING_0_5: u32 = 0x3285;
2700
2701    pub(crate) const PLATFORM_GBM_KHR: u32 = 0x31D7;
2702
2703    pub(crate) const PLATFORM_DEVICE_EXT: u32 = 0x313F;
2704}
2705
2706fn generate_vertex_shader() -> &'static str {
2707    "\
2708#version 300 es
2709precision mediump float;
2710layout(location = 0) in vec3 pos;
2711layout(location = 1) in vec2 texCoord;
2712
2713out vec3 fragPos;
2714out vec2 tc;
2715
2716void main() {
2717    fragPos = pos;
2718    tc = texCoord;
2719
2720    gl_Position = vec4(pos, 1.0);
2721}
2722"
2723}
2724
2725fn generate_texture_fragment_shader() -> &'static str {
2726    "\
2727#version 300 es
2728
2729precision mediump float;
2730uniform sampler2D tex;
2731in vec3 fragPos;
2732in vec2 tc;
2733
2734out vec4 color;
2735
2736void main(){
2737    color = texture(tex, tc);
2738}
2739"
2740}
2741
2742fn generate_texture_fragment_shader_yuv() -> &'static str {
2743    "\
2744#version 300 es
2745#extension GL_OES_EGL_image_external_essl3 : require
2746precision mediump float;
2747uniform samplerExternalOES tex;
2748in vec3 fragPos;
2749in vec2 tc;
2750
2751out vec4 color;
2752
2753void main(){
2754    color = texture(tex, tc);
2755}
2756"
2757}
2758
2759fn generate_planar_rgb_shader() -> &'static str {
2760    "\
2761#version 300 es
2762#extension GL_OES_EGL_image_external_essl3 : require
2763precision mediump float;
2764uniform samplerExternalOES tex;
2765in vec3 fragPos;
2766in vec2 tc;
2767
2768out vec4 color;
2769
2770void main(){
2771    color = texture(tex, tc);
2772}
2773"
2774}
2775
2776/// this shader requires a reshape of the segmentation output tensor to (H, W,
2777/// C/4, 4)
2778fn generate_segmentation_shader() -> &'static str {
2779    "\
2780#version 300 es
2781precision mediump float;
2782precision mediump sampler2DArray;
2783
2784uniform sampler2DArray tex;
2785uniform vec4 colors[20];
2786uniform int background_index;
2787
2788in vec3 fragPos;
2789in vec2 tc;
2790in vec4 fragColor;
2791
2792out vec4 color;
2793
2794float max_arg(const in vec4 args, out int argmax) {
2795    if (args[0] >= args[1] && args[0] >= args[2] && args[0] >= args[3]) {
2796        argmax = 0;
2797        return args[0];
2798    }
2799    if (args[1] >= args[0] && args[1] >= args[2] && args[1] >= args[3]) {
2800        argmax = 1;
2801        return args[1];
2802    }
2803    if (args[2] >= args[0] && args[2] >= args[1] && args[2] >= args[3]) {
2804        argmax = 2;
2805        return args[2];
2806    }
2807    argmax = 3;
2808    return args[3];
2809}
2810
2811void main() {
2812    mediump int layers = textureSize(tex, 0).z;
2813    float max_all = -4.0;
2814    int max_ind = 0;
2815    for (int i = 0; i < layers; i++) {
2816        vec4 d = texture(tex, vec3(tc, i));
2817        int max_ind_ = 0;
2818        float max_ = max_arg(d, max_ind_);
2819        if (max_ <= max_all) { continue; }
2820        max_all = max_;
2821        max_ind = i*4 + max_ind_;
2822    }
2823    if (max_ind == background_index) {
2824        discard;
2825    }
2826    max_ind = max_ind % 20;
2827    color = colors[max_ind];
2828}
2829"
2830}
2831
2832fn generate_instanced_segmentation_shader() -> &'static str {
2833    "\
2834#version 300 es
2835precision mediump float;
2836uniform sampler2D mask0;
2837uniform vec4 colors[20];
2838uniform int class_index;
2839in vec3 fragPos;
2840in vec2 tc;
2841in vec4 fragColor;
2842
2843out vec4 color;
2844void main() {
2845    float r0 = texture(mask0, tc).r;
2846    int arg = int(r0>=0.5);
2847    if (arg == 0) {
2848        discard;
2849    }
2850    color = colors[class_index % 20];
2851}
2852"
2853}
2854
2855fn generate_color_shader() -> &'static str {
2856    "\
2857#version 300 es
2858precision mediump float;
2859uniform vec4 colors[20];
2860uniform int class_index;
2861
2862out vec4 color;
2863void main() {
2864    int index = class_index % 20;
2865    color = colors[index];
2866}
2867"
2868}
2869
2870#[cfg(test)]
2871#[cfg(feature = "opengl")]
2872mod gl_tests {
2873    use super::*;
2874    use crate::{TensorImage, RGBA};
2875    #[cfg(feature = "dma_test_formats")]
2876    use crate::{NV12, YUYV};
2877    use edgefirst_tensor::TensorTrait;
2878    #[cfg(feature = "dma_test_formats")]
2879    use edgefirst_tensor::{TensorMapTrait, TensorMemory};
2880    use image::buffer::ConvertBuffer;
2881    use ndarray::Array3;
2882
2883    #[test]
2884    #[cfg(feature = "decoder")]
2885    fn test_segmentation() {
2886        use edgefirst_decoder::Segmentation;
2887
2888        if !is_opengl_available() {
2889            eprintln!("SKIPPED: {} - OpenGL not available", function!());
2890            return;
2891        }
2892
2893        let mut image = TensorImage::load(
2894            include_bytes!("../../../testdata/giraffe.jpg"),
2895            Some(RGBA),
2896            None,
2897        )
2898        .unwrap();
2899
2900        let mut segmentation = Array3::from_shape_vec(
2901            (2, 160, 160),
2902            include_bytes!("../../../testdata/modelpack_seg_2x160x160.bin").to_vec(),
2903        )
2904        .unwrap();
2905        segmentation.swap_axes(0, 1);
2906        segmentation.swap_axes(1, 2);
2907        let segmentation = segmentation.as_standard_layout().to_owned();
2908
2909        let seg = Segmentation {
2910            segmentation,
2911            xmin: 0.0,
2912            ymin: 0.0,
2913            xmax: 1.0,
2914            ymax: 1.0,
2915        };
2916
2917        let mut renderer = GLProcessorThreaded::new().unwrap();
2918        renderer.render_to_image(&mut image, &[], &[seg]).unwrap();
2919    }
2920
2921    #[test]
2922    #[cfg(feature = "decoder")]
2923    fn test_segmentation_mem() {
2924        use edgefirst_decoder::Segmentation;
2925
2926        if !is_opengl_available() {
2927            eprintln!("SKIPPED: {} - OpenGL not available", function!());
2928            return;
2929        }
2930
2931        let mut image = TensorImage::load(
2932            include_bytes!("../../../testdata/giraffe.jpg"),
2933            Some(RGBA),
2934            Some(edgefirst_tensor::TensorMemory::Mem),
2935        )
2936        .unwrap();
2937
2938        let mut segmentation = Array3::from_shape_vec(
2939            (2, 160, 160),
2940            include_bytes!("../../../testdata/modelpack_seg_2x160x160.bin").to_vec(),
2941        )
2942        .unwrap();
2943        segmentation.swap_axes(0, 1);
2944        segmentation.swap_axes(1, 2);
2945        let segmentation = segmentation.as_standard_layout().to_owned();
2946
2947        let seg = Segmentation {
2948            segmentation,
2949            xmin: 0.0,
2950            ymin: 0.0,
2951            xmax: 1.0,
2952            ymax: 1.0,
2953        };
2954
2955        let mut renderer = GLProcessorThreaded::new().unwrap();
2956        renderer.render_to_image(&mut image, &[], &[seg]).unwrap();
2957    }
2958
2959    #[test]
2960    #[cfg(feature = "decoder")]
2961    fn test_segmentation_yolo() {
2962        use edgefirst_decoder::Segmentation;
2963        use ndarray::Array3;
2964
2965        if !is_opengl_available() {
2966            eprintln!("SKIPPED: {} - OpenGL not available", function!());
2967            return;
2968        }
2969
2970        let mut image = TensorImage::load(
2971            include_bytes!("../../../testdata/giraffe.jpg"),
2972            Some(RGBA),
2973            None,
2974        )
2975        .unwrap();
2976
2977        let segmentation = Array3::from_shape_vec(
2978            (76, 55, 1),
2979            include_bytes!("../../../testdata/yolov8_seg_crop_76x55.bin").to_vec(),
2980        )
2981        .unwrap();
2982
2983        let detect = DetectBox {
2984            bbox: [0.59375, 0.25, 0.9375, 0.725].into(),
2985            score: 0.99,
2986            label: 1,
2987        };
2988
2989        let seg = Segmentation {
2990            segmentation,
2991            xmin: 0.59375,
2992            ymin: 0.25,
2993            xmax: 0.9375,
2994            ymax: 0.725,
2995        };
2996
2997        let mut renderer = GLProcessorThreaded::new().unwrap();
2998        renderer
2999            .set_class_colors(&[[255, 255, 0, 233], [128, 128, 255, 100]])
3000            .unwrap();
3001        renderer
3002            .render_to_image(&mut image, &[detect], &[seg])
3003            .unwrap();
3004
3005        let expected = TensorImage::load(
3006            include_bytes!("../../../testdata/output_render_gl.jpg"),
3007            Some(RGBA),
3008            None,
3009        )
3010        .unwrap();
3011
3012        compare_images(&image, &expected, 0.99, function!());
3013    }
3014
3015    #[test]
3016    #[cfg(feature = "decoder")]
3017    fn test_boxes() {
3018        use edgefirst_decoder::DetectBox;
3019
3020        if !is_opengl_available() {
3021            eprintln!("SKIPPED: {} - OpenGL not available", function!());
3022            return;
3023        }
3024
3025        let mut image = TensorImage::load(
3026            include_bytes!("../../../testdata/giraffe.jpg"),
3027            Some(RGBA),
3028            None,
3029        )
3030        .unwrap();
3031
3032        let detect = DetectBox {
3033            bbox: [0.59375, 0.25, 0.9375, 0.725].into(),
3034            score: 0.99,
3035            label: 0,
3036        };
3037        let mut renderer = GLProcessorThreaded::new().unwrap();
3038        renderer
3039            .set_class_colors(&[[255, 255, 0, 233], [128, 128, 255, 100]])
3040            .unwrap();
3041        renderer
3042            .render_to_image(&mut image, &[detect], &[])
3043            .unwrap();
3044    }
3045
3046    static GL_AVAILABLE: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
3047    // Helper function to check if OpenGL is available
3048    fn is_opengl_available() -> bool {
3049        #[cfg(all(target_os = "linux", feature = "opengl"))]
3050        {
3051            *GL_AVAILABLE.get_or_init(|| GLProcessorThreaded::new().is_ok())
3052        }
3053
3054        #[cfg(not(all(target_os = "linux", feature = "opengl")))]
3055        {
3056            false
3057        }
3058    }
3059
3060    fn compare_images(img1: &TensorImage, img2: &TensorImage, threshold: f64, name: &str) {
3061        assert_eq!(img1.height(), img2.height(), "Heights differ");
3062        assert_eq!(img1.width(), img2.width(), "Widths differ");
3063        assert_eq!(img1.fourcc(), img2.fourcc(), "FourCC differ");
3064        assert!(
3065            matches!(img1.fourcc(), RGB | RGBA | GREY | PLANAR_RGB),
3066            "FourCC must be RGB or RGBA for comparison"
3067        );
3068
3069        let image1 = match img1.fourcc() {
3070            RGB => image::RgbImage::from_vec(
3071                img1.width() as u32,
3072                img1.height() as u32,
3073                img1.tensor().map().unwrap().to_vec(),
3074            )
3075            .unwrap(),
3076            RGBA => image::RgbaImage::from_vec(
3077                img1.width() as u32,
3078                img1.height() as u32,
3079                img1.tensor().map().unwrap().to_vec(),
3080            )
3081            .unwrap()
3082            .convert(),
3083            GREY => image::GrayImage::from_vec(
3084                img1.width() as u32,
3085                img1.height() as u32,
3086                img1.tensor().map().unwrap().to_vec(),
3087            )
3088            .unwrap()
3089            .convert(),
3090            PLANAR_RGB => image::GrayImage::from_vec(
3091                img1.width() as u32,
3092                (img1.height() * 3) as u32,
3093                img1.tensor().map().unwrap().to_vec(),
3094            )
3095            .unwrap()
3096            .convert(),
3097            _ => return,
3098        };
3099
3100        let image2 = match img2.fourcc() {
3101            RGB => image::RgbImage::from_vec(
3102                img2.width() as u32,
3103                img2.height() as u32,
3104                img2.tensor().map().unwrap().to_vec(),
3105            )
3106            .unwrap(),
3107            RGBA => image::RgbaImage::from_vec(
3108                img2.width() as u32,
3109                img2.height() as u32,
3110                img2.tensor().map().unwrap().to_vec(),
3111            )
3112            .unwrap()
3113            .convert(),
3114            GREY => image::GrayImage::from_vec(
3115                img2.width() as u32,
3116                img2.height() as u32,
3117                img2.tensor().map().unwrap().to_vec(),
3118            )
3119            .unwrap()
3120            .convert(),
3121            PLANAR_RGB => image::GrayImage::from_vec(
3122                img2.width() as u32,
3123                (img2.height() * 3) as u32,
3124                img2.tensor().map().unwrap().to_vec(),
3125            )
3126            .unwrap()
3127            .convert(),
3128            _ => return,
3129        };
3130
3131        let similarity = image_compare::rgb_similarity_structure(
3132            &image_compare::Algorithm::RootMeanSquared,
3133            &image1,
3134            &image2,
3135        )
3136        .expect("Image Comparison failed");
3137        if similarity.score < threshold {
3138            // image1.save(format!("{name}_1.png"));
3139            // image2.save(format!("{name}_2.png"));
3140            similarity
3141                .image
3142                .to_color_map()
3143                .save(format!("{name}.png"))
3144                .unwrap();
3145            panic!(
3146                "{name}: converted image and target image have similarity score too low: {} < {}",
3147                similarity.score, threshold
3148            )
3149        }
3150    }
3151
3152    // =========================================================================
3153    // NV12 Reference Validation Tests
3154    // These tests compare OpenGL NV12 conversions against ffmpeg-generated
3155    // references
3156    // =========================================================================
3157
3158    #[cfg(feature = "dma_test_formats")]
3159    fn load_raw_image(
3160        width: usize,
3161        height: usize,
3162        fourcc: FourCharCode,
3163        memory: Option<TensorMemory>,
3164        bytes: &[u8],
3165    ) -> Result<TensorImage, crate::Error> {
3166        let img = TensorImage::new(width, height, fourcc, memory)?;
3167        let mut map = img.tensor().map()?;
3168        map.as_mut_slice()[..bytes.len()].copy_from_slice(bytes);
3169        Ok(img)
3170    }
3171
3172    /// Test OpenGL NV12→RGBA conversion against ffmpeg reference
3173    #[test]
3174    #[cfg(all(target_os = "linux", feature = "dma_test_formats"))]
3175    fn test_opengl_nv12_to_rgba_reference() {
3176        // Load NV12 source with DMA
3177        let src = load_raw_image(
3178            1280,
3179            720,
3180            NV12,
3181            Some(TensorMemory::Dma),
3182            include_bytes!("../../../testdata/camera720p.nv12"),
3183        )
3184        .unwrap();
3185
3186        // Load RGBA reference (ffmpeg-generated)
3187        let reference = load_raw_image(
3188            1280,
3189            720,
3190            RGBA,
3191            None,
3192            include_bytes!("../../../testdata/camera720p.rgba"),
3193        )
3194        .unwrap();
3195
3196        // Convert using OpenGL
3197        let mut dst = TensorImage::new(1280, 720, RGBA, Some(TensorMemory::Dma)).unwrap();
3198        let mut gl = GLProcessorThreaded::new().unwrap();
3199        gl.convert(&src, &mut dst, Rotation::None, Flip::None, Crop::no_crop())
3200            .unwrap();
3201
3202        // Copy to CPU for comparison
3203        let cpu_dst = TensorImage::new(1280, 720, RGBA, None).unwrap();
3204        cpu_dst
3205            .tensor()
3206            .map()
3207            .unwrap()
3208            .as_mut_slice()
3209            .copy_from_slice(dst.tensor().map().unwrap().as_slice());
3210
3211        compare_images(&reference, &cpu_dst, 0.98, "opengl_nv12_to_rgba_reference");
3212    }
3213
3214    /// Test OpenGL YUYV→RGBA conversion against ffmpeg reference
3215    #[test]
3216    #[cfg(all(target_os = "linux", feature = "dma_test_formats"))]
3217    fn test_opengl_yuyv_to_rgba_reference() {
3218        // Load YUYV source with DMA
3219        let src = load_raw_image(
3220            1280,
3221            720,
3222            YUYV,
3223            Some(TensorMemory::Dma),
3224            include_bytes!("../../../testdata/camera720p.yuyv"),
3225        )
3226        .unwrap();
3227
3228        // Load RGBA reference (ffmpeg-generated)
3229        let reference = load_raw_image(
3230            1280,
3231            720,
3232            RGBA,
3233            None,
3234            include_bytes!("../../../testdata/camera720p.rgba"),
3235        )
3236        .unwrap();
3237
3238        // Convert using OpenGL
3239        let mut dst = TensorImage::new(1280, 720, RGBA, Some(TensorMemory::Dma)).unwrap();
3240        let mut gl = GLProcessorThreaded::new().unwrap();
3241        gl.convert(&src, &mut dst, Rotation::None, Flip::None, Crop::no_crop())
3242            .unwrap();
3243
3244        // Copy to CPU for comparison
3245        let cpu_dst = TensorImage::new(1280, 720, RGBA, None).unwrap();
3246        cpu_dst
3247            .tensor()
3248            .map()
3249            .unwrap()
3250            .as_mut_slice()
3251            .copy_from_slice(dst.tensor().map().unwrap().as_slice());
3252
3253        compare_images(&reference, &cpu_dst, 0.98, "opengl_yuyv_to_rgba_reference");
3254    }
3255}