Skip to main content

edgefirst_image/
opengl_headless.rs

1// SPDX-FileCopyrightText: Copyright 2025 Au-Zone Technologies
2// SPDX-License-Identifier: Apache-2.0
3
4#![cfg(target_os = "linux")]
5#![cfg(feature = "opengl")]
6
7use edgefirst_decoder::DetectBox;
8#[cfg(feature = "decoder")]
9use edgefirst_decoder::Segmentation;
10use edgefirst_tensor::{TensorMemory, TensorTrait};
11use four_char_code::FourCharCode;
12use gbm::{
13    drm::{buffer::DrmFourcc, control::Device as DrmControlDevice, Device as DrmDevice},
14    AsRaw, Device,
15};
16use khronos_egl::{self as egl, Attrib, Display, Dynamic, Instance, EGL1_4};
17use log::{debug, error};
18use std::{
19    collections::BTreeSet,
20    ffi::{c_char, c_void, CStr, CString},
21    mem::ManuallyDrop,
22    os::fd::AsRawFd,
23    ptr::{null, null_mut, NonNull},
24    rc::Rc,
25    str::FromStr,
26    sync::OnceLock,
27    thread::JoinHandle,
28    time::Instant,
29};
30use tokio::sync::mpsc::Sender;
31
32macro_rules! function {
33    () => {{
34        fn f() {}
35        fn type_name_of<T>(_: T) -> &'static str {
36            std::any::type_name::<T>()
37        }
38        let name = type_name_of(f);
39
40        // Find and cut the rest of the path
41        match &name[..name.len() - 3].rfind(':') {
42            Some(pos) => &name[pos + 1..name.len() - 3],
43            None => &name[..name.len() - 3],
44        }
45    }};
46}
47
48#[cfg(feature = "decoder")]
49use crate::DEFAULT_COLORS;
50use crate::{
51    CPUProcessor, Crop, Error, Flip, ImageProcessorTrait, Rect, Rotation, TensorImage,
52    TensorImageRef, GREY, NV12, PLANAR_RGB, PLANAR_RGBA, RGB, RGBA, YUYV,
53};
54
55/// EGL library handle. Intentionally leaked (never dlclose'd) to avoid SIGBUS
56/// on process exit: GPU drivers may keep internal state that outlives explicit
57/// EGL cleanup, and dlclose can unmap memory still referenced by the driver.
58static EGL_LIB: OnceLock<&'static libloading::Library> = OnceLock::new();
59
60fn get_egl_lib() -> Result<&'static libloading::Library, crate::Error> {
61    if let Some(egl) = EGL_LIB.get() {
62        Ok(egl)
63    } else {
64        let egl = unsafe { libloading::Library::new("libEGL.so.1")? };
65        // Leak the library to prevent dlclose on process exit
66        let egl: &'static libloading::Library = Box::leak(Box::new(egl));
67        Ok(EGL_LIB.get_or_init(|| egl))
68    }
69}
70
71type Egl = Instance<Dynamic<&'static libloading::Library, EGL1_4>>;
72pub(crate) struct GlContext {
73    pub(crate) support_dma: bool,
74    pub(crate) surface: Option<egl::Surface>,
75    pub(crate) display: EglDisplayType,
76    pub(crate) ctx: egl::Context,
77    /// Wrapped in ManuallyDrop because the khronos-egl Dynamic instance's
78    /// Drop calls eglReleaseThread() which can panic during process shutdown
79    /// if the EGL library has been partially unloaded. We drop it explicitly
80    /// inside catch_unwind in GlContext::drop.
81    pub(crate) egl: ManuallyDrop<Rc<Egl>>,
82}
83
84pub(crate) enum EglDisplayType {
85    Default(egl::Display),
86    Gbm(egl::Display, #[allow(dead_code)] Device<Card>),
87    PlatformDisplay(egl::Display),
88}
89
90impl EglDisplayType {
91    fn as_display(&self) -> egl::Display {
92        match self {
93            EglDisplayType::Default(disp) => *disp,
94            EglDisplayType::Gbm(disp, _) => *disp,
95            EglDisplayType::PlatformDisplay(disp) => *disp,
96        }
97    }
98}
99
100impl GlContext {
101    pub(crate) fn new() -> Result<GlContext, crate::Error> {
102        // Create an EGL API instance.
103        let egl: Rc<Egl> =
104            Rc::new(unsafe { Instance::<Dynamic<_, EGL1_4>>::load_required_from(get_egl_lib()?)? });
105
106        // Try headless-friendly EGL methods first (GBM/DRM, device enumeration)
107        // before the default display, which may block if a compositor (Wayland)
108        // is expected but not running.
109        if let Ok(headless) = Self::try_initialize_egl(egl.clone(), Self::egl_get_gbm_display) {
110            return Ok(headless);
111        } else {
112            log::debug!("Didn't initialize EGL with GBM Display");
113        }
114
115        if let Ok(headless) =
116            Self::try_initialize_egl(egl.clone(), Self::egl_get_platform_display_from_device)
117        {
118            return Ok(headless);
119        } else {
120            log::debug!("Didn't initialize EGL with platform display from device enumeration");
121        }
122
123        if let Ok(headless) = Self::try_initialize_egl(egl.clone(), Self::egl_get_default_display) {
124            return Ok(headless);
125        } else {
126            log::debug!("Didn't initialize EGL with Default Display");
127        }
128
129        Err(Error::OpenGl(
130            "Could not initialize EGL with any known method".to_string(),
131        ))
132    }
133
134    fn try_initialize_egl(
135        egl: Rc<Egl>,
136        display_fn: impl Fn(&Egl) -> Result<EglDisplayType, crate::Error>,
137    ) -> Result<GlContext, crate::Error> {
138        let display = display_fn(&egl)?;
139        log::debug!("egl initialize with display: {:x?}", display.as_display());
140        egl.initialize(display.as_display())?;
141        let attributes = [
142            egl::SURFACE_TYPE,
143            egl::PBUFFER_BIT,
144            egl::RENDERABLE_TYPE,
145            egl::OPENGL_ES3_BIT,
146            egl::RED_SIZE,
147            8,
148            egl::GREEN_SIZE,
149            8,
150            egl::BLUE_SIZE,
151            8,
152            egl::ALPHA_SIZE,
153            8,
154            egl::NONE,
155        ];
156
157        let config =
158            if let Some(config) = egl.choose_first_config(display.as_display(), &attributes)? {
159                config
160            } else {
161                return Err(crate::Error::NotImplemented(
162                    "Did not find valid OpenGL ES config".to_string(),
163                ));
164            };
165
166        debug!("config: {config:?}");
167
168        let surface = Some(egl.create_pbuffer_surface(
169            display.as_display(),
170            config,
171            &[egl::WIDTH, 64, egl::HEIGHT, 64, egl::NONE],
172        )?);
173
174        egl.bind_api(egl::OPENGL_ES_API)?;
175        let context_attributes = [egl::CONTEXT_MAJOR_VERSION, 3, egl::NONE, egl::NONE];
176
177        let ctx = egl.create_context(display.as_display(), config, None, &context_attributes)?;
178        debug!("ctx: {ctx:?}");
179
180        egl.make_current(display.as_display(), surface, surface, Some(ctx))?;
181
182        let support_dma = Self::egl_check_support_dma(&egl).is_ok();
183        let headless = GlContext {
184            display,
185            ctx,
186            egl: ManuallyDrop::new(egl),
187            surface,
188            support_dma,
189        };
190        Ok(headless)
191    }
192
193    fn egl_get_default_display(egl: &Egl) -> Result<EglDisplayType, crate::Error> {
194        // get the default display
195        if let Some(display) = unsafe { egl.get_display(egl::DEFAULT_DISPLAY) } {
196            debug!("default display: {display:?}");
197            return Ok(EglDisplayType::Default(display));
198        }
199
200        Err(Error::OpenGl(
201            "Could not obtain EGL Default Display".to_string(),
202        ))
203    }
204
205    fn egl_get_gbm_display(egl: &Egl) -> Result<EglDisplayType, crate::Error> {
206        // init a GBM device
207        let gbm = Device::new(Card::open_global()?)?;
208
209        debug!("gbm: {gbm:?}");
210        let display = Self::egl_get_platform_display_with_fallback(
211            egl,
212            egl_ext::PLATFORM_GBM_KHR,
213            gbm.as_raw() as *mut c_void,
214            &[egl::ATTRIB_NONE],
215        )?;
216
217        Ok(EglDisplayType::Gbm(display, gbm))
218    }
219
220    fn egl_get_platform_display_from_device(egl: &Egl) -> Result<EglDisplayType, crate::Error> {
221        let extensions = egl.query_string(None, egl::EXTENSIONS)?;
222        let extensions = extensions.to_string_lossy();
223        log::debug!("EGL Extensions: {}", extensions);
224
225        if !extensions.contains("EGL_EXT_device_enumeration") {
226            return Err(Error::GLVersion(
227                "EGL doesn't supported EGL_EXT_device_enumeration extension".to_string(),
228            ));
229        }
230
231        type EGLDeviceEXT = *mut c_void;
232        let devices = if let Some(ext) = egl.get_proc_address("eglQueryDevicesEXT") {
233            let func: unsafe extern "system" fn(
234                max_devices: egl::Int,
235                devices: *mut EGLDeviceEXT,
236                num_devices: *mut egl::Int,
237            ) -> *const c_char = unsafe { std::mem::transmute(ext) };
238            let mut devices = [std::ptr::null_mut(); 10];
239            let mut num_devices = 0;
240            unsafe { func(devices.len() as i32, devices.as_mut_ptr(), &mut num_devices) };
241            for i in 0..num_devices {
242                log::debug!("EGL device: {:?}", devices[i as usize]);
243            }
244            devices[0..num_devices as usize].to_vec()
245        } else {
246            return Err(Error::GLVersion(
247                "EGL doesn't supported eglQueryDevicesEXT function".to_string(),
248            ));
249        };
250
251        if !extensions.contains("EGL_EXT_platform_device") {
252            return Err(Error::GLVersion(
253                "EGL doesn't supported EGL_EXT_platform_device extension".to_string(),
254            ));
255        }
256
257        // just use the first device?
258        let disp = Self::egl_get_platform_display_with_fallback(
259            egl,
260            egl_ext::PLATFORM_DEVICE_EXT,
261            devices[0],
262            &[egl::ATTRIB_NONE],
263        )?;
264        Ok(EglDisplayType::PlatformDisplay(disp))
265    }
266
267    fn egl_check_support_dma(egl: &Egl) -> Result<(), crate::Error> {
268        let extensions = egl.query_string(None, egl::EXTENSIONS)?;
269        let extensions = extensions.to_string_lossy();
270        log::debug!("EGL Extensions: {}", extensions);
271
272        if egl.upcast::<egl::EGL1_5>().is_some() {
273            return Ok(());
274        }
275
276        if !extensions.contains("EGL_EXT_image_dma_buf_import") {
277            return Err(crate::Error::GLVersion(
278                "EGL does not support EGL_EXT_image_dma_buf_import extension".to_string(),
279            ));
280        }
281
282        if egl.get_proc_address("eglCreateImageKHR").is_none() {
283            return Err(crate::Error::GLVersion(
284                "EGL does not support eglCreateImageKHR function".to_string(),
285            ));
286        }
287
288        if egl.get_proc_address("eglDestroyImageKHR").is_none() {
289            return Err(crate::Error::GLVersion(
290                "EGL does not support eglDestroyImageKHR function".to_string(),
291            ));
292        }
293        // Err(crate::Error::GLVersion("EGL Version too low".to_string()))
294        Ok(())
295    }
296
297    fn egl_get_platform_display_with_fallback(
298        egl: &Egl,
299        platform: egl::Enum,
300        native_display: *mut c_void,
301        attrib_list: &[Attrib],
302    ) -> Result<Display, Error> {
303        if let Some(egl) = egl.upcast::<egl::EGL1_5>() {
304            unsafe { egl.get_platform_display(platform, native_display, attrib_list) }
305                .map_err(|e| e.into())
306        } else if let Some(ext) = egl.get_proc_address("eglGetPlatformDisplayEXT") {
307            let func: unsafe extern "system" fn(
308                platform: egl::Enum,
309                native_display: *mut c_void,
310                attrib_list: *const Attrib,
311            ) -> egl::EGLDisplay = unsafe { std::mem::transmute(ext) };
312            let disp = unsafe { func(platform, native_display, attrib_list.as_ptr()) };
313            if disp != egl::NO_DISPLAY {
314                Ok(unsafe { Display::from_ptr(disp) })
315            } else {
316                Err(egl.get_error().map(|e| e.into()).unwrap_or(Error::Internal(
317                    "EGL failed but no error was reported".to_owned(),
318                )))
319            }
320        } else {
321            Err(Error::EGLLoad(egl::LoadError::InvalidVersion {
322                provided: egl.version(),
323                required: khronos_egl::Version::EGL1_5,
324            }))
325        }
326    }
327
328    fn egl_create_image_with_fallback(
329        egl: &Egl,
330        display: Display,
331        ctx: egl::Context,
332        target: egl::Enum,
333        buffer: egl::ClientBuffer,
334        attrib_list: &[Attrib],
335    ) -> Result<egl::Image, Error> {
336        if let Some(egl) = egl.upcast::<egl::EGL1_5>() {
337            egl.create_image(display, ctx, target, buffer, attrib_list)
338                .map_err(|e| e.into())
339        } else if let Some(ext) = egl.get_proc_address("eglCreateImageKHR") {
340            log::trace!("eglCreateImageKHR addr: {:?}", ext);
341            let func: unsafe extern "system" fn(
342                display: egl::EGLDisplay,
343                ctx: egl::EGLContext,
344                target: egl::Enum,
345                buffer: egl::EGLClientBuffer,
346                attrib_list: *const egl::Int,
347            ) -> egl::EGLImage = unsafe { std::mem::transmute(ext) };
348            let new_attrib_list = attrib_list
349                .iter()
350                .map(|x| *x as egl::Int)
351                .collect::<Vec<_>>();
352
353            let image = unsafe {
354                func(
355                    display.as_ptr(),
356                    ctx.as_ptr(),
357                    target,
358                    buffer.as_ptr(),
359                    new_attrib_list.as_ptr(),
360                )
361            };
362            if image != egl::NO_IMAGE {
363                Ok(unsafe { egl::Image::from_ptr(image) })
364            } else {
365                Err(egl.get_error().map(|e| e.into()).unwrap_or(Error::Internal(
366                    "EGL failed but no error was reported".to_owned(),
367                )))
368            }
369        } else {
370            Err(Error::EGLLoad(egl::LoadError::InvalidVersion {
371                provided: egl.version(),
372                required: khronos_egl::Version::EGL1_5,
373            }))
374        }
375    }
376
377    fn egl_destory_image_with_fallback(
378        egl: &Egl,
379        display: Display,
380        image: egl::Image,
381    ) -> Result<(), Error> {
382        if let Some(egl) = egl.upcast::<egl::EGL1_5>() {
383            egl.destroy_image(display, image).map_err(|e| e.into())
384        } else if let Some(ext) = egl.get_proc_address("eglDestroyImageKHR") {
385            let func: unsafe extern "system" fn(
386                display: egl::EGLDisplay,
387                image: egl::EGLImage,
388            ) -> egl::Boolean = unsafe { std::mem::transmute(ext) };
389            let res = unsafe { func(display.as_ptr(), image.as_ptr()) };
390            if res == egl::TRUE {
391                Ok(())
392            } else {
393                Err(egl.get_error().map(|e| e.into()).unwrap_or(Error::Internal(
394                    "EGL failed but no error was reported".to_owned(),
395                )))
396            }
397        } else {
398            Err(Error::EGLLoad(egl::LoadError::InvalidVersion {
399                provided: egl.version(),
400                required: khronos_egl::Version::EGL1_5,
401            }))
402        }
403    }
404}
405
406impl Drop for GlContext {
407    fn drop(&mut self) {
408        // During process shutdown (e.g. Python interpreter exit), the EGL/GL
409        // shared libraries may already be partially unloaded, causing panics
410        // or heap corruption when calling cleanup functions. We suppress
411        // panic output and catch panics to prevent propagation.
412        let prev_hook = std::panic::take_hook();
413        std::panic::set_hook(Box::new(|_| {}));
414        let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
415            let _ = self
416                .egl
417                .make_current(self.display.as_display(), None, None, None);
418
419            let _ = self
420                .egl
421                .destroy_context(self.display.as_display(), self.ctx);
422
423            if let Some(surface) = self.surface.take() {
424                let _ = self.egl.destroy_surface(self.display.as_display(), surface);
425            }
426
427            // Note: eglTerminate is intentionally omitted. The context and
428            // surface are already destroyed above, and calling terminate after
429            // individual resource destruction can cause double-free issues on
430            // some EGL drivers (observed as heap corruption on ARM targets
431            // during process shutdown).
432        }));
433        std::panic::set_hook(prev_hook);
434
435        // The Rc<Egl> (ManuallyDrop) is intentionally NOT dropped. The
436        // khronos-egl Dynamic instance's Drop calls eglReleaseThread() which
437        // panics if the EGL library has been unloaded (local/x86_64) or
438        // causes heap corruption by calling into invalid memory (ARM). Since
439        // EGL display connections are process-scoped singletons, leaking the
440        // Rc is harmless — the OS reclaims all resources on process exit.
441    }
442}
443
444#[derive(Debug)]
445/// A simple wrapper for a device node.
446pub(crate) struct Card(std::fs::File);
447
448/// Implementing `AsFd` is a prerequisite to implementing the traits found
449/// in this crate. Here, we are just calling `as_fd()` on the inner File.
450impl std::os::unix::io::AsFd for Card {
451    fn as_fd(&self) -> std::os::unix::io::BorrowedFd<'_> {
452        self.0.as_fd()
453    }
454}
455
456/// With `AsFd` implemented, we can now implement `drm::Device`.
457impl DrmDevice for Card {}
458impl DrmControlDevice for Card {}
459
460/// Simple helper methods for opening a `Card`.
461impl Card {
462    pub fn open(path: &str) -> Result<Self, crate::Error> {
463        let mut options = std::fs::OpenOptions::new();
464        options.read(true);
465        options.write(true);
466        let c = options.open(path);
467        match c {
468            Ok(c) => Ok(Card(c)),
469            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
470                Err(Error::NotFound(format!("File not found: {path}")))
471            }
472            Err(e) => Err(e.into()),
473        }
474    }
475
476    pub fn open_global() -> Result<Self, crate::Error> {
477        let targets = ["/dev/dri/renderD128", "/dev/dri/card0", "/dev/dri/card1"];
478        let e = Self::open(targets[0]);
479        if let Ok(t) = e {
480            return Ok(t);
481        }
482        for t in &targets[1..] {
483            if let Ok(t) = Self::open(t) {
484                return Ok(t);
485            }
486        }
487        e
488    }
489}
490
491#[derive(Debug, Clone, Copy)]
492struct RegionOfInterest {
493    left: f32,
494    top: f32,
495    right: f32,
496    bottom: f32,
497}
498
499enum GLProcessorMessage {
500    ImageConvert(
501        SendablePtr<TensorImage>,
502        SendablePtr<TensorImage>,
503        Rotation,
504        Flip,
505        Crop,
506        tokio::sync::oneshot::Sender<Result<(), Error>>,
507    ),
508    SetColors(
509        Vec<[u8; 4]>,
510        tokio::sync::oneshot::Sender<Result<(), Error>>,
511    ),
512    ImageRender(
513        SendablePtr<TensorImage>,
514        SendablePtr<DetectBox>,
515        SendablePtr<Segmentation>,
516        tokio::sync::oneshot::Sender<Result<(), Error>>,
517    ),
518}
519
520/// OpenGL multi-threaded image converter. The actual conversion is done in a
521/// separate rendering thread, as OpenGL contexts are not thread-safe. This can
522/// be safely sent between threads. The `convert()` call sends the conversion
523/// request to the rendering thread and waits for the result.
524#[derive(Debug)]
525pub struct GLProcessorThreaded {
526    // This is only None when the converter is being dropped.
527    handle: Option<JoinHandle<()>>,
528
529    // This is only None when the converter is being dropped.
530    sender: Option<Sender<GLProcessorMessage>>,
531    support_dma: bool,
532}
533
534unsafe impl Send for GLProcessorThreaded {}
535unsafe impl Sync for GLProcessorThreaded {}
536
537struct SendablePtr<T: Send> {
538    ptr: NonNull<T>,
539    len: usize,
540}
541
542unsafe impl<T> Send for SendablePtr<T> where T: Send {}
543
544impl GLProcessorThreaded {
545    /// Creates a new OpenGL multi-threaded image converter.
546    pub fn new() -> Result<Self, Error> {
547        let (send, mut recv) = tokio::sync::mpsc::channel::<GLProcessorMessage>(1);
548
549        let (create_ctx_send, create_ctx_recv) = tokio::sync::oneshot::channel();
550
551        let func = move || {
552            let mut gl_converter = match GLProcessorST::new() {
553                Ok(gl) => gl,
554                Err(e) => {
555                    let _ = create_ctx_send.send(Err(e));
556                    return;
557                }
558            };
559            let _ = create_ctx_send.send(Ok(gl_converter.gl_context.support_dma));
560            while let Some(msg) = recv.blocking_recv() {
561                match msg {
562                    GLProcessorMessage::ImageConvert(src, mut dst, rotation, flip, crop, resp) => {
563                        // SAFETY: This is safe because the convert() function waits for the resp to
564                        // be sent before dropping the borrow for src and dst
565                        let src = unsafe { src.ptr.as_ref() };
566                        let dst = unsafe { dst.ptr.as_mut() };
567                        let res = gl_converter.convert(src, dst, rotation, flip, crop);
568                        let _ = resp.send(res);
569                    }
570                    GLProcessorMessage::ImageRender(mut dst, det, seg, resp) => {
571                        // SAFETY: This is safe because the render_to_image() function waits for the
572                        // resp to be sent before dropping the borrow for dst, detect, and
573                        // segmentation
574                        let dst = unsafe { dst.ptr.as_mut() };
575                        let det = unsafe { std::slice::from_raw_parts(det.ptr.as_ptr(), det.len) };
576                        let seg = unsafe { std::slice::from_raw_parts(seg.ptr.as_ptr(), seg.len) };
577                        let res = gl_converter.render_to_image(dst, det, seg);
578                        let _ = resp.send(res);
579                    }
580                    GLProcessorMessage::SetColors(colors, resp) => {
581                        let res = gl_converter.set_class_colors(&colors);
582                        let _ = resp.send(res);
583                    }
584                }
585            }
586        };
587
588        // let handle = tokio::task::spawn(func());
589        let handle = std::thread::spawn(func);
590
591        let support_dma = match create_ctx_recv.blocking_recv() {
592            Ok(Err(e)) => return Err(e),
593            Err(_) => {
594                return Err(Error::Internal(
595                    "GL converter error messaging closed without update".to_string(),
596                ));
597            }
598            Ok(Ok(supports_dma)) => supports_dma,
599        };
600
601        Ok(Self {
602            handle: Some(handle),
603            sender: Some(send),
604            support_dma,
605        })
606    }
607}
608
609impl ImageProcessorTrait for GLProcessorThreaded {
610    fn convert(
611        &mut self,
612        src: &TensorImage,
613        dst: &mut TensorImage,
614        rotation: crate::Rotation,
615        flip: Flip,
616        crop: Crop,
617    ) -> crate::Result<()> {
618        crop.check_crop(src, dst)?;
619        if !GLProcessorST::check_src_format_supported(self.support_dma, src) {
620            return Err(crate::Error::NotSupported(format!(
621                "Opengl doesn't support {} source texture",
622                src.fourcc().display()
623            )));
624        }
625
626        if !GLProcessorST::check_dst_format_supported(self.support_dma, dst) {
627            return Err(crate::Error::NotSupported(format!(
628                "Opengl doesn't support {} destination texture",
629                dst.fourcc().display()
630            )));
631        }
632
633        let (err_send, err_recv) = tokio::sync::oneshot::channel();
634        self.sender
635            .as_ref()
636            .unwrap()
637            .blocking_send(GLProcessorMessage::ImageConvert(
638                SendablePtr {
639                    ptr: src.into(),
640                    len: 1,
641                },
642                SendablePtr {
643                    ptr: dst.into(),
644                    len: 1,
645                },
646                rotation,
647                flip,
648                crop,
649                err_send,
650            ))
651            .map_err(|_| Error::Internal("GL converter thread exited".to_string()))?;
652        err_recv.blocking_recv().map_err(|_| {
653            Error::Internal("GL converter error messaging closed without update".to_string())
654        })?
655    }
656
657    fn convert_ref(
658        &mut self,
659        src: &TensorImage,
660        dst: &mut TensorImageRef<'_>,
661        rotation: Rotation,
662        flip: Flip,
663        crop: Crop,
664    ) -> crate::Result<()> {
665        // OpenGL doesn't support PLANAR_RGB output, delegate to CPU
666        let mut cpu = CPUProcessor::new();
667        cpu.convert_ref(src, dst, rotation, flip, crop)
668    }
669
670    #[cfg(feature = "decoder")]
671    fn render_to_image(
672        &mut self,
673        dst: &mut TensorImage,
674        detect: &[crate::DetectBox],
675        segmentation: &[crate::Segmentation],
676    ) -> crate::Result<()> {
677        let (err_send, err_recv) = tokio::sync::oneshot::channel();
678        self.sender
679            .as_ref()
680            .unwrap()
681            .blocking_send(GLProcessorMessage::ImageRender(
682                SendablePtr {
683                    ptr: dst.into(),
684                    len: 1,
685                },
686                SendablePtr {
687                    ptr: NonNull::new(detect.as_ptr() as *mut DetectBox).unwrap(),
688                    len: detect.len(),
689                },
690                SendablePtr {
691                    ptr: NonNull::new(segmentation.as_ptr() as *mut Segmentation).unwrap(),
692                    len: segmentation.len(),
693                },
694                err_send,
695            ))
696            .map_err(|_| Error::Internal("GL converter thread exited".to_string()))?;
697        err_recv.blocking_recv().map_err(|_| {
698            Error::Internal("GL converter error messaging closed without update".to_string())
699        })?
700    }
701
702    #[cfg(feature = "decoder")]
703    fn set_class_colors(&mut self, colors: &[[u8; 4]]) -> Result<(), crate::Error> {
704        let (err_send, err_recv) = tokio::sync::oneshot::channel();
705        self.sender
706            .as_ref()
707            .unwrap()
708            .blocking_send(GLProcessorMessage::SetColors(colors.to_vec(), err_send))
709            .map_err(|_| Error::Internal("GL converter thread exited".to_string()))?;
710        err_recv.blocking_recv().map_err(|_| {
711            Error::Internal("GL converter error messaging closed without update".to_string())
712        })?
713    }
714}
715
716impl Drop for GLProcessorThreaded {
717    fn drop(&mut self) {
718        drop(self.sender.take());
719        let _ = self.handle.take().and_then(|h| h.join().ok());
720    }
721}
722
723/// OpenGL single-threaded image converter.
724pub struct GLProcessorST {
725    camera_eglimage_texture: Texture,
726    camera_normal_texture: Texture,
727    render_texture: Texture,
728    #[cfg(feature = "decoder")]
729    segmentation_texture: Texture,
730    #[cfg(feature = "decoder")]
731    segmentation_program: GlProgram,
732    #[cfg(feature = "decoder")]
733    instanced_segmentation_program: GlProgram,
734    #[cfg(feature = "decoder")]
735    color_program: GlProgram,
736    vertex_buffer: Buffer,
737    texture_buffer: Buffer,
738    texture_program: GlProgram,
739    texture_program_yuv: GlProgram,
740    texture_program_planar: GlProgram,
741    gl_context: GlContext,
742}
743
744impl ImageProcessorTrait for GLProcessorST {
745    fn convert(
746        &mut self,
747        src: &TensorImage,
748        dst: &mut TensorImage,
749        rotation: crate::Rotation,
750        flip: Flip,
751        crop: Crop,
752    ) -> crate::Result<()> {
753        crop.check_crop(src, dst)?;
754        if !Self::check_src_format_supported(self.gl_context.support_dma, src) {
755            return Err(crate::Error::NotSupported(format!(
756                "Opengl doesn't support {} source texture",
757                src.fourcc().display()
758            )));
759        }
760
761        if !Self::check_dst_format_supported(self.gl_context.support_dma, dst) {
762            return Err(crate::Error::NotSupported(format!(
763                "Opengl doesn't support {} destination texture",
764                dst.fourcc().display()
765            )));
766        }
767        log::debug!(
768            "dst tensor: {:?} src tensor :{:?}",
769            dst.tensor().memory(),
770            src.tensor().memory()
771        );
772        check_gl_error(function!(), line!())?;
773        if self.gl_context.support_dma
774            && dst.tensor().memory() == TensorMemory::Dma
775            && dst.fourcc() != RGB
776        // DMA generally doesn't support RGB
777        {
778            let res = self.convert_dest_dma(dst, src, rotation, flip, crop);
779            return res;
780        }
781        let start = Instant::now();
782        let res = self.convert_dest_non_dma(dst, src, rotation, flip, crop);
783        log::debug!("convert_dest_non_dma takes {:?}", start.elapsed());
784        res
785    }
786
787    fn convert_ref(
788        &mut self,
789        src: &TensorImage,
790        dst: &mut TensorImageRef<'_>,
791        rotation: Rotation,
792        flip: Flip,
793        crop: Crop,
794    ) -> crate::Result<()> {
795        // OpenGL doesn't support PLANAR_RGB output, delegate to CPU
796        let mut cpu = CPUProcessor::new();
797        cpu.convert_ref(src, dst, rotation, flip, crop)
798    }
799
800    #[cfg(feature = "decoder")]
801    fn render_to_image(
802        &mut self,
803        dst: &mut TensorImage,
804        detect: &[DetectBox],
805        segmentation: &[Segmentation],
806    ) -> Result<(), crate::Error> {
807        use crate::FunctionTimer;
808
809        let _timer = FunctionTimer::new("GLProcessorST::render_to_image");
810        if !matches!(dst.fourcc(), RGBA | RGB) {
811            return Err(crate::Error::NotSupported(
812                "Opengl image rendering only supports RGBA or RGB images".to_string(),
813            ));
814        }
815
816        let (_render_buffer, is_dma) = match dst.tensor.memory() {
817            edgefirst_tensor::TensorMemory::Dma => {
818                if let Ok(render_buffer) = self.setup_renderbuffer_dma(dst) {
819                    (render_buffer, true)
820                } else {
821                    (
822                        self.setup_renderbuffer_non_dma(
823                            dst,
824                            Crop::new().with_dst_rect(Some(Rect::new(0, 0, 0, 0))),
825                        )?,
826                        false,
827                    )
828                }
829            }
830            _ => (
831                self.setup_renderbuffer_non_dma(
832                    dst,
833                    Crop::new().with_dst_rect(Some(Rect::new(0, 0, 0, 0))),
834                )?,
835                false,
836            ), // Add dest rect to make sure dst is rendered fully
837        };
838
839        gls::enable(gls::gl::BLEND);
840        gls::blend_func_separate(
841            gls::gl::SRC_ALPHA,
842            gls::gl::ONE_MINUS_SRC_ALPHA,
843            gls::gl::ZERO,
844            gls::gl::ONE,
845        );
846
847        self.render_box(dst, detect)?;
848        self.render_segmentation(detect, segmentation)?;
849
850        gls::finish();
851        if !is_dma {
852            let mut dst_map = dst.tensor().map()?;
853            let format = match dst.fourcc() {
854                RGB => gls::gl::RGB,
855                RGBA => gls::gl::RGBA,
856                _ => unreachable!(),
857            };
858            unsafe {
859                gls::gl::ReadBuffer(gls::gl::COLOR_ATTACHMENT0);
860                gls::gl::ReadnPixels(
861                    0,
862                    0,
863                    dst.width() as i32,
864                    dst.height() as i32,
865                    format,
866                    gls::gl::UNSIGNED_BYTE,
867                    dst.tensor.len() as i32,
868                    dst_map.as_mut_ptr() as *mut c_void,
869                );
870            }
871        }
872
873        Ok(())
874    }
875
876    #[cfg(feature = "decoder")]
877    fn set_class_colors(&mut self, colors: &[[u8; 4]]) -> crate::Result<()> {
878        if colors.is_empty() {
879            return Ok(());
880        }
881        let mut colors_f32 = colors
882            .iter()
883            .map(|c| {
884                [
885                    c[0] as f32 / 255.0,
886                    c[1] as f32 / 255.0,
887                    c[2] as f32 / 255.0,
888                    c[3] as f32 / 255.0,
889                ]
890            })
891            .take(20)
892            .collect::<Vec<[f32; 4]>>();
893
894        self.segmentation_program
895            .load_uniform_4fv(c"colors", &colors_f32)?;
896        self.instanced_segmentation_program
897            .load_uniform_4fv(c"colors", &colors_f32)?;
898
899        colors_f32.iter_mut().for_each(|c| {
900            c[3] = 1.0; // set alpha to 1.0 for color rendering
901        });
902        self.color_program
903            .load_uniform_4fv(c"colors", &colors_f32)?;
904
905        Ok(())
906    }
907}
908
909impl GLProcessorST {
910    pub fn new() -> Result<GLProcessorST, crate::Error> {
911        let gl_context = GlContext::new()?;
912        gls::load_with(|s| {
913            gl_context
914                .egl
915                .get_proc_address(s)
916                .map_or(std::ptr::null(), |p| p as *const _)
917        });
918
919        Self::gl_check_support()?;
920
921        // Uploads and downloads are all packed with no alignment requirements
922        unsafe {
923            gls::gl::PixelStorei(gls::gl::PACK_ALIGNMENT, 1);
924            gls::gl::PixelStorei(gls::gl::UNPACK_ALIGNMENT, 1);
925        }
926
927        let texture_program_planar =
928            GlProgram::new(generate_vertex_shader(), generate_planar_rgb_shader())?;
929
930        let texture_program =
931            GlProgram::new(generate_vertex_shader(), generate_texture_fragment_shader())?;
932
933        let texture_program_yuv = GlProgram::new(
934            generate_vertex_shader(),
935            generate_texture_fragment_shader_yuv(),
936        )?;
937
938        #[cfg(feature = "decoder")]
939        let segmentation_program =
940            GlProgram::new(generate_vertex_shader(), generate_segmentation_shader())?;
941        #[cfg(feature = "decoder")]
942        segmentation_program.load_uniform_4fv(c"colors", &DEFAULT_COLORS)?;
943        #[cfg(feature = "decoder")]
944        let instanced_segmentation_program = GlProgram::new(
945            generate_vertex_shader(),
946            generate_instanced_segmentation_shader(),
947        )?;
948        #[cfg(feature = "decoder")]
949        instanced_segmentation_program.load_uniform_4fv(c"colors", &DEFAULT_COLORS)?;
950
951        #[cfg(feature = "decoder")]
952        let color_program = GlProgram::new(generate_vertex_shader(), generate_color_shader())?;
953        #[cfg(feature = "decoder")]
954        color_program.load_uniform_4fv(c"colors", &DEFAULT_COLORS)?;
955
956        let camera_eglimage_texture = Texture::new();
957        let camera_normal_texture = Texture::new();
958        let render_texture = Texture::new();
959        let segmentation_texture = Texture::new();
960        let vertex_buffer = Buffer::new(0, 3, 100);
961        let texture_buffer = Buffer::new(1, 2, 100);
962
963        let converter = GLProcessorST {
964            gl_context,
965            texture_program,
966            texture_program_yuv,
967            texture_program_planar,
968            camera_eglimage_texture,
969            camera_normal_texture,
970            #[cfg(feature = "decoder")]
971            segmentation_texture,
972            vertex_buffer,
973            texture_buffer,
974            render_texture,
975            #[cfg(feature = "decoder")]
976            segmentation_program,
977            #[cfg(feature = "decoder")]
978            instanced_segmentation_program,
979            #[cfg(feature = "decoder")]
980            color_program,
981        };
982        check_gl_error(function!(), line!())?;
983
984        log::debug!("GLConverter created");
985        Ok(converter)
986    }
987
988    fn check_src_format_supported(support_dma: bool, img: &TensorImage) -> bool {
989        if support_dma && img.tensor().memory() == TensorMemory::Dma {
990            // EGLImage supports RGBA, GREY, YUYV, and NV12 for DMA buffers
991            matches!(img.fourcc(), RGBA | GREY | YUYV | NV12)
992        } else {
993            matches!(img.fourcc(), RGB | RGBA | GREY)
994        }
995    }
996
997    fn check_dst_format_supported(support_dma: bool, img: &TensorImage) -> bool {
998        if support_dma && img.tensor().memory() == TensorMemory::Dma {
999            // generally EGLImage doesn't support RGB
1000            matches!(img.fourcc(), RGBA | GREY | PLANAR_RGB)
1001        } else {
1002            matches!(img.fourcc(), RGB | RGBA | GREY)
1003        }
1004    }
1005
1006    fn gl_check_support() -> Result<(), crate::Error> {
1007        if let Ok(version) = gls::get_string(gls::gl::SHADING_LANGUAGE_VERSION) {
1008            log::debug!("GL Shading Language Version: {version:?}");
1009        } else {
1010            log::warn!("Could not get GL Shading Language Version");
1011        }
1012
1013        let extensions = unsafe {
1014            let str = gls::gl::GetString(gls::gl::EXTENSIONS);
1015            if str.is_null() {
1016                return Err(crate::Error::GLVersion(
1017                    "GL returned no supported extensions".to_string(),
1018                ));
1019            }
1020            CStr::from_ptr(str as *const c_char)
1021                .to_string_lossy()
1022                .to_string()
1023        };
1024        log::debug!("GL Extensions: {extensions}");
1025        let required_ext = [
1026            "GL_OES_EGL_image_external_essl3",
1027            "GL_OES_surfaceless_context",
1028        ];
1029        let extensions = extensions.split_ascii_whitespace().collect::<BTreeSet<_>>();
1030        for required in required_ext {
1031            if !extensions.contains(required) {
1032                return Err(crate::Error::GLVersion(format!(
1033                    "GL does not support {required} extension",
1034                )));
1035            }
1036        }
1037
1038        Ok(())
1039    }
1040
1041    fn setup_renderbuffer_dma(&mut self, dst: &TensorImage) -> crate::Result<FrameBuffer> {
1042        let frame_buffer = FrameBuffer::new();
1043        frame_buffer.bind();
1044
1045        let (width, height) = if matches!(dst.fourcc(), PLANAR_RGB) {
1046            let width = dst.width();
1047            let height = dst.height() * 3;
1048            (width as i32, height as i32)
1049        } else {
1050            (dst.width() as i32, dst.height() as i32)
1051        };
1052        let dest_img = self.create_image_from_dma2(dst)?;
1053        unsafe {
1054            gls::gl::UseProgram(self.texture_program_yuv.id);
1055            gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1056            gls::gl::BindTexture(gls::gl::TEXTURE_2D, self.render_texture.id);
1057            gls::gl::TexParameteri(
1058                gls::gl::TEXTURE_2D,
1059                gls::gl::TEXTURE_MIN_FILTER,
1060                gls::gl::LINEAR as i32,
1061            );
1062            gls::gl::TexParameteri(
1063                gls::gl::TEXTURE_2D,
1064                gls::gl::TEXTURE_MAG_FILTER,
1065                gls::gl::LINEAR as i32,
1066            );
1067            gls::gl::EGLImageTargetTexture2DOES(gls::gl::TEXTURE_2D, dest_img.egl_image.as_ptr());
1068            gls::gl::FramebufferTexture2D(
1069                gls::gl::FRAMEBUFFER,
1070                gls::gl::COLOR_ATTACHMENT0,
1071                gls::gl::TEXTURE_2D,
1072                self.render_texture.id,
1073                0,
1074            );
1075            check_gl_error(function!(), line!())?;
1076            gls::gl::Viewport(0, 0, width, height);
1077        }
1078        Ok(frame_buffer)
1079    }
1080
1081    fn convert_dest_dma(
1082        &mut self,
1083        dst: &mut TensorImage,
1084        src: &TensorImage,
1085        rotation: crate::Rotation,
1086        flip: Flip,
1087        crop: Crop,
1088    ) -> crate::Result<()> {
1089        assert!(self.gl_context.support_dma);
1090        let _framebuffer = self.setup_renderbuffer_dma(dst)?;
1091        if dst.is_planar() {
1092            self.convert_to_planar(src, dst, rotation, flip, crop)
1093        } else {
1094            self.convert_to(src, dst, rotation, flip, crop)
1095        }
1096    }
1097
1098    fn setup_renderbuffer_non_dma(
1099        &mut self,
1100        dst: &TensorImage,
1101        crop: Crop,
1102    ) -> crate::Result<FrameBuffer> {
1103        debug_assert!(matches!(dst.fourcc(), RGB | RGBA | GREY | PLANAR_RGB));
1104        let (width, height) = if dst.is_planar() {
1105            let width = dst.width() / 4;
1106            let height = match dst.fourcc() {
1107                RGBA => dst.height() * 4,
1108                RGB => dst.height() * 3,
1109                GREY => dst.height(),
1110                _ => unreachable!(),
1111            };
1112            (width as i32, height as i32)
1113        } else {
1114            (dst.width() as i32, dst.height() as i32)
1115        };
1116
1117        let format = if dst.is_planar() {
1118            gls::gl::RED
1119        } else {
1120            match dst.fourcc() {
1121                RGB => gls::gl::RGB,
1122                RGBA => gls::gl::RGBA,
1123                GREY => gls::gl::RED,
1124                _ => unreachable!(),
1125            }
1126        };
1127
1128        let start = Instant::now();
1129        let frame_buffer = FrameBuffer::new();
1130        frame_buffer.bind();
1131
1132        let map;
1133
1134        let pixels = if crop.dst_rect.is_none_or(|crop| {
1135            crop.top == 0
1136                && crop.left == 0
1137                && crop.height == dst.height()
1138                && crop.width == dst.width()
1139        }) {
1140            std::ptr::null()
1141        } else {
1142            map = dst.tensor().map()?;
1143            map.as_ptr() as *const c_void
1144        };
1145        unsafe {
1146            gls::gl::UseProgram(self.texture_program.id);
1147            gls::gl::BindTexture(gls::gl::TEXTURE_2D, self.render_texture.id);
1148            gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1149            gls::gl::TexParameteri(
1150                gls::gl::TEXTURE_2D,
1151                gls::gl::TEXTURE_MIN_FILTER,
1152                gls::gl::LINEAR as i32,
1153            );
1154            gls::gl::TexParameteri(
1155                gls::gl::TEXTURE_2D,
1156                gls::gl::TEXTURE_MAG_FILTER,
1157                gls::gl::LINEAR as i32,
1158            );
1159
1160            gls::gl::TexImage2D(
1161                gls::gl::TEXTURE_2D,
1162                0,
1163                format as i32,
1164                width,
1165                height,
1166                0,
1167                format,
1168                gls::gl::UNSIGNED_BYTE,
1169                pixels,
1170            );
1171            check_gl_error(function!(), line!())?;
1172            gls::gl::FramebufferTexture2D(
1173                gls::gl::FRAMEBUFFER,
1174                gls::gl::COLOR_ATTACHMENT0,
1175                gls::gl::TEXTURE_2D,
1176                self.render_texture.id,
1177                0,
1178            );
1179            check_gl_error(function!(), line!())?;
1180            gls::gl::Viewport(0, 0, width, height);
1181        }
1182        log::debug!("Set up framebuffer takes {:?}", start.elapsed());
1183        Ok(frame_buffer)
1184    }
1185
1186    fn convert_dest_non_dma(
1187        &mut self,
1188        dst: &mut TensorImage,
1189        src: &TensorImage,
1190        rotation: crate::Rotation,
1191        flip: Flip,
1192        crop: Crop,
1193    ) -> crate::Result<()> {
1194        let _framebuffer = self.setup_renderbuffer_non_dma(dst, crop)?;
1195        let start = Instant::now();
1196        if dst.is_planar() {
1197            self.convert_to_planar(src, dst, rotation, flip, crop)?;
1198        } else {
1199            self.convert_to(src, dst, rotation, flip, crop)?;
1200        }
1201        log::debug!("Draw to framebuffer takes {:?}", start.elapsed());
1202        let start = Instant::now();
1203        let dest_format = match dst.fourcc() {
1204            RGB => gls::gl::RGB,
1205            RGBA => gls::gl::RGBA,
1206            GREY => gls::gl::RED,
1207            _ => unreachable!(),
1208        };
1209
1210        unsafe {
1211            let mut dst_map = dst.tensor().map()?;
1212            gls::gl::ReadBuffer(gls::gl::COLOR_ATTACHMENT0);
1213            gls::gl::ReadnPixels(
1214                0,
1215                0,
1216                dst.width() as i32,
1217                dst.height() as i32,
1218                dest_format,
1219                gls::gl::UNSIGNED_BYTE,
1220                dst.tensor.len() as i32,
1221                dst_map.as_mut_ptr() as *mut c_void,
1222            );
1223        }
1224        log::debug!("Read from framebuffer takes {:?}", start.elapsed());
1225        Ok(())
1226    }
1227
1228    fn convert_to(
1229        &mut self,
1230        src: &TensorImage,
1231        dst: &TensorImage,
1232        rotation: crate::Rotation,
1233        flip: Flip,
1234        crop: Crop,
1235    ) -> Result<(), crate::Error> {
1236        check_gl_error(function!(), line!())?;
1237
1238        let has_crop = crop.dst_rect.is_some_and(|x| {
1239            x.left != 0 || x.top != 0 || x.width != dst.width() || x.height != dst.height()
1240        });
1241        if has_crop {
1242            if let Some(dst_color) = crop.dst_color {
1243                unsafe {
1244                    gls::gl::ClearColor(
1245                        dst_color[0] as f32 / 255.0,
1246                        dst_color[1] as f32 / 255.0,
1247                        dst_color[2] as f32 / 255.0,
1248                        dst_color[3] as f32 / 255.0,
1249                    );
1250                    gls::gl::Clear(gls::gl::COLOR_BUFFER_BIT);
1251                };
1252            }
1253        }
1254
1255        // top and bottom are flipped because OpenGL uses 0,0 as bottom left
1256        let src_roi = if let Some(crop) = crop.src_rect {
1257            RegionOfInterest {
1258                left: crop.left as f32 / src.width() as f32,
1259                top: (crop.top + crop.height) as f32 / src.height() as f32,
1260                right: (crop.left + crop.width) as f32 / src.width() as f32,
1261                bottom: crop.top as f32 / src.height() as f32,
1262            }
1263        } else {
1264            RegionOfInterest {
1265                left: 0.,
1266                top: 1.,
1267                right: 1.,
1268                bottom: 0.,
1269            }
1270        };
1271
1272        // top and bottom are flipped because OpenGL uses 0,0 as bottom left
1273        let cvt_screen_coord = |normalized| normalized * 2.0 - 1.0;
1274        let dst_roi = if let Some(crop) = crop.dst_rect {
1275            RegionOfInterest {
1276                left: cvt_screen_coord(crop.left as f32 / dst.width() as f32),
1277                top: cvt_screen_coord((crop.top + crop.height) as f32 / dst.height() as f32),
1278                right: cvt_screen_coord((crop.left + crop.width) as f32 / dst.width() as f32),
1279                bottom: cvt_screen_coord(crop.top as f32 / dst.height() as f32),
1280            }
1281        } else {
1282            RegionOfInterest {
1283                left: -1.,
1284                top: 1.,
1285                right: 1.,
1286                bottom: -1.,
1287            }
1288        };
1289        let rotation_offset = match rotation {
1290            crate::Rotation::None => 0,
1291            crate::Rotation::Clockwise90 => 1,
1292            crate::Rotation::Rotate180 => 2,
1293            crate::Rotation::CounterClockwise90 => 3,
1294        };
1295        if self.gl_context.support_dma && src.tensor().memory() == TensorMemory::Dma {
1296            match self.create_image_from_dma2(src) {
1297                Ok(new_egl_image) => self.draw_camera_texture_eglimage(
1298                    src,
1299                    &new_egl_image,
1300                    src_roi,
1301                    dst_roi,
1302                    rotation_offset,
1303                    flip,
1304                )?,
1305                Err(e) => {
1306                    log::warn!("EGL image creation failed for {:?}: {:?}", src.fourcc(), e);
1307                    let start = Instant::now();
1308                    self.draw_src_texture(src, src_roi, dst_roi, rotation_offset, flip)?;
1309                    log::debug!("draw_src_texture takes {:?}", start.elapsed());
1310                }
1311            }
1312        } else {
1313            let start = Instant::now();
1314            self.draw_src_texture(src, src_roi, dst_roi, rotation_offset, flip)?;
1315            log::debug!("draw_src_texture takes {:?}", start.elapsed());
1316        }
1317
1318        let start = Instant::now();
1319        unsafe { gls::gl::Finish() };
1320        log::debug!("gl_Finish takes {:?}", start.elapsed());
1321        check_gl_error(function!(), line!())?;
1322        Ok(())
1323    }
1324
1325    fn convert_to_planar(
1326        &self,
1327        src: &TensorImage,
1328        dst: &TensorImage,
1329        rotation: crate::Rotation,
1330        flip: Flip,
1331        crop: Crop,
1332    ) -> Result<(), crate::Error> {
1333        // if let Some(crop) = crop.src_rect
1334        //     && (crop.left > 0
1335        //         || crop.top > 0
1336        //         || crop.height < src.height()
1337        //         || crop.width < src.width())
1338        // {
1339        //     return Err(crate::Error::NotSupported(
1340        //         "Cropping in planar RGB mode is not supported".to_string(),
1341        //     ));
1342        // }
1343
1344        // if let Some(crop) = crop.dst_rect
1345        //     && (crop.left > 0
1346        //         || crop.top > 0
1347        //         || crop.height < src.height()
1348        //         || crop.width < src.width())
1349        // {
1350        //     return Err(crate::Error::NotSupported(
1351        //         "Cropping in planar RGB mode is not supported".to_string(),
1352        //     ));
1353        // }
1354
1355        let alpha = match dst.fourcc() {
1356            PLANAR_RGB => false,
1357            PLANAR_RGBA => true,
1358            _ => {
1359                return Err(crate::Error::NotSupported(
1360                    "Destination format must be PLANAR_RGB or PLANAR_RGBA".to_string(),
1361                ));
1362            }
1363        };
1364
1365        // top and bottom are flipped because OpenGL uses 0,0 as bottom left
1366        let src_roi = if let Some(crop) = crop.src_rect {
1367            RegionOfInterest {
1368                left: crop.left as f32 / src.width() as f32,
1369                top: (crop.top + crop.height) as f32 / src.height() as f32,
1370                right: (crop.left + crop.width) as f32 / src.width() as f32,
1371                bottom: crop.top as f32 / src.height() as f32,
1372            }
1373        } else {
1374            RegionOfInterest {
1375                left: 0.,
1376                top: 1.,
1377                right: 1.,
1378                bottom: 0.,
1379            }
1380        };
1381
1382        // top and bottom are flipped because OpenGL uses 0,0 as bottom left
1383        let cvt_screen_coord = |normalized| normalized * 2.0 - 1.0;
1384        let dst_roi = if let Some(crop) = crop.dst_rect {
1385            RegionOfInterest {
1386                left: cvt_screen_coord(crop.left as f32 / dst.width() as f32),
1387                top: cvt_screen_coord((crop.top + crop.height) as f32 / dst.height() as f32),
1388                right: cvt_screen_coord((crop.left + crop.width) as f32 / dst.width() as f32),
1389                bottom: cvt_screen_coord(crop.top as f32 / dst.height() as f32),
1390            }
1391        } else {
1392            RegionOfInterest {
1393                left: -1.,
1394                top: 1.,
1395                right: 1.,
1396                bottom: -1.,
1397            }
1398        };
1399        let rotation_offset = match rotation {
1400            crate::Rotation::None => 0,
1401            crate::Rotation::Clockwise90 => 1,
1402            crate::Rotation::Rotate180 => 2,
1403            crate::Rotation::CounterClockwise90 => 3,
1404        };
1405
1406        let has_crop = crop.dst_rect.is_some_and(|x| {
1407            x.left != 0 || x.top != 0 || x.width != dst.width() || x.height != dst.height()
1408        });
1409        if has_crop {
1410            if let Some(dst_color) = crop.dst_color {
1411                self.clear_rect_planar(
1412                    dst.width(),
1413                    dst.height(),
1414                    dst_roi,
1415                    [
1416                        dst_color[0] as f32 / 255.0,
1417                        dst_color[1] as f32 / 255.0,
1418                        dst_color[2] as f32 / 255.0,
1419                        dst_color[3] as f32 / 255.0,
1420                    ],
1421                    alpha,
1422                )?;
1423            }
1424        }
1425
1426        let new_egl_image = self.create_image_from_dma2(src)?;
1427
1428        self.draw_camera_texture_to_rgb_planar(
1429            &new_egl_image,
1430            src_roi,
1431            dst_roi,
1432            rotation_offset,
1433            flip,
1434            alpha,
1435        )?;
1436        unsafe { gls::gl::Finish() };
1437
1438        Ok(())
1439    }
1440
1441    fn clear_rect_planar(
1442        &self,
1443        width: usize,
1444        height: usize,
1445        dst_roi: RegionOfInterest,
1446        color: [f32; 4],
1447        alpha: bool,
1448    ) -> Result<(), Error> {
1449        if !alpha && color[0] == color[1] && color[1] == color[2] {
1450            unsafe {
1451                gls::gl::ClearColor(color[0], color[0], color[0], 1.0);
1452                gls::gl::Clear(gls::gl::COLOR_BUFFER_BIT);
1453            };
1454        }
1455
1456        let split = if alpha { 4 } else { 3 };
1457
1458        unsafe {
1459            gls::gl::Enable(gls::gl::SCISSOR_TEST);
1460            let x = (((dst_roi.left + 1.0) / 2.0) * width as f32).round() as i32;
1461            let y = (((dst_roi.bottom + 1.0) / 2.0) * height as f32).round() as i32;
1462            let width = (((dst_roi.right - dst_roi.left) / 2.0) * width as f32).round() as i32;
1463            let height = (((dst_roi.top - dst_roi.bottom) / 2.0) * height as f32 / split as f32)
1464                .round() as i32;
1465            for (i, c) in color.iter().enumerate().take(split) {
1466                gls::gl::Scissor(x, y + i as i32 * height, width, height);
1467                gls::gl::ClearColor(*c, *c, *c, 1.0);
1468                gls::gl::Clear(gls::gl::COLOR_BUFFER_BIT);
1469            }
1470            gls::gl::Disable(gls::gl::SCISSOR_TEST);
1471        }
1472        Ok(())
1473    }
1474
1475    #[allow(clippy::too_many_arguments)]
1476    fn draw_camera_texture_to_rgb_planar(
1477        &self,
1478        egl_img: &EglImage,
1479        src_roi: RegionOfInterest,
1480        mut dst_roi: RegionOfInterest,
1481        rotation_offset: usize,
1482        flip: Flip,
1483        alpha: bool,
1484    ) -> Result<(), Error> {
1485        let texture_target = gls::gl::TEXTURE_EXTERNAL_OES;
1486        match flip {
1487            Flip::None => {}
1488            Flip::Vertical => {
1489                std::mem::swap(&mut dst_roi.top, &mut dst_roi.bottom);
1490            }
1491            Flip::Horizontal => {
1492                std::mem::swap(&mut dst_roi.left, &mut dst_roi.right);
1493            }
1494        }
1495        unsafe {
1496            // self.texture_program.load_uniform_1f(c"width", width as f32);
1497            gls::gl::UseProgram(self.texture_program_planar.id);
1498            gls::gl::BindTexture(texture_target, self.camera_eglimage_texture.id);
1499            gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1500            gls::gl::TexParameteri(
1501                texture_target,
1502                gls::gl::TEXTURE_MIN_FILTER,
1503                gls::gl::LINEAR as i32,
1504            );
1505            gls::gl::TexParameteri(
1506                texture_target,
1507                gls::gl::TEXTURE_MAG_FILTER,
1508                gls::gl::LINEAR as i32,
1509            );
1510            gls::gl::TexParameteri(
1511                texture_target,
1512                gls::gl::TEXTURE_WRAP_S,
1513                gls::gl::CLAMP_TO_EDGE as i32,
1514            );
1515
1516            gls::gl::TexParameteri(
1517                texture_target,
1518                gls::gl::TEXTURE_WRAP_T,
1519                gls::gl::CLAMP_TO_EDGE as i32,
1520            );
1521
1522            gls::egl_image_target_texture_2d_oes(texture_target, egl_img.egl_image.as_ptr());
1523            check_gl_error(function!(), line!())?;
1524            let y_centers = if alpha {
1525                vec![-3.0 / 4.0, -1.0 / 4.0, 1.0 / 4.0, 3.0 / 4.0]
1526            } else {
1527                vec![-2.0 / 3.0, 0.0, 2.0 / 3.0]
1528            };
1529            let swizzles = [gls::gl::RED, gls::gl::GREEN, gls::gl::BLUE, gls::gl::ALPHA];
1530            // starts from bottom
1531            for (i, y_center) in y_centers.iter().enumerate() {
1532                gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
1533                gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
1534                let camera_vertices: [f32; 12] = [
1535                    dst_roi.left,
1536                    dst_roi.top / 3.0 + y_center,
1537                    0., // left top
1538                    dst_roi.right,
1539                    dst_roi.top / 3.0 + y_center,
1540                    0., // right top
1541                    dst_roi.right,
1542                    dst_roi.bottom / 3.0 + y_center,
1543                    0., // right bottom
1544                    dst_roi.left,
1545                    dst_roi.bottom / 3.0 + y_center,
1546                    0., // left bottom
1547                ];
1548                gls::gl::BufferData(
1549                    gls::gl::ARRAY_BUFFER,
1550                    (size_of::<f32>() * camera_vertices.len()) as isize,
1551                    camera_vertices.as_ptr() as *const c_void,
1552                    gls::gl::DYNAMIC_DRAW,
1553                );
1554
1555                gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
1556                gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
1557                let texture_vertices: [f32; 16] = [
1558                    src_roi.left,
1559                    src_roi.top,
1560                    src_roi.right,
1561                    src_roi.top,
1562                    src_roi.right,
1563                    src_roi.bottom,
1564                    src_roi.left,
1565                    src_roi.bottom,
1566                    src_roi.left,
1567                    src_roi.top,
1568                    src_roi.right,
1569                    src_roi.top,
1570                    src_roi.right,
1571                    src_roi.bottom,
1572                    src_roi.left,
1573                    src_roi.bottom,
1574                ];
1575
1576                gls::gl::BufferData(
1577                    gls::gl::ARRAY_BUFFER,
1578                    (size_of::<f32>() * 8) as isize,
1579                    (texture_vertices[(rotation_offset * 2)..]).as_ptr() as *const c_void,
1580                    gls::gl::DYNAMIC_DRAW,
1581                );
1582                let vertices_index: [u32; 4] = [0, 1, 2, 3];
1583                // self.texture_program_planar
1584                //     .load_uniform_1i(c"color_index", 2 - i as i32);
1585
1586                gls::gl::TexParameteri(
1587                    texture_target,
1588                    gls::gl::TEXTURE_SWIZZLE_R,
1589                    swizzles[i] as i32,
1590                );
1591
1592                gls::gl::DrawElements(
1593                    gls::gl::TRIANGLE_FAN,
1594                    vertices_index.len() as i32,
1595                    gls::gl::UNSIGNED_INT,
1596                    vertices_index.as_ptr() as *const c_void,
1597                );
1598            }
1599            check_gl_error(function!(), line!())?;
1600        }
1601        Ok(())
1602    }
1603
1604    fn draw_src_texture(
1605        &mut self,
1606        src: &TensorImage,
1607        src_roi: RegionOfInterest,
1608        mut dst_roi: RegionOfInterest,
1609        rotation_offset: usize,
1610        flip: Flip,
1611    ) -> Result<(), Error> {
1612        let texture_target = gls::gl::TEXTURE_2D;
1613        let texture_format = match src.fourcc() {
1614            RGB => gls::gl::RGB,
1615            RGBA => gls::gl::RGBA,
1616            GREY => gls::gl::RED,
1617            _ => {
1618                return Err(Error::NotSupported(format!(
1619                    "draw_src_texture does not support {:?} (use DMA-BUF path for YUV)",
1620                    src.fourcc()
1621                )));
1622            }
1623        };
1624        unsafe {
1625            gls::gl::UseProgram(self.texture_program.id);
1626            gls::gl::BindTexture(texture_target, self.camera_normal_texture.id);
1627            gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1628            gls::gl::TexParameteri(
1629                texture_target,
1630                gls::gl::TEXTURE_MIN_FILTER,
1631                gls::gl::LINEAR as i32,
1632            );
1633            gls::gl::TexParameteri(
1634                texture_target,
1635                gls::gl::TEXTURE_MAG_FILTER,
1636                gls::gl::LINEAR as i32,
1637            );
1638            if src.fourcc() == GREY {
1639                for swizzle in [
1640                    gls::gl::TEXTURE_SWIZZLE_R,
1641                    gls::gl::TEXTURE_SWIZZLE_G,
1642                    gls::gl::TEXTURE_SWIZZLE_B,
1643                ] {
1644                    gls::gl::TexParameteri(gls::gl::TEXTURE_2D, swizzle, gls::gl::RED as i32);
1645                }
1646            } else {
1647                for (swizzle, src) in [
1648                    (gls::gl::TEXTURE_SWIZZLE_R, gls::gl::RED),
1649                    (gls::gl::TEXTURE_SWIZZLE_G, gls::gl::GREEN),
1650                    (gls::gl::TEXTURE_SWIZZLE_B, gls::gl::BLUE),
1651                ] {
1652                    gls::gl::TexParameteri(gls::gl::TEXTURE_2D, swizzle, src as i32);
1653                }
1654            }
1655            self.camera_normal_texture.update_texture(
1656                texture_target,
1657                src.width(),
1658                src.height(),
1659                texture_format,
1660                &src.tensor().map()?,
1661            );
1662
1663            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
1664            gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
1665
1666            match flip {
1667                Flip::None => {}
1668                Flip::Vertical => {
1669                    std::mem::swap(&mut dst_roi.top, &mut dst_roi.bottom);
1670                }
1671                Flip::Horizontal => {
1672                    std::mem::swap(&mut dst_roi.left, &mut dst_roi.right);
1673                }
1674            }
1675
1676            let camera_vertices: [f32; 12] = [
1677                dst_roi.left,
1678                dst_roi.top,
1679                0., // left top
1680                dst_roi.right,
1681                dst_roi.top,
1682                0., // right top
1683                dst_roi.right,
1684                dst_roi.bottom,
1685                0., // right bottom
1686                dst_roi.left,
1687                dst_roi.bottom,
1688                0., // left bottom
1689            ];
1690            gls::gl::BufferData(
1691                gls::gl::ARRAY_BUFFER,
1692                (size_of::<f32>() * camera_vertices.len()) as isize,
1693                camera_vertices.as_ptr() as *const c_void,
1694                gls::gl::DYNAMIC_DRAW,
1695            );
1696            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
1697            gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
1698            let texture_vertices: [f32; 16] = [
1699                src_roi.left,
1700                src_roi.top,
1701                src_roi.right,
1702                src_roi.top,
1703                src_roi.right,
1704                src_roi.bottom,
1705                src_roi.left,
1706                src_roi.bottom,
1707                src_roi.left,
1708                src_roi.top,
1709                src_roi.right,
1710                src_roi.top,
1711                src_roi.right,
1712                src_roi.bottom,
1713                src_roi.left,
1714                src_roi.bottom,
1715            ];
1716
1717            gls::gl::BufferData(
1718                gls::gl::ARRAY_BUFFER,
1719                (size_of::<f32>() * 8) as isize,
1720                (texture_vertices[(rotation_offset * 2)..]).as_ptr() as *const c_void,
1721                gls::gl::DYNAMIC_DRAW,
1722            );
1723            let vertices_index: [u32; 4] = [0, 1, 2, 3];
1724            gls::gl::DrawElements(
1725                gls::gl::TRIANGLE_FAN,
1726                vertices_index.len() as i32,
1727                gls::gl::UNSIGNED_INT,
1728                vertices_index.as_ptr() as *const c_void,
1729            );
1730            check_gl_error(function!(), line!())?;
1731
1732            Ok(())
1733        }
1734    }
1735
1736    fn draw_camera_texture_eglimage(
1737        &self,
1738        src: &TensorImage,
1739        egl_img: &EglImage,
1740        src_roi: RegionOfInterest,
1741        mut dst_roi: RegionOfInterest,
1742        rotation_offset: usize,
1743        flip: Flip,
1744    ) -> Result<(), Error> {
1745        // let texture_target = gls::gl::TEXTURE_2D;
1746        let texture_target = gls::gl::TEXTURE_EXTERNAL_OES;
1747        unsafe {
1748            gls::gl::UseProgram(self.texture_program_yuv.id);
1749            gls::gl::BindTexture(texture_target, self.camera_eglimage_texture.id);
1750            gls::gl::ActiveTexture(gls::gl::TEXTURE0);
1751            gls::gl::TexParameteri(
1752                texture_target,
1753                gls::gl::TEXTURE_MIN_FILTER,
1754                gls::gl::LINEAR as i32,
1755            );
1756            gls::gl::TexParameteri(
1757                texture_target,
1758                gls::gl::TEXTURE_MAG_FILTER,
1759                gls::gl::LINEAR as i32,
1760            );
1761
1762            if src.fourcc() == GREY {
1763                for swizzle in [
1764                    gls::gl::TEXTURE_SWIZZLE_R,
1765                    gls::gl::TEXTURE_SWIZZLE_G,
1766                    gls::gl::TEXTURE_SWIZZLE_B,
1767                ] {
1768                    gls::gl::TexParameteri(gls::gl::TEXTURE_2D, swizzle, gls::gl::RED as i32);
1769                }
1770            } else {
1771                for (swizzle, src) in [
1772                    (gls::gl::TEXTURE_SWIZZLE_R, gls::gl::RED),
1773                    (gls::gl::TEXTURE_SWIZZLE_G, gls::gl::GREEN),
1774                    (gls::gl::TEXTURE_SWIZZLE_B, gls::gl::BLUE),
1775                ] {
1776                    gls::gl::TexParameteri(gls::gl::TEXTURE_2D, swizzle, src as i32);
1777                }
1778            }
1779
1780            gls::egl_image_target_texture_2d_oes(texture_target, egl_img.egl_image.as_ptr());
1781            check_gl_error(function!(), line!())?;
1782            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
1783            gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
1784
1785            match flip {
1786                Flip::None => {}
1787                Flip::Vertical => {
1788                    std::mem::swap(&mut dst_roi.top, &mut dst_roi.bottom);
1789                }
1790                Flip::Horizontal => {
1791                    std::mem::swap(&mut dst_roi.left, &mut dst_roi.right);
1792                }
1793            }
1794
1795            let camera_vertices: [f32; 12] = [
1796                dst_roi.left,
1797                dst_roi.top,
1798                0., // left top
1799                dst_roi.right,
1800                dst_roi.top,
1801                0., // right top
1802                dst_roi.right,
1803                dst_roi.bottom,
1804                0., // right bottom
1805                dst_roi.left,
1806                dst_roi.bottom,
1807                0., // left bottom
1808            ];
1809            gls::gl::BufferSubData(
1810                gls::gl::ARRAY_BUFFER,
1811                0,
1812                (size_of::<f32>() * camera_vertices.len()) as isize,
1813                camera_vertices.as_ptr() as *const c_void,
1814            );
1815
1816            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
1817            gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
1818
1819            let texture_vertices: [f32; 16] = [
1820                src_roi.left,
1821                src_roi.top,
1822                src_roi.right,
1823                src_roi.top,
1824                src_roi.right,
1825                src_roi.bottom,
1826                src_roi.left,
1827                src_roi.bottom,
1828                src_roi.left,
1829                src_roi.top,
1830                src_roi.right,
1831                src_roi.top,
1832                src_roi.right,
1833                src_roi.bottom,
1834                src_roi.left,
1835                src_roi.bottom,
1836            ];
1837            gls::gl::BufferSubData(
1838                gls::gl::ARRAY_BUFFER,
1839                0,
1840                (size_of::<f32>() * 8) as isize,
1841                (texture_vertices[(rotation_offset * 2)..]).as_ptr() as *const c_void,
1842            );
1843
1844            let vertices_index: [u32; 4] = [0, 1, 2, 3];
1845            gls::gl::DrawElements(
1846                gls::gl::TRIANGLE_FAN,
1847                vertices_index.len() as i32,
1848                gls::gl::UNSIGNED_INT,
1849                vertices_index.as_ptr() as *const c_void,
1850            );
1851        }
1852        check_gl_error(function!(), line!())?;
1853        Ok(())
1854    }
1855
1856    fn create_image_from_dma2(&self, src: &TensorImage) -> Result<EglImage, crate::Error> {
1857        let width;
1858        let height;
1859        let format;
1860        let channels;
1861
1862        // NV12 is semi-planar but handled specially via EGL multi-plane import
1863        if src.fourcc() == NV12 {
1864            if !src.width().is_multiple_of(4) {
1865                return Err(Error::NotSupported(
1866                    "OpenGL EGLImage doesn't support image widths which are not multiples of 4"
1867                        .to_string(),
1868                ));
1869            }
1870            width = src.width();
1871            height = src.height();
1872            format = fourcc_to_drm(NV12);
1873            channels = 1; // Y plane pitch is 1 byte per pixel
1874        } else if src.is_planar() {
1875            if !src.width().is_multiple_of(16) {
1876                return Err(Error::NotSupported(
1877                    "OpenGL Planar RGB EGLImage doesn't support image widths which are not multiples of 16"
1878                        .to_string(),
1879                ));
1880            }
1881            match src.fourcc() {
1882                PLANAR_RGB => {
1883                    format = DrmFourcc::R8;
1884                    width = src.width();
1885                    height = src.height() * 3;
1886                    channels = 1;
1887                }
1888                fourcc => {
1889                    return Err(crate::Error::NotSupported(format!(
1890                        "Unsupported Planar FourCC {fourcc:?}"
1891                    )));
1892                }
1893            };
1894        } else {
1895            if !src.width().is_multiple_of(4) {
1896                return Err(Error::NotSupported(
1897                    "OpenGL EGLImage doesn't support image widths which are not multiples of 4"
1898                        .to_string(),
1899                ));
1900            }
1901            width = src.width();
1902            height = src.height();
1903            format = fourcc_to_drm(src.fourcc());
1904            channels = src.channels();
1905        }
1906
1907        let fd = match &src.tensor {
1908            edgefirst_tensor::Tensor::Dma(dma_tensor) => dma_tensor.fd.as_raw_fd(),
1909            edgefirst_tensor::Tensor::Shm(_) => {
1910                return Err(Error::NotImplemented(
1911                    "OpenGL EGLImage doesn't support SHM".to_string(),
1912                ));
1913            }
1914            edgefirst_tensor::Tensor::Mem(_) => {
1915                return Err(Error::NotImplemented(
1916                    "OpenGL EGLImage doesn't support MEM".to_string(),
1917                ));
1918            }
1919        };
1920
1921        // For NV12, plane0 pitch is width (Y is 1 byte/pixel)
1922        // For other formats, pitch is width * channels
1923        let plane0_pitch = if src.fourcc() == NV12 {
1924            width
1925        } else {
1926            width * channels
1927        };
1928
1929        let mut egl_img_attr = vec![
1930            egl_ext::LINUX_DRM_FOURCC as Attrib,
1931            format as Attrib,
1932            khronos_egl::WIDTH as Attrib,
1933            width as Attrib,
1934            khronos_egl::HEIGHT as Attrib,
1935            height as Attrib,
1936            egl_ext::DMA_BUF_PLANE0_PITCH as Attrib,
1937            plane0_pitch as Attrib,
1938            egl_ext::DMA_BUF_PLANE0_OFFSET as Attrib,
1939            0 as Attrib,
1940            egl_ext::DMA_BUF_PLANE0_FD as Attrib,
1941            fd as Attrib,
1942            egl::IMAGE_PRESERVED as Attrib,
1943            egl::TRUE as Attrib,
1944        ];
1945
1946        // NV12 requires a second plane for UV data
1947        if src.fourcc() == NV12 {
1948            let uv_offset = width * height; // Y plane size
1949            egl_img_attr.append(&mut vec![
1950                egl_ext::DMA_BUF_PLANE1_FD as Attrib,
1951                fd as Attrib,
1952                egl_ext::DMA_BUF_PLANE1_OFFSET as Attrib,
1953                uv_offset as Attrib,
1954                egl_ext::DMA_BUF_PLANE1_PITCH as Attrib,
1955                width as Attrib, // UV plane has same width as Y plane
1956            ]);
1957        }
1958
1959        if matches!(src.fourcc(), YUYV | NV12) {
1960            egl_img_attr.append(&mut vec![
1961                egl_ext::YUV_COLOR_SPACE_HINT as Attrib,
1962                egl_ext::ITU_REC709 as Attrib,
1963                egl_ext::SAMPLE_RANGE_HINT as Attrib,
1964                egl_ext::YUV_NARROW_RANGE as Attrib,
1965            ]);
1966        }
1967
1968        egl_img_attr.push(khronos_egl::NONE as Attrib);
1969
1970        match self.new_egl_image_owned(egl_ext::LINUX_DMA_BUF, &egl_img_attr) {
1971            Ok(v) => Ok(v),
1972            Err(e) => Err(e),
1973        }
1974    }
1975
1976    fn new_egl_image_owned(
1977        &'_ self,
1978        target: egl::Enum,
1979        attrib_list: &[Attrib],
1980    ) -> Result<EglImage, Error> {
1981        let image = GlContext::egl_create_image_with_fallback(
1982            &self.gl_context.egl,
1983            self.gl_context.display.as_display(),
1984            unsafe { egl::Context::from_ptr(egl::NO_CONTEXT) },
1985            target,
1986            unsafe { egl::ClientBuffer::from_ptr(null_mut()) },
1987            attrib_list,
1988        )?;
1989        Ok(EglImage {
1990            egl_image: image,
1991            display: self.gl_context.display.as_display(),
1992            egl: Rc::clone(&self.gl_context.egl),
1993        })
1994    }
1995
1996    // Reshapes the segmentation to be compatible with RGBA texture array rendering.
1997    fn reshape_segmentation_to_rgba(&self, segmentation: &[u8], shape: [usize; 3]) -> Vec<u8> {
1998        let [height, width, classes] = shape;
1999
2000        let n_layer_stride = height * width * 4;
2001        let n_row_stride = width * 4;
2002        let n_col_stride = 4;
2003        let row_stride = width * classes;
2004        let col_stride = classes;
2005
2006        let mut new_segmentation = vec![0u8; n_layer_stride * classes.div_ceil(4)];
2007
2008        for i in 0..height {
2009            for j in 0..width {
2010                for k in 0..classes.div_ceil(4) * 4 {
2011                    if k >= classes {
2012                        new_segmentation[n_layer_stride * (k / 4)
2013                            + i * n_row_stride
2014                            + j * n_col_stride
2015                            + k % 4] = 0;
2016                    } else {
2017                        new_segmentation[n_layer_stride * (k / 4)
2018                            + i * n_row_stride
2019                            + j * n_col_stride
2020                            + k % 4] = segmentation[i * row_stride + j * col_stride + k];
2021                    }
2022                }
2023            }
2024        }
2025
2026        new_segmentation
2027    }
2028
2029    #[cfg(feature = "decoder")]
2030    fn render_modelpack_segmentation(
2031        &mut self,
2032        dst_roi: RegionOfInterest,
2033        segmentation: &[u8],
2034        shape: [usize; 3],
2035    ) -> Result<(), crate::Error> {
2036        log::debug!("start render_segmentation_to_image");
2037
2038        // TODO: Implement specialization for 2 classes and 4 classes which shouldn't
2039        // need rearranging the data
2040        let new_segmentation = self.reshape_segmentation_to_rgba(segmentation, shape);
2041
2042        let [height, width, classes] = shape;
2043
2044        let format = gls::gl::RGBA;
2045        let texture_target = gls::gl::TEXTURE_2D_ARRAY;
2046        self.segmentation_program
2047            .load_uniform_1i(c"background_index", shape[2] as i32 - 1)?;
2048
2049        gls::use_program(self.segmentation_program.id);
2050
2051        gls::bind_texture(texture_target, self.segmentation_texture.id);
2052        gls::active_texture(gls::gl::TEXTURE0);
2053        gls::tex_parameteri(
2054            texture_target,
2055            gls::gl::TEXTURE_MIN_FILTER,
2056            gls::gl::LINEAR as i32,
2057        );
2058        gls::tex_parameteri(
2059            texture_target,
2060            gls::gl::TEXTURE_MAG_FILTER,
2061            gls::gl::LINEAR as i32,
2062        );
2063        gls::tex_parameteri(
2064            texture_target,
2065            gls::gl::TEXTURE_WRAP_S,
2066            gls::gl::CLAMP_TO_EDGE as i32,
2067        );
2068
2069        gls::tex_parameteri(
2070            texture_target,
2071            gls::gl::TEXTURE_WRAP_T,
2072            gls::gl::CLAMP_TO_EDGE as i32,
2073        );
2074
2075        gls::tex_image3d(
2076            texture_target,
2077            0,
2078            format as i32,
2079            width as i32,
2080            height as i32,
2081            classes.div_ceil(4) as i32,
2082            0,
2083            format,
2084            gls::gl::UNSIGNED_BYTE,
2085            Some(&new_segmentation),
2086        );
2087
2088        let src_roi = RegionOfInterest {
2089            left: 0.,
2090            top: 1.,
2091            right: 1.,
2092            bottom: 0.,
2093        };
2094
2095        unsafe {
2096            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
2097            gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
2098
2099            let camera_vertices: [f32; 12] = [
2100                dst_roi.left,
2101                dst_roi.top,
2102                0., // left top
2103                dst_roi.right,
2104                dst_roi.top,
2105                0., // right top
2106                dst_roi.right,
2107                dst_roi.bottom,
2108                0., // right bottom
2109                dst_roi.left,
2110                dst_roi.bottom,
2111                0., // left bottom
2112            ];
2113            gls::gl::BufferSubData(
2114                gls::gl::ARRAY_BUFFER,
2115                0,
2116                (size_of::<f32>() * camera_vertices.len()) as isize,
2117                camera_vertices.as_ptr() as *const c_void,
2118            );
2119
2120            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
2121            gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
2122
2123            let texture_vertices: [f32; 8] = [
2124                src_roi.left,
2125                src_roi.top,
2126                src_roi.right,
2127                src_roi.top,
2128                src_roi.right,
2129                src_roi.bottom,
2130                src_roi.left,
2131                src_roi.bottom,
2132            ];
2133            gls::gl::BufferSubData(
2134                gls::gl::ARRAY_BUFFER,
2135                0,
2136                (size_of::<f32>() * 8) as isize,
2137                (texture_vertices[0..]).as_ptr() as *const c_void,
2138            );
2139
2140            let vertices_index: [u32; 4] = [0, 1, 2, 3];
2141            gls::gl::DrawElements(
2142                gls::gl::TRIANGLE_FAN,
2143                vertices_index.len() as i32,
2144                gls::gl::UNSIGNED_INT,
2145                vertices_index.as_ptr() as *const c_void,
2146            );
2147        }
2148
2149        Ok(())
2150    }
2151
2152    #[cfg(feature = "decoder")]
2153    fn render_yolo_segmentation(
2154        &mut self,
2155        dst_roi: RegionOfInterest,
2156        segmentation: &[u8],
2157        shape: [usize; 2],
2158        class: usize,
2159    ) -> Result<(), crate::Error> {
2160        log::debug!("start render_yolo_segmentation");
2161
2162        let [height, width] = shape;
2163
2164        let format = gls::gl::RED;
2165        let texture_target = gls::gl::TEXTURE_2D;
2166        gls::use_program(self.instanced_segmentation_program.id);
2167        self.instanced_segmentation_program
2168            .load_uniform_1i(c"class_index", class as i32)?;
2169        gls::bind_texture(texture_target, self.segmentation_texture.id);
2170        gls::active_texture(gls::gl::TEXTURE0);
2171        gls::tex_parameteri(
2172            texture_target,
2173            gls::gl::TEXTURE_MIN_FILTER,
2174            gls::gl::LINEAR as i32,
2175        );
2176        gls::tex_parameteri(
2177            texture_target,
2178            gls::gl::TEXTURE_MAG_FILTER,
2179            gls::gl::LINEAR as i32,
2180        );
2181        gls::tex_parameteri(
2182            texture_target,
2183            gls::gl::TEXTURE_WRAP_S,
2184            gls::gl::CLAMP_TO_EDGE as i32,
2185        );
2186
2187        gls::tex_parameteri(
2188            texture_target,
2189            gls::gl::TEXTURE_WRAP_T,
2190            gls::gl::CLAMP_TO_EDGE as i32,
2191        );
2192
2193        gls::tex_image2d(
2194            texture_target,
2195            0,
2196            format as i32,
2197            width as i32,
2198            height as i32,
2199            0,
2200            format,
2201            gls::gl::UNSIGNED_BYTE,
2202            Some(segmentation),
2203        );
2204
2205        let src_roi = RegionOfInterest {
2206            left: 0.,
2207            top: 1.,
2208            right: 1.,
2209            bottom: 0.,
2210        };
2211
2212        unsafe {
2213            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
2214            gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
2215
2216            let camera_vertices: [f32; 12] = [
2217                dst_roi.left,
2218                dst_roi.top,
2219                0., // left top
2220                dst_roi.right,
2221                dst_roi.top,
2222                0., // right top
2223                dst_roi.right,
2224                dst_roi.bottom,
2225                0., // right bottom
2226                dst_roi.left,
2227                dst_roi.bottom,
2228                0., // left bottom
2229            ];
2230            gls::gl::BufferSubData(
2231                gls::gl::ARRAY_BUFFER,
2232                0,
2233                (size_of::<f32>() * camera_vertices.len()) as isize,
2234                camera_vertices.as_ptr() as *const c_void,
2235            );
2236
2237            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.texture_buffer.id);
2238            gls::gl::EnableVertexAttribArray(self.texture_buffer.buffer_index);
2239
2240            let texture_vertices: [f32; 8] = [
2241                src_roi.left,
2242                src_roi.top,
2243                src_roi.right,
2244                src_roi.top,
2245                src_roi.right,
2246                src_roi.bottom,
2247                src_roi.left,
2248                src_roi.bottom,
2249            ];
2250            gls::gl::BufferSubData(
2251                gls::gl::ARRAY_BUFFER,
2252                0,
2253                (size_of::<f32>() * 8) as isize,
2254                (texture_vertices).as_ptr() as *const c_void,
2255            );
2256
2257            let vertices_index: [u32; 4] = [0, 1, 2, 3];
2258            gls::gl::DrawElements(
2259                gls::gl::TRIANGLE_FAN,
2260                vertices_index.len() as i32,
2261                gls::gl::UNSIGNED_INT,
2262                vertices_index.as_ptr() as *const c_void,
2263            );
2264            gls::gl::Finish();
2265        }
2266
2267        Ok(())
2268    }
2269
2270    fn render_segmentation(
2271        &mut self,
2272        detect: &[DetectBox],
2273        segmentation: &[Segmentation],
2274    ) -> crate::Result<()> {
2275        if segmentation.is_empty() {
2276            return Ok(());
2277        }
2278
2279        let is_modelpack = segmentation[0].segmentation.shape()[2] > 1;
2280        // top and bottom are flipped because OpenGL uses 0,0 as bottom left
2281        let cvt_screen_coord = |normalized| normalized * 2.0 - 1.0;
2282        if is_modelpack {
2283            let seg = &segmentation[0];
2284            let dst_roi = RegionOfInterest {
2285                left: cvt_screen_coord(seg.xmin),
2286                top: cvt_screen_coord(seg.ymax),
2287                right: cvt_screen_coord(seg.xmax),
2288                bottom: cvt_screen_coord(seg.ymin),
2289            };
2290            let segment = seg.segmentation.as_standard_layout();
2291            let slice = segment.as_slice().ok_or(Error::Internal(
2292                "Cannot get slice of segmentation".to_owned(),
2293            ))?;
2294
2295            self.render_modelpack_segmentation(
2296                dst_roi,
2297                slice,
2298                [
2299                    seg.segmentation.shape()[0],
2300                    seg.segmentation.shape()[1],
2301                    seg.segmentation.shape()[2],
2302                ],
2303            )?;
2304        } else {
2305            for (seg, det) in segmentation.iter().zip(detect) {
2306                let dst_roi = RegionOfInterest {
2307                    left: cvt_screen_coord(seg.xmin),
2308                    top: cvt_screen_coord(seg.ymax),
2309                    right: cvt_screen_coord(seg.xmax),
2310                    bottom: cvt_screen_coord(seg.ymin),
2311                };
2312
2313                let segment = seg.segmentation.as_standard_layout();
2314                let slice = segment.as_slice().ok_or(Error::Internal(
2315                    "Cannot get slice of segmentation".to_owned(),
2316                ))?;
2317
2318                self.render_yolo_segmentation(
2319                    dst_roi,
2320                    slice,
2321                    [seg.segmentation.shape()[0], seg.segmentation.shape()[1]],
2322                    det.label,
2323                )?;
2324            }
2325        }
2326
2327        gls::disable(gls::gl::BLEND);
2328        Ok(())
2329    }
2330
2331    fn render_box(&mut self, dst: &TensorImage, detect: &[DetectBox]) -> Result<(), Error> {
2332        unsafe {
2333            gls::gl::UseProgram(self.color_program.id);
2334            let rescale = |x: f32| x * 2.0 - 1.0;
2335            let thickness = 3.0;
2336            for d in detect {
2337                self.color_program
2338                    .load_uniform_1i(c"class_index", d.label as i32)?;
2339                gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, self.vertex_buffer.id);
2340                gls::gl::EnableVertexAttribArray(self.vertex_buffer.buffer_index);
2341                let bbox: [f32; 4] = d.bbox.into();
2342                let outer_box = [
2343                    bbox[0] - thickness / dst.width() as f32,
2344                    bbox[1] - thickness / dst.height() as f32,
2345                    bbox[2] + thickness / dst.width() as f32,
2346                    bbox[3] + thickness / dst.height() as f32,
2347                ];
2348                let camera_vertices: [f32; 24] = [
2349                    rescale(bbox[0]),
2350                    rescale(bbox[3]),
2351                    0., // bottom left
2352                    rescale(bbox[2]),
2353                    rescale(bbox[3]),
2354                    0., // bottom right
2355                    rescale(bbox[2]),
2356                    rescale(bbox[1]),
2357                    0., // top right
2358                    rescale(bbox[0]),
2359                    rescale(bbox[1]),
2360                    0., // top left
2361                    rescale(outer_box[0]),
2362                    rescale(outer_box[3]),
2363                    0., // bottom left
2364                    rescale(outer_box[2]),
2365                    rescale(outer_box[3]),
2366                    0., // bottom right
2367                    rescale(outer_box[2]),
2368                    rescale(outer_box[1]),
2369                    0., // top right
2370                    rescale(outer_box[0]),
2371                    rescale(outer_box[1]),
2372                    0., // top left
2373                ];
2374                gls::gl::BufferData(
2375                    gls::gl::ARRAY_BUFFER,
2376                    (size_of::<f32>() * camera_vertices.len()) as isize,
2377                    camera_vertices.as_ptr() as *const c_void,
2378                    gls::gl::DYNAMIC_DRAW,
2379                );
2380
2381                let vertices_index: [u32; 10] = [0, 1, 5, 2, 6, 3, 7, 0, 4, 5];
2382                gls::gl::DrawElements(
2383                    gls::gl::TRIANGLE_STRIP,
2384                    vertices_index.len() as i32,
2385                    gls::gl::UNSIGNED_INT,
2386                    vertices_index.as_ptr() as *const c_void,
2387                );
2388            }
2389        }
2390        check_gl_error(function!(), line!())?;
2391        Ok(())
2392    }
2393}
2394struct EglImage {
2395    egl_image: egl::Image,
2396    egl: Rc<Egl>,
2397    display: egl::Display,
2398}
2399
2400impl Drop for EglImage {
2401    fn drop(&mut self) {
2402        if self.egl_image.as_ptr() == egl::NO_IMAGE {
2403            return;
2404        }
2405
2406        let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
2407            let e =
2408                GlContext::egl_destory_image_with_fallback(&self.egl, self.display, self.egl_image);
2409            if let Err(e) = e {
2410                error!("Could not destroy EGL image: {e:?}");
2411            }
2412        }));
2413    }
2414}
2415
2416struct Texture {
2417    id: u32,
2418    target: gls::gl::types::GLenum,
2419    width: usize,
2420    height: usize,
2421    format: gls::gl::types::GLenum,
2422}
2423
2424impl Default for Texture {
2425    fn default() -> Self {
2426        Self::new()
2427    }
2428}
2429
2430impl Texture {
2431    fn new() -> Self {
2432        let mut id = 0;
2433        unsafe { gls::gl::GenTextures(1, &raw mut id) };
2434        Self {
2435            id,
2436            target: 0,
2437            width: 0,
2438            height: 0,
2439            format: 0,
2440        }
2441    }
2442
2443    fn update_texture(
2444        &mut self,
2445        target: gls::gl::types::GLenum,
2446        width: usize,
2447        height: usize,
2448        format: gls::gl::types::GLenum,
2449        data: &[u8],
2450    ) {
2451        if target != self.target
2452            || width != self.width
2453            || height != self.height
2454            || format != self.format
2455        {
2456            unsafe {
2457                gls::gl::TexImage2D(
2458                    target,
2459                    0,
2460                    format as i32,
2461                    width as i32,
2462                    height as i32,
2463                    0,
2464                    format,
2465                    gls::gl::UNSIGNED_BYTE,
2466                    data.as_ptr() as *const c_void,
2467                );
2468            }
2469            self.target = target;
2470            self.format = format;
2471            self.width = width;
2472            self.height = height;
2473        } else {
2474            unsafe {
2475                gls::gl::TexSubImage2D(
2476                    target,
2477                    0,
2478                    0,
2479                    0,
2480                    width as i32,
2481                    height as i32,
2482                    format,
2483                    gls::gl::UNSIGNED_BYTE,
2484                    data.as_ptr() as *const c_void,
2485                );
2486            }
2487        }
2488    }
2489}
2490
2491impl Drop for Texture {
2492    fn drop(&mut self) {
2493        let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| unsafe {
2494            gls::gl::DeleteTextures(1, &raw mut self.id)
2495        }));
2496    }
2497}
2498
2499struct Buffer {
2500    id: u32,
2501    buffer_index: u32,
2502}
2503
2504impl Buffer {
2505    fn new(buffer_index: u32, size_per_point: usize, max_points: usize) -> Buffer {
2506        let mut id = 0;
2507        unsafe {
2508            gls::gl::EnableVertexAttribArray(buffer_index);
2509            gls::gl::GenBuffers(1, &raw mut id);
2510            gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, id);
2511            gls::gl::VertexAttribPointer(
2512                buffer_index,
2513                size_per_point as i32,
2514                gls::gl::FLOAT,
2515                gls::gl::FALSE,
2516                0,
2517                null(),
2518            );
2519            gls::gl::BufferData(
2520                gls::gl::ARRAY_BUFFER,
2521                (size_of::<f32>() * size_per_point * max_points) as isize,
2522                null(),
2523                gls::gl::DYNAMIC_DRAW,
2524            );
2525        }
2526
2527        Buffer { id, buffer_index }
2528    }
2529}
2530
2531impl Drop for Buffer {
2532    fn drop(&mut self) {
2533        let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| unsafe {
2534            gls::gl::DeleteBuffers(1, &raw mut self.id)
2535        }));
2536    }
2537}
2538
2539struct FrameBuffer {
2540    id: u32,
2541}
2542
2543impl FrameBuffer {
2544    fn new() -> FrameBuffer {
2545        let mut id = 0;
2546        unsafe {
2547            gls::gl::GenFramebuffers(1, &raw mut id);
2548        }
2549
2550        FrameBuffer { id }
2551    }
2552
2553    fn bind(&self) {
2554        unsafe { gls::gl::BindFramebuffer(gls::gl::FRAMEBUFFER, self.id) };
2555    }
2556
2557    fn unbind(&self) {
2558        unsafe { gls::gl::BindFramebuffer(gls::gl::FRAMEBUFFER, 0) };
2559    }
2560}
2561
2562impl Drop for FrameBuffer {
2563    fn drop(&mut self) {
2564        let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
2565            self.unbind();
2566            unsafe {
2567                gls::gl::DeleteFramebuffers(1, &raw mut self.id);
2568            }
2569        }));
2570    }
2571}
2572
2573pub struct GlProgram {
2574    id: u32,
2575    vertex_id: u32,
2576    fragment_id: u32,
2577}
2578
2579impl GlProgram {
2580    fn new(vertex_shader: &str, fragment_shader: &str) -> Result<Self, crate::Error> {
2581        let id = unsafe { gls::gl::CreateProgram() };
2582        let vertex_id = unsafe { gls::gl::CreateShader(gls::gl::VERTEX_SHADER) };
2583        if compile_shader_from_str(vertex_id, vertex_shader, "shader_vert").is_err() {
2584            log::debug!("Vertex shader source:\n{}", vertex_shader);
2585            return Err(crate::Error::OpenGl(format!(
2586                "Shader compile error: {vertex_shader}"
2587            )));
2588        }
2589        unsafe {
2590            gls::gl::AttachShader(id, vertex_id);
2591        }
2592
2593        let fragment_id = unsafe { gls::gl::CreateShader(gls::gl::FRAGMENT_SHADER) };
2594        if compile_shader_from_str(fragment_id, fragment_shader, "shader_frag").is_err() {
2595            log::debug!("Fragment shader source:\n{}", fragment_shader);
2596            return Err(crate::Error::OpenGl(format!(
2597                "Shader compile error: {fragment_shader}"
2598            )));
2599        }
2600
2601        unsafe {
2602            gls::gl::AttachShader(id, fragment_id);
2603            gls::gl::LinkProgram(id);
2604            gls::gl::UseProgram(id);
2605        }
2606
2607        Ok(Self {
2608            id,
2609            vertex_id,
2610            fragment_id,
2611        })
2612    }
2613
2614    #[allow(dead_code)]
2615    fn load_uniform_1f(&self, name: &CStr, value: f32) -> Result<(), crate::Error> {
2616        unsafe {
2617            gls::gl::UseProgram(self.id);
2618            let location = gls::gl::GetUniformLocation(self.id, name.as_ptr());
2619            gls::gl::Uniform1f(location, value);
2620        }
2621        Ok(())
2622    }
2623
2624    #[allow(dead_code)]
2625    fn load_uniform_1i(&self, name: &CStr, value: i32) -> Result<(), crate::Error> {
2626        unsafe {
2627            gls::gl::UseProgram(self.id);
2628            let location = gls::gl::GetUniformLocation(self.id, name.as_ptr());
2629            gls::gl::Uniform1i(location, value);
2630        }
2631        Ok(())
2632    }
2633
2634    fn load_uniform_4fv(&self, name: &CStr, value: &[[f32; 4]]) -> Result<(), crate::Error> {
2635        unsafe {
2636            gls::gl::UseProgram(self.id);
2637            let location = gls::gl::GetUniformLocation(self.id, name.as_ptr());
2638            if location == -1 {
2639                return Err(crate::Error::OpenGl(format!(
2640                    "Could not find uniform location for '{}'",
2641                    name.to_string_lossy().into_owned()
2642                )));
2643            }
2644            gls::gl::Uniform4fv(location, value.len() as i32, value.as_flattened().as_ptr());
2645        }
2646        check_gl_error(function!(), line!())?;
2647        Ok(())
2648    }
2649}
2650
2651impl Drop for GlProgram {
2652    fn drop(&mut self) {
2653        let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| unsafe {
2654            gls::gl::DeleteProgram(self.id);
2655            gls::gl::DeleteShader(self.fragment_id);
2656            gls::gl::DeleteShader(self.vertex_id);
2657        }));
2658    }
2659}
2660
2661fn compile_shader_from_str(shader: u32, shader_source: &str, shader_name: &str) -> Result<(), ()> {
2662    let src = match CString::from_str(shader_source) {
2663        Ok(v) => v,
2664        Err(_) => return Err(()),
2665    };
2666    let src_ptr = src.as_ptr();
2667    unsafe {
2668        gls::gl::ShaderSource(shader, 1, &raw const src_ptr, null());
2669        gls::gl::CompileShader(shader);
2670        let mut is_compiled = 0;
2671        gls::gl::GetShaderiv(shader, gls::gl::COMPILE_STATUS, &raw mut is_compiled);
2672        if is_compiled == 0 {
2673            let mut max_length = 0;
2674            gls::gl::GetShaderiv(shader, gls::gl::INFO_LOG_LENGTH, &raw mut max_length);
2675            let mut error_log: Vec<u8> = vec![0; max_length as usize];
2676            gls::gl::GetShaderInfoLog(
2677                shader,
2678                max_length,
2679                &raw mut max_length,
2680                error_log.as_mut_ptr() as *mut c_char,
2681            );
2682            error!(
2683                "Shader '{}' failed: {:?}\n",
2684                shader_name,
2685                CString::from_vec_with_nul(error_log)
2686                    .unwrap()
2687                    .into_string()
2688                    .unwrap()
2689            );
2690            gls::gl::DeleteShader(shader);
2691            return Err(());
2692        }
2693        Ok(())
2694    }
2695}
2696
2697fn check_gl_error(name: &str, line: u32) -> Result<(), Error> {
2698    unsafe {
2699        let err = gls::gl::GetError();
2700        if err != gls::gl::NO_ERROR {
2701            error!("GL Error: {name}:{line}: {err:#X}");
2702            // panic!("GL Error: {err}");
2703            return Err(Error::OpenGl(format!("{err:#X}")));
2704        }
2705    }
2706    Ok(())
2707}
2708
2709fn fourcc_to_drm(fourcc: FourCharCode) -> DrmFourcc {
2710    match fourcc {
2711        RGBA => DrmFourcc::Abgr8888,
2712        YUYV => DrmFourcc::Yuyv,
2713        RGB => DrmFourcc::Bgr888,
2714        GREY => DrmFourcc::R8,
2715        NV12 => DrmFourcc::Nv12,
2716        _ => todo!(),
2717    }
2718}
2719
2720mod egl_ext {
2721    #![allow(dead_code)]
2722    pub(crate) const LINUX_DMA_BUF: u32 = 0x3270;
2723    pub(crate) const LINUX_DRM_FOURCC: u32 = 0x3271;
2724    pub(crate) const DMA_BUF_PLANE0_FD: u32 = 0x3272;
2725    pub(crate) const DMA_BUF_PLANE0_OFFSET: u32 = 0x3273;
2726    pub(crate) const DMA_BUF_PLANE0_PITCH: u32 = 0x3274;
2727    pub(crate) const DMA_BUF_PLANE1_FD: u32 = 0x3275;
2728    pub(crate) const DMA_BUF_PLANE1_OFFSET: u32 = 0x3276;
2729    pub(crate) const DMA_BUF_PLANE1_PITCH: u32 = 0x3277;
2730    pub(crate) const DMA_BUF_PLANE2_FD: u32 = 0x3278;
2731    pub(crate) const DMA_BUF_PLANE2_OFFSET: u32 = 0x3279;
2732    pub(crate) const DMA_BUF_PLANE2_PITCH: u32 = 0x327A;
2733    pub(crate) const YUV_COLOR_SPACE_HINT: u32 = 0x327B;
2734    pub(crate) const SAMPLE_RANGE_HINT: u32 = 0x327C;
2735    pub(crate) const YUV_CHROMA_HORIZONTAL_SITING_HINT: u32 = 0x327D;
2736    pub(crate) const YUV_CHROMA_VERTICAL_SITING_HINT: u32 = 0x327E;
2737
2738    pub(crate) const ITU_REC601: u32 = 0x327F;
2739    pub(crate) const ITU_REC709: u32 = 0x3280;
2740    pub(crate) const ITU_REC2020: u32 = 0x3281;
2741
2742    pub(crate) const YUV_FULL_RANGE: u32 = 0x3282;
2743    pub(crate) const YUV_NARROW_RANGE: u32 = 0x3283;
2744
2745    pub(crate) const YUV_CHROMA_SITING_0: u32 = 0x3284;
2746    pub(crate) const YUV_CHROMA_SITING_0_5: u32 = 0x3285;
2747
2748    pub(crate) const PLATFORM_GBM_KHR: u32 = 0x31D7;
2749
2750    pub(crate) const PLATFORM_DEVICE_EXT: u32 = 0x313F;
2751}
2752
2753fn generate_vertex_shader() -> &'static str {
2754    "\
2755#version 300 es
2756precision mediump float;
2757layout(location = 0) in vec3 pos;
2758layout(location = 1) in vec2 texCoord;
2759
2760out vec3 fragPos;
2761out vec2 tc;
2762
2763void main() {
2764    fragPos = pos;
2765    tc = texCoord;
2766
2767    gl_Position = vec4(pos, 1.0);
2768}
2769"
2770}
2771
2772fn generate_texture_fragment_shader() -> &'static str {
2773    "\
2774#version 300 es
2775
2776precision mediump float;
2777uniform sampler2D tex;
2778in vec3 fragPos;
2779in vec2 tc;
2780
2781out vec4 color;
2782
2783void main(){
2784    color = texture(tex, tc);
2785}
2786"
2787}
2788
2789fn generate_texture_fragment_shader_yuv() -> &'static str {
2790    "\
2791#version 300 es
2792#extension GL_OES_EGL_image_external_essl3 : require
2793precision mediump float;
2794uniform samplerExternalOES tex;
2795in vec3 fragPos;
2796in vec2 tc;
2797
2798out vec4 color;
2799
2800void main(){
2801    color = texture(tex, tc);
2802}
2803"
2804}
2805
2806fn generate_planar_rgb_shader() -> &'static str {
2807    "\
2808#version 300 es
2809#extension GL_OES_EGL_image_external_essl3 : require
2810precision mediump float;
2811uniform samplerExternalOES tex;
2812in vec3 fragPos;
2813in vec2 tc;
2814
2815out vec4 color;
2816
2817void main(){
2818    color = texture(tex, tc);
2819}
2820"
2821}
2822
2823/// this shader requires a reshape of the segmentation output tensor to (H, W,
2824/// C/4, 4)
2825fn generate_segmentation_shader() -> &'static str {
2826    "\
2827#version 300 es
2828precision mediump float;
2829precision mediump sampler2DArray;
2830
2831uniform sampler2DArray tex;
2832uniform vec4 colors[20];
2833uniform int background_index;
2834
2835in vec3 fragPos;
2836in vec2 tc;
2837in vec4 fragColor;
2838
2839out vec4 color;
2840
2841float max_arg(const in vec4 args, out int argmax) {
2842    if (args[0] >= args[1] && args[0] >= args[2] && args[0] >= args[3]) {
2843        argmax = 0;
2844        return args[0];
2845    }
2846    if (args[1] >= args[0] && args[1] >= args[2] && args[1] >= args[3]) {
2847        argmax = 1;
2848        return args[1];
2849    }
2850    if (args[2] >= args[0] && args[2] >= args[1] && args[2] >= args[3]) {
2851        argmax = 2;
2852        return args[2];
2853    }
2854    argmax = 3;
2855    return args[3];
2856}
2857
2858void main() {
2859    mediump int layers = textureSize(tex, 0).z;
2860    float max_all = -4.0;
2861    int max_ind = 0;
2862    for (int i = 0; i < layers; i++) {
2863        vec4 d = texture(tex, vec3(tc, i));
2864        int max_ind_ = 0;
2865        float max_ = max_arg(d, max_ind_);
2866        if (max_ <= max_all) { continue; }
2867        max_all = max_;
2868        max_ind = i*4 + max_ind_;
2869    }
2870    if (max_ind == background_index) {
2871        discard;
2872    }
2873    max_ind = max_ind % 20;
2874    color = colors[max_ind];
2875}
2876"
2877}
2878
2879fn generate_instanced_segmentation_shader() -> &'static str {
2880    "\
2881#version 300 es
2882precision mediump float;
2883uniform sampler2D mask0;
2884uniform vec4 colors[20];
2885uniform int class_index;
2886in vec3 fragPos;
2887in vec2 tc;
2888in vec4 fragColor;
2889
2890out vec4 color;
2891void main() {
2892    float r0 = texture(mask0, tc).r;
2893    int arg = int(r0>=0.5);
2894    if (arg == 0) {
2895        discard;
2896    }
2897    color = colors[class_index % 20];
2898}
2899"
2900}
2901
2902fn generate_color_shader() -> &'static str {
2903    "\
2904#version 300 es
2905precision mediump float;
2906uniform vec4 colors[20];
2907uniform int class_index;
2908
2909out vec4 color;
2910void main() {
2911    int index = class_index % 20;
2912    color = colors[index];
2913}
2914"
2915}
2916
2917#[cfg(test)]
2918#[cfg(feature = "opengl")]
2919mod gl_tests {
2920    use super::*;
2921    use crate::{TensorImage, RGBA};
2922    #[cfg(feature = "dma_test_formats")]
2923    use crate::{NV12, YUYV};
2924    use edgefirst_tensor::TensorTrait;
2925    #[cfg(feature = "dma_test_formats")]
2926    use edgefirst_tensor::{is_dma_available, TensorMapTrait, TensorMemory};
2927    use image::buffer::ConvertBuffer;
2928    use ndarray::Array3;
2929
2930    #[test]
2931    #[cfg(feature = "decoder")]
2932    fn test_segmentation() {
2933        use edgefirst_decoder::Segmentation;
2934
2935        if !is_opengl_available() {
2936            eprintln!("SKIPPED: {} - OpenGL not available", function!());
2937            return;
2938        }
2939
2940        let mut image = TensorImage::load(
2941            include_bytes!("../../../testdata/giraffe.jpg"),
2942            Some(RGBA),
2943            None,
2944        )
2945        .unwrap();
2946
2947        let mut segmentation = Array3::from_shape_vec(
2948            (2, 160, 160),
2949            include_bytes!("../../../testdata/modelpack_seg_2x160x160.bin").to_vec(),
2950        )
2951        .unwrap();
2952        segmentation.swap_axes(0, 1);
2953        segmentation.swap_axes(1, 2);
2954        let segmentation = segmentation.as_standard_layout().to_owned();
2955
2956        let seg = Segmentation {
2957            segmentation,
2958            xmin: 0.0,
2959            ymin: 0.0,
2960            xmax: 1.0,
2961            ymax: 1.0,
2962        };
2963
2964        let mut renderer = GLProcessorThreaded::new().unwrap();
2965        renderer.render_to_image(&mut image, &[], &[seg]).unwrap();
2966    }
2967
2968    #[test]
2969    #[cfg(feature = "decoder")]
2970    fn test_segmentation_mem() {
2971        use edgefirst_decoder::Segmentation;
2972
2973        if !is_opengl_available() {
2974            eprintln!("SKIPPED: {} - OpenGL not available", function!());
2975            return;
2976        }
2977
2978        let mut image = TensorImage::load(
2979            include_bytes!("../../../testdata/giraffe.jpg"),
2980            Some(RGBA),
2981            Some(edgefirst_tensor::TensorMemory::Mem),
2982        )
2983        .unwrap();
2984
2985        let mut segmentation = Array3::from_shape_vec(
2986            (2, 160, 160),
2987            include_bytes!("../../../testdata/modelpack_seg_2x160x160.bin").to_vec(),
2988        )
2989        .unwrap();
2990        segmentation.swap_axes(0, 1);
2991        segmentation.swap_axes(1, 2);
2992        let segmentation = segmentation.as_standard_layout().to_owned();
2993
2994        let seg = Segmentation {
2995            segmentation,
2996            xmin: 0.0,
2997            ymin: 0.0,
2998            xmax: 1.0,
2999            ymax: 1.0,
3000        };
3001
3002        let mut renderer = GLProcessorThreaded::new().unwrap();
3003        renderer.render_to_image(&mut image, &[], &[seg]).unwrap();
3004    }
3005
3006    #[test]
3007    #[cfg(feature = "decoder")]
3008    fn test_segmentation_yolo() {
3009        use edgefirst_decoder::Segmentation;
3010        use ndarray::Array3;
3011
3012        if !is_opengl_available() {
3013            eprintln!("SKIPPED: {} - OpenGL not available", function!());
3014            return;
3015        }
3016
3017        let mut image = TensorImage::load(
3018            include_bytes!("../../../testdata/giraffe.jpg"),
3019            Some(RGBA),
3020            None,
3021        )
3022        .unwrap();
3023
3024        let segmentation = Array3::from_shape_vec(
3025            (76, 55, 1),
3026            include_bytes!("../../../testdata/yolov8_seg_crop_76x55.bin").to_vec(),
3027        )
3028        .unwrap();
3029
3030        let detect = DetectBox {
3031            bbox: [0.59375, 0.25, 0.9375, 0.725].into(),
3032            score: 0.99,
3033            label: 1,
3034        };
3035
3036        let seg = Segmentation {
3037            segmentation,
3038            xmin: 0.59375,
3039            ymin: 0.25,
3040            xmax: 0.9375,
3041            ymax: 0.725,
3042        };
3043
3044        let mut renderer = GLProcessorThreaded::new().unwrap();
3045        renderer
3046            .set_class_colors(&[[255, 255, 0, 233], [128, 128, 255, 100]])
3047            .unwrap();
3048        renderer
3049            .render_to_image(&mut image, &[detect], &[seg])
3050            .unwrap();
3051
3052        let expected = TensorImage::load(
3053            include_bytes!("../../../testdata/output_render_gl.jpg"),
3054            Some(RGBA),
3055            None,
3056        )
3057        .unwrap();
3058
3059        compare_images(&image, &expected, 0.99, function!());
3060    }
3061
3062    #[test]
3063    #[cfg(feature = "decoder")]
3064    fn test_boxes() {
3065        use edgefirst_decoder::DetectBox;
3066
3067        if !is_opengl_available() {
3068            eprintln!("SKIPPED: {} - OpenGL not available", function!());
3069            return;
3070        }
3071
3072        let mut image = TensorImage::load(
3073            include_bytes!("../../../testdata/giraffe.jpg"),
3074            Some(RGBA),
3075            None,
3076        )
3077        .unwrap();
3078
3079        let detect = DetectBox {
3080            bbox: [0.59375, 0.25, 0.9375, 0.725].into(),
3081            score: 0.99,
3082            label: 0,
3083        };
3084        let mut renderer = GLProcessorThreaded::new().unwrap();
3085        renderer
3086            .set_class_colors(&[[255, 255, 0, 233], [128, 128, 255, 100]])
3087            .unwrap();
3088        renderer
3089            .render_to_image(&mut image, &[detect], &[])
3090            .unwrap();
3091    }
3092
3093    static GL_AVAILABLE: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
3094    // Helper function to check if OpenGL is available
3095    fn is_opengl_available() -> bool {
3096        #[cfg(all(target_os = "linux", feature = "opengl"))]
3097        {
3098            *GL_AVAILABLE.get_or_init(|| GLProcessorThreaded::new().is_ok())
3099        }
3100
3101        #[cfg(not(all(target_os = "linux", feature = "opengl")))]
3102        {
3103            false
3104        }
3105    }
3106
3107    fn compare_images(img1: &TensorImage, img2: &TensorImage, threshold: f64, name: &str) {
3108        assert_eq!(img1.height(), img2.height(), "Heights differ");
3109        assert_eq!(img1.width(), img2.width(), "Widths differ");
3110        assert_eq!(img1.fourcc(), img2.fourcc(), "FourCC differ");
3111        assert!(
3112            matches!(img1.fourcc(), RGB | RGBA | GREY | PLANAR_RGB),
3113            "FourCC must be RGB or RGBA for comparison"
3114        );
3115
3116        let image1 = match img1.fourcc() {
3117            RGB => image::RgbImage::from_vec(
3118                img1.width() as u32,
3119                img1.height() as u32,
3120                img1.tensor().map().unwrap().to_vec(),
3121            )
3122            .unwrap(),
3123            RGBA => image::RgbaImage::from_vec(
3124                img1.width() as u32,
3125                img1.height() as u32,
3126                img1.tensor().map().unwrap().to_vec(),
3127            )
3128            .unwrap()
3129            .convert(),
3130            GREY => image::GrayImage::from_vec(
3131                img1.width() as u32,
3132                img1.height() as u32,
3133                img1.tensor().map().unwrap().to_vec(),
3134            )
3135            .unwrap()
3136            .convert(),
3137            PLANAR_RGB => image::GrayImage::from_vec(
3138                img1.width() as u32,
3139                (img1.height() * 3) as u32,
3140                img1.tensor().map().unwrap().to_vec(),
3141            )
3142            .unwrap()
3143            .convert(),
3144            _ => return,
3145        };
3146
3147        let image2 = match img2.fourcc() {
3148            RGB => image::RgbImage::from_vec(
3149                img2.width() as u32,
3150                img2.height() as u32,
3151                img2.tensor().map().unwrap().to_vec(),
3152            )
3153            .unwrap(),
3154            RGBA => image::RgbaImage::from_vec(
3155                img2.width() as u32,
3156                img2.height() as u32,
3157                img2.tensor().map().unwrap().to_vec(),
3158            )
3159            .unwrap()
3160            .convert(),
3161            GREY => image::GrayImage::from_vec(
3162                img2.width() as u32,
3163                img2.height() as u32,
3164                img2.tensor().map().unwrap().to_vec(),
3165            )
3166            .unwrap()
3167            .convert(),
3168            PLANAR_RGB => image::GrayImage::from_vec(
3169                img2.width() as u32,
3170                (img2.height() * 3) as u32,
3171                img2.tensor().map().unwrap().to_vec(),
3172            )
3173            .unwrap()
3174            .convert(),
3175            _ => return,
3176        };
3177
3178        let similarity = image_compare::rgb_similarity_structure(
3179            &image_compare::Algorithm::RootMeanSquared,
3180            &image1,
3181            &image2,
3182        )
3183        .expect("Image Comparison failed");
3184        if similarity.score < threshold {
3185            // image1.save(format!("{name}_1.png"));
3186            // image2.save(format!("{name}_2.png"));
3187            similarity
3188                .image
3189                .to_color_map()
3190                .save(format!("{name}.png"))
3191                .unwrap();
3192            panic!(
3193                "{name}: converted image and target image have similarity score too low: {} < {}",
3194                similarity.score, threshold
3195            )
3196        }
3197    }
3198
3199    // =========================================================================
3200    // NV12 Reference Validation Tests
3201    // These tests compare OpenGL NV12 conversions against ffmpeg-generated
3202    // references
3203    // =========================================================================
3204
3205    #[cfg(feature = "dma_test_formats")]
3206    fn load_raw_image(
3207        width: usize,
3208        height: usize,
3209        fourcc: FourCharCode,
3210        memory: Option<TensorMemory>,
3211        bytes: &[u8],
3212    ) -> Result<TensorImage, crate::Error> {
3213        let img = TensorImage::new(width, height, fourcc, memory)?;
3214        let mut map = img.tensor().map()?;
3215        map.as_mut_slice()[..bytes.len()].copy_from_slice(bytes);
3216        Ok(img)
3217    }
3218
3219    /// Test OpenGL NV12→RGBA conversion against ffmpeg reference
3220    #[test]
3221    #[cfg(all(target_os = "linux", feature = "dma_test_formats"))]
3222    fn test_opengl_nv12_to_rgba_reference() {
3223        if !is_dma_available() {
3224            return;
3225        }
3226        // Load NV12 source with DMA
3227        let src = load_raw_image(
3228            1280,
3229            720,
3230            NV12,
3231            Some(TensorMemory::Dma),
3232            include_bytes!("../../../testdata/camera720p.nv12"),
3233        )
3234        .unwrap();
3235
3236        // Load RGBA reference (ffmpeg-generated)
3237        let reference = load_raw_image(
3238            1280,
3239            720,
3240            RGBA,
3241            None,
3242            include_bytes!("../../../testdata/camera720p.rgba"),
3243        )
3244        .unwrap();
3245
3246        // Convert using OpenGL
3247        let mut dst = TensorImage::new(1280, 720, RGBA, Some(TensorMemory::Dma)).unwrap();
3248        let mut gl = GLProcessorThreaded::new().unwrap();
3249        gl.convert(&src, &mut dst, Rotation::None, Flip::None, Crop::no_crop())
3250            .unwrap();
3251
3252        // Copy to CPU for comparison
3253        let cpu_dst = TensorImage::new(1280, 720, RGBA, None).unwrap();
3254        cpu_dst
3255            .tensor()
3256            .map()
3257            .unwrap()
3258            .as_mut_slice()
3259            .copy_from_slice(dst.tensor().map().unwrap().as_slice());
3260
3261        compare_images(&reference, &cpu_dst, 0.98, "opengl_nv12_to_rgba_reference");
3262    }
3263
3264    /// Test OpenGL YUYV→RGBA conversion against ffmpeg reference
3265    #[test]
3266    #[cfg(all(target_os = "linux", feature = "dma_test_formats"))]
3267    fn test_opengl_yuyv_to_rgba_reference() {
3268        if !is_dma_available() {
3269            return;
3270        }
3271        // Load YUYV source with DMA
3272        let src = load_raw_image(
3273            1280,
3274            720,
3275            YUYV,
3276            Some(TensorMemory::Dma),
3277            include_bytes!("../../../testdata/camera720p.yuyv"),
3278        )
3279        .unwrap();
3280
3281        // Load RGBA reference (ffmpeg-generated)
3282        let reference = load_raw_image(
3283            1280,
3284            720,
3285            RGBA,
3286            None,
3287            include_bytes!("../../../testdata/camera720p.rgba"),
3288        )
3289        .unwrap();
3290
3291        // Convert using OpenGL
3292        let mut dst = TensorImage::new(1280, 720, RGBA, Some(TensorMemory::Dma)).unwrap();
3293        let mut gl = GLProcessorThreaded::new().unwrap();
3294        gl.convert(&src, &mut dst, Rotation::None, Flip::None, Crop::no_crop())
3295            .unwrap();
3296
3297        // Copy to CPU for comparison
3298        let cpu_dst = TensorImage::new(1280, 720, RGBA, None).unwrap();
3299        cpu_dst
3300            .tensor()
3301            .map()
3302            .unwrap()
3303            .as_mut_slice()
3304            .copy_from_slice(dst.tensor().map().unwrap().as_slice());
3305
3306        compare_images(&reference, &cpu_dst, 0.98, "opengl_yuyv_to_rgba_reference");
3307    }
3308}