#![cfg(target_os = "macos")]
use super::iosurface_import;
use super::platform::macos::MacosPlatform;
use super::Egl;
use crate::{Crop, Error, Flip, ImageProcessorTrait, MaskOverlay, Result, Rotation};
use edgefirst_decoder::{DetectBox, ProtoData, Segmentation};
use edgefirst_tensor::{PixelFormat, TensorDyn};
use khronos_egl as egl;
use log::debug;
use std::collections::HashMap;
use std::ffi::{c_void, CString};
use std::sync::{Mutex, MutexGuard, OnceLock};
const EGL_OPENGL_ES3_BIT: i32 = 0x0040;
const EGL_PBUFFER_BIT: i32 = 0x0001;
const EGL_RENDERABLE_TYPE: i32 = 0x3040;
const EGL_SURFACE_TYPE: i32 = 0x3033;
const EGL_RED_SIZE: i32 = 0x3024;
const EGL_GREEN_SIZE: i32 = 0x3023;
const EGL_BLUE_SIZE: i32 = 0x3022;
const EGL_ALPHA_SIZE: i32 = 0x3021;
const EGL_CONTEXT_CLIENT_VERSION: i32 = 0x3098;
const EGL_BACK_BUFFER: i32 = 0x3084;
const VERTEX_SHADER: &str = r#"#version 300 es
precision mediump float;
layout(location = 0) in vec2 pos;
layout(location = 1) in vec2 uv_in;
out vec2 v_uv;
void main() {
v_uv = uv_in;
gl_Position = vec4(pos, 0.0, 1.0);
}
"#;
const YUYV_TO_RGBA_FRAGMENT: &str = r#"#version 300 es
precision mediump float;
uniform sampler2D src;
uniform vec2 src_size;
in vec2 v_uv;
out vec4 frag;
void main() {
vec2 texel = vec2(1.0) / src_size;
vec2 col = floor(v_uv * src_size);
bool even = mod(col.x, 2.0) < 0.5;
vec2 self_uv = (col + vec2(0.5)) * texel;
vec2 pair_uv = (col + vec2(even ? 1.5 : -0.5, 0.5)) * texel;
vec4 self_rg = texture(src, self_uv);
vec4 pair_rg = texture(src, pair_uv);
float y = self_rg.r;
float u, v;
if (even) { u = self_rg.g; v = pair_rg.g; }
else { v = self_rg.g; u = pair_rg.g; }
float yp = (y * 255.0 - 16.0) * (1.164 / 255.0);
float up = u - 128.0/255.0;
float vp = v - 128.0/255.0;
float r = clamp(yp + 1.793 * vp, 0.0, 1.0);
float g = clamp(yp - 0.213 * up - 0.533 * vp, 0.0, 1.0);
float b = clamp(yp + 2.112 * up, 0.0, 1.0);
frag = vec4(r, g, b, 1.0);
}
"#;
static GL_LOADED: OnceLock<()> = OnceLock::new();
fn load_gl_once(egl: &Egl) {
GL_LOADED.get_or_init(|| {
gls::load_with(|name| match egl.get_proc_address(name) {
Some(ptr) => ptr as *const c_void,
None => std::ptr::null(),
});
});
}
struct SharedAngleDisplay {
egl: Egl,
display: egl::Display,
config: egl::Config,
context: egl::Context,
dummy_pbuffer: egl::Surface,
}
unsafe impl Send for SharedAngleDisplay {}
unsafe impl Sync for SharedAngleDisplay {}
static SHARED_DISPLAY: OnceLock<std::result::Result<SharedAngleDisplay, String>> = OnceLock::new();
static GL_MUTEX: Mutex<()> = Mutex::new(());
fn shared_display() -> Result<&'static SharedAngleDisplay> {
SHARED_DISPLAY
.get_or_init(|| init_shared_display().map_err(|e| e.to_string()))
.as_ref()
.map_err(|s| Error::Io(std::io::Error::other(s.clone())))
}
fn init_shared_display() -> Result<SharedAngleDisplay> {
let _span =
tracing::info_span!("image.gl_init", platform = "macos", backend = "iosurface",).entered();
let egl_lib = MacosPlatform::load_egl_lib()
.map_err(|e| Error::Io(std::io::Error::other(format!("ANGLE libEGL: {e}"))))?;
let egl: Egl = unsafe {
khronos_egl::Instance::<
khronos_egl::Dynamic<&'static libloading::Library, khronos_egl::EGL1_4>,
>::load_required_from(egl_lib)
}
.map_err(|e| Error::Io(std::io::Error::other(format!("EGL load: {e:?}"))))?;
let display = MacosPlatform::create_display(&egl)?;
let (maj, min) = egl
.initialize(display)
.map_err(|e| Error::Io(std::io::Error::other(format!("eglInitialize: {e:?}"))))?;
debug!("MacosGlProcessor: EGL {maj}.{min} initialised via ANGLE (process-global)");
egl.bind_api(egl::OPENGL_ES_API)
.map_err(|e| Error::Io(std::io::Error::other(format!("eglBindAPI: {e:?}"))))?;
let cfg_attribs = [
EGL_RENDERABLE_TYPE,
EGL_OPENGL_ES3_BIT,
EGL_SURFACE_TYPE,
EGL_PBUFFER_BIT,
EGL_RED_SIZE,
8,
EGL_GREEN_SIZE,
8,
EGL_BLUE_SIZE,
8,
EGL_ALPHA_SIZE,
8,
iosurface_import::EGL_BIND_TO_TEXTURE_TARGET_ANGLE,
0x305F, egl::NONE,
];
let config = egl
.choose_first_config(display, &cfg_attribs)
.map_err(|e| Error::Io(std::io::Error::other(format!("eglChooseConfig: {e:?}"))))?
.ok_or_else(|| {
Error::NotSupported("no EGL config with GLES3+PBUFFER+TEXTURE_2D bind".into())
})?;
let ctx_attribs = [EGL_CONTEXT_CLIENT_VERSION, 3, egl::NONE];
let context = egl
.create_context(display, config, None, &ctx_attribs)
.map_err(|e| Error::Io(std::io::Error::other(format!("eglCreateContext: {e:?}"))))?;
let dummy_attribs = [egl::WIDTH, 16, egl::HEIGHT, 16, egl::NONE];
let dummy_pbuffer = egl
.create_pbuffer_surface(display, config, &dummy_attribs)
.map_err(|e| {
let _ = egl.destroy_context(display, context);
Error::Io(std::io::Error::other(format!(
"eglCreatePbufferSurface(dummy): {e:?}"
)))
})?;
if let Err(e) = egl.make_current(
display,
Some(dummy_pbuffer),
Some(dummy_pbuffer),
Some(context),
) {
let _ = egl.destroy_surface(display, dummy_pbuffer);
let _ = egl.destroy_context(display, context);
return Err(Error::Io(std::io::Error::other(format!(
"eglMakeCurrent(dummy): {e:?}"
))));
}
load_gl_once(&egl);
let _ = egl.make_current(display, None, None, None);
Ok(SharedAngleDisplay {
egl,
display,
config,
context,
dummy_pbuffer,
})
}
fn lock_gl() -> MutexGuard<'static, ()> {
GL_MUTEX.lock().unwrap_or_else(|p| p.into_inner())
}
pub struct MacosGlProcessor {
program_yuyv_to_rgba: u32,
uniform_src: i32,
uniform_src_size: i32,
vao: u32,
vbo: u32,
fbo: u32,
src_tex: u32,
dst_tex: u32,
pbuf_cache: Mutex<HashMap<PbufferCacheKey, egl::Surface>>,
}
#[derive(Hash, Eq, PartialEq, Clone, Copy, Debug)]
struct PbufferCacheKey {
iosurface_id: u32,
format_disc: u8,
}
impl std::fmt::Debug for MacosGlProcessor {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("MacosGlProcessor")
.field("backend", &"ANGLE+IOSurface")
.finish()
}
}
struct MakeCurrentGuard<'d> {
egl: &'d Egl,
display: egl::Display,
}
impl<'d> MakeCurrentGuard<'d> {
fn new(d: &'d SharedAngleDisplay) -> Result<Self> {
d.egl
.make_current(
d.display,
Some(d.dummy_pbuffer),
Some(d.dummy_pbuffer),
Some(d.context),
)
.map_err(|e| Error::Io(std::io::Error::other(format!("eglMakeCurrent: {e:?}"))))?;
Ok(Self {
egl: &d.egl,
display: d.display,
})
}
}
impl Drop for MakeCurrentGuard<'_> {
fn drop(&mut self) {
let _ = self.egl.make_current(self.display, None, None, None);
}
}
struct BoundTexImage<'d> {
egl: &'d Egl,
display: egl::Display,
pbuf: egl::Surface,
bound: bool,
}
impl<'d> BoundTexImage<'d> {
fn bind(d: &'d SharedAngleDisplay, pbuf: egl::Surface) -> Result<Self> {
d.egl
.bind_tex_image(d.display, pbuf, EGL_BACK_BUFFER)
.map_err(|e| Error::Io(std::io::Error::other(format!("eglBindTexImage: {e:?}"))))?;
Ok(Self {
egl: &d.egl,
display: d.display,
pbuf,
bound: true,
})
}
}
impl Drop for BoundTexImage<'_> {
fn drop(&mut self) {
if self.bound {
let _ = self
.egl
.release_tex_image(self.display, self.pbuf, EGL_BACK_BUFFER);
}
}
}
impl MacosGlProcessor {
pub fn new() -> Result<Self> {
let d = shared_display()?;
let _guard = lock_gl();
let _current = MakeCurrentGuard::new(d)?;
unsafe {
let program = compile_program(VERTEX_SHADER, YUYV_TO_RGBA_FRAGMENT)?;
struct InstanceCleanup {
program: Option<u32>,
vbo: Option<u32>,
vao: Option<u32>,
fbo: Option<u32>,
src_tex: Option<u32>,
dst_tex: Option<u32>,
}
impl Drop for InstanceCleanup {
fn drop(&mut self) {
unsafe {
if let Some(p) = self.program {
gls::gl::DeleteProgram(p);
}
if let Some(b) = self.vbo {
gls::gl::DeleteBuffers(1, &b);
}
if let Some(a) = self.vao {
gls::gl::DeleteVertexArrays(1, &a);
}
if let Some(f) = self.fbo {
gls::gl::DeleteFramebuffers(1, &f);
}
if let Some(t) = self.src_tex {
gls::gl::DeleteTextures(1, &t);
}
if let Some(t) = self.dst_tex {
gls::gl::DeleteTextures(1, &t);
}
}
}
}
let mut cleanup = InstanceCleanup {
program: Some(program),
vbo: None,
vao: None,
fbo: None,
src_tex: None,
dst_tex: None,
};
let (uniform_src, uniform_src_size) = {
let loc_src = gls::gl::GetUniformLocation(program, c"src".as_ptr() as *const _);
let loc_size =
gls::gl::GetUniformLocation(program, c"src_size".as_ptr() as *const _);
(loc_src, loc_size)
};
#[rustfmt::skip]
let quad: [f32; 16] = [
-1.0,-1.0, 0.0, 0.0,
1.0,-1.0, 1.0, 0.0,
-1.0, 1.0, 0.0, 1.0,
1.0, 1.0, 1.0, 1.0,
];
let mut vbo = 0u32;
let mut vao = 0u32;
gls::gl::GenBuffers(1, &mut vbo);
cleanup.vbo = Some(vbo);
gls::gl::BindBuffer(gls::gl::ARRAY_BUFFER, vbo);
gls::gl::BufferData(
gls::gl::ARRAY_BUFFER,
std::mem::size_of_val(&quad) as isize,
quad.as_ptr() as *const _,
gls::gl::STATIC_DRAW,
);
gls::gl::GenVertexArrays(1, &mut vao);
cleanup.vao = Some(vao);
gls::gl::BindVertexArray(vao);
gls::gl::VertexAttribPointer(0, 2, gls::gl::FLOAT, 0, 16, std::ptr::null());
gls::gl::EnableVertexAttribArray(0);
gls::gl::VertexAttribPointer(1, 2, gls::gl::FLOAT, 0, 16, 8 as *const _);
gls::gl::EnableVertexAttribArray(1);
let mut fbo = 0u32;
let mut src_tex = 0u32;
let mut dst_tex = 0u32;
gls::gl::GenFramebuffers(1, &mut fbo);
cleanup.fbo = Some(fbo);
gls::gl::GenTextures(1, &mut src_tex);
cleanup.src_tex = Some(src_tex);
gls::gl::GenTextures(1, &mut dst_tex);
cleanup.dst_tex = Some(dst_tex);
for tex in [src_tex, dst_tex] {
gls::gl::BindTexture(gls::gl::TEXTURE_2D, tex);
gls::gl::TexParameteri(
gls::gl::TEXTURE_2D,
gls::gl::TEXTURE_MIN_FILTER,
gls::gl::NEAREST as i32,
);
gls::gl::TexParameteri(
gls::gl::TEXTURE_2D,
gls::gl::TEXTURE_MAG_FILTER,
gls::gl::NEAREST as i32,
);
gls::gl::TexParameteri(
gls::gl::TEXTURE_2D,
gls::gl::TEXTURE_WRAP_S,
gls::gl::CLAMP_TO_EDGE as i32,
);
gls::gl::TexParameteri(
gls::gl::TEXTURE_2D,
gls::gl::TEXTURE_WRAP_T,
gls::gl::CLAMP_TO_EDGE as i32,
);
}
let program = cleanup.program.take().unwrap();
let vbo = cleanup.vbo.take().unwrap();
let vao = cleanup.vao.take().unwrap();
let fbo = cleanup.fbo.take().unwrap();
let src_tex = cleanup.src_tex.take().unwrap();
let dst_tex = cleanup.dst_tex.take().unwrap();
std::mem::forget(cleanup);
Ok(Self {
program_yuyv_to_rgba: program,
uniform_src,
uniform_src_size,
vao,
vbo,
fbo,
src_tex,
dst_tex,
pbuf_cache: Mutex::new(HashMap::new()),
})
}
}
pub fn supports(src_fmt: PixelFormat, dst_fmt: PixelFormat) -> bool {
matches!((src_fmt, dst_fmt), (PixelFormat::Yuyv, PixelFormat::Rgba))
}
fn convert_yuyv_to_rgba(
&self,
src: &TensorDyn,
dst: &mut TensorDyn,
src_fmt: PixelFormat,
dst_fmt: PixelFormat,
) -> Result<()> {
let _span = tracing::trace_span!(
"image.convert",
backend = "gl",
platform = "macos",
src_fmt = ?src_fmt,
dst_fmt = ?dst_fmt,
)
.entered();
let src_w = src
.width()
.ok_or_else(|| Error::InvalidShape("src width".into()))?;
let src_h = src
.height()
.ok_or_else(|| Error::InvalidShape("src height".into()))?;
let dst_w = dst
.width()
.ok_or_else(|| Error::InvalidShape("dst width".into()))?;
let dst_h = dst
.height()
.ok_or_else(|| Error::InvalidShape("dst height".into()))?;
if src_w != dst_w || src_h != dst_h {
return Err(Error::NotImplemented(format!(
"MacosGlProcessor: resize not yet supported (src {src_w}×{src_h} → dst {dst_w}×{dst_h}); CPU fallback handles this"
)));
}
let src_u8 = src
.as_u8()
.ok_or_else(|| Error::NotSupported("GL backend requires u8 source tensor".into()))?;
let dst_u8 = dst.as_u8_mut().ok_or_else(|| {
Error::NotSupported("GL backend requires u8 destination tensor".into())
})?;
let src_iosurface = src_u8.iosurface_ref().ok_or_else(|| {
Error::NotSupported("GL convert: source tensor is not IOSurface-backed".into())
})?;
let dst_iosurface = dst_u8.iosurface_ref().ok_or_else(|| {
Error::NotSupported("GL convert: destination tensor is not IOSurface-backed".into())
})?;
let src_id = src_u8.iosurface_id().unwrap_or(0);
let dst_id = dst_u8.iosurface_id().unwrap_or(0);
let d = shared_display()?;
let _gl_guard = lock_gl();
let _current = MakeCurrentGuard::new(d)?;
let src_pbuf =
self.get_or_create_pbuffer(d, src_id, src_iosurface, src_fmt, src_w, src_h)?;
let dst_pbuf =
self.get_or_create_pbuffer(d, dst_id, dst_iosurface, dst_fmt, dst_w, dst_h)?;
unsafe {
gls::gl::BindTexture(gls::gl::TEXTURE_2D, self.src_tex);
let _src_bound = BoundTexImage::bind(d, src_pbuf)?;
gls::gl::BindTexture(gls::gl::TEXTURE_2D, self.dst_tex);
let _dst_bound = BoundTexImage::bind(d, dst_pbuf)?;
gls::gl::BindFramebuffer(gls::gl::FRAMEBUFFER, self.fbo);
gls::gl::FramebufferTexture2D(
gls::gl::FRAMEBUFFER,
gls::gl::COLOR_ATTACHMENT0,
gls::gl::TEXTURE_2D,
self.dst_tex,
0,
);
let fbo_status = gls::gl::CheckFramebufferStatus(gls::gl::FRAMEBUFFER);
if fbo_status != gls::gl::FRAMEBUFFER_COMPLETE {
return Err(Error::Io(std::io::Error::other(format!(
"FBO incomplete: 0x{fbo_status:x}"
))));
}
gls::gl::Viewport(0, 0, dst_w as i32, dst_h as i32);
gls::gl::UseProgram(self.program_yuyv_to_rgba);
gls::gl::ActiveTexture(gls::gl::TEXTURE0);
gls::gl::BindTexture(gls::gl::TEXTURE_2D, self.src_tex);
gls::gl::Uniform1i(self.uniform_src, 0);
gls::gl::Uniform2f(self.uniform_src_size, src_w as f32, src_h as f32);
gls::gl::BindVertexArray(self.vao);
gls::gl::DrawArrays(gls::gl::TRIANGLE_STRIP, 0, 4);
gls::gl::Finish();
}
Ok(())
}
fn get_or_create_pbuffer(
&self,
d: &SharedAngleDisplay,
iosurface_id: u32,
surface_ref: *mut c_void,
format: PixelFormat,
width: usize,
height: usize,
) -> Result<egl::Surface> {
let key = PbufferCacheKey {
iosurface_id,
format_disc: pixel_format_discriminant(format),
};
if iosurface_id != 0 {
let cache = self.pbuf_cache.lock().unwrap_or_else(|p| p.into_inner());
if let Some(&pbuf) = cache.get(&key) {
return Ok(pbuf);
}
}
let pbuf = unsafe {
iosurface_import::create_iosurface_pbuffer(
&d.egl,
d.display,
d.config,
surface_ref,
format,
width,
height,
)?
};
if iosurface_id != 0 {
let mut cache = self.pbuf_cache.lock().unwrap_or_else(|p| p.into_inner());
cache.insert(key, pbuf);
}
Ok(pbuf)
}
}
fn pixel_format_discriminant(fmt: PixelFormat) -> u8 {
fmt as u8
}
impl Drop for MacosGlProcessor {
fn drop(&mut self) {
let Ok(d) = shared_display() else {
return; };
let _gl_guard = lock_gl();
let _current = match MakeCurrentGuard::new(d) {
Ok(g) => g,
Err(_) => return,
};
unsafe {
let mut cache = self
.pbuf_cache
.get_mut()
.map(std::mem::take)
.unwrap_or_default();
for (_, pbuf) in cache.drain() {
let _ = d.egl.destroy_surface(d.display, pbuf);
}
gls::gl::DeleteFramebuffers(1, &self.fbo);
gls::gl::DeleteTextures(1, &self.src_tex);
gls::gl::DeleteTextures(1, &self.dst_tex);
gls::gl::DeleteBuffers(1, &self.vbo);
gls::gl::DeleteVertexArrays(1, &self.vao);
gls::gl::DeleteProgram(self.program_yuyv_to_rgba);
}
}
}
impl ImageProcessorTrait for MacosGlProcessor {
fn convert(
&mut self,
src: &TensorDyn,
dst: &mut TensorDyn,
rotation: Rotation,
flip: Flip,
crop: Crop,
) -> Result<()> {
if !matches!(rotation, Rotation::None) || !matches!(flip, Flip::None) {
return Err(Error::NotImplemented(
"MacosGlProcessor: rotation/flip not yet supported; CPU fallback handles this"
.into(),
));
}
if crop.src_rect.is_some() || crop.dst_rect.is_some() {
return Err(Error::NotImplemented(
"MacosGlProcessor: crop not yet supported; CPU fallback handles this".into(),
));
}
let (src_fmt, dst_fmt) = match (src.format(), dst.format()) {
(Some(s), Some(d)) => (s, d),
_ => {
return Err(Error::NotSupported(
"MacosGlProcessor: untyped tensors (None format) not supported".into(),
));
}
};
if !Self::supports(src_fmt, dst_fmt) {
return Err(Error::NotSupported(format!(
"MacosGlProcessor: {src_fmt:?} → {dst_fmt:?} not in the initial GL coverage set"
)));
}
self.convert_yuyv_to_rgba(src, dst, src_fmt, dst_fmt)
}
fn draw_decoded_masks(
&mut self,
_dst: &mut TensorDyn,
_detect: &[DetectBox],
_segmentation: &[Segmentation],
_overlay: MaskOverlay<'_>,
) -> Result<()> {
Err(Error::NotImplemented(
"MacosGlProcessor: draw_decoded_masks not yet ported (use CPU backend)".into(),
))
}
fn draw_proto_masks(
&mut self,
_dst: &mut TensorDyn,
_detect: &[DetectBox],
_proto_data: &ProtoData,
_overlay: MaskOverlay<'_>,
) -> Result<()> {
Err(Error::NotImplemented(
"MacosGlProcessor: draw_proto_masks not yet ported (use CPU backend)".into(),
))
}
fn set_class_colors(&mut self, _colors: &[[u8; 4]]) -> Result<()> {
Ok(())
}
}
unsafe fn compile_program(vertex_src: &str, fragment_src: &str) -> Result<u32> {
let vs = compile_shader(gls::gl::VERTEX_SHADER, vertex_src)?;
struct ProgramBuild {
vs: Option<u32>,
fs: Option<u32>,
program: Option<u32>,
}
impl Drop for ProgramBuild {
fn drop(&mut self) {
unsafe {
if let Some(p) = self.program {
gls::gl::DeleteProgram(p);
}
if let Some(s) = self.fs {
gls::gl::DeleteShader(s);
}
if let Some(s) = self.vs {
gls::gl::DeleteShader(s);
}
}
}
}
let mut state = ProgramBuild {
vs: Some(vs),
fs: None,
program: None,
};
let fs = compile_shader(gls::gl::FRAGMENT_SHADER, fragment_src)?;
state.fs = Some(fs);
let program = gls::gl::CreateProgram();
state.program = Some(program);
gls::gl::AttachShader(program, vs);
gls::gl::AttachShader(program, fs);
gls::gl::LinkProgram(program);
let mut ok = 0i32;
gls::gl::GetProgramiv(program, gls::gl::LINK_STATUS, &mut ok);
if ok == 0 {
let mut log = [0u8; 4096];
let mut len = 0i32;
gls::gl::GetProgramInfoLog(
program,
log.len() as i32,
&mut len,
log.as_mut_ptr() as *mut _,
);
return Err(Error::Internal(format!(
"program link failed: {}",
String::from_utf8_lossy(&log[..len.max(0) as usize])
)));
}
gls::gl::DeleteShader(state.vs.take().unwrap());
gls::gl::DeleteShader(state.fs.take().unwrap());
let program = state.program.take().unwrap();
std::mem::forget(state);
Ok(program)
}
unsafe fn compile_shader(kind: u32, src: &str) -> Result<u32> {
let shader = gls::gl::CreateShader(kind);
let c = CString::new(src).map_err(|e| Error::Internal(format!("shader CString: {e}")))?;
let ptr = c.as_ptr();
let len = src.len() as i32;
gls::gl::ShaderSource(shader, 1, &ptr, &len);
gls::gl::CompileShader(shader);
let mut ok = 0i32;
gls::gl::GetShaderiv(shader, gls::gl::COMPILE_STATUS, &mut ok);
if ok == 0 {
let mut log = [0u8; 4096];
let mut len = 0i32;
gls::gl::GetShaderInfoLog(
shader,
log.len() as i32,
&mut len,
log.as_mut_ptr() as *mut _,
);
return Err(Error::Internal(format!(
"shader compile failed (kind=0x{kind:x}): {}",
String::from_utf8_lossy(&log[..len.max(0) as usize])
)));
}
Ok(shader)
}