use super::compositor::BlendMode;
use super::frame::{Frame, FrameStatus, PixelBuffer, PixelFormat};
use eframe::glow::{self, HasContext};
use log::{trace, warn};
use std::collections::HashMap;
use std::sync::Arc;
struct TextureGuard {
gl: Arc<glow::Context>,
textures: Vec<glow::Texture>,
}
impl TextureGuard {
fn new(gl: Arc<glow::Context>) -> Self {
Self { gl, textures: Vec::new() }
}
fn push(&mut self, texture: glow::Texture) {
self.textures.push(texture);
}
fn delete(&mut self, texture: glow::Texture) {
if let Some(pos) = self.textures.iter().position(|t| *t == texture) {
self.textures.remove(pos);
unsafe { self.gl.delete_texture(texture); }
}
}
}
impl Drop for TextureGuard {
fn drop(&mut self) {
for texture in self.textures.drain(..) {
unsafe { self.gl.delete_texture(texture); }
}
}
}
pub struct GpuCompositor {
gl: Arc<glow::Context>,
fbo: Option<glow::Framebuffer>,
blend_program: Option<glow::Program>,
vao: Option<glow::VertexArray>,
vbo: Option<glow::Buffer>,
#[allow(dead_code)]
texture_cache: Arc<std::sync::Mutex<HashMap<u64, glow::Texture>>>,
}
impl GpuCompositor {
pub fn new(gl: Arc<glow::Context>) -> Self {
trace!("GpuCompositor::new() - initializing");
Self {
gl,
fbo: None,
blend_program: None,
vao: None,
vbo: None,
texture_cache: Arc::new(std::sync::Mutex::new(HashMap::new())),
}
}
fn ensure_initialized(&mut self) -> Result<(), String> {
if self.blend_program.is_some() && self.vao.is_some() && self.fbo.is_some() {
return Ok(());
}
trace!("GpuCompositor::ensure_initialized() - creating OpenGL resources");
unsafe {
let gl = &self.gl;
self.blend_program = Some(self.compile_blend_shader()?);
let vao = gl
.create_vertex_array()
.map_err(|e| format!("Failed to create VAO: {}", e))?;
gl.bind_vertex_array(Some(vao));
let vbo = gl
.create_buffer()
.map_err(|e| format!("Failed to create VBO: {}", e))?;
gl.bind_buffer(glow::ARRAY_BUFFER, Some(vbo));
#[rustfmt::skip]
let vertices: [f32; 16] = [
-1.0, -1.0, 0.0, 0.0,
1.0, -1.0, 1.0, 0.0,
1.0, 1.0, 1.0, 1.0,
-1.0, 1.0, 0.0, 1.0,
];
gl.buffer_data_u8_slice(
glow::ARRAY_BUFFER,
bytemuck::cast_slice(&vertices),
glow::STATIC_DRAW,
);
let stride = 4 * std::mem::size_of::<f32>() as i32;
gl.vertex_attrib_pointer_f32(0, 2, glow::FLOAT, false, stride, 0);
gl.enable_vertex_attrib_array(0);
gl.vertex_attrib_pointer_f32(1, 2, glow::FLOAT, false, stride, 2 * std::mem::size_of::<f32>() as i32);
gl.enable_vertex_attrib_array(1);
self.vao = Some(vao);
self.vbo = Some(vbo);
let fbo = gl
.create_framebuffer()
.map_err(|e| format!("Failed to create FBO: {}", e))?;
self.fbo = Some(fbo);
trace!("GpuCompositor initialized successfully");
Ok(())
}
}
fn compile_blend_shader(&self) -> Result<glow::Program, String> {
let gl = &self.gl;
let vertex_src = r#"#version 330 core
layout(location = 0) in vec2 a_pos;
layout(location = 1) in vec2 a_texcoord;
out vec2 v_texcoord;
void main() {
gl_Position = vec4(a_pos, 0.0, 1.0);
v_texcoord = a_texcoord;
}
"#;
let fragment_src = r#"#version 330 core
uniform sampler2D u_bottom;
uniform sampler2D u_top;
uniform float u_opacity;
uniform int u_blend_mode;
uniform mat3 u_top_transform; // Inverse transform for sampling top layer
uniform vec2 u_canvas_size; // Output canvas size in pixels
uniform vec2 u_top_size; // Top layer size in pixels
in vec2 v_texcoord;
out vec4 frag_color;
// Blend mode implementation
vec3 blend(vec3 bottom, vec3 top, int mode) {
if (mode == 0) { // Normal
return top;
} else if (mode == 1) { // Screen
return vec3(1.0) - (vec3(1.0) - bottom) * (vec3(1.0) - top);
} else if (mode == 2) { // Add
return min(bottom + top, vec3(1.0));
} else if (mode == 3) { // Subtract
return max(bottom - top, vec3(0.0));
} else if (mode == 4) { // Multiply
return bottom * top;
} else if (mode == 5) { // Divide
return min(bottom / max(top, vec3(0.00001)), vec3(1.0));
} else if (mode == 6) { // Difference
return abs(bottom - top);
} else if (mode == 7) { // Overlay
// Multiply if base < 0.5, Screen if base >= 0.5
vec3 result;
result.r = bottom.r < 0.5 ? 2.0 * bottom.r * top.r : 1.0 - 2.0 * (1.0 - bottom.r) * (1.0 - top.r);
result.g = bottom.g < 0.5 ? 2.0 * bottom.g * top.g : 1.0 - 2.0 * (1.0 - bottom.g) * (1.0 - top.g);
result.b = bottom.b < 0.5 ? 2.0 * bottom.b * top.b : 1.0 - 2.0 * (1.0 - bottom.b) * (1.0 - top.b);
return result;
}
return top; // Fallback to normal
}
void main() {
vec4 bottom_color = texture(u_bottom, v_texcoord);
// Convert UV (0-1) to pixel coordinates on canvas
vec2 canvas_pixel = v_texcoord * u_canvas_size;
// Apply inverse transform (in pixel space)
vec3 transformed = u_top_transform * vec3(canvas_pixel, 1.0);
// Convert back to UV coordinates for top layer
vec2 top_texcoord = transformed.xy / u_top_size;
// Sample top with bounds check (transparent outside 0-1 range)
vec4 top_color;
if (top_texcoord.x < 0.0 || top_texcoord.x > 1.0 ||
top_texcoord.y < 0.0 || top_texcoord.y > 1.0) {
top_color = vec4(0.0);
} else {
top_color = texture(u_top, top_texcoord);
}
float top_alpha = top_color.a * u_opacity;
vec3 blended = blend(bottom_color.rgb, top_color.rgb, u_blend_mode);
// Alpha compositing
vec3 result_rgb = bottom_color.rgb * (1.0 - top_alpha) + blended * top_alpha;
float result_alpha = bottom_color.a * (1.0 - top_alpha) + top_alpha;
frag_color = vec4(result_rgb, result_alpha);
}
"#;
unsafe {
let vertex_shader = gl
.create_shader(glow::VERTEX_SHADER)
.map_err(|e| format!("Failed to create vertex shader: {}", e))?;
gl.shader_source(vertex_shader, vertex_src);
gl.compile_shader(vertex_shader);
if !gl.get_shader_compile_status(vertex_shader) {
let log = gl.get_shader_info_log(vertex_shader);
gl.delete_shader(vertex_shader);
return Err(format!("Vertex shader compilation failed: {}", log));
}
let fragment_shader = gl
.create_shader(glow::FRAGMENT_SHADER)
.map_err(|e| format!("Failed to create fragment shader: {}", e))?;
gl.shader_source(fragment_shader, fragment_src);
gl.compile_shader(fragment_shader);
if !gl.get_shader_compile_status(fragment_shader) {
let log = gl.get_shader_info_log(fragment_shader);
gl.delete_shader(vertex_shader);
gl.delete_shader(fragment_shader);
return Err(format!("Fragment shader compilation failed: {}", log));
}
let program = gl
.create_program()
.map_err(|e| format!("Failed to create program: {}", e))?;
gl.attach_shader(program, vertex_shader);
gl.attach_shader(program, fragment_shader);
gl.link_program(program);
if !gl.get_program_link_status(program) {
let log = gl.get_program_info_log(program);
gl.delete_shader(vertex_shader);
gl.delete_shader(fragment_shader);
gl.delete_program(program);
return Err(format!("Shader program linking failed: {}", log));
}
gl.delete_shader(vertex_shader);
gl.delete_shader(fragment_shader);
trace!("Blend shader compiled successfully");
Ok(program)
}
}
fn upload_frame_to_texture(&self, frame: &Frame) -> Result<glow::Texture, String> {
let gl = &self.gl;
let width = frame.width() as i32;
let height = frame.height() as i32;
unsafe {
let texture = gl
.create_texture()
.map_err(|e| format!("Failed to create texture: {}", e))?;
gl.bind_texture(glow::TEXTURE_2D, Some(texture));
gl.tex_parameter_i32(glow::TEXTURE_2D, glow::TEXTURE_MIN_FILTER, glow::LINEAR as i32);
gl.tex_parameter_i32(glow::TEXTURE_2D, glow::TEXTURE_MAG_FILTER, glow::LINEAR as i32);
gl.tex_parameter_i32(glow::TEXTURE_2D, glow::TEXTURE_WRAP_S, glow::CLAMP_TO_EDGE as i32);
gl.tex_parameter_i32(glow::TEXTURE_2D, glow::TEXTURE_WRAP_T, glow::CLAMP_TO_EDGE as i32);
match (&*frame.buffer(), frame.pixel_format()) {
(PixelBuffer::F32(data), PixelFormat::RgbaF32) => {
gl.tex_image_2d(
glow::TEXTURE_2D,
0,
glow::RGBA32F as i32,
width,
height,
0,
glow::RGBA,
glow::FLOAT,
glow::PixelUnpackData::Slice(Some(bytemuck::cast_slice(data))),
);
}
(PixelBuffer::F16(data), PixelFormat::RgbaF16) => {
let u16_data: Vec<u16> = data.iter().map(|f| f.to_bits()).collect();
gl.tex_image_2d(
glow::TEXTURE_2D,
0,
glow::RGBA16F as i32,
width,
height,
0,
glow::RGBA,
glow::HALF_FLOAT,
glow::PixelUnpackData::Slice(Some(bytemuck::cast_slice(&u16_data))),
);
}
(PixelBuffer::U8(data), PixelFormat::Rgba8) => {
gl.tex_image_2d(
glow::TEXTURE_2D,
0,
glow::RGBA8 as i32,
width,
height,
0,
glow::RGBA,
glow::UNSIGNED_BYTE,
glow::PixelUnpackData::Slice(Some(data)),
);
}
_ => {
gl.delete_texture(texture);
return Err("Pixel format mismatch".to_string());
}
}
Ok(texture)
}
}
fn download_texture_to_frame(
&self,
texture: glow::Texture,
width: usize,
height: usize,
format: PixelFormat,
status: FrameStatus,
) -> Result<Frame, String> {
let gl = &self.gl;
unsafe {
gl.bind_texture(glow::TEXTURE_2D, Some(texture));
match format {
PixelFormat::RgbaF32 => {
let mut data = vec![0.0f32; width * height * 4];
gl.get_tex_image(
glow::TEXTURE_2D,
0,
glow::RGBA,
glow::FLOAT,
glow::PixelPackData::Slice(Some(bytemuck::cast_slice_mut(&mut data))),
);
Ok(Frame::from_f32_buffer_with_status(data, width, height, status))
}
PixelFormat::RgbaF16 => {
let mut u16_data = vec![0u16; width * height * 4];
gl.get_tex_image(
glow::TEXTURE_2D,
0,
glow::RGBA,
glow::HALF_FLOAT,
glow::PixelPackData::Slice(Some(bytemuck::cast_slice_mut(&mut u16_data))),
);
let f16_data: Vec<half::f16> = u16_data.iter().map(|&u| half::f16::from_bits(u)).collect();
Ok(Frame::from_f16_buffer_with_status(f16_data, width, height, status))
}
PixelFormat::Rgba8 => {
let mut data = vec![0u8; width * height * 4];
gl.get_tex_image(
glow::TEXTURE_2D,
0,
glow::RGBA,
glow::UNSIGNED_BYTE,
glow::PixelPackData::Slice(Some(&mut data)),
);
Ok(Frame::from_u8_buffer_with_status(data, width, height, status))
}
}
}
}
fn blend_textures(
&mut self,
bottom: glow::Texture,
top: glow::Texture,
opacity: f32,
mode: &BlendMode,
transform: &[f32; 9],
top_size: (usize, usize),
width: usize,
height: usize,
format: PixelFormat,
) -> Result<glow::Texture, String> {
let gl = &self.gl;
let program = self.blend_program.ok_or("Program not initialized")?;
let fbo = self.fbo.ok_or("FBO not initialized")?;
let vao = self.vao.ok_or("VAO not initialized")?;
unsafe {
let output_texture = self.create_empty_texture(width, height, format)?;
gl.bind_framebuffer(glow::FRAMEBUFFER, Some(fbo));
gl.framebuffer_texture_2d(
glow::FRAMEBUFFER,
glow::COLOR_ATTACHMENT0,
glow::TEXTURE_2D,
Some(output_texture),
0,
);
if gl.check_framebuffer_status(glow::FRAMEBUFFER) != glow::FRAMEBUFFER_COMPLETE {
return Err("Framebuffer incomplete".to_string());
}
gl.viewport(0, 0, width as i32, height as i32);
gl.use_program(Some(program));
gl.active_texture(glow::TEXTURE0);
gl.bind_texture(glow::TEXTURE_2D, Some(bottom));
gl.active_texture(glow::TEXTURE1);
gl.bind_texture(glow::TEXTURE_2D, Some(top));
if let Some(loc) = gl.get_uniform_location(program, "u_bottom") {
gl.uniform_1_i32(Some(&loc), 0);
}
if let Some(loc) = gl.get_uniform_location(program, "u_top") {
gl.uniform_1_i32(Some(&loc), 1);
}
if let Some(loc) = gl.get_uniform_location(program, "u_opacity") {
gl.uniform_1_f32(Some(&loc), opacity);
}
if let Some(loc) = gl.get_uniform_location(program, "u_blend_mode") {
let mode_id = match mode {
BlendMode::Normal => 0,
BlendMode::Screen => 1,
BlendMode::Add => 2,
BlendMode::Subtract => 3,
BlendMode::Multiply => 4,
BlendMode::Divide => 5,
BlendMode::Difference => 6,
BlendMode::Overlay => 7,
};
gl.uniform_1_i32(Some(&loc), mode_id);
}
if let Some(loc) = gl.get_uniform_location(program, "u_top_transform") {
gl.uniform_matrix_3_f32_slice(Some(&loc), false, transform);
}
if let Some(loc) = gl.get_uniform_location(program, "u_canvas_size") {
gl.uniform_2_f32(Some(&loc), width as f32, height as f32);
}
if let Some(loc) = gl.get_uniform_location(program, "u_top_size") {
gl.uniform_2_f32(Some(&loc), top_size.0 as f32, top_size.1 as f32);
}
gl.bind_vertex_array(Some(vao));
gl.draw_arrays(glow::TRIANGLE_FAN, 0, 4);
gl.bind_framebuffer(glow::FRAMEBUFFER, None);
gl.bind_vertex_array(None);
Ok(output_texture)
}
}
fn create_empty_texture(
&self,
width: usize,
height: usize,
format: PixelFormat,
) -> Result<glow::Texture, String> {
let gl = &self.gl;
unsafe {
let texture = gl
.create_texture()
.map_err(|e| format!("Failed to create texture: {}", e))?;
gl.bind_texture(glow::TEXTURE_2D, Some(texture));
gl.tex_parameter_i32(glow::TEXTURE_2D, glow::TEXTURE_MIN_FILTER, glow::LINEAR as i32);
gl.tex_parameter_i32(glow::TEXTURE_2D, glow::TEXTURE_MAG_FILTER, glow::LINEAR as i32);
gl.tex_parameter_i32(glow::TEXTURE_2D, glow::TEXTURE_WRAP_S, glow::CLAMP_TO_EDGE as i32);
gl.tex_parameter_i32(glow::TEXTURE_2D, glow::TEXTURE_WRAP_T, glow::CLAMP_TO_EDGE as i32);
match format {
PixelFormat::RgbaF32 => {
gl.tex_image_2d(
glow::TEXTURE_2D,
0,
glow::RGBA32F as i32,
width as i32,
height as i32,
0,
glow::RGBA,
glow::FLOAT,
glow::PixelUnpackData::Slice(None),
);
}
PixelFormat::RgbaF16 => {
gl.tex_image_2d(
glow::TEXTURE_2D,
0,
glow::RGBA16F as i32,
width as i32,
height as i32,
0,
glow::RGBA,
glow::HALF_FLOAT,
glow::PixelUnpackData::Slice(None),
);
}
PixelFormat::Rgba8 => {
gl.tex_image_2d(
glow::TEXTURE_2D,
0,
glow::RGBA8 as i32,
width as i32,
height as i32,
0,
glow::RGBA,
glow::UNSIGNED_BYTE,
glow::PixelUnpackData::Slice(None),
);
}
}
Ok(texture)
}
}
pub(crate) fn blend(&mut self, frames: Vec<(Frame, f32, BlendMode, [f32; 9])>) -> Option<Frame> {
match self.blend_impl(frames.clone()) {
Ok(result) => Some(result),
Err(e) => {
warn!("GPU compositor failed: {}, falling back to CPU", e);
use super::compositor::CpuCompositor;
CpuCompositor.blend(frames)
}
}
}
fn blend_impl(&mut self, frames: Vec<(Frame, f32, BlendMode, [f32; 9])>) -> Result<Frame, String> {
use crate::entities::frame::FrameStatus;
if frames.is_empty() {
return Err("No frames to blend".to_string());
}
let min_status = frames
.iter()
.map(|(f, _, _, _)| f.status())
.min_by_key(|s| match s {
FrameStatus::Error => 0,
FrameStatus::Placeholder => 1,
FrameStatus::Header => 2,
FrameStatus::Loading | FrameStatus::Composing | FrameStatus::Expired => 3,
FrameStatus::Loaded => 4,
})
.unwrap_or(FrameStatus::Placeholder);
self.ensure_initialized()?;
let (first_frame, _, _, _) = &frames[0];
let width = first_frame.width();
let height = first_frame.height();
let format = first_frame.pixel_format();
trace!(
"GPU blend: {} frames, {}x{}, format: {:?}, min_status: {:?}",
frames.len(),
width,
height,
format,
min_status
);
let mut guard = TextureGuard::new(Arc::clone(&self.gl));
for (frame, _, _, _) in &frames {
let texture = self.upload_frame_to_texture(frame)?;
guard.push(texture);
}
let mut result_texture = guard.textures[0];
for i in 1..guard.textures.len() {
let top_texture = guard.textures[i];
let (top_frame, opacity, mode, transform) = &frames[i];
let top_size = (top_frame.width(), top_frame.height());
let new_result = self.blend_textures(
result_texture,
top_texture,
*opacity,
mode,
transform,
top_size,
width,
height,
format,
)?;
guard.push(new_result);
if i > 1 {
guard.delete(result_texture);
}
result_texture = new_result;
}
let result_frame = self.download_texture_to_frame(result_texture, width, height, format, min_status)?;
drop(guard);
trace!("GPU blend completed successfully with status: {:?}", min_status);
Ok(result_frame)
}
pub(crate) fn blend_with_dim(
&mut self,
frames: Vec<(Frame, f32, BlendMode, [f32; 9])>,
dim: (usize, usize),
) -> Option<Frame> {
let result = self.blend(frames)?;
let cropped = result;
cropped.crop(dim.0, dim.1, super::frame::CropAlign::LeftTop);
Some(cropped)
}
}
impl std::fmt::Debug for GpuCompositor {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("GpuCompositor")
.field("initialized", &self.blend_program.is_some())
.finish()
}
}
impl Drop for GpuCompositor {
fn drop(&mut self) {
unsafe {
if let Some(program) = self.blend_program.take() {
self.gl.delete_program(program);
}
if let Some(vao) = self.vao.take() {
self.gl.delete_vertex_array(vao);
}
if let Some(vbo) = self.vbo.take() {
self.gl.delete_buffer(vbo);
}
if let Some(fbo) = self.fbo.take() {
self.gl.delete_framebuffer(fbo);
}
}
trace!("GpuCompositor dropped and resources cleaned up");
}
}