use eframe::{
egui,
egui_wgpu::{self, wgpu},
};
use openipc_video::{DecodedFrame, DecodedSurface as _, PixelFormat};
const SHADER: &str = r#"
@group(0) @binding(0) var video_sampler: sampler;
@group(0) @binding(1) var y_texture: texture_2d<f32>;
@group(0) @binding(2) var uv_texture: texture_2d<f32>;
struct VertexOutput {
@builtin(position) position: vec4<f32>,
@location(0) uv: vec2<f32>,
};
@vertex
fn vertex(@builtin(vertex_index) index: u32) -> VertexOutput {
var positions = array<vec2<f32>, 3>(
vec2<f32>(-1.0, -1.0),
vec2<f32>(3.0, -1.0),
vec2<f32>(-1.0, 3.0),
);
var output: VertexOutput;
output.position = vec4<f32>(positions[index], 0.0, 1.0);
output.uv = vec2<f32>(
(positions[index].x + 1.0) * 0.5,
1.0 - (positions[index].y + 1.0) * 0.5,
);
return output;
}
@fragment
fn fragment(input: VertexOutput) -> @location(0) vec4<f32> {
let y = textureSample(y_texture, video_sampler, input.uv).r;
let chroma = textureSample(uv_texture, video_sampler, input.uv).rg - vec2<f32>(0.5, 0.5);
let luma = max(y - 16.0 / 255.0, 0.0) * 1.1643836;
let rgb = vec3<f32>(
luma + 1.5960272 * chroma.y,
luma - 0.3917623 * chroma.x - 0.8129683 * chroma.y,
luma + 2.0172321 * chroma.x,
);
return vec4<f32>(rgb, 1.0);
}
"#;
pub(crate) struct NativeNv12Renderer {
render_state: egui_wgpu::RenderState,
}
struct Nv12Resources {
pipeline: wgpu::RenderPipeline,
bind_group_layout: wgpu::BindGroupLayout,
sampler: wgpu::Sampler,
frame: Option<GpuFrame>,
}
struct GpuFrame {
y_texture: wgpu::Texture,
uv_texture: wgpu::Texture,
bind_group: wgpu::BindGroup,
dimensions: [u32; 2],
}
struct Nv12PaintCallback;
impl NativeNv12Renderer {
pub(crate) fn new(context: &eframe::CreationContext<'_>) -> Result<Self, String> {
let render_state = context
.wgpu_render_state
.clone()
.ok_or_else(|| "native NV12 renderer requires the wgpu backend".to_owned())?;
let resources = Nv12Resources::new(&render_state.device, render_state.target_format);
render_state
.renderer
.write()
.callback_resources
.insert(resources);
Ok(Self { render_state })
}
#[cfg(target_os = "macos")]
pub(crate) fn upload(
&self,
frame: &DecodedFrame<openipc_video::MacOsVideoFrame>,
) -> Result<(), String> {
if frame.surface.pixel_format() != PixelFormat::Nv12VideoRange {
return Err(format!(
"GPU presenter requires video-range NV12, received {:?}",
frame.surface.pixel_format()
));
}
let dimensions = frame.dimensions();
frame
.surface
.with_mapped_planes(|planes| {
let [y, uv, ..] = planes else {
return Err("VideoToolbox NV12 frame did not expose two planes".to_owned());
};
self.upload_planes(
[dimensions.width, dimensions.height],
y.data(),
y.stride(),
uv.data(),
uv.stride(),
)
})
.map_err(|error| error.to_string())?
}
#[cfg(target_os = "linux")]
pub(crate) fn upload(
&self,
frame: &DecodedFrame<openipc_video::LinuxVideoFrame>,
) -> Result<(), String> {
if frame.surface.pixel_format() != PixelFormat::Nv12VideoRange {
return Err(format!(
"GPU presenter requires video-range NV12, received {:?}",
frame.surface.pixel_format()
));
}
let dimensions = frame.dimensions();
let pitches = frame.surface.plane_pitches();
frame
.surface
.with_mapped_planes(|planes| {
let [y, uv, ..] = planes else {
return Err("VA-API NV12 frame did not expose two planes".to_owned());
};
let [y_stride, uv_stride, ..] = pitches.as_slice() else {
return Err("VA-API NV12 frame did not expose two pitches".to_owned());
};
self.upload_planes(
[dimensions.width, dimensions.height],
y,
*y_stride,
uv,
*uv_stride,
)
})
.map_err(|error| error.to_string())?
}
#[cfg(target_os = "windows")]
pub(crate) fn upload(
&self,
frame: &DecodedFrame<openipc_video::WindowsVideoFrame>,
) -> Result<(), String> {
if frame.surface.pixel_format() != PixelFormat::Nv12VideoRange {
return Err(format!(
"GPU presenter requires video-range NV12, received {:?}",
frame.surface.pixel_format()
));
}
let mapped = frame
.surface
.copy_nv12()
.map_err(|error| error.to_string())?;
let dimensions = mapped.dimensions();
self.upload_planes(
[dimensions.width, dimensions.height],
mapped.y_plane(),
mapped.stride(),
mapped.uv_plane(),
mapped.stride(),
)
}
fn upload_planes(
&self,
dimensions: [u32; 2],
y: &[u8],
y_stride: usize,
uv: &[u8],
uv_stride: usize,
) -> Result<(), String> {
let mut renderer = self.render_state.renderer.write();
let resources = renderer
.callback_resources
.get_mut::<Nv12Resources>()
.ok_or_else(|| "NV12 GPU resources are unavailable".to_owned())?;
resources.upload(
&self.render_state.device,
&self.render_state.queue,
dimensions,
y,
y_stride,
uv,
uv_stride,
)
}
pub(crate) fn paint(&self, painter: &egui::Painter, rect: egui::Rect) {
painter.add(egui_wgpu::Callback::new_paint_callback(
rect,
Nv12PaintCallback,
));
}
}
impl Nv12Resources {
fn new(device: &wgpu::Device, target_format: wgpu::TextureFormat) -> Self {
let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("nebulus-nv12-layout"),
entries: &[
wgpu::BindGroupLayoutEntry {
binding: 0,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
count: None,
},
texture_layout(1),
texture_layout(2),
],
});
let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("nebulus-nv12-pipeline-layout"),
bind_group_layouts: &[Some(&bind_group_layout)],
immediate_size: 0,
});
let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("nebulus-nv12-shader"),
source: wgpu::ShaderSource::Wgsl(SHADER.into()),
});
let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
label: Some("nebulus-nv12-pipeline"),
layout: Some(&pipeline_layout),
vertex: wgpu::VertexState {
module: &shader,
entry_point: Some("vertex"),
compilation_options: Default::default(),
buffers: &[],
},
fragment: Some(wgpu::FragmentState {
module: &shader,
entry_point: Some("fragment"),
compilation_options: Default::default(),
targets: &[Some(wgpu::ColorTargetState {
format: target_format,
blend: Some(wgpu::BlendState::REPLACE),
write_mask: wgpu::ColorWrites::ALL,
})],
}),
primitive: Default::default(),
depth_stencil: None,
multisample: Default::default(),
multiview_mask: None,
cache: None,
});
let sampler = device.create_sampler(&wgpu::SamplerDescriptor {
label: Some("nebulus-nv12-sampler"),
mag_filter: wgpu::FilterMode::Linear,
min_filter: wgpu::FilterMode::Linear,
..Default::default()
});
Self {
pipeline,
bind_group_layout,
sampler,
frame: None,
}
}
#[allow(clippy::too_many_arguments)]
fn upload(
&mut self,
device: &wgpu::Device,
queue: &wgpu::Queue,
dimensions: [u32; 2],
y: &[u8],
y_stride: usize,
uv: &[u8],
uv_stride: usize,
) -> Result<(), String> {
let [width, height] = dimensions;
if let Some(frame) = self
.frame
.as_ref()
.filter(|frame| frame.dimensions == dimensions)
{
write_plane(queue, &frame.y_texture, y, y_stride, width, height, 1)?;
write_plane(
queue,
&frame.uv_texture,
uv,
uv_stride,
width.div_ceil(2),
height.div_ceil(2),
2,
)?;
return Ok(());
}
let y_texture = create_texture(
device,
"nebulus-nv12-y",
width,
height,
wgpu::TextureFormat::R8Unorm,
);
let uv_texture = create_texture(
device,
"nebulus-nv12-uv",
width.div_ceil(2),
height.div_ceil(2),
wgpu::TextureFormat::Rg8Unorm,
);
write_plane(queue, &y_texture, y, y_stride, width, height, 1)?;
write_plane(
queue,
&uv_texture,
uv,
uv_stride,
width.div_ceil(2),
height.div_ceil(2),
2,
)?;
let y_view = y_texture.create_view(&Default::default());
let uv_view = uv_texture.create_view(&Default::default());
let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("nebulus-nv12-bind-group"),
layout: &self.bind_group_layout,
entries: &[
wgpu::BindGroupEntry {
binding: 0,
resource: wgpu::BindingResource::Sampler(&self.sampler),
},
wgpu::BindGroupEntry {
binding: 1,
resource: wgpu::BindingResource::TextureView(&y_view),
},
wgpu::BindGroupEntry {
binding: 2,
resource: wgpu::BindingResource::TextureView(&uv_view),
},
],
});
self.frame = Some(GpuFrame {
y_texture,
uv_texture,
bind_group,
dimensions,
});
Ok(())
}
}
impl egui_wgpu::CallbackTrait for Nv12PaintCallback {
fn paint(
&self,
_info: egui::PaintCallbackInfo,
render_pass: &mut wgpu::RenderPass<'static>,
callback_resources: &egui_wgpu::CallbackResources,
) {
let Some(resources) = callback_resources.get::<Nv12Resources>() else {
return;
};
let Some(frame) = resources.frame.as_ref() else {
return;
};
render_pass.set_pipeline(&resources.pipeline);
render_pass.set_bind_group(0, &frame.bind_group, &[]);
render_pass.draw(0..3, 0..1);
}
}
fn texture_layout(binding: u32) -> wgpu::BindGroupLayoutEntry {
wgpu::BindGroupLayoutEntry {
binding,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Texture {
sample_type: wgpu::TextureSampleType::Float { filterable: true },
view_dimension: wgpu::TextureViewDimension::D2,
multisampled: false,
},
count: None,
}
}
fn create_texture(
device: &wgpu::Device,
label: &'static str,
width: u32,
height: u32,
format: wgpu::TextureFormat,
) -> wgpu::Texture {
device.create_texture(&wgpu::TextureDescriptor {
label: Some(label),
size: wgpu::Extent3d {
width,
height,
depth_or_array_layers: 1,
},
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[],
})
}
fn write_plane(
queue: &wgpu::Queue,
texture: &wgpu::Texture,
bytes: &[u8],
stride: usize,
width: u32,
height: u32,
bytes_per_texel: usize,
) -> Result<(), String> {
let required_row_bytes = usize::try_from(width)
.ok()
.and_then(|width| width.checked_mul(bytes_per_texel))
.ok_or_else(|| "video plane row size overflows usize".to_owned())?;
let required_len = required_plane_len(stride, required_row_bytes, height)?;
if bytes.len() < required_len {
return Err(format!(
"video plane is truncated: received {} bytes, need at least {required_len}",
bytes.len()
));
}
let stride = u32::try_from(stride).map_err(|_| "video plane stride exceeds u32")?;
queue.write_texture(
wgpu::TexelCopyTextureInfo {
texture,
mip_level: 0,
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
bytes,
wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: Some(stride),
rows_per_image: Some(height),
},
wgpu::Extent3d {
width,
height,
depth_or_array_layers: 1,
},
);
Ok(())
}
fn required_plane_len(
stride: usize,
required_row_bytes: usize,
height: u32,
) -> Result<usize, String> {
if height == 0 || required_row_bytes == 0 {
return Err("video plane dimensions must be non-zero".to_owned());
}
if stride < required_row_bytes {
return Err(format!(
"video plane stride {stride} is smaller than its {required_row_bytes}-byte row"
));
}
usize::try_from(height - 1)
.ok()
.and_then(|rows| rows.checked_mul(stride))
.and_then(|prefix| prefix.checked_add(required_row_bytes))
.ok_or_else(|| "video plane byte length overflows usize".to_owned())
}
#[cfg(test)]
mod tests {
use super::required_plane_len;
#[test]
fn plane_length_allows_padding_after_every_nonfinal_row() {
assert_eq!(required_plane_len(8, 6, 3), Ok(22));
}
#[test]
fn plane_length_rejects_short_stride() {
assert!(required_plane_len(5, 6, 3).unwrap_err().contains("smaller"));
}
}