use super::{Texture, TextureFilter, TextureFormat};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum YuvMatrix {
Bt601,
#[default]
Bt709,
Bt2020,
}
#[derive(Debug, Clone)]
pub enum Samples {
Bytes(Vec<u8>),
Words(Vec<u16>),
}
impl Default for Samples {
fn default() -> Self {
Samples::Words(Vec::new())
}
}
impl From<Vec<u8>> for Samples {
fn from(v: Vec<u8>) -> Self {
Samples::Bytes(v)
}
}
impl From<Vec<u16>> for Samples {
fn from(v: Vec<u16>) -> Self {
Samples::Words(v)
}
}
impl Samples {
pub fn len(&self) -> usize {
match self {
Samples::Bytes(v) => v.len(),
Samples::Words(v) => v.len(),
}
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
#[inline]
pub fn get(&self, i: usize) -> u32 {
match self {
Samples::Bytes(v) => v[i] as u32,
Samples::Words(v) => v[i] as u32,
}
}
pub fn bytes_per_sample(&self) -> u32 {
match self {
Samples::Bytes(_) => 1,
Samples::Words(_) => 2,
}
}
pub fn as_bytes(&self) -> &[u8] {
match self {
Samples::Bytes(v) => v,
Samples::Words(v) => unsafe {
std::slice::from_raw_parts(
v.as_ptr().cast::<u8>(),
std::mem::size_of_val(v.as_slice()),
)
},
}
}
}
#[derive(Debug, Clone, Default)]
pub struct YuvFrame {
pub width: u32,
pub height: u32,
pub bit_depth: u8,
pub subsampling: (u8, u8),
pub y: Samples,
pub u: Samples,
pub v: Samples,
pub interleaved: bool,
pub matrix: YuvMatrix,
pub full_range: bool,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct YuvCoefficients {
pub y_off: f32,
pub y_scale: f32,
pub c_mid: f32,
pub rv: f32,
pub gu: f32,
pub gv: f32,
pub bu: f32,
}
impl YuvFrame {
pub fn is_monochrome(&self) -> bool {
self.u.is_empty()
}
pub fn chroma_size(&self) -> (u32, u32) {
if self.is_monochrome() {
return (0, 0);
}
let (sw, sh) = self.subsampling;
(
self.width.div_ceil(sw.max(1) as u32),
self.height.div_ceil(sh.max(1) as u32),
)
}
pub fn is_valid(&self) -> bool {
let (cw, ch) = self.chroma_size();
let chroma = cw as usize * ch as usize;
let bytes = self.y.bytes_per_sample();
if self.interleaved && !(self.v.is_empty() && self.u.len() == 2 * chroma) {
return false;
}
matches!(self.subsampling, (1 | 2, 1 | 2))
&& (8..=16).contains(&self.bit_depth)
&& (bytes == 2 || self.bit_depth == 8)
&& (self.u.is_empty() || self.u.bytes_per_sample() == bytes)
&& (self.v.is_empty() || self.v.bytes_per_sample() == bytes)
&& self.y.len() == self.width as usize * self.height as usize
&& (self.interleaved
|| (self.u.len() == self.v.len() && (self.u.is_empty() || self.u.len() == chroma)))
}
pub fn coefficients(&self) -> YuvCoefficients {
let depth = self.bit_depth.clamp(8, 16) as u32;
let max = ((1u64 << depth) - 1) as f64;
let s = (1u64 << (depth - 8)) as f64;
let (y_off, y_scale, c_scale) = if self.full_range {
(0.0, 1.0 / max, 1.0 / max)
} else {
(16.0 * s, 1.0 / (219.0 * s), 1.0 / (224.0 * s))
};
let (kr, kb) = match self.matrix {
YuvMatrix::Bt601 => (0.299, 0.114),
YuvMatrix::Bt709 => (0.2126, 0.0722),
YuvMatrix::Bt2020 => (0.2627, 0.0593),
};
let kg = 1.0 - kr - kb;
YuvCoefficients {
y_off: y_off as f32,
y_scale: y_scale as f32,
c_mid: (1u64 << (depth - 1)) as f32,
rv: (2.0 * (1.0 - kr) * c_scale) as f32,
gu: (2.0 * kb * (1.0 - kb) / kg * c_scale) as f32,
gv: (2.0 * kr * (1.0 - kr) / kg * c_scale) as f32,
bu: (2.0 * (1.0 - kb) * c_scale) as f32,
}
}
pub fn to_rgba_into(&self, out: &mut Vec<u8>) {
let (w, h) = (self.width as usize, self.height as usize);
out.clear();
out.resize(w * h * 4, 255);
if !self.is_valid() {
return;
}
let k = self.coefficients();
let (cw, ch) = self.chroma_size();
let (cw, ch) = (cw as usize, ch as usize);
let (sw, sh) = (self.subsampling.0 == 2, self.subsampling.1 == 2);
let to8 = |v: f32| (v.clamp(0.0, 1.0) * 255.0 + 0.5) as u8;
for y in 0..h {
let rows = chroma_rows(y, ch, sh);
for x in 0..w {
let (cb, cr) = if self.is_monochrome() {
(0.0, 0.0)
} else {
let cols = chroma_cols(x, cw, sw);
let at = |which: usize, i: usize| {
if self.interleaved {
self.u.get(2 * i + which)
} else if which == 0 {
self.u.get(i)
} else {
self.v.get(i)
}
};
let tap = |which: usize| {
let mut sum = 0u32;
for (r, wr) in [(rows.0, rows.2), (rows.1, rows.3)] {
for (c, wc) in [(cols.0, cols.2), (cols.1, cols.3)] {
sum += wr * wc * at(which, r * cw + c);
}
}
sum as f32 / 8.0 - k.c_mid
};
(tap(0), tap(1))
};
let l = k.y_scale * (self.y.get(y * w + x) as f32 - k.y_off);
let px = &mut out[(y * w + x) * 4..][..3];
px[0] = to8(l + k.rv * cr);
px[1] = to8(l - k.gu * cb - k.gv * cr);
px[2] = to8(l + k.bu * cb);
}
}
}
}
fn chroma_rows(y: usize, ch: usize, halved: bool) -> (usize, usize, u32, u32) {
if !halved {
let r = y.min(ch.saturating_sub(1));
return (r, r, 4, 0);
}
let c = (y / 2).min(ch.saturating_sub(1));
let other = if y.is_multiple_of(2) {
c.saturating_sub(1)
} else {
(c + 1).min(ch.saturating_sub(1))
};
(c, other, 3, 1)
}
fn chroma_cols(x: usize, cw: usize, halved: bool) -> (usize, usize, u32, u32) {
if !halved {
let c = x.min(cw.saturating_sub(1));
return (c, c, 2, 0);
}
let j = x / 2;
if x.is_multiple_of(2) {
(j, j, 2, 0)
} else {
(j, (j + 1).min(cw.saturating_sub(1)), 1, 1)
}
}
#[derive(Debug, Clone, Copy)]
pub enum StreamFrame<'a> {
Rgba(&'a [u8]),
Yuv(&'a YuvFrame),
#[cfg(target_arch = "wasm32")]
Browser(&'a BrowserImage),
}
#[cfg(target_arch = "wasm32")]
#[derive(Debug, Clone)]
pub enum BrowserImage {
Video(web_sys::HtmlVideoElement),
Frame(web_sys::VideoFrame),
}
#[cfg(target_arch = "wasm32")]
impl BrowserImage {
pub fn size(&self) -> (u32, u32) {
match self {
BrowserImage::Video(v) => (v.video_width(), v.video_height()),
BrowserImage::Frame(f) => (f.display_width(), f.display_height()),
}
}
fn release(&self) {
if let BrowserImage::Frame(f) = self {
f.close();
}
}
}
#[cfg(target_arch = "wasm32")]
struct OnMainThread(BrowserImage);
#[cfg(target_arch = "wasm32")]
unsafe impl Send for OnMainThread {}
#[cfg(target_arch = "wasm32")]
unsafe impl Sync for OnMainThread {}
#[cfg(target_arch = "wasm32")]
impl Drop for OnMainThread {
fn drop(&mut self) {
self.0.release();
}
}
struct Held {
frame: Kind,
rgba: Option<(u64, Vec<u8>)>,
}
enum Kind {
Rgba(Vec<u8>),
Yuv(YuvFrame),
#[cfg(target_arch = "wasm32")]
Browser(OnMainThread),
}
pub struct TextureStream {
width: u32,
height: u32,
held: Mutex<Held>,
seq: AtomicU64,
}
impl std::fmt::Debug for TextureStream {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TextureStream")
.field("width", &self.width)
.field("height", &self.height)
.field("frames", &self.frames())
.finish()
}
}
impl TextureStream {
pub fn new(width: u32, height: u32) -> Self {
Self {
width,
height,
held: Mutex::new(Held {
frame: Kind::Rgba(Vec::new()),
rgba: None,
}),
seq: AtomicU64::new(0),
}
}
pub fn size(&self) -> (u32, u32) {
(self.width, self.height)
}
pub fn frames(&self) -> u64 {
self.seq.load(Ordering::Acquire)
}
fn lock(&self) -> std::sync::MutexGuard<'_, Held> {
self.held.lock().unwrap_or_else(|e| e.into_inner())
}
pub fn write_rgba(&self, rgba: &[u8]) -> bool {
self.write_with(|frame| {
if frame.len() != rgba.len() {
return false;
}
frame.copy_from_slice(rgba);
true
})
}
pub fn write_with(&self, fill: impl FnOnce(&mut [u8]) -> bool) -> bool {
let mut held = self.lock();
if !matches!(held.frame, Kind::Rgba(_)) {
let buffer = held.rgba.take().map(|(_, b)| b);
let size = self.width as usize * self.height as usize * 4;
held.frame = Kind::Rgba(
buffer
.filter(|b| b.len() == size)
.unwrap_or_else(|| vec![0; size]),
);
}
let Kind::Rgba(frame) = &mut held.frame else {
unreachable!()
};
let size = self.width as usize * self.height as usize * 4;
if frame.len() != size {
*frame = black(self.width, self.height);
}
let wrote = fill(frame);
if wrote {
self.seq.fetch_add(1, Ordering::Release);
}
wrote
}
pub fn write_yuv(&self, frame: YuvFrame) -> bool {
self.swap_yuv(frame).is_ok()
}
pub fn swap_yuv(&self, frame: YuvFrame) -> Result<Option<YuvFrame>, YuvFrame> {
if (frame.width, frame.height) != (self.width, self.height) || !frame.is_valid() {
return Err(frame);
}
let mut held = self.lock();
let replaced = match std::mem::replace(&mut held.frame, Kind::Yuv(frame)) {
Kind::Rgba(old) => {
if held.rgba.is_none() && !old.is_empty() {
held.rgba = Some((u64::MAX, old));
}
None
}
Kind::Yuv(old) => Some(old),
#[cfg(target_arch = "wasm32")]
Kind::Browser(_) => None,
};
self.seq.fetch_add(1, Ordering::Release);
Ok(replaced)
}
#[cfg(target_arch = "wasm32")]
pub fn write_browser(&self, image: BrowserImage) -> bool {
if image.size() != (self.width, self.height) {
image.release();
return false;
}
self.lock().frame = Kind::Browser(OnMainThread(image));
self.seq.fetch_add(1, Ordering::Release);
true
}
pub fn read<R>(&self, read: impl FnOnce(&[u8], u64) -> R) -> R {
let mut held = self.lock();
let seq = self.seq.load(Ordering::Acquire);
let Held { frame, rgba } = &mut *held;
match frame {
Kind::Rgba(bytes) if bytes.is_empty() => {
if rgba.is_none() {
*rgba = Some((u64::MAX, black(self.width, self.height)));
}
read(&rgba.as_ref().expect("just made").1, seq)
}
Kind::Rgba(bytes) => read(bytes, seq),
#[cfg(target_arch = "wasm32")]
Kind::Browser(_) => {
let (w, h) = (self.width, self.height);
let buffer = rgba.get_or_insert_with(|| (u64::MAX, black(w, h)));
if buffer.0 != seq {
*buffer = (seq, black(w, h));
}
read(&buffer.1, seq)
}
Kind::Yuv(yuv) => {
let (at, buffer) = rgba.get_or_insert_with(|| (u64::MAX, Vec::new()));
if *at != seq {
yuv.to_rgba_into(buffer);
*at = seq;
}
read(buffer, seq)
}
}
}
pub fn read_frame<R>(&self, read: impl FnOnce(StreamFrame<'_>, u64) -> R) -> R {
let held = self.lock();
let seq = self.seq.load(Ordering::Acquire);
match &held.frame {
Kind::Rgba(bytes) => read(StreamFrame::Rgba(bytes), seq),
Kind::Yuv(yuv) => read(StreamFrame::Yuv(yuv), seq),
#[cfg(target_arch = "wasm32")]
Kind::Browser(image) => read(StreamFrame::Browser(&image.0), seq),
}
}
}
fn black(width: u32, height: u32) -> Vec<u8> {
let mut black = vec![0u8; width as usize * height as usize * 4];
black
.as_chunks_mut::<4>()
.0
.iter_mut()
.for_each(|p| p[3] = 255);
black
}
#[derive(Debug, Clone)]
pub struct VideoTexture {
texture: Arc<Texture>,
stream: Arc<TextureStream>,
}
impl VideoTexture {
pub fn new(width: u32, height: u32) -> Self {
let stream = Arc::new(TextureStream::new(width, height));
let mut texture = Texture::new(width, height, TextureFormat::Rgba8UnormSrgb, Vec::new());
texture.mipmap_filter = Some(TextureFilter::Linear);
texture.min_filter = TextureFilter::Linear;
texture.mag_filter = TextureFilter::Linear;
texture.flip_y = true;
texture.stream = Some(stream.clone());
Self {
texture: Arc::new(texture),
stream,
}
}
pub fn without_mipmaps(self) -> Self {
let mut texture = (*self.texture).clone();
texture.mipmap_filter = None;
texture.id = super::texture::next_texture_id();
Self {
texture: Arc::new(texture),
stream: self.stream,
}
}
pub fn texture(&self) -> Arc<Texture> {
self.texture.clone()
}
pub fn stream(&self) -> &Arc<TextureStream> {
&self.stream
}
pub fn size(&self) -> (u32, u32) {
self.stream.size()
}
pub fn write_rgba(&self, rgba: &[u8]) -> bool {
self.stream.write_rgba(rgba)
}
pub fn write_yuv(&self, frame: YuvFrame) -> bool {
self.stream.write_yuv(frame)
}
}