use ::rlx::{DType, Dim, Shape};
use crate::core::{BufferAttribute, BufferGeometry};
use crate::textures::{pack_rgba16f, Texture, TextureFormat};
use super::{linear_to_srgb, srgb_to_linear};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Layout {
Nhwc,
Nchw,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ColorSpace {
Srgb,
Linear,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum TensorError {
ShapeMismatch { dims: Vec<usize>, len: usize },
DynamicDim,
NotF32(DType),
Rank { expected: &'static str, got: usize },
Channels(usize),
FormatChannels {
format: TextureFormat,
channels: usize,
},
ByteCount { expected: usize, got: usize },
Empty,
}
impl std::fmt::Display for TensorError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::ShapeMismatch { dims, len } => {
write!(f, "{len} values do not fill dims {dims:?}")
}
Self::DynamicDim => write!(f, "shape has a dynamic dimension"),
Self::NotF32(d) => write!(f, "expected an f32 shape, got {d:?}"),
Self::Rank { expected, got } => write!(f, "expected rank {expected}, got {got}"),
Self::Channels(c) => write!(f, "{c} channels is not 1, 3 or 4"),
Self::FormatChannels { format, channels } => {
write!(f, "{channels} channels cannot be written as {format:?}")
}
Self::ByteCount { expected, got } => {
write!(f, "expected {expected} bytes, got {got}")
}
Self::Empty => write!(f, "nothing to work on"),
}
}
}
impl std::error::Error for TensorError {}
#[derive(Debug, Clone, PartialEq)]
pub struct Tensor {
data: Vec<f32>,
dims: Vec<usize>,
}
impl Tensor {
pub fn new(data: Vec<f32>, dims: &[usize]) -> Result<Self, TensorError> {
let want: usize = dims.iter().product();
if want != data.len() {
return Err(TensorError::ShapeMismatch {
dims: dims.to_vec(),
len: data.len(),
});
}
Ok(Self {
data,
dims: dims.to_vec(),
})
}
pub fn zeros(dims: &[usize]) -> Self {
Self {
data: vec![0.0; dims.iter().product()],
dims: dims.to_vec(),
}
}
pub fn from_flat(data: Vec<f32>) -> Self {
Self {
dims: vec![data.len()],
data,
}
}
pub fn from_shape(shape: &Shape, data: Vec<f32>) -> Result<Self, TensorError> {
if shape.dtype() != DType::F32 {
return Err(TensorError::NotF32(shape.dtype()));
}
let mut dims = Vec::with_capacity(shape.rank());
for d in shape.dims() {
match d {
Dim::Static(n) => dims.push(*n),
Dim::Dynamic(_) => return Err(TensorError::DynamicDim),
}
}
Self::new(data, &dims)
}
pub fn shape(&self) -> Shape {
Shape::new(&self.dims, DType::F32)
}
pub fn dims(&self) -> &[usize] {
&self.dims
}
pub fn rank(&self) -> usize {
self.dims.len()
}
pub fn len(&self) -> usize {
self.data.len()
}
pub fn is_empty(&self) -> bool {
self.data.is_empty()
}
pub fn data(&self) -> &[f32] {
&self.data
}
pub fn as_mut_slice(&mut self) -> &mut [f32] {
&mut self.data
}
pub fn into_data(self) -> Vec<f32> {
self.data
}
pub fn reshape(self, dims: &[usize]) -> Result<Self, TensorError> {
Self::new(self.data, dims)
}
}
impl From<&BufferAttribute> for Tensor {
fn from(attr: &BufferAttribute) -> Self {
Self {
dims: vec![attr.count(), attr.item_size],
data: attr.array.clone(),
}
}
}
impl TryFrom<&Tensor> for BufferAttribute {
type Error = TensorError;
fn try_from(t: &Tensor) -> Result<Self, Self::Error> {
if t.rank() != 2 {
return Err(TensorError::Rank {
expected: "2 ([count, item_size])",
got: t.rank(),
});
}
Ok(BufferAttribute::new(t.data.clone(), t.dims[1].max(1)))
}
}
pub fn geometry_to_tensor(geometry: &BufferGeometry, name: &str) -> Option<Tensor> {
geometry.get_attribute(name).map(Tensor::from)
}
pub fn tensor_to_geometry(
geometry: &mut BufferGeometry,
name: &str,
tensor: &Tensor,
) -> Result<(), TensorError> {
let attr = BufferAttribute::try_from(tensor)?;
geometry.set_attribute(name, attr);
Ok(())
}
pub fn frame_to_tensor(
rgba: &[u8],
width: u32,
height: u32,
layout: Layout,
space: ColorSpace,
) -> Result<Tensor, TensorError> {
let expected = width as usize * height as usize * 4;
if rgba.len() != expected {
return Err(TensorError::ByteCount {
expected,
got: rgba.len(),
});
}
let decode = matches!(space, ColorSpace::Linear);
Ok(pack(width, height, 4, layout, |i, c| {
let v = rgba[i * 4 + c] as f32 / 255.0;
if decode && c < 3 {
srgb_to_linear(v)
} else {
v
}
}))
}
pub fn tensor_to_frame(
tensor: &Tensor,
layout: Layout,
space: ColorSpace,
) -> Result<(Vec<u8>, u32, u32), TensorError> {
let (width, height, channels) = image_dims(tensor, layout)?;
let encode = matches!(space, ColorSpace::Linear);
let mut out = vec![0u8; width as usize * height as usize * 4];
for i in 0..(width as usize * height as usize) {
for c in 0..4 {
out[i * 4 + c] = if c < channels {
let v = at(tensor, layout, width, height, channels, i, c);
let v = if encode && c < 3 {
linear_to_srgb(v)
} else {
v
};
quantize(v)
} else if c == 3 {
255
} else {
let v = at(tensor, layout, width, height, channels, i, 0);
let v = if encode { linear_to_srgb(v) } else { v };
quantize(v)
};
}
}
Ok((out, width, height))
}
pub fn texture_to_tensor(
texture: &Texture,
layout: Layout,
space: ColorSpace,
) -> Result<Tensor, TensorError> {
let (width, height) = (texture.width, texture.height);
let pixels = width as usize * height as usize;
let bpp = texture.bytes_per_pixel();
let expected = pixels * bpp;
if texture.data.len() != expected {
return Err(TensorError::ByteCount {
expected,
got: texture.data.len(),
});
}
let channels = match texture.format {
TextureFormat::R8Unorm => 1,
_ => 4,
};
let stored_srgb = matches!(texture.format, TextureFormat::Rgba8UnormSrgb);
let want_linear = matches!(space, ColorSpace::Linear);
let bytes = &texture.data;
Ok(pack(width, height, channels, layout, |i, c| {
let v = match texture.format {
TextureFormat::Rgba16Float => {
let o = i * 8 + c * 2;
f16_to_f32(u16::from_le_bytes([bytes[o], bytes[o + 1]]))
}
_ => bytes[i * bpp + c] as f32 / 255.0,
};
match (stored_srgb, want_linear, c) {
(_, _, 3) => v,
(true, true, _) => srgb_to_linear(v),
(false, false, _) => linear_to_srgb(v),
_ => v,
}
}))
}
pub fn tensor_to_texture(
tensor: &Tensor,
format: TextureFormat,
layout: Layout,
space: ColorSpace,
) -> Result<Texture, TensorError> {
let (width, height, channels) = image_dims(tensor, layout)?;
let pixels = width as usize * height as usize;
let fits = match format {
TextureFormat::R8Unorm => channels == 1,
_ => channels == 3 || channels == 4,
};
if !fits {
return Err(TensorError::FormatChannels { format, channels });
}
let stored_srgb = matches!(format, TextureFormat::Rgba8UnormSrgb);
let want_linear = matches!(space, ColorSpace::Linear);
let sample = |i: usize, c: usize| -> f32 {
let v = at(tensor, layout, width, height, channels, i, c);
match (stored_srgb, want_linear, c) {
(_, _, 3) => v,
(true, true, _) => linear_to_srgb(v),
(false, false, _) => srgb_to_linear(v),
_ => v,
}
};
let data = match format {
TextureFormat::R8Unorm => (0..pixels).map(|i| quantize(sample(i, 0))).collect(),
TextureFormat::Rgba16Float => {
let mut rgba = Vec::with_capacity(pixels * 4);
for i in 0..pixels {
for c in 0..4 {
rgba.push(if c < channels { sample(i, c) } else { 1.0 });
}
}
pack_rgba16f(&rgba)
}
_ => {
let mut out = Vec::with_capacity(pixels * 4);
for i in 0..pixels {
for c in 0..4 {
out.push(if c < channels {
quantize(sample(i, c))
} else {
255
});
}
}
out
}
};
Ok(Texture::new(width, height, format, data))
}
fn pack(
width: u32,
height: u32,
channels: usize,
layout: Layout,
value: impl Fn(usize, usize) -> f32,
) -> Tensor {
let pixels = width as usize * height as usize;
let mut data = Vec::with_capacity(pixels * channels);
match layout {
Layout::Nhwc => {
for i in 0..pixels {
for c in 0..channels {
data.push(value(i, c));
}
}
}
Layout::Nchw => {
for c in 0..channels {
for i in 0..pixels {
data.push(value(i, c));
}
}
}
}
let dims = match layout {
Layout::Nhwc => vec![1, height as usize, width as usize, channels],
Layout::Nchw => vec![1, channels, height as usize, width as usize],
};
Tensor { data, dims }
}
fn at(
tensor: &Tensor,
layout: Layout,
width: u32,
height: u32,
channels: usize,
pixel: usize,
channel: usize,
) -> f32 {
let pixels = width as usize * height as usize;
let index = match layout {
Layout::Nhwc => pixel * channels + channel,
Layout::Nchw => channel * pixels + pixel,
};
tensor.data[index]
}
fn image_dims(tensor: &Tensor, layout: Layout) -> Result<(u32, u32, usize), TensorError> {
let d = tensor.dims();
let spatial = match d.len() {
4 => &d[1..],
3 => d,
got => {
return Err(TensorError::Rank {
expected: "3 or 4 (an image)",
got,
})
}
};
let (height, width, channels) = match layout {
Layout::Nhwc => (spatial[0], spatial[1], spatial[2]),
Layout::Nchw => (spatial[1], spatial[2], spatial[0]),
};
if !matches!(channels, 1 | 3 | 4) {
return Err(TensorError::Channels(channels));
}
Ok((width as u32, height as u32, channels))
}
fn quantize(v: f32) -> u8 {
(v.clamp(0.0, 1.0) * 255.0 + 0.5) as u8
}
fn f16_to_f32(bits: u16) -> f32 {
let sign = ((bits >> 15) & 1) as u32;
let exp = ((bits >> 10) & 0x1f) as u32;
let mant = (bits & 0x3ff) as u32;
let out = match exp {
0 => {
if mant == 0 {
sign << 31
} else {
let v = mant as f32 * (1.0 / 16_777_216.0); return if sign == 1 { -v } else { v };
}
}
0x1f => (sign << 31) | 0x7f80_0000 | (mant << 13),
_ => (sign << 31) | ((exp + 112) << 23) | (mant << 13),
};
f32::from_bits(out)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn an_attribute_round_trips_through_a_tensor() {
let attr = BufferAttribute::new(vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0], 3);
let t = Tensor::from(&attr);
assert_eq!(t.dims(), &[2, 3]);
let back = BufferAttribute::try_from(&t).unwrap();
assert_eq!(back.array, attr.array);
assert_eq!(back.item_size, 3);
}
#[test]
fn a_shape_with_a_dynamic_dim_is_refused() {
let shape = Shape::from_dims(&[Dim::Static(2), Dim::Dynamic(0)], DType::F32);
assert_eq!(
Tensor::from_shape(&shape, vec![0.0; 4]),
Err(TensorError::DynamicDim)
);
}
#[test]
#[allow(clippy::identity_op)]
fn the_two_layouts_hold_the_same_pixels_in_a_different_order() {
let rgba: Vec<u8> = (0..(2 * 2 * 4) as u8).collect();
let nhwc = frame_to_tensor(&rgba, 2, 2, Layout::Nhwc, ColorSpace::Srgb).unwrap();
let nchw = frame_to_tensor(&rgba, 2, 2, Layout::Nchw, ColorSpace::Srgb).unwrap();
assert_eq!(nhwc.dims(), &[1, 2, 2, 4]);
assert_eq!(nchw.dims(), &[1, 4, 2, 2]);
assert_eq!(nhwc.data()[1 * 4 + 2], nchw.data()[2 * 4 + 1]);
for layout in [Layout::Nhwc, Layout::Nchw] {
let t = frame_to_tensor(&rgba, 2, 2, layout, ColorSpace::Srgb).unwrap();
let (back, w, h) = tensor_to_frame(&t, layout, ColorSpace::Srgb).unwrap();
assert_eq!((w, h), (2, 2));
assert_eq!(back, rgba);
}
}
#[test]
fn an_srgb_texture_decodes_to_linear_and_back() {
let tex = Texture::new(1, 1, TextureFormat::Rgba8UnormSrgb, vec![188, 128, 0, 200]);
let t = texture_to_tensor(&tex, Layout::Nhwc, ColorSpace::Linear).unwrap();
assert!((t.data()[0] - 0.5).abs() < 0.01, "{}", t.data()[0]);
assert!((t.data()[3] - 200.0 / 255.0).abs() < 1e-6);
let back = tensor_to_texture(
&t,
TextureFormat::Rgba8UnormSrgb,
Layout::Nhwc,
ColorSpace::Linear,
)
.unwrap();
assert_eq!(*back.data, *tex.data);
}
#[test]
fn a_linear_texture_is_copied_not_transferred() {
let tex = Texture::new(1, 1, TextureFormat::Rgba8Unorm, vec![188, 128, 0, 200]);
let t = texture_to_tensor(&tex, Layout::Nhwc, ColorSpace::Linear).unwrap();
assert!((t.data()[0] - 188.0 / 255.0).abs() < 1e-6);
}
#[test]
fn half_floats_survive_the_round_trip() {
let tex = Texture::new(
1,
1,
TextureFormat::Rgba16Float,
pack_rgba16f(&[4.5, 0.25, 0.0, 1.0]),
);
let t = texture_to_tensor(&tex, Layout::Nhwc, ColorSpace::Linear).unwrap();
assert_eq!(t.data(), &[4.5, 0.25, 0.0, 1.0]);
let back = tensor_to_texture(
&t,
TextureFormat::Rgba16Float,
Layout::Nhwc,
ColorSpace::Linear,
)
.unwrap();
assert_eq!(*back.data, *tex.data);
}
#[test]
fn a_wrong_length_buffer_is_refused_rather_than_padded() {
assert_eq!(
Tensor::new(vec![0.0; 5], &[2, 3]),
Err(TensorError::ShapeMismatch {
dims: vec![2, 3],
len: 5
})
);
assert!(frame_to_tensor(&[0; 3], 2, 2, Layout::Nhwc, ColorSpace::Srgb).is_err());
}
}