use ::rlx::{grad_with_loss, DType, Device, Graph, GraphExt, Session, Shape};
use crate::textures::{Texture, TextureFormat};
use super::session::GraphRunner;
use super::tensor::{ColorSpace, Layout, Tensor, TensorError};
use super::FrameFilter;
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct ColorGrade {
pub matrix: [[f32; 3]; 3],
pub bias: [f32; 3],
}
impl Default for ColorGrade {
fn default() -> Self {
Self::IDENTITY
}
}
impl ColorGrade {
pub const IDENTITY: Self = Self {
matrix: [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]],
bias: [0.0; 3],
};
pub fn apply_rgb(&self, rgb: [f32; 3]) -> [f32; 3] {
let mut out = self.bias;
for (o, row) in self.matrix.iter().enumerate() {
for (i, m) in row.iter().enumerate() {
out[o] += m * rgb[i];
}
}
out
}
pub fn apply(&self, rgba: &[u8]) -> Vec<u8> {
self.apply_in(rgba, ColorSpace::Linear)
}
pub fn apply_in(&self, rgba: &[u8], space: ColorSpace) -> Vec<u8> {
let linear = matches!(space, ColorSpace::Linear);
let mut out = Vec::with_capacity(rgba.len());
for px in rgba.chunks_exact(4) {
let mut rgb = [0.0f32; 3];
for (c, v) in rgb.iter_mut().enumerate() {
let raw = px[c] as f32 / 255.0;
*v = if linear {
super::srgb_to_linear(raw)
} else {
raw
};
}
let graded = self.apply_rgb(rgb);
for v in graded {
let v = if linear { super::linear_to_srgb(v) } else { v };
out.push((v.clamp(0.0, 1.0) * 255.0 + 0.5) as u8);
}
out.push(px[3]);
}
out
}
pub fn apply_to_texture(&self, rgba: &[u8], width: u32, height: u32) -> Texture {
Texture::new(
width,
height,
TextureFormat::Rgba8UnormSrgb,
self.apply(rgba),
)
}
pub fn filter(&self, width: u32, height: u32, device: Device) -> FrameFilter {
let mut filter = FrameFilter::new(apply_graph(width, height), device, "frame")
.layout(Layout::Nhwc)
.color_space(ColorSpace::Linear);
self.bind(filter.runner_mut());
filter
}
fn bind(&self, runner: &mut GraphRunner) {
let mut m = [0.0f32; 16];
for o in 0..3 {
for i in 0..3 {
m[i * 4 + o] = self.matrix[o][i];
}
}
m[15] = 1.0;
let b = [self.bias[0], self.bias[1], self.bias[2], 0.0];
runner.set_param("grade", &m);
runner.set_param("offset", &b);
}
pub fn fit(
source: &[u8],
target: &[u8],
options: &FitOptions,
device: Device,
) -> Result<FitReport, TensorError> {
if source.len() != target.len() {
return Err(TensorError::ByteCount {
expected: source.len(),
got: target.len(),
});
}
let (x, t) = sample_pairs(source, target, options);
let n = x.dims()[0];
if n == 0 {
return Err(TensorError::Empty);
}
let mut forward = Graph::new("grade_loss");
let xs = forward.input("source", Shape::new(&[n, 3], DType::F32));
let ts = forward.input("target", Shape::new(&[n, 3], DType::F32));
let m = forward.param("grade", Shape::new(&[3, 3], DType::F32));
let b = forward.param("offset", Shape::new(&[1, 3], DType::F32));
let mapped = forward.mm(xs, m);
let mapped = forward.add(mapped, b);
let residual = forward.sub(mapped, ts);
let squared = forward.mul(residual, residual);
let loss = forward.mean(squared, vec![0, 1], false);
forward.set_outputs(vec![loss]);
let backward = grad_with_loss(&forward, &[m, b]);
let mut compiled = Session::new(device).compile(backward);
let mut matrix = [1.0f32, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0];
let mut offset = [0.0f32; 3];
let seed = [1.0f32];
let inputs: [(&str, &[f32]); 3] = [
("source", x.data()),
("target", t.data()),
("d_output", &seed),
];
let mut v_matrix = [0.0f32; 9];
let mut v_offset = [0.0f32; 3];
let mut initial_loss = f32::NAN;
let mut final_loss = f32::NAN;
let mut diverged = false;
let mut last_good = (matrix, offset);
for step in 0..options.iterations {
compiled.set_param("grade", &matrix);
compiled.set_param("offset", &offset);
let out = compiled.run(&inputs);
let loss = out[0].first().copied().unwrap_or(f32::NAN);
if step == 0 {
initial_loss = loss;
}
if !loss.is_finite() {
diverged = true;
let (m, o) = last_good;
matrix = m;
offset = o;
break;
}
final_loss = loss;
last_good = (matrix, offset);
descend(
&mut matrix,
&mut v_matrix,
&out[1],
options.learning_rate,
options.momentum,
);
descend(
&mut offset,
&mut v_offset,
&out[2],
options.learning_rate,
options.momentum,
);
}
let mut grade = ColorGrade {
matrix: [[0.0; 3]; 3],
bias: offset,
};
for o in 0..3 {
for i in 0..3 {
grade.matrix[o][i] = matrix[i * 3 + o];
}
}
Ok(FitReport {
grade,
initial_loss,
final_loss,
samples: n,
diverged,
})
}
}
fn descend(weights: &mut [f32], velocity: &mut [f32], gradient: &[f32], rate: f32, beta: f32) {
for ((w, v), g) in weights.iter_mut().zip(velocity.iter_mut()).zip(gradient) {
*v = beta * *v + g;
*w -= rate * *v;
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct FitOptions {
pub iterations: usize,
pub learning_rate: f32,
pub momentum: f32,
pub samples: usize,
pub color_space: ColorSpace,
}
impl Default for FitOptions {
fn default() -> Self {
Self {
iterations: 400,
learning_rate: 0.5,
momentum: 0.9,
samples: 20_000,
color_space: ColorSpace::Linear,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct FitReport {
pub grade: ColorGrade,
pub initial_loss: f32,
pub final_loss: f32,
pub samples: usize,
pub diverged: bool,
}
fn apply_graph(width: u32, height: u32) -> Graph {
let pixels = width as usize * height as usize;
let mut g = Graph::new("grade_apply");
let x = g.input(
"frame",
Shape::new(&[1, height as usize, width as usize, 4], DType::F32),
);
let m = g.param("grade", Shape::new(&[4, 4], DType::F32));
let b = g.param("offset", Shape::new(&[1, 4], DType::F32));
let flat = g.reshape_(x, vec![pixels as i64, 4]);
let mapped = g.mm(flat, m);
let mapped = g.add(mapped, b);
let out = g.reshape_(mapped, vec![1, height as i64, width as i64, 4]);
g.set_outputs(vec![out]);
g
}
fn sample_pairs(source: &[u8], target: &[u8], options: &FitOptions) -> (Tensor, Tensor) {
let pixels = source.len() / 4;
let wanted = options.samples.clamp(1, pixels.max(1));
let stride = (pixels / wanted).max(1);
let linear = matches!(options.color_space, ColorSpace::Linear);
let mut xs = Vec::with_capacity(wanted * 3);
let mut ts = Vec::with_capacity(wanted * 3);
let mut taken = 0;
for p in (0..pixels).step_by(stride) {
for c in 0..3 {
let s = source[p * 4 + c] as f32 / 255.0;
let t = target[p * 4 + c] as f32 / 255.0;
if linear {
xs.push(super::srgb_to_linear(s));
ts.push(super::srgb_to_linear(t));
} else {
xs.push(s);
ts.push(t);
}
}
taken += 1;
}
(
Tensor::new(xs, &[taken, 3]).expect("three per sample"),
Tensor::new(ts, &[taken, 3]).expect("three per sample"),
)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::rlx::preferred_device;
fn swatches() -> Vec<u8> {
let mut out = Vec::new();
let mut state = 0x1234_5678u64;
for _ in 0..1024 {
let mut next = || {
state = state.wrapping_mul(6364136223846793005).wrapping_add(1);
((state >> 33) % 256) as u8
};
out.extend_from_slice(&[next(), next(), next(), 255]);
}
out
}
#[test]
fn a_known_grade_is_recovered_from_the_images_it_produced() {
let known = ColorGrade {
matrix: [[0.9, 0.05, 0.0], [0.0, 0.85, 0.05], [0.02, 0.0, 0.88]],
bias: [0.01, 0.01, 0.01],
};
let source = swatches();
let target = known.apply(&source);
let report = ColorGrade::fit(
&source,
&target,
&FitOptions {
iterations: 600,
..Default::default()
},
preferred_device(),
)
.unwrap();
assert!(
report.final_loss < report.initial_loss * 0.01,
"loss barely moved: {} → {}",
report.initial_loss,
report.final_loss
);
for o in 0..3 {
for i in 0..3 {
assert!(
(report.grade.matrix[o][i] - known.matrix[o][i]).abs() < 0.05,
"matrix[{o}][{i}] = {}, wanted {}",
report.grade.matrix[o][i],
known.matrix[o][i]
);
}
assert!(
(report.grade.bias[o] - known.bias[o]).abs() < 0.05,
"bias[{o}] = {}, wanted {}",
report.grade.bias[o],
known.bias[o]
);
}
}
#[test]
fn the_identity_grade_leaves_a_frame_alone() {
let frame = swatches();
assert_eq!(ColorGrade::IDENTITY.apply(&frame), frame);
}
#[test]
fn fitting_a_frame_to_itself_finds_the_identity() {
let frame = swatches();
let report =
ColorGrade::fit(&frame, &frame, &FitOptions::default(), preferred_device()).unwrap();
assert!(report.final_loss < 1e-6, "loss {}", report.final_loss);
let graded = report.grade.apply(&frame);
for (a, b) in graded.iter().zip(frame.iter()) {
assert!((*a as i32 - *b as i32).abs() <= 2, "{a} vs {b}");
}
}
#[test]
fn the_compiled_filter_agrees_with_the_host_path() {
let known = ColorGrade {
matrix: [[1.2, 0.0, 0.0], [0.0, 0.85, 0.1], [0.0, 0.0, 1.05]],
bias: [0.02, 0.0, -0.01],
};
let (w, h) = (16u32, 16u32);
let frame: Vec<u8> = (0..(w * h * 4)).map(|i| ((i * 37) % 251) as u8).collect();
let host = known.apply(&frame);
let mut filter = known.filter(w, h, preferred_device());
let (device_side, _, _) = filter.apply(&frame, w, h).unwrap();
for (i, (a, b)) in host.iter().zip(device_side.iter()).enumerate() {
assert!(
(*a as i32 - *b as i32).abs() <= 1,
"byte {i}: host {a}, graph {b}"
);
}
}
#[test]
fn an_empty_frame_is_refused_rather_than_compiled() {
assert_eq!(
ColorGrade::fit(&[], &[], &FitOptions::default(), preferred_device()),
Err(TensorError::Empty)
);
}
#[test]
fn a_diverging_fit_reports_itself_and_returns_the_last_good_grade() {
let source = swatches();
let target: Vec<u8> = source.iter().map(|v| v.wrapping_add(40)).collect();
let report = ColorGrade::fit(
&source,
&target,
&FitOptions {
learning_rate: 80.0,
iterations: 200,
..Default::default()
},
preferred_device(),
)
.unwrap();
assert!(report.diverged, "an 80.0 learning rate did not diverge");
assert!(report.final_loss.is_finite(), "reported a NaN loss");
for row in report.grade.matrix {
for v in row {
assert!(v.is_finite(), "returned a NaN grade: {:?}", report.grade);
}
}
let out = report.grade.apply(&source);
assert_eq!(out.len(), source.len());
}
#[test]
fn mismatched_frames_are_refused() {
assert!(
ColorGrade::fit(&[0; 8], &[0; 4], &FitOptions::default(), preferred_device()).is_err()
);
}
}