use super::denoise_net::{Denoiser, Guides, OUT_CHANNELS};
const CORE_CHANNELS: usize = 9;
const IN_CHANNELS: usize = 11;
use super::denoise::DenoiseGuides;
use super::film::Film;
const TILE: usize = 256;
const GUIDES: Guides = Guides::RelativeError;
const MARGIN: usize = 32;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum Passes {
#[default]
Single,
Mirrored,
}
impl Passes {
fn transforms(self) -> &'static [(bool, bool)] {
match self {
Passes::Single => &[(false, false)],
Passes::Mirrored => &[(false, false), (true, false), (false, true), (true, true)],
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum Prefilter {
#[default]
None,
Guides,
Atrous,
}
pub struct LearnedDenoiser {
denoiser: Denoiser,
inputs: usize,
passes: Passes,
prefilter: Prefilter,
}
impl std::fmt::Debug for LearnedDenoiser {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("LearnedDenoiser")
.field("tile", &TILE)
.finish()
}
}
impl LearnedDenoiser {
pub fn open(path: impl AsRef<std::path::Path>) -> Result<Self, String> {
Self::load(path)
}
pub fn load(path: impl AsRef<std::path::Path>) -> Result<Self, String> {
let bytes = std::fs::read(path.as_ref()).map_err(|e| e.to_string())?;
let net = Denoiser::from_bytes(&bytes).map_err(|e| e.to_string())?;
let inputs = net.inputs();
if inputs != CORE_CHANNELS && inputs != IN_CHANNELS {
return Err(format!(
"these weights take {inputs} input planes; this renderer fills \
{CORE_CHANNELS} or {IN_CHANNELS}"
));
}
if inputs > CORE_CHANNELS && net.guides() != GUIDES {
return Err(format!(
"these weights were trained on {:?} guides, this renderer produces {GUIDES:?}",
net.guides()
));
}
Ok(Self {
denoiser: net,
inputs,
passes: Passes::default(),
prefilter: Prefilter::default(),
})
}
pub fn set_prefilter(&mut self, prefilter: Prefilter) {
self.prefilter = prefilter;
}
pub fn set_passes(&mut self, passes: Passes) {
self.passes = passes;
}
pub fn apply(&mut self, film: &Film, scene_scale: f32) -> Result<Vec<f32>, String> {
let (w, h) = (film.width() as usize, film.height() as usize);
let hdr = film.resolve_hdr();
if w == 0 || h == 0 || film.samples() == 0 {
return Ok(hdr);
}
let mut planes = Planes::gather(film, scene_scale);
match self.prefilter {
Prefilter::None => {}
Prefilter::Guides => self.prefilter_guides(&mut planes, w, h)?,
Prefilter::Atrous => prefilter_atrous(&mut planes, film, w, h, scene_scale),
}
let transforms = self.passes.transforms();
let mut total = vec![0.0f32; w * h * OUT_CHANNELS];
for &mirror in transforms {
let mut sum = vec![0.0f32; w * h * OUT_CHANNELS];
let mut weight = vec![0.0f32; w * h];
let step = TILE.saturating_sub(2 * MARGIN).max(1);
let mut y0 = 0usize;
loop {
let mut x0 = 0usize;
loop {
self.run_tile(
&planes,
Frame { w, h },
(x0, y0),
mirror,
&mut sum,
&mut weight,
)?;
if x0 + TILE >= w {
break;
}
x0 = (x0 + step).min(w.saturating_sub(TILE));
}
if y0 + TILE >= h {
break;
}
y0 = (y0 + step).min(h.saturating_sub(TILE));
}
for i in 0..w * h {
if weight[i] <= 0.0 {
continue;
}
for c in 0..OUT_CHANNELS {
total[i * OUT_CHANNELS + c] += sum[i * OUT_CHANNELS + c] / weight[i];
}
}
}
let passes = transforms.len() as f32;
let mut out = hdr;
for i in 0..w * h {
for c in 0..OUT_CHANNELS {
out[i * 4 + c] = decompress(total[i * OUT_CHANNELS + c] / passes);
}
}
Ok(out)
}
fn prefilter_guides(&mut self, planes: &mut Planes, w: usize, h: usize) -> Result<(), String> {
let albedo = self.denoise_plane(planes, w, h, Guide::Albedo)?;
let normal = self.denoise_plane(planes, w, h, Guide::Normal)?;
planes.albedo = albedo;
planes.normal = normal
.into_iter()
.map(|n| {
let len = (n[0] * n[0] + n[1] * n[1] + n[2] * n[2]).sqrt();
if len > 1e-6 {
[n[0] / len, n[1] / len, n[2] / len]
} else {
[0.0, 1.0, 0.0]
}
})
.collect();
Ok(())
}
fn denoise_plane(
&mut self,
planes: &Planes,
w: usize,
h: usize,
guide: Guide,
) -> Result<Vec<[f32; 3]>, String> {
let mut sum = vec![0.0f32; w * h * OUT_CHANNELS];
let mut weight = vec![0.0f32; w * h];
let step = TILE.saturating_sub(2 * MARGIN).max(1);
let mut y0 = 0usize;
loop {
let mut x0 = 0usize;
loop {
self.run_guide_tile(
planes,
guide,
Frame { w, h },
(x0, y0),
&mut sum,
&mut weight,
)?;
if x0 + TILE >= w {
break;
}
x0 = (x0 + step).min(w.saturating_sub(TILE));
}
if y0 + TILE >= h {
break;
}
y0 = (y0 + step).min(h.saturating_sub(TILE));
}
let mut out = vec![[0.0f32; 3]; w * h];
for i in 0..w * h {
let raw = guide.raw(planes, i);
if weight[i] <= 0.0 {
out[i] = raw;
continue;
}
for c in 0..OUT_CHANNELS {
let v = sum[i * OUT_CHANNELS + c] / weight[i];
out[i][c] = guide.decode(v);
}
}
Ok(out)
}
#[allow(clippy::too_many_arguments)]
fn run_guide_tile(
&mut self,
planes: &Planes,
guide: Guide,
frame: Frame,
origin: (usize, usize),
sum: &mut [f32],
weight: &mut [f32],
) -> Result<(), String> {
let Frame { w, h } = frame;
let (x0, y0) = origin;
let pixels = TILE * TILE;
let mut input = vec![0.0f32; self.inputs * pixels];
for c in 0..self.inputs {
for ty in 0..TILE {
let sy = (y0 + ty).min(h - 1);
for tx in 0..TILE {
let sx = (x0 + tx).min(w - 1);
let i = sy * w + sx;
input[c * pixels + ty * TILE + tx] = if c < OUT_CHANNELS {
guide.encode(guide.raw(planes, i)[c])
} else {
planes.at(c, i)
};
}
}
}
let out = self
.denoiser
.denoise(&input, TILE, TILE)
.map_err(|e| e.to_string())?;
for ty in 0..TILE {
let sy = y0 + ty;
if sy >= h {
continue;
}
for tx in 0..TILE {
let sx = x0 + tx;
if sx >= w {
continue;
}
let f = feather(tx, TILE, x0 == 0, x0 + TILE >= w)
* feather(ty, TILE, y0 == 0, y0 + TILE >= h);
if f <= 0.0 {
continue;
}
let i = sy * w + sx;
weight[i] += f;
for c in 0..OUT_CHANNELS {
sum[i * OUT_CHANNELS + c] += f * out[c * pixels + ty * TILE + tx];
}
}
}
Ok(())
}
fn run_tile(
&mut self,
planes: &Planes,
frame: Frame,
origin: (usize, usize),
mirror: (bool, bool),
sum: &mut [f32],
weight: &mut [f32],
) -> Result<(), String> {
let Frame { w, h } = frame;
let (x0, y0) = origin;
let (flip_x, flip_y) = mirror;
let src = |x: usize, y: usize| {
let sx = if flip_x { w - 1 - x } else { x };
let sy = if flip_y { h - 1 - y } else { y };
sy * w + sx
};
let pixels = TILE * TILE;
let mut input = vec![0.0f32; self.inputs * pixels];
for c in 0..self.inputs {
for ty in 0..TILE {
let sy = (y0 + ty).min(h - 1);
for tx in 0..TILE {
let sx = (x0 + tx).min(w - 1);
input[c * pixels + ty * TILE + tx] = planes.at(c, src(sx, sy));
}
}
}
let out = self
.denoiser
.denoise(&input, TILE, TILE)
.map_err(|e| e.to_string())?;
for ty in 0..TILE {
let sy = y0 + ty;
if sy >= h {
continue;
}
for tx in 0..TILE {
let sx = x0 + tx;
if sx >= w {
continue;
}
let f = feather(tx, TILE, x0 == 0, x0 + TILE >= w)
* feather(ty, TILE, y0 == 0, y0 + TILE >= h);
if f <= 0.0 {
continue;
}
let i = src(sx, sy);
weight[i] += f;
for c in 0..OUT_CHANNELS {
sum[i * OUT_CHANNELS + c] += f * out[c * pixels + ty * TILE + tx];
}
}
}
Ok(())
}
}
fn feather(i: usize, tile: usize, at_start: bool, at_end: bool) -> f32 {
let from_start = i;
let from_end = tile - 1 - i;
let lead = if at_start { MARGIN } else { from_start };
let trail = if at_end { MARGIN } else { from_end };
let edge = lead.min(trail);
if edge >= MARGIN {
1.0
} else {
let t = (edge as f32 + 0.5) / MARGIN as f32;
t * t * (3.0 - 2.0 * t)
}
}
fn prefilter_atrous(planes: &mut Planes, film: &Film, w: usize, h: usize, scene_scale: f32) {
let depth = film.resolve_depth();
let variance = film.resolve_variance();
let sample_counts = film.resolve_sample_counts();
let params = super::denoise::DenoiseParams {
iterations: 2,
..Default::default()
};
let smooth = |source: &[[f32; 3]]| -> Vec<[f32; 3]> {
let rgba: Vec<f32> = source
.iter()
.flat_map(|v| [v[0], v[1], v[2], 1.0])
.collect();
let out = super::denoise::denoise(
w as u32,
h as u32,
&rgba,
&DenoiseGuides {
albedo: &planes.albedo,
normal: &planes.normal,
depth: &depth,
variance: &variance,
sample_counts: Some(&sample_counts),
scene_scale,
},
¶ms,
);
(0..w * h)
.map(|i| [out[i * 4], out[i * 4 + 1], out[i * 4 + 2]])
.collect()
};
let albedo = smooth(&planes.albedo);
let normal = smooth(&planes.normal);
planes.albedo = albedo;
planes.normal = normal
.into_iter()
.map(|n| {
let len = (n[0] * n[0] + n[1] * n[1] + n[2] * n[2]).sqrt();
if len > 1e-6 {
[n[0] / len, n[1] / len, n[2] / len]
} else {
[0.0, 1.0, 0.0]
}
})
.collect();
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum Guide {
Albedo,
Normal,
}
impl Guide {
fn raw(self, planes: &Planes, i: usize) -> [f32; 3] {
match self {
Guide::Albedo => planes.albedo[i],
Guide::Normal => planes.normal[i],
}
}
fn encode(self, v: f32) -> f32 {
match self {
Guide::Albedo => v.clamp(0.0, 1.0),
Guide::Normal => 0.5 + 0.5 * v,
}
}
fn decode(self, v: f32) -> f32 {
match self {
Guide::Albedo => v.clamp(0.0, 1.0),
Guide::Normal => (2.0 * v - 1.0).clamp(-1.0, 1.0),
}
}
}
#[derive(Clone, Copy)]
struct Frame {
w: usize,
h: usize,
}
pub(crate) struct Planes {
colour: Vec<f32>,
albedo: Vec<[f32; 3]>,
normal: Vec<[f32; 3]>,
depth: Vec<f32>,
error: Vec<f32>,
}
impl Planes {
fn gather(film: &Film, scene_scale: f32) -> Self {
let scale = scene_scale.max(1e-6);
Self {
colour: film.resolve_hdr(),
albedo: film.resolve_albedo(),
normal: film.resolve_normal(),
depth: film
.resolve_depth()
.into_iter()
.map(|d| {
if d.is_finite() {
compress(d / scale)
} else {
1.0
}
})
.collect(),
error: film.resolve_error().into_iter().map(compress).collect(),
}
}
fn at(&self, channel: usize, i: usize) -> f32 {
match channel {
0..=2 => compress(self.colour[i * 4 + channel]),
3..=5 => self.albedo[i][channel - 3],
6..=8 => self.normal[i][channel - 6],
9 => self.depth[i],
_ => self.error[i],
}
}
}
fn compress(v: f32) -> f32 {
if !v.is_finite() {
return if v > 0.0 { 1.0 } else { 0.0 };
}
v.max(0.0) / (1.0 + v.max(0.0))
}
fn decompress(v: f32) -> f32 {
let v = v.clamp(0.0, 0.999_999);
v / (1.0 - v)
}
#[cfg(test)]
mod tests {
fn write_identity_checkpoint(path: &std::path::Path, inputs: usize) {
write_checkpoint(path, inputs, 1)
}
fn write_checkpoint(path: &std::path::Path, inputs: usize, guides: u32) {
const K: usize = 3;
let (w0, w1, w2, w3) = (8usize, 12, 16, 20);
let shapes: [[usize; 2]; 12] = [
[w0, inputs],
[w1, w0],
[w2, w1],
[w3, w2],
[w3, w3],
[w2, w3],
[w2, w2 * 2],
[w1, w2],
[w1, w1 * 2],
[w0, w1],
[w0, w0 * 2],
[3, w0],
];
let mut b: Vec<u8> = Vec::new();
b.extend_from_slice(b"RLXDW004");
b.extend_from_slice(&(inputs as u32).to_le_bytes());
for v in [w0, w1, w2, w3] {
b.extend_from_slice(&(v as u32).to_le_bytes());
}
for v in [0u32, 0, 1, guides] {
b.extend_from_slice(&v.to_le_bytes());
}
b.extend_from_slice(&(shapes.len() as u32).to_le_bytes());
for [oc, ic] in shapes {
let elems = oc * ic * K * K;
b.extend_from_slice(&(elems as u32).to_le_bytes());
b.extend_from_slice(&vec![0u8; elems * 4]);
}
std::fs::write(path, b).expect("write checkpoint");
}
use super::*;
#[test]
fn compression_round_trips() {
for v in [0.0f32, 0.25, 1.0, 7.5, 1000.0] {
let back = decompress(compress(v));
assert!(
(back - v).abs() <= 1e-3 * v.max(1.0),
"{v} came back as {back}"
);
}
}
#[test]
fn compression_is_total() {
assert_eq!(compress(f32::INFINITY), 1.0);
assert_eq!(compress(f32::NEG_INFINITY), 0.0);
assert_eq!(compress(f32::NAN), 0.0);
assert!(compress(f32::INFINITY).is_finite());
}
#[test]
fn compression_is_bounded() {
assert!(compress(f32::MAX) <= 1.0);
assert!(compress((1u32 << 24) as f32) >= 1.0);
assert!(
compress(1e6) < 1.0,
"an ordinary bright pixel must stay inside"
);
assert_eq!(compress(-5.0), 0.0);
assert!(
decompress(compress(f32::MAX)).is_finite(),
"the pole must not come back as infinity"
);
}
#[test]
fn feather_is_flat_inside_and_ramps_at_a_shared_edge() {
assert_eq!(feather(TILE / 2, TILE, false, false), 1.0);
assert_eq!(feather(MARGIN, TILE, false, false), 1.0);
let first = feather(0, TILE, false, false);
assert!(first > 0.0 && first < 0.1, "edge weight was {first}");
for i in 0..MARGIN {
let a = feather(i, TILE, false, false);
let b = feather(i + 1, TILE, false, false);
assert!(b >= a, "weight fell going inward at {i}: {a} -> {b}");
}
}
#[test]
fn the_frames_outer_edge_keeps_full_weight() {
assert_eq!(feather(0, TILE, true, false), 1.0);
assert_eq!(feather(TILE - 1, TILE, false, true), 1.0);
}
#[test]
fn an_untrained_network_returns_the_film_it_was_given() {
use crate::math::Vector3;
let (w, h) = (40u32, 24u32);
let mut film = Film::new(w, h);
for (i, p) in film.pixels_mut().iter_mut().enumerate() {
let v = (i % 17) as f32 / 17.0;
p.add(Vector3::new(v, v * 0.5 + 0.1, 2.0 - v), 1.0);
p.add(Vector3::new(v * 1.2, v * 0.4 + 0.1, 1.8 - v), 1.0);
}
let expected = film.resolve_hdr();
let path = std::env::temp_dir().join("threers_learned_identity.bin");
write_identity_checkpoint(&path, CORE_CHANNELS);
let mut denoiser = LearnedDenoiser::load(&path).expect("load");
let out = denoiser.apply(&film, 4.0).expect("apply");
let _ = std::fs::remove_file(path);
assert_eq!(out.len(), expected.len());
let worst = out
.chunks(4)
.zip(expected.chunks(4))
.flat_map(|(a, b)| (0..3).map(move |c| (a[c] - b[c]).abs()))
.fold(0.0f32, f32::max);
assert!(worst < 2e-4, "the path changed the image by {worst}");
}
#[test]
fn prefiltering_leaves_an_identity_network_alone() {
use crate::math::Vector3;
let (w, h) = (32u32, 24u32);
let mut film = Film::new(w, h);
for (i, p) in film.pixels_mut().iter_mut().enumerate() {
let v = (i % 13) as f32 / 13.0;
p.add(Vector3::new(v, 0.4 + 0.5 * v, 1.0 - v), 1.0);
p.add(Vector3::new(v * 1.1, 0.4 + 0.4 * v, 0.9 - v), 1.0);
}
let expected = film.resolve_hdr();
let path = std::env::temp_dir().join("threers_prefilter.bin");
write_identity_checkpoint(&path, CORE_CHANNELS);
let mut denoiser = LearnedDenoiser::load(&path).expect("load");
denoiser.set_prefilter(Prefilter::Guides);
let out = denoiser.apply(&film, 3.0).expect("apply");
let _ = std::fs::remove_file(path);
let worst = out
.chunks(4)
.zip(expected.chunks(4))
.flat_map(|(a, b)| (0..3).map(move |c| (a[c] - b[c]).abs()))
.fold(0.0f32, f32::max);
assert!(worst < 2e-4, "prefiltering changed the image by {worst}");
}
#[test]
fn a_normal_round_trips_through_the_colour_slot() {
for v in [-1.0f32, -0.5, 0.0, 0.25, 1.0] {
let back = Guide::Normal.decode(Guide::Normal.encode(v));
assert!((back - v).abs() < 1e-6, "{v} came back as {back}");
}
for v in [0.0f32, 0.5, 1.0] {
let back = Guide::Albedo.decode(Guide::Albedo.encode(v));
assert!((back - v).abs() < 1e-6, "{v} came back as {back}");
}
}
#[test]
fn mirrored_passes_leave_an_identity_network_alone() {
use crate::math::Vector3;
let (w, h) = (36u32, 28u32);
let mut film = Film::new(w, h);
for (i, p) in film.pixels_mut().iter_mut().enumerate() {
let x = (i as u32 % w) as f32 / w as f32;
let y = (i as u32 / w) as f32 / h as f32;
p.add(Vector3::new(x, y, 0.3 + 0.5 * x * y), 1.0);
p.add(Vector3::new(x * 0.9, y * 1.1, 0.3 + 0.4 * x * y), 1.0);
}
let expected = film.resolve_hdr();
let path = std::env::temp_dir().join("threers_tta.bin");
write_identity_checkpoint(&path, CORE_CHANNELS);
let mut denoiser = LearnedDenoiser::load(&path).expect("load");
denoiser.set_passes(Passes::Mirrored);
let out = denoiser.apply(&film, 3.0).expect("apply");
let _ = std::fs::remove_file(path);
let worst = out
.chunks(4)
.zip(expected.chunks(4))
.flat_map(|(a, b)| (0..3).map(move |c| (a[c] - b[c]).abs()))
.fold(0.0f32, f32::max);
assert!(worst < 2e-4, "mirrored passes changed the image by {worst}");
}
#[test]
fn a_core_guide_model_loads_and_returns_its_input() {
use crate::math::Vector3;
let (w, h) = (32u32, 20u32);
let mut film = Film::new(w, h);
for (i, p) in film.pixels_mut().iter_mut().enumerate() {
let v = (i % 11) as f32 / 11.0;
p.add(Vector3::new(v, v * 0.6, 1.0 - v), 1.0);
p.add(Vector3::new(v * 1.1, v * 0.5, 0.9 - v), 1.0);
}
let expected = film.resolve_hdr();
let path = std::env::temp_dir().join("threers_core_guides.bin");
write_identity_checkpoint(&path, CORE_CHANNELS);
let mut denoiser = LearnedDenoiser::load(&path)
.expect("a core-plane model must load whatever the guide field says");
let out = denoiser.apply(&film, 3.0).expect("apply");
let _ = std::fs::remove_file(path);
let worst = out
.chunks(4)
.zip(expected.chunks(4))
.flat_map(|(a, b)| (0..3).map(move |c| (a[c] - b[c]).abs()))
.fold(0.0f32, f32::max);
assert!(worst < 2e-4, "the path changed the image by {worst}");
}
#[test]
fn weights_from_another_guide_encoding_are_refused() {
let path = std::env::temp_dir().join("threers_guide_mismatch.bin");
write_checkpoint(&path, IN_CHANNELS, 0);
let err = LearnedDenoiser::load(&path).expect_err("a mismatched encoding must not load");
assert!(
err.contains("AbsoluteError"),
"the error should say which: {err}"
);
let _ = std::fs::remove_file(path);
}
#[test]
fn tiles_step_by_less_than_their_width() {
let step = TILE - 2 * MARGIN;
assert!(step > 0, "the margins consume the whole tile");
assert!(TILE.is_multiple_of(8), "the encoder halves three times");
}
}