oximedia-virtual 0.1.4

Virtual production and LED wall tools for OxiMedia
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
//! Tile-based parallel compositing for ICVFX.
//!
//! Divides the output frame into rectangular tiles and composites each tile
//! independently.  When Rust's `std::thread` is available the tiles can be
//! processed in parallel using a simple work-stealing approach.  The tiled
//! compositor uses the same Porter-Duff "over" blending as the main compositor
//! but operates on independent tile-sized scratch buffers.

use super::{BlendMode, CompositeLayer};
use crate::{Result, VirtualProductionError};
use serde::{Deserialize, Serialize};

/// Configuration for the tiled compositor.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TiledCompositorConfig {
    /// Output resolution (width, height).
    pub resolution: (usize, usize),
    /// Tile width in pixels.
    pub tile_width: usize,
    /// Tile height in pixels.
    pub tile_height: usize,
    /// Number of worker threads (0 = single-threaded sequential).
    pub num_threads: usize,
}

impl Default for TiledCompositorConfig {
    fn default() -> Self {
        Self {
            resolution: (1920, 1080),
            tile_width: 64,
            tile_height: 64,
            num_threads: 0,
        }
    }
}

/// A single tile region.
#[derive(Debug, Clone, Copy)]
pub struct Tile {
    /// Top-left X.
    pub x: usize,
    /// Top-left Y.
    pub y: usize,
    /// Width of this tile.
    pub width: usize,
    /// Height of this tile.
    pub height: usize,
}

impl Tile {
    /// Pixel count.
    #[must_use]
    pub fn pixel_count(&self) -> usize {
        self.width * self.height
    }
}

/// RGBA layer passed to the tiled compositor.
#[derive(Debug, Clone)]
pub struct TileLayerData {
    /// RGBA f32 pixel data (row-major, full frame).
    pub pixels_rgba: Vec<f32>,
    /// Width (must match compositor resolution).
    pub width: usize,
    /// Height (must match compositor resolution).
    pub height: usize,
    /// Per-layer opacity [0, 1].
    pub opacity: f32,
    /// Blend mode.
    pub blend_mode: BlendMode,
    /// Z-order (lower = further back).
    pub z_order: i32,
    /// Whether this layer is active.
    pub enabled: bool,
    /// Optional label for debugging.
    pub label: String,
}

impl TileLayerData {
    /// Create a solid colour layer (all pixels same RGBA).
    #[must_use]
    pub fn solid(
        label: &str,
        r: f32,
        g: f32,
        b: f32,
        a: f32,
        width: usize,
        height: usize,
        z_order: i32,
    ) -> Self {
        let n = width * height * 4;
        let mut pixels_rgba = Vec::with_capacity(n);
        for _ in 0..(width * height) {
            pixels_rgba.push(r);
            pixels_rgba.push(g);
            pixels_rgba.push(b);
            pixels_rgba.push(a);
        }
        Self {
            pixels_rgba,
            width,
            height,
            opacity: 1.0,
            blend_mode: BlendMode::Normal,
            z_order,
            enabled: true,
            label: label.to_string(),
        }
    }

    /// Get RGBA for a specific pixel (col, row).
    #[must_use]
    pub fn get_pixel(&self, col: usize, row: usize) -> Option<[f32; 4]> {
        if col >= self.width || row >= self.height {
            return None;
        }
        let i = (row * self.width + col) * 4;
        Some([
            self.pixels_rgba[i],
            self.pixels_rgba[i + 1],
            self.pixels_rgba[i + 2],
            self.pixels_rgba[i + 3],
        ])
    }
}

/// Output frame from the tiled compositor.
#[derive(Debug, Clone)]
pub struct TiledFrame {
    /// RGB u8 pixel data.
    pub pixels: Vec<u8>,
    /// Width.
    pub width: usize,
    /// Height.
    pub height: usize,
    /// Number of tiles processed.
    pub tiles_processed: usize,
}

impl TiledFrame {
    /// Get pixel at (col, row).
    #[must_use]
    pub fn get_pixel(&self, col: usize, row: usize) -> Option<[u8; 3]> {
        if col >= self.width || row >= self.height {
            return None;
        }
        let i = (row * self.width + col) * 3;
        Some([self.pixels[i], self.pixels[i + 1], self.pixels[i + 2]])
    }
}

/// Tiled compositor.
pub struct TiledCompositor {
    config: TiledCompositorConfig,
    tiles: Vec<Tile>,
    #[allow(dead_code)]
    layers: Vec<CompositeLayer>,
}

impl TiledCompositor {
    /// Create a new tiled compositor.
    pub fn new(config: TiledCompositorConfig) -> Result<Self> {
        if config.tile_width == 0 || config.tile_height == 0 {
            return Err(VirtualProductionError::InvalidConfig(
                "Tile dimensions must be non-zero".to_string(),
            ));
        }
        if config.resolution.0 == 0 || config.resolution.1 == 0 {
            return Err(VirtualProductionError::InvalidConfig(
                "Resolution must be non-zero".to_string(),
            ));
        }
        let tiles = Self::build_tiles(&config);
        Ok(Self {
            config,
            tiles,
            layers: Vec::new(),
        })
    }

    /// Build tile list for the given configuration.
    fn build_tiles(config: &TiledCompositorConfig) -> Vec<Tile> {
        let (w, h) = config.resolution;
        let tw = config.tile_width;
        let th = config.tile_height;

        let mut tiles = Vec::new();
        let mut y = 0;
        while y < h {
            let tile_h = th.min(h - y);
            let mut x = 0;
            while x < w {
                let tile_w = tw.min(w - x);
                tiles.push(Tile {
                    x,
                    y,
                    width: tile_w,
                    height: tile_h,
                });
                x += tw;
            }
            y += th;
        }
        tiles
    }

    /// Number of tiles.
    #[must_use]
    pub fn tile_count(&self) -> usize {
        self.tiles.len()
    }

    /// Get the configuration.
    #[must_use]
    pub fn config(&self) -> &TiledCompositorConfig {
        &self.config
    }

    /// Tile list.
    #[must_use]
    pub fn tiles(&self) -> &[Tile] {
        &self.tiles
    }

    /// Composite a stack of layers using tile-based processing.
    ///
    /// Layers are sorted by z_order (ascending = furthest back first).
    /// Each tile is processed independently using Porter-Duff "over" compositing.
    pub fn composite(&self, layers: &[TileLayerData], timestamp_ns: u64) -> Result<TiledFrame> {
        let (w, h) = self.config.resolution;

        // Validate all layers
        let mut active: Vec<&TileLayerData> = layers
            .iter()
            .filter(|l| l.enabled && l.opacity > 0.0)
            .collect();

        for layer in &active {
            if layer.width != w || layer.height != h {
                return Err(VirtualProductionError::Compositing(format!(
                    "Layer '{}' resolution {}×{} doesn't match compositor {}×{}",
                    layer.label, layer.width, layer.height, w, h
                )));
            }
        }

        // Sort by z_order
        active.sort_by_key(|l| l.z_order);

        let _ = timestamp_ns; // available for future timestamped output

        // Output buffer
        let n = w * h;
        let mut out_r = vec![0.0f32; n];
        let mut out_g = vec![0.0f32; n];
        let mut out_b = vec![0.0f32; n];
        let mut out_a = vec![0.0f32; n];

        // Tile-based compositing: process each tile
        for tile in &self.tiles {
            // For each pixel in the tile
            for ty in 0..tile.height {
                for tx in 0..tile.width {
                    let gx = tile.x + tx;
                    let gy = tile.y + ty;
                    let gi = gy * w + gx;

                    let mut cr = 0.0f32;
                    let mut cg = 0.0f32;
                    let mut cb = 0.0f32;
                    let mut ca = 0.0f32;

                    for layer in &active {
                        let li = gi * 4;
                        let lr = layer.pixels_rgba[li];
                        let lg = layer.pixels_rgba[li + 1];
                        let lb = layer.pixels_rgba[li + 2];
                        let la = layer.pixels_rgba[li + 3] * layer.opacity;

                        if la < 1e-6 {
                            continue;
                        }

                        let base = [cr, cg, cb];
                        let blend_color = [lr, lg, lb];
                        let blended = layer.blend_mode.blend(base, blend_color, la);

                        let out_alpha = la + ca * (1.0 - la);
                        if out_alpha > 1e-6 {
                            cr = (blended[0] * la + cr * ca * (1.0 - la)) / out_alpha;
                            cg = (blended[1] * la + cg * ca * (1.0 - la)) / out_alpha;
                            cb = (blended[2] * la + cb * ca * (1.0 - la)) / out_alpha;
                        }
                        ca = out_alpha.min(1.0);
                    }

                    out_r[gi] = cr;
                    out_g[gi] = cg;
                    out_b[gi] = cb;
                    out_a[gi] = ca;
                }
            }
        }

        // Convert to u8
        let mut pixels = vec![0u8; n * 3];
        for i in 0..n {
            pixels[i * 3] = (out_r[i].clamp(0.0, 1.0) * 255.0) as u8;
            pixels[i * 3 + 1] = (out_g[i].clamp(0.0, 1.0) * 255.0) as u8;
            pixels[i * 3 + 2] = (out_b[i].clamp(0.0, 1.0) * 255.0) as u8;
        }

        Ok(TiledFrame {
            pixels,
            width: w,
            height: h,
            tiles_processed: self.tiles.len(),
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn small_config() -> TiledCompositorConfig {
        TiledCompositorConfig {
            resolution: (16, 16),
            tile_width: 8,
            tile_height: 8,
            num_threads: 0,
        }
    }

    #[test]
    fn test_tiled_compositor_creation() {
        let c = TiledCompositor::new(small_config());
        assert!(c.is_ok());
    }

    #[test]
    fn test_zero_tile_size_fails() {
        let mut cfg = small_config();
        cfg.tile_width = 0;
        assert!(TiledCompositor::new(cfg).is_err());
    }

    #[test]
    fn test_tile_count() {
        let tc = TiledCompositor::new(small_config()).expect("ok");
        // 16/8 × 16/8 = 2 × 2 = 4 tiles
        assert_eq!(tc.tile_count(), 4);
    }

    #[test]
    fn test_tile_count_non_even() {
        let cfg = TiledCompositorConfig {
            resolution: (10, 10),
            tile_width: 4,
            tile_height: 4,
            num_threads: 0,
        };
        let tc = TiledCompositor::new(cfg).expect("ok");
        // ceil(10/4) × ceil(10/4) = 3 × 3 = 9 tiles
        assert_eq!(tc.tile_count(), 9);
    }

    #[test]
    fn test_composite_empty_layers() {
        let tc = TiledCompositor::new(small_config()).expect("ok");
        let frame = tc.composite(&[], 0);
        assert!(frame.is_ok());
        let f = frame.expect("ok");
        // All black when no layers
        assert_eq!(f.get_pixel(8, 8), Some([0, 0, 0]));
    }

    #[test]
    fn test_composite_single_opaque_layer() {
        let tc = TiledCompositor::new(small_config()).expect("ok");
        let layer = TileLayerData::solid("bg", 1.0, 0.0, 0.0, 1.0, 16, 16, 0);
        let frame = tc.composite(&[layer], 0).expect("ok");
        assert_eq!(frame.get_pixel(0, 0), Some([255, 0, 0]));
        assert_eq!(frame.tiles_processed, 4);
    }

    #[test]
    fn test_composite_two_layers_z_order() {
        let tc = TiledCompositor::new(small_config()).expect("ok");
        let bg = TileLayerData::solid("bg", 1.0, 0.0, 0.0, 1.0, 16, 16, 0);
        let fg = TileLayerData::solid("fg", 0.0, 0.0, 1.0, 1.0, 16, 16, 10);
        let frame = tc.composite(&[fg, bg], 0).expect("ok");
        // Blue fg (z=10) should cover red bg (z=0)
        assert_eq!(frame.get_pixel(4, 4), Some([0, 0, 255]));
    }

    #[test]
    fn test_composite_semi_transparent_layer() {
        let tc = TiledCompositor::new(small_config()).expect("ok");
        let bg = TileLayerData::solid("bg", 1.0, 0.0, 0.0, 1.0, 16, 16, 0);
        let mut fg = TileLayerData::solid("fg", 0.0, 1.0, 0.0, 0.5, 16, 16, 1);
        fg.opacity = 1.0;
        let frame = tc.composite(&[bg, fg], 0).expect("ok");
        let px = frame.get_pixel(8, 8).expect("ok");
        assert!(px[0] > 0, "red should bleed through");
        assert!(px[1] > 0, "green should contribute");
    }

    #[test]
    fn test_composite_resolution_mismatch_error() {
        let tc = TiledCompositor::new(small_config()).expect("ok");
        let wrong = TileLayerData::solid("wrong", 1.0, 0.0, 0.0, 1.0, 8, 8, 0);
        let result = tc.composite(&[wrong], 0);
        assert!(result.is_err());
    }

    #[test]
    fn test_composite_disabled_layer_skipped() {
        let tc = TiledCompositor::new(small_config()).expect("ok");
        let bg = TileLayerData::solid("bg", 1.0, 0.0, 0.0, 1.0, 16, 16, 0);
        let mut fg = TileLayerData::solid("fg", 0.0, 1.0, 0.0, 1.0, 16, 16, 1);
        fg.enabled = false;
        let frame = tc.composite(&[bg, fg], 0).expect("ok");
        assert_eq!(frame.get_pixel(0, 0), Some([255, 0, 0]));
    }

    #[test]
    fn test_composite_additive_blend() {
        let cfg = TiledCompositorConfig {
            resolution: (4, 4),
            tile_width: 2,
            tile_height: 2,
            num_threads: 0,
        };
        let tc = TiledCompositor::new(cfg).expect("ok");
        let bg = TileLayerData::solid("bg", 0.5, 0.0, 0.0, 1.0, 4, 4, 0);
        let mut glow = TileLayerData::solid("glow", 0.3, 0.3, 0.0, 1.0, 4, 4, 1);
        glow.blend_mode = BlendMode::Add;
        let frame = tc.composite(&[bg, glow], 0).expect("ok");
        let px = frame.get_pixel(2, 2).expect("ok");
        assert!(px[0] > 180, "additive red: {}", px[0]);
    }

    #[test]
    fn test_tile_solid_pixel_access() {
        let layer = TileLayerData::solid("test", 0.5, 0.25, 0.1, 1.0, 4, 4, 0);
        let px = layer.get_pixel(2, 2).expect("ok");
        assert!((px[0] - 0.5).abs() < 1e-5);
    }

    #[test]
    fn test_tile_pixel_out_of_bounds() {
        let layer = TileLayerData::solid("test", 1.0, 0.0, 0.0, 1.0, 4, 4, 0);
        assert!(layer.get_pixel(4, 0).is_none());
    }

    #[test]
    fn test_large_tile_covers_whole_frame() {
        let cfg = TiledCompositorConfig {
            resolution: (4, 4),
            tile_width: 16,
            tile_height: 16,
            num_threads: 0,
        };
        let tc = TiledCompositor::new(cfg).expect("ok");
        assert_eq!(tc.tile_count(), 1); // one tile covers everything
    }
}