alpha2-render 0.2.2

Render graph and effect registry for alpha2.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
use std::collections::HashMap;
use std::num::NonZeroU64;
use std::sync::Mutex;

use crate::Shared;
use crate::retire::{Retired, Retirement};

/// The bytes one effect parameter takes in the generated `EffectParams`
/// struct: every type is padded to one 16-byte slot.
pub(crate) const PARAM_SLOT_BYTES: usize = 16;

/// The bytes of an effect's parameter uniform: one slot per parameter, and
/// one for an effect without any, since a uniform cannot be empty. The graph
/// reserves this much and the pass writes it, so both read it from here.
pub(crate) const fn param_uniform_bytes(count: usize) -> usize {
    let slots = if count == 0 { 1 } else { count };
    slots.saturating_mul(PARAM_SLOT_BYTES)
}

/// The 32-bit words of a runtime effect pass's runtime uniform: two `vec4f`
/// of values, then a `vec4u` holding the time in ticks.
pub(crate) const RUNTIME_UNIFORM_WORDS: usize = 12;

/// The `f32` values of one composite pass's `BatchParams` uniform
/// (composite.wesl): three `vec4f` of controls, then four for each of
/// [`crate::composite::COMPOSITE_BATCH_LAYERS`] layers.
pub(crate) const BATCH_UNIFORM_FLOATS: usize = 12 + 16 * crate::composite::COMPOSITE_BATCH_LAYERS;

/// The bytes of `values` floats, as the graph reserves them.
pub(crate) const fn float_uniform_bytes(values: usize) -> u64 {
    (values * size_of::<f32>()) as u64
}

// Idle arenas are retained up to the most of one capacity in use at once, so a frame with
// many sources reuses them all; these bound what an unusual frame can leave behind.
const MAX_IDLE_UNIFORM_BUFFERS: usize = 64;
const MAX_IDLE_UNIFORM_BYTES: u64 = 4 * 1024 * 1024;

pub(crate) fn f32_bytes<const N: usize>(values: [f32; N]) -> Vec<u8> {
    values
        .into_iter()
        .flat_map(f32::to_ne_bytes)
        .collect::<Vec<_>>()
}

/// Allocation and retention counters for submission-scoped uniform arenas.
#[derive(Copy, Clone, Debug, Default, Eq, PartialEq)]
pub struct UniformBufferPoolStats {
    /// Physical buffers created since renderer construction.
    pub created: u64,
    /// Graph executions served by an idle buffer.
    pub reused: u64,
    /// Buffers dropped instead of retained, including failed graph encodes.
    pub discarded: u64,
    /// Arenas currently being encoded or awaiting submission.
    pub active: usize,
    /// Bytes currently owned by active arenas.
    pub active_bytes: u64,
    /// Maximum active bytes observed since renderer construction.
    pub high_water_bytes: u64,
    /// Unmapped buffers currently available for reuse.
    pub available: usize,
    /// Bytes retained by idle reusable buffers.
    pub available_bytes: u64,
}

#[derive(Debug)]
pub(crate) struct UniformBufferPool {
    /// The device's `min_uniform_buffer_offset_alignment`.
    alignment: u64,
    state: Mutex<UniformBufferPoolState>,
    retirement: Shared<Retirement>,
}

#[derive(Debug, Default)]
struct UniformBufferPoolState {
    /// Oldest release first: capacity, buffer, its bytes, and the
    /// [`Retirement::generation`] it was created in.
    available: Vec<(u64, wgpu::Buffer, Vec<u8>, u64)>,
    /// The generation whose untrusted idle buffers were last dropped.
    checked_generation: u64,
    active_by_capacity: HashMap<u64, usize>,
    peak_active_by_capacity: HashMap<u64, usize>,
    created: u64,
    reused: u64,
    discarded: u64,
    active: usize,
    active_bytes: u64,
    high_water_bytes: u64,
    available_bytes: u64,
}

#[derive(Debug)]
pub(crate) struct UniformArenaLease {
    pool: Shared<UniformBufferPool>,
    buffer: Option<wgpu::Buffer>,
    capacity: u64,
    alignment: u64,
    cursor: u64,
    bytes: Vec<u8>,
    generation: u64,
}

#[derive(Copy, Clone, Debug)]
pub(crate) struct UniformSlice {
    offset: u64,
    size: NonZeroU64,
}

impl UniformBufferPool {
    pub(crate) fn new(alignment: u64, retirement: Shared<Retirement>) -> Self {
        Self {
            alignment: alignment.max(1),
            state: Mutex::default(),
            retirement,
        }
    }

    /// Leases an arena of at least `capacity` bytes.
    ///
    /// Capacities round up to a power of two, so graphs whose uniform needs
    /// differ a little share arenas instead of each wanting an exact size.
    pub(crate) fn acquire(
        self: &Shared<Self>,
        device: &wgpu::Device,
        capacity: u64,
    ) -> UniformArenaLease {
        let alignment = self.alignment;
        let capacity = align_up(capacity.max(1), alignment)
            .checked_next_power_of_two()
            .unwrap_or(u64::MAX);
        let reused = {
            let mut state = self
                .state
                .lock()
                .unwrap_or_else(std::sync::PoisonError::into_inner);
            let untrusted = self.drop_untrusted(&mut state);
            let index = state
                .available
                .iter()
                .position(|(available_capacity, _, _, _)| *available_capacity == capacity);
            let reused = index.map(|index| {
                state.reused = state.reused.saturating_add(1);
                state.available_bytes = state.available_bytes.saturating_sub(capacity);
                let (_, buffer, bytes, generation) = state.available.remove(index);
                (buffer, bytes, generation)
            });
            drop(state);
            self.retirement.retire(untrusted);
            reused
        };
        let (buffer, bytes, generation) = reused.unwrap_or_else(|| {
            let mut state = self
                .state
                .lock()
                .unwrap_or_else(std::sync::PoisonError::into_inner);
            state.created = state.created.saturating_add(1);
            drop(state);
            let generation = self.retirement.generation();
            (
                device.create_buffer(&wgpu::BufferDescriptor {
                    label: Some("alpha2 uniform arena"),
                    size: capacity,
                    usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
                    mapped_at_creation: false,
                }),
                vec![0; usize::try_from(capacity).expect("uniform arena fits host indexing")],
                generation,
            )
        });
        {
            let mut state = self
                .state
                .lock()
                .unwrap_or_else(std::sync::PoisonError::into_inner);
            state.active = state.active.saturating_add(1);
            state.active_bytes = state.active_bytes.saturating_add(capacity);
            state.high_water_bytes = state.high_water_bytes.max(state.active_bytes);
            let active = state.active_by_capacity.entry(capacity).or_default();
            *active = active.saturating_add(1);
            let active = *active;
            let peak = state.peak_active_by_capacity.entry(capacity).or_default();
            *peak = (*peak).max(active);
        }
        UniformArenaLease {
            pool: Shared::clone(self),
            buffer: Some(buffer),
            capacity,
            alignment,
            cursor: 0,
            bytes,
            generation,
        }
    }

    /// Takes the idle arenas created before the GPU last reported an error
    /// out of the pool, once per such report, for the caller to retire.
    fn drop_untrusted(&self, state: &mut UniformBufferPoolState) -> Vec<(Retired, u64)> {
        let generation = self.retirement.generation();
        if state.checked_generation == generation {
            return Vec::new();
        }
        state.checked_generation = generation;
        let (kept, untrusted): (Vec<_>, Vec<_>) = std::mem::take(&mut state.available)
            .into_iter()
            .partition(|(_, _, _, created)| *created == generation);
        state.available = kept;
        state.available_bytes = state
            .available
            .iter()
            .map(|(capacity, _, _, _)| *capacity)
            .sum();
        state.discarded = state
            .discarded
            .saturating_add(u64::try_from(untrusted.len()).unwrap_or(u64::MAX));
        untrusted
            .into_iter()
            .map(|(capacity, buffer, _, _)| {
                prune_capacity(state, capacity);
                (Retired::Buffer(buffer), capacity)
            })
            .collect()
    }

    /// Destroys every idle uniform buffer.
    pub(crate) fn clear(&self) {
        let mut state = self
            .state
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner);
        let discarded = u64::try_from(state.available.len()).unwrap_or(u64::MAX);
        let idle = std::mem::take(&mut state.available);
        state.discarded = state.discarded.saturating_add(discarded);
        state.available_bytes = 0;
        drop(state);
        self.retirement.retire(
            idle.into_iter()
                .map(|(capacity, buffer, _, _)| (Retired::Buffer(buffer), capacity)),
        );
    }

    pub(crate) fn stats(&self) -> UniformBufferPoolStats {
        let state = self
            .state
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner);
        UniformBufferPoolStats {
            created: state.created,
            reused: state.reused,
            discarded: state.discarded,
            active: state.active,
            active_bytes: state.active_bytes,
            high_water_bytes: state.high_water_bytes,
            available: state.available.len(),
            available_bytes: state.available_bytes,
        }
    }

    /// Keeps a finished arena for reuse while its capacity has fewer idle
    /// arenas than were ever in use at once, within the idle bounds, which
    /// evict the oldest release first. What the pool does not keep is
    /// destroyed.
    fn release(&self, capacity: u64, buffer: wgpu::Buffer, bytes: Vec<u8>, generation: u64) {
        if !self.retirement.trusts(generation) {
            self.discard(capacity, buffer);
            return;
        }
        let mut retired = Vec::new();
        self.keep_or_retire(capacity, buffer, bytes, generation, &mut retired);
        self.retirement.retire(retired);
    }

    fn keep_or_retire(
        &self,
        capacity: u64,
        buffer: wgpu::Buffer,
        bytes: Vec<u8>,
        generation: u64,
        retired: &mut Vec<(Retired, u64)>,
    ) {
        let mut state = self
            .state
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner);
        finish_active(&mut state, capacity);
        let retention_target = state
            .peak_active_by_capacity
            .get(&capacity)
            .copied()
            .unwrap_or(1);
        let idle_at_capacity = state
            .available
            .iter()
            .filter(|(available, _, _, _)| *available == capacity)
            .count();
        if idle_at_capacity >= retention_target || capacity > MAX_IDLE_UNIFORM_BYTES {
            state.discarded = state.discarded.saturating_add(1);
            retired.push((Retired::Buffer(buffer), capacity));
            return;
        }
        while state.available.len() >= MAX_IDLE_UNIFORM_BUFFERS
            || state.available_bytes.saturating_add(capacity) > MAX_IDLE_UNIFORM_BYTES
        {
            let (discarded, discarded_buffer, _bytes, _) = state.available.remove(0);
            state.available_bytes = state.available_bytes.saturating_sub(discarded);
            state.discarded = state.discarded.saturating_add(1);
            prune_capacity(&mut state, discarded);
            retired.push((Retired::Buffer(discarded_buffer), discarded));
        }
        state.available.push((capacity, buffer, bytes, generation));
        state.available_bytes = state.available_bytes.saturating_add(capacity);
    }

    /// Takes back an arena whose encode failed or was abandoned.
    fn discard(&self, capacity: u64, buffer: wgpu::Buffer) {
        let mut state = self
            .state
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner);
        finish_active(&mut state, capacity);
        state.discarded = state.discarded.saturating_add(1);
        prune_capacity(&mut state, capacity);
        drop(state);
        self.retirement
            .retire([(Retired::Buffer(buffer), capacity)]);
    }
}

impl UniformArenaLease {
    pub(crate) fn write(&mut self, contents: &[u8]) -> UniformSlice {
        let offset = align_up(self.cursor, self.alignment);
        let size = u64::try_from(contents.len()).expect("uniform contents fit u64");
        let end = offset.checked_add(size).expect("uniform arena offset");
        assert!(end <= self.capacity, "uniform arena capacity was reserved");
        let start = usize::try_from(offset).expect("uniform offset fits host indexing");
        let end = usize::try_from(end).expect("uniform end fits host indexing");
        self.bytes[start..end].copy_from_slice(contents);
        self.cursor = u64::try_from(end).expect("uniform cursor fits u64");
        UniformSlice {
            offset,
            size: NonZeroU64::new(size).expect("uniform binding is non-empty"),
        }
    }

    pub(crate) fn binding(&self, slice: UniformSlice) -> wgpu::BindingResource<'_> {
        wgpu::BindingResource::Buffer(wgpu::BufferBinding {
            buffer: self.buffer.as_ref().expect("live uniform arena"),
            offset: slice.offset,
            size: Some(slice.size),
        })
    }

    #[cfg(test)]
    pub(crate) fn buffer(&self) -> &wgpu::Buffer {
        self.buffer.as_ref().expect("live uniform arena")
    }

    pub(crate) fn flush(&self, queue: &wgpu::Queue) {
        if self.cursor > 0 {
            let used = usize::try_from(self.cursor).expect("uniform cursor fits host indexing");
            queue.write_buffer(
                self.buffer.as_ref().expect("live uniform arena"),
                0,
                &self.bytes[..used],
            );
        }
    }

    pub(crate) fn recycle(mut self) {
        let buffer = self.buffer.take().expect("live uniform arena");
        let bytes = std::mem::take(&mut self.bytes);
        self.pool
            .release(self.capacity, buffer, bytes, self.generation);
    }
}

impl Drop for UniformArenaLease {
    fn drop(&mut self) {
        if let Some(buffer) = self.buffer.take() {
            self.pool.discard(self.capacity, buffer);
        }
    }
}

fn finish_active(state: &mut UniformBufferPoolState, capacity: u64) {
    state.active = state.active.saturating_sub(1);
    state.active_bytes = state.active_bytes.saturating_sub(capacity);
    if let Some(active) = state.active_by_capacity.get_mut(&capacity) {
        *active = active.saturating_sub(1);
        if *active == 0 {
            state.active_by_capacity.remove(&capacity);
        }
    }
}

/// Forgets a capacity's peak once nothing of it is active or idle, so the
/// retention map stays as small as the capacities in use.
fn prune_capacity(state: &mut UniformBufferPoolState, capacity: u64) {
    if !state.active_by_capacity.contains_key(&capacity)
        && !state
            .available
            .iter()
            .any(|(available, _, _, _)| *available == capacity)
    {
        state.peak_active_by_capacity.remove(&capacity);
    }
}

pub(crate) const fn aligned_uniform_end(cursor: u64, size: u64, alignment: u64) -> u64 {
    align_up(cursor, alignment).saturating_add(size)
}

const fn align_up(value: u64, alignment: u64) -> u64 {
    value
        .saturating_add(alignment.saturating_sub(1))
        .div_euclid(alignment)
        .saturating_mul(alignment)
}