1use std::sync::Arc;
2
3use super::{
4 FrameBinding, FrameComposerError, FrameItem, ReadyItem, RetainedResourceKey, ScalarLutPlan,
5 create_frame_binding, uniform_uploads,
6};
7use crate::renderer::{ImageSampling, ImageUniform, PreparedDrawBatch, WgpuRenderer};
8
9#[cfg(test)]
10pub(in crate::renderer) use tests::assert_gpu_binding_sharing_contract;
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq)]
19pub struct FrameCacheBudget {
20 max_cpu_bytes: usize,
21 max_uniform_bytes: usize,
22 max_texture_bytes: usize,
23 max_bindings: usize,
24 max_upload_staging_bytes: usize,
25}
26
27impl FrameCacheBudget {
28 pub const fn new(
31 cpu_bytes: usize,
32 uniform_bytes: usize,
33 texture_bytes: usize,
34 bindings: usize,
35 ) -> Self {
36 Self {
37 max_cpu_bytes: cpu_bytes,
38 max_uniform_bytes: uniform_bytes,
39 max_texture_bytes: texture_bytes,
40 max_bindings: bindings,
41 max_upload_staging_bytes: 0,
42 }
43 }
44 pub const fn max_cpu_bytes(self) -> usize {
46 self.max_cpu_bytes
47 }
48 pub const fn max_uniform_bytes(self) -> usize {
50 self.max_uniform_bytes
51 }
52 pub const fn max_texture_bytes(self) -> usize {
54 self.max_texture_bytes
55 }
56 pub const fn max_bindings(self) -> usize {
58 self.max_bindings
59 }
60 pub const fn with_upload_staging_bytes(mut self, bytes: usize) -> Self {
67 self.max_upload_staging_bytes = bytes;
68 self
69 }
70 pub const fn max_upload_staging_bytes(self) -> usize {
73 self.max_upload_staging_bytes
74 }
75}
76
77impl Default for FrameCacheBudget {
78 fn default() -> Self {
79 Self::new(8 * 1024 * 1024, 1024 * 1024, 256 * 1024 * 1024, 4096)
80 .with_upload_staging_bytes(256 * 1024)
81 }
82}
83
84#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
88pub struct FrameCacheStatistics {
89 pub(super) cpu_bytes: usize,
90 pub(super) uniform_bytes: usize,
91 pub(super) texture_bytes: usize,
92 pub(super) binding_count: usize,
93 pub(super) created_buffers: usize,
94 pub(super) created_bind_groups: usize,
95 pub(super) reused_bind_groups: usize,
96 pub(super) shared_bind_groups: usize,
97 pub(super) uploaded_uniform_bytes: usize,
98 pub(super) uniform_upload_calls: usize,
99 pub(super) batched_uniform_copies: usize,
100 pub(super) batched_uniform_bytes: usize,
101 pub(super) upload_staging_bytes: usize,
102 pub(super) peak_upload_staging_bytes: usize,
103 pub(super) created_upload_staging_buffers: usize,
104 pub(super) peak_cpu_bytes: usize,
105 pub(super) peak_uniform_bytes: usize,
106 pub(super) peak_texture_bytes: usize,
107}
108
109impl FrameCacheStatistics {
110 pub const fn cpu_bytes(self) -> usize {
112 self.cpu_bytes
113 }
114 pub const fn uniform_bytes(self) -> usize {
116 self.uniform_bytes
117 }
118 pub const fn texture_bytes(self) -> usize {
120 self.texture_bytes
121 }
122 pub const fn binding_count(self) -> usize {
124 self.binding_count
125 }
126 pub const fn created_buffers(self) -> usize {
128 self.created_buffers
129 }
130 pub const fn created_bind_groups(self) -> usize {
132 self.created_bind_groups
133 }
134 pub const fn reused_bind_groups(self) -> usize {
136 self.reused_bind_groups
137 }
138 pub const fn shared_bind_groups(self) -> usize {
142 self.shared_bind_groups
143 }
144 pub const fn uploaded_uniform_bytes(self) -> usize {
146 self.uploaded_uniform_bytes
147 }
148 pub const fn uniform_upload_calls(self) -> usize {
151 self.uniform_upload_calls
152 }
153 pub const fn batched_uniform_copies(self) -> usize {
156 self.batched_uniform_copies
157 }
158 pub const fn batched_uniform_bytes(self) -> usize {
162 self.batched_uniform_bytes
163 }
164 pub const fn upload_staging_bytes(self) -> usize {
166 self.upload_staging_bytes
167 }
168 pub const fn peak_upload_staging_bytes(self) -> usize {
170 self.peak_upload_staging_bytes
171 }
172 pub const fn created_upload_staging_buffers(self) -> usize {
174 self.created_upload_staging_buffers
175 }
176 pub const fn peak_cpu_bytes(self) -> usize {
180 self.peak_cpu_bytes
181 }
182 pub const fn peak_uniform_bytes(self) -> usize {
184 self.peak_uniform_bytes
185 }
186 pub const fn peak_texture_bytes(self) -> usize {
188 self.peak_texture_bytes
189 }
190}
191
192#[derive(Clone)]
193enum BindingKey {
194 Camera,
195 Image {
196 identity: Arc<()>,
197 sampling: ImageSampling,
198 texture_bytes: usize,
199 },
200 Target {
201 identity: Arc<()>,
202 texture_bytes: usize,
203 },
204}
205
206#[derive(Clone, Copy)]
209enum BindingKeyRef<'a> {
210 Camera,
211 Image {
212 identity: &'a Arc<()>,
213 sampling: ImageSampling,
214 texture_bytes: usize,
215 },
216 Target {
217 identity: &'a Arc<()>,
218 texture_bytes: usize,
219 },
220}
221
222impl<'a> BindingKeyRef<'a> {
223 fn matches(self, other: BindingKeyRef<'_>) -> bool {
224 match (self, other) {
225 (Self::Camera, BindingKeyRef::Camera) => true,
226 (
227 Self::Image {
228 identity: a,
229 sampling: sa,
230 ..
231 },
232 BindingKeyRef::Image {
233 identity: b,
234 sampling: sb,
235 ..
236 },
237 ) => Arc::ptr_eq(a, b) && sa == sb,
238 (Self::Target { identity: a, .. }, BindingKeyRef::Target { identity: b, .. }) => {
239 Arc::ptr_eq(a, b)
240 }
241 _ => false,
242 }
243 }
244
245 fn texture(self) -> Option<(&'a Arc<()>, usize)> {
246 match self {
247 Self::Camera => None,
248 Self::Image {
249 identity,
250 texture_bytes,
251 ..
252 }
253 | Self::Target {
254 identity,
255 texture_bytes,
256 } => Some((identity, texture_bytes)),
257 }
258 }
259
260 fn into_owned(self) -> BindingKey {
261 match self {
262 Self::Camera => BindingKey::Camera,
263 Self::Image {
264 identity,
265 sampling,
266 texture_bytes,
267 } => BindingKey::Image {
268 identity: Arc::clone(identity),
269 sampling,
270 texture_bytes,
271 },
272 Self::Target {
273 identity,
274 texture_bytes,
275 } => BindingKey::Target {
276 identity: Arc::clone(identity),
277 texture_bytes,
278 },
279 }
280 }
281}
282
283impl BindingKey {
284 fn as_ref(&self) -> BindingKeyRef<'_> {
285 match self {
286 Self::Camera => BindingKeyRef::Camera,
287 Self::Image {
288 identity,
289 sampling,
290 texture_bytes,
291 } => BindingKeyRef::Image {
292 identity,
293 sampling: *sampling,
294 texture_bytes: *texture_bytes,
295 },
296 Self::Target {
297 identity,
298 texture_bytes,
299 } => BindingKeyRef::Target {
300 identity,
301 texture_bytes: *texture_bytes,
302 },
303 }
304 }
305
306 fn texture(&self) -> Option<(&Arc<()>, usize)> {
307 self.as_ref().texture()
308 }
309 fn matches(&self, other: BindingKeyRef<'_>) -> bool {
310 self.as_ref().matches(other)
311 }
312}
313
314type BindingDescription<'a> = (BindingKeyRef<'a>, &'a [u8]);
315
316#[derive(Clone, Copy)]
317struct SharedBinding<'a> {
318 description: BindingDescription<'a>,
319 index: usize,
320}
321
322#[derive(Default)]
326pub(super) struct FrameBindingSharing<'a> {
327 entries: [Option<SharedBinding<'a>>; 8],
328 next: usize,
329}
330
331impl<'a> FrameBindingSharing<'a> {
332 fn find(&self, (key, bytes): BindingDescription<'_>) -> Option<usize> {
333 self.entries.iter().flatten().find_map(|entry| {
334 (entry.description.0.matches(key) && entry.description.1 == bytes)
335 .then_some(entry.index)
336 })
337 }
338
339 fn remember(&mut self, description: Option<BindingDescription<'a>>, index: usize) {
340 if let Some(description) = description {
341 self.entries[self.next] = Some(SharedBinding { description, index });
342 self.next = (self.next + 1) % self.entries.len();
343 }
344 }
345}
346
347struct CachedBinding {
348 key: BindingKey,
349 binding: FrameBinding,
350 bytes: [u8; std::mem::size_of::<ImageUniform>()],
351 len: usize,
352}
353
354#[derive(Default)]
355pub(in crate::renderer) struct FrameCache {
356 pub(super) items: Vec<FrameItem<'static>>,
357 pub(super) retained_resources: Vec<RetainedResourceKey>,
358 pub(super) scalar_luts: Vec<ScalarLutPlan>,
359 pub(super) ready: Vec<ReadyItem<'static>>,
360 pub(super) bindings: Vec<FrameBinding>,
361 pub(super) batches: Vec<Vec<PreparedDrawBatch>>,
362 slots: Vec<Option<CachedBinding>>,
363 uniform_uploads: uniform_uploads::UniformUploadBatch,
364 budget: FrameCacheBudget,
365 statistics: FrameCacheStatistics,
366 initial_cpu_bytes: usize,
367}
368
369impl FrameCache {
370 pub(super) fn uploaded_uniform_bytes(&self) -> usize {
371 self.statistics.uploaded_uniform_bytes
372 }
373 pub(super) fn begin(&mut self) {
374 self.uniform_uploads.begin();
375 self.initial_cpu_bytes = self.cpu_bytes();
376 self.statistics.created_buffers = 0;
377 self.statistics.created_bind_groups = 0;
378 self.statistics.reused_bind_groups = 0;
379 self.statistics.shared_bind_groups = 0;
380 self.statistics.uploaded_uniform_bytes = 0;
381 self.statistics.uniform_upload_calls = 0;
382 self.statistics.batched_uniform_copies = 0;
383 self.statistics.batched_uniform_bytes = 0;
384 self.statistics.created_upload_staging_buffers = 0;
385 self.statistics.peak_upload_staging_bytes = self.uniform_uploads.allocation_bytes();
386 self.statistics.peak_cpu_bytes = self.initial_cpu_bytes;
387 self.statistics.peak_uniform_bytes = self.statistics.uniform_bytes;
388 self.statistics.peak_texture_bytes = self.statistics.texture_bytes;
389 }
390
391 fn cpu_bytes(&self) -> usize {
392 vec_bytes(&self.items)
393 .saturating_add(vec_bytes(&self.retained_resources))
394 .saturating_add(vec_bytes(&self.scalar_luts))
395 .saturating_add(vec_bytes(&self.ready))
396 .saturating_add(vec_bytes(&self.bindings))
397 .saturating_add(vec_bytes(&self.slots))
398 .saturating_add(vec_bytes(&self.batches))
399 .saturating_add(self.batches.iter().map(vec_bytes).sum::<usize>())
400 .saturating_add(self.uniform_uploads.cpu_bytes())
401 }
402
403 pub(super) fn finish(&mut self) {
404 self.uniform_uploads.begin();
407 let mut cpu_bytes = self.cpu_bytes();
408 self.statistics.peak_cpu_bytes = self
409 .initial_cpu_bytes
410 .saturating_add(cpu_bytes.saturating_mul(2));
411 if cpu_bytes > self.budget.max_cpu_bytes {
412 self.items = Vec::new();
413 self.retained_resources = Vec::new();
414 self.scalar_luts = Vec::new();
415 self.ready = Vec::new();
416 self.bindings = Vec::new();
417 self.batches = Vec::new();
418 self.uniform_uploads.clear();
419 cpu_bytes = self.cpu_bytes();
420 }
421 if cpu_bytes > self.budget.max_cpu_bytes {
422 self.slots = Vec::new();
423 self.statistics.uniform_bytes = 0;
424 self.statistics.texture_bytes = 0;
425 cpu_bytes = 0;
426 }
427 self.statistics.cpu_bytes = cpu_bytes;
428 self.statistics.upload_staging_bytes = self.uniform_uploads.allocation_bytes();
429 self.statistics.binding_count = self.slots.iter().flatten().count();
430 }
431
432 pub(in crate::renderer) fn clear(&mut self) {
433 let budget = self.budget;
434 *self = Self {
435 budget,
436 ..Self::default()
437 };
438 }
439
440 pub(super) fn reserve_slots(&mut self, count: usize) -> Result<(), FrameComposerError> {
441 let count = count.min(self.budget.max_bindings);
442 if count > self.slots.len() {
443 self.slots
444 .try_reserve_exact(count - self.slots.len())
445 .map_err(|_| FrameComposerError::AllocationFailed {
446 requested_bytes: count
447 .saturating_mul(std::mem::size_of::<Option<CachedBinding>>()),
448 })?;
449 self.slots.resize_with(count, || None);
450 }
451 Ok(())
452 }
453
454 pub(super) fn prepare_uniform_uploads(
455 &mut self,
456 ready: &[ReadyItem<'_>],
457 max_buffer_size: u64,
458 ) {
459 let mut size = 0usize;
460 let mut count = 0usize;
461 if self.budget.max_upload_staging_bytes != 0 {
462 for (item, cached) in ready.iter().zip(&self.slots) {
463 if let Some((key, bytes)) = binding_description(item)
464 && let Some(cached) = cached
465 && cached.key.matches(key)
466 && cached.len == bytes.len()
467 && cached.bytes[..cached.len] != *bytes
468 {
469 let Some(next) = size.checked_add(bytes.len()) else {
470 self.uniform_uploads.begin();
471 return;
472 };
473 size = next;
474 count += 1;
475 }
476 }
477 }
478 self.uniform_uploads.prepare(
481 size,
482 count,
483 self.budget.max_upload_staging_bytes,
484 max_buffer_size,
485 );
486 }
487
488 pub(super) fn flush_uniform_uploads(
489 &mut self,
490 device: &wgpu::Device,
491 queue: &wgpu::Queue,
492 encoder: &mut wgpu::CommandEncoder,
493 bindings: &[FrameBinding],
494 ) {
495 let report = self.uniform_uploads.flush(device, queue, encoder, bindings);
496 self.statistics.created_upload_staging_buffers += report.created_buffers;
497 self.statistics.upload_staging_bytes = report.buffer_bytes;
498 self.statistics.peak_upload_staging_bytes = self
499 .statistics
500 .peak_upload_staging_bytes
501 .max(report.peak_buffer_bytes);
502 self.statistics.batched_uniform_copies += report.copy_count;
503 self.statistics.batched_uniform_bytes += report.uploaded_bytes;
504 self.statistics.uniform_upload_calls += usize::from(report.uploaded_bytes != 0);
505 }
506
507 fn other_owns_texture(&self, slot: usize, identity: &Arc<()>) -> bool {
508 self.slots.iter().enumerate().any(|(index, cached)| {
509 index != slot
510 && cached
511 .as_ref()
512 .and_then(|cached| cached.key.texture())
513 .is_some_and(|(other, _)| Arc::ptr_eq(identity, other))
514 })
515 }
516
517 fn remove_slot(&mut self, index: usize) {
518 if let Some(old) = self.slots[index].take() {
519 self.statistics.uniform_bytes = self
520 .statistics
521 .uniform_bytes
522 .saturating_sub(old.binding._buffer.size() as usize);
523 if let Some((identity, bytes)) = old.key.texture()
524 && !self.other_owns_texture(index, identity)
525 {
526 self.statistics.texture_bytes = self.statistics.texture_bytes.saturating_sub(bytes);
527 }
528 }
529 }
530
531 pub(super) fn binding<'a>(
532 &mut self,
533 renderer: &mut WgpuRenderer,
534 item: &'a ReadyItem<'_>,
535 slot: usize,
536 sharing: &mut FrameBindingSharing<'a>,
537 bindings: &[FrameBinding],
538 ) -> FrameBinding {
539 let description = binding_description(item);
540 if let Some(binding) =
541 self.reuse_binding(&renderer.queue, description, slot, sharing, bindings)
542 {
543 return binding;
544 }
545 let binding = create_frame_binding(renderer, item);
546 self.retain_new_binding(binding, description, slot, sharing)
547 }
548
549 fn reuse_binding<'a>(
550 &mut self,
551 queue: &wgpu::Queue,
552 description: Option<BindingDescription<'a>>,
553 slot: usize,
554 sharing: &mut FrameBindingSharing<'a>,
555 bindings: &[FrameBinding],
556 ) -> Option<FrameBinding> {
557 if let Some((key, bytes)) = &description
558 && let Some(Some(cached)) = self.slots.get_mut(slot)
559 && cached.key.matches(*key)
560 && cached.len == bytes.len()
561 {
562 if &cached.bytes[..cached.len] != *bytes {
563 if !self.uniform_uploads.stage(slot, bytes) {
564 queue.write_buffer(&cached.binding._buffer, 0, bytes);
565 self.statistics.uniform_upload_calls += 1;
566 }
567 cached.bytes[..cached.len].copy_from_slice(bytes);
568 self.statistics.uploaded_uniform_bytes += bytes.len();
569 }
570 self.statistics.reused_bind_groups += 1;
571 sharing.remember(description, slot);
572 return Some(cached.binding.clone());
573 }
574 if let Some(description) = description
575 && let Some(index) = sharing.find(description)
576 && let Some(binding) = bindings.get(index)
577 {
578 self.statistics.reused_bind_groups += 1;
581 self.statistics.shared_bind_groups += 1;
582 sharing.remember(Some(description), slot);
583 return Some(binding.clone());
584 }
585 None
586 }
587
588 fn retain_new_binding<'a>(
589 &mut self,
590 binding: FrameBinding,
591 description: Option<BindingDescription<'a>>,
592 slot: usize,
593 sharing: &mut FrameBindingSharing<'a>,
594 ) -> FrameBinding {
595 sharing.remember(description, slot);
596 let uniform_size = binding._buffer.size() as usize;
597 self.statistics.created_buffers += 1;
598 self.statistics.created_bind_groups += 1;
599 self.statistics.uploaded_uniform_bytes += uniform_size;
600 self.statistics.uniform_upload_calls += 1;
601 self.statistics.peak_uniform_bytes = self
602 .statistics
603 .peak_uniform_bytes
604 .saturating_add(uniform_size);
605 let Some((key, bytes)) = description else {
606 return binding;
607 };
608 if slot >= self.slots.len() {
609 return binding;
610 }
611 let new_texture = key.texture().map_or(0, |(identity, size)| {
612 if self.other_owns_texture(slot, identity)
613 || self.slots[slot]
614 .as_ref()
615 .and_then(|cached| cached.key.texture())
616 .is_some_and(|(old, _)| Arc::ptr_eq(identity, old))
617 {
618 0
619 } else {
620 size
621 }
622 });
623 let retained_uniform = self
624 .statistics
625 .uniform_bytes
626 .saturating_sub(
627 self.slots[slot]
628 .as_ref()
629 .map_or(0, |old| old.binding._buffer.size() as usize),
630 )
631 .saturating_add(uniform_size);
632 let texture_peak = self.statistics.texture_bytes.saturating_add(new_texture);
633 self.statistics.peak_texture_bytes = self.statistics.peak_texture_bytes.max(texture_peak);
634 if retained_uniform > self.budget.max_uniform_bytes
635 || texture_peak > self.budget.max_texture_bytes
636 {
637 self.remove_slot(slot);
638 return binding;
639 }
640 self.remove_slot(slot);
641 if let Some((identity, size)) = key.texture()
642 && !self.other_owns_texture(slot, identity)
643 {
644 self.statistics.texture_bytes = self.statistics.texture_bytes.saturating_add(size);
645 }
646 self.statistics.uniform_bytes = self.statistics.uniform_bytes.saturating_add(uniform_size);
647 let mut stored = [0; std::mem::size_of::<ImageUniform>()];
648 stored[..bytes.len()].copy_from_slice(bytes);
649 self.slots[slot] = Some(CachedBinding {
650 key: key.into_owned(),
651 binding: binding.clone(),
652 bytes: stored,
653 len: bytes.len(),
654 });
655 binding
656 }
657}
658
659fn binding_description<'a>(item: &'a ReadyItem<'_>) -> Option<(BindingKeyRef<'a>, &'a [u8])> {
660 match item {
661 ReadyItem::Geometry(geometry) => Some((
662 BindingKeyRef::Camera,
663 bytemuck::bytes_of(&geometry.camera_uniform),
664 )),
665 ReadyItem::Particle { camera_uniform, .. } => {
666 Some((BindingKeyRef::Camera, bytemuck::bytes_of(camera_uniform)))
667 }
668 ReadyItem::Image {
669 image,
670 uniform,
671 sampling,
672 ..
673 }
674 | ReadyItem::ImageBatch {
675 image,
676 uniform,
677 sampling,
678 ..
679 } => Some((
680 BindingKeyRef::Image {
681 identity: &image.resource_identity,
682 sampling: *sampling,
683 texture_bytes: image.gpu_allocation_bytes(),
684 },
685 bytemuck::bytes_of(uniform),
686 )),
687 ReadyItem::Target {
688 target, uniform, ..
689 } => Some((
690 BindingKeyRef::Target {
691 identity: &target.resource_identity,
692 texture_bytes: target.allocation_bytes,
693 },
694 bytemuck::bytes_of(uniform),
695 )),
696 ReadyItem::Scalar { .. } => None,
697 }
698}
699
700fn vec_bytes<T>(values: &Vec<T>) -> usize {
701 values.capacity().saturating_mul(std::mem::size_of::<T>())
702}
703
704#[allow(
708 clippy::unnecessary_filter_map,
709 reason = "filter cannot change the empty Vec's element lifetime"
710)]
711pub(super) fn recycle_items<'source, 'target>(
712 values: Vec<FrameItem<'source>>,
713) -> Vec<FrameItem<'target>> {
714 values
715 .into_iter()
716 .filter_map(|_| None::<FrameItem<'target>>)
717 .collect()
718}
719
720#[cfg(test)]
721#[allow(
722 clippy::unnecessary_filter_map,
723 reason = "filter cannot change the empty Vec's element lifetime"
724)]
725pub(super) fn recycle_ready<'source, 'target>(
726 values: Vec<ReadyItem<'source>>,
727) -> Vec<ReadyItem<'target>> {
728 values
729 .into_iter()
730 .filter_map(|_| None::<ReadyItem<'target>>)
731 .collect()
732}
733
734pub(super) fn recycle_ready_with_batches<'source, 'target>(
735 values: Vec<ReadyItem<'source>>,
736 batches: &mut Vec<Vec<PreparedDrawBatch>>,
737) -> Vec<ReadyItem<'target>> {
738 values
739 .into_iter()
740 .filter_map(|value| {
741 if let ReadyItem::Geometry(mut geometry) = value
742 && batches.len() < batches.capacity()
743 {
744 geometry.batches.clear();
745 batches.push(geometry.batches);
746 }
747 None::<ReadyItem<'target>>
748 })
749 .collect()
750}
751
752impl WgpuRenderer {
753 pub fn frame_cache_budget(&self) -> FrameCacheBudget {
755 self.frame_cache.budget
756 }
757 pub fn set_frame_cache_budget(&mut self, budget: FrameCacheBudget) {
760 self.frame_cache.clear();
761 self.frame_cache.budget = budget;
762 }
763 pub fn clear_frame_cache(&mut self) {
765 self.frame_cache.clear();
766 }
767 pub fn frame_cache_statistics(&self) -> FrameCacheStatistics {
770 self.frame_cache.statistics
771 }
772}
773
774#[cfg(test)]
775mod tests;