Skip to main content

sui_render_wgpu/
capture.rs

1use crate::WgpuRenderer;
2use crate::output::DebugCaptureEncoding;
3use crate::output::DebugCaptureRequest;
4use crate::output::DebugCaptureStage;
5use crate::output::DebugSdrVisualization;
6use crate::output::DisplayColorPrimaries;
7use crate::resources::OffscreenTarget;
8use crate::surface::normalize_framebuffer_size;
9use half::f16;
10use std::sync::{Arc, Mutex};
11use sui_core::Error;
12use sui_core::Result;
13use sui_core::WindowId;
14
15#[derive(Debug, Clone, PartialEq, Eq)]
16pub struct RgbaImage {
17    pub(crate) width: u32,
18    pub(crate) height: u32,
19    pub(crate) pixels: Vec<u8>,
20}
21
22#[derive(Debug, Clone, PartialEq)]
23pub struct HdrRgbaImage {
24    pub(crate) width: u32,
25    pub(crate) height: u32,
26    pub(crate) pixels: Vec<f32>,
27}
28
29#[derive(Debug, Clone, PartialEq)]
30pub enum DebugCaptureArtifact {
31    SdrRgba8(RgbaImage),
32    HdrLinearRgbaF32(HdrRgbaImage),
33}
34
35impl RgbaImage {
36    pub fn new(width: u32, height: u32, pixels: Vec<u8>) -> Result<Self> {
37        let expected_len = width as usize * height as usize * 4;
38        if pixels.len() != expected_len {
39            return Err(Error::new(format!(
40                "RGBA image pixel buffer length {} does not match {}x{} image size",
41                pixels.len(),
42                width,
43                height
44            )));
45        }
46
47        Ok(Self {
48            width,
49            height,
50            pixels,
51        })
52    }
53
54    pub const fn width(&self) -> u32 {
55        self.width
56    }
57
58    pub const fn height(&self) -> u32 {
59        self.height
60    }
61
62    pub fn pixels(&self) -> &[u8] {
63        &self.pixels
64    }
65
66    pub fn into_pixels(self) -> Vec<u8> {
67        self.pixels
68    }
69}
70
71impl HdrRgbaImage {
72    pub fn new(width: u32, height: u32, pixels: Vec<f32>) -> Result<Self> {
73        let expected_len = width as usize * height as usize * 4;
74        if pixels.len() != expected_len {
75            return Err(Error::new(format!(
76                "HDR RGBA image pixel buffer length {} does not match {}x{} image size",
77                pixels.len(),
78                width,
79                height
80            )));
81        }
82
83        Ok(Self {
84            width,
85            height,
86            pixels,
87        })
88    }
89
90    pub const fn width(&self) -> u32 {
91        self.width
92    }
93
94    pub const fn height(&self) -> u32 {
95        self.height
96    }
97
98    pub fn pixels(&self) -> &[f32] {
99        &self.pixels
100    }
101
102    pub fn into_pixels(self) -> Vec<f32> {
103        self.pixels
104    }
105}
106
107pub(crate) fn strip_padded_readback_rows(
108    mapped: &[u8],
109    bytes_per_row: usize,
110    padded_bytes_per_row: usize,
111    rows: usize,
112) -> Vec<u8> {
113    let mut tightly_packed = Vec::with_capacity(bytes_per_row * rows);
114    for row in 0..rows {
115        let start = row * padded_bytes_per_row;
116        tightly_packed.extend_from_slice(&mapped[start..start + bytes_per_row]);
117    }
118    tightly_packed
119}
120
121pub(crate) fn decode_rgba16f_pixels(raw: &[u8]) -> Vec<f32> {
122    let mut pixels = Vec::with_capacity(raw.len() / 2);
123    for chunk in raw.chunks_exact(2) {
124        pixels.push(f16::from_bits(u16::from_le_bytes([chunk[0], chunk[1]])).to_f32());
125    }
126    pixels
127}
128
129pub(crate) fn hdr_image_to_sdr_rgba(
130    image: &HdrRgbaImage,
131    visualization: DebugSdrVisualization,
132    reference_white: f32,
133    output_primaries: DisplayColorPrimaries,
134    fit: crate::output::SdrFit,
135) -> Result<RgbaImage> {
136    let reference_white = if reference_white.is_finite() && reference_white > 0.0 {
137        reference_white
138    } else {
139        1.0
140    };
141    let mut pixels = Vec::with_capacity((image.width() * image.height() * 4) as usize);
142    for rgba in image.pixels().chunks_exact(4) {
143        let normalized = linear_output_primaries_to_srgb(
144            [
145                rgba[0] / reference_white,
146                rgba[1] / reference_white,
147                rgba[2] / reference_white,
148            ],
149            output_primaries,
150        );
151        match visualization {
152            DebugSdrVisualization::ToneMappedColor => {
153                // Native HDR final targets store SDR white above 1.0. Divide that
154                // headroom back out, convert the output primaries to linear sRGB,
155                // then fit highlights as the window's SDR output would.
156                // SDR-authored sRGB colors round back to their original PNG bytes.
157                let fitted = fit.apply(normalized.map(|channel| channel.max(0.0)));
158                pixels.extend_from_slice(&[
159                    linear_to_srgb_capture_u8(fitted[0]),
160                    linear_to_srgb_capture_u8(fitted[1]),
161                    linear_to_srgb_capture_u8(fitted[2]),
162                    linear_alpha_to_capture_u8(rgba[3]),
163                ]);
164            }
165            DebugSdrVisualization::LuminanceHeatmap => {
166                let luminance =
167                    (normalized[0] * 0.2126 + normalized[1] * 0.7152 + normalized[2] * 0.0722)
168                        .max(0.0);
169                let normalized = (luminance / (1.0 + luminance)).clamp(0.0, 1.0);
170                let value = (normalized * 255.0).round() as u8;
171                pixels.extend_from_slice(&[value, value, value, 255]);
172            }
173            DebugSdrVisualization::HeadroomHeatmap => {
174                let headroom = normalized[0].max(normalized[1]).max(normalized[2]).max(0.0);
175                let normalized = (headroom / (1.0 + headroom)).clamp(0.0, 1.0);
176                let red = (normalized * 255.0).round() as u8;
177                let blue = ((1.0 - normalized) * 96.0).round() as u8;
178                pixels.extend_from_slice(&[red, 32, blue, 255]);
179            }
180            DebugSdrVisualization::ClipMask => {
181                let clipped = normalized[0].max(normalized[1]).max(normalized[2]) > 1.0;
182                if clipped {
183                    pixels.extend_from_slice(&[255, 64, 64, 255]);
184                } else {
185                    pixels.extend_from_slice(&[0, 0, 0, 255]);
186                }
187            }
188        }
189    }
190    RgbaImage::new(image.width(), image.height(), pixels)
191}
192
193pub(crate) fn linear_output_primaries_to_srgb(
194    color: [f32; 3],
195    output_primaries: DisplayColorPrimaries,
196) -> [f32; 3] {
197    match output_primaries {
198        DisplayColorPrimaries::Srgb => color,
199        DisplayColorPrimaries::DisplayP3 => {
200            let det = (0.822_461_96 * 0.966_805_76) - (0.177_538_02 * 0.033_194_2);
201            let red = (0.966_805_76 * color[0] - 0.177_538_02 * color[1]) / det;
202            let green = (-0.033_194_2 * color[0] + 0.822_461_96 * color[1]) / det;
203            let blue = (color[2] - (0.017_082_63 * red) - (0.072_397_43 * green)) / 0.910_519_96;
204            [red, green, blue]
205        }
206    }
207}
208
209pub(crate) fn encode_hdr_debug_artifact(
210    image: HdrRgbaImage,
211    request: DebugCaptureRequest,
212    sdr_reference_white: f32,
213    output_primaries: DisplayColorPrimaries,
214    fit: crate::output::SdrFit,
215) -> Result<DebugCaptureArtifact> {
216    match request.encoding {
217        DebugCaptureEncoding::Exr => Ok(DebugCaptureArtifact::HdrLinearRgbaF32(image)),
218        DebugCaptureEncoding::Png => hdr_image_to_sdr_rgba(
219            &image,
220            request.sdr_visualization,
221            sdr_reference_white,
222            output_primaries,
223            fit,
224        )
225        .map(DebugCaptureArtifact::SdrRgba8),
226    }
227}
228
229pub(crate) fn linear_to_srgb_capture_u8(channel: f32) -> u8 {
230    let value = channel.clamp(0.0, 1.0);
231    let encoded = if value <= 0.003_130_8 {
232        value * 12.92
233    } else {
234        (1.055 * value.powf(1.0 / 2.4)) - 0.055
235    };
236    (encoded.clamp(0.0, 1.0) * 255.0).round() as u8
237}
238
239pub(crate) fn linear_alpha_to_capture_u8(alpha: f32) -> u8 {
240    (alpha.clamp(0.0, 1.0) * 255.0).round() as u8
241}
242/// Identifies a debug capture the renderer began, until it is collected with
243/// [`WgpuRenderer::take_finished_debug_captures`].
244#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
245pub struct DebugCaptureId(u64);
246
247impl DebugCaptureId {
248    pub const fn new(value: u64) -> Self {
249        Self(value)
250    }
251
252    pub const fn get(self) -> u64 {
253        self.0
254    }
255}
256
257/// Pixels being copied back from a texture. The copy finishes on the GPU's
258/// schedule; nothing here waits for it.
259pub(crate) struct Readback {
260    buffer: wgpu::Buffer,
261    bytes_per_row: usize,
262    padded_bytes_per_row: usize,
263    rows: usize,
264    /// Set by the map callback: whether the buffer can be read.
265    mapped: Arc<Mutex<Option<std::result::Result<(), String>>>>,
266}
267
268impl Readback {
269    /// Whether the copy has come back, successfully or not.
270    fn is_back(&self) -> bool {
271        self.mapped
272            .lock()
273            .unwrap_or_else(std::sync::PoisonError::into_inner)
274            .is_some()
275    }
276
277    /// The pixels, tightly packed, once the copy is back; `None` while it is
278    /// still on its way.
279    fn bytes(&self) -> Option<Result<Vec<u8>>> {
280        let mapped = self
281            .mapped
282            .lock()
283            .unwrap_or_else(std::sync::PoisonError::into_inner)
284            .clone()?;
285        Some(mapped.map_err(Error::new).and_then(|()| {
286            let range = self.buffer.slice(..).get_mapped_range().map_err(|error| {
287                Error::new(format!(
288                    "failed to access the mapped readback buffer: {error}"
289                ))
290            })?;
291            Ok(strip_padded_readback_rows(
292                &range,
293                self.bytes_per_row,
294                self.padded_bytes_per_row,
295                self.rows,
296            ))
297        }))
298    }
299}
300
301/// How a capture's pixels become its artifact.
302#[derive(Debug, Clone, Copy)]
303enum CaptureDecoding {
304    /// 8-bit sRGB-encoded color, in BGRA order when `bgra`.
305    Sdr { bgra: bool },
306    /// RGBA16 float, encoded as the request asks.
307    Hdr {
308        sdr_reference_white: f32,
309        primaries: DisplayColorPrimaries,
310        fit: crate::output::SdrFit,
311    },
312}
313
314/// A capture whose pixels are being copied back.
315/// How long [`WgpuRenderer::wait_for_readbacks_until`] waits for a copy to be
316/// reported before leaving the caller to say it did not finish.
317#[cfg(not(target_arch = "wasm32"))]
318const READBACK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
319
320pub(crate) struct InFlightCapture {
321    id: DebugCaptureId,
322    pub(crate) window_id: WindowId,
323    request: DebugCaptureRequest,
324    size: (u32, u32),
325    decoding: CaptureDecoding,
326    readback: Readback,
327}
328
329impl InFlightCapture {
330    /// The artifact, once the pixels are back.
331    fn artifact(&self) -> Option<Result<DebugCaptureArtifact>> {
332        let bytes = self.readback.bytes()?;
333        Some(bytes.and_then(|bytes| {
334            let (width, height) = self.size;
335            match self.decoding {
336                CaptureDecoding::Sdr { bgra } => {
337                    let pixels = if bgra { bgra_to_rgba(&bytes) } else { bytes };
338                    RgbaImage::new(width, height, pixels).map(DebugCaptureArtifact::SdrRgba8)
339                }
340                CaptureDecoding::Hdr {
341                    sdr_reference_white,
342                    primaries,
343                    fit,
344                } => HdrRgbaImage::new(width, height, decode_rgba16f_pixels(&bytes)).and_then(
345                    |image| {
346                        encode_hdr_debug_artifact(
347                            image,
348                            self.request,
349                            sdr_reference_white,
350                            primaries,
351                            fit,
352                        )
353                    },
354                ),
355            }
356        }))
357    }
358}
359
360fn bgra_to_rgba(bytes: &[u8]) -> Vec<u8> {
361    let mut pixels = Vec::with_capacity(bytes.len());
362    for chunk in bytes.chunks_exact(4) {
363        pixels.extend_from_slice(&[chunk[2], chunk[1], chunk[0], chunk[3]]);
364    }
365    pixels
366}
367
368impl WgpuRenderer {
369    pub fn capture_last_frame_rgba(&mut self, window_id: WindowId) -> Result<RgbaImage> {
370        match self.capture_last_frame_debug(window_id, DebugCaptureRequest::default())? {
371            DebugCaptureArtifact::SdrRgba8(image) => Ok(image),
372            DebugCaptureArtifact::HdrLinearRgbaF32(_) => Err(Error::new(format!(
373                "window {} returned an HDR debug artifact when SDR RGBA capture was requested",
374                window_id.get()
375            ))),
376        }
377    }
378
379    /// Capture `window_id`'s last frame at the requested stage, waiting for
380    /// the GPU to copy it back. Browsers cannot wait; begin captures there
381    /// with [`begin_debug_capture`](Self::begin_debug_capture).
382    pub fn capture_last_frame_debug(
383        &mut self,
384        window_id: WindowId,
385        request: DebugCaptureRequest,
386    ) -> Result<DebugCaptureArtifact> {
387        self.render_last_frame_for_capture(window_id, request)?;
388        self.capture_debug_frame(window_id, request)
389    }
390
391    /// Start capturing `window_id`'s last frame at the requested stage: draw
392    /// it again offscreen and begin copying it back. Nothing waits for the
393    /// GPU, so this works in browsers too. Collect the artifact with
394    /// [`take_finished_debug_captures`](Self::take_finished_debug_captures)
395    /// once the copy is back.
396    pub fn begin_debug_capture(
397        &mut self,
398        window_id: WindowId,
399        request: DebugCaptureRequest,
400    ) -> Result<DebugCaptureId> {
401        self.render_last_frame_for_capture(window_id, request)?;
402        let (target, decoding) = self.capture_target(window_id, request)?;
403        let size = target.size;
404        let bytes_per_pixel = match decoding {
405            CaptureDecoding::Sdr { .. } => 4,
406            CaptureDecoding::Hdr { .. } => 8,
407        };
408        let readback = self.start_readback(
409            &target.texture,
410            size,
411            bytes_per_pixel,
412            "SUI debug capture readback",
413        )?;
414        let id = DebugCaptureId(self.next_debug_capture_id);
415        self.next_debug_capture_id += 1;
416        self.debug_captures_in_flight.push(InFlightCapture {
417            id,
418            window_id,
419            request,
420            size,
421            decoding,
422            readback,
423        });
424        Ok(id)
425    }
426
427    /// Whether captures of `window_id` are still being copied back.
428    pub fn has_debug_captures_in_flight(&self, window_id: WindowId) -> bool {
429        self.debug_captures_in_flight
430            .iter()
431            .any(|capture| capture.window_id == window_id)
432    }
433
434    /// The captures of `window_id` whose pixels are back, each handed out
435    /// once. Never waits for the GPU.
436    pub fn take_finished_debug_captures(
437        &mut self,
438        window_id: WindowId,
439    ) -> Vec<(DebugCaptureId, Result<DebugCaptureArtifact>)> {
440        // Native devices deliver map callbacks when polled; browsers deliver
441        // them on their own.
442        if let Some(shared) = &self.shared {
443            let _ = shared.device.poll(wgpu::PollType::Poll);
444        }
445        let mut finished = Vec::new();
446        self.debug_captures_in_flight.retain(|capture| {
447            if capture.window_id != window_id {
448                return true;
449            }
450            match capture.artifact() {
451                Some(artifact) => {
452                    finished.push((capture.id, artifact));
453                    false
454                }
455                None => true,
456            }
457        });
458        finished
459    }
460
461    /// Wait until the GPU has copied back every capture begun. Browsers
462    /// cannot wait, so this is native-only.
463    #[cfg(not(target_arch = "wasm32"))]
464    pub fn wait_for_debug_captures(&self) -> Result<()> {
465        self.wait_for_readbacks_until(|| {
466            self.debug_captures_in_flight
467                .iter()
468                .all(|capture| capture.readback.is_back())
469        })
470    }
471
472    fn render_last_frame_for_capture(
473        &mut self,
474        window_id: WindowId,
475        request: DebugCaptureRequest,
476    ) -> Result<()> {
477        let frame = self.last_frames.get(&window_id).cloned().ok_or_else(|| {
478            Error::new(format!(
479                "window {} does not have a previously rendered frame available for capture",
480                window_id.get()
481            ))
482        })?;
483
484        let size = normalize_framebuffer_size(frame.surface_size).ok_or_else(|| {
485            Error::new(format!(
486                "window {} last rendered frame has an invalid framebuffer size",
487                window_id.get()
488            ))
489        })?;
490
491        self.render_debug_capture_stage(&frame, size, request)
492            .map(|_| ())
493    }
494
495    /// The target a capture at `request`'s stage reads, and how its pixels
496    /// become the artifact.
497    fn capture_target(
498        &self,
499        window_id: WindowId,
500        request: DebugCaptureRequest,
501    ) -> Result<(&OffscreenTarget, CaptureDecoding)> {
502        let fit = self.sdr_preview_fit(window_id);
503        match request.stage {
504            DebugCaptureStage::FinalComposed => {
505                let target = self.offscreen_targets.get(&window_id).ok_or_else(|| {
506                    Error::new(format!(
507                        "window {} does not have an offscreen target available for final composed debug capture",
508                        window_id.get()
509                    ))
510                })?;
511                let decoding = match target.format {
512                    // Unorm targets hold color the output shader encoded, the
513                    // same bytes an sRGB target stores.
514                    wgpu::TextureFormat::Bgra8UnormSrgb | wgpu::TextureFormat::Bgra8Unorm => {
515                        CaptureDecoding::Sdr { bgra: true }
516                    }
517                    wgpu::TextureFormat::Rgba8UnormSrgb | wgpu::TextureFormat::Rgba8Unorm => {
518                        CaptureDecoding::Sdr { bgra: false }
519                    }
520                    wgpu::TextureFormat::Rgba16Float => CaptureDecoding::Hdr {
521                        sdr_reference_white: self.final_composed_sdr_reference_white(window_id),
522                        primaries: self.final_composed_output_primaries(window_id),
523                        fit,
524                    },
525                    other => {
526                        return Err(Error::new(format!(
527                            "window {} uses unsupported final composed debug capture format {other:?}",
528                            window_id.get()
529                        )));
530                    }
531                };
532                Ok((target, decoding))
533            }
534            DebugCaptureStage::HdrIntermediate => {
535                let target = self.intermediate_targets.get(&window_id).ok_or_else(|| {
536                    Error::new(format!(
537                        "window {} does not have an HDR intermediate target available for debug capture",
538                        window_id.get()
539                    ))
540                })?;
541                Ok((
542                    target,
543                    CaptureDecoding::Hdr {
544                        sdr_reference_white: 1.0,
545                        primaries: DisplayColorPrimaries::Srgb,
546                        fit,
547                    },
548                ))
549            }
550        }
551    }
552
553    /// Capture the frame last drawn at `request`'s stage, waiting for the
554    /// GPU to copy it back.
555    pub fn capture_debug_frame(
556        &self,
557        window_id: WindowId,
558        request: DebugCaptureRequest,
559    ) -> Result<DebugCaptureArtifact> {
560        let (target, decoding) = self.capture_target(window_id, request)?;
561        let size = target.size;
562        let bytes_per_pixel = match decoding {
563            CaptureDecoding::Sdr { .. } => 4,
564            CaptureDecoding::Hdr { .. } => 8,
565        };
566        let readback = self.start_readback(
567            &target.texture,
568            size,
569            bytes_per_pixel,
570            "SUI debug capture readback",
571        )?;
572        self.wait_for_readbacks_until(|| readback.is_back())?;
573        InFlightCapture {
574            id: DebugCaptureId(0),
575            window_id,
576            request,
577            size,
578            decoding,
579            readback,
580        }
581        .artifact()
582        .unwrap_or_else(|| Err(Error::new("the debug capture readback did not finish")))
583    }
584
585    pub fn capture_rgba(&self, window_id: WindowId) -> Result<RgbaImage> {
586        let target = self.offscreen_targets.get(&window_id).ok_or_else(|| {
587            Error::new(format!(
588                "window {} does not have an offscreen render target available for screenshot capture",
589                window_id.get()
590            ))
591        })?;
592        let raw =
593            self.readback_target_bytes(&target.texture, target.size, 4, "SUI screenshot readback")?;
594        let pixels = match target.format {
595            wgpu::TextureFormat::Rgba8UnormSrgb | wgpu::TextureFormat::Rgba8Unorm => raw,
596            _ => bgra_to_rgba(&raw),
597        };
598        RgbaImage::new(target.size.0, target.size.1, pixels)
599    }
600
601    pub fn capture_hdr_intermediate_rgba_f32(&self, window_id: WindowId) -> Result<HdrRgbaImage> {
602        let target = self.intermediate_targets.get(&window_id).ok_or_else(|| {
603            Error::new(format!(
604                "window {} does not have an HDR intermediate target available for debug capture",
605                window_id.get()
606            ))
607        })?;
608        let raw = self.readback_target_bytes(
609            &target.texture,
610            target.size,
611            8,
612            "SUI HDR intermediate readback",
613        )?;
614        let pixels = decode_rgba16f_pixels(&raw);
615        HdrRgbaImage::new(target.size.0, target.size.1, pixels)
616    }
617
618    #[cfg(test)]
619    pub(crate) fn capture_hdr_offscreen_rgba_f32(
620        &self,
621        window_id: WindowId,
622    ) -> Result<HdrRgbaImage> {
623        let target = self.offscreen_targets.get(&window_id).ok_or_else(|| {
624            Error::new(format!(
625                "window {} does not have an offscreen HDR target available for debug capture",
626                window_id.get()
627            ))
628        })?;
629        let raw =
630            self.readback_target_bytes(&target.texture, target.size, 8, "SUI HDR final readback")?;
631        let pixels = decode_rgba16f_pixels(&raw);
632        HdrRgbaImage::new(target.size.0, target.size.1, pixels)
633    }
634
635    /// How SDR previews of HDR captures fit highlights: as the window's own
636    /// SDR output would.
637    pub(crate) fn sdr_preview_fit(&self, window_id: WindowId) -> crate::output::SdrFit {
638        crate::output::SdrFit::for_tone_mapping(
639            self.window_color_management(window_id).tone_mapping,
640        )
641    }
642
643    pub(crate) fn final_composed_sdr_reference_white(&self, window_id: WindowId) -> f32 {
644        self.window_output_strategy(window_id)
645            .map(|strategy| {
646                crate::output::output_sdr_content_scale(
647                    strategy,
648                    self.window_color_management(window_id)
649                        .sdr_content_brightness_nits,
650                    self.window_display_capabilities(window_id)
651                        .and_then(|capabilities| capabilities.sdr_white_nits),
652                )
653            })
654            .filter(|scale| scale.is_finite() && *scale > 0.0)
655            .unwrap_or(1.0)
656    }
657
658    pub(crate) fn final_composed_output_primaries(
659        &self,
660        window_id: WindowId,
661    ) -> DisplayColorPrimaries {
662        self.window_output_strategy(window_id)
663            .map(crate::output::output_primaries)
664            .unwrap_or(DisplayColorPrimaries::Srgb)
665    }
666
667    /// Copy `texture` into a buffer and ask for it to be mapped, without
668    /// waiting for either.
669    pub(crate) fn start_readback(
670        &self,
671        texture: &wgpu::Texture,
672        size: (u32, u32),
673        bytes_per_pixel: u32,
674        label: &'static str,
675    ) -> Result<Readback> {
676        let shared = self
677            .shared
678            .as_ref()
679            .ok_or_else(|| Error::new("renderer has not initialized a wgpu device yet"))?;
680        let bytes_per_row = size.0 * bytes_per_pixel;
681        let padded_bytes_per_row = bytes_per_row.div_ceil(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT)
682            * wgpu::COPY_BYTES_PER_ROW_ALIGNMENT;
683        let buffer = shared.device.create_buffer(&wgpu::BufferDescriptor {
684            label: Some(label),
685            size: padded_bytes_per_row as u64 * size.1 as u64,
686            usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
687            mapped_at_creation: false,
688        });
689
690        let mut encoder = shared
691            .device
692            .create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some(label) });
693        encoder.copy_texture_to_buffer(
694            wgpu::TexelCopyTextureInfo {
695                texture,
696                mip_level: 0,
697                origin: wgpu::Origin3d::ZERO,
698                aspect: wgpu::TextureAspect::All,
699            },
700            wgpu::TexelCopyBufferInfo {
701                buffer: &buffer,
702                layout: wgpu::TexelCopyBufferLayout {
703                    offset: 0,
704                    bytes_per_row: Some(padded_bytes_per_row),
705                    rows_per_image: Some(size.1),
706                },
707            },
708            wgpu::Extent3d {
709                width: size.0,
710                height: size.1,
711                depth_or_array_layers: 1,
712            },
713        );
714        shared.queue.submit([encoder.finish()]);
715
716        let mapped = Arc::new(Mutex::new(None));
717        let callback_mapped = Arc::clone(&mapped);
718        buffer
719            .slice(..)
720            .map_async(wgpu::MapMode::Read, move |result| {
721                *callback_mapped
722                    .lock()
723                    .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(
724                    result.map_err(|error| format!("failed to map the readback buffer: {error}")),
725                );
726            });
727        Ok(Readback {
728            buffer,
729            bytes_per_row: bytes_per_row as usize,
730            padded_bytes_per_row: padded_bytes_per_row as usize,
731            rows: size.1 as usize,
732            mapped,
733        })
734    }
735
736    /// Wait for the GPU to finish the readbacks it was given, until `back`
737    /// says the ones the caller wants have been reported. Renderers without
738    /// a window share their device across threads, and another thread's poll
739    /// can collect a finished copy and still be reporting it when this poll
740    /// returns, so this polls again until the report is in, for at most
741    /// [`READBACK_TIMEOUT`]. Browsers return at once, before they finish.
742    fn wait_for_readbacks_until(&self, back: impl Fn() -> bool) -> Result<()> {
743        let shared = self
744            .shared
745            .as_ref()
746            .ok_or_else(|| Error::new("renderer has not initialized a wgpu device yet"))?;
747        let poll = || {
748            shared
749                .device
750                .poll(wgpu::PollType::wait_indefinitely())
751                .map(|_| ())
752                .map_err(|error| Error::new(format!("failed to wait for a readback: {error}")))
753        };
754        poll()?;
755        #[cfg(not(target_arch = "wasm32"))]
756        {
757            let deadline = std::time::Instant::now() + READBACK_TIMEOUT;
758            while !back() && std::time::Instant::now() < deadline {
759                std::thread::sleep(std::time::Duration::from_millis(1));
760                poll()?;
761            }
762        }
763        #[cfg(target_arch = "wasm32")]
764        let _ = back;
765        Ok(())
766    }
767
768    /// Copy `texture` back, waiting for the GPU.
769    pub(crate) fn readback_target_bytes(
770        &self,
771        texture: &wgpu::Texture,
772        size: (u32, u32),
773        bytes_per_pixel: u32,
774        label: &'static str,
775    ) -> Result<Vec<u8>> {
776        let readback = self.start_readback(texture, size, bytes_per_pixel, label)?;
777        self.wait_for_readbacks_until(|| readback.is_back())?;
778        readback
779            .bytes()
780            .unwrap_or_else(|| Err(Error::new("the readback did not finish")))
781    }
782}