Skip to main content

wgpu_core/device/
queue.rs

1use alloc::{boxed::Box, string::ToString, sync::Arc, vec, vec::Vec};
2use core::{
3    iter,
4    mem::{self, ManuallyDrop},
5    num::NonZeroU64,
6    ptr::NonNull,
7    sync::atomic::Ordering,
8};
9use smallvec::SmallVec;
10use thiserror::Error;
11use wgt::{
12    error::{ErrorType, WebGpuError},
13    AccelerationStructureFlags,
14};
15
16use super::{life::LifetimeTracker, Device};
17#[cfg(feature = "trace")]
18use crate::device::trace::{Action, IntoTrace};
19use crate::{
20    api_log,
21    command::{
22        extract_texture_selector, validate_linear_texture_data, validate_texture_buffer_copy,
23        validate_texture_copy_dst_format, validate_texture_copy_range, ClearError,
24        CommandAllocator, CommandBuffer, CommandEncoder, CommandEncoderError, CopySide,
25        TransferError,
26    },
27    device::{DeviceError, WaitIdleError},
28    get_lowest_common_denom,
29    global::Global,
30    hal_label,
31    id::{self, BlasId, QueueId},
32    init_tracker::{has_copy_partial_init_tracker_coverage, TextureInitRange},
33    lock::{rank, Mutex, MutexGuard, RwLock, RwLockWriteGuard},
34    ray_tracing::{BlasCompactReadyPendingClosure, CompactBlasError},
35    resource::{
36        Blas, BlasCompactState, Buffer, BufferAccessError, BufferMapState, DestroyedBuffer,
37        DestroyedQuerySet, DestroyedResourceError, DestroyedTexture, Fallible,
38        FlushedStagingBuffer, InvalidOrDestroyedResourceError, InvalidResourceError, Labeled,
39        ParentDevice, ResourceErrorIdent, StagingBuffer, Texture, TextureInner, Trackable,
40        TrackingData,
41    },
42    resource_log,
43    scratch::ScratchBuffer,
44    snatch::{SnatchGuard, Snatchable},
45    track::{self, Tracker, TrackerIndex},
46    FastHashMap, SubmissionIndex,
47};
48use crate::{device::resource::CommandIndices, resource::RawResourceAccess};
49
50pub struct Queue {
51    raw: Box<dyn hal::DynQueue>,
52    pub(crate) pending_writes: Mutex<PendingWrites>,
53    life_tracker: Mutex<LifetimeTracker>,
54    // The device needs to be dropped last (`Device.zero_buffer` might be referenced by the encoder in pending writes).
55    pub(crate) device: Arc<Device>,
56}
57
58impl Queue {
59    pub(crate) fn new(
60        device: Arc<Device>,
61        raw: Box<dyn hal::DynQueue>,
62        instance_flags: wgt::InstanceFlags,
63    ) -> Result<Self, DeviceError> {
64        let pending_encoder = device
65            .command_allocator
66            .acquire_encoder(device.raw(), raw.as_ref())
67            .map_err(DeviceError::from_hal);
68
69        let pending_encoder = match pending_encoder {
70            Ok(pending_encoder) => pending_encoder,
71            Err(e) => {
72                return Err(e);
73            }
74        };
75
76        let mut pending_writes = PendingWrites::new(pending_encoder, instance_flags);
77
78        let zero_buffer = device.zero_buffer.as_ref();
79        pending_writes.activate();
80        unsafe {
81            pending_writes
82                .command_encoder
83                .transition_buffers(&[hal::BufferBarrier {
84                    buffer: zero_buffer,
85                    usage: hal::StateTransition {
86                        from: wgt::BufferUses::empty(),
87                        to: wgt::BufferUses::COPY_DST,
88                    },
89                }]);
90            pending_writes
91                .command_encoder
92                .clear_buffer(zero_buffer, 0..super::ZERO_BUFFER_SIZE);
93            pending_writes
94                .command_encoder
95                .transition_buffers(&[hal::BufferBarrier {
96                    buffer: zero_buffer,
97                    usage: hal::StateTransition {
98                        from: wgt::BufferUses::COPY_DST,
99                        to: wgt::BufferUses::COPY_SRC,
100                    },
101                }]);
102        }
103
104        Ok(Queue {
105            raw,
106            device,
107            pending_writes: Mutex::new(rank::QUEUE_PENDING_WRITES, pending_writes),
108            life_tracker: Mutex::new(rank::QUEUE_LIFE_TRACKER, LifetimeTracker::new()),
109        })
110    }
111
112    pub(crate) fn raw(&self) -> &dyn hal::DynQueue {
113        self.raw.as_ref()
114    }
115
116    #[track_caller]
117    pub(crate) fn lock_life<'a>(&'a self) -> MutexGuard<'a, LifetimeTracker> {
118        self.life_tracker.lock()
119    }
120
121    /// Ensure the surface texture is in the PRESENT state, clearing it if it was never rendered to.
122    /// Submits any necessary work to the GPU before the HAL present call.
123    ///
124    /// See <https://github.com/gfx-rs/wgpu/issues/6748>
125    pub(crate) fn prepare_surface_texture_for_present(
126        &self,
127        texture: &Arc<Texture>,
128    ) -> Result<(), DeviceError> {
129        let snatch_guard = self.device.snatchable_lock.read();
130        let submission = self
131            .allocate_submission(snatch_guard)
132            .map_err(|(_index, e)| e)?;
133        let device = &self.device;
134
135        // If the texture is uninitialized it needs to be cleared before presenting
136        let needs_clear = {
137            let status = texture.initialization_status.read();
138            status
139                .mips
140                .first()
141                .is_some_and(|mip| mip.check(0..1).is_some())
142        };
143
144        let mut pending_writes = self.pending_writes.lock();
145
146        if needs_clear {
147            // After encoding the clear operation, we must not return without
148            // adding the texture to `pending_writes`.
149            let encoder = pending_writes.activate();
150            let mut trackers = device.trackers.lock();
151            crate::command::clear_texture(
152                texture,
153                TextureInitRange {
154                    mip_range: 0..1,
155                    layer_range: 0..1,
156                },
157                encoder,
158                &mut trackers.textures,
159                &device.alignments,
160                device.zero_buffer.as_ref(),
161                &submission.snatch_guard,
162                device.instance_flags,
163            )
164            .map_err(|e| match e {
165                ClearError::Device(e) => e,
166                _ => DeviceError::Lost,
167            })?;
168            texture.initialization_status.write().mips[0].drain(0..1);
169        }
170
171        // Transition the texture to PRESENT in the device tracker.
172        // If it's already in PRESENT, this produces no barriers and we can skip the submission.
173        //
174        // This has to be after any clear_texture call because clear_texture modifies the tracker state internally.
175        // Computing transitions afterward ensures they reflect the actual current state.
176        let pending = {
177            let mut trackers = device.trackers.lock();
178            let pending: Vec<track::PendingTransition<wgt::TextureUses>> = trackers
179                .textures
180                .set_single(
181                    texture,
182                    texture.full_range.clone(),
183                    wgt::TextureUses::PRESENT,
184                )
185                .collect();
186            pending
187        };
188
189        // A kept image (see `Surface::keep_presented_images`) that no
190        // submission used is already in PRESENT, but a submission must still
191        // wait for its acquisition before it is presented.
192        let awaits_acquire = match texture.inner.get(&submission.snatch_guard).maybe_valid() {
193            Some(TextureInner::Surface { raw }) => raw.awaits_acquire(),
194            _ => false,
195        };
196        if pending.is_empty() && !awaits_acquire {
197            // This assert checks that we don't return here if we encoded a
198            // clear operation for the texture, which would be a problem since
199            // we haven't done anything yet to ensure it stays alive. If we
200            // cleared the texture, then we must have produced a barrier to put
201            // it in PRESENT state, so `pending` will not be empty.
202            debug_assert!(!needs_clear);
203            return Ok(());
204        }
205
206        // Emit the transition barriers to PRESENT.
207        {
208            let raw_texture = texture
209                .raw(&submission.snatch_guard)
210                .ok_or(DeviceError::Lost)?;
211            let barriers: Vec<hal::TextureBarrier<'_, dyn hal::DynTexture>> = pending
212                .into_iter()
213                .map(|pt| pt.into_hal(raw_texture))
214                .collect();
215
216            let encoder = pending_writes.activate();
217            // SAFETY:
218            // - The encoder is in the recording state after `activate()`
219            // - The texture is kept alive by adding it to `PendingWrites` below
220            unsafe {
221                encoder.transition_textures(&barriers);
222            }
223        }
224
225        // Add the texture to `PendingWrites`. This will cause `submit()` to:
226        // - Flush any pending writes to the texture.
227        // - Include the texture in `surface_textures` for the submission.
228        // - Keep the texture alive so the texture and its clear_view aren't
229        //   destroyed before the GPU finishes the `clear_texture` operation
230        //   encoded above.
231        pending_writes.insert_texture(texture);
232
233        submission.submit(pending_writes)?;
234
235        Ok(())
236    }
237
238    /// Maintains the queue's list of finished command buffers.
239    ///
240    /// Command buffers submitted before `submission_index` was submitted
241    /// stop being tracked and callbacks which are waiting for them are
242    /// returned. Also returned is whether the queue is empty. This may
243    /// be stale unless new submissions are prevented by locking
244    /// [`Device::command_indices`]
245    pub(crate) fn maintain(
246        &self,
247        submission_index: u64,
248        snatch_guard: &SnatchGuard,
249    ) -> (
250        SmallVec<[SubmittedWorkDoneClosure; 1]>,
251        Vec<super::BufferMapPendingClosure>,
252        Vec<BlasCompactReadyPendingClosure>,
253        bool,
254    ) {
255        let mut life_tracker = self.lock_life();
256        let submission_closures = life_tracker.triage_submissions(submission_index);
257
258        let mapping_closures = life_tracker.handle_mapping(snatch_guard);
259        let blas_closures = life_tracker.handle_compact_read_back();
260
261        let queue_empty = life_tracker.queue_empty();
262
263        (
264            submission_closures,
265            mapping_closures,
266            blas_closures,
267            queue_empty,
268        )
269    }
270}
271
272crate::impl_resource_type!(Queue);
273// TODO: https://github.com/gfx-rs/wgpu/issues/4014
274impl Labeled for Queue {
275    fn label(&self) -> &str {
276        ""
277    }
278}
279crate::impl_parent_device!(Queue);
280crate::impl_storage_item!(Queue);
281
282impl Drop for Queue {
283    fn drop(&mut self) {
284        resource_log!("Drop {}", self.error_ident());
285
286        // On Vulkan, pending presents are not tracked by fences.
287        // wait_for_idle covers both fence-tracked submissions and pending presents.
288        match unsafe { self.raw.wait_for_idle() } {
289            Ok(()) => {}
290            Err(hal::DeviceError::Lost) => {
291                self.device.handle_hal_error(hal::DeviceError::Lost);
292            }
293            Err(e) => {
294                panic!("Unexpected error while waiting for queue idle on drop: {e:?}");
295            }
296        }
297
298        let last_successful_submission_index = self
299            .device
300            .last_successful_submission_index
301            .load(Ordering::Acquire);
302
303        let snatch_guard = self.device.snatchable_lock.read();
304        let (submission_closures, mapping_closures, blas_compact_ready_closures, queue_empty) =
305            self.maintain(last_successful_submission_index, &snatch_guard);
306        drop(snatch_guard);
307
308        assert!(queue_empty);
309
310        let closures = crate::device::UserClosures {
311            mappings: mapping_closures,
312            blas_compact_ready: blas_compact_ready_closures,
313            submissions: submission_closures,
314            device_lost_invocations: SmallVec::new(),
315        };
316
317        closures.fire();
318    }
319}
320
321#[cfg(send_sync)]
322pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + Send + 'static>;
323#[cfg(not(send_sync))]
324pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + 'static>;
325
326/// A texture or buffer to be freed soon.
327///
328/// This is just a tagged raw texture or buffer, generally about to be added to
329/// some other more specific container like:
330///
331/// - `PendingWrites::temp_resources`: resources used by queue writes and
332///   unmaps, waiting to be folded in with the next queue submission
333///
334/// - `ActiveSubmission::temp_resources`: temporary resources used by a queue
335///   submission, to be freed when it completes
336#[derive(Debug)]
337pub enum TempResource {
338    StagingBuffer(FlushedStagingBuffer),
339    ScratchBuffer(ScratchBuffer),
340    DestroyedBuffer(DestroyedBuffer),
341    DestroyedTexture(DestroyedTexture),
342    DestroyedQuerySet(DestroyedQuerySet),
343}
344
345/// A series of raw [`CommandBuffer`]s that have been submitted to a
346/// queue, and the [`wgpu_hal::CommandEncoder`] that built them.
347///
348/// [`CommandBuffer`]: hal::Api::CommandBuffer
349/// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder
350pub(crate) struct EncoderInFlight {
351    inner: crate::command::InnerCommandEncoder,
352    pub(crate) trackers: Tracker,
353    pub(crate) temp_resources: Vec<TempResource>,
354    /// We only need to keep these resources alive.
355    _indirect_draw_validation_resources: crate::indirect_validation::DrawResources,
356
357    /// These are the buffers that have been tracked by `PendingWrites`.
358    pub(crate) pending_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
359    /// These are the textures that have been tracked by `PendingWrites`.
360    pub(crate) pending_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
361    /// These are the BLASes that have been tracked by `PendingWrites`.
362    pub(crate) pending_blas_s: FastHashMap<TrackerIndex, Arc<Blas>>,
363}
364
365/// A private command encoder for writes made directly on the device
366/// or queue.
367///
368/// Operations like `buffer_unmap`, `queue_write_buffer`, and
369/// `queue_write_texture` need to copy data to the GPU. At the hal
370/// level, this must be done by encoding and submitting commands, but
371/// these operations are not associated with any specific wgpu command
372/// buffer.
373///
374/// Instead, `Device::pending_writes` owns one of these values, which
375/// has its own hal command encoder and resource lists. The commands
376/// accumulated here are automatically submitted to the queue at the
377/// sooner of:
378///
379/// 1. The user's next wgpu command buffer submission. (Pending writes
380///    are inserted ahead of the user's commands.)
381/// 2. The next `mapAsync` request for a buffer that has pending
382///    writes.
383///
384/// Important:
385/// When locking pending_writes be sure that tracker is not locked
386/// and try to lock trackers for the minimum timespan possible
387///
388/// All uses of [`StagingBuffer`]s end up here.
389#[derive(Debug)]
390pub(crate) struct PendingWrites {
391    // The command encoder needs to be destroyed before any other resource in pending writes.
392    pub command_encoder: Box<dyn hal::DynCommandEncoder>,
393
394    /// True if `command_encoder` is in the "recording" state, as
395    /// described in the docs for the [`wgpu_hal::CommandEncoder`]
396    /// trait.
397    ///
398    /// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder
399    pub is_recording: bool,
400
401    temp_resources: Vec<TempResource>,
402    dst_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
403    dst_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
404    copied_blas_s: FastHashMap<TrackerIndex, Arc<Blas>>,
405    instance_flags: wgt::InstanceFlags,
406}
407
408impl PendingWrites {
409    pub fn new(
410        command_encoder: Box<dyn hal::DynCommandEncoder>,
411        instance_flags: wgt::InstanceFlags,
412    ) -> Self {
413        Self {
414            command_encoder,
415            is_recording: false,
416            temp_resources: Vec::new(),
417            dst_buffers: FastHashMap::default(),
418            dst_textures: FastHashMap::default(),
419            copied_blas_s: FastHashMap::default(),
420            instance_flags,
421        }
422    }
423
424    pub fn insert_buffer(&mut self, buffer: &Arc<Buffer>) {
425        self.dst_buffers
426            .insert(buffer.tracker_index(), buffer.clone());
427    }
428
429    pub fn insert_texture(&mut self, texture: &Arc<Texture>) {
430        self.dst_textures
431            .insert(texture.tracker_index(), texture.clone());
432    }
433
434    pub fn insert_blas(&mut self, blas: &Arc<Blas>) {
435        self.copied_blas_s
436            .insert(blas.tracker_index(), blas.clone());
437    }
438
439    pub fn contains_buffer(&self, buffer: &Arc<Buffer>) -> bool {
440        self.dst_buffers.contains_key(&buffer.tracker_index())
441    }
442
443    pub fn contains_texture(&self, texture: &Arc<Texture>) -> bool {
444        self.dst_textures.contains_key(&texture.tracker_index())
445    }
446
447    pub fn consume_temp(&mut self, resource: TempResource) {
448        self.temp_resources.push(resource);
449    }
450
451    pub fn consume(&mut self, buffer: FlushedStagingBuffer) {
452        self.temp_resources
453            .push(TempResource::StagingBuffer(buffer));
454    }
455
456    fn pre_submit(
457        &mut self,
458        command_allocator: &CommandAllocator,
459        device: &Arc<Device>,
460        queue: &Queue,
461    ) -> Result<Option<EncoderInFlight>, DeviceError> {
462        if self.is_recording {
463            let pending_buffers = mem::take(&mut self.dst_buffers);
464            let pending_textures = mem::take(&mut self.dst_textures);
465            let pending_blas_s = mem::take(&mut self.copied_blas_s);
466
467            let cmd_buf = unsafe { self.command_encoder.end_encoding() }
468                .map_err(|e| device.handle_hal_error(e))?;
469            self.is_recording = false;
470
471            let new_encoder = command_allocator
472                .acquire_encoder(device.raw(), queue.raw())
473                .map_err(|e| device.handle_hal_error(e))?;
474
475            let encoder = EncoderInFlight {
476                inner: crate::command::InnerCommandEncoder {
477                    raw: ManuallyDrop::new(mem::replace(&mut self.command_encoder, new_encoder)),
478                    list: vec![cmd_buf],
479                    device: device.clone(),
480                    is_open: false,
481                    api: crate::command::EncodingApi::InternalUse,
482                    label: "(wgpu internal) PendingWrites command encoder".into(),
483                },
484                trackers: Tracker::new(device.ordered_buffer_usages, device.ordered_texture_usages),
485                temp_resources: mem::take(&mut self.temp_resources),
486                _indirect_draw_validation_resources: crate::indirect_validation::DrawResources::new(
487                    device.clone(),
488                ),
489                pending_buffers,
490                pending_textures,
491                pending_blas_s,
492            };
493            Ok(Some(encoder))
494        } else {
495            self.dst_buffers.clear();
496            self.dst_textures.clear();
497            self.copied_blas_s.clear();
498            Ok(None)
499        }
500    }
501
502    pub fn activate(&mut self) -> &mut dyn hal::DynCommandEncoder {
503        if !self.is_recording {
504            unsafe {
505                self.command_encoder
506                    .begin_encoding(hal_label(
507                        Some("(wgpu internal) PendingWrites"),
508                        self.instance_flags,
509                    ))
510                    .unwrap();
511            }
512            self.is_recording = true;
513        }
514        self.command_encoder.as_mut()
515    }
516}
517
518impl Drop for PendingWrites {
519    fn drop(&mut self) {
520        unsafe {
521            if self.is_recording {
522                self.command_encoder.discard_encoding();
523            }
524        }
525    }
526}
527
528#[derive(Clone, Debug, Error)]
529#[non_exhaustive]
530pub enum QueueWriteError {
531    #[error(transparent)]
532    Queue(#[from] DeviceError),
533    #[error(transparent)]
534    Transfer(#[from] TransferError),
535    #[error(transparent)]
536    MemoryInitFailure(#[from] ClearError),
537    #[error(transparent)]
538    DestroyedResource(#[from] DestroyedResourceError),
539    #[error(transparent)]
540    InvalidResource(#[from] InvalidResourceError),
541}
542
543impl From<InvalidOrDestroyedResourceError> for QueueWriteError {
544    fn from(e: InvalidOrDestroyedResourceError) -> Self {
545        match e {
546            InvalidOrDestroyedResourceError::InvalidResource(e) => Self::InvalidResource(e),
547            InvalidOrDestroyedResourceError::DestroyedResource(e) => Self::DestroyedResource(e),
548        }
549    }
550}
551
552impl WebGpuError for QueueWriteError {
553    fn webgpu_error_type(&self) -> ErrorType {
554        match self {
555            Self::Queue(e) => e.webgpu_error_type(),
556            Self::Transfer(e) => e.webgpu_error_type(),
557            Self::MemoryInitFailure(e) => e.webgpu_error_type(),
558            Self::DestroyedResource(e) => e.webgpu_error_type(),
559            Self::InvalidResource(e) => e.webgpu_error_type(),
560        }
561    }
562}
563
564#[derive(Clone, Debug, Error)]
565#[non_exhaustive]
566pub enum QueueSubmitError {
567    #[error(transparent)]
568    Queue(#[from] DeviceError),
569    #[error(transparent)]
570    DestroyedResource(#[from] DestroyedResourceError),
571    #[error("{0} is still mapped")]
572    BufferStillMapped(ResourceErrorIdent),
573    #[error(transparent)]
574    InvalidResource(#[from] InvalidResourceError),
575    #[error(transparent)]
576    CommandEncoder(#[from] CommandEncoderError),
577    #[error(transparent)]
578    ValidateAsActionsError(#[from] crate::ray_tracing::ValidateAsActionsError),
579}
580
581impl From<InvalidOrDestroyedResourceError> for QueueSubmitError {
582    fn from(e: InvalidOrDestroyedResourceError) -> Self {
583        match e {
584            InvalidOrDestroyedResourceError::InvalidResource(e) => Self::InvalidResource(e),
585            InvalidOrDestroyedResourceError::DestroyedResource(e) => Self::DestroyedResource(e),
586        }
587    }
588}
589
590impl WebGpuError for QueueSubmitError {
591    fn webgpu_error_type(&self) -> ErrorType {
592        match self {
593            Self::Queue(e) => e.webgpu_error_type(),
594            Self::CommandEncoder(e) => e.webgpu_error_type(),
595            Self::ValidateAsActionsError(e) => e.webgpu_error_type(),
596            Self::InvalidResource(e) => e.webgpu_error_type(),
597            Self::DestroyedResource(_) | Self::BufferStillMapped(_) => ErrorType::Validation,
598        }
599    }
600}
601
602/// A partially-assembled submission.
603///
604/// Returned from [`Queue::allocate_submission`] and consumed by [`submit`].
605/// These are internal APIs used in `Queue::submit` and other places within
606/// `wgpu-core` that need to submit work.
607///
608/// [`submit`]: `PendingSubmission::submit`
609pub(crate) struct PendingSubmission<'a> {
610    queue: &'a Queue,
611    snatch_guard: SnatchGuard<'a>,
612    command_index_guard: RwLockWriteGuard<'a, CommandIndices>,
613    // Command buffers to be executed, along with trackers for the resources they use.
614    pub executions: Vec<EncoderInFlight>,
615    // Surface textures referenced by command buffers in this submission. These need to be
616    // passed to the HAL `submit` call. Deduplicated using a hashmap to avoid vulkan
617    // deadlocking from the same surface texture being submitted multiple times.
618    surface_textures: FastHashMap<*const Texture, Arc<Texture>>,
619    pub index: SubmissionIndex,
620}
621
622pub(crate) struct SubmissionResult<'a> {
623    pub snatch_guard: SnatchGuard<'a>,
624}
625
626impl<'a> PendingSubmission<'a> {
627    fn submit(
628        self,
629        pending_writes: MutexGuard<'a, PendingWrites>,
630    ) -> Result<SubmissionResult<'a>, DeviceError> {
631        self.queue.submit_pending_submission(pending_writes, self)
632    }
633}
634
635//TODO: move out common parts of write_xxx.
636
637impl Queue {
638    pub fn write_buffer(
639        &self,
640        buffer: Arc<Buffer>,
641        buffer_offset: wgt::BufferAddress,
642        data: &[u8],
643    ) -> Result<(), QueueWriteError> {
644        profiling::scope!("Queue::write_buffer");
645        api_log!("Queue::write_buffer");
646
647        self.device.check_is_valid()?;
648
649        let data_size = data.len() as wgt::BufferAddress;
650
651        self.same_device_as(buffer.as_ref())?;
652
653        let data_size = if let Some(data_size) = wgt::BufferSize::new(data_size) {
654            data_size
655        } else {
656            // even though a zero-length write is a no-op and no copy operation will occur,
657            // we must still validate the copy operation. This ensures that invalid
658            // API calls—like writing to a mapped buffer or out-of-bounds offsets—are
659            // caught consistently, even if no data is actually moved.
660            self.validate_write_buffer_impl(buffer.as_ref(), buffer_offset, 0)?;
661
662            log::trace!("Ignoring write_buffer of size 0");
663            return Ok(());
664        };
665
666        // Platform validation requires that the staging buffer always be
667        // freed, even if an error occurs. All paths from here must call
668        // `device.pending_writes.consume`.
669        let mut staging_buffer = StagingBuffer::new(&self.device, data_size)?;
670
671        let staging_buffer = {
672            profiling::scope!("copy");
673            staging_buffer.write(data);
674            staging_buffer.flush()
675        };
676
677        let snatch_guard = self.device.snatchable_lock.read();
678        let mut pending_writes = self.pending_writes.lock();
679
680        let result = self.write_staging_buffer_impl(
681            &snatch_guard,
682            &mut pending_writes,
683            &staging_buffer,
684            buffer,
685            buffer_offset,
686        );
687
688        drop(snatch_guard);
689
690        pending_writes.consume(staging_buffer);
691
692        drop(pending_writes);
693
694        result
695    }
696
697    pub fn create_staging_buffer(
698        &self,
699        buffer_size: wgt::BufferSize,
700    ) -> Result<(StagingBuffer, NonNull<u8>), QueueWriteError> {
701        profiling::scope!("Queue::create_staging_buffer");
702        resource_log!("Queue::create_staging_buffer");
703
704        self.device.check_is_valid()?;
705
706        let staging_buffer = StagingBuffer::new(&self.device, buffer_size)?;
707        let ptr = unsafe { staging_buffer.ptr() };
708
709        Ok((staging_buffer, ptr))
710    }
711
712    pub fn write_staging_buffer(
713        &self,
714        buffer: Fallible<Buffer>,
715        buffer_offset: wgt::BufferAddress,
716        staging_buffer: StagingBuffer,
717    ) -> Result<(), QueueWriteError> {
718        profiling::scope!("Queue::write_staging_buffer");
719
720        self.device.check_is_valid()?;
721
722        let buffer = buffer.get()?;
723
724        // At this point, we have taken ownership of the staging_buffer from the
725        // user. Platform validation requires that the staging buffer always
726        // be freed, even if an error occurs. All paths from here must call
727        // `device.pending_writes.consume`.
728        let staging_buffer = staging_buffer.flush();
729
730        let snatch_guard = self.device.snatchable_lock.read();
731        let mut pending_writes = self.pending_writes.lock();
732
733        let result = self.write_staging_buffer_impl(
734            &snatch_guard,
735            &mut pending_writes,
736            &staging_buffer,
737            buffer,
738            buffer_offset,
739        );
740
741        drop(snatch_guard);
742
743        pending_writes.consume(staging_buffer);
744
745        drop(pending_writes);
746
747        result
748    }
749
750    pub fn validate_write_buffer(
751        &self,
752        buffer: Fallible<Buffer>,
753        buffer_offset: u64,
754        buffer_size: wgt::BufferSize,
755    ) -> Result<(), QueueWriteError> {
756        profiling::scope!("Queue::validate_write_buffer");
757
758        self.device.check_is_valid()?;
759
760        let buffer = buffer.get()?;
761
762        self.validate_write_buffer_impl(&buffer, buffer_offset, buffer_size.into())?;
763
764        Ok(())
765    }
766
767    fn validate_write_buffer_impl(
768        &self,
769        buffer: &Buffer,
770        buffer_offset: u64,
771        buffer_size: u64,
772    ) -> Result<(), TransferError> {
773        if !matches!(&*buffer.map_state.lock(), BufferMapState::Idle) {
774            return Err(TransferError::BufferNotAvailable);
775        }
776        buffer.check_usage(wgt::BufferUsages::COPY_DST)?;
777        if !buffer_size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
778            return Err(TransferError::UnalignedCopySize(buffer_size));
779        }
780        if !buffer_offset.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
781            return Err(TransferError::UnalignedBufferOffset(buffer_offset));
782        }
783
784        if buffer_offset > buffer.size {
785            return Err(TransferError::BufferStartOffsetOverrun {
786                start_offset: buffer_offset,
787                buffer_size: buffer.size,
788                side: CopySide::Destination,
789            });
790        }
791        if buffer_size > buffer.size - buffer_offset {
792            return Err(TransferError::BufferEndOffsetOverrun {
793                start_offset: buffer_offset,
794                size: buffer_size,
795                buffer_size: buffer.size,
796                side: CopySide::Destination,
797            });
798        }
799
800        Ok(())
801    }
802
803    fn write_staging_buffer_impl(
804        &self,
805        snatch_guard: &SnatchGuard,
806        pending_writes: &mut PendingWrites,
807        staging_buffer: &FlushedStagingBuffer,
808        buffer: Arc<Buffer>,
809        buffer_offset: u64,
810    ) -> Result<(), QueueWriteError> {
811        self.device.check_is_valid()?;
812
813        let transition = {
814            let mut trackers = self.device.trackers.lock();
815            trackers
816                .buffers
817                .set_single(&buffer, wgt::BufferUses::COPY_DST)
818        };
819
820        let dst_raw = buffer.try_raw(snatch_guard)?;
821
822        self.same_device_as(buffer.as_ref())?;
823
824        self.validate_write_buffer_impl(&buffer, buffer_offset, staging_buffer.size.into())?;
825
826        let region = hal::BufferCopy {
827            src_offset: 0,
828            dst_offset: buffer_offset,
829            size: staging_buffer.size,
830        };
831        let barriers = iter::once(hal::BufferBarrier {
832            buffer: staging_buffer.raw(),
833            usage: hal::StateTransition {
834                from: wgt::BufferUses::MAP_WRITE,
835                to: wgt::BufferUses::COPY_SRC,
836            },
837        })
838        .chain(transition.map(|pending| pending.into_hal(&buffer, snatch_guard)))
839        .collect::<Vec<_>>();
840        let encoder = pending_writes.activate();
841        unsafe {
842            encoder.transition_buffers(&barriers);
843            encoder.copy_buffer_to_buffer(staging_buffer.raw(), dst_raw, &[region]);
844        }
845
846        pending_writes.insert_buffer(&buffer);
847
848        // Ensure the overwritten bytes are marked as initialized so
849        // they don't need to be nulled prior to mapping or binding.
850        {
851            buffer
852                .initialization_status
853                .write()
854                .drain(buffer_offset..(buffer_offset + staging_buffer.size.get()));
855        }
856
857        Ok(())
858    }
859
860    pub fn write_texture(
861        &self,
862        destination: wgt::TexelCopyTextureInfo<Arc<Texture>>,
863        data: &[u8],
864        data_layout: &wgt::TexelCopyBufferLayout,
865        size: &wgt::Extent3d,
866    ) -> Result<(), QueueWriteError> {
867        profiling::scope!("Queue::write_texture");
868        api_log!("Queue::write_texture");
869
870        self.device.check_is_valid()?;
871
872        let dst = destination.texture;
873        let destination = wgt::TexelCopyTextureInfo {
874            texture: (),
875            mip_level: destination.mip_level,
876            origin: destination.origin,
877            aspect: destination.aspect,
878        };
879
880        self.same_device_as(dst.as_ref())?;
881
882        dst.check_usage(wgt::TextureUsages::COPY_DST)
883            .map_err(TransferError::MissingTextureUsage)?;
884
885        // Note: Doing the copy range validation early is important because ensures that the
886        // dimensions are not going to cause overflow in other parts of the validation.
887        let (hal_copy_size, array_layer_count) =
888            validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, size)?;
889
890        let (selector, dst_base) = extract_texture_selector(&destination, size, &dst)?;
891
892        validate_texture_copy_dst_format(dst.desc.format, destination.aspect)?;
893
894        validate_texture_buffer_copy(
895            &destination,
896            dst_base.aspect,
897            &dst.desc,
898            data_layout,
899            false, // alignment not required for buffer offset or bytes per row
900        )?;
901
902        // Note: `_source_bytes_per_array_layer` is ignored since we
903        // have a staging copy, and it can have a different value.
904        let (required_bytes_in_copy, _source_bytes_per_array_layer, _) =
905            validate_linear_texture_data(
906                data_layout,
907                dst.desc.format,
908                destination.aspect,
909                data.len() as wgt::BufferAddress,
910                CopySide::Source,
911                size,
912            )?;
913
914        if dst.desc.format.is_depth_stencil_format() {
915            self.device
916                .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
917                .map_err(TransferError::from)?;
918        }
919
920        let snatch_guard = self.device.snatchable_lock.read();
921
922        let dst_raw = dst.try_inner(&snatch_guard)?.raw();
923
924        // This must happen after parameter validation (so that errors are reported
925        // as required by the spec), but before any side effects.
926        if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
927            log::trace!("Ignoring write_texture of size 0");
928            return Ok(());
929        }
930
931        let mut pending_writes = self.pending_writes.lock();
932        let encoder = pending_writes.activate();
933
934        // If the copy does not fully cover the layers, we need to initialize to
935        // zero *first* as we don't keep track of partial texture layer inits.
936        //
937        // Strictly speaking we only need to clear the areas of a layer
938        // untouched, but this would get increasingly messy.
939        let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
940            // volume textures don't have a layer range as array volumes aren't supported
941            0..1
942        } else {
943            destination.origin.z..destination.origin.z + size.depth_or_array_layers
944        };
945        let mut dst_initialization_status = dst.initialization_status.write();
946        if dst_initialization_status.mips[destination.mip_level as usize]
947            .check(init_layer_range.clone())
948            .is_some()
949        {
950            if has_copy_partial_init_tracker_coverage(size, &destination, &dst.desc) {
951                for layer_range in dst_initialization_status.mips[destination.mip_level as usize]
952                    .drain(init_layer_range)
953                    .collect::<Vec<core::ops::Range<u32>>>()
954                {
955                    let mut trackers = self.device.trackers.lock();
956                    crate::command::clear_texture(
957                        &dst,
958                        TextureInitRange {
959                            mip_range: destination.mip_level..(destination.mip_level + 1),
960                            layer_range,
961                        },
962                        encoder,
963                        &mut trackers.textures,
964                        &self.device.alignments,
965                        self.device.zero_buffer.as_ref(),
966                        &snatch_guard,
967                        self.device.instance_flags,
968                    )
969                    .map_err(QueueWriteError::from)?;
970                }
971            } else {
972                dst_initialization_status.mips[destination.mip_level as usize]
973                    .drain(init_layer_range);
974            }
975        }
976
977        let (block_width, block_height) = dst.desc.format.block_dimensions();
978        let width_in_blocks = size.width / block_width;
979        let height_in_blocks = size.height / block_height;
980
981        let block_size = dst
982            .desc
983            .format
984            .block_copy_size(Some(destination.aspect))
985            .unwrap();
986        let bytes_in_last_row = width_in_blocks * block_size;
987
988        let bytes_per_row = data_layout.bytes_per_row.unwrap_or(bytes_in_last_row);
989        let rows_per_image = data_layout.rows_per_image.unwrap_or(height_in_blocks);
990
991        let bytes_per_row_alignment = get_lowest_common_denom(
992            self.device.alignments.buffer_copy_pitch.get() as u32,
993            block_size,
994        );
995        assert!(u32::MAX - bytes_in_last_row >= bytes_per_row_alignment);
996        let stage_bytes_per_row = wgt::math::align_to(bytes_in_last_row, bytes_per_row_alignment);
997
998        // Platform validation requires that the staging buffer always be
999        // freed, even if an error occurs. All paths from here must call
1000        // `device.pending_writes.consume`.
1001        let staging_buffer = if stage_bytes_per_row == bytes_per_row {
1002            profiling::scope!("copy aligned");
1003            // Fast path if the data is already being aligned optimally.
1004            let stage_size = wgt::BufferSize::new(required_bytes_in_copy).unwrap();
1005            let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?;
1006            staging_buffer.write(&data[data_layout.offset as usize..]);
1007            staging_buffer
1008        } else {
1009            profiling::scope!("copy chunked");
1010            // Copy row by row into the optimal alignment.
1011            let block_rows_in_copy = u64::from(size.depth_or_array_layers - 1)
1012                * u64::from(rows_per_image)
1013                + u64::from(height_in_blocks);
1014            // The copy size was validated against the source buffer, however,
1015            // `stage_bytes_per_row` can differ, so let's be paranoid.
1016            let stage_size = u64::from(stage_bytes_per_row)
1017                .checked_mul(block_rows_in_copy)
1018                .and_then(wgt::BufferSize::new)
1019                .unwrap();
1020            let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?;
1021            for layer in 0..u64::from(size.depth_or_array_layers) {
1022                let rows_offset = layer * u64::from(rows_per_image);
1023                for row in rows_offset..rows_offset + u64::from(height_in_blocks) {
1024                    let src_offset = data_layout.offset + row * u64::from(bytes_per_row);
1025                    let dst_offset = row * u64::from(stage_bytes_per_row);
1026                    unsafe {
1027                        staging_buffer.write_with_offset(
1028                            data,
1029                            src_offset as isize,
1030                            dst_offset as isize,
1031                            bytes_in_last_row as usize,
1032                        )
1033                    }
1034                }
1035            }
1036            staging_buffer
1037        };
1038
1039        let staging_buffer = staging_buffer.flush();
1040
1041        let regions = (0..array_layer_count)
1042            .map(|array_layer_offset| {
1043                let mut texture_base = dst_base.clone();
1044                texture_base.array_layer += array_layer_offset;
1045                hal::BufferTextureCopy {
1046                    buffer_layout: wgt::TexelCopyBufferLayout {
1047                        offset: array_layer_offset as u64
1048                            * rows_per_image as u64
1049                            * stage_bytes_per_row as u64,
1050                        bytes_per_row: Some(stage_bytes_per_row),
1051                        rows_per_image: Some(rows_per_image),
1052                    },
1053                    texture_base,
1054                    size: hal_copy_size,
1055                }
1056            })
1057            .collect::<Vec<_>>();
1058
1059        {
1060            let buffer_barrier = hal::BufferBarrier {
1061                buffer: staging_buffer.raw(),
1062                usage: hal::StateTransition {
1063                    from: wgt::BufferUses::MAP_WRITE,
1064                    to: wgt::BufferUses::COPY_SRC,
1065                },
1066            };
1067
1068            let mut trackers = self.device.trackers.lock();
1069            let transition =
1070                trackers
1071                    .textures
1072                    .set_single(&dst, selector, wgt::TextureUses::COPY_DST);
1073            let texture_barriers = transition
1074                .map(|pending| pending.into_hal(dst_raw))
1075                .collect::<Vec<_>>();
1076
1077            unsafe {
1078                encoder.transition_textures(&texture_barriers);
1079                encoder.transition_buffers(&[buffer_barrier]);
1080                encoder.copy_buffer_to_texture(staging_buffer.raw(), dst_raw, &regions);
1081            }
1082        }
1083
1084        pending_writes.consume(staging_buffer);
1085        pending_writes.insert_texture(&dst);
1086
1087        Ok(())
1088    }
1089
1090    #[cfg(webgl)]
1091    pub fn copy_external_image_to_texture(
1092        &self,
1093        source: &wgt::CopyExternalImageSourceInfo,
1094        destination: wgt::CopyExternalImageDestInfo<Arc<Texture>>,
1095        size: wgt::Extent3d,
1096    ) -> Result<(), QueueWriteError> {
1097        use crate::conv;
1098
1099        profiling::scope!("Queue::copy_external_image_to_texture");
1100
1101        self.device.check_is_valid()?;
1102
1103        let mut needs_flag = false;
1104        needs_flag |= matches!(source.source, wgt::ExternalImageSource::OffscreenCanvas(_));
1105        needs_flag |= source.origin != wgt::Origin2d::ZERO;
1106        needs_flag |= destination.color_space != wgt::PredefinedColorSpace::Srgb;
1107        #[allow(clippy::bool_comparison)]
1108        if matches!(source.source, wgt::ExternalImageSource::ImageBitmap(_)) {
1109            needs_flag |= source.flip_y != false;
1110            needs_flag |= destination.premultiplied_alpha != false;
1111        }
1112
1113        if needs_flag {
1114            self.device
1115                .require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_EXTERNAL_TEXTURE_COPIES)
1116                .map_err(TransferError::from)?;
1117        }
1118
1119        let src_width = source.source.width();
1120        let src_height = source.source.height();
1121
1122        let dst = destination.texture;
1123        let premultiplied_alpha = destination.premultiplied_alpha;
1124        let destination = wgt::TexelCopyTextureInfo {
1125            texture: (),
1126            mip_level: destination.mip_level,
1127            origin: destination.origin,
1128            aspect: destination.aspect,
1129        };
1130
1131        if !conv::is_valid_external_image_copy_dst_texture_format(dst.desc.format) {
1132            return Err(
1133                TransferError::ExternalCopyToForbiddenTextureFormat(dst.desc.format).into(),
1134            );
1135        }
1136        if dst.desc.dimension != wgt::TextureDimension::D2 {
1137            return Err(TransferError::InvalidDimensionExternal.into());
1138        }
1139        dst.check_usage(wgt::TextureUsages::COPY_DST | wgt::TextureUsages::RENDER_ATTACHMENT)
1140            .map_err(TransferError::MissingTextureUsage)?;
1141        if dst.desc.sample_count != 1 {
1142            return Err(TransferError::InvalidSampleCount {
1143                sample_count: dst.desc.sample_count,
1144            }
1145            .into());
1146        }
1147
1148        if source.origin.x > src_width || src_width - source.origin.x < size.width {
1149            return Err(TransferError::TextureOverrun {
1150                start_offset: source.origin.x,
1151                end_offset: source.origin.x.saturating_add(size.width),
1152                texture_size: src_width,
1153                dimension: crate::resource::TextureErrorDimension::X,
1154                side: CopySide::Source,
1155            }
1156            .into());
1157        }
1158        if source.origin.y > src_height || src_height - source.origin.y < size.height {
1159            return Err(TransferError::TextureOverrun {
1160                start_offset: source.origin.y,
1161                end_offset: source.origin.y.saturating_add(size.height),
1162                texture_size: src_height,
1163                dimension: crate::resource::TextureErrorDimension::Y,
1164                side: CopySide::Source,
1165            }
1166            .into());
1167        }
1168        if size.depth_or_array_layers != 1 {
1169            return Err(TransferError::TextureOverrun {
1170                start_offset: 0,
1171                end_offset: size.depth_or_array_layers,
1172                texture_size: 1,
1173                dimension: crate::resource::TextureErrorDimension::Z,
1174                side: CopySide::Source,
1175            }
1176            .into());
1177        }
1178
1179        // Note: Doing the copy range validation early is important because ensures that the
1180        // dimensions are not going to cause overflow in other parts of the validation.
1181        let (hal_copy_size, _) =
1182            validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, &size)?;
1183
1184        let (selector, dst_base) = extract_texture_selector(&destination, &size, &dst)?;
1185
1186        // This must happen after parameter validation (so that errors are reported
1187        // as required by the spec), but before any side effects.
1188        if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
1189            log::trace!("Ignoring copy_external_image_to_texture of size 0");
1190            return Ok(());
1191        }
1192
1193        let mut pending_writes = self.pending_writes.lock();
1194        let encoder = pending_writes.activate();
1195
1196        // If the copy does not fully cover the layers, we need to initialize to
1197        // zero *first* as we don't keep track of partial texture layer inits.
1198        //
1199        // Strictly speaking we only need to clear the areas of a layer
1200        // untouched, but this would get increasingly messy.
1201        let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
1202            // volume textures don't have a layer range as array volumes aren't supported
1203            0..1
1204        } else {
1205            destination.origin.z..destination.origin.z + size.depth_or_array_layers
1206        };
1207        let mut dst_initialization_status = dst.initialization_status.write();
1208        if dst_initialization_status.mips[destination.mip_level as usize]
1209            .check(init_layer_range.clone())
1210            .is_some()
1211        {
1212            if has_copy_partial_init_tracker_coverage(&size, &destination, &dst.desc) {
1213                for layer_range in dst_initialization_status.mips[destination.mip_level as usize]
1214                    .drain(init_layer_range)
1215                    .collect::<Vec<core::ops::Range<u32>>>()
1216                {
1217                    let mut trackers = self.device.trackers.lock();
1218                    crate::command::clear_texture(
1219                        &dst,
1220                        TextureInitRange {
1221                            mip_range: destination.mip_level..(destination.mip_level + 1),
1222                            layer_range,
1223                        },
1224                        encoder,
1225                        &mut trackers.textures,
1226                        &self.device.alignments,
1227                        self.device.zero_buffer.as_ref(),
1228                        &self.device.snatchable_lock.read(),
1229                        self.device.instance_flags,
1230                    )
1231                    .map_err(QueueWriteError::from)?;
1232                }
1233            } else {
1234                dst_initialization_status.mips[destination.mip_level as usize]
1235                    .drain(init_layer_range);
1236            }
1237        }
1238
1239        let snatch_guard = self.device.snatchable_lock.read();
1240        let dst_raw = dst.try_raw(&snatch_guard)?;
1241
1242        let regions = hal::TextureCopy {
1243            src_base: hal::TextureCopyBase {
1244                mip_level: 0,
1245                array_layer: 0,
1246                origin: source.origin.to_3d(0),
1247                aspect: hal::FormatAspects::COLOR,
1248            },
1249            dst_base,
1250            size: hal_copy_size,
1251        };
1252
1253        let mut trackers = self.device.trackers.lock();
1254        let transitions = trackers
1255            .textures
1256            .set_single(&dst, selector, wgt::TextureUses::COPY_DST);
1257
1258        // `copy_external_image_to_texture` is exclusive to the WebGL backend.
1259        // Don't go through the `DynCommandEncoder` abstraction and directly to the WebGL backend.
1260        let encoder_webgl = encoder
1261            .as_any_mut()
1262            .downcast_mut::<hal::gles::CommandEncoder>()
1263            .unwrap();
1264        let dst_raw_webgl = dst_raw
1265            .as_any()
1266            .downcast_ref::<hal::gles::Texture>()
1267            .unwrap();
1268        let transitions_webgl = transitions.map(|pending| {
1269            let dyn_transition = pending.into_hal(dst_raw);
1270            hal::TextureBarrier {
1271                texture: dst_raw_webgl,
1272                range: dyn_transition.range,
1273                usage: dyn_transition.usage,
1274            }
1275        });
1276
1277        use hal::CommandEncoder as _;
1278        unsafe {
1279            encoder_webgl.transition_textures(transitions_webgl);
1280            encoder_webgl.copy_external_image_to_texture(
1281                source,
1282                dst_raw_webgl,
1283                premultiplied_alpha,
1284                iter::once(regions),
1285            );
1286        }
1287
1288        pending_writes.insert_texture(&dst);
1289
1290        Ok(())
1291    }
1292
1293    /// Flush `PendingWrites` if it contains a write to `buffer`.
1294    pub fn flush_writes_for_buffer(
1295        &self,
1296        buffer: &Arc<Buffer>,
1297        snatch_guard: SnatchGuard,
1298    ) -> Result<(), BufferAccessError> {
1299        let submission = self
1300            .allocate_submission(snatch_guard)
1301            .map_err(|(_index, e)| e)?;
1302
1303        let pending_writes = self.pending_writes.lock();
1304        if !pending_writes.contains_buffer(buffer) {
1305            return Ok(());
1306        }
1307
1308        submission.submit(pending_writes)?;
1309
1310        Ok(())
1311    }
1312
1313    fn flush_pending_writes(&self) -> Result<Option<SubmissionIndex>, DeviceError> {
1314        let snatch_guard = self.device.snatchable_lock.read();
1315        let submission = self
1316            .allocate_submission(snatch_guard)
1317            .map_err(|(_index, e)| e)?;
1318        let submit_index = submission.index;
1319        let pending_writes = self.pending_writes.lock();
1320        if pending_writes.is_recording {
1321            submission.submit(pending_writes)?;
1322            Ok(Some(submit_index))
1323        } else {
1324            Ok(None)
1325        }
1326    }
1327
1328    #[cfg(feature = "trace")]
1329    fn trace_submission(
1330        &self,
1331        submit_index: SubmissionIndex,
1332        commands: Vec<crate::command::Command<crate::command::PointerReferences>>,
1333    ) {
1334        if let Some(ref mut trace) = *self.device.trace.lock() {
1335            trace.add(Action::Submit(submit_index, commands));
1336        }
1337    }
1338
1339    #[cfg(feature = "trace")]
1340    fn trace_failed_submission(
1341        &self,
1342        submit_index: SubmissionIndex,
1343        commands: Option<Vec<crate::command::Command<crate::command::PointerReferences>>>,
1344        error: alloc::string::String,
1345    ) {
1346        if let Some(ref mut trace) = *self.device.trace.lock() {
1347            trace.add(Action::FailedCommands {
1348                commands,
1349                failed_at_submit: Some(submit_index),
1350                error,
1351            });
1352        }
1353    }
1354
1355    pub fn submit(
1356        &self,
1357        command_buffers: &[Arc<CommandBuffer>],
1358    ) -> Result<SubmissionIndex, (SubmissionIndex, QueueSubmitError)> {
1359        profiling::scope!("Queue::submit");
1360        api_log!("Queue::submit");
1361
1362        let snatch_guard = self.device.snatchable_lock.read();
1363        let mut submission = self
1364            .allocate_submission(snatch_guard)
1365            .map_err(|(index, e)| (index, e.into()))?;
1366        let submit_index = submission.index;
1367
1368        let res = 'error: {
1369            let mut used_surface_textures = track::TextureUsageScope::default();
1370
1371            {
1372                if !command_buffers.is_empty() {
1373                    profiling::scope!("prepare");
1374
1375                    let mut first_error = None;
1376
1377                    //TODO: if multiple command buffers are submitted, we can re-use the last
1378                    // native command buffer of the previous chain instead of always creating
1379                    // a temporary one, since the chains are not finished.
1380
1381                    // finish all the command buffers first
1382                    for command_buffer in command_buffers {
1383                        profiling::scope!("process command buffer");
1384
1385                        // we reset the used surface textures every time we use
1386                        // it, so make sure to set_size on it.
1387                        used_surface_textures.set_size(self.device.tracker_indices.textures.size());
1388
1389                        // Note that we are required to invalidate all command buffers in both the success and failure paths.
1390                        // This is why we `continue` and don't early return via `?`.
1391                        #[allow(unused_mut)]
1392                        let mut cmd_buf_data = command_buffer.take_finished();
1393
1394                        if first_error.is_some() {
1395                            continue;
1396                        }
1397
1398                        #[cfg(feature = "trace")]
1399                        let trace_commands = cmd_buf_data
1400                            .as_mut()
1401                            .ok()
1402                            .and_then(|data| mem::take(&mut data.trace_commands));
1403
1404                        let mut baked = match cmd_buf_data {
1405                            Ok(cmd_buf_data) => {
1406                                let res = validate_command_buffer(
1407                                    command_buffer,
1408                                    self,
1409                                    &cmd_buf_data,
1410                                    &submission.snatch_guard,
1411                                    &mut submission.surface_textures,
1412                                    &mut used_surface_textures,
1413                                    &mut submission.command_index_guard,
1414                                );
1415                                if let Err(err) = res {
1416                                    #[cfg(feature = "trace")]
1417                                    self.trace_failed_submission(
1418                                        submit_index,
1419                                        trace_commands,
1420                                        err.to_string(),
1421                                    );
1422                                    first_error.get_or_insert(err);
1423                                    continue;
1424                                }
1425
1426                                #[cfg(feature = "trace")]
1427                                if let Some(commands) = trace_commands {
1428                                    self.trace_submission(submit_index, commands);
1429                                }
1430
1431                                cmd_buf_data.set_acceleration_structure_dependencies(
1432                                    &submission.snatch_guard,
1433                                );
1434                                cmd_buf_data.into_baked_commands()
1435                            }
1436                            Err(err) => {
1437                                #[cfg(feature = "trace")]
1438                                self.trace_failed_submission(
1439                                    submit_index,
1440                                    trace_commands,
1441                                    err.to_string(),
1442                                );
1443                                first_error.get_or_insert(err.into());
1444                                continue;
1445                            }
1446                        };
1447
1448                        if let Err(e) = baked.process_deferred_query_set_resolves(
1449                            &self.device,
1450                            &submission.snatch_guard,
1451                        ) {
1452                            break 'error Err(e.into());
1453                        }
1454
1455                        // execute resource transitions
1456                        if let Err(e) = baked.encoder.open_pass(hal_label(
1457                            Some("(wgpu internal) Transit"),
1458                            self.device.instance_flags,
1459                        )) {
1460                            break 'error Err(e.into());
1461                        }
1462
1463                        //Note: locking the trackers has to be done after the storages
1464                        let mut trackers = self.device.trackers.lock();
1465                        if let Err(e) =
1466                            baked.initialize_buffer_memory(&mut trackers, &submission.snatch_guard)
1467                        {
1468                            break 'error Err(e.into());
1469                        }
1470                        if let Err(e) = baked.initialize_texture_memory(
1471                            &mut trackers,
1472                            &self.device,
1473                            &submission.snatch_guard,
1474                        ) {
1475                            break 'error Err(e.into());
1476                        }
1477
1478                        //Note: stateless trackers are not merged:
1479                        // device already knows these resources exist.
1480                        CommandEncoder::insert_barriers_from_device_tracker(
1481                            baked.encoder.raw.as_mut(),
1482                            &mut trackers,
1483                            &baked.trackers,
1484                            &submission.snatch_guard,
1485                        );
1486
1487                        if let Err(e) = baked.encoder.close_and_push_front() {
1488                            break 'error Err(e.into());
1489                        }
1490
1491                        // Transition surface textures into `Present` state.
1492                        // Note: we could technically do it after all of the command buffers,
1493                        // but here we have a command encoder by hand, so it's easier to use it.
1494                        if !used_surface_textures.is_empty() {
1495                            if let Err(e) = baked.encoder.open_pass(hal_label(
1496                                Some("(wgpu internal) Present"),
1497                                self.device.instance_flags,
1498                            )) {
1499                                break 'error Err(e.into());
1500                            }
1501                            let texture_barriers = trackers
1502                                .textures
1503                                .set_from_usage_scope_and_drain_transitions(
1504                                    &used_surface_textures,
1505                                    &submission.snatch_guard,
1506                                )
1507                                .collect::<Vec<_>>();
1508                            unsafe {
1509                                baked.encoder.raw.transition_textures(&texture_barriers);
1510                            };
1511                            if let Err(e) = baked.encoder.close() {
1512                                break 'error Err(e.into());
1513                            }
1514                            used_surface_textures = track::TextureUsageScope::default();
1515                        }
1516
1517                        // done
1518                        submission.executions.push(EncoderInFlight {
1519                            inner: baked.encoder,
1520                            trackers: baked.trackers,
1521                            temp_resources: baked.temp_resources,
1522                            _indirect_draw_validation_resources: baked
1523                                .indirect_draw_validation_resources,
1524                            pending_buffers: FastHashMap::default(),
1525                            pending_textures: FastHashMap::default(),
1526                            pending_blas_s: FastHashMap::default(),
1527                        });
1528                    }
1529
1530                    if let Some(first_error) = first_error {
1531                        break 'error Err(first_error);
1532                    }
1533                }
1534            }
1535
1536            let pending_writes = self.pending_writes.lock();
1537
1538            let SubmissionResult { snatch_guard } = match submission.submit(pending_writes) {
1539                Ok(result) => result,
1540                Err(e) => break 'error Err(e.into()),
1541            };
1542
1543            profiling::scope!("cleanup");
1544
1545            // This will schedule destruction of all resources that are no longer needed
1546            // by the user but used in the command stream, among other things.
1547            // `device.maintain` consumes and will release the snatch guard.
1548            let (closures, result) = self.device.maintain(wgt::PollType::Poll, snatch_guard);
1549            match result {
1550                Ok(status) => {
1551                    debug_assert!(matches!(
1552                        status,
1553                        wgt::PollStatus::QueueEmpty | wgt::PollStatus::Poll
1554                    ));
1555                }
1556                Err(WaitIdleError::Device(err)) => break 'error Err(QueueSubmitError::Queue(err)),
1557                Err(WaitIdleError::WrongSubmissionIndex(..)) => {
1558                    unreachable!("Cannot get WrongSubmissionIndex from Poll")
1559                }
1560                Err(WaitIdleError::Timeout) => unreachable!("Cannot get Timeout from Poll"),
1561            };
1562
1563            Ok(closures)
1564        };
1565
1566        let callbacks = match res {
1567            Ok(ok) => ok,
1568            Err(e) => return Err((submit_index, e)),
1569        };
1570
1571        // the closures should execute with nothing locked!
1572        callbacks.fire();
1573
1574        self.device.lose_if_oom();
1575
1576        api_log!("Queue::submit returned submit index {submit_index}");
1577
1578        Ok(submit_index)
1579    }
1580
1581    /// Allocate a submission index and prepare for a submission.
1582    ///
1583    /// This is an internal API used in [`Queue::submit`] and other places within
1584    /// `wgpu-core` that need to submit work.
1585    ///
1586    /// Returns the index and a [`PendingSubmission`].
1587    ///
1588    /// The caller passes in the already-acquired [`SnatchGuard`]. This function acquires
1589    /// the fence lock and the command index lock.
1590    ///
1591    /// The caller should update [`PendingSubmission::executions`] with details of the
1592    /// submission.
1593    ///
1594    /// To finalize and submit the submission, call [`PendingSubmission::submit`] (which is
1595    /// a convenience wrapper around [`Queue::submit_pending_submission`]).
1596    ///
1597    /// After calling this function and before submitting, the caller must acquire the
1598    /// pending writes lock, and pass it to `submit`.
1599    ///
1600    /// It is also acceptable to drop the `PendingSubmission` without submitting. This may
1601    /// be necessary when locks are required to access the state that determines whether a
1602    /// submission is needed.
1603    fn allocate_submission<'a>(
1604        &'a self,
1605        snatch_guard: SnatchGuard<'a>,
1606    ) -> Result<PendingSubmission<'a>, (SubmissionIndex, DeviceError)> {
1607        let mut command_index_guard = self.device.command_indices.write();
1608        command_index_guard.active_submission_index += 1;
1609        let index = command_index_guard.active_submission_index;
1610
1611        if let Err(e) = self.device.check_is_valid() {
1612            return Err((index, e));
1613        }
1614
1615        let submission = PendingSubmission {
1616            queue: self,
1617            snatch_guard,
1618            command_index_guard,
1619            executions: Vec::new(),
1620            surface_textures: FastHashMap::default(),
1621            index,
1622        };
1623
1624        Ok(submission)
1625    }
1626
1627    /// Finalize and submit a [`PendingSubmission`] that was returned by
1628    /// [`Queue::allocate_submission`].
1629    ///
1630    /// This is an internal API used in `Queue::submit` and other places within
1631    /// `wgpu-core` that need to submit work. See [`Queue::allocate_submission`]
1632    /// for more details.
1633    ///
1634    /// This function:
1635    ///
1636    /// - Performs a HAL submission of the pending writes command
1637    ///   encoder and any other command encoders that were added to the
1638    ///   [`PendingSubmission`].
1639    /// - Advances `last_successful_submission_index` and registers the
1640    ///   submission with the lifetime tracker.
1641    /// - Returns a [`SubmissionResult`], which contains the snatch guard.
1642    fn submit_pending_submission<'a>(
1643        &self,
1644        mut pending_writes: MutexGuard<'_, PendingWrites>,
1645        prepared: PendingSubmission<'a>,
1646    ) -> Result<SubmissionResult<'a>, DeviceError> {
1647        let PendingSubmission {
1648            queue: _,
1649            snatch_guard,
1650            command_index_guard,
1651            mut executions,
1652            mut surface_textures,
1653            index: submit_index,
1654        } = prepared;
1655
1656        let mut used_surface_textures = track::TextureUsageScope::default();
1657        used_surface_textures.set_size(self.device.tracker_indices.textures.size());
1658        for texture in pending_writes.dst_textures.values() {
1659            match texture.try_inner(&snatch_guard) {
1660                Ok(TextureInner::Native { .. }) => {}
1661                Ok(TextureInner::Surface { .. }) => {
1662                    // Compare the Arcs by pointer as Textures don't implement Eq
1663                    surface_textures.insert(Arc::as_ptr(texture), texture.clone());
1664
1665                    unsafe {
1666                        used_surface_textures
1667                            .merge_single(texture, None, wgt::TextureUses::PRESENT)
1668                            .unwrap()
1669                    };
1670                }
1671                // The texture must not have been destroyed when its usage here was
1672                // encoded. If it was destroyed after that, then it was transferred
1673                // to `pending_writes.temp_resources` at the time of destruction, so
1674                // we are still okay to use it.
1675                Err(InvalidOrDestroyedResourceError::DestroyedResource(_)) => {}
1676                Err(InvalidOrDestroyedResourceError::InvalidResource(_)) => {
1677                    unreachable!()
1678                }
1679            }
1680        }
1681
1682        if !used_surface_textures.is_empty() {
1683            let mut trackers = self.device.trackers.lock();
1684
1685            let texture_barriers = trackers
1686                .textures
1687                .set_from_usage_scope_and_drain_transitions(&used_surface_textures, &snatch_guard)
1688                .collect::<Vec<_>>();
1689            unsafe {
1690                pending_writes
1691                    .command_encoder
1692                    .transition_textures(&texture_barriers);
1693            };
1694        }
1695
1696        match pending_writes.pre_submit(&self.device.command_allocator, &self.device, self) {
1697            Ok(Some(pending_execution)) => {
1698                executions.insert(0, pending_execution);
1699            }
1700            Ok(None) => {}
1701            Err(e) => return Err(e),
1702        }
1703        let hal_command_buffers = executions
1704            .iter()
1705            .flat_map(|e| e.inner.list.iter().map(|b| b.as_ref()))
1706            .collect::<Vec<_>>();
1707
1708        {
1709            let mut submit_surface_textures =
1710                SmallVec::<[&dyn hal::DynSurfaceTexture; 2]>::with_capacity(surface_textures.len());
1711            for texture in surface_textures.values() {
1712                let raw = match texture.inner.get(&snatch_guard).maybe_valid() {
1713                    Some(TextureInner::Surface { raw, .. }) => raw.as_ref(),
1714                    _ => unreachable!(),
1715                };
1716                submit_surface_textures.push(raw);
1717            }
1718
1719            unsafe {
1720                self.raw().submit(
1721                    &hal_command_buffers,
1722                    &submit_surface_textures,
1723                    (self.device.fence.as_ref(), submit_index),
1724                )
1725            }
1726            .map_err(|e| self.device.handle_hal_error(e))?;
1727
1728            // Submissions must have strictly increasing indices, so we must hold the
1729            // command index guard until we have submitted, to prevent another submission
1730            // from claiming the next index and reaching `submit` before we do.
1731            drop(pending_writes);
1732
1733            // Advance the successful submission index.
1734            self.device
1735                .last_successful_submission_index
1736                .fetch_max(submit_index, Ordering::SeqCst);
1737        }
1738
1739        // this will register the new submission to the life time tracker
1740        self.lock_life().track_submission(submit_index, executions);
1741
1742        // `device.maintain` relies on being able to prevent new submissions by
1743        // using `command_index_guard` while also checking whether there are
1744        // no tracked submissions to guarantee no new submissions will happen
1745        // after a device is lost. This requires `command_index_guard` to be
1746        // held over `self.lock_life()`
1747        drop(command_index_guard);
1748
1749        Ok(SubmissionResult { snatch_guard })
1750    }
1751
1752    pub fn get_timestamp_period(&self) -> f32 {
1753        unsafe { self.raw().get_timestamp_period() }
1754    }
1755
1756    /// `closure` is guaranteed to be called.
1757    pub fn on_submitted_work_done(
1758        &self,
1759        closure: SubmittedWorkDoneClosure,
1760    ) -> Option<SubmissionIndex> {
1761        api_log!("Queue::on_submitted_work_done");
1762
1763        // A `DeviceError` means we're losing the device anyways, so we can ignore it here
1764        // (mostly to avoid a breaking change to the `on_submitted_work_done` signature
1765        // for an error case that it is unlikely the caller will be able to handle).
1766        let _: Result<_, DeviceError> = self.flush_pending_writes();
1767
1768        self.lock_life().add_work_done_closure(closure)
1769    }
1770
1771    pub fn compact_blas(&self, blas: &Arc<Blas>) -> Result<Arc<Blas>, CompactBlasError> {
1772        profiling::scope!("Queue::compact_blas");
1773        api_log!("Queue::compact_blas");
1774
1775        let new_label = blas.label.clone() + " (compacted)";
1776
1777        self.device.check_is_valid()?;
1778        self.same_device_as(blas.as_ref())?;
1779
1780        let device = blas.device.clone();
1781
1782        let snatch_guard = device.snatchable_lock.read();
1783
1784        let BlasCompactState::Ready { size } = *blas.compacted_state.lock() else {
1785            return Err(CompactBlasError::BlasNotReady);
1786        };
1787
1788        let mut size_info = blas.size_info;
1789        size_info.acceleration_structure_size = size;
1790
1791        let mut pending_writes = self.pending_writes.lock();
1792        let cmd_buf_raw = pending_writes.activate();
1793
1794        let raw = unsafe {
1795            device
1796                .raw()
1797                .create_acceleration_structure(&hal::AccelerationStructureDescriptor {
1798                    label: hal_label(Some(&new_label), device.instance_flags),
1799                    size: size_info.acceleration_structure_size,
1800                    format: hal::AccelerationStructureFormat::BottomLevel,
1801                    allow_compaction: false,
1802                })
1803        }
1804        .map_err(DeviceError::from_hal)?;
1805
1806        let src_raw = blas.try_raw(&snatch_guard)?;
1807
1808        unsafe {
1809            cmd_buf_raw.copy_acceleration_structure_to_acceleration_structure(
1810                src_raw,
1811                raw.as_ref(),
1812                wgt::AccelerationStructureCopy::Compact,
1813            )
1814        };
1815
1816        let handle = unsafe {
1817            device
1818                .raw()
1819                .get_acceleration_structure_device_address(raw.as_ref())
1820        };
1821
1822        drop(snatch_guard);
1823
1824        let mut command_indices_lock = device.command_indices.write();
1825        command_indices_lock.next_acceleration_structure_build_command_index += 1;
1826        let built_index =
1827            NonZeroU64::new(command_indices_lock.next_acceleration_structure_build_command_index)
1828                .unwrap();
1829
1830        let new_blas = Arc::new(Blas {
1831            raw: Snatchable::new(raw),
1832            device: device.clone(),
1833            size_info,
1834            sizes: blas.sizes.clone(),
1835            flags: blas.flags & !AccelerationStructureFlags::ALLOW_COMPACTION,
1836            update_mode: blas.update_mode,
1837            // Bypass the submit checks which update this because we don't submit this normally.
1838            built_index: RwLock::new(rank::BLAS_BUILT_INDEX, Some(built_index)),
1839            handle,
1840            label: new_label,
1841            tracking_data: TrackingData::new(blas.device.tracker_indices.blas_s.clone()),
1842            compaction_buffer: None,
1843            compacted_state: Mutex::new(rank::BLAS_COMPACTION_STATE, BlasCompactState::Compacted),
1844        });
1845
1846        pending_writes.insert_blas(blas);
1847        pending_writes.insert_blas(&new_blas);
1848
1849        Ok(new_blas)
1850    }
1851}
1852
1853impl Global {
1854    pub fn queue_write_buffer(
1855        &self,
1856        queue_id: QueueId,
1857        buffer_id: id::BufferId,
1858        buffer_offset: wgt::BufferAddress,
1859        data: &[u8],
1860    ) -> Result<(), QueueWriteError> {
1861        let queue = self.hub.queues.get(queue_id);
1862        let buffer = self.hub.buffers.get(buffer_id).get()?;
1863
1864        #[cfg(feature = "trace")]
1865        if let Some(ref mut trace) = *queue.device.trace.lock() {
1866            use crate::device::trace::DataKind;
1867            let size = data.len() as u64;
1868            let data = trace.make_binary(DataKind::Bin, data);
1869            trace.add(Action::WriteBuffer {
1870                id: buffer.to_trace(),
1871                data,
1872                offset: buffer_offset,
1873                size,
1874                queued: true,
1875            });
1876        }
1877
1878        queue.write_buffer(buffer, buffer_offset, data)
1879    }
1880
1881    pub fn queue_create_staging_buffer(
1882        &self,
1883        queue_id: QueueId,
1884        buffer_size: wgt::BufferSize,
1885        id_in: Option<id::StagingBufferId>,
1886    ) -> Result<(id::StagingBufferId, NonNull<u8>), QueueWriteError> {
1887        let queue = self.hub.queues.get(queue_id);
1888        let (staging_buffer, ptr) = queue.create_staging_buffer(buffer_size)?;
1889
1890        let fid = self.hub.staging_buffers.prepare(id_in);
1891        let id = fid.assign(staging_buffer);
1892
1893        Ok((id, ptr))
1894    }
1895
1896    pub fn queue_write_staging_buffer(
1897        &self,
1898        queue_id: QueueId,
1899        buffer_id: id::BufferId,
1900        buffer_offset: wgt::BufferAddress,
1901        staging_buffer_id: id::StagingBufferId,
1902    ) -> Result<(), QueueWriteError> {
1903        let queue = self.hub.queues.get(queue_id);
1904        let buffer = self.hub.buffers.get(buffer_id);
1905        let staging_buffer = self.hub.staging_buffers.remove(staging_buffer_id);
1906        queue.write_staging_buffer(buffer, buffer_offset, staging_buffer)
1907    }
1908
1909    pub fn queue_validate_write_buffer(
1910        &self,
1911        queue_id: QueueId,
1912        buffer_id: id::BufferId,
1913        buffer_offset: u64,
1914        buffer_size: wgt::BufferSize,
1915    ) -> Result<(), QueueWriteError> {
1916        let queue = self.hub.queues.get(queue_id);
1917        let buffer = self.hub.buffers.get(buffer_id);
1918        queue.validate_write_buffer(buffer, buffer_offset, buffer_size)
1919    }
1920
1921    pub fn queue_write_texture(
1922        &self,
1923        queue_id: QueueId,
1924        destination: &wgt::TexelCopyTextureInfo<id::TextureId>,
1925        data: &[u8],
1926        data_layout: &wgt::TexelCopyBufferLayout,
1927        size: &wgt::Extent3d,
1928    ) -> Result<(), QueueWriteError> {
1929        let queue = self.hub.queues.get(queue_id);
1930        let texture = self.hub.textures.get(destination.texture);
1931        let destination = wgt::TexelCopyTextureInfo {
1932            texture,
1933            mip_level: destination.mip_level,
1934            origin: destination.origin,
1935            aspect: destination.aspect,
1936        };
1937
1938        #[cfg(feature = "trace")]
1939        if let Some(ref mut trace) = *queue.device.trace.lock() {
1940            use crate::device::trace::DataKind;
1941            let data = trace.make_binary(DataKind::Bin, data);
1942            trace.add(Action::WriteTexture {
1943                to: destination.to_trace(),
1944                data,
1945                layout: *data_layout,
1946                size: *size,
1947            });
1948        }
1949
1950        queue.write_texture(destination, data, data_layout, size)
1951    }
1952
1953    #[cfg(webgl)]
1954    pub fn queue_copy_external_image_to_texture(
1955        &self,
1956        queue_id: QueueId,
1957        source: &wgt::CopyExternalImageSourceInfo,
1958        destination: crate::command::CopyExternalImageDestInfo,
1959        size: wgt::Extent3d,
1960    ) -> Result<(), QueueWriteError> {
1961        let queue = self.hub.queues.get(queue_id);
1962        let destination = wgt::CopyExternalImageDestInfo {
1963            texture: self.hub.textures.get(destination.texture),
1964            mip_level: destination.mip_level,
1965            origin: destination.origin,
1966            aspect: destination.aspect,
1967            color_space: destination.color_space,
1968            premultiplied_alpha: destination.premultiplied_alpha,
1969        };
1970        queue.copy_external_image_to_texture(source, destination, size)
1971    }
1972
1973    pub fn queue_submit(
1974        &self,
1975        queue_id: QueueId,
1976        command_buffer_ids: &[id::CommandBufferId],
1977    ) -> Result<SubmissionIndex, (SubmissionIndex, QueueSubmitError)> {
1978        let queue = self.hub.queues.get(queue_id);
1979        let command_buffer_guard = self.hub.command_buffers.read();
1980        let command_buffers = command_buffer_ids
1981            .iter()
1982            .map(|id| command_buffer_guard.get(*id))
1983            .collect::<Vec<_>>();
1984        drop(command_buffer_guard);
1985        queue.submit(&command_buffers)
1986    }
1987
1988    pub fn queue_get_timestamp_period(&self, queue_id: QueueId) -> f32 {
1989        let queue = self.hub.queues.get(queue_id);
1990
1991        if queue.device.timestamp_normalizer.get().unwrap().enabled() {
1992            return 1.0;
1993        }
1994
1995        queue.get_timestamp_period()
1996    }
1997
1998    pub fn queue_on_submitted_work_done(
1999        &self,
2000        queue_id: QueueId,
2001        closure: SubmittedWorkDoneClosure,
2002    ) -> SubmissionIndex {
2003        api_log!("Queue::on_submitted_work_done {queue_id:?}");
2004
2005        let queue = self.hub.queues.get(queue_id);
2006        let result = queue.on_submitted_work_done(closure);
2007        result.unwrap_or(0) // '0' means no wait is necessary
2008    }
2009
2010    pub fn queue_compact_blas(
2011        &self,
2012        queue_id: QueueId,
2013        blas_id: BlasId,
2014        id_in: Option<BlasId>,
2015    ) -> (BlasId, Option<u64>, Option<CompactBlasError>) {
2016        api_log!("Queue::compact_blas {queue_id:?}, {blas_id:?}");
2017
2018        let fid = self.hub.blas_s.prepare(id_in);
2019
2020        let queue = self.hub.queues.get(queue_id);
2021        let blas = self.hub.blas_s.get(blas_id);
2022        let device = &queue.device;
2023
2024        // TODO: Tracing
2025
2026        let error = 'error: {
2027            match device.require_features(wgpu_types::Features::EXPERIMENTAL_RAY_QUERY) {
2028                Ok(_) => {}
2029                Err(err) => break 'error err.into(),
2030            }
2031
2032            let blas = match blas.get() {
2033                Ok(blas) => blas,
2034                Err(err) => break 'error err.into(),
2035            };
2036
2037            let new_blas = match queue.compact_blas(&blas) {
2038                Ok(blas) => blas,
2039                Err(err) => break 'error err,
2040            };
2041
2042            // We should have no more errors after this because we have marked the command encoder as successful.
2043            let old_blas_size = blas.size_info.acceleration_structure_size;
2044            let new_blas_size = new_blas.size_info.acceleration_structure_size;
2045            let handle = new_blas.handle;
2046
2047            let id = fid.assign(Fallible::Valid(new_blas));
2048
2049            api_log!("CommandEncoder::compact_blas {blas_id:?} (size: {old_blas_size}) -> {id:?} (size: {new_blas_size})");
2050
2051            return (id, Some(handle), None);
2052        };
2053
2054        let id = fid.assign(Fallible::Invalid(Arc::new(error.to_string())));
2055
2056        (id, None, Some(error))
2057    }
2058}
2059
2060fn validate_command_buffer(
2061    command_buffer: &CommandBuffer,
2062    queue: &Queue,
2063    cmd_buf_data: &crate::command::CommandBufferMutable,
2064    snatch_guard: &SnatchGuard,
2065    surface_textures: &mut FastHashMap<*const Texture, Arc<Texture>>,
2066    used_surface_textures: &mut track::TextureUsageScope,
2067    command_index_guard: &mut RwLockWriteGuard<CommandIndices>,
2068) -> Result<(), QueueSubmitError> {
2069    command_buffer.same_device_as(queue)?;
2070
2071    {
2072        profiling::scope!("check resource state");
2073
2074        {
2075            profiling::scope!("buffers");
2076            for buffer in cmd_buf_data.trackers.buffers.used_resources() {
2077                buffer.check_destroyed(snatch_guard)?;
2078
2079                match *buffer.map_state.lock() {
2080                    BufferMapState::Idle => (),
2081                    _ => return Err(QueueSubmitError::BufferStillMapped(buffer.error_ident())),
2082                }
2083            }
2084        }
2085        {
2086            profiling::scope!("textures");
2087            for texture in cmd_buf_data.trackers.textures.used_resources() {
2088                let should_extend = match texture.try_inner(snatch_guard)? {
2089                    TextureInner::Native { .. } => false,
2090                    TextureInner::Surface { .. } => {
2091                        // Compare the Arcs by pointer as Textures don't implement Eq.
2092                        surface_textures.insert(Arc::as_ptr(texture), texture.clone());
2093
2094                        true
2095                    }
2096                };
2097                if should_extend {
2098                    unsafe {
2099                        used_surface_textures
2100                            .merge_single(texture, None, wgt::TextureUses::PRESENT)
2101                            .unwrap();
2102                    };
2103                }
2104            }
2105        }
2106        {
2107            profiling::scope!("query sets");
2108            for query_set in cmd_buf_data.trackers.query_sets.used_resources() {
2109                query_set.try_raw(snatch_guard)?;
2110            }
2111        }
2112        // WebGPU requires that we check every bind group referenced during
2113        // encoding, even ones that may have been replaced before being used.
2114        // TODO(<https://github.com/gfx-rs/wgpu/issues/8510>): Optimize this.
2115        {
2116            profiling::scope!("bind groups");
2117            for bind_group in &cmd_buf_data.trackers.bind_groups {
2118                // This checks the bind group and all resources it references.
2119                bind_group.try_raw(snatch_guard)?;
2120            }
2121        }
2122
2123        if let Err(e) =
2124            cmd_buf_data.validate_acceleration_structure_actions(snatch_guard, command_index_guard)
2125        {
2126            return Err(e.into());
2127        }
2128    }
2129    Ok(())
2130}