1use alloc::{boxed::Box, string::ToString, sync::Arc, vec, vec::Vec};
2use core::{
3 iter,
4 mem::{self, ManuallyDrop},
5 num::NonZeroU64,
6 ptr::NonNull,
7 sync::atomic::Ordering,
8};
9use smallvec::SmallVec;
10use thiserror::Error;
11use wgt::{
12 error::{ErrorType, WebGpuError},
13 AccelerationStructureFlags,
14};
15
16use super::{life::LifetimeTracker, Device};
17#[cfg(feature = "trace")]
18use crate::device::trace::{Action, IntoTrace};
19use crate::{
20 api_log,
21 command::{
22 extract_texture_selector, validate_linear_texture_data, validate_texture_buffer_copy,
23 validate_texture_copy_dst_format, validate_texture_copy_range, ClearError,
24 CommandAllocator, CommandBuffer, CommandEncoder, CommandEncoderError, CopySide,
25 TransferError,
26 },
27 device::{DeviceError, WaitIdleError},
28 get_lowest_common_denom,
29 global::Global,
30 hal_label,
31 id::{self, BlasId, QueueId},
32 init_tracker::{has_copy_partial_init_tracker_coverage, TextureInitRange},
33 lock::{rank, Mutex, MutexGuard, RwLock, RwLockWriteGuard},
34 ray_tracing::{BlasCompactReadyPendingClosure, CompactBlasError},
35 resource::{
36 Blas, BlasCompactState, Buffer, BufferAccessError, BufferMapState, DestroyedBuffer,
37 DestroyedQuerySet, DestroyedResourceError, DestroyedTexture, Fallible,
38 FlushedStagingBuffer, InvalidOrDestroyedResourceError, InvalidResourceError, Labeled,
39 ParentDevice, ResourceErrorIdent, StagingBuffer, Texture, TextureInner, Trackable,
40 TrackingData,
41 },
42 resource_log,
43 scratch::ScratchBuffer,
44 snatch::{SnatchGuard, Snatchable},
45 track::{self, Tracker, TrackerIndex},
46 FastHashMap, SubmissionIndex,
47};
48use crate::{device::resource::CommandIndices, resource::RawResourceAccess};
49
50pub struct Queue {
51 raw: Box<dyn hal::DynQueue>,
52 pub(crate) pending_writes: Mutex<PendingWrites>,
53 life_tracker: Mutex<LifetimeTracker>,
54 pub(crate) device: Arc<Device>,
56}
57
58impl Queue {
59 pub(crate) fn new(
60 device: Arc<Device>,
61 raw: Box<dyn hal::DynQueue>,
62 instance_flags: wgt::InstanceFlags,
63 ) -> Result<Self, DeviceError> {
64 let pending_encoder = device
65 .command_allocator
66 .acquire_encoder(device.raw(), raw.as_ref())
67 .map_err(DeviceError::from_hal);
68
69 let pending_encoder = match pending_encoder {
70 Ok(pending_encoder) => pending_encoder,
71 Err(e) => {
72 return Err(e);
73 }
74 };
75
76 let mut pending_writes = PendingWrites::new(pending_encoder, instance_flags);
77
78 let zero_buffer = device.zero_buffer.as_ref();
79 pending_writes.activate();
80 unsafe {
81 pending_writes
82 .command_encoder
83 .transition_buffers(&[hal::BufferBarrier {
84 buffer: zero_buffer,
85 usage: hal::StateTransition {
86 from: wgt::BufferUses::empty(),
87 to: wgt::BufferUses::COPY_DST,
88 },
89 }]);
90 pending_writes
91 .command_encoder
92 .clear_buffer(zero_buffer, 0..super::ZERO_BUFFER_SIZE);
93 pending_writes
94 .command_encoder
95 .transition_buffers(&[hal::BufferBarrier {
96 buffer: zero_buffer,
97 usage: hal::StateTransition {
98 from: wgt::BufferUses::COPY_DST,
99 to: wgt::BufferUses::COPY_SRC,
100 },
101 }]);
102 }
103
104 Ok(Queue {
105 raw,
106 device,
107 pending_writes: Mutex::new(rank::QUEUE_PENDING_WRITES, pending_writes),
108 life_tracker: Mutex::new(rank::QUEUE_LIFE_TRACKER, LifetimeTracker::new()),
109 })
110 }
111
112 pub(crate) fn raw(&self) -> &dyn hal::DynQueue {
113 self.raw.as_ref()
114 }
115
116 #[track_caller]
117 pub(crate) fn lock_life<'a>(&'a self) -> MutexGuard<'a, LifetimeTracker> {
118 self.life_tracker.lock()
119 }
120
121 pub(crate) fn prepare_surface_texture_for_present(
126 &self,
127 texture: &Arc<Texture>,
128 ) -> Result<(), DeviceError> {
129 let snatch_guard = self.device.snatchable_lock.read();
130 let submission = self
131 .allocate_submission(snatch_guard)
132 .map_err(|(_index, e)| e)?;
133 let device = &self.device;
134
135 let needs_clear = {
137 let status = texture.initialization_status.read();
138 status
139 .mips
140 .first()
141 .is_some_and(|mip| mip.check(0..1).is_some())
142 };
143
144 let mut pending_writes = self.pending_writes.lock();
145
146 if needs_clear {
147 let encoder = pending_writes.activate();
150 let mut trackers = device.trackers.lock();
151 crate::command::clear_texture(
152 texture,
153 TextureInitRange {
154 mip_range: 0..1,
155 layer_range: 0..1,
156 },
157 encoder,
158 &mut trackers.textures,
159 &device.alignments,
160 device.zero_buffer.as_ref(),
161 &submission.snatch_guard,
162 device.instance_flags,
163 )
164 .map_err(|e| match e {
165 ClearError::Device(e) => e,
166 _ => DeviceError::Lost,
167 })?;
168 texture.initialization_status.write().mips[0].drain(0..1);
169 }
170
171 let pending = {
177 let mut trackers = device.trackers.lock();
178 let pending: Vec<track::PendingTransition<wgt::TextureUses>> = trackers
179 .textures
180 .set_single(
181 texture,
182 texture.full_range.clone(),
183 wgt::TextureUses::PRESENT,
184 )
185 .collect();
186 pending
187 };
188
189 let awaits_acquire = match texture.inner.get(&submission.snatch_guard).maybe_valid() {
193 Some(TextureInner::Surface { raw }) => raw.awaits_acquire(),
194 _ => false,
195 };
196 if pending.is_empty() && !awaits_acquire {
197 debug_assert!(!needs_clear);
203 return Ok(());
204 }
205
206 {
208 let raw_texture = texture
209 .raw(&submission.snatch_guard)
210 .ok_or(DeviceError::Lost)?;
211 let barriers: Vec<hal::TextureBarrier<'_, dyn hal::DynTexture>> = pending
212 .into_iter()
213 .map(|pt| pt.into_hal(raw_texture))
214 .collect();
215
216 let encoder = pending_writes.activate();
217 unsafe {
221 encoder.transition_textures(&barriers);
222 }
223 }
224
225 pending_writes.insert_texture(texture);
232
233 submission.submit(pending_writes)?;
234
235 Ok(())
236 }
237
238 pub(crate) fn maintain(
246 &self,
247 submission_index: u64,
248 snatch_guard: &SnatchGuard,
249 ) -> (
250 SmallVec<[SubmittedWorkDoneClosure; 1]>,
251 Vec<super::BufferMapPendingClosure>,
252 Vec<BlasCompactReadyPendingClosure>,
253 bool,
254 ) {
255 let mut life_tracker = self.lock_life();
256 let submission_closures = life_tracker.triage_submissions(submission_index);
257
258 let mapping_closures = life_tracker.handle_mapping(snatch_guard);
259 let blas_closures = life_tracker.handle_compact_read_back();
260
261 let queue_empty = life_tracker.queue_empty();
262
263 (
264 submission_closures,
265 mapping_closures,
266 blas_closures,
267 queue_empty,
268 )
269 }
270}
271
272crate::impl_resource_type!(Queue);
273impl Labeled for Queue {
275 fn label(&self) -> &str {
276 ""
277 }
278}
279crate::impl_parent_device!(Queue);
280crate::impl_storage_item!(Queue);
281
282impl Drop for Queue {
283 fn drop(&mut self) {
284 resource_log!("Drop {}", self.error_ident());
285
286 match unsafe { self.raw.wait_for_idle() } {
289 Ok(()) => {}
290 Err(hal::DeviceError::Lost) => {
291 self.device.handle_hal_error(hal::DeviceError::Lost);
292 }
293 Err(e) => {
294 panic!("Unexpected error while waiting for queue idle on drop: {e:?}");
295 }
296 }
297
298 let last_successful_submission_index = self
299 .device
300 .last_successful_submission_index
301 .load(Ordering::Acquire);
302
303 let snatch_guard = self.device.snatchable_lock.read();
304 let (submission_closures, mapping_closures, blas_compact_ready_closures, queue_empty) =
305 self.maintain(last_successful_submission_index, &snatch_guard);
306 drop(snatch_guard);
307
308 assert!(queue_empty);
309
310 let closures = crate::device::UserClosures {
311 mappings: mapping_closures,
312 blas_compact_ready: blas_compact_ready_closures,
313 submissions: submission_closures,
314 device_lost_invocations: SmallVec::new(),
315 };
316
317 closures.fire();
318 }
319}
320
321#[cfg(send_sync)]
322pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + Send + 'static>;
323#[cfg(not(send_sync))]
324pub type SubmittedWorkDoneClosure = Box<dyn FnOnce() + 'static>;
325
326#[derive(Debug)]
337pub enum TempResource {
338 StagingBuffer(FlushedStagingBuffer),
339 ScratchBuffer(ScratchBuffer),
340 DestroyedBuffer(DestroyedBuffer),
341 DestroyedTexture(DestroyedTexture),
342 DestroyedQuerySet(DestroyedQuerySet),
343}
344
345pub(crate) struct EncoderInFlight {
351 inner: crate::command::InnerCommandEncoder,
352 pub(crate) trackers: Tracker,
353 pub(crate) temp_resources: Vec<TempResource>,
354 _indirect_draw_validation_resources: crate::indirect_validation::DrawResources,
356
357 pub(crate) pending_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
359 pub(crate) pending_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
361 pub(crate) pending_blas_s: FastHashMap<TrackerIndex, Arc<Blas>>,
363}
364
365#[derive(Debug)]
390pub(crate) struct PendingWrites {
391 pub command_encoder: Box<dyn hal::DynCommandEncoder>,
393
394 pub is_recording: bool,
400
401 temp_resources: Vec<TempResource>,
402 dst_buffers: FastHashMap<TrackerIndex, Arc<Buffer>>,
403 dst_textures: FastHashMap<TrackerIndex, Arc<Texture>>,
404 copied_blas_s: FastHashMap<TrackerIndex, Arc<Blas>>,
405 instance_flags: wgt::InstanceFlags,
406}
407
408impl PendingWrites {
409 pub fn new(
410 command_encoder: Box<dyn hal::DynCommandEncoder>,
411 instance_flags: wgt::InstanceFlags,
412 ) -> Self {
413 Self {
414 command_encoder,
415 is_recording: false,
416 temp_resources: Vec::new(),
417 dst_buffers: FastHashMap::default(),
418 dst_textures: FastHashMap::default(),
419 copied_blas_s: FastHashMap::default(),
420 instance_flags,
421 }
422 }
423
424 pub fn insert_buffer(&mut self, buffer: &Arc<Buffer>) {
425 self.dst_buffers
426 .insert(buffer.tracker_index(), buffer.clone());
427 }
428
429 pub fn insert_texture(&mut self, texture: &Arc<Texture>) {
430 self.dst_textures
431 .insert(texture.tracker_index(), texture.clone());
432 }
433
434 pub fn insert_blas(&mut self, blas: &Arc<Blas>) {
435 self.copied_blas_s
436 .insert(blas.tracker_index(), blas.clone());
437 }
438
439 pub fn contains_buffer(&self, buffer: &Arc<Buffer>) -> bool {
440 self.dst_buffers.contains_key(&buffer.tracker_index())
441 }
442
443 pub fn contains_texture(&self, texture: &Arc<Texture>) -> bool {
444 self.dst_textures.contains_key(&texture.tracker_index())
445 }
446
447 pub fn consume_temp(&mut self, resource: TempResource) {
448 self.temp_resources.push(resource);
449 }
450
451 pub fn consume(&mut self, buffer: FlushedStagingBuffer) {
452 self.temp_resources
453 .push(TempResource::StagingBuffer(buffer));
454 }
455
456 fn pre_submit(
457 &mut self,
458 command_allocator: &CommandAllocator,
459 device: &Arc<Device>,
460 queue: &Queue,
461 ) -> Result<Option<EncoderInFlight>, DeviceError> {
462 if self.is_recording {
463 let pending_buffers = mem::take(&mut self.dst_buffers);
464 let pending_textures = mem::take(&mut self.dst_textures);
465 let pending_blas_s = mem::take(&mut self.copied_blas_s);
466
467 let cmd_buf = unsafe { self.command_encoder.end_encoding() }
468 .map_err(|e| device.handle_hal_error(e))?;
469 self.is_recording = false;
470
471 let new_encoder = command_allocator
472 .acquire_encoder(device.raw(), queue.raw())
473 .map_err(|e| device.handle_hal_error(e))?;
474
475 let encoder = EncoderInFlight {
476 inner: crate::command::InnerCommandEncoder {
477 raw: ManuallyDrop::new(mem::replace(&mut self.command_encoder, new_encoder)),
478 list: vec![cmd_buf],
479 device: device.clone(),
480 is_open: false,
481 api: crate::command::EncodingApi::InternalUse,
482 label: "(wgpu internal) PendingWrites command encoder".into(),
483 },
484 trackers: Tracker::new(device.ordered_buffer_usages, device.ordered_texture_usages),
485 temp_resources: mem::take(&mut self.temp_resources),
486 _indirect_draw_validation_resources: crate::indirect_validation::DrawResources::new(
487 device.clone(),
488 ),
489 pending_buffers,
490 pending_textures,
491 pending_blas_s,
492 };
493 Ok(Some(encoder))
494 } else {
495 self.dst_buffers.clear();
496 self.dst_textures.clear();
497 self.copied_blas_s.clear();
498 Ok(None)
499 }
500 }
501
502 pub fn activate(&mut self) -> &mut dyn hal::DynCommandEncoder {
503 if !self.is_recording {
504 unsafe {
505 self.command_encoder
506 .begin_encoding(hal_label(
507 Some("(wgpu internal) PendingWrites"),
508 self.instance_flags,
509 ))
510 .unwrap();
511 }
512 self.is_recording = true;
513 }
514 self.command_encoder.as_mut()
515 }
516}
517
518impl Drop for PendingWrites {
519 fn drop(&mut self) {
520 unsafe {
521 if self.is_recording {
522 self.command_encoder.discard_encoding();
523 }
524 }
525 }
526}
527
528#[derive(Clone, Debug, Error)]
529#[non_exhaustive]
530pub enum QueueWriteError {
531 #[error(transparent)]
532 Queue(#[from] DeviceError),
533 #[error(transparent)]
534 Transfer(#[from] TransferError),
535 #[error(transparent)]
536 MemoryInitFailure(#[from] ClearError),
537 #[error(transparent)]
538 DestroyedResource(#[from] DestroyedResourceError),
539 #[error(transparent)]
540 InvalidResource(#[from] InvalidResourceError),
541}
542
543impl From<InvalidOrDestroyedResourceError> for QueueWriteError {
544 fn from(e: InvalidOrDestroyedResourceError) -> Self {
545 match e {
546 InvalidOrDestroyedResourceError::InvalidResource(e) => Self::InvalidResource(e),
547 InvalidOrDestroyedResourceError::DestroyedResource(e) => Self::DestroyedResource(e),
548 }
549 }
550}
551
552impl WebGpuError for QueueWriteError {
553 fn webgpu_error_type(&self) -> ErrorType {
554 match self {
555 Self::Queue(e) => e.webgpu_error_type(),
556 Self::Transfer(e) => e.webgpu_error_type(),
557 Self::MemoryInitFailure(e) => e.webgpu_error_type(),
558 Self::DestroyedResource(e) => e.webgpu_error_type(),
559 Self::InvalidResource(e) => e.webgpu_error_type(),
560 }
561 }
562}
563
564#[derive(Clone, Debug, Error)]
565#[non_exhaustive]
566pub enum QueueSubmitError {
567 #[error(transparent)]
568 Queue(#[from] DeviceError),
569 #[error(transparent)]
570 DestroyedResource(#[from] DestroyedResourceError),
571 #[error("{0} is still mapped")]
572 BufferStillMapped(ResourceErrorIdent),
573 #[error(transparent)]
574 InvalidResource(#[from] InvalidResourceError),
575 #[error(transparent)]
576 CommandEncoder(#[from] CommandEncoderError),
577 #[error(transparent)]
578 ValidateAsActionsError(#[from] crate::ray_tracing::ValidateAsActionsError),
579}
580
581impl From<InvalidOrDestroyedResourceError> for QueueSubmitError {
582 fn from(e: InvalidOrDestroyedResourceError) -> Self {
583 match e {
584 InvalidOrDestroyedResourceError::InvalidResource(e) => Self::InvalidResource(e),
585 InvalidOrDestroyedResourceError::DestroyedResource(e) => Self::DestroyedResource(e),
586 }
587 }
588}
589
590impl WebGpuError for QueueSubmitError {
591 fn webgpu_error_type(&self) -> ErrorType {
592 match self {
593 Self::Queue(e) => e.webgpu_error_type(),
594 Self::CommandEncoder(e) => e.webgpu_error_type(),
595 Self::ValidateAsActionsError(e) => e.webgpu_error_type(),
596 Self::InvalidResource(e) => e.webgpu_error_type(),
597 Self::DestroyedResource(_) | Self::BufferStillMapped(_) => ErrorType::Validation,
598 }
599 }
600}
601
602pub(crate) struct PendingSubmission<'a> {
610 queue: &'a Queue,
611 snatch_guard: SnatchGuard<'a>,
612 command_index_guard: RwLockWriteGuard<'a, CommandIndices>,
613 pub executions: Vec<EncoderInFlight>,
615 surface_textures: FastHashMap<*const Texture, Arc<Texture>>,
619 pub index: SubmissionIndex,
620}
621
622pub(crate) struct SubmissionResult<'a> {
623 pub snatch_guard: SnatchGuard<'a>,
624}
625
626impl<'a> PendingSubmission<'a> {
627 fn submit(
628 self,
629 pending_writes: MutexGuard<'a, PendingWrites>,
630 ) -> Result<SubmissionResult<'a>, DeviceError> {
631 self.queue.submit_pending_submission(pending_writes, self)
632 }
633}
634
635impl Queue {
638 pub fn write_buffer(
639 &self,
640 buffer: Arc<Buffer>,
641 buffer_offset: wgt::BufferAddress,
642 data: &[u8],
643 ) -> Result<(), QueueWriteError> {
644 profiling::scope!("Queue::write_buffer");
645 api_log!("Queue::write_buffer");
646
647 self.device.check_is_valid()?;
648
649 let data_size = data.len() as wgt::BufferAddress;
650
651 self.same_device_as(buffer.as_ref())?;
652
653 let data_size = if let Some(data_size) = wgt::BufferSize::new(data_size) {
654 data_size
655 } else {
656 self.validate_write_buffer_impl(buffer.as_ref(), buffer_offset, 0)?;
661
662 log::trace!("Ignoring write_buffer of size 0");
663 return Ok(());
664 };
665
666 let mut staging_buffer = StagingBuffer::new(&self.device, data_size)?;
670
671 let staging_buffer = {
672 profiling::scope!("copy");
673 staging_buffer.write(data);
674 staging_buffer.flush()
675 };
676
677 let snatch_guard = self.device.snatchable_lock.read();
678 let mut pending_writes = self.pending_writes.lock();
679
680 let result = self.write_staging_buffer_impl(
681 &snatch_guard,
682 &mut pending_writes,
683 &staging_buffer,
684 buffer,
685 buffer_offset,
686 );
687
688 drop(snatch_guard);
689
690 pending_writes.consume(staging_buffer);
691
692 drop(pending_writes);
693
694 result
695 }
696
697 pub fn create_staging_buffer(
698 &self,
699 buffer_size: wgt::BufferSize,
700 ) -> Result<(StagingBuffer, NonNull<u8>), QueueWriteError> {
701 profiling::scope!("Queue::create_staging_buffer");
702 resource_log!("Queue::create_staging_buffer");
703
704 self.device.check_is_valid()?;
705
706 let staging_buffer = StagingBuffer::new(&self.device, buffer_size)?;
707 let ptr = unsafe { staging_buffer.ptr() };
708
709 Ok((staging_buffer, ptr))
710 }
711
712 pub fn write_staging_buffer(
713 &self,
714 buffer: Fallible<Buffer>,
715 buffer_offset: wgt::BufferAddress,
716 staging_buffer: StagingBuffer,
717 ) -> Result<(), QueueWriteError> {
718 profiling::scope!("Queue::write_staging_buffer");
719
720 self.device.check_is_valid()?;
721
722 let buffer = buffer.get()?;
723
724 let staging_buffer = staging_buffer.flush();
729
730 let snatch_guard = self.device.snatchable_lock.read();
731 let mut pending_writes = self.pending_writes.lock();
732
733 let result = self.write_staging_buffer_impl(
734 &snatch_guard,
735 &mut pending_writes,
736 &staging_buffer,
737 buffer,
738 buffer_offset,
739 );
740
741 drop(snatch_guard);
742
743 pending_writes.consume(staging_buffer);
744
745 drop(pending_writes);
746
747 result
748 }
749
750 pub fn validate_write_buffer(
751 &self,
752 buffer: Fallible<Buffer>,
753 buffer_offset: u64,
754 buffer_size: wgt::BufferSize,
755 ) -> Result<(), QueueWriteError> {
756 profiling::scope!("Queue::validate_write_buffer");
757
758 self.device.check_is_valid()?;
759
760 let buffer = buffer.get()?;
761
762 self.validate_write_buffer_impl(&buffer, buffer_offset, buffer_size.into())?;
763
764 Ok(())
765 }
766
767 fn validate_write_buffer_impl(
768 &self,
769 buffer: &Buffer,
770 buffer_offset: u64,
771 buffer_size: u64,
772 ) -> Result<(), TransferError> {
773 if !matches!(&*buffer.map_state.lock(), BufferMapState::Idle) {
774 return Err(TransferError::BufferNotAvailable);
775 }
776 buffer.check_usage(wgt::BufferUsages::COPY_DST)?;
777 if !buffer_size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
778 return Err(TransferError::UnalignedCopySize(buffer_size));
779 }
780 if !buffer_offset.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
781 return Err(TransferError::UnalignedBufferOffset(buffer_offset));
782 }
783
784 if buffer_offset > buffer.size {
785 return Err(TransferError::BufferStartOffsetOverrun {
786 start_offset: buffer_offset,
787 buffer_size: buffer.size,
788 side: CopySide::Destination,
789 });
790 }
791 if buffer_size > buffer.size - buffer_offset {
792 return Err(TransferError::BufferEndOffsetOverrun {
793 start_offset: buffer_offset,
794 size: buffer_size,
795 buffer_size: buffer.size,
796 side: CopySide::Destination,
797 });
798 }
799
800 Ok(())
801 }
802
803 fn write_staging_buffer_impl(
804 &self,
805 snatch_guard: &SnatchGuard,
806 pending_writes: &mut PendingWrites,
807 staging_buffer: &FlushedStagingBuffer,
808 buffer: Arc<Buffer>,
809 buffer_offset: u64,
810 ) -> Result<(), QueueWriteError> {
811 self.device.check_is_valid()?;
812
813 let transition = {
814 let mut trackers = self.device.trackers.lock();
815 trackers
816 .buffers
817 .set_single(&buffer, wgt::BufferUses::COPY_DST)
818 };
819
820 let dst_raw = buffer.try_raw(snatch_guard)?;
821
822 self.same_device_as(buffer.as_ref())?;
823
824 self.validate_write_buffer_impl(&buffer, buffer_offset, staging_buffer.size.into())?;
825
826 let region = hal::BufferCopy {
827 src_offset: 0,
828 dst_offset: buffer_offset,
829 size: staging_buffer.size,
830 };
831 let barriers = iter::once(hal::BufferBarrier {
832 buffer: staging_buffer.raw(),
833 usage: hal::StateTransition {
834 from: wgt::BufferUses::MAP_WRITE,
835 to: wgt::BufferUses::COPY_SRC,
836 },
837 })
838 .chain(transition.map(|pending| pending.into_hal(&buffer, snatch_guard)))
839 .collect::<Vec<_>>();
840 let encoder = pending_writes.activate();
841 unsafe {
842 encoder.transition_buffers(&barriers);
843 encoder.copy_buffer_to_buffer(staging_buffer.raw(), dst_raw, &[region]);
844 }
845
846 pending_writes.insert_buffer(&buffer);
847
848 {
851 buffer
852 .initialization_status
853 .write()
854 .drain(buffer_offset..(buffer_offset + staging_buffer.size.get()));
855 }
856
857 Ok(())
858 }
859
860 pub fn write_texture(
861 &self,
862 destination: wgt::TexelCopyTextureInfo<Arc<Texture>>,
863 data: &[u8],
864 data_layout: &wgt::TexelCopyBufferLayout,
865 size: &wgt::Extent3d,
866 ) -> Result<(), QueueWriteError> {
867 profiling::scope!("Queue::write_texture");
868 api_log!("Queue::write_texture");
869
870 self.device.check_is_valid()?;
871
872 let dst = destination.texture;
873 let destination = wgt::TexelCopyTextureInfo {
874 texture: (),
875 mip_level: destination.mip_level,
876 origin: destination.origin,
877 aspect: destination.aspect,
878 };
879
880 self.same_device_as(dst.as_ref())?;
881
882 dst.check_usage(wgt::TextureUsages::COPY_DST)
883 .map_err(TransferError::MissingTextureUsage)?;
884
885 let (hal_copy_size, array_layer_count) =
888 validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, size)?;
889
890 let (selector, dst_base) = extract_texture_selector(&destination, size, &dst)?;
891
892 validate_texture_copy_dst_format(dst.desc.format, destination.aspect)?;
893
894 validate_texture_buffer_copy(
895 &destination,
896 dst_base.aspect,
897 &dst.desc,
898 data_layout,
899 false, )?;
901
902 let (required_bytes_in_copy, _source_bytes_per_array_layer, _) =
905 validate_linear_texture_data(
906 data_layout,
907 dst.desc.format,
908 destination.aspect,
909 data.len() as wgt::BufferAddress,
910 CopySide::Source,
911 size,
912 )?;
913
914 if dst.desc.format.is_depth_stencil_format() {
915 self.device
916 .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
917 .map_err(TransferError::from)?;
918 }
919
920 let snatch_guard = self.device.snatchable_lock.read();
921
922 let dst_raw = dst.try_inner(&snatch_guard)?.raw();
923
924 if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
927 log::trace!("Ignoring write_texture of size 0");
928 return Ok(());
929 }
930
931 let mut pending_writes = self.pending_writes.lock();
932 let encoder = pending_writes.activate();
933
934 let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
940 0..1
942 } else {
943 destination.origin.z..destination.origin.z + size.depth_or_array_layers
944 };
945 let mut dst_initialization_status = dst.initialization_status.write();
946 if dst_initialization_status.mips[destination.mip_level as usize]
947 .check(init_layer_range.clone())
948 .is_some()
949 {
950 if has_copy_partial_init_tracker_coverage(size, &destination, &dst.desc) {
951 for layer_range in dst_initialization_status.mips[destination.mip_level as usize]
952 .drain(init_layer_range)
953 .collect::<Vec<core::ops::Range<u32>>>()
954 {
955 let mut trackers = self.device.trackers.lock();
956 crate::command::clear_texture(
957 &dst,
958 TextureInitRange {
959 mip_range: destination.mip_level..(destination.mip_level + 1),
960 layer_range,
961 },
962 encoder,
963 &mut trackers.textures,
964 &self.device.alignments,
965 self.device.zero_buffer.as_ref(),
966 &snatch_guard,
967 self.device.instance_flags,
968 )
969 .map_err(QueueWriteError::from)?;
970 }
971 } else {
972 dst_initialization_status.mips[destination.mip_level as usize]
973 .drain(init_layer_range);
974 }
975 }
976
977 let (block_width, block_height) = dst.desc.format.block_dimensions();
978 let width_in_blocks = size.width / block_width;
979 let height_in_blocks = size.height / block_height;
980
981 let block_size = dst
982 .desc
983 .format
984 .block_copy_size(Some(destination.aspect))
985 .unwrap();
986 let bytes_in_last_row = width_in_blocks * block_size;
987
988 let bytes_per_row = data_layout.bytes_per_row.unwrap_or(bytes_in_last_row);
989 let rows_per_image = data_layout.rows_per_image.unwrap_or(height_in_blocks);
990
991 let bytes_per_row_alignment = get_lowest_common_denom(
992 self.device.alignments.buffer_copy_pitch.get() as u32,
993 block_size,
994 );
995 assert!(u32::MAX - bytes_in_last_row >= bytes_per_row_alignment);
996 let stage_bytes_per_row = wgt::math::align_to(bytes_in_last_row, bytes_per_row_alignment);
997
998 let staging_buffer = if stage_bytes_per_row == bytes_per_row {
1002 profiling::scope!("copy aligned");
1003 let stage_size = wgt::BufferSize::new(required_bytes_in_copy).unwrap();
1005 let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?;
1006 staging_buffer.write(&data[data_layout.offset as usize..]);
1007 staging_buffer
1008 } else {
1009 profiling::scope!("copy chunked");
1010 let block_rows_in_copy = u64::from(size.depth_or_array_layers - 1)
1012 * u64::from(rows_per_image)
1013 + u64::from(height_in_blocks);
1014 let stage_size = u64::from(stage_bytes_per_row)
1017 .checked_mul(block_rows_in_copy)
1018 .and_then(wgt::BufferSize::new)
1019 .unwrap();
1020 let mut staging_buffer = StagingBuffer::new(&self.device, stage_size)?;
1021 for layer in 0..u64::from(size.depth_or_array_layers) {
1022 let rows_offset = layer * u64::from(rows_per_image);
1023 for row in rows_offset..rows_offset + u64::from(height_in_blocks) {
1024 let src_offset = data_layout.offset + row * u64::from(bytes_per_row);
1025 let dst_offset = row * u64::from(stage_bytes_per_row);
1026 unsafe {
1027 staging_buffer.write_with_offset(
1028 data,
1029 src_offset as isize,
1030 dst_offset as isize,
1031 bytes_in_last_row as usize,
1032 )
1033 }
1034 }
1035 }
1036 staging_buffer
1037 };
1038
1039 let staging_buffer = staging_buffer.flush();
1040
1041 let regions = (0..array_layer_count)
1042 .map(|array_layer_offset| {
1043 let mut texture_base = dst_base.clone();
1044 texture_base.array_layer += array_layer_offset;
1045 hal::BufferTextureCopy {
1046 buffer_layout: wgt::TexelCopyBufferLayout {
1047 offset: array_layer_offset as u64
1048 * rows_per_image as u64
1049 * stage_bytes_per_row as u64,
1050 bytes_per_row: Some(stage_bytes_per_row),
1051 rows_per_image: Some(rows_per_image),
1052 },
1053 texture_base,
1054 size: hal_copy_size,
1055 }
1056 })
1057 .collect::<Vec<_>>();
1058
1059 {
1060 let buffer_barrier = hal::BufferBarrier {
1061 buffer: staging_buffer.raw(),
1062 usage: hal::StateTransition {
1063 from: wgt::BufferUses::MAP_WRITE,
1064 to: wgt::BufferUses::COPY_SRC,
1065 },
1066 };
1067
1068 let mut trackers = self.device.trackers.lock();
1069 let transition =
1070 trackers
1071 .textures
1072 .set_single(&dst, selector, wgt::TextureUses::COPY_DST);
1073 let texture_barriers = transition
1074 .map(|pending| pending.into_hal(dst_raw))
1075 .collect::<Vec<_>>();
1076
1077 unsafe {
1078 encoder.transition_textures(&texture_barriers);
1079 encoder.transition_buffers(&[buffer_barrier]);
1080 encoder.copy_buffer_to_texture(staging_buffer.raw(), dst_raw, ®ions);
1081 }
1082 }
1083
1084 pending_writes.consume(staging_buffer);
1085 pending_writes.insert_texture(&dst);
1086
1087 Ok(())
1088 }
1089
1090 #[cfg(webgl)]
1091 pub fn copy_external_image_to_texture(
1092 &self,
1093 source: &wgt::CopyExternalImageSourceInfo,
1094 destination: wgt::CopyExternalImageDestInfo<Arc<Texture>>,
1095 size: wgt::Extent3d,
1096 ) -> Result<(), QueueWriteError> {
1097 use crate::conv;
1098
1099 profiling::scope!("Queue::copy_external_image_to_texture");
1100
1101 self.device.check_is_valid()?;
1102
1103 let mut needs_flag = false;
1104 needs_flag |= matches!(source.source, wgt::ExternalImageSource::OffscreenCanvas(_));
1105 needs_flag |= source.origin != wgt::Origin2d::ZERO;
1106 needs_flag |= destination.color_space != wgt::PredefinedColorSpace::Srgb;
1107 #[allow(clippy::bool_comparison)]
1108 if matches!(source.source, wgt::ExternalImageSource::ImageBitmap(_)) {
1109 needs_flag |= source.flip_y != false;
1110 needs_flag |= destination.premultiplied_alpha != false;
1111 }
1112
1113 if needs_flag {
1114 self.device
1115 .require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_EXTERNAL_TEXTURE_COPIES)
1116 .map_err(TransferError::from)?;
1117 }
1118
1119 let src_width = source.source.width();
1120 let src_height = source.source.height();
1121
1122 let dst = destination.texture;
1123 let premultiplied_alpha = destination.premultiplied_alpha;
1124 let destination = wgt::TexelCopyTextureInfo {
1125 texture: (),
1126 mip_level: destination.mip_level,
1127 origin: destination.origin,
1128 aspect: destination.aspect,
1129 };
1130
1131 if !conv::is_valid_external_image_copy_dst_texture_format(dst.desc.format) {
1132 return Err(
1133 TransferError::ExternalCopyToForbiddenTextureFormat(dst.desc.format).into(),
1134 );
1135 }
1136 if dst.desc.dimension != wgt::TextureDimension::D2 {
1137 return Err(TransferError::InvalidDimensionExternal.into());
1138 }
1139 dst.check_usage(wgt::TextureUsages::COPY_DST | wgt::TextureUsages::RENDER_ATTACHMENT)
1140 .map_err(TransferError::MissingTextureUsage)?;
1141 if dst.desc.sample_count != 1 {
1142 return Err(TransferError::InvalidSampleCount {
1143 sample_count: dst.desc.sample_count,
1144 }
1145 .into());
1146 }
1147
1148 if source.origin.x > src_width || src_width - source.origin.x < size.width {
1149 return Err(TransferError::TextureOverrun {
1150 start_offset: source.origin.x,
1151 end_offset: source.origin.x.saturating_add(size.width),
1152 texture_size: src_width,
1153 dimension: crate::resource::TextureErrorDimension::X,
1154 side: CopySide::Source,
1155 }
1156 .into());
1157 }
1158 if source.origin.y > src_height || src_height - source.origin.y < size.height {
1159 return Err(TransferError::TextureOverrun {
1160 start_offset: source.origin.y,
1161 end_offset: source.origin.y.saturating_add(size.height),
1162 texture_size: src_height,
1163 dimension: crate::resource::TextureErrorDimension::Y,
1164 side: CopySide::Source,
1165 }
1166 .into());
1167 }
1168 if size.depth_or_array_layers != 1 {
1169 return Err(TransferError::TextureOverrun {
1170 start_offset: 0,
1171 end_offset: size.depth_or_array_layers,
1172 texture_size: 1,
1173 dimension: crate::resource::TextureErrorDimension::Z,
1174 side: CopySide::Source,
1175 }
1176 .into());
1177 }
1178
1179 let (hal_copy_size, _) =
1182 validate_texture_copy_range(&destination, &dst.desc, CopySide::Destination, &size)?;
1183
1184 let (selector, dst_base) = extract_texture_selector(&destination, &size, &dst)?;
1185
1186 if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 {
1189 log::trace!("Ignoring copy_external_image_to_texture of size 0");
1190 return Ok(());
1191 }
1192
1193 let mut pending_writes = self.pending_writes.lock();
1194 let encoder = pending_writes.activate();
1195
1196 let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 {
1202 0..1
1204 } else {
1205 destination.origin.z..destination.origin.z + size.depth_or_array_layers
1206 };
1207 let mut dst_initialization_status = dst.initialization_status.write();
1208 if dst_initialization_status.mips[destination.mip_level as usize]
1209 .check(init_layer_range.clone())
1210 .is_some()
1211 {
1212 if has_copy_partial_init_tracker_coverage(&size, &destination, &dst.desc) {
1213 for layer_range in dst_initialization_status.mips[destination.mip_level as usize]
1214 .drain(init_layer_range)
1215 .collect::<Vec<core::ops::Range<u32>>>()
1216 {
1217 let mut trackers = self.device.trackers.lock();
1218 crate::command::clear_texture(
1219 &dst,
1220 TextureInitRange {
1221 mip_range: destination.mip_level..(destination.mip_level + 1),
1222 layer_range,
1223 },
1224 encoder,
1225 &mut trackers.textures,
1226 &self.device.alignments,
1227 self.device.zero_buffer.as_ref(),
1228 &self.device.snatchable_lock.read(),
1229 self.device.instance_flags,
1230 )
1231 .map_err(QueueWriteError::from)?;
1232 }
1233 } else {
1234 dst_initialization_status.mips[destination.mip_level as usize]
1235 .drain(init_layer_range);
1236 }
1237 }
1238
1239 let snatch_guard = self.device.snatchable_lock.read();
1240 let dst_raw = dst.try_raw(&snatch_guard)?;
1241
1242 let regions = hal::TextureCopy {
1243 src_base: hal::TextureCopyBase {
1244 mip_level: 0,
1245 array_layer: 0,
1246 origin: source.origin.to_3d(0),
1247 aspect: hal::FormatAspects::COLOR,
1248 },
1249 dst_base,
1250 size: hal_copy_size,
1251 };
1252
1253 let mut trackers = self.device.trackers.lock();
1254 let transitions = trackers
1255 .textures
1256 .set_single(&dst, selector, wgt::TextureUses::COPY_DST);
1257
1258 let encoder_webgl = encoder
1261 .as_any_mut()
1262 .downcast_mut::<hal::gles::CommandEncoder>()
1263 .unwrap();
1264 let dst_raw_webgl = dst_raw
1265 .as_any()
1266 .downcast_ref::<hal::gles::Texture>()
1267 .unwrap();
1268 let transitions_webgl = transitions.map(|pending| {
1269 let dyn_transition = pending.into_hal(dst_raw);
1270 hal::TextureBarrier {
1271 texture: dst_raw_webgl,
1272 range: dyn_transition.range,
1273 usage: dyn_transition.usage,
1274 }
1275 });
1276
1277 use hal::CommandEncoder as _;
1278 unsafe {
1279 encoder_webgl.transition_textures(transitions_webgl);
1280 encoder_webgl.copy_external_image_to_texture(
1281 source,
1282 dst_raw_webgl,
1283 premultiplied_alpha,
1284 iter::once(regions),
1285 );
1286 }
1287
1288 pending_writes.insert_texture(&dst);
1289
1290 Ok(())
1291 }
1292
1293 pub fn flush_writes_for_buffer(
1295 &self,
1296 buffer: &Arc<Buffer>,
1297 snatch_guard: SnatchGuard,
1298 ) -> Result<(), BufferAccessError> {
1299 let submission = self
1300 .allocate_submission(snatch_guard)
1301 .map_err(|(_index, e)| e)?;
1302
1303 let pending_writes = self.pending_writes.lock();
1304 if !pending_writes.contains_buffer(buffer) {
1305 return Ok(());
1306 }
1307
1308 submission.submit(pending_writes)?;
1309
1310 Ok(())
1311 }
1312
1313 fn flush_pending_writes(&self) -> Result<Option<SubmissionIndex>, DeviceError> {
1314 let snatch_guard = self.device.snatchable_lock.read();
1315 let submission = self
1316 .allocate_submission(snatch_guard)
1317 .map_err(|(_index, e)| e)?;
1318 let submit_index = submission.index;
1319 let pending_writes = self.pending_writes.lock();
1320 if pending_writes.is_recording {
1321 submission.submit(pending_writes)?;
1322 Ok(Some(submit_index))
1323 } else {
1324 Ok(None)
1325 }
1326 }
1327
1328 #[cfg(feature = "trace")]
1329 fn trace_submission(
1330 &self,
1331 submit_index: SubmissionIndex,
1332 commands: Vec<crate::command::Command<crate::command::PointerReferences>>,
1333 ) {
1334 if let Some(ref mut trace) = *self.device.trace.lock() {
1335 trace.add(Action::Submit(submit_index, commands));
1336 }
1337 }
1338
1339 #[cfg(feature = "trace")]
1340 fn trace_failed_submission(
1341 &self,
1342 submit_index: SubmissionIndex,
1343 commands: Option<Vec<crate::command::Command<crate::command::PointerReferences>>>,
1344 error: alloc::string::String,
1345 ) {
1346 if let Some(ref mut trace) = *self.device.trace.lock() {
1347 trace.add(Action::FailedCommands {
1348 commands,
1349 failed_at_submit: Some(submit_index),
1350 error,
1351 });
1352 }
1353 }
1354
1355 pub fn submit(
1356 &self,
1357 command_buffers: &[Arc<CommandBuffer>],
1358 ) -> Result<SubmissionIndex, (SubmissionIndex, QueueSubmitError)> {
1359 profiling::scope!("Queue::submit");
1360 api_log!("Queue::submit");
1361
1362 let snatch_guard = self.device.snatchable_lock.read();
1363 let mut submission = self
1364 .allocate_submission(snatch_guard)
1365 .map_err(|(index, e)| (index, e.into()))?;
1366 let submit_index = submission.index;
1367
1368 let res = 'error: {
1369 let mut used_surface_textures = track::TextureUsageScope::default();
1370
1371 {
1372 if !command_buffers.is_empty() {
1373 profiling::scope!("prepare");
1374
1375 let mut first_error = None;
1376
1377 for command_buffer in command_buffers {
1383 profiling::scope!("process command buffer");
1384
1385 used_surface_textures.set_size(self.device.tracker_indices.textures.size());
1388
1389 #[allow(unused_mut)]
1392 let mut cmd_buf_data = command_buffer.take_finished();
1393
1394 if first_error.is_some() {
1395 continue;
1396 }
1397
1398 #[cfg(feature = "trace")]
1399 let trace_commands = cmd_buf_data
1400 .as_mut()
1401 .ok()
1402 .and_then(|data| mem::take(&mut data.trace_commands));
1403
1404 let mut baked = match cmd_buf_data {
1405 Ok(cmd_buf_data) => {
1406 let res = validate_command_buffer(
1407 command_buffer,
1408 self,
1409 &cmd_buf_data,
1410 &submission.snatch_guard,
1411 &mut submission.surface_textures,
1412 &mut used_surface_textures,
1413 &mut submission.command_index_guard,
1414 );
1415 if let Err(err) = res {
1416 #[cfg(feature = "trace")]
1417 self.trace_failed_submission(
1418 submit_index,
1419 trace_commands,
1420 err.to_string(),
1421 );
1422 first_error.get_or_insert(err);
1423 continue;
1424 }
1425
1426 #[cfg(feature = "trace")]
1427 if let Some(commands) = trace_commands {
1428 self.trace_submission(submit_index, commands);
1429 }
1430
1431 cmd_buf_data.set_acceleration_structure_dependencies(
1432 &submission.snatch_guard,
1433 );
1434 cmd_buf_data.into_baked_commands()
1435 }
1436 Err(err) => {
1437 #[cfg(feature = "trace")]
1438 self.trace_failed_submission(
1439 submit_index,
1440 trace_commands,
1441 err.to_string(),
1442 );
1443 first_error.get_or_insert(err.into());
1444 continue;
1445 }
1446 };
1447
1448 if let Err(e) = baked.process_deferred_query_set_resolves(
1449 &self.device,
1450 &submission.snatch_guard,
1451 ) {
1452 break 'error Err(e.into());
1453 }
1454
1455 if let Err(e) = baked.encoder.open_pass(hal_label(
1457 Some("(wgpu internal) Transit"),
1458 self.device.instance_flags,
1459 )) {
1460 break 'error Err(e.into());
1461 }
1462
1463 let mut trackers = self.device.trackers.lock();
1465 if let Err(e) =
1466 baked.initialize_buffer_memory(&mut trackers, &submission.snatch_guard)
1467 {
1468 break 'error Err(e.into());
1469 }
1470 if let Err(e) = baked.initialize_texture_memory(
1471 &mut trackers,
1472 &self.device,
1473 &submission.snatch_guard,
1474 ) {
1475 break 'error Err(e.into());
1476 }
1477
1478 CommandEncoder::insert_barriers_from_device_tracker(
1481 baked.encoder.raw.as_mut(),
1482 &mut trackers,
1483 &baked.trackers,
1484 &submission.snatch_guard,
1485 );
1486
1487 if let Err(e) = baked.encoder.close_and_push_front() {
1488 break 'error Err(e.into());
1489 }
1490
1491 if !used_surface_textures.is_empty() {
1495 if let Err(e) = baked.encoder.open_pass(hal_label(
1496 Some("(wgpu internal) Present"),
1497 self.device.instance_flags,
1498 )) {
1499 break 'error Err(e.into());
1500 }
1501 let texture_barriers = trackers
1502 .textures
1503 .set_from_usage_scope_and_drain_transitions(
1504 &used_surface_textures,
1505 &submission.snatch_guard,
1506 )
1507 .collect::<Vec<_>>();
1508 unsafe {
1509 baked.encoder.raw.transition_textures(&texture_barriers);
1510 };
1511 if let Err(e) = baked.encoder.close() {
1512 break 'error Err(e.into());
1513 }
1514 used_surface_textures = track::TextureUsageScope::default();
1515 }
1516
1517 submission.executions.push(EncoderInFlight {
1519 inner: baked.encoder,
1520 trackers: baked.trackers,
1521 temp_resources: baked.temp_resources,
1522 _indirect_draw_validation_resources: baked
1523 .indirect_draw_validation_resources,
1524 pending_buffers: FastHashMap::default(),
1525 pending_textures: FastHashMap::default(),
1526 pending_blas_s: FastHashMap::default(),
1527 });
1528 }
1529
1530 if let Some(first_error) = first_error {
1531 break 'error Err(first_error);
1532 }
1533 }
1534 }
1535
1536 let pending_writes = self.pending_writes.lock();
1537
1538 let SubmissionResult { snatch_guard } = match submission.submit(pending_writes) {
1539 Ok(result) => result,
1540 Err(e) => break 'error Err(e.into()),
1541 };
1542
1543 profiling::scope!("cleanup");
1544
1545 let (closures, result) = self.device.maintain(wgt::PollType::Poll, snatch_guard);
1549 match result {
1550 Ok(status) => {
1551 debug_assert!(matches!(
1552 status,
1553 wgt::PollStatus::QueueEmpty | wgt::PollStatus::Poll
1554 ));
1555 }
1556 Err(WaitIdleError::Device(err)) => break 'error Err(QueueSubmitError::Queue(err)),
1557 Err(WaitIdleError::WrongSubmissionIndex(..)) => {
1558 unreachable!("Cannot get WrongSubmissionIndex from Poll")
1559 }
1560 Err(WaitIdleError::Timeout) => unreachable!("Cannot get Timeout from Poll"),
1561 };
1562
1563 Ok(closures)
1564 };
1565
1566 let callbacks = match res {
1567 Ok(ok) => ok,
1568 Err(e) => return Err((submit_index, e)),
1569 };
1570
1571 callbacks.fire();
1573
1574 self.device.lose_if_oom();
1575
1576 api_log!("Queue::submit returned submit index {submit_index}");
1577
1578 Ok(submit_index)
1579 }
1580
1581 fn allocate_submission<'a>(
1604 &'a self,
1605 snatch_guard: SnatchGuard<'a>,
1606 ) -> Result<PendingSubmission<'a>, (SubmissionIndex, DeviceError)> {
1607 let mut command_index_guard = self.device.command_indices.write();
1608 command_index_guard.active_submission_index += 1;
1609 let index = command_index_guard.active_submission_index;
1610
1611 if let Err(e) = self.device.check_is_valid() {
1612 return Err((index, e));
1613 }
1614
1615 let submission = PendingSubmission {
1616 queue: self,
1617 snatch_guard,
1618 command_index_guard,
1619 executions: Vec::new(),
1620 surface_textures: FastHashMap::default(),
1621 index,
1622 };
1623
1624 Ok(submission)
1625 }
1626
1627 fn submit_pending_submission<'a>(
1643 &self,
1644 mut pending_writes: MutexGuard<'_, PendingWrites>,
1645 prepared: PendingSubmission<'a>,
1646 ) -> Result<SubmissionResult<'a>, DeviceError> {
1647 let PendingSubmission {
1648 queue: _,
1649 snatch_guard,
1650 command_index_guard,
1651 mut executions,
1652 mut surface_textures,
1653 index: submit_index,
1654 } = prepared;
1655
1656 let mut used_surface_textures = track::TextureUsageScope::default();
1657 used_surface_textures.set_size(self.device.tracker_indices.textures.size());
1658 for texture in pending_writes.dst_textures.values() {
1659 match texture.try_inner(&snatch_guard) {
1660 Ok(TextureInner::Native { .. }) => {}
1661 Ok(TextureInner::Surface { .. }) => {
1662 surface_textures.insert(Arc::as_ptr(texture), texture.clone());
1664
1665 unsafe {
1666 used_surface_textures
1667 .merge_single(texture, None, wgt::TextureUses::PRESENT)
1668 .unwrap()
1669 };
1670 }
1671 Err(InvalidOrDestroyedResourceError::DestroyedResource(_)) => {}
1676 Err(InvalidOrDestroyedResourceError::InvalidResource(_)) => {
1677 unreachable!()
1678 }
1679 }
1680 }
1681
1682 if !used_surface_textures.is_empty() {
1683 let mut trackers = self.device.trackers.lock();
1684
1685 let texture_barriers = trackers
1686 .textures
1687 .set_from_usage_scope_and_drain_transitions(&used_surface_textures, &snatch_guard)
1688 .collect::<Vec<_>>();
1689 unsafe {
1690 pending_writes
1691 .command_encoder
1692 .transition_textures(&texture_barriers);
1693 };
1694 }
1695
1696 match pending_writes.pre_submit(&self.device.command_allocator, &self.device, self) {
1697 Ok(Some(pending_execution)) => {
1698 executions.insert(0, pending_execution);
1699 }
1700 Ok(None) => {}
1701 Err(e) => return Err(e),
1702 }
1703 let hal_command_buffers = executions
1704 .iter()
1705 .flat_map(|e| e.inner.list.iter().map(|b| b.as_ref()))
1706 .collect::<Vec<_>>();
1707
1708 {
1709 let mut submit_surface_textures =
1710 SmallVec::<[&dyn hal::DynSurfaceTexture; 2]>::with_capacity(surface_textures.len());
1711 for texture in surface_textures.values() {
1712 let raw = match texture.inner.get(&snatch_guard).maybe_valid() {
1713 Some(TextureInner::Surface { raw, .. }) => raw.as_ref(),
1714 _ => unreachable!(),
1715 };
1716 submit_surface_textures.push(raw);
1717 }
1718
1719 unsafe {
1720 self.raw().submit(
1721 &hal_command_buffers,
1722 &submit_surface_textures,
1723 (self.device.fence.as_ref(), submit_index),
1724 )
1725 }
1726 .map_err(|e| self.device.handle_hal_error(e))?;
1727
1728 drop(pending_writes);
1732
1733 self.device
1735 .last_successful_submission_index
1736 .fetch_max(submit_index, Ordering::SeqCst);
1737 }
1738
1739 self.lock_life().track_submission(submit_index, executions);
1741
1742 drop(command_index_guard);
1748
1749 Ok(SubmissionResult { snatch_guard })
1750 }
1751
1752 pub fn get_timestamp_period(&self) -> f32 {
1753 unsafe { self.raw().get_timestamp_period() }
1754 }
1755
1756 pub fn on_submitted_work_done(
1758 &self,
1759 closure: SubmittedWorkDoneClosure,
1760 ) -> Option<SubmissionIndex> {
1761 api_log!("Queue::on_submitted_work_done");
1762
1763 let _: Result<_, DeviceError> = self.flush_pending_writes();
1767
1768 self.lock_life().add_work_done_closure(closure)
1769 }
1770
1771 pub fn compact_blas(&self, blas: &Arc<Blas>) -> Result<Arc<Blas>, CompactBlasError> {
1772 profiling::scope!("Queue::compact_blas");
1773 api_log!("Queue::compact_blas");
1774
1775 let new_label = blas.label.clone() + " (compacted)";
1776
1777 self.device.check_is_valid()?;
1778 self.same_device_as(blas.as_ref())?;
1779
1780 let device = blas.device.clone();
1781
1782 let snatch_guard = device.snatchable_lock.read();
1783
1784 let BlasCompactState::Ready { size } = *blas.compacted_state.lock() else {
1785 return Err(CompactBlasError::BlasNotReady);
1786 };
1787
1788 let mut size_info = blas.size_info;
1789 size_info.acceleration_structure_size = size;
1790
1791 let mut pending_writes = self.pending_writes.lock();
1792 let cmd_buf_raw = pending_writes.activate();
1793
1794 let raw = unsafe {
1795 device
1796 .raw()
1797 .create_acceleration_structure(&hal::AccelerationStructureDescriptor {
1798 label: hal_label(Some(&new_label), device.instance_flags),
1799 size: size_info.acceleration_structure_size,
1800 format: hal::AccelerationStructureFormat::BottomLevel,
1801 allow_compaction: false,
1802 })
1803 }
1804 .map_err(DeviceError::from_hal)?;
1805
1806 let src_raw = blas.try_raw(&snatch_guard)?;
1807
1808 unsafe {
1809 cmd_buf_raw.copy_acceleration_structure_to_acceleration_structure(
1810 src_raw,
1811 raw.as_ref(),
1812 wgt::AccelerationStructureCopy::Compact,
1813 )
1814 };
1815
1816 let handle = unsafe {
1817 device
1818 .raw()
1819 .get_acceleration_structure_device_address(raw.as_ref())
1820 };
1821
1822 drop(snatch_guard);
1823
1824 let mut command_indices_lock = device.command_indices.write();
1825 command_indices_lock.next_acceleration_structure_build_command_index += 1;
1826 let built_index =
1827 NonZeroU64::new(command_indices_lock.next_acceleration_structure_build_command_index)
1828 .unwrap();
1829
1830 let new_blas = Arc::new(Blas {
1831 raw: Snatchable::new(raw),
1832 device: device.clone(),
1833 size_info,
1834 sizes: blas.sizes.clone(),
1835 flags: blas.flags & !AccelerationStructureFlags::ALLOW_COMPACTION,
1836 update_mode: blas.update_mode,
1837 built_index: RwLock::new(rank::BLAS_BUILT_INDEX, Some(built_index)),
1839 handle,
1840 label: new_label,
1841 tracking_data: TrackingData::new(blas.device.tracker_indices.blas_s.clone()),
1842 compaction_buffer: None,
1843 compacted_state: Mutex::new(rank::BLAS_COMPACTION_STATE, BlasCompactState::Compacted),
1844 });
1845
1846 pending_writes.insert_blas(blas);
1847 pending_writes.insert_blas(&new_blas);
1848
1849 Ok(new_blas)
1850 }
1851}
1852
1853impl Global {
1854 pub fn queue_write_buffer(
1855 &self,
1856 queue_id: QueueId,
1857 buffer_id: id::BufferId,
1858 buffer_offset: wgt::BufferAddress,
1859 data: &[u8],
1860 ) -> Result<(), QueueWriteError> {
1861 let queue = self.hub.queues.get(queue_id);
1862 let buffer = self.hub.buffers.get(buffer_id).get()?;
1863
1864 #[cfg(feature = "trace")]
1865 if let Some(ref mut trace) = *queue.device.trace.lock() {
1866 use crate::device::trace::DataKind;
1867 let size = data.len() as u64;
1868 let data = trace.make_binary(DataKind::Bin, data);
1869 trace.add(Action::WriteBuffer {
1870 id: buffer.to_trace(),
1871 data,
1872 offset: buffer_offset,
1873 size,
1874 queued: true,
1875 });
1876 }
1877
1878 queue.write_buffer(buffer, buffer_offset, data)
1879 }
1880
1881 pub fn queue_create_staging_buffer(
1882 &self,
1883 queue_id: QueueId,
1884 buffer_size: wgt::BufferSize,
1885 id_in: Option<id::StagingBufferId>,
1886 ) -> Result<(id::StagingBufferId, NonNull<u8>), QueueWriteError> {
1887 let queue = self.hub.queues.get(queue_id);
1888 let (staging_buffer, ptr) = queue.create_staging_buffer(buffer_size)?;
1889
1890 let fid = self.hub.staging_buffers.prepare(id_in);
1891 let id = fid.assign(staging_buffer);
1892
1893 Ok((id, ptr))
1894 }
1895
1896 pub fn queue_write_staging_buffer(
1897 &self,
1898 queue_id: QueueId,
1899 buffer_id: id::BufferId,
1900 buffer_offset: wgt::BufferAddress,
1901 staging_buffer_id: id::StagingBufferId,
1902 ) -> Result<(), QueueWriteError> {
1903 let queue = self.hub.queues.get(queue_id);
1904 let buffer = self.hub.buffers.get(buffer_id);
1905 let staging_buffer = self.hub.staging_buffers.remove(staging_buffer_id);
1906 queue.write_staging_buffer(buffer, buffer_offset, staging_buffer)
1907 }
1908
1909 pub fn queue_validate_write_buffer(
1910 &self,
1911 queue_id: QueueId,
1912 buffer_id: id::BufferId,
1913 buffer_offset: u64,
1914 buffer_size: wgt::BufferSize,
1915 ) -> Result<(), QueueWriteError> {
1916 let queue = self.hub.queues.get(queue_id);
1917 let buffer = self.hub.buffers.get(buffer_id);
1918 queue.validate_write_buffer(buffer, buffer_offset, buffer_size)
1919 }
1920
1921 pub fn queue_write_texture(
1922 &self,
1923 queue_id: QueueId,
1924 destination: &wgt::TexelCopyTextureInfo<id::TextureId>,
1925 data: &[u8],
1926 data_layout: &wgt::TexelCopyBufferLayout,
1927 size: &wgt::Extent3d,
1928 ) -> Result<(), QueueWriteError> {
1929 let queue = self.hub.queues.get(queue_id);
1930 let texture = self.hub.textures.get(destination.texture);
1931 let destination = wgt::TexelCopyTextureInfo {
1932 texture,
1933 mip_level: destination.mip_level,
1934 origin: destination.origin,
1935 aspect: destination.aspect,
1936 };
1937
1938 #[cfg(feature = "trace")]
1939 if let Some(ref mut trace) = *queue.device.trace.lock() {
1940 use crate::device::trace::DataKind;
1941 let data = trace.make_binary(DataKind::Bin, data);
1942 trace.add(Action::WriteTexture {
1943 to: destination.to_trace(),
1944 data,
1945 layout: *data_layout,
1946 size: *size,
1947 });
1948 }
1949
1950 queue.write_texture(destination, data, data_layout, size)
1951 }
1952
1953 #[cfg(webgl)]
1954 pub fn queue_copy_external_image_to_texture(
1955 &self,
1956 queue_id: QueueId,
1957 source: &wgt::CopyExternalImageSourceInfo,
1958 destination: crate::command::CopyExternalImageDestInfo,
1959 size: wgt::Extent3d,
1960 ) -> Result<(), QueueWriteError> {
1961 let queue = self.hub.queues.get(queue_id);
1962 let destination = wgt::CopyExternalImageDestInfo {
1963 texture: self.hub.textures.get(destination.texture),
1964 mip_level: destination.mip_level,
1965 origin: destination.origin,
1966 aspect: destination.aspect,
1967 color_space: destination.color_space,
1968 premultiplied_alpha: destination.premultiplied_alpha,
1969 };
1970 queue.copy_external_image_to_texture(source, destination, size)
1971 }
1972
1973 pub fn queue_submit(
1974 &self,
1975 queue_id: QueueId,
1976 command_buffer_ids: &[id::CommandBufferId],
1977 ) -> Result<SubmissionIndex, (SubmissionIndex, QueueSubmitError)> {
1978 let queue = self.hub.queues.get(queue_id);
1979 let command_buffer_guard = self.hub.command_buffers.read();
1980 let command_buffers = command_buffer_ids
1981 .iter()
1982 .map(|id| command_buffer_guard.get(*id))
1983 .collect::<Vec<_>>();
1984 drop(command_buffer_guard);
1985 queue.submit(&command_buffers)
1986 }
1987
1988 pub fn queue_get_timestamp_period(&self, queue_id: QueueId) -> f32 {
1989 let queue = self.hub.queues.get(queue_id);
1990
1991 if queue.device.timestamp_normalizer.get().unwrap().enabled() {
1992 return 1.0;
1993 }
1994
1995 queue.get_timestamp_period()
1996 }
1997
1998 pub fn queue_on_submitted_work_done(
1999 &self,
2000 queue_id: QueueId,
2001 closure: SubmittedWorkDoneClosure,
2002 ) -> SubmissionIndex {
2003 api_log!("Queue::on_submitted_work_done {queue_id:?}");
2004
2005 let queue = self.hub.queues.get(queue_id);
2006 let result = queue.on_submitted_work_done(closure);
2007 result.unwrap_or(0) }
2009
2010 pub fn queue_compact_blas(
2011 &self,
2012 queue_id: QueueId,
2013 blas_id: BlasId,
2014 id_in: Option<BlasId>,
2015 ) -> (BlasId, Option<u64>, Option<CompactBlasError>) {
2016 api_log!("Queue::compact_blas {queue_id:?}, {blas_id:?}");
2017
2018 let fid = self.hub.blas_s.prepare(id_in);
2019
2020 let queue = self.hub.queues.get(queue_id);
2021 let blas = self.hub.blas_s.get(blas_id);
2022 let device = &queue.device;
2023
2024 let error = 'error: {
2027 match device.require_features(wgpu_types::Features::EXPERIMENTAL_RAY_QUERY) {
2028 Ok(_) => {}
2029 Err(err) => break 'error err.into(),
2030 }
2031
2032 let blas = match blas.get() {
2033 Ok(blas) => blas,
2034 Err(err) => break 'error err.into(),
2035 };
2036
2037 let new_blas = match queue.compact_blas(&blas) {
2038 Ok(blas) => blas,
2039 Err(err) => break 'error err,
2040 };
2041
2042 let old_blas_size = blas.size_info.acceleration_structure_size;
2044 let new_blas_size = new_blas.size_info.acceleration_structure_size;
2045 let handle = new_blas.handle;
2046
2047 let id = fid.assign(Fallible::Valid(new_blas));
2048
2049 api_log!("CommandEncoder::compact_blas {blas_id:?} (size: {old_blas_size}) -> {id:?} (size: {new_blas_size})");
2050
2051 return (id, Some(handle), None);
2052 };
2053
2054 let id = fid.assign(Fallible::Invalid(Arc::new(error.to_string())));
2055
2056 (id, None, Some(error))
2057 }
2058}
2059
2060fn validate_command_buffer(
2061 command_buffer: &CommandBuffer,
2062 queue: &Queue,
2063 cmd_buf_data: &crate::command::CommandBufferMutable,
2064 snatch_guard: &SnatchGuard,
2065 surface_textures: &mut FastHashMap<*const Texture, Arc<Texture>>,
2066 used_surface_textures: &mut track::TextureUsageScope,
2067 command_index_guard: &mut RwLockWriteGuard<CommandIndices>,
2068) -> Result<(), QueueSubmitError> {
2069 command_buffer.same_device_as(queue)?;
2070
2071 {
2072 profiling::scope!("check resource state");
2073
2074 {
2075 profiling::scope!("buffers");
2076 for buffer in cmd_buf_data.trackers.buffers.used_resources() {
2077 buffer.check_destroyed(snatch_guard)?;
2078
2079 match *buffer.map_state.lock() {
2080 BufferMapState::Idle => (),
2081 _ => return Err(QueueSubmitError::BufferStillMapped(buffer.error_ident())),
2082 }
2083 }
2084 }
2085 {
2086 profiling::scope!("textures");
2087 for texture in cmd_buf_data.trackers.textures.used_resources() {
2088 let should_extend = match texture.try_inner(snatch_guard)? {
2089 TextureInner::Native { .. } => false,
2090 TextureInner::Surface { .. } => {
2091 surface_textures.insert(Arc::as_ptr(texture), texture.clone());
2093
2094 true
2095 }
2096 };
2097 if should_extend {
2098 unsafe {
2099 used_surface_textures
2100 .merge_single(texture, None, wgt::TextureUses::PRESENT)
2101 .unwrap();
2102 };
2103 }
2104 }
2105 }
2106 {
2107 profiling::scope!("query sets");
2108 for query_set in cmd_buf_data.trackers.query_sets.used_resources() {
2109 query_set.try_raw(snatch_guard)?;
2110 }
2111 }
2112 {
2116 profiling::scope!("bind groups");
2117 for bind_group in &cmd_buf_data.trackers.bind_groups {
2118 bind_group.try_raw(snatch_guard)?;
2120 }
2121 }
2122
2123 if let Err(e) =
2124 cmd_buf_data.validate_acceleration_structure_actions(snatch_guard, command_index_guard)
2125 {
2126 return Err(e.into());
2127 }
2128 }
2129 Ok(())
2130}