1use alloc::{
2 borrow::Cow,
3 boxed::Box,
4 string::{String, ToString as _},
5 sync::{Arc, Weak},
6 vec::Vec,
7};
8use core::{
9 fmt,
10 mem::{self, ManuallyDrop},
11 num::NonZeroU32,
12 sync::atomic::{AtomicBool, Ordering},
13};
14use hal::ShouldBeNonZeroExt;
15
16use arrayvec::ArrayVec;
17use bitflags::Flags;
18use smallvec::SmallVec;
19use wgt::{
20 math::align_to, DeviceLostReason, TextureFormat, TextureSampleType, TextureSelector,
21 TextureViewDimension,
22};
23
24#[cfg(feature = "trace")]
25use crate::device::trace;
26use crate::{
27 api_log,
28 binding_model::{
29 self, BindGroup, BindGroupLateBufferBindingInfo, BindGroupLayout,
30 BindGroupLayoutEntryError, BindGroupLayoutState, CreateBindGroupError,
31 CreateBindGroupLayoutError,
32 },
33 command, conv,
34 device::{
35 bgl, create_validator, features_to_naga_capabilities, life::WaitIdleError, map_buffer,
36 AttachmentData, DeviceLostInvocation, HostMap, MissingDownlevelFlags, MissingFeatures,
37 RenderPassContext,
38 },
39 hal_label,
40 init_tracker::{
41 BufferInitTracker, BufferInitTrackerAction, MemoryInitKind, TextureInitRange,
42 TextureInitTrackerAction,
43 },
44 instance::{Adapter, RequestDeviceError},
45 lock::{rank, Mutex, RwLock},
46 pipeline::{self, ColorStateError},
47 pool::ResourcePool,
48 present,
49 resource::{
50 self, Buffer, ExternalTexture, Fallible, Labeled, ParentDevice, QuerySet,
51 RawResourceAccess, ResourceState, Sampler, StagingBuffer, Texture, TextureView,
52 TextureViewNotRenderableReason, Tlas, TrackingData,
53 },
54 resource_log,
55 snatch::{SnatchGuard, SnatchLock, Snatchable},
56 timestamp_normalization::TIMESTAMP_NORMALIZATION_BUFFER_USES,
57 track::{BindGroupStates, DeviceTracker, TrackerIndexAllocators, UsageScope, UsageScopePool},
58 validation::{self, check_color_attachment_count, PassthroughInterface, ShaderMetaData},
59 weak_vec::WeakVec,
60 FastHashMap, LabelHelpers, OnceCellOrLock,
61};
62
63use super::{
64 queue::Queue, surface_config::validate_surface_configuration, DeviceDescriptor, DeviceError,
65 DeviceLostClosure, UserClosures, ENTRYPOINT_FAILURE_ERROR, ZERO_BUFFER_SIZE,
66};
67
68#[cfg(supports_64bit_atomics)]
69use core::sync::atomic::AtomicU64;
70#[cfg(not(supports_64bit_atomics))]
71use portable_atomic::AtomicU64;
72
73pub(crate) struct CommandIndices {
74 pub(crate) active_submission_index: hal::FenceValue,
82 pub(crate) next_acceleration_structure_build_command_index: u64,
83}
84
85#[repr(C)]
92#[derive(Copy, Clone, bytemuck::Zeroable, bytemuck::Pod)]
93pub struct ExternalTextureParams {
94 pub yuv_conversion_matrix: [f32; 16],
99
100 pub gamut_conversion_matrix: [f32; 12],
113
114 pub src_transfer_function: wgt::ExternalTextureTransferFunction,
118
119 pub dst_transfer_function: wgt::ExternalTextureTransferFunction,
122
123 pub sample_transform: [f32; 6],
136
137 pub load_transform: [f32; 6],
153
154 pub size: [u32; 2],
167
168 pub num_planes: u32,
172 pub _padding: [u8; 4],
174}
175
176impl ExternalTextureParams {
177 pub fn from_desc<L>(desc: &wgt::ExternalTextureDescriptor<L>) -> Self {
178 let gamut_conversion_matrix = [
179 desc.gamut_conversion_matrix[0],
180 desc.gamut_conversion_matrix[1],
181 desc.gamut_conversion_matrix[2],
182 0.0, desc.gamut_conversion_matrix[3],
184 desc.gamut_conversion_matrix[4],
185 desc.gamut_conversion_matrix[5],
186 0.0, desc.gamut_conversion_matrix[6],
188 desc.gamut_conversion_matrix[7],
189 desc.gamut_conversion_matrix[8],
190 0.0, ];
192
193 Self {
194 yuv_conversion_matrix: desc.yuv_conversion_matrix,
195 gamut_conversion_matrix,
196 src_transfer_function: desc.src_transfer_function,
197 dst_transfer_function: desc.dst_transfer_function,
198 size: [desc.width, desc.height],
199 sample_transform: desc.sample_transform,
200 load_transform: desc.load_transform,
201 num_planes: desc.num_planes() as u32,
202 _padding: Default::default(),
203 }
204 }
205}
206
207pub struct Device {
210 raw: Box<dyn hal::DynDevice>,
211 pub(crate) adapter: Arc<Adapter>,
212 pub(crate) queue: OnceCellOrLock<Weak<Queue>>,
213 pub(crate) zero_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
214 pub(crate) empty_bgl: ManuallyDrop<Box<dyn hal::DynBindGroupLayout>>,
215 label: String,
217
218 pub(crate) command_allocator: command::CommandAllocator,
219
220 pub(crate) command_indices: RwLock<CommandIndices>,
221
222 pub(crate) last_successful_submission_index: hal::AtomicFenceValue,
232
233 pub(crate) fence: ManuallyDrop<Box<dyn hal::DynFence>>,
234 pub(crate) snatchable_lock: SnatchLock,
235
236 pub(crate) valid: AtomicBool,
248
249 pub(crate) device_lost_closure: Mutex<Option<DeviceLostClosure>>,
253
254 pub(crate) trackers: Mutex<DeviceTracker>,
256 pub(crate) tracker_indices: TrackerIndexAllocators,
257 pub(crate) bgl_pool: ResourcePool<bgl::EntryMap, BindGroupLayout>,
259 pub(crate) alignments: hal::Alignments,
260 pub(crate) limits: wgt::Limits,
261 pub(crate) features: wgt::Features,
262 pub(crate) downlevel: wgt::DownlevelCapabilities,
263 pub(crate) ordered_buffer_usages: wgt::BufferUses,
268 pub(crate) ordered_texture_usages: wgt::TextureUses,
273 pub(crate) instance_flags: wgt::InstanceFlags,
274 pub(crate) deferred_destroy: Mutex<Vec<DeferredDestroy>>,
275 pub(crate) usage_scopes: UsageScopePool,
276 pub(crate) indirect_validation: Option<crate::indirect_validation::IndirectValidation>,
277 pub(crate) timestamp_normalizer:
279 OnceCellOrLock<crate::timestamp_normalization::TimestampNormalizer>,
280 pub(crate) default_external_texture_params_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
285 #[cfg(feature = "trace")]
287 pub(crate) trace: Mutex<Option<Box<dyn trace::Trace + Send + Sync + 'static>>>,
288}
289
290pub(crate) enum DeferredDestroy {
291 TextureViews(WeakVec<TextureView>),
292 BindGroups(WeakVec<BindGroup>),
293}
294
295impl fmt::Debug for Device {
296 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
297 f.debug_struct("Device")
298 .field("label", &self.label())
299 .field("limits", &self.limits)
300 .field("features", &self.features)
301 .field("downlevel", &self.downlevel)
302 .finish()
303 }
304}
305
306impl Drop for Device {
307 fn drop(&mut self) {
308 resource_log!("Drop {}", self.error_ident());
309
310 let zero_buffer = unsafe { ManuallyDrop::take(&mut self.zero_buffer) };
313 let empty_bgl = unsafe { ManuallyDrop::take(&mut self.empty_bgl) };
315 let default_external_texture_params_buffer =
318 unsafe { ManuallyDrop::take(&mut self.default_external_texture_params_buffer) };
319 let fence = unsafe { ManuallyDrop::take(&mut self.fence) };
321 if let Some(indirect_validation) = self.indirect_validation.take() {
322 indirect_validation.dispose(self.raw.as_ref());
323 }
324 if let Some(timestamp_normalizer) = self.timestamp_normalizer.take() {
325 timestamp_normalizer.dispose(self.raw.as_ref());
326 }
327 unsafe {
328 self.raw.destroy_buffer(zero_buffer);
329 self.raw.destroy_bind_group_layout(empty_bgl);
330 self.raw
331 .destroy_buffer(default_external_texture_params_buffer);
332 self.raw.destroy_fence(fence);
333 }
334 }
335}
336
337impl Device {
338 pub(crate) fn raw(&self) -> &dyn hal::DynDevice {
339 self.raw.as_ref()
340 }
341 pub(crate) fn require_features(&self, feature: wgt::Features) -> Result<(), MissingFeatures> {
342 if self.features.contains(feature) {
343 Ok(())
344 } else {
345 Err(MissingFeatures(feature))
346 }
347 }
348
349 pub(crate) fn require_downlevel_flags(
350 &self,
351 flags: wgt::DownlevelFlags,
352 ) -> Result<(), MissingDownlevelFlags> {
353 if self.downlevel.flags.contains(flags) {
354 Ok(())
355 } else {
356 Err(MissingDownlevelFlags(flags))
357 }
358 }
359
360 pub unsafe fn start_graphics_debugger_capture(&self) {
366 api_log!("Device::start_graphics_debugger_capture");
367
368 if !self.is_valid() {
369 return;
370 }
371 unsafe { self.raw().start_graphics_debugger_capture() };
372 }
373
374 pub unsafe fn stop_graphics_debugger_capture(&self) {
380 api_log!("Device::stop_graphics_debugger_capture");
381
382 if !self.is_valid() {
383 return;
384 }
385 unsafe { self.raw().stop_graphics_debugger_capture() };
386 }
387}
388
389impl Device {
390 pub(crate) fn new(
391 raw_device: Box<dyn hal::DynDevice>,
392 adapter: &Arc<Adapter>,
393 desc: &DeviceDescriptor,
394 instance_flags: wgt::InstanceFlags,
395 ) -> Result<Self, DeviceError> {
396 #[cfg(not(feature = "trace"))]
397 match &desc.trace {
398 wgt::Trace::Off => {}
399 _ => {
400 log::error!("wgpu-core feature 'trace' is not enabled");
401 }
402 };
403 #[cfg(feature = "trace")]
404 let trace: Option<Box<dyn trace::Trace + Send + Sync + 'static>> = match &desc.trace {
405 wgt::Trace::Off => None,
406 wgt::Trace::Directory(dir) => match trace::DiskTrace::new(dir.clone()) {
407 Ok(mut trace) => {
408 trace::Trace::add(
409 &mut trace,
410 trace::Action::Init {
411 desc: wgt::DeviceDescriptor {
412 trace: wgt::Trace::Off,
413 ..desc.clone()
414 },
415 backend: adapter.backend(),
416 },
417 );
418 Some(Box::new(trace))
419 }
420 Err(e) => {
421 log::error!("Unable to start a trace in '{dir:?}': {e}");
422 None
423 }
424 },
425 wgt::Trace::Memory => {
426 let mut trace = trace::MemoryTrace::new();
427 trace::Trace::add(
428 &mut trace,
429 trace::Action::Init {
430 desc: wgt::DeviceDescriptor {
431 trace: wgt::Trace::Off,
432 ..desc.clone()
433 },
434 backend: adapter.backend(),
435 },
436 );
437 Some(Box::new(trace))
438 }
439 t => {
442 log::error!("unimplemented wgpu_types::Trace variant {t:?}");
443 None
444 }
445 };
446
447 let ordered_buffer_usages = adapter.raw.adapter.get_ordered_buffer_usages();
448 let ordered_texture_usages = adapter.raw.adapter.get_ordered_texture_usages();
449
450 let fence = unsafe { raw_device.create_fence() }.map_err(DeviceError::from_hal)?;
451
452 let command_allocator = command::CommandAllocator::new();
453
454 let rt_uses = if desc
455 .required_features
456 .intersects(wgt::Features::EXPERIMENTAL_RAY_QUERY)
457 {
458 wgt::BufferUses::TOP_LEVEL_ACCELERATION_STRUCTURE_INPUT
459 } else {
460 wgt::BufferUses::empty()
461 };
462
463 let zero_buffer = unsafe {
465 raw_device.create_buffer(&hal::BufferDescriptor {
466 label: hal_label(Some("(wgpu internal) zero init buffer"), instance_flags),
467 size: ZERO_BUFFER_SIZE,
468 usage: wgt::BufferUses::COPY_SRC | wgt::BufferUses::COPY_DST | rt_uses,
469 memory_flags: hal::MemoryFlags::empty(),
470 })
471 }
472 .map_err(DeviceError::from_hal)?;
473
474 let empty_bgl = unsafe {
475 raw_device.create_bind_group_layout(&hal::BindGroupLayoutDescriptor {
476 label: None,
477 flags: hal::BindGroupLayoutFlags::empty(),
478 entries: &[],
479 })
480 }
481 .map_err(DeviceError::from_hal)?;
482
483 let default_external_texture_params_buffer = unsafe {
484 raw_device.create_buffer(&hal::BufferDescriptor {
485 label: hal_label(
486 Some("(wgpu internal) default external texture params buffer"),
487 instance_flags,
488 ),
489 size: size_of::<ExternalTextureParams>() as _,
490 usage: wgt::BufferUses::COPY_DST | wgt::BufferUses::UNIFORM,
491 memory_flags: hal::MemoryFlags::empty(),
492 })
493 }
494 .map_err(DeviceError::from_hal)?;
495
496 let alignments = adapter.raw.capabilities.alignments.clone();
498 let downlevel = adapter.raw.capabilities.downlevel.clone();
499 let limits = &adapter.raw.capabilities.limits;
500
501 let enable_indirect_validation = instance_flags
502 .contains(wgt::InstanceFlags::VALIDATION_INDIRECT_CALL)
503 && downlevel.flags.contains(
504 wgt::DownlevelFlags::INDIRECT_EXECUTION | wgt::DownlevelFlags::COMPUTE_SHADERS,
505 )
506 && limits.max_storage_buffers_per_shader_stage >= 2;
507
508 let indirect_validation = if enable_indirect_validation {
509 Some(crate::indirect_validation::IndirectValidation::new(
510 raw_device.as_ref(),
511 &desc.required_limits,
512 &desc.required_features,
513 instance_flags,
514 adapter.backend(),
515 )?)
516 } else {
517 None
518 };
519
520 Ok(Self {
521 raw: raw_device,
522 adapter: adapter.clone(),
523 queue: OnceCellOrLock::new(),
524 zero_buffer: ManuallyDrop::new(zero_buffer),
525 empty_bgl: ManuallyDrop::new(empty_bgl),
526 default_external_texture_params_buffer: ManuallyDrop::new(
527 default_external_texture_params_buffer,
528 ),
529 label: desc.label.to_string(),
530 command_allocator,
531 command_indices: RwLock::new(
532 rank::DEVICE_COMMAND_INDICES,
533 CommandIndices {
534 active_submission_index: 0,
535 next_acceleration_structure_build_command_index: 1,
537 },
538 ),
539 last_successful_submission_index: AtomicU64::new(0),
540 fence: ManuallyDrop::new(fence),
541 snatchable_lock: unsafe { SnatchLock::new(rank::DEVICE_SNATCHABLE_LOCK) },
542 valid: AtomicBool::new(true),
543 device_lost_closure: Mutex::new(rank::DEVICE_LOST_CLOSURE, None),
544 trackers: Mutex::new(
545 rank::DEVICE_TRACKERS,
546 DeviceTracker::new(ordered_buffer_usages, ordered_texture_usages),
547 ),
548 tracker_indices: TrackerIndexAllocators::new(),
549 bgl_pool: ResourcePool::new(),
550 #[cfg(feature = "trace")]
551 trace: Mutex::new(rank::DEVICE_TRACE, trace),
552 alignments,
553 limits: desc.required_limits.clone(),
554 features: desc.required_features,
555 downlevel,
556 ordered_buffer_usages,
557 ordered_texture_usages,
558 instance_flags,
559 deferred_destroy: Mutex::new(rank::DEVICE_DEFERRED_DESTROY, Vec::new()),
560 usage_scopes: Mutex::new(rank::DEVICE_USAGE_SCOPES, Default::default()),
561 timestamp_normalizer: OnceCellOrLock::new(),
562 indirect_validation,
563 })
564 }
565
566 fn init_default_external_texture_params_buffer(self: &Arc<Self>) -> Result<(), DeviceError> {
571 let data = ExternalTextureParams {
572 #[rustfmt::skip]
573 yuv_conversion_matrix: [
574 1.0, 0.0, 0.0, 0.0,
575 0.0, 1.0, 0.0, 0.0,
576 0.0, 0.0, 1.0, 0.0,
577 0.0, 0.0, 0.0, 1.0,
578 ],
579 #[rustfmt::skip]
580 gamut_conversion_matrix: [
581 1.0, 0.0, 0.0, 0.0,
582 0.0, 1.0, 0.0, 0.0,
583 0.0, 0.0, 1.0, 0.0,
584 ],
585 src_transfer_function: Default::default(),
586 dst_transfer_function: Default::default(),
587 size: [0, 0],
588 #[rustfmt::skip]
589 sample_transform: [
590 1.0, 0.0,
591 0.0, 1.0,
592 0.0, 0.0
593 ],
594 #[rustfmt::skip]
595 load_transform: [
596 1.0, 0.0,
597 0.0, 1.0,
598 0.0, 0.0
599 ],
600 num_planes: 1,
601 _padding: Default::default(),
602 };
603 let mut staging_buffer =
604 StagingBuffer::new(self, wgt::BufferSize::new(size_of_val(&data) as _).unwrap())?;
605 staging_buffer.write(bytemuck::bytes_of(&data));
606 let staging_buffer = staging_buffer.flush();
607
608 let params_buffer = self.default_external_texture_params_buffer.as_ref();
609 let queue = self.get_queue().unwrap();
610 let mut pending_writes = queue.pending_writes.lock();
611
612 unsafe {
613 pending_writes
614 .command_encoder
615 .transition_buffers(&[hal::BufferBarrier {
616 buffer: params_buffer,
617 usage: hal::StateTransition {
618 from: wgt::BufferUses::MAP_WRITE,
619 to: wgt::BufferUses::COPY_DST,
620 },
621 }]);
622 pending_writes.command_encoder.copy_buffer_to_buffer(
623 staging_buffer.raw(),
624 params_buffer,
625 &[hal::BufferCopy {
626 src_offset: 0,
627 dst_offset: 0,
628 size: staging_buffer.size,
629 }],
630 );
631 pending_writes.consume(staging_buffer);
632 pending_writes
633 .command_encoder
634 .transition_buffers(&[hal::BufferBarrier {
635 buffer: params_buffer,
636 usage: hal::StateTransition {
637 from: wgt::BufferUses::COPY_DST,
638 to: wgt::BufferUses::UNIFORM,
639 },
640 }]);
641 }
642
643 Ok(())
644 }
645
646 pub fn late_init_resources_with_queue(self: &Arc<Self>) -> Result<(), RequestDeviceError> {
647 let queue = self.get_queue().unwrap();
648
649 let timestamp_normalizer = crate::timestamp_normalization::TimestampNormalizer::new(
650 self,
651 queue.get_timestamp_period(),
652 )?;
653
654 self.timestamp_normalizer
655 .set(timestamp_normalizer)
656 .unwrap_or_else(|_| panic!("Called late_init_resources_with_queue twice"));
657
658 self.init_default_external_texture_params_buffer()?;
659
660 Ok(())
661 }
662
663 pub fn backend(&self) -> wgt::Backend {
665 self.adapter.backend()
666 }
667
668 pub fn is_valid(&self) -> bool {
669 self.valid.load(Ordering::Acquire)
670 }
671
672 pub fn check_is_valid(&self) -> Result<(), DeviceError> {
673 if self.is_valid() {
674 Ok(())
675 } else {
676 Err(DeviceError::Lost)
677 }
678 }
679
680 #[cfg(feature = "trace")]
684 pub fn take_trace(&self) -> Option<Box<dyn trace::Trace + Send + Sync + 'static>> {
685 self.trace.lock().take()
686 }
687
688 pub fn lose_if_oom(&self) {
695 let _ = self
696 .raw()
697 .check_if_oom()
698 .map_err(|e| self.handle_hal_error(e));
699 }
700
701 pub fn handle_hal_error(&self, error: hal::DeviceError) -> DeviceError {
702 match error {
703 hal::DeviceError::OutOfMemory
704 | hal::DeviceError::Lost
705 | hal::DeviceError::Unexpected => {
706 self.lose(&error.to_string());
707 }
708 }
709 DeviceError::from_hal(error)
710 }
711
712 pub fn handle_hal_error_with_nonfatal_oom(&self, error: hal::DeviceError) -> DeviceError {
713 match error {
714 hal::DeviceError::OutOfMemory => DeviceError::from_hal(error),
715 error => self.handle_hal_error(error),
716 }
717 }
718
719 pub(crate) fn deferred_resource_destruction(&self) {
727 let deferred_destroy = mem::take(&mut *self.deferred_destroy.lock());
729 for item in deferred_destroy {
730 match item {
731 DeferredDestroy::TextureViews(views) => {
732 for view in views {
733 let Some(view) = view.upgrade() else {
734 continue;
735 };
736 let Some(raw_view) = view.raw.snatch(&mut self.snatchable_lock.write())
737 else {
738 continue;
739 };
740
741 resource_log!("Destroy raw {}", view.error_ident());
742
743 unsafe {
744 self.raw().destroy_texture_view(raw_view);
745 }
746 }
747 }
748 DeferredDestroy::BindGroups(bind_groups) => {
749 for bind_group in bind_groups {
750 let Some(bind_group) = bind_group.upgrade() else {
751 continue;
752 };
753 let Some(raw_bind_group) =
754 bind_group.raw.snatch(&mut self.snatchable_lock.write())
755 else {
756 continue;
757 };
758
759 resource_log!("Destroy raw {}", bind_group.error_ident());
760
761 unsafe {
762 self.raw().destroy_bind_group(raw_bind_group);
763 }
764 }
765 }
766 }
767 }
768 }
769
770 pub fn get_queue(&self) -> Option<Arc<Queue>> {
771 self.queue.get().as_ref()?.upgrade()
772 }
773
774 pub fn set_queue(&self, queue: &Arc<Queue>) {
775 assert!(self.queue.set(Arc::downgrade(queue)).is_ok());
776 }
777
778 pub fn poll(
779 &self,
780 poll_type: wgt::PollType<crate::SubmissionIndex>,
781 ) -> Result<wgt::PollStatus, WaitIdleError> {
782 let (user_closures, result) = self.poll_and_return_closures(poll_type);
783 user_closures.fire();
784 result
785 }
786
787 pub(crate) fn poll_and_return_closures(
794 &self,
795 poll_type: wgt::PollType<crate::SubmissionIndex>,
796 ) -> (UserClosures, Result<wgt::PollStatus, WaitIdleError>) {
797 let snatch_guard = self.snatchable_lock.read();
798 let maintain_result = self.maintain(poll_type, snatch_guard);
799
800 self.lose_if_oom();
801
802 self.deferred_resource_destruction();
805
806 maintain_result
807 }
808
809 pub(crate) fn maintain<'this>(
827 &'this self,
828 poll_type: wgt::PollType<crate::SubmissionIndex>,
829 snatch_guard: SnatchGuard,
830 ) -> (UserClosures, Result<wgt::PollStatus, WaitIdleError>) {
831 profiling::scope!("Device::maintain");
832
833 let mut user_closures = UserClosures::default();
834
835 let wait_submission_index = match poll_type {
837 wgt::PollType::Wait {
838 submission_index: Some(submission_index),
839 ..
840 } => {
841 let last_successful_submission_index = self
842 .last_successful_submission_index
843 .load(Ordering::Acquire);
844
845 if submission_index > last_successful_submission_index {
846 let result = Err(WaitIdleError::WrongSubmissionIndex(
847 submission_index,
848 last_successful_submission_index,
849 ));
850
851 return (user_closures, result);
852 }
853
854 Some(submission_index)
855 }
856 wgt::PollType::Wait {
857 submission_index: None,
858 ..
859 } => Some(
860 self.last_successful_submission_index
861 .load(Ordering::Acquire),
862 ),
863 wgt::PollType::Poll => None,
864 };
865
866 if let Some(target_submission_index) = wait_submission_index {
868 log::trace!("Device::maintain: waiting for submission index {target_submission_index}");
869
870 let wait_timeout = match poll_type {
871 wgt::PollType::Wait { timeout, .. } => timeout,
872 wgt::PollType::Poll => unreachable!(
873 "`wait_submission_index` index for poll type `Poll` should be None"
874 ),
875 };
876
877 let wait_result = unsafe {
878 self.raw()
879 .wait(self.fence.as_ref(), target_submission_index, wait_timeout)
880 };
881
882 if let Err(e) = wait_result {
885 let hal_error: WaitIdleError = self.handle_hal_error(e).into();
886 return (user_closures, Err(hal_error));
887 }
888 }
889
890 let fence_value_result = unsafe { self.raw().get_fence_value(self.fence.as_ref()) };
893 let current_finished_submission = match fence_value_result {
894 Ok(fence_value) => fence_value,
895 Err(e) => {
896 let hal_error: WaitIdleError = self.handle_hal_error(e).into();
897 return (user_closures, Err(hal_error));
898 }
899 };
900
901 let command_indices = self.command_indices.read();
903 let device_valid = self.is_valid();
908 drop(command_indices);
909
910 let mut queue_empty = false;
916 if let Some(queue) = self.get_queue() {
917 let queue_result = queue.maintain(current_finished_submission, &snatch_guard);
918 (
919 user_closures.submissions,
920 user_closures.mappings,
921 user_closures.blas_compact_ready,
922 queue_empty,
923 ) = queue_result;
924 drop(snatch_guard);
938 } else {
939 drop(snatch_guard);
940 };
941
942 let result = if queue_empty {
945 if let Some(wait_submission_index) = wait_submission_index {
946 assert!(
949 current_finished_submission >= wait_submission_index,
950 concat!(
951 "If the queue is empty, the current submission index ",
952 "({}) should be at least the wait submission index ({})",
953 ),
954 current_finished_submission,
955 wait_submission_index,
956 );
957 }
958
959 Ok(wgt::PollStatus::QueueEmpty)
960 } else if let Some(wait_submission_index) = wait_submission_index {
961 if current_finished_submission >= wait_submission_index {
965 Ok(wgt::PollStatus::WaitSucceeded)
966 } else {
967 Err(WaitIdleError::Timeout)
968 }
969 } else {
970 Ok(wgt::PollStatus::Poll)
971 };
972
973 let mut should_release_gpu_resource = false;
980 if !device_valid && queue_empty {
981 should_release_gpu_resource = true;
984
985 if let Some(device_lost_closure) = self.device_lost_closure.lock().take() {
988 user_closures
989 .device_lost_invocations
990 .push(DeviceLostInvocation {
991 closure: device_lost_closure,
992 reason: DeviceLostReason::Destroyed,
993 message: String::new(),
994 });
995 }
996 }
997
998 if should_release_gpu_resource {
999 self.release_gpu_resources();
1000 }
1001
1002 (user_closures, result)
1003 }
1004
1005 pub fn create_buffer(
1006 self: &Arc<Self>,
1007 desc: &resource::BufferDescriptor,
1008 ) -> Result<Arc<Buffer>, resource::CreateBufferError> {
1009 self.check_is_valid()?;
1010
1011 if desc.size > self.limits.max_buffer_size {
1012 return Err(resource::CreateBufferError::MaxBufferSize {
1013 requested: desc.size,
1014 maximum: self.limits.max_buffer_size,
1015 });
1016 }
1017
1018 if desc
1019 .usage
1020 .intersects(wgt::BufferUsages::BLAS_INPUT | wgt::BufferUsages::TLAS_INPUT)
1021 {
1022 self.require_features(wgt::Features::EXPERIMENTAL_RAY_QUERY)?;
1023 }
1024
1025 if desc.usage.contains(wgt::BufferUsages::INDEX)
1026 && desc.usage.contains(
1027 wgt::BufferUsages::VERTEX
1028 | wgt::BufferUsages::UNIFORM
1029 | wgt::BufferUsages::INDIRECT
1030 | wgt::BufferUsages::STORAGE,
1031 )
1032 {
1033 self.require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)?;
1034 }
1035
1036 if desc.usage.is_empty() || desc.usage.contains_unknown_bits() {
1037 return Err(resource::CreateBufferError::InvalidUsage(desc.usage));
1038 }
1039
1040 if !self
1041 .features
1042 .contains(wgt::Features::MAPPABLE_PRIMARY_BUFFERS)
1043 {
1044 use wgt::BufferUsages as Bu;
1045 let write_mismatch = desc.usage.contains(Bu::MAP_WRITE)
1046 && !(Bu::MAP_WRITE | Bu::COPY_SRC).contains(desc.usage);
1047 let read_mismatch = desc.usage.contains(Bu::MAP_READ)
1048 && !(Bu::MAP_READ | Bu::COPY_DST).contains(desc.usage);
1049 if write_mismatch || read_mismatch {
1050 return Err(resource::CreateBufferError::UsageMismatch(desc.usage));
1051 }
1052 }
1053
1054 let mut usage = conv::map_buffer_usage(desc.usage);
1055
1056 if desc.usage.contains(wgt::BufferUsages::INDIRECT) {
1057 self.require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)?;
1058 usage |= wgt::BufferUses::STORAGE_READ_ONLY | wgt::BufferUses::STORAGE_READ_WRITE;
1061 }
1062
1063 if desc.usage.contains(wgt::BufferUsages::QUERY_RESOLVE) {
1064 usage |= TIMESTAMP_NORMALIZATION_BUFFER_USES;
1065 }
1066
1067 if desc.mapped_at_creation {
1068 if !desc.size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
1069 return Err(resource::CreateBufferError::UnalignedSize);
1070 }
1071 if !desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
1072 usage |= wgt::BufferUses::COPY_DST;
1074 }
1075 } else {
1076 usage |= wgt::BufferUses::COPY_DST;
1079 }
1080
1081 let actual_size = if desc.size == 0 {
1082 wgt::COPY_BUFFER_ALIGNMENT
1083 } else if desc.usage.contains(wgt::BufferUsages::VERTEX) {
1084 desc.size + 1
1087 } else {
1088 desc.size
1089 };
1090 let clear_remainder = actual_size % wgt::COPY_BUFFER_ALIGNMENT;
1091 let aligned_size = if clear_remainder != 0 {
1092 actual_size + wgt::COPY_BUFFER_ALIGNMENT - clear_remainder
1093 } else {
1094 actual_size
1095 };
1096
1097 let hal_desc = hal::BufferDescriptor {
1098 label: desc.label.to_hal(self.instance_flags),
1099 size: aligned_size,
1100 usage,
1101 memory_flags: hal::MemoryFlags::empty(),
1102 };
1103 let buffer = unsafe { self.raw().create_buffer(&hal_desc) }
1104 .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
1105
1106 let timestamp_normalization_bind_group = Snatchable::new(unsafe {
1107 self.timestamp_normalizer
1109 .get()
1110 .unwrap()
1111 .create_normalization_bind_group(
1112 self,
1113 &*buffer,
1114 desc.label.as_deref(),
1115 wgt::BufferSize::new(hal_desc.size).unwrap(),
1116 desc.usage,
1117 )
1118 }?);
1119
1120 let indirect_validation_bind_groups =
1121 self.create_indirect_validation_bind_groups(buffer.as_ref(), desc.size, desc.usage)?;
1122
1123 let buffer = Buffer {
1124 raw: Snatchable::new(buffer),
1125 device: self.clone(),
1126 usage: desc.usage,
1127 size: desc.size,
1128 initialization_status: RwLock::new(
1129 rank::BUFFER_INITIALIZATION_STATUS,
1130 BufferInitTracker::new(aligned_size),
1131 ),
1132 map_state: Mutex::new(rank::BUFFER_MAP_STATE, resource::BufferMapState::Idle),
1133 label: desc.label.to_string(),
1134 tracking_data: TrackingData::new(self.tracker_indices.buffers.clone()),
1135 bind_groups: Mutex::new(rank::BUFFER_BIND_GROUPS, WeakVec::new()),
1136 timestamp_normalization_bind_group,
1137 indirect_validation_bind_groups,
1138 };
1139
1140 let buffer = Arc::new(buffer);
1141
1142 let buffer_use = if !desc.mapped_at_creation {
1143 wgt::BufferUses::empty()
1144 } else if desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
1145 let map_size = buffer.size;
1147 let mapping = if map_size == 0 {
1148 hal::BufferMapping {
1149 ptr: core::ptr::NonNull::dangling(),
1150 is_coherent: true,
1151 }
1152 } else {
1153 let snatch_guard: SnatchGuard = self.snatchable_lock.read();
1154 map_buffer(&buffer, 0, map_size, HostMap::Write, &snatch_guard)?
1155 };
1156 *buffer.map_state.lock() = resource::BufferMapState::Active {
1157 mapping,
1158 range: 0..map_size,
1159 host: HostMap::Write,
1160 };
1161 wgt::BufferUses::MAP_WRITE
1162 } else {
1163 let mut staging_buffer =
1164 StagingBuffer::new(self, wgt::BufferSize::new(aligned_size).unwrap())?;
1165
1166 staging_buffer.write_zeros();
1169 buffer.initialization_status.write().drain(0..aligned_size);
1170
1171 *buffer.map_state.lock() = resource::BufferMapState::Init { staging_buffer };
1172 wgt::BufferUses::COPY_DST
1173 };
1174
1175 self.trackers
1176 .lock()
1177 .buffers
1178 .insert_single(&buffer, buffer_use);
1179
1180 Ok(buffer)
1181 }
1182
1183 #[cfg(feature = "replay")]
1184 pub fn set_buffer_data(
1185 self: &Arc<Self>,
1186 buffer: &Arc<Buffer>,
1187 offset: wgt::BufferAddress,
1188 data: &[u8],
1189 ) -> resource::BufferAccessResult {
1190 use crate::resource::RawResourceAccess;
1191
1192 let device = &buffer.device;
1193
1194 device.check_is_valid()?;
1195 buffer.check_usage(wgt::BufferUsages::MAP_WRITE)?;
1196
1197 let last_submission = device
1198 .get_queue()
1199 .and_then(|queue| queue.lock_life().get_buffer_latest_submission_index(buffer));
1200
1201 if let Some(last_submission) = last_submission {
1202 device.wait_for_submit(last_submission)?;
1203 }
1204
1205 let snatch_guard = device.snatchable_lock.read();
1206 let raw_buf = buffer.try_raw(&snatch_guard)?;
1207
1208 if offset > buffer.size {
1209 return Err(resource::BufferAccessError::OutOfBoundsStartOffsetOverrun {
1210 index: offset,
1211 max: buffer.size,
1212 });
1213 } else if buffer.size - offset < u64::try_from(data.len()).unwrap() {
1214 return Err(resource::BufferAccessError::OutOfBoundsEndOffsetOverrun {
1215 index: offset,
1216 size: u64::try_from(data.len()).unwrap(),
1217 max: buffer.size,
1218 });
1219 }
1220
1221 let mapping = unsafe {
1222 device
1223 .raw()
1224 .map_buffer(raw_buf, offset..offset + u64::try_from(data.len()).unwrap())
1225 }
1226 .map_err(|e| device.handle_hal_error(e))?;
1227
1228 unsafe { core::ptr::copy_nonoverlapping(data.as_ptr(), mapping.ptr.as_ptr(), data.len()) };
1229
1230 if !mapping.is_coherent {
1231 #[allow(clippy::single_range_in_vec_init)]
1232 unsafe {
1233 device
1234 .raw()
1235 .flush_mapped_ranges(raw_buf, &[offset..offset + data.len() as u64])
1236 };
1237 }
1238
1239 unsafe { device.raw().unmap_buffer(raw_buf) };
1240
1241 Ok(())
1242 }
1243
1244 pub(crate) fn create_texture_from_hal(
1245 self: &Arc<Self>,
1246 hal_texture: Box<dyn hal::DynTexture>,
1247 desc: &resource::TextureDescriptor,
1248 initial_state: wgt::TextureUses,
1249 ) -> Result<Arc<Texture>, resource::CreateTextureError> {
1250 let format_features = self
1251 .describe_format_features(desc.format)
1252 .map_err(|error| resource::CreateTextureError::MissingFeatures(desc.format, error))?;
1253
1254 unsafe { self.raw().add_raw_texture(&*hal_texture) };
1255
1256 let texture = Texture::new(
1257 self,
1258 resource::TextureInner::Native { raw: hal_texture },
1259 conv::map_texture_usage(desc.usage, desc.format.into(), format_features.flags),
1260 desc,
1261 format_features,
1262 resource::TextureClearMode::None,
1263 false,
1264 );
1265
1266 let texture = Arc::new(texture);
1267
1268 self.trackers
1269 .lock()
1270 .textures
1271 .insert_single(&texture, initial_state);
1272
1273 Ok(texture)
1274 }
1275
1276 pub(crate) unsafe fn create_buffer_from_hal(
1283 self: &Arc<Self>,
1284 hal_buffer: Box<dyn hal::DynBuffer>,
1285 desc: &resource::BufferDescriptor,
1286 ) -> (Fallible<Buffer>, Option<resource::CreateBufferError>) {
1287 let timestamp_normalization_bind_group = unsafe {
1288 match self
1289 .timestamp_normalizer
1290 .get()
1291 .unwrap()
1292 .create_normalization_bind_group(
1293 self,
1294 &*hal_buffer,
1295 desc.label.as_deref(),
1296 wgt::BufferSize::new(desc.size).unwrap(),
1297 desc.usage,
1298 ) {
1299 Ok(bg) => Snatchable::new(bg),
1300 Err(e) => {
1301 return (
1302 Fallible::Invalid(Arc::new(desc.label.to_string())),
1303 Some(e.into()),
1304 )
1305 }
1306 }
1307 };
1308
1309 let indirect_validation_bind_groups = match self.create_indirect_validation_bind_groups(
1310 hal_buffer.as_ref(),
1311 desc.size,
1312 desc.usage,
1313 ) {
1314 Ok(ok) => ok,
1315 Err(e) => return (Fallible::Invalid(Arc::new(desc.label.to_string())), Some(e)),
1316 };
1317
1318 unsafe { self.raw().add_raw_buffer(&*hal_buffer) };
1319
1320 let buffer = Buffer {
1321 raw: Snatchable::new(hal_buffer),
1322 device: self.clone(),
1323 usage: desc.usage,
1324 size: desc.size,
1325 initialization_status: RwLock::new(
1326 rank::BUFFER_INITIALIZATION_STATUS,
1327 BufferInitTracker::new(0),
1328 ),
1329 map_state: Mutex::new(rank::BUFFER_MAP_STATE, resource::BufferMapState::Idle),
1330 label: desc.label.to_string(),
1331 tracking_data: TrackingData::new(self.tracker_indices.buffers.clone()),
1332 bind_groups: Mutex::new(rank::BUFFER_BIND_GROUPS, WeakVec::new()),
1333 timestamp_normalization_bind_group,
1334 indirect_validation_bind_groups,
1335 };
1336
1337 let buffer = Arc::new(buffer);
1338
1339 self.trackers
1340 .lock()
1341 .buffers
1342 .insert_single(&buffer, wgt::BufferUses::empty());
1343
1344 (Fallible::Valid(buffer), None)
1345 }
1346
1347 fn create_indirect_validation_bind_groups(
1348 &self,
1349 raw_buffer: &dyn hal::DynBuffer,
1350 buffer_size: u64,
1351 usage: wgt::BufferUsages,
1352 ) -> Result<Snatchable<crate::indirect_validation::BindGroups>, resource::CreateBufferError>
1353 {
1354 if !usage.contains(wgt::BufferUsages::INDIRECT) {
1355 return Ok(Snatchable::empty());
1356 }
1357
1358 let Some(ref indirect_validation) = self.indirect_validation else {
1359 return Ok(Snatchable::empty());
1360 };
1361
1362 let bind_groups = crate::indirect_validation::BindGroups::new(
1363 indirect_validation,
1364 self,
1365 buffer_size,
1366 raw_buffer,
1367 )
1368 .map_err(resource::CreateBufferError::IndirectValidationBindGroup)?;
1369
1370 if let Some(bind_groups) = bind_groups {
1371 Ok(Snatchable::new(bind_groups))
1372 } else {
1373 Ok(Snatchable::empty())
1374 }
1375 }
1376
1377 fn create_texture_inner(
1378 self: &Arc<Self>,
1379 desc: &resource::TextureDescriptor,
1380 ) -> Result<Arc<Texture>, resource::CreateTextureError> {
1381 use resource::{CreateTextureError, TextureDimensionError};
1382
1383 self.check_is_valid()?;
1384
1385 if desc.usage.is_empty() || desc.usage.contains_unknown_bits() {
1386 return Err(CreateTextureError::InvalidUsage(desc.usage));
1387 }
1388
1389 conv::check_texture_dimension_size(
1390 desc.dimension,
1391 desc.size,
1392 desc.sample_count,
1393 &self.limits,
1394 )?;
1395
1396 if desc.dimension != wgt::TextureDimension::D2 {
1397 if desc.format.is_depth_stencil_format() {
1399 return Err(CreateTextureError::InvalidDepthDimension(
1400 desc.dimension,
1401 desc.format,
1402 ));
1403 }
1404 if desc
1406 .usage
1407 .contains(wgt::TextureUsages::TRANSIENT_ATTACHMENT)
1408 {
1409 return Err(CreateTextureError::InvalidDimensionUsages(
1410 wgt::TextureUsages::TRANSIENT_ATTACHMENT,
1411 desc.dimension,
1412 ));
1413 }
1414 }
1415
1416 if desc.dimension != wgt::TextureDimension::D2
1417 && desc.dimension != wgt::TextureDimension::D3
1418 {
1419 if desc.format.is_compressed() {
1421 return Err(CreateTextureError::InvalidCompressedDimension(
1422 desc.dimension,
1423 desc.format,
1424 ));
1425 }
1426
1427 if desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
1429 return Err(CreateTextureError::InvalidDimensionUsages(
1430 wgt::TextureUsages::RENDER_ATTACHMENT,
1431 desc.dimension,
1432 ));
1433 }
1434 }
1435
1436 if desc.format.is_compressed() {
1437 let (block_width, block_height) = desc.format.block_dimensions();
1438
1439 if !desc.size.width.is_multiple_of(block_width) {
1440 return Err(CreateTextureError::InvalidDimension(
1441 TextureDimensionError::NotMultipleOfBlockWidth {
1442 width: desc.size.width,
1443 block_width,
1444 format: desc.format,
1445 },
1446 ));
1447 }
1448
1449 if !desc.size.height.is_multiple_of(block_height) {
1450 return Err(CreateTextureError::InvalidDimension(
1451 TextureDimensionError::NotMultipleOfBlockHeight {
1452 height: desc.size.height,
1453 block_height,
1454 format: desc.format,
1455 },
1456 ));
1457 }
1458
1459 if desc.dimension == wgt::TextureDimension::D3 {
1460 if desc.format.is_bcn() {
1462 self.require_features(wgt::Features::TEXTURE_COMPRESSION_BC_SLICED_3D)
1463 .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1464 } else if desc.format.is_astc() {
1465 self.require_features(wgt::Features::TEXTURE_COMPRESSION_ASTC_SLICED_3D)
1466 .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1467 } else {
1468 return Err(CreateTextureError::InvalidCompressedDimension(
1469 desc.dimension,
1470 desc.format,
1471 ));
1472 }
1473 }
1474 }
1475
1476 let mips = desc.mip_level_count;
1477 let max_levels_allowed = desc.size.max_mips(desc.dimension).min(hal::MAX_MIP_LEVELS);
1478 if mips == 0 || mips > max_levels_allowed {
1479 return Err(CreateTextureError::InvalidMipLevelCount {
1480 requested: mips,
1481 maximum: max_levels_allowed,
1482 });
1483 }
1484
1485 {
1486 let (mut width_multiple, mut height_multiple) = desc.format.size_multiple_requirement();
1487
1488 if desc.format.is_multi_planar_format() {
1489 width_multiple <<= desc.mip_level_count.saturating_sub(1);
1493 height_multiple <<= desc.mip_level_count.saturating_sub(1);
1494 }
1495
1496 if !desc.size.width.is_multiple_of(width_multiple) {
1497 return Err(CreateTextureError::InvalidDimension(
1498 TextureDimensionError::WidthNotMultipleOf {
1499 width: desc.size.width,
1500 multiple: width_multiple,
1501 format: desc.format,
1502 },
1503 ));
1504 }
1505
1506 if !desc.size.height.is_multiple_of(height_multiple) {
1507 return Err(CreateTextureError::InvalidDimension(
1508 TextureDimensionError::HeightNotMultipleOf {
1509 height: desc.size.height,
1510 multiple: height_multiple,
1511 format: desc.format,
1512 },
1513 ));
1514 }
1515 }
1516
1517 if desc
1518 .usage
1519 .contains(wgt::TextureUsages::TRANSIENT_ATTACHMENT)
1520 {
1521 if desc.usage
1522 != (wgt::TextureUsages::TRANSIENT_ATTACHMENT
1523 | wgt::TextureUsages::RENDER_ATTACHMENT)
1524 {
1525 return Err(CreateTextureError::InvalidTransientTextureUsage(desc.usage));
1526 }
1527
1528 if desc.mip_level_count != 1 {
1529 return Err(CreateTextureError::InvalidTransientTextureMipLevelCount(
1530 desc.mip_level_count,
1531 ));
1532 }
1533
1534 if desc.size.depth_or_array_layers != 1 {
1535 return Err(CreateTextureError::InvalidTransientTextureLayerCount(
1536 desc.size.depth_or_array_layers,
1537 ));
1538 }
1539
1540 if !desc.view_formats.is_empty() {
1541 return Err(CreateTextureError::InvalidTransientTextureViewFormats);
1542 }
1543 }
1544
1545 let format_features = self
1546 .describe_format_features(desc.format)
1547 .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1548
1549 if desc.sample_count > 1 {
1550 if desc.mip_level_count != 1 {
1556 return Err(CreateTextureError::InvalidMipLevelCount {
1557 requested: desc.mip_level_count,
1558 maximum: 1,
1559 });
1560 }
1561
1562 if desc.size.depth_or_array_layers != 1
1563 && !self.features.contains(wgt::Features::MULTISAMPLE_ARRAY)
1564 {
1565 return Err(CreateTextureError::InvalidDimension(
1566 TextureDimensionError::MultisampledDepthOrArrayLayer(
1567 desc.size.depth_or_array_layers,
1568 ),
1569 ));
1570 }
1571
1572 if desc.usage.contains(wgt::TextureUsages::STORAGE_BINDING) {
1573 return Err(CreateTextureError::InvalidMultisampledStorageBinding);
1574 }
1575
1576 if !desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
1577 return Err(CreateTextureError::MultisampledNotRenderAttachment);
1578 }
1579
1580 if !format_features.flags.intersects(
1581 wgt::TextureFormatFeatureFlags::MULTISAMPLE_X4
1582 | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X2
1583 | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X8
1584 | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X16,
1585 ) {
1586 return Err(CreateTextureError::InvalidMultisampledFormat(desc.format));
1587 }
1588
1589 if !format_features
1590 .flags
1591 .sample_count_supported(desc.sample_count)
1592 {
1593 return Err(CreateTextureError::InvalidSampleCount(
1594 desc.sample_count,
1595 desc.format,
1596 desc.format
1597 .guaranteed_format_features(self.features)
1598 .flags
1599 .supported_sample_counts(),
1600 self.adapter
1601 .get_texture_format_features(desc.format)
1602 .flags
1603 .supported_sample_counts(),
1604 ));
1605 };
1606 }
1607
1608 let missing_allowed_usages = match desc.format.planes() {
1609 Some(planes) => {
1610 let mut planes_usages = wgt::TextureUsages::all();
1611 for plane in 0..planes {
1612 let aspect = wgt::TextureAspect::from_plane(plane).unwrap();
1613 let format = desc.format.aspect_specific_format(aspect).unwrap();
1614 let format_features = self
1615 .describe_format_features(format)
1616 .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1617
1618 planes_usages &= format_features.allowed_usages;
1619 }
1620
1621 desc.usage - planes_usages
1622 }
1623 None => desc.usage - format_features.allowed_usages,
1624 };
1625
1626 if !missing_allowed_usages.is_empty() {
1627 let wgpu_allowed_usages = desc
1629 .format
1630 .guaranteed_format_features(self.features)
1631 .allowed_usages;
1632 let wgpu_missing_usages = desc.usage - wgpu_allowed_usages;
1633 return Err(CreateTextureError::InvalidFormatUsages(
1634 missing_allowed_usages,
1635 desc.format,
1636 wgpu_missing_usages.is_empty(),
1637 ));
1638 }
1639
1640 let mut hal_view_formats = Vec::new();
1641 for format in desc.view_formats.iter() {
1642 if desc.format == *format {
1643 continue;
1644 }
1645 if desc.format.remove_srgb_suffix() != format.remove_srgb_suffix() {
1646 return Err(CreateTextureError::InvalidViewFormat(*format, desc.format));
1647 }
1648 hal_view_formats.push(*format);
1649 }
1650 if !hal_view_formats.is_empty() {
1651 self.require_downlevel_flags(wgt::DownlevelFlags::VIEW_FORMATS)?;
1652 }
1653
1654 let hal_usage = conv::map_texture_usage_for_texture(desc, &format_features);
1655
1656 let hal_desc = hal::TextureDescriptor {
1657 label: desc.label.to_hal(self.instance_flags),
1658 size: desc.size,
1659 mip_level_count: desc.mip_level_count,
1660 sample_count: desc.sample_count,
1661 dimension: desc.dimension,
1662 format: desc.format,
1663 usage: hal_usage,
1664 memory_flags: hal::MemoryFlags::empty(),
1665 view_formats: hal_view_formats,
1666 };
1667
1668 let raw_texture = unsafe { self.raw().create_texture(&hal_desc) }
1669 .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
1670
1671 let clear_mode = if hal_usage
1672 .intersects(wgt::TextureUses::DEPTH_STENCIL_WRITE | wgt::TextureUses::COLOR_TARGET)
1673 && desc.dimension == wgt::TextureDimension::D2
1674 {
1675 let (is_color, usage) = if desc.format.is_depth_stencil_format() {
1676 (false, wgt::TextureUses::DEPTH_STENCIL_WRITE)
1677 } else {
1678 (true, wgt::TextureUses::COLOR_TARGET)
1679 };
1680
1681 let clear_label = hal_label(
1682 Some("(wgpu internal) clear texture view"),
1683 self.instance_flags,
1684 );
1685
1686 let mut clear_views = SmallVec::new();
1687 for mip_level in 0..desc.mip_level_count {
1688 for array_layer in 0..desc.size.depth_or_array_layers {
1689 macro_rules! push_clear_view {
1690 ($format:expr, $aspect:expr) => {
1691 let desc = hal::TextureViewDescriptor {
1692 label: clear_label,
1693 format: $format,
1694 dimension: TextureViewDimension::D2,
1695 usage,
1696 range: wgt::ImageSubresourceRange {
1697 aspect: $aspect,
1698 base_mip_level: mip_level,
1699 mip_level_count: Some(1),
1700 base_array_layer: array_layer,
1701 array_layer_count: Some(1),
1702 },
1703 };
1704 clear_views.push(ManuallyDrop::new(
1705 unsafe {
1706 self.raw().create_texture_view(raw_texture.as_ref(), &desc)
1707 }
1708 .map_err(|e| self.handle_hal_error(e))?,
1709 ));
1710 };
1711 }
1712
1713 if let Some(planes) = desc.format.planes() {
1714 for plane in 0..planes {
1715 let aspect = wgt::TextureAspect::from_plane(plane).unwrap();
1716 let format = desc.format.aspect_specific_format(aspect).unwrap();
1717 push_clear_view!(format, aspect);
1718 }
1719 } else {
1720 push_clear_view!(desc.format, wgt::TextureAspect::All);
1721 }
1722 }
1723 }
1724 resource::TextureClearMode::RenderPass {
1725 clear_views,
1726 is_color,
1727 }
1728 } else {
1729 resource::TextureClearMode::BufferCopy
1730 };
1731
1732 let texture = Texture::new(
1733 self,
1734 resource::TextureInner::Native { raw: raw_texture },
1735 hal_usage,
1736 desc,
1737 format_features,
1738 clear_mode,
1739 true,
1740 );
1741
1742 let texture = Arc::new(texture);
1743
1744 self.trackers
1745 .lock()
1746 .textures
1747 .insert_single(&texture, wgt::TextureUses::UNINITIALIZED);
1748
1749 Ok(texture)
1750 }
1751
1752 pub fn create_texture(
1753 self: &Arc<Self>,
1754 desc: &resource::TextureDescriptor,
1755 ) -> (Arc<Texture>, Option<resource::CreateTextureError>) {
1756 let (texture, error) = match self.create_texture_inner(desc) {
1757 Ok(texture) => (texture, None),
1758 Err(e) => {
1759 let texture = Texture::invalid(self, desc);
1760 (Arc::new(texture), Some(e))
1761 }
1762 };
1763 api_log!(
1764 "Device::create_texture({desc:?}) -> {:?}",
1765 Arc::as_ptr(&texture)
1766 );
1767
1768 #[cfg(feature = "trace")]
1769 if let Some(ref mut trace) = *self.trace.lock() {
1770 use crate::device::trace::IntoTrace as _;
1771
1772 trace.add(trace::Action::CreateTexture(
1773 texture.to_trace(),
1774 desc.clone(),
1775 ));
1776 }
1777 (texture, error)
1778 }
1779
1780 pub fn create_texture_error(
1782 self: &Arc<Self>,
1783 desc: &resource::TextureDescriptor,
1784 ) -> Arc<Texture> {
1785 let texture = Arc::new(Texture::invalid(self, desc));
1786 #[cfg(feature = "trace")]
1787 if let Some(ref mut trace) = *self.trace.lock() {
1788 use crate::device::trace::IntoTrace as _;
1789
1790 trace.add(trace::Action::CreateTextureError(
1791 texture.to_trace(),
1792 desc.clone(),
1793 ));
1794 }
1795 texture
1796 }
1797
1798 pub fn create_texture_view(
1799 self: &Arc<Self>,
1800 texture: &Arc<Texture>,
1801 desc: &resource::TextureViewDescriptor,
1802 ) -> Result<Arc<TextureView>, resource::CreateTextureViewError> {
1803 self.check_is_valid()?;
1804
1805 let snatch_guard = texture.device.snatchable_lock.read();
1806
1807 let texture_raw = texture.try_inner(&snatch_guard)?.raw();
1808
1809 let resolved_format = desc.format.unwrap_or_else(|| {
1812 texture
1813 .desc
1814 .format
1815 .aspect_specific_format(desc.range.aspect)
1816 .unwrap_or(texture.desc.format)
1817 });
1818
1819 let resolved_dimension = desc
1820 .dimension
1821 .unwrap_or_else(|| match texture.desc.dimension {
1822 wgt::TextureDimension::D1 => TextureViewDimension::D1,
1823 wgt::TextureDimension::D2 => {
1824 if texture.desc.array_layer_count() == 1 {
1825 TextureViewDimension::D2
1826 } else {
1827 TextureViewDimension::D2Array
1828 }
1829 }
1830 wgt::TextureDimension::D3 => TextureViewDimension::D3,
1831 });
1832
1833 let resolved_mip_level_count = desc.range.mip_level_count.unwrap_or_else(|| {
1834 texture
1835 .desc
1836 .mip_level_count
1837 .saturating_sub(desc.range.base_mip_level)
1838 });
1839
1840 let resolved_array_layer_count =
1841 desc.range
1842 .array_layer_count
1843 .unwrap_or_else(|| match resolved_dimension {
1844 TextureViewDimension::D1
1845 | TextureViewDimension::D2
1846 | TextureViewDimension::D3 => 1,
1847 TextureViewDimension::Cube => 6,
1848 TextureViewDimension::D2Array | TextureViewDimension::CubeArray => texture
1849 .desc
1850 .array_layer_count()
1851 .saturating_sub(desc.range.base_array_layer),
1852 });
1853
1854 let resolved_usage = {
1855 let usage = desc.usage.unwrap_or(wgt::TextureUsages::empty());
1856 if usage.is_empty() {
1857 texture.desc.usage
1858 } else if texture.desc.usage.contains(usage) {
1859 if texture
1861 .desc
1862 .usage
1863 .contains(wgt::TextureUsages::TRANSIENT_ATTACHMENT)
1864 && texture.desc.usage != usage
1865 {
1866 return Err(
1867 resource::CreateTextureViewError::InvalidTransientTextureViewUsage {
1868 texture: texture.desc.usage,
1869 view: usage,
1870 },
1871 );
1872 }
1873
1874 usage
1875 } else {
1876 return Err(resource::CreateTextureViewError::InvalidTextureViewUsage {
1877 view: usage,
1878 texture: texture.desc.usage,
1879 });
1880 }
1881 };
1882
1883 let format_features = self.describe_format_features(resolved_format)?;
1884 let allowed_format_usages = format_features.allowed_usages;
1885 if resolved_usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT)
1886 && !allowed_format_usages.contains(wgt::TextureUsages::RENDER_ATTACHMENT)
1887 {
1888 return Err(
1889 resource::CreateTextureViewError::TextureViewFormatNotRenderable(resolved_format),
1890 );
1891 }
1892
1893 if resolved_usage.contains(wgt::TextureUsages::STORAGE_BINDING)
1894 && !allowed_format_usages.contains(wgt::TextureUsages::STORAGE_BINDING)
1895 {
1896 return Err(
1897 resource::CreateTextureViewError::TextureViewFormatNotStorage(resolved_format),
1898 );
1899 }
1900
1901 let aspects = hal::FormatAspects::new(texture.desc.format, desc.range.aspect);
1904 if aspects.is_empty() {
1905 return Err(resource::CreateTextureViewError::InvalidAspect {
1906 texture_format: texture.desc.format,
1907 requested_aspect: desc.range.aspect,
1908 });
1909 }
1910
1911 let format_is_good = if desc.range.aspect == wgt::TextureAspect::All {
1912 resolved_format == texture.desc.format
1913 || texture.desc.view_formats.contains(&resolved_format)
1914 } else {
1915 Some(resolved_format)
1916 == texture
1917 .desc
1918 .format
1919 .aspect_specific_format(desc.range.aspect)
1920 };
1921 if !format_is_good {
1922 return Err(resource::CreateTextureViewError::FormatReinterpretation {
1923 texture: texture.desc.format,
1924 view: resolved_format,
1925 });
1926 }
1927
1928 if texture.desc.sample_count > 1 && resolved_dimension != TextureViewDimension::D2 {
1930 let multisample_array_exception = resolved_dimension == TextureViewDimension::D2Array
1932 && self.features.contains(wgt::Features::MULTISAMPLE_ARRAY);
1933
1934 if !multisample_array_exception {
1935 return Err(
1936 resource::CreateTextureViewError::InvalidMultisampledTextureViewDimension(
1937 resolved_dimension,
1938 ),
1939 );
1940 }
1941 }
1942
1943 if texture.desc.dimension != resolved_dimension.compatible_texture_dimension() {
1945 return Err(
1946 resource::CreateTextureViewError::InvalidTextureViewDimension {
1947 view: resolved_dimension,
1948 texture: texture.desc.dimension,
1949 },
1950 );
1951 }
1952
1953 match resolved_dimension {
1954 TextureViewDimension::D1 | TextureViewDimension::D2 | TextureViewDimension::D3 => {
1955 if resolved_array_layer_count != 1 {
1956 return Err(resource::CreateTextureViewError::InvalidArrayLayerCount {
1957 requested: resolved_array_layer_count,
1958 dim: resolved_dimension,
1959 });
1960 }
1961 }
1962 TextureViewDimension::Cube => {
1963 if resolved_array_layer_count != 6 {
1964 return Err(
1965 resource::CreateTextureViewError::InvalidCubemapTextureDepth {
1966 depth: resolved_array_layer_count,
1967 },
1968 );
1969 }
1970 }
1971 TextureViewDimension::CubeArray => {
1972 if !resolved_array_layer_count.is_multiple_of(6) {
1973 return Err(
1974 resource::CreateTextureViewError::InvalidCubemapArrayTextureDepth {
1975 depth: resolved_array_layer_count,
1976 },
1977 );
1978 }
1979 }
1980 _ => {}
1981 }
1982
1983 match resolved_dimension {
1984 TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
1985 if texture.desc.size.width != texture.desc.size.height {
1986 return Err(resource::CreateTextureViewError::InvalidCubeTextureViewSize);
1987 }
1988 }
1989 _ => {}
1990 }
1991
1992 if resolved_mip_level_count == 0 {
1993 return Err(resource::CreateTextureViewError::ZeroMipLevelCount);
1994 }
1995
1996 let mip_level_end = desc
1997 .range
1998 .base_mip_level
1999 .saturating_add(resolved_mip_level_count);
2000
2001 let level_end = texture.desc.mip_level_count;
2002 if mip_level_end > level_end {
2003 return Err(resource::CreateTextureViewError::TooManyMipLevels {
2004 base_mip_level: desc.range.base_mip_level,
2005 mip_level_count: resolved_mip_level_count,
2006 total: level_end,
2007 });
2008 }
2009
2010 if resolved_array_layer_count == 0 {
2011 return Err(resource::CreateTextureViewError::ZeroArrayLayerCount);
2012 }
2013
2014 let array_layer_end = desc
2015 .range
2016 .base_array_layer
2017 .saturating_add(resolved_array_layer_count);
2018
2019 let layer_end = texture.desc.array_layer_count();
2020 if array_layer_end > layer_end {
2021 return Err(resource::CreateTextureViewError::TooManyArrayLayers {
2022 base_array_layer: desc.range.base_array_layer,
2023 array_layer_count: resolved_array_layer_count,
2024 total: layer_end,
2025 });
2026 };
2027
2028 let render_extent = 'error: {
2030 if !resolved_usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
2031 break 'error Err(TextureViewNotRenderableReason::Usage(resolved_usage));
2032 }
2033
2034 let allowed_view_dimensions = [
2035 TextureViewDimension::D2,
2036 TextureViewDimension::D2Array,
2037 TextureViewDimension::D3,
2038 ];
2039 if !allowed_view_dimensions.contains(&resolved_dimension) {
2040 break 'error Err(TextureViewNotRenderableReason::Dimension(
2041 resolved_dimension,
2042 ));
2043 }
2044
2045 if resolved_mip_level_count != 1 {
2046 break 'error Err(TextureViewNotRenderableReason::MipLevelCount(
2047 resolved_mip_level_count,
2048 ));
2049 }
2050
2051 if resolved_array_layer_count != 1
2052 && !(self.features.contains(wgt::Features::MULTIVIEW))
2053 {
2054 break 'error Err(TextureViewNotRenderableReason::ArrayLayerCount(
2055 resolved_array_layer_count,
2056 ));
2057 }
2058
2059 if !texture.desc.format.is_multi_planar_format()
2060 && aspects != hal::FormatAspects::from(texture.desc.format)
2061 {
2062 break 'error Err(TextureViewNotRenderableReason::Aspects(aspects));
2063 }
2064
2065 Ok(texture
2066 .desc
2067 .compute_render_extent(desc.range.base_mip_level, desc.range.aspect.to_plane()))
2068 };
2069
2070 let usage = {
2072 let resolved_hal_usage = conv::map_texture_usage(
2073 resolved_usage,
2074 resolved_format.into(),
2075 format_features.flags,
2076 );
2077 let mask_copy = !(wgt::TextureUses::COPY_SRC | wgt::TextureUses::COPY_DST);
2078 let mask_dimension = match resolved_dimension {
2079 TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
2080 wgt::TextureUses::RESOURCE
2081 }
2082 TextureViewDimension::D3 => {
2083 wgt::TextureUses::RESOURCE
2084 | wgt::TextureUses::STORAGE_READ_ONLY
2085 | wgt::TextureUses::STORAGE_WRITE_ONLY
2086 | wgt::TextureUses::STORAGE_READ_WRITE
2087 }
2088 _ => wgt::TextureUses::all(),
2089 };
2090 let mask_mip_level = if resolved_mip_level_count == 1 {
2091 wgt::TextureUses::all()
2092 } else {
2093 wgt::TextureUses::RESOURCE
2094 };
2095 resolved_hal_usage & mask_copy & mask_dimension & mask_mip_level
2096 };
2097
2098 let format = if resolved_format.is_depth_stencil_component(texture.desc.format) {
2100 texture.desc.format
2101 } else {
2102 resolved_format
2103 };
2104
2105 let resolved_range = wgt::ImageSubresourceRange {
2106 aspect: desc.range.aspect,
2107 base_mip_level: desc.range.base_mip_level,
2108 mip_level_count: Some(resolved_mip_level_count),
2109 base_array_layer: desc.range.base_array_layer,
2110 array_layer_count: Some(resolved_array_layer_count),
2111 };
2112
2113 let hal_desc = hal::TextureViewDescriptor {
2114 label: desc.label.to_hal(self.instance_flags),
2115 format,
2116 dimension: resolved_dimension,
2117 usage,
2118 range: resolved_range,
2119 };
2120
2121 let raw = unsafe { self.raw().create_texture_view(texture_raw, &hal_desc) }
2122 .map_err(|e| self.handle_hal_error(e))?;
2123
2124 let selector = TextureSelector {
2125 mips: desc.range.base_mip_level..mip_level_end,
2126 layers: desc.range.base_array_layer..array_layer_end,
2127 };
2128
2129 let view = TextureView {
2130 raw: Snatchable::new(raw),
2131 parent: texture.clone(),
2132 device: self.clone(),
2133 desc: resource::HalTextureViewDescriptor {
2134 texture_format: texture.desc.format,
2135 format: resolved_format,
2136 dimension: resolved_dimension,
2137 usage: resolved_usage,
2138 range: resolved_range,
2139 },
2140 format_features: texture.format_features,
2141 render_extent,
2142 samples: texture.desc.sample_count,
2143 selector,
2144 label: desc.label.to_string(),
2145 };
2146
2147 let view = Arc::new(view);
2148
2149 {
2150 let mut views = texture.views.lock();
2151 views.push(Arc::downgrade(&view));
2152 }
2153
2154 Ok(view)
2155 }
2156
2157 pub fn create_external_texture(
2158 self: &Arc<Self>,
2159 desc: &resource::ExternalTextureDescriptor,
2160 planes: &[Arc<TextureView>],
2161 ) -> Result<Arc<ExternalTexture>, resource::CreateExternalTextureError> {
2162 use resource::CreateExternalTextureError;
2163 self.require_features(wgt::Features::EXTERNAL_TEXTURE)?;
2164 self.check_is_valid()?;
2165
2166 if desc.num_planes() != planes.len() {
2167 return Err(CreateExternalTextureError::IncorrectPlaneCount {
2168 format: desc.format,
2169 expected: desc.num_planes(),
2170 provided: planes.len(),
2171 });
2172 }
2173
2174 let planes = planes
2175 .iter()
2176 .enumerate()
2177 .map(|(i, plane)| {
2178 if plane.samples != 1 {
2179 return Err(CreateExternalTextureError::InvalidPlaneMultisample(
2180 plane.samples,
2181 ));
2182 }
2183
2184 let sample_type = plane
2185 .desc
2186 .format
2187 .sample_type(Some(plane.desc.range.aspect), Some(self.features))
2188 .unwrap();
2189 if !matches!(sample_type, TextureSampleType::Float { filterable: true }) {
2190 return Err(CreateExternalTextureError::InvalidPlaneSampleType {
2191 format: plane.desc.format,
2192 sample_type,
2193 });
2194 }
2195
2196 if plane.desc.dimension != TextureViewDimension::D2 {
2197 return Err(CreateExternalTextureError::InvalidPlaneDimension(
2198 plane.desc.dimension,
2199 ));
2200 }
2201
2202 let expected_components = match desc.format {
2203 wgt::ExternalTextureFormat::Rgba => 4,
2204 wgt::ExternalTextureFormat::Nv12 => match i {
2205 0 => 1,
2206 1 => 2,
2207 _ => unreachable!(),
2208 },
2209 wgt::ExternalTextureFormat::Yu12 => 1,
2210 };
2211 if plane.desc.format.components() != expected_components {
2212 return Err(CreateExternalTextureError::InvalidPlaneFormat {
2213 format: desc.format,
2214 plane: i,
2215 expected: expected_components,
2216 provided: plane.desc.format,
2217 });
2218 }
2219
2220 plane.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
2221 Ok(plane.clone())
2222 })
2223 .collect::<Result<_, _>>()?;
2224
2225 let params_data = ExternalTextureParams::from_desc(desc);
2226 let label = desc.label.as_ref().map(|l| alloc::format!("{l} params"));
2227 let params_desc = resource::BufferDescriptor {
2228 label: label.map(Cow::Owned),
2229 size: size_of_val(¶ms_data) as wgt::BufferAddress,
2230 usage: wgt::BufferUsages::UNIFORM | wgt::BufferUsages::COPY_DST,
2231 mapped_at_creation: false,
2232 };
2233 let params = self.create_buffer(¶ms_desc)?;
2234 self.get_queue().unwrap().write_buffer(
2235 params.clone(),
2236 0,
2237 bytemuck::bytes_of(¶ms_data),
2238 )?;
2239
2240 let external_texture = ExternalTexture {
2241 device: self.clone(),
2242 planes,
2243 params,
2244 label: desc.label.to_string(),
2245 tracking_data: TrackingData::new(self.tracker_indices.external_textures.clone()),
2246 };
2247 let external_texture = Arc::new(external_texture);
2248
2249 Ok(external_texture)
2250 }
2251
2252 pub fn create_sampler(
2253 self: &Arc<Self>,
2254 desc: &resource::SamplerDescriptor,
2255 ) -> Result<Arc<Sampler>, resource::CreateSamplerError> {
2256 self.check_is_valid()?;
2257
2258 if desc
2259 .address_modes
2260 .iter()
2261 .any(|am| am == &wgt::AddressMode::ClampToBorder)
2262 {
2263 self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_BORDER)?;
2264 }
2265
2266 if desc.border_color == Some(wgt::SamplerBorderColor::Zero) {
2267 self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_ZERO)?;
2268 }
2269
2270 if desc.lod_min_clamp < 0.0 {
2271 return Err(resource::CreateSamplerError::InvalidLodMinClamp(
2272 desc.lod_min_clamp,
2273 ));
2274 }
2275 if desc.lod_max_clamp < desc.lod_min_clamp {
2276 return Err(resource::CreateSamplerError::InvalidLodMaxClamp {
2277 lod_min_clamp: desc.lod_min_clamp,
2278 lod_max_clamp: desc.lod_max_clamp,
2279 });
2280 }
2281
2282 if desc.anisotropy_clamp < 1 {
2283 return Err(resource::CreateSamplerError::InvalidAnisotropy(
2284 desc.anisotropy_clamp,
2285 ));
2286 }
2287
2288 if desc.anisotropy_clamp != 1 {
2289 if !matches!(desc.min_filter, wgt::FilterMode::Linear) {
2290 return Err(
2291 resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
2292 filter_type: resource::SamplerFilterErrorType::MinFilter,
2293 filter_mode: desc.min_filter,
2294 anisotropic_clamp: desc.anisotropy_clamp,
2295 },
2296 );
2297 }
2298 if !matches!(desc.mag_filter, wgt::FilterMode::Linear) {
2299 return Err(
2300 resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
2301 filter_type: resource::SamplerFilterErrorType::MagFilter,
2302 filter_mode: desc.mag_filter,
2303 anisotropic_clamp: desc.anisotropy_clamp,
2304 },
2305 );
2306 }
2307 if !matches!(desc.mipmap_filter, wgt::MipmapFilterMode::Linear) {
2308 return Err(
2309 resource::CreateSamplerError::InvalidMipmapFilterModeWithAnisotropy {
2310 filter_type: resource::SamplerFilterErrorType::MipmapFilter,
2311 filter_mode: desc.mipmap_filter,
2312 anisotropic_clamp: desc.anisotropy_clamp,
2313 },
2314 );
2315 }
2316 }
2317
2318 let anisotropy_clamp = if self
2319 .downlevel
2320 .flags
2321 .contains(wgt::DownlevelFlags::ANISOTROPIC_FILTERING)
2322 {
2323 desc.anisotropy_clamp.min(16)
2325 } else {
2326 1
2328 };
2329
2330 let hal_desc = hal::SamplerDescriptor {
2333 label: desc.label.to_hal(self.instance_flags),
2334 address_modes: desc.address_modes,
2335 mag_filter: desc.mag_filter,
2336 min_filter: desc.min_filter,
2337 mipmap_filter: desc.mipmap_filter,
2338 lod_clamp: desc.lod_min_clamp..desc.lod_max_clamp,
2339 compare: desc.compare,
2340 anisotropy_clamp,
2341 border_color: desc.border_color,
2342 };
2343
2344 let raw = unsafe { self.raw().create_sampler(&hal_desc) }
2345 .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
2346
2347 let sampler = Sampler {
2348 raw: ManuallyDrop::new(raw),
2349 device: self.clone(),
2350 label: desc.label.to_string(),
2351 tracking_data: TrackingData::new(self.tracker_indices.samplers.clone()),
2352 comparison: desc.compare.is_some(),
2353 filtering: desc.min_filter == wgt::FilterMode::Linear
2354 || desc.mag_filter == wgt::FilterMode::Linear
2355 || desc.mipmap_filter == wgt::MipmapFilterMode::Linear,
2356 };
2357
2358 let sampler = Arc::new(sampler);
2359
2360 Ok(sampler)
2361 }
2362
2363 pub fn create_shader_module<'a>(
2364 self: &Arc<Self>,
2365 desc: &pipeline::ShaderModuleDescriptor<'a>,
2366 source: pipeline::ShaderModuleSource<'a>,
2367 ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
2368 self.check_is_valid()?;
2369
2370 let (module, source) = match source {
2371 #[cfg(feature = "wgsl")]
2372 pipeline::ShaderModuleSource::Wgsl(code) => {
2373 profiling::scope!("naga::front::wgsl::parse");
2374 let capabilities =
2375 features_to_naga_capabilities(self.features, self.downlevel.flags);
2376 let mut options = naga::front::wgsl::Options::new();
2377 options.capabilities = capabilities;
2378 let mut frontend = naga::front::wgsl::Frontend::new_with_options(options);
2379 let module = frontend.parse(&code).map_err(|inner| {
2380 pipeline::CreateShaderModuleError::Parsing(naga::error::ShaderError {
2381 source: code.to_string(),
2382 label: desc.label.as_ref().map(|l| l.to_string()),
2383 inner: Box::new(inner),
2384 })
2385 })?;
2386 (Cow::Owned(module), code.into_owned())
2387 }
2388 #[cfg(feature = "spirv")]
2389 pipeline::ShaderModuleSource::SpirV(spv, options) => {
2390 let parser = naga::front::spv::Frontend::new(spv.iter().cloned(), &options);
2391 profiling::scope!("naga::front::spv::Frontend");
2392 let module = parser.parse().map_err(|inner| {
2393 pipeline::CreateShaderModuleError::ParsingSpirV(naga::error::ShaderError {
2394 source: String::new(),
2395 label: desc.label.as_ref().map(|l| l.to_string()),
2396 inner: Box::new(inner),
2397 })
2398 })?;
2399 (Cow::Owned(module), String::new())
2400 }
2401 #[cfg(feature = "glsl")]
2402 pipeline::ShaderModuleSource::Glsl(code, options) => {
2403 let mut parser = naga::front::glsl::Frontend::default();
2404 profiling::scope!("naga::front::glsl::Frontend.parse");
2405 let module = parser.parse(&options, &code).map_err(|inner| {
2406 pipeline::CreateShaderModuleError::ParsingGlsl(naga::error::ShaderError {
2407 source: code.to_string(),
2408 label: desc.label.as_ref().map(|l| l.to_string()),
2409 inner: Box::new(inner),
2410 })
2411 })?;
2412 (Cow::Owned(module), code.into_owned())
2413 }
2414 pipeline::ShaderModuleSource::Naga(module) => (module, String::new()),
2415 pipeline::ShaderModuleSource::Dummy(_) => panic!("found `ShaderModuleSource::Dummy`"),
2416 };
2417 for (_, var) in module.global_variables.iter() {
2418 match var.binding {
2419 Some(br) if br.group >= self.limits.max_bind_groups => {
2420 return Err(pipeline::CreateShaderModuleError::InvalidGroupIndex {
2421 bind: br,
2422 group: br.group,
2423 limit: self.limits.max_bind_groups,
2424 });
2425 }
2426 _ => continue,
2427 };
2428 }
2429
2430 profiling::scope!("naga::validate");
2431 let debug_source =
2432 if self.instance_flags.contains(wgt::InstanceFlags::DEBUG) && !source.is_empty() {
2433 Some(hal::DebugSource {
2434 file_name: Cow::Owned(
2435 desc.label
2436 .as_ref()
2437 .map_or("shader".to_string(), |l| l.to_string()),
2438 ),
2439 source_code: Cow::Owned(source.clone()),
2440 })
2441 } else {
2442 None
2443 };
2444
2445 let info = create_validator(
2446 self.features,
2447 self.downlevel.flags,
2448 naga::valid::ValidationFlags::all(),
2449 )
2450 .validate(&module)
2451 .map_err(|inner| {
2452 pipeline::CreateShaderModuleError::Validation(naga::error::ShaderError {
2453 source,
2454 label: desc.label.as_ref().map(|l| l.to_string()),
2455 inner: Box::new(inner),
2456 })
2457 })?;
2458
2459 let interface = validation::Interface::new(&module, &info, self.limits.clone());
2460 let hal_shader = hal::ShaderInput::Naga(hal::NagaShader {
2461 module,
2462 info,
2463 debug_source,
2464 });
2465 let hal_desc = hal::ShaderModuleDescriptor {
2466 label: desc.label.to_hal(self.instance_flags),
2467 runtime_checks: desc.runtime_checks,
2468 };
2469 let raw = match unsafe { self.raw().create_shader_module(&hal_desc, hal_shader) } {
2470 Ok(raw) => raw,
2471 Err(error) => {
2472 return Err(match error {
2473 hal::ShaderError::Device(error) => {
2474 pipeline::CreateShaderModuleError::Device(self.handle_hal_error(error))
2475 }
2476 hal::ShaderError::Compilation(ref msg) => {
2477 log::error!("Shader error: {msg}");
2478 pipeline::CreateShaderModuleError::Generation
2479 }
2480 })
2481 }
2482 };
2483
2484 let module = pipeline::ShaderModule {
2485 raw: ManuallyDrop::new(raw),
2486 device: self.clone(),
2487 interface: ShaderMetaData::Interface(interface),
2488 label: desc.label.to_string(),
2489 };
2490
2491 let module = Arc::new(module);
2492
2493 Ok(module)
2494 }
2495
2496 #[allow(unused_unsafe)]
2498 #[doc(hidden)]
2499 pub unsafe fn create_shader_module_passthrough<'a>(
2500 self: &Arc<Self>,
2501 descriptor: &pipeline::ShaderModuleDescriptorPassthrough<'a>,
2502 ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
2503 self.check_is_valid()?;
2504 self.require_features(wgt::Features::PASSTHROUGH_SHADERS)?;
2505
2506 if (descriptor.dxil.is_some() || descriptor.glsl.is_some())
2508 && descriptor.entry_points.len() != 1
2509 {
2510 return Err(pipeline::CreateShaderModuleError::IncorrectPassthroughEntryPointCount);
2511 }
2512
2513 let entry_point_hashmap = || {
2514 descriptor
2515 .entry_points
2516 .iter()
2517 .map(|e| (e.name.to_string(), e.workgroup_size))
2518 .collect()
2519 };
2520
2521 let hal_shader = match self.backend() {
2522 wgt::Backend::Vulkan => hal::ShaderInput::SpirV(
2523 descriptor
2524 .spirv
2525 .as_ref()
2526 .ok_or(pipeline::CreateShaderModuleError::NotCompiledForBackend)?,
2527 ),
2528 wgt::Backend::Dx12 => {
2529 if let Some(dxil) = &descriptor.dxil {
2530 hal::ShaderInput::Dxil { shader: dxil }
2531 } else if let Some(hlsl) = &descriptor.hlsl {
2532 hal::ShaderInput::Hlsl { shader: hlsl }
2533 } else {
2534 return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend);
2535 }
2536 }
2537 wgt::Backend::Metal => {
2538 if let Some(metallib) = &descriptor.metallib {
2539 hal::ShaderInput::MetalLib {
2540 file: metallib,
2541 num_workgroups: entry_point_hashmap(),
2542 }
2543 } else if let Some(msl) = &descriptor.msl {
2544 hal::ShaderInput::Msl {
2545 shader: msl,
2546 num_workgroups: entry_point_hashmap(),
2547 }
2548 } else {
2549 return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend);
2550 }
2551 }
2552 wgt::Backend::Gl => hal::ShaderInput::Glsl {
2553 shader: descriptor
2554 .glsl
2555 .as_ref()
2556 .ok_or(pipeline::CreateShaderModuleError::NotCompiledForBackend)?,
2557 },
2558 wgt::Backend::Noop => {
2559 return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend)
2560 }
2561 wgt::Backend::BrowserWebGpu => unreachable!(),
2562 };
2563
2564 let hal_desc = hal::ShaderModuleDescriptor {
2565 label: descriptor.label.to_hal(self.instance_flags),
2566 runtime_checks: wgt::ShaderRuntimeChecks::unchecked(),
2567 };
2568
2569 let raw = match unsafe { self.raw().create_shader_module(&hal_desc, hal_shader) } {
2570 Ok(raw) => raw,
2571 Err(error) => {
2572 return Err(match error {
2573 hal::ShaderError::Device(error) => {
2574 pipeline::CreateShaderModuleError::Device(self.handle_hal_error(error))
2575 }
2576 hal::ShaderError::Compilation(ref msg) => {
2577 log::error!("Shader error: {msg}");
2578 pipeline::CreateShaderModuleError::Generation
2579 }
2580 })
2581 }
2582 };
2583
2584 let module = pipeline::ShaderModule {
2585 raw: ManuallyDrop::new(raw),
2586 device: self.clone(),
2587 interface: ShaderMetaData::Passthrough(PassthroughInterface {
2588 entry_point_names: descriptor
2589 .entry_points
2590 .iter()
2591 .map(|e| e.name.to_string())
2592 .collect(),
2593 }),
2594 label: descriptor.label.to_string(),
2595 };
2596
2597 Ok(Arc::new(module))
2598 }
2599
2600 pub fn read_program_links(&self) -> bool {
2604 #[cfg(gles)]
2605 if let Some(device) = self.raw().as_any().downcast_ref::<hal::gles::Device>() {
2606 return device.read_program_links();
2607 }
2608 false
2609 }
2610
2611 pub unsafe fn create_shader_module_glsl_templates(
2618 self: &Arc<Self>,
2619 label: &crate::Label<'_>,
2620 templates: &'static [hal::GlslTemplate],
2621 ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
2622 self.check_is_valid()?;
2623 if self.backend() != wgt::Backend::Gl {
2624 return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend);
2625 }
2626 let hal_desc = hal::ShaderModuleDescriptor {
2627 label: label.to_hal(self.instance_flags),
2628 runtime_checks: wgt::ShaderRuntimeChecks::unchecked(),
2629 };
2630 let raw = unsafe {
2631 self.raw()
2632 .create_shader_module(&hal_desc, hal::ShaderInput::GlslTemplates(templates))
2633 }
2634 .map_err(|error| match error {
2635 hal::ShaderError::Device(error) => {
2636 pipeline::CreateShaderModuleError::Device(self.handle_hal_error(error))
2637 }
2638 hal::ShaderError::Compilation(ref msg) => {
2639 log::error!("Shader error: {msg}");
2640 pipeline::CreateShaderModuleError::Generation
2641 }
2642 })?;
2643 Ok(Arc::new(pipeline::ShaderModule {
2644 raw: ManuallyDrop::new(raw),
2645 device: self.clone(),
2646 interface: ShaderMetaData::Passthrough(PassthroughInterface {
2647 entry_point_names: templates
2648 .iter()
2649 .map(|template| template.entry_point.to_string())
2650 .collect(),
2651 }),
2652 label: label.to_string(),
2653 }))
2654 }
2655
2656 pub(crate) fn create_command_encoder(
2657 self: &Arc<Self>,
2658 label: &crate::Label,
2659 ) -> Result<Arc<command::CommandEncoder>, DeviceError> {
2660 self.check_is_valid()?;
2661
2662 let queue = self.get_queue().unwrap();
2663
2664 let encoder = self
2665 .command_allocator
2666 .acquire_encoder(self.raw(), queue.raw())
2667 .map_err(|e| self.handle_hal_error(e))?;
2668
2669 let cmd_enc = command::CommandEncoder::new(encoder, self, label);
2670
2671 let cmd_enc = Arc::new(cmd_enc);
2672
2673 Ok(cmd_enc)
2674 }
2675
2676 fn make_late_sized_buffer_groups(
2679 shader_binding_sizes: &FastHashMap<naga::ResourceBinding, wgt::BufferSize>,
2680 layout: &binding_model::PipelineLayout,
2681 ) -> ArrayVec<pipeline::LateSizedBufferGroup, { hal::MAX_BIND_GROUPS }> {
2682 layout
2686 .bind_group_layouts
2687 .iter()
2688 .enumerate()
2689 .map(|(group_index, bgl)| {
2690 let Some(bgl) = bgl else {
2691 return pipeline::LateSizedBufferGroup::default();
2692 };
2693
2694 let shader_sizes = bgl
2695 .entries
2696 .values()
2697 .filter_map(|entry| match entry.ty {
2698 wgt::BindingType::Buffer {
2699 min_binding_size: None,
2700 ..
2701 } => {
2702 let rb = naga::ResourceBinding {
2703 group: group_index as u32,
2704 binding: entry.binding,
2705 };
2706 let shader_size =
2707 shader_binding_sizes.get(&rb).map_or(0, |nz| nz.get());
2708 Some(shader_size)
2709 }
2710 _ => None,
2711 })
2712 .collect();
2713 pipeline::LateSizedBufferGroup { shader_sizes }
2714 })
2715 .collect()
2716 }
2717
2718 pub fn create_bind_group_layout(
2719 self: &Arc<Self>,
2720 desc: &binding_model::BindGroupLayoutDescriptor,
2721 ) -> (Arc<BindGroupLayout>, Option<CreateBindGroupLayoutError>) {
2722 let (bgl, error) = match self.create_bind_group_layout_inner(desc) {
2723 Ok(layout) => (layout, None),
2724 Err(e) => (
2725 BindGroupLayout::invalid(self, desc.label.to_string()),
2726 Some(e),
2727 ),
2728 };
2729 #[cfg(feature = "trace")]
2730 if let Some(ref mut trace) = *self.trace.lock() {
2731 use crate::device::trace::IntoTrace;
2732
2733 trace.add(trace::Action::CreateBindGroupLayout(
2734 bgl.to_trace(),
2735 desc.clone(),
2736 ));
2737 }
2738 (bgl, error)
2739 }
2740
2741 fn create_bind_group_layout_inner(
2742 self: &Arc<Device>,
2743 desc: &binding_model::BindGroupLayoutDescriptor,
2744 ) -> Result<Arc<BindGroupLayout>, CreateBindGroupLayoutError> {
2745 self.check_is_valid()?;
2746
2747 let entry_map = bgl::EntryMap::from_entries(&desc.entries)?;
2748
2749 let bgl_result = self.bgl_pool.get_or_init(entry_map, |entry_map| {
2750 let bgl =
2751 self.create_bind_group_layout_impl(&desc.label, entry_map, bgl::Origin::Pool)?;
2752 bgl.exclusive_pipeline
2753 .set(binding_model::ExclusivePipeline::None)
2754 .unwrap();
2755 Ok(bgl)
2756 });
2757
2758 match bgl_result {
2759 Ok(layout) => Ok(layout),
2760 Err(e) => Err(e),
2761 }
2762 }
2763
2764 fn create_bind_group_layout_impl(
2765 self: &Arc<Self>,
2766 label: &crate::Label,
2767 entry_map: bgl::EntryMap,
2768 origin: bgl::Origin,
2769 ) -> Result<Arc<BindGroupLayout>, CreateBindGroupLayoutError> {
2770 #[derive(PartialEq)]
2771 enum WritableStorage {
2772 Yes,
2773 No,
2774 }
2775
2776 for entry in entry_map.values() {
2777 if entry.binding >= self.limits.max_bindings_per_bind_group {
2778 return Err(CreateBindGroupLayoutError::InvalidBindingIndex {
2779 binding: entry.binding,
2780 maximum: self.limits.max_bindings_per_bind_group,
2781 });
2782 }
2783
2784 use wgt::BindingType as Bt;
2785
2786 let mut required_features = wgt::Features::empty();
2787 let mut required_downlevel_flags = wgt::DownlevelFlags::empty();
2788 let (array_feature, writable_storage) = match entry.ty {
2789 Bt::Buffer {
2790 ty: wgt::BufferBindingType::Uniform,
2791 has_dynamic_offset: false,
2792 min_binding_size: _,
2793 } => (
2794 Some(wgt::Features::BUFFER_BINDING_ARRAY),
2795 WritableStorage::No,
2796 ),
2797 Bt::Buffer {
2798 ty: wgt::BufferBindingType::Uniform,
2799 has_dynamic_offset: true,
2800 min_binding_size: _,
2801 } => (
2802 Some(wgt::Features::BUFFER_BINDING_ARRAY),
2803 WritableStorage::No,
2804 ),
2805 Bt::Buffer {
2806 ty: wgt::BufferBindingType::Storage { read_only },
2807 ..
2808 } => (
2809 Some(
2810 wgt::Features::BUFFER_BINDING_ARRAY
2811 | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY,
2812 ),
2813 match read_only {
2814 true => WritableStorage::No,
2815 false => WritableStorage::Yes,
2816 },
2817 ),
2818 Bt::Sampler { .. } => (
2819 Some(wgt::Features::TEXTURE_BINDING_ARRAY),
2820 WritableStorage::No,
2821 ),
2822 Bt::Texture {
2823 multisampled: true,
2824 sample_type: TextureSampleType::Float { filterable: true },
2825 ..
2826 } => {
2827 return Err(CreateBindGroupLayoutError::Entry {
2828 binding: entry.binding,
2829 error:
2830 BindGroupLayoutEntryError::SampleTypeFloatFilterableBindingMultisampled,
2831 });
2832 }
2833 Bt::Texture {
2834 multisampled,
2835 view_dimension,
2836 ..
2837 } => {
2838 if multisampled && view_dimension != TextureViewDimension::D2 {
2839 return Err(CreateBindGroupLayoutError::Entry {
2840 binding: entry.binding,
2841 error: BindGroupLayoutEntryError::Non2DMultisampled(view_dimension),
2842 });
2843 }
2844
2845 (
2846 Some(wgt::Features::TEXTURE_BINDING_ARRAY),
2847 WritableStorage::No,
2848 )
2849 }
2850 Bt::StorageTexture {
2851 access,
2852 view_dimension,
2853 format,
2854 } => {
2855 use wgt::{StorageTextureAccess as Access, TextureFormatFeatureFlags as Flags};
2856
2857 match view_dimension {
2858 TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
2859 return Err(CreateBindGroupLayoutError::Entry {
2860 binding: entry.binding,
2861 error: BindGroupLayoutEntryError::StorageTextureCube,
2862 })
2863 }
2864 _ => (),
2865 }
2866 match access {
2867 wgt::StorageTextureAccess::Atomic
2868 if !self.features.contains(wgt::Features::TEXTURE_ATOMIC) =>
2869 {
2870 return Err(CreateBindGroupLayoutError::Entry {
2871 binding: entry.binding,
2872 error: BindGroupLayoutEntryError::StorageTextureAtomic,
2873 });
2874 }
2875 _ => (),
2876 }
2877
2878 let format_features =
2879 self.describe_format_features(format).map_err(|error| {
2880 CreateBindGroupLayoutError::Entry {
2881 binding: entry.binding,
2882 error: BindGroupLayoutEntryError::MissingFeatures(error),
2883 }
2884 })?;
2885
2886 let required_feature_flag = match access {
2887 Access::WriteOnly => Flags::STORAGE_WRITE_ONLY,
2888 Access::ReadOnly => Flags::STORAGE_READ_ONLY,
2889 Access::ReadWrite => Flags::STORAGE_READ_WRITE,
2890 Access::Atomic => Flags::STORAGE_ATOMIC,
2891 };
2892
2893 if !format_features.flags.contains(required_feature_flag) {
2894 return Err(
2895 CreateBindGroupLayoutError::UnsupportedStorageTextureAccess {
2896 binding: entry.binding,
2897 access,
2898 format,
2899 },
2900 );
2901 }
2902
2903 (
2904 Some(
2905 wgt::Features::TEXTURE_BINDING_ARRAY
2906 | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY,
2907 ),
2908 match access {
2909 wgt::StorageTextureAccess::WriteOnly => WritableStorage::Yes,
2910 wgt::StorageTextureAccess::ReadOnly => WritableStorage::No,
2911 wgt::StorageTextureAccess::ReadWrite => WritableStorage::Yes,
2912 wgt::StorageTextureAccess::Atomic => {
2913 required_features |= wgt::Features::TEXTURE_ATOMIC;
2914 WritableStorage::Yes
2915 }
2916 },
2917 )
2918 }
2919 Bt::AccelerationStructure { vertex_return } => {
2920 self.require_features(wgt::Features::EXPERIMENTAL_RAY_QUERY)
2921 .map_err(|e| CreateBindGroupLayoutError::Entry {
2922 binding: entry.binding,
2923 error: e.into(),
2924 })?;
2925 if vertex_return {
2926 self.require_features(wgt::Features::EXPERIMENTAL_RAY_HIT_VERTEX_RETURN)
2927 .map_err(|e| CreateBindGroupLayoutError::Entry {
2928 binding: entry.binding,
2929 error: e.into(),
2930 })?;
2931 }
2932 (
2933 Some(wgt::Features::ACCELERATION_STRUCTURE_BINDING_ARRAY),
2934 WritableStorage::No,
2935 )
2936 }
2937 Bt::ExternalTexture => {
2938 self.require_features(wgt::Features::EXTERNAL_TEXTURE)
2939 .map_err(|e| CreateBindGroupLayoutError::Entry {
2940 binding: entry.binding,
2941 error: e.into(),
2942 })?;
2943 (None, WritableStorage::No)
2944 }
2945 };
2946
2947 if entry.count.is_some() {
2949 required_features |= array_feature
2950 .ok_or(BindGroupLayoutEntryError::ArrayUnsupported)
2951 .map_err(|error| CreateBindGroupLayoutError::Entry {
2952 binding: entry.binding,
2953 error,
2954 })?;
2955 }
2956
2957 if entry.visibility.contains_unknown_bits() {
2958 return Err(CreateBindGroupLayoutError::InvalidVisibility(
2959 entry.visibility,
2960 ));
2961 }
2962
2963 if entry.visibility.contains(wgt::ShaderStages::VERTEX) {
2964 if writable_storage == WritableStorage::Yes {
2965 required_features |= wgt::Features::VERTEX_WRITABLE_STORAGE;
2966 }
2967 if let Bt::Buffer {
2968 ty: wgt::BufferBindingType::Storage { .. },
2969 ..
2970 } = entry.ty
2971 {
2972 required_downlevel_flags |= wgt::DownlevelFlags::VERTEX_STORAGE;
2973 }
2974 }
2975 if writable_storage == WritableStorage::Yes
2976 && entry.visibility.contains(wgt::ShaderStages::FRAGMENT)
2977 {
2978 required_downlevel_flags |= wgt::DownlevelFlags::FRAGMENT_WRITABLE_STORAGE;
2979 }
2980
2981 self.require_features(required_features)
2982 .map_err(BindGroupLayoutEntryError::MissingFeatures)
2983 .map_err(|error| CreateBindGroupLayoutError::Entry {
2984 binding: entry.binding,
2985 error,
2986 })?;
2987 self.require_downlevel_flags(required_downlevel_flags)
2988 .map_err(BindGroupLayoutEntryError::MissingDownlevelFlags)
2989 .map_err(|error| CreateBindGroupLayoutError::Entry {
2990 binding: entry.binding,
2991 error,
2992 })?;
2993 }
2994
2995 let bgl_flags = conv::bind_group_layout_flags(self.features);
2996
2997 let hal_bindings = entry_map.values().copied().collect::<Vec<_>>();
2998 let hal_desc = hal::BindGroupLayoutDescriptor {
2999 label: label.to_hal(self.instance_flags),
3000 flags: bgl_flags,
3001 entries: &hal_bindings,
3002 };
3003
3004 let mut count_validator = binding_model::BindingTypeMaxCountValidator::default();
3005 for entry in entry_map.values() {
3006 count_validator.add_binding(entry);
3007 }
3008 count_validator
3011 .validate(&self.limits, self.instance_flags)
3012 .map_err(CreateBindGroupLayoutError::TooManyBindings)?;
3013
3014 count_validator.validate_binding_arrays()?;
3016
3017 let raw = unsafe { self.raw().create_bind_group_layout(&hal_desc) }
3018 .map_err(|e| self.handle_hal_error(e))?;
3019
3020 let bgl = BindGroupLayout {
3021 state: ResourceState::Valid(BindGroupLayoutState {
3022 raw: binding_model::RawBindGroupLayout::Owning(ManuallyDrop::new(raw)),
3023 origin,
3024 binding_count_validator: count_validator,
3025 }),
3026 device: self.clone(),
3027 entries: entry_map,
3028 exclusive_pipeline: OnceCellOrLock::new(),
3029 label: label.to_string(),
3030 };
3031
3032 let bgl = Arc::new(bgl);
3033
3034 Ok(bgl)
3035 }
3036
3037 fn create_buffer_binding<'a>(
3038 &self,
3039 bb: &'a binding_model::ResolvedBufferBinding,
3040 binding: u32,
3041 decl: &wgt::BindGroupLayoutEntry,
3042 buffer_init_actions: &mut Vec<BufferInitTrackerAction>,
3043 dynamic_binding_info: &mut Vec<binding_model::BindGroupDynamicBindingData>,
3044 late_buffer_binding_sizes: &mut FastHashMap<u32, wgt::BufferSize>,
3045 used: &mut BindGroupStates,
3046 snatch_guard: &'a SnatchGuard<'a>,
3047 ) -> Result<hal::BufferBinding<'a, dyn hal::DynBuffer>, CreateBindGroupError> {
3048 use crate::binding_model::CreateBindGroupError as Error;
3049
3050 let (binding_ty, dynamic, min_size) = match decl.ty {
3051 wgt::BindingType::Buffer {
3052 ty,
3053 has_dynamic_offset,
3054 min_binding_size,
3055 } => (ty, has_dynamic_offset, min_binding_size),
3056 _ => {
3057 return Err(Error::WrongBindingType {
3058 binding,
3059 actual: decl.ty,
3060 expected: "UniformBuffer, StorageBuffer or ReadonlyStorageBuffer",
3061 })
3062 }
3063 };
3064
3065 let (pub_usage, internal_use, range_limit) = match binding_ty {
3066 wgt::BufferBindingType::Uniform => (
3067 wgt::BufferUsages::UNIFORM,
3068 wgt::BufferUses::UNIFORM,
3069 self.limits.max_uniform_buffer_binding_size,
3070 ),
3071 wgt::BufferBindingType::Storage { read_only } => (
3072 wgt::BufferUsages::STORAGE,
3073 if read_only {
3074 wgt::BufferUses::STORAGE_READ_ONLY
3075 } else {
3076 wgt::BufferUses::STORAGE_READ_WRITE
3077 },
3078 self.limits.max_storage_buffer_binding_size,
3079 ),
3080 };
3081
3082 let (align, align_limit_name) =
3083 binding_model::buffer_binding_type_alignment(&self.limits, binding_ty);
3084 if !bb.offset.is_multiple_of(align as u64) {
3085 return Err(Error::UnalignedBufferOffset(
3086 bb.offset,
3087 align_limit_name,
3088 align,
3089 ));
3090 }
3091
3092 let buffer = &bb.buffer;
3093
3094 used.buffers.insert_single(buffer.clone(), internal_use);
3095
3096 buffer.same_device(self)?;
3097
3098 buffer.check_usage(pub_usage)?;
3099
3100 let req_size = match bb.size.map(wgt::BufferSize::new) {
3101 Some(non_zero @ Some(_)) => non_zero,
3103 None => None,
3105 Some(None) => return Err(CreateBindGroupError::BindingZeroSize(buffer.error_ident())),
3107 };
3108 let (bb, bind_size) = buffer.binding(bb.offset, req_size, snatch_guard)?;
3109
3110 if matches!(binding_ty, wgt::BufferBindingType::Storage { .. })
3111 && bind_size % u64::from(wgt::STORAGE_BINDING_SIZE_ALIGNMENT) != 0
3112 {
3113 return Err(Error::UnalignedEffectiveBufferBindingSizeForStorage {
3114 alignment: wgt::STORAGE_BINDING_SIZE_ALIGNMENT,
3115 size: bind_size,
3116 });
3117 }
3118
3119 let bind_end = bb.offset + bind_size;
3120
3121 if bind_size > range_limit {
3122 return Err(Error::BufferRangeTooLarge {
3123 binding,
3124 given: bind_size,
3125 limit: range_limit,
3126 });
3127 }
3128
3129 if dynamic {
3131 dynamic_binding_info.push(binding_model::BindGroupDynamicBindingData {
3132 binding_idx: binding,
3133 buffer_size: buffer.size,
3134 binding_range: bb.offset..bind_end,
3135 maximum_dynamic_offset: buffer.size - bind_end,
3136 binding_type: binding_ty,
3137 });
3138 }
3139
3140 if let Some(non_zero) = min_size {
3141 let min_size = non_zero.get();
3142 if min_size > bind_size {
3143 return Err(Error::BindingSizeTooSmall {
3144 buffer: buffer.error_ident(),
3145 actual: bind_size,
3146 min: min_size,
3147 });
3148 }
3149 } else {
3150 let late_size = wgt::BufferSize::new(bind_size)
3151 .ok_or_else(|| Error::BindingZeroSize(buffer.error_ident()))?;
3152 late_buffer_binding_sizes.insert(binding, late_size);
3153 }
3154
3155 assert_eq!(bb.offset % wgt::COPY_BUFFER_ALIGNMENT, 0);
3158
3159 let init_range = if dynamic {
3160 0..buffer.size
3163 } else {
3164 let bounds_check_alignment = binding_model::buffer_binding_type_bounds_check_alignment(
3169 &self.alignments,
3170 binding_ty,
3171 );
3172 let visible_size = align_to(bind_size, bounds_check_alignment);
3173
3174 bb.offset..bb.offset + visible_size
3175 };
3176
3177 buffer_init_actions.extend(buffer.initialization_status.read().create_action(
3178 buffer,
3179 init_range,
3180 MemoryInitKind::NeedsInitializedMemory,
3181 ));
3182
3183 Ok(bb)
3184 }
3185
3186 fn create_sampler_binding<'a>(
3187 &self,
3188 used: &mut BindGroupStates,
3189 binding: u32,
3190 decl: &wgt::BindGroupLayoutEntry,
3191 sampler: &'a Arc<Sampler>,
3192 ) -> Result<&'a dyn hal::DynSampler, CreateBindGroupError> {
3193 use crate::binding_model::CreateBindGroupError as Error;
3194
3195 used.samplers.insert_single(sampler.clone());
3196
3197 sampler.same_device(self)?;
3198
3199 match decl.ty {
3200 wgt::BindingType::Sampler(ty) => {
3201 let (allowed_filtering, allowed_comparison) = match ty {
3202 wgt::SamplerBindingType::Filtering => (None, false),
3203 wgt::SamplerBindingType::NonFiltering => (Some(false), false),
3204 wgt::SamplerBindingType::Comparison => (None, true),
3205 };
3206 if let Some(allowed_filtering) = allowed_filtering {
3207 if allowed_filtering != sampler.filtering {
3208 return Err(Error::WrongSamplerFiltering {
3209 binding,
3210 layout_flt: allowed_filtering,
3211 sampler_flt: sampler.filtering,
3212 });
3213 }
3214 }
3215 if allowed_comparison != sampler.comparison {
3216 return Err(Error::WrongSamplerComparison {
3217 binding,
3218 layout_cmp: allowed_comparison,
3219 sampler_cmp: sampler.comparison,
3220 });
3221 }
3222 }
3223 _ => {
3224 return Err(Error::WrongBindingType {
3225 binding,
3226 actual: decl.ty,
3227 expected: "Sampler",
3228 })
3229 }
3230 }
3231
3232 Ok(sampler.raw())
3233 }
3234
3235 fn create_texture_binding<'a>(
3236 &self,
3237 binding: u32,
3238 decl: &wgt::BindGroupLayoutEntry,
3239 view: &'a Arc<TextureView>,
3240 used: &mut BindGroupStates,
3241 texture_init_actions: &mut Vec<TextureInitTrackerAction>,
3242 snatch_guard: &'a SnatchGuard<'a>,
3243 ) -> Result<hal::TextureBinding<'a, dyn hal::DynTextureView>, CreateBindGroupError> {
3244 view.same_device(self)?;
3245
3246 let internal_use = self.texture_use_parameters(
3247 binding,
3248 decl,
3249 view,
3250 "SampledTexture, ReadonlyStorageTexture or WriteonlyStorageTexture",
3251 )?;
3252
3253 used.views.insert_single(view.clone(), internal_use);
3254
3255 let texture = &view.parent;
3256
3257 texture_init_actions.push(TextureInitTrackerAction {
3258 texture: texture.clone(),
3259 range: TextureInitRange {
3260 mip_range: view.desc.range.mip_range(texture.desc.mip_level_count),
3261 layer_range: view
3262 .desc
3263 .range
3264 .layer_range(texture.desc.array_layer_count()),
3265 },
3266 kind: MemoryInitKind::NeedsInitializedMemory,
3267 });
3268
3269 Ok(hal::TextureBinding {
3270 view: view.try_raw(snatch_guard)?,
3271 usage: internal_use,
3272 })
3273 }
3274
3275 fn create_tlas_binding<'a>(
3276 self: &Arc<Self>,
3277 used: &mut BindGroupStates,
3278 binding: u32,
3279 decl: &wgt::BindGroupLayoutEntry,
3280 tlas: &'a Arc<Tlas>,
3281 snatch_guard: &'a SnatchGuard<'a>,
3282 ) -> Result<&'a dyn hal::DynAccelerationStructure, CreateBindGroupError> {
3283 use crate::binding_model::CreateBindGroupError as Error;
3284
3285 used.acceleration_structures.insert_single(tlas.clone());
3286
3287 tlas.same_device(self)?;
3288
3289 match decl.ty {
3290 wgt::BindingType::AccelerationStructure { vertex_return } => {
3291 if vertex_return
3292 && !tlas.flags.contains(
3293 wgpu_types::AccelerationStructureFlags::ALLOW_RAY_HIT_VERTEX_RETURN,
3294 )
3295 {
3296 return Err(Error::MissingTLASVertexReturn { binding });
3297 }
3298 }
3299 _ => {
3300 return Err(Error::WrongBindingType {
3301 binding,
3302 actual: decl.ty,
3303 expected: "Tlas",
3304 });
3305 }
3306 }
3307
3308 Ok(tlas.try_raw(snatch_guard)?)
3309 }
3310
3311 #[inline(never)]
3312 fn create_external_texture_binding<'a>(
3313 &'a self,
3314 binding: u32,
3315 decl: &wgt::BindGroupLayoutEntry,
3316 external_texture: &'a Arc<ExternalTexture>,
3317 used: &mut BindGroupStates,
3318 snatch_guard: &'a SnatchGuard,
3319 ) -> Result<
3320 hal::ExternalTextureBinding<'a, dyn hal::DynBuffer, dyn hal::DynTextureView>,
3321 CreateBindGroupError,
3322 > {
3323 use crate::binding_model::CreateBindGroupError as Error;
3324
3325 external_texture.same_device(self)?;
3326
3327 used.external_textures
3328 .insert_single(external_texture.clone());
3329
3330 match decl.ty {
3331 wgt::BindingType::ExternalTexture => {}
3332 _ => {
3333 return Err(Error::WrongBindingType {
3334 binding,
3335 actual: decl.ty,
3336 expected: "ExternalTexture",
3337 });
3338 }
3339 }
3340
3341 let planes = (0..3)
3342 .map(|i| {
3343 let plane = external_texture
3347 .planes
3348 .get(i)
3349 .unwrap_or(&external_texture.planes[0]);
3350 let internal_use = wgt::TextureUses::RESOURCE;
3351 used.views.insert_single(plane.clone(), internal_use);
3352 let view = plane.try_raw(snatch_guard)?;
3353 Ok(hal::TextureBinding {
3354 view,
3355 usage: internal_use,
3356 })
3357 })
3358 .collect::<Result<Vec<_>, Error>>()?;
3361 let planes = planes.try_into().unwrap();
3362
3363 used.buffers
3364 .insert_single(external_texture.params.clone(), wgt::BufferUses::UNIFORM);
3365 let params = external_texture.params.binding(0, None, snatch_guard)?.0;
3366
3367 Ok(hal::ExternalTextureBinding { planes, params })
3368 }
3369
3370 fn create_external_texture_binding_from_view<'a>(
3371 &'a self,
3372 binding: u32,
3373 decl: &wgt::BindGroupLayoutEntry,
3374 view: &'a Arc<TextureView>,
3375 used: &mut BindGroupStates,
3376 snatch_guard: &'a SnatchGuard,
3377 ) -> Result<
3378 hal::ExternalTextureBinding<'a, dyn hal::DynBuffer, dyn hal::DynTextureView>,
3379 CreateBindGroupError,
3380 > {
3381 use crate::binding_model::CreateBindGroupError as Error;
3382
3383 view.same_device(self)?;
3384
3385 let internal_use = self.texture_use_parameters(binding, decl, view, "SampledTexture")?;
3386 used.views.insert_single(view.clone(), internal_use);
3387
3388 match decl.ty {
3389 wgt::BindingType::ExternalTexture => {}
3390 _ => {
3391 return Err(Error::WrongBindingType {
3392 binding,
3393 actual: decl.ty,
3394 expected: "ExternalTexture",
3395 });
3396 }
3397 }
3398
3399 let planes = [
3401 hal::TextureBinding {
3402 view: view.try_raw(snatch_guard)?,
3403 usage: internal_use,
3404 },
3405 hal::TextureBinding {
3406 view: view.try_raw(snatch_guard)?,
3407 usage: internal_use,
3408 },
3409 hal::TextureBinding {
3410 view: view.try_raw(snatch_guard)?,
3411 usage: internal_use,
3412 },
3413 ];
3414 let params = hal::BufferBinding::new_unchecked(
3415 self.default_external_texture_params_buffer.as_ref(),
3416 0,
3417 None,
3418 );
3419
3420 Ok(hal::ExternalTextureBinding { planes, params })
3421 }
3422
3423 pub fn create_bind_group(
3426 self: &Arc<Self>,
3427 desc: binding_model::ResolvedBindGroupDescriptor,
3428 ) -> Result<Arc<BindGroup>, CreateBindGroupError> {
3429 use crate::binding_model::{CreateBindGroupError as Error, ResolvedBindingResource as Br};
3430
3431 let layout = desc.layout;
3432
3433 self.check_is_valid()?;
3434 layout.same_device(self)?;
3435 layout.check_is_valid()?;
3436
3437 {
3438 let actual = desc.entries.len();
3441 let expected = layout.entries.len();
3442 if actual != expected {
3443 return Err(Error::BindingsNumMismatch { expected, actual });
3444 }
3445 }
3446
3447 let mut dynamic_binding_info = Vec::new();
3450 let mut late_buffer_binding_sizes = FastHashMap::default();
3454 let mut used = BindGroupStates::new();
3456
3457 let mut buffer_init_actions = Vec::new();
3458 let mut texture_init_actions = Vec::new();
3459 let mut hal_entries = Vec::with_capacity(desc.entries.len());
3460 let mut hal_buffers = Vec::new();
3461 let mut hal_samplers = Vec::new();
3462 let mut hal_textures = Vec::new();
3463 let mut hal_tlas_s = Vec::new();
3464 let mut hal_external_textures = Vec::new();
3465 let snatch_guard = self.snatchable_lock.read();
3466 for entry in desc.entries.iter() {
3467 let binding = entry.binding;
3468 let decl = layout
3470 .entries
3471 .get(binding)
3472 .ok_or(Error::MissingBindingDeclaration(binding))?;
3473 let (res_index, count) = match entry.resource {
3474 Br::Buffer(ref bb) => {
3475 let bb = self.create_buffer_binding(
3476 bb,
3477 binding,
3478 decl,
3479 &mut buffer_init_actions,
3480 &mut dynamic_binding_info,
3481 &mut late_buffer_binding_sizes,
3482 &mut used,
3483 &snatch_guard,
3484 )?;
3485
3486 let res_index = hal_buffers.len();
3487 hal_buffers.push(bb);
3488 (res_index, 1)
3489 }
3490 Br::BufferArray(ref bindings_array) => {
3491 let num_bindings = bindings_array.len();
3492 Self::check_array_binding(self.features, decl.count, num_bindings)?;
3493
3494 let res_index = hal_buffers.len();
3495 for bb in bindings_array.iter() {
3496 let bb = self.create_buffer_binding(
3497 bb,
3498 binding,
3499 decl,
3500 &mut buffer_init_actions,
3501 &mut dynamic_binding_info,
3502 &mut late_buffer_binding_sizes,
3503 &mut used,
3504 &snatch_guard,
3505 )?;
3506 hal_buffers.push(bb);
3507 }
3508 (res_index, num_bindings)
3509 }
3510 Br::Sampler(ref sampler) => {
3511 let sampler = self.create_sampler_binding(&mut used, binding, decl, sampler)?;
3512
3513 let res_index = hal_samplers.len();
3514 hal_samplers.push(sampler);
3515 (res_index, 1)
3516 }
3517 Br::SamplerArray(ref samplers) => {
3518 let num_bindings = samplers.len();
3519 Self::check_array_binding(self.features, decl.count, num_bindings)?;
3520
3521 let res_index = hal_samplers.len();
3522 for sampler in samplers.iter() {
3523 let sampler =
3524 self.create_sampler_binding(&mut used, binding, decl, sampler)?;
3525
3526 hal_samplers.push(sampler);
3527 }
3528
3529 (res_index, num_bindings)
3530 }
3531 Br::TextureView(ref view) => match decl.ty {
3532 wgt::BindingType::ExternalTexture => {
3533 let et = self.create_external_texture_binding_from_view(
3534 binding,
3535 decl,
3536 view,
3537 &mut used,
3538 &snatch_guard,
3539 )?;
3540 let res_index = hal_external_textures.len();
3541 hal_external_textures.push(et);
3542 (res_index, 1)
3543 }
3544 _ => {
3545 let tb = self.create_texture_binding(
3546 binding,
3547 decl,
3548 view,
3549 &mut used,
3550 &mut texture_init_actions,
3551 &snatch_guard,
3552 )?;
3553 let res_index = hal_textures.len();
3554 hal_textures.push(tb);
3555 (res_index, 1)
3556 }
3557 },
3558 Br::TextureViewArray(ref views) => {
3559 let num_bindings = views.len();
3560 Self::check_array_binding(self.features, decl.count, num_bindings)?;
3561
3562 let res_index = hal_textures.len();
3563 for view in views.iter() {
3564 let tb = self.create_texture_binding(
3565 binding,
3566 decl,
3567 view,
3568 &mut used,
3569 &mut texture_init_actions,
3570 &snatch_guard,
3571 )?;
3572
3573 hal_textures.push(tb);
3574 }
3575
3576 (res_index, num_bindings)
3577 }
3578 Br::AccelerationStructure(ref tlas) => {
3579 let tlas =
3580 self.create_tlas_binding(&mut used, binding, decl, tlas, &snatch_guard)?;
3581 let res_index = hal_tlas_s.len();
3582 hal_tlas_s.push(tlas);
3583 (res_index, 1)
3584 }
3585 Br::AccelerationStructureArray(ref tlas_array) => {
3586 let num_bindings = tlas_array.len();
3590 Self::check_array_binding(self.features, decl.count, num_bindings)?;
3591
3592 let res_index = hal_tlas_s.len();
3593 for tlas in tlas_array.iter() {
3594 let tlas = self.create_tlas_binding(
3595 &mut used,
3596 binding,
3597 decl,
3598 tlas,
3599 &snatch_guard,
3600 )?;
3601 hal_tlas_s.push(tlas);
3602 }
3603 (res_index, num_bindings)
3604 }
3605 Br::ExternalTexture(ref et) => {
3606 let et = self.create_external_texture_binding(
3607 binding,
3608 decl,
3609 et,
3610 &mut used,
3611 &snatch_guard,
3612 )?;
3613 let res_index = hal_external_textures.len();
3614 hal_external_textures.push(et);
3615 (res_index, 1)
3616 }
3617 };
3618
3619 hal_entries.push(hal::BindGroupEntry {
3620 binding,
3621 resource_index: res_index as u32,
3622 count: count as u32,
3623 });
3624 }
3625
3626 used.optimize();
3627
3628 hal_entries.sort_by_key(|entry| entry.binding);
3629 for (a, b) in hal_entries.iter().zip(hal_entries.iter().skip(1)) {
3630 if a.binding == b.binding {
3631 return Err(Error::DuplicateBinding(a.binding));
3632 }
3633 }
3634
3635 dynamic_binding_info.sort_by_key(|i| i.binding_idx);
3636
3637 let hal_desc = hal::BindGroupDescriptor {
3638 label: desc.label.to_hal(self.instance_flags),
3639 layout: layout.try_raw()?,
3640 entries: &hal_entries,
3641 buffers: &hal_buffers,
3642 samplers: &hal_samplers,
3643 textures: &hal_textures,
3644 acceleration_structures: &hal_tlas_s,
3645 external_textures: &hal_external_textures,
3646 };
3647 let raw = unsafe { self.raw().create_bind_group(&hal_desc) }
3648 .map_err(|e| self.handle_hal_error(e))?;
3649
3650 let late_buffer_binding_infos = layout
3652 .entries
3653 .indices()
3654 .flat_map(|binding| {
3655 let size = late_buffer_binding_sizes.get(&binding).cloned()?;
3656 Some(BindGroupLateBufferBindingInfo {
3657 binding_index: binding,
3658 size,
3659 })
3660 })
3661 .collect();
3662
3663 let bind_group = BindGroup {
3664 raw: Snatchable::new(raw),
3665 device: self.clone(),
3666 layout,
3667 label: desc.label.to_string(),
3668 tracking_data: TrackingData::new(self.tracker_indices.bind_groups.clone()),
3669 used,
3670 buffer_init_actions,
3671 texture_init_actions,
3672 dynamic_binding_info,
3673 late_buffer_binding_infos,
3674 };
3675
3676 let bind_group = Arc::new(bind_group);
3677
3678 let weak_ref = Arc::downgrade(&bind_group);
3679 for texture in bind_group.used.views.used_textures() {
3680 let mut bind_groups = texture.bind_groups.lock();
3681 bind_groups.push(weak_ref.clone());
3682 }
3683 for buffer in bind_group.used.buffers.used_resources() {
3684 let mut bind_groups = buffer.bind_groups.lock();
3685 bind_groups.push(weak_ref.clone());
3686 }
3687
3688 Ok(bind_group)
3689 }
3690
3691 fn check_array_binding(
3692 features: wgt::Features,
3693 count: Option<NonZeroU32>,
3694 num_bindings: usize,
3695 ) -> Result<(), CreateBindGroupError> {
3696 use super::binding_model::CreateBindGroupError as Error;
3697
3698 if let Some(count) = count {
3699 let count = count.get() as usize;
3700 if count < num_bindings {
3701 return Err(Error::BindingArrayPartialLengthMismatch {
3702 actual: num_bindings,
3703 expected: count,
3704 });
3705 }
3706 if count != num_bindings
3707 && !features.contains(wgt::Features::PARTIALLY_BOUND_BINDING_ARRAY)
3708 {
3709 return Err(Error::BindingArrayLengthMismatch {
3710 actual: num_bindings,
3711 expected: count,
3712 });
3713 }
3714 if num_bindings == 0 {
3715 return Err(Error::BindingArrayZeroLength);
3716 }
3717 } else {
3718 return Err(Error::SingleBindingExpected);
3719 };
3720
3721 Ok(())
3722 }
3723
3724 fn texture_use_parameters(
3725 &self,
3726 binding: u32,
3727 decl: &wgt::BindGroupLayoutEntry,
3728 view: &TextureView,
3729 expected: &'static str,
3730 ) -> Result<wgt::TextureUses, CreateBindGroupError> {
3731 use crate::binding_model::CreateBindGroupError as Error;
3732 if view
3733 .desc
3734 .aspects()
3735 .contains(hal::FormatAspects::DEPTH | hal::FormatAspects::STENCIL)
3736 {
3737 return Err(Error::DepthStencilAspect);
3738 }
3739 match decl.ty {
3740 wgt::BindingType::Texture {
3741 sample_type,
3742 view_dimension,
3743 multisampled,
3744 } => {
3745 use wgt::TextureSampleType as Tst;
3746 if multisampled != (view.samples != 1) {
3747 return Err(Error::InvalidTextureMultisample {
3748 binding,
3749 layout_multisampled: multisampled,
3750 view_samples: view.samples,
3751 });
3752 }
3753 let compat_sample_type = view
3754 .desc
3755 .format
3756 .sample_type(Some(view.desc.range.aspect), Some(self.features))
3757 .unwrap();
3758 match (sample_type, compat_sample_type) {
3759 (Tst::Uint, Tst::Uint) |
3760 (Tst::Sint, Tst::Sint) |
3761 (Tst::Depth, Tst::Depth) |
3762 (Tst::Float { filterable: false }, Tst::Float { .. }) |
3764 (Tst::Float { filterable: true }, Tst::Float { filterable: true }) |
3766 (Tst::Float { filterable: false }, Tst::Depth) => {}
3768 (Tst::Float { filterable: true }, Tst::Float { .. })
3773 if view.format_features.flags
3774 .contains(wgt::TextureFormatFeatureFlags::FILTERABLE) => {}
3775 _ => {
3776 return Err(Error::InvalidTextureSampleType {
3777 binding,
3778 layout_sample_type: sample_type,
3779 view_format: view.desc.format,
3780 view_sample_type: compat_sample_type,
3781 })
3782 }
3783 }
3784 if view_dimension != view.desc.dimension {
3785 return Err(Error::InvalidTextureDimension {
3786 binding,
3787 layout_dimension: view_dimension,
3788 view_dimension: view.desc.dimension,
3789 });
3790 }
3791 view.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
3792 Ok(wgt::TextureUses::RESOURCE)
3793 }
3794 wgt::BindingType::StorageTexture {
3795 access,
3796 format,
3797 view_dimension,
3798 } => {
3799 if format != view.desc.format {
3800 return Err(Error::InvalidStorageTextureFormat {
3801 binding,
3802 layout_format: format,
3803 view_format: view.desc.format,
3804 });
3805 }
3806 if view_dimension != view.desc.dimension {
3807 return Err(Error::InvalidTextureDimension {
3808 binding,
3809 layout_dimension: view_dimension,
3810 view_dimension: view.desc.dimension,
3811 });
3812 }
3813
3814 let mip_level_count = view.selector.mips.end - view.selector.mips.start;
3815 if mip_level_count != 1 {
3816 return Err(Error::InvalidStorageTextureMipLevelCount {
3817 binding,
3818 mip_level_count,
3819 });
3820 }
3821
3822 view.check_usage(wgt::TextureUsages::STORAGE_BINDING)?;
3823
3824 Ok(match access {
3825 wgt::StorageTextureAccess::ReadOnly => wgt::TextureUses::STORAGE_READ_ONLY,
3826 wgt::StorageTextureAccess::WriteOnly => wgt::TextureUses::STORAGE_WRITE_ONLY,
3827 wgt::StorageTextureAccess::ReadWrite => wgt::TextureUses::STORAGE_READ_WRITE,
3828 wgt::StorageTextureAccess::Atomic => wgt::TextureUses::STORAGE_ATOMIC,
3829 })
3830 }
3831 wgt::BindingType::ExternalTexture => {
3832 if view.desc.dimension != TextureViewDimension::D2 {
3833 return Err(Error::InvalidTextureDimension {
3834 binding,
3835 layout_dimension: TextureViewDimension::D2,
3836 view_dimension: view.desc.dimension,
3837 });
3838 }
3839 let mip_level_count = view.selector.mips.end - view.selector.mips.start;
3840 if mip_level_count != 1 {
3841 return Err(Error::InvalidExternalTextureMipLevelCount {
3842 binding,
3843 mip_level_count,
3844 });
3845 }
3846 if view.desc.format != TextureFormat::Rgba8Unorm
3847 && view.desc.format != TextureFormat::Bgra8Unorm
3848 && view.desc.format != TextureFormat::Rgba16Float
3849 {
3850 return Err(Error::InvalidExternalTextureFormat {
3851 binding,
3852 format: view.desc.format,
3853 });
3854 }
3855 if view.samples != 1 {
3856 return Err(Error::InvalidTextureMultisample {
3857 binding,
3858 layout_multisampled: false,
3859 view_samples: view.samples,
3860 });
3861 }
3862
3863 view.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
3864 Ok(wgt::TextureUses::RESOURCE)
3865 }
3866 _ => Err(Error::WrongBindingType {
3867 binding,
3868 actual: decl.ty,
3869 expected,
3870 }),
3871 }
3872 }
3873
3874 pub fn create_pipeline_layout(
3875 self: &Arc<Self>,
3876 desc: &binding_model::ResolvedPipelineLayoutDescriptor,
3877 ) -> (
3878 Arc<binding_model::PipelineLayout>,
3879 Option<binding_model::CreatePipelineLayoutError>,
3880 ) {
3881 let (layout, error) = match self.create_pipeline_layout_impl(desc, false) {
3882 Ok(layout) => (layout, None),
3883 Err(e) => (
3884 binding_model::PipelineLayout::invalid(Arc::clone(self), desc.label.to_string()),
3885 Some(e),
3886 ),
3887 };
3888 #[cfg(feature = "trace")]
3889 if let Some(ref mut trace) = *self.trace.lock() {
3890 use crate::device::trace::IntoTrace;
3891 trace.add(trace::Action::CreatePipelineLayout(
3892 layout.to_trace(),
3893 desc.to_trace(),
3894 ));
3895 }
3896 api_log!(
3897 "Device::create_pipeline_layout -> {:?}",
3898 Arc::as_ptr(&layout)
3899 );
3900 (layout, error)
3901 }
3902
3903 fn create_pipeline_layout_impl(
3904 self: &Arc<Self>,
3905 desc: &binding_model::ResolvedPipelineLayoutDescriptor,
3906 ignore_exclusive_pipeline_check: bool,
3907 ) -> Result<Arc<binding_model::PipelineLayout>, binding_model::CreatePipelineLayoutError> {
3908 use crate::binding_model::CreatePipelineLayoutError as Error;
3909
3910 self.check_is_valid()?;
3911
3912 let bind_group_layouts_count = desc.bind_group_layouts.len();
3913 let device_max_bind_groups = self.limits.max_bind_groups as usize;
3914 if bind_group_layouts_count > device_max_bind_groups {
3915 return Err(Error::TooManyGroups {
3916 actual: bind_group_layouts_count,
3917 max: device_max_bind_groups,
3918 });
3919 }
3920
3921 if desc.immediate_size != 0 {
3922 self.require_features(wgt::Features::IMMEDIATES)?;
3923 }
3924 if self.limits.max_immediate_size < desc.immediate_size {
3925 return Err(Error::ImmediateRangeTooLarge {
3926 size: desc.immediate_size,
3927 max: self.limits.max_immediate_size,
3928 });
3929 }
3930 if !desc
3931 .immediate_size
3932 .is_multiple_of(wgt::IMMEDIATE_DATA_ALIGNMENT)
3933 {
3934 return Err(Error::MisalignedImmediateSize {
3935 size: desc.immediate_size,
3936 });
3937 }
3938
3939 let mut count_validator = binding_model::BindingTypeMaxCountValidator::default();
3940
3941 for (index, bgl) in desc.bind_group_layouts.iter().enumerate() {
3942 let Some(bgl) = bgl else {
3943 continue;
3944 };
3945
3946 bgl.same_device(self)?;
3947
3948 if !ignore_exclusive_pipeline_check {
3949 let exclusive_pipeline = bgl.exclusive_pipeline.get().unwrap();
3950 if !matches!(exclusive_pipeline, binding_model::ExclusivePipeline::None) {
3951 return Err(Error::BglHasExclusivePipeline {
3952 index,
3953 pipeline: alloc::format!("{exclusive_pipeline}"),
3954 });
3955 }
3956 }
3957
3958 count_validator.merge(&bgl.state()?.binding_count_validator);
3959 }
3960
3961 count_validator
3962 .validate(&self.limits, self.instance_flags)
3963 .map_err(Error::TooManyBindings)?;
3964
3965 let buffers_and_acceleration_structures_in_vertex_stage =
3966 count_validator.buffers_and_acceleration_structures_in_vertex_stage();
3967
3968 let get_bgl_iter = || {
3969 desc.bind_group_layouts
3970 .iter()
3971 .map(|bgl| bgl.as_ref().filter(|bgl| !bgl.entries.is_empty()))
3972 };
3973
3974 let bind_group_layouts = get_bgl_iter()
3975 .map(|bgl| bgl.cloned())
3976 .collect::<ArrayVec<_, { hal::MAX_BIND_GROUPS }>>();
3977
3978 let raw_bind_group_layouts = get_bgl_iter()
3979 .map(|bgl| bgl.map(|bgl| bgl.try_raw()).transpose())
3980 .collect::<Result<ArrayVec<_, { hal::MAX_BIND_GROUPS }>, _>>()?;
3981
3982 let additional_flags = if self.indirect_validation.is_some() {
3983 hal::PipelineLayoutFlags::INDIRECT_BUILTIN_UPDATE
3984 } else {
3985 hal::PipelineLayoutFlags::empty()
3986 };
3987
3988 let hal_desc = hal::PipelineLayoutDescriptor {
3989 label: desc.label.to_hal(self.instance_flags),
3990 flags: hal::PipelineLayoutFlags::FIRST_VERTEX_INSTANCE
3991 | hal::PipelineLayoutFlags::NUM_WORK_GROUPS
3992 | additional_flags,
3993 bind_group_layouts: &raw_bind_group_layouts,
3994 immediate_size: desc.immediate_size,
3995 };
3996
3997 let raw = unsafe { self.raw().create_pipeline_layout(&hal_desc) }
3998 .map_err(|e| self.handle_hal_error(e))?;
3999
4000 drop(raw_bind_group_layouts);
4001
4002 let layout = binding_model::PipelineLayout {
4003 raw: ResourceState::Valid(raw),
4004 device: self.clone(),
4005 label: desc.label.to_string(),
4006 bind_group_layouts,
4007 immediate_size: desc.immediate_size,
4008 buffers_and_acceleration_structures_in_vertex_stage,
4009 };
4010
4011 let layout = Arc::new(layout);
4012
4013 Ok(layout)
4014 }
4015
4016 fn create_derived_pipeline_layout(
4017 self: &Arc<Self>,
4018 mut derived_group_layouts: Box<ArrayVec<bgl::EntryMap, { hal::MAX_BIND_GROUPS }>>,
4019 immediate_size: u32,
4020 ) -> Result<Arc<binding_model::PipelineLayout>, pipeline::ImplicitLayoutError> {
4021 while derived_group_layouts
4022 .last()
4023 .is_some_and(|map| map.is_empty())
4024 {
4025 derived_group_layouts.pop();
4026 }
4027
4028 let mut unique_bind_group_layouts = FastHashMap::default();
4029
4030 let bind_group_layouts = derived_group_layouts
4031 .into_iter()
4032 .map(|mut bgl_entry_map| {
4033 if bgl_entry_map.is_empty() {
4034 return Ok(None);
4035 }
4036
4037 bgl_entry_map.sort();
4038 match unique_bind_group_layouts.entry(bgl_entry_map) {
4039 hashbrown::hash_map::Entry::Occupied(v) => Ok(Some(Arc::clone(v.get()))),
4040 hashbrown::hash_map::Entry::Vacant(e) => {
4041 match self.create_bind_group_layout_impl(
4042 &None,
4043 e.key().clone(),
4044 bgl::Origin::Derived,
4045 ) {
4046 Ok(bgl) => {
4047 e.insert(bgl.clone());
4048 Ok(Some(bgl))
4049 }
4050 Err(e) => Err(e),
4051 }
4052 }
4053 }
4054 })
4055 .collect::<Result<Vec<_>, _>>()?;
4056
4057 let layout_desc = binding_model::ResolvedPipelineLayoutDescriptor {
4058 label: None,
4059 bind_group_layouts: Cow::Owned(bind_group_layouts),
4060 immediate_size,
4061 };
4062
4063 let layout = self.create_pipeline_layout_impl(&layout_desc, true)?;
4064 Ok(layout)
4065 }
4066
4067 pub fn create_compute_pipeline(
4068 self: &Arc<Self>,
4069 desc: pipeline::ResolvedComputePipelineDescriptor,
4070 ) -> (
4071 Arc<pipeline::ComputePipeline>,
4072 Option<pipeline::CreateComputePipelineError>,
4073 ) {
4074 let (compute_pipeline, error) = match self.create_compute_pipeline_inner(desc.clone()) {
4075 Ok(compute_pipeline) => (compute_pipeline, None),
4076 Err(error) => (
4077 pipeline::ComputePipeline::invalid(self.clone(), desc.label.to_string()),
4078 Some(error),
4079 ),
4080 };
4081 #[cfg(feature = "trace")]
4082 if let Some(ref mut trace) = *self.trace.lock() {
4083 use crate::device::trace;
4084 use crate::device::trace::IntoTrace;
4085 trace.add(trace::Action::CreateComputePipeline {
4086 id: compute_pipeline.to_trace(),
4087 desc: desc.to_trace(),
4088 });
4089 }
4090 (compute_pipeline, error)
4091 }
4092
4093 pub fn create_compute_pipeline_inner(
4094 self: &Arc<Self>,
4095 desc: pipeline::ResolvedComputePipelineDescriptor,
4096 ) -> Result<Arc<pipeline::ComputePipeline>, pipeline::CreateComputePipelineError> {
4097 self.check_is_valid()?;
4098
4099 self.require_downlevel_flags(wgt::DownlevelFlags::COMPUTE_SHADERS)?;
4100
4101 let shader_module = desc.stage.module;
4102
4103 shader_module.same_device(self)?;
4104
4105 let is_auto_layout = desc.layout.is_none();
4106
4107 let pipeline_layout = match desc.layout {
4109 Some(pipeline_layout) => {
4110 pipeline_layout.same_device(self)?;
4111 pipeline_layout.check_valid()?;
4112 Some(pipeline_layout)
4113 }
4114 None => None,
4115 };
4116
4117 let mut binding_layout_source = match pipeline_layout {
4118 Some(pipeline_layout) => validation::BindingLayoutSource::Provided(pipeline_layout),
4119 None => validation::BindingLayoutSource::new_derived(&self.limits),
4120 };
4121 let mut shader_binding_sizes = FastHashMap::default();
4122 let io = validation::StageIo::default();
4123
4124 let final_entry_point_name;
4125
4126 {
4127 let stage = validation::ShaderStageForValidation::Compute;
4128
4129 final_entry_point_name = shader_module.finalize_entry_point_name(
4130 stage.to_naga(),
4131 desc.stage.entry_point.as_ref().map(|ep| ep.as_ref()),
4132 )?;
4133
4134 if let Some(interface) = shader_module.interface.interface() {
4135 let _ = interface.check_stage(
4136 &mut binding_layout_source,
4137 &mut shader_binding_sizes,
4138 &final_entry_point_name,
4139 stage,
4140 io,
4141 None,
4142 )?;
4143 }
4144 }
4145
4146 let pipeline_layout = match binding_layout_source {
4147 validation::BindingLayoutSource::Provided(pipeline_layout) => pipeline_layout,
4148 validation::BindingLayoutSource::Derived(entries) => {
4149 let immediate_size = shader_module
4150 .interface
4151 .interface()
4152 .map_or(0, |i| i.immediate_size);
4153 self.create_derived_pipeline_layout(entries, immediate_size)?
4154 }
4155 };
4156
4157 let late_sized_buffer_groups =
4158 Device::make_late_sized_buffer_groups(&shader_binding_sizes, &pipeline_layout);
4159
4160 let cache = match desc.cache {
4161 Some(cache) => {
4162 cache.same_device(self)?;
4163 Some(cache)
4164 }
4165 None => None,
4166 };
4167
4168 let pipeline_desc = hal::ComputePipelineDescriptor {
4169 label: desc.label.to_hal(self.instance_flags),
4170 layout: pipeline_layout.raw()?,
4171 stage: hal::ProgrammableStage {
4172 module: shader_module.raw(),
4173 entry_point: final_entry_point_name.as_ref(),
4174 constants: &desc.stage.constants,
4175 zero_initialize_workgroup_memory: desc.stage.zero_initialize_workgroup_memory,
4176 },
4177 cache: cache.as_ref().map(|it| it.raw()),
4178 };
4179
4180 let raw =
4181 unsafe { self.raw().create_compute_pipeline(&pipeline_desc) }.map_err(
4182 |err| match err {
4183 hal::PipelineError::Device(error) => {
4184 pipeline::CreateComputePipelineError::Device(self.handle_hal_error(error))
4185 }
4186 hal::PipelineError::Linkage(_stages, msg) => {
4187 pipeline::CreateComputePipelineError::Internal(msg)
4188 }
4189 hal::PipelineError::EntryPoint(_stage) => {
4190 pipeline::CreateComputePipelineError::Internal(
4191 ENTRYPOINT_FAILURE_ERROR.to_string(),
4192 )
4193 }
4194 hal::PipelineError::PipelineConstants(_stages, msg) => {
4195 pipeline::CreateComputePipelineError::PipelineConstants(msg)
4196 }
4197 },
4198 )?;
4199
4200 let immediate_slots_required =
4201 shader_module
4202 .interface
4203 .interface()
4204 .map_or(Default::default(), |iface| {
4205 iface.immediate_slots_required(
4206 naga::ShaderStage::Compute,
4207 &final_entry_point_name,
4208 )
4209 });
4210
4211 let pipeline = pipeline::ComputePipeline {
4212 state: ResourceState::Valid(pipeline::ComputePipelineState {
4213 raw: ManuallyDrop::new(raw),
4214 layout: pipeline_layout.clone(),
4215 _shader_module: shader_module,
4216 }),
4217 device: self.clone(),
4218 late_sized_buffer_groups,
4219 immediate_slots_required,
4220 label: desc.label.to_string(),
4221 tracking_data: TrackingData::new(self.tracker_indices.compute_pipelines.clone()),
4222 };
4223
4224 let pipeline = Arc::new(pipeline);
4225
4226 if is_auto_layout {
4227 for bgl in pipeline_layout.bind_group_layouts.iter() {
4228 let Some(bgl) = bgl else {
4229 continue;
4230 };
4231
4232 let _ = bgl.exclusive_pipeline.set((&pipeline).into());
4235 }
4236 }
4237
4238 Ok(pipeline)
4239 }
4240
4241 pub fn create_render_pipeline(
4242 self: &Arc<Self>,
4243 desc: pipeline::ResolvedGeneralRenderPipelineDescriptor,
4244 ) -> (
4245 Arc<pipeline::RenderPipeline>,
4246 Option<pipeline::CreateRenderPipelineError>,
4247 ) {
4248 let (render_pipeline, error) = match self.create_render_pipeline_inner(desc.clone()) {
4249 Ok(pipeline) => (pipeline, None),
4250 Err(e) => (
4251 pipeline::RenderPipeline::invalid(self.clone(), desc.label.to_string()),
4252 Some(e),
4253 ),
4254 };
4255 #[cfg(feature = "trace")]
4256 if let Some(ref mut trace) = *self.trace.lock() {
4257 use crate::device::trace::IntoTrace;
4258 trace.add(trace::Action::CreateGeneralRenderPipeline {
4259 id: render_pipeline.to_trace(),
4260 desc: desc.to_trace(),
4261 });
4262 }
4263 (render_pipeline, error)
4264 }
4265
4266 pub fn create_render_pipeline_inner(
4267 self: &Arc<Self>,
4268 desc: pipeline::ResolvedGeneralRenderPipelineDescriptor,
4269 ) -> Result<Arc<pipeline::RenderPipeline>, pipeline::CreateRenderPipelineError> {
4270 use wgt::TextureFormatFeatureFlags as Tfff;
4271
4272 self.check_is_valid()?;
4273
4274 let mut shader_binding_sizes = FastHashMap::default();
4275
4276 let color_targets = desc
4277 .fragment
4278 .as_ref()
4279 .map_or(&[][..], |fragment| &fragment.targets);
4280 let depth_stencil_state = desc.depth_stencil.as_ref();
4281
4282 check_color_attachment_count(color_targets.len(), self.limits.max_color_attachments)?;
4283
4284 {
4285 let cts: ArrayVec<_, { hal::MAX_COLOR_ATTACHMENTS }> =
4286 color_targets.iter().filter_map(|x| x.as_ref()).collect();
4287 if !cts.is_empty() && {
4288 let first = &cts[0];
4289 cts[1..]
4290 .iter()
4291 .any(|ct| ct.write_mask != first.write_mask || ct.blend != first.blend)
4292 } {
4293 self.require_downlevel_flags(wgt::DownlevelFlags::INDEPENDENT_BLEND)?;
4294 }
4295 }
4296
4297 let mut io = validation::StageIo::default();
4298 let mut validated_stages = wgt::ShaderStages::empty();
4299
4300 let mut vertex_steps;
4301 let mut hal_vertex_buffer_layouts;
4302 let mut total_attributes;
4303 let mut dual_source_blending = false;
4304 let mut has_depth_attachment = false;
4305 if let pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) = desc.vertex {
4306 if vertex.buffers.len() > self.limits.max_vertex_buffers as usize {
4307 return Err(pipeline::CreateRenderPipelineError::TooManyVertexBuffers {
4308 given: vertex.buffers.len() as u32,
4309 limit: self.limits.max_vertex_buffers,
4310 });
4311 }
4312
4313 vertex_steps = Vec::with_capacity(vertex.buffers.len());
4314 hal_vertex_buffer_layouts = Vec::with_capacity(vertex.buffers.len());
4315 total_attributes = 0;
4316 for (i, vb_state) in vertex.buffers.iter().enumerate() {
4317 let Some(vb_state) = vb_state else {
4318 vertex_steps.push(None);
4319 hal_vertex_buffer_layouts.push(None);
4320 continue;
4321 };
4322
4323 if vb_state.array_stride > self.limits.max_vertex_buffer_array_stride as u64 {
4326 return Err(pipeline::CreateRenderPipelineError::VertexStrideTooLarge {
4327 index: i as u32,
4328 given: vb_state.array_stride as u32,
4329 limit: self.limits.max_vertex_buffer_array_stride,
4330 });
4331 }
4332 if vb_state.array_stride % wgt::VERTEX_ALIGNMENT != 0 {
4333 return Err(pipeline::CreateRenderPipelineError::UnalignedVertexStride {
4334 index: i as u32,
4335 stride: vb_state.array_stride,
4336 });
4337 }
4338
4339 let max_stride = if vb_state.array_stride == 0 {
4340 self.limits.max_vertex_buffer_array_stride as u64
4341 } else {
4342 vb_state.array_stride
4343 };
4344 let mut last_stride = 0;
4345 for attribute in vb_state.attributes.iter() {
4346 let attribute_stride = attribute.offset + attribute.format.size();
4347 if attribute_stride > max_stride {
4348 return Err(
4349 pipeline::CreateRenderPipelineError::VertexAttributeStrideTooLarge {
4350 location: attribute.shader_location,
4351 given: attribute_stride as u32,
4352 limit: max_stride as u32,
4353 },
4354 );
4355 }
4356
4357 let required_offset_alignment = attribute.format.size().min(4);
4358 if attribute.offset % required_offset_alignment != 0 {
4359 return Err(
4360 pipeline::CreateRenderPipelineError::InvalidVertexAttributeOffset {
4361 location: attribute.shader_location,
4362 offset: attribute.offset,
4363 },
4364 );
4365 }
4366
4367 if attribute.shader_location >= self.limits.max_vertex_attributes {
4368 return Err(
4369 pipeline::CreateRenderPipelineError::VertexAttributeLocationTooLarge {
4370 given: attribute.shader_location,
4371 limit: self.limits.max_vertex_attributes,
4372 },
4373 );
4374 }
4375
4376 last_stride = last_stride.max(attribute_stride);
4377 }
4378
4379 vertex_steps.push(Some(pipeline::VertexStep {
4380 stride: vb_state.array_stride,
4381 last_stride,
4382 mode: vb_state.step_mode,
4383 }));
4384 hal_vertex_buffer_layouts.push(if vb_state.attributes.is_empty() {
4385 None
4386 } else {
4387 Some(hal::VertexBufferLayout {
4388 array_stride: vb_state.array_stride,
4389 step_mode: vb_state.step_mode,
4390 attributes: vb_state.attributes.as_ref(),
4391 })
4392 });
4393
4394 for attribute in vb_state.attributes.iter() {
4395 if attribute.offset >= 0x10000000 {
4396 return Err(
4397 pipeline::CreateRenderPipelineError::InvalidVertexAttributeOffset {
4398 location: attribute.shader_location,
4399 offset: attribute.offset,
4400 },
4401 );
4402 }
4403
4404 if let wgt::VertexFormat::Float64
4405 | wgt::VertexFormat::Float64x2
4406 | wgt::VertexFormat::Float64x3
4407 | wgt::VertexFormat::Float64x4 = attribute.format
4408 {
4409 self.require_features(wgt::Features::VERTEX_ATTRIBUTE_64BIT)?;
4410 }
4411
4412 let previous = io.varyings.insert(
4413 attribute.shader_location,
4414 validation::InterfaceVar::vertex_attribute(attribute.format),
4415 );
4416
4417 if previous.is_some() {
4418 return Err(pipeline::CreateRenderPipelineError::ShaderLocationClash(
4419 attribute.shader_location,
4420 ));
4421 }
4422 }
4423 total_attributes += vb_state.attributes.len();
4424 }
4425
4426 if total_attributes > self.limits.max_vertex_attributes as usize {
4427 return Err(
4428 pipeline::CreateRenderPipelineError::TooManyVertexAttributes {
4429 given: total_attributes as u32,
4430 limit: self.limits.max_vertex_attributes,
4431 },
4432 );
4433 }
4434 } else {
4435 vertex_steps = Vec::new();
4436 hal_vertex_buffer_layouts = Vec::new();
4437 };
4438
4439 if desc.primitive.strip_index_format.is_some() && !desc.primitive.topology.is_strip() {
4440 return Err(
4441 pipeline::CreateRenderPipelineError::StripIndexFormatForNonStripTopology {
4442 strip_index_format: desc.primitive.strip_index_format,
4443 topology: desc.primitive.topology,
4444 },
4445 );
4446 }
4447
4448 if desc.primitive.unclipped_depth {
4449 self.require_features(wgt::Features::DEPTH_CLIP_CONTROL)?;
4450 }
4451
4452 if desc.primitive.polygon_mode == wgt::PolygonMode::Line {
4453 self.require_features(wgt::Features::POLYGON_MODE_LINE)?;
4454 }
4455 if desc.primitive.polygon_mode == wgt::PolygonMode::Point {
4456 self.require_features(wgt::Features::POLYGON_MODE_POINT)?;
4457 }
4458
4459 if desc.primitive.conservative {
4460 self.require_features(wgt::Features::CONSERVATIVE_RASTERIZATION)?;
4461 }
4462
4463 if desc.primitive.conservative && desc.primitive.polygon_mode != wgt::PolygonMode::Fill {
4464 return Err(
4465 pipeline::CreateRenderPipelineError::ConservativeRasterizationNonFillPolygonMode,
4466 );
4467 }
4468
4469 let mut target_specified = false;
4470
4471 for (i, cs) in color_targets.iter().enumerate() {
4472 if let Some(cs) = cs.as_ref() {
4473 target_specified = true;
4474 let error = 'error: {
4475 if cs.write_mask.contains_unknown_bits() {
4479 break 'error Some(ColorStateError::InvalidWriteMask(cs.write_mask));
4480 }
4481
4482 let format_features = self.describe_format_features(cs.format)?;
4483 if !format_features
4484 .allowed_usages
4485 .contains(wgt::TextureUsages::RENDER_ATTACHMENT)
4486 {
4487 break 'error Some(ColorStateError::FormatNotRenderable(cs.format));
4488 }
4489 if cs.blend.is_some() && !format_features.flags.contains(Tfff::BLENDABLE) {
4490 break 'error Some(ColorStateError::FormatNotBlendable(cs.format));
4491 }
4492 if !hal::FormatAspects::from(cs.format).contains(hal::FormatAspects::COLOR) {
4493 break 'error Some(ColorStateError::FormatNotColor(cs.format));
4494 }
4495
4496 if desc.multisample.count > 1
4497 && !format_features
4498 .flags
4499 .sample_count_supported(desc.multisample.count)
4500 {
4501 break 'error Some(ColorStateError::InvalidSampleCount(
4502 desc.multisample.count,
4503 cs.format,
4504 cs.format
4505 .guaranteed_format_features(self.features)
4506 .flags
4507 .supported_sample_counts(),
4508 self.adapter
4509 .get_texture_format_features(cs.format)
4510 .flags
4511 .supported_sample_counts(),
4512 ));
4513 }
4514
4515 if let Some(blend_mode) = cs.blend {
4516 for component in [&blend_mode.color, &blend_mode.alpha] {
4517 for factor in [component.src_factor, component.dst_factor] {
4518 if factor.ref_second_blend_source() {
4519 self.require_features(wgt::Features::DUAL_SOURCE_BLENDING)?;
4520 if i == 0 {
4521 dual_source_blending = true;
4522 } else {
4523 break 'error Some(
4524 ColorStateError::BlendFactorOnUnsupportedTarget {
4525 factor,
4526 target: i as u32,
4527 },
4528 );
4529 }
4530 }
4531
4532 if [wgt::BlendOperation::Min, wgt::BlendOperation::Max]
4533 .contains(&component.operation)
4534 && factor != wgt::BlendFactor::One
4535 {
4536 break 'error Some(ColorStateError::InvalidMinMaxBlendFactor {
4537 factor,
4538 target: i as u32,
4539 });
4540 }
4541 }
4542 }
4543 }
4544
4545 break 'error None;
4546 };
4547 if let Some(e) = error {
4548 return Err(pipeline::CreateRenderPipelineError::ColorState(i as u8, e));
4549 }
4550 }
4551 }
4552
4553 if dual_source_blending && color_targets.len() > 1 {
4554 return Err(
4555 pipeline::CreateRenderPipelineError::DualSourceBlendingWithMultipleColorTargets {
4556 count: color_targets.len(),
4557 },
4558 );
4559 }
4560
4561 validation::validate_color_attachment_bytes_per_sample(
4562 color_targets.iter().flatten().map(|cs| cs.format),
4563 self.limits.max_color_attachment_bytes_per_sample,
4564 )
4565 .map_err(pipeline::CreateRenderPipelineError::ColorAttachment)?;
4566
4567 if let Some(ds) = depth_stencil_state {
4568 target_specified = true;
4570 let error = 'error: {
4571 if !ds.format.is_depth_stencil_format() {
4572 break 'error Some(pipeline::DepthStencilStateError::FormatNotDepthOrStencil(
4575 ds.format,
4576 ));
4577 }
4578
4579 let format_features = self.describe_format_features(ds.format)?;
4580 if !format_features
4581 .allowed_usages
4582 .contains(wgt::TextureUsages::RENDER_ATTACHMENT)
4583 {
4584 break 'error Some(pipeline::DepthStencilStateError::FormatNotRenderable(
4585 ds.format,
4586 ));
4587 }
4588
4589 let aspect = hal::FormatAspects::from(ds.format);
4590 if aspect.contains(hal::FormatAspects::DEPTH) {
4591 has_depth_attachment = true;
4592 } else if ds.is_depth_enabled() {
4593 break 'error Some(pipeline::DepthStencilStateError::FormatNotDepth(ds.format));
4594 }
4595 if has_depth_attachment {
4596 let Some(depth_write_enabled) = ds.depth_write_enabled else {
4597 break 'error Some(
4598 pipeline::DepthStencilStateError::MissingDepthWriteEnabled(ds.format),
4599 );
4600 };
4601
4602 let depth_compare_required = depth_write_enabled
4603 || ds.stencil.front.depth_fail_op != wgt::StencilOperation::Keep
4604 || ds.stencil.back.depth_fail_op != wgt::StencilOperation::Keep;
4605 if depth_compare_required && ds.depth_compare.is_none() {
4606 break 'error Some(pipeline::DepthStencilStateError::MissingDepthCompare(
4607 ds.format,
4608 ));
4609 }
4610 }
4611
4612 if ds.stencil.is_enabled() && !aspect.contains(hal::FormatAspects::STENCIL) {
4613 break 'error Some(pipeline::DepthStencilStateError::FormatNotStencil(
4614 ds.format,
4615 ));
4616 }
4617 if desc.multisample.count > 1
4618 && !format_features
4619 .flags
4620 .sample_count_supported(desc.multisample.count)
4621 {
4622 break 'error Some(pipeline::DepthStencilStateError::InvalidSampleCount(
4623 desc.multisample.count,
4624 ds.format,
4625 ds.format
4626 .guaranteed_format_features(self.features)
4627 .flags
4628 .supported_sample_counts(),
4629 self.adapter
4630 .get_texture_format_features(ds.format)
4631 .flags
4632 .supported_sample_counts(),
4633 ));
4634 }
4635
4636 break 'error None;
4637 };
4638 if let Some(e) = error {
4639 return Err(pipeline::CreateRenderPipelineError::DepthStencilState(e));
4640 }
4641
4642 if ds.bias.clamp != 0.0 {
4643 self.require_downlevel_flags(wgt::DownlevelFlags::DEPTH_BIAS_CLAMP)?;
4644 }
4645
4646 if (ds.bias.is_enabled() || ds.bias.clamp != 0.0)
4647 && !desc.primitive.topology.is_triangles()
4648 {
4649 return Err(pipeline::CreateRenderPipelineError::DepthStencilState(
4650 pipeline::DepthStencilStateError::DepthBiasWithIncompatibleTopology(
4651 desc.primitive.topology,
4652 ),
4653 ));
4654 }
4655 }
4656
4657 if !target_specified {
4658 return Err(pipeline::CreateRenderPipelineError::NoTargetSpecified);
4659 }
4660
4661 let is_auto_layout = desc.layout.is_none();
4662
4663 let pipeline_layout = match desc.layout {
4665 Some(pipeline_layout) => {
4666 pipeline_layout.same_device(self)?;
4667 pipeline_layout.check_valid()?;
4668 Some(pipeline_layout)
4669 }
4670 None => None,
4671 };
4672
4673 let mut binding_layout_source = match pipeline_layout {
4674 Some(pipeline_layout) => validation::BindingLayoutSource::Provided(pipeline_layout),
4675 None => validation::BindingLayoutSource::new_derived(&self.limits),
4676 };
4677
4678 let samples = {
4679 let sc = desc.multisample.count;
4680 if sc == 0 || sc > 32 || !sc.is_power_of_two() {
4681 return Err(pipeline::CreateRenderPipelineError::InvalidSampleCount(sc));
4682 }
4683 sc
4684 };
4685
4686 let mut vertex_stage = None;
4687 let mut task_stage = None;
4688 let mut mesh_stage = None;
4689 let mut _vertex_entry_point_name = String::new();
4690 let mut _task_entry_point_name = String::new();
4691 let mut _mesh_entry_point_name = String::new();
4692 let mut immediate_slots_required = naga::valid::ImmediateSlots::default();
4693 match desc.vertex {
4694 pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) => {
4695 vertex_stage = {
4696 let stage_desc = &vertex.stage;
4697 let stage = validation::ShaderStageForValidation::Vertex {
4698 topology: desc.primitive.topology,
4699 compare_function: desc.depth_stencil.as_ref().and_then(|d| d.depth_compare),
4700 };
4701 let stage_bit = stage.to_wgt_bit();
4702
4703 let vertex_shader_module = &stage_desc.module;
4704 vertex_shader_module.same_device(self)?;
4705
4706 let stage_err = |error| pipeline::CreateRenderPipelineError::Stage {
4707 stage: stage_bit,
4708 error,
4709 };
4710
4711 _vertex_entry_point_name = vertex_shader_module
4712 .finalize_entry_point_name(
4713 stage.to_naga(),
4714 stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4715 )
4716 .map_err(stage_err)?;
4717
4718 if let Some(interface) = vertex_shader_module.interface.interface() {
4719 immediate_slots_required |= interface
4720 .immediate_slots_required(stage.to_naga(), &_vertex_entry_point_name);
4721 io = interface
4722 .check_stage(
4723 &mut binding_layout_source,
4724 &mut shader_binding_sizes,
4725 &_vertex_entry_point_name,
4726 stage,
4727 io,
4728 Some(desc.primitive.topology),
4729 )
4730 .map_err(stage_err)?;
4731 validated_stages |= stage_bit;
4732 }
4733 Some(hal::ProgrammableStage {
4734 module: vertex_shader_module.raw(),
4735 entry_point: &_vertex_entry_point_name,
4736 constants: &stage_desc.constants,
4737 zero_initialize_workgroup_memory: stage_desc
4738 .zero_initialize_workgroup_memory,
4739 })
4740 };
4741 }
4742 pipeline::RenderPipelineVertexProcessor::Mesh(ref task, ref mesh) => {
4743 self.require_features(wgt::Features::EXPERIMENTAL_MESH_SHADER)?;
4744
4745 task_stage = if let Some(task) = task {
4746 let stage_desc = &task.stage;
4747 let stage = validation::ShaderStageForValidation::Task;
4748 let stage_bit = stage.to_wgt_bit();
4749 let task_shader_module = &stage_desc.module;
4750 task_shader_module.same_device(self)?;
4751
4752 let stage_err = |error| pipeline::CreateRenderPipelineError::Stage {
4753 stage: stage_bit,
4754 error,
4755 };
4756
4757 _task_entry_point_name = task_shader_module
4758 .finalize_entry_point_name(
4759 stage.to_naga(),
4760 stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4761 )
4762 .map_err(stage_err)?;
4763
4764 if let Some(interface) = task_shader_module.interface.interface() {
4765 immediate_slots_required |= interface
4766 .immediate_slots_required(stage.to_naga(), &_task_entry_point_name);
4767 io = interface
4768 .check_stage(
4769 &mut binding_layout_source,
4770 &mut shader_binding_sizes,
4771 &_task_entry_point_name,
4772 stage,
4773 io,
4774 Some(desc.primitive.topology),
4775 )
4776 .map_err(stage_err)?;
4777 validated_stages |= stage_bit;
4778 }
4779 Some(hal::ProgrammableStage {
4780 module: task_shader_module.raw(),
4781 entry_point: &_task_entry_point_name,
4782 constants: &stage_desc.constants,
4783 zero_initialize_workgroup_memory: stage_desc
4784 .zero_initialize_workgroup_memory,
4785 })
4786 } else {
4787 None
4788 };
4789 mesh_stage = {
4790 let stage_desc = &mesh.stage;
4791 let stage = validation::ShaderStageForValidation::Mesh;
4792 let stage_bit = stage.to_wgt_bit();
4793 let mesh_shader_module = &stage_desc.module;
4794 mesh_shader_module.same_device(self)?;
4795
4796 let stage_err = |error| pipeline::CreateRenderPipelineError::Stage {
4797 stage: stage_bit,
4798 error,
4799 };
4800
4801 _mesh_entry_point_name = mesh_shader_module
4802 .finalize_entry_point_name(
4803 stage.to_naga(),
4804 stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4805 )
4806 .map_err(stage_err)?;
4807
4808 if let Some(interface) = mesh_shader_module.interface.interface() {
4809 immediate_slots_required |= interface
4810 .immediate_slots_required(stage.to_naga(), &_mesh_entry_point_name);
4811 io = interface
4812 .check_stage(
4813 &mut binding_layout_source,
4814 &mut shader_binding_sizes,
4815 &_mesh_entry_point_name,
4816 stage,
4817 io,
4818 Some(desc.primitive.topology),
4819 )
4820 .map_err(stage_err)?;
4821 validated_stages |= stage_bit;
4822 }
4823 Some(hal::ProgrammableStage {
4824 module: mesh_shader_module.raw(),
4825 entry_point: &_mesh_entry_point_name,
4826 constants: &stage_desc.constants,
4827 zero_initialize_workgroup_memory: stage_desc
4828 .zero_initialize_workgroup_memory,
4829 })
4830 };
4831 }
4832 }
4833
4834 let fragment_entry_point_name;
4835 let fragment_stage = match desc.fragment {
4836 Some(ref fragment_state) => {
4837 let stage = validation::ShaderStageForValidation::Fragment {
4838 dual_source_blending,
4839 has_depth_attachment,
4840 };
4841 let stage_bit = stage.to_wgt_bit();
4842
4843 let shader_module = &fragment_state.stage.module;
4844 shader_module.same_device(self)?;
4845
4846 let stage_err = |error| pipeline::CreateRenderPipelineError::Stage {
4847 stage: stage_bit,
4848 error,
4849 };
4850
4851 fragment_entry_point_name = shader_module
4852 .finalize_entry_point_name(
4853 stage.to_naga(),
4854 fragment_state
4855 .stage
4856 .entry_point
4857 .as_ref()
4858 .map(|ep| ep.as_ref()),
4859 )
4860 .map_err(stage_err)?;
4861
4862 if let Some(interface) = shader_module.interface.interface() {
4863 immediate_slots_required |= interface
4864 .immediate_slots_required(stage.to_naga(), &fragment_entry_point_name);
4865 io = interface
4866 .check_stage(
4867 &mut binding_layout_source,
4868 &mut shader_binding_sizes,
4869 &fragment_entry_point_name,
4870 stage,
4871 io,
4872 Some(desc.primitive.topology),
4873 )
4874 .map_err(stage_err)?;
4875 validated_stages |= stage_bit;
4876 }
4877
4878 Some(hal::ProgrammableStage {
4879 module: shader_module.raw(),
4880 entry_point: &fragment_entry_point_name,
4881 constants: &fragment_state.stage.constants,
4882 zero_initialize_workgroup_memory: fragment_state
4883 .stage
4884 .zero_initialize_workgroup_memory,
4885 })
4886 }
4887 None => None,
4888 };
4889
4890 if validated_stages.contains(wgt::ShaderStages::FRAGMENT) {
4891 for (i, output) in io.varyings.iter() {
4892 match color_targets.get(*i as usize) {
4893 Some(Some(state)) => {
4894 validation::check_texture_format(state.format, &output.ty).map_err(
4895 |pipeline| {
4896 pipeline::CreateRenderPipelineError::ColorState(
4897 *i as u8,
4898 ColorStateError::IncompatibleFormat {
4899 pipeline,
4900 shader: output.ty,
4901 },
4902 )
4903 },
4904 )?;
4905 }
4906 _ => {
4907 log::debug!(
4908 "The fragment stage {:?} output @location({}) values are ignored",
4909 fragment_stage
4910 .as_ref()
4911 .map_or("", |stage| stage.entry_point),
4912 i
4913 );
4914 }
4915 }
4916 }
4917 }
4918 let last_stage = match desc.fragment {
4919 Some(_) => wgt::ShaderStages::FRAGMENT,
4920 None => wgt::ShaderStages::VERTEX,
4921 };
4922 if is_auto_layout && !validated_stages.contains(last_stage) {
4923 return Err(pipeline::ImplicitLayoutError::ReflectionError(last_stage).into());
4924 }
4925
4926 let pipeline_layout = match binding_layout_source {
4927 validation::BindingLayoutSource::Provided(pipeline_layout) => pipeline_layout,
4928 validation::BindingLayoutSource::Derived(entries) => {
4929 let immediate_size = {
4930 let immediate_size_of = |sm: &pipeline::ShaderModule| {
4931 sm.interface.interface().map(|i| i.immediate_size)
4932 };
4933 let vertex = match desc.vertex {
4934 pipeline::RenderPipelineVertexProcessor::Vertex(ref v) => {
4935 immediate_size_of(&v.stage.module)
4936 }
4937 pipeline::RenderPipelineVertexProcessor::Mesh(ref task, ref mesh) => task
4938 .as_ref()
4939 .and_then(|t| immediate_size_of(&t.stage.module))
4940 .max(immediate_size_of(&mesh.stage.module)),
4941 };
4942 let fragment = desc
4943 .fragment
4944 .as_ref()
4945 .and_then(|f| immediate_size_of(&f.stage.module));
4946 vertex.max(fragment).unwrap_or(0)
4947 };
4948 self.create_derived_pipeline_layout(entries, immediate_size)?
4949 }
4950 };
4951
4952 if let pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) = desc.vertex {
4953 let bind_groups_plus_vertex_buffers =
4954 u32::try_from(pipeline_layout.bind_group_layouts.len() + vertex.buffers.len())
4955 .unwrap();
4956 if bind_groups_plus_vertex_buffers > self.limits.max_bind_groups_plus_vertex_buffers {
4957 return Err(
4958 pipeline::CreateRenderPipelineError::TooManyBindGroupsPlusVertexBuffers {
4959 given: bind_groups_plus_vertex_buffers,
4960 limit: self.limits.max_bind_groups_plus_vertex_buffers,
4961 },
4962 );
4963 }
4964
4965 let given = pipeline_layout
4966 .buffers_and_acceleration_structures_in_vertex_stage
4967 .saturating_add(vertex.buffers.len() as u32);
4968 if !self
4969 .instance_flags
4970 .contains(wgt::InstanceFlags::STRICT_WEBGPU_COMPLIANCE)
4971 {
4972 let limit = self
4973 .limits
4974 .max_buffers_and_acceleration_structures_per_shader_stage;
4975 if given > limit {
4976 return Err(
4977 pipeline::CreateRenderPipelineError::TooManyBuffersAndAccelerationStructuresInVertexStage {
4978 given,
4979 limit,
4980 },
4981 );
4982 }
4983 }
4984 }
4985
4986 if let Some(mv_mask) = desc.multiview_mask {
4988 self.require_features(wgt::Features::MULTIVIEW)?;
4989 if !(mv_mask.get() + 1).is_power_of_two() {
4990 self.require_features(wgt::Features::SELECTIVE_MULTIVIEW)?;
4991 }
4992 }
4993
4994 if !self
4995 .downlevel
4996 .flags
4997 .contains(wgt::DownlevelFlags::BUFFER_BINDINGS_NOT_16_BYTE_ALIGNED)
4998 {
4999 for (binding, size) in shader_binding_sizes.iter() {
5000 if size.get() % 16 != 0 {
5001 return Err(pipeline::CreateRenderPipelineError::UnalignedShader {
5002 binding: binding.binding,
5003 group: binding.group,
5004 size: size.get(),
5005 });
5006 }
5007 }
5008 }
5009
5010 let late_sized_buffer_groups =
5011 Device::make_late_sized_buffer_groups(&shader_binding_sizes, &pipeline_layout);
5012
5013 let cache = match desc.cache {
5014 Some(cache) => {
5015 cache.same_device(self)?;
5016 Some(cache)
5017 }
5018 None => None,
5019 };
5020
5021 let is_mesh = mesh_stage.is_some();
5022 let has_task_shader = task_stage.is_some();
5023 let raw = {
5024 let pipeline_desc = hal::RenderPipelineDescriptor {
5025 label: desc.label.to_hal(self.instance_flags),
5026 layout: pipeline_layout.raw()?,
5027 vertex_processor: match vertex_stage {
5028 Some(vertex_stage) => hal::VertexProcessor::Standard {
5029 vertex_buffers: &hal_vertex_buffer_layouts,
5030 vertex_stage,
5031 },
5032 None => hal::VertexProcessor::Mesh {
5033 task_stage,
5034 mesh_stage: mesh_stage.unwrap(),
5035 },
5036 },
5037 primitive: desc.primitive,
5038 depth_stencil: desc.depth_stencil.clone(),
5039 multisample: desc.multisample,
5040 fragment_stage,
5041 color_targets,
5042 multiview_mask: desc.multiview_mask,
5043 cache: cache.as_ref().map(|it| it.raw()),
5044 };
5045 unsafe { self.raw().create_render_pipeline(&pipeline_desc) }.map_err(
5046 |err| match err {
5047 hal::PipelineError::Device(error) => {
5048 pipeline::CreateRenderPipelineError::Device(self.handle_hal_error(error))
5049 }
5050 hal::PipelineError::Linkage(stage, msg) => {
5051 pipeline::CreateRenderPipelineError::Internal { stage, error: msg }
5052 }
5053 hal::PipelineError::EntryPoint(stage) => {
5054 pipeline::CreateRenderPipelineError::Internal {
5055 stage: hal::auxil::map_naga_stage(stage),
5056 error: ENTRYPOINT_FAILURE_ERROR.to_string(),
5057 }
5058 }
5059 hal::PipelineError::PipelineConstants(stage, error) => {
5060 pipeline::CreateRenderPipelineError::PipelineConstants { stage, error }
5061 }
5062 },
5063 )?
5064 };
5065
5066 let pass_context = RenderPassContext {
5067 attachments: AttachmentData {
5068 colors: color_targets
5069 .iter()
5070 .map(|state| state.as_ref().map(|s| s.format))
5071 .collect(),
5072 resolves: ArrayVec::new(),
5073 depth_stencil: depth_stencil_state.as_ref().map(|state| state.format),
5074 },
5075 sample_count: samples,
5076 multiview_mask: desc.multiview_mask,
5077 };
5078
5079 let mut flags = pipeline::PipelineFlags::empty();
5080 for state in color_targets.iter().filter_map(|s| s.as_ref()) {
5081 if let Some(ref bs) = state.blend {
5082 if bs.color.uses_constant() | bs.alpha.uses_constant() {
5083 flags |= pipeline::PipelineFlags::BLEND_CONSTANT;
5084 }
5085 }
5086 }
5087 if let Some(ds) = depth_stencil_state.as_ref() {
5088 if ds.stencil.is_enabled() && ds.stencil.needs_ref_value() {
5089 flags |= pipeline::PipelineFlags::STENCIL_REFERENCE;
5090 }
5091 if !ds.is_depth_read_only() {
5092 flags |= pipeline::PipelineFlags::WRITES_DEPTH;
5093 }
5094 if !ds.is_stencil_read_only(desc.primitive.cull_mode) {
5095 flags |= pipeline::PipelineFlags::WRITES_STENCIL;
5096 }
5097 }
5098 let shader_modules = if self.backend() == wgt::Backend::Vulkan {
5103 ArrayVec::new()
5104 } else {
5105 let mut shader_modules = ArrayVec::new();
5106 match desc.vertex {
5107 pipeline::RenderPipelineVertexProcessor::Vertex(vertex) => {
5108 shader_modules.push(vertex.stage.module)
5109 }
5110 pipeline::RenderPipelineVertexProcessor::Mesh(task, mesh) => {
5111 if let Some(task) = task {
5112 shader_modules.push(task.stage.module);
5113 }
5114 shader_modules.push(mesh.stage.module);
5115 }
5116 }
5117 shader_modules.extend(desc.fragment.map(|f| f.stage.module));
5118 shader_modules
5119 };
5120
5121 let pipeline = pipeline::RenderPipeline {
5122 state: ResourceState::Valid(pipeline::RenderPipelineState {
5123 raw: ManuallyDrop::new(raw),
5124 layout: pipeline_layout.clone(),
5125 }),
5126 device: self.clone(),
5127 pass_context,
5128 _shader_modules: shader_modules,
5129 flags,
5130 topology: desc.primitive.topology,
5131 strip_index_format: desc.primitive.strip_index_format,
5132 vertex_steps,
5133 late_sized_buffer_groups,
5134 immediate_slots_required,
5135 label: desc.label.to_string(),
5136 tracking_data: TrackingData::new(self.tracker_indices.render_pipelines.clone()),
5137 is_mesh,
5138 has_task_shader,
5139 };
5140
5141 let pipeline = Arc::new(pipeline);
5142
5143 if is_auto_layout {
5144 for bgl in pipeline_layout.bind_group_layouts.iter() {
5145 let Some(bgl) = bgl else {
5146 continue;
5147 };
5148
5149 let _ = bgl.exclusive_pipeline.set((&pipeline).into());
5152 }
5153 }
5154
5155 Ok(pipeline)
5156 }
5157
5158 pub unsafe fn create_pipeline_cache(
5162 self: &Arc<Self>,
5163 desc: &pipeline::PipelineCacheDescriptor,
5164 ) -> Result<Arc<pipeline::PipelineCache>, pipeline::CreatePipelineCacheError> {
5165 use crate::pipeline_cache;
5166
5167 self.check_is_valid()?;
5168
5169 self.require_features(wgt::Features::PIPELINE_CACHE)?;
5170 let data = if let Some((data, validation_key)) = desc
5171 .data
5172 .as_ref()
5173 .zip(self.raw().pipeline_cache_validation_key())
5174 {
5175 let data = pipeline_cache::validate_pipeline_cache(
5176 data,
5177 &self.adapter.raw.info,
5178 validation_key,
5179 );
5180 match data {
5181 Ok(data) => Some(data),
5182 Err(e) if e.was_avoidable() || !desc.fallback => return Err(e.into()),
5183 Err(_) => None,
5185 }
5186 } else {
5187 None
5188 };
5189 let cache_desc = hal::PipelineCacheDescriptor {
5190 data,
5191 label: desc.label.to_hal(self.instance_flags),
5192 };
5193 let raw = match unsafe { self.raw().create_pipeline_cache(&cache_desc) } {
5194 Ok(raw) => raw,
5195 Err(e) => match e {
5196 hal::PipelineCacheError::Device(e) => return Err(self.handle_hal_error(e).into()),
5197 },
5198 };
5199 let cache = pipeline::PipelineCache {
5200 device: self.clone(),
5201 label: desc.label.to_string(),
5202 raw: ManuallyDrop::new(raw),
5204 };
5205
5206 let cache = Arc::new(cache);
5207
5208 Ok(cache)
5209 }
5210
5211 fn get_texture_format_features(&self, format: TextureFormat) -> wgt::TextureFormatFeatures {
5212 use wgt::TextureFormatFeatureFlags as tfsc;
5214 let mut format_features = self.adapter.get_texture_format_features(format);
5215 if (format == TextureFormat::R32Float
5216 || format == TextureFormat::Rg32Float
5217 || format == TextureFormat::Rgba32Float)
5218 && !self.features.contains(wgt::Features::FLOAT32_FILTERABLE)
5219 {
5220 format_features.flags.set(tfsc::FILTERABLE, false);
5221 }
5222 format_features
5223 }
5224
5225 pub(crate) fn describe_format_features(
5226 &self,
5227 format: TextureFormat,
5228 ) -> Result<wgt::TextureFormatFeatures, MissingFeatures> {
5229 self.require_features(format.required_features())?;
5230
5231 let using_device_features = self
5232 .features
5233 .contains(wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES);
5234 let downlevel = !self
5237 .downlevel
5238 .flags
5239 .contains(wgt::DownlevelFlags::WEBGPU_TEXTURE_FORMAT_SUPPORT);
5240
5241 if using_device_features || downlevel {
5242 Ok(self.get_texture_format_features(format))
5243 } else {
5244 Ok(format.guaranteed_format_features(self.features))
5245 }
5246 }
5247
5248 #[cfg(feature = "replay")]
5249 pub(crate) fn wait_for_submit(
5250 &self,
5251 submission_index: crate::SubmissionIndex,
5252 ) -> Result<(), DeviceError> {
5253 let last_done_index = unsafe { self.raw().get_fence_value(self.fence.as_ref()) }
5254 .map_err(|e| self.handle_hal_error(e))?;
5255 if last_done_index < submission_index {
5256 unsafe { self.raw().wait(self.fence.as_ref(), submission_index, None) }
5257 .map_err(|e| self.handle_hal_error(e))?;
5258 if let Some(queue) = self.get_queue() {
5259 let closures = queue.lock_life().triage_submissions(submission_index);
5260 assert!(
5261 closures.is_empty(),
5262 "wait_for_submit is not expected to work with closures"
5263 );
5264 }
5265 }
5266 Ok(())
5267 }
5268
5269 pub fn create_query_set(
5270 self: &Arc<Self>,
5271 desc: &resource::QuerySetDescriptor,
5272 ) -> Result<Arc<QuerySet>, resource::CreateQuerySetError> {
5273 use resource::CreateQuerySetError as Error;
5274
5275 self.check_is_valid()?;
5276
5277 match desc.ty {
5278 wgt::QueryType::Occlusion => {}
5279 wgt::QueryType::Timestamp => {
5280 self.require_features(wgt::Features::TIMESTAMP_QUERY)?;
5281 }
5282 wgt::QueryType::PipelineStatistics(..) => {
5283 self.require_features(wgt::Features::PIPELINE_STATISTICS_QUERY)?;
5284 }
5285 }
5286
5287 if desc.count == 0 {
5288 return Err(Error::ZeroCount);
5289 }
5290
5291 if desc.count > wgt::QUERY_SET_MAX_QUERIES {
5292 return Err(Error::TooManyQueries {
5293 count: desc.count,
5294 maximum: wgt::QUERY_SET_MAX_QUERIES,
5295 });
5296 }
5297
5298 let hal_desc = desc.map_label(|label| label.to_hal(self.instance_flags));
5299
5300 let raw = unsafe { self.raw().create_query_set(&hal_desc) }
5301 .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
5302
5303 let query_set = QuerySet {
5304 raw: Snatchable::new(raw),
5305 device: self.clone(),
5306 label: desc.label.to_string(),
5307 tracking_data: TrackingData::new(self.tracker_indices.query_sets.clone()),
5308 desc: desc.map_label(|_| ()),
5309 initialized_slots: Mutex::new(
5310 rank::QUERY_SET_INITIALIZED_SLOTS,
5311 bit_vec::BitVec::from_elem(desc.count as usize, false),
5312 ),
5313 };
5314
5315 let query_set = Arc::new(query_set);
5316
5317 Ok(query_set)
5318 }
5319
5320 pub fn configure_surface(
5321 self: &Arc<Self>,
5322 surface: &crate::instance::Surface,
5323 config: &wgt::SurfaceConfiguration<Vec<TextureFormat>>,
5324 ) -> Option<present::ConfigureSurfaceError> {
5325 use present::ConfigureSurfaceError as E;
5326 profiling::scope!("surface_configure");
5327
5328 log::debug!("configuring surface with {config:?}");
5329
5330 let error = 'error: {
5331 let user_callbacks;
5333 {
5334 if let Err(e) = self.check_is_valid() {
5335 break 'error e.into();
5336 }
5337
5338 let caps = match surface.get_capabilities(&self.adapter) {
5339 Ok(caps) => caps,
5340 Err(_) => break 'error E::UnsupportedQueueFamily,
5341 };
5342
5343 let mut hal_view_formats = Vec::new();
5344 for format in config.view_formats.iter() {
5345 if *format == config.format {
5346 continue;
5347 }
5348 if !caps.formats.iter().any(|fc| fc.format == config.format) {
5349 break 'error E::UnsupportedFormat {
5350 requested: config.format,
5351 available: caps.texture_formats().collect(),
5352 };
5353 }
5354 if config.format.remove_srgb_suffix() != format.remove_srgb_suffix() {
5355 break 'error E::InvalidViewFormat(*format, config.format);
5356 }
5357 hal_view_formats.push(*format);
5358 }
5359
5360 if !hal_view_formats.is_empty() {
5361 if let Err(missing_flag) =
5362 self.require_downlevel_flags(wgt::DownlevelFlags::SURFACE_VIEW_FORMATS)
5363 {
5364 break 'error E::MissingDownlevelFlags(missing_flag);
5365 }
5366 }
5367
5368 let maximum_frame_latency = config.desired_maximum_frame_latency.clamp(
5369 *caps.maximum_frame_latency.start(),
5370 *caps.maximum_frame_latency.end(),
5371 );
5372 let mut hal_config = hal::SurfaceConfiguration {
5373 maximum_frame_latency,
5374 present_mode: config.present_mode,
5375 composite_alpha_mode: config.alpha_mode,
5376 format: config.format,
5377 color_space: config.color_space,
5378 extent: wgt::Extent3d {
5379 width: config.width,
5380 height: config.height,
5381 depth_or_array_layers: 1,
5382 },
5383 usage: conv::map_texture_usage(
5384 config.usage,
5385 hal::FormatAspects::COLOR,
5386 wgt::TextureFormatFeatureFlags::STORAGE_READ_ONLY
5387 | wgt::TextureFormatFeatureFlags::STORAGE_WRITE_ONLY
5388 | wgt::TextureFormatFeatureFlags::STORAGE_READ_WRITE,
5389 ),
5390 view_formats: hal_view_formats,
5391 keep_presented: surface
5392 .keep_images
5393 .load(Ordering::Acquire),
5394 };
5395
5396 if let Err(error) = validate_surface_configuration(
5397 &mut hal_config,
5398 &caps,
5399 self.limits.max_texture_dimension_2d,
5400 ) {
5401 break 'error error;
5402 }
5403
5404 let snatch_guard = self.snatchable_lock.read();
5406
5407 let maintain_result;
5408 (user_callbacks, maintain_result) =
5409 self.maintain(wgt::PollType::wait_indefinitely(), snatch_guard);
5410
5411 match maintain_result {
5412 Ok(wgt::PollStatus::QueueEmpty | wgt::PollStatus::WaitSucceeded) => {}
5419 Ok(wgt::PollStatus::Poll) => {
5420 unreachable!("Cannot get a Poll result from a Wait action.")
5421 }
5422 Err(WaitIdleError::Timeout) if cfg!(target_arch = "wasm32") => {
5423 }
5428 Err(e) => {
5429 break 'error e.into();
5430 }
5431 }
5432
5433 if let Some(present) = surface.presentation.lock().take() {
5435 if present.acquired_texture.is_some() {
5436 break 'error E::PreviousOutputExists;
5437 }
5438 }
5439
5440 let surface_raw = surface.raw(self.backend()).unwrap();
5447 match unsafe { surface_raw.configure(self.raw(), &hal_config) } {
5448 Ok(()) => (),
5449 Err(error) => {
5450 break 'error match error {
5451 hal::SurfaceError::Outdated
5452 | hal::SurfaceError::Lost
5453 | hal::SurfaceError::Occluded
5454 | hal::SurfaceError::Timeout => E::InvalidSurface,
5455 hal::SurfaceError::Device(error) => {
5456 E::Device(self.handle_hal_error(error))
5457 }
5458 hal::SurfaceError::Other(message) => {
5459 log::error!("surface configuration failed: {message}");
5460 E::InvalidSurface
5461 }
5462 }
5463 }
5464 }
5465
5466 let mut presentation = surface.presentation.lock();
5467 *presentation = Some(present::Presentation {
5468 device: Arc::clone(self),
5469 config: config.clone(),
5470 acquired_texture: None,
5471 acquired_image: None,
5472 images: present::PresentedImages::default(),
5473 });
5474 }
5475
5476 user_callbacks.fire();
5477 return None;
5478 };
5479
5480 Some(error)
5481 }
5482
5483 fn lose(&self, message: &str) {
5484 self.valid.store(false, Ordering::Release);
5489
5490 if let Some(device_lost_closure) = self.device_lost_closure.lock().take() {
5492 device_lost_closure(DeviceLostReason::Unknown, message.to_string());
5493 }
5494
5495 }
5503
5504 fn release_gpu_resources(&self) {
5505 let trackers = self.trackers.lock();
5515 for buffer in trackers.buffers.used_resources() {
5516 if let Some(buffer) = Weak::upgrade(buffer) {
5517 buffer.destroy();
5518 }
5519 }
5520 for texture in trackers.textures.used_resources() {
5521 if let Some(texture) = Weak::upgrade(texture) {
5522 texture.destroy();
5523 }
5524 }
5525 }
5526
5527 pub(crate) fn new_usage_scope(&self) -> UsageScope<'_> {
5528 UsageScope::new_pooled(
5529 &self.usage_scopes,
5530 &self.tracker_indices,
5531 self.ordered_buffer_usages,
5532 self.ordered_texture_usages,
5533 )
5534 }
5535
5536 pub fn get_hal_counters(&self) -> wgt::HalCounters {
5537 self.raw().get_internal_counters()
5538 }
5539
5540 pub fn generate_allocator_report(&self) -> Option<wgt::AllocatorReport> {
5541 self.raw().generate_allocator_report()
5542 }
5543}
5544
5545crate::impl_resource_type!(Device);
5546crate::impl_labeled!(Device);
5547crate::impl_storage_item!(Device);