Skip to main content

wgpu_core/device/
resource.rs

1use alloc::{
2    borrow::Cow,
3    boxed::Box,
4    string::{String, ToString as _},
5    sync::{Arc, Weak},
6    vec::Vec,
7};
8use core::{
9    fmt,
10    mem::{self, ManuallyDrop},
11    num::NonZeroU32,
12    sync::atomic::{AtomicBool, Ordering},
13};
14use hal::ShouldBeNonZeroExt;
15
16use arrayvec::ArrayVec;
17use bitflags::Flags;
18use smallvec::SmallVec;
19use wgt::{
20    math::align_to, DeviceLostReason, TextureFormat, TextureSampleType, TextureSelector,
21    TextureViewDimension,
22};
23
24#[cfg(feature = "trace")]
25use crate::device::trace;
26use crate::{
27    api_log,
28    binding_model::{
29        self, BindGroup, BindGroupLateBufferBindingInfo, BindGroupLayout,
30        BindGroupLayoutEntryError, BindGroupLayoutState, CreateBindGroupError,
31        CreateBindGroupLayoutError,
32    },
33    command, conv,
34    device::{
35        bgl, create_validator, features_to_naga_capabilities, life::WaitIdleError, map_buffer,
36        AttachmentData, DeviceLostInvocation, HostMap, MissingDownlevelFlags, MissingFeatures,
37        RenderPassContext,
38    },
39    hal_label,
40    init_tracker::{
41        BufferInitTracker, BufferInitTrackerAction, MemoryInitKind, TextureInitRange,
42        TextureInitTrackerAction,
43    },
44    instance::{Adapter, RequestDeviceError},
45    lock::{rank, Mutex, RwLock},
46    pipeline::{self, ColorStateError},
47    pool::ResourcePool,
48    present,
49    resource::{
50        self, Buffer, ExternalTexture, Fallible, Labeled, ParentDevice, QuerySet,
51        RawResourceAccess, ResourceState, Sampler, StagingBuffer, Texture, TextureView,
52        TextureViewNotRenderableReason, Tlas, TrackingData,
53    },
54    resource_log,
55    snatch::{SnatchGuard, SnatchLock, Snatchable},
56    timestamp_normalization::TIMESTAMP_NORMALIZATION_BUFFER_USES,
57    track::{BindGroupStates, DeviceTracker, TrackerIndexAllocators, UsageScope, UsageScopePool},
58    validation::{self, check_color_attachment_count, PassthroughInterface, ShaderMetaData},
59    weak_vec::WeakVec,
60    FastHashMap, LabelHelpers, OnceCellOrLock,
61};
62
63use super::{
64    queue::Queue, surface_config::validate_surface_configuration, DeviceDescriptor, DeviceError,
65    DeviceLostClosure, UserClosures, ENTRYPOINT_FAILURE_ERROR, ZERO_BUFFER_SIZE,
66};
67
68#[cfg(supports_64bit_atomics)]
69use core::sync::atomic::AtomicU64;
70#[cfg(not(supports_64bit_atomics))]
71use portable_atomic::AtomicU64;
72
73pub(crate) struct CommandIndices {
74    /// The index of the last command submission that was attempted.
75    ///
76    /// Note that `fence` may never be signalled with this value, if the command
77    /// submission failed. If you need to wait for everything running on a
78    /// `Queue` to complete, wait for [`last_successful_submission_index`].
79    ///
80    /// [`last_successful_submission_index`]: Device::last_successful_submission_index
81    pub(crate) active_submission_index: hal::FenceValue,
82    pub(crate) next_acceleration_structure_build_command_index: u64,
83}
84
85/// Parameters provided to shaders via a uniform buffer of the type
86/// [`NagaExternalTextureParams`], describing an [`ExternalTexture`] resource
87/// binding.
88///
89/// [`NagaExternalTextureParams`]: naga::SpecialTypes::external_texture_params
90/// [`ExternalTexture`]: binding_model::BindingResource::ExternalTexture
91#[repr(C)]
92#[derive(Copy, Clone, bytemuck::Zeroable, bytemuck::Pod)]
93pub struct ExternalTextureParams {
94    /// 4x4 column-major matrix with which to convert sampled YCbCr values
95    /// to RGBA.
96    ///
97    /// This is ignored when `num_planes` is 1.
98    pub yuv_conversion_matrix: [f32; 16],
99
100    /// 3x3 column-major matrix to transform linear RGB values in the source
101    /// color space to linear RGB values in the destination color space. In
102    /// combination with [`Self::src_transfer_function`] and
103    /// [`Self::dst_transfer_function`] this can be used to ensure that
104    /// [`ImageSample`] and [`ImageLoad`] operations return values in the
105    /// desired destination color space rather than the source color space of
106    /// the underlying planes.
107    ///
108    /// Includes a padding element after each column.
109    ///
110    /// [`ImageSample`]: naga::ir::Expression::ImageSample
111    /// [`ImageLoad`]: naga::ir::Expression::ImageLoad
112    pub gamut_conversion_matrix: [f32; 12],
113
114    /// Transfer function for the source color space. The *inverse* of this
115    /// will be applied to decode non-linear RGB to linear RGB in the source
116    /// color space.
117    pub src_transfer_function: wgt::ExternalTextureTransferFunction,
118
119    /// Transfer function for the destination color space. This will be applied
120    /// to encode linear RGB to non-linear RGB in the destination color space.
121    pub dst_transfer_function: wgt::ExternalTextureTransferFunction,
122
123    /// Transform to apply to [`ImageSample`] coordinates.
124    ///
125    /// This is a 3x2 column-major matrix representing an affine transform from
126    /// normalized texture coordinates to the normalized coordinates that should
127    /// be sampled from the external texture's underlying plane(s).
128    ///
129    /// This transform may scale, translate, flip, and rotate in 90-degree
130    /// increments, but the result of transforming the rectangle (0,0)..(1,1)
131    /// must be an axis-aligned rectangle that falls within the bounds of
132    /// (0,0)..(1,1).
133    ///
134    /// [`ImageSample`]: naga::ir::Expression::ImageSample
135    pub sample_transform: [f32; 6],
136
137    /// Transform to apply to [`ImageLoad`] coordinates.
138    ///
139    /// This is a 3x2 column-major matrix representing an affine transform from
140    /// non-normalized texel coordinates to the non-normalized coordinates of
141    /// the texel that should be loaded from the external texture's underlying
142    /// plane 0. For planes 1 and 2, if present, plane 0's coordinates are
143    /// scaled according to the textures' relative sizes.
144    ///
145    /// This transform may scale, translate, flip, and rotate in 90-degree
146    /// increments, but the result of transforming the rectangle (0,0)..[`size`]
147    /// must be an axis-aligned rectangle that falls within the bounds of
148    /// (0,0)..[`size`].
149    ///
150    /// [`ImageLoad`]: naga::ir::Expression::ImageLoad
151    /// [`size`]: Self::size
152    pub load_transform: [f32; 6],
153
154    /// Size of the external texture.
155    ///
156    /// This is the value that should be returned by size queries in shader
157    /// code; it does not necessarily match the dimensions of the underlying
158    /// texture(s). As a special case, if this is `[0, 0]`, the actual size of
159    /// plane 0 should be used instead.
160    ///
161    /// This must be consistent with [`sample_transform`]: it should be the size
162    /// in texels of the rectangle covered by the square (0,0)..(1,1) after
163    /// [`sample_transform`] has been applied to it.
164    ///
165    /// [`sample_transform`]: Self::sample_transform
166    pub size: [u32; 2],
167
168    /// Number of planes. 1 indicates a single RGBA plane. 2 indicates a Y
169    /// plane and an interleaved CbCr plane. 3 indicates separate Y, Cb, and Cr
170    /// planes.
171    pub num_planes: u32,
172    // Ensure the size of this struct matches the type generated by Naga.
173    pub _padding: [u8; 4],
174}
175
176impl ExternalTextureParams {
177    pub fn from_desc<L>(desc: &wgt::ExternalTextureDescriptor<L>) -> Self {
178        let gamut_conversion_matrix = [
179            desc.gamut_conversion_matrix[0],
180            desc.gamut_conversion_matrix[1],
181            desc.gamut_conversion_matrix[2],
182            0.0, // padding
183            desc.gamut_conversion_matrix[3],
184            desc.gamut_conversion_matrix[4],
185            desc.gamut_conversion_matrix[5],
186            0.0, // padding
187            desc.gamut_conversion_matrix[6],
188            desc.gamut_conversion_matrix[7],
189            desc.gamut_conversion_matrix[8],
190            0.0, // padding
191        ];
192
193        Self {
194            yuv_conversion_matrix: desc.yuv_conversion_matrix,
195            gamut_conversion_matrix,
196            src_transfer_function: desc.src_transfer_function,
197            dst_transfer_function: desc.dst_transfer_function,
198            size: [desc.width, desc.height],
199            sample_transform: desc.sample_transform,
200            load_transform: desc.load_transform,
201            num_planes: desc.num_planes() as u32,
202            _padding: Default::default(),
203        }
204    }
205}
206
207/// Structure describing a logical device. Some members are internally mutable,
208/// stored behind mutexes.
209pub struct Device {
210    raw: Box<dyn hal::DynDevice>,
211    pub(crate) adapter: Arc<Adapter>,
212    pub(crate) queue: OnceCellOrLock<Weak<Queue>>,
213    pub(crate) zero_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
214    pub(crate) empty_bgl: ManuallyDrop<Box<dyn hal::DynBindGroupLayout>>,
215    /// The `label` from the descriptor used to create the resource.
216    label: String,
217
218    pub(crate) command_allocator: command::CommandAllocator,
219
220    pub(crate) command_indices: RwLock<CommandIndices>,
221
222    /// The index of the last successful submission to this device's
223    /// [`hal::Queue`].
224    ///
225    /// Unlike [`active_submission_index`], which is incremented each time
226    /// submission is attempted, this is updated only when submission succeeds,
227    /// so waiting for this value won't hang waiting for work that was never
228    /// submitted.
229    ///
230    /// [`active_submission_index`]: CommandIndices::active_submission_index
231    pub(crate) last_successful_submission_index: hal::AtomicFenceValue,
232
233    pub(crate) fence: ManuallyDrop<Box<dyn hal::DynFence>>,
234    pub(crate) snatchable_lock: SnatchLock,
235
236    /// Is this device valid? Valid is closely associated with "lose the device",
237    /// which can be triggered by various methods, including at the end of device
238    /// destroy, and by any GPU errors that cause us to no longer trust the state
239    /// of the device. Ideally we would like to fold valid into the storage of
240    /// the device itself (for example as an Error enum), but unfortunately we
241    /// need to continue to be able to retrieve the device in poll_devices to
242    /// determine if it can be dropped. If our internal accesses of devices were
243    /// done through ref-counted references and external accesses checked for
244    /// Error enums, we wouldn't need this. For now, we need it. All the call
245    /// sites where we check it are areas that should be revisited if we start
246    /// using ref-counted references for internal access.
247    pub(crate) valid: AtomicBool,
248
249    /// Closure to be called on "lose the device". This is invoked directly by
250    /// device.lose or by the UserCallbacks returned from maintain when the device
251    /// has been destroyed and its queues are empty.
252    pub(crate) device_lost_closure: Mutex<Option<DeviceLostClosure>>,
253
254    /// Stores the state of buffers and textures.
255    pub(crate) trackers: Mutex<DeviceTracker>,
256    pub(crate) tracker_indices: TrackerIndexAllocators,
257    /// Pool of bind group layouts, allowing deduplication.
258    pub(crate) bgl_pool: ResourcePool<bgl::EntryMap, BindGroupLayout>,
259    pub(crate) alignments: hal::Alignments,
260    pub(crate) limits: wgt::Limits,
261    pub(crate) features: wgt::Features,
262    pub(crate) downlevel: wgt::DownlevelCapabilities,
263    /// Buffer uses listed here, are expected to be ordered by the underlying hardware.
264    /// If a usage is ordered, then if the buffer state doesn't change between draw calls,
265    /// there are no barriers needed for synchronization.
266    /// See the implementations of [`hal::Adapter::get_ordered_buffer_usages`] for hardware specific info
267    pub(crate) ordered_buffer_usages: wgt::BufferUses,
268    /// Texture uses listed here, are expected to be ordered by the underlying hardware.
269    /// If a usage is ordered, then if the buffer state doesn't change between draw calls,
270    /// there are no barriers needed for synchronization.
271    /// See the implementations of [`hal::Adapter::get_ordered_texture_usages`] for hardware specific info
272    pub(crate) ordered_texture_usages: wgt::TextureUses,
273    pub(crate) instance_flags: wgt::InstanceFlags,
274    pub(crate) deferred_destroy: Mutex<Vec<DeferredDestroy>>,
275    pub(crate) usage_scopes: UsageScopePool,
276    pub(crate) indirect_validation: Option<crate::indirect_validation::IndirectValidation>,
277    // Optional so that we can late-initialize this after the queue is created.
278    pub(crate) timestamp_normalizer:
279        OnceCellOrLock<crate::timestamp_normalization::TimestampNormalizer>,
280    /// Uniform buffer containing [`ExternalTextureParams`] with values such
281    /// that a [`TextureView`] bound to a [`wgt::BindingType::ExternalTexture`]
282    /// binding point will be rendered correctly. Intended to be used as the
283    /// [`hal::ExternalTextureBinding::params`] field.
284    pub(crate) default_external_texture_params_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
285    // needs to be dropped last
286    #[cfg(feature = "trace")]
287    pub(crate) trace: Mutex<Option<Box<dyn trace::Trace + Send + Sync + 'static>>>,
288}
289
290pub(crate) enum DeferredDestroy {
291    TextureViews(WeakVec<TextureView>),
292    BindGroups(WeakVec<BindGroup>),
293}
294
295impl fmt::Debug for Device {
296    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
297        f.debug_struct("Device")
298            .field("label", &self.label())
299            .field("limits", &self.limits)
300            .field("features", &self.features)
301            .field("downlevel", &self.downlevel)
302            .finish()
303    }
304}
305
306impl Drop for Device {
307    fn drop(&mut self) {
308        resource_log!("Drop {}", self.error_ident());
309
310        // SAFETY: We are in the Drop impl and we don't use self.zero_buffer anymore after this
311        // point.
312        let zero_buffer = unsafe { ManuallyDrop::take(&mut self.zero_buffer) };
313        // SAFETY: We are in the Drop impl and we don't use self.empty_bgl anymore after this point.
314        let empty_bgl = unsafe { ManuallyDrop::take(&mut self.empty_bgl) };
315        // SAFETY: We are in the Drop impl and we don't use
316        // self.default_external_texture_params_buffer anymore after this point.
317        let default_external_texture_params_buffer =
318            unsafe { ManuallyDrop::take(&mut self.default_external_texture_params_buffer) };
319        // SAFETY: We are in the Drop impl and we don't use self.fence anymore after this point.
320        let fence = unsafe { ManuallyDrop::take(&mut self.fence) };
321        if let Some(indirect_validation) = self.indirect_validation.take() {
322            indirect_validation.dispose(self.raw.as_ref());
323        }
324        if let Some(timestamp_normalizer) = self.timestamp_normalizer.take() {
325            timestamp_normalizer.dispose(self.raw.as_ref());
326        }
327        unsafe {
328            self.raw.destroy_buffer(zero_buffer);
329            self.raw.destroy_bind_group_layout(empty_bgl);
330            self.raw
331                .destroy_buffer(default_external_texture_params_buffer);
332            self.raw.destroy_fence(fence);
333        }
334    }
335}
336
337impl Device {
338    pub(crate) fn raw(&self) -> &dyn hal::DynDevice {
339        self.raw.as_ref()
340    }
341    pub(crate) fn require_features(&self, feature: wgt::Features) -> Result<(), MissingFeatures> {
342        if self.features.contains(feature) {
343            Ok(())
344        } else {
345            Err(MissingFeatures(feature))
346        }
347    }
348
349    pub(crate) fn require_downlevel_flags(
350        &self,
351        flags: wgt::DownlevelFlags,
352    ) -> Result<(), MissingDownlevelFlags> {
353        if self.downlevel.flags.contains(flags) {
354            Ok(())
355        } else {
356            Err(MissingDownlevelFlags(flags))
357        }
358    }
359
360    /// # Safety
361    ///
362    /// - See [wgpu::Device::start_graphics_debugger_capture][api] for details the safety.
363    ///
364    /// [api]: ../../wgpu/struct.Device.html#method.start_graphics_debugger_capture
365    pub unsafe fn start_graphics_debugger_capture(&self) {
366        api_log!("Device::start_graphics_debugger_capture");
367
368        if !self.is_valid() {
369            return;
370        }
371        unsafe { self.raw().start_graphics_debugger_capture() };
372    }
373
374    /// # Safety
375    ///
376    /// - See [wgpu::Device::stop_graphics_debugger_capture][api] for details the safety.
377    ///
378    /// [api]: ../../wgpu/struct.Device.html#method.stop_graphics_debugger_capture
379    pub unsafe fn stop_graphics_debugger_capture(&self) {
380        api_log!("Device::stop_graphics_debugger_capture");
381
382        if !self.is_valid() {
383            return;
384        }
385        unsafe { self.raw().stop_graphics_debugger_capture() };
386    }
387}
388
389impl Device {
390    pub(crate) fn new(
391        raw_device: Box<dyn hal::DynDevice>,
392        adapter: &Arc<Adapter>,
393        desc: &DeviceDescriptor,
394        instance_flags: wgt::InstanceFlags,
395    ) -> Result<Self, DeviceError> {
396        #[cfg(not(feature = "trace"))]
397        match &desc.trace {
398            wgt::Trace::Off => {}
399            _ => {
400                log::error!("wgpu-core feature 'trace' is not enabled");
401            }
402        };
403        #[cfg(feature = "trace")]
404        let trace: Option<Box<dyn trace::Trace + Send + Sync + 'static>> = match &desc.trace {
405            wgt::Trace::Off => None,
406            wgt::Trace::Directory(dir) => match trace::DiskTrace::new(dir.clone()) {
407                Ok(mut trace) => {
408                    trace::Trace::add(
409                        &mut trace,
410                        trace::Action::Init {
411                            desc: wgt::DeviceDescriptor {
412                                trace: wgt::Trace::Off,
413                                ..desc.clone()
414                            },
415                            backend: adapter.backend(),
416                        },
417                    );
418                    Some(Box::new(trace))
419                }
420                Err(e) => {
421                    log::error!("Unable to start a trace in '{dir:?}': {e}");
422                    None
423                }
424            },
425            wgt::Trace::Memory => {
426                let mut trace = trace::MemoryTrace::new();
427                trace::Trace::add(
428                    &mut trace,
429                    trace::Action::Init {
430                        desc: wgt::DeviceDescriptor {
431                            trace: wgt::Trace::Off,
432                            ..desc.clone()
433                        },
434                        backend: adapter.backend(),
435                    },
436                );
437                Some(Box::new(trace))
438            }
439            // The enum is non_exhaustive, so we must have a fallback arm (that should be
440            // unreachable in practice).
441            t => {
442                log::error!("unimplemented wgpu_types::Trace variant {t:?}");
443                None
444            }
445        };
446
447        let ordered_buffer_usages = adapter.raw.adapter.get_ordered_buffer_usages();
448        let ordered_texture_usages = adapter.raw.adapter.get_ordered_texture_usages();
449
450        let fence = unsafe { raw_device.create_fence() }.map_err(DeviceError::from_hal)?;
451
452        let command_allocator = command::CommandAllocator::new();
453
454        let rt_uses = if desc
455            .required_features
456            .intersects(wgt::Features::EXPERIMENTAL_RAY_QUERY)
457        {
458            wgt::BufferUses::TOP_LEVEL_ACCELERATION_STRUCTURE_INPUT
459        } else {
460            wgt::BufferUses::empty()
461        };
462
463        // Create zeroed buffer used for texture clears (and raytracing if required).
464        let zero_buffer = unsafe {
465            raw_device.create_buffer(&hal::BufferDescriptor {
466                label: hal_label(Some("(wgpu internal) zero init buffer"), instance_flags),
467                size: ZERO_BUFFER_SIZE,
468                usage: wgt::BufferUses::COPY_SRC | wgt::BufferUses::COPY_DST | rt_uses,
469                memory_flags: hal::MemoryFlags::empty(),
470            })
471        }
472        .map_err(DeviceError::from_hal)?;
473
474        let empty_bgl = unsafe {
475            raw_device.create_bind_group_layout(&hal::BindGroupLayoutDescriptor {
476                label: None,
477                flags: hal::BindGroupLayoutFlags::empty(),
478                entries: &[],
479            })
480        }
481        .map_err(DeviceError::from_hal)?;
482
483        let default_external_texture_params_buffer = unsafe {
484            raw_device.create_buffer(&hal::BufferDescriptor {
485                label: hal_label(
486                    Some("(wgpu internal) default external texture params buffer"),
487                    instance_flags,
488                ),
489                size: size_of::<ExternalTextureParams>() as _,
490                usage: wgt::BufferUses::COPY_DST | wgt::BufferUses::UNIFORM,
491                memory_flags: hal::MemoryFlags::empty(),
492            })
493        }
494        .map_err(DeviceError::from_hal)?;
495
496        // Cloned as we need them below anyway.
497        let alignments = adapter.raw.capabilities.alignments.clone();
498        let downlevel = adapter.raw.capabilities.downlevel.clone();
499        let limits = &adapter.raw.capabilities.limits;
500
501        let enable_indirect_validation = instance_flags
502            .contains(wgt::InstanceFlags::VALIDATION_INDIRECT_CALL)
503            && downlevel.flags.contains(
504                wgt::DownlevelFlags::INDIRECT_EXECUTION | wgt::DownlevelFlags::COMPUTE_SHADERS,
505            )
506            && limits.max_storage_buffers_per_shader_stage >= 2;
507
508        let indirect_validation = if enable_indirect_validation {
509            Some(crate::indirect_validation::IndirectValidation::new(
510                raw_device.as_ref(),
511                &desc.required_limits,
512                &desc.required_features,
513                instance_flags,
514                adapter.backend(),
515            )?)
516        } else {
517            None
518        };
519
520        Ok(Self {
521            raw: raw_device,
522            adapter: adapter.clone(),
523            queue: OnceCellOrLock::new(),
524            zero_buffer: ManuallyDrop::new(zero_buffer),
525            empty_bgl: ManuallyDrop::new(empty_bgl),
526            default_external_texture_params_buffer: ManuallyDrop::new(
527                default_external_texture_params_buffer,
528            ),
529            label: desc.label.to_string(),
530            command_allocator,
531            command_indices: RwLock::new(
532                rank::DEVICE_COMMAND_INDICES,
533                CommandIndices {
534                    active_submission_index: 0,
535                    // By starting at one, we can put the result in a NonZeroU64.
536                    next_acceleration_structure_build_command_index: 1,
537                },
538            ),
539            last_successful_submission_index: AtomicU64::new(0),
540            fence: ManuallyDrop::new(fence),
541            snatchable_lock: unsafe { SnatchLock::new(rank::DEVICE_SNATCHABLE_LOCK) },
542            valid: AtomicBool::new(true),
543            device_lost_closure: Mutex::new(rank::DEVICE_LOST_CLOSURE, None),
544            trackers: Mutex::new(
545                rank::DEVICE_TRACKERS,
546                DeviceTracker::new(ordered_buffer_usages, ordered_texture_usages),
547            ),
548            tracker_indices: TrackerIndexAllocators::new(),
549            bgl_pool: ResourcePool::new(),
550            #[cfg(feature = "trace")]
551            trace: Mutex::new(rank::DEVICE_TRACE, trace),
552            alignments,
553            limits: desc.required_limits.clone(),
554            features: desc.required_features,
555            downlevel,
556            ordered_buffer_usages,
557            ordered_texture_usages,
558            instance_flags,
559            deferred_destroy: Mutex::new(rank::DEVICE_DEFERRED_DESTROY, Vec::new()),
560            usage_scopes: Mutex::new(rank::DEVICE_USAGE_SCOPES, Default::default()),
561            timestamp_normalizer: OnceCellOrLock::new(),
562            indirect_validation,
563        })
564    }
565
566    /// Initializes [`Device::default_external_texture_params_buffer`] with
567    /// required values such that a [`TextureView`] bound to a
568    /// [`wgt::BindingType::ExternalTexture`] binding point will be rendered
569    /// correctly.
570    fn init_default_external_texture_params_buffer(self: &Arc<Self>) -> Result<(), DeviceError> {
571        let data = ExternalTextureParams {
572            #[rustfmt::skip]
573            yuv_conversion_matrix: [
574                1.0, 0.0, 0.0, 0.0,
575                0.0, 1.0, 0.0, 0.0,
576                0.0, 0.0, 1.0, 0.0,
577                0.0, 0.0, 0.0, 1.0,
578            ],
579            #[rustfmt::skip]
580            gamut_conversion_matrix: [
581                1.0, 0.0, 0.0, /* padding */ 0.0,
582                0.0, 1.0, 0.0, /* padding */ 0.0,
583                0.0, 0.0, 1.0, /* padding */ 0.0,
584            ],
585            src_transfer_function: Default::default(),
586            dst_transfer_function: Default::default(),
587            size: [0, 0],
588            #[rustfmt::skip]
589            sample_transform: [
590                1.0, 0.0,
591                0.0, 1.0,
592                0.0, 0.0
593            ],
594            #[rustfmt::skip]
595            load_transform: [
596                1.0, 0.0,
597                0.0, 1.0,
598                0.0, 0.0
599            ],
600            num_planes: 1,
601            _padding: Default::default(),
602        };
603        let mut staging_buffer =
604            StagingBuffer::new(self, wgt::BufferSize::new(size_of_val(&data) as _).unwrap())?;
605        staging_buffer.write(bytemuck::bytes_of(&data));
606        let staging_buffer = staging_buffer.flush();
607
608        let params_buffer = self.default_external_texture_params_buffer.as_ref();
609        let queue = self.get_queue().unwrap();
610        let mut pending_writes = queue.pending_writes.lock();
611
612        unsafe {
613            pending_writes
614                .command_encoder
615                .transition_buffers(&[hal::BufferBarrier {
616                    buffer: params_buffer,
617                    usage: hal::StateTransition {
618                        from: wgt::BufferUses::MAP_WRITE,
619                        to: wgt::BufferUses::COPY_DST,
620                    },
621                }]);
622            pending_writes.command_encoder.copy_buffer_to_buffer(
623                staging_buffer.raw(),
624                params_buffer,
625                &[hal::BufferCopy {
626                    src_offset: 0,
627                    dst_offset: 0,
628                    size: staging_buffer.size,
629                }],
630            );
631            pending_writes.consume(staging_buffer);
632            pending_writes
633                .command_encoder
634                .transition_buffers(&[hal::BufferBarrier {
635                    buffer: params_buffer,
636                    usage: hal::StateTransition {
637                        from: wgt::BufferUses::COPY_DST,
638                        to: wgt::BufferUses::UNIFORM,
639                    },
640                }]);
641        }
642
643        Ok(())
644    }
645
646    pub fn late_init_resources_with_queue(self: &Arc<Self>) -> Result<(), RequestDeviceError> {
647        let queue = self.get_queue().unwrap();
648
649        let timestamp_normalizer = crate::timestamp_normalization::TimestampNormalizer::new(
650            self,
651            queue.get_timestamp_period(),
652        )?;
653
654        self.timestamp_normalizer
655            .set(timestamp_normalizer)
656            .unwrap_or_else(|_| panic!("Called late_init_resources_with_queue twice"));
657
658        self.init_default_external_texture_params_buffer()?;
659
660        Ok(())
661    }
662
663    /// Returns the backend this device is using.
664    pub fn backend(&self) -> wgt::Backend {
665        self.adapter.backend()
666    }
667
668    pub fn is_valid(&self) -> bool {
669        self.valid.load(Ordering::Acquire)
670    }
671
672    pub fn check_is_valid(&self) -> Result<(), DeviceError> {
673        if self.is_valid() {
674            Ok(())
675        } else {
676            Err(DeviceError::Lost)
677        }
678    }
679
680    /// Stop tracing and return the trace object.
681    ///
682    /// This is mostly useful for in-memory traces.
683    #[cfg(feature = "trace")]
684    pub fn take_trace(&self) -> Option<Box<dyn trace::Trace + Send + Sync + 'static>> {
685        self.trace.lock().take()
686    }
687
688    /// Checks that we are operating within the memory budget reported by the native APIs.
689    ///
690    /// If we are not, the device gets invalidated.
691    ///
692    /// The budget might fluctuate over the lifetime of the application, so it should be checked
693    /// somewhat frequently.
694    pub fn lose_if_oom(&self) {
695        let _ = self
696            .raw()
697            .check_if_oom()
698            .map_err(|e| self.handle_hal_error(e));
699    }
700
701    pub fn handle_hal_error(&self, error: hal::DeviceError) -> DeviceError {
702        match error {
703            hal::DeviceError::OutOfMemory
704            | hal::DeviceError::Lost
705            | hal::DeviceError::Unexpected => {
706                self.lose(&error.to_string());
707            }
708        }
709        DeviceError::from_hal(error)
710    }
711
712    pub fn handle_hal_error_with_nonfatal_oom(&self, error: hal::DeviceError) -> DeviceError {
713        match error {
714            hal::DeviceError::OutOfMemory => DeviceError::from_hal(error),
715            error => self.handle_hal_error(error),
716        }
717    }
718
719    /// Run some destroy operations that were deferred.
720    ///
721    /// Destroying the resources requires taking a write lock on the device's snatch lock,
722    /// so a good reason for deferring resource destruction is when we don't know for sure
723    /// how risky it is to take the lock (typically, it shouldn't be taken from the drop
724    /// implementation of a reference-counted structure).
725    /// The snatch lock must not be held while this function is called.
726    pub(crate) fn deferred_resource_destruction(&self) {
727        // Note that the deferred_destroy list may contain duplicate entries.
728        let deferred_destroy = mem::take(&mut *self.deferred_destroy.lock());
729        for item in deferred_destroy {
730            match item {
731                DeferredDestroy::TextureViews(views) => {
732                    for view in views {
733                        let Some(view) = view.upgrade() else {
734                            continue;
735                        };
736                        let Some(raw_view) = view.raw.snatch(&mut self.snatchable_lock.write())
737                        else {
738                            continue;
739                        };
740
741                        resource_log!("Destroy raw {}", view.error_ident());
742
743                        unsafe {
744                            self.raw().destroy_texture_view(raw_view);
745                        }
746                    }
747                }
748                DeferredDestroy::BindGroups(bind_groups) => {
749                    for bind_group in bind_groups {
750                        let Some(bind_group) = bind_group.upgrade() else {
751                            continue;
752                        };
753                        let Some(raw_bind_group) =
754                            bind_group.raw.snatch(&mut self.snatchable_lock.write())
755                        else {
756                            continue;
757                        };
758
759                        resource_log!("Destroy raw {}", bind_group.error_ident());
760
761                        unsafe {
762                            self.raw().destroy_bind_group(raw_bind_group);
763                        }
764                    }
765                }
766            }
767        }
768    }
769
770    pub fn get_queue(&self) -> Option<Arc<Queue>> {
771        self.queue.get().as_ref()?.upgrade()
772    }
773
774    pub fn set_queue(&self, queue: &Arc<Queue>) {
775        assert!(self.queue.set(Arc::downgrade(queue)).is_ok());
776    }
777
778    pub fn poll(
779        &self,
780        poll_type: wgt::PollType<crate::SubmissionIndex>,
781    ) -> Result<wgt::PollStatus, WaitIdleError> {
782        let (user_closures, result) = self.poll_and_return_closures(poll_type);
783        user_closures.fire();
784        result
785    }
786
787    /// Poll the device, returning any `UserClosures` that need to be executed.
788    ///
789    /// The caller must invoke the `UserClosures` even if this function returns
790    /// an error. This is an internal helper, used by `Device::poll` and
791    /// `Global::poll_all_devices`, so that `poll_all_devices` can invoke
792    /// closures once after all devices have been polled.
793    pub(crate) fn poll_and_return_closures(
794        &self,
795        poll_type: wgt::PollType<crate::SubmissionIndex>,
796    ) -> (UserClosures, Result<wgt::PollStatus, WaitIdleError>) {
797        let snatch_guard = self.snatchable_lock.read();
798        let maintain_result = self.maintain(poll_type, snatch_guard);
799
800        self.lose_if_oom();
801
802        // Some deferred destroys are scheduled in maintain so run this right after
803        // to avoid holding on to them until the next device poll.
804        self.deferred_resource_destruction();
805
806        maintain_result
807    }
808
809    /// Check the current status of the GPU and process any submissions that have
810    /// finished.
811    ///
812    /// The `poll_type` argument tells if this function should wait for a particular
813    /// submission index to complete, or if it should just poll the current status.
814    ///
815    /// This will process _all_ completed submissions, even if the caller only asked
816    /// us to poll to a given submission index.
817    ///
818    /// Return a pair `(closures, result)`, where:
819    ///
820    /// - `closures` is a list of callbacks that need to be invoked informing the user
821    ///   about various things occurring. These happen and should be handled even if
822    ///   this function returns an error, hence they are outside of the result.
823    ///
824    /// - `results` is a boolean indicating the result of the wait operation, including
825    ///   if there was a timeout or a validation error.
826    pub(crate) fn maintain<'this>(
827        &'this self,
828        poll_type: wgt::PollType<crate::SubmissionIndex>,
829        snatch_guard: SnatchGuard,
830    ) -> (UserClosures, Result<wgt::PollStatus, WaitIdleError>) {
831        profiling::scope!("Device::maintain");
832
833        let mut user_closures = UserClosures::default();
834
835        // If a wait was requested, determine which submission index to wait for.
836        let wait_submission_index = match poll_type {
837            wgt::PollType::Wait {
838                submission_index: Some(submission_index),
839                ..
840            } => {
841                let last_successful_submission_index = self
842                    .last_successful_submission_index
843                    .load(Ordering::Acquire);
844
845                if submission_index > last_successful_submission_index {
846                    let result = Err(WaitIdleError::WrongSubmissionIndex(
847                        submission_index,
848                        last_successful_submission_index,
849                    ));
850
851                    return (user_closures, result);
852                }
853
854                Some(submission_index)
855            }
856            wgt::PollType::Wait {
857                submission_index: None,
858                ..
859            } => Some(
860                self.last_successful_submission_index
861                    .load(Ordering::Acquire),
862            ),
863            wgt::PollType::Poll => None,
864        };
865
866        // Wait for the submission index if requested.
867        if let Some(target_submission_index) = wait_submission_index {
868            log::trace!("Device::maintain: waiting for submission index {target_submission_index}");
869
870            let wait_timeout = match poll_type {
871                wgt::PollType::Wait { timeout, .. } => timeout,
872                wgt::PollType::Poll => unreachable!(
873                    "`wait_submission_index` index for poll type `Poll` should be None"
874                ),
875            };
876
877            let wait_result = unsafe {
878                self.raw()
879                    .wait(self.fence.as_ref(), target_submission_index, wait_timeout)
880            };
881
882            // This error match is only about `DeviceErrors`. At this stage we do not care if
883            // the wait succeeded or not, and the `Ok(bool)`` variant is ignored.
884            if let Err(e) = wait_result {
885                let hal_error: WaitIdleError = self.handle_hal_error(e).into();
886                return (user_closures, Err(hal_error));
887            }
888        }
889
890        // Get the currently finished submission index. This may be higher than the requested
891        // wait, or it may be less than the requested wait if the wait failed.
892        let fence_value_result = unsafe { self.raw().get_fence_value(self.fence.as_ref()) };
893        let current_finished_submission = match fence_value_result {
894            Ok(fence_value) => fence_value,
895            Err(e) => {
896                let hal_error: WaitIdleError = self.handle_hal_error(e).into();
897                return (user_closures, Err(hal_error));
898            }
899        };
900
901        // Prevent new commands from being submitted as we want to act on `queue_empty`.
902        let command_indices = self.command_indices.read();
903        // Check that the device is valid. This is combined with queue empty to decide whether
904        // to destroy all resources. Queue.submit blocks on command indices being writable
905        // and rejects if invalid so if the device in now invalid, and all submissions are
906        // finished, there will be no more submissions.
907        let device_valid = self.is_valid();
908        drop(command_indices);
909
910        // Maintain all finished submissions on the queue, updating the relevant user closures and
911        // collecting if the queue is empty.
912        //
913        // We don't use the result of the wait here, as we want to progress forward as far as
914        // possible and the wait could have been for submissions that finished long ago.
915        let mut queue_empty = false;
916        if let Some(queue) = self.get_queue() {
917            let queue_result = queue.maintain(current_finished_submission, &snatch_guard);
918            (
919                user_closures.submissions,
920                user_closures.mappings,
921                user_closures.blas_compact_ready,
922                queue_empty,
923            ) = queue_result;
924            // DEADLOCK PREVENTION: We must drop `snatch_guard` before `queue` goes out of scope.
925            //
926            // `Queue::drop` acquires the snatch guard. If we still hold it when `queue` is dropped
927            // at the end of this block, we would deadlock. This can happen in the following
928            // scenario:
929            //
930            // - Thread A calls `Device::maintain` while Thread B holds the last strong ref to the
931            //   queue.
932            // - Thread A calls `self.get_queue()`, obtaining a new strong ref, and enters this
933            //   branch.
934            // - Thread B drops its strong ref, making Thread A's ref the last one.
935            // - When `queue` goes out of scope here, `Queue::drop` runs and tries to acquire the
936            //   snatch guard — but Thread A (this thread) still holds it, causing a deadlock.
937            drop(snatch_guard);
938        } else {
939            drop(snatch_guard);
940        };
941
942        // Based on the queue empty status, and the current finished submission index, determine
943        // the result of the poll.
944        let result = if queue_empty {
945            if let Some(wait_submission_index) = wait_submission_index {
946                // Assert to ensure that if we received a queue empty status, the fence shows the
947                // correct value. This is defensive, as this should never be hit.
948                assert!(
949                    current_finished_submission >= wait_submission_index,
950                    concat!(
951                        "If the queue is empty, the current submission index ",
952                        "({}) should be at least the wait submission index ({})",
953                    ),
954                    current_finished_submission,
955                    wait_submission_index,
956                );
957            }
958
959            Ok(wgt::PollStatus::QueueEmpty)
960        } else if let Some(wait_submission_index) = wait_submission_index {
961            // This is theoretically possible to succeed more than checking on the poll result
962            // as submissions could have finished in the time between the timeout resolving,
963            // the thread getting scheduled again, and us checking the fence value.
964            if current_finished_submission >= wait_submission_index {
965                Ok(wgt::PollStatus::WaitSucceeded)
966            } else {
967                Err(WaitIdleError::Timeout)
968            }
969        } else {
970            Ok(wgt::PollStatus::Poll)
971        };
972
973        // Detect if we have been destroyed and now need to lose the device.
974        //
975        // If we are invalid (set at start of destroy) and our queue is empty,
976        // and we have a DeviceLostClosure, return the closure to be called by
977        // our caller. This will complete the steps for both destroy and for
978        // "lose the device".
979        let mut should_release_gpu_resource = false;
980        if !device_valid && queue_empty {
981            // We can release gpu resources associated with this device (but not
982            // while holding the life_tracker lock).
983            should_release_gpu_resource = true;
984
985            // If we have a DeviceLostClosure, build an invocation with the
986            // reason DeviceLostReason::Destroyed and no message.
987            if let Some(device_lost_closure) = self.device_lost_closure.lock().take() {
988                user_closures
989                    .device_lost_invocations
990                    .push(DeviceLostInvocation {
991                        closure: device_lost_closure,
992                        reason: DeviceLostReason::Destroyed,
993                        message: String::new(),
994                    });
995            }
996        }
997
998        if should_release_gpu_resource {
999            self.release_gpu_resources();
1000        }
1001
1002        (user_closures, result)
1003    }
1004
1005    pub fn create_buffer(
1006        self: &Arc<Self>,
1007        desc: &resource::BufferDescriptor,
1008    ) -> Result<Arc<Buffer>, resource::CreateBufferError> {
1009        self.check_is_valid()?;
1010
1011        if desc.size > self.limits.max_buffer_size {
1012            return Err(resource::CreateBufferError::MaxBufferSize {
1013                requested: desc.size,
1014                maximum: self.limits.max_buffer_size,
1015            });
1016        }
1017
1018        if desc
1019            .usage
1020            .intersects(wgt::BufferUsages::BLAS_INPUT | wgt::BufferUsages::TLAS_INPUT)
1021        {
1022            self.require_features(wgt::Features::EXPERIMENTAL_RAY_QUERY)?;
1023        }
1024
1025        if desc.usage.contains(wgt::BufferUsages::INDEX)
1026            && desc.usage.contains(
1027                wgt::BufferUsages::VERTEX
1028                    | wgt::BufferUsages::UNIFORM
1029                    | wgt::BufferUsages::INDIRECT
1030                    | wgt::BufferUsages::STORAGE,
1031            )
1032        {
1033            self.require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)?;
1034        }
1035
1036        if desc.usage.is_empty() || desc.usage.contains_unknown_bits() {
1037            return Err(resource::CreateBufferError::InvalidUsage(desc.usage));
1038        }
1039
1040        if !self
1041            .features
1042            .contains(wgt::Features::MAPPABLE_PRIMARY_BUFFERS)
1043        {
1044            use wgt::BufferUsages as Bu;
1045            let write_mismatch = desc.usage.contains(Bu::MAP_WRITE)
1046                && !(Bu::MAP_WRITE | Bu::COPY_SRC).contains(desc.usage);
1047            let read_mismatch = desc.usage.contains(Bu::MAP_READ)
1048                && !(Bu::MAP_READ | Bu::COPY_DST).contains(desc.usage);
1049            if write_mismatch || read_mismatch {
1050                return Err(resource::CreateBufferError::UsageMismatch(desc.usage));
1051            }
1052        }
1053
1054        let mut usage = conv::map_buffer_usage(desc.usage);
1055
1056        if desc.usage.contains(wgt::BufferUsages::INDIRECT) {
1057            self.require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)?;
1058            // We are going to be reading from it, internally;
1059            // when validating the content of the buffer
1060            usage |= wgt::BufferUses::STORAGE_READ_ONLY | wgt::BufferUses::STORAGE_READ_WRITE;
1061        }
1062
1063        if desc.usage.contains(wgt::BufferUsages::QUERY_RESOLVE) {
1064            usage |= TIMESTAMP_NORMALIZATION_BUFFER_USES;
1065        }
1066
1067        if desc.mapped_at_creation {
1068            if !desc.size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
1069                return Err(resource::CreateBufferError::UnalignedSize);
1070            }
1071            if !desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
1072                // we are going to be copying into it, internally
1073                usage |= wgt::BufferUses::COPY_DST;
1074            }
1075        } else {
1076            // We are required to zero out (initialize) all memory. This is done
1077            // on demand using clear_buffer which requires write transfer usage!
1078            usage |= wgt::BufferUses::COPY_DST;
1079        }
1080
1081        let actual_size = if desc.size == 0 {
1082            wgt::COPY_BUFFER_ALIGNMENT
1083        } else if desc.usage.contains(wgt::BufferUsages::VERTEX) {
1084            // Bumping the size by 1 so that we can bind an empty range at the
1085            // end of the buffer.
1086            desc.size + 1
1087        } else {
1088            desc.size
1089        };
1090        let clear_remainder = actual_size % wgt::COPY_BUFFER_ALIGNMENT;
1091        let aligned_size = if clear_remainder != 0 {
1092            actual_size + wgt::COPY_BUFFER_ALIGNMENT - clear_remainder
1093        } else {
1094            actual_size
1095        };
1096
1097        let hal_desc = hal::BufferDescriptor {
1098            label: desc.label.to_hal(self.instance_flags),
1099            size: aligned_size,
1100            usage,
1101            memory_flags: hal::MemoryFlags::empty(),
1102        };
1103        let buffer = unsafe { self.raw().create_buffer(&hal_desc) }
1104            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
1105
1106        let timestamp_normalization_bind_group = Snatchable::new(unsafe {
1107            // SAFETY: The size passed here must not overflow the buffer.
1108            self.timestamp_normalizer
1109                .get()
1110                .unwrap()
1111                .create_normalization_bind_group(
1112                    self,
1113                    &*buffer,
1114                    desc.label.as_deref(),
1115                    wgt::BufferSize::new(hal_desc.size).unwrap(),
1116                    desc.usage,
1117                )
1118        }?);
1119
1120        let indirect_validation_bind_groups =
1121            self.create_indirect_validation_bind_groups(buffer.as_ref(), desc.size, desc.usage)?;
1122
1123        let buffer = Buffer {
1124            raw: Snatchable::new(buffer),
1125            device: self.clone(),
1126            usage: desc.usage,
1127            size: desc.size,
1128            initialization_status: RwLock::new(
1129                rank::BUFFER_INITIALIZATION_STATUS,
1130                BufferInitTracker::new(aligned_size),
1131            ),
1132            map_state: Mutex::new(rank::BUFFER_MAP_STATE, resource::BufferMapState::Idle),
1133            label: desc.label.to_string(),
1134            tracking_data: TrackingData::new(self.tracker_indices.buffers.clone()),
1135            bind_groups: Mutex::new(rank::BUFFER_BIND_GROUPS, WeakVec::new()),
1136            timestamp_normalization_bind_group,
1137            indirect_validation_bind_groups,
1138        };
1139
1140        let buffer = Arc::new(buffer);
1141
1142        let buffer_use = if !desc.mapped_at_creation {
1143            wgt::BufferUses::empty()
1144        } else if desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
1145            // buffer is mappable, so we are just doing that at start
1146            let map_size = buffer.size;
1147            let mapping = if map_size == 0 {
1148                hal::BufferMapping {
1149                    ptr: core::ptr::NonNull::dangling(),
1150                    is_coherent: true,
1151                }
1152            } else {
1153                let snatch_guard: SnatchGuard = self.snatchable_lock.read();
1154                map_buffer(&buffer, 0, map_size, HostMap::Write, &snatch_guard)?
1155            };
1156            *buffer.map_state.lock() = resource::BufferMapState::Active {
1157                mapping,
1158                range: 0..map_size,
1159                host: HostMap::Write,
1160            };
1161            wgt::BufferUses::MAP_WRITE
1162        } else {
1163            let mut staging_buffer =
1164                StagingBuffer::new(self, wgt::BufferSize::new(aligned_size).unwrap())?;
1165
1166            // Zero initialize memory and then mark the buffer as initialized
1167            // (it's guaranteed that this is the case by the time the buffer is usable)
1168            staging_buffer.write_zeros();
1169            buffer.initialization_status.write().drain(0..aligned_size);
1170
1171            *buffer.map_state.lock() = resource::BufferMapState::Init { staging_buffer };
1172            wgt::BufferUses::COPY_DST
1173        };
1174
1175        self.trackers
1176            .lock()
1177            .buffers
1178            .insert_single(&buffer, buffer_use);
1179
1180        Ok(buffer)
1181    }
1182
1183    #[cfg(feature = "replay")]
1184    pub fn set_buffer_data(
1185        self: &Arc<Self>,
1186        buffer: &Arc<Buffer>,
1187        offset: wgt::BufferAddress,
1188        data: &[u8],
1189    ) -> resource::BufferAccessResult {
1190        use crate::resource::RawResourceAccess;
1191
1192        let device = &buffer.device;
1193
1194        device.check_is_valid()?;
1195        buffer.check_usage(wgt::BufferUsages::MAP_WRITE)?;
1196
1197        let last_submission = device
1198            .get_queue()
1199            .and_then(|queue| queue.lock_life().get_buffer_latest_submission_index(buffer));
1200
1201        if let Some(last_submission) = last_submission {
1202            device.wait_for_submit(last_submission)?;
1203        }
1204
1205        let snatch_guard = device.snatchable_lock.read();
1206        let raw_buf = buffer.try_raw(&snatch_guard)?;
1207
1208        if offset > buffer.size {
1209            return Err(resource::BufferAccessError::OutOfBoundsStartOffsetOverrun {
1210                index: offset,
1211                max: buffer.size,
1212            });
1213        } else if buffer.size - offset < u64::try_from(data.len()).unwrap() {
1214            return Err(resource::BufferAccessError::OutOfBoundsEndOffsetOverrun {
1215                index: offset,
1216                size: u64::try_from(data.len()).unwrap(),
1217                max: buffer.size,
1218            });
1219        }
1220
1221        let mapping = unsafe {
1222            device
1223                .raw()
1224                .map_buffer(raw_buf, offset..offset + u64::try_from(data.len()).unwrap())
1225        }
1226        .map_err(|e| device.handle_hal_error(e))?;
1227
1228        unsafe { core::ptr::copy_nonoverlapping(data.as_ptr(), mapping.ptr.as_ptr(), data.len()) };
1229
1230        if !mapping.is_coherent {
1231            #[allow(clippy::single_range_in_vec_init)]
1232            unsafe {
1233                device
1234                    .raw()
1235                    .flush_mapped_ranges(raw_buf, &[offset..offset + data.len() as u64])
1236            };
1237        }
1238
1239        unsafe { device.raw().unmap_buffer(raw_buf) };
1240
1241        Ok(())
1242    }
1243
1244    pub(crate) fn create_texture_from_hal(
1245        self: &Arc<Self>,
1246        hal_texture: Box<dyn hal::DynTexture>,
1247        desc: &resource::TextureDescriptor,
1248        initial_state: wgt::TextureUses,
1249    ) -> Result<Arc<Texture>, resource::CreateTextureError> {
1250        let format_features = self
1251            .describe_format_features(desc.format)
1252            .map_err(|error| resource::CreateTextureError::MissingFeatures(desc.format, error))?;
1253
1254        unsafe { self.raw().add_raw_texture(&*hal_texture) };
1255
1256        let texture = Texture::new(
1257            self,
1258            resource::TextureInner::Native { raw: hal_texture },
1259            conv::map_texture_usage(desc.usage, desc.format.into(), format_features.flags),
1260            desc,
1261            format_features,
1262            resource::TextureClearMode::None,
1263            false,
1264        );
1265
1266        let texture = Arc::new(texture);
1267
1268        self.trackers
1269            .lock()
1270            .textures
1271            .insert_single(&texture, initial_state);
1272
1273        Ok(texture)
1274    }
1275
1276    /// # Safety
1277    ///
1278    /// - `hal_buffer` must have been created on this device.
1279    /// - `hal_buffer` must have been created respecting `desc` (in particular, the size).
1280    /// - `hal_buffer` must be initialized.
1281    /// - `hal_buffer` must not have zero size.
1282    pub(crate) unsafe fn create_buffer_from_hal(
1283        self: &Arc<Self>,
1284        hal_buffer: Box<dyn hal::DynBuffer>,
1285        desc: &resource::BufferDescriptor,
1286    ) -> (Fallible<Buffer>, Option<resource::CreateBufferError>) {
1287        let timestamp_normalization_bind_group = unsafe {
1288            match self
1289                .timestamp_normalizer
1290                .get()
1291                .unwrap()
1292                .create_normalization_bind_group(
1293                    self,
1294                    &*hal_buffer,
1295                    desc.label.as_deref(),
1296                    wgt::BufferSize::new(desc.size).unwrap(),
1297                    desc.usage,
1298                ) {
1299                Ok(bg) => Snatchable::new(bg),
1300                Err(e) => {
1301                    return (
1302                        Fallible::Invalid(Arc::new(desc.label.to_string())),
1303                        Some(e.into()),
1304                    )
1305                }
1306            }
1307        };
1308
1309        let indirect_validation_bind_groups = match self.create_indirect_validation_bind_groups(
1310            hal_buffer.as_ref(),
1311            desc.size,
1312            desc.usage,
1313        ) {
1314            Ok(ok) => ok,
1315            Err(e) => return (Fallible::Invalid(Arc::new(desc.label.to_string())), Some(e)),
1316        };
1317
1318        unsafe { self.raw().add_raw_buffer(&*hal_buffer) };
1319
1320        let buffer = Buffer {
1321            raw: Snatchable::new(hal_buffer),
1322            device: self.clone(),
1323            usage: desc.usage,
1324            size: desc.size,
1325            initialization_status: RwLock::new(
1326                rank::BUFFER_INITIALIZATION_STATUS,
1327                BufferInitTracker::new(0),
1328            ),
1329            map_state: Mutex::new(rank::BUFFER_MAP_STATE, resource::BufferMapState::Idle),
1330            label: desc.label.to_string(),
1331            tracking_data: TrackingData::new(self.tracker_indices.buffers.clone()),
1332            bind_groups: Mutex::new(rank::BUFFER_BIND_GROUPS, WeakVec::new()),
1333            timestamp_normalization_bind_group,
1334            indirect_validation_bind_groups,
1335        };
1336
1337        let buffer = Arc::new(buffer);
1338
1339        self.trackers
1340            .lock()
1341            .buffers
1342            .insert_single(&buffer, wgt::BufferUses::empty());
1343
1344        (Fallible::Valid(buffer), None)
1345    }
1346
1347    fn create_indirect_validation_bind_groups(
1348        &self,
1349        raw_buffer: &dyn hal::DynBuffer,
1350        buffer_size: u64,
1351        usage: wgt::BufferUsages,
1352    ) -> Result<Snatchable<crate::indirect_validation::BindGroups>, resource::CreateBufferError>
1353    {
1354        if !usage.contains(wgt::BufferUsages::INDIRECT) {
1355            return Ok(Snatchable::empty());
1356        }
1357
1358        let Some(ref indirect_validation) = self.indirect_validation else {
1359            return Ok(Snatchable::empty());
1360        };
1361
1362        let bind_groups = crate::indirect_validation::BindGroups::new(
1363            indirect_validation,
1364            self,
1365            buffer_size,
1366            raw_buffer,
1367        )
1368        .map_err(resource::CreateBufferError::IndirectValidationBindGroup)?;
1369
1370        if let Some(bind_groups) = bind_groups {
1371            Ok(Snatchable::new(bind_groups))
1372        } else {
1373            Ok(Snatchable::empty())
1374        }
1375    }
1376
1377    fn create_texture_inner(
1378        self: &Arc<Self>,
1379        desc: &resource::TextureDescriptor,
1380    ) -> Result<Arc<Texture>, resource::CreateTextureError> {
1381        use resource::{CreateTextureError, TextureDimensionError};
1382
1383        self.check_is_valid()?;
1384
1385        if desc.usage.is_empty() || desc.usage.contains_unknown_bits() {
1386            return Err(CreateTextureError::InvalidUsage(desc.usage));
1387        }
1388
1389        conv::check_texture_dimension_size(
1390            desc.dimension,
1391            desc.size,
1392            desc.sample_count,
1393            &self.limits,
1394        )?;
1395
1396        if desc.dimension != wgt::TextureDimension::D2 {
1397            // Depth textures can only be 2D
1398            if desc.format.is_depth_stencil_format() {
1399                return Err(CreateTextureError::InvalidDepthDimension(
1400                    desc.dimension,
1401                    desc.format,
1402                ));
1403            }
1404            // Transient textures can only be 2D
1405            if desc
1406                .usage
1407                .contains(wgt::TextureUsages::TRANSIENT_ATTACHMENT)
1408            {
1409                return Err(CreateTextureError::InvalidDimensionUsages(
1410                    wgt::TextureUsages::TRANSIENT_ATTACHMENT,
1411                    desc.dimension,
1412                ));
1413            }
1414        }
1415
1416        if desc.dimension != wgt::TextureDimension::D2
1417            && desc.dimension != wgt::TextureDimension::D3
1418        {
1419            // Compressed textures can only be 2D or 3D
1420            if desc.format.is_compressed() {
1421                return Err(CreateTextureError::InvalidCompressedDimension(
1422                    desc.dimension,
1423                    desc.format,
1424                ));
1425            }
1426
1427            // Renderable textures can only be 2D or 3D
1428            if desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
1429                return Err(CreateTextureError::InvalidDimensionUsages(
1430                    wgt::TextureUsages::RENDER_ATTACHMENT,
1431                    desc.dimension,
1432                ));
1433            }
1434        }
1435
1436        if desc.format.is_compressed() {
1437            let (block_width, block_height) = desc.format.block_dimensions();
1438
1439            if !desc.size.width.is_multiple_of(block_width) {
1440                return Err(CreateTextureError::InvalidDimension(
1441                    TextureDimensionError::NotMultipleOfBlockWidth {
1442                        width: desc.size.width,
1443                        block_width,
1444                        format: desc.format,
1445                    },
1446                ));
1447            }
1448
1449            if !desc.size.height.is_multiple_of(block_height) {
1450                return Err(CreateTextureError::InvalidDimension(
1451                    TextureDimensionError::NotMultipleOfBlockHeight {
1452                        height: desc.size.height,
1453                        block_height,
1454                        format: desc.format,
1455                    },
1456                ));
1457            }
1458
1459            if desc.dimension == wgt::TextureDimension::D3 {
1460                // Only BCn formats with Sliced 3D feature can be used for 3D textures
1461                if desc.format.is_bcn() {
1462                    self.require_features(wgt::Features::TEXTURE_COMPRESSION_BC_SLICED_3D)
1463                        .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1464                } else if desc.format.is_astc() {
1465                    self.require_features(wgt::Features::TEXTURE_COMPRESSION_ASTC_SLICED_3D)
1466                        .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1467                } else {
1468                    return Err(CreateTextureError::InvalidCompressedDimension(
1469                        desc.dimension,
1470                        desc.format,
1471                    ));
1472                }
1473            }
1474        }
1475
1476        let mips = desc.mip_level_count;
1477        let max_levels_allowed = desc.size.max_mips(desc.dimension).min(hal::MAX_MIP_LEVELS);
1478        if mips == 0 || mips > max_levels_allowed {
1479            return Err(CreateTextureError::InvalidMipLevelCount {
1480                requested: mips,
1481                maximum: max_levels_allowed,
1482            });
1483        }
1484
1485        {
1486            let (mut width_multiple, mut height_multiple) = desc.format.size_multiple_requirement();
1487
1488            if desc.format.is_multi_planar_format() {
1489                // TODO(https://github.com/gfx-rs/wgpu/issues/8491): fix
1490                // `mip_level_size` calculation for these formats and relax this
1491                // restriction.
1492                width_multiple <<= desc.mip_level_count.saturating_sub(1);
1493                height_multiple <<= desc.mip_level_count.saturating_sub(1);
1494            }
1495
1496            if !desc.size.width.is_multiple_of(width_multiple) {
1497                return Err(CreateTextureError::InvalidDimension(
1498                    TextureDimensionError::WidthNotMultipleOf {
1499                        width: desc.size.width,
1500                        multiple: width_multiple,
1501                        format: desc.format,
1502                    },
1503                ));
1504            }
1505
1506            if !desc.size.height.is_multiple_of(height_multiple) {
1507                return Err(CreateTextureError::InvalidDimension(
1508                    TextureDimensionError::HeightNotMultipleOf {
1509                        height: desc.size.height,
1510                        multiple: height_multiple,
1511                        format: desc.format,
1512                    },
1513                ));
1514            }
1515        }
1516
1517        if desc
1518            .usage
1519            .contains(wgt::TextureUsages::TRANSIENT_ATTACHMENT)
1520        {
1521            if desc.usage
1522                != (wgt::TextureUsages::TRANSIENT_ATTACHMENT
1523                    | wgt::TextureUsages::RENDER_ATTACHMENT)
1524            {
1525                return Err(CreateTextureError::InvalidTransientTextureUsage(desc.usage));
1526            }
1527
1528            if desc.mip_level_count != 1 {
1529                return Err(CreateTextureError::InvalidTransientTextureMipLevelCount(
1530                    desc.mip_level_count,
1531                ));
1532            }
1533
1534            if desc.size.depth_or_array_layers != 1 {
1535                return Err(CreateTextureError::InvalidTransientTextureLayerCount(
1536                    desc.size.depth_or_array_layers,
1537                ));
1538            }
1539
1540            if !desc.view_formats.is_empty() {
1541                return Err(CreateTextureError::InvalidTransientTextureViewFormats);
1542            }
1543        }
1544
1545        let format_features = self
1546            .describe_format_features(desc.format)
1547            .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1548
1549        if desc.sample_count > 1 {
1550            // <https://www.w3.org/TR/2025/CRD-webgpu-20251120/#:~:text=If%20descriptor%2EsampleCount%20%3E%201>
1551            //
1552            // Note that there are also some checks related to the sample count
1553            // in [`conv::check_texture_dimension_size`].
1554
1555            if desc.mip_level_count != 1 {
1556                return Err(CreateTextureError::InvalidMipLevelCount {
1557                    requested: desc.mip_level_count,
1558                    maximum: 1,
1559                });
1560            }
1561
1562            if desc.size.depth_or_array_layers != 1
1563                && !self.features.contains(wgt::Features::MULTISAMPLE_ARRAY)
1564            {
1565                return Err(CreateTextureError::InvalidDimension(
1566                    TextureDimensionError::MultisampledDepthOrArrayLayer(
1567                        desc.size.depth_or_array_layers,
1568                    ),
1569                ));
1570            }
1571
1572            if desc.usage.contains(wgt::TextureUsages::STORAGE_BINDING) {
1573                return Err(CreateTextureError::InvalidMultisampledStorageBinding);
1574            }
1575
1576            if !desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
1577                return Err(CreateTextureError::MultisampledNotRenderAttachment);
1578            }
1579
1580            if !format_features.flags.intersects(
1581                wgt::TextureFormatFeatureFlags::MULTISAMPLE_X4
1582                    | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X2
1583                    | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X8
1584                    | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X16,
1585            ) {
1586                return Err(CreateTextureError::InvalidMultisampledFormat(desc.format));
1587            }
1588
1589            if !format_features
1590                .flags
1591                .sample_count_supported(desc.sample_count)
1592            {
1593                return Err(CreateTextureError::InvalidSampleCount(
1594                    desc.sample_count,
1595                    desc.format,
1596                    desc.format
1597                        .guaranteed_format_features(self.features)
1598                        .flags
1599                        .supported_sample_counts(),
1600                    self.adapter
1601                        .get_texture_format_features(desc.format)
1602                        .flags
1603                        .supported_sample_counts(),
1604                ));
1605            };
1606        }
1607
1608        let missing_allowed_usages = match desc.format.planes() {
1609            Some(planes) => {
1610                let mut planes_usages = wgt::TextureUsages::all();
1611                for plane in 0..planes {
1612                    let aspect = wgt::TextureAspect::from_plane(plane).unwrap();
1613                    let format = desc.format.aspect_specific_format(aspect).unwrap();
1614                    let format_features = self
1615                        .describe_format_features(format)
1616                        .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1617
1618                    planes_usages &= format_features.allowed_usages;
1619                }
1620
1621                desc.usage - planes_usages
1622            }
1623            None => desc.usage - format_features.allowed_usages,
1624        };
1625
1626        if !missing_allowed_usages.is_empty() {
1627            // detect downlevel incompatibilities
1628            let wgpu_allowed_usages = desc
1629                .format
1630                .guaranteed_format_features(self.features)
1631                .allowed_usages;
1632            let wgpu_missing_usages = desc.usage - wgpu_allowed_usages;
1633            return Err(CreateTextureError::InvalidFormatUsages(
1634                missing_allowed_usages,
1635                desc.format,
1636                wgpu_missing_usages.is_empty(),
1637            ));
1638        }
1639
1640        let mut hal_view_formats = Vec::new();
1641        for format in desc.view_formats.iter() {
1642            if desc.format == *format {
1643                continue;
1644            }
1645            if desc.format.remove_srgb_suffix() != format.remove_srgb_suffix() {
1646                return Err(CreateTextureError::InvalidViewFormat(*format, desc.format));
1647            }
1648            hal_view_formats.push(*format);
1649        }
1650        if !hal_view_formats.is_empty() {
1651            self.require_downlevel_flags(wgt::DownlevelFlags::VIEW_FORMATS)?;
1652        }
1653
1654        let hal_usage = conv::map_texture_usage_for_texture(desc, &format_features);
1655
1656        let hal_desc = hal::TextureDescriptor {
1657            label: desc.label.to_hal(self.instance_flags),
1658            size: desc.size,
1659            mip_level_count: desc.mip_level_count,
1660            sample_count: desc.sample_count,
1661            dimension: desc.dimension,
1662            format: desc.format,
1663            usage: hal_usage,
1664            memory_flags: hal::MemoryFlags::empty(),
1665            view_formats: hal_view_formats,
1666        };
1667
1668        let raw_texture = unsafe { self.raw().create_texture(&hal_desc) }
1669            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
1670
1671        let clear_mode = if hal_usage
1672            .intersects(wgt::TextureUses::DEPTH_STENCIL_WRITE | wgt::TextureUses::COLOR_TARGET)
1673            && desc.dimension == wgt::TextureDimension::D2
1674        {
1675            let (is_color, usage) = if desc.format.is_depth_stencil_format() {
1676                (false, wgt::TextureUses::DEPTH_STENCIL_WRITE)
1677            } else {
1678                (true, wgt::TextureUses::COLOR_TARGET)
1679            };
1680
1681            let clear_label = hal_label(
1682                Some("(wgpu internal) clear texture view"),
1683                self.instance_flags,
1684            );
1685
1686            let mut clear_views = SmallVec::new();
1687            for mip_level in 0..desc.mip_level_count {
1688                for array_layer in 0..desc.size.depth_or_array_layers {
1689                    macro_rules! push_clear_view {
1690                        ($format:expr, $aspect:expr) => {
1691                            let desc = hal::TextureViewDescriptor {
1692                                label: clear_label,
1693                                format: $format,
1694                                dimension: TextureViewDimension::D2,
1695                                usage,
1696                                range: wgt::ImageSubresourceRange {
1697                                    aspect: $aspect,
1698                                    base_mip_level: mip_level,
1699                                    mip_level_count: Some(1),
1700                                    base_array_layer: array_layer,
1701                                    array_layer_count: Some(1),
1702                                },
1703                            };
1704                            clear_views.push(ManuallyDrop::new(
1705                                unsafe {
1706                                    self.raw().create_texture_view(raw_texture.as_ref(), &desc)
1707                                }
1708                                .map_err(|e| self.handle_hal_error(e))?,
1709                            ));
1710                        };
1711                    }
1712
1713                    if let Some(planes) = desc.format.planes() {
1714                        for plane in 0..planes {
1715                            let aspect = wgt::TextureAspect::from_plane(plane).unwrap();
1716                            let format = desc.format.aspect_specific_format(aspect).unwrap();
1717                            push_clear_view!(format, aspect);
1718                        }
1719                    } else {
1720                        push_clear_view!(desc.format, wgt::TextureAspect::All);
1721                    }
1722                }
1723            }
1724            resource::TextureClearMode::RenderPass {
1725                clear_views,
1726                is_color,
1727            }
1728        } else {
1729            resource::TextureClearMode::BufferCopy
1730        };
1731
1732        let texture = Texture::new(
1733            self,
1734            resource::TextureInner::Native { raw: raw_texture },
1735            hal_usage,
1736            desc,
1737            format_features,
1738            clear_mode,
1739            true,
1740        );
1741
1742        let texture = Arc::new(texture);
1743
1744        self.trackers
1745            .lock()
1746            .textures
1747            .insert_single(&texture, wgt::TextureUses::UNINITIALIZED);
1748
1749        Ok(texture)
1750    }
1751
1752    pub fn create_texture(
1753        self: &Arc<Self>,
1754        desc: &resource::TextureDescriptor,
1755    ) -> (Arc<Texture>, Option<resource::CreateTextureError>) {
1756        let (texture, error) = match self.create_texture_inner(desc) {
1757            Ok(texture) => (texture, None),
1758            Err(e) => {
1759                let texture = Texture::invalid(self, desc);
1760                (Arc::new(texture), Some(e))
1761            }
1762        };
1763        api_log!(
1764            "Device::create_texture({desc:?}) -> {:?}",
1765            Arc::as_ptr(&texture)
1766        );
1767
1768        #[cfg(feature = "trace")]
1769        if let Some(ref mut trace) = *self.trace.lock() {
1770            use crate::device::trace::IntoTrace as _;
1771
1772            trace.add(trace::Action::CreateTexture(
1773                texture.to_trace(),
1774                desc.clone(),
1775            ));
1776        }
1777        (texture, error)
1778    }
1779
1780    /// Creates a texture that is guaranteed to be invalid
1781    pub fn create_texture_error(
1782        self: &Arc<Self>,
1783        desc: &resource::TextureDescriptor,
1784    ) -> Arc<Texture> {
1785        let texture = Arc::new(Texture::invalid(self, desc));
1786        #[cfg(feature = "trace")]
1787        if let Some(ref mut trace) = *self.trace.lock() {
1788            use crate::device::trace::IntoTrace as _;
1789
1790            trace.add(trace::Action::CreateTextureError(
1791                texture.to_trace(),
1792                desc.clone(),
1793            ));
1794        }
1795        texture
1796    }
1797
1798    pub fn create_texture_view(
1799        self: &Arc<Self>,
1800        texture: &Arc<Texture>,
1801        desc: &resource::TextureViewDescriptor,
1802    ) -> Result<Arc<TextureView>, resource::CreateTextureViewError> {
1803        self.check_is_valid()?;
1804
1805        let snatch_guard = texture.device.snatchable_lock.read();
1806
1807        let texture_raw = texture.try_inner(&snatch_guard)?.raw();
1808
1809        // resolve TextureViewDescriptor defaults
1810        // https://gpuweb.github.io/gpuweb/#abstract-opdef-resolving-gputextureviewdescriptor-defaults
1811        let resolved_format = desc.format.unwrap_or_else(|| {
1812            texture
1813                .desc
1814                .format
1815                .aspect_specific_format(desc.range.aspect)
1816                .unwrap_or(texture.desc.format)
1817        });
1818
1819        let resolved_dimension = desc
1820            .dimension
1821            .unwrap_or_else(|| match texture.desc.dimension {
1822                wgt::TextureDimension::D1 => TextureViewDimension::D1,
1823                wgt::TextureDimension::D2 => {
1824                    if texture.desc.array_layer_count() == 1 {
1825                        TextureViewDimension::D2
1826                    } else {
1827                        TextureViewDimension::D2Array
1828                    }
1829                }
1830                wgt::TextureDimension::D3 => TextureViewDimension::D3,
1831            });
1832
1833        let resolved_mip_level_count = desc.range.mip_level_count.unwrap_or_else(|| {
1834            texture
1835                .desc
1836                .mip_level_count
1837                .saturating_sub(desc.range.base_mip_level)
1838        });
1839
1840        let resolved_array_layer_count =
1841            desc.range
1842                .array_layer_count
1843                .unwrap_or_else(|| match resolved_dimension {
1844                    TextureViewDimension::D1
1845                    | TextureViewDimension::D2
1846                    | TextureViewDimension::D3 => 1,
1847                    TextureViewDimension::Cube => 6,
1848                    TextureViewDimension::D2Array | TextureViewDimension::CubeArray => texture
1849                        .desc
1850                        .array_layer_count()
1851                        .saturating_sub(desc.range.base_array_layer),
1852                });
1853
1854        let resolved_usage = {
1855            let usage = desc.usage.unwrap_or(wgt::TextureUsages::empty());
1856            if usage.is_empty() {
1857                texture.desc.usage
1858            } else if texture.desc.usage.contains(usage) {
1859                // Transient texture usage subsetting is disallowed
1860                if texture
1861                    .desc
1862                    .usage
1863                    .contains(wgt::TextureUsages::TRANSIENT_ATTACHMENT)
1864                    && texture.desc.usage != usage
1865                {
1866                    return Err(
1867                        resource::CreateTextureViewError::InvalidTransientTextureViewUsage {
1868                            texture: texture.desc.usage,
1869                            view: usage,
1870                        },
1871                    );
1872                }
1873
1874                usage
1875            } else {
1876                return Err(resource::CreateTextureViewError::InvalidTextureViewUsage {
1877                    view: usage,
1878                    texture: texture.desc.usage,
1879                });
1880            }
1881        };
1882
1883        let format_features = self.describe_format_features(resolved_format)?;
1884        let allowed_format_usages = format_features.allowed_usages;
1885        if resolved_usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT)
1886            && !allowed_format_usages.contains(wgt::TextureUsages::RENDER_ATTACHMENT)
1887        {
1888            return Err(
1889                resource::CreateTextureViewError::TextureViewFormatNotRenderable(resolved_format),
1890            );
1891        }
1892
1893        if resolved_usage.contains(wgt::TextureUsages::STORAGE_BINDING)
1894            && !allowed_format_usages.contains(wgt::TextureUsages::STORAGE_BINDING)
1895        {
1896            return Err(
1897                resource::CreateTextureViewError::TextureViewFormatNotStorage(resolved_format),
1898            );
1899        }
1900
1901        // validate TextureViewDescriptor
1902
1903        let aspects = hal::FormatAspects::new(texture.desc.format, desc.range.aspect);
1904        if aspects.is_empty() {
1905            return Err(resource::CreateTextureViewError::InvalidAspect {
1906                texture_format: texture.desc.format,
1907                requested_aspect: desc.range.aspect,
1908            });
1909        }
1910
1911        let format_is_good = if desc.range.aspect == wgt::TextureAspect::All {
1912            resolved_format == texture.desc.format
1913                || texture.desc.view_formats.contains(&resolved_format)
1914        } else {
1915            Some(resolved_format)
1916                == texture
1917                    .desc
1918                    .format
1919                    .aspect_specific_format(desc.range.aspect)
1920        };
1921        if !format_is_good {
1922            return Err(resource::CreateTextureViewError::FormatReinterpretation {
1923                texture: texture.desc.format,
1924                view: resolved_format,
1925            });
1926        }
1927
1928        // check if multisampled texture is seen as anything but 2D
1929        if texture.desc.sample_count > 1 && resolved_dimension != TextureViewDimension::D2 {
1930            // Multisample is allowed on 2D arrays, only if explicitly supported
1931            let multisample_array_exception = resolved_dimension == TextureViewDimension::D2Array
1932                && self.features.contains(wgt::Features::MULTISAMPLE_ARRAY);
1933
1934            if !multisample_array_exception {
1935                return Err(
1936                    resource::CreateTextureViewError::InvalidMultisampledTextureViewDimension(
1937                        resolved_dimension,
1938                    ),
1939                );
1940            }
1941        }
1942
1943        // check if the dimension is compatible with the texture
1944        if texture.desc.dimension != resolved_dimension.compatible_texture_dimension() {
1945            return Err(
1946                resource::CreateTextureViewError::InvalidTextureViewDimension {
1947                    view: resolved_dimension,
1948                    texture: texture.desc.dimension,
1949                },
1950            );
1951        }
1952
1953        match resolved_dimension {
1954            TextureViewDimension::D1 | TextureViewDimension::D2 | TextureViewDimension::D3 => {
1955                if resolved_array_layer_count != 1 {
1956                    return Err(resource::CreateTextureViewError::InvalidArrayLayerCount {
1957                        requested: resolved_array_layer_count,
1958                        dim: resolved_dimension,
1959                    });
1960                }
1961            }
1962            TextureViewDimension::Cube => {
1963                if resolved_array_layer_count != 6 {
1964                    return Err(
1965                        resource::CreateTextureViewError::InvalidCubemapTextureDepth {
1966                            depth: resolved_array_layer_count,
1967                        },
1968                    );
1969                }
1970            }
1971            TextureViewDimension::CubeArray => {
1972                if !resolved_array_layer_count.is_multiple_of(6) {
1973                    return Err(
1974                        resource::CreateTextureViewError::InvalidCubemapArrayTextureDepth {
1975                            depth: resolved_array_layer_count,
1976                        },
1977                    );
1978                }
1979            }
1980            _ => {}
1981        }
1982
1983        match resolved_dimension {
1984            TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
1985                if texture.desc.size.width != texture.desc.size.height {
1986                    return Err(resource::CreateTextureViewError::InvalidCubeTextureViewSize);
1987                }
1988            }
1989            _ => {}
1990        }
1991
1992        if resolved_mip_level_count == 0 {
1993            return Err(resource::CreateTextureViewError::ZeroMipLevelCount);
1994        }
1995
1996        let mip_level_end = desc
1997            .range
1998            .base_mip_level
1999            .saturating_add(resolved_mip_level_count);
2000
2001        let level_end = texture.desc.mip_level_count;
2002        if mip_level_end > level_end {
2003            return Err(resource::CreateTextureViewError::TooManyMipLevels {
2004                base_mip_level: desc.range.base_mip_level,
2005                mip_level_count: resolved_mip_level_count,
2006                total: level_end,
2007            });
2008        }
2009
2010        if resolved_array_layer_count == 0 {
2011            return Err(resource::CreateTextureViewError::ZeroArrayLayerCount);
2012        }
2013
2014        let array_layer_end = desc
2015            .range
2016            .base_array_layer
2017            .saturating_add(resolved_array_layer_count);
2018
2019        let layer_end = texture.desc.array_layer_count();
2020        if array_layer_end > layer_end {
2021            return Err(resource::CreateTextureViewError::TooManyArrayLayers {
2022                base_array_layer: desc.range.base_array_layer,
2023                array_layer_count: resolved_array_layer_count,
2024                total: layer_end,
2025            });
2026        };
2027
2028        // https://gpuweb.github.io/gpuweb/#abstract-opdef-renderable-texture-view
2029        let render_extent = 'error: {
2030            if !resolved_usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
2031                break 'error Err(TextureViewNotRenderableReason::Usage(resolved_usage));
2032            }
2033
2034            let allowed_view_dimensions = [
2035                TextureViewDimension::D2,
2036                TextureViewDimension::D2Array,
2037                TextureViewDimension::D3,
2038            ];
2039            if !allowed_view_dimensions.contains(&resolved_dimension) {
2040                break 'error Err(TextureViewNotRenderableReason::Dimension(
2041                    resolved_dimension,
2042                ));
2043            }
2044
2045            if resolved_mip_level_count != 1 {
2046                break 'error Err(TextureViewNotRenderableReason::MipLevelCount(
2047                    resolved_mip_level_count,
2048                ));
2049            }
2050
2051            if resolved_array_layer_count != 1
2052                && !(self.features.contains(wgt::Features::MULTIVIEW))
2053            {
2054                break 'error Err(TextureViewNotRenderableReason::ArrayLayerCount(
2055                    resolved_array_layer_count,
2056                ));
2057            }
2058
2059            if !texture.desc.format.is_multi_planar_format()
2060                && aspects != hal::FormatAspects::from(texture.desc.format)
2061            {
2062                break 'error Err(TextureViewNotRenderableReason::Aspects(aspects));
2063            }
2064
2065            Ok(texture
2066                .desc
2067                .compute_render_extent(desc.range.base_mip_level, desc.range.aspect.to_plane()))
2068        };
2069
2070        // filter the usages based on the other criteria
2071        let usage = {
2072            let resolved_hal_usage = conv::map_texture_usage(
2073                resolved_usage,
2074                resolved_format.into(),
2075                format_features.flags,
2076            );
2077            let mask_copy = !(wgt::TextureUses::COPY_SRC | wgt::TextureUses::COPY_DST);
2078            let mask_dimension = match resolved_dimension {
2079                TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
2080                    wgt::TextureUses::RESOURCE
2081                }
2082                TextureViewDimension::D3 => {
2083                    wgt::TextureUses::RESOURCE
2084                        | wgt::TextureUses::STORAGE_READ_ONLY
2085                        | wgt::TextureUses::STORAGE_WRITE_ONLY
2086                        | wgt::TextureUses::STORAGE_READ_WRITE
2087                }
2088                _ => wgt::TextureUses::all(),
2089            };
2090            let mask_mip_level = if resolved_mip_level_count == 1 {
2091                wgt::TextureUses::all()
2092            } else {
2093                wgt::TextureUses::RESOURCE
2094            };
2095            resolved_hal_usage & mask_copy & mask_dimension & mask_mip_level
2096        };
2097
2098        // use the combined depth-stencil format for the view
2099        let format = if resolved_format.is_depth_stencil_component(texture.desc.format) {
2100            texture.desc.format
2101        } else {
2102            resolved_format
2103        };
2104
2105        let resolved_range = wgt::ImageSubresourceRange {
2106            aspect: desc.range.aspect,
2107            base_mip_level: desc.range.base_mip_level,
2108            mip_level_count: Some(resolved_mip_level_count),
2109            base_array_layer: desc.range.base_array_layer,
2110            array_layer_count: Some(resolved_array_layer_count),
2111        };
2112
2113        let hal_desc = hal::TextureViewDescriptor {
2114            label: desc.label.to_hal(self.instance_flags),
2115            format,
2116            dimension: resolved_dimension,
2117            usage,
2118            range: resolved_range,
2119        };
2120
2121        let raw = unsafe { self.raw().create_texture_view(texture_raw, &hal_desc) }
2122            .map_err(|e| self.handle_hal_error(e))?;
2123
2124        let selector = TextureSelector {
2125            mips: desc.range.base_mip_level..mip_level_end,
2126            layers: desc.range.base_array_layer..array_layer_end,
2127        };
2128
2129        let view = TextureView {
2130            raw: Snatchable::new(raw),
2131            parent: texture.clone(),
2132            device: self.clone(),
2133            desc: resource::HalTextureViewDescriptor {
2134                texture_format: texture.desc.format,
2135                format: resolved_format,
2136                dimension: resolved_dimension,
2137                usage: resolved_usage,
2138                range: resolved_range,
2139            },
2140            format_features: texture.format_features,
2141            render_extent,
2142            samples: texture.desc.sample_count,
2143            selector,
2144            label: desc.label.to_string(),
2145        };
2146
2147        let view = Arc::new(view);
2148
2149        {
2150            let mut views = texture.views.lock();
2151            views.push(Arc::downgrade(&view));
2152        }
2153
2154        Ok(view)
2155    }
2156
2157    pub fn create_external_texture(
2158        self: &Arc<Self>,
2159        desc: &resource::ExternalTextureDescriptor,
2160        planes: &[Arc<TextureView>],
2161    ) -> Result<Arc<ExternalTexture>, resource::CreateExternalTextureError> {
2162        use resource::CreateExternalTextureError;
2163        self.require_features(wgt::Features::EXTERNAL_TEXTURE)?;
2164        self.check_is_valid()?;
2165
2166        if desc.num_planes() != planes.len() {
2167            return Err(CreateExternalTextureError::IncorrectPlaneCount {
2168                format: desc.format,
2169                expected: desc.num_planes(),
2170                provided: planes.len(),
2171            });
2172        }
2173
2174        let planes = planes
2175            .iter()
2176            .enumerate()
2177            .map(|(i, plane)| {
2178                if plane.samples != 1 {
2179                    return Err(CreateExternalTextureError::InvalidPlaneMultisample(
2180                        plane.samples,
2181                    ));
2182                }
2183
2184                let sample_type = plane
2185                    .desc
2186                    .format
2187                    .sample_type(Some(plane.desc.range.aspect), Some(self.features))
2188                    .unwrap();
2189                if !matches!(sample_type, TextureSampleType::Float { filterable: true }) {
2190                    return Err(CreateExternalTextureError::InvalidPlaneSampleType {
2191                        format: plane.desc.format,
2192                        sample_type,
2193                    });
2194                }
2195
2196                if plane.desc.dimension != TextureViewDimension::D2 {
2197                    return Err(CreateExternalTextureError::InvalidPlaneDimension(
2198                        plane.desc.dimension,
2199                    ));
2200                }
2201
2202                let expected_components = match desc.format {
2203                    wgt::ExternalTextureFormat::Rgba => 4,
2204                    wgt::ExternalTextureFormat::Nv12 => match i {
2205                        0 => 1,
2206                        1 => 2,
2207                        _ => unreachable!(),
2208                    },
2209                    wgt::ExternalTextureFormat::Yu12 => 1,
2210                };
2211                if plane.desc.format.components() != expected_components {
2212                    return Err(CreateExternalTextureError::InvalidPlaneFormat {
2213                        format: desc.format,
2214                        plane: i,
2215                        expected: expected_components,
2216                        provided: plane.desc.format,
2217                    });
2218                }
2219
2220                plane.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
2221                Ok(plane.clone())
2222            })
2223            .collect::<Result<_, _>>()?;
2224
2225        let params_data = ExternalTextureParams::from_desc(desc);
2226        let label = desc.label.as_ref().map(|l| alloc::format!("{l} params"));
2227        let params_desc = resource::BufferDescriptor {
2228            label: label.map(Cow::Owned),
2229            size: size_of_val(&params_data) as wgt::BufferAddress,
2230            usage: wgt::BufferUsages::UNIFORM | wgt::BufferUsages::COPY_DST,
2231            mapped_at_creation: false,
2232        };
2233        let params = self.create_buffer(&params_desc)?;
2234        self.get_queue().unwrap().write_buffer(
2235            params.clone(),
2236            0,
2237            bytemuck::bytes_of(&params_data),
2238        )?;
2239
2240        let external_texture = ExternalTexture {
2241            device: self.clone(),
2242            planes,
2243            params,
2244            label: desc.label.to_string(),
2245            tracking_data: TrackingData::new(self.tracker_indices.external_textures.clone()),
2246        };
2247        let external_texture = Arc::new(external_texture);
2248
2249        Ok(external_texture)
2250    }
2251
2252    pub fn create_sampler(
2253        self: &Arc<Self>,
2254        desc: &resource::SamplerDescriptor,
2255    ) -> Result<Arc<Sampler>, resource::CreateSamplerError> {
2256        self.check_is_valid()?;
2257
2258        if desc
2259            .address_modes
2260            .iter()
2261            .any(|am| am == &wgt::AddressMode::ClampToBorder)
2262        {
2263            self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_BORDER)?;
2264        }
2265
2266        if desc.border_color == Some(wgt::SamplerBorderColor::Zero) {
2267            self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_ZERO)?;
2268        }
2269
2270        if desc.lod_min_clamp < 0.0 {
2271            return Err(resource::CreateSamplerError::InvalidLodMinClamp(
2272                desc.lod_min_clamp,
2273            ));
2274        }
2275        if desc.lod_max_clamp < desc.lod_min_clamp {
2276            return Err(resource::CreateSamplerError::InvalidLodMaxClamp {
2277                lod_min_clamp: desc.lod_min_clamp,
2278                lod_max_clamp: desc.lod_max_clamp,
2279            });
2280        }
2281
2282        if desc.anisotropy_clamp < 1 {
2283            return Err(resource::CreateSamplerError::InvalidAnisotropy(
2284                desc.anisotropy_clamp,
2285            ));
2286        }
2287
2288        if desc.anisotropy_clamp != 1 {
2289            if !matches!(desc.min_filter, wgt::FilterMode::Linear) {
2290                return Err(
2291                    resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
2292                        filter_type: resource::SamplerFilterErrorType::MinFilter,
2293                        filter_mode: desc.min_filter,
2294                        anisotropic_clamp: desc.anisotropy_clamp,
2295                    },
2296                );
2297            }
2298            if !matches!(desc.mag_filter, wgt::FilterMode::Linear) {
2299                return Err(
2300                    resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
2301                        filter_type: resource::SamplerFilterErrorType::MagFilter,
2302                        filter_mode: desc.mag_filter,
2303                        anisotropic_clamp: desc.anisotropy_clamp,
2304                    },
2305                );
2306            }
2307            if !matches!(desc.mipmap_filter, wgt::MipmapFilterMode::Linear) {
2308                return Err(
2309                    resource::CreateSamplerError::InvalidMipmapFilterModeWithAnisotropy {
2310                        filter_type: resource::SamplerFilterErrorType::MipmapFilter,
2311                        filter_mode: desc.mipmap_filter,
2312                        anisotropic_clamp: desc.anisotropy_clamp,
2313                    },
2314                );
2315            }
2316        }
2317
2318        let anisotropy_clamp = if self
2319            .downlevel
2320            .flags
2321            .contains(wgt::DownlevelFlags::ANISOTROPIC_FILTERING)
2322        {
2323            // Clamp anisotropy clamp to [1, 16] per the wgpu-hal interface
2324            desc.anisotropy_clamp.min(16)
2325        } else {
2326            // If it isn't supported, set this unconditionally to 1
2327            1
2328        };
2329
2330        //TODO: check for wgt::DownlevelFlags::COMPARISON_SAMPLERS
2331
2332        let hal_desc = hal::SamplerDescriptor {
2333            label: desc.label.to_hal(self.instance_flags),
2334            address_modes: desc.address_modes,
2335            mag_filter: desc.mag_filter,
2336            min_filter: desc.min_filter,
2337            mipmap_filter: desc.mipmap_filter,
2338            lod_clamp: desc.lod_min_clamp..desc.lod_max_clamp,
2339            compare: desc.compare,
2340            anisotropy_clamp,
2341            border_color: desc.border_color,
2342        };
2343
2344        let raw = unsafe { self.raw().create_sampler(&hal_desc) }
2345            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
2346
2347        let sampler = Sampler {
2348            raw: ManuallyDrop::new(raw),
2349            device: self.clone(),
2350            label: desc.label.to_string(),
2351            tracking_data: TrackingData::new(self.tracker_indices.samplers.clone()),
2352            comparison: desc.compare.is_some(),
2353            filtering: desc.min_filter == wgt::FilterMode::Linear
2354                || desc.mag_filter == wgt::FilterMode::Linear
2355                || desc.mipmap_filter == wgt::MipmapFilterMode::Linear,
2356        };
2357
2358        let sampler = Arc::new(sampler);
2359
2360        Ok(sampler)
2361    }
2362
2363    pub fn create_shader_module<'a>(
2364        self: &Arc<Self>,
2365        desc: &pipeline::ShaderModuleDescriptor<'a>,
2366        source: pipeline::ShaderModuleSource<'a>,
2367    ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
2368        self.check_is_valid()?;
2369
2370        let (module, source) = match source {
2371            #[cfg(feature = "wgsl")]
2372            pipeline::ShaderModuleSource::Wgsl(code) => {
2373                profiling::scope!("naga::front::wgsl::parse");
2374                let capabilities =
2375                    features_to_naga_capabilities(self.features, self.downlevel.flags);
2376                let mut options = naga::front::wgsl::Options::new();
2377                options.capabilities = capabilities;
2378                let mut frontend = naga::front::wgsl::Frontend::new_with_options(options);
2379                let module = frontend.parse(&code).map_err(|inner| {
2380                    pipeline::CreateShaderModuleError::Parsing(naga::error::ShaderError {
2381                        source: code.to_string(),
2382                        label: desc.label.as_ref().map(|l| l.to_string()),
2383                        inner: Box::new(inner),
2384                    })
2385                })?;
2386                (Cow::Owned(module), code.into_owned())
2387            }
2388            #[cfg(feature = "spirv")]
2389            pipeline::ShaderModuleSource::SpirV(spv, options) => {
2390                let parser = naga::front::spv::Frontend::new(spv.iter().cloned(), &options);
2391                profiling::scope!("naga::front::spv::Frontend");
2392                let module = parser.parse().map_err(|inner| {
2393                    pipeline::CreateShaderModuleError::ParsingSpirV(naga::error::ShaderError {
2394                        source: String::new(),
2395                        label: desc.label.as_ref().map(|l| l.to_string()),
2396                        inner: Box::new(inner),
2397                    })
2398                })?;
2399                (Cow::Owned(module), String::new())
2400            }
2401            #[cfg(feature = "glsl")]
2402            pipeline::ShaderModuleSource::Glsl(code, options) => {
2403                let mut parser = naga::front::glsl::Frontend::default();
2404                profiling::scope!("naga::front::glsl::Frontend.parse");
2405                let module = parser.parse(&options, &code).map_err(|inner| {
2406                    pipeline::CreateShaderModuleError::ParsingGlsl(naga::error::ShaderError {
2407                        source: code.to_string(),
2408                        label: desc.label.as_ref().map(|l| l.to_string()),
2409                        inner: Box::new(inner),
2410                    })
2411                })?;
2412                (Cow::Owned(module), code.into_owned())
2413            }
2414            pipeline::ShaderModuleSource::Naga(module) => (module, String::new()),
2415            pipeline::ShaderModuleSource::Dummy(_) => panic!("found `ShaderModuleSource::Dummy`"),
2416        };
2417        for (_, var) in module.global_variables.iter() {
2418            match var.binding {
2419                Some(br) if br.group >= self.limits.max_bind_groups => {
2420                    return Err(pipeline::CreateShaderModuleError::InvalidGroupIndex {
2421                        bind: br,
2422                        group: br.group,
2423                        limit: self.limits.max_bind_groups,
2424                    });
2425                }
2426                _ => continue,
2427            };
2428        }
2429
2430        profiling::scope!("naga::validate");
2431        let debug_source =
2432            if self.instance_flags.contains(wgt::InstanceFlags::DEBUG) && !source.is_empty() {
2433                Some(hal::DebugSource {
2434                    file_name: Cow::Owned(
2435                        desc.label
2436                            .as_ref()
2437                            .map_or("shader".to_string(), |l| l.to_string()),
2438                    ),
2439                    source_code: Cow::Owned(source.clone()),
2440                })
2441            } else {
2442                None
2443            };
2444
2445        let info = create_validator(
2446            self.features,
2447            self.downlevel.flags,
2448            naga::valid::ValidationFlags::all(),
2449        )
2450        .validate(&module)
2451        .map_err(|inner| {
2452            pipeline::CreateShaderModuleError::Validation(naga::error::ShaderError {
2453                source,
2454                label: desc.label.as_ref().map(|l| l.to_string()),
2455                inner: Box::new(inner),
2456            })
2457        })?;
2458
2459        let interface = validation::Interface::new(&module, &info, self.limits.clone());
2460        let hal_shader = hal::ShaderInput::Naga(hal::NagaShader {
2461            module,
2462            info,
2463            debug_source,
2464        });
2465        let hal_desc = hal::ShaderModuleDescriptor {
2466            label: desc.label.to_hal(self.instance_flags),
2467            runtime_checks: desc.runtime_checks,
2468        };
2469        let raw = match unsafe { self.raw().create_shader_module(&hal_desc, hal_shader) } {
2470            Ok(raw) => raw,
2471            Err(error) => {
2472                return Err(match error {
2473                    hal::ShaderError::Device(error) => {
2474                        pipeline::CreateShaderModuleError::Device(self.handle_hal_error(error))
2475                    }
2476                    hal::ShaderError::Compilation(ref msg) => {
2477                        log::error!("Shader error: {msg}");
2478                        pipeline::CreateShaderModuleError::Generation
2479                    }
2480                })
2481            }
2482        };
2483
2484        let module = pipeline::ShaderModule {
2485            raw: ManuallyDrop::new(raw),
2486            device: self.clone(),
2487            interface: ShaderMetaData::Interface(interface),
2488            label: desc.label.to_string(),
2489        };
2490
2491        let module = Arc::new(module);
2492
2493        Ok(module)
2494    }
2495
2496    /// Not a public API. For use by `player` only.
2497    #[allow(unused_unsafe)]
2498    #[doc(hidden)]
2499    pub unsafe fn create_shader_module_passthrough<'a>(
2500        self: &Arc<Self>,
2501        descriptor: &pipeline::ShaderModuleDescriptorPassthrough<'a>,
2502    ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
2503        self.check_is_valid()?;
2504        self.require_features(wgt::Features::PASSTHROUGH_SHADERS)?;
2505
2506        // Mainly important for GLSL or SPIR-V or DXIL, which each take exactly 1 entry point.
2507        if (descriptor.dxil.is_some() || descriptor.glsl.is_some())
2508            && descriptor.entry_points.len() != 1
2509        {
2510            return Err(pipeline::CreateShaderModuleError::IncorrectPassthroughEntryPointCount);
2511        }
2512
2513        let entry_point_hashmap = || {
2514            descriptor
2515                .entry_points
2516                .iter()
2517                .map(|e| (e.name.to_string(), e.workgroup_size))
2518                .collect()
2519        };
2520
2521        let hal_shader = match self.backend() {
2522            wgt::Backend::Vulkan => hal::ShaderInput::SpirV(
2523                descriptor
2524                    .spirv
2525                    .as_ref()
2526                    .ok_or(pipeline::CreateShaderModuleError::NotCompiledForBackend)?,
2527            ),
2528            wgt::Backend::Dx12 => {
2529                if let Some(dxil) = &descriptor.dxil {
2530                    hal::ShaderInput::Dxil { shader: dxil }
2531                } else if let Some(hlsl) = &descriptor.hlsl {
2532                    hal::ShaderInput::Hlsl { shader: hlsl }
2533                } else {
2534                    return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend);
2535                }
2536            }
2537            wgt::Backend::Metal => {
2538                if let Some(metallib) = &descriptor.metallib {
2539                    hal::ShaderInput::MetalLib {
2540                        file: metallib,
2541                        num_workgroups: entry_point_hashmap(),
2542                    }
2543                } else if let Some(msl) = &descriptor.msl {
2544                    hal::ShaderInput::Msl {
2545                        shader: msl,
2546                        num_workgroups: entry_point_hashmap(),
2547                    }
2548                } else {
2549                    return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend);
2550                }
2551            }
2552            wgt::Backend::Gl => hal::ShaderInput::Glsl {
2553                shader: descriptor
2554                    .glsl
2555                    .as_ref()
2556                    .ok_or(pipeline::CreateShaderModuleError::NotCompiledForBackend)?,
2557            },
2558            wgt::Backend::Noop => {
2559                return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend)
2560            }
2561            wgt::Backend::BrowserWebGpu => unreachable!(),
2562        };
2563
2564        let hal_desc = hal::ShaderModuleDescriptor {
2565            label: descriptor.label.to_hal(self.instance_flags),
2566            runtime_checks: wgt::ShaderRuntimeChecks::unchecked(),
2567        };
2568
2569        let raw = match unsafe { self.raw().create_shader_module(&hal_desc, hal_shader) } {
2570            Ok(raw) => raw,
2571            Err(error) => {
2572                return Err(match error {
2573                    hal::ShaderError::Device(error) => {
2574                        pipeline::CreateShaderModuleError::Device(self.handle_hal_error(error))
2575                    }
2576                    hal::ShaderError::Compilation(ref msg) => {
2577                        log::error!("Shader error: {msg}");
2578                        pipeline::CreateShaderModuleError::Generation
2579                    }
2580                })
2581            }
2582        };
2583
2584        let module = pipeline::ShaderModule {
2585            raw: ManuallyDrop::new(raw),
2586            device: self.clone(),
2587            interface: ShaderMetaData::Passthrough(PassthroughInterface {
2588                entry_point_names: descriptor
2589                    .entry_points
2590                    .iter()
2591                    .map(|e| e.name.to_string())
2592                    .collect(),
2593            }),
2594            label: descriptor.label.to_string(),
2595        };
2596
2597        Ok(Arc::new(module))
2598    }
2599
2600    /// Reads the links of the GL programs no submission read yet, and
2601    /// returns whether there was one; nothing on another backend. See
2602    /// `hal::gles::Device::read_program_links`.
2603    pub fn read_program_links(&self) -> bool {
2604        #[cfg(gles)]
2605        if let Some(device) = self.raw().as_any().downcast_ref::<hal::gles::Device>() {
2606            return device.read_program_links();
2607        }
2608        false
2609    }
2610
2611    /// A module of GLSL templates, one per entry point, for the GL backend.
2612    ///
2613    /// # Safety
2614    ///
2615    /// No naga checks the templates: a driver gets their GLSL as written.
2616    /// On WebGL the browser checks it.
2617    pub unsafe fn create_shader_module_glsl_templates(
2618        self: &Arc<Self>,
2619        label: &crate::Label<'_>,
2620        templates: &'static [hal::GlslTemplate],
2621    ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
2622        self.check_is_valid()?;
2623        if self.backend() != wgt::Backend::Gl {
2624            return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend);
2625        }
2626        let hal_desc = hal::ShaderModuleDescriptor {
2627            label: label.to_hal(self.instance_flags),
2628            runtime_checks: wgt::ShaderRuntimeChecks::unchecked(),
2629        };
2630        let raw = unsafe {
2631            self.raw()
2632                .create_shader_module(&hal_desc, hal::ShaderInput::GlslTemplates(templates))
2633        }
2634        .map_err(|error| match error {
2635            hal::ShaderError::Device(error) => {
2636                pipeline::CreateShaderModuleError::Device(self.handle_hal_error(error))
2637            }
2638            hal::ShaderError::Compilation(ref msg) => {
2639                log::error!("Shader error: {msg}");
2640                pipeline::CreateShaderModuleError::Generation
2641            }
2642        })?;
2643        Ok(Arc::new(pipeline::ShaderModule {
2644            raw: ManuallyDrop::new(raw),
2645            device: self.clone(),
2646            interface: ShaderMetaData::Passthrough(PassthroughInterface {
2647                entry_point_names: templates
2648                    .iter()
2649                    .map(|template| template.entry_point.to_string())
2650                    .collect(),
2651            }),
2652            label: label.to_string(),
2653        }))
2654    }
2655
2656    pub(crate) fn create_command_encoder(
2657        self: &Arc<Self>,
2658        label: &crate::Label,
2659    ) -> Result<Arc<command::CommandEncoder>, DeviceError> {
2660        self.check_is_valid()?;
2661
2662        let queue = self.get_queue().unwrap();
2663
2664        let encoder = self
2665            .command_allocator
2666            .acquire_encoder(self.raw(), queue.raw())
2667            .map_err(|e| self.handle_hal_error(e))?;
2668
2669        let cmd_enc = command::CommandEncoder::new(encoder, self, label);
2670
2671        let cmd_enc = Arc::new(cmd_enc);
2672
2673        Ok(cmd_enc)
2674    }
2675
2676    /// Generate information about late-validated buffer bindings for pipelines.
2677    //TODO: should this be combined with `get_introspection_bind_group_layouts` in some way?
2678    fn make_late_sized_buffer_groups(
2679        shader_binding_sizes: &FastHashMap<naga::ResourceBinding, wgt::BufferSize>,
2680        layout: &binding_model::PipelineLayout,
2681    ) -> ArrayVec<pipeline::LateSizedBufferGroup, { hal::MAX_BIND_GROUPS }> {
2682        // Given the shader-required binding sizes and the pipeline layout,
2683        // return the filtered list of them in the layout order,
2684        // removing those with given `min_binding_size`.
2685        layout
2686            .bind_group_layouts
2687            .iter()
2688            .enumerate()
2689            .map(|(group_index, bgl)| {
2690                let Some(bgl) = bgl else {
2691                    return pipeline::LateSizedBufferGroup::default();
2692                };
2693
2694                let shader_sizes = bgl
2695                    .entries
2696                    .values()
2697                    .filter_map(|entry| match entry.ty {
2698                        wgt::BindingType::Buffer {
2699                            min_binding_size: None,
2700                            ..
2701                        } => {
2702                            let rb = naga::ResourceBinding {
2703                                group: group_index as u32,
2704                                binding: entry.binding,
2705                            };
2706                            let shader_size =
2707                                shader_binding_sizes.get(&rb).map_or(0, |nz| nz.get());
2708                            Some(shader_size)
2709                        }
2710                        _ => None,
2711                    })
2712                    .collect();
2713                pipeline::LateSizedBufferGroup { shader_sizes }
2714            })
2715            .collect()
2716    }
2717
2718    pub fn create_bind_group_layout(
2719        self: &Arc<Self>,
2720        desc: &binding_model::BindGroupLayoutDescriptor,
2721    ) -> (Arc<BindGroupLayout>, Option<CreateBindGroupLayoutError>) {
2722        let (bgl, error) = match self.create_bind_group_layout_inner(desc) {
2723            Ok(layout) => (layout, None),
2724            Err(e) => (
2725                BindGroupLayout::invalid(self, desc.label.to_string()),
2726                Some(e),
2727            ),
2728        };
2729        #[cfg(feature = "trace")]
2730        if let Some(ref mut trace) = *self.trace.lock() {
2731            use crate::device::trace::IntoTrace;
2732
2733            trace.add(trace::Action::CreateBindGroupLayout(
2734                bgl.to_trace(),
2735                desc.clone(),
2736            ));
2737        }
2738        (bgl, error)
2739    }
2740
2741    fn create_bind_group_layout_inner(
2742        self: &Arc<Device>,
2743        desc: &binding_model::BindGroupLayoutDescriptor,
2744    ) -> Result<Arc<BindGroupLayout>, CreateBindGroupLayoutError> {
2745        self.check_is_valid()?;
2746
2747        let entry_map = bgl::EntryMap::from_entries(&desc.entries)?;
2748
2749        let bgl_result = self.bgl_pool.get_or_init(entry_map, |entry_map| {
2750            let bgl =
2751                self.create_bind_group_layout_impl(&desc.label, entry_map, bgl::Origin::Pool)?;
2752            bgl.exclusive_pipeline
2753                .set(binding_model::ExclusivePipeline::None)
2754                .unwrap();
2755            Ok(bgl)
2756        });
2757
2758        match bgl_result {
2759            Ok(layout) => Ok(layout),
2760            Err(e) => Err(e),
2761        }
2762    }
2763
2764    fn create_bind_group_layout_impl(
2765        self: &Arc<Self>,
2766        label: &crate::Label,
2767        entry_map: bgl::EntryMap,
2768        origin: bgl::Origin,
2769    ) -> Result<Arc<BindGroupLayout>, CreateBindGroupLayoutError> {
2770        #[derive(PartialEq)]
2771        enum WritableStorage {
2772            Yes,
2773            No,
2774        }
2775
2776        for entry in entry_map.values() {
2777            if entry.binding >= self.limits.max_bindings_per_bind_group {
2778                return Err(CreateBindGroupLayoutError::InvalidBindingIndex {
2779                    binding: entry.binding,
2780                    maximum: self.limits.max_bindings_per_bind_group,
2781                });
2782            }
2783
2784            use wgt::BindingType as Bt;
2785
2786            let mut required_features = wgt::Features::empty();
2787            let mut required_downlevel_flags = wgt::DownlevelFlags::empty();
2788            let (array_feature, writable_storage) = match entry.ty {
2789                Bt::Buffer {
2790                    ty: wgt::BufferBindingType::Uniform,
2791                    has_dynamic_offset: false,
2792                    min_binding_size: _,
2793                } => (
2794                    Some(wgt::Features::BUFFER_BINDING_ARRAY),
2795                    WritableStorage::No,
2796                ),
2797                Bt::Buffer {
2798                    ty: wgt::BufferBindingType::Uniform,
2799                    has_dynamic_offset: true,
2800                    min_binding_size: _,
2801                } => (
2802                    Some(wgt::Features::BUFFER_BINDING_ARRAY),
2803                    WritableStorage::No,
2804                ),
2805                Bt::Buffer {
2806                    ty: wgt::BufferBindingType::Storage { read_only },
2807                    ..
2808                } => (
2809                    Some(
2810                        wgt::Features::BUFFER_BINDING_ARRAY
2811                            | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY,
2812                    ),
2813                    match read_only {
2814                        true => WritableStorage::No,
2815                        false => WritableStorage::Yes,
2816                    },
2817                ),
2818                Bt::Sampler { .. } => (
2819                    Some(wgt::Features::TEXTURE_BINDING_ARRAY),
2820                    WritableStorage::No,
2821                ),
2822                Bt::Texture {
2823                    multisampled: true,
2824                    sample_type: TextureSampleType::Float { filterable: true },
2825                    ..
2826                } => {
2827                    return Err(CreateBindGroupLayoutError::Entry {
2828                        binding: entry.binding,
2829                        error:
2830                            BindGroupLayoutEntryError::SampleTypeFloatFilterableBindingMultisampled,
2831                    });
2832                }
2833                Bt::Texture {
2834                    multisampled,
2835                    view_dimension,
2836                    ..
2837                } => {
2838                    if multisampled && view_dimension != TextureViewDimension::D2 {
2839                        return Err(CreateBindGroupLayoutError::Entry {
2840                            binding: entry.binding,
2841                            error: BindGroupLayoutEntryError::Non2DMultisampled(view_dimension),
2842                        });
2843                    }
2844
2845                    (
2846                        Some(wgt::Features::TEXTURE_BINDING_ARRAY),
2847                        WritableStorage::No,
2848                    )
2849                }
2850                Bt::StorageTexture {
2851                    access,
2852                    view_dimension,
2853                    format,
2854                } => {
2855                    use wgt::{StorageTextureAccess as Access, TextureFormatFeatureFlags as Flags};
2856
2857                    match view_dimension {
2858                        TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
2859                            return Err(CreateBindGroupLayoutError::Entry {
2860                                binding: entry.binding,
2861                                error: BindGroupLayoutEntryError::StorageTextureCube,
2862                            })
2863                        }
2864                        _ => (),
2865                    }
2866                    match access {
2867                        wgt::StorageTextureAccess::Atomic
2868                            if !self.features.contains(wgt::Features::TEXTURE_ATOMIC) =>
2869                        {
2870                            return Err(CreateBindGroupLayoutError::Entry {
2871                                binding: entry.binding,
2872                                error: BindGroupLayoutEntryError::StorageTextureAtomic,
2873                            });
2874                        }
2875                        _ => (),
2876                    }
2877
2878                    let format_features =
2879                        self.describe_format_features(format).map_err(|error| {
2880                            CreateBindGroupLayoutError::Entry {
2881                                binding: entry.binding,
2882                                error: BindGroupLayoutEntryError::MissingFeatures(error),
2883                            }
2884                        })?;
2885
2886                    let required_feature_flag = match access {
2887                        Access::WriteOnly => Flags::STORAGE_WRITE_ONLY,
2888                        Access::ReadOnly => Flags::STORAGE_READ_ONLY,
2889                        Access::ReadWrite => Flags::STORAGE_READ_WRITE,
2890                        Access::Atomic => Flags::STORAGE_ATOMIC,
2891                    };
2892
2893                    if !format_features.flags.contains(required_feature_flag) {
2894                        return Err(
2895                            CreateBindGroupLayoutError::UnsupportedStorageTextureAccess {
2896                                binding: entry.binding,
2897                                access,
2898                                format,
2899                            },
2900                        );
2901                    }
2902
2903                    (
2904                        Some(
2905                            wgt::Features::TEXTURE_BINDING_ARRAY
2906                                | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY,
2907                        ),
2908                        match access {
2909                            wgt::StorageTextureAccess::WriteOnly => WritableStorage::Yes,
2910                            wgt::StorageTextureAccess::ReadOnly => WritableStorage::No,
2911                            wgt::StorageTextureAccess::ReadWrite => WritableStorage::Yes,
2912                            wgt::StorageTextureAccess::Atomic => {
2913                                required_features |= wgt::Features::TEXTURE_ATOMIC;
2914                                WritableStorage::Yes
2915                            }
2916                        },
2917                    )
2918                }
2919                Bt::AccelerationStructure { vertex_return } => {
2920                    self.require_features(wgt::Features::EXPERIMENTAL_RAY_QUERY)
2921                        .map_err(|e| CreateBindGroupLayoutError::Entry {
2922                            binding: entry.binding,
2923                            error: e.into(),
2924                        })?;
2925                    if vertex_return {
2926                        self.require_features(wgt::Features::EXPERIMENTAL_RAY_HIT_VERTEX_RETURN)
2927                            .map_err(|e| CreateBindGroupLayoutError::Entry {
2928                                binding: entry.binding,
2929                                error: e.into(),
2930                            })?;
2931                    }
2932                    (
2933                        Some(wgt::Features::ACCELERATION_STRUCTURE_BINDING_ARRAY),
2934                        WritableStorage::No,
2935                    )
2936                }
2937                Bt::ExternalTexture => {
2938                    self.require_features(wgt::Features::EXTERNAL_TEXTURE)
2939                        .map_err(|e| CreateBindGroupLayoutError::Entry {
2940                            binding: entry.binding,
2941                            error: e.into(),
2942                        })?;
2943                    (None, WritableStorage::No)
2944                }
2945            };
2946
2947            // Validate the count parameter
2948            if entry.count.is_some() {
2949                required_features |= array_feature
2950                    .ok_or(BindGroupLayoutEntryError::ArrayUnsupported)
2951                    .map_err(|error| CreateBindGroupLayoutError::Entry {
2952                        binding: entry.binding,
2953                        error,
2954                    })?;
2955            }
2956
2957            if entry.visibility.contains_unknown_bits() {
2958                return Err(CreateBindGroupLayoutError::InvalidVisibility(
2959                    entry.visibility,
2960                ));
2961            }
2962
2963            if entry.visibility.contains(wgt::ShaderStages::VERTEX) {
2964                if writable_storage == WritableStorage::Yes {
2965                    required_features |= wgt::Features::VERTEX_WRITABLE_STORAGE;
2966                }
2967                if let Bt::Buffer {
2968                    ty: wgt::BufferBindingType::Storage { .. },
2969                    ..
2970                } = entry.ty
2971                {
2972                    required_downlevel_flags |= wgt::DownlevelFlags::VERTEX_STORAGE;
2973                }
2974            }
2975            if writable_storage == WritableStorage::Yes
2976                && entry.visibility.contains(wgt::ShaderStages::FRAGMENT)
2977            {
2978                required_downlevel_flags |= wgt::DownlevelFlags::FRAGMENT_WRITABLE_STORAGE;
2979            }
2980
2981            self.require_features(required_features)
2982                .map_err(BindGroupLayoutEntryError::MissingFeatures)
2983                .map_err(|error| CreateBindGroupLayoutError::Entry {
2984                    binding: entry.binding,
2985                    error,
2986                })?;
2987            self.require_downlevel_flags(required_downlevel_flags)
2988                .map_err(BindGroupLayoutEntryError::MissingDownlevelFlags)
2989                .map_err(|error| CreateBindGroupLayoutError::Entry {
2990                    binding: entry.binding,
2991                    error,
2992                })?;
2993        }
2994
2995        let bgl_flags = conv::bind_group_layout_flags(self.features);
2996
2997        let hal_bindings = entry_map.values().copied().collect::<Vec<_>>();
2998        let hal_desc = hal::BindGroupLayoutDescriptor {
2999            label: label.to_hal(self.instance_flags),
3000            flags: bgl_flags,
3001            entries: &hal_bindings,
3002        };
3003
3004        let mut count_validator = binding_model::BindingTypeMaxCountValidator::default();
3005        for entry in entry_map.values() {
3006            count_validator.add_binding(entry);
3007        }
3008        // If a single bind group layout violates limits, the pipeline layout is
3009        // definitely going to violate limits too, lets catch it now.
3010        count_validator
3011            .validate(&self.limits, self.instance_flags)
3012            .map_err(CreateBindGroupLayoutError::TooManyBindings)?;
3013
3014        // Validate that binding arrays don't conflict with dynamic offsets.
3015        count_validator.validate_binding_arrays()?;
3016
3017        let raw = unsafe { self.raw().create_bind_group_layout(&hal_desc) }
3018            .map_err(|e| self.handle_hal_error(e))?;
3019
3020        let bgl = BindGroupLayout {
3021            state: ResourceState::Valid(BindGroupLayoutState {
3022                raw: binding_model::RawBindGroupLayout::Owning(ManuallyDrop::new(raw)),
3023                origin,
3024                binding_count_validator: count_validator,
3025            }),
3026            device: self.clone(),
3027            entries: entry_map,
3028            exclusive_pipeline: OnceCellOrLock::new(),
3029            label: label.to_string(),
3030        };
3031
3032        let bgl = Arc::new(bgl);
3033
3034        Ok(bgl)
3035    }
3036
3037    fn create_buffer_binding<'a>(
3038        &self,
3039        bb: &'a binding_model::ResolvedBufferBinding,
3040        binding: u32,
3041        decl: &wgt::BindGroupLayoutEntry,
3042        buffer_init_actions: &mut Vec<BufferInitTrackerAction>,
3043        dynamic_binding_info: &mut Vec<binding_model::BindGroupDynamicBindingData>,
3044        late_buffer_binding_sizes: &mut FastHashMap<u32, wgt::BufferSize>,
3045        used: &mut BindGroupStates,
3046        snatch_guard: &'a SnatchGuard<'a>,
3047    ) -> Result<hal::BufferBinding<'a, dyn hal::DynBuffer>, CreateBindGroupError> {
3048        use crate::binding_model::CreateBindGroupError as Error;
3049
3050        let (binding_ty, dynamic, min_size) = match decl.ty {
3051            wgt::BindingType::Buffer {
3052                ty,
3053                has_dynamic_offset,
3054                min_binding_size,
3055            } => (ty, has_dynamic_offset, min_binding_size),
3056            _ => {
3057                return Err(Error::WrongBindingType {
3058                    binding,
3059                    actual: decl.ty,
3060                    expected: "UniformBuffer, StorageBuffer or ReadonlyStorageBuffer",
3061                })
3062            }
3063        };
3064
3065        let (pub_usage, internal_use, range_limit) = match binding_ty {
3066            wgt::BufferBindingType::Uniform => (
3067                wgt::BufferUsages::UNIFORM,
3068                wgt::BufferUses::UNIFORM,
3069                self.limits.max_uniform_buffer_binding_size,
3070            ),
3071            wgt::BufferBindingType::Storage { read_only } => (
3072                wgt::BufferUsages::STORAGE,
3073                if read_only {
3074                    wgt::BufferUses::STORAGE_READ_ONLY
3075                } else {
3076                    wgt::BufferUses::STORAGE_READ_WRITE
3077                },
3078                self.limits.max_storage_buffer_binding_size,
3079            ),
3080        };
3081
3082        let (align, align_limit_name) =
3083            binding_model::buffer_binding_type_alignment(&self.limits, binding_ty);
3084        if !bb.offset.is_multiple_of(align as u64) {
3085            return Err(Error::UnalignedBufferOffset(
3086                bb.offset,
3087                align_limit_name,
3088                align,
3089            ));
3090        }
3091
3092        let buffer = &bb.buffer;
3093
3094        used.buffers.insert_single(buffer.clone(), internal_use);
3095
3096        buffer.same_device(self)?;
3097
3098        buffer.check_usage(pub_usage)?;
3099
3100        let req_size = match bb.size.map(wgt::BufferSize::new) {
3101            // Requested a non-zero size
3102            Some(non_zero @ Some(_)) => non_zero,
3103            // Requested size not specified
3104            None => None,
3105            // Requested zero size
3106            Some(None) => return Err(CreateBindGroupError::BindingZeroSize(buffer.error_ident())),
3107        };
3108        let (bb, bind_size) = buffer.binding(bb.offset, req_size, snatch_guard)?;
3109
3110        if matches!(binding_ty, wgt::BufferBindingType::Storage { .. })
3111            && bind_size % u64::from(wgt::STORAGE_BINDING_SIZE_ALIGNMENT) != 0
3112        {
3113            return Err(Error::UnalignedEffectiveBufferBindingSizeForStorage {
3114                alignment: wgt::STORAGE_BINDING_SIZE_ALIGNMENT,
3115                size: bind_size,
3116            });
3117        }
3118
3119        let bind_end = bb.offset + bind_size;
3120
3121        if bind_size > range_limit {
3122            return Err(Error::BufferRangeTooLarge {
3123                binding,
3124                given: bind_size,
3125                limit: range_limit,
3126            });
3127        }
3128
3129        // Record binding info for validating dynamic offsets
3130        if dynamic {
3131            dynamic_binding_info.push(binding_model::BindGroupDynamicBindingData {
3132                binding_idx: binding,
3133                buffer_size: buffer.size,
3134                binding_range: bb.offset..bind_end,
3135                maximum_dynamic_offset: buffer.size - bind_end,
3136                binding_type: binding_ty,
3137            });
3138        }
3139
3140        if let Some(non_zero) = min_size {
3141            let min_size = non_zero.get();
3142            if min_size > bind_size {
3143                return Err(Error::BindingSizeTooSmall {
3144                    buffer: buffer.error_ident(),
3145                    actual: bind_size,
3146                    min: min_size,
3147                });
3148            }
3149        } else {
3150            let late_size = wgt::BufferSize::new(bind_size)
3151                .ok_or_else(|| Error::BindingZeroSize(buffer.error_ident()))?;
3152            late_buffer_binding_sizes.insert(binding, late_size);
3153        }
3154
3155        // This was checked against the device's alignment requirements above,
3156        // which should always be a multiple of `COPY_BUFFER_ALIGNMENT`.
3157        assert_eq!(bb.offset % wgt::COPY_BUFFER_ALIGNMENT, 0);
3158
3159        let init_range = if dynamic {
3160            // We don't know what part of the buffer will be bound, so require that it
3161            // is fully initialized.
3162            0..buffer.size
3163        } else {
3164            // `wgpu_hal` only restricts shader access to bound buffer regions with
3165            // a certain resolution. For the sake of lazy initialization, round up
3166            // the size of the bound range to reflect how much of the buffer is
3167            // actually going to be visible to the shader.
3168            let bounds_check_alignment = binding_model::buffer_binding_type_bounds_check_alignment(
3169                &self.alignments,
3170                binding_ty,
3171            );
3172            let visible_size = align_to(bind_size, bounds_check_alignment);
3173
3174            bb.offset..bb.offset + visible_size
3175        };
3176
3177        buffer_init_actions.extend(buffer.initialization_status.read().create_action(
3178            buffer,
3179            init_range,
3180            MemoryInitKind::NeedsInitializedMemory,
3181        ));
3182
3183        Ok(bb)
3184    }
3185
3186    fn create_sampler_binding<'a>(
3187        &self,
3188        used: &mut BindGroupStates,
3189        binding: u32,
3190        decl: &wgt::BindGroupLayoutEntry,
3191        sampler: &'a Arc<Sampler>,
3192    ) -> Result<&'a dyn hal::DynSampler, CreateBindGroupError> {
3193        use crate::binding_model::CreateBindGroupError as Error;
3194
3195        used.samplers.insert_single(sampler.clone());
3196
3197        sampler.same_device(self)?;
3198
3199        match decl.ty {
3200            wgt::BindingType::Sampler(ty) => {
3201                let (allowed_filtering, allowed_comparison) = match ty {
3202                    wgt::SamplerBindingType::Filtering => (None, false),
3203                    wgt::SamplerBindingType::NonFiltering => (Some(false), false),
3204                    wgt::SamplerBindingType::Comparison => (None, true),
3205                };
3206                if let Some(allowed_filtering) = allowed_filtering {
3207                    if allowed_filtering != sampler.filtering {
3208                        return Err(Error::WrongSamplerFiltering {
3209                            binding,
3210                            layout_flt: allowed_filtering,
3211                            sampler_flt: sampler.filtering,
3212                        });
3213                    }
3214                }
3215                if allowed_comparison != sampler.comparison {
3216                    return Err(Error::WrongSamplerComparison {
3217                        binding,
3218                        layout_cmp: allowed_comparison,
3219                        sampler_cmp: sampler.comparison,
3220                    });
3221                }
3222            }
3223            _ => {
3224                return Err(Error::WrongBindingType {
3225                    binding,
3226                    actual: decl.ty,
3227                    expected: "Sampler",
3228                })
3229            }
3230        }
3231
3232        Ok(sampler.raw())
3233    }
3234
3235    fn create_texture_binding<'a>(
3236        &self,
3237        binding: u32,
3238        decl: &wgt::BindGroupLayoutEntry,
3239        view: &'a Arc<TextureView>,
3240        used: &mut BindGroupStates,
3241        texture_init_actions: &mut Vec<TextureInitTrackerAction>,
3242        snatch_guard: &'a SnatchGuard<'a>,
3243    ) -> Result<hal::TextureBinding<'a, dyn hal::DynTextureView>, CreateBindGroupError> {
3244        view.same_device(self)?;
3245
3246        let internal_use = self.texture_use_parameters(
3247            binding,
3248            decl,
3249            view,
3250            "SampledTexture, ReadonlyStorageTexture or WriteonlyStorageTexture",
3251        )?;
3252
3253        used.views.insert_single(view.clone(), internal_use);
3254
3255        let texture = &view.parent;
3256
3257        texture_init_actions.push(TextureInitTrackerAction {
3258            texture: texture.clone(),
3259            range: TextureInitRange {
3260                mip_range: view.desc.range.mip_range(texture.desc.mip_level_count),
3261                layer_range: view
3262                    .desc
3263                    .range
3264                    .layer_range(texture.desc.array_layer_count()),
3265            },
3266            kind: MemoryInitKind::NeedsInitializedMemory,
3267        });
3268
3269        Ok(hal::TextureBinding {
3270            view: view.try_raw(snatch_guard)?,
3271            usage: internal_use,
3272        })
3273    }
3274
3275    fn create_tlas_binding<'a>(
3276        self: &Arc<Self>,
3277        used: &mut BindGroupStates,
3278        binding: u32,
3279        decl: &wgt::BindGroupLayoutEntry,
3280        tlas: &'a Arc<Tlas>,
3281        snatch_guard: &'a SnatchGuard<'a>,
3282    ) -> Result<&'a dyn hal::DynAccelerationStructure, CreateBindGroupError> {
3283        use crate::binding_model::CreateBindGroupError as Error;
3284
3285        used.acceleration_structures.insert_single(tlas.clone());
3286
3287        tlas.same_device(self)?;
3288
3289        match decl.ty {
3290            wgt::BindingType::AccelerationStructure { vertex_return } => {
3291                if vertex_return
3292                    && !tlas.flags.contains(
3293                        wgpu_types::AccelerationStructureFlags::ALLOW_RAY_HIT_VERTEX_RETURN,
3294                    )
3295                {
3296                    return Err(Error::MissingTLASVertexReturn { binding });
3297                }
3298            }
3299            _ => {
3300                return Err(Error::WrongBindingType {
3301                    binding,
3302                    actual: decl.ty,
3303                    expected: "Tlas",
3304                });
3305            }
3306        }
3307
3308        Ok(tlas.try_raw(snatch_guard)?)
3309    }
3310
3311    #[inline(never)]
3312    fn create_external_texture_binding<'a>(
3313        &'a self,
3314        binding: u32,
3315        decl: &wgt::BindGroupLayoutEntry,
3316        external_texture: &'a Arc<ExternalTexture>,
3317        used: &mut BindGroupStates,
3318        snatch_guard: &'a SnatchGuard,
3319    ) -> Result<
3320        hal::ExternalTextureBinding<'a, dyn hal::DynBuffer, dyn hal::DynTextureView>,
3321        CreateBindGroupError,
3322    > {
3323        use crate::binding_model::CreateBindGroupError as Error;
3324
3325        external_texture.same_device(self)?;
3326
3327        used.external_textures
3328            .insert_single(external_texture.clone());
3329
3330        match decl.ty {
3331            wgt::BindingType::ExternalTexture => {}
3332            _ => {
3333                return Err(Error::WrongBindingType {
3334                    binding,
3335                    actual: decl.ty,
3336                    expected: "ExternalTexture",
3337                });
3338            }
3339        }
3340
3341        let planes = (0..3)
3342            .map(|i| {
3343                // We always need 3 bindings. If we have fewer than 3 planes
3344                // just bind plane 0 multiple times. The shader will only
3345                // sample from valid planes anyway.
3346                let plane = external_texture
3347                    .planes
3348                    .get(i)
3349                    .unwrap_or(&external_texture.planes[0]);
3350                let internal_use = wgt::TextureUses::RESOURCE;
3351                used.views.insert_single(plane.clone(), internal_use);
3352                let view = plane.try_raw(snatch_guard)?;
3353                Ok(hal::TextureBinding {
3354                    view,
3355                    usage: internal_use,
3356                })
3357            })
3358            // We can remove this intermediate Vec by using
3359            // array::try_from_fn() above, once it stabilizes.
3360            .collect::<Result<Vec<_>, Error>>()?;
3361        let planes = planes.try_into().unwrap();
3362
3363        used.buffers
3364            .insert_single(external_texture.params.clone(), wgt::BufferUses::UNIFORM);
3365        let params = external_texture.params.binding(0, None, snatch_guard)?.0;
3366
3367        Ok(hal::ExternalTextureBinding { planes, params })
3368    }
3369
3370    fn create_external_texture_binding_from_view<'a>(
3371        &'a self,
3372        binding: u32,
3373        decl: &wgt::BindGroupLayoutEntry,
3374        view: &'a Arc<TextureView>,
3375        used: &mut BindGroupStates,
3376        snatch_guard: &'a SnatchGuard,
3377    ) -> Result<
3378        hal::ExternalTextureBinding<'a, dyn hal::DynBuffer, dyn hal::DynTextureView>,
3379        CreateBindGroupError,
3380    > {
3381        use crate::binding_model::CreateBindGroupError as Error;
3382
3383        view.same_device(self)?;
3384
3385        let internal_use = self.texture_use_parameters(binding, decl, view, "SampledTexture")?;
3386        used.views.insert_single(view.clone(), internal_use);
3387
3388        match decl.ty {
3389            wgt::BindingType::ExternalTexture => {}
3390            _ => {
3391                return Err(Error::WrongBindingType {
3392                    binding,
3393                    actual: decl.ty,
3394                    expected: "ExternalTexture",
3395                });
3396            }
3397        }
3398
3399        // We need 3 bindings, so just repeat the same texture view 3 times.
3400        let planes = [
3401            hal::TextureBinding {
3402                view: view.try_raw(snatch_guard)?,
3403                usage: internal_use,
3404            },
3405            hal::TextureBinding {
3406                view: view.try_raw(snatch_guard)?,
3407                usage: internal_use,
3408            },
3409            hal::TextureBinding {
3410                view: view.try_raw(snatch_guard)?,
3411                usage: internal_use,
3412            },
3413        ];
3414        let params = hal::BufferBinding::new_unchecked(
3415            self.default_external_texture_params_buffer.as_ref(),
3416            0,
3417            None,
3418        );
3419
3420        Ok(hal::ExternalTextureBinding { planes, params })
3421    }
3422
3423    // This function expects the provided bind group layout to be resolved
3424    // (not passing a duplicate) beforehand.
3425    pub fn create_bind_group(
3426        self: &Arc<Self>,
3427        desc: binding_model::ResolvedBindGroupDescriptor,
3428    ) -> Result<Arc<BindGroup>, CreateBindGroupError> {
3429        use crate::binding_model::{CreateBindGroupError as Error, ResolvedBindingResource as Br};
3430
3431        let layout = desc.layout;
3432
3433        self.check_is_valid()?;
3434        layout.same_device(self)?;
3435        layout.check_is_valid()?;
3436
3437        {
3438            // Check that the number of entries in the descriptor matches
3439            // the number of entries in the layout.
3440            let actual = desc.entries.len();
3441            let expected = layout.entries.len();
3442            if actual != expected {
3443                return Err(Error::BindingsNumMismatch { expected, actual });
3444            }
3445        }
3446
3447        // TODO: arrayvec/smallvec, or re-use allocations
3448        // Record binding info for dynamic offset validation
3449        let mut dynamic_binding_info = Vec::new();
3450        // Map of binding -> shader reflected size
3451        //Note: we can't collect into a vector right away because
3452        // it needs to be in BGL iteration order, not BG entry order.
3453        let mut late_buffer_binding_sizes = FastHashMap::default();
3454        // fill out the descriptors
3455        let mut used = BindGroupStates::new();
3456
3457        let mut buffer_init_actions = Vec::new();
3458        let mut texture_init_actions = Vec::new();
3459        let mut hal_entries = Vec::with_capacity(desc.entries.len());
3460        let mut hal_buffers = Vec::new();
3461        let mut hal_samplers = Vec::new();
3462        let mut hal_textures = Vec::new();
3463        let mut hal_tlas_s = Vec::new();
3464        let mut hal_external_textures = Vec::new();
3465        let snatch_guard = self.snatchable_lock.read();
3466        for entry in desc.entries.iter() {
3467            let binding = entry.binding;
3468            // Find the corresponding declaration in the layout
3469            let decl = layout
3470                .entries
3471                .get(binding)
3472                .ok_or(Error::MissingBindingDeclaration(binding))?;
3473            let (res_index, count) = match entry.resource {
3474                Br::Buffer(ref bb) => {
3475                    let bb = self.create_buffer_binding(
3476                        bb,
3477                        binding,
3478                        decl,
3479                        &mut buffer_init_actions,
3480                        &mut dynamic_binding_info,
3481                        &mut late_buffer_binding_sizes,
3482                        &mut used,
3483                        &snatch_guard,
3484                    )?;
3485
3486                    let res_index = hal_buffers.len();
3487                    hal_buffers.push(bb);
3488                    (res_index, 1)
3489                }
3490                Br::BufferArray(ref bindings_array) => {
3491                    let num_bindings = bindings_array.len();
3492                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
3493
3494                    let res_index = hal_buffers.len();
3495                    for bb in bindings_array.iter() {
3496                        let bb = self.create_buffer_binding(
3497                            bb,
3498                            binding,
3499                            decl,
3500                            &mut buffer_init_actions,
3501                            &mut dynamic_binding_info,
3502                            &mut late_buffer_binding_sizes,
3503                            &mut used,
3504                            &snatch_guard,
3505                        )?;
3506                        hal_buffers.push(bb);
3507                    }
3508                    (res_index, num_bindings)
3509                }
3510                Br::Sampler(ref sampler) => {
3511                    let sampler = self.create_sampler_binding(&mut used, binding, decl, sampler)?;
3512
3513                    let res_index = hal_samplers.len();
3514                    hal_samplers.push(sampler);
3515                    (res_index, 1)
3516                }
3517                Br::SamplerArray(ref samplers) => {
3518                    let num_bindings = samplers.len();
3519                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
3520
3521                    let res_index = hal_samplers.len();
3522                    for sampler in samplers.iter() {
3523                        let sampler =
3524                            self.create_sampler_binding(&mut used, binding, decl, sampler)?;
3525
3526                        hal_samplers.push(sampler);
3527                    }
3528
3529                    (res_index, num_bindings)
3530                }
3531                Br::TextureView(ref view) => match decl.ty {
3532                    wgt::BindingType::ExternalTexture => {
3533                        let et = self.create_external_texture_binding_from_view(
3534                            binding,
3535                            decl,
3536                            view,
3537                            &mut used,
3538                            &snatch_guard,
3539                        )?;
3540                        let res_index = hal_external_textures.len();
3541                        hal_external_textures.push(et);
3542                        (res_index, 1)
3543                    }
3544                    _ => {
3545                        let tb = self.create_texture_binding(
3546                            binding,
3547                            decl,
3548                            view,
3549                            &mut used,
3550                            &mut texture_init_actions,
3551                            &snatch_guard,
3552                        )?;
3553                        let res_index = hal_textures.len();
3554                        hal_textures.push(tb);
3555                        (res_index, 1)
3556                    }
3557                },
3558                Br::TextureViewArray(ref views) => {
3559                    let num_bindings = views.len();
3560                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
3561
3562                    let res_index = hal_textures.len();
3563                    for view in views.iter() {
3564                        let tb = self.create_texture_binding(
3565                            binding,
3566                            decl,
3567                            view,
3568                            &mut used,
3569                            &mut texture_init_actions,
3570                            &snatch_guard,
3571                        )?;
3572
3573                        hal_textures.push(tb);
3574                    }
3575
3576                    (res_index, num_bindings)
3577                }
3578                Br::AccelerationStructure(ref tlas) => {
3579                    let tlas =
3580                        self.create_tlas_binding(&mut used, binding, decl, tlas, &snatch_guard)?;
3581                    let res_index = hal_tlas_s.len();
3582                    hal_tlas_s.push(tlas);
3583                    (res_index, 1)
3584                }
3585                Br::AccelerationStructureArray(ref tlas_array) => {
3586                    // Feature validation for TLAS binding arrays happens at bind group layout
3587                    // creation time (mirroring other binding-array resource types). By the time we
3588                    // get here, `decl.count` has already been validated against device features.
3589                    let num_bindings = tlas_array.len();
3590                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
3591
3592                    let res_index = hal_tlas_s.len();
3593                    for tlas in tlas_array.iter() {
3594                        let tlas = self.create_tlas_binding(
3595                            &mut used,
3596                            binding,
3597                            decl,
3598                            tlas,
3599                            &snatch_guard,
3600                        )?;
3601                        hal_tlas_s.push(tlas);
3602                    }
3603                    (res_index, num_bindings)
3604                }
3605                Br::ExternalTexture(ref et) => {
3606                    let et = self.create_external_texture_binding(
3607                        binding,
3608                        decl,
3609                        et,
3610                        &mut used,
3611                        &snatch_guard,
3612                    )?;
3613                    let res_index = hal_external_textures.len();
3614                    hal_external_textures.push(et);
3615                    (res_index, 1)
3616                }
3617            };
3618
3619            hal_entries.push(hal::BindGroupEntry {
3620                binding,
3621                resource_index: res_index as u32,
3622                count: count as u32,
3623            });
3624        }
3625
3626        used.optimize();
3627
3628        hal_entries.sort_by_key(|entry| entry.binding);
3629        for (a, b) in hal_entries.iter().zip(hal_entries.iter().skip(1)) {
3630            if a.binding == b.binding {
3631                return Err(Error::DuplicateBinding(a.binding));
3632            }
3633        }
3634
3635        dynamic_binding_info.sort_by_key(|i| i.binding_idx);
3636
3637        let hal_desc = hal::BindGroupDescriptor {
3638            label: desc.label.to_hal(self.instance_flags),
3639            layout: layout.try_raw()?,
3640            entries: &hal_entries,
3641            buffers: &hal_buffers,
3642            samplers: &hal_samplers,
3643            textures: &hal_textures,
3644            acceleration_structures: &hal_tlas_s,
3645            external_textures: &hal_external_textures,
3646        };
3647        let raw = unsafe { self.raw().create_bind_group(&hal_desc) }
3648            .map_err(|e| self.handle_hal_error(e))?;
3649
3650        // collect in the order of BGL iteration
3651        let late_buffer_binding_infos = layout
3652            .entries
3653            .indices()
3654            .flat_map(|binding| {
3655                let size = late_buffer_binding_sizes.get(&binding).cloned()?;
3656                Some(BindGroupLateBufferBindingInfo {
3657                    binding_index: binding,
3658                    size,
3659                })
3660            })
3661            .collect();
3662
3663        let bind_group = BindGroup {
3664            raw: Snatchable::new(raw),
3665            device: self.clone(),
3666            layout,
3667            label: desc.label.to_string(),
3668            tracking_data: TrackingData::new(self.tracker_indices.bind_groups.clone()),
3669            used,
3670            buffer_init_actions,
3671            texture_init_actions,
3672            dynamic_binding_info,
3673            late_buffer_binding_infos,
3674        };
3675
3676        let bind_group = Arc::new(bind_group);
3677
3678        let weak_ref = Arc::downgrade(&bind_group);
3679        for texture in bind_group.used.views.used_textures() {
3680            let mut bind_groups = texture.bind_groups.lock();
3681            bind_groups.push(weak_ref.clone());
3682        }
3683        for buffer in bind_group.used.buffers.used_resources() {
3684            let mut bind_groups = buffer.bind_groups.lock();
3685            bind_groups.push(weak_ref.clone());
3686        }
3687
3688        Ok(bind_group)
3689    }
3690
3691    fn check_array_binding(
3692        features: wgt::Features,
3693        count: Option<NonZeroU32>,
3694        num_bindings: usize,
3695    ) -> Result<(), CreateBindGroupError> {
3696        use super::binding_model::CreateBindGroupError as Error;
3697
3698        if let Some(count) = count {
3699            let count = count.get() as usize;
3700            if count < num_bindings {
3701                return Err(Error::BindingArrayPartialLengthMismatch {
3702                    actual: num_bindings,
3703                    expected: count,
3704                });
3705            }
3706            if count != num_bindings
3707                && !features.contains(wgt::Features::PARTIALLY_BOUND_BINDING_ARRAY)
3708            {
3709                return Err(Error::BindingArrayLengthMismatch {
3710                    actual: num_bindings,
3711                    expected: count,
3712                });
3713            }
3714            if num_bindings == 0 {
3715                return Err(Error::BindingArrayZeroLength);
3716            }
3717        } else {
3718            return Err(Error::SingleBindingExpected);
3719        };
3720
3721        Ok(())
3722    }
3723
3724    fn texture_use_parameters(
3725        &self,
3726        binding: u32,
3727        decl: &wgt::BindGroupLayoutEntry,
3728        view: &TextureView,
3729        expected: &'static str,
3730    ) -> Result<wgt::TextureUses, CreateBindGroupError> {
3731        use crate::binding_model::CreateBindGroupError as Error;
3732        if view
3733            .desc
3734            .aspects()
3735            .contains(hal::FormatAspects::DEPTH | hal::FormatAspects::STENCIL)
3736        {
3737            return Err(Error::DepthStencilAspect);
3738        }
3739        match decl.ty {
3740            wgt::BindingType::Texture {
3741                sample_type,
3742                view_dimension,
3743                multisampled,
3744            } => {
3745                use wgt::TextureSampleType as Tst;
3746                if multisampled != (view.samples != 1) {
3747                    return Err(Error::InvalidTextureMultisample {
3748                        binding,
3749                        layout_multisampled: multisampled,
3750                        view_samples: view.samples,
3751                    });
3752                }
3753                let compat_sample_type = view
3754                    .desc
3755                    .format
3756                    .sample_type(Some(view.desc.range.aspect), Some(self.features))
3757                    .unwrap();
3758                match (sample_type, compat_sample_type) {
3759                    (Tst::Uint, Tst::Uint) |
3760                        (Tst::Sint, Tst::Sint) |
3761                        (Tst::Depth, Tst::Depth) |
3762                        // if we expect non-filterable, accept anything float
3763                        (Tst::Float { filterable: false }, Tst::Float { .. }) |
3764                        // if we expect filterable, require it
3765                        (Tst::Float { filterable: true }, Tst::Float { filterable: true }) |
3766                        // if we expect non-filterable, also accept depth
3767                        (Tst::Float { filterable: false }, Tst::Depth) => {}
3768                    // if we expect filterable, also accept Float that is defined as
3769                    // unfilterable if filterable feature is explicitly enabled (only hit
3770                    // if wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES is
3771                    // enabled)
3772                    (Tst::Float { filterable: true }, Tst::Float { .. })
3773                        if view.format_features.flags
3774                            .contains(wgt::TextureFormatFeatureFlags::FILTERABLE) => {}
3775                    _ => {
3776                        return Err(Error::InvalidTextureSampleType {
3777                            binding,
3778                            layout_sample_type: sample_type,
3779                            view_format: view.desc.format,
3780                            view_sample_type: compat_sample_type,
3781                        })
3782                    }
3783                }
3784                if view_dimension != view.desc.dimension {
3785                    return Err(Error::InvalidTextureDimension {
3786                        binding,
3787                        layout_dimension: view_dimension,
3788                        view_dimension: view.desc.dimension,
3789                    });
3790                }
3791                view.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
3792                Ok(wgt::TextureUses::RESOURCE)
3793            }
3794            wgt::BindingType::StorageTexture {
3795                access,
3796                format,
3797                view_dimension,
3798            } => {
3799                if format != view.desc.format {
3800                    return Err(Error::InvalidStorageTextureFormat {
3801                        binding,
3802                        layout_format: format,
3803                        view_format: view.desc.format,
3804                    });
3805                }
3806                if view_dimension != view.desc.dimension {
3807                    return Err(Error::InvalidTextureDimension {
3808                        binding,
3809                        layout_dimension: view_dimension,
3810                        view_dimension: view.desc.dimension,
3811                    });
3812                }
3813
3814                let mip_level_count = view.selector.mips.end - view.selector.mips.start;
3815                if mip_level_count != 1 {
3816                    return Err(Error::InvalidStorageTextureMipLevelCount {
3817                        binding,
3818                        mip_level_count,
3819                    });
3820                }
3821
3822                view.check_usage(wgt::TextureUsages::STORAGE_BINDING)?;
3823
3824                Ok(match access {
3825                    wgt::StorageTextureAccess::ReadOnly => wgt::TextureUses::STORAGE_READ_ONLY,
3826                    wgt::StorageTextureAccess::WriteOnly => wgt::TextureUses::STORAGE_WRITE_ONLY,
3827                    wgt::StorageTextureAccess::ReadWrite => wgt::TextureUses::STORAGE_READ_WRITE,
3828                    wgt::StorageTextureAccess::Atomic => wgt::TextureUses::STORAGE_ATOMIC,
3829                })
3830            }
3831            wgt::BindingType::ExternalTexture => {
3832                if view.desc.dimension != TextureViewDimension::D2 {
3833                    return Err(Error::InvalidTextureDimension {
3834                        binding,
3835                        layout_dimension: TextureViewDimension::D2,
3836                        view_dimension: view.desc.dimension,
3837                    });
3838                }
3839                let mip_level_count = view.selector.mips.end - view.selector.mips.start;
3840                if mip_level_count != 1 {
3841                    return Err(Error::InvalidExternalTextureMipLevelCount {
3842                        binding,
3843                        mip_level_count,
3844                    });
3845                }
3846                if view.desc.format != TextureFormat::Rgba8Unorm
3847                    && view.desc.format != TextureFormat::Bgra8Unorm
3848                    && view.desc.format != TextureFormat::Rgba16Float
3849                {
3850                    return Err(Error::InvalidExternalTextureFormat {
3851                        binding,
3852                        format: view.desc.format,
3853                    });
3854                }
3855                if view.samples != 1 {
3856                    return Err(Error::InvalidTextureMultisample {
3857                        binding,
3858                        layout_multisampled: false,
3859                        view_samples: view.samples,
3860                    });
3861                }
3862
3863                view.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
3864                Ok(wgt::TextureUses::RESOURCE)
3865            }
3866            _ => Err(Error::WrongBindingType {
3867                binding,
3868                actual: decl.ty,
3869                expected,
3870            }),
3871        }
3872    }
3873
3874    pub fn create_pipeline_layout(
3875        self: &Arc<Self>,
3876        desc: &binding_model::ResolvedPipelineLayoutDescriptor,
3877    ) -> (
3878        Arc<binding_model::PipelineLayout>,
3879        Option<binding_model::CreatePipelineLayoutError>,
3880    ) {
3881        let (layout, error) = match self.create_pipeline_layout_impl(desc, false) {
3882            Ok(layout) => (layout, None),
3883            Err(e) => (
3884                binding_model::PipelineLayout::invalid(Arc::clone(self), desc.label.to_string()),
3885                Some(e),
3886            ),
3887        };
3888        #[cfg(feature = "trace")]
3889        if let Some(ref mut trace) = *self.trace.lock() {
3890            use crate::device::trace::IntoTrace;
3891            trace.add(trace::Action::CreatePipelineLayout(
3892                layout.to_trace(),
3893                desc.to_trace(),
3894            ));
3895        }
3896        api_log!(
3897            "Device::create_pipeline_layout -> {:?}",
3898            Arc::as_ptr(&layout)
3899        );
3900        (layout, error)
3901    }
3902
3903    fn create_pipeline_layout_impl(
3904        self: &Arc<Self>,
3905        desc: &binding_model::ResolvedPipelineLayoutDescriptor,
3906        ignore_exclusive_pipeline_check: bool,
3907    ) -> Result<Arc<binding_model::PipelineLayout>, binding_model::CreatePipelineLayoutError> {
3908        use crate::binding_model::CreatePipelineLayoutError as Error;
3909
3910        self.check_is_valid()?;
3911
3912        let bind_group_layouts_count = desc.bind_group_layouts.len();
3913        let device_max_bind_groups = self.limits.max_bind_groups as usize;
3914        if bind_group_layouts_count > device_max_bind_groups {
3915            return Err(Error::TooManyGroups {
3916                actual: bind_group_layouts_count,
3917                max: device_max_bind_groups,
3918            });
3919        }
3920
3921        if desc.immediate_size != 0 {
3922            self.require_features(wgt::Features::IMMEDIATES)?;
3923        }
3924        if self.limits.max_immediate_size < desc.immediate_size {
3925            return Err(Error::ImmediateRangeTooLarge {
3926                size: desc.immediate_size,
3927                max: self.limits.max_immediate_size,
3928            });
3929        }
3930        if !desc
3931            .immediate_size
3932            .is_multiple_of(wgt::IMMEDIATE_DATA_ALIGNMENT)
3933        {
3934            return Err(Error::MisalignedImmediateSize {
3935                size: desc.immediate_size,
3936            });
3937        }
3938
3939        let mut count_validator = binding_model::BindingTypeMaxCountValidator::default();
3940
3941        for (index, bgl) in desc.bind_group_layouts.iter().enumerate() {
3942            let Some(bgl) = bgl else {
3943                continue;
3944            };
3945
3946            bgl.same_device(self)?;
3947
3948            if !ignore_exclusive_pipeline_check {
3949                let exclusive_pipeline = bgl.exclusive_pipeline.get().unwrap();
3950                if !matches!(exclusive_pipeline, binding_model::ExclusivePipeline::None) {
3951                    return Err(Error::BglHasExclusivePipeline {
3952                        index,
3953                        pipeline: alloc::format!("{exclusive_pipeline}"),
3954                    });
3955                }
3956            }
3957
3958            count_validator.merge(&bgl.state()?.binding_count_validator);
3959        }
3960
3961        count_validator
3962            .validate(&self.limits, self.instance_flags)
3963            .map_err(Error::TooManyBindings)?;
3964
3965        let buffers_and_acceleration_structures_in_vertex_stage =
3966            count_validator.buffers_and_acceleration_structures_in_vertex_stage();
3967
3968        let get_bgl_iter = || {
3969            desc.bind_group_layouts
3970                .iter()
3971                .map(|bgl| bgl.as_ref().filter(|bgl| !bgl.entries.is_empty()))
3972        };
3973
3974        let bind_group_layouts = get_bgl_iter()
3975            .map(|bgl| bgl.cloned())
3976            .collect::<ArrayVec<_, { hal::MAX_BIND_GROUPS }>>();
3977
3978        let raw_bind_group_layouts = get_bgl_iter()
3979            .map(|bgl| bgl.map(|bgl| bgl.try_raw()).transpose())
3980            .collect::<Result<ArrayVec<_, { hal::MAX_BIND_GROUPS }>, _>>()?;
3981
3982        let additional_flags = if self.indirect_validation.is_some() {
3983            hal::PipelineLayoutFlags::INDIRECT_BUILTIN_UPDATE
3984        } else {
3985            hal::PipelineLayoutFlags::empty()
3986        };
3987
3988        let hal_desc = hal::PipelineLayoutDescriptor {
3989            label: desc.label.to_hal(self.instance_flags),
3990            flags: hal::PipelineLayoutFlags::FIRST_VERTEX_INSTANCE
3991                | hal::PipelineLayoutFlags::NUM_WORK_GROUPS
3992                | additional_flags,
3993            bind_group_layouts: &raw_bind_group_layouts,
3994            immediate_size: desc.immediate_size,
3995        };
3996
3997        let raw = unsafe { self.raw().create_pipeline_layout(&hal_desc) }
3998            .map_err(|e| self.handle_hal_error(e))?;
3999
4000        drop(raw_bind_group_layouts);
4001
4002        let layout = binding_model::PipelineLayout {
4003            raw: ResourceState::Valid(raw),
4004            device: self.clone(),
4005            label: desc.label.to_string(),
4006            bind_group_layouts,
4007            immediate_size: desc.immediate_size,
4008            buffers_and_acceleration_structures_in_vertex_stage,
4009        };
4010
4011        let layout = Arc::new(layout);
4012
4013        Ok(layout)
4014    }
4015
4016    fn create_derived_pipeline_layout(
4017        self: &Arc<Self>,
4018        mut derived_group_layouts: Box<ArrayVec<bgl::EntryMap, { hal::MAX_BIND_GROUPS }>>,
4019        immediate_size: u32,
4020    ) -> Result<Arc<binding_model::PipelineLayout>, pipeline::ImplicitLayoutError> {
4021        while derived_group_layouts
4022            .last()
4023            .is_some_and(|map| map.is_empty())
4024        {
4025            derived_group_layouts.pop();
4026        }
4027
4028        let mut unique_bind_group_layouts = FastHashMap::default();
4029
4030        let bind_group_layouts = derived_group_layouts
4031            .into_iter()
4032            .map(|mut bgl_entry_map| {
4033                if bgl_entry_map.is_empty() {
4034                    return Ok(None);
4035                }
4036
4037                bgl_entry_map.sort();
4038                match unique_bind_group_layouts.entry(bgl_entry_map) {
4039                    hashbrown::hash_map::Entry::Occupied(v) => Ok(Some(Arc::clone(v.get()))),
4040                    hashbrown::hash_map::Entry::Vacant(e) => {
4041                        match self.create_bind_group_layout_impl(
4042                            &None,
4043                            e.key().clone(),
4044                            bgl::Origin::Derived,
4045                        ) {
4046                            Ok(bgl) => {
4047                                e.insert(bgl.clone());
4048                                Ok(Some(bgl))
4049                            }
4050                            Err(e) => Err(e),
4051                        }
4052                    }
4053                }
4054            })
4055            .collect::<Result<Vec<_>, _>>()?;
4056
4057        let layout_desc = binding_model::ResolvedPipelineLayoutDescriptor {
4058            label: None,
4059            bind_group_layouts: Cow::Owned(bind_group_layouts),
4060            immediate_size,
4061        };
4062
4063        let layout = self.create_pipeline_layout_impl(&layout_desc, true)?;
4064        Ok(layout)
4065    }
4066
4067    pub fn create_compute_pipeline(
4068        self: &Arc<Self>,
4069        desc: pipeline::ResolvedComputePipelineDescriptor,
4070    ) -> (
4071        Arc<pipeline::ComputePipeline>,
4072        Option<pipeline::CreateComputePipelineError>,
4073    ) {
4074        let (compute_pipeline, error) = match self.create_compute_pipeline_inner(desc.clone()) {
4075            Ok(compute_pipeline) => (compute_pipeline, None),
4076            Err(error) => (
4077                pipeline::ComputePipeline::invalid(self.clone(), desc.label.to_string()),
4078                Some(error),
4079            ),
4080        };
4081        #[cfg(feature = "trace")]
4082        if let Some(ref mut trace) = *self.trace.lock() {
4083            use crate::device::trace;
4084            use crate::device::trace::IntoTrace;
4085            trace.add(trace::Action::CreateComputePipeline {
4086                id: compute_pipeline.to_trace(),
4087                desc: desc.to_trace(),
4088            });
4089        }
4090        (compute_pipeline, error)
4091    }
4092
4093    pub fn create_compute_pipeline_inner(
4094        self: &Arc<Self>,
4095        desc: pipeline::ResolvedComputePipelineDescriptor,
4096    ) -> Result<Arc<pipeline::ComputePipeline>, pipeline::CreateComputePipelineError> {
4097        self.check_is_valid()?;
4098
4099        self.require_downlevel_flags(wgt::DownlevelFlags::COMPUTE_SHADERS)?;
4100
4101        let shader_module = desc.stage.module;
4102
4103        shader_module.same_device(self)?;
4104
4105        let is_auto_layout = desc.layout.is_none();
4106
4107        // Get the pipeline layout from the desc if it is provided.
4108        let pipeline_layout = match desc.layout {
4109            Some(pipeline_layout) => {
4110                pipeline_layout.same_device(self)?;
4111                pipeline_layout.check_valid()?;
4112                Some(pipeline_layout)
4113            }
4114            None => None,
4115        };
4116
4117        let mut binding_layout_source = match pipeline_layout {
4118            Some(pipeline_layout) => validation::BindingLayoutSource::Provided(pipeline_layout),
4119            None => validation::BindingLayoutSource::new_derived(&self.limits),
4120        };
4121        let mut shader_binding_sizes = FastHashMap::default();
4122        let io = validation::StageIo::default();
4123
4124        let final_entry_point_name;
4125
4126        {
4127            let stage = validation::ShaderStageForValidation::Compute;
4128
4129            final_entry_point_name = shader_module.finalize_entry_point_name(
4130                stage.to_naga(),
4131                desc.stage.entry_point.as_ref().map(|ep| ep.as_ref()),
4132            )?;
4133
4134            if let Some(interface) = shader_module.interface.interface() {
4135                let _ = interface.check_stage(
4136                    &mut binding_layout_source,
4137                    &mut shader_binding_sizes,
4138                    &final_entry_point_name,
4139                    stage,
4140                    io,
4141                    None,
4142                )?;
4143            }
4144        }
4145
4146        let pipeline_layout = match binding_layout_source {
4147            validation::BindingLayoutSource::Provided(pipeline_layout) => pipeline_layout,
4148            validation::BindingLayoutSource::Derived(entries) => {
4149                let immediate_size = shader_module
4150                    .interface
4151                    .interface()
4152                    .map_or(0, |i| i.immediate_size);
4153                self.create_derived_pipeline_layout(entries, immediate_size)?
4154            }
4155        };
4156
4157        let late_sized_buffer_groups =
4158            Device::make_late_sized_buffer_groups(&shader_binding_sizes, &pipeline_layout);
4159
4160        let cache = match desc.cache {
4161            Some(cache) => {
4162                cache.same_device(self)?;
4163                Some(cache)
4164            }
4165            None => None,
4166        };
4167
4168        let pipeline_desc = hal::ComputePipelineDescriptor {
4169            label: desc.label.to_hal(self.instance_flags),
4170            layout: pipeline_layout.raw()?,
4171            stage: hal::ProgrammableStage {
4172                module: shader_module.raw(),
4173                entry_point: final_entry_point_name.as_ref(),
4174                constants: &desc.stage.constants,
4175                zero_initialize_workgroup_memory: desc.stage.zero_initialize_workgroup_memory,
4176            },
4177            cache: cache.as_ref().map(|it| it.raw()),
4178        };
4179
4180        let raw =
4181            unsafe { self.raw().create_compute_pipeline(&pipeline_desc) }.map_err(
4182                |err| match err {
4183                    hal::PipelineError::Device(error) => {
4184                        pipeline::CreateComputePipelineError::Device(self.handle_hal_error(error))
4185                    }
4186                    hal::PipelineError::Linkage(_stages, msg) => {
4187                        pipeline::CreateComputePipelineError::Internal(msg)
4188                    }
4189                    hal::PipelineError::EntryPoint(_stage) => {
4190                        pipeline::CreateComputePipelineError::Internal(
4191                            ENTRYPOINT_FAILURE_ERROR.to_string(),
4192                        )
4193                    }
4194                    hal::PipelineError::PipelineConstants(_stages, msg) => {
4195                        pipeline::CreateComputePipelineError::PipelineConstants(msg)
4196                    }
4197                },
4198            )?;
4199
4200        let immediate_slots_required =
4201            shader_module
4202                .interface
4203                .interface()
4204                .map_or(Default::default(), |iface| {
4205                    iface.immediate_slots_required(
4206                        naga::ShaderStage::Compute,
4207                        &final_entry_point_name,
4208                    )
4209                });
4210
4211        let pipeline = pipeline::ComputePipeline {
4212            state: ResourceState::Valid(pipeline::ComputePipelineState {
4213                raw: ManuallyDrop::new(raw),
4214                layout: pipeline_layout.clone(),
4215                _shader_module: shader_module,
4216            }),
4217            device: self.clone(),
4218            late_sized_buffer_groups,
4219            immediate_slots_required,
4220            label: desc.label.to_string(),
4221            tracking_data: TrackingData::new(self.tracker_indices.compute_pipelines.clone()),
4222        };
4223
4224        let pipeline = Arc::new(pipeline);
4225
4226        if is_auto_layout {
4227            for bgl in pipeline_layout.bind_group_layouts.iter() {
4228                let Some(bgl) = bgl else {
4229                    continue;
4230                };
4231
4232                // `bind_group_layouts` might contain duplicate entries, so we need to ignore the
4233                // result.
4234                let _ = bgl.exclusive_pipeline.set((&pipeline).into());
4235            }
4236        }
4237
4238        Ok(pipeline)
4239    }
4240
4241    pub fn create_render_pipeline(
4242        self: &Arc<Self>,
4243        desc: pipeline::ResolvedGeneralRenderPipelineDescriptor,
4244    ) -> (
4245        Arc<pipeline::RenderPipeline>,
4246        Option<pipeline::CreateRenderPipelineError>,
4247    ) {
4248        let (render_pipeline, error) = match self.create_render_pipeline_inner(desc.clone()) {
4249            Ok(pipeline) => (pipeline, None),
4250            Err(e) => (
4251                pipeline::RenderPipeline::invalid(self.clone(), desc.label.to_string()),
4252                Some(e),
4253            ),
4254        };
4255        #[cfg(feature = "trace")]
4256        if let Some(ref mut trace) = *self.trace.lock() {
4257            use crate::device::trace::IntoTrace;
4258            trace.add(trace::Action::CreateGeneralRenderPipeline {
4259                id: render_pipeline.to_trace(),
4260                desc: desc.to_trace(),
4261            });
4262        }
4263        (render_pipeline, error)
4264    }
4265
4266    pub fn create_render_pipeline_inner(
4267        self: &Arc<Self>,
4268        desc: pipeline::ResolvedGeneralRenderPipelineDescriptor,
4269    ) -> Result<Arc<pipeline::RenderPipeline>, pipeline::CreateRenderPipelineError> {
4270        use wgt::TextureFormatFeatureFlags as Tfff;
4271
4272        self.check_is_valid()?;
4273
4274        let mut shader_binding_sizes = FastHashMap::default();
4275
4276        let color_targets = desc
4277            .fragment
4278            .as_ref()
4279            .map_or(&[][..], |fragment| &fragment.targets);
4280        let depth_stencil_state = desc.depth_stencil.as_ref();
4281
4282        check_color_attachment_count(color_targets.len(), self.limits.max_color_attachments)?;
4283
4284        {
4285            let cts: ArrayVec<_, { hal::MAX_COLOR_ATTACHMENTS }> =
4286                color_targets.iter().filter_map(|x| x.as_ref()).collect();
4287            if !cts.is_empty() && {
4288                let first = &cts[0];
4289                cts[1..]
4290                    .iter()
4291                    .any(|ct| ct.write_mask != first.write_mask || ct.blend != first.blend)
4292            } {
4293                self.require_downlevel_flags(wgt::DownlevelFlags::INDEPENDENT_BLEND)?;
4294            }
4295        }
4296
4297        let mut io = validation::StageIo::default();
4298        let mut validated_stages = wgt::ShaderStages::empty();
4299
4300        let mut vertex_steps;
4301        let mut hal_vertex_buffer_layouts;
4302        let mut total_attributes;
4303        let mut dual_source_blending = false;
4304        let mut has_depth_attachment = false;
4305        if let pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) = desc.vertex {
4306            if vertex.buffers.len() > self.limits.max_vertex_buffers as usize {
4307                return Err(pipeline::CreateRenderPipelineError::TooManyVertexBuffers {
4308                    given: vertex.buffers.len() as u32,
4309                    limit: self.limits.max_vertex_buffers,
4310                });
4311            }
4312
4313            vertex_steps = Vec::with_capacity(vertex.buffers.len());
4314            hal_vertex_buffer_layouts = Vec::with_capacity(vertex.buffers.len());
4315            total_attributes = 0;
4316            for (i, vb_state) in vertex.buffers.iter().enumerate() {
4317                let Some(vb_state) = vb_state else {
4318                    vertex_steps.push(None);
4319                    hal_vertex_buffer_layouts.push(None);
4320                    continue;
4321                };
4322
4323                // https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-gpuvertexbufferlayout
4324
4325                if vb_state.array_stride > self.limits.max_vertex_buffer_array_stride as u64 {
4326                    return Err(pipeline::CreateRenderPipelineError::VertexStrideTooLarge {
4327                        index: i as u32,
4328                        given: vb_state.array_stride as u32,
4329                        limit: self.limits.max_vertex_buffer_array_stride,
4330                    });
4331                }
4332                if vb_state.array_stride % wgt::VERTEX_ALIGNMENT != 0 {
4333                    return Err(pipeline::CreateRenderPipelineError::UnalignedVertexStride {
4334                        index: i as u32,
4335                        stride: vb_state.array_stride,
4336                    });
4337                }
4338
4339                let max_stride = if vb_state.array_stride == 0 {
4340                    self.limits.max_vertex_buffer_array_stride as u64
4341                } else {
4342                    vb_state.array_stride
4343                };
4344                let mut last_stride = 0;
4345                for attribute in vb_state.attributes.iter() {
4346                    let attribute_stride = attribute.offset + attribute.format.size();
4347                    if attribute_stride > max_stride {
4348                        return Err(
4349                            pipeline::CreateRenderPipelineError::VertexAttributeStrideTooLarge {
4350                                location: attribute.shader_location,
4351                                given: attribute_stride as u32,
4352                                limit: max_stride as u32,
4353                            },
4354                        );
4355                    }
4356
4357                    let required_offset_alignment = attribute.format.size().min(4);
4358                    if attribute.offset % required_offset_alignment != 0 {
4359                        return Err(
4360                            pipeline::CreateRenderPipelineError::InvalidVertexAttributeOffset {
4361                                location: attribute.shader_location,
4362                                offset: attribute.offset,
4363                            },
4364                        );
4365                    }
4366
4367                    if attribute.shader_location >= self.limits.max_vertex_attributes {
4368                        return Err(
4369                            pipeline::CreateRenderPipelineError::VertexAttributeLocationTooLarge {
4370                                given: attribute.shader_location,
4371                                limit: self.limits.max_vertex_attributes,
4372                            },
4373                        );
4374                    }
4375
4376                    last_stride = last_stride.max(attribute_stride);
4377                }
4378
4379                vertex_steps.push(Some(pipeline::VertexStep {
4380                    stride: vb_state.array_stride,
4381                    last_stride,
4382                    mode: vb_state.step_mode,
4383                }));
4384                hal_vertex_buffer_layouts.push(if vb_state.attributes.is_empty() {
4385                    None
4386                } else {
4387                    Some(hal::VertexBufferLayout {
4388                        array_stride: vb_state.array_stride,
4389                        step_mode: vb_state.step_mode,
4390                        attributes: vb_state.attributes.as_ref(),
4391                    })
4392                });
4393
4394                for attribute in vb_state.attributes.iter() {
4395                    if attribute.offset >= 0x10000000 {
4396                        return Err(
4397                            pipeline::CreateRenderPipelineError::InvalidVertexAttributeOffset {
4398                                location: attribute.shader_location,
4399                                offset: attribute.offset,
4400                            },
4401                        );
4402                    }
4403
4404                    if let wgt::VertexFormat::Float64
4405                    | wgt::VertexFormat::Float64x2
4406                    | wgt::VertexFormat::Float64x3
4407                    | wgt::VertexFormat::Float64x4 = attribute.format
4408                    {
4409                        self.require_features(wgt::Features::VERTEX_ATTRIBUTE_64BIT)?;
4410                    }
4411
4412                    let previous = io.varyings.insert(
4413                        attribute.shader_location,
4414                        validation::InterfaceVar::vertex_attribute(attribute.format),
4415                    );
4416
4417                    if previous.is_some() {
4418                        return Err(pipeline::CreateRenderPipelineError::ShaderLocationClash(
4419                            attribute.shader_location,
4420                        ));
4421                    }
4422                }
4423                total_attributes += vb_state.attributes.len();
4424            }
4425
4426            if total_attributes > self.limits.max_vertex_attributes as usize {
4427                return Err(
4428                    pipeline::CreateRenderPipelineError::TooManyVertexAttributes {
4429                        given: total_attributes as u32,
4430                        limit: self.limits.max_vertex_attributes,
4431                    },
4432                );
4433            }
4434        } else {
4435            vertex_steps = Vec::new();
4436            hal_vertex_buffer_layouts = Vec::new();
4437        };
4438
4439        if desc.primitive.strip_index_format.is_some() && !desc.primitive.topology.is_strip() {
4440            return Err(
4441                pipeline::CreateRenderPipelineError::StripIndexFormatForNonStripTopology {
4442                    strip_index_format: desc.primitive.strip_index_format,
4443                    topology: desc.primitive.topology,
4444                },
4445            );
4446        }
4447
4448        if desc.primitive.unclipped_depth {
4449            self.require_features(wgt::Features::DEPTH_CLIP_CONTROL)?;
4450        }
4451
4452        if desc.primitive.polygon_mode == wgt::PolygonMode::Line {
4453            self.require_features(wgt::Features::POLYGON_MODE_LINE)?;
4454        }
4455        if desc.primitive.polygon_mode == wgt::PolygonMode::Point {
4456            self.require_features(wgt::Features::POLYGON_MODE_POINT)?;
4457        }
4458
4459        if desc.primitive.conservative {
4460            self.require_features(wgt::Features::CONSERVATIVE_RASTERIZATION)?;
4461        }
4462
4463        if desc.primitive.conservative && desc.primitive.polygon_mode != wgt::PolygonMode::Fill {
4464            return Err(
4465                pipeline::CreateRenderPipelineError::ConservativeRasterizationNonFillPolygonMode,
4466            );
4467        }
4468
4469        let mut target_specified = false;
4470
4471        for (i, cs) in color_targets.iter().enumerate() {
4472            if let Some(cs) = cs.as_ref() {
4473                target_specified = true;
4474                let error = 'error: {
4475                    // This is expected to be the operative check for illegal write mask
4476                    // values (larger than 15), because WebGPU requires that it be validated
4477                    // on the device timeline.
4478                    if cs.write_mask.contains_unknown_bits() {
4479                        break 'error Some(ColorStateError::InvalidWriteMask(cs.write_mask));
4480                    }
4481
4482                    let format_features = self.describe_format_features(cs.format)?;
4483                    if !format_features
4484                        .allowed_usages
4485                        .contains(wgt::TextureUsages::RENDER_ATTACHMENT)
4486                    {
4487                        break 'error Some(ColorStateError::FormatNotRenderable(cs.format));
4488                    }
4489                    if cs.blend.is_some() && !format_features.flags.contains(Tfff::BLENDABLE) {
4490                        break 'error Some(ColorStateError::FormatNotBlendable(cs.format));
4491                    }
4492                    if !hal::FormatAspects::from(cs.format).contains(hal::FormatAspects::COLOR) {
4493                        break 'error Some(ColorStateError::FormatNotColor(cs.format));
4494                    }
4495
4496                    if desc.multisample.count > 1
4497                        && !format_features
4498                            .flags
4499                            .sample_count_supported(desc.multisample.count)
4500                    {
4501                        break 'error Some(ColorStateError::InvalidSampleCount(
4502                            desc.multisample.count,
4503                            cs.format,
4504                            cs.format
4505                                .guaranteed_format_features(self.features)
4506                                .flags
4507                                .supported_sample_counts(),
4508                            self.adapter
4509                                .get_texture_format_features(cs.format)
4510                                .flags
4511                                .supported_sample_counts(),
4512                        ));
4513                    }
4514
4515                    if let Some(blend_mode) = cs.blend {
4516                        for component in [&blend_mode.color, &blend_mode.alpha] {
4517                            for factor in [component.src_factor, component.dst_factor] {
4518                                if factor.ref_second_blend_source() {
4519                                    self.require_features(wgt::Features::DUAL_SOURCE_BLENDING)?;
4520                                    if i == 0 {
4521                                        dual_source_blending = true;
4522                                    } else {
4523                                        break 'error Some(
4524                                            ColorStateError::BlendFactorOnUnsupportedTarget {
4525                                                factor,
4526                                                target: i as u32,
4527                                            },
4528                                        );
4529                                    }
4530                                }
4531
4532                                if [wgt::BlendOperation::Min, wgt::BlendOperation::Max]
4533                                    .contains(&component.operation)
4534                                    && factor != wgt::BlendFactor::One
4535                                {
4536                                    break 'error Some(ColorStateError::InvalidMinMaxBlendFactor {
4537                                        factor,
4538                                        target: i as u32,
4539                                    });
4540                                }
4541                            }
4542                        }
4543                    }
4544
4545                    break 'error None;
4546                };
4547                if let Some(e) = error {
4548                    return Err(pipeline::CreateRenderPipelineError::ColorState(i as u8, e));
4549                }
4550            }
4551        }
4552
4553        if dual_source_blending && color_targets.len() > 1 {
4554            return Err(
4555                pipeline::CreateRenderPipelineError::DualSourceBlendingWithMultipleColorTargets {
4556                    count: color_targets.len(),
4557                },
4558            );
4559        }
4560
4561        validation::validate_color_attachment_bytes_per_sample(
4562            color_targets.iter().flatten().map(|cs| cs.format),
4563            self.limits.max_color_attachment_bytes_per_sample,
4564        )
4565        .map_err(pipeline::CreateRenderPipelineError::ColorAttachment)?;
4566
4567        if let Some(ds) = depth_stencil_state {
4568            // See <https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-gpudepthstencilstate>.
4569            target_specified = true;
4570            let error = 'error: {
4571                if !ds.format.is_depth_stencil_format() {
4572                    // This error case is not redundant with the aspect check below when
4573                    // neither depth nor stencil is enabled at all.
4574                    break 'error Some(pipeline::DepthStencilStateError::FormatNotDepthOrStencil(
4575                        ds.format,
4576                    ));
4577                }
4578
4579                let format_features = self.describe_format_features(ds.format)?;
4580                if !format_features
4581                    .allowed_usages
4582                    .contains(wgt::TextureUsages::RENDER_ATTACHMENT)
4583                {
4584                    break 'error Some(pipeline::DepthStencilStateError::FormatNotRenderable(
4585                        ds.format,
4586                    ));
4587                }
4588
4589                let aspect = hal::FormatAspects::from(ds.format);
4590                if aspect.contains(hal::FormatAspects::DEPTH) {
4591                    has_depth_attachment = true;
4592                } else if ds.is_depth_enabled() {
4593                    break 'error Some(pipeline::DepthStencilStateError::FormatNotDepth(ds.format));
4594                }
4595                if has_depth_attachment {
4596                    let Some(depth_write_enabled) = ds.depth_write_enabled else {
4597                        break 'error Some(
4598                            pipeline::DepthStencilStateError::MissingDepthWriteEnabled(ds.format),
4599                        );
4600                    };
4601
4602                    let depth_compare_required = depth_write_enabled
4603                        || ds.stencil.front.depth_fail_op != wgt::StencilOperation::Keep
4604                        || ds.stencil.back.depth_fail_op != wgt::StencilOperation::Keep;
4605                    if depth_compare_required && ds.depth_compare.is_none() {
4606                        break 'error Some(pipeline::DepthStencilStateError::MissingDepthCompare(
4607                            ds.format,
4608                        ));
4609                    }
4610                }
4611
4612                if ds.stencil.is_enabled() && !aspect.contains(hal::FormatAspects::STENCIL) {
4613                    break 'error Some(pipeline::DepthStencilStateError::FormatNotStencil(
4614                        ds.format,
4615                    ));
4616                }
4617                if desc.multisample.count > 1
4618                    && !format_features
4619                        .flags
4620                        .sample_count_supported(desc.multisample.count)
4621                {
4622                    break 'error Some(pipeline::DepthStencilStateError::InvalidSampleCount(
4623                        desc.multisample.count,
4624                        ds.format,
4625                        ds.format
4626                            .guaranteed_format_features(self.features)
4627                            .flags
4628                            .supported_sample_counts(),
4629                        self.adapter
4630                            .get_texture_format_features(ds.format)
4631                            .flags
4632                            .supported_sample_counts(),
4633                    ));
4634                }
4635
4636                break 'error None;
4637            };
4638            if let Some(e) = error {
4639                return Err(pipeline::CreateRenderPipelineError::DepthStencilState(e));
4640            }
4641
4642            if ds.bias.clamp != 0.0 {
4643                self.require_downlevel_flags(wgt::DownlevelFlags::DEPTH_BIAS_CLAMP)?;
4644            }
4645
4646            if (ds.bias.is_enabled() || ds.bias.clamp != 0.0)
4647                && !desc.primitive.topology.is_triangles()
4648            {
4649                return Err(pipeline::CreateRenderPipelineError::DepthStencilState(
4650                    pipeline::DepthStencilStateError::DepthBiasWithIncompatibleTopology(
4651                        desc.primitive.topology,
4652                    ),
4653                ));
4654            }
4655        }
4656
4657        if !target_specified {
4658            return Err(pipeline::CreateRenderPipelineError::NoTargetSpecified);
4659        }
4660
4661        let is_auto_layout = desc.layout.is_none();
4662
4663        // Get the pipeline layout from the desc if it is provided.
4664        let pipeline_layout = match desc.layout {
4665            Some(pipeline_layout) => {
4666                pipeline_layout.same_device(self)?;
4667                pipeline_layout.check_valid()?;
4668                Some(pipeline_layout)
4669            }
4670            None => None,
4671        };
4672
4673        let mut binding_layout_source = match pipeline_layout {
4674            Some(pipeline_layout) => validation::BindingLayoutSource::Provided(pipeline_layout),
4675            None => validation::BindingLayoutSource::new_derived(&self.limits),
4676        };
4677
4678        let samples = {
4679            let sc = desc.multisample.count;
4680            if sc == 0 || sc > 32 || !sc.is_power_of_two() {
4681                return Err(pipeline::CreateRenderPipelineError::InvalidSampleCount(sc));
4682            }
4683            sc
4684        };
4685
4686        let mut vertex_stage = None;
4687        let mut task_stage = None;
4688        let mut mesh_stage = None;
4689        let mut _vertex_entry_point_name = String::new();
4690        let mut _task_entry_point_name = String::new();
4691        let mut _mesh_entry_point_name = String::new();
4692        let mut immediate_slots_required = naga::valid::ImmediateSlots::default();
4693        match desc.vertex {
4694            pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) => {
4695                vertex_stage = {
4696                    let stage_desc = &vertex.stage;
4697                    let stage = validation::ShaderStageForValidation::Vertex {
4698                        topology: desc.primitive.topology,
4699                        compare_function: desc.depth_stencil.as_ref().and_then(|d| d.depth_compare),
4700                    };
4701                    let stage_bit = stage.to_wgt_bit();
4702
4703                    let vertex_shader_module = &stage_desc.module;
4704                    vertex_shader_module.same_device(self)?;
4705
4706                    let stage_err = |error| pipeline::CreateRenderPipelineError::Stage {
4707                        stage: stage_bit,
4708                        error,
4709                    };
4710
4711                    _vertex_entry_point_name = vertex_shader_module
4712                        .finalize_entry_point_name(
4713                            stage.to_naga(),
4714                            stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4715                        )
4716                        .map_err(stage_err)?;
4717
4718                    if let Some(interface) = vertex_shader_module.interface.interface() {
4719                        immediate_slots_required |= interface
4720                            .immediate_slots_required(stage.to_naga(), &_vertex_entry_point_name);
4721                        io = interface
4722                            .check_stage(
4723                                &mut binding_layout_source,
4724                                &mut shader_binding_sizes,
4725                                &_vertex_entry_point_name,
4726                                stage,
4727                                io,
4728                                Some(desc.primitive.topology),
4729                            )
4730                            .map_err(stage_err)?;
4731                        validated_stages |= stage_bit;
4732                    }
4733                    Some(hal::ProgrammableStage {
4734                        module: vertex_shader_module.raw(),
4735                        entry_point: &_vertex_entry_point_name,
4736                        constants: &stage_desc.constants,
4737                        zero_initialize_workgroup_memory: stage_desc
4738                            .zero_initialize_workgroup_memory,
4739                    })
4740                };
4741            }
4742            pipeline::RenderPipelineVertexProcessor::Mesh(ref task, ref mesh) => {
4743                self.require_features(wgt::Features::EXPERIMENTAL_MESH_SHADER)?;
4744
4745                task_stage = if let Some(task) = task {
4746                    let stage_desc = &task.stage;
4747                    let stage = validation::ShaderStageForValidation::Task;
4748                    let stage_bit = stage.to_wgt_bit();
4749                    let task_shader_module = &stage_desc.module;
4750                    task_shader_module.same_device(self)?;
4751
4752                    let stage_err = |error| pipeline::CreateRenderPipelineError::Stage {
4753                        stage: stage_bit,
4754                        error,
4755                    };
4756
4757                    _task_entry_point_name = task_shader_module
4758                        .finalize_entry_point_name(
4759                            stage.to_naga(),
4760                            stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4761                        )
4762                        .map_err(stage_err)?;
4763
4764                    if let Some(interface) = task_shader_module.interface.interface() {
4765                        immediate_slots_required |= interface
4766                            .immediate_slots_required(stage.to_naga(), &_task_entry_point_name);
4767                        io = interface
4768                            .check_stage(
4769                                &mut binding_layout_source,
4770                                &mut shader_binding_sizes,
4771                                &_task_entry_point_name,
4772                                stage,
4773                                io,
4774                                Some(desc.primitive.topology),
4775                            )
4776                            .map_err(stage_err)?;
4777                        validated_stages |= stage_bit;
4778                    }
4779                    Some(hal::ProgrammableStage {
4780                        module: task_shader_module.raw(),
4781                        entry_point: &_task_entry_point_name,
4782                        constants: &stage_desc.constants,
4783                        zero_initialize_workgroup_memory: stage_desc
4784                            .zero_initialize_workgroup_memory,
4785                    })
4786                } else {
4787                    None
4788                };
4789                mesh_stage = {
4790                    let stage_desc = &mesh.stage;
4791                    let stage = validation::ShaderStageForValidation::Mesh;
4792                    let stage_bit = stage.to_wgt_bit();
4793                    let mesh_shader_module = &stage_desc.module;
4794                    mesh_shader_module.same_device(self)?;
4795
4796                    let stage_err = |error| pipeline::CreateRenderPipelineError::Stage {
4797                        stage: stage_bit,
4798                        error,
4799                    };
4800
4801                    _mesh_entry_point_name = mesh_shader_module
4802                        .finalize_entry_point_name(
4803                            stage.to_naga(),
4804                            stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4805                        )
4806                        .map_err(stage_err)?;
4807
4808                    if let Some(interface) = mesh_shader_module.interface.interface() {
4809                        immediate_slots_required |= interface
4810                            .immediate_slots_required(stage.to_naga(), &_mesh_entry_point_name);
4811                        io = interface
4812                            .check_stage(
4813                                &mut binding_layout_source,
4814                                &mut shader_binding_sizes,
4815                                &_mesh_entry_point_name,
4816                                stage,
4817                                io,
4818                                Some(desc.primitive.topology),
4819                            )
4820                            .map_err(stage_err)?;
4821                        validated_stages |= stage_bit;
4822                    }
4823                    Some(hal::ProgrammableStage {
4824                        module: mesh_shader_module.raw(),
4825                        entry_point: &_mesh_entry_point_name,
4826                        constants: &stage_desc.constants,
4827                        zero_initialize_workgroup_memory: stage_desc
4828                            .zero_initialize_workgroup_memory,
4829                    })
4830                };
4831            }
4832        }
4833
4834        let fragment_entry_point_name;
4835        let fragment_stage = match desc.fragment {
4836            Some(ref fragment_state) => {
4837                let stage = validation::ShaderStageForValidation::Fragment {
4838                    dual_source_blending,
4839                    has_depth_attachment,
4840                };
4841                let stage_bit = stage.to_wgt_bit();
4842
4843                let shader_module = &fragment_state.stage.module;
4844                shader_module.same_device(self)?;
4845
4846                let stage_err = |error| pipeline::CreateRenderPipelineError::Stage {
4847                    stage: stage_bit,
4848                    error,
4849                };
4850
4851                fragment_entry_point_name = shader_module
4852                    .finalize_entry_point_name(
4853                        stage.to_naga(),
4854                        fragment_state
4855                            .stage
4856                            .entry_point
4857                            .as_ref()
4858                            .map(|ep| ep.as_ref()),
4859                    )
4860                    .map_err(stage_err)?;
4861
4862                if let Some(interface) = shader_module.interface.interface() {
4863                    immediate_slots_required |= interface
4864                        .immediate_slots_required(stage.to_naga(), &fragment_entry_point_name);
4865                    io = interface
4866                        .check_stage(
4867                            &mut binding_layout_source,
4868                            &mut shader_binding_sizes,
4869                            &fragment_entry_point_name,
4870                            stage,
4871                            io,
4872                            Some(desc.primitive.topology),
4873                        )
4874                        .map_err(stage_err)?;
4875                    validated_stages |= stage_bit;
4876                }
4877
4878                Some(hal::ProgrammableStage {
4879                    module: shader_module.raw(),
4880                    entry_point: &fragment_entry_point_name,
4881                    constants: &fragment_state.stage.constants,
4882                    zero_initialize_workgroup_memory: fragment_state
4883                        .stage
4884                        .zero_initialize_workgroup_memory,
4885                })
4886            }
4887            None => None,
4888        };
4889
4890        if validated_stages.contains(wgt::ShaderStages::FRAGMENT) {
4891            for (i, output) in io.varyings.iter() {
4892                match color_targets.get(*i as usize) {
4893                    Some(Some(state)) => {
4894                        validation::check_texture_format(state.format, &output.ty).map_err(
4895                            |pipeline| {
4896                                pipeline::CreateRenderPipelineError::ColorState(
4897                                    *i as u8,
4898                                    ColorStateError::IncompatibleFormat {
4899                                        pipeline,
4900                                        shader: output.ty,
4901                                    },
4902                                )
4903                            },
4904                        )?;
4905                    }
4906                    _ => {
4907                        log::debug!(
4908                            "The fragment stage {:?} output @location({}) values are ignored",
4909                            fragment_stage
4910                                .as_ref()
4911                                .map_or("", |stage| stage.entry_point),
4912                            i
4913                        );
4914                    }
4915                }
4916            }
4917        }
4918        let last_stage = match desc.fragment {
4919            Some(_) => wgt::ShaderStages::FRAGMENT,
4920            None => wgt::ShaderStages::VERTEX,
4921        };
4922        if is_auto_layout && !validated_stages.contains(last_stage) {
4923            return Err(pipeline::ImplicitLayoutError::ReflectionError(last_stage).into());
4924        }
4925
4926        let pipeline_layout = match binding_layout_source {
4927            validation::BindingLayoutSource::Provided(pipeline_layout) => pipeline_layout,
4928            validation::BindingLayoutSource::Derived(entries) => {
4929                let immediate_size = {
4930                    let immediate_size_of = |sm: &pipeline::ShaderModule| {
4931                        sm.interface.interface().map(|i| i.immediate_size)
4932                    };
4933                    let vertex = match desc.vertex {
4934                        pipeline::RenderPipelineVertexProcessor::Vertex(ref v) => {
4935                            immediate_size_of(&v.stage.module)
4936                        }
4937                        pipeline::RenderPipelineVertexProcessor::Mesh(ref task, ref mesh) => task
4938                            .as_ref()
4939                            .and_then(|t| immediate_size_of(&t.stage.module))
4940                            .max(immediate_size_of(&mesh.stage.module)),
4941                    };
4942                    let fragment = desc
4943                        .fragment
4944                        .as_ref()
4945                        .and_then(|f| immediate_size_of(&f.stage.module));
4946                    vertex.max(fragment).unwrap_or(0)
4947                };
4948                self.create_derived_pipeline_layout(entries, immediate_size)?
4949            }
4950        };
4951
4952        if let pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) = desc.vertex {
4953            let bind_groups_plus_vertex_buffers =
4954                u32::try_from(pipeline_layout.bind_group_layouts.len() + vertex.buffers.len())
4955                    .unwrap();
4956            if bind_groups_plus_vertex_buffers > self.limits.max_bind_groups_plus_vertex_buffers {
4957                return Err(
4958                    pipeline::CreateRenderPipelineError::TooManyBindGroupsPlusVertexBuffers {
4959                        given: bind_groups_plus_vertex_buffers,
4960                        limit: self.limits.max_bind_groups_plus_vertex_buffers,
4961                    },
4962                );
4963            }
4964
4965            let given = pipeline_layout
4966                .buffers_and_acceleration_structures_in_vertex_stage
4967                .saturating_add(vertex.buffers.len() as u32);
4968            if !self
4969                .instance_flags
4970                .contains(wgt::InstanceFlags::STRICT_WEBGPU_COMPLIANCE)
4971            {
4972                let limit = self
4973                    .limits
4974                    .max_buffers_and_acceleration_structures_per_shader_stage;
4975                if given > limit {
4976                    return Err(
4977                    pipeline::CreateRenderPipelineError::TooManyBuffersAndAccelerationStructuresInVertexStage {
4978                        given,
4979                        limit,
4980                    },
4981                );
4982                }
4983            }
4984        }
4985
4986        // Multiview is only supported if the feature is enabled
4987        if let Some(mv_mask) = desc.multiview_mask {
4988            self.require_features(wgt::Features::MULTIVIEW)?;
4989            if !(mv_mask.get() + 1).is_power_of_two() {
4990                self.require_features(wgt::Features::SELECTIVE_MULTIVIEW)?;
4991            }
4992        }
4993
4994        if !self
4995            .downlevel
4996            .flags
4997            .contains(wgt::DownlevelFlags::BUFFER_BINDINGS_NOT_16_BYTE_ALIGNED)
4998        {
4999            for (binding, size) in shader_binding_sizes.iter() {
5000                if size.get() % 16 != 0 {
5001                    return Err(pipeline::CreateRenderPipelineError::UnalignedShader {
5002                        binding: binding.binding,
5003                        group: binding.group,
5004                        size: size.get(),
5005                    });
5006                }
5007            }
5008        }
5009
5010        let late_sized_buffer_groups =
5011            Device::make_late_sized_buffer_groups(&shader_binding_sizes, &pipeline_layout);
5012
5013        let cache = match desc.cache {
5014            Some(cache) => {
5015                cache.same_device(self)?;
5016                Some(cache)
5017            }
5018            None => None,
5019        };
5020
5021        let is_mesh = mesh_stage.is_some();
5022        let has_task_shader = task_stage.is_some();
5023        let raw = {
5024            let pipeline_desc = hal::RenderPipelineDescriptor {
5025                label: desc.label.to_hal(self.instance_flags),
5026                layout: pipeline_layout.raw()?,
5027                vertex_processor: match vertex_stage {
5028                    Some(vertex_stage) => hal::VertexProcessor::Standard {
5029                        vertex_buffers: &hal_vertex_buffer_layouts,
5030                        vertex_stage,
5031                    },
5032                    None => hal::VertexProcessor::Mesh {
5033                        task_stage,
5034                        mesh_stage: mesh_stage.unwrap(),
5035                    },
5036                },
5037                primitive: desc.primitive,
5038                depth_stencil: desc.depth_stencil.clone(),
5039                multisample: desc.multisample,
5040                fragment_stage,
5041                color_targets,
5042                multiview_mask: desc.multiview_mask,
5043                cache: cache.as_ref().map(|it| it.raw()),
5044            };
5045            unsafe { self.raw().create_render_pipeline(&pipeline_desc) }.map_err(
5046                |err| match err {
5047                    hal::PipelineError::Device(error) => {
5048                        pipeline::CreateRenderPipelineError::Device(self.handle_hal_error(error))
5049                    }
5050                    hal::PipelineError::Linkage(stage, msg) => {
5051                        pipeline::CreateRenderPipelineError::Internal { stage, error: msg }
5052                    }
5053                    hal::PipelineError::EntryPoint(stage) => {
5054                        pipeline::CreateRenderPipelineError::Internal {
5055                            stage: hal::auxil::map_naga_stage(stage),
5056                            error: ENTRYPOINT_FAILURE_ERROR.to_string(),
5057                        }
5058                    }
5059                    hal::PipelineError::PipelineConstants(stage, error) => {
5060                        pipeline::CreateRenderPipelineError::PipelineConstants { stage, error }
5061                    }
5062                },
5063            )?
5064        };
5065
5066        let pass_context = RenderPassContext {
5067            attachments: AttachmentData {
5068                colors: color_targets
5069                    .iter()
5070                    .map(|state| state.as_ref().map(|s| s.format))
5071                    .collect(),
5072                resolves: ArrayVec::new(),
5073                depth_stencil: depth_stencil_state.as_ref().map(|state| state.format),
5074            },
5075            sample_count: samples,
5076            multiview_mask: desc.multiview_mask,
5077        };
5078
5079        let mut flags = pipeline::PipelineFlags::empty();
5080        for state in color_targets.iter().filter_map(|s| s.as_ref()) {
5081            if let Some(ref bs) = state.blend {
5082                if bs.color.uses_constant() | bs.alpha.uses_constant() {
5083                    flags |= pipeline::PipelineFlags::BLEND_CONSTANT;
5084                }
5085            }
5086        }
5087        if let Some(ds) = depth_stencil_state.as_ref() {
5088            if ds.stencil.is_enabled() && ds.stencil.needs_ref_value() {
5089                flags |= pipeline::PipelineFlags::STENCIL_REFERENCE;
5090            }
5091            if !ds.is_depth_read_only() {
5092                flags |= pipeline::PipelineFlags::WRITES_DEPTH;
5093            }
5094            if !ds.is_stencil_read_only(desc.primitive.cull_mode) {
5095                flags |= pipeline::PipelineFlags::WRITES_STENCIL;
5096            }
5097        }
5098        // A Vulkan pipeline needs none of its shader modules once it is
5099        // made, and a module made from naga IR keeps that IR in memory, so
5100        // on Vulkan the pipeline lets its modules go and the caller's
5101        // handles alone keep them.
5102        let shader_modules = if self.backend() == wgt::Backend::Vulkan {
5103            ArrayVec::new()
5104        } else {
5105            let mut shader_modules = ArrayVec::new();
5106            match desc.vertex {
5107                pipeline::RenderPipelineVertexProcessor::Vertex(vertex) => {
5108                    shader_modules.push(vertex.stage.module)
5109                }
5110                pipeline::RenderPipelineVertexProcessor::Mesh(task, mesh) => {
5111                    if let Some(task) = task {
5112                        shader_modules.push(task.stage.module);
5113                    }
5114                    shader_modules.push(mesh.stage.module);
5115                }
5116            }
5117            shader_modules.extend(desc.fragment.map(|f| f.stage.module));
5118            shader_modules
5119        };
5120
5121        let pipeline = pipeline::RenderPipeline {
5122            state: ResourceState::Valid(pipeline::RenderPipelineState {
5123                raw: ManuallyDrop::new(raw),
5124                layout: pipeline_layout.clone(),
5125            }),
5126            device: self.clone(),
5127            pass_context,
5128            _shader_modules: shader_modules,
5129            flags,
5130            topology: desc.primitive.topology,
5131            strip_index_format: desc.primitive.strip_index_format,
5132            vertex_steps,
5133            late_sized_buffer_groups,
5134            immediate_slots_required,
5135            label: desc.label.to_string(),
5136            tracking_data: TrackingData::new(self.tracker_indices.render_pipelines.clone()),
5137            is_mesh,
5138            has_task_shader,
5139        };
5140
5141        let pipeline = Arc::new(pipeline);
5142
5143        if is_auto_layout {
5144            for bgl in pipeline_layout.bind_group_layouts.iter() {
5145                let Some(bgl) = bgl else {
5146                    continue;
5147                };
5148
5149                // `bind_group_layouts` might contain duplicate entries, so we need to ignore the
5150                // result.
5151                let _ = bgl.exclusive_pipeline.set((&pipeline).into());
5152            }
5153        }
5154
5155        Ok(pipeline)
5156    }
5157
5158    /// # Safety
5159    /// The `data` field on `desc` must have previously been returned from
5160    /// [`crate::global::Global::pipeline_cache_get_data`]
5161    pub unsafe fn create_pipeline_cache(
5162        self: &Arc<Self>,
5163        desc: &pipeline::PipelineCacheDescriptor,
5164    ) -> Result<Arc<pipeline::PipelineCache>, pipeline::CreatePipelineCacheError> {
5165        use crate::pipeline_cache;
5166
5167        self.check_is_valid()?;
5168
5169        self.require_features(wgt::Features::PIPELINE_CACHE)?;
5170        let data = if let Some((data, validation_key)) = desc
5171            .data
5172            .as_ref()
5173            .zip(self.raw().pipeline_cache_validation_key())
5174        {
5175            let data = pipeline_cache::validate_pipeline_cache(
5176                data,
5177                &self.adapter.raw.info,
5178                validation_key,
5179            );
5180            match data {
5181                Ok(data) => Some(data),
5182                Err(e) if e.was_avoidable() || !desc.fallback => return Err(e.into()),
5183                // If the error was unavoidable and we are asked to fallback, do so
5184                Err(_) => None,
5185            }
5186        } else {
5187            None
5188        };
5189        let cache_desc = hal::PipelineCacheDescriptor {
5190            data,
5191            label: desc.label.to_hal(self.instance_flags),
5192        };
5193        let raw = match unsafe { self.raw().create_pipeline_cache(&cache_desc) } {
5194            Ok(raw) => raw,
5195            Err(e) => match e {
5196                hal::PipelineCacheError::Device(e) => return Err(self.handle_hal_error(e).into()),
5197            },
5198        };
5199        let cache = pipeline::PipelineCache {
5200            device: self.clone(),
5201            label: desc.label.to_string(),
5202            // This would be none in the error condition, which we don't implement yet
5203            raw: ManuallyDrop::new(raw),
5204        };
5205
5206        let cache = Arc::new(cache);
5207
5208        Ok(cache)
5209    }
5210
5211    fn get_texture_format_features(&self, format: TextureFormat) -> wgt::TextureFormatFeatures {
5212        // Variant of adapter.get_texture_format_features that takes device features into account
5213        use wgt::TextureFormatFeatureFlags as tfsc;
5214        let mut format_features = self.adapter.get_texture_format_features(format);
5215        if (format == TextureFormat::R32Float
5216            || format == TextureFormat::Rg32Float
5217            || format == TextureFormat::Rgba32Float)
5218            && !self.features.contains(wgt::Features::FLOAT32_FILTERABLE)
5219        {
5220            format_features.flags.set(tfsc::FILTERABLE, false);
5221        }
5222        format_features
5223    }
5224
5225    pub(crate) fn describe_format_features(
5226        &self,
5227        format: TextureFormat,
5228    ) -> Result<wgt::TextureFormatFeatures, MissingFeatures> {
5229        self.require_features(format.required_features())?;
5230
5231        let using_device_features = self
5232            .features
5233            .contains(wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES);
5234        // If we're running downlevel, we need to manually ask the backend what
5235        // we can use as we can't trust WebGPU.
5236        let downlevel = !self
5237            .downlevel
5238            .flags
5239            .contains(wgt::DownlevelFlags::WEBGPU_TEXTURE_FORMAT_SUPPORT);
5240
5241        if using_device_features || downlevel {
5242            Ok(self.get_texture_format_features(format))
5243        } else {
5244            Ok(format.guaranteed_format_features(self.features))
5245        }
5246    }
5247
5248    #[cfg(feature = "replay")]
5249    pub(crate) fn wait_for_submit(
5250        &self,
5251        submission_index: crate::SubmissionIndex,
5252    ) -> Result<(), DeviceError> {
5253        let last_done_index = unsafe { self.raw().get_fence_value(self.fence.as_ref()) }
5254            .map_err(|e| self.handle_hal_error(e))?;
5255        if last_done_index < submission_index {
5256            unsafe { self.raw().wait(self.fence.as_ref(), submission_index, None) }
5257                .map_err(|e| self.handle_hal_error(e))?;
5258            if let Some(queue) = self.get_queue() {
5259                let closures = queue.lock_life().triage_submissions(submission_index);
5260                assert!(
5261                    closures.is_empty(),
5262                    "wait_for_submit is not expected to work with closures"
5263                );
5264            }
5265        }
5266        Ok(())
5267    }
5268
5269    pub fn create_query_set(
5270        self: &Arc<Self>,
5271        desc: &resource::QuerySetDescriptor,
5272    ) -> Result<Arc<QuerySet>, resource::CreateQuerySetError> {
5273        use resource::CreateQuerySetError as Error;
5274
5275        self.check_is_valid()?;
5276
5277        match desc.ty {
5278            wgt::QueryType::Occlusion => {}
5279            wgt::QueryType::Timestamp => {
5280                self.require_features(wgt::Features::TIMESTAMP_QUERY)?;
5281            }
5282            wgt::QueryType::PipelineStatistics(..) => {
5283                self.require_features(wgt::Features::PIPELINE_STATISTICS_QUERY)?;
5284            }
5285        }
5286
5287        if desc.count == 0 {
5288            return Err(Error::ZeroCount);
5289        }
5290
5291        if desc.count > wgt::QUERY_SET_MAX_QUERIES {
5292            return Err(Error::TooManyQueries {
5293                count: desc.count,
5294                maximum: wgt::QUERY_SET_MAX_QUERIES,
5295            });
5296        }
5297
5298        let hal_desc = desc.map_label(|label| label.to_hal(self.instance_flags));
5299
5300        let raw = unsafe { self.raw().create_query_set(&hal_desc) }
5301            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
5302
5303        let query_set = QuerySet {
5304            raw: Snatchable::new(raw),
5305            device: self.clone(),
5306            label: desc.label.to_string(),
5307            tracking_data: TrackingData::new(self.tracker_indices.query_sets.clone()),
5308            desc: desc.map_label(|_| ()),
5309            initialized_slots: Mutex::new(
5310                rank::QUERY_SET_INITIALIZED_SLOTS,
5311                bit_vec::BitVec::from_elem(desc.count as usize, false),
5312            ),
5313        };
5314
5315        let query_set = Arc::new(query_set);
5316
5317        Ok(query_set)
5318    }
5319
5320    pub fn configure_surface(
5321        self: &Arc<Self>,
5322        surface: &crate::instance::Surface,
5323        config: &wgt::SurfaceConfiguration<Vec<TextureFormat>>,
5324    ) -> Option<present::ConfigureSurfaceError> {
5325        use present::ConfigureSurfaceError as E;
5326        profiling::scope!("surface_configure");
5327
5328        log::debug!("configuring surface with {config:?}");
5329
5330        let error = 'error: {
5331            // User callbacks must not be called while we are holding locks.
5332            let user_callbacks;
5333            {
5334                if let Err(e) = self.check_is_valid() {
5335                    break 'error e.into();
5336                }
5337
5338                let caps = match surface.get_capabilities(&self.adapter) {
5339                    Ok(caps) => caps,
5340                    Err(_) => break 'error E::UnsupportedQueueFamily,
5341                };
5342
5343                let mut hal_view_formats = Vec::new();
5344                for format in config.view_formats.iter() {
5345                    if *format == config.format {
5346                        continue;
5347                    }
5348                    if !caps.formats.iter().any(|fc| fc.format == config.format) {
5349                        break 'error E::UnsupportedFormat {
5350                            requested: config.format,
5351                            available: caps.texture_formats().collect(),
5352                        };
5353                    }
5354                    if config.format.remove_srgb_suffix() != format.remove_srgb_suffix() {
5355                        break 'error E::InvalidViewFormat(*format, config.format);
5356                    }
5357                    hal_view_formats.push(*format);
5358                }
5359
5360                if !hal_view_formats.is_empty() {
5361                    if let Err(missing_flag) =
5362                        self.require_downlevel_flags(wgt::DownlevelFlags::SURFACE_VIEW_FORMATS)
5363                    {
5364                        break 'error E::MissingDownlevelFlags(missing_flag);
5365                    }
5366                }
5367
5368                let maximum_frame_latency = config.desired_maximum_frame_latency.clamp(
5369                    *caps.maximum_frame_latency.start(),
5370                    *caps.maximum_frame_latency.end(),
5371                );
5372                let mut hal_config = hal::SurfaceConfiguration {
5373                    maximum_frame_latency,
5374                    present_mode: config.present_mode,
5375                    composite_alpha_mode: config.alpha_mode,
5376                    format: config.format,
5377                    color_space: config.color_space,
5378                    extent: wgt::Extent3d {
5379                        width: config.width,
5380                        height: config.height,
5381                        depth_or_array_layers: 1,
5382                    },
5383                    usage: conv::map_texture_usage(
5384                        config.usage,
5385                        hal::FormatAspects::COLOR,
5386                        wgt::TextureFormatFeatureFlags::STORAGE_READ_ONLY
5387                            | wgt::TextureFormatFeatureFlags::STORAGE_WRITE_ONLY
5388                            | wgt::TextureFormatFeatureFlags::STORAGE_READ_WRITE,
5389                    ),
5390                    view_formats: hal_view_formats,
5391                    keep_presented: surface
5392                        .keep_images
5393                        .load(Ordering::Acquire),
5394                };
5395
5396                if let Err(error) = validate_surface_configuration(
5397                    &mut hal_config,
5398                    &caps,
5399                    self.limits.max_texture_dimension_2d,
5400                ) {
5401                    break 'error error;
5402                }
5403
5404                // Wait for all work to finish before configuring the surface.
5405                let snatch_guard = self.snatchable_lock.read();
5406
5407                let maintain_result;
5408                (user_callbacks, maintain_result) =
5409                    self.maintain(wgt::PollType::wait_indefinitely(), snatch_guard);
5410
5411                match maintain_result {
5412                    // Every submission made before this call has finished. A queue that is
5413                    // not empty means another thread submitted since, which happens when
5414                    // an engine renders on its own thread with this device. That work
5415                    // cannot use this surface's textures: none is acquired, which is
5416                    // checked below, and a presented one cannot be submitted. Upstream
5417                    // wgpu accepts both results too.
5418                    Ok(wgt::PollStatus::QueueEmpty | wgt::PollStatus::WaitSucceeded) => {}
5419                    Ok(wgt::PollStatus::Poll) => {
5420                        unreachable!("Cannot get a Poll result from a Wait action.")
5421                    }
5422                    Err(WaitIdleError::Timeout) if cfg!(target_arch = "wasm32") => {
5423                        // On wasm, you cannot actually successfully wait for the surface.
5424                        // However WebGL does not actually require you do this, so ignoring
5425                        // the failure is totally fine. See
5426                        // https://github.com/gfx-rs/wgpu/issues/7363
5427                    }
5428                    Err(e) => {
5429                        break 'error e.into();
5430                    }
5431                }
5432
5433                // All textures must be destroyed before the surface can be re-configured.
5434                if let Some(present) = surface.presentation.lock().take() {
5435                    if present.acquired_texture.is_some() {
5436                        break 'error E::PreviousOutputExists;
5437                    }
5438                }
5439
5440                // TODO: Texture views may still be alive that point to the texture.
5441                // this will allow the user to render to the surface texture, long after
5442                // it has been removed.
5443                //
5444                // https://github.com/gfx-rs/wgpu/issues/4105
5445
5446                let surface_raw = surface.raw(self.backend()).unwrap();
5447                match unsafe { surface_raw.configure(self.raw(), &hal_config) } {
5448                    Ok(()) => (),
5449                    Err(error) => {
5450                        break 'error match error {
5451                            hal::SurfaceError::Outdated
5452                            | hal::SurfaceError::Lost
5453                            | hal::SurfaceError::Occluded
5454                            | hal::SurfaceError::Timeout => E::InvalidSurface,
5455                            hal::SurfaceError::Device(error) => {
5456                                E::Device(self.handle_hal_error(error))
5457                            }
5458                            hal::SurfaceError::Other(message) => {
5459                                log::error!("surface configuration failed: {message}");
5460                                E::InvalidSurface
5461                            }
5462                        }
5463                    }
5464                }
5465
5466                let mut presentation = surface.presentation.lock();
5467                *presentation = Some(present::Presentation {
5468                    device: Arc::clone(self),
5469                    config: config.clone(),
5470                    acquired_texture: None,
5471                    acquired_image: None,
5472                    images: present::PresentedImages::default(),
5473                });
5474            }
5475
5476            user_callbacks.fire();
5477            return None;
5478        };
5479
5480        Some(error)
5481    }
5482
5483    fn lose(&self, message: &str) {
5484        // Follow the steps at https://gpuweb.github.io/gpuweb/#lose-the-device.
5485
5486        // Mark the device explicitly as invalid. This is checked in various
5487        // places to prevent new work from being submitted.
5488        self.valid.store(false, Ordering::Release);
5489
5490        // 1) Resolve the GPUDevice device.lost promise.
5491        if let Some(device_lost_closure) = self.device_lost_closure.lock().take() {
5492            device_lost_closure(DeviceLostReason::Unknown, message.to_string());
5493        }
5494
5495        // 2) Complete any outstanding mapAsync() steps.
5496        // 3) Complete any outstanding onSubmittedWorkDone() steps.
5497
5498        // These parts are passively accomplished by setting valid to false,
5499        // since that will prevent any new work from being added to the queues.
5500        // Future calls to poll_devices will continue to check the work queues
5501        // until they are cleared, and then drop the device.
5502    }
5503
5504    fn release_gpu_resources(&self) {
5505        // This is called when the device is lost, which makes every associated
5506        // resource invalid and unusable. This is an opportunity to release all of
5507        // the underlying gpu resources, even though the objects remain visible to
5508        // the user agent. We purge this memory naturally when resources have been
5509        // moved into the appropriate buckets, so this function just needs to
5510        // initiate movement into those buckets, and it can do that by calling
5511        // "destroy" on all the resources we know about.
5512
5513        // During these iterations, we discard all errors. We don't care!
5514        let trackers = self.trackers.lock();
5515        for buffer in trackers.buffers.used_resources() {
5516            if let Some(buffer) = Weak::upgrade(buffer) {
5517                buffer.destroy();
5518            }
5519        }
5520        for texture in trackers.textures.used_resources() {
5521            if let Some(texture) = Weak::upgrade(texture) {
5522                texture.destroy();
5523            }
5524        }
5525    }
5526
5527    pub(crate) fn new_usage_scope(&self) -> UsageScope<'_> {
5528        UsageScope::new_pooled(
5529            &self.usage_scopes,
5530            &self.tracker_indices,
5531            self.ordered_buffer_usages,
5532            self.ordered_texture_usages,
5533        )
5534    }
5535
5536    pub fn get_hal_counters(&self) -> wgt::HalCounters {
5537        self.raw().get_internal_counters()
5538    }
5539
5540    pub fn generate_allocator_report(&self) -> Option<wgt::AllocatorReport> {
5541        self.raw().generate_allocator_report()
5542    }
5543}
5544
5545crate::impl_resource_type!(Device);
5546crate::impl_labeled!(Device);
5547crate::impl_storage_item!(Device);