wgpu_hal/vulkan/
command.rs

1use super::conv;
2use arrayvec::ArrayVec;
3use ash::vk;
4use core::{mem, ops::Range};
5use hashbrown::hash_map::Entry;
6
7const ALLOCATION_GRANULARITY: u32 = 16;
8const DST_IMAGE_LAYOUT: vk::ImageLayout = vk::ImageLayout::TRANSFER_DST_OPTIMAL;
9
10impl super::Texture {
11    fn map_buffer_copies<T>(&self, regions: T) -> impl Iterator<Item = vk::BufferImageCopy>
12    where
13        T: Iterator<Item = crate::BufferTextureCopy>,
14    {
15        let (block_width, block_height) = self.format.block_dimensions();
16        let format = self.format;
17        let copy_size = self.copy_size;
18        regions.map(move |r| {
19            let extent = r.texture_base.max_copy_size(&copy_size).min(&r.size);
20            let (image_subresource, image_offset) = conv::map_subresource_layers(&r.texture_base);
21            vk::BufferImageCopy {
22                buffer_offset: r.buffer_layout.offset,
23                buffer_row_length: r.buffer_layout.bytes_per_row.map_or(0, |bpr| {
24                    let block_size = format
25                        .block_copy_size(Some(r.texture_base.aspect.map()))
26                        .unwrap();
27                    block_width * (bpr / block_size)
28                }),
29                buffer_image_height: r
30                    .buffer_layout
31                    .rows_per_image
32                    .map_or(0, |rpi| rpi * block_height),
33                image_subresource,
34                image_offset,
35                image_extent: conv::map_copy_extent(&extent),
36            }
37        })
38    }
39}
40
41impl super::CommandEncoder {
42    fn write_pass_end_timestamp_if_requested(&mut self) {
43        if let Some((query_set, index)) = self.end_of_pass_timer_query.take() {
44            unsafe {
45                self.device.raw.cmd_write_timestamp(
46                    self.active,
47                    vk::PipelineStageFlags::BOTTOM_OF_PIPE,
48                    query_set,
49                    index,
50                );
51            }
52        }
53    }
54
55    fn make_framebuffer(
56        &mut self,
57        key: super::FramebufferKey,
58    ) -> Result<vk::Framebuffer, crate::DeviceError> {
59        Ok(match self.framebuffers.entry(key) {
60            Entry::Occupied(e) => *e.get(),
61            Entry::Vacant(e) => {
62                let super::FramebufferKey {
63                    raw_pass,
64                    ref attachments,
65                    extent,
66                } = *e.key();
67
68                let vk_info = vk::FramebufferCreateInfo::default()
69                    .render_pass(raw_pass)
70                    .width(extent.width)
71                    .height(extent.height)
72                    .layers(extent.depth_or_array_layers)
73                    .attachments(attachments);
74
75                let raw = unsafe { self.device.raw.create_framebuffer(&vk_info, None).unwrap() };
76                *e.insert(raw)
77            }
78        })
79    }
80
81    fn make_temp_texture_view(
82        &mut self,
83        key: super::TempTextureViewKey,
84    ) -> Result<vk::ImageView, crate::DeviceError> {
85        Ok(match self.temp_texture_views.entry(key) {
86            Entry::Occupied(e) => *e.get(),
87            Entry::Vacant(e) => {
88                let super::TempTextureViewKey {
89                    texture,
90                    format,
91                    mip_level,
92                    depth_slice,
93                } = *e.key();
94
95                let vk_info = vk::ImageViewCreateInfo::default()
96                    .image(texture)
97                    .view_type(vk::ImageViewType::TYPE_2D)
98                    .format(format)
99                    .subresource_range(vk::ImageSubresourceRange {
100                        aspect_mask: vk::ImageAspectFlags::COLOR,
101                        base_mip_level: mip_level,
102                        level_count: 1,
103                        base_array_layer: depth_slice,
104                        layer_count: 1,
105                    });
106                let raw = unsafe { self.device.raw.create_image_view(&vk_info, None) }
107                    .map_err(super::map_host_device_oom_and_ioca_err)?;
108                *e.insert(raw)
109            }
110        })
111    }
112}
113
114impl crate::CommandEncoder for super::CommandEncoder {
115    type A = super::Api;
116
117    unsafe fn begin_encoding(&mut self, label: crate::Label) -> Result<(), crate::DeviceError> {
118        if self.free.is_empty() {
119            let vk_info = vk::CommandBufferAllocateInfo::default()
120                .command_pool(self.raw)
121                .command_buffer_count(ALLOCATION_GRANULARITY);
122            let cmd_buf_vec = unsafe {
123                self.device
124                    .raw
125                    .allocate_command_buffers(&vk_info)
126                    .map_err(super::map_host_device_oom_err)?
127            };
128            self.free.extend(cmd_buf_vec);
129        }
130        let raw = self.free.pop().unwrap();
131
132        // Set the name unconditionally, since there might be a
133        // previous name assigned to this.
134        unsafe { self.device.set_object_name(raw, label.unwrap_or_default()) };
135
136        // Reset this in case the last renderpass was never ended.
137        self.rpass_debug_marker_active = false;
138
139        let vk_info = vk::CommandBufferBeginInfo::default()
140            .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT);
141        unsafe { self.device.raw.begin_command_buffer(raw, &vk_info) }
142            .map_err(super::map_host_device_oom_err)?;
143        self.active = raw;
144
145        Ok(())
146    }
147
148    unsafe fn end_encoding(&mut self) -> Result<super::CommandBuffer, crate::DeviceError> {
149        let raw = self.active;
150        self.active = vk::CommandBuffer::null();
151        unsafe { self.device.raw.end_command_buffer(raw) }.map_err(map_err)?;
152        fn map_err(err: vk::Result) -> crate::DeviceError {
153            // We don't use VK_KHR_video_encode_queue
154            // VK_ERROR_INVALID_VIDEO_STD_PARAMETERS_KHR
155            super::map_host_device_oom_err(err)
156        }
157        Ok(super::CommandBuffer { raw })
158    }
159
160    unsafe fn discard_encoding(&mut self) {
161        // Safe use requires this is not called in the "closed" state, so the buffer
162        // shouldn't be null. Assert this to make sure we're not pushing null
163        // buffers to the discard pile.
164        assert_ne!(self.active, vk::CommandBuffer::null());
165
166        self.discarded.push(self.active);
167        self.active = vk::CommandBuffer::null();
168    }
169
170    unsafe fn reset_all<I>(&mut self, cmd_bufs: I)
171    where
172        I: Iterator<Item = super::CommandBuffer>,
173    {
174        self.temp.clear();
175        self.free
176            .extend(cmd_bufs.into_iter().map(|cmd_buf| cmd_buf.raw));
177        self.free.append(&mut self.discarded);
178        let _ = unsafe {
179            self.device
180                .raw
181                .reset_command_pool(self.raw, vk::CommandPoolResetFlags::default())
182        };
183    }
184
185    unsafe fn transition_buffers<'a, T>(&mut self, barriers: T)
186    where
187        T: Iterator<Item = crate::BufferBarrier<'a, super::Buffer>>,
188    {
189        //Note: this is done so that we never end up with empty stage flags
190        let mut src_stages = vk::PipelineStageFlags::TOP_OF_PIPE;
191        let mut dst_stages = vk::PipelineStageFlags::BOTTOM_OF_PIPE;
192        let vk_barriers = &mut self.temp.buffer_barriers;
193        vk_barriers.clear();
194
195        for bar in barriers {
196            let (src_stage, src_access) = conv::map_buffer_usage_to_barrier(bar.usage.from);
197            src_stages |= src_stage;
198            let (dst_stage, dst_access) = conv::map_buffer_usage_to_barrier(bar.usage.to);
199            dst_stages |= dst_stage;
200
201            vk_barriers.push(
202                vk::BufferMemoryBarrier::default()
203                    .buffer(bar.buffer.raw)
204                    .size(vk::WHOLE_SIZE)
205                    .src_access_mask(src_access)
206                    .dst_access_mask(dst_access),
207            )
208        }
209
210        if !vk_barriers.is_empty() {
211            unsafe {
212                self.device.raw.cmd_pipeline_barrier(
213                    self.active,
214                    src_stages,
215                    dst_stages,
216                    vk::DependencyFlags::empty(),
217                    &[],
218                    vk_barriers,
219                    &[],
220                )
221            };
222        }
223    }
224
225    unsafe fn transition_textures<'a, T>(&mut self, barriers: T)
226    where
227        T: Iterator<Item = crate::TextureBarrier<'a, super::Texture>>,
228    {
229        let mut src_stages = vk::PipelineStageFlags::empty();
230        let mut dst_stages = vk::PipelineStageFlags::empty();
231        let vk_barriers = &mut self.temp.image_barriers;
232        vk_barriers.clear();
233
234        for bar in barriers {
235            let range = conv::map_subresource_range_combined_aspect(
236                &bar.range,
237                bar.texture.format,
238                &self.device.private_caps,
239            );
240            let (src_stage, src_access) = conv::map_texture_usage_to_barrier(bar.usage.from);
241            let src_layout = conv::derive_image_layout(bar.usage.from, bar.texture.format);
242            src_stages |= src_stage;
243            let (dst_stage, dst_access) = conv::map_texture_usage_to_barrier(bar.usage.to);
244            let dst_layout = conv::derive_image_layout(bar.usage.to, bar.texture.format);
245            dst_stages |= dst_stage;
246
247            vk_barriers.push(
248                vk::ImageMemoryBarrier::default()
249                    .image(bar.texture.raw)
250                    .subresource_range(range)
251                    .src_access_mask(src_access)
252                    .dst_access_mask(dst_access)
253                    .old_layout(src_layout)
254                    .new_layout(dst_layout),
255            );
256        }
257
258        if !vk_barriers.is_empty() {
259            unsafe {
260                self.device.raw.cmd_pipeline_barrier(
261                    self.active,
262                    src_stages,
263                    dst_stages,
264                    vk::DependencyFlags::empty(),
265                    &[],
266                    &[],
267                    vk_barriers,
268                )
269            };
270        }
271    }
272
273    unsafe fn clear_buffer(&mut self, buffer: &super::Buffer, range: crate::MemoryRange) {
274        let range_size = range.end - range.start;
275        if self.device.workarounds.contains(
276            super::Workarounds::FORCE_FILL_BUFFER_WITH_SIZE_GREATER_4096_ALIGNED_OFFSET_16,
277        ) && range_size >= 4096
278            && range.start % 16 != 0
279        {
280            let rounded_start = wgt::math::align_to(range.start, 16);
281            let prefix_size = rounded_start - range.start;
282
283            unsafe {
284                self.device.raw.cmd_fill_buffer(
285                    self.active,
286                    buffer.raw,
287                    range.start,
288                    prefix_size,
289                    0,
290                )
291            };
292
293            // This will never be zero, as rounding can only add up to 12 bytes, and the total size is 4096.
294            let suffix_size = range.end - rounded_start;
295
296            unsafe {
297                self.device.raw.cmd_fill_buffer(
298                    self.active,
299                    buffer.raw,
300                    rounded_start,
301                    suffix_size,
302                    0,
303                )
304            };
305        } else {
306            unsafe {
307                self.device
308                    .raw
309                    .cmd_fill_buffer(self.active, buffer.raw, range.start, range_size, 0)
310            };
311        }
312    }
313
314    unsafe fn copy_buffer_to_buffer<T>(
315        &mut self,
316        src: &super::Buffer,
317        dst: &super::Buffer,
318        regions: T,
319    ) where
320        T: Iterator<Item = crate::BufferCopy>,
321    {
322        let vk_regions_iter = regions.map(|r| vk::BufferCopy {
323            src_offset: r.src_offset,
324            dst_offset: r.dst_offset,
325            size: r.size.get(),
326        });
327
328        unsafe {
329            self.device.raw.cmd_copy_buffer(
330                self.active,
331                src.raw,
332                dst.raw,
333                &smallvec::SmallVec::<[vk::BufferCopy; 32]>::from_iter(vk_regions_iter),
334            )
335        };
336    }
337
338    unsafe fn copy_texture_to_texture<T>(
339        &mut self,
340        src: &super::Texture,
341        src_usage: wgt::TextureUses,
342        dst: &super::Texture,
343        regions: T,
344    ) where
345        T: Iterator<Item = crate::TextureCopy>,
346    {
347        let src_layout = conv::derive_image_layout(src_usage, src.format);
348
349        let vk_regions_iter = regions.map(|r| {
350            let (src_subresource, src_offset) = conv::map_subresource_layers(&r.src_base);
351            let (dst_subresource, dst_offset) = conv::map_subresource_layers(&r.dst_base);
352            let extent = r
353                .size
354                .min(&r.src_base.max_copy_size(&src.copy_size))
355                .min(&r.dst_base.max_copy_size(&dst.copy_size));
356            vk::ImageCopy {
357                src_subresource,
358                src_offset,
359                dst_subresource,
360                dst_offset,
361                extent: conv::map_copy_extent(&extent),
362            }
363        });
364
365        unsafe {
366            self.device.raw.cmd_copy_image(
367                self.active,
368                src.raw,
369                src_layout,
370                dst.raw,
371                DST_IMAGE_LAYOUT,
372                &smallvec::SmallVec::<[vk::ImageCopy; 32]>::from_iter(vk_regions_iter),
373            )
374        };
375    }
376
377    unsafe fn copy_buffer_to_texture<T>(
378        &mut self,
379        src: &super::Buffer,
380        dst: &super::Texture,
381        regions: T,
382    ) where
383        T: Iterator<Item = crate::BufferTextureCopy>,
384    {
385        let vk_regions_iter = dst.map_buffer_copies(regions);
386
387        unsafe {
388            self.device.raw.cmd_copy_buffer_to_image(
389                self.active,
390                src.raw,
391                dst.raw,
392                DST_IMAGE_LAYOUT,
393                &smallvec::SmallVec::<[vk::BufferImageCopy; 32]>::from_iter(vk_regions_iter),
394            )
395        };
396    }
397
398    unsafe fn copy_texture_to_buffer<T>(
399        &mut self,
400        src: &super::Texture,
401        src_usage: wgt::TextureUses,
402        dst: &super::Buffer,
403        regions: T,
404    ) where
405        T: Iterator<Item = crate::BufferTextureCopy>,
406    {
407        let src_layout = conv::derive_image_layout(src_usage, src.format);
408        let vk_regions_iter = src.map_buffer_copies(regions);
409
410        unsafe {
411            self.device.raw.cmd_copy_image_to_buffer(
412                self.active,
413                src.raw,
414                src_layout,
415                dst.raw,
416                &smallvec::SmallVec::<[vk::BufferImageCopy; 32]>::from_iter(vk_regions_iter),
417            )
418        };
419    }
420
421    unsafe fn begin_query(&mut self, set: &super::QuerySet, index: u32) {
422        unsafe {
423            self.device.raw.cmd_begin_query(
424                self.active,
425                set.raw,
426                index,
427                vk::QueryControlFlags::empty(),
428            )
429        };
430    }
431    unsafe fn end_query(&mut self, set: &super::QuerySet, index: u32) {
432        unsafe { self.device.raw.cmd_end_query(self.active, set.raw, index) };
433    }
434    unsafe fn write_timestamp(&mut self, set: &super::QuerySet, index: u32) {
435        unsafe {
436            self.device.raw.cmd_write_timestamp(
437                self.active,
438                vk::PipelineStageFlags::BOTTOM_OF_PIPE,
439                set.raw,
440                index,
441            )
442        };
443    }
444    unsafe fn read_acceleration_structure_compact_size(
445        &mut self,
446        acceleration_structure: &super::AccelerationStructure,
447        buffer: &super::Buffer,
448    ) {
449        let ray_tracing_functions = self
450            .device
451            .extension_fns
452            .ray_tracing
453            .as_ref()
454            .expect("Feature `RAY_TRACING` not enabled");
455        let query_pool = acceleration_structure
456            .compacted_size_query
457            .as_ref()
458            .unwrap();
459        unsafe {
460            self.device
461                .raw
462                .cmd_reset_query_pool(self.active, *query_pool, 0, 1);
463            ray_tracing_functions
464                .acceleration_structure
465                .cmd_write_acceleration_structures_properties(
466                    self.active,
467                    &[acceleration_structure.raw],
468                    vk::QueryType::ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR,
469                    *query_pool,
470                    0,
471                );
472            self.device.raw.cmd_copy_query_pool_results(
473                self.active,
474                *query_pool,
475                0,
476                1,
477                buffer.raw,
478                0,
479                wgt::QUERY_SIZE as vk::DeviceSize,
480                vk::QueryResultFlags::TYPE_64 | vk::QueryResultFlags::WAIT,
481            )
482        };
483    }
484    unsafe fn reset_queries(&mut self, set: &super::QuerySet, range: Range<u32>) {
485        unsafe {
486            self.device.raw.cmd_reset_query_pool(
487                self.active,
488                set.raw,
489                range.start,
490                range.end - range.start,
491            )
492        };
493    }
494    unsafe fn copy_query_results(
495        &mut self,
496        set: &super::QuerySet,
497        range: Range<u32>,
498        buffer: &super::Buffer,
499        offset: wgt::BufferAddress,
500        stride: wgt::BufferSize,
501    ) {
502        unsafe {
503            self.device.raw.cmd_copy_query_pool_results(
504                self.active,
505                set.raw,
506                range.start,
507                range.end - range.start,
508                buffer.raw,
509                offset,
510                stride.get(),
511                vk::QueryResultFlags::TYPE_64 | vk::QueryResultFlags::WAIT,
512            )
513        };
514    }
515
516    unsafe fn build_acceleration_structures<'a, T>(&mut self, descriptor_count: u32, descriptors: T)
517    where
518        super::Api: 'a,
519        T: IntoIterator<
520            Item = crate::BuildAccelerationStructureDescriptor<
521                'a,
522                super::Buffer,
523                super::AccelerationStructure,
524            >,
525        >,
526    {
527        const CAPACITY_OUTER: usize = 8;
528        const CAPACITY_INNER: usize = 1;
529        let descriptor_count = descriptor_count as usize;
530
531        let ray_tracing_functions = self
532            .device
533            .extension_fns
534            .ray_tracing
535            .as_ref()
536            .expect("Feature `RAY_TRACING` not enabled");
537
538        let get_device_address = |buffer: Option<&super::Buffer>| unsafe {
539            match buffer {
540                Some(buffer) => ray_tracing_functions
541                    .buffer_device_address
542                    .get_buffer_device_address(
543                        &vk::BufferDeviceAddressInfo::default().buffer(buffer.raw),
544                    ),
545                None => panic!("Buffers are required to build acceleration structures"),
546            }
547        };
548
549        // storage to all the data required for cmd_build_acceleration_structures
550        let mut ranges_storage = smallvec::SmallVec::<
551            [smallvec::SmallVec<[vk::AccelerationStructureBuildRangeInfoKHR; CAPACITY_INNER]>;
552                CAPACITY_OUTER],
553        >::with_capacity(descriptor_count);
554        let mut geometries_storage = smallvec::SmallVec::<
555            [smallvec::SmallVec<[vk::AccelerationStructureGeometryKHR; CAPACITY_INNER]>;
556                CAPACITY_OUTER],
557        >::with_capacity(descriptor_count);
558
559        // pointers to all the data required for cmd_build_acceleration_structures
560        let mut geometry_infos = smallvec::SmallVec::<
561            [vk::AccelerationStructureBuildGeometryInfoKHR; CAPACITY_OUTER],
562        >::with_capacity(descriptor_count);
563        let mut ranges_ptrs = smallvec::SmallVec::<
564            [&[vk::AccelerationStructureBuildRangeInfoKHR]; CAPACITY_OUTER],
565        >::with_capacity(descriptor_count);
566
567        for desc in descriptors {
568            let (geometries, ranges) = match *desc.entries {
569                crate::AccelerationStructureEntries::Instances(ref instances) => {
570                    let instance_data = vk::AccelerationStructureGeometryInstancesDataKHR::default(
571                    // TODO: Code is so large that rustfmt refuses to treat this... :(
572                    )
573                    .data(vk::DeviceOrHostAddressConstKHR {
574                        device_address: get_device_address(instances.buffer),
575                    });
576
577                    let geometry = vk::AccelerationStructureGeometryKHR::default()
578                        .geometry_type(vk::GeometryTypeKHR::INSTANCES)
579                        .geometry(vk::AccelerationStructureGeometryDataKHR {
580                            instances: instance_data,
581                        });
582
583                    let range = vk::AccelerationStructureBuildRangeInfoKHR::default()
584                        .primitive_count(instances.count)
585                        .primitive_offset(instances.offset);
586
587                    (smallvec::smallvec![geometry], smallvec::smallvec![range])
588                }
589                crate::AccelerationStructureEntries::Triangles(ref in_geometries) => {
590                    let mut ranges = smallvec::SmallVec::<
591                        [vk::AccelerationStructureBuildRangeInfoKHR; CAPACITY_INNER],
592                    >::with_capacity(in_geometries.len());
593                    let mut geometries = smallvec::SmallVec::<
594                        [vk::AccelerationStructureGeometryKHR; CAPACITY_INNER],
595                    >::with_capacity(in_geometries.len());
596                    for triangles in in_geometries {
597                        let mut triangle_data =
598                            vk::AccelerationStructureGeometryTrianglesDataKHR::default()
599                                // IndexType::NONE_KHR is not set by default (due to being provided by VK_KHR_acceleration_structure) but unless there is an
600                                // index buffer we need to have IndexType::NONE_KHR as our index type.
601                                .index_type(vk::IndexType::NONE_KHR)
602                                .vertex_data(vk::DeviceOrHostAddressConstKHR {
603                                    device_address: get_device_address(triangles.vertex_buffer),
604                                })
605                                .vertex_format(conv::map_vertex_format(triangles.vertex_format))
606                                .max_vertex(triangles.vertex_count)
607                                .vertex_stride(triangles.vertex_stride);
608
609                        let mut range = vk::AccelerationStructureBuildRangeInfoKHR::default();
610
611                        if let Some(ref indices) = triangles.indices {
612                            triangle_data = triangle_data
613                                .index_data(vk::DeviceOrHostAddressConstKHR {
614                                    device_address: get_device_address(indices.buffer),
615                                })
616                                .index_type(conv::map_index_format(indices.format));
617
618                            range = range
619                                .primitive_count(indices.count / 3)
620                                .primitive_offset(indices.offset)
621                                .first_vertex(triangles.first_vertex);
622                        } else {
623                            range = range
624                                .primitive_count(triangles.vertex_count)
625                                .first_vertex(triangles.first_vertex);
626                        }
627
628                        if let Some(ref transform) = triangles.transform {
629                            let transform_device_address = unsafe {
630                                ray_tracing_functions
631                                    .buffer_device_address
632                                    .get_buffer_device_address(
633                                        &vk::BufferDeviceAddressInfo::default()
634                                            .buffer(transform.buffer.raw),
635                                    )
636                            };
637                            triangle_data =
638                                triangle_data.transform_data(vk::DeviceOrHostAddressConstKHR {
639                                    device_address: transform_device_address,
640                                });
641
642                            range = range.transform_offset(transform.offset);
643                        }
644
645                        let geometry = vk::AccelerationStructureGeometryKHR::default()
646                            .geometry_type(vk::GeometryTypeKHR::TRIANGLES)
647                            .geometry(vk::AccelerationStructureGeometryDataKHR {
648                                triangles: triangle_data,
649                            })
650                            .flags(conv::map_acceleration_structure_geometry_flags(
651                                triangles.flags,
652                            ));
653
654                        geometries.push(geometry);
655                        ranges.push(range);
656                    }
657                    (geometries, ranges)
658                }
659                crate::AccelerationStructureEntries::AABBs(ref in_geometries) => {
660                    let mut ranges = smallvec::SmallVec::<
661                        [vk::AccelerationStructureBuildRangeInfoKHR; CAPACITY_INNER],
662                    >::with_capacity(in_geometries.len());
663                    let mut geometries = smallvec::SmallVec::<
664                        [vk::AccelerationStructureGeometryKHR; CAPACITY_INNER],
665                    >::with_capacity(in_geometries.len());
666                    for aabb in in_geometries {
667                        let aabbs_data = vk::AccelerationStructureGeometryAabbsDataKHR::default()
668                            .data(vk::DeviceOrHostAddressConstKHR {
669                                device_address: get_device_address(aabb.buffer),
670                            })
671                            .stride(aabb.stride);
672
673                        let range = vk::AccelerationStructureBuildRangeInfoKHR::default()
674                            .primitive_count(aabb.count)
675                            .primitive_offset(aabb.offset);
676
677                        let geometry = vk::AccelerationStructureGeometryKHR::default()
678                            .geometry_type(vk::GeometryTypeKHR::AABBS)
679                            .geometry(vk::AccelerationStructureGeometryDataKHR {
680                                aabbs: aabbs_data,
681                            })
682                            .flags(conv::map_acceleration_structure_geometry_flags(aabb.flags));
683
684                        geometries.push(geometry);
685                        ranges.push(range);
686                    }
687                    (geometries, ranges)
688                }
689            };
690
691            ranges_storage.push(ranges);
692            geometries_storage.push(geometries);
693
694            let scratch_device_address = unsafe {
695                ray_tracing_functions
696                    .buffer_device_address
697                    .get_buffer_device_address(
698                        &vk::BufferDeviceAddressInfo::default().buffer(desc.scratch_buffer.raw),
699                    )
700            };
701            let ty = match *desc.entries {
702                crate::AccelerationStructureEntries::Instances(_) => {
703                    vk::AccelerationStructureTypeKHR::TOP_LEVEL
704                }
705                _ => vk::AccelerationStructureTypeKHR::BOTTOM_LEVEL,
706            };
707            let mut geometry_info = vk::AccelerationStructureBuildGeometryInfoKHR::default()
708                .ty(ty)
709                .mode(conv::map_acceleration_structure_build_mode(desc.mode))
710                .flags(conv::map_acceleration_structure_flags(desc.flags))
711                .dst_acceleration_structure(desc.destination_acceleration_structure.raw)
712                .scratch_data(vk::DeviceOrHostAddressKHR {
713                    device_address: scratch_device_address + desc.scratch_buffer_offset,
714                });
715
716            if desc.mode == crate::AccelerationStructureBuildMode::Update {
717                geometry_info.src_acceleration_structure = desc
718                    .source_acceleration_structure
719                    .unwrap_or(desc.destination_acceleration_structure)
720                    .raw;
721            }
722
723            geometry_infos.push(geometry_info);
724        }
725
726        for (i, geometry_info) in geometry_infos.iter_mut().enumerate() {
727            geometry_info.geometry_count = geometries_storage[i].len() as u32;
728            geometry_info.p_geometries = geometries_storage[i].as_ptr();
729            ranges_ptrs.push(&ranges_storage[i]);
730        }
731
732        unsafe {
733            ray_tracing_functions
734                .acceleration_structure
735                .cmd_build_acceleration_structures(self.active, &geometry_infos, &ranges_ptrs);
736        }
737    }
738
739    unsafe fn place_acceleration_structure_barrier(
740        &mut self,
741        barrier: crate::AccelerationStructureBarrier,
742    ) {
743        let (src_stage, src_access) = conv::map_acceleration_structure_usage_to_barrier(
744            barrier.usage.from,
745            self.device.features,
746        );
747        let (dst_stage, dst_access) = conv::map_acceleration_structure_usage_to_barrier(
748            barrier.usage.to,
749            self.device.features,
750        );
751
752        unsafe {
753            self.device.raw.cmd_pipeline_barrier(
754                self.active,
755                src_stage | vk::PipelineStageFlags::TOP_OF_PIPE,
756                dst_stage | vk::PipelineStageFlags::BOTTOM_OF_PIPE,
757                vk::DependencyFlags::empty(),
758                &[vk::MemoryBarrier::default()
759                    .src_access_mask(src_access)
760                    .dst_access_mask(dst_access)],
761                &[],
762                &[],
763            )
764        };
765    }
766    // render
767
768    unsafe fn begin_render_pass(
769        &mut self,
770        desc: &crate::RenderPassDescriptor<super::QuerySet, super::TextureView>,
771    ) -> Result<(), crate::DeviceError> {
772        let mut vk_clear_values =
773            ArrayVec::<vk::ClearValue, { super::MAX_TOTAL_ATTACHMENTS }>::new();
774        let mut rp_key = super::RenderPassKey {
775            colors: ArrayVec::default(),
776            depth_stencil: None,
777            sample_count: desc.sample_count,
778            multiview: desc.multiview,
779        };
780        let mut fb_key = super::FramebufferKey {
781            raw_pass: vk::RenderPass::null(),
782            attachments: ArrayVec::default(),
783            extent: desc.extent,
784        };
785
786        for cat in desc.color_attachments {
787            if let Some(cat) = cat.as_ref() {
788                let color_view = if cat.target.view.dimension == wgt::TextureViewDimension::D3 {
789                    let key = super::TempTextureViewKey {
790                        texture: cat.target.view.raw_texture,
791                        format: cat.target.view.raw_format,
792                        mip_level: cat.target.view.base_mip_level,
793                        depth_slice: cat.depth_slice.unwrap(),
794                    };
795                    self.make_temp_texture_view(key)?
796                } else {
797                    cat.target.view.raw
798                };
799
800                vk_clear_values.push(vk::ClearValue {
801                    color: unsafe { cat.make_vk_clear_color() },
802                });
803                let color = super::ColorAttachmentKey {
804                    base: cat.target.make_attachment_key(cat.ops),
805                    resolve: cat
806                        .resolve_target
807                        .as_ref()
808                        .map(|target| target.make_attachment_key(crate::AttachmentOps::STORE)),
809                };
810
811                rp_key.colors.push(Some(color));
812                fb_key.attachments.push(color_view);
813                if let Some(ref at) = cat.resolve_target {
814                    vk_clear_values.push(unsafe { mem::zeroed() });
815                    fb_key.attachments.push(at.view.raw);
816                }
817
818                // Assert this attachment is valid for the detected multiview, as a sanity check
819                // The driver crash for this is really bad on AMD, so the check is worth it
820                if let Some(multiview) = desc.multiview {
821                    assert_eq!(cat.target.view.layers, multiview);
822                    if let Some(ref resolve_target) = cat.resolve_target {
823                        assert_eq!(resolve_target.view.layers, multiview);
824                    }
825                }
826            } else {
827                rp_key.colors.push(None);
828            }
829        }
830        if let Some(ref ds) = desc.depth_stencil_attachment {
831            vk_clear_values.push(vk::ClearValue {
832                depth_stencil: vk::ClearDepthStencilValue {
833                    depth: ds.clear_value.0,
834                    stencil: ds.clear_value.1,
835                },
836            });
837            rp_key.depth_stencil = Some(super::DepthStencilAttachmentKey {
838                base: ds.target.make_attachment_key(ds.depth_ops),
839                stencil_ops: ds.stencil_ops,
840            });
841            fb_key.attachments.push(ds.target.view.raw);
842
843            // Assert this attachment is valid for the detected multiview, as a sanity check
844            // The driver crash for this is really bad on AMD, so the check is worth it
845            if let Some(multiview) = desc.multiview {
846                assert_eq!(ds.target.view.layers, multiview);
847            }
848        }
849
850        let render_area = vk::Rect2D {
851            offset: vk::Offset2D { x: 0, y: 0 },
852            extent: vk::Extent2D {
853                width: desc.extent.width,
854                height: desc.extent.height,
855            },
856        };
857        let vk_viewports = [vk::Viewport {
858            x: 0.0,
859            y: desc.extent.height as f32,
860            width: desc.extent.width as f32,
861            height: -(desc.extent.height as f32),
862            min_depth: 0.0,
863            max_depth: 1.0,
864        }];
865
866        let raw_pass = self.device.make_render_pass(rp_key).unwrap();
867        fb_key.raw_pass = raw_pass;
868        let raw_framebuffer = self.make_framebuffer(fb_key).unwrap();
869
870        let vk_info = vk::RenderPassBeginInfo::default()
871            .render_pass(raw_pass)
872            .render_area(render_area)
873            .clear_values(&vk_clear_values)
874            .framebuffer(raw_framebuffer);
875
876        if let Some(label) = desc.label {
877            unsafe { self.begin_debug_marker(label) };
878            self.rpass_debug_marker_active = true;
879        }
880
881        // Start timestamp if any (before all other commands but after debug marker)
882        if let Some(timestamp_writes) = desc.timestamp_writes.as_ref() {
883            if let Some(index) = timestamp_writes.beginning_of_pass_write_index {
884                unsafe {
885                    self.write_timestamp(timestamp_writes.query_set, index);
886                }
887            }
888            self.end_of_pass_timer_query = timestamp_writes
889                .end_of_pass_write_index
890                .map(|index| (timestamp_writes.query_set.raw, index));
891        }
892
893        unsafe {
894            self.device
895                .raw
896                .cmd_set_viewport(self.active, 0, &vk_viewports);
897            self.device
898                .raw
899                .cmd_set_scissor(self.active, 0, &[render_area]);
900            self.device.raw.cmd_begin_render_pass(
901                self.active,
902                &vk_info,
903                vk::SubpassContents::INLINE,
904            );
905        };
906
907        self.bind_point = vk::PipelineBindPoint::GRAPHICS;
908
909        Ok(())
910    }
911    unsafe fn end_render_pass(&mut self) {
912        unsafe {
913            self.device.raw.cmd_end_render_pass(self.active);
914        }
915
916        // After all other commands but before debug marker, so this is still seen as part of this pass.
917        self.write_pass_end_timestamp_if_requested();
918
919        if self.rpass_debug_marker_active {
920            unsafe {
921                self.end_debug_marker();
922            }
923            self.rpass_debug_marker_active = false;
924        }
925    }
926
927    unsafe fn set_bind_group(
928        &mut self,
929        layout: &super::PipelineLayout,
930        index: u32,
931        group: &super::BindGroup,
932        dynamic_offsets: &[wgt::DynamicOffset],
933    ) {
934        let sets = [*group.set.raw()];
935        unsafe {
936            self.device.raw.cmd_bind_descriptor_sets(
937                self.active,
938                self.bind_point,
939                layout.raw,
940                index,
941                &sets,
942                dynamic_offsets,
943            )
944        };
945    }
946    unsafe fn set_push_constants(
947        &mut self,
948        layout: &super::PipelineLayout,
949        stages: wgt::ShaderStages,
950        offset_bytes: u32,
951        data: &[u32],
952    ) {
953        unsafe {
954            self.device.raw.cmd_push_constants(
955                self.active,
956                layout.raw,
957                conv::map_shader_stage(stages),
958                offset_bytes,
959                bytemuck::cast_slice(data),
960            )
961        };
962    }
963
964    unsafe fn insert_debug_marker(&mut self, label: &str) {
965        if let Some(ext) = self.device.extension_fns.debug_utils.as_ref() {
966            let cstr = self.temp.make_c_str(label);
967            let vk_label = vk::DebugUtilsLabelEXT::default().label_name(cstr);
968            unsafe { ext.cmd_insert_debug_utils_label(self.active, &vk_label) };
969        }
970    }
971    unsafe fn begin_debug_marker(&mut self, group_label: &str) {
972        if let Some(ext) = self.device.extension_fns.debug_utils.as_ref() {
973            let cstr = self.temp.make_c_str(group_label);
974            let vk_label = vk::DebugUtilsLabelEXT::default().label_name(cstr);
975            unsafe { ext.cmd_begin_debug_utils_label(self.active, &vk_label) };
976        }
977    }
978    unsafe fn end_debug_marker(&mut self) {
979        if let Some(ext) = self.device.extension_fns.debug_utils.as_ref() {
980            unsafe { ext.cmd_end_debug_utils_label(self.active) };
981        }
982    }
983
984    unsafe fn set_render_pipeline(&mut self, pipeline: &super::RenderPipeline) {
985        unsafe {
986            self.device.raw.cmd_bind_pipeline(
987                self.active,
988                vk::PipelineBindPoint::GRAPHICS,
989                pipeline.raw,
990            )
991        };
992    }
993
994    unsafe fn set_index_buffer<'a>(
995        &mut self,
996        binding: crate::BufferBinding<'a, super::Buffer>,
997        format: wgt::IndexFormat,
998    ) {
999        unsafe {
1000            self.device.raw.cmd_bind_index_buffer(
1001                self.active,
1002                binding.buffer.raw,
1003                binding.offset,
1004                conv::map_index_format(format),
1005            )
1006        };
1007    }
1008    unsafe fn set_vertex_buffer<'a>(
1009        &mut self,
1010        index: u32,
1011        binding: crate::BufferBinding<'a, super::Buffer>,
1012    ) {
1013        let vk_buffers = [binding.buffer.raw];
1014        let vk_offsets = [binding.offset];
1015        unsafe {
1016            self.device
1017                .raw
1018                .cmd_bind_vertex_buffers(self.active, index, &vk_buffers, &vk_offsets)
1019        };
1020    }
1021    unsafe fn set_viewport(&mut self, rect: &crate::Rect<f32>, depth_range: Range<f32>) {
1022        let vk_viewports = [vk::Viewport {
1023            x: rect.x,
1024            y: rect.y + rect.h,
1025            width: rect.w,
1026            height: -rect.h, // flip Y
1027            min_depth: depth_range.start,
1028            max_depth: depth_range.end,
1029        }];
1030        unsafe {
1031            self.device
1032                .raw
1033                .cmd_set_viewport(self.active, 0, &vk_viewports)
1034        };
1035    }
1036    unsafe fn set_scissor_rect(&mut self, rect: &crate::Rect<u32>) {
1037        let vk_scissors = [vk::Rect2D {
1038            offset: vk::Offset2D {
1039                x: rect.x as i32,
1040                y: rect.y as i32,
1041            },
1042            extent: vk::Extent2D {
1043                width: rect.w,
1044                height: rect.h,
1045            },
1046        }];
1047        unsafe {
1048            self.device
1049                .raw
1050                .cmd_set_scissor(self.active, 0, &vk_scissors)
1051        };
1052    }
1053    unsafe fn set_stencil_reference(&mut self, value: u32) {
1054        unsafe {
1055            self.device.raw.cmd_set_stencil_reference(
1056                self.active,
1057                vk::StencilFaceFlags::FRONT_AND_BACK,
1058                value,
1059            )
1060        };
1061    }
1062    unsafe fn set_blend_constants(&mut self, color: &[f32; 4]) {
1063        unsafe { self.device.raw.cmd_set_blend_constants(self.active, color) };
1064    }
1065
1066    unsafe fn draw(
1067        &mut self,
1068        first_vertex: u32,
1069        vertex_count: u32,
1070        first_instance: u32,
1071        instance_count: u32,
1072    ) {
1073        unsafe {
1074            self.device.raw.cmd_draw(
1075                self.active,
1076                vertex_count,
1077                instance_count,
1078                first_vertex,
1079                first_instance,
1080            )
1081        };
1082    }
1083    unsafe fn draw_indexed(
1084        &mut self,
1085        first_index: u32,
1086        index_count: u32,
1087        base_vertex: i32,
1088        first_instance: u32,
1089        instance_count: u32,
1090    ) {
1091        unsafe {
1092            self.device.raw.cmd_draw_indexed(
1093                self.active,
1094                index_count,
1095                instance_count,
1096                first_index,
1097                base_vertex,
1098                first_instance,
1099            )
1100        };
1101    }
1102    unsafe fn draw_mesh_tasks(
1103        &mut self,
1104        group_count_x: u32,
1105        group_count_y: u32,
1106        group_count_z: u32,
1107    ) {
1108        if let Some(ref t) = self.device.extension_fns.mesh_shading {
1109            unsafe {
1110                t.cmd_draw_mesh_tasks(self.active, group_count_x, group_count_y, group_count_z);
1111            };
1112        } else {
1113            panic!("Feature `MESH_SHADING` not enabled");
1114        }
1115    }
1116    unsafe fn draw_indirect(
1117        &mut self,
1118        buffer: &super::Buffer,
1119        offset: wgt::BufferAddress,
1120        draw_count: u32,
1121    ) {
1122        unsafe {
1123            self.device.raw.cmd_draw_indirect(
1124                self.active,
1125                buffer.raw,
1126                offset,
1127                draw_count,
1128                size_of::<wgt::DrawIndirectArgs>() as u32,
1129            )
1130        };
1131    }
1132    unsafe fn draw_indexed_indirect(
1133        &mut self,
1134        buffer: &super::Buffer,
1135        offset: wgt::BufferAddress,
1136        draw_count: u32,
1137    ) {
1138        unsafe {
1139            self.device.raw.cmd_draw_indexed_indirect(
1140                self.active,
1141                buffer.raw,
1142                offset,
1143                draw_count,
1144                size_of::<wgt::DrawIndexedIndirectArgs>() as u32,
1145            )
1146        };
1147    }
1148    unsafe fn draw_mesh_tasks_indirect(
1149        &mut self,
1150        buffer: &<Self::A as crate::Api>::Buffer,
1151        offset: wgt::BufferAddress,
1152        draw_count: u32,
1153    ) {
1154        if let Some(ref t) = self.device.extension_fns.mesh_shading {
1155            unsafe {
1156                t.cmd_draw_mesh_tasks_indirect(
1157                    self.active,
1158                    buffer.raw,
1159                    offset,
1160                    draw_count,
1161                    size_of::<wgt::DispatchIndirectArgs>() as u32,
1162                );
1163            };
1164        } else {
1165            panic!("Feature `MESH_SHADING` not enabled");
1166        }
1167    }
1168    unsafe fn draw_indirect_count(
1169        &mut self,
1170        buffer: &super::Buffer,
1171        offset: wgt::BufferAddress,
1172        count_buffer: &super::Buffer,
1173        count_offset: wgt::BufferAddress,
1174        max_count: u32,
1175    ) {
1176        let stride = size_of::<wgt::DrawIndirectArgs>() as u32;
1177        match self.device.extension_fns.draw_indirect_count {
1178            Some(ref t) => {
1179                unsafe {
1180                    t.cmd_draw_indirect_count(
1181                        self.active,
1182                        buffer.raw,
1183                        offset,
1184                        count_buffer.raw,
1185                        count_offset,
1186                        max_count,
1187                        stride,
1188                    )
1189                };
1190            }
1191            None => panic!("Feature `DRAW_INDIRECT_COUNT` not enabled"),
1192        }
1193    }
1194    unsafe fn draw_indexed_indirect_count(
1195        &mut self,
1196        buffer: &super::Buffer,
1197        offset: wgt::BufferAddress,
1198        count_buffer: &super::Buffer,
1199        count_offset: wgt::BufferAddress,
1200        max_count: u32,
1201    ) {
1202        let stride = size_of::<wgt::DrawIndexedIndirectArgs>() as u32;
1203        match self.device.extension_fns.draw_indirect_count {
1204            Some(ref t) => {
1205                unsafe {
1206                    t.cmd_draw_indexed_indirect_count(
1207                        self.active,
1208                        buffer.raw,
1209                        offset,
1210                        count_buffer.raw,
1211                        count_offset,
1212                        max_count,
1213                        stride,
1214                    )
1215                };
1216            }
1217            None => panic!("Feature `DRAW_INDIRECT_COUNT` not enabled"),
1218        }
1219    }
1220    unsafe fn draw_mesh_tasks_indirect_count(
1221        &mut self,
1222        buffer: &<Self::A as crate::Api>::Buffer,
1223        offset: wgt::BufferAddress,
1224        count_buffer: &super::Buffer,
1225        count_offset: wgt::BufferAddress,
1226        max_count: u32,
1227    ) {
1228        if self.device.extension_fns.draw_indirect_count.is_none() {
1229            panic!("Feature `DRAW_INDIRECT_COUNT` not enabled");
1230        }
1231        if let Some(ref t) = self.device.extension_fns.mesh_shading {
1232            unsafe {
1233                t.cmd_draw_mesh_tasks_indirect_count(
1234                    self.active,
1235                    buffer.raw,
1236                    offset,
1237                    count_buffer.raw,
1238                    count_offset,
1239                    max_count,
1240                    size_of::<wgt::DispatchIndirectArgs>() as u32,
1241                );
1242            };
1243        } else {
1244            panic!("Feature `MESH_SHADING` not enabled");
1245        }
1246    }
1247
1248    // compute
1249
1250    unsafe fn begin_compute_pass(
1251        &mut self,
1252        desc: &crate::ComputePassDescriptor<'_, super::QuerySet>,
1253    ) {
1254        self.bind_point = vk::PipelineBindPoint::COMPUTE;
1255        if let Some(label) = desc.label {
1256            unsafe { self.begin_debug_marker(label) };
1257            self.rpass_debug_marker_active = true;
1258        }
1259
1260        if let Some(timestamp_writes) = desc.timestamp_writes.as_ref() {
1261            if let Some(index) = timestamp_writes.beginning_of_pass_write_index {
1262                unsafe {
1263                    self.write_timestamp(timestamp_writes.query_set, index);
1264                }
1265            }
1266            self.end_of_pass_timer_query = timestamp_writes
1267                .end_of_pass_write_index
1268                .map(|index| (timestamp_writes.query_set.raw, index));
1269        }
1270    }
1271    unsafe fn end_compute_pass(&mut self) {
1272        self.write_pass_end_timestamp_if_requested();
1273
1274        if self.rpass_debug_marker_active {
1275            unsafe { self.end_debug_marker() };
1276            self.rpass_debug_marker_active = false
1277        }
1278    }
1279
1280    unsafe fn set_compute_pipeline(&mut self, pipeline: &super::ComputePipeline) {
1281        unsafe {
1282            self.device.raw.cmd_bind_pipeline(
1283                self.active,
1284                vk::PipelineBindPoint::COMPUTE,
1285                pipeline.raw,
1286            )
1287        };
1288    }
1289
1290    unsafe fn dispatch(&mut self, count: [u32; 3]) {
1291        unsafe {
1292            self.device
1293                .raw
1294                .cmd_dispatch(self.active, count[0], count[1], count[2])
1295        };
1296    }
1297    unsafe fn dispatch_indirect(&mut self, buffer: &super::Buffer, offset: wgt::BufferAddress) {
1298        unsafe {
1299            self.device
1300                .raw
1301                .cmd_dispatch_indirect(self.active, buffer.raw, offset)
1302        }
1303    }
1304
1305    unsafe fn copy_acceleration_structure_to_acceleration_structure(
1306        &mut self,
1307        src: &super::AccelerationStructure,
1308        dst: &super::AccelerationStructure,
1309        copy: wgt::AccelerationStructureCopy,
1310    ) {
1311        let ray_tracing_functions = self
1312            .device
1313            .extension_fns
1314            .ray_tracing
1315            .as_ref()
1316            .expect("Feature `RAY_TRACING` not enabled");
1317
1318        let mode = match copy {
1319            wgt::AccelerationStructureCopy::Clone => vk::CopyAccelerationStructureModeKHR::CLONE,
1320            wgt::AccelerationStructureCopy::Compact => {
1321                vk::CopyAccelerationStructureModeKHR::COMPACT
1322            }
1323        };
1324
1325        unsafe {
1326            ray_tracing_functions
1327                .acceleration_structure
1328                .cmd_copy_acceleration_structure(
1329                    self.active,
1330                    &vk::CopyAccelerationStructureInfoKHR {
1331                        s_type: vk::StructureType::COPY_ACCELERATION_STRUCTURE_INFO_KHR,
1332                        p_next: core::ptr::null(),
1333                        src: src.raw,
1334                        dst: dst.raw,
1335                        mode,
1336                        _marker: Default::default(),
1337                    },
1338                );
1339        }
1340    }
1341}
1342
1343#[test]
1344fn check_dst_image_layout() {
1345    assert_eq!(
1346        conv::derive_image_layout(wgt::TextureUses::COPY_DST, wgt::TextureFormat::Rgba8Unorm),
1347        DST_IMAGE_LAYOUT
1348    );
1349}
wgpu_hal/vulkan/command.rs

wgpu_hal/vulkan/
command.rs
