use super::conv;
use arrayvec::ArrayVec;
use ash::{extensions::ext, vk};
use std::{mem, ops::Range, slice};
const ALLOCATION_GRANULARITY: u32 = 16;
const DST_IMAGE_LAYOUT: vk::ImageLayout = vk::ImageLayout::TRANSFER_DST_OPTIMAL;
impl super::Texture {
fn map_buffer_copies<T>(&self, regions: T) -> impl Iterator<Item = vk::BufferImageCopy>
where
T: Iterator<Item = crate::BufferTextureCopy>,
{
let aspects = self.aspects;
let fi = self.format_info;
let copy_size = self.copy_size;
regions.map(move |r| {
let extent = r.texture_base.max_copy_size(©_size).min(&r.size);
let (image_subresource, image_offset) =
conv::map_subresource_layers(&r.texture_base, aspects);
vk::BufferImageCopy {
buffer_offset: r.buffer_layout.offset,
buffer_row_length: r.buffer_layout.bytes_per_row.map_or(0, |bpr| {
fi.block_dimensions.0 as u32 * (bpr.get() / fi.block_size as u32)
}),
buffer_image_height: r
.buffer_layout
.rows_per_image
.map_or(0, |rpi| rpi.get() * fi.block_dimensions.1 as u32),
image_subresource,
image_offset,
image_extent: conv::map_copy_extent(&extent),
}
})
}
}
impl super::DeviceShared {
fn debug_messenger(&self) -> Option<&ext::DebugUtils> {
Some(&self.instance.debug_utils.as_ref()?.extension)
}
}
impl crate::CommandEncoder<super::Api> for super::CommandEncoder {
unsafe fn begin_encoding(&mut self, label: crate::Label) -> Result<(), crate::DeviceError> {
if self.free.is_empty() {
let vk_info = vk::CommandBufferAllocateInfo::builder()
.command_pool(self.raw)
.command_buffer_count(ALLOCATION_GRANULARITY)
.build();
let cmd_buf_vec = unsafe { self.device.raw.allocate_command_buffers(&vk_info)? };
self.free.extend(cmd_buf_vec);
}
let raw = self.free.pop().unwrap();
unsafe {
self.device.set_object_name(
vk::ObjectType::COMMAND_BUFFER,
raw,
label.unwrap_or_default(),
)
};
self.rpass_debug_marker_active = false;
let vk_info = vk::CommandBufferBeginInfo::builder()
.flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT)
.build();
unsafe { self.device.raw.begin_command_buffer(raw, &vk_info) }?;
self.active = raw;
Ok(())
}
unsafe fn end_encoding(&mut self) -> Result<super::CommandBuffer, crate::DeviceError> {
let raw = self.active;
self.active = vk::CommandBuffer::null();
unsafe { self.device.raw.end_command_buffer(raw) }?;
Ok(super::CommandBuffer { raw })
}
unsafe fn discard_encoding(&mut self) {
self.discarded.push(self.active);
self.active = vk::CommandBuffer::null();
}
unsafe fn reset_all<I>(&mut self, cmd_bufs: I)
where
I: Iterator<Item = super::CommandBuffer>,
{
self.temp.clear();
self.free
.extend(cmd_bufs.into_iter().map(|cmd_buf| cmd_buf.raw));
self.free.append(&mut self.discarded);
let _ = unsafe {
self.device
.raw
.reset_command_pool(self.raw, vk::CommandPoolResetFlags::default())
};
}
unsafe fn transition_buffers<'a, T>(&mut self, barriers: T)
where
T: Iterator<Item = crate::BufferBarrier<'a, super::Api>>,
{
let mut src_stages = vk::PipelineStageFlags::TOP_OF_PIPE;
let mut dst_stages = vk::PipelineStageFlags::BOTTOM_OF_PIPE;
let vk_barriers = &mut self.temp.buffer_barriers;
vk_barriers.clear();
for bar in barriers {
let (src_stage, src_access) = conv::map_buffer_usage_to_barrier(bar.usage.start);
src_stages |= src_stage;
let (dst_stage, dst_access) = conv::map_buffer_usage_to_barrier(bar.usage.end);
dst_stages |= dst_stage;
vk_barriers.push(
vk::BufferMemoryBarrier::builder()
.buffer(bar.buffer.raw)
.size(vk::WHOLE_SIZE)
.src_access_mask(src_access)
.dst_access_mask(dst_access)
.build(),
)
}
if !vk_barriers.is_empty() {
unsafe {
self.device.raw.cmd_pipeline_barrier(
self.active,
src_stages,
dst_stages,
vk::DependencyFlags::empty(),
&[],
vk_barriers,
&[],
)
};
}
}
unsafe fn transition_textures<'a, T>(&mut self, barriers: T)
where
T: Iterator<Item = crate::TextureBarrier<'a, super::Api>>,
{
let mut src_stages = vk::PipelineStageFlags::empty();
let mut dst_stages = vk::PipelineStageFlags::empty();
let vk_barriers = &mut self.temp.image_barriers;
vk_barriers.clear();
for bar in barriers {
let range = conv::map_subresource_range(&bar.range, bar.texture.aspects);
let (src_stage, src_access) = conv::map_texture_usage_to_barrier(bar.usage.start);
let src_layout = conv::derive_image_layout(bar.usage.start, bar.texture.aspects);
src_stages |= src_stage;
let (dst_stage, dst_access) = conv::map_texture_usage_to_barrier(bar.usage.end);
let dst_layout = conv::derive_image_layout(bar.usage.end, bar.texture.aspects);
dst_stages |= dst_stage;
vk_barriers.push(
vk::ImageMemoryBarrier::builder()
.image(bar.texture.raw)
.subresource_range(range)
.src_access_mask(src_access)
.dst_access_mask(dst_access)
.old_layout(src_layout)
.new_layout(dst_layout)
.build(),
);
}
if !vk_barriers.is_empty() {
unsafe {
self.device.raw.cmd_pipeline_barrier(
self.active,
src_stages,
dst_stages,
vk::DependencyFlags::empty(),
&[],
&[],
vk_barriers,
)
};
}
}
unsafe fn clear_buffer(&mut self, buffer: &super::Buffer, range: crate::MemoryRange) {
unsafe {
self.device.raw.cmd_fill_buffer(
self.active,
buffer.raw,
range.start,
range.end - range.start,
0,
)
};
}
unsafe fn copy_buffer_to_buffer<T>(
&mut self,
src: &super::Buffer,
dst: &super::Buffer,
regions: T,
) where
T: Iterator<Item = crate::BufferCopy>,
{
let vk_regions_iter = regions.map(|r| vk::BufferCopy {
src_offset: r.src_offset,
dst_offset: r.dst_offset,
size: r.size.get(),
});
unsafe {
self.device.raw.cmd_copy_buffer(
self.active,
src.raw,
dst.raw,
&smallvec::SmallVec::<[vk::BufferCopy; 32]>::from_iter(vk_regions_iter),
)
};
}
unsafe fn copy_texture_to_texture<T>(
&mut self,
src: &super::Texture,
src_usage: crate::TextureUses,
dst: &super::Texture,
regions: T,
) where
T: Iterator<Item = crate::TextureCopy>,
{
let src_layout = conv::derive_image_layout(src_usage, src.aspects);
let vk_regions_iter = regions.map(|r| {
let (src_subresource, src_offset) =
conv::map_subresource_layers(&r.src_base, src.aspects);
let (dst_subresource, dst_offset) =
conv::map_subresource_layers(&r.dst_base, dst.aspects);
let extent = r
.size
.min(&r.src_base.max_copy_size(&src.copy_size))
.min(&r.dst_base.max_copy_size(&dst.copy_size));
vk::ImageCopy {
src_subresource,
src_offset,
dst_subresource,
dst_offset,
extent: conv::map_copy_extent(&extent),
}
});
unsafe {
self.device.raw.cmd_copy_image(
self.active,
src.raw,
src_layout,
dst.raw,
DST_IMAGE_LAYOUT,
&smallvec::SmallVec::<[vk::ImageCopy; 32]>::from_iter(vk_regions_iter),
)
};
}
unsafe fn copy_buffer_to_texture<T>(
&mut self,
src: &super::Buffer,
dst: &super::Texture,
regions: T,
) where
T: Iterator<Item = crate::BufferTextureCopy>,
{
let vk_regions_iter = dst.map_buffer_copies(regions);
unsafe {
self.device.raw.cmd_copy_buffer_to_image(
self.active,
src.raw,
dst.raw,
DST_IMAGE_LAYOUT,
&smallvec::SmallVec::<[vk::BufferImageCopy; 32]>::from_iter(vk_regions_iter),
)
};
}
unsafe fn copy_texture_to_buffer<T>(
&mut self,
src: &super::Texture,
src_usage: crate::TextureUses,
dst: &super::Buffer,
regions: T,
) where
T: Iterator<Item = crate::BufferTextureCopy>,
{
let src_layout = conv::derive_image_layout(src_usage, src.aspects);
let vk_regions_iter = src.map_buffer_copies(regions);
unsafe {
self.device.raw.cmd_copy_image_to_buffer(
self.active,
src.raw,
src_layout,
dst.raw,
&smallvec::SmallVec::<[vk::BufferImageCopy; 32]>::from_iter(vk_regions_iter),
)
};
}
unsafe fn begin_query(&mut self, set: &super::QuerySet, index: u32) {
unsafe {
self.device.raw.cmd_begin_query(
self.active,
set.raw,
index,
vk::QueryControlFlags::empty(),
)
};
}
unsafe fn end_query(&mut self, set: &super::QuerySet, index: u32) {
unsafe { self.device.raw.cmd_end_query(self.active, set.raw, index) };
}
unsafe fn write_timestamp(&mut self, set: &super::QuerySet, index: u32) {
unsafe {
self.device.raw.cmd_write_timestamp(
self.active,
vk::PipelineStageFlags::BOTTOM_OF_PIPE,
set.raw,
index,
)
};
}
unsafe fn reset_queries(&mut self, set: &super::QuerySet, range: Range<u32>) {
unsafe {
self.device.raw.cmd_reset_query_pool(
self.active,
set.raw,
range.start,
range.end - range.start,
)
};
}
unsafe fn copy_query_results(
&mut self,
set: &super::QuerySet,
range: Range<u32>,
buffer: &super::Buffer,
offset: wgt::BufferAddress,
stride: wgt::BufferSize,
) {
unsafe {
self.device.raw.cmd_copy_query_pool_results(
self.active,
set.raw,
range.start,
range.end - range.start,
buffer.raw,
offset,
stride.get(),
vk::QueryResultFlags::TYPE_64 | vk::QueryResultFlags::WAIT,
)
};
}
unsafe fn begin_render_pass(&mut self, desc: &crate::RenderPassDescriptor<super::Api>) {
let mut vk_clear_values =
ArrayVec::<vk::ClearValue, { super::MAX_TOTAL_ATTACHMENTS }>::new();
let mut vk_image_views = ArrayVec::<vk::ImageView, { super::MAX_TOTAL_ATTACHMENTS }>::new();
let mut rp_key = super::RenderPassKey::default();
let mut fb_key = super::FramebufferKey {
attachments: ArrayVec::default(),
extent: desc.extent,
sample_count: desc.sample_count,
};
let caps = &self.device.private_caps;
for cat in desc.color_attachments {
if let Some(cat) = cat.as_ref() {
vk_clear_values.push(vk::ClearValue {
color: unsafe { cat.make_vk_clear_color() },
});
vk_image_views.push(cat.target.view.raw);
let color = super::ColorAttachmentKey {
base: cat.target.make_attachment_key(cat.ops, caps),
resolve: cat.resolve_target.as_ref().map(|target| {
target.make_attachment_key(crate::AttachmentOps::STORE, caps)
}),
};
rp_key.colors.push(Some(color));
fb_key.attachments.push(cat.target.view.attachment.clone());
if let Some(ref at) = cat.resolve_target {
vk_clear_values.push(unsafe { mem::zeroed() });
vk_image_views.push(at.view.raw);
fb_key.attachments.push(at.view.attachment.clone());
}
if let Some(multiview) = desc.multiview {
assert_eq!(cat.target.view.layers, multiview);
if let Some(ref resolve_target) = cat.resolve_target {
assert_eq!(resolve_target.view.layers, multiview);
}
}
} else {
rp_key.colors.push(None);
}
}
if let Some(ref ds) = desc.depth_stencil_attachment {
vk_clear_values.push(vk::ClearValue {
depth_stencil: vk::ClearDepthStencilValue {
depth: ds.clear_value.0,
stencil: ds.clear_value.1,
},
});
vk_image_views.push(ds.target.view.raw);
rp_key.depth_stencil = Some(super::DepthStencilAttachmentKey {
base: ds.target.make_attachment_key(ds.depth_ops, caps),
stencil_ops: ds.stencil_ops,
});
fb_key.attachments.push(ds.target.view.attachment.clone());
if let Some(multiview) = desc.multiview {
assert_eq!(ds.target.view.layers, multiview);
}
}
rp_key.sample_count = fb_key.sample_count;
rp_key.multiview = desc.multiview;
let render_area = vk::Rect2D {
offset: vk::Offset2D { x: 0, y: 0 },
extent: vk::Extent2D {
width: desc.extent.width,
height: desc.extent.height,
},
};
let vk_viewports = [vk::Viewport {
x: 0.0,
y: if self.device.private_caps.flip_y_requires_shift {
desc.extent.height as f32
} else {
0.0
},
width: desc.extent.width as f32,
height: -(desc.extent.height as f32),
min_depth: 0.0,
max_depth: 1.0,
}];
let raw_pass = self.device.make_render_pass(rp_key).unwrap();
let raw_framebuffer = self
.device
.make_framebuffer(fb_key, raw_pass, desc.label)
.unwrap();
let mut vk_info = vk::RenderPassBeginInfo::builder()
.render_pass(raw_pass)
.render_area(render_area)
.clear_values(&vk_clear_values)
.framebuffer(raw_framebuffer);
let mut vk_attachment_info = if caps.imageless_framebuffers {
Some(
vk::RenderPassAttachmentBeginInfo::builder()
.attachments(&vk_image_views)
.build(),
)
} else {
None
};
if let Some(attachment_info) = vk_attachment_info.as_mut() {
vk_info = vk_info.push_next(attachment_info);
}
if let Some(label) = desc.label {
unsafe { self.begin_debug_marker(label) };
self.rpass_debug_marker_active = true;
}
unsafe {
self.device
.raw
.cmd_set_viewport(self.active, 0, &vk_viewports);
self.device
.raw
.cmd_set_scissor(self.active, 0, &[render_area]);
self.device.raw.cmd_begin_render_pass(
self.active,
&vk_info,
vk::SubpassContents::INLINE,
);
};
self.bind_point = vk::PipelineBindPoint::GRAPHICS;
}
unsafe fn end_render_pass(&mut self) {
unsafe {
self.device.raw.cmd_end_render_pass(self.active);
if self.rpass_debug_marker_active {
self.end_debug_marker();
self.rpass_debug_marker_active = false;
}
}
}
unsafe fn set_bind_group(
&mut self,
layout: &super::PipelineLayout,
index: u32,
group: &super::BindGroup,
dynamic_offsets: &[wgt::DynamicOffset],
) {
let sets = [*group.set.raw()];
unsafe {
self.device.raw.cmd_bind_descriptor_sets(
self.active,
self.bind_point,
layout.raw,
index,
&sets,
dynamic_offsets,
)
};
}
unsafe fn set_push_constants(
&mut self,
layout: &super::PipelineLayout,
stages: wgt::ShaderStages,
offset: u32,
data: &[u32],
) {
unsafe {
self.device.raw.cmd_push_constants(
self.active,
layout.raw,
conv::map_shader_stage(stages),
offset,
slice::from_raw_parts(data.as_ptr() as _, data.len() * 4),
)
};
}
unsafe fn insert_debug_marker(&mut self, label: &str) {
if let Some(ext) = self.device.debug_messenger() {
let cstr = self.temp.make_c_str(label);
let vk_label = vk::DebugUtilsLabelEXT::builder().label_name(cstr).build();
unsafe { ext.cmd_insert_debug_utils_label(self.active, &vk_label) };
}
}
unsafe fn begin_debug_marker(&mut self, group_label: &str) {
if let Some(ext) = self.device.debug_messenger() {
let cstr = self.temp.make_c_str(group_label);
let vk_label = vk::DebugUtilsLabelEXT::builder().label_name(cstr).build();
unsafe { ext.cmd_begin_debug_utils_label(self.active, &vk_label) };
}
}
unsafe fn end_debug_marker(&mut self) {
if let Some(ext) = self.device.debug_messenger() {
unsafe { ext.cmd_end_debug_utils_label(self.active) };
}
}
unsafe fn set_render_pipeline(&mut self, pipeline: &super::RenderPipeline) {
unsafe {
self.device.raw.cmd_bind_pipeline(
self.active,
vk::PipelineBindPoint::GRAPHICS,
pipeline.raw,
)
};
}
unsafe fn set_index_buffer<'a>(
&mut self,
binding: crate::BufferBinding<'a, super::Api>,
format: wgt::IndexFormat,
) {
unsafe {
self.device.raw.cmd_bind_index_buffer(
self.active,
binding.buffer.raw,
binding.offset,
conv::map_index_format(format),
)
};
}
unsafe fn set_vertex_buffer<'a>(
&mut self,
index: u32,
binding: crate::BufferBinding<'a, super::Api>,
) {
let vk_buffers = [binding.buffer.raw];
let vk_offsets = [binding.offset];
unsafe {
self.device
.raw
.cmd_bind_vertex_buffers(self.active, index, &vk_buffers, &vk_offsets)
};
}
unsafe fn set_viewport(&mut self, rect: &crate::Rect<f32>, depth_range: Range<f32>) {
let vk_viewports = [vk::Viewport {
x: rect.x,
y: if self.device.private_caps.flip_y_requires_shift {
rect.y + rect.h
} else {
rect.y
},
width: rect.w,
height: -rect.h, min_depth: depth_range.start,
max_depth: depth_range.end,
}];
unsafe {
self.device
.raw
.cmd_set_viewport(self.active, 0, &vk_viewports)
};
}
unsafe fn set_scissor_rect(&mut self, rect: &crate::Rect<u32>) {
let vk_scissors = [vk::Rect2D {
offset: vk::Offset2D {
x: rect.x as i32,
y: rect.y as i32,
},
extent: vk::Extent2D {
width: rect.w,
height: rect.h,
},
}];
unsafe {
self.device
.raw
.cmd_set_scissor(self.active, 0, &vk_scissors)
};
}
unsafe fn set_stencil_reference(&mut self, value: u32) {
unsafe {
self.device.raw.cmd_set_stencil_reference(
self.active,
vk::StencilFaceFlags::FRONT_AND_BACK,
value,
)
};
}
unsafe fn set_blend_constants(&mut self, color: &[f32; 4]) {
unsafe { self.device.raw.cmd_set_blend_constants(self.active, color) };
}
unsafe fn draw(
&mut self,
start_vertex: u32,
vertex_count: u32,
start_instance: u32,
instance_count: u32,
) {
unsafe {
self.device.raw.cmd_draw(
self.active,
vertex_count,
instance_count,
start_vertex,
start_instance,
)
};
}
unsafe fn draw_indexed(
&mut self,
start_index: u32,
index_count: u32,
base_vertex: i32,
start_instance: u32,
instance_count: u32,
) {
unsafe {
self.device.raw.cmd_draw_indexed(
self.active,
index_count,
instance_count,
start_index,
base_vertex,
start_instance,
)
};
}
unsafe fn draw_indirect(
&mut self,
buffer: &super::Buffer,
offset: wgt::BufferAddress,
draw_count: u32,
) {
unsafe {
self.device.raw.cmd_draw_indirect(
self.active,
buffer.raw,
offset,
draw_count,
mem::size_of::<wgt::DrawIndirectArgs>() as u32,
)
};
}
unsafe fn draw_indexed_indirect(
&mut self,
buffer: &super::Buffer,
offset: wgt::BufferAddress,
draw_count: u32,
) {
unsafe {
self.device.raw.cmd_draw_indexed_indirect(
self.active,
buffer.raw,
offset,
draw_count,
mem::size_of::<wgt::DrawIndexedIndirectArgs>() as u32,
)
};
}
unsafe fn draw_indirect_count(
&mut self,
buffer: &super::Buffer,
offset: wgt::BufferAddress,
count_buffer: &super::Buffer,
count_offset: wgt::BufferAddress,
max_count: u32,
) {
let stride = mem::size_of::<wgt::DrawIndirectArgs>() as u32;
match self.device.extension_fns.draw_indirect_count {
Some(ref t) => {
unsafe {
t.cmd_draw_indirect_count(
self.active,
buffer.raw,
offset,
count_buffer.raw,
count_offset,
max_count,
stride,
)
};
}
None => panic!("Feature `DRAW_INDIRECT_COUNT` not enabled"),
}
}
unsafe fn draw_indexed_indirect_count(
&mut self,
buffer: &super::Buffer,
offset: wgt::BufferAddress,
count_buffer: &super::Buffer,
count_offset: wgt::BufferAddress,
max_count: u32,
) {
let stride = mem::size_of::<wgt::DrawIndexedIndirectArgs>() as u32;
match self.device.extension_fns.draw_indirect_count {
Some(ref t) => {
unsafe {
t.cmd_draw_indexed_indirect_count(
self.active,
buffer.raw,
offset,
count_buffer.raw,
count_offset,
max_count,
stride,
)
};
}
None => panic!("Feature `DRAW_INDIRECT_COUNT` not enabled"),
}
}
unsafe fn begin_compute_pass(&mut self, desc: &crate::ComputePassDescriptor) {
self.bind_point = vk::PipelineBindPoint::COMPUTE;
if let Some(label) = desc.label {
unsafe { self.begin_debug_marker(label) };
self.rpass_debug_marker_active = true;
}
}
unsafe fn end_compute_pass(&mut self) {
if self.rpass_debug_marker_active {
unsafe { self.end_debug_marker() };
self.rpass_debug_marker_active = false
}
}
unsafe fn set_compute_pipeline(&mut self, pipeline: &super::ComputePipeline) {
unsafe {
self.device.raw.cmd_bind_pipeline(
self.active,
vk::PipelineBindPoint::COMPUTE,
pipeline.raw,
)
};
}
unsafe fn dispatch(&mut self, count: [u32; 3]) {
unsafe {
self.device
.raw
.cmd_dispatch(self.active, count[0], count[1], count[2])
};
}
unsafe fn dispatch_indirect(&mut self, buffer: &super::Buffer, offset: wgt::BufferAddress) {
unsafe {
self.device
.raw
.cmd_dispatch_indirect(self.active, buffer.raw, offset)
}
}
}
#[test]
fn check_dst_image_layout() {
assert_eq!(
conv::derive_image_layout(crate::TextureUses::COPY_DST, crate::FormatAspects::empty()),
DST_IMAGE_LAYOUT
);
}