use super::super::shared;
use super::super::shared::DISPATCH_BATCH_STRIDE;
use super::staging;
use super::submit_session::{VulkanSubmitScope, VulkanSubmitView};
use super::types::{
BufferState, ComputePipelineState, LogicalDevice, PushLayout, SharedBufferTable, SharedComputePipelineTable,
SharedPipelineTable, SlotKey, TimelineWaitTarget,
};
use super::{BufferHandle, ComputePipelineHandle, DeviceHandle};
use crate::backend::submission_worker::allocate_timeline_value;
use crate::backend::{GpuCommand, GraphCommand, RenderCommand, SubmitSync};
use crate::gpu_profiler::{self, DispatchGpuNs};
use crate::task_graph::{NodeAccessUnion, SlotUsageSet, UsageKindFlags};
use crate::timeline::TimelineValue;
use crate::tracy_zone;
use anyhow::{Context, Result};
use ash::vk;
use std::collections::HashMap;
use std::sync::atomic::Ordering;
use std::sync::Arc;
fn slot_key_from_category(cat: crate::types::ResourceCategory, index: u32) -> Option<SlotKey> {
use crate::types::ResourceCategory;
match cat {
ResourceCategory::Scattered => Some(SlotKey::StorageBuffer(index)),
ResourceCategory::Broadcast => Some(SlotKey::UniformBuffer(index)),
ResourceCategory::Texture => Some(SlotKey::SampledTexture(index)),
ResourceCategory::StorageImage => Some(SlotKey::StorageImage(index)),
ResourceCategory::Sampler => Some(SlotKey::Sampler(index)),
}
}
fn buffer_stride_for_bindless_index(
buffers: &HashMap<BufferHandle, BufferState>,
device_handle: DeviceHandle,
index: u32,
cat: crate::types::ResourceCategory,
) -> Option<u32> {
for b in buffers.values() {
if b.device_handle != device_handle {
continue;
}
match cat {
crate::types::ResourceCategory::Scattered if b.is_storage && b.bindless_index == Some(index) => {
return b.element_stride;
}
crate::types::ResourceCategory::Broadcast if !b.is_storage && b.bindless_index == Some(index) => {
return b.element_stride;
}
_ => {}
}
}
None
}
fn collect_slots_from_raw_bind(indices: &[u32], categories: &[Option<crate::types::ResourceCategory>]) -> Vec<SlotKey> {
let mut slots = Vec::new();
for (i, &idx) in indices.iter().enumerate() {
if let Some(Some(cat)) = categories.get(i) {
if let Some(key) = slot_key_from_category(*cat, idx) {
slots.push(key);
}
}
}
slots
}
fn collect_slot_keys_from_gpu_commands(
commands: &[GpuCommand],
compute_pipelines: &SharedComputePipelineTable,
_buffers: &SharedBufferTable,
) -> Vec<SlotKey> {
let mut current_pipeline = None;
let mut slots = Vec::new();
let compute_read = compute_pipelines.read().unwrap();
for cmd in commands {
match cmd {
GpuCommand::SetPipeline(p) => current_pipeline = Some(*p),
GpuCommand::BindResourcesRaw { indices, .. } => {
if let Some(h) = current_pipeline {
if let Some(p) = compute_read.entries.get(&h) {
slots.extend(collect_slots_from_raw_bind(indices, &p.push_constant_categories));
}
}
}
GpuCommand::DispatchBatch { arg_data, count, .. } => {
if let Some(h) = current_pipeline {
if let Some(p) = compute_read.entries.get(&h) {
let layout_size = std::mem::size_of::<PushLayout>();
for i in 0..*count as usize {
let base = i * DISPATCH_BATCH_STRIDE;
if base + layout_size <= arg_data.len() {
let layout: &PushLayout = bytemuck::from_bytes(&arg_data[base..base + layout_size]);
for (slot_i, &idx) in layout.bindless.iter().enumerate() {
if let Some(Some(cat)) = p.push_constant_categories.get(slot_i).copied() {
if let Some(key) = slot_key_from_category(cat, idx as u32) {
slots.push(key);
}
}
}
}
}
}
}
}
_ => {}
}
}
slots
}
fn collect_slot_keys_from_graph_commands(
commands: &[GraphCommand],
compute_pipelines: &SharedComputePipelineTable,
pipelines: &SharedPipelineTable,
buffers: &SharedBufferTable,
) -> Vec<SlotKey> {
let mut slots = Vec::new();
let mut current_compute_pipeline = None;
let mut current_render_pipeline = None;
let compute_read = compute_pipelines.read().unwrap();
let pipelines_read = pipelines.read().unwrap();
let buffers_read = buffers.read().unwrap();
for gc in commands {
match gc {
GraphCommand::Compute(cmd) => match cmd {
GpuCommand::SetPipeline(p) => current_compute_pipeline = Some(*p),
GpuCommand::BindResourcesRaw { indices, .. } => {
if let Some(h) = current_compute_pipeline {
if let Some(p) = compute_read.entries.get(&h) {
slots.extend(collect_slots_from_raw_bind(indices, &p.push_constant_categories));
}
}
}
GpuCommand::DispatchBatch { arg_data, count, .. } => {
if let Some(h) = current_compute_pipeline {
if let Some(p) = compute_read.entries.get(&h) {
let layout_size = std::mem::size_of::<PushLayout>();
for i in 0..*count as usize {
let base = i * DISPATCH_BATCH_STRIDE;
if base + layout_size <= arg_data.len() {
let layout: &PushLayout = bytemuck::from_bytes(&arg_data[base..base + layout_size]);
for (slot_i, &idx) in layout.bindless.iter().enumerate() {
if let Some(Some(cat)) = p.push_constant_categories.get(slot_i).copied() {
if let Some(key) = slot_key_from_category(cat, idx as u32) {
slots.push(key);
}
}
}
}
}
}
}
}
_ => {}
},
GraphCommand::Render {
commands: render_cmds, ..
} => {
for rc in render_cmds {
match rc {
RenderCommand::SetPipeline(p) => current_render_pipeline = Some(*p),
RenderCommand::BindResources { buffers: buf_handles } => {
for h in buf_handles {
if let Some(idx) = buffers_read.entries.get(h).and_then(|b| b.bindless_index) {
slots.push(SlotKey::StorageBuffer(idx));
}
}
}
RenderCommand::BindResourcesRaw { indices, .. } => {
if let Some(h) = current_render_pipeline {
if let Some(p) = pipelines_read.entries.get(&h) {
slots.extend(collect_slots_from_raw_bind(indices, &p.push_constant_categories));
}
}
}
RenderCommand::BindResourcesTyped { handles } => {
for h in handles {
if let Some(key) = slot_key_from_category(h.category(), h.index()) {
slots.push(key);
}
}
}
_ => {}
}
}
}
}
}
slots
}
fn slot_usage_to_vk_stage(usage: &SlotUsageSet, on_graphics_queue: bool) -> vk::PipelineStageFlags2 {
if usage.kinds.is_empty() {
return vk::PipelineStageFlags2::ALL_COMMANDS;
}
let mut flags = vk::PipelineStageFlags2::empty();
if usage.kinds.contains(UsageKindFlags::COMPUTE) {
flags |= vk::PipelineStageFlags2::COMPUTE_SHADER | vk::PipelineStageFlags2::DRAW_INDIRECT;
}
if usage.kinds.contains(UsageKindFlags::TRANSFER) {
flags |= vk::PipelineStageFlags2::TRANSFER;
}
if usage.kinds.contains(UsageKindFlags::RENDER) {
if on_graphics_queue {
flags |= vk::PipelineStageFlags2::ALL_GRAPHICS;
} else {
flags |= vk::PipelineStageFlags2::COMPUTE_SHADER;
}
}
flags
}
fn record_legacy_acquire_barrier(device: &ash::Device, cmd: vk::CommandBuffer, on_graphics_queue: bool) {
let src_graphics = if on_graphics_queue {
vk::PipelineStageFlags2::ALL_GRAPHICS
} else {
vk::PipelineStageFlags2::empty()
};
let dst_graphics = if on_graphics_queue {
vk::PipelineStageFlags2::VERTEX_SHADER
| vk::PipelineStageFlags2::FRAGMENT_SHADER
| vk::PipelineStageFlags2::VERTEX_INPUT
} else {
vk::PipelineStageFlags2::empty()
};
let dst_access = if on_graphics_queue {
vk::AccessFlags2::SHADER_READ
| vk::AccessFlags2::TRANSFER_READ
| vk::AccessFlags2::INDIRECT_COMMAND_READ
| vk::AccessFlags2::VERTEX_ATTRIBUTE_READ
} else {
vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::TRANSFER_READ | vk::AccessFlags2::INDIRECT_COMMAND_READ
};
let acquire = vk::MemoryBarrier2::default()
.src_stage_mask(vk::PipelineStageFlags2::COMPUTE_SHADER | vk::PipelineStageFlags2::TRANSFER | src_graphics)
.src_access_mask(vk::AccessFlags2::SHADER_WRITE | vk::AccessFlags2::TRANSFER_WRITE)
.dst_stage_mask(
vk::PipelineStageFlags2::COMPUTE_SHADER
| vk::PipelineStageFlags2::TRANSFER
| vk::PipelineStageFlags2::DRAW_INDIRECT
| dst_graphics,
)
.dst_access_mask(dst_access);
let dep = vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&acquire));
unsafe {
device.cmd_pipeline_barrier2(cmd, &dep);
}
}
fn record_legacy_release_barrier(device: &ash::Device, cmd: vk::CommandBuffer, on_graphics_queue: bool) {
let dst_graphics = if on_graphics_queue {
vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT
} else {
vk::PipelineStageFlags2::empty()
};
let dst_access = if on_graphics_queue {
vk::AccessFlags2::SHADER_READ
| vk::AccessFlags2::SHADER_WRITE
| vk::AccessFlags2::TRANSFER_READ
| vk::AccessFlags2::INDIRECT_COMMAND_READ
| vk::AccessFlags2::COLOR_ATTACHMENT_READ
} else {
vk::AccessFlags2::SHADER_READ
| vk::AccessFlags2::SHADER_WRITE
| vk::AccessFlags2::TRANSFER_READ
| vk::AccessFlags2::INDIRECT_COMMAND_READ
};
let release = vk::MemoryBarrier2::default()
.src_stage_mask(vk::PipelineStageFlags2::COMPUTE_SHADER | vk::PipelineStageFlags2::TRANSFER)
.src_access_mask(vk::AccessFlags2::SHADER_WRITE | vk::AccessFlags2::TRANSFER_WRITE)
.dst_stage_mask(
vk::PipelineStageFlags2::COMPUTE_SHADER
| vk::PipelineStageFlags2::TRANSFER
| vk::PipelineStageFlags2::DRAW_INDIRECT
| dst_graphics,
)
.dst_access_mask(dst_access);
let dep = vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&release));
unsafe {
device.cmd_pipeline_barrier2(cmd, &dep);
}
}
fn slot_usage_to_vk_access(usage: &SlotUsageSet, for_buffer: bool, on_graphics_queue: bool) -> vk::AccessFlags2 {
if usage.kinds.is_empty() {
return vk::AccessFlags2::MEMORY_READ | vk::AccessFlags2::MEMORY_WRITE;
}
let mut flags = vk::AccessFlags2::empty();
if usage.kinds.contains(UsageKindFlags::COMPUTE) {
if usage.access == NodeAccessUnion::Write {
flags |= vk::AccessFlags2::SHADER_WRITE | vk::AccessFlags2::SHADER_READ;
} else {
flags |= vk::AccessFlags2::SHADER_READ;
}
flags |= vk::AccessFlags2::INDIRECT_COMMAND_READ;
}
if usage.kinds.contains(UsageKindFlags::TRANSFER) {
if usage.access == NodeAccessUnion::Write {
flags |= vk::AccessFlags2::TRANSFER_WRITE;
} else {
flags |= vk::AccessFlags2::TRANSFER_READ;
}
}
if usage.kinds.contains(UsageKindFlags::RENDER) {
if for_buffer || !on_graphics_queue {
if usage.access == NodeAccessUnion::Write {
flags |= vk::AccessFlags2::SHADER_WRITE | vk::AccessFlags2::SHADER_READ;
} else {
flags |= vk::AccessFlags2::SHADER_READ;
}
} else {
flags |= vk::AccessFlags2::COLOR_ATTACHMENT_WRITE | vk::AccessFlags2::DEPTH_STENCIL_ATTACHMENT_WRITE;
}
}
flags
}
fn acquire_cmd_buffer(ld: &LogicalDevice, sc: &mut super::types::SubmissionContext) -> Result<vk::CommandBuffer> {
if let Some(cb) = sc.free_cmd_buffers.pop() {
return Ok(cb);
}
let alloc_info = vk::CommandBufferAllocateInfo::default()
.command_pool(sc.command_pool)
.level(vk::CommandBufferLevel::PRIMARY)
.command_buffer_count(1);
let cbs =
unsafe { ld.device.allocate_command_buffers(&alloc_info) }.context("Failed to allocate command buffer")?;
Ok(cbs[0])
}
fn context_queue_target(scope: &VulkanSubmitScope<'_>) -> (vk::Queue, std::sync::Arc<std::sync::Mutex<()>>) {
let sc = scope.sc.lock().unwrap();
(sc.queue, std::sync::Arc::clone(&sc.queue_lock))
}
fn device_graphics_queue_target(ld: &LogicalDevice) -> (vk::Queue, std::sync::Arc<std::sync::Mutex<()>>) {
(ld.queue, std::sync::Arc::clone(&ld.queue_lock))
}
fn record_last_submitted(scope: &VulkanSubmitScope<'_>, route_device: bool, signal_value: u64) {
scope.sc.lock().unwrap().last_submitted_seq = signal_value;
if route_device {
if let Some(owner) = scope.device_owner {
if let Some(owner_arc) = scope.view.contexts.read().unwrap().get(&owner) {
owner_arc.lock().unwrap().last_submitted_seq = signal_value;
}
}
}
}
fn build_submit_signal_infos(
scope: &VulkanSubmitScope<'_>,
route_device: bool,
signal_value: u64,
) -> Result<Vec<vk::SemaphoreSubmitInfo<'static>>> {
let submit_sem = scope.sc.lock().unwrap().timeline_semaphore;
let mut infos = vec![vk::SemaphoreSubmitInfo::default()
.semaphore(submit_sem)
.value(signal_value)
.stage_mask(vk::PipelineStageFlags2::ALL_COMMANDS)];
if route_device {
let owner = scope
.device_owner
.context("device owner missing for device-queue render submit")?;
let owner_sem = scope
.view
.contexts
.read()
.unwrap()
.get(&owner)
.context("invalid device owner context")?
.lock()
.unwrap()
.timeline_semaphore;
infos.push(
vk::SemaphoreSubmitInfo::default()
.semaphore(owner_sem)
.value(signal_value)
.stage_mask(vk::PipelineStageFlags2::ALL_COMMANDS),
);
}
Ok(infos)
}
fn register_submit_timeline(ld: &LogicalDevice, value: u64, route_device: bool, ctx: super::ContextHandle) {
let target = if route_device {
TimelineWaitTarget::DeviceOwner
} else {
TimelineWaitTarget::Context(ctx)
};
super::context::register_timeline_wait_target(ld, value, target);
}
#[derive(Debug)]
pub(super) struct VulkanGpuProfilePool {
pool: vk::QueryPool,
query_count: u32,
dispatch_labels: Vec<Option<&'static str>>,
period_ns: f32,
valid_bits: u32,
}
fn collect_dispatch_labels_compute(commands: &[GpuCommand]) -> (usize, Vec<Option<&'static str>>) {
let mut labels = Vec::new();
for c in commands {
match c {
GpuCommand::Dispatch { label, .. }
| GpuCommand::DispatchIndirect { label, .. }
| GpuCommand::DispatchBatch { label, .. } => {
labels.push(*label);
}
_ => {}
}
}
let n = labels.len();
(n, labels)
}
fn collect_dispatch_labels_graph(commands: &[GraphCommand]) -> (usize, Vec<Option<&'static str>>) {
let mut labels = Vec::new();
for gc in commands {
if let GraphCommand::Compute(
GpuCommand::Dispatch { label, .. }
| GpuCommand::DispatchIndirect { label, .. }
| GpuCommand::DispatchBatch { label, .. },
) = gc
{
labels.push(*label);
}
}
let n = labels.len();
(n, labels)
}
unsafe fn create_vulkan_gpu_profile_pool(
ld: &LogicalDevice,
defer_present: bool,
dispatch_count: usize,
dispatch_labels: Vec<Option<&'static str>>,
) -> Result<Option<VulkanGpuProfilePool>> {
if defer_present || !gpu_profiler::gpu_profile_enabled() || !ld.vk_timestamp_compute_and_graphics {
return Ok(None);
}
debug_assert_eq!(dispatch_labels.len(), dispatch_count);
let query_count = 2u32.saturating_add((dispatch_count as u32).saturating_mul(2));
let pool = ld.device.create_query_pool(
&vk::QueryPoolCreateInfo::default()
.query_type(vk::QueryType::TIMESTAMP)
.query_count(query_count),
None,
)?;
Ok(Some(VulkanGpuProfilePool {
pool,
query_count,
dispatch_labels,
period_ns: ld.vk_timestamp_period_ns,
valid_bits: 64,
}))
}
fn vulkan_decode_duration_ns(start: u64, end: u64, valid_bits: u32, period_ns: f32) -> u64 {
let mask = if valid_bits >= 64 {
u64::MAX
} else {
(1u64 << valid_bits) - 1
};
let a = start & mask;
let b = end & mask;
let delta = b.wrapping_sub(a);
let ns_f = (delta as f64) * f64::from(period_ns);
if ns_f <= 0.0 {
0
} else if ns_f >= u64::MAX as f64 {
u64::MAX
} else {
ns_f as u64
}
}
pub(super) unsafe fn vulkan_readback_gpu_profile(
device: &ash::Device,
signal_value: TimelineValue,
profile: VulkanGpuProfilePool,
) -> Result<()> {
let mut raw = vec![0u64; profile.query_count as usize];
if let Err(e) = device.get_query_pool_results(
profile.pool,
0,
&mut raw,
vk::QueryResultFlags::WAIT | vk::QueryResultFlags::TYPE_64,
) {
unsafe {
device.destroy_query_pool(profile.pool, None);
}
return Err(anyhow::anyhow!("get_query_pool_results: {:?}", e));
}
let cb_ns = vulkan_decode_duration_ns(raw[0], raw[1], profile.valid_bits, profile.period_ns);
gpu_profiler::log_cb_timing("vulkan", signal_value, cb_ns as f64 / 1_000_000.0);
let n = profile.dispatch_labels.len();
if n > 0 {
let mut dispatches = Vec::with_capacity(n);
for i in 0..n {
let si = 2 + 2 * i;
let ns = vulkan_decode_duration_ns(raw[si], raw[si + 1], profile.valid_bits, profile.period_ns);
let label = profile.dispatch_labels[i].unwrap_or("dispatch");
dispatches.push(DispatchGpuNs { label, gpu_ns: ns });
}
gpu_profiler::log_dispatch_timings("vulkan", signal_value, &dispatches);
}
device.destroy_query_pool(profile.pool, None);
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn enqueue_vulkan_compute_with_housekeeping(
scope: &super::submit_session::VulkanSubmitScope<'_>,
device_handle: super::DeviceHandle,
queue: ash::vk::Queue,
queue_lock: Arc<std::sync::Mutex<()>>,
timeline_sem: ash::vk::Semaphore,
signal_value: crate::timeline::TimelineValue,
signal_semaphore_infos: Vec<ash::vk::SemaphoreSubmitInfo<'static>>,
cmd: Option<ash::vk::CommandBuffer>,
sync: Option<&crate::backend::SubmitSync>,
staging_belt_finish: bool,
texture_staging_entries: Vec<super::staging::TextureStagingEntry>,
gpu_profile: Option<super::pending_submit::VulkanGpuProfileWork>,
) -> Result<()> {
let view = &scope.view;
let ld = view.devices.get(&device_handle).context("Invalid device handle")?;
let completed = scope.completed_timeline_value();
super::pending_submit::vulkan_drain_context_deletion_up_to(ld, view.contexts, device_handle, &scope.sc, completed);
{
let mut sc_guard = scope.sc.lock().unwrap();
super::pending_submit::vulkan_drain_pending_gpu_profiles_up_to(ld, &mut sc_guard, completed);
}
super::pending_submit::enqueue_vulkan_submit(
ld,
view.contexts,
view.buffers,
queue,
queue_lock,
timeline_sem,
signal_value,
signal_semaphore_infos,
cmd,
sync,
)?;
if let Some(prof) = gpu_profile {
scope.sc.lock().unwrap().pending_gpu_profiles.push((signal_value, prof));
}
super::pending_submit::vulkan_finish_staging_after_enqueue(
&scope.sc,
signal_value,
staging_belt_finish,
texture_staging_entries,
);
Ok(())
}
fn reap_signaled_fences(view: &VulkanSubmitView<'_>) {
let signaled: Vec<u64> = {
let pool = view.compute_fence_pool.lock().unwrap();
pool.iter()
.filter_map(|(token, (device_handle, fence, _))| {
let logical_device = view.devices.get(device_handle)?;
let signaled = unsafe { logical_device.device.get_fence_status(*fence) }.unwrap_or(false);
if signaled {
Some(*token)
} else {
None
}
})
.collect()
};
let mut pool = view.compute_fence_pool.lock().unwrap();
for token in signaled {
if let Some((device_handle, fence, cmd_buf)) = pool.remove(&token) {
if let Some(logical_device) = view.devices.get(&device_handle) {
if let Some(cb) = cmd_buf {
logical_device.free_device_cmd_buffers_now(&[cb]);
}
unsafe {
logical_device.device.destroy_fence(fence, None);
}
}
}
}
}
pub(super) fn create(
devices: &HashMap<DeviceHandle, super::types::SharedLogicalDevice>,
compute_pipelines: &SharedComputePipelineTable,
device_handle: DeviceHandle,
cs_module: vk::ShaderModule,
shader_debug_name: String,
) -> Result<ComputePipelineHandle> {
let logical_device = devices.get(&device_handle).context("Invalid device handle")?;
let pipeline_layout = logical_device
.bindless_pipeline_layout
.context("Bindless pipeline layout required")?;
let owns_layout = false;
let cs_stage = vk::PipelineShaderStageCreateInfo::default()
.stage(vk::ShaderStageFlags::COMPUTE)
.module(cs_module)
.name(c"main");
let mut robustness = vk::PipelineRobustnessCreateInfoEXT::default()
.storage_buffers(vk::PipelineRobustnessBufferBehaviorEXT::ROBUST_BUFFER_ACCESS_2)
.uniform_buffers(vk::PipelineRobustnessBufferBehaviorEXT::ROBUST_BUFFER_ACCESS_2)
.images(vk::PipelineRobustnessImageBehaviorEXT::ROBUST_IMAGE_ACCESS_2);
let pipeline_info = vk::ComputePipelineCreateInfo::default()
.stage(cs_stage)
.layout(pipeline_layout)
.push_next(&mut robustness);
let pipelines = unsafe {
logical_device
.device
.create_compute_pipelines(logical_device.pipeline_cache, &[pipeline_info], None)
}
.map_err(|(_, e)| anyhow::anyhow!("Failed to create compute pipeline: {:?}", e))?;
let handle = compute_pipelines.write().unwrap().alloc_handle();
compute_pipelines.write().unwrap().entries.insert(
handle,
ComputePipelineState {
device_handle,
pipeline: pipelines[0],
layout: pipeline_layout,
owns_layout,
parameter_block_layouts: Vec::new(),
push_constant_categories: Vec::new(),
binding_element_strides: Vec::new(),
shader_debug_name,
},
);
tracing::debug!("Created compute pipeline (handle={})", handle);
Ok(handle)
}
pub(super) fn destroy(
devices: &HashMap<DeviceHandle, super::types::SharedLogicalDevice>,
compute_pipelines: &SharedComputePipelineTable,
pipeline_handle: ComputePipelineHandle,
) {
if let Some(pipeline) = compute_pipelines.write().unwrap().entries.remove(&pipeline_handle) {
if let Some(logical_device) = devices.get(&pipeline.device_handle) {
unsafe {
let _ = logical_device.synchronized_device_wait_idle();
logical_device.device.destroy_pipeline(pipeline.pipeline, None);
if pipeline.owns_layout {
logical_device.device.destroy_pipeline_layout(pipeline.layout, None);
}
}
}
}
}
pub(super) fn submit_with_scope(
scope: &VulkanSubmitScope<'_>,
ctx: super::ContextHandle,
commands: &[GpuCommand],
sync: Option<&SubmitSync>,
) -> Result<TimelineValue> {
scope.assert_ctx(ctx);
let view = &scope.view;
let device_handle = scope.device_handle;
let mut commands = commands.to_vec();
crate::frame_table::lower_gpu_commands(&mut commands);
let _tz = tracy_zone!("vk.submit");
let has_write_buffer = commands.iter().any(|c| matches!(c, GpuCommand::WriteBuffer { .. }));
let has_write_texture = commands.iter().any(|c| {
matches!(
c,
GpuCommand::WriteTexture { .. }
| GpuCommand::WriteTextureRegion { .. }
| GpuCommand::CopyBufferToTexture { .. }
)
});
if has_write_buffer || has_write_texture {
let _rz = tracy_zone!("vk.submit.belt_reclaim");
let completed_timeline = scope.completed_timeline_value();
if has_write_buffer {
{
scope.sc.lock().unwrap().staging_belt.reclaim(
view.compute_fence_pool,
view.devices,
completed_timeline,
)?;
}
reap_signaled_fences(view);
}
if has_write_texture {
{
scope
.sc
.lock()
.unwrap()
.texture_staging_pool
.reclaim(completed_timeline);
}
}
}
let mut belt_slices: Vec<(vk::Buffer, u64)> = Vec::new();
if has_write_buffer {
for command in &commands {
if let GpuCommand::WriteBuffer {
buffer: buf_handle,
offset,
data,
} = command
{
let (host_mapped, is_storage, buf_device, buf_memory) = {
let buffers_read = view.buffers.read().unwrap();
let buf = buffers_read
.entries
.get(buf_handle)
.context("WriteBuffer: invalid buffer handle")?;
(buf.host_mapped, buf.is_storage, buf.device_handle, buf.memory)
};
if let Some(base) = host_mapped {
let p = base as *mut u8;
unsafe {
std::ptr::copy_nonoverlapping(data.as_ptr(), p.add(*offset as usize), data.len());
}
} else if !is_storage {
let dev = view.devices.get(&buf_device).context("WriteBuffer: device invalid")?;
unsafe {
let ptr = dev
.map_memory2(buf_memory, *offset, data.len() as u64)
.context("WriteBuffer: map failed")?;
std::ptr::copy_nonoverlapping(data.as_ptr(), ptr as *mut u8, data.len());
dev.unmap_memory2(buf_memory).context("WriteBuffer: unmap failed")?;
}
} else {
let dev = view.devices.get(&buf_device).context("WriteBuffer: device invalid")?;
let mut sc = scope.sc.lock().unwrap();
let (stg_buf, stg_off) = sc.staging_belt.write(view.instance, dev, data)?;
belt_slices.push((stg_buf, stg_off));
}
}
}
}
if commands.is_empty() {
let ld = view.devices.get(&device_handle).context("Invalid device handle")?;
let signal_value = allocate_timeline_value(&ld.timeline_next);
register_submit_timeline(ld, signal_value, false, ctx);
let (queue, queue_lock) = context_queue_target(scope);
let signal_infos = build_submit_signal_infos(scope, false, signal_value)?;
let timeline_sem = scope.sc.lock().unwrap().timeline_semaphore;
record_last_submitted(scope, false, signal_value);
enqueue_vulkan_compute_with_housekeeping(
scope,
device_handle,
queue,
queue_lock,
timeline_sem,
signal_value,
signal_infos,
None,
sync,
false,
Vec::new(),
None,
)?;
return Ok(signal_value);
}
let compute_pipelines = &view.compute_pipelines;
let buffers = &view.buffers;
let mut texture_upload_scratch: Vec<super::texture::ComputeTextureScratch> = Vec::new();
for command in &commands {
match command {
GpuCommand::WriteTexture {
texture,
data,
width,
height,
} => {
let mut sc_guard = scope.sc.lock().unwrap();
let pool = &mut sc_guard.texture_staging_pool;
texture_upload_scratch.push(super::texture::allocate_compute_texture_staging(
view.instance,
view.devices,
view.textures,
pool,
*texture,
data,
0,
0,
*width,
*height,
)?);
}
GpuCommand::WriteTextureRegion {
texture,
x,
y,
width,
height,
data,
} => {
let mut sc_guard = scope.sc.lock().unwrap();
let pool = &mut sc_guard.texture_staging_pool;
texture_upload_scratch.push(super::texture::allocate_compute_texture_staging(
view.instance,
view.devices,
view.textures,
pool,
*texture,
data,
*x,
*y,
*width,
*height,
)?);
}
GpuCommand::CopyBufferToTexture {
src,
src_offset,
dst,
x,
y,
width,
height,
..
} => {
let flat = super::texture::copy_buffer_to_texture_flat_bytes(
view.textures,
view.buffers,
*src,
*src_offset,
*dst,
*width,
*height,
)?;
let mut sc_guard = scope.sc.lock().unwrap();
texture_upload_scratch.push(super::texture::allocate_compute_texture_staging(
view.instance,
view.devices,
view.textures,
&mut sc_guard.texture_staging_pool,
*dst,
&flat,
*x,
*y,
*width,
*height,
)?);
}
_ => {}
}
}
let (dispatch_count, dispatch_labels) = collect_dispatch_labels_compute(&commands);
let vk_gpu_profile = unsafe {
let ld = view.devices.get(&device_handle).context("Invalid device handle")?;
create_vulkan_gpu_profile_pool(ld, false, dispatch_count, dispatch_labels)?
};
let cmd = {
let ld = view.devices.get(&device_handle).context("Invalid device handle")?;
let mut sc = scope.sc.lock().unwrap();
let cb = acquire_cmd_buffer(ld, &mut sc)?;
let begin_info = vk::CommandBufferBeginInfo::default().flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT);
if let Err(e) = unsafe { ld.device.begin_command_buffer(cb, &begin_info) } {
sc.free_cmd_buffers.push(cb);
return Err(anyhow::anyhow!("Failed to begin command buffer: {:?}", e));
}
cb
};
let (cmd, belt_idx, _texture_upload_idx, frame_table_row) = {
let logical_device = view.devices.get(&device_handle).context("Invalid device handle")?;
if SubmitSync::use_legacy_acquire_from(sync) {
record_legacy_acquire_barrier(&logical_device.device, cmd, false);
}
unsafe {
if let (Some(bindless_set), Some(bindless_layout)) = (
logical_device.bindless_descriptor_set,
logical_device.bindless_pipeline_layout,
) {
logical_device.device.cmd_bind_descriptor_sets(
cmd,
vk::PipelineBindPoint::COMPUTE,
bindless_layout,
0,
std::slice::from_ref(&bindless_set),
&[],
);
}
}
let mut vk_dispatch_idx = 0usize;
if let Some(ref prof) = vk_gpu_profile {
unsafe {
logical_device
.device
.cmd_reset_query_pool(cmd, prof.pool, 0, prof.query_count);
logical_device
.device
.cmd_write_timestamp2(cmd, vk::PipelineStageFlags2::TOP_OF_PIPE, prof.pool, 0);
}
}
let mut current_pipeline: Option<ComputePipelineHandle> = None;
let mut belt_idx = 0usize;
let mut texture_upload_idx = 0usize;
let mut row_guard = super::frame_table::RowReservation::new(&scope.frame_table);
for command in &commands {
match command {
GpuCommand::FrameTableStaging { data } => {
let row = super::frame_table::record_prologue(
view.contexts,
ctx,
super::frame_table::PrologueRecording {
frame_table: &scope.frame_table,
buffers: view.buffers,
ld: logical_device,
cmd,
on_graphics_queue: false,
},
data,
)?;
row_guard.set(row);
}
GpuCommand::SetPipeline(handle) => {
let _tz = tracy_zone!("vk.set_pipeline");
if let Some(pipeline_state) = compute_pipelines.read().unwrap().entries.get(handle) {
unsafe {
logical_device.device.cmd_bind_pipeline(
cmd,
vk::PipelineBindPoint::COMPUTE,
pipeline_state.pipeline,
);
}
current_pipeline = Some(*handle);
}
}
GpuCommand::BindResourcesRaw {
indices: raw_indices,
user: raw_user,
frame_table_base,
} => {
let pipelines_read = compute_pipelines.read().unwrap();
if let Some(pipeline) = current_pipeline.and_then(|h| pipelines_read.entries.get(&h)) {
crate::backend::with_layout_validation(|| {
crate::backend::validate_raw_binding_strides(
raw_indices,
&pipeline.push_constant_categories,
&pipeline.binding_element_strides,
|idx, cat| {
buffer_stride_for_bindless_index(
&buffers.read().unwrap().entries,
device_handle,
idx,
cat,
)
},
&pipeline.shader_debug_name,
)
})?;
let mut layout = PushLayout::default();
shared::fill_frame_table_dispatch(&mut layout, *frame_table_base, raw_user);
shared::set_frame_table_slots(
&mut layout,
scope.frame_table.selector_slot,
scope.frame_table.table_slot,
);
unsafe {
logical_device.device.cmd_push_constants(
cmd,
pipeline.layout,
vk::ShaderStageFlags::ALL,
0,
layout.as_bytes(),
);
}
}
}
GpuCommand::Dispatch {
label: _label,
workgroups_x,
workgroups_y,
workgroups_z,
} => {
let _tz = tracy_zone!("vk.dispatch");
unsafe {
if let Some(ref prof) = vk_gpu_profile {
let base = 2u32 + (vk_dispatch_idx as u32) * 2;
logical_device.device.cmd_write_timestamp2(
cmd,
vk::PipelineStageFlags2::TOP_OF_PIPE,
prof.pool,
base,
);
}
logical_device
.device
.cmd_dispatch(cmd, *workgroups_x, *workgroups_y, *workgroups_z);
if let Some(ref prof) = vk_gpu_profile {
let base = 2u32 + (vk_dispatch_idx as u32) * 2;
logical_device.device.cmd_write_timestamp2(
cmd,
vk::PipelineStageFlags2::BOTTOM_OF_PIPE,
prof.pool,
base + 1,
);
}
vk_dispatch_idx += 1;
}
}
GpuCommand::DispatchBatch {
label: _,
arg_data,
count,
} => {
let _tz = tracy_zone!("vk.dispatch_batch");
let push_size = std::mem::size_of::<crate::backend::shared::PushLayout>();
let stride = crate::backend::shared::DISPATCH_BATCH_STRIDE;
let pipelines_read = compute_pipelines.read().unwrap();
let pipeline_layout = current_pipeline
.and_then(|h| pipelines_read.entries.get(&h))
.map(|p| p.layout);
let mut patched_args = arg_data.to_vec();
crate::backend::shared::patch_dispatch_batch_frame_table_slots(
&mut patched_args,
*count as usize,
scope.frame_table.selector_slot,
scope.frame_table.table_slot,
);
for i in 0..*count as usize {
let base = i * stride;
let layout_bytes = &patched_args[base..base + push_size];
let wg_off = base + push_size;
let wg_x = u32::from_ne_bytes(arg_data[wg_off..wg_off + 4].try_into().unwrap());
let wg_y = u32::from_ne_bytes(arg_data[wg_off + 4..wg_off + 8].try_into().unwrap());
let wg_z = u32::from_ne_bytes(arg_data[wg_off + 8..wg_off + 12].try_into().unwrap());
unsafe {
if let Some(layout) = pipeline_layout {
logical_device.device.cmd_push_constants(
cmd,
layout,
vk::ShaderStageFlags::ALL,
0,
layout_bytes,
);
}
logical_device.device.cmd_dispatch(cmd, wg_x, wg_y, wg_z);
}
vk_dispatch_idx += 1;
}
}
GpuCommand::DispatchIndirect {
label: _label,
buffer,
offset,
} => {
let _tz = tracy_zone!("vk.dispatch_indirect");
let vk_buf = buffers
.read()
.unwrap()
.entries
.get(buffer)
.context("DispatchIndirect: invalid buffer handle")?
.buffer;
unsafe {
if let Some(ref prof) = vk_gpu_profile {
let base = 2u32 + (vk_dispatch_idx as u32) * 2;
logical_device.device.cmd_write_timestamp2(
cmd,
vk::PipelineStageFlags2::TOP_OF_PIPE,
prof.pool,
base,
);
}
logical_device.device.cmd_dispatch_indirect(cmd, vk_buf, *offset);
if let Some(ref prof) = vk_gpu_profile {
let base = 2u32 + (vk_dispatch_idx as u32) * 2;
logical_device.device.cmd_write_timestamp2(
cmd,
vk::PipelineStageFlags2::BOTTOM_OF_PIPE,
prof.pool,
base + 1,
);
}
}
vk_dispatch_idx += 1;
}
GpuCommand::ResourceBarrier {
buffers: buf_entries,
textures: tex_entries,
} => {
let _tz = tracy_zone!("vk.resource_barrier");
unsafe {
let buffers_guard = buffers.read().unwrap();
let buf_barriers: Vec<vk::BufferMemoryBarrier2> = buf_entries
.iter()
.filter_map(|(h, usage)| {
buffers_guard.entries.get(h).map(|bs| {
vk::BufferMemoryBarrier2::default()
.src_stage_mask(slot_usage_to_vk_stage(&usage.src, false))
.src_access_mask(slot_usage_to_vk_access(&usage.src, true, false))
.dst_stage_mask(slot_usage_to_vk_stage(&usage.dst, false))
.dst_access_mask(slot_usage_to_vk_access(&usage.dst, true, false))
.buffer(bs.buffer)
.offset(0)
.size(vk::WHOLE_SIZE)
})
})
.collect();
let tex_img: Vec<vk::ImageMemoryBarrier2> = tex_entries
.iter()
.filter_map(|(h, usage)| {
view.textures.read().unwrap().entries.get(h).map(|ts| {
let old_layout = ts.image_layout();
ts.set_image_layout(vk::ImageLayout::GENERAL);
vk::ImageMemoryBarrier2::default()
.src_stage_mask(slot_usage_to_vk_stage(&usage.src, false))
.src_access_mask(slot_usage_to_vk_access(&usage.src, false, false))
.dst_stage_mask(slot_usage_to_vk_stage(&usage.dst, false))
.dst_access_mask(slot_usage_to_vk_access(&usage.dst, false, false))
.old_layout(old_layout)
.new_layout(vk::ImageLayout::GENERAL)
.src_queue_family_index(vk::QUEUE_FAMILY_IGNORED)
.dst_queue_family_index(vk::QUEUE_FAMILY_IGNORED)
.image(ts.image)
.subresource_range(vk::ImageSubresourceRange {
aspect_mask: vk::ImageAspectFlags::COLOR,
base_mip_level: 0,
level_count: 1,
base_array_layer: 0,
layer_count: 1,
})
})
})
.collect();
let dep_info = if tex_img.is_empty() {
vk::DependencyInfo::default().buffer_memory_barriers(&buf_barriers)
} else {
vk::DependencyInfo::default()
.buffer_memory_barriers(&buf_barriers)
.image_memory_barriers(&tex_img)
};
logical_device.device.cmd_pipeline_barrier2(cmd, &dep_info);
}
}
GpuCommand::ClearBuffer { buffer, offset, size } => {
let _tz = tracy_zone!("vk.clear_buffer");
let (vk_buf, buf_size) = {
let buffers_guard = buffers.read().unwrap();
let bs = buffers_guard
.entries
.get(buffer)
.context("ClearBuffer: invalid buffer handle")?;
(bs.buffer, bs.size)
};
let clear_size = if *size == 0 {
buf_size.saturating_sub(*offset)
} else {
*size
};
if clear_size > 0 {
unsafe {
logical_device
.device
.cmd_fill_buffer(cmd, vk_buf, *offset, clear_size, 0);
let mem_barrier = vk::MemoryBarrier2::default()
.src_stage_mask(vk::PipelineStageFlags2::TRANSFER)
.src_access_mask(vk::AccessFlags2::TRANSFER_WRITE)
.dst_stage_mask(vk::PipelineStageFlags2::COMPUTE_SHADER)
.dst_access_mask(vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::SHADER_WRITE);
let dep_info =
vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&mem_barrier));
logical_device.device.cmd_pipeline_barrier2(cmd, &dep_info);
}
}
}
GpuCommand::WriteBuffer {
buffer: buf_handle,
offset,
data,
} => {
let _tz = tracy_zone!("vk.write_buffer");
let (is_storage, host_mapped, vk_buf) = {
let buffers_guard = buffers.read().unwrap();
let bs = buffers_guard
.entries
.get(buf_handle)
.context("WriteBuffer: invalid buffer handle")?;
(bs.is_storage, bs.host_mapped, bs.buffer)
};
if is_storage && host_mapped.is_none() {
let (stg, stg_off) = belt_slices
.get(belt_idx)
.context("WriteBuffer: belt slice missing (internal error)")?;
belt_idx += 1;
let region = vk::BufferCopy {
src_offset: *stg_off,
dst_offset: *offset,
size: data.len() as u64,
};
unsafe {
logical_device
.device
.cmd_copy_buffer(cmd, *stg, vk_buf, std::slice::from_ref(®ion));
let mem_barrier = vk::MemoryBarrier2::default()
.src_stage_mask(vk::PipelineStageFlags2::TRANSFER)
.src_access_mask(vk::AccessFlags2::TRANSFER_WRITE)
.dst_stage_mask(vk::PipelineStageFlags2::COMPUTE_SHADER)
.dst_access_mask(vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::SHADER_WRITE);
let dep_info =
vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&mem_barrier));
logical_device.device.cmd_pipeline_barrier2(cmd, &dep_info);
}
}
}
GpuCommand::WriteTexture { .. }
| GpuCommand::WriteTextureRegion { .. }
| GpuCommand::CopyBufferToTexture { .. } => {
let _tz = tracy_zone!("vk.write_texture");
let scratch = texture_upload_scratch
.get(texture_upload_idx)
.context("WriteTexture: scratch missing (internal)")?;
texture_upload_idx += 1;
super::texture::record_compute_texture_upload(view.devices, view.textures, cmd, scratch)?;
}
GpuCommand::CopyTexture { src, dst } => {
let _tz = tracy_zone!("vk.copy_texture");
let (src_image, width, height, dst_image) = {
let textures_read = view.textures.read().unwrap();
let ts = textures_read
.entries
.get(src)
.context("CopyTexture: src texture not found")?;
let dst_image = textures_read
.entries
.get(dst)
.context("CopyTexture: dst texture not found")?
.image;
(ts.image, ts.width, ts.height, dst_image)
};
unsafe {
let mem_barrier = vk::MemoryBarrier2::default()
.src_stage_mask(vk::PipelineStageFlags2::COMPUTE_SHADER | vk::PipelineStageFlags2::TRANSFER)
.src_access_mask(vk::AccessFlags2::SHADER_WRITE | vk::AccessFlags2::TRANSFER_WRITE)
.dst_stage_mask(vk::PipelineStageFlags2::TRANSFER)
.dst_access_mask(vk::AccessFlags2::TRANSFER_READ | vk::AccessFlags2::TRANSFER_WRITE);
let dep_info =
vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&mem_barrier));
logical_device.device.cmd_pipeline_barrier2(cmd, &dep_info);
let region = vk::ImageCopy {
src_subresource: vk::ImageSubresourceLayers {
aspect_mask: vk::ImageAspectFlags::COLOR,
mip_level: 0,
base_array_layer: 0,
layer_count: 1,
},
src_offset: vk::Offset3D::default(),
dst_subresource: vk::ImageSubresourceLayers {
aspect_mask: vk::ImageAspectFlags::COLOR,
mip_level: 0,
base_array_layer: 0,
layer_count: 1,
},
dst_offset: vk::Offset3D::default(),
extent: vk::Extent3D {
width,
height,
depth: 1,
},
};
logical_device.device.cmd_copy_image(
cmd,
src_image,
vk::ImageLayout::GENERAL,
dst_image,
vk::ImageLayout::GENERAL,
std::slice::from_ref(®ion),
);
let mem_barrier2 = vk::MemoryBarrier2::default()
.src_stage_mask(vk::PipelineStageFlags2::TRANSFER)
.src_access_mask(vk::AccessFlags2::TRANSFER_WRITE)
.dst_stage_mask(
vk::PipelineStageFlags2::COMPUTE_SHADER | vk::PipelineStageFlags2::ALL_COMMANDS,
)
.dst_access_mask(vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::SHADER_WRITE);
let dep_info2 =
vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&mem_barrier2));
logical_device.device.cmd_pipeline_barrier2(cmd, &dep_info2);
}
}
GpuCommand::CopyBuffer {
src,
src_offset,
dst,
dst_offset,
size,
} => {
let _tz = tracy_zone!("vk.copy_buffer");
let (src_buf, dst_buf) = {
let buffers_read = view.buffers.read().unwrap();
let src_state = buffers_read.entries.get(src).context("CopyBuffer: invalid src")?;
let dst_state = buffers_read.entries.get(dst).context("CopyBuffer: invalid dst")?;
if src_offset.saturating_add(*size) > src_state.size
|| dst_offset.saturating_add(*size) > dst_state.size
{
anyhow::bail!("CopyBuffer: size exceeds buffer bounds");
}
(src_state.buffer, dst_state.buffer)
};
unsafe {
let mem_barrier = vk::MemoryBarrier2::default()
.src_stage_mask(vk::PipelineStageFlags2::COMPUTE_SHADER | vk::PipelineStageFlags2::TRANSFER)
.src_access_mask(vk::AccessFlags2::SHADER_WRITE | vk::AccessFlags2::TRANSFER_WRITE)
.dst_stage_mask(vk::PipelineStageFlags2::TRANSFER)
.dst_access_mask(vk::AccessFlags2::TRANSFER_READ | vk::AccessFlags2::TRANSFER_WRITE);
let dep_info =
vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&mem_barrier));
logical_device.device.cmd_pipeline_barrier2(cmd, &dep_info);
let region = vk::BufferCopy {
src_offset: *src_offset,
dst_offset: *dst_offset,
size: *size,
};
logical_device
.device
.cmd_copy_buffer(cmd, src_buf, dst_buf, std::slice::from_ref(®ion));
}
}
GpuCommand::CopyTextureToReadback { src, dst, layout } => {
let _tz = tracy_zone!("vk.copy_texture_to_readback");
let staging_buffer = {
let buffers_read = view.buffers.read().unwrap();
buffers_read
.entries
.get(dst)
.context("CopyTextureToReadback: invalid dst")?
.buffer
};
super::texture::record_copy_texture_to_readback(
cmd,
logical_device,
view.textures,
staging_buffer,
*src,
*layout,
)?;
}
GpuCommand::CopyRenderTarget { src, dst } => {
let _tz = tracy_zone!("vk.copy_render_target");
let (src_image, width, height, dst_image) = {
let render_targets_read = view.render_targets.read().unwrap();
let rt = render_targets_read
.entries
.get(src)
.context("CopyRenderTarget: src render target not found")?;
let textures_read = view.textures.read().unwrap();
let dst_image = textures_read
.entries
.get(dst)
.context("CopyRenderTarget: dst texture not found")?
.image;
(rt.image, rt.width, rt.height, dst_image)
};
unsafe {
let mem_barrier = vk::MemoryBarrier2::default()
.src_stage_mask(vk::PipelineStageFlags2::TRANSFER)
.src_access_mask(vk::AccessFlags2::TRANSFER_READ)
.dst_stage_mask(vk::PipelineStageFlags2::TRANSFER)
.dst_access_mask(vk::AccessFlags2::TRANSFER_WRITE);
let dep = vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&mem_barrier));
logical_device.device.cmd_pipeline_barrier2(cmd, &dep);
let region = vk::ImageCopy {
src_subresource: vk::ImageSubresourceLayers {
aspect_mask: vk::ImageAspectFlags::COLOR,
mip_level: 0,
base_array_layer: 0,
layer_count: 1,
},
src_offset: vk::Offset3D::default(),
dst_subresource: vk::ImageSubresourceLayers {
aspect_mask: vk::ImageAspectFlags::COLOR,
mip_level: 0,
base_array_layer: 0,
layer_count: 1,
},
dst_offset: vk::Offset3D::default(),
extent: vk::Extent3D {
width,
height,
depth: 1,
},
};
logical_device.device.cmd_copy_image(
cmd,
src_image,
vk::ImageLayout::TRANSFER_SRC_OPTIMAL,
dst_image,
vk::ImageLayout::GENERAL,
std::slice::from_ref(®ion),
);
let mem_barrier2 = vk::MemoryBarrier2::default()
.src_stage_mask(vk::PipelineStageFlags2::TRANSFER)
.src_access_mask(vk::AccessFlags2::TRANSFER_WRITE)
.dst_stage_mask(
vk::PipelineStageFlags2::COMPUTE_SHADER | vk::PipelineStageFlags2::ALL_COMMANDS,
)
.dst_access_mask(vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::SHADER_WRITE);
let dep_info2 =
vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&mem_barrier2));
logical_device.device.cmd_pipeline_barrier2(cmd, &dep_info2);
}
}
}
}
debug_assert_eq!(
texture_upload_idx,
texture_upload_scratch.len(),
"WriteTexture commands mismatch texture scratch pre-pass"
);
record_legacy_release_barrier(&logical_device.device, cmd, false);
if let Some(ref prof) = vk_gpu_profile {
unsafe {
logical_device
.device
.cmd_write_timestamp2(cmd, vk::PipelineStageFlags2::BOTTOM_OF_PIPE, prof.pool, 1);
}
}
if let Err(e) = unsafe { logical_device.device.end_command_buffer(cmd) } {
let mut sc = scope.sc.lock().unwrap();
sc.free_cmd_buffers.push(cmd);
return Err(anyhow::anyhow!("Failed to end command buffer: {:?}", e));
}
(cmd, belt_idx, texture_upload_idx, row_guard.take())
};
let mut row_guard = super::frame_table::RowReservation::new(&scope.frame_table);
if let Some(row) = frame_table_row {
row_guard.set(row);
}
let ld = view.devices.get(&device_handle).context("Invalid device handle")?;
let signal_value = allocate_timeline_value(&ld.timeline_next);
register_submit_timeline(ld, signal_value, false, ctx);
let used_slots = collect_slot_keys_from_gpu_commands(&commands, view.compute_pipelines, view.buffers);
ld.descriptors
.lock()
.unwrap()
.record_slot_usage(ctx, signal_value, used_slots);
let (queue, queue_lock) = context_queue_target(scope);
let signal_infos = build_submit_signal_infos(scope, false, signal_value)?;
let timeline_sem = scope.sc.lock().unwrap().timeline_semaphore;
row_guard.commit(signal_value);
record_last_submitted(scope, false, signal_value);
{
let mut sc = scope.sc.lock().unwrap();
sc.timeline_cmd_buffers.entry(signal_value).or_default().push(cmd);
}
debug_assert_eq!(
belt_idx,
belt_slices.len(),
"WriteBuffer DEVICE_LOCAL count must match belt pre-pass"
);
let texture_entries: Vec<staging::TextureStagingEntry> =
texture_upload_scratch.into_iter().map(|s| s.entry).collect();
let gpu_profile_work = vk_gpu_profile.map(|prof| super::pending_submit::VulkanGpuProfileWork { ctx, cmd, prof });
enqueue_vulkan_compute_with_housekeeping(
scope,
device_handle,
queue,
queue_lock,
timeline_sem,
signal_value,
signal_infos,
Some(cmd),
sync,
true,
texture_entries,
gpu_profile_work,
)?;
Ok(signal_value)
}
pub(super) fn submit(
state: &super::types::VulkanState,
ctx: super::ContextHandle,
commands: &[GpuCommand],
sync: Option<&SubmitSync>,
) -> Result<TimelineValue> {
submit_with_scope(
&super::submit_session::scope_from_state(state, ctx)?,
ctx,
commands,
sync,
)
}
pub(super) fn submit_graph(
state: &super::types::VulkanState,
ctx: super::ContextHandle,
commands: &[GraphCommand],
sync: Option<&SubmitSync>,
) -> Result<TimelineValue> {
submit_graph_with_scope(
&super::submit_session::scope_from_state(state, ctx)?,
ctx,
commands,
None,
sync,
)
}
pub(super) fn submit_graph_with_scope(
scope: &VulkanSubmitScope<'_>,
ctx: super::ContextHandle,
commands: &[GraphCommand],
retain_key: Option<u64>,
sync: Option<&SubmitSync>,
) -> Result<TimelineValue> {
scope.assert_ctx(ctx);
let view = &scope.view;
let device_handle = scope.device_handle;
let _tz = tracy_zone!("vk.submit_graph");
let has_render = commands.iter().any(|c| matches!(c, GraphCommand::Render { .. }));
let route_device = has_render;
let has_upload = commands.iter().any(|c| {
matches!(
c,
GraphCommand::Compute(
GpuCommand::WriteBuffer { .. }
| GpuCommand::WriteTexture { .. }
| GpuCommand::WriteTextureRegion { .. }
| GpuCommand::CopyBufferToTexture { .. }
)
)
});
let has_write_texture_graph = commands.iter().any(|c| {
matches!(
c,
GraphCommand::Compute(
GpuCommand::WriteTexture { .. }
| GpuCommand::WriteTextureRegion { .. }
| GpuCommand::CopyBufferToTexture { .. }
)
)
});
if has_upload {
let _rz = tracy_zone!("vk.submit_graph.belt_reclaim");
let completed_timeline = scope.completed_timeline_value();
{
{
scope.sc.lock().unwrap().staging_belt.reclaim(
view.compute_fence_pool,
view.devices,
completed_timeline,
)?;
}
}
reap_signaled_fences(view);
if has_write_texture_graph {
{
scope
.sc
.lock()
.unwrap()
.texture_staging_pool
.reclaim(completed_timeline);
}
}
}
let mut belt_slices: Vec<(vk::Buffer, u64)> = Vec::new();
let mut texture_upload_scratch: Vec<super::texture::ComputeTextureScratch> = Vec::new();
if has_upload {
for graph_cmd in commands {
if let GraphCommand::Compute(gpu_cmd) = graph_cmd {
match gpu_cmd {
GpuCommand::WriteBuffer {
buffer: buf_handle,
offset,
data,
} => {
let (host_mapped, is_storage, buf_device, buf_memory) = {
let buffers_read = view.buffers.read().unwrap();
let buf = buffers_read
.entries
.get(buf_handle)
.context("WriteBuffer: invalid buffer handle")?;
(buf.host_mapped, buf.is_storage, buf.device_handle, buf.memory)
};
if let Some(base) = host_mapped {
let p = base as *mut u8;
unsafe {
std::ptr::copy_nonoverlapping(data.as_ptr(), p.add(*offset as usize), data.len());
}
} else if !is_storage {
let dev = view.devices.get(&buf_device).context("WriteBuffer: device invalid")?;
unsafe {
let ptr = dev
.map_memory2(buf_memory, *offset, data.len() as u64)
.context("WriteBuffer: map failed")?;
std::ptr::copy_nonoverlapping(data.as_ptr(), ptr as *mut u8, data.len());
dev.unmap_memory2(buf_memory).context("WriteBuffer: unmap failed")?;
}
} else {
let dev = view.devices.get(&buf_device).context("WriteBuffer: device invalid")?;
let mut sc = scope.sc.lock().unwrap();
let (stg_buf, stg_off) = sc.staging_belt.write(view.instance, dev, data)?;
belt_slices.push((stg_buf, stg_off));
}
}
GpuCommand::WriteTexture {
texture,
data,
width,
height,
} => {
let mut sc_guard = scope.sc.lock().unwrap();
let pool = &mut sc_guard.texture_staging_pool;
texture_upload_scratch.push(super::texture::allocate_compute_texture_staging(
view.instance,
view.devices,
view.textures,
pool,
*texture,
data,
0,
0,
*width,
*height,
)?);
}
GpuCommand::WriteTextureRegion {
texture,
x,
y,
width,
height,
data,
} => {
let mut sc_guard = scope.sc.lock().unwrap();
let pool = &mut sc_guard.texture_staging_pool;
texture_upload_scratch.push(super::texture::allocate_compute_texture_staging(
view.instance,
view.devices,
view.textures,
pool,
*texture,
data,
*x,
*y,
*width,
*height,
)?);
}
GpuCommand::CopyBufferToTexture {
src,
src_offset,
dst,
x,
y,
width,
height,
..
} => {
let flat = super::texture::copy_buffer_to_texture_flat_bytes(
view.textures,
view.buffers,
*src,
*src_offset,
*dst,
*width,
*height,
)?;
let mut sc_guard = scope.sc.lock().unwrap();
texture_upload_scratch.push(super::texture::allocate_compute_texture_staging(
view.instance,
view.devices,
view.textures,
&mut sc_guard.texture_staging_pool,
*dst,
&flat,
*x,
*y,
*width,
*height,
)?);
}
_ => {}
}
}
}
}
let cmd = {
let ld = view.devices.get(&device_handle).context("Invalid device handle")?;
let flags = if retain_key.is_none() {
vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT
} else {
vk::CommandBufferUsageFlags::SIMULTANEOUS_USE
};
let begin_info = vk::CommandBufferBeginInfo::default().flags(flags);
if route_device {
let cb = ld.acquire_device_cmd_buffer()?;
if let Err(e) = unsafe { ld.device.begin_command_buffer(cb, &begin_info) } {
ld.recycle_device_cmd_buffer(cb);
return Err(anyhow::anyhow!("Failed to begin device command buffer: {:?}", e));
}
cb
} else {
let mut sc = scope.sc.lock().unwrap();
let cb = acquire_cmd_buffer(ld, &mut sc)?;
if let Err(e) = unsafe { ld.device.begin_command_buffer(cb, &begin_info) } {
sc.free_cmd_buffers.push(cb);
return Err(anyhow::anyhow!("Failed to begin command buffer: {:?}", e));
}
cb
}
};
let logical_device = view.devices.get(&device_handle).context("Invalid device handle")?;
let (dispatch_count_graph, dispatch_labels_graph) = collect_dispatch_labels_graph(commands);
let vk_gpu_profile =
unsafe { create_vulkan_gpu_profile_pool(logical_device, false, dispatch_count_graph, dispatch_labels_graph)? };
if SubmitSync::use_legacy_acquire_from(sync) {
record_legacy_acquire_barrier(&logical_device.device, cmd, route_device);
}
unsafe {
if let (Some(bindless_set), Some(bindless_layout)) = (
logical_device.bindless_descriptor_set,
logical_device.bindless_pipeline_layout,
) {
logical_device.device.cmd_bind_descriptor_sets(
cmd,
vk::PipelineBindPoint::COMPUTE,
bindless_layout,
0,
std::slice::from_ref(&bindless_set),
&[],
);
}
}
let mut vk_dispatch_idx = 0usize;
if let Some(ref prof) = vk_gpu_profile {
unsafe {
logical_device
.device
.cmd_reset_query_pool(cmd, prof.pool, 0, prof.query_count);
logical_device
.device
.cmd_write_timestamp2(cmd, vk::PipelineStageFlags2::TOP_OF_PIPE, prof.pool, 0);
}
}
let compute_pipelines = &view.compute_pipelines;
let buffers = &view.buffers;
let mut current_compute_pipeline: Option<ComputePipelineHandle> = None;
let mut belt_idx = 0usize;
let mut texture_upload_idx = 0usize;
let mut frame_table_prologue_in_cb = false;
let mut frame_table_row: Option<u32> = None;
let mut row_guard = super::frame_table::RowReservation::new(&scope.frame_table);
for graph_cmd in commands {
match graph_cmd {
GraphCommand::Compute(gpu_cmd) => match gpu_cmd {
GpuCommand::FrameTableStaging { data } => {
frame_table_prologue_in_cb = true;
let row = super::frame_table::record_prologue(
view.contexts,
ctx,
super::frame_table::PrologueRecording {
frame_table: &scope.frame_table,
buffers: view.buffers,
ld: logical_device,
cmd,
on_graphics_queue: route_device,
},
data,
)?;
frame_table_row = Some(row);
row_guard.set(row);
}
GpuCommand::SetPipeline(handle) => {
let _tz = tracy_zone!("vk.set_pipeline");
if let Some(pipeline_state) = compute_pipelines.read().unwrap().entries.get(handle) {
unsafe {
logical_device.device.cmd_bind_pipeline(
cmd,
vk::PipelineBindPoint::COMPUTE,
pipeline_state.pipeline,
);
}
current_compute_pipeline = Some(*handle);
}
}
GpuCommand::BindResourcesRaw {
indices: raw_indices,
user: raw_user,
frame_table_base,
} => {
let pipelines_read = compute_pipelines.read().unwrap();
if let Some(pipeline) = current_compute_pipeline.and_then(|p| pipelines_read.entries.get(&p)) {
crate::backend::with_layout_validation(|| {
crate::backend::validate_raw_binding_strides(
raw_indices,
&pipeline.push_constant_categories,
&pipeline.binding_element_strides,
|idx, cat| {
buffer_stride_for_bindless_index(
&buffers.read().unwrap().entries,
device_handle,
idx,
cat,
)
},
&pipeline.shader_debug_name,
)
})?;
let mut layout = PushLayout::default();
shared::fill_frame_table_dispatch(&mut layout, *frame_table_base, raw_user);
shared::set_frame_table_slots(
&mut layout,
scope.frame_table.selector_slot,
scope.frame_table.table_slot,
);
unsafe {
logical_device.device.cmd_push_constants(
cmd,
pipeline.layout,
vk::ShaderStageFlags::ALL,
0,
layout.as_bytes(),
);
}
}
}
GpuCommand::Dispatch {
label: _label,
workgroups_x,
workgroups_y,
workgroups_z,
} => {
let _tz = tracy_zone!("vk.dispatch");
unsafe {
if let Some(ref prof) = vk_gpu_profile {
let base = 2u32 + (vk_dispatch_idx as u32) * 2;
logical_device.device.cmd_write_timestamp2(
cmd,
vk::PipelineStageFlags2::TOP_OF_PIPE,
prof.pool,
base,
);
}
logical_device
.device
.cmd_dispatch(cmd, *workgroups_x, *workgroups_y, *workgroups_z);
if let Some(ref prof) = vk_gpu_profile {
let base = 2u32 + (vk_dispatch_idx as u32) * 2;
logical_device.device.cmd_write_timestamp2(
cmd,
vk::PipelineStageFlags2::BOTTOM_OF_PIPE,
prof.pool,
base + 1,
);
}
vk_dispatch_idx += 1;
}
}
GpuCommand::DispatchBatch {
label: _,
arg_data,
count,
} => {
let _tz = tracy_zone!("vk.dispatch_batch");
let push_size = std::mem::size_of::<crate::backend::shared::PushLayout>();
let stride = crate::backend::shared::DISPATCH_BATCH_STRIDE;
let pipelines_read = compute_pipelines.read().unwrap();
let pipeline_layout = current_compute_pipeline
.and_then(|h| pipelines_read.entries.get(&h))
.map(|p| p.layout);
let mut patched_args = arg_data.to_vec();
crate::backend::shared::patch_dispatch_batch_frame_table_slots(
&mut patched_args,
*count as usize,
scope.frame_table.selector_slot,
scope.frame_table.table_slot,
);
for i in 0..*count as usize {
let base = i * stride;
let layout_bytes = &patched_args[base..base + push_size];
let wg_off = base + push_size;
let wg_x = u32::from_ne_bytes(arg_data[wg_off..wg_off + 4].try_into().unwrap());
let wg_y = u32::from_ne_bytes(arg_data[wg_off + 4..wg_off + 8].try_into().unwrap());
let wg_z = u32::from_ne_bytes(arg_data[wg_off + 8..wg_off + 12].try_into().unwrap());
unsafe {
if let Some(layout) = pipeline_layout {
logical_device.device.cmd_push_constants(
cmd,
layout,
vk::ShaderStageFlags::ALL,
0,
layout_bytes,
);
}
logical_device.device.cmd_dispatch(cmd, wg_x, wg_y, wg_z);
}
vk_dispatch_idx += 1;
}
}
GpuCommand::DispatchIndirect {
label: _label,
buffer,
offset,
} => {
let _tz = tracy_zone!("vk.dispatch_indirect");
let vk_buf = buffers
.read()
.unwrap()
.entries
.get(buffer)
.context("DispatchIndirect: invalid buffer handle")?
.buffer;
unsafe {
if let Some(ref prof) = vk_gpu_profile {
let base = 2u32 + (vk_dispatch_idx as u32) * 2;
logical_device.device.cmd_write_timestamp2(
cmd,
vk::PipelineStageFlags2::TOP_OF_PIPE,
prof.pool,
base,
);
}
logical_device.device.cmd_dispatch_indirect(cmd, vk_buf, *offset);
if let Some(ref prof) = vk_gpu_profile {
let base = 2u32 + (vk_dispatch_idx as u32) * 2;
logical_device.device.cmd_write_timestamp2(
cmd,
vk::PipelineStageFlags2::BOTTOM_OF_PIPE,
prof.pool,
base + 1,
);
}
}
vk_dispatch_idx += 1;
}
GpuCommand::ResourceBarrier {
buffers: buf_entries,
textures: tex_entries,
} => {
let _tz = tracy_zone!("vk.resource_barrier");
unsafe {
let buffers_guard = buffers.read().unwrap();
let buf_barriers: Vec<vk::BufferMemoryBarrier2> = buf_entries
.iter()
.filter_map(|(h, usage)| {
buffers_guard.entries.get(h).map(|bs| {
vk::BufferMemoryBarrier2::default()
.src_stage_mask(slot_usage_to_vk_stage(&usage.src, route_device))
.src_access_mask(slot_usage_to_vk_access(&usage.src, true, route_device))
.dst_stage_mask(slot_usage_to_vk_stage(&usage.dst, route_device))
.dst_access_mask(slot_usage_to_vk_access(&usage.dst, true, route_device))
.buffer(bs.buffer)
.offset(0)
.size(vk::WHOLE_SIZE)
})
})
.collect();
let tex_img: Vec<vk::ImageMemoryBarrier2> = tex_entries
.iter()
.filter_map(|(h, usage)| {
view.textures.read().unwrap().entries.get(h).map(|ts| {
let old_layout = ts.image_layout();
ts.set_image_layout(vk::ImageLayout::GENERAL);
vk::ImageMemoryBarrier2::default()
.src_stage_mask(slot_usage_to_vk_stage(&usage.src, route_device))
.src_access_mask(slot_usage_to_vk_access(&usage.src, false, route_device))
.dst_stage_mask(slot_usage_to_vk_stage(&usage.dst, route_device))
.dst_access_mask(slot_usage_to_vk_access(&usage.dst, false, route_device))
.old_layout(old_layout)
.new_layout(vk::ImageLayout::GENERAL)
.src_queue_family_index(vk::QUEUE_FAMILY_IGNORED)
.dst_queue_family_index(vk::QUEUE_FAMILY_IGNORED)
.image(ts.image)
.subresource_range(vk::ImageSubresourceRange {
aspect_mask: vk::ImageAspectFlags::COLOR,
base_mip_level: 0,
level_count: 1,
base_array_layer: 0,
layer_count: 1,
})
})
})
.collect();
let dep_info = if tex_img.is_empty() {
vk::DependencyInfo::default().buffer_memory_barriers(&buf_barriers)
} else {
vk::DependencyInfo::default()
.buffer_memory_barriers(&buf_barriers)
.image_memory_barriers(&tex_img)
};
logical_device.device.cmd_pipeline_barrier2(cmd, &dep_info);
}
}
GpuCommand::ClearBuffer { buffer, offset, size } => {
let _tz = tracy_zone!("vk.clear_buffer");
let (vk_buf, buf_size) = {
let buffers_guard = buffers.read().unwrap();
let bs = buffers_guard
.entries
.get(buffer)
.context("ClearBuffer: invalid buffer handle")?;
(bs.buffer, bs.size)
};
let clear_size = if *size == 0 {
buf_size.saturating_sub(*offset)
} else {
*size
};
if clear_size > 0 {
unsafe {
logical_device
.device
.cmd_fill_buffer(cmd, vk_buf, *offset, clear_size, 0);
let mem_barrier = vk::MemoryBarrier2::default()
.src_stage_mask(vk::PipelineStageFlags2::TRANSFER)
.src_access_mask(vk::AccessFlags2::TRANSFER_WRITE)
.dst_stage_mask(vk::PipelineStageFlags2::COMPUTE_SHADER)
.dst_access_mask(vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::SHADER_WRITE);
let dep_info =
vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&mem_barrier));
logical_device.device.cmd_pipeline_barrier2(cmd, &dep_info);
}
}
}
GpuCommand::WriteBuffer {
buffer: buf_handle,
offset,
data,
} => {
let _tz = tracy_zone!("vk.write_buffer");
let (is_storage, host_mapped, vk_buf) = {
let buffers_guard = buffers.read().unwrap();
let bs = buffers_guard
.entries
.get(buf_handle)
.context("WriteBuffer: invalid buffer handle")?;
(bs.is_storage, bs.host_mapped, bs.buffer)
};
if is_storage && host_mapped.is_none() {
let (stg, stg_off) = belt_slices
.get(belt_idx)
.context("WriteBuffer: belt slice missing (internal error)")?;
belt_idx += 1;
let region = vk::BufferCopy {
src_offset: *stg_off,
dst_offset: *offset,
size: data.len() as u64,
};
unsafe {
logical_device
.device
.cmd_copy_buffer(cmd, *stg, vk_buf, std::slice::from_ref(®ion));
let mem_barrier = vk::MemoryBarrier2::default()
.src_stage_mask(vk::PipelineStageFlags2::TRANSFER)
.src_access_mask(vk::AccessFlags2::TRANSFER_WRITE)
.dst_stage_mask(vk::PipelineStageFlags2::COMPUTE_SHADER)
.dst_access_mask(vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::SHADER_WRITE);
let dep_info =
vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&mem_barrier));
logical_device.device.cmd_pipeline_barrier2(cmd, &dep_info);
}
}
}
GpuCommand::WriteTexture { .. }
| GpuCommand::WriteTextureRegion { .. }
| GpuCommand::CopyBufferToTexture { .. } => {
let _tz = tracy_zone!("vk.write_texture");
let scratch = texture_upload_scratch
.get(texture_upload_idx)
.context("WriteTexture: scratch missing (internal)")?;
texture_upload_idx += 1;
super::texture::record_compute_texture_upload(view.devices, view.textures, cmd, scratch)?;
}
GpuCommand::CopyTexture { src, dst } => {
let _tz = tracy_zone!("vk.copy_texture");
let (src_image, width, height, dst_image) = {
let textures_read = view.textures.read().unwrap();
let ts = textures_read
.entries
.get(src)
.context("CopyTexture: src texture not found")?;
let dst_image = textures_read
.entries
.get(dst)
.context("CopyTexture: dst texture not found")?
.image;
(ts.image, ts.width, ts.height, dst_image)
};
unsafe {
let mem_barrier = vk::MemoryBarrier2::default()
.src_stage_mask(vk::PipelineStageFlags2::COMPUTE_SHADER | vk::PipelineStageFlags2::TRANSFER)
.src_access_mask(vk::AccessFlags2::SHADER_WRITE | vk::AccessFlags2::TRANSFER_WRITE)
.dst_stage_mask(vk::PipelineStageFlags2::TRANSFER)
.dst_access_mask(vk::AccessFlags2::TRANSFER_READ | vk::AccessFlags2::TRANSFER_WRITE);
let dep = vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&mem_barrier));
logical_device.device.cmd_pipeline_barrier2(cmd, &dep);
let region = vk::ImageCopy {
src_subresource: vk::ImageSubresourceLayers {
aspect_mask: vk::ImageAspectFlags::COLOR,
mip_level: 0,
base_array_layer: 0,
layer_count: 1,
},
src_offset: vk::Offset3D::default(),
dst_subresource: vk::ImageSubresourceLayers {
aspect_mask: vk::ImageAspectFlags::COLOR,
mip_level: 0,
base_array_layer: 0,
layer_count: 1,
},
dst_offset: vk::Offset3D::default(),
extent: vk::Extent3D {
width,
height,
depth: 1,
},
};
logical_device.device.cmd_copy_image(
cmd,
src_image,
vk::ImageLayout::GENERAL,
dst_image,
vk::ImageLayout::GENERAL,
std::slice::from_ref(®ion),
);
let mem_barrier2 = vk::MemoryBarrier2::default()
.src_stage_mask(vk::PipelineStageFlags2::TRANSFER)
.src_access_mask(vk::AccessFlags2::TRANSFER_WRITE)
.dst_stage_mask(
vk::PipelineStageFlags2::COMPUTE_SHADER | vk::PipelineStageFlags2::ALL_COMMANDS,
)
.dst_access_mask(vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::SHADER_WRITE);
let dep2 = vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&mem_barrier2));
logical_device.device.cmd_pipeline_barrier2(cmd, &dep2);
}
}
GpuCommand::CopyBuffer {
src,
src_offset,
dst,
dst_offset,
size,
} => {
let _tz = tracy_zone!("vk.copy_buffer");
let (src_buf, dst_buf) = {
let buffers_read = view.buffers.read().unwrap();
let src_state = buffers_read.entries.get(src).context("CopyBuffer: invalid src")?;
let dst_state = buffers_read.entries.get(dst).context("CopyBuffer: invalid dst")?;
if src_offset.saturating_add(*size) > src_state.size
|| dst_offset.saturating_add(*size) > dst_state.size
{
anyhow::bail!("CopyBuffer: size exceeds buffer bounds");
}
(src_state.buffer, dst_state.buffer)
};
unsafe {
let mem_barrier = vk::MemoryBarrier2::default()
.src_stage_mask(vk::PipelineStageFlags2::COMPUTE_SHADER | vk::PipelineStageFlags2::TRANSFER)
.src_access_mask(vk::AccessFlags2::SHADER_WRITE | vk::AccessFlags2::TRANSFER_WRITE)
.dst_stage_mask(vk::PipelineStageFlags2::TRANSFER)
.dst_access_mask(vk::AccessFlags2::TRANSFER_READ | vk::AccessFlags2::TRANSFER_WRITE);
let dep_info =
vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&mem_barrier));
logical_device.device.cmd_pipeline_barrier2(cmd, &dep_info);
let region = vk::BufferCopy {
src_offset: *src_offset,
dst_offset: *dst_offset,
size: *size,
};
logical_device
.device
.cmd_copy_buffer(cmd, src_buf, dst_buf, std::slice::from_ref(®ion));
}
}
GpuCommand::CopyTextureToReadback { src, dst, layout } => {
let _tz = tracy_zone!("vk.copy_texture_to_readback");
let staging_buffer = {
let buffers_read = view.buffers.read().unwrap();
buffers_read
.entries
.get(dst)
.context("CopyTextureToReadback: invalid dst")?
.buffer
};
super::texture::record_copy_texture_to_readback(
cmd,
logical_device,
view.textures,
staging_buffer,
*src,
*layout,
)?;
}
GpuCommand::CopyRenderTarget { src, dst } => {
let _tz = tracy_zone!("vk.copy_render_target");
let (src_image, width, height, dst_image) = {
let render_targets_read = view.render_targets.read().unwrap();
let rt = render_targets_read
.entries
.get(src)
.context("CopyRenderTarget: src render target not found")?;
let textures_read = view.textures.read().unwrap();
let dst_image = textures_read
.entries
.get(dst)
.context("CopyRenderTarget: dst texture not found")?
.image;
(rt.image, rt.width, rt.height, dst_image)
};
unsafe {
let mem_barrier = vk::MemoryBarrier2::default()
.src_stage_mask(vk::PipelineStageFlags2::TRANSFER)
.src_access_mask(vk::AccessFlags2::TRANSFER_READ)
.dst_stage_mask(vk::PipelineStageFlags2::TRANSFER)
.dst_access_mask(vk::AccessFlags2::TRANSFER_WRITE);
let dep = vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&mem_barrier));
logical_device.device.cmd_pipeline_barrier2(cmd, &dep);
let region = vk::ImageCopy {
src_subresource: vk::ImageSubresourceLayers {
aspect_mask: vk::ImageAspectFlags::COLOR,
mip_level: 0,
base_array_layer: 0,
layer_count: 1,
},
src_offset: vk::Offset3D::default(),
dst_subresource: vk::ImageSubresourceLayers {
aspect_mask: vk::ImageAspectFlags::COLOR,
mip_level: 0,
base_array_layer: 0,
layer_count: 1,
},
dst_offset: vk::Offset3D::default(),
extent: vk::Extent3D {
width,
height,
depth: 1,
},
};
logical_device.device.cmd_copy_image(
cmd,
src_image,
vk::ImageLayout::TRANSFER_SRC_OPTIMAL,
dst_image,
vk::ImageLayout::GENERAL,
std::slice::from_ref(®ion),
);
let mem_barrier2 = vk::MemoryBarrier2::default()
.src_stage_mask(vk::PipelineStageFlags2::TRANSFER)
.src_access_mask(vk::AccessFlags2::TRANSFER_WRITE)
.dst_stage_mask(
vk::PipelineStageFlags2::COMPUTE_SHADER | vk::PipelineStageFlags2::ALL_COMMANDS,
)
.dst_access_mask(vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::SHADER_WRITE);
let dep2 = vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&mem_barrier2));
logical_device.device.cmd_pipeline_barrier2(cmd, &dep2);
}
}
},
GraphCommand::Render {
target,
color_load,
commands: render_cmds,
} => {
let _tz = tracy_zone!("vk.render_pass");
unsafe {
let barrier = vk::MemoryBarrier2::default()
.src_stage_mask(vk::PipelineStageFlags2::COMPUTE_SHADER | vk::PipelineStageFlags2::TRANSFER)
.src_access_mask(vk::AccessFlags2::SHADER_WRITE | vk::AccessFlags2::TRANSFER_WRITE)
.dst_stage_mask(
vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT
| vk::PipelineStageFlags2::EARLY_FRAGMENT_TESTS
| vk::PipelineStageFlags2::LATE_FRAGMENT_TESTS
| vk::PipelineStageFlags2::VERTEX_SHADER
| vk::PipelineStageFlags2::FRAGMENT_SHADER,
)
.dst_access_mask(
vk::AccessFlags2::COLOR_ATTACHMENT_WRITE
| vk::AccessFlags2::DEPTH_STENCIL_ATTACHMENT_WRITE
| vk::AccessFlags2::SHADER_READ
| vk::AccessFlags2::MEMORY_READ,
);
let dep = vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&barrier));
logical_device.device.cmd_pipeline_barrier2(cmd, &dep);
}
let (staging_data, lowered, has_render_bindings) =
super::frame_table::prepare_render_commands(buffers, view.pipelines, render_cmds)?;
if has_render_bindings {
if frame_table_prologue_in_cb {
let graph_staging = super::frame_table::extract_staging_from_graph(commands)
.map(|data| data.to_vec())
.unwrap_or_else(|| vec![0u32; crate::frame_table::FRAME_TABLE_TABLE_U32S]);
let sync_data =
super::frame_table::merge_staging_for_render_sync(&graph_staging, &staging_data);
super::frame_table::sync_table_row_to_device(
&scope.frame_table,
view.buffers,
logical_device,
cmd,
&sync_data,
route_device,
)?;
} else {
let row = super::frame_table::record_prologue(
view.contexts,
ctx,
super::frame_table::PrologueRecording {
frame_table: &scope.frame_table,
buffers: view.buffers,
ld: logical_device,
cmd,
on_graphics_queue: route_device,
},
&staging_data,
)?;
frame_table_row = Some(row);
row_guard.set(row);
}
}
super::render_target::record_render_pass_to_buffer(
view.devices,
view.render_targets,
device_handle,
*target,
*color_load,
&lowered,
cmd,
|cb, cmds, ld, cur_pipe| {
let pipelines_read = view.pipelines.read().unwrap();
let buffers_read = view.buffers.read().unwrap();
super::render_commands::record(
cb,
cmds,
ld,
&pipelines_read.entries,
&buffers_read.entries,
cur_pipe,
(scope.frame_table.selector_slot, scope.frame_table.table_slot),
)
},
)?;
unsafe {
let barrier = vk::MemoryBarrier2::default()
.src_stage_mask(
vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT
| vk::PipelineStageFlags2::LATE_FRAGMENT_TESTS,
)
.src_access_mask(
vk::AccessFlags2::COLOR_ATTACHMENT_WRITE | vk::AccessFlags2::DEPTH_STENCIL_ATTACHMENT_WRITE,
)
.dst_stage_mask(vk::PipelineStageFlags2::COMPUTE_SHADER | vk::PipelineStageFlags2::TRANSFER)
.dst_access_mask(
vk::AccessFlags2::SHADER_READ
| vk::AccessFlags2::SHADER_WRITE
| vk::AccessFlags2::TRANSFER_READ
| vk::AccessFlags2::TRANSFER_WRITE,
);
let dep = vk::DependencyInfo::default().memory_barriers(std::slice::from_ref(&barrier));
logical_device.device.cmd_pipeline_barrier2(cmd, &dep);
}
}
}
}
record_legacy_release_barrier(&logical_device.device, cmd, route_device);
if let Some(ref prof) = vk_gpu_profile {
unsafe {
logical_device
.device
.cmd_write_timestamp2(cmd, vk::PipelineStageFlags2::BOTTOM_OF_PIPE, prof.pool, 1);
}
}
if let Err(e) = unsafe { logical_device.device.end_command_buffer(cmd) } {
if route_device {
logical_device.recycle_device_cmd_buffer(cmd);
} else {
let mut sc = scope.sc.lock().unwrap();
sc.free_cmd_buffers.push(cmd);
}
return Err(anyhow::anyhow!("Failed to end command buffer: {:?}", e));
}
let used_slots =
collect_slot_keys_from_graph_commands(commands, view.compute_pipelines, view.pipelines, view.buffers);
let submit_device = view.devices.get(&device_handle).context("Invalid device handle")?;
let (queue, queue_lock) = if route_device {
device_graphics_queue_target(submit_device)
} else {
context_queue_target(scope)
};
let retain_plan = if let Some(key) = retain_key {
let ft = &scope.frame_table;
let pin_row_index = super::frame_table::extract_staging_from_graph(commands)
.is_some()
.then_some(frame_table_row)
.flatten();
if let Some(row) = pin_row_index {
super::frame_table::pin_row(ft, row)?;
}
Some((key, pin_row_index))
} else {
None
};
let signal_value = {
let _queue_guard = queue_lock.lock().unwrap();
if route_device {
super::context::reserve_device_owner_timeline_locked(submit_device)
} else {
let value = submit_device.timeline_next.fetch_add(1, Ordering::Relaxed);
register_submit_timeline(submit_device, value, false, ctx);
value
}
};
submit_device
.descriptors
.lock()
.unwrap()
.record_slot_usage(ctx, signal_value, used_slots.iter().copied());
let signal_infos = build_submit_signal_infos(scope, route_device, signal_value)?;
let timeline_sem = scope.sc.lock().unwrap().timeline_semaphore;
row_guard.commit(signal_value);
record_last_submitted(scope, route_device, signal_value);
{
let mut sc = scope.sc.lock().unwrap();
if let Some((key, frame_table_row)) = retain_plan {
let pin_slots = used_slots.clone();
let replaced = sc.retained_compute_cbs.insert(
key,
super::types::RetainedVkCb {
command_buffer: cmd,
used_slots,
frame_table_row,
last_signal_value: signal_value,
on_graphics_queue: route_device,
},
);
let unpin_slots = replaced.map(|old| old.used_slots).unwrap_or_default();
drop(sc);
if !unpin_slots.is_empty() || !pin_slots.is_empty() {
let ld = scope
.view
.devices
.get(&scope.device_handle)
.expect("submit scope device handle must exist");
let mut registry = ld.descriptors.lock().unwrap();
registry.unpin_retained_slots(unpin_slots);
registry.pin_retained_slots(pin_slots);
}
} else if route_device {
sc.graphics_timeline_cmd_buffers
.entry(signal_value)
.or_default()
.push(cmd);
} else {
sc.timeline_cmd_buffers.entry(signal_value).or_default().push(cmd);
}
}
let texture_entries: Vec<staging::TextureStagingEntry> =
texture_upload_scratch.into_iter().map(|s| s.entry).collect();
let gpu_profile_work = vk_gpu_profile.map(|prof| super::pending_submit::VulkanGpuProfileWork { ctx, cmd, prof });
enqueue_vulkan_compute_with_housekeeping(
scope,
device_handle,
queue,
queue_lock,
timeline_sem,
signal_value,
signal_infos,
Some(cmd),
sync,
true,
texture_entries,
gpu_profile_work,
)?;
Ok(signal_value)
}
pub(super) fn submit_graph_and_retain(
state: &super::types::VulkanState,
ctx: super::ContextHandle,
commands: &[GraphCommand],
key: u64,
sync: Option<&SubmitSync>,
) -> Result<TimelineValue> {
let scope = super::submit_session::scope_from_state(state, ctx)?;
evict_retained_with_scope(&scope, ctx, key);
submit_graph_with_scope(&scope, ctx, commands, Some(key), sync)
}
pub(super) fn try_resubmit_retained(
state: &super::types::VulkanState,
ctx: super::ContextHandle,
key: u64,
sync: Option<&SubmitSync>,
) -> Result<Option<TimelineValue>> {
try_resubmit_retained_with_scope(&super::submit_session::scope_from_state(state, ctx)?, ctx, key, sync)
}
pub(super) fn try_resubmit_retained_with_scope(
scope: &VulkanSubmitScope<'_>,
ctx: super::ContextHandle,
key: u64,
sync: Option<&SubmitSync>,
) -> Result<Option<TimelineValue>> {
scope.assert_ctx(ctx);
let view = &scope.view;
let device_handle = scope.device_handle;
let retained = {
let sc = scope.sc.lock().unwrap();
sc.retained_compute_cbs.get(&key).map(|r| {
(
r.command_buffer,
r.used_slots.clone(),
r.frame_table_row,
r.on_graphics_queue,
)
})
};
let Some((cmd, used_slots, frame_table_row, on_graphics_queue)) = retained else {
return Ok(None);
};
let submit_device = view.devices.get(&device_handle).context("Invalid device handle")?;
let (queue, queue_lock) = if on_graphics_queue {
device_graphics_queue_target(submit_device)
} else {
context_queue_target(scope)
};
let (signal_value, signal_infos) = {
let _tz = tracy_zone!("vk.resubmit_retained");
let _queue_guard = queue_lock.lock().unwrap();
let signal_value = if on_graphics_queue {
super::context::reserve_device_owner_timeline_locked(submit_device)
} else {
let value = submit_device.timeline_next.fetch_add(1, Ordering::Relaxed);
register_submit_timeline(submit_device, value, false, ctx);
value
};
let signal_infos = build_submit_signal_infos(scope, on_graphics_queue, signal_value)?;
(signal_value, signal_infos)
};
record_last_submitted(scope, on_graphics_queue, signal_value);
submit_device
.descriptors
.lock()
.unwrap()
.record_slot_usage(ctx, signal_value, used_slots);
{
let mut sc = scope.sc.lock().unwrap();
if let Some(retained) = sc.retained_compute_cbs.values_mut().find(|r| r.command_buffer == cmd) {
retained.last_signal_value = signal_value;
}
}
if let Some(row) = frame_table_row {
super::frame_table::record_submission(&scope.frame_table, row, signal_value);
}
let timeline_sem = scope.sc.lock().unwrap().timeline_semaphore;
enqueue_vulkan_compute_with_housekeeping(
scope,
device_handle,
queue,
queue_lock,
timeline_sem,
signal_value,
signal_infos,
Some(cmd),
sync,
false,
Vec::new(),
None,
)?;
Ok(Some(signal_value))
}
fn evict_retained_on_context(
frame_table: &super::frame_table::ContextFrameTable,
ld: &super::types::LogicalDevice,
ctx: super::ContextHandle,
key: u64,
contexts: &super::types::SharedContextMap,
) {
let removed = if let Some(sc_arc) = contexts.read().unwrap().get(&ctx) {
let mut sc = sc_arc.lock().unwrap();
sc.retained_compute_cbs.remove(&key)
} else {
None
};
if let Some(old) = removed {
ld.descriptors.lock().unwrap().unpin_retained_slots(old.used_slots);
if let Some(row) = old.frame_table_row {
super::frame_table::unpin_row(frame_table, row);
}
if let Some(sc_arc) = contexts.read().unwrap().get(&ctx) {
let mut sc = sc_arc.lock().unwrap();
let target = if old.on_graphics_queue {
sc.graphics_timeline_cmd_buffers
.entry(old.last_signal_value)
.or_default()
} else {
sc.timeline_cmd_buffers.entry(old.last_signal_value).or_default()
};
target.push(old.command_buffer);
}
}
}
pub(super) fn evict_retained_pinning_row_for_context(
contexts: &super::types::SharedContextMap,
frame_table: &super::frame_table::ContextFrameTable,
ld: &super::types::LogicalDevice,
ctx: super::ContextHandle,
row: u32,
) {
let keys: Vec<u64> = {
let contexts_read = contexts.read().unwrap();
let Some(sc_arc) = contexts_read.get(&ctx) else {
return;
};
let sc = sc_arc.lock().unwrap();
sc.retained_compute_cbs
.iter()
.filter(|(_, g)| g.frame_table_row == Some(row))
.map(|(k, _)| *k)
.collect()
};
for key in keys {
evict_retained_on_context(frame_table, ld, ctx, key, contexts);
}
}
pub(super) fn evict_retained_with_scope(scope: &VulkanSubmitScope<'_>, ctx: super::ContextHandle, key: u64) {
scope.assert_ctx(ctx);
let ld = scope
.view
.devices
.get(&scope.device_handle)
.expect("submit scope device handle must exist");
evict_retained_on_context(&scope.frame_table, ld, ctx, key, scope.view.contexts);
}
pub(super) fn evict_retained(state: &super::types::VulkanState, ctx: super::ContextHandle, key: u64) {
if let Ok(scope) = super::submit_session::scope_from_state(state, ctx) {
evict_retained_with_scope(&scope, ctx, key);
}
}
pub(super) fn evict_retained_graphs_using_slots(
state: &super::types::VulkanState,
device: super::DeviceHandle,
slots: &[super::types::SlotKey],
) {
if slots.is_empty() {
return;
}
let slot_set: std::collections::HashSet<_> = slots.iter().copied().collect();
let to_evict: Vec<(super::ContextHandle, u64)> = {
let contexts = state.contexts.read().unwrap();
let mut out = Vec::new();
for (&ctx, sc_arc) in contexts.iter() {
if super::context::context_device(state, ctx) != device {
continue;
}
let sc = sc_arc.lock().unwrap();
for (&key, retained) in &sc.retained_compute_cbs {
if retained.used_slots.iter().any(|s| slot_set.contains(s)) {
out.push((ctx, key));
}
}
}
out
};
for (ctx, key) in to_evict {
evict_retained(state, ctx, key);
}
}
fn reap_timeline_cmd_buffers_up_to_with_view(
view: &VulkanSubmitView<'_>,
ctx: super::ContextHandle,
max_completed_value: u64,
) {
let (device, _ctx_pool, ctx_keys, gfx_keys): (DeviceHandle, vk::CommandPool, Vec<u64>, Vec<u64>) = {
let contexts = view.contexts.read().unwrap();
let sc_arc = match contexts.get(&ctx) {
Some(s) => s,
None => return,
};
let sc = sc_arc.lock().unwrap();
if sc.timeline_cmd_buffers.is_empty() && sc.graphics_timeline_cmd_buffers.is_empty() {
return;
}
let ctx_keys: Vec<u64> = sc
.timeline_cmd_buffers
.keys()
.copied()
.filter(|k| *k <= max_completed_value)
.collect();
let gfx_keys: Vec<u64> = sc
.graphics_timeline_cmd_buffers
.keys()
.copied()
.filter(|k| *k <= max_completed_value)
.collect();
(sc.device, sc.command_pool, ctx_keys, gfx_keys)
};
let (ctx_cbs, gfx_cbs): (Vec<vk::CommandBuffer>, Vec<vk::CommandBuffer>) = {
let contexts = view.contexts.read().unwrap();
let sc_arc = contexts.get(&ctx).expect("context");
let mut sc = sc_arc.lock().unwrap();
let mut ctx_cbs = Vec::new();
let mut gfx_cbs = Vec::new();
for k in &ctx_keys {
if let Some(cbs) = sc.timeline_cmd_buffers.remove(k) {
ctx_cbs.extend(cbs);
}
}
for k in &gfx_keys {
if let Some(cbs) = sc.graphics_timeline_cmd_buffers.remove(k) {
gfx_cbs.extend(cbs);
}
}
(ctx_cbs, gfx_cbs)
};
if let Some(ld) = view.devices.get(&device) {
if !ctx_cbs.is_empty() {
if let Some(sc_arc) = view.contexts.read().unwrap().get(&ctx) {
sc_arc.lock().unwrap().free_cmd_buffers.extend(ctx_cbs);
}
}
if !gfx_cbs.is_empty() {
ld.recycle_device_cmd_buffers(&gfx_cbs);
}
}
}
pub(super) fn reap_timeline_cmd_buffers_up_to(
state: &super::types::VulkanState,
ctx: super::ContextHandle,
max_completed_value: u64,
) {
reap_timeline_cmd_buffers_up_to_with_view(&state.submit_view(), ctx, max_completed_value);
}