pub use self::{
fence_signal::{FenceSignalFuture, FenceSignalFutureBehavior},
join::JoinFuture,
now::{now, NowFuture},
semaphore_signal::SemaphoreSignalFuture,
};
use super::{fence::Fence, semaphore::Semaphore};
use crate::{
buffer::{Buffer, BufferState},
command_buffer::{
CommandBufferExecError, CommandBufferExecFuture, CommandBufferResourcesUsage,
CommandBufferState, CommandBufferSubmitInfo, CommandBufferUsage,
PrimaryCommandBufferAbstract, SubmitInfo,
},
device::{DeviceOwned, Queue},
image::{Image, ImageLayout, ImageState},
memory::sparse::BindSparseInfo,
swapchain::{self, PresentFuture, PresentInfo, Swapchain, SwapchainPresentInfo},
DeviceSize, Validated, ValidationError, VulkanError, VulkanObject,
};
use foldhash::HashMap;
use parking_lot::MutexGuard;
use smallvec::{smallvec, SmallVec};
use std::{
error::Error,
fmt::{Display, Error as FmtError, Formatter},
ops::Range,
sync::{atomic::Ordering, Arc},
};
mod fence_signal;
mod join;
mod now;
mod semaphore_signal;
pub unsafe trait GpuFuture: DeviceOwned {
fn cleanup_finished(&mut self);
unsafe fn build_submission(&self) -> Result<SubmitAnyBuilder, Validated<VulkanError>>;
fn flush(&self) -> Result<(), Validated<VulkanError>>;
unsafe fn signal_finished(&self);
fn queue(&self) -> Option<Arc<Queue>>;
fn queue_change_allowed(&self) -> bool;
fn check_buffer_access(
&self,
buffer: &Buffer,
range: Range<DeviceSize>,
exclusive: bool,
queue: &Queue,
) -> Result<(), AccessCheckError>;
fn check_image_access(
&self,
image: &Image,
range: Range<DeviceSize>,
exclusive: bool,
expected_layout: ImageLayout,
queue: &Queue,
) -> Result<(), AccessCheckError>;
fn check_swapchain_image_acquired(
&self,
swapchain: &Swapchain,
image_index: u32,
before: bool,
) -> Result<(), AccessCheckError>;
fn join<F>(self, other: F) -> JoinFuture<Self, F>
where
Self: Sized,
F: GpuFuture,
{
join::join(self, other)
}
fn then_execute(
self,
queue: Arc<Queue>,
command_buffer: Arc<impl PrimaryCommandBufferAbstract + 'static>,
) -> Result<CommandBufferExecFuture<Self>, CommandBufferExecError>
where
Self: Sized,
{
command_buffer.execute_after(self, queue)
}
fn then_execute_same_queue(
self,
command_buffer: Arc<impl PrimaryCommandBufferAbstract + 'static>,
) -> Result<CommandBufferExecFuture<Self>, CommandBufferExecError>
where
Self: Sized,
{
let queue = self.queue().unwrap();
command_buffer.execute_after(self, queue)
}
#[inline]
fn then_signal_semaphore(self) -> SemaphoreSignalFuture<Self>
where
Self: Sized,
{
semaphore_signal::then_signal_semaphore(self)
}
#[inline]
fn then_signal_semaphore_and_flush(
self,
) -> Result<SemaphoreSignalFuture<Self>, Validated<VulkanError>>
where
Self: Sized,
{
let f = self.then_signal_semaphore();
f.flush()?;
Ok(f)
}
#[inline]
fn then_signal_fence(self) -> FenceSignalFuture<Self>
where
Self: Sized,
{
fence_signal::then_signal_fence(self, FenceSignalFutureBehavior::Continue)
}
#[inline]
fn then_signal_fence_and_flush(self) -> Result<FenceSignalFuture<Self>, Validated<VulkanError>>
where
Self: Sized,
{
let f = self.then_signal_fence();
f.flush()?;
Ok(f)
}
#[inline]
fn then_swapchain_present(
self,
queue: Arc<Queue>,
swapchain_info: SwapchainPresentInfo,
) -> PresentFuture<Self>
where
Self: Sized,
{
swapchain::present(self, queue, swapchain_info)
}
#[inline]
fn boxed(self) -> Box<dyn GpuFuture>
where
Self: Sized + 'static,
{
Box::new(self) as _
}
#[inline]
fn boxed_send(self) -> Box<dyn GpuFuture + Send>
where
Self: Sized + Send + 'static,
{
Box::new(self) as _
}
#[inline]
fn boxed_sync(self) -> Box<dyn GpuFuture + Sync>
where
Self: Sized + Sync + 'static,
{
Box::new(self) as _
}
#[inline]
fn boxed_send_sync(self) -> Box<dyn GpuFuture + Send + Sync>
where
Self: Sized + Send + Sync + 'static,
{
Box::new(self) as _
}
}
unsafe impl<F: ?Sized> GpuFuture for Box<F>
where
F: GpuFuture,
{
fn cleanup_finished(&mut self) {
(**self).cleanup_finished()
}
unsafe fn build_submission(&self) -> Result<SubmitAnyBuilder, Validated<VulkanError>> {
unsafe { (**self).build_submission() }
}
fn flush(&self) -> Result<(), Validated<VulkanError>> {
(**self).flush()
}
unsafe fn signal_finished(&self) {
unsafe { (**self).signal_finished() }
}
fn queue_change_allowed(&self) -> bool {
(**self).queue_change_allowed()
}
fn queue(&self) -> Option<Arc<Queue>> {
(**self).queue()
}
fn check_buffer_access(
&self,
buffer: &Buffer,
range: Range<DeviceSize>,
exclusive: bool,
queue: &Queue,
) -> Result<(), AccessCheckError> {
(**self).check_buffer_access(buffer, range, exclusive, queue)
}
fn check_image_access(
&self,
image: &Image,
range: Range<DeviceSize>,
exclusive: bool,
expected_layout: ImageLayout,
queue: &Queue,
) -> Result<(), AccessCheckError> {
(**self).check_image_access(image, range, exclusive, expected_layout, queue)
}
#[inline]
fn check_swapchain_image_acquired(
&self,
swapchain: &Swapchain,
image_index: u32,
before: bool,
) -> Result<(), AccessCheckError> {
(**self).check_swapchain_image_acquired(swapchain, image_index, before)
}
}
#[derive(Debug)]
pub enum SubmitAnyBuilder {
Empty,
SemaphoresWait(SmallVec<[Arc<Semaphore>; 8]>),
CommandBuffer(SubmitInfo, Option<Arc<Fence>>),
QueuePresent(PresentInfo),
BindSparse(SmallVec<[BindSparseInfo; 1]>, Option<Arc<Fence>>),
}
impl SubmitAnyBuilder {
#[inline]
pub fn is_empty(&self) -> bool {
matches!(self, SubmitAnyBuilder::Empty)
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum AccessError {
AlreadyInUse,
UnexpectedImageLayout {
allowed: ImageLayout,
requested: ImageLayout,
},
ImageNotInitialized {
requested: ImageLayout,
},
SwapchainImageNotAcquired,
}
impl Error for AccessError {}
impl Display for AccessError {
fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), FmtError> {
let value = match self {
AccessError::AlreadyInUse => {
"the resource is already in use, and there is no tracking of concurrent usages"
}
AccessError::UnexpectedImageLayout { allowed, requested } => {
return write!(
f,
"unexpected image layout: requested {:?}, allowed {:?}",
allowed, requested
)
}
AccessError::ImageNotInitialized { .. } => {
"trying to use an image without transitioning it from the undefined or \
preinitialized layouts first"
}
AccessError::SwapchainImageNotAcquired => {
"trying to use a swapchain image without depending on a corresponding acquire \
image future"
}
};
write!(f, "{}", value,)
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum AccessCheckError {
Denied(AccessError),
Unknown,
}
impl Error for AccessCheckError {}
impl Display for AccessCheckError {
fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), FmtError> {
match self {
AccessCheckError::Denied(err) => {
write!(f, "access to the resource has been denied: {}", err)
}
AccessCheckError::Unknown => write!(f, "the resource is unknown"),
}
}
}
impl From<AccessError> for AccessCheckError {
fn from(err: AccessError) -> AccessCheckError {
AccessCheckError::Denied(err)
}
}
pub(crate) unsafe fn queue_bind_sparse(
queue: &Arc<Queue>,
bind_infos: impl IntoIterator<Item = BindSparseInfo>,
fence: Option<Arc<Fence>>,
) -> Result<(), Validated<VulkanError>> {
let bind_infos: SmallVec<[_; 4]> = bind_infos.into_iter().collect();
queue
.with(|mut queue_guard| unsafe { queue_guard.bind_sparse(&bind_infos, fence.as_ref()) })?;
Ok(())
}
pub(crate) unsafe fn queue_present(
queue: &Arc<Queue>,
present_info: PresentInfo,
) -> Result<impl ExactSizeIterator<Item = Result<bool, VulkanError>>, Validated<VulkanError>> {
let results: SmallVec<[_; 1]> = queue
.with(|mut queue_guard| unsafe { queue_guard.present(&present_info) })?
.collect();
let PresentInfo {
wait_semaphores: _,
swapchain_infos: swapchains,
_ne: _,
} = &present_info;
for (&result, swapchain_info) in results.iter().zip(swapchains) {
if result == Err(VulkanError::FullScreenExclusiveModeLost) {
unsafe { swapchain_info.swapchain.full_screen_exclusive_held() }
.store(false, Ordering::SeqCst);
}
}
Ok(results.into_iter())
}
pub(crate) unsafe fn queue_submit(
queue: &Arc<Queue>,
submit_info: SubmitInfo,
fence: Option<Arc<Fence>>,
future: &dyn GpuFuture,
) -> Result<(), Validated<VulkanError>> {
let submit_infos: SmallVec<[_; 4]> = smallvec![submit_info];
let mut states = States::from_submit_infos(&submit_infos);
for submit_info in &submit_infos {
for command_buffer_submit_info in &submit_info.command_buffers {
let &CommandBufferSubmitInfo {
ref command_buffer,
_ne: _,
} = command_buffer_submit_info;
let state = states
.command_buffers
.get(&command_buffer.handle())
.unwrap();
match command_buffer.usage() {
CommandBufferUsage::OneTimeSubmit => {
if state.has_been_submitted() {
return Err(Box::new(ValidationError {
problem: "a command buffer, or one of the secondary \
command buffers it executes, was created with the \
`CommandBufferUsage::OneTimeSubmit` usage, but \
it has already been submitted in the past"
.into(),
vuids: &["VUID-vkQueueSubmit2-commandBuffer-03874"],
..Default::default()
})
.into());
}
}
CommandBufferUsage::MultipleSubmit => {
if state.is_submit_pending() {
return Err(Box::new(ValidationError {
problem: "a command buffer, or one of the secondary \
command buffers it executes, was not created with the \
`CommandBufferUsage::SimultaneousUse` usage, but \
it is already in use by the device"
.into(),
vuids: &["VUID-vkQueueSubmit2-commandBuffer-03875"],
..Default::default()
})
.into());
}
}
CommandBufferUsage::SimultaneousUse => (),
}
let CommandBufferResourcesUsage {
buffers,
images,
buffer_indices: _,
image_indices: _,
} = command_buffer.resources_usage();
for usage in buffers {
let state = states.buffers.get_mut(&usage.buffer.handle()).unwrap();
for (range, range_usage) in usage.ranges.iter() {
match future.check_buffer_access(
&usage.buffer,
range.clone(),
range_usage.mutable,
queue,
) {
Err(AccessCheckError::Denied(error)) => {
return Err(Box::new(ValidationError {
problem: format!(
"access to a resource has been denied \
(resource use: {:?}, error: {})",
range_usage.first_use, error
)
.into(),
..Default::default()
})
.into());
}
Err(AccessCheckError::Unknown) => {
let result = if range_usage.mutable {
state.check_gpu_write(range.clone())
} else {
state.check_gpu_read(range.clone())
};
if let Err(error) = result {
return Err(Box::new(ValidationError {
problem: format!(
"access to a resource has been denied \
(resource use: {:?}, error: {})",
range_usage.first_use, error
)
.into(),
..Default::default()
})
.into());
}
}
_ => (),
}
}
}
for usage in images {
let state = states.images.get_mut(&usage.image.handle()).unwrap();
for (range, range_usage) in usage.ranges.iter() {
match future.check_image_access(
&usage.image,
range.clone(),
range_usage.mutable,
range_usage.expected_layout,
queue,
) {
Err(AccessCheckError::Denied(error)) => {
return Err(Box::new(ValidationError {
problem: format!(
"access to a resource has been denied \
(resource use: {:?}, error: {})",
range_usage.first_use, error
)
.into(),
..Default::default()
})
.into());
}
Err(AccessCheckError::Unknown) => {
let result = if range_usage.mutable {
state.check_gpu_write(range.clone(), range_usage.expected_layout)
} else {
state.check_gpu_read(range.clone(), range_usage.expected_layout)
};
if let Err(error) = result {
return Err(Box::new(ValidationError {
problem: format!(
"access to a resource has been denied \
(resource use: {:?}, error: {})",
range_usage.first_use, error
)
.into(),
..Default::default()
})
.into());
}
}
_ => (),
};
}
}
}
}
queue.with(|mut queue_guard| unsafe { queue_guard.submit(&submit_infos, fence.as_ref()) })?;
for submit_info in &submit_infos {
let SubmitInfo {
wait_semaphores: _,
command_buffers,
signal_semaphores: _,
_ne: _,
} = submit_info;
for command_buffer_submit_info in command_buffers {
let CommandBufferSubmitInfo {
command_buffer,
_ne: _,
} = command_buffer_submit_info;
let state = states
.command_buffers
.get_mut(&command_buffer.handle())
.unwrap();
unsafe { state.add_queue_submit() };
let CommandBufferResourcesUsage {
buffers,
images,
buffer_indices: _,
image_indices: _,
} = command_buffer.resources_usage();
for usage in buffers {
let state = states.buffers.get_mut(&usage.buffer.handle()).unwrap();
for (range, range_usage) in usage.ranges.iter() {
if range_usage.mutable {
unsafe { state.gpu_write_lock(range.clone()) };
} else {
unsafe { state.gpu_read_lock(range.clone()) };
}
}
}
for usage in images {
let state = states.images.get_mut(&usage.image.handle()).unwrap();
for (range, range_usage) in usage.ranges.iter() {
if range_usage.mutable {
unsafe { state.gpu_write_lock(range.clone(), range_usage.final_layout) };
} else {
unsafe { state.gpu_read_lock(range.clone()) };
}
}
}
}
}
Ok(())
}
#[derive(Debug)]
struct States<'a> {
buffers: HashMap<ash::vk::Buffer, MutexGuard<'a, BufferState>>,
command_buffers: HashMap<ash::vk::CommandBuffer, MutexGuard<'a, CommandBufferState>>,
images: HashMap<ash::vk::Image, MutexGuard<'a, ImageState>>,
}
impl<'a> States<'a> {
fn from_submit_infos(submit_infos: &'a [SubmitInfo]) -> Self {
let mut buffers = HashMap::default();
let mut command_buffers = HashMap::default();
let mut images = HashMap::default();
for submit_info in submit_infos {
let SubmitInfo {
wait_semaphores: _,
command_buffers: info_command_buffers,
signal_semaphores: _,
_ne: _,
} = submit_info;
for command_buffer_submit_info in info_command_buffers {
let &CommandBufferSubmitInfo {
ref command_buffer,
_ne: _,
} = command_buffer_submit_info;
command_buffers
.entry(command_buffer.handle())
.or_insert_with(|| command_buffer.state());
let CommandBufferResourcesUsage {
buffers: buffers_usage,
images: images_usage,
buffer_indices: _,
image_indices: _,
} = command_buffer.resources_usage();
for usage in buffers_usage {
let buffer = &usage.buffer;
buffers
.entry(buffer.handle())
.or_insert_with(|| buffer.state());
}
for usage in images_usage {
let image = &usage.image;
images
.entry(image.handle())
.or_insert_with(|| image.state());
}
}
}
Self {
buffers,
command_buffers,
images,
}
}
}