use super::*;
struct Requirements {
memory: vk::MemoryRequirements,
dedication: Dedication,
dedicated_supported: bool,
}
unsafe fn requirements(
allocator: &Allocator<'_>,
resource: Resource,
) -> Result<Requirements, AllocError> {
unsafe { requirements_with_plane(allocator, resource, None) }
}
unsafe fn requirements_with_plane(
allocator: &Allocator<'_>,
resource: Resource,
plane: Option<vk::ImageAspectFlagBits>,
) -> Result<Requirements, AllocError> {
let route = allocator.requirements_route(matches!(resource, Resource::Image(_)))?;
unsafe { query_requirements(allocator.device.as_device(), route, resource, plane) }
}
unsafe fn query_requirements(
device: &vk::Device,
route: RequirementsRoute,
resource: Resource,
plane: Option<vk::ImageAspectFlagBits>,
) -> Result<Requirements, AllocError> {
if plane.is_some() && matches!(route, RequirementsRoute::Legacy) {
return Err(AllocError::UnsupportedCapability(
"disjoint planes require memory requirements2",
));
}
let plane_info = vk::ImagePlaneMemoryRequirementsInfo {
plane_aspect: plane.unwrap_or_default(),
..Default::default()
};
let mut dedication = vk::MemoryDedicatedRequirements::new();
let mut output = vk::MemoryRequirements2 {
p_next: std::ptr::from_mut(&mut dedication).cast(),
..Default::default()
};
unsafe {
match resource {
Resource::Buffer(buffer) => {
let query = vk::BufferMemoryRequirementsInfo2 {
buffer,
..Default::default()
};
match route {
RequirementsRoute::Legacy => {
output.memory_requirements = device.get_buffer_memory_requirements(buffer)?
}
RequirementsRoute::Core => device.get_buffer_memory_requirements2(&query, &mut output)?,
RequirementsRoute::Khr => {
device.get_buffer_memory_requirements2_khr(&query, &mut output)?
}
}
}
Resource::Image(image) => {
let query = vk::ImageMemoryRequirementsInfo2 {
p_next: if plane.is_some() {
std::ptr::from_ref(&plane_info).cast()
} else {
std::ptr::null()
},
image,
..Default::default()
};
match route {
RequirementsRoute::Legacy => {
device.get_image_memory_requirements(image, &mut output.memory_requirements)?
}
RequirementsRoute::Core => device.get_image_memory_requirements2(&query, &mut output)?,
RequirementsRoute::Khr => {
device.get_image_memory_requirements2_khr(&query, &mut output)?
}
}
}
}
}
if output.memory_requirements.size == 0 || !output.memory_requirements.alignment.is_power_of_two()
{
return Err(AllocError::InvalidRequest(
"invalid native resource requirements",
));
}
Ok(Requirements {
memory: output.memory_requirements,
dedication: Dedication::new(&dedication),
dedicated_supported: !matches!(route, RequirementsRoute::Legacy),
})
}
pub(in crate::mem) fn plane_slot(plane: Option<vk::ImageAspectFlagBits>) -> usize {
match plane {
Some(vk::ImageAspectFlagBits::PLANE_0_BIT) => 1,
Some(vk::ImageAspectFlagBits::PLANE_1_BIT) => 2,
Some(vk::ImageAspectFlagBits::PLANE_2_BIT) => 3,
None => 0,
_ => unreachable!("canonical binding validates plane aspects"),
}
}
pub(super) struct Associations<'device> {
buffers: HostMap<'device, [u64; 4]>,
images: HostMap<'device, [u64; 4]>,
}
impl<'device> Associations<'device> {
pub fn new(host: HostMemory<'device>) -> Self {
Self {
buffers: HostMap::new(host),
images: HostMap::new(host),
}
}
fn registry(&self, resource: Resource) -> (&HostMap<'device, [u64; 4]>, u64) {
match resource {
Resource::Buffer(buffer) => (&self.buffers, buffer.as_raw()),
Resource::Image(image) => (&self.images, image.as_raw()),
}
}
fn registry_mut(&mut self, resource: Resource) -> (&mut HostMap<'device, [u64; 4]>, u64) {
match resource {
Resource::Buffer(buffer) => (&mut self.buffers, buffer.as_raw()),
Resource::Image(image) => (&mut self.images, image.as_raw()),
}
}
pub fn associated(&self, resource: Resource) -> bool {
let (registry, key) = self.registry(resource);
registry.get(&key).is_some()
}
pub fn conflicts(&self, resource: Resource, plane: Option<vk::ImageAspectFlagBits>) -> bool {
let (registry, key) = self.registry(resource);
let Some(slots) = registry.get(&key) else {
return false;
};
match plane {
None => true,
Some(_) => slots[0] != 0 || slots[plane_slot(plane)] != 0,
}
}
pub fn try_reserve_for(&mut self, resource: Resource) -> Result<(), AllocError> {
let (registry, _) = self.registry_mut(resource);
registry.try_reserve(1)
}
pub fn try_reserve(&mut self, additional: usize) -> Result<(), AllocError> {
self.buffers.try_reserve(additional)?;
self.images.try_reserve(additional)
}
pub fn record(
&mut self,
resource: Resource,
plane: Option<vk::ImageAspectFlagBits>,
serial: u64,
) {
let (registry, key) = self.registry_mut(resource);
registry.get_or_insert_reserved_with(key, || [0; 4])[plane_slot(plane)] = serial;
}
pub fn release(
&mut self,
resource: Resource,
plane: Option<vk::ImageAspectFlagBits>,
serial: u64,
) {
let (registry, key) = self.registry_mut(resource);
registry.remove_if(&key, |slots| {
let slot = plane_slot(plane);
if slots[slot] == serial {
slots[slot] = 0;
}
slots.iter().all(|owner| *owner == 0)
});
}
}
pub(super) fn associated(allocator: &Allocator<'_>, resource: Resource) -> bool {
if let Some(shared) = allocator.parallel_state() {
let domain = allocator.partition_id;
if shared.associated(resource, domain) {
return true;
}
}
allocator.associated.associated(resource)
}
pub(super) unsafe fn allocate_prepared_resource(
allocator: &mut Allocator<'_>,
resource: Resource,
options: &AllocationCreateInfo,
preferences: (
vk::MemoryPropertyFlags,
vk::MemoryPropertyFlags,
vk::MemoryPropertyFlags,
),
class: ResourceClass,
device_address: bool,
) -> Result<u64, AllocError> {
let (required, preferred, avoided) = preferences;
if allocator.parallel.is_none() {
allocator.associated.try_reserve_for(resource)?;
}
let claim = parallel::claim(allocator, resource, None, None)?;
let imported = options
.memory_allocate_next
.and_then(|chain| chain.data.import_info);
let requirements = if let Some(info) = imported.filter(|info| info.resource.is_some()) {
if import_resource(info.resource) != Some(resource) {
return Err(AllocError::WrongResource);
}
Requirements {
memory: vk::MemoryRequirements {
size: info.allocation_size,
alignment: 1,
memory_type_bits: info.memory_type_bits,
},
dedication: Dedication::Required,
dedicated_supported: true,
}
} else {
if options
.memory_allocate_next
.is_some_and(|chain| chain.data.import_kind.requires_info())
{
return Err(AllocError::InvalidRequest(
"platform resource imports require exact resource metadata",
));
}
unsafe { requirements(allocator, resource) }?
};
let request = AllocRequest {
size: requirements.memory.size,
alignment: requirements
.memory
.alignment
.max(options.min_allocation_alignment),
memory_type_bits: requirements.memory.memory_type_bits & options.memory_type_bits,
required_flags: required,
preferred_flags: preferred,
};
let mut dedicated = vk::MemoryDedicatedAllocateInfo::new();
match resource {
Resource::Buffer(buffer) => dedicated.buffer = buffer,
Resource::Image(image) => dedicated.image = image,
}
let serial = unsafe {
allocate_managed(
allocator,
&request,
StorageOptions {
chain: &super::super::chain::ChainData::EMPTY,
dedicated: requirements.dedicated_supported.then_some(&dedicated),
device_address,
priority: options.priority,
within_budget: options.within_budget,
},
requirements.dedication,
options,
class,
avoided,
)
}?;
let live = allocator.live.get_mut(&serial).expect("new allocation");
live.resource = Some(resource);
live.class = if options.can_alias {
ResourceClass::Unknown
} else {
class
};
live.can_alias = options.can_alias;
if let Some(claim) = claim {
claim.commit(serial);
} else {
allocator.associated.record(resource, None, serial);
}
Ok(serial)
}
pub(super) unsafe fn persistent(
allocator: &mut Allocator<'_>,
allocation: &Allocation,
options: &AllocationCreateInfo,
) -> Result<(), AllocError> {
unsafe { guard::initialize(allocator, allocation) }?;
if options.persistent_map {
unsafe { map(allocator, allocation) }?;
let live = allocator
.live
.get_mut(&allocation.serial)
.expect("new allocation");
live.map_count = 0;
live.persistent = true;
}
Ok(())
}
unsafe fn allocate_existing(
allocator: &mut Allocator<'_>,
resource: Resource,
options: &AllocationCreateInfo,
) -> Result<Allocation, AllocError> {
allocator.ensure_open()?;
if associated(allocator, resource) {
return Err(AllocError::InvalidRequest(
"resource already has an allocation",
));
}
if options.usage != super::super::MemoryUsage::Explicit {
return Err(AllocError::UnsupportedCapability(
"existing resource usage is unknown; use explicit memory flags",
));
}
let preferences = super::super::policy::preferences(options, false, allocator.integrated)?;
let class = match resource {
Resource::Buffer(_) => ResourceClass::Buffer,
Resource::Image(_) => ResourceClass::UnknownImage,
};
let address = matches!(resource, Resource::Buffer(_)) && allocator.buffer_device_address;
if resource.is_null() {
return Err(AllocError::InvalidRequest("null resource"));
}
request::validate_options(allocator, options, preferences.0)?;
if options.persistent_map {
require(allocator.device.fp().map_memory.is_some(), "vkMapMemory")?;
require(
allocator.device.fp().unmap_memory.is_some(),
"vkUnmapMemory",
)?;
}
allocator.reserve()?;
let serial = unsafe {
allocate_prepared_resource(allocator, resource, options, preferences, class, address)
}?;
let mut allocation = allocator.published(serial);
if let Err(error) = unsafe { persistent(allocator, &allocation, options) } {
unsafe { rollback_allocation(allocator, &mut allocation) }
.expect("preflighted allocation rollback");
return Err(error);
}
Ok(allocation)
}
pub unsafe fn allocate_for_buffer(
allocator: &mut Allocator<'_>,
buffer: vk::Buffer,
options: &AllocationCreateInfo,
) -> Result<Allocation, AllocError> {
unsafe { allocate_existing(allocator, Resource::Buffer(buffer), options) }
}
pub unsafe fn allocate_for_image(
allocator: &mut Allocator<'_>,
image: vk::Image,
options: &AllocationCreateInfo,
) -> Result<Allocation, AllocError> {
unsafe { allocate_existing(allocator, Resource::Image(image), options) }
}
struct BindChain {
status: Option<*mut vk::Result>,
plane: Option<vk::ImageAspectFlagBits>,
}
#[inline]
unsafe fn binding_status(next: *const std::ffi::c_void) -> Result<BindChain, AllocError> {
if next.is_null() {
return Ok(BindChain {
status: None,
plane: None,
});
}
unsafe { nonempty_binding_status(next) }
}
#[inline(never)]
unsafe fn nonempty_binding_status(
mut next: *const std::ffi::c_void,
) -> Result<BindChain, AllocError> {
let mut seen = [std::ptr::null(); 64];
let mut count = 0;
let mut status = None;
let mut plane = None;
let mut tags = [vk::StructureType::from_raw(0); 64];
while !next.is_null() {
if count == seen.len() || seen[..count].contains(&next) {
return Err(AllocError::InvalidRequest("cyclic/excessive bind chain"));
}
seen[count] = next;
let node = unsafe { &*next.cast::<vk::BaseInStructure>() };
if tags[..count].contains(&node.s_type) {
return Err(AllocError::InvalidRequest("duplicate bind chain record"));
}
tags[count] = node.s_type;
count += 1;
if node.s_type == vk::StructureType::BIND_IMAGE_PLANE_MEMORY_INFO {
let aspect = unsafe { (*next.cast::<vk::BindImagePlaneMemoryInfo>()).plane_aspect };
use crate::ImageAspectFlagBits as A;
if !matches!(aspect, A::PLANE_0_BIT | A::PLANE_1_BIT | A::PLANE_2_BIT) {
return Err(AllocError::UnsupportedCapability(
"ordinary disjoint image plane aspect required",
));
}
plane = Some(aspect);
}
if node.s_type == vk::StructureType::BIND_MEMORY_STATUS {
let pointer = unsafe { (*next.cast::<vk::BindMemoryStatus>()).p_result };
if status.is_some() || pointer.is_null() {
return Err(AllocError::InvalidRequest(
"duplicate/null bind status output",
));
}
status = Some(pointer);
}
next = node.p_next.cast();
}
Ok(BindChain { status, plane })
}
pub unsafe fn bind_buffer_memory2(
allocator: &mut Allocator<'_>,
buffer: vk::Buffer,
allocation: &Allocation,
local_offset: u64,
next: *const std::ffi::c_void,
) -> Result<(), AllocError> {
unsafe {
bind_resource(
allocator,
Resource::Buffer(buffer),
allocation,
local_offset,
next,
true,
false,
)
}
}
pub unsafe fn bind_image_memory2(
allocator: &mut Allocator<'_>,
image: vk::Image,
allocation: &Allocation,
local_offset: u64,
next: *const std::ffi::c_void,
) -> Result<(), AllocError> {
unsafe {
bind_resource(
allocator,
Resource::Image(image),
allocation,
local_offset,
next,
true,
false,
)
}
}
pub(super) unsafe fn bind_alias(
allocator: &mut Allocator<'_>,
resource: Resource,
allocation: &Allocation,
local_offset: u64,
) -> Result<(), AllocError> {
unsafe {
bind_resource(
allocator,
resource,
allocation,
local_offset,
std::ptr::null(),
false,
true,
)
}
}
unsafe fn bind_resource(
allocator: &mut Allocator<'_>,
resource: Resource,
allocation: &Allocation,
local_offset: u64,
next: *const std::ffi::c_void,
extended: bool,
alias: bool,
) -> Result<(), AllocError> {
let operation = unsafe {
prepare_bind(
allocator,
resource,
allocation,
local_offset,
next,
extended,
alias,
)
}?;
let backing = allocator.mapping(allocation);
unsafe { operation.execute(allocator.device.as_device()) }?;
drop(backing);
operation.commit(allocator)
}
pub(super) unsafe fn bind_prepared(
allocator: &mut Allocator<'_>,
resource: Resource,
allocation: &Allocation,
) -> Result<(), AllocError> {
let operation = BindOperation {
route: RequirementsRoute::Legacy,
requirements_route: None,
resource,
serial: allocation.serial,
handle: allocation.handle,
native_offset: allocation.offset,
remaining_size: allocation.size,
memory_type_index: allocation.memory_type_index,
next: std::ptr::null(),
status: None,
plane: None,
alias: false,
claim: None,
};
let backing = allocator.mapping(allocation);
unsafe { operation.execute(allocator.device.as_device()) }?;
drop(backing);
operation.commit(allocator)
}
pub(super) struct BindOperation<'device> {
route: RequirementsRoute,
requirements_route: Option<RequirementsRoute>,
resource: Resource,
serial: u64,
handle: vk::DeviceMemory,
native_offset: u64,
remaining_size: u64,
memory_type_index: u32,
next: *const std::ffi::c_void,
status: Option<*mut vk::Result>,
pub(super) plane: Option<vk::ImageAspectFlagBits>,
alias: bool,
claim: Option<parallel::resource::Claim<'device>>,
}
pub(super) unsafe fn prepare_bind<'device>(
allocator: &mut Allocator<'device>,
resource: Resource,
allocation: &Allocation,
local_offset: u64,
next: *const std::ffi::c_void,
extended: bool,
alias: bool,
) -> Result<BindOperation<'device>, AllocError> {
let route = if !extended {
RequirementsRoute::Legacy
} else {
let capabilities =
allocator
.device
.as_device()
.capabilities()
.ok_or(AllocError::UnsupportedCapability(
"extended binding requires creation metadata",
))?;
if capabilities.api_version() >= vk::API_VERSION_1_1 {
RequirementsRoute::Core
} else if super::super::resource::extension(
allocator.device.as_device(),
c"VK_KHR_bind_memory2",
) {
RequirementsRoute::Khr
} else {
return Err(AllocError::UnsupportedCapability(
"bind_memory2 is not enabled",
));
}
};
let BindChain { status, plane } = unsafe { binding_status(next) }?;
let live = allocator.validate(allocation)?;
if live.native_users != 0 {
return Err(AllocError::InvalidRequest("native operation in progress"));
}
if plane.is_some() {
if !extended || !matches!(resource, Resource::Image(_)) || live.resource.is_some() || alias {
return Err(AllocError::InvalidRequest(
"plane binding requires unassociated raw storage and image bind2",
));
}
if !allocator
.device
.as_device()
.capabilities()
.is_some_and(|caps| caps.api_version() >= vk::API_VERSION_1_1)
&& !super::super::resource::extension(
allocator.device.as_device(),
c"VK_KHR_sampler_ycbcr_conversion",
) {
return Err(AllocError::UnsupportedCapability(
"disjoint plane binding is not enabled",
));
}
}
if resource.is_null() {
return Err(AllocError::InvalidRequest("null resource"));
}
if alias && !live.can_alias {
return Err(AllocError::UnsupportedCapability(
"allocation was not created for aliasing",
));
}
if !alias && live.bound {
return Err(AllocError::InvalidRequest("allocation already bound"));
}
if !alias && live.resource.is_some_and(|target| target != resource) {
return Err(AllocError::WrongResource);
}
let raw = alias || live.resource.is_none();
let (handle, offset, size, memory_type_index) =
(live.handle, live.offset, live.size, live.memory_type_index);
if raw && allocator.associated.conflicts(resource, plane) {
return Err(AllocError::WrongResource);
}
allocator.associated.try_reserve_for(resource)?;
let claim = parallel::claim(
allocator,
resource,
plane,
(!raw).then_some(allocation.serial),
)?;
let fp = allocator.device.fp();
match (resource, route) {
(Resource::Buffer(_), RequirementsRoute::Legacy) => {
require(fp.bind_buffer_memory.is_some(), "vkBindBufferMemory")?
}
(Resource::Image(_), RequirementsRoute::Legacy) => {
require(fp.bind_image_memory.is_some(), "vkBindImageMemory")?
}
(Resource::Buffer(_), RequirementsRoute::Core) => {
require(fp.bind_buffer_memory2.is_some(), "vkBindBufferMemory2")?
}
(Resource::Image(_), RequirementsRoute::Core) => {
require(fp.bind_image_memory2.is_some(), "vkBindImageMemory2")?
}
(Resource::Buffer(_), RequirementsRoute::Khr) => require(
fp.bind_buffer_memory2_khr.is_some(),
"vkBindBufferMemory2KHR",
)?,
(Resource::Image(_), RequirementsRoute::Khr) => {
require(fp.bind_image_memory2_khr.is_some(), "vkBindImageMemory2KHR")?
}
}
let native_offset = offset
.checked_add(local_offset)
.ok_or(AllocError::InvalidRange)?;
if local_offset > size || (!raw && local_offset != 0) {
return Err(AllocError::InvalidRange);
}
let requirements_route = if raw {
Some(allocator.requirements_route(matches!(resource, Resource::Image(_)))?)
} else {
None
};
Ok(BindOperation {
route,
requirements_route,
resource,
serial: allocation.serial,
handle,
native_offset,
remaining_size: size - local_offset,
memory_type_index,
next,
status,
plane,
alias,
claim,
})
}
impl BindOperation<'_> {
pub(super) unsafe fn execute(&self, device: &vk::Device) -> Result<(), AllocError> {
let (route, resource, handle, native_offset, next, status, plane) = (
self.route,
self.resource,
self.handle,
self.native_offset,
self.next,
self.status,
self.plane,
);
if let Some(requirements_route) = self.requirements_route {
let requirements =
unsafe { query_requirements(device, requirements_route, resource, plane) }?;
if requirements.dedication == Dedication::Required {
return Err(AllocError::UnsupportedCapability(
"raw allocation cannot satisfy required resource dedication",
));
}
if requirements.memory.memory_type_bits & (1 << self.memory_type_index) == 0 {
return Err(AllocError::NoCompatibleMemoryType);
}
if requirements.memory.size > self.remaining_size
|| !native_offset.is_multiple_of(requirements.memory.alignment)
{
return Err(AllocError::InvalidRange);
}
}
if let Some(status) = status {
unsafe {
*status = vk::Result::NOT_READY;
}
}
unsafe {
match resource {
Resource::Buffer(buffer) => {
let info = vk::BindBufferMemoryInfo {
buffer,
memory: handle,
memory_offset: native_offset,
p_next: next,
..Default::default()
};
match route {
RequirementsRoute::Legacy => {
device.bind_buffer_memory(buffer, handle, native_offset)?
}
RequirementsRoute::Core => device.bind_buffer_memory2(&[info])?,
RequirementsRoute::Khr => device.bind_buffer_memory2_khr(&[info])?,
}
}
Resource::Image(image) => {
let info = vk::BindImageMemoryInfo {
image,
memory: handle,
memory_offset: native_offset,
p_next: next,
..Default::default()
};
match route {
RequirementsRoute::Legacy => device.bind_image_memory(image, handle, native_offset)?,
RequirementsRoute::Core => device.bind_image_memory2(&[info])?,
RequirementsRoute::Khr => device.bind_image_memory2_khr(&[info])?,
}
}
}
}
if let Some(status) = status {
let result = unsafe { *status };
if result != vk::Result::SUCCESS {
return Err(
if result.as_raw() < 0 {
CallError::Vulkan(result)
} else {
CallError::InvalidOutput("native binding did not publish success status")
}
.into(),
);
}
}
Ok(())
}
pub(super) fn commit(self, allocator: &mut Allocator<'_>) -> Result<(), AllocError> {
if self.alias {
return Ok(());
}
let live = allocator
.live
.get_mut(&self.serial)
.expect("validated allocation");
let already_associated = live.resource == Some(self.resource) && live.plane == self.plane;
live.resource = Some(self.resource);
live.plane = self.plane;
live.bound = true;
if let Some(claim) = self.claim {
claim.commit(self.serial);
} else if !already_associated {
allocator
.associated
.record(self.resource, self.plane, self.serial);
}
Ok(())
}
}
pub unsafe fn bind_buffer_memory(
allocator: &mut Allocator<'_>,
buffer: vk::Buffer,
allocation: &Allocation,
) -> Result<(), AllocError> {
unsafe {
bind_resource(
allocator,
Resource::Buffer(buffer),
allocation,
0,
std::ptr::null(),
false,
false,
)
}
}
pub unsafe fn bind_image_memory(
allocator: &mut Allocator<'_>,
image: vk::Image,
allocation: &Allocation,
) -> Result<(), AllocError> {
unsafe {
bind_resource(
allocator,
Resource::Image(image),
allocation,
0,
std::ptr::null(),
false,
false,
)
}
}
impl Allocator<'_> {
pub unsafe fn bind_buffer_memory2(
&mut self,
buffer: vk::Buffer,
allocation: &Allocation,
local_offset: u64,
next: *const std::ffi::c_void,
) -> Result<(), AllocError> {
unsafe { bind_buffer_memory2(self, buffer, allocation, local_offset, next) }
}
pub unsafe fn bind_image_memory2(
&mut self,
image: vk::Image,
allocation: &Allocation,
local_offset: u64,
next: *const std::ffi::c_void,
) -> Result<(), AllocError> {
unsafe { bind_image_memory2(self, image, allocation, local_offset, next) }
}
pub unsafe fn allocate_for_buffer(
&mut self,
buffer: vk::Buffer,
options: &AllocationCreateInfo,
) -> Result<Allocation, AllocError> {
unsafe { allocate_for_buffer(self, buffer, options) }
}
pub unsafe fn allocate_for_image(
&mut self,
image: vk::Image,
options: &AllocationCreateInfo,
) -> Result<Allocation, AllocError> {
unsafe { allocate_for_image(self, image, options) }
}
pub unsafe fn bind_buffer_memory(
&mut self,
buffer: vk::Buffer,
allocation: &Allocation,
) -> Result<(), AllocError> {
unsafe { bind_buffer_memory(self, buffer, allocation) }
}
pub unsafe fn bind_image_memory(
&mut self,
image: vk::Image,
allocation: &Allocation,
) -> Result<(), AllocError> {
unsafe { bind_image_memory(self, image, allocation) }
}
}
#[cfg(test)]
#[path = "../../../test/rs/mem/test_associated.rs"]
mod tests;