gpu/
vk_dev.rs

1// vk_dev -- Vulkan device and queue: creation, the timeline, deferred destruction, command
2// buffer slots, staged uploads, submission
3//
4//   adapter.create_device(&DeviceDesc) -> Result<Device>
5//        one queue (the first family with graphics + compute, presentation checked per
6//        surface); level Voll when the adapter has it and desc.max_level allows it.  Enabled:
7//        timeline semaphores, dynamic rendering (core 1.3 or the KHR extensions on 1.1 / 1.2),
8//        VK_KHR_swapchain when the instance has surfaces, the optional features of Features,
9//        the Vulkan memory model only together with its device scope (else Device-scope
10//        atomics, as in the benchmark's cull, would be invalid); on Voll also descriptor
11//        indexing (update after bind, partially bound, non-uniform indexing) and
12//        draw-indirect-count; maintenance4 on Vulkan 1.3 (LocalSizeId); for shaders compiled
13//        from WGSL (lib/wgsl): f16 in uniform buffers and push constants, the standard uniform
14//        buffer layout, demote-to-helper (discard on Vulkan 1.3), clip distances, the
15//        primitive index (geometryShader), sample-rate shading, dual-source blending in
16//        shaders, the extended storage image formats, storage reads without format.  Not
17//        enabled: robust buffer access (speed; lib/wgsl's back end clamps indices instead).
18//   device.queue() -> &Queue        queue.submit([cmd, ..]) -> Result<SubmitId>
19//   queue.write_buffer(&buf, off, &[T])            staged into an upload command buffer that
20//   queue.write_texture(&region, &[u8], layout, before, after)   runs first in the next submit
21//   queue.write_texture_rects(&tex, &rects, &[u8], stride, before, after)   rectangles of a frame
22//   queue.write_texture_rects_at(&tex, &rects, origin, &[u8], stride, before, after)   of a larger image
23//   queue.read_buffer(&buf, off, &mut [T])          copies, submits and waits (slow path)
24//   queue.flush() -> Result<SubmitId>               submit the pending uploads now
25//   device.wait(id, timeout_ns) -> Result<bool>     false: timed out
26//   device.wait_idle(), poll() (frees what the GPU has finished with), is_lost(),
27//   test_lose() (a test aid: the device behaves as lost from now on)
28//
29// The timeline: one timeline semaphore per device; every submit signals the next value
30// (SubmitId).  completed caches the last value seen reached (vkGetSemaphoreCounterValue at
31// submits, polls and waits).  A dropped resource goes to the garbage with the value of the
32// latest submit and is destroyed once that value is reached -- earlier if no submit is
33// pending; a buffer or image, which the open upload command buffer may name, with the value of
34// the next submit, which runs it.  Command buffers come from slots (a transient command pool with one buffer each),
35// reused once their submission completed.  Uploads: the data goes into the staging ring, the
36// copies into one upload command buffer that ends with a global barrier (copy writes -> every
37// later access) and is put before the caller's command buffers in the next vkQueueSubmit.
38// The staging ring: 256 KiB chunks of Upload memory, filled front to back across submissions;
39// a full chunk is reused once the GPU has passed the last submission that reads from it (two
40// are kept free, more go back).  An upload above half a chunk gets a buffer of its own, reused
41// by the next one that fits and released when none came for a second (Device::poll): a frame of
42// small rectangles costs what they hold, not a full frame or a new allocation.
43// Device lost: VK_ERROR_DEVICE_LOST from any call sets the device's lost flag; every later
44// call returns GpuError::DeviceLost (objects are still destroyed normally).
45
46use alloc::sync::Arc;
47use alloc::vec::Vec;
48use core::ffi::c_void;
49use core::sync::atomic::{AtomicBool, AtomicU64, Ordering};
50
51use rt::sync::SpinLock;
52
53use crate::vk_ffi as vk;
54use crate::vk_inst::{Adapter, InstShared, err};
55use crate::vk_mem::{Alloc, Allocator};
56use crate::{AdapterInfo, DataLayout, DeviceDesc, Features, GpuError, Layout, Level, Limits, MemoryKind, Pod, Result,
57            SubmitId, TextureRegion};
58
59/// what waits for the GPU before it is destroyed
60pub(crate) enum Garbage {
61    Buffer(vk::Handle, Alloc),
62    Image(vk::Handle, Option<Alloc>),
63    View(vk::Handle),
64    Sampler(vk::Handle),
65    Shader(vk::Handle),
66    SetLayout(vk::Handle),
67    PipeLayout(vk::Handle),
68    Pipeline(vk::Handle),
69    Set(vk::Handle, vk::Handle),
70    Pool(vk::Handle),
71    Query(vk::Handle),
72    #[cfg_attr(jfgui_glibc, allow(dead_code))] // (vk_swap.rs)
73    Semaphore(vk::Handle),
74    #[cfg_attr(jfgui_glibc, allow(dead_code))]
75    Swapchain(vk::Handle),
76    #[cfg_attr(not(jfgui_glibc), allow(dead_code))] // (vk_dmabuf.rs: an image's dedicated allocation)
77    Memory(vk::Handle),
78}
79
80/// a command pool with its one command buffer
81pub(crate) struct Slot {
82    pub pool: vk::Handle,
83    pub cb: vk::CommandBuffer,
84}
85
86/// a buffer of the staging ring
87struct Chunk {
88    buf: vk::Handle,
89    a: Alloc,
90    used: u64, // bytes used, front to back
91}
92
93/// the pending staged uploads: the command buffer, the chunks full since the last submission
94/// that it reads from
95struct Upload {
96    slot: Slot,
97    retired: Vec<Chunk>,
98}
99
100/// the staging ring
101#[derive(Default)]
102struct Ring {
103    cur: Option<Chunk>, // the chunk being filled (what is behind `used` stays free across submissions)
104    cur_value: u64,     // the last submission that reads from it (0: none)
105    cur_open: bool,     // the open upload command buffer reads from it
106    busy: Vec<(u64, Chunk)>, // retired: free once the GPU reached the value
107    free: Vec<Chunk>,
108    big_at: u64,        // plat::time_us of the last upload above half a chunk
109    big_size: u64,      // its bytes
110}
111
112pub(crate) struct State {
113    pub mem: Allocator,
114    garbage: Vec<(u64, Garbage)>,
115    free: Vec<Slot>,
116    busy: Vec<(u64, Slot)>,
117    pools: Vec<vk::Handle>, // descriptor pools of the bind groups (the last one is filled)
118    upload: Option<Upload>,
119    ring: Ring,
120}
121
122/// a chunk of the staging ring
123const CHUNK: u64 = 256 << 10;
124/// a large chunk stays while large uploads need at least this part of it
125const BIG_KEEP_RATIO: u64 = 8;
126/// free chunks kept beyond the one being filled
127const CHUNKS_FREE: usize = 2;
128/// an upload above half a chunk keeps its buffer this long after the last one (us)
129const BIG_KEEP_US: u64 = 1_000_000;
130
131pub(crate) struct Shared {
132    pub inst: Arc<InstShared>,
133    pub pd: vk::PhysicalDevice,
134    pub dev: vk::Device,
135    pub f: vk::DeviceFns,
136    pub queue: vk::Queue,
137    pub family: u32,
138    pub timeline: vk::Handle,
139    pub info: AdapterInfo,
140    pub features: Features,
141    pub limits: Limits,
142    pub level: Level,
143    pub raw: vk::PhysicalDeviceLimits,
144    pub ts_mask: u64,
145    #[cfg_attr(jfgui_glibc, allow(dead_code))] // (vk_swap.rs)
146    pub swapchain: bool,
147    /// images for a window go to VK_QUEUE_FAMILY_FOREIGN_EXT (vk_dmabuf.rs)
148    #[cfg_attr(not(jfgui_glibc), allow(dead_code))]
149    pub queue_foreign: bool,
150    /// the first timestamp tick read on the device: Timestamps::read_ms counts from it
151    pub ts_base: AtomicU64,
152    /// vkResetQueryPool from the host (hostQueryReset enabled)
153    pub host_query_reset: bool,
154    /// maintenance4 enabled (Vulkan 1.3): shaders may set their workgroup size with LocalSizeId
155    pub maintenance4: bool,
156    lost: AtomicBool,
157    submitted: AtomicU64,
158    completed: AtomicU64,
159    qlock: SpinLock<()>,
160    pub st: SpinLock<State>,
161}
162
163/// a logical GPU device: creates resources, records commands, owns the queue
164#[derive(Clone)]
165pub struct Device {
166    pub(crate) sh: Arc<Shared>,
167    queue: Queue,
168}
169
170/// the device's queue (graphics, compute, copies, presentation)
171#[derive(Clone)]
172pub struct Queue {
173    pub(crate) sh: Arc<Shared>,
174}
175
176// features enabled when present (indices into the VkBool32 arrays)
177const E10: &[usize] = &[
178    vk::F10_FULL_DRAW_INDEX_UINT32,
179    vk::F10_IMAGE_CUBE_ARRAY,
180    vk::F10_INDEPENDENT_BLEND,
181    vk::F10_GEOMETRY_SHADER,
182    vk::F10_SAMPLE_RATE_SHADING,
183    vk::F10_DUAL_SRC_BLEND,
184    vk::F10_MULTI_DRAW_INDIRECT,
185    vk::F10_DRAW_INDIRECT_FIRST_INSTANCE,
186    vk::F10_DEPTH_CLAMP,
187    vk::F10_FILL_MODE_NON_SOLID,
188    vk::F10_SAMPLER_ANISOTROPY,
189    vk::F10_TEXTURE_COMPRESSION_BC,
190    vk::F10_VERTEX_PIPELINE_STORES_AND_ATOMICS,
191    vk::F10_FRAGMENT_STORES_AND_ATOMICS,
192    vk::F10_SHADER_STORAGE_IMAGE_EXTENDED_FORMATS,
193    vk::F10_SHADER_STORAGE_IMAGE_READ_WITHOUT_FORMAT,
194    vk::F10_SHADER_STORAGE_IMAGE_WRITE_WITHOUT_FORMAT,
195    vk::F10_SHADER_UNIFORM_BUFFER_ARRAY_DYNAMIC_INDEXING,
196    vk::F10_SHADER_SAMPLED_IMAGE_ARRAY_DYNAMIC_INDEXING,
197    vk::F10_SHADER_STORAGE_BUFFER_ARRAY_DYNAMIC_INDEXING,
198    vk::F10_SHADER_STORAGE_IMAGE_ARRAY_DYNAMIC_INDEXING,
199    vk::F10_SHADER_CLIP_DISTANCE,
200    vk::F10_SHADER_FLOAT64,
201    vk::F10_SHADER_INT64,
202    vk::F10_SHADER_INT16,
203];
204const E11: &[usize] = &[
205    vk::F11_STORAGE_BUFFER_16BIT_ACCESS,
206    vk::F11_UNIFORM_AND_STORAGE_BUFFER_16BIT_ACCESS,
207    vk::F11_STORAGE_PUSH_CONSTANT16,
208    vk::F11_SHADER_DRAW_PARAMETERS,
209];
210const E12: &[usize] = &[
211    vk::F12_STORAGE_BUFFER_8BIT_ACCESS,
212    vk::F12_SHADER_FLOAT16,
213    vk::F12_SHADER_INT8,
214    vk::F12_SCALAR_BLOCK_LAYOUT,
215    vk::F12_UNIFORM_BUFFER_STANDARD_LAYOUT,
216    vk::F12_SHADER_SUBGROUP_EXTENDED_TYPES,
217    vk::F12_HOST_QUERY_RESET,
218    vk::F12_TIMELINE_SEMAPHORE,
219    vk::F12_BUFFER_DEVICE_ADDRESS,
220    vk::F12_VULKAN_MEMORY_MODEL,
221    vk::F12_VULKAN_MEMORY_MODEL_DEVICE_SCOPE,
222];
223const E12_VOLL: &[usize] = &[
224    vk::F12_DRAW_INDIRECT_COUNT,
225    vk::F12_DESCRIPTOR_INDEXING,
226    vk::F12_SHADER_SAMPLED_IMAGE_ARRAY_NON_UNIFORM_INDEXING,
227    vk::F12_SHADER_STORAGE_BUFFER_ARRAY_NON_UNIFORM_INDEXING,
228    vk::F12_SHADER_STORAGE_IMAGE_ARRAY_NON_UNIFORM_INDEXING,
229    vk::F12_DESCRIPTOR_BINDING_SAMPLED_IMAGE_UPDATE_AFTER_BIND,
230    vk::F12_DESCRIPTOR_BINDING_STORAGE_IMAGE_UPDATE_AFTER_BIND,
231    vk::F12_DESCRIPTOR_BINDING_STORAGE_BUFFER_UPDATE_AFTER_BIND,
232    vk::F12_DESCRIPTOR_BINDING_UPDATE_UNUSED_WHILE_PENDING,
233    vk::F12_DESCRIPTOR_BINDING_PARTIALLY_BOUND,
234    vk::F12_RUNTIME_DESCRIPTOR_ARRAY,
235];
236const E13: &[usize] = &[
237    vk::F13_SHADER_DEMOTE_TO_HELPER_INVOCATION,
238    vk::F13_SUBGROUP_SIZE_CONTROL,
239    vk::F13_SYNCHRONIZATION2,
240    vk::F13_DYNAMIC_RENDERING,
241    vk::F13_MAINTENANCE4,
242];
243
244fn pick<const N: usize>(have: &[u32; N], want: &[usize]) -> [u32; N] {
245    let mut out = [0u32; N];
246    for &i in want {
247        out[i] = have[i];
248    }
249    out
250}
251
252impl Adapter {
253    /// a device of this adapter (level: the highest up to desc.max_level)
254    pub fn create_device(&self, desc: &DeviceDesc) -> Result<Device> {
255        if let Some(why) = self.info.missing {
256            return Err(GpuError::Unsupported(why));
257        }
258        let inst = &self.inst;
259        let level = if self.voll && desc.max_level == Level::Voll { Level::Voll } else { Level::Basis };
260        let voll = level == Level::Voll;
261        let v12 = self.api >= vk::API_VERSION_1_2;
262        let v13 = self.api >= vk::API_VERSION_1_3;
263        let mut e10 = pick(&self.f10, E10);
264        if !voll {
265            e10[vk::F10_MULTI_DRAW_INDIRECT] = 0;
266        }
267        let e11 = pick(&self.f11, E11);
268        let mut e12 = pick(&self.f12, E12);
269        // with the Vulkan memory model on, Device-scope atomics (any shader's, also under the
270        // GLSL450 model) need its device scope (VUID-RuntimeSpirv-vulkanMemoryModel-06265)
271        if e12[vk::F12_VULKAN_MEMORY_MODEL_DEVICE_SCOPE] == 0 {
272            e12[vk::F12_VULKAN_MEMORY_MODEL] = 0;
273        }
274        if voll {
275            for &i in E12_VOLL {
276                e12[i] = self.f12[i];
277            }
278        }
279        let e13 = pick(&self.f13, E13);
280        let mut f13: vk::PhysicalDeviceVulkan13Features = vk::Zero::zero();
281        f13.s_type = vk::STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES;
282        f13.f = e13;
283        let mut f12: vk::PhysicalDeviceVulkan12Features = vk::Zero::zero();
284        f12.s_type = vk::STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES;
285        f12.f = e12;
286        let mut f11: vk::PhysicalDeviceVulkan11Features = vk::Zero::zero();
287        f11.s_type = vk::STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES;
288        f11.f = e11;
289        let mut dynr: vk::PhysicalDeviceOneFeature = vk::Zero::zero();
290        dynr.s_type = vk::STRUCTURE_TYPE_PHYSICAL_DEVICE_DYNAMIC_RENDERING_FEATURES;
291        dynr.f = 1;
292        let mut tl: vk::PhysicalDeviceOneFeature = vk::Zero::zero();
293        tl.s_type = vk::STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES;
294        tl.f = 1;
295        let mut f2: vk::PhysicalDeviceFeatures2 = vk::Zero::zero();
296        f2.s_type = vk::STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2;
297        f2.f = e10;
298        let mut names: Vec<*const u8> = Vec::new();
299        // the chain is linked from its tail: no structure is written after a pointer to it exists
300        if v12 {
301            if v13 {
302                f12.p_next = &mut f13 as *mut _ as *mut c_void;
303            } else {
304                f12.p_next = &mut dynr as *mut _ as *mut c_void;
305                names.push(b"VK_KHR_dynamic_rendering\0".as_ptr());
306            }
307            f11.p_next = &mut f12 as *mut _ as *mut c_void;
308            f2.p_next = &mut f11 as *mut _ as *mut c_void;
309        } else {
310            // Vulkan 1.1: the extensions and their dependencies
311            tl.p_next = &mut dynr as *mut _ as *mut c_void;
312            f2.p_next = &mut tl as *mut _ as *mut c_void;
313            names.push(b"VK_KHR_timeline_semaphore\0".as_ptr());
314            names.push(b"VK_KHR_create_renderpass2\0".as_ptr());
315            names.push(b"VK_KHR_depth_stencil_resolve\0".as_ptr());
316            names.push(b"VK_KHR_dynamic_rendering\0".as_ptr());
317        }
318        let swapchain = self.exts.swapchain;
319        if swapchain {
320            names.push(b"VK_KHR_swapchain\0".as_ptr());
321        }
322        let dmabuf = self.exts.dmabuf;
323        if dmabuf {
324            names.push(b"VK_KHR_external_memory_fd\0".as_ptr());
325            names.push(b"VK_EXT_external_memory_dma_buf\0".as_ptr());
326            names.push(b"VK_EXT_image_drm_format_modifier\0".as_ptr());
327            if !v12 {
328                names.push(b"VK_KHR_image_format_list\0".as_ptr());
329            }
330        }
331        let queue_foreign = dmabuf && self.exts.queue_foreign;
332        if queue_foreign {
333            names.push(b"VK_EXT_queue_family_foreign\0".as_ptr());
334        }
335        let prio = 1.0f32;
336        let qi = vk::DeviceQueueCreateInfo {
337            s_type: vk::STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,
338            queue_family_index: self.family,
339            queue_count: 1,
340            p_queue_priorities: &prio,
341            ..vk::Zero::zero()
342        };
343        let info = vk::DeviceCreateInfo {
344            s_type: vk::STRUCTURE_TYPE_DEVICE_CREATE_INFO,
345            p_next: &f2 as *const _ as *const c_void,
346            queue_create_info_count: 1,
347            p_queue_create_infos: &qi,
348            enabled_extension_count: names.len() as u32,
349            pp_enabled_extension_names: names.as_ptr(),
350            ..vk::Zero::zero()
351        };
352        let mut dev: vk::Device = 0;
353        // SAFETY: the chain and the arrays live until the call returns.
354        let r = unsafe { (inst.f.create_device)(self.pd, &info, core::ptr::null(), &mut dev) };
355        if r != vk::SUCCESS {
356            return Err(err(r));
357        }
358        let gdpa = inst.f.get_device_proc_addr;
359        // SAFETY: names are NUL-terminated; dev is the device just created.
360        let f = match vk::DeviceFns::load(&|n| unsafe { gdpa(dev, n.as_ptr()) }) {
361            Ok(f) => f,
362            Err(name) => {
363                // SAFETY: vkDestroyDevice of the fresh device (nothing created from it).
364                let d = unsafe { gdpa(dev, b"vkDestroyDevice\0".as_ptr()) };
365                if d != 0 {
366                    // SAFETY: vkDestroyDevice's prototype.
367                    unsafe {
368                        core::mem::transmute::<usize, unsafe extern "system" fn(vk::Device, *const c_void)>(d)(
369                            dev,
370                            core::ptr::null(),
371                        )
372                    };
373                }
374                return Err(GpuError::Unsupported(name));
375            }
376        };
377        let mut queue: vk::Queue = 0;
378        // SAFETY: family / index 0 were requested above.
379        unsafe { (f.get_device_queue)(dev, self.family, 0, &mut queue) };
380        let ti = vk::SemaphoreTypeCreateInfo {
381            s_type: vk::STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
382            semaphore_type: vk::SEMAPHORE_TYPE_TIMELINE,
383            initial_value: 0,
384            ..vk::Zero::zero()
385        };
386        let si = vk::SemaphoreCreateInfo {
387            s_type: vk::STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
388            p_next: &ti as *const _ as *const c_void,
389            flags: 0,
390        };
391        let mut timeline = 0;
392        // SAFETY: valid create info.
393        let r = unsafe { (f.create_semaphore)(dev, &si, core::ptr::null(), &mut timeline) };
394        if r != vk::SUCCESS {
395            // SAFETY: nothing else was created from the device.
396            unsafe { (f.destroy_device)(dev, core::ptr::null()) };
397            return Err(err(r));
398        }
399        let mut features = self.features;
400        if !voll {
401            features.bindless = false;
402            features.draw_indirect_count = false;
403            features.multi_draw_indirect = false;
404        }
405        let mut limits = self.limits;
406        if !voll {
407            limits.max_bindless_textures = 0;
408            limits.max_bindless_samplers = 0;
409            limits.max_bindless_buffers = 0;
410            limits.max_bindless_storage_textures = 0;
411        }
412        let ts_mask = match self.ts_bits {
413            0 => 0,
414            64.. => !0,
415            b => (1u64 << b) - 1,
416        };
417        let sh = Arc::new(Shared {
418            inst: inst.clone(),
419            pd: self.pd,
420            dev,
421            f,
422            queue,
423            family: self.family,
424            timeline,
425            info: self.info.clone(),
426            features,
427            limits,
428            level,
429            raw: self.raw,
430            ts_mask,
431            swapchain,
432            queue_foreign,
433            ts_base: AtomicU64::new(0),
434            host_query_reset: v12 && e12[vk::F12_HOST_QUERY_RESET] != 0,
435            maintenance4: v13 && e13[vk::F13_MAINTENANCE4] != 0,
436            lost: AtomicBool::new(false),
437            submitted: AtomicU64::new(0),
438            completed: AtomicU64::new(0),
439            qlock: SpinLock::new(()),
440            st: SpinLock::new(State {
441                mem: Allocator::new(self.mem),
442                garbage: Vec::new(),
443                free: Vec::new(),
444                busy: Vec::new(),
445                pools: Vec::new(),
446                upload: None,
447                ring: Ring::default(),
448            }),
449        });
450        Ok(Device { queue: Queue { sh: sh.clone() }, sh })
451    }
452}
453
454impl Shared {
455    /// Err(DeviceLost) once the device is lost
456    #[inline]
457    pub fn check(&self) -> Result<()> {
458        if self.lost.load(Ordering::Relaxed) { Err(GpuError::DeviceLost) } else { Ok(()) }
459    }
460
461    /// a VkResult as Result (DEVICE_LOST marks the device)
462    pub fn vk(&self, r: vk::VkResult) -> Result<()> {
463        if r == vk::SUCCESS {
464            return Ok(());
465        }
466        if r == vk::ERROR_DEVICE_LOST {
467            self.lost.store(true, Ordering::Relaxed);
468        }
469        Err(err(r))
470    }
471
472    /// run f with the queue held (vkQueuePresentKHR and vkQueueSubmit need it alone)
473    #[cfg_attr(jfgui_glibc, allow(dead_code))]
474    pub fn with_queue<R>(&self, f: impl FnOnce(vk::Queue) -> R) -> R {
475        let _q = self.qlock.lock();
476        f(self.queue)
477    }
478
479    /// wait until the queue has finished everything, presents included (vkQueueWaitIdle;
480    /// also on a lost device, where it returns at once)
481    #[cfg_attr(jfgui_glibc, allow(dead_code))]
482    pub fn queue_idle(&self) -> Result<()> {
483        // the value is read while the queue is held: nothing is submitted until the wait ends
484        // SAFETY: the queue is held while the driver waits on it.
485        let (r, v) = self.with_queue(|q| (unsafe { (self.f.queue_wait_idle)(q) }, self.submitted()));
486        self.vk(r)?;
487        self.completed.fetch_max(v, Ordering::AcqRel);
488        let mut st = self.st.lock();
489        self.collect(&mut st);
490        Ok(())
491    }
492
493    pub fn mark_lost(&self) {
494        self.lost.store(true, Ordering::Relaxed);
495    }
496
497    pub fn submitted(&self) -> u64 {
498        self.submitted.load(Ordering::Acquire)
499    }
500
501    /// the timeline value the GPU has reached (asks the driver)
502    pub fn refresh(&self) -> u64 {
503        let mut v = 0u64;
504        // SAFETY: the device's timeline semaphore.
505        let r = unsafe { (self.f.get_semaphore_counter_value)(self.dev, self.timeline, &mut v) };
506        if r != vk::SUCCESS {
507            if r == vk::ERROR_DEVICE_LOST {
508                self.lost.store(true, Ordering::Relaxed);
509            }
510            return self.completed.load(Ordering::Acquire);
511        }
512        self.completed.fetch_max(v, Ordering::AcqRel);
513        v
514    }
515
516    /// destroy g once the GPU has finished every submission made so far; a buffer or image also
517    /// the open upload buffer, whose copies may name it (a texture written and dropped before the
518    /// next submit).  Nothing else waits for that: a swapchain must go before its surface, which
519    /// its drop releases right after this
520    pub fn defer(&self, g: Garbage) {
521        let mut st = self.st.lock();
522        let named = st.upload.is_some() && matches!(g, Garbage::Buffer(..) | Garbage::Image(..));
523        let v = self.submitted() + named as u64;
524        if v <= self.completed.load(Ordering::Acquire) || self.lost.load(Ordering::Relaxed) {
525            self.destroy(&mut st, g);
526        } else {
527            st.garbage.push((v, g));
528        }
529    }
530
531    fn destroy(&self, st: &mut State, g: Garbage) {
532        let (f, d, n) = (&self.f, self.dev, core::ptr::null());
533        // SAFETY: the GPU no longer uses the object (its timeline value is reached, or no
534        // submission is pending, or the device is lost); each handle is destroyed once.
535        unsafe {
536            match g {
537                Garbage::Buffer(h, a) => {
538                    (f.destroy_buffer)(d, h, n);
539                    st.mem.free(f, d, &a);
540                }
541                Garbage::Image(h, a) => {
542                    (f.destroy_image)(d, h, n);
543                    if let Some(a) = a {
544                        st.mem.free(f, d, &a);
545                    }
546                }
547                Garbage::View(h) => (f.destroy_image_view)(d, h, n),
548                Garbage::Sampler(h) => (f.destroy_sampler)(d, h, n),
549                Garbage::Shader(h) => (f.destroy_shader_module)(d, h, n),
550                Garbage::SetLayout(h) => (f.destroy_descriptor_set_layout)(d, h, n),
551                Garbage::PipeLayout(h) => (f.destroy_pipeline_layout)(d, h, n),
552                Garbage::Pipeline(h) => (f.destroy_pipeline)(d, h, n),
553                Garbage::Set(pool, set) => {
554                    (f.free_descriptor_sets)(d, pool, 1, &set);
555                }
556                Garbage::Pool(h) => (f.destroy_descriptor_pool)(d, h, n),
557                Garbage::Query(h) => (f.destroy_query_pool)(d, h, n),
558                Garbage::Semaphore(h) => (f.destroy_semaphore)(d, h, n),
559                Garbage::Swapchain(h) => {
560                    if let Some(ds) = f.destroy_swapchain_khr {
561                        ds(d, h, n);
562                    }
563                }
564                Garbage::Memory(h) => (f.free_memory)(d, h, n),
565            }
566        }
567    }
568
569    /// free the garbage and the command slots whose submissions completed
570    fn collect(&self, st: &mut State) {
571        if st.garbage.is_empty() && st.busy.is_empty() && st.ring.busy.is_empty() && st.ring.free.is_empty() {
572            return;
573        }
574        let done = if self.lost.load(Ordering::Relaxed) { u64::MAX } else { self.refresh() };
575        let mut i = 0;
576        while i < st.ring.busy.len() {
577            if st.ring.busy[i].0 <= done {
578                let (_, mut c) = st.ring.busy.swap_remove(i);
579                c.used = 0;
580                st.ring.free.push(c);
581            } else {
582                i += 1;
583            }
584        }
585        self.ring_trim(st);
586        let mut i = 0;
587        while i < st.busy.len() {
588            if st.busy[i].0 <= done {
589                let (_, s) = st.busy.swap_remove(i);
590                // SAFETY: the pool's command buffer finished executing.
591                unsafe { (self.f.reset_command_pool)(self.dev, s.pool, 0) };
592                st.free.push(s);
593            } else {
594                i += 1;
595            }
596        }
597        let mut i = 0;
598        while i < st.garbage.len() {
599            if st.garbage[i].0 <= done {
600                let (_, g) = st.garbage.swap_remove(i);
601                self.destroy(st, g);
602            } else {
603                i += 1;
604            }
605        }
606    }
607
608    /// a command buffer slot (reused or new), begun for one submission
609    pub fn take_slot(&self) -> Result<Slot> {
610        self.check()?;
611        let slot = {
612            let mut st = self.st.lock();
613            if st.free.is_empty() {
614                self.collect(&mut st);
615            }
616            st.free.pop()
617        };
618        let slot = match slot {
619            Some(s) => s,
620            None => self.new_slot()?,
621        };
622        let bi = vk::CommandBufferBeginInfo {
623            s_type: vk::STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
624            flags: vk::COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,
625            ..vk::Zero::zero()
626        };
627        // SAFETY: the slot's buffer is in the initial state (new or its pool reset).
628        let r = unsafe { (self.f.begin_command_buffer)(slot.cb, &bi) };
629        if let Err(e) = self.vk(r) {
630            self.give_slot(slot);
631            return Err(e);
632        }
633        Ok(slot)
634    }
635
636    fn new_slot(&self) -> Result<Slot> {
637        let pi = vk::CommandPoolCreateInfo {
638            s_type: vk::STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
639            flags: vk::COMMAND_POOL_CREATE_TRANSIENT_BIT,
640            queue_family_index: self.family,
641            ..vk::Zero::zero()
642        };
643        let mut pool = 0;
644        // SAFETY: valid create info.
645        self.vk(unsafe { (self.f.create_command_pool)(self.dev, &pi, core::ptr::null(), &mut pool) })?;
646        let ai = vk::CommandBufferAllocateInfo {
647            s_type: vk::STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
648            command_pool: pool,
649            level: vk::COMMAND_BUFFER_LEVEL_PRIMARY,
650            command_buffer_count: 1,
651            ..vk::Zero::zero()
652        };
653        let mut cb = 0;
654        // SAFETY: one buffer from the new pool.
655        let r = unsafe { (self.f.allocate_command_buffers)(self.dev, &ai, &mut cb) };
656        if let Err(e) = self.vk(r) {
657            // SAFETY: the pool is unused.
658            unsafe { (self.f.destroy_command_pool)(self.dev, pool, core::ptr::null()) };
659            return Err(e);
660        }
661        Ok(Slot { pool, cb })
662    }
663
664    /// a slot that was never submitted: reset, back to the free list
665    pub fn give_slot(&self, s: Slot) {
666        // SAFETY: the buffer is not pending (never submitted).
667        unsafe { (self.f.reset_command_pool)(self.dev, s.pool, 0) };
668        self.st.lock().free.push(s);
669    }
670
671    /// the free chunks beyond what the ring keeps go back: ordinary ones above CHUNKS_FREE,
672    /// large ones a second after the last large upload (all but the largest at once; the largest
673    /// too when the last large upload was far smaller: one big image's buffer does not stay
674    /// around for the frames after it)
675    fn ring_trim(&self, st: &mut State) {
676        let now = plat::time_us();
677        let keep_big = now.saturating_sub(st.ring.big_at) < BIG_KEEP_US;
678        let largest = st.ring.free.iter().map(|c| c.a.size).max().unwrap_or(0);
679        let mut small = 0;
680        let mut i = 0;
681        while i < st.ring.free.len() {
682            let size = st.ring.free[i].a.size;
683            let keep = if size >= CHUNK * 2 {
684                keep_big && size == largest && size <= st.ring.big_size.saturating_mul(BIG_KEEP_RATIO).max(CHUNK * 2)
685            } else {
686                small += 1;
687                small <= CHUNKS_FREE
688            };
689            if keep {
690                i += 1;
691            } else {
692                let c = st.ring.free.swap_remove(i);
693                self.destroy(st, Garbage::Buffer(c.buf, c.a));
694            }
695        }
696    }
697
698    /// a chunk of at least size bytes, reused or new (its `used` is 0); the ring's own chunks
699    /// (size = CHUNK) are never large ones, those would stay around
700    fn ring_chunk(&self, st: &mut State, size: u64) -> Result<Chunk> {
701        let top = if size == CHUNK { CHUNK * 2 } else { u64::MAX };
702        let best = st.ring.free.iter().enumerate().filter(|(_, c)| c.a.size >= size && c.a.size < top).min_by_key(|(_, c)| c.a.size).map(|(i, _)| i);
703        if let Some(i) = best {
704            return Ok(st.ring.free.swap_remove(i));
705        }
706        let (buf, a) = self.raw_buffer(st, size, vk::BUFFER_USAGE_TRANSFER_SRC_BIT, MemoryKind::Upload)?;
707        Ok(Chunk { buf, a, used: 0 })
708    }
709
710    /// the upload command buffer (begun when there is none): it runs first in the next submit
711    pub fn upload_cb(&self, st: &mut State) -> Result<vk::CommandBuffer> {
712        self.stage(st, 0)?;
713        Ok(st.upload.as_ref().map_or(0, |u| u.slot.cb))
714    }
715
716    /// staging space of n bytes for the upload command buffer -> (buffer, offset, address)
717    /// (n = 0: only the command buffer)
718    fn stage(&self, st: &mut State, n: u64) -> Result<(vk::Handle, u64, usize)> {
719        if st.upload.is_none() {
720            // the upload buffer's slot: from the free list or new (the lock is held: no collect)
721            let slot = match st.free.pop() {
722                Some(s) => s,
723                None => self.new_slot()?,
724            };
725            let bi = vk::CommandBufferBeginInfo {
726                s_type: vk::STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
727                flags: vk::COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,
728                ..vk::Zero::zero()
729            };
730            // SAFETY: a slot in the initial state.
731            let r = unsafe { (self.f.begin_command_buffer)(slot.cb, &bi) };
732            if let Err(e) = self.vk(r) {
733                st.free.push(slot);
734                return Err(e);
735            }
736            st.upload = Some(Upload { slot, retired: Vec::new() });
737        }
738        if n == 0 {
739            return Ok((0, 0, 0));
740        }
741        if n > CHUNK / 2 {
742            // a buffer of its own: free once the next submission is done
743            st.ring.big_at = plat::time_us();
744            st.ring.big_size = n;
745            let size = (n + 0xFFFF) & !0xFFFF;
746            let mut c = self.ring_chunk(st, size.max(CHUNK * 2))?;
747            c.used = n;
748            let (buf, ptr) = (c.buf, c.a.ptr);
749            st.upload.as_mut().unwrap_or_else(|| unreachable!()).retired.push(c);
750            return Ok((buf, 0, ptr));
751        }
752        let room = st.ring.cur.as_ref().is_some_and(|c| c.used + n <= c.a.size);
753        if !room {
754            if let Some(c) = st.ring.cur.take() {
755                if st.ring.cur_open {
756                    st.upload.as_mut().unwrap_or_else(|| unreachable!()).retired.push(c);
757                } else {
758                    let v = st.ring.cur_value;
759                    st.ring.busy.push((v, c));
760                }
761            }
762            let c = self.ring_chunk(st, CHUNK)?;
763            st.ring.cur = Some(c);
764            st.ring.cur_open = false;
765        }
766        st.ring.cur_open = true;
767        let c = st.ring.cur.as_mut().unwrap_or_else(|| unreachable!());
768        let off = c.used;
769        c.used = (off + n + 15) & !15;
770        Ok((c.buf, off, c.a.ptr + off as usize))
771    }
772
773    /// a VkBuffer with memory of kind (bound) -> (buffer, allocation)
774    pub fn raw_buffer(&self, st: &mut State, size: u64, usage: u32, kind: MemoryKind) -> Result<(vk::Handle, Alloc)> {
775        let bi = vk::BufferCreateInfo {
776            s_type: vk::STRUCTURE_TYPE_BUFFER_CREATE_INFO,
777            size,
778            usage,
779            sharing_mode: vk::SHARING_MODE_EXCLUSIVE,
780            ..vk::Zero::zero()
781        };
782        let mut buf = 0;
783        // SAFETY: valid create info.
784        self.vk(unsafe { (self.f.create_buffer)(self.dev, &bi, core::ptr::null(), &mut buf) })?;
785        let mut req: vk::MemoryRequirements = vk::Zero::zero();
786        // SAFETY: the buffer just created.
787        unsafe { (self.f.get_buffer_memory_requirements)(self.dev, buf, &mut req) };
788        let a = match st.mem.alloc(&self.f, self.dev, &req, kind, true) {
789            Ok(a) => a,
790            Err(e) => {
791                // SAFETY: unbound, unused buffer.
792                unsafe { (self.f.destroy_buffer)(self.dev, buf, core::ptr::null()) };
793                return Err(e);
794            }
795        };
796        // SAFETY: the allocation satisfies the buffer's requirements.
797        let r = unsafe { (self.f.bind_buffer_memory)(self.dev, buf, a.mem, a.offset) };
798        if let Err(e) = self.vk(r) {
799            // SAFETY: as above.
800            unsafe { (self.f.destroy_buffer)(self.dev, buf, core::ptr::null()) };
801            st.mem.free(&self.f, self.dev, &a);
802            return Err(e);
803        }
804        Ok((buf, a))
805    }
806
807    /// submit command buffers (their slots), after the pending uploads; wait: a binary
808    /// semaphore and its stages, signal: a binary semaphore -> the timeline value
809    pub fn submit(&self, cbs: &[vk::CommandBuffer], slots: Vec<Slot>, wait: Option<(vk::Handle, u32)>,
810                  signal: Option<vk::Handle>) -> Result<u64> {
811        let mut slots = slots;
812        if let Err(e) = self.check() {
813            for s in slots {
814                self.give_slot(s);
815            }
816            return Err(e);
817        }
818        let up = self.st.lock().upload.take();
819        let mut all: Vec<vk::CommandBuffer> = Vec::with_capacity(cbs.len() + 1);
820        let mut chunks: Vec<Chunk> = Vec::new();
821        let uploads = up.is_some();
822        if let Some(u) = up {
823            let mb = vk::MemoryBarrier {
824                s_type: vk::STRUCTURE_TYPE_MEMORY_BARRIER,
825                src_access_mask: vk::ACCESS_TRANSFER_WRITE_BIT,
826                dst_access_mask: vk::ACCESS_MEMORY_READ_BIT | vk::ACCESS_MEMORY_WRITE_BIT,
827                ..vk::Zero::zero()
828            };
829            // SAFETY: the upload buffer is recording; its copies are done before what follows.
830            unsafe {
831                (self.f.cmd_pipeline_barrier)(u.slot.cb, vk::PIPELINE_STAGE_TRANSFER_BIT,
832                    vk::PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &mb, 0, core::ptr::null(), 0, core::ptr::null());
833            }
834            // SAFETY: as above.
835            let r = unsafe { (self.f.end_command_buffer)(u.slot.cb) };
836            all.push(u.slot.cb);
837            slots.push(u.slot);
838            chunks = u.retired;
839            if let Err(e) = self.vk(r) {
840                self.fail_submit(slots, chunks);
841                return Err(e);
842            }
843        }
844        all.extend_from_slice(cbs);
845        let q = self.qlock.lock();
846        let value = self.submitted.load(Ordering::Relaxed) + 1;
847        let sems = [self.timeline, signal.unwrap_or(0)];
848        let values = [value, 0];
849        let nsig = 1 + signal.is_some() as u32;
850        let tsi = vk::TimelineSemaphoreSubmitInfo {
851            s_type: vk::STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO,
852            signal_semaphore_value_count: nsig,
853            p_signal_semaphore_values: values.as_ptr(),
854            ..vk::Zero::zero()
855        };
856        let (ws, wst) = wait.map_or(([0u64], [0u32]), |(s, st)| ([s], [st]));
857        let si = vk::SubmitInfo {
858            s_type: vk::STRUCTURE_TYPE_SUBMIT_INFO,
859            p_next: &tsi as *const _ as *const c_void,
860            wait_semaphore_count: wait.is_some() as u32,
861            p_wait_semaphores: ws.as_ptr(),
862            p_wait_dst_stage_mask: wst.as_ptr(),
863            command_buffer_count: all.len() as u32,
864            p_command_buffers: all.as_ptr(),
865            signal_semaphore_count: nsig,
866            p_signal_semaphores: sems.as_ptr(),
867        };
868        // SAFETY: every command buffer is executable; the queue is held (qlock).
869        let r = unsafe { (self.f.queue_submit)(self.queue, 1, &si, 0) };
870        if r == vk::SUCCESS {
871            self.submitted.store(value, Ordering::Release);
872        }
873        drop(q);
874        if let Err(e) = self.vk(r) {
875            self.fail_submit(slots, chunks);
876            return Err(e);
877        }
878        let mut st = self.st.lock();
879        for s in slots {
880            st.busy.push((value, s));
881        }
882        for c in chunks {
883            st.ring.busy.push((value, c));
884        }
885        if uploads && st.ring.cur_open {
886            st.ring.cur_value = value;
887            st.ring.cur_open = false;
888        }
889        self.collect(&mut st);
890        Ok(value)
891    }
892
893    fn fail_submit(&self, slots: Vec<Slot>, chunks: Vec<Chunk>) {
894        for s in slots {
895            self.give_slot(s);
896        }
897        let mut st = self.st.lock();
898        st.ring.cur_open = false; // its copies never ran
899        for c in chunks {
900            self.destroy(&mut st, Garbage::Buffer(c.buf, c.a));
901        }
902    }
903
904    /// wait until the timeline reaches v -> false on timeout
905    pub fn wait_value(&self, v: u64, timeout_ns: u64) -> Result<bool> {
906        self.check()?;
907        if v <= self.completed.load(Ordering::Acquire) {
908            return Ok(true);
909        }
910        if v > self.submitted() {
911            return Err(GpuError::Invalid("wait: not submitted yet"));
912        }
913        let wi = vk::SemaphoreWaitInfo {
914            s_type: vk::STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
915            semaphore_count: 1,
916            p_semaphores: &self.timeline,
917            p_values: &v,
918            ..vk::Zero::zero()
919        };
920        // SAFETY: the device's timeline semaphore and a value already submitted (or later).
921        let r = unsafe { (self.f.wait_semaphores)(self.dev, &wi, timeout_ns) };
922        if r == vk::TIMEOUT {
923            return Ok(false);
924        }
925        self.vk(r)?;
926        self.completed.fetch_max(v, Ordering::AcqRel);
927        let mut st = self.st.lock();
928        self.collect(&mut st);
929        Ok(true)
930    }
931
932    /// bind-group descriptor sets: a set of layout from the pools (a new pool when all are full)
933    pub fn alloc_set(&self, layout: vk::Handle, needs: &[vk::DescriptorPoolSize]) -> Result<(vk::Handle, vk::Handle)> {
934        let mut st = self.st.lock();
935        let alloc = |pool: vk::Handle| {
936            let ai = vk::DescriptorSetAllocateInfo {
937                s_type: vk::STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO,
938                descriptor_pool: pool,
939                descriptor_set_count: 1,
940                p_set_layouts: &layout,
941                ..vk::Zero::zero()
942            };
943            let mut set = 0;
944            // SAFETY: the pool belongs to this device; the state lock serialises pool use.
945            let r = unsafe { (self.f.allocate_descriptor_sets)(self.dev, &ai, &mut set) };
946            (r, set)
947        };
948        for &pool in st.pools.iter().rev() {
949            let (r, set) = alloc(pool);
950            if r == vk::SUCCESS {
951                return Ok((pool, set));
952            }
953            if r != vk::ERROR_OUT_OF_POOL_MEMORY && r != vk::ERROR_FRAGMENTED_POOL {
954                self.vk(r)?;
955            }
956        }
957        // a new pool, large enough for 64 groups of this layout as well
958        let mut sizes: Vec<vk::DescriptorPoolSize> = [
959            (vk::DESCRIPTOR_TYPE_UNIFORM_BUFFER, 256),
960            (vk::DESCRIPTOR_TYPE_STORAGE_BUFFER, 512),
961            (vk::DESCRIPTOR_TYPE_SAMPLED_IMAGE, 512),
962            (vk::DESCRIPTOR_TYPE_STORAGE_IMAGE, 128),
963            (vk::DESCRIPTOR_TYPE_SAMPLER, 256),
964        ]
965        .iter()
966        .map(|&(t, n)| vk::DescriptorPoolSize { type_: t, descriptor_count: n })
967        .collect();
968        for s in sizes.iter_mut() {
969            let need: u32 = needs.iter().filter(|x| x.type_ == s.type_).map(|x| x.descriptor_count).sum();
970            s.descriptor_count = s.descriptor_count.max(need.saturating_mul(64));
971        }
972        let pi = vk::DescriptorPoolCreateInfo {
973            s_type: vk::STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO,
974            flags: vk::DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT,
975            max_sets: 256,
976            pool_size_count: sizes.len() as u32,
977            p_pool_sizes: sizes.as_ptr(),
978            ..vk::Zero::zero()
979        };
980        let mut pool = 0;
981        // SAFETY: valid create info.
982        self.vk(unsafe { (self.f.create_descriptor_pool)(self.dev, &pi, core::ptr::null(), &mut pool) })?;
983        st.pools.push(pool);
984        let (r, set) = alloc(pool);
985        self.vk(r)?;
986        Ok((pool, set))
987    }
988}
989
990impl Drop for Shared {
991    fn drop(&mut self) {
992        let (f, d, n) = (&self.f, self.dev, core::ptr::null());
993        // SAFETY: the last handle of the device is gone: wait for the GPU, then destroy every
994        // object the device still owns, the device last.
995        unsafe {
996            (f.device_wait_idle)(d);
997            let mut st = self.st.lock();
998            if let Some(u) = st.upload.take() {
999                for c in u.retired {
1000                    st.garbage.push((0, Garbage::Buffer(c.buf, c.a)));
1001                }
1002                st.free.push(u.slot);
1003            }
1004            let ring = core::mem::take(&mut st.ring);
1005            for c in ring.cur.into_iter().chain(ring.free).chain(ring.busy.into_iter().map(|(_, c)| c)) {
1006                st.garbage.push((0, Garbage::Buffer(c.buf, c.a)));
1007            }
1008            let garbage = core::mem::take(&mut st.garbage);
1009            for (_, g) in garbage {
1010                self.destroy(&mut st, g);
1011            }
1012            for (_, s) in core::mem::take(&mut st.busy) {
1013                st.free.push(s);
1014            }
1015            for s in st.free.drain(..) {
1016                (f.destroy_command_pool)(d, s.pool, n);
1017            }
1018            for p in st.pools.drain(..) {
1019                (f.destroy_descriptor_pool)(d, p, n);
1020            }
1021            st.mem.destroy_all(f, d);
1022            (f.destroy_semaphore)(d, self.timeline, n);
1023            drop(st);
1024            (f.destroy_device)(d, n);
1025        }
1026    }
1027}
1028
1029impl Device {
1030    pub fn queue(&self) -> &Queue {
1031        &self.queue
1032    }
1033    pub fn info(&self) -> &AdapterInfo {
1034        &self.sh.info
1035    }
1036    /// what this device has enabled
1037    pub fn features(&self) -> &Features {
1038        &self.sh.features
1039    }
1040    pub fn limits(&self) -> &Limits {
1041        &self.sh.limits
1042    }
1043    pub fn level(&self) -> Level {
1044        self.sh.level
1045    }
1046    /// the device was lost: every call fails with DeviceLost; create a new device
1047    pub fn is_lost(&self) -> bool {
1048        self.sh.check().is_err()
1049    }
1050    /// (test aid) behave as a lost device from now on (a lost device runs nothing any more and
1051    /// its resources may go at once; here the GPU may still work on the last submission: it
1052    /// finishes first)
1053    pub fn test_lose(&self) {
1054        let _ = self.wait_idle();
1055        self.sh.mark_lost();
1056    }
1057    /// the validation layer watches this device
1058    pub fn validating(&self) -> bool {
1059        self.sh.inst.validate
1060    }
1061    /// free what the GPU has finished with (also done by submit and wait)
1062    pub fn poll(&self) -> Result<()> {
1063        self.sh.check()?;
1064        let mut st = self.sh.st.lock();
1065        self.sh.collect(&mut st);
1066        drop(st);
1067        self.sh.check()
1068    }
1069    /// wait for a submission -> false when timeout_ns passed first
1070    pub fn wait(&self, id: SubmitId, timeout_ns: u64) -> Result<bool> {
1071        self.sh.wait_value(id.0, timeout_ns)
1072    }
1073    /// wait for every submission so far
1074    pub fn wait_idle(&self) -> Result<()> {
1075        let v = self.sh.submitted();
1076        if self.sh.wait_value(v, 10_000_000_000)? { Ok(()) } else { Err(GpuError::Timeout) }
1077    }
1078    /// the last submission the GPU has finished
1079    pub fn completed(&self) -> SubmitId {
1080        SubmitId(self.sh.refresh())
1081    }
1082    /// the submission id is done on the GPU (asks the driver; never waits)
1083    pub fn is_done(&self, id: SubmitId) -> bool {
1084        self.sh.refresh() >= id.0
1085    }
1086    /// bytes of device and host memory the device holds in blocks
1087    pub fn memory_reserved(&self) -> u64 {
1088        self.sh.st.lock().mem.reserved
1089    }
1090}
1091
1092impl Queue {
1093    /// run command buffers in order (after the pending uploads) -> their timeline value
1094    pub fn submit<'a, I: IntoIterator<Item = crate::CommandBuffer<'a>>>(&self, cmds: I) -> Result<SubmitId> {
1095        let mut cbs = Vec::new();
1096        let mut slots = Vec::new();
1097        for c in cmds {
1098            let s = c.into_slot();
1099            cbs.push(s.cb);
1100            slots.push(s);
1101        }
1102        self.sh.submit(&cbs, slots, None, None).map(SubmitId)
1103    }
1104
1105    /// submit the pending uploads now
1106    pub fn flush(&self) -> Result<SubmitId> {
1107        self.sh.submit(&[], Vec::new(), None, None).map(SubmitId)
1108    }
1109
1110    /// write data into a buffer at offset before the next submission (staged)
1111    pub fn write_buffer<T: Pod>(&self, buf: &crate::Buffer, offset: u64, data: &[T]) -> Result<()> {
1112        let bytes = crate::bytes_of(data);
1113        let n = bytes.len() as u64;
1114        if offset.checked_add(n).is_none_or(|e| e > buf.size()) {
1115            return Err(GpuError::Invalid("write_buffer: range outside the buffer"));
1116        }
1117        if n == 0 {
1118            return Ok(());
1119        }
1120        let sh = &self.sh;
1121        sh.check()?;
1122        let mut st = sh.st.lock();
1123        let (sbuf, soff, ptr) = sh.stage(&mut st, n)?;
1124        let cb = st.upload.as_ref().map_or(0, |u| u.slot.cb);
1125        // SAFETY: ptr addresses n mapped bytes of the staging chunk; the copy is recorded
1126        // into the upload buffer (recording, guarded by the lock).
1127        unsafe {
1128            core::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr as *mut u8, bytes.len());
1129            let region = vk::BufferCopy { src_offset: soff, dst_offset: offset, size: n };
1130            (sh.f.cmd_copy_buffer)(cb, sbuf, buf.handle(), 1, &region);
1131        }
1132        Ok(())
1133    }
1134
1135    /// write texels into a texture region before the next submission (staged): the region
1136    /// goes from `before` to CopyDst, gets the data, then goes to `after`
1137    pub fn write_texture(&self, dst: &TextureRegion, data: &[u8], layout: DataLayout, before: Layout,
1138                         after: Layout) -> Result<()> {
1139        let t = dst.texture;
1140        let copy = crate::vk_res::copy_region(dst, &layout, data.len() as u64)?;
1141        let need = copy.1;
1142        let sh = &self.sh;
1143        sh.check()?;
1144        let mut st = sh.st.lock();
1145        let (sbuf, soff, ptr) = sh.stage(&mut st, need)?;
1146        let cb = st.upload.as_ref().map_or(0, |u| u.slot.cb);
1147        let mut region = copy.0;
1148        region.buffer_offset = soff;
1149        let range = crate::vk_res::subresources(t, dst.mip, 1, dst.base_layer, dst.layers);
1150        // SAFETY: as in write_buffer; data holds at least `need` bytes after layout.offset
1151        // (copy_region checked it).
1152        unsafe {
1153            core::ptr::copy_nonoverlapping(data.as_ptr().add(layout.offset as usize), ptr as *mut u8, need as usize);
1154            crate::vk_cmd::image_barrier(&sh.f, cb, t.image(), range, before, Layout::CopyDst);
1155            (sh.f.cmd_copy_buffer_to_image)(cb, sbuf, t.image(), vk::IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &region);
1156            crate::vk_cmd::image_barrier(&sh.f, cb, t.image(), range, Layout::CopyDst, after);
1157        }
1158        Ok(())
1159    }
1160
1161    /// write rectangles (x, y, w, h; clipped to the texture, empty ones skipped) of mip 0, layer 0
1162    /// of a 2D texture from `data`, an image of the texture's size in rows of `stride` bytes (a
1163    /// framebuffer), before the next submission; the staged bytes are those of the rectangles.
1164    /// The whole level goes from `before` to CopyDst and then to `after`, once for all of them
1165    pub fn write_texture_rects(&self, t: &crate::Texture, rects: &[[i32; 4]], data: &[u8], stride: u32, before: Layout,
1166                               after: Layout) -> Result<()> {
1167        let f = t.format();
1168        let bb = f.block_bytes() as u64;
1169        if f.is_compressed() || f.is_depth() || t.depth() != 1 {
1170            return Err(GpuError::Invalid("write_texture_rects: an uncompressed 2D colour texture"));
1171        }
1172        let (tw, th) = (t.width() as i32, t.height() as i32);
1173        if (stride as u64) < tw as u64 * bb || (data.len() as u64) < stride as u64 * th as u64 {
1174            return Err(GpuError::Invalid("write_texture_rects: data smaller than the texture"));
1175        }
1176        self.write_texture_rects_at(t, rects, (0, 0), data, stride, before, after)
1177    }
1178
1179    /// write_texture_rects from an image of another size: rectangles (x, y, w, h) of `data`, an
1180    /// image in rows of `stride` bytes, go to (x - origin.0, y - origin.1) of the texture; what
1181    /// lies outside the image or the texture is skipped
1182    #[allow(clippy::too_many_arguments)]
1183    pub fn write_texture_rects_at(&self, t: &crate::Texture, rects: &[[i32; 4]], origin: (i32, i32), data: &[u8], stride: u32, before: Layout,
1184                                  after: Layout) -> Result<()> {
1185        let f = t.format();
1186        let bb = f.block_bytes() as u64;
1187        if f.is_compressed() || f.is_depth() || t.depth() != 1 {
1188            return Err(GpuError::Invalid("write_texture_rects: an uncompressed 2D colour texture"));
1189        }
1190        let (tw, th) = (t.width() as i32, t.height() as i32);
1191        if stride == 0 {
1192            return Err(GpuError::Invalid("write_texture_rects: stride 0"));
1193        }
1194        // the image's size: rows that lie completely in data, as many pixels as a row holds
1195        let (iw, ih) = ((stride as u64 / bb).min(i32::MAX as u64) as i64, (data.len() as u64 / stride as u64).min(i32::MAX as u64) as i64);
1196        let mut regions: Vec<vk::BufferImageCopy> = Vec::with_capacity(rects.len());
1197        let mut total = 0u64;
1198        let sub = crate::vk_res::subresources(t, 0, 1, 0, 1);
1199        let (ox, oy) = (origin.0 as i64, origin.1 as i64);
1200        for r in rects {
1201            // in the texture's coordinates, cut to the texture and the image
1202            let x0 = (r[0] as i64 - ox).max(0).max(-ox);
1203            let y0 = (r[1] as i64 - oy).max(0).max(-oy);
1204            let x1 = (r[0] as i64 + r[2] as i64 - ox).min(tw as i64).min(iw - ox);
1205            let y1 = (r[1] as i64 + r[3] as i64 - oy).min(th as i64).min(ih - oy);
1206            if x1 <= x0 || y1 <= y0 {
1207                continue;
1208            }
1209            let (x0, y0, x1, y1) = (x0 as i32, y0 as i32, x1 as i32, y1 as i32);
1210            let (w, h) = ((x1 - x0) as u32, (y1 - y0) as u32);
1211            regions.push(vk::BufferImageCopy {
1212                buffer_offset: total,
1213                buffer_row_length: w,
1214                buffer_image_height: h,
1215                image_subresource: vk::ImageSubresourceLayers { aspect_mask: sub.aspect_mask, mip_level: 0, base_array_layer: 0, layer_count: 1 },
1216                image_offset: vk::Offset3D { x: x0, y: y0, z: 0 },
1217                image_extent: vk::Extent3D { width: w, height: h, depth: 1 },
1218            });
1219            total += w as u64 * h as u64 * bb;
1220        }
1221        if regions.is_empty() {
1222            return Ok(());
1223        }
1224        let sh = &self.sh;
1225        sh.check()?;
1226        let mut st = sh.st.lock();
1227        let (sbuf, soff, ptr) = sh.stage(&mut st, total)?;
1228        let cb = st.upload.as_ref().map_or(0, |u| u.slot.cb);
1229        for g in regions.iter_mut() {
1230            let (x, y) = ((g.image_offset.x as i64 + ox) as usize, (g.image_offset.y as i64 + oy) as usize);
1231            let (w, h) = (g.image_extent.width as usize, g.image_extent.height as usize);
1232            let row = w * bb as usize;
1233            let mut dst = ptr + g.buffer_offset as usize;
1234            for yy in y..y + h {
1235                let from = yy * stride as usize + x * bb as usize;
1236                // SAFETY: the row lies in data (cut to the image above) and in the staged range
1237                // of `total` bytes, whose rectangles follow each other.
1238                unsafe { core::ptr::copy_nonoverlapping(data.as_ptr().add(from), dst as *mut u8, row) };
1239                dst += row;
1240            }
1241            g.buffer_offset += soff;
1242        }
1243        // SAFETY: the upload buffer is recording (guarded by the lock); the regions lie in the
1244        // texture (clipped above) and the staged range.
1245        unsafe {
1246            crate::vk_cmd::image_barrier(&sh.f, cb, t.image(), sub, before, Layout::CopyDst);
1247            (sh.f.cmd_copy_buffer_to_image)(cb, sbuf, t.image(), vk::IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, regions.len() as u32, regions.as_ptr());
1248            crate::vk_cmd::image_barrier(&sh.f, cb, t.image(), sub, Layout::CopyDst, after);
1249        }
1250        Ok(())
1251    }
1252
1253    /// read a buffer's contents at offset (copies through a readback buffer when needed,
1254    /// submits and waits for the GPU: the slow path, for tools and tests)
1255    pub fn read_buffer<T: Pod>(&self, buf: &crate::Buffer, offset: u64, out: &mut [T]) -> Result<()> {
1256        let n = core::mem::size_of_val(out) as u64;
1257        if offset.checked_add(n).is_none_or(|e| e > buf.size()) {
1258            return Err(GpuError::Invalid("read_buffer: range outside the buffer"));
1259        }
1260        if buf.memory() == MemoryKind::Readback {
1261            let id = self.flush()?;
1262            self.sh.wait_value(id.0, u64::MAX)?;
1263            return buf.read(offset, out);
1264        }
1265        let dev = Device { sh: self.sh.clone(), queue: self.clone() };
1266        let rb = dev.create_buffer(&crate::BufferDesc { size: n.max(4), usage: crate::BufferUsage::NONE, memory: MemoryKind::Readback })?;
1267        let mut enc = dev.encoder()?;
1268        enc.copy_buffer(buf, offset, &rb, 0, n);
1269        let id = self.submit([enc.finish()?])?;
1270        if !self.sh.wait_value(id.0, 10_000_000_000)? {
1271            return Err(GpuError::Timeout);
1272        }
1273        rb.read(0, out)
1274    }
1275}
gpu/vk_dev.rs

gpu/
vk_dev.rs
