1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
// Copyright (c) 2016 The vulkano developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use std::ptr;
use std::sync::Arc;
use std::time::Duration;
use smallvec::SmallVec;

use command_buffer::pool::CommandPool;
use command_buffer::sys::Kind;
use command_buffer::sys::Flags;
use command_buffer::sys::PipelineBarrierBuilder;
use command_buffer::sys::UnsafeCommandBufferBuilder;
use command_buffer::sys::UnsafeCommandBuffer;
use device::Device;
use device::Queue;
use framebuffer::EmptySinglePassRenderPass;
use framebuffer::Framebuffer as OldFramebuffer;
use sync::Fence;
use sync::PipelineStages;
use sync::Semaphore;

use check_errors;
use vk;
use VulkanObject;
use VulkanPointers;
use SynchronizedVulkanObject;

/// Trait for objects that represent commands ready to be executed by the GPU.
pub unsafe trait CommandBuffer {
    /// Submits the command buffer.
    ///
    /// Note that since submitting has a fixed overhead, you should try, if possible, to submit
    /// multiple command buffers at once instead.
    ///
    /// This is a simple shortcut for creating a `Submit` object.
    // TODO: remove 'static
    #[inline]
    fn submit(self, queue: &Arc<Queue>) -> Submission where Self: Sized + 'static {
        Submit::new().add(self).submit(queue)
    }

    /// Type of the pool that was used to allocate the command buffer.
    type Pool: CommandPool;

    /// Iterator that returns the list of semaphores to wait upon before the command buffer is
    /// submitted.
    type SemaphoresWaitIterator: Iterator<Item = (Arc<Semaphore>, PipelineStages)>;

    /// Iterator that returns the list of semaphores to signal after the command buffer has
    /// finished execution.
    type SemaphoresSignalIterator: Iterator<Item = Arc<Semaphore>>;

    /// Returns the inner object.
    fn inner(&self) -> &UnsafeCommandBuffer<Self::Pool>;

    /// Called slightly before the command buffer is submitted. Signals the command buffers that it
    /// is going to be submitted on the given queue. The function must return the list of
    /// semaphores to wait upon and transitions to perform.
    ///
    /// The `fence` parameter is a closure that can be used to pull a fence if required. If a fence
    /// is pulled, it is guaranteed that it will be signaled after the command buffer ends.
    ///
    /// # Safety for the caller
    ///
    /// This function must only be called if there's actually a submission that follows. If a
    /// fence is pulled, then it must eventually be signaled. All the semaphores that are waited
    /// upon must become unsignaled, and all the semaphores that are supposed to be signaled must
    /// become signaled.
    ///
    /// This function is supposed to be called only by vulkano's internals. It is recommended
    /// that you never call it.
    ///
    /// # Safety for the implementation
    ///
    /// The implementation must ensure that the command buffer doesn't get destroyed before the
    /// fence is signaled, or before a fence of a later submission to the same queue is signaled.
    ///
    unsafe fn on_submit<F>(&self, queue: &Arc<Queue>, fence: F)
                           -> SubmitInfo<Self::SemaphoresWaitIterator,
                                         Self::SemaphoresSignalIterator>
        where F: FnMut() -> Arc<Fence>;
}

/// Information about how the submitting function should synchronize the submission.
pub struct SubmitInfo<Swi, Ssi> {
    /// List of semaphores to wait upon before the command buffer starts execution.
    pub semaphores_wait: Swi,
    /// List of semaphores to signal after the command buffer has finished.
    pub semaphores_signal: Ssi,
    /// Pipeline barrier to execute on the queue and immediately before the command buffer.
    /// Ignored if empty.
    pub pre_pipeline_barrier: PipelineBarrierBuilder,
    /// Pipeline barrier to execute on the queue and immediately after the command buffer.
    /// Ignored if empty.
    pub post_pipeline_barrier: PipelineBarrierBuilder,
}

/// Returned when you submit one or multiple command buffers.
///
/// This object holds the resources that are used by the GPU and that must be kept alive for at
/// least as long as the GPU is executing the submission. Therefore destroying a `Submission`
/// object will block until the GPU is finished executing.
///
/// Whenever you submit a command buffer, you are encouraged to store the returned `Submission`
/// in a long-living container such as a `Vec`. From time to time, you can clean the obsolete
/// objects by checking whether `destroying_would_block()` returns false. For example, if you use
/// a `Vec` you can do `vec.retain(|s| s.destroying_would_block())`.
// TODO: docs
// # Leak safety
//
// The `Submission` object can hold borrows of command buffers. In order for it to be safe to leak
// a `Submission`, the borrowed object themselves must be protected by a fence.
#[must_use]
pub struct Submission {
    fence: Arc<Fence>,      // TODO: make optional
    keep_alive: SmallVec<[Arc<KeepAlive>; 4]>,
}

impl Submission {
    /// Returns `true` if destroying this `Submission` object would block the CPU for some time.
    #[inline]
    pub fn destroying_would_block(&self) -> bool {
        !self.finished()
    }

    /// Returns `true` if the GPU has finished executing this submission.
    #[inline]
    pub fn finished(&self) -> bool {
        self.fence.ready().unwrap_or(false)     // TODO: what to do in case of error?   
    }
}

impl Drop for Submission {
    fn drop(&mut self) {
        self.fence.wait(Duration::from_secs(10)).unwrap();      // TODO: handle some errors
    }
}

trait KeepAlive {}
impl<T> KeepAlive for T {}

#[derive(Debug, Copy, Clone)]
pub struct Submit<L> {
    list: L,
}

impl Submit<()> {
    /// Builds an empty submission list.
    #[inline]
    pub fn new() -> Submit<()> {
        Submit { list: () }
    }
}

impl<L> Submit<L> where L: SubmitList {
    /// Adds a command buffer to submit to the list.
    ///
    /// In the Vulkan API, a submission is divided into batches that each contain one or more
    /// command buffers. Vulkano will automatically determine which command buffers can be grouped
    /// into the same batch.
    // TODO: remove 'static
    #[inline]
    pub fn add<C>(self, command_buffer: C) -> Submit<(C, L)> where C: CommandBuffer + 'static {
        Submit { list: (command_buffer, self.list) }
    }

    /// Submits the list of command buffers.
    pub fn submit(self, queue: &Arc<Queue>) -> Submission {
        let SubmitListOpaque { fence, wait_semaphores, wait_stages, command_buffers,
                               signal_semaphores, mut submits, keep_alive }
                             = self.list.infos(queue);

        // TODO: for now we always create a Fence in order to put it in the submission
        let fence = fence.unwrap_or_else(|| Fence::new(queue.device().clone()));

        // Filling the pointers inside `submits`.
        unsafe {
            debug_assert_eq!(wait_semaphores.len(), wait_stages.len());

            let mut next_wait = 0;
            let mut next_cb = 0;
            let mut next_signal = 0;

            for submit in submits.iter_mut() {
                debug_assert!(submit.waitSemaphoreCount as usize + next_wait as usize <=
                              wait_semaphores.len());
                debug_assert!(submit.commandBufferCount as usize + next_cb as usize <=
                              command_buffers.len());
                debug_assert!(submit.signalSemaphoreCount as usize + next_signal as usize <=
                              signal_semaphores.len());

                submit.pWaitSemaphores = wait_semaphores.as_ptr().offset(next_wait);
                submit.pWaitDstStageMask = wait_stages.as_ptr().offset(next_wait);
                submit.pCommandBuffers = command_buffers.as_ptr().offset(next_cb);
                submit.pSignalSemaphores = signal_semaphores.as_ptr().offset(next_signal);

                next_wait += submit.waitSemaphoreCount as isize;
                next_cb += submit.commandBufferCount as isize;
                next_signal += submit.signalSemaphoreCount as isize;
            }

            debug_assert_eq!(next_wait as usize, wait_semaphores.len());
            debug_assert_eq!(next_wait as usize, wait_stages.len());
            debug_assert_eq!(next_cb as usize, command_buffers.len());
            debug_assert_eq!(next_signal as usize, signal_semaphores.len());
        }

        unsafe {
            let vk = queue.device().pointers();
            let queue = queue.internal_object_guard();
            //let fence = fence.as_ref().map(|f| f.internal_object()).unwrap_or(0);
            let fence = fence.internal_object();
            check_errors(vk.QueueSubmit(*queue, submits.len() as u32, submits.as_ptr(),
                                        fence)).unwrap();        // TODO: handle errors (trickier than it looks)
        }

        Submission {
            keep_alive: keep_alive,
            fence: fence,
        }
    }
}

/* TODO: All that stuff below is undocumented */

pub struct SubmitListOpaque {
    fence: Option<Arc<Fence>>,
    wait_semaphores: SmallVec<[vk::Semaphore; 16]>,
    wait_stages: SmallVec<[vk::PipelineStageFlags; 16]>,
    command_buffers: SmallVec<[vk::CommandBuffer; 16]>,
    signal_semaphores: SmallVec<[vk::Semaphore; 16]>,
    submits: SmallVec<[vk::SubmitInfo; 8]>,
    keep_alive: SmallVec<[Arc<KeepAlive>; 4]>,
}

pub unsafe trait SubmitList {
    fn infos(self, queue: &Arc<Queue>) -> SubmitListOpaque;
}

unsafe impl SubmitList for () {
    fn infos(self, queue: &Arc<Queue>) -> SubmitListOpaque {
        SubmitListOpaque {
            fence: None,
            wait_semaphores: SmallVec::new(),
            wait_stages: SmallVec::new(),
            command_buffers: SmallVec::new(),
            signal_semaphores: SmallVec::new(),
            submits: SmallVec::new(),
            keep_alive: SmallVec::new(),
        }
    }
}

// TODO: remove 'static
unsafe impl<C, R> SubmitList for (C, R) where C: CommandBuffer + 'static, R: SubmitList {
    fn infos(self, queue: &Arc<Queue>) -> SubmitListOpaque {
        // TODO: attempt to group multiple submits into one when possible

        let (current, rest) = self;

        let mut infos = rest.infos(queue);
        let device = current.inner().device().clone();
        let current_infos = unsafe { current.on_submit(queue, || {
            if let Some(fence) = infos.fence.as_ref() {
                return fence.clone();
            }

            let new_fence = Fence::new(device.clone());
            infos.fence = Some(new_fence.clone());
            new_fence
        })};

        let mut new_submit = vk::SubmitInfo {
            sType: vk::STRUCTURE_TYPE_SUBMIT_INFO,
            pNext: ptr::null(),
            waitSemaphoreCount: 0,
            pWaitSemaphores: ptr::null(),
            pWaitDstStageMask: ptr::null(),
            commandBufferCount: 1,
            pCommandBuffers: ptr::null(),
            signalSemaphoreCount: 0,
            pSignalSemaphores: ptr::null(),
        };

        if !current_infos.pre_pipeline_barrier.is_empty() {
            let mut cb = UnsafeCommandBufferBuilder::new(Device::standard_command_pool(&device, queue.family()),
                                                         Kind::Primary::<EmptySinglePassRenderPass,
                                                                         OldFramebuffer<EmptySinglePassRenderPass>>,
                                                         Flags::OneTimeSubmit).unwrap();
            cb.pipeline_barrier(current_infos.pre_pipeline_barrier);
            new_submit.commandBufferCount += 1;
            infos.command_buffers.push(cb.internal_object());
            infos.keep_alive.push(Arc::new(cb) as Arc<_>);
        }

        infos.command_buffers.push(current.inner().internal_object());

        if !current_infos.post_pipeline_barrier.is_empty() {
            let mut cb = UnsafeCommandBufferBuilder::new(Device::standard_command_pool(&device, queue.family()),
                                                         Kind::Primary::<EmptySinglePassRenderPass,
                                                                         OldFramebuffer<EmptySinglePassRenderPass>>,
                                                         Flags::OneTimeSubmit).unwrap();
            cb.pipeline_barrier(current_infos.post_pipeline_barrier);
            new_submit.commandBufferCount += 1;
            infos.command_buffers.push(cb.internal_object());
            infos.keep_alive.push(Arc::new(cb) as Arc<_>);
        }

        for (semaphore, stage) in current_infos.semaphores_wait {
            infos.wait_semaphores.push(semaphore.internal_object());
            infos.wait_stages.push(stage.into());
            infos.keep_alive.push(semaphore);
            new_submit.waitSemaphoreCount += 1;
        }

        for semaphore in current_infos.semaphores_signal {
            infos.signal_semaphores.push(semaphore.internal_object());
            infos.keep_alive.push(semaphore);
            new_submit.signalSemaphoreCount += 1;
        }

        infos.submits.push(new_submit);
        infos.keep_alive.push(Arc::new(current) as Arc<_>);

        infos
    }
}

#[cfg(test)]
mod tests {
    use std::iter;
    use std::iter::Empty;
    use std::sync::Arc;

    use command_buffer::pool::StandardCommandPool;
    use command_buffer::submit::CommandBuffer;
    use command_buffer::submit::SubmitInfo;
    use command_buffer::sys::Kind;
    use command_buffer::sys::Flags;
    use command_buffer::sys::PipelineBarrierBuilder;
    use command_buffer::sys::UnsafeCommandBuffer;
    use command_buffer::sys::UnsafeCommandBufferBuilder;
    use device::Device;
    use device::Queue;
    use framebuffer::EmptySinglePassRenderPass;
    use framebuffer::Framebuffer as OldFramebuffer;
    use sync::Fence;
    use sync::PipelineStages;
    use sync::Semaphore;

    #[test]
    fn basic_submit() {
        struct Basic { inner: UnsafeCommandBuffer<Arc<StandardCommandPool>> }
        unsafe impl CommandBuffer for Basic {
            type Pool = Arc<StandardCommandPool>;
            type SemaphoresWaitIterator = Empty<(Arc<Semaphore>, PipelineStages)>;
            type SemaphoresSignalIterator = Empty<Arc<Semaphore>>;

            fn inner(&self) -> &UnsafeCommandBuffer<Self::Pool> { &self.inner }

            unsafe fn on_submit<F>(&self, _: &Arc<Queue>, fence: F)
                                   -> SubmitInfo<Self::SemaphoresWaitIterator,
                                                 Self::SemaphoresSignalIterator>
                where F: FnOnce() -> Arc<Fence>
            {
                SubmitInfo {
                    semaphores_wait: iter::empty(),
                    semaphores_signal: iter::empty(),
                    pre_pipeline_barrier: PipelineBarrierBuilder::new(),
                    post_pipeline_barrier: PipelineBarrierBuilder::new(),
                }
            }
        }

        let (device, queue) = gfx_dev_and_queue!();

        let pool = Device::standard_command_pool(&device, queue.family());
        let kind = Kind::Primary::<EmptySinglePassRenderPass, OldFramebuffer<EmptySinglePassRenderPass>>;

        let cb = UnsafeCommandBufferBuilder::new(pool, kind, Flags::OneTimeSubmit).unwrap();
        let cb = Basic { inner: cb.build().unwrap() };

        let _s = cb.submit(&queue);
    }
}