use core::ffi::c_void;
use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, MutexGuard, OnceLock};
use std::time::Duration;
use serde_json::Value;
use windows::Win32::Foundation::{CloseHandle, WAIT_OBJECT_0};
use windows::Win32::Graphics::Direct3D12::*;
use windows::Win32::System::Threading::{CreateEventW, WaitForSingleObject};
use windows::core::Interface;
use super::states::{Access, Read};
const BUDGET: u64 = 4 << 30;
static BYTES: AtomicU64 = AtomicU64::new(0);
pub type Key = (u64, u64);
#[derive(Clone, Debug)]
pub struct Input {
pub part: &'static str,
pub geometry: Option<usize>,
pub resource: u64,
pub offset: u64,
pub bytes: u64,
at: u64,
}
pub struct Build {
pub dest: Key,
pub description: Value,
pub inputs: Vec<Input>,
pub error: Option<String>,
copy: Option<Copy>,
gather: Option<super::gather::Gather>,
last: Mutex<Option<(ID3D12Fence, u64)>>,
}
struct Copy {
buffer: ID3D12Resource,
bytes: u64,
}
impl Drop for Copy {
fn drop(&mut self) {
BYTES.fetch_sub(self.bytes, Ordering::AcqRel);
}
}
impl Build {
fn done(&self) -> bool {
self.last
.lock()
.unwrap_or_else(|e| e.into_inner())
.as_ref()
.is_none_or(|(f, v)| unsafe { f.GetCompletedValue() } >= *v)
}
pub fn read(&self, patience: Duration) -> Result<Vec<Vec<u8>>, String> {
if let Some(why) = &self.error {
return Err(why.clone());
}
let copy = self.copy.as_ref().ok_or("its inputs were not copied")?;
let analysis = match &self.gather {
Some(g) => {
let analysis = g.analysis.get().ok_or("its list was never submitted")?.as_ref()?;
if let Some(why) = g.failed.lock().unwrap_or_else(|e| e.into_inner()).clone() {
return Err(format!("its instances' gather was not made: {why}"));
}
Some((g.count, analysis))
}
None => None,
};
let last = self.last.lock().unwrap_or_else(|e| e.into_inner()).clone();
let Some((fence, value)) = last else {
return Err("its list was never submitted".into());
};
unsafe { wait(&fence, value, patience) }?;
let mut mapped = core::ptr::null_mut();
unsafe { copy.buffer.Map(0, None, Some(&mut mapped)) }.map_err(|e| format!("Map: {e}"))?;
let all = unsafe { core::slice::from_raw_parts(mapped.cast::<u8>(), copy.bytes as usize) };
let parts = analysis
.map_or(Ok(()), |(count, a)| super::gather::verdict(all, count, a))
.map(|()| {
self.inputs
.iter()
.map(|i| all[i.at as usize..(i.at + i.bytes) as usize].to_vec())
.collect()
});
unsafe { copy.buffer.Unmap(0, None) };
parts
}
}
unsafe fn wait(fence: &ID3D12Fence, value: u64, patience: Duration) -> Result<(), String> {
if unsafe { fence.GetCompletedValue() } >= value {
return Ok(());
}
let event = unsafe { CreateEventW(None, false, false, None) }.map_err(|e| format!("an event: {e}"))?;
let waited = unsafe { fence.SetEventOnCompletion(value, event) }
.map_err(|e| format!("SetEventOnCompletion: {e}"))
.and_then(|()| {
(unsafe { WaitForSingleObject(event, patience.as_millis() as u32) } == WAIT_OBJECT_0)
.then_some(())
.ok_or_else(|| format!("the GPU did not copy its inputs within {} s", patience.as_secs()))
});
unsafe { CloseHandle(event) }.ok();
waited
}
enum Step {
Build(Arc<Build>),
Copy(Key, Key, i32),
}
#[derive(Default)]
struct State {
lists: HashMap<u64, Vec<Step>>,
latest: HashMap<Key, Arc<Build>>,
leaving: Vec<Arc<Build>>,
}
fn state() -> MutexGuard<'static, State> {
static STATE: OnceLock<Mutex<State>> = OnceLock::new();
STATE
.get_or_init(Default::default)
.lock()
.unwrap_or_else(|e| e.into_inner())
}
fn key(address: u64) -> Option<Key> {
super::slots::unique_address(address).or_else(|| super::slots::buffers_at(address).first().copied())
}
fn extent(count: u64, stride: u64, last: u64) -> u64 {
if count == 0 {
0
} else {
(count - 1) * stride + last
}
}
type Range = (&'static str, Option<usize>, u64, u64);
unsafe fn ranges(i: &D3D12_BUILD_RAYTRACING_ACCELERATION_STRUCTURE_INPUTS) -> Result<Vec<Range>, String> {
let n = i.NumDescs as usize;
if i.Type == D3D12_RAYTRACING_ACCELERATION_STRUCTURE_TYPE_TOP_LEVEL {
let bytes = n as u64 * size_of::<D3D12_RAYTRACING_INSTANCE_DESC>() as u64;
return Ok(vec![(
"instances",
None,
unsafe { i.Anonymous.InstanceDescs },
bytes,
)]);
}
let geometry: Vec<&D3D12_RAYTRACING_GEOMETRY_DESC> = if i.DescsLayout == D3D12_ELEMENTS_LAYOUT_ARRAY {
unsafe { super::values::slice(i.Anonymous.pGeometryDescs, n) }
.iter()
.collect()
} else {
unsafe { super::values::slice(i.Anonymous.ppGeometryDescs, n) }
.iter()
.map(|g| unsafe { g.as_ref() }.ok_or("a null geometry description"))
.collect::<Result<_, _>>()?
};
let mut out = Vec::new();
for (k, g) in geometry.into_iter().enumerate() {
if g.Type == D3D12_RAYTRACING_GEOMETRY_TYPE_TRIANGLES {
let t = unsafe { &g.Anonymous.Triangles };
let vertex = locum_capture::format::layout(t.VertexFormat.0 as u32)
.map(|f| u64::from(f.block_bytes))
.ok_or_else(|| format!("vertex format {} has no size locum knows", t.VertexFormat.0))?;
out.push((
"vertices",
Some(k),
t.VertexBuffer.StartAddress,
extent(u64::from(t.VertexCount), t.VertexBuffer.StrideInBytes, vertex),
));
if t.IndexBuffer != 0 && t.IndexCount > 0 {
let index = if t.IndexFormat == windows::Win32::Graphics::Dxgi::Common::DXGI_FORMAT_R16_UINT {
2
} else {
4
};
out.push(("indices", Some(k), t.IndexBuffer, u64::from(t.IndexCount) * index));
}
if t.Transform3x4 != 0 {
out.push(("transform", Some(k), t.Transform3x4, 48));
}
} else if g.Type == D3D12_RAYTRACING_GEOMETRY_TYPE_PROCEDURAL_PRIMITIVE_AABBS {
let a = unsafe { &g.Anonymous.AABBs };
out.push((
"aabbs",
Some(k),
a.AABBs.StartAddress,
extent(
a.AABBCount,
a.AABBs.StrideInBytes,
size_of::<D3D12_RAYTRACING_AABB>() as u64,
),
));
} else {
return Err(format!("geometry type {} is not represented", g.Type.0));
}
}
Ok(out)
}
fn access(commands: &[locum_capture::d3d12::Command], id: u64, real: &ID3D12Resource) -> Read {
let mut known = None;
let mut enhanced = false;
for c in commands {
match c.call {
"ResourceBarrier" => {
for b in c.args[0].as_array().into_iter().flatten() {
let p = &b["payload"];
if b["type"] == 0 && b["flags"] != 1 && p["resource"].as_u64() == Some(id) {
known = p["after"].as_i64().map(|s| Access::State(s as i32));
}
}
}
"Barrier" => enhanced = true,
_ => {}
}
}
Read::of(known, enhanced, real)
}
fn predication(commands: &[locum_capture::d3d12::Command]) -> Option<(u64, u64, i32)> {
let last = commands.iter().rev().find(|c| c.call == "SetPredication")?;
let a = &last.args;
a[0].as_u64()
.filter(|r| *r != 0)
.map(|r| (r, a[1].as_u64().unwrap_or(0), a[2].as_i64().unwrap_or(0) as i32))
}
pub unsafe fn built(
data: &super::journal::Data,
real: *mut c_void,
p: *const D3D12_BUILD_RAYTRACING_ACCELERATION_STRUCTURE_DESC,
) {
if !super::journal::enabled() {
return;
}
let Some(d) = (unsafe { p.as_ref() }) else {
return;
};
let Some(dest) = key(d.DestAccelerationStructureData) else {
return;
};
let description = unsafe { super::values::acceleration_structure_build(p) };
let (inputs, copy, gather, error) = match unsafe { copy_inputs(data, real, &d.Inputs) } {
Ok((inputs, copy, gather)) => (inputs, copy, gather, None),
Err(why) => (Vec::new(), None, None, Some(why)),
};
let build = Arc::new(Build {
dest,
description,
inputs,
error,
copy,
gather,
last: Mutex::new(None),
});
state().lists.entry(data.id).or_default().push(Step::Build(build));
}
type Copied = (Vec<Input>, Option<Copy>, Option<super::gather::Gather>);
unsafe fn readback(list: &ID3D12GraphicsCommandList, bytes: u64) -> Result<(ID3D12Device, Copy), String> {
if BYTES.fetch_add(bytes, Ordering::AcqRel) + bytes > BUDGET {
BYTES.fetch_sub(bytes, Ordering::AcqRel);
return Err(format!(
"the {} GiB kept of builds' inputs are spent",
BUDGET >> 30
));
}
let mut device: Option<ID3D12Device> = None;
let made = unsafe { list.GetDevice(&mut device) }
.map_err(|e| e.to_string())
.and_then(|()| device.ok_or_else(|| "no device".to_string()))
.and_then(|d| unsafe { super::passes::readback_resource(&d, bytes) }.map(|b| (d, b)));
match made {
Ok((device, buffer)) => Ok((device, Copy { buffer, bytes })),
Err(why) => {
BYTES.fetch_sub(bytes, Ordering::AcqRel);
Err(why)
}
}
}
unsafe fn copy_inputs(
data: &super::journal::Data,
real: *mut c_void,
i: &D3D12_BUILD_RAYTRACING_ACCELERATION_STRUCTURE_INPUTS,
) -> Result<Copied, String> {
let list = unsafe { ID3D12GraphicsCommandList::from_raw_borrowed(&real) }.ok_or("no list")?;
let commands = data
.recording
.lock()
.unwrap_or_else(|e| e.into_inner())
.as_ref()
.map(|r| r.snapshot())
.ok_or("its list is not recorded")?;
let commands = &commands.commands;
if i.Type == D3D12_RAYTRACING_ACCELERATION_STRUCTURE_TYPE_TOP_LEVEL
&& i.DescsLayout == D3D12_ELEMENTS_LAYOUT_ARRAY_OF_POINTERS
{
return unsafe { gathered(data, list, commands, i) };
}
let ranges = unsafe { ranges(i) }?;
let mut inputs = Vec::new();
let mut copies = Vec::new();
let mut at = 0;
for (part, geometry, address, bytes) in ranges {
let (resource, offset) =
key(address).ok_or_else(|| format!("its {part} are in no live buffer locum knows"))?;
let real = super::real::<ID3D12Resource>(resource).ok_or("an input buffer was destroyed")?;
copies.push(super::passes::Source {
resource,
read: access(commands, resource, &real),
real,
offset,
bytes,
});
inputs.push(Input {
part,
geometry,
resource,
offset,
bytes,
at,
});
at += bytes;
}
if at == 0 {
return Ok((inputs, None, None));
}
let (_, copy) = unsafe { readback(list, at) }?;
let predicate = predication(commands)
.and_then(|(r, offset, op)| super::real::<ID3D12Resource>(r).map(|r| (r, offset, op)));
let promoted = unsafe {
super::passes::record_ranges(
list,
&copies,
D3D12_BARRIER_ACCESS_SHADER_RESOURCE,
predicate,
©.buffer,
)
}?;
promote(data, promoted);
Ok((inputs, Some(copy), None))
}
fn promote(data: &super::journal::Data, promoted: Vec<u64>) {
if let Some(r) = data.recording.lock().unwrap_or_else(|e| e.into_inner()).as_mut() {
for resource in promoted {
r.promote(resource);
}
}
}
unsafe fn gathered(
data: &super::journal::Data,
list: &ID3D12GraphicsCommandList,
commands: &[locum_capture::d3d12::Command],
i: &D3D12_BUILD_RAYTRACING_ACCELERATION_STRUCTURE_INPUTS,
) -> Result<Copied, String> {
let count = u64::from(i.NumDescs);
if count == 0 {
return Ok((Vec::new(), None, None));
}
if predication(commands).is_some() {
return Err(
"its build is predicated (SetPredication), so locum does not know whether it ran, nor gathers its \
instances, which are given as an array of pointers"
.into(),
);
}
let address = unsafe { i.Anonymous.InstanceDescs };
let (resource, offset) = key(address).ok_or("its instance pointers are in no live buffer locum knows")?;
let real =
super::real::<ID3D12Resource>(resource).ok_or("its instance pointers' buffer was destroyed")?;
let source = super::passes::Source {
resource,
read: access(commands, resource, &real),
real,
offset,
bytes: count * 8,
};
let (device, copy) = unsafe { readback(list, super::gather::bytes(count)) }?;
let instances = count * size_of::<D3D12_RAYTRACING_INSTANCE_DESC>() as u64;
let inputs = vec![
Input {
part: "instances",
geometry: None,
resource,
offset,
bytes: instances,
at: 0,
},
Input {
part: "pointers",
geometry: None,
resource,
offset,
bytes: count * 8,
at: instances,
},
];
let (gather, promoted) = unsafe {
super::gather::prepare(
&device,
list,
source,
i.NumDescs,
commands.len().saturating_sub(1),
)
}?;
promote(data, promoted);
Ok((inputs, Some(copy), Some(gather)))
}
pub fn copied(list: u64, dest: u64, source: u64, mode: D3D12_RAYTRACING_ACCELERATION_STRUCTURE_COPY_MODE) {
if !super::journal::enabled() {
return;
}
if let (Some(dest), Some(source)) = (key(dest), key(source)) {
state()
.lists
.entry(list)
.or_default()
.push(Step::Copy(dest, source, mode.0));
}
}
pub fn retire(list: u64) {
let mut s = state();
if let Some(steps) = s.lists.remove(&list) {
for step in steps {
if let Step::Build(b) = step {
s.leaving.push(b);
}
}
}
sweep(&mut s);
}
fn sweep(s: &mut State) {
s.leaving.retain(|b| Arc::strong_count(b) > 1 || !b.done());
}
pub fn destroyed(id: u64) {
let mut s = state();
let gone: Vec<Key> = s.latest.keys().filter(|k| k.0 == id).copied().collect();
for k in gone {
if let Some(b) = s.latest.remove(&k) {
s.leaving.push(b);
}
}
}
pub fn executing(lists: &[u64]) -> Vec<Arc<Build>> {
let mut guard = state();
let State {
lists: steps,
latest,
leaving,
} = &mut *guard;
let mut builds = Vec::new();
for step in lists.iter().flat_map(|l| steps.get(l)).flatten() {
let (dest, now) = match step {
Step::Build(b) => {
builds.push(b.clone());
(b.dest, Some(b.clone()))
}
Step::Copy(dest, source, mode)
if *mode == D3D12_RAYTRACING_ACCELERATION_STRUCTURE_COPY_MODE_CLONE.0
|| *mode == D3D12_RAYTRACING_ACCELERATION_STRUCTURE_COPY_MODE_COMPACT.0 =>
{
(*dest, latest.get(source).cloned())
}
Step::Copy(dest, ..) => (*dest, None),
};
let replaced = match now {
Some(b) => latest.insert(dest, b),
None => latest.remove(&dest),
};
leaving.extend(replaced);
}
sweep(&mut guard);
builds
}
pub fn gathers(lists: &[(usize, &locum_capture::d3d12::List)]) -> Vec<(usize, Vec<Arc<Build>>)> {
let s = state();
let mut out = Vec::new();
for &(at, list) in lists {
let builds: Vec<Arc<Build>> = s
.lists
.get(&list.id)
.into_iter()
.flatten()
.filter_map(|step| match step {
Step::Build(b) => b.gather.as_ref().map(|g| (b, g)),
Step::Copy(..) => None,
})
.filter(|(_, g)| {
g.analysis
.get_or_init(|| super::gather::analyse(&list.commands, g.at, &super::slots::buffer_range))
.is_ok()
})
.map(|(b, _)| b.clone())
.collect();
if !builds.is_empty() {
out.push((at, builds));
}
}
out
}
pub unsafe fn record_gathers(list: &ID3D12GraphicsCommandList, builds: &[Arc<Build>]) {
let gathers: Vec<_> = builds
.iter()
.filter_map(|b| {
let g = b.gather.as_ref()?;
let analysis = g.analysis.get()?.as_ref().ok()?;
Some((g, analysis, &b.copy.as_ref()?.buffer))
})
.collect();
unsafe { super::gather::record(list, &gathers) };
}
pub fn gathered_at_submission(builds: &[Arc<Build>], why: Option<&str>) {
for b in builds {
if let Some(g) = &b.gather {
*g.failed.lock().unwrap_or_else(|e| e.into_inner()) = why.map(str::to_string);
}
}
}
pub unsafe fn submitted(queue_id: u64, queue: *mut c_void, builds: Vec<Arc<Build>>) {
if builds.is_empty() {
return;
}
match unsafe { super::passes::mark(queue_id, queue) } {
Ok(marked) => {
for b in builds {
*b.last.lock().unwrap_or_else(|e| e.into_inner()) = Some(marked.clone());
}
}
Err(why) => crate::log::log!("D3D12 acceleration structures: {why} on queue {queue_id}"),
}
}
pub fn latest() -> Vec<(Key, Arc<Build>)> {
let s = state();
let mut all: Vec<_> = s.latest.iter().map(|(k, b)| (*k, b.clone())).collect();
all.sort_by_key(|(k, _)| *k);
all
}
const PATIENCE: Duration = Duration::from_secs(10);
pub fn record(key: Key, b: &Build) -> locum_capture::d3d12::AccelerationStructure {
let inputs = b.read(PATIENCE).map(|parts| {
let blobs = &crate::settings::settings().blobs;
Value::Array(
b.inputs
.iter()
.zip(parts)
.map(|(i, bytes)| {
let hash = locum_capture::hash::key(&bytes);
locum_capture::writer::blob(blobs, hash.clone(), bytes, true);
serde_json::json!({"part":i.part,"geometry":i.geometry,"resource":i.resource,"offset":i.offset,
"bytes":i.bytes,"blob":hash})
})
.collect(),
)
});
locum_capture::d3d12::AccelerationStructure {
resource: key.0,
offset: key.1,
build: b.description.clone(),
inputs,
}
}