concinnity-device 0.19.119

//! Backing store for the render graph's transient render targets on D3D12. The
//! shared `gfx::render_graph::alias` planner decides which transients can share
//! physical memory; this pool realizes that on D3D12 with placed resources on an
//! `ID3D12Heap` (the analogue of Vulkan's aliased `VkImage`s on a shared
//! `VkDeviceMemory`). Features stop owning these resources and read them back by
//! label, so the pool can repoint several labels at one heap region without
//! touching the features.
//!
//! Buffering: D3D12 is single-buffered for these targets. The command queue runs
//! frames in submission order and the per-resource state-transition barriers
//! serialize a frame's writes against a prior frame's reads of the same resource,
//! so a single resource is safe across frames in flight (unlike Vulkan, whose
//! explicit-layout model led that backend to per-frame buffer its bloom chain).
//! So a DX alias slot is ONE shared heap region (not per-frame): making the
//! members per-frame would multiply already-single-buffered resources and cost
//! more memory than aliasing saves. The cross-frame reuse ordering is carried by
//! aliasing barriers at both reuse boundaries (added when sharing lands).
//!
//! A resource is "managed" iff its owning feature is enabled at build time (e.g.
//! `ao_output` only when SSAO is on); `resource_for` returns `None` otherwise and
//! the consumer keeps its disabled-feature fallback.

use concinnity_core::render::error::{RenderError, RenderResult};
use concinnity_core::render::render_graph::{
    ClearValue, PixelFormat, TextureUsage, TransientSlot, TransientTexture,
};
use std::collections::HashMap;
use windows::Win32::Graphics::Direct3D12::*;
use windows::Win32::Graphics::Dxgi::Common::*;

use super::texture::{one_shot_submit, transition_barrier};
use crate::directx::error::map_hresult;

// Everything about one label that is fixed once the pool is built: which placed
// resource backs it, and which member it reclaims heap memory from. The executor
// resolves both for every graph resource every frame, so they are indexed here
// rather than re-scanned per lookup. Mirrors the Vulkan pool's `by_label`.
struct LabelEntry {
    // Index into `resources`.
    resource: usize,
    alias_predecessor: Option<&'static str>,
}

// The transient render-target pool owned by `DxContext`. Resolution-dependent,
// so it is rebuilt on swapchain resize. COM resources release on drop, so the
// old heaps + resources free when the pool is reassigned (the caller has already
// idled the device).
pub(super) struct TransientResourcePool {
    // One heap per slot. Held only to keep the heaps alive: a placed resource
    // does not keep its heap alive (D3D12 requires the heap to outlive the
    // resource), so the pool must retain them. Never read after construction.
    #[expect(
        dead_code,
        reason = "a placed resource does not keep its heap alive, so the pool retains the heaps"
    )]
    heaps: Vec<ID3D12Heap>,
    // Every placed member, in slot-major member order -- the order `index_labels`
    // assigns `LabelEntry::resource` from.
    resources: Vec<ID3D12Resource>,
    by_label: HashMap<&'static str, LabelEntry>,
    // The member labels of each slot, in the order they reuse its heap region.
    // Read back by the executor's per-frame soundness assertion, which runs
    // under `debug_assertions`, and so does its only source of truth.
    #[cfg(debug_assertions)]
    slot_labels: Vec<Vec<&'static str>>,
    // The pool's aliased footprint: the sum of its slot heap sizes. Reported to
    // the memory ledger, which would otherwise not see this pool at all -- it
    // deliberately sits off the device allocator.
    allocated_bytes: u64,
}

impl TransientResourcePool {
    // Allocate one heap per slot, sized to the largest member, and place every
    // member resource at offset 0. Each is created in its `initial_state` with
    // its optimized clear value, exactly as the committed version was, so its
    // first-use barrier is unchanged.
    pub(super) fn build(
        device: &ID3D12Device,
        queue: &ID3D12CommandQueue,
        slots: &[TransientSlot],
    ) -> RenderResult<Self> {
        let mut heaps = Vec::new();
        let mut resources = Vec::new();
        // `allocated_bytes` is what the pool really reserves (one heap per slot,
        // sized to its largest member); `unaliased_bytes` is what the same
        // members would cost one heap each. Their difference is the aliasing
        // saving, which is otherwise invisible: the ledger sees only the total,
        // so a plan that quietly stopped sharing would look like a bigger scene.
        let mut allocated_bytes: u64 = 0;
        let mut unaliased_bytes: u64 = 0;
        // (resource, resting state) for the one-shot init below. A placed
        // render-target resource is NOT auto-zeroed like a committed one, so
        // D3D12 rejects its first draw/sample until a Clear/Discard/Copy
        // initializes it. Discard suffices (no need to define the contents):
        // every managed transient is fully written each frame before it is read
        // (the SSAO blur writes `ao_output`, the bloom prefilter writes
        // `bloom_top`), and a consumer that may run while a target is unwritten
        // guards its read (the composite skips `bloom_top` when bloom is off),
        // so the undefined initial contents are never observed. Only single-
        // member slots are initialized here; a shared slot's members are
        // re-initialized per frame by the executor's aliasing barrier + Discard
        // before each first write (Discarding them here, on shared memory with no
        // aliasing barrier between, would itself be an aliasing hazard).
        let mut to_init: Vec<(ID3D12Resource, D3D12_RESOURCE_STATES)> = Vec::new();
        for slot in slots {
            let shared = slot.members.len() > 1;
            // Size the heap to the largest member; offset 0 satisfies every
            // member's alignment, so aliased members all place there.
            let mut slot_size: u64 = 0;
            let mut slot_align: u64 = D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT as u64;
            let descs: Vec<(&TransientTexture, D3D12_RESOURCE_DESC)> = slot
                .members
                .iter()
                .map(|m| {
                    let desc = rt_desc(m);
                    // SAFETY: a query on a live COM object; the descriptor it reads and the out-
                    // parameters it fills are live locals that outlive the call.
                    let info = unsafe { device.GetResourceAllocationInfo(0, &[desc]) };
                    slot_size = slot_size.max(info.SizeInBytes);
                    slot_align = slot_align.max(info.Alignment);
                    unaliased_bytes += info.SizeInBytes;
                    (m, desc)
                })
                .collect();

            let heap_desc = D3D12_HEAP_DESC {
                SizeInBytes: slot_size,
                Properties: D3D12_HEAP_PROPERTIES {
                    Type: D3D12_HEAP_TYPE_DEFAULT,
                    ..Default::default()
                },
                Alignment: slot_align,
                // These targets are all render targets, so a heap restricted to
                // RT/DS textures is valid on every resource-heap tier.
                Flags: D3D12_HEAP_FLAG_ALLOW_ONLY_RT_DS_TEXTURES,
            };
            allocated_bytes += slot_size;
            let mut heap: Option<ID3D12Heap> = None;
            // SAFETY: the create descriptor and every pointer it borrows are live for the call, and
            // the new COM object lands in a binding that owns it.
            unsafe { device.CreateHeap(&heap_desc, &mut heap) }
                .map_err(|e| map_hresult(e.code(), "transient pool heap"))?;
            let heap =
                heap.ok_or_else(|| RenderError::Other("transient pool heap returned None".into()))?;

            for (m, desc) in &descs {
                let clear = clear_value(m);
                let mut res: Option<ID3D12Resource> = None;
                // SAFETY: the create descriptor and every pointer it borrows are live for the call,
                // and the new COM object lands in a binding that owns it.
                unsafe {
                    device.CreatePlacedResource(
                        &heap,
                        0,
                        desc,
                        resting_state(m),
                        Some(&clear),
                        &mut res,
                    )
                }
                .map_err(|e| map_hresult(e.code(), &format!("transient pool place {}", m.label)))?;
                let resource = res.ok_or_else(|| {
                    RenderError::Other("transient pool placed resource None".into())
                })?;
                if !shared {
                    to_init.push((resource.clone(), resting_state(m)));
                }
                resources.push(resource);
            }
            heaps.push(heap);
        }

        // Initialize every placed resource (Discard in its RENDER_TARGET state,
        // then back to its resting state) so its first real use is legal.
        if !to_init.is_empty() {
            one_shot_submit(device, queue, |cmd| {
                for (res, resting) in &to_init {
                    // SAFETY: the command list is in the recording state, and every resource,
                    // descriptor and slice these commands name is live for the call.
                    unsafe {
                        cmd.ResourceBarrier(&[transition_barrier(
                            res,
                            *resting,
                            D3D12_RESOURCE_STATE_RENDER_TARGET,
                        )]);
                        cmd.DiscardResource(res, None);
                        cmd.ResourceBarrier(&[transition_barrier(
                            res,
                            D3D12_RESOURCE_STATE_RENDER_TARGET,
                            *resting,
                        )]);
                    }
                }
            })?;
        }

        tracing::info!(
            "transient heap pool: {} heap allocation(s), {} KiB ({} KiB saved by aliasing)",
            heaps.len(),
            allocated_bytes / 1024,
            unaliased_bytes.saturating_sub(allocated_bytes) / 1024,
        );
        let by_label = index_labels(slots);
        debug_assert_eq!(by_label.len(), resources.len(), "a label was placed twice");
        Ok(Self {
            heaps,
            resources,
            by_label,
            #[cfg(debug_assertions)]
            slot_labels: slots.iter().map(|s| s.labels()).collect(),
            allocated_bytes,
        })
    }

    // The managed resource for `label`, or `None` when the owning feature was
    // disabled at build time (so nothing was placed).
    pub(super) fn resource_for(&self, label: &str) -> Option<&ID3D12Resource> {
        self.resources.get(self.by_label.get(label)?.resource)
    }

    // The label of the resource whose heap memory `label` reclaims (its cyclic
    // slot predecessor), or `None` when `label` is not a shared-slot member (so
    // it is not aliased and needs no aliasing barrier). The executor emits an
    // aliasing barrier before the pass that first-writes any resource for which
    // this returns `Some`.
    pub(super) fn alias_predecessor(&self, label: &str) -> Option<&'static str> {
        self.by_label.get(label)?.alias_predecessor
    }

    // The pool's aliased footprint in bytes, for the memory ledger.
    pub(super) fn allocated_bytes(&self) -> u64 {
        self.allocated_bytes
    }

    // The three pooled G-buffer color targets, or `None` when the pool was
    // built without the G-buffer gate (no screen-space consumer, so the
    // pre-pass node is absent and nothing was placed). All three are placed
    // together or not at all, so a partial result is a planner bug rather than
    // a state a caller should handle.
    pub(super) fn gbuffer_pooled(&self) -> Option<super::post::gbuffer::GbufferPooled> {
        Some(super::post::gbuffer::GbufferPooled {
            normal_depth: self.resource_for("gbuffer_normal_depth")?.clone(),
            roughness: self.resource_for("gbuffer_roughness")?.clone(),
            velocity: self.resource_for("gbuffer_velocity")?.clone(),
        })
    }

    // The member labels of each slot, for the executor's per-frame check that
    // no slot has two resources live at once in the graph it is about to run.
    #[cfg(debug_assertions)]
    pub(super) fn slot_labels(&self) -> &[Vec<&'static str>] {
        &self.slot_labels
    }

    // Rebuild every managed resource at a new extent. The caller has already
    // idled the device; reassigning drops the old heaps + placed resources
    // (COM release), so any feature descriptor that referenced them must be
    // rewritten by the caller afterward.
    pub(super) fn rebuild(
        &mut self,
        device: &ID3D12Device,
        queue: &ID3D12CommandQueue,
        slots: &[TransientSlot],
    ) -> RenderResult<()> {
        *self = Self::build(device, queue, slots)?;
        Ok(())
    }
}

// Resolve each placed label to its `resources` index and its cyclic slot
// predecessor: the member whose heap memory it reclaims. Cyclic because D3D12 is
// single-buffered, so the first member of a shared slot reclaims from the last
// across the frame boundary (the wrap), giving every shared member a
// predecessor; a slot with one member shares nothing and needs no aliasing
// barrier. `build` places every member of every slot in this order, so the
// running index is the resource's position.
fn index_labels(slots: &[TransientSlot]) -> HashMap<&'static str, LabelEntry> {
    let mut by_label = HashMap::new();
    let mut next = 0;
    for slot in slots {
        let n = slot.members.len();
        for (pos, m) in slot.members.iter().enumerate() {
            by_label.insert(
                m.label,
                LabelEntry {
                    resource: next,
                    alias_predecessor: (n > 1).then(|| slot.members[(pos + n - 1) % n].label),
                },
            );
            next += 1;
        }
    }
    by_label
}

// The optimized clear value a pooled target is created with, from the graph's
// desc. D3D12 matches this against the value a real `Clear*View` passes: a
// mismatch is a debug-layer warning and costs the fast-clear path, so it is the
// graph's business rather than a constant here. A depth target carries a depth
// value; every color target carries four floats.
fn clear_value(m: &TransientTexture) -> D3D12_CLEAR_VALUE {
    let format = dxgi_format(m.format);
    match m.clear {
        ClearValue::Color(color) => D3D12_CLEAR_VALUE {
            Format: format,
            Anonymous: D3D12_CLEAR_VALUE_0 { Color: color },
        },
        ClearValue::Depth(depth) => D3D12_CLEAR_VALUE {
            Format: format,
            Anonymous: D3D12_CLEAR_VALUE_0 {
                DepthStencil: D3D12_DEPTH_STENCIL_VALUE {
                    Depth: depth,
                    Stencil: 0,
                },
            },
        },
    }
}

// Translate one graph-declared transient into its D3D12 resource desc. This is
// the backend's whole share of describing a pooled resource: the extent,
// format, mip count and flags all come from the graph, so there is no second
// table here that could disagree with it. `Alignment` 0 lets the runtime pick
// the default (64 KiB) placement alignment.
fn rt_desc(m: &TransientTexture) -> D3D12_RESOURCE_DESC {
    let (dimension, depth_or_array) = if m.depth.max(1) > 1 {
        (D3D12_RESOURCE_DIMENSION_TEXTURE3D, m.depth.max(1))
    } else {
        (D3D12_RESOURCE_DIMENSION_TEXTURE2D, m.array_layers.max(1))
    };
    D3D12_RESOURCE_DESC {
        Dimension: dimension,
        Alignment: 0,
        Width: m.width.max(1) as u64,
        Height: m.height.max(1),
        DepthOrArraySize: depth_or_array as u16,
        MipLevels: m.mip_levels.max(1) as u16,
        Format: dxgi_format(m.format),
        SampleDesc: DXGI_SAMPLE_DESC {
            Count: m.sample_count.max(1),
            Quality: 0,
        },
        Layout: D3D12_TEXTURE_LAYOUT_UNKNOWN,
        Flags: resource_flags(m.usage),
    }
}

pub(in crate::directx) fn dxgi_format(format: PixelFormat) -> DXGI_FORMAT {
    match format {
        PixelFormat::Rgba16Float => DXGI_FORMAT_R16G16B16A16_FLOAT,
        PixelFormat::Rgba8Unorm => DXGI_FORMAT_R8G8B8A8_UNORM,
        PixelFormat::Rg16Float => DXGI_FORMAT_R16G16_FLOAT,
        PixelFormat::R8Unorm => DXGI_FORMAT_R8_UNORM,
        PixelFormat::R16Float => DXGI_FORMAT_R16_FLOAT,
        PixelFormat::R32Float => DXGI_FORMAT_R32_FLOAT,
        PixelFormat::Rg32Float => DXGI_FORMAT_R32G32_FLOAT,
        PixelFormat::Depth32Float => DXGI_FORMAT_D32_FLOAT,
        PixelFormat::BgraSwapchain => DXGI_FORMAT_B8G8R8A8_UNORM,
    }
}

fn resource_flags(usage: TextureUsage) -> D3D12_RESOURCE_FLAGS {
    let mut flags = D3D12_RESOURCE_FLAG_NONE;
    if usage.contains(TextureUsage::RENDER_TARGET) {
        flags |= D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET;
    }
    if usage.contains(TextureUsage::DEPTH_STENCIL) {
        flags |= D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL;
    }
    if usage.contains(TextureUsage::STORAGE) {
        flags |= D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS;
    }
    flags
}

// Where a pooled target sits between frames, which is also the state it is
// created in so its first derived transition names a state it is really in.
// Color targets rest sampled, matching the barrier registry; a depth target
// would rest as its attachment, which is why this follows the declared usage
// rather than being one constant.
fn resting_state(m: &TransientTexture) -> D3D12_RESOURCE_STATES {
    if m.usage.contains(TextureUsage::DEPTH_STENCIL) {
        D3D12_RESOURCE_STATE_DEPTH_WRITE
    } else {
        D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE
    }
}

#[cfg(test)]
mod tests {
    use super::super::post::gbuffer::GBUFFER_ROUGHNESS_CLEAR;
    use super::*;
    use concinnity_core::render::depth::DEPTH_CLEAR;
    use concinnity_core::render::post::device::PostExtent;
    use concinnity_core::render::post::{bloom, ssao};
    use concinnity_core::render::render_graph::{self, PoolGates, plan_pool_slots};

    fn planned(render: (u32, u32), output: (u32, u32)) -> Vec<TransientSlot> {
        let gates = PoolGates {
            ssao: true,
            gbuffer: true,
        };
        plan_pool_slots(gates, render, output).expect("plans")
    }

    #[test]
    fn the_label_index_resolves_placement_order_and_the_cyclic_predecessor() {
        // What `resource_for` / `alias_predecessor` answer with, without a
        // device: `build` places every member of every slot in this order, so a
        // label's index is its position in that walk. The predecessors wrap,
        // because a shared slot's first member reclaims the last member's memory
        // across the frame boundary.
        let slots = planned((1024, 768), (1024, 768));
        let index = index_labels(&slots);
        let mut placed = 0;
        for slot in &slots {
            for m in &slot.members {
                assert_eq!(index[m.label].resource, placed, "{}", m.label);
                placed += 1;
                let pred = index[m.label].alias_predecessor;
                if slot.members.len() > 1 {
                    assert!(
                        slot.labels().contains(&pred.expect("shared member")),
                        "{} reclaims from outside its slot",
                        m.label
                    );
                } else {
                    assert_eq!(pred, None, "{} sits alone", m.label);
                }
            }
        }
        assert_eq!(index.len(), placed, "a label was placed twice");
        // The saving this pool exists for, spelled out: `bloom_top` reclaims an
        // earlier member's region rather than owning one.
        assert!(index["bloom_top"].alias_predecessor.is_some());
    }

    #[test]
    fn the_roughness_clear_matches_the_feature_constant() {
        // The reason `TextureDesc` models a clear value at all. D3D12 bakes an
        // optimized clear into a placed resource, and roughness clears to 1.0
        // (fully rough) rather than 0: a mismatch here costs the fast-clear path
        // and, if the pool won, would make untouched pixels mirror-smooth.
        let slots = planned((1024, 768), (1024, 768));
        let roughness = slots
            .iter()
            .flat_map(|s| &s.members)
            .find(|m| m.label == "gbuffer_roughness")
            .expect("roughness pooled");
        assert_eq!(
            roughness.clear,
            render_graph::ClearValue::Color(GBUFFER_ROUGHNESS_CLEAR)
        );
    }

    #[test]
    fn translated_descs_match_the_feature_formats() {
        // The graph is the single source of the shape now, so what this pins is
        // the *translation*: a divergence from each feature's own constant
        // would silently mis-back the resource that feature binds.
        let slots = planned((1024, 768), (1920, 1080));
        let member = |label: &str| {
            slots
                .iter()
                .flat_map(|s| &s.members)
                .find(|m| m.label == label)
                .unwrap_or_else(|| panic!("{label} pooled"))
                .clone()
        };

        // `ao_output` follows the render extent; `bloom_top` is half the output
        // extent, the octave above the bloom chain's own.
        let ao = member("ao_output");
        let ao_desc = rt_desc(&ao);
        assert_eq!((ao_desc.Width, ao_desc.Height), (1024, 768));
        assert_eq!(ao_desc.Format, dxgi_format(ssao::OCCLUSION_FORMAT));
        assert_eq!(ao_desc.MipLevels, 1);
        assert_eq!(ao_desc.SampleDesc.Count, 1);
        assert_eq!(ao_desc.Dimension, D3D12_RESOURCE_DIMENSION_TEXTURE2D);
        assert_eq!(ao_desc.Flags, D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET);
        // Rests sampled, matching the barrier registry's resting state.
        assert_eq!(
            resting_state(&ao),
            D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE
        );

        let bloom = member("bloom_top");
        let bloom_desc = rt_desc(&bloom);
        let output = PostExtent {
            width: 1920,
            height: 1080,
        };
        let top = bloom::top_extent(output);
        assert_eq!(
            (bloom_desc.Width, bloom_desc.Height),
            (u64::from(top.width), top.height)
        );
        assert_eq!(
            bloom_desc.Format,
            dxgi_format(bloom::chain_desc(output).format)
        );

        // The G-buffer color targets. A format divergence here would silently
        // mis-back the resource the pre-pass MRT binds, and the render-target
        // flag is what makes it bindable at all.
        use super::super::post::gbuffer::{
            GBUFFER_NORMAL_DEPTH_FORMAT, GBUFFER_ROUGHNESS_FORMAT, GBUFFER_VELOCITY_FORMAT,
        };
        for (label, format) in [
            ("gbuffer_normal_depth", GBUFFER_NORMAL_DEPTH_FORMAT),
            ("gbuffer_roughness", GBUFFER_ROUGHNESS_FORMAT),
            ("gbuffer_velocity", GBUFFER_VELOCITY_FORMAT),
        ] {
            let desc = rt_desc(&member(label));
            assert_eq!(desc.Format, format, "{label}");
            // Render resolution, not the drawable: the pre-pass rasterizes at
            // the scene resolution, which differs under temporal upscaling.
            assert_eq!((desc.Width, desc.Height), (1024, 768), "{label}");
            assert_eq!(desc.SampleDesc.Count, 1, "{label} rasterizes once");
            assert_eq!(
                desc.Flags, D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET,
                "{label}"
            );
        }
    }

    #[test]
    fn depth_and_volume_shapes_translate() {
        // Nothing pooled needs these yet, but the descs the graph now carries
        // do, so the translator has to be right before they can be pooled. A
        // depth transient in particular rests in its attachment state, not
        // sampled, which is why resting is derived rather than constant.
        let depth = TransientTexture {
            label: "probe_depth",
            width: 8,
            height: 8,
            depth: 1,
            format: PixelFormat::Depth32Float,
            sample_count: 4,
            array_layers: 1,
            mip_levels: 1,
            usage: TextureUsage::DEPTH_STENCIL.union(TextureUsage::SHADER_READ),
            clear: ClearValue::DEPTH,
        };
        let desc = rt_desc(&depth);
        assert_eq!(desc.Format, DXGI_FORMAT_D32_FLOAT);
        assert_eq!(desc.SampleDesc.Count, 4);
        assert_eq!(desc.Flags, D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL);
        assert_eq!(resting_state(&depth), D3D12_RESOURCE_STATE_DEPTH_WRITE);
        // A depth target's optimized clear must be the depth arm: handing
        // D3D12 a color for a D32 resource is a creation failure.
        assert_eq!(
            // SAFETY: the union arm is the one `clear_value` just wrote for a
            // `ClearValue::Depth`, which the assertion above pins.
            unsafe { clear_value(&depth).Anonymous.DepthStencil.Depth },
            DEPTH_CLEAR
        );

        let volume = TransientTexture {
            label: "probe_volume",
            depth: 64,
            format: PixelFormat::Rgba16Float,
            sample_count: 1,
            usage: TextureUsage::STORAGE.union(TextureUsage::SHADER_READ),
            clear: ClearValue::Color([0.0; 4]),
            ..depth
        };
        let desc = rt_desc(&volume);
        assert_eq!(desc.Dimension, D3D12_RESOURCE_DIMENSION_TEXTURE3D);
        assert_eq!(desc.DepthOrArraySize, 64);
        assert_eq!(desc.Flags, D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS);
    }
}