fuel-dispatch 0.19.1

// SPDX-License-Identifier: MIT OR Apache-2.0
//! Pipelined compile + execute. Phase 7.5 B4.
//!
//! [`PipelinedExecutor::realize`] runs compilation and execution on
//! separate threads connected by a channel: a compiler thread walks
//! the graph in topological order and emits work items; the
//! executor (this thread) consumes them, allocates output Storage,
//! calls the kernel, and stores the result in an internal cache.
//! Both threads run concurrently so execution can begin while
//! compilation is still resolving later nodes.
//!
//! Today's "compile" step is a single binding-table lookup —
//! roughly nanoseconds per node. The threading delivers no
//! measurable speedup in this regime. The pipelining infrastructure
//! exists *now* — built on the [`compile_node`] / [`crate::compiled::execute_compiled`]
//! interface from B5 — so future work that grows the compile step
//! (residency-aware planning, transfer-cost minimization, kernel
//! auto-tuning) plugs in without revisiting call sites.
//!
//! ## Storage during the migration
//!
//! `fuel_graph::Graph::storage_map` uses the legacy
//! `fuel_backend_contract::Storage` (the `Box<dyn DynBackendStorage>`
//! newtype). The pipelined executor uses the new
//! `fuel_memory::Storage` (BackendStorage enum + dtype). During
//! the migration the two coexist — neither is converted on the fly.
//! The pipelined executor takes pre-realized inputs as a
//! `HashMap<NodeId, Arc<RwLock<fuel_memory::Storage>>>` rather
//! than reading from the graph's storage_map. Phase D unifies the
//! two paths once kernel migration completes.
//!
//! ## Op coverage
//!
//! B4 supports `Op::Const` (input-cache adoption — no kernel call)
//! and `Op::Add` on f32 (mapped to `OpKind::AddElementwise`).
//! Phase C adds the rest as more (op, dtype) bindings register.

use std::collections::{HashMap, HashSet};
use std::sync::mpsc::{Sender, channel};
use std::sync::{Arc, RwLock};
use std::thread;

use fuel_graph::opt::{execution_plan, insert_safety_copies};
use fuel_graph::registry::FusedOpParams;
use fuel_graph::{Graph, Node, NodeId, Op, PickedRoute};
use fuel_ir::dispatch::OpKind;
use fuel_ir::probe::BackendId;
use fuel_ir::{DType, DeviceLocation, Error, Layout, Result, SymEnv};

use crate::compiled::{
    CompiledNode, CompletionHandle, compile_node, execute_compiled_with_wait_hint,
};
use crate::dispatch::global_bindings;
use crate::kernel::{KernelBindingTable, KernelDTypes, MatmulM, OpParams};
use crate::optimize::OptimizedGraph;
use crate::ranker::{BackendRuntimeLookup, RuntimeSelector, resolve_branch};
// `pick_route` is called only by the streamed-vs-one-shot equivalence test
// below; the executor itself resolves branch-by-branch via `resolve_branch`.
#[cfg(test)]
use crate::ranker::pick_route;
use fuel_memory::Storage;

/// How the executor derives its dispatch order + topology-generation
/// stamp for the `TopologyChanged` chunk-boundary check.
///
/// PR-A3b-1 of the "plan IS the graph" rebuild: the realize bridge can
/// now drive the executor from `optimize_graph`'s in-place form via
/// [`OptimizedGraph::dispatch_order`] (`extract_runs`/`lower_run`)
/// instead of the legacy `ExecutionPlan`-supplied order. Both arms
/// compute the *same* `NodeId` sequence on a branchless graph (the
/// A3a equivalence gate proved `dispatch_order == compile_plan(...).order`);
/// the only difference is the source of truth.
///
/// - [`OrderSource::Default`] — recompute via `execution_plan` inside
///   the executor (the pre-A3b-1 behavior; the legacy `ExecutionPlan`
///   path and the plain `realize`/`realize_many` entries use this).
/// - [`OrderSource::Optimized`] — derive from the `OptimizedGraph`'s
///   run lowering, computed AFTER `insert_safety_copies` so the order
///   covers any safety-copy nodes the executor just inserted. Carries
///   the optimize-time `generation` so the chunk-boundary
///   `TopologyChanged` check still fires (the new path dispatches with
///   `plan: None`, so the generation must come from here instead).
///
///   PR-C1: it also carries an optional **route** ([`PickedRoute`]) — the
///   runtime route picker (Picker 2) chose one arm per `Op::Branch`. When
///   `Some`, the order is the route-aware lowering
///   ([`fuel_graph::lower_picked_route`]); when `None` (or empty) it is
///   the arm-0 lowering (`lower_runs_arm0`), byte-identical to Phase B.
enum OrderSource<'a> {
    Default,
    Optimized {
        optimized: &'a OptimizedGraph,
        /// The route the picker resolved (one arm per branch). `None` ⇒
        /// arm-0 everywhere (no picker / no pressure ⇒ Phase B behavior).
        route: Option<&'a PickedRoute>,
    },
    /// Step E Phase C, PR C1 — the **streaming run-walk**. Instead of a
    /// route resolved up front, the compiler thread resolves each branch
    /// **lazily, as the topo walk reaches its arm-entry**, via
    /// [`resolve_branch`] over the threaded [`StreamingPick`]. The order is
    /// never pre-flattened: the compiler thread interleaves arm resolution,
    /// node compilation, and dispatch (so C2 can later read live device
    /// load at the moment a branch resolves). Still VRAM-only selection in
    /// C1, so the streamed order equals the one-shot
    /// [`OrderSource::Optimized`]`{route: Some(pick_route(..))}` order
    /// byte-for-byte.
    Streaming {
        optimized: &'a OptimizedGraph,
        pick: &'a StreamingPick,
    },
    /// `order_for` fix track (`docs/design/incremental-consumer-index.md`
    /// §5): memoizes the `Optimized { route: None }` arm's dispatch order
    /// WHOLESALE in `order_cache`, computed once and reused by the caller
    /// across repeated calls over an unchanged graph/roots — the plan-once
    /// persistent-decode shape, where only leaf values are rebound between
    /// tokens, never the graph's structure. Safe because that recompute is
    /// a pure function of (graph structure, roots, generation) when none of
    /// those change — proven, not assumed, by
    /// `dispatch_order_is_stable_across_repeated_calls_with_no_mutation`
    /// (`fuel-dispatch/src/optimize.rs`). A cache miss falls back to the
    /// exact `Optimized { route: None }` computation and populates the
    /// cache for next time; `generation_for` still reports `optimized`'s
    /// generation, so the existing `TopologyChanged` chunk-boundary check
    /// fires exactly as it would for `Optimized` — if the topology changes
    /// underneath a cached session, realize fails loudly (the existing
    /// contract), it does not silently serve a stale order.
    Cached {
        optimized: &'a OptimizedGraph,
        order_cache: &'a std::sync::OnceLock<Vec<NodeId>>,
    },
}

/// Owned arm-resolution config threaded into the compiler thread for the
/// [`OrderSource::Streaming`] walk (Step E Phase C, PR C1). The selector +
/// lookup are owned `Arc`s (not borrows) so they can be `move`d into the
/// spawned compiler thread; both are `Send + Sync`. The per-arm candidate
/// bindings are read from [`global_bindings`] inside the thread (as the
/// eager `pick_route_for` does), so they are not carried here.
///
/// The streaming walk consults the SAME selector chain as the one-shot
/// picker (the production `ChainedSelector`). As of C2 that chain is
/// load-aware: its key is `(pressure_tier, load_tier, latency_ns,
/// original_index)`, and the `load_tier` leg reads B1's in-flight counter
/// through the selector's own `BackendRuntimeLookup` handle (the
/// `BackendStreams` seam). Under no load / no pressure every leg is the
/// no-signal default, so the lazy resolution yields the same arm as the
/// eager `pick_route` on every input — byte-identical to pre-C2.
#[derive(Clone)]
pub(crate) struct StreamingPick {
    /// The runtime selector the streaming walk consults at each branch —
    /// the production `ChainedSelector` (VRAM-pressure guard + load tier +
    /// Judge rank). It holds its own `BackendRuntimeLookup`, so it reads
    /// both VRAM fit and live device load off that lookup's handles.
    pub selector: Arc<dyn RuntimeSelector>,
    /// The live per-tier free-memory + load lookup the picker fingerprints
    /// telemetry through (consulted *through* the selector). `None` ⇒ no
    /// pressure/load signal ⇒ arm-0 (the no-telemetry contract). The
    /// streaming walk consults VRAM + load through the selector (the
    /// production `ChainedSelector` holds its own lookup), so this is
    /// carried for the `RouteCache`-fingerprint parity / future direct
    /// reads but not re-queried in the walk (mirrors `pick_route`'s
    /// `let _ = lookup`).
    #[allow(dead_code)]
    pub lookup: Option<BackendRuntimeLookup>,
}

/// Map from NodeId to a realized Storage Arc. Used both as the
/// input cache (passed in by the caller for pre-realized leaves)
/// and as the output cache (built up during execution).
pub type StorageCache = HashMap<NodeId, Arc<RwLock<Storage>>>;

/// Persistent per-node output buffers for CUDA-graph capture (the
/// "CapturedRun executor" build-out). Maps each compute node's [`NodeId`]
/// to a FIXED-ADDRESS output [`Storage`] Arc that survives across realizes,
/// so a captured graph's baked kernel-argument addresses stay valid on
/// replay.
///
/// Two modes, both driven by the executor's Kernel arm:
/// - **`Record`** (warm run): the executor allocates each kernel output as
///   usual AND records the Arc here. Produces the fixed buffers.
/// - **`Reuse`** (capture / replay run): the executor REUSES the recorded
///   Arc for each node — no allocation (mandatory inside a capture scope) —
///   and the kernel's write-into path writes into it in place.
///
/// `None` (the param default at every existing realize call site) preserves
/// byte-identical behavior: fresh alloc every realize, nothing recorded.
pub struct PersistentOutputs {
    map: HashMap<NodeId, Arc<RwLock<Storage>>>,
    mode: PersistentMode,
}

#[derive(Clone, Copy, PartialEq, Eq)]
enum PersistentMode {
    /// Warm run: allocate + record each kernel output.
    Record,
    /// Capture / replay run: reuse the recorded buffer (no alloc).
    Reuse,
}

impl PersistentOutputs {
    /// A fresh recording map for the warm run.
    pub fn recording() -> Self {
        Self {
            map: HashMap::new(),
            mode: PersistentMode::Record,
        }
    }

    /// A reuse map for the capture / replay run, seeded with the buffers
    /// a prior warm run recorded.
    pub fn reuse(map: HashMap<NodeId, Arc<RwLock<Storage>>>) -> Self {
        Self {
            map,
            mode: PersistentMode::Reuse,
        }
    }

    /// Consume, yielding the recorded NodeId→buffer map (to seed a
    /// [`reuse`](Self::reuse) map for the capture run).
    pub fn into_map(self) -> HashMap<NodeId, Arc<RwLock<Storage>>> {
        self.map
    }
}

/// Whether a kernel op's CUDA dispatch wrapper writes its result INTO the
/// executor-provided `outputs[0]` buffer (rather than allocating a fresh
/// output and replacing it). Only write-into ops are safe in the
/// executor's Reuse (capture) mode: a replace-the-buffer wrapper would
/// allocate inside the capture scope AND change the output's device
/// address, invalidating the captured graph. [`capture_decode`] rejects
/// any graph whose compute nodes fall outside this set — a surfaced
/// capability gap (telemetry-worthy), never a silent miscompile.
///
/// The set is the full decode op-set, all VERIFIED to replay bit-exact under
/// capture (each with a forced-freed-memory-reuse test — see the `diag_*` and
/// `capture_decode_*` GPU tests).
///
/// **History / the big correction (2026-07-10):** `MatMul` and `IndexSelect`
/// were *briefly* excluded on the belief they weren't capture-safe (matmul's
/// "cuBLAS internal state", index_select's "host arrays / driver anomaly").
/// That was ALL WRONG — it was a single Fuel-side bug: `capture_decode`
/// consumed `inputs` and retained only compute OUTPUTS, so any input buffer
/// (weights, embedding table) the caller didn't independently hold was FREED
/// after capture, and the captured graph replayed against freed/reused memory
/// → intermittent partial-wrong output (cuBLAS "split-K columns", index_select
/// "element 0"). Fixed by `CapturedDecode::retained_inputs`. cuBLAS gemm +
/// index_select then replay bit-exact (0/50 even with forced reuse), so the
/// baracuda GEMV + gather_rows are NOT needed for capture-safety.
///
/// **`Rope` (`OpKind::Rope`, corrected 2026-07-11)**: previously excluded on
/// the (now-disproven) assumption that decode's rope always goes through
/// `rope_with_tables` (decomposed to elementwise ops including `Concat`, see
/// below) rather than this fused single-input kernel. Tracing a real Llama
/// decode graph's capture_decode rejection (`capture_decode_matches_persistent`
/// GPU test, `fuel-core`) found `Op::Rope` nodes placed by the optimizer
/// (`rope_with_tables` is not always selected — shape-dependent). The
/// dispatch wrapper (`cuda_rope_baracuda_wrapper!` in
/// `baracuda_dispatch.rs`) already delegates to baracuda's write-into
/// `rope_*_into` FFI — that mechanism predates this predicate fix, it was
/// simply never added here. `rope.fkc.md`'s CUDA kernel is audited
/// (`docs/architecture/10-decisions-log.md`, 2026-07-11): grid-stride,
/// one thread per output element, reads only its own rotation-pair partner,
/// no atomics/shared-memory/cross-thread state — genuinely capture-safe.
///
/// **`Concat` (4b-γ) is capture-safe ONLY for N==2 inputs** — baracuda's
/// `concat2_into` write-into path covers exactly the decode graph's only
/// Concat (the rope rotate-half's 2-input `concat(-x2, x1)`, from
/// `rope_with_tables`'s CUDA decomposition). This predicate alone can't
/// express the N-arity condition (a pure `OpKind -> bool` fn has no access
/// to `OpParams`); `capture_decode`'s validation loop adds an explicit N>2
/// rejection alongside the `op_kind_is_capture_writeinto` check so N>2
/// Concat is refused at validation time, before the warm pass ever runs —
/// the write-into wrapper's N>2 fallback (which still allocates internally
/// for its N-2 intermediates) is therefore provably never invoked while
/// `persistent.mode == Reuse`. N-ary write-into concat is a real future gap
/// if a consumer needs it; not built here (no consumer today).
///
/// **`Op::Contiguize` (4b-γ)** — `WorkItemKind::ContiguizeOf`, not a
/// `Kernel` work item, so this predicate never even sees it — became
/// capture-safe via a separate write-into path (`contiguize_into`) wired
/// directly into the `ContiguizeOf` executor arm's `PersistentOutputs`
/// consultation, mirroring how `Kernel` nodes consult this predicate. No
/// change needed here or in `capture_decode`'s validation loop: a given
/// `ContiguizeOf` node's allocate-vs-zero-copy branch is a static
/// layout/stride property, identical across the warm and capture passes
/// (both replay the same compiled item list), so it's unconditionally safe
/// once wired to `PersistentOutputs`.
///
/// LESSON: a bug reproduced through the full harness but not in isolation is
/// the harness's fault, not the callee's — chase the differentiator (here,
/// input lifetime), don't conclude "kernel/library not capture-safe" from it.
///
/// [`capture_decode`]: PipelinedExecutor::capture_decode
/// GAP-217b: gated on `cuda` because its ONLY caller is
/// `capture_decode` (`#[cfg(feature = "cuda")]`, ~:1266). Without this it
/// reports as dead code on every default build — a false signal on a
/// predicate whose doc comment is a record of hard-won capture-safety
/// findings, i.e. exactly the kind of item someone eventually deletes for
/// being "unused". Gating it makes the reachability claim honest instead of
/// leaving a warning to argue the opposite.
#[cfg(feature = "cuda")]
fn op_kind_is_capture_writeinto(op: OpKind) -> bool {
    matches!(
        op,
        // Binary elementwise — residual + gating.
        OpKind::AddElementwise
            | OpKind::SubElementwise
            | OpKind::MulElementwise
            | OpKind::DivElementwise
            | OpKind::MaximumElementwise
            | OpKind::MinimumElementwise
            | OpKind::PowElementwise
            | OpKind::RemElementwise
            // Softmax.
            | OpKind::SoftmaxLastDim
            | OpKind::LogSoftmaxLastDim
            // Affine (mul_scalar/add_scalar — the 1/√d attention scale).
            | OpKind::Affine
            // Dense matmul (cuBLAS gemm_dense — capture-safe; the earlier
            // "not replay-stable" was the dangling-input bug, now fixed).
            | OpKind::MatMul
            // Embedding lookup (was the same dangling-input bug, not a kernel issue).
            | OpKind::IndexSelect
            // RMS-norm: capture-safe via the device's fixed workspace for its
            // per-row RMS scratch (warm sizes it; capture + replay reuse, no alloc).
            | OpKind::RmsNormLastDim
            // Unary activations (write-into + workspace-free).
            | OpKind::SiluElementwise
            | OpKind::GeluTanhElementwise
            | OpKind::GeluElementwise
            // Negation (write-into + workspace-free) — the rope rotate-half
            // decomposition's `neg_second` half (CapturedRun 4b-resume: rope
            // runs decomposed on CUDA since there's no rotate-half fused kernel).
            | OpKind::NegElementwise
            // N==2 concat only (rope rotate-half) — see the doc comment
            // above; N>2 is rejected explicitly in capture_decode's
            // validation loop, not by this predicate.
            | OpKind::Concat
            // Rotary position embedding (fused single-input kernel) — see
            // the doc comment above.
            | OpKind::Rope
    )
}

/// The product of [`PipelinedExecutor::capture_decode`]: a replayable
/// CUDA-graph capture of a single-device decode step plus the handle to
/// the fixed output buffer the captured launches write into.
///
/// Replay contract: update the fixed INPUT buffers (the Arcs passed into
/// `capture_decode`) in place via H2D, call [`run`](Self::run)`.replay()`,
/// then read [`output`](Self::output) — the captured launches recompute
/// into it at the same device addresses. `persistent` holds every compute
/// node's fixed buffer (the same addresses baked into the graph), exposed
/// so a caller can inspect / retain them across replays.
#[cfg(feature = "cuda")]
pub struct CapturedDecode {
    /// The instantiated, replayable capture.
    pub run: fuel_cuda_backend::CapturedRun,
    /// The realize target's fixed output buffer (written by replay).
    pub output: Arc<RwLock<Storage>>,
    /// The target's resolved output [`Layout`].
    pub output_layout: Layout,
    /// Every compute node's fixed output buffer (the capture-baked
    /// addresses). Retained so they outlive the [`CapturedRun`].
    pub persistent: HashMap<NodeId, Arc<RwLock<Storage>>>,
    /// EVERY input buffer's Arc (weights, KV, per-token inputs), retained so
    /// their device addresses — baked into the captured graph — stay valid
    /// for the life of the capture. Without this, an input buffer the CALLER
    /// doesn't independently hold is freed when `capture_decode` returns
    /// (it consumes `inputs`), and replay reads freed/reused memory. This is
    /// the fix for the "index_select / matmul mis-replay" — a dangling input
    /// pointer, NOT a kernel/cuBLAS/driver capture issue.
    pub retained_inputs: Vec<Arc<RwLock<Storage>>>,
}

/// Warm→capture→replay orchestration over [`capture_decode`] for a
/// persistent decode loop (the CapturedRun build-out, Increment 4a).
///
/// Owns the FIXED per-token INPUT buffers — the ones that change each token
/// (token-id, rope position, mask, KV write offset) — at stable device
/// addresses, plus the [`CapturedDecode`]. Each generated token is one
/// [`replay_token`](Self::replay_token): H2D-update the fixed input buffers
/// IN PLACE (same addresses the captured graph reads), replay the captured
/// graph (one `cuGraphLaunch`, no per-op launch overhead), then read the
/// output at its fixed address.
///
/// The mechanism is graph-agnostic: [`capture`](Self::capture) runs a
/// single-device CUDA decode graph through the persistent-output executor;
/// the model layer (fuel-core's `DecodeSession`) supplies the held graph +
/// the fixed buffers and drives the per-token updates. Every compute op must
/// be capture-safe (`op_kind_is_capture_writeinto`) — `capture` surfaces any
/// that are not (today: matmul, index_select — pending baracuda kernels).
///
/// [`capture_decode`]: PipelinedExecutor::capture_decode
#[cfg(feature = "cuda")]
pub struct CapturedDecodeSession {
    device: fuel_cuda_backend::CudaDevice,
    captured: CapturedDecode,
    /// Per-token input buffers, keyed by NodeId — fixed device addresses
    /// updated in place each token (the input addresses the captured graph
    /// reads). A subset of the `inputs` handed to [`capture`](Self::capture).
    per_token_inputs: HashMap<NodeId, Arc<RwLock<Storage>>>,
    /// EVERY input buffer's Arc (weights, KV, per-token), retained so their
    /// device addresses stay valid for the life of the capture. `capture_decode`
    /// consumes `inputs` and only `CapturedDecode` retains the compute-output
    /// buffers, so without this the WEIGHT input buffers would be freed and the
    /// captured graph would replay against freed memory.
    _retained_inputs: Vec<Arc<RwLock<Storage>>>,
}

#[cfg(feature = "cuda")]
impl CapturedDecodeSession {
    /// Capture the decode graph once against the given fixed input buffers.
    ///
    /// `inputs` are ALL the graph's leaves (weights, KV, per-token data) at
    /// fixed device addresses; `per_token_input_ids` names the subset that
    /// changes each token. The per-token Arcs are snapshotted (shared, not
    /// copied) so overwriting them in place later feeds the captured graph's
    /// fixed input reads. Returns a session ready for
    /// [`replay_token`](Self::replay_token). Errors (never panics) if the
    /// graph carries a capture-unsafe op or a per-token id is not an input.
    pub fn capture(
        graph: Arc<RwLock<Graph>>,
        target: NodeId,
        inputs: StorageCache,
        per_token_input_ids: &[NodeId],
        sym_env: SymEnv,
    ) -> Result<Self> {
        use fuel_memory::BackendStorage;
        // Snapshot the per-token input Arcs + retain EVERY input Arc BEFORE
        // `inputs` is consumed by capture_decode (so no input buffer's device
        // address is freed while the captured graph still bakes it).
        let retained_inputs: Vec<Arc<RwLock<Storage>>> = inputs.values().cloned().collect();
        let mut per_token_inputs = HashMap::with_capacity(per_token_input_ids.len());
        for &id in per_token_input_ids {
            let arc = inputs.get(&id).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "CapturedDecodeSession::capture: per-token input {:?} not in inputs",
                    id,
                ))
                .bt()
            })?;
            per_token_inputs.insert(id, arc);
        }
        // Derive the CUDA device from any input.
        let device = {
            let any = per_token_inputs
                .values()
                .next()
                .or_else(|| inputs.values().next())
                .ok_or_else(|| {
                    Error::Msg("CapturedDecodeSession::capture: no inputs".into()).bt()
                })?;
            let guard = any.read().map_err(|_| poisoned("input storage"))?;
            match &guard.inner {
                BackendStorage::Cuda(c) => c.device().clone(),
                other => {
                    return Err(Error::Msg(format!(
                        "CapturedDecodeSession::capture: input is BackendStorage::{:?}, not Cuda",
                        std::mem::discriminant(other),
                    ))
                    .bt());
                }
            }
        };
        let captured = PipelinedExecutor::capture_decode(graph, target, inputs, sym_env)?;
        Ok(Self {
            device,
            captured,
            per_token_inputs,
            _retained_inputs: retained_inputs,
        })
    }

    /// Replay one token. For each `(node_id, host_bytes)` in `updates`,
    /// overwrite that per-token input buffer IN PLACE (H2D, same device
    /// address), then replay the captured graph and return the output buffer
    /// (now holding this token's result). `host_bytes.len()` must equal the
    /// buffer's byte length.
    /// **Every compute node's output buffer, keyed by `NodeId`** — the
    /// capture's inspection surface.
    ///
    /// Capture is usually described as a launch-overhead optimization, and that
    /// is what Fuel currently exploits (measured 4.28x at k=1). But recording
    /// the graph has a second, unexploited consequence: **every intermediate
    /// gets a FIXED device address that survives across replays.** On the
    /// ordinary path intermediates are transient — allocated, consumed, freed —
    /// so there is nothing to look at after the fact.
    ///
    /// That makes a captured session a **whole-graph, per-node, addressable
    /// snapshot of what actually executed**, which is a different artifact from
    /// anything the planner records. `OptimizedGraph::placement_of` says what
    /// the optimizer DECIDED; this says what the device HELD. Those can
    /// disagree, and an independent second instrument is the only way to notice.
    ///
    /// Uses this enables, none of which need re-running the model:
    /// - read any intermediate after a token;
    /// - compare per-node against a CPU oracle — whole-graph differential,
    ///   where FKC verification today is per-KERNEL;
    /// - measure real per-op divergence for the tolerance/Judge work instead of
    ///   modelling it.
    ///
    /// Buffers are live device memory owned by the capture; they are overwritten
    /// by the next `replay_token`. Read them before replaying again.
    pub fn node_outputs(&self) -> &HashMap<NodeId, Arc<RwLock<Storage>>> {
        &self.captured.persistent
    }

    pub fn replay_token(&self, updates: &[(NodeId, &[u8])]) -> Result<Arc<RwLock<Storage>>> {
        use fuel_memory::BackendStorage;
        for (id, bytes) in updates {
            let arc = self.per_token_inputs.get(id).ok_or_else(|| {
                Error::Msg(format!(
                    "CapturedDecodeSession::replay_token: {:?} is not a per-token input",
                    id,
                ))
                .bt()
            })?;
            let guard = arc.read().map_err(|_| poisoned("per-token input"))?;
            match &guard.inner {
                BackendStorage::Cuda(c) => c.write_from_host(bytes)?,
                other => {
                    return Err(Error::Msg(format!(
                        "CapturedDecodeSession::replay_token: input {:?} is \
                         BackendStorage::{:?}, not Cuda",
                        id,
                        std::mem::discriminant(other),
                    ))
                    .bt());
                }
            }
        }
        self.captured.run.replay(&self.device)?;
        self.device.synchronize()?;
        Ok(Arc::clone(&self.captured.output))
    }

    /// The fixed output buffer, updated by each
    /// [`replay_token`](Self::replay_token). Holds the warm-run result until
    /// the first replay.
    pub fn output(&self) -> &Arc<RwLock<Storage>> {
        &self.captured.output
    }

    /// The output's resolved [`Layout`].
    pub fn output_layout(&self) -> &Layout {
        &self.captured.output_layout
    }
}

/// What flavor of work item the executor is processing.
/// Disambiguates the four cases:
enum WorkItemKind {
    /// `Op::Const` — its Storage Arc is already in the input cache.
    /// Executor verifies the entry exists and moves on.
    ConstAdopt,
    /// Metadata-only view op (`Op::Transpose`, `Op::Permute`,
    /// `Op::BroadcastTo`): the output's Storage Arc IS the
    /// input's Storage Arc (bytes shared); `output_layout`
    /// describes the strided view.
    ViewOf { input: NodeId },
    /// Reshape-style adoption: the output is contiguous in
    /// `output_layout.shape()`. If the input is already contiguous
    /// + zero offset, the output Arc is the input Arc (zero copy).
    ///
    /// Otherwise, the executor auto-contiguizes the input into a
    /// fresh Arc and uses that.
    ContiguizeOf { input: NodeId },
    /// Computational kernel: allocate output, run the compiled
    /// kernel, store the result. `compiled` is `Some(...)`.
    Kernel,
    /// `Op::Iota { len }` — 0-input F32 position generator
    /// `[0.0, 1.0, …, (len-1)]`. Direct executor dispatch (no kernel /
    /// binding-table lookup), like `WorkItemKind::Alloc`: a leaf has no
    /// `inputs[0]` to inherit a backend from, so it can't take the normal
    /// kernel path. The executor fills the sequence on CPU.
    Iota { len: usize },
    /// `Op::Release` — metadata-only "this storage is no longer
    /// needed" directive. Per `Op::Release`'s contract: the output
    /// is a zero-element marker (NodeId placeholder for graph
    /// bookkeeping; never read by any consumer). The executor emits
    /// a zero-byte CPU `Storage` at the node's slot so downstream
    /// cache lookups don't surface "missing slot" errors; the actual
    /// deallocation of `inputs[0]` is driven by `destructive_input`
    /// in the realize loop (Phase B), which drops the Arc held by
    /// the cache. The held Arc's Drop chain frees device memory
    /// (sync for CPU, stream-deferred for async backends).
    ReleaseMarker,
    /// `Op::WriteSlice` — in-place scatter write. The output adopts
    /// the destination input's Storage Arc (zero-copy alias); the
    /// kernel mutates that Arc's bytes via a write lock. The source
    /// input's slab is copied into the destination's rectangular
    /// region defined by `OpParams::WriteSlice`'s `ranges`.
    ///
    /// Carries `dest`/`source` NodeIds so the executor can:
    /// 1. Adopt the dest's Arc as the kernel's output slot.
    /// 2. Pass only the source as a kernel input (the dest is not
    ///    a read-input to the kernel; it's the destination buffer
    ///    being mutated in place).
    ///
    /// The realize loop's `destructive_input` cleanup evicts the
    /// `dest` NodeId from the cache after this op runs (per
    /// `Op::WriteSlice`'s destructive_input == Some(0) contract).
    /// Downstream consumers read post-write bytes via this op's
    /// own NodeId, not the dest's.
    WriteSlice { dest: NodeId, source: NodeId },
    /// `Op::WriteSliceRotating` — like `WriteSlice` but the rotating
    /// axis wraps modulo `modulus`. Carries `dest`/`source`/`position`
    /// NodeIds; the kernel reads the U32 position from `position`'s
    /// storage and splits the write across the ring boundary if
    /// `position % modulus + slab_len > modulus`.
    WriteSliceRotating {
        dest: NodeId,
        source: NodeId,
        position: NodeId,
    },
    /// `Op::WriteSliceDoff` — like `WriteSlice` but the start on `axis`
    /// is read from a device-resident rank-0 `I64` scalar (`offset`)
    /// at kernel launch, not host-baked. Carries `dest`/`source`/
    /// `offset` NodeIds; the CUDA wrapper threads `offset`'s device
    /// pointer straight to the kernel (no D2H) so a captured graph
    /// replays at the host-updated position. No ring wrap.
    WriteSliceDoff {
        dest: NodeId,
        source: NodeId,
        offset: NodeId,
    },
    /// `Op::Copy { target }` — produce a fresh Storage on
    /// `target_location`, copying bytes from `inputs[0]`'s residency.
    ///
    /// The kernel lookup goes through the standard binding-table path
    /// at `(OpKind::Copy, [dt, dt], source_backend)`, so D2H is a peer
    /// of every other op (architecture identity check #1). The
    /// dedicated WorkItemKind exists because Op::Copy is the one op
    /// whose output's device differs from `target_backend`:
    /// `target_backend` is the **source** backend (where the kernel
    /// runs — it owns the download path), while `target_location`
    /// drives output allocation. WorkItemKind::Kernel always allocates
    /// output on `target_backend`; Copy needs the override.
    ///
    /// Phase 2 of the bridge-retirement trajectory. Replaces the
    /// per-variant `match self` in `BackendStorage::read_to_cpu_bytes`
    /// (deleted alongside this commit) with a graph-level node the
    /// optimizer can see, cost, and eventually fuse.
    Copy { target_location: DeviceLocation },
    /// `Op::Move { target }` — `Op::Copy`'s destructive sibling:
    /// produce a fresh Storage on `target_location` via the same
    /// data-movement kernel (binding-table lookup at
    /// `(OpKind::Copy, [dt, dt], source_backend)`; output allocation
    /// driven by `target_location`), then release the source. The
    /// release half is NOT handled in the executor's arm — it rides
    /// the realize loop's `destructive_input` cleanup
    /// (`Op::Move::destructive_input() == Some(0)`), exactly like
    /// `Op::Release`: the cache's Arc to the source drops after the
    /// move runs and the Drop chain frees device memory (sync for
    /// CPU, stream-deferred for async backends).
    ///
    /// Ordering safety is the graph's job, not this arm's:
    /// `execution_plan` integrates `derive_ordering`, which pins the
    /// Move AFTER every non-destructive reader of the source's alias
    /// set, and `insert_safety_copies` snapshots the source for any
    /// reader that data-flow-depends on the Move's output. A Move
    /// therefore never strands a sibling consumer of its source.
    ///
    /// Same-device moves are legal and match the legacy
    /// `GraphExecutor` contract: a plain copy producing fresh
    /// storage on the same device, with the source still evicted
    /// afterward.
    Move { target_location: DeviceLocation },
    /// `Op::Alloc { target }` — produce a freshly-allocated, zero-
    /// initialized Storage on `target_location` with the node's shape
    /// + dtype. Zero inputs.
    ///
    /// Doesn't dispatch through the binding table — the executor's
    /// arm calls each backend's native allocator directly. For
    /// non-CPU targets it derives the device handle by searching the
    /// input cache for any storage on `target_location`'s backend
    /// (callers seed this via
    /// `fuel-core::pipelined_bridge::device_seed_storage`).
    ///
    /// Phase 3a of the bridge-retirement trajectory (post-9c).
    /// Replaces `fuel-core::inference_context::alloc_zeroed_on`'s
    /// per-`DeviceLocation` match with a graph node the optimizer
    /// can see; the per-backend match moves from fuel-core's bridge
    /// layer into fuel-storage's executor (the architectural dispatch
    /// layer).
    Alloc { target_location: DeviceLocation },
    /// `Op::ZeroFill` — fill the input's storage bytes with zero,
    /// in place. Adopts the input's Storage Arc as the output (same
    /// Storage; bytes mutated). Destructive on `inputs[0]`.
    ///
    /// Direct executor dispatch (no binding-table lookup) per the
    /// same rationale as `WorkItemKind::Alloc`: structural op,
    /// per-backend dispatch. Per-backend behaviour:
    /// - CPU: `bytes_mut().fill(0)` via the CoW path.
    /// - CUDA: `CudaStorageBytes::zero_async` via baracuda alpha.30's
    ///   `DeviceBuffer::zero_async` (cuMemsetD8Async, in-place).
    /// - Vulkan: `VulkanBackend::fill_bytes_zero` via
    ///   `vkCmdFillBuffer` — device-side, ~2× the bandwidth of the
    ///   old host-staged zeros path that `alloc_zeroed_on` used.
    ///
    /// Phase 3a follow-up of bridge-retirement (post-9c). Pairs with
    /// the uninit-alloc `WorkItemKind::Alloc` to give the
    /// architecturally clean "Op::Alloc (uninit) → Op::ZeroFill
    /// (explicit fill)" pipeline.
    ZeroFill,
    /// In-place kernel — the output adopts the input at index
    /// `target_idx`'s Storage Arc and the kernel mutates that Arc's
    /// bytes through a single write lock. Phase 3 of the in-place ops
    /// infrastructure
    /// (`docs/session-prompts/in-place-ops-infrastructure.md`).
    ///
    /// Kernel lookup goes through the standard binding table at
    /// `(op_to_op_kind, [target_dtype], target_backend)`. The wrapper's
    /// `inputs` slice is empty; `outputs[0]` is the target Arc. The
    /// wrapper acquires `outputs[0]`'s write lock and calls the
    /// underlying single-pointer kernel (e.g. baracuda's
    /// `affine_inplace_*` on CUDA, or the chassis with `src == dst`
    /// on CPU).
    ///
    /// Output Arc adoption: identical to `WorkItemKind::WriteSlice`
    /// (the output's slot = the target's Arc; the realize loop's
    /// `destructive_input` cleanup evicts the target's NodeId from the
    /// cache afterward). Downstream consumers read post-mutation bytes
    /// through this op's NodeId, not the target's.
    InplaceKernel {
        /// Index into `inputs` whose Storage Arc the output adopts.
        /// Always equal to the source `Op::destructive_input().unwrap()`;
        /// stored explicitly so the executor doesn't re-derive it.
        target_idx: usize,
    },
    /// `Op::View { slot }` — multi-output projection (Option C,
    /// Session 4). The output's Storage Arc IS the producer's Arc;
    /// the WorkItem's `output_layout` was prepared by
    /// [`fuel_graph::NodeHandle::view`] with the slot's `byte_offset`
    /// baked into `Layout::start_offset` so downstream kernels reading
    /// the producer's bytes as `slot_dtype` elements land on the
    /// slot's first byte. Structurally identical to [`Self::ViewOf`]
    /// at execute time — kept distinct so error messages and
    /// telemetry have a clean dispatch point.
    SlotView { producer: NodeId },
    /// `Op::ViewOwned { slot }` — multi-output projection with an
    /// independent destination buffer. At realize time, allocate a
    /// fresh contiguous Storage of the slot's `(shape, dtype)` on
    /// the producer's device, then memcpy
    /// `producer.bytes[slot.byte_offset .. slot.byte_offset + slot.len_bytes]`
    /// into the new storage. Session 4 ships the CPU path; non-CPU
    /// backends return a typed error until their copy-with-offset
    /// hooks land in the followup session.
    SlotOwn { producer: NodeId, slot: u32 },
}

/// One unit of work emitted by the compiler thread to the executor
/// thread.
struct WorkItem {
    node_id: NodeId,
    inputs: Vec<NodeId>,
    /// Number of elements in the output (for output Storage
    /// allocation; multiplied by dtype size at allocation time).
    elem_count: usize,
    dtype: DType,
    target_backend: BackendId,
    /// What kind of work this represents (kernel vs adopt vs view).
    kind: WorkItemKind,
    /// `Some` for [`WorkItemKind::Kernel`]; `None` for the other
    /// two. Carries the resolved kernel ref + op_params.
    compiled: Option<CompiledNode>,
    /// The output's [`Layout`]. For kernels: always
    /// `Layout::contiguous(node.shape)`. For metadata-only view
    /// ops: a strided/broadcast Layout pointing at the input's
    /// Storage. Carried so the executor can publish the right
    /// Layout into its layout cache and ultimately return it from
    /// [`PipelinedExecutor::realize`].
    output_layout: Layout,
    /// Index into `inputs` whose storage gets destroyed by this op
    /// (`Op::Release` / `Op::Move` → `Some(0)`; non-destructive ops
    /// → `None`). The executor evicts the destroyed input from the
    /// cache after the op runs, unless it's also in the realize
    /// target set. Snapshot of `node.op.destructive_input()` at
    /// compile time.
    destructive_input: Option<usize>,
    /// Multi-output bundle metadata (Option C, item 3). `Some(_)`
    /// when the node was declared multi-output via
    /// `Graph::set_output_views`; the Kernel arm allocates one
    /// contiguous output Storage sized to fit every slot's bytes,
    /// then attaches the bundle via `Storage::with_bundle` so
    /// downstream `Op::View`/`Op::ViewOwned` nodes resolve
    /// correctly.
    output_bundle: Option<Arc<[fuel_ir::storage::OutputView]>>,
}

/// Merge the graph's side-effect roots into the caller's requested
/// roots. Preserves caller order (user roots come first), dedupes,
/// and appends side-effect-bearing nodes (`Op::Print`, `Op::Save`,
/// etc.) so their effects fire even when not reachable from the
/// caller's roots. Mirrors `fuel-graph-executor::extend_with_side_effect_roots`.
///
/// Short-circuits to a borrow-equivalent `Vec<NodeId>` when the graph
/// has no side-effect roots — the common case during inference.
fn extend_with_side_effect_roots(graph: &Graph, user_roots: &[NodeId]) -> Vec<NodeId> {
    let side = graph.side_effect_roots();
    if side.is_empty() {
        return user_roots.to_vec();
    }
    let mut out = Vec::with_capacity(user_roots.len() + side.len());
    out.extend_from_slice(user_roots);
    for &s in side {
        if !out.contains(&s) {
            out.push(s);
        }
    }
    out
}

/// Pipelined executor: walks a graph, compiles each node in a
/// dedicated thread, executes the compiled stream on the calling
/// thread.
///
/// # Architectural commitments (Phase 7.6 step 9c)
///
/// ## Fail-fast dispatch — no runtime CPU fallback
///
/// The binding-table lookup is fail-fast. If
/// `global_bindings().lookup(op_kind, dtypes, backend)` returns
/// `None`, `realize` / `realize_many` surface a typed `Error::Msg`
/// at compile time. There is **no runtime fallback** to a CPU
/// sibling when a registered kernel returns `Err`; we commit to the
/// backend contract that registered kernels must succeed for shapes
/// matching their declared coverage.
///
/// Rationale: per-decision-point alternatives (architecture v1.0
/// §04) already let multiple backends register at the same key. If
/// a backend's coverage is partial, the right answer is to register
/// only the shapes it actually handles — not to silently fall back
/// at runtime, which masks bugs and slows the hot path. The
/// long-term home for "this op on this device" is graph-level
/// dispatch insertion (Op::Copy edges to a backend that does
/// support the shape), which makes the routing decision visible in
/// the IR rather than hidden in the executor. Picker-arc step 4b
/// landed exactly that: `compile_plan`'s off-device fallback
/// (`PlanOptions::fallback_placements_for`) admits a candidate on
/// another device when the pinned device lacks the implementation,
/// and the bridge's cross-device-copy pass stitches residency
/// around the off-device winner. There is exactly ONE fallback
/// owner — the plan-time picker; this executor still dispatches
/// fail-fast, and ops with no implementation anywhere surface the
/// plan-time `NoBackendForOp` error.
///
/// This is a deliberate departure from the legacy executor's
/// `cpu_fallback(op, inputs, shape, dtype, cache)` semantics.
///
/// ## Optimizer separation — callers compose
///
/// The pipelined executor does **not** run `fuel_graph::opt` or any
/// `RuleRegistry` pass itself. Callers that want graph-level
/// optimization compose it at the call site:
///
/// ```ignore
/// let optimized_targets = registry.optimize_to_fixpoint(&graph, &targets);
/// PipelinedExecutor::realize_many(graph, &optimized_targets, inputs)?;
/// ```
///
/// Rationale: the legacy `GraphExecutor<B>::with_optimization(bool)`
/// coupling reflected its per-typed-entry API surface
/// (`realize_f32`, etc.) where the convenience of a single
/// optimize+realize call was load-bearing. The pipelined path has
/// a uniform `realize_many` that composes trivially — no need to
/// re-couple them at the executor layer. This keeps the executor
/// focused on execution and lets the optimizer evolve
/// independently (different rule registries per call site, mid-
/// pipeline opt+realize+opt+realize patterns, etc.).
pub struct PipelinedExecutor;

impl PipelinedExecutor {
    /// Realize `target` and every transitive dependency. Compilation
    /// runs in a worker thread; execution runs on the calling
    /// thread.
    ///
    /// Pre-conditions:
    ///
    /// - Every reachable `Op::Const` node must have a corresponding
    ///   entry in `inputs` (pre-realized Storage Arc).
    /// - Every reachable non-`Const` node must have its
    ///   `target_backend` set in the graph
    ///   (`Graph::set_target_backend`).
    /// - The op + dtype must be registered in `global_bindings()`.
    ///
    /// Returns the realized `Storage` Arc for `target` plus its
    /// resolved [`Layout`]. The Layout is contiguous for kernel
    /// outputs; for graphs whose target is a metadata-only view
    /// op (`Op::Transpose`, `Op::Permute`, `Op::BroadcastTo`),
    /// the returned Storage shares its bytes with an upstream
    /// node and the Layout encodes the view's strides + offset.
    ///
    /// Production-correct: errors on any unmet precondition rather
    /// than panicking.
    pub fn realize(
        graph: Arc<RwLock<Graph>>,
        target: NodeId,
        inputs: StorageCache,
    ) -> Result<(Arc<RwLock<Storage>>, Layout)> {
        Self::realize_inner(
            graph,
            target,
            inputs,
            OrderSource::Default,
            SymEnv::default(),
        )
        .map(|(s, l, _produced)| (s, l))
    }

    /// Env-carrying sibling of [`Self::realize`]: realize `target` with a
    /// per-pass [`SymEnv`] supplying the runtime bindings for any
    /// `DynScalar` op params (today: `Op::WriteSlice`'s dynamic start
    /// offset — Phase D symbolic extents). An **empty** env is
    /// byte-identical to [`Self::realize`]; the env is consulted only by ops
    /// that carry a `DynScalar`, so a graph with none ignores it. Uses
    /// the default (`execution_plan`) dispatch order.
    ///
    /// This is the input-determined path for persistent decode (the
    /// per-token `cached_len` write offset). The `OptimizedGraph`/route
    /// entry points keep an empty env until the realize bridge threads
    /// the session env through them (Phase D step 1, fuel-core).
    pub fn realize_with_env(
        graph: Arc<RwLock<Graph>>,
        target: NodeId,
        inputs: StorageCache,
        sym_env: SymEnv,
    ) -> Result<(Arc<RwLock<Storage>>, Layout)> {
        Self::realize_inner(graph, target, inputs, OrderSource::Default, sym_env)
            .map(|(s, l, _produced)| (s, l))
    }

    /// Producing sibling of [`Self::realize_with_env`]: realize `target` and
    /// ALSO return the [`SymEnv`] of symbols BOUND BY PRODUCERS during the
    /// pass — data-determined dynamic shapes (`Op::NonZeroIndices`'s
    /// runtime count, etc.). The returned env is disjoint from the input
    /// `sym_env` (which is input-determined). Empty when the graph has no
    /// data-determined producer. This is the observation point for the
    /// increment-2a bind seam; a downstream pass consumes these bindings
    /// as dynamic extents.
    pub fn realize_with_env_producing(
        graph: Arc<RwLock<Graph>>,
        target: NodeId,
        inputs: StorageCache,
        sym_env: SymEnv,
    ) -> Result<(Arc<RwLock<Storage>>, Layout, SymEnv)> {
        Self::realize_inner(graph, target, inputs, OrderSource::Default, sym_env)
    }

    /// PR-A3b-1 entry: realize `target` driven from `optimized`'s
    /// in-place "plan IS the graph" form. The dispatch order is the
    /// `OptimizedGraph`'s `extract_runs`/`lower_run` lowering (computed
    /// after safety-copy insertion); per-node kernel resolution uses
    /// the binding-table-lookup path (no `ExecutionPlan`, so no
    /// route-picking — A3b-1 is branchless). The optimize-time
    /// `generation` drives the `TopologyChanged` chunk-boundary check.
    ///
    /// Pre-conditions match [`Self::realize`]: every reachable kernel-bearing
    /// node must have `target_backend` set (the bridge's
    /// `stamp_plan_backends` does this) and a registered binding.
    pub fn realize_with_optimized(
        graph: Arc<RwLock<Graph>>,
        target: NodeId,
        inputs: StorageCache,
        optimized: &OptimizedGraph,
    ) -> Result<(Arc<RwLock<Storage>>, Layout)> {
        Self::realize_inner(
            graph,
            target,
            inputs,
            OrderSource::Optimized {
                optimized,
                route: None,
            },
            SymEnv::default(),
        )
        .map(|(s, l, _produced)| (s, l))
    }

    /// Env-carrying sibling of [`Self::realize_with_optimized`]: same
    /// optimized-graph dispatch, but with a per-pass [`SymEnv`] supplying
    /// the runtime bindings for `DynScalar` op params (Phase D symbolic
    /// extents). An empty env is byte-identical to
    /// [`Self::realize_with_optimized`]. The realize bridge threads its session
    /// env through here for persistent decode.
    pub fn realize_with_optimized_env(
        graph: Arc<RwLock<Graph>>,
        target: NodeId,
        inputs: StorageCache,
        optimized: &OptimizedGraph,
        sym_env: SymEnv,
    ) -> Result<(Arc<RwLock<Storage>>, Layout)> {
        Self::realize_inner(
            graph,
            target,
            inputs,
            OrderSource::Optimized {
                optimized,
                route: None,
            },
            sym_env,
        )
        .map(|(s, l, _produced)| (s, l))
    }

    /// PR-C1 entry: realize `target` driven from `optimized`'s in-place
    /// form **following the runtime route picker's chosen arms**. The
    /// dispatch order is the route-aware lowering
    /// ([`fuel_graph::lower_picked_route`]) over `route` — the per-branch
    /// arm the picker (Picker 2) selected by live telemetry. A branch
    /// absent from `route` defaults to arm 0, so an **empty** route is
    /// byte-identical to [`Self::realize_with_optimized`] (the no-pressure /
    /// no-telemetry contract). A branchless graph has no branches ⇒ the
    /// route is empty ⇒ this is exactly the arm-0 path.
    pub fn realize_with_optimized_route(
        graph: Arc<RwLock<Graph>>,
        target: NodeId,
        inputs: StorageCache,
        optimized: &OptimizedGraph,
        route: &PickedRoute,
    ) -> Result<(Arc<RwLock<Storage>>, Layout)> {
        Self::realize_inner(
            graph,
            target,
            inputs,
            OrderSource::Optimized {
                optimized,
                route: Some(route),
            },
            SymEnv::default(),
        )
        .map(|(s, l, _produced)| (s, l))
    }

    /// Env-carrying sibling of [`Self::realize_with_optimized_route`] — the
    /// route-aware lowering with a per-pass [`SymEnv`]. An empty env is
    /// byte-identical to [`Self::realize_with_optimized_route`].
    pub fn realize_with_optimized_route_env(
        graph: Arc<RwLock<Graph>>,
        target: NodeId,
        inputs: StorageCache,
        optimized: &OptimizedGraph,
        route: &PickedRoute,
        sym_env: SymEnv,
    ) -> Result<(Arc<RwLock<Storage>>, Layout)> {
        Self::realize_inner(
            graph,
            target,
            inputs,
            OrderSource::Optimized {
                optimized,
                route: Some(route),
            },
            sym_env,
        )
        .map(|(s, l, _produced)| (s, l))
    }

    /// Cleanup Step C/D entry: the executor OWNS `Op::Branch` arm-selection.
    /// Given the runtime `selector` + the live device `lookup` (built
    /// bridge-side from the realize `Device` + Judge oracle and passed in,
    /// exactly like the residency `input_residency` provider), pick one arm per
    /// branch HERE — at dispatch — then lower the chosen route. The per-arm
    /// candidates are re-enumerated from the runtime binding registry + the
    /// graph (Step D: no threaded `ExecutionPlan`).
    ///
    /// No `selector` (the `runtime_selector_disabled()` opt-out) or a
    /// branchless graph yields the arm-0 lowering, byte-identical to Phase B.
    ///
    /// **Step E Phase C, PR C1 — STREAMING.** A branched graph WITH a
    /// selector no longer resolves the whole route up front: it routes to the
    /// `OrderSource::Streaming` walk, where the compiler thread resolves
    /// each branch lazily as the frontier reaches it (via
    /// [`resolve_branch`]). C1 keeps the SAME VRAM-only selector chain — so
    /// the streamed route equals the one-shot `pick_route` route byte-for-byte
    /// — and only changes *when* a branch resolves; C2 makes that moment read
    /// live device load.
    pub fn realize_with_optimized_picking_env(
        graph: Arc<RwLock<Graph>>,
        target: NodeId,
        inputs: StorageCache,
        optimized: &OptimizedGraph,
        selector: Option<Arc<dyn RuntimeSelector>>,
        lookup: Option<BackendRuntimeLookup>,
        sym_env: SymEnv,
    ) -> Result<(Arc<RwLock<Storage>>, Layout)> {
        match Self::streaming_pick_for(&graph, selector, lookup)? {
            Some(pick) => Self::realize_inner(
                graph,
                target,
                inputs,
                OrderSource::Streaming {
                    optimized,
                    pick: &pick,
                },
                sym_env,
            )
            .map(|(s, l, _produced)| (s, l)),
            None => Self::realize_with_optimized_env(graph, target, inputs, optimized, sym_env),
        }
    }

    /// `order_for` fix track (`docs/design/incremental-consumer-index.md`
    /// §5) — cached sibling of [`Self::realize_with_optimized_picking_env`]
    /// for the plan-once persistent-decode caller, which holds `order_cache`
    /// across repeated calls over the SAME graph/roots/`optimized` (one per
    /// `DecodeSession`, initially empty). On a branchless graph with no
    /// selector pressure (the decode shape this is for), dispatches via
    /// [`OrderSource::Cached`] — computed once, reused thereafter, see that
    /// variant's doc for the safety argument. A branched graph WITH a
    /// resolved selector pick falls back to the uncached
    /// [`OrderSource::Streaming`] path unchanged (caching doesn't apply
    /// there; this entry point exists for the decode case, not to cover
    /// every caller).
    pub fn realize_with_optimized_picking_cached_env(
        graph: Arc<RwLock<Graph>>,
        target: NodeId,
        inputs: StorageCache,
        optimized: &OptimizedGraph,
        order_cache: &std::sync::OnceLock<Vec<NodeId>>,
        selector: Option<Arc<dyn RuntimeSelector>>,
        lookup: Option<BackendRuntimeLookup>,
        sym_env: SymEnv,
    ) -> Result<(Arc<RwLock<Storage>>, Layout)> {
        match Self::streaming_pick_for(&graph, selector, lookup)? {
            Some(pick) => Self::realize_inner(
                graph,
                target,
                inputs,
                OrderSource::Streaming {
                    optimized,
                    pick: &pick,
                },
                sym_env,
            )
            .map(|(s, l, _produced)| (s, l)),
            None => Self::realize_inner(
                graph,
                target,
                inputs,
                OrderSource::Cached {
                    optimized,
                    order_cache,
                },
                sym_env,
            )
            .map(|(s, l, _produced)| (s, l)),
        }
    }

    /// Build the [`StreamingPick`] for the executor's picking entry points —
    /// the streaming successor of the one-shot `pick_route_for` (which was the
    /// relocated body of the bridge's old `resolve_runtime_route`). Returns
    /// `None` (⇒ arm-0 lowering, the untouched `route: None` /
    /// `dispatch_order` path) when there is no `selector` (the disabled
    /// opt-out) or the graph has no `Op::Branch` (the branchless fast-path:
    /// skip the streaming machinery entirely so the common case is unchanged
    /// and byte-identical). Otherwise it carries the selector + lookup into
    /// the [`OrderSource::Streaming`] walk, which resolves each branch at the
    /// frontier (PR C1). The per-arm candidates are re-enumerated from the
    /// runtime binding registry inside the compiler thread (Step D — was the
    /// threaded `ExecutionPlan`).
    fn streaming_pick_for(
        graph: &Arc<RwLock<Graph>>,
        selector: Option<Arc<dyn RuntimeSelector>>,
        lookup: Option<BackendRuntimeLookup>,
    ) -> Result<Option<StreamingPick>> {
        let Some(selector) = selector else {
            return Ok(None);
        };
        let g = graph
            .read()
            .map_err(|_| Error::Msg("graph lock poisoned".into()).bt())?;
        let has_branch = (0..g.len()).any(|i| matches!(g.node(NodeId(i)).op, Op::Branch { .. }));
        if !has_branch {
            return Ok(None);
        }
        Ok(Some(StreamingPick { selector, lookup }))
    }

    /// Capture a single-device CUDA decode step's kernel launches into a
    /// replayable [`CapturedRun`] (the CapturedRun executor build-out).
    ///
    /// Runs the graph TWICE against fixed-address buffers:
    ///   1. **Warm (record)**: realize normally into a fresh cache cloned
    ///      from `inputs`, recording every compute node's output Arc as its
    ///      fixed capture buffer, and synchronizing so the result is stable.
    ///   2. **Capture (reuse)**: re-issue the same launch sequence — each
    ///      node writing INTO its recorded buffer via the write-into kernel
    ///      path (no allocation) — inside [`CudaDevice::capture_run`],
    ///      producing an instantiated [`CapturedRun`].
    ///
    /// Both passes share the SAME input Arcs (cloned map, shared buffers)
    /// and the SAME recorded output buffers, so every device address the
    /// captured graph bakes in is stable for replay. To replay a later
    /// token: overwrite the fixed input buffers in place (H2D), call
    /// `run.replay(dev)`, then read `output`.
    ///
    /// Contract (surfaced as `Err`, never a panic or a mid-capture alloc):
    ///   - CUDA-only, single device: every compute node targets `Cuda`;
    ///     no cross-device `Op::Copy`/`Move` in the graph.
    ///   - Every compute node's op must have a write-into CUDA wrapper
    ///     (`op_kind_is_capture_writeinto`); an unsupported op surfaces a
    ///     clear capability gap rather than silently breaking the capture.
    ///   - Inputs must be contiguous (no auto-Contiguize alloc in capture).
    #[cfg(feature = "cuda")]
    pub fn capture_decode(
        graph: Arc<RwLock<Graph>>,
        target: NodeId,
        inputs: StorageCache,
        sym_env: SymEnv,
    ) -> Result<CapturedDecode> {
        use fuel_memory::BackendStorage;

        // Front matter mirrors realize_inner: safety copies + frozen order.
        {
            let mut g = graph.write().map_err(|_| poisoned("graph lock"))?;
            let effective_roots = extend_with_side_effect_roots(&g, &[target]);
            insert_safety_copies(&mut g, &effective_roots);
        }
        let (compiler_work, _wait_set, seed_layouts) = {
            let g = graph.read().map_err(|_| poisoned("graph lock"))?;
            let effective_roots = extend_with_side_effect_roots(&g, &[target]);
            let (work, wait_set) =
                compiler_work_and_wait_set_for(&g, &effective_roots, &OrderSource::Default);
            let mut layouts: HashMap<NodeId, Layout> = HashMap::with_capacity(inputs.len());
            for &id in inputs.keys() {
                layouts.insert(id, g.layout(id));
            }
            (work, wait_set, layouts)
        };

        // Compile every WorkItem once (one compiler thread) and collect them
        // so both passes replay the identical sequence.
        let (tx, rx) = channel::<Result<WorkItem>>();
        let graph_for_compiler = Arc::clone(&graph);
        let compiler = thread::spawn(move || {
            compiler_thread_body(graph_for_compiler, compiler_work, sym_env, tx);
        });
        let mut items: Vec<WorkItem> = Vec::new();
        for item in rx {
            items.push(item?);
        }
        compiler
            .join()
            .map_err(|_| Error::Msg("compiler thread panicked".to_string()).bt())?;

        // CapturedRun 4b Step-4 diagnostic: name every kernel placed OFF Cuda
        // (an unseeded / audited-loser op that forces a cross-device copy and
        // blocks capture), so verification-ledger seeding can target it
        // precisely. Fires only when a capture is attempted AND something is
        // mis-placed — otherwise silent.
        {
            let off: Vec<String> = items
                .iter()
                .filter_map(|it| match &it.kind {
                    WorkItemKind::Kernel if it.target_backend != BackendId::Cuda => {
                        let desc = it
                            .compiled
                            .as_ref()
                            .map(|c| format!("{:?} dtypes={:?}", c.op, c.dtypes))
                            .unwrap_or_else(|| "?".to_string());
                        Some(format!(
                            "node {:?}: {desc} on {:?}",
                            it.node_id, it.target_backend
                        ))
                    }
                    _ => None,
                })
                .collect();
            if !off.is_empty() {
                eprintln!(
                    "[capture-diag] {} kernel(s) placed OFF Cuda (these block capture): {}",
                    off.len(),
                    off.join("; "),
                );
            }
        }

        // Capture-safety validation — surface every gap up front, before any
        // launch, so a capture scope is never entered on an unsupported graph.
        for item in &items {
            match &item.kind {
                // Op::Move: the destructive half (releasing the source) is
                // driven by the realize loop's `destructive_input` cleanup,
                // which runs once per REALIZE call — not once per `replay()`.
                // A captured graph's `replay()` re-executes the same baked
                // launches repeatedly without re-running that surrounding
                // cleanup between replays, so whether a repeat-replayed Move
                // is even meaningful is unclear; out of scope here (4b-ζ) —
                // keep rejecting every Move unconditionally.
                WorkItemKind::Move { .. } => {
                    return Err(Error::Msg(format!(
                        "capture_decode: graph has an Op::Move at node {:?}; \
                         Move's destructive source-release half is driven by \
                         per-realize cleanup, not per-replay, so capture is \
                         not supported for Move (out of scope — see the \
                         `capture_decode` validation loop's Move arm).",
                        item.node_id,
                    ))
                    .bt());
                }
                // Op::Copy: a same-device CUDA→CUDA copy (e.g. an
                // `insert_safety_copies` snapshot) has a write-into wrapper
                // (`copy_from_cuda_wrapper`'s `BackendStorage::Cuda` branch,
                // via `CudaStorageBytes::copy_from_device`) and is
                // capture-safe. A copy whose `target_location` is `Cpu` or
                // `Vulkan` is a genuine host round-trip / cross-backend
                // transfer and can never be part of a CUDA graph capture —
                // keep rejecting those. (`gpu_id` is not cross-checked
                // against the capture device, matching the `Kernel` arm's
                // `target_backend` check just above: today's decode graphs
                // are single-GPU only.)
                WorkItemKind::Copy { target_location } => {
                    if !matches!(target_location, DeviceLocation::Cuda { .. }) {
                        return Err(Error::Msg(format!(
                            "capture_decode: graph has a cross-device Op::Copy at \
                             node {:?} (target {:?}); capture requires a \
                             single-device CUDA decode step — only same-device \
                             CUDA→CUDA copies are capture-safe.",
                            item.node_id, target_location,
                        ))
                        .bt());
                    }
                }
                WorkItemKind::Kernel => {
                    if item.target_backend != BackendId::Cuda {
                        return Err(Error::Msg(format!(
                            "capture_decode: kernel node {:?} targets {:?}, not Cuda",
                            item.node_id, item.target_backend,
                        ))
                        .bt());
                    }
                    let compiled = item.compiled.as_ref().ok_or_else(|| {
                        Error::Msg(format!(
                            "capture_decode: kernel node {:?} has no compiled op",
                            item.node_id,
                        ))
                        .bt()
                    })?;
                    let op = compiled.op;
                    if !op_kind_is_capture_writeinto(op) {
                        return Err(Error::Msg(format!(
                            "capture_decode: op {:?} (node {:?}) has no write-into CUDA \
                             wrapper yet — capture unsupported for this op. Extend \
                             op_kind_is_capture_writeinto once its write-into variant lands.",
                            op, item.node_id,
                        ))
                        .bt());
                    }
                    // Concat's capture-safety is conditional on arity — N==2
                    // has a write-into path (concat2_into), N>2 does not (its
                    // wrapper falls back to an allocating N-1 chain). This
                    // can't be expressed by op_kind_is_capture_writeinto (a
                    // pure OpKind -> bool predicate with no OpParams access),
                    // so reject N>2 explicitly here, BEFORE the warm pass —
                    // see op_kind_is_capture_writeinto's doc comment.
                    if op == OpKind::Concat {
                        if let OpParams::Concat {
                            input_dim_sizes, ..
                        } = &compiled.op_params
                        {
                            if input_dim_sizes.len() != 2 {
                                return Err(Error::Msg(format!(
                                    "capture_decode: Concat node {:?} has {} inputs; only \
                                     2-input concat (the rope rotate-half shape) has a \
                                     write-into CUDA path today — N-ary (N>2) concat would \
                                     allocate mid-capture.",
                                    item.node_id,
                                    input_dim_sizes.len(),
                                ))
                                .bt());
                            }
                        }
                    }
                }
                // View / adopt / slot kinds: no alloc, no launch — capture-safe.
                _ => {}
            }
        }

        // ---- Pass 1: warm (record) — fixed buffers + a synchronized result --
        let mut persistent = PersistentOutputs::recording();
        let mut warm_cache: StorageCache = inputs.clone();
        let mut warm_layouts = seed_layouts.clone();
        let mut warm_syms = SymEnv::new();
        for item in &items {
            let _handle = execute_work_item(
                item,
                &mut warm_cache,
                &mut warm_layouts,
                &mut warm_syms,
                false,
                Some(&mut persistent),
            )
            .map_err(|e| with_node_location(&graph, item.node_id, e))?;
        }
        // Flush all warm launches before capture (device idle; result stable).
        sync_active_cuda_devices(&warm_cache)?;

        let output = warm_cache.get(&target).cloned().ok_or_else(|| {
            Error::Msg(format!(
                "capture_decode: target {:?} not realized in warm run",
                target,
            ))
            .bt()
        })?;
        let output_layout = warm_layouts.get(&target).cloned().ok_or_else(|| {
            Error::Msg(format!("capture_decode: target {:?} has no layout", target)).bt()
        })?;

        // Derive the CUDA device from the target's storage for capture_run.
        let device = {
            let guard = output.read().map_err(|_| poisoned("output storage"))?;
            match &guard.inner {
                BackendStorage::Cuda(c) => c.device().clone(),
                other => {
                    return Err(Error::Msg(format!(
                        "capture_decode: target {:?} is BackendStorage::{:?}, not Cuda",
                        target,
                        std::mem::discriminant(other),
                    ))
                    .bt());
                }
            }
        };

        // ---- Pass 2: capture (reuse) — record the launch sequence ----------
        let mut reuse = PersistentOutputs::reuse(persistent.into_map());
        // Retain EVERY input buffer before `inputs` is consumed: the captured
        // graph bakes their device addresses, so they must outlive it even if
        // the caller doesn't independently hold them.
        let retained_inputs: Vec<Arc<RwLock<Storage>>> = inputs.values().cloned().collect();
        let mut cap_cache: StorageCache = inputs;
        let mut cap_layouts = seed_layouts;
        let mut cap_syms = SymEnv::new();
        let mut inner_err: Option<Error> = None;
        let items_ref = &items;
        let graph_ref = &graph;
        let run = device.capture_run(|_stream| {
            // Inside the capture scope: NO device alloc, NO host sync. Each
            // kernel launches on device.stream() (now capturing) and writes
            // into its reused fixed buffer.
            for item in items_ref {
                match execute_work_item(
                    item,
                    &mut cap_cache,
                    &mut cap_layouts,
                    &mut cap_syms,
                    false,
                    Some(&mut reuse),
                ) {
                    Ok(_handle) => {}
                    Err(e) => {
                        inner_err = Some(with_node_location(graph_ref, item.node_id, e));
                        break;
                    }
                }
            }
            Ok(())
        })?;
        if let Some(e) = inner_err {
            return Err(e);
        }

        Ok(CapturedDecode {
            run,
            output,
            output_layout,
            retained_inputs,
            persistent: reuse.into_map(),
        })
    }

    fn realize_inner(
        graph: Arc<RwLock<Graph>>,
        target: NodeId,
        inputs: StorageCache,
        order_source: OrderSource<'_>,
        sym_env: SymEnv,
    ) -> Result<(Arc<RwLock<Storage>>, Layout, SymEnv)> {
        // Auto-insert safety copies for in-place ops whose target
        // has additional readers in this realize set (residual-
        // connection cycle break). No-op when no destructive ops
        // are present.
        {
            let mut g = graph.write().map_err(|_| poisoned("graph lock"))?;
            let effective_roots = extend_with_side_effect_roots(&g, &[target]);
            insert_safety_copies(&mut g, &effective_roots);
        }

        // Execution plan + initial layouts for the input cache
        // entries, computed on the calling thread to keep the
        // compiler thread free of graph-locking responsibilities.
        // Side-effect roots (Op::Print, Op::Save, etc.) are merged
        // into the walk so their effects fire even when not
        // reachable from the user's `target`. `execution_plan`
        // integrates `derive_ordering`'s view-aware pinning so
        // destructive ops run AFTER non-destructive readers of
        // their targets.
        let (compiler_work, wait_set, mut layout_cache): (
            CompilerWork,
            WaitSet,
            HashMap<NodeId, Layout>,
        ) = {
            let g = graph.read().map_err(|_| poisoned("graph lock"))?;
            let effective_roots = extend_with_side_effect_roots(&g, &[target]);
            // PR-A3b-1: the OptimizedGraph path lowers via
            // `extract_runs`/`lower_run` (computed here, AFTER
            // `insert_safety_copies`, so the order covers freshly
            // inserted safety-copy nodes). The default path keeps the
            // pre-A3b-1 `execution_plan` walk. On a branchless graph the
            // two sequences are identical (A3a equivalence gate).
            //
            // PR-C1: a `Streaming` source is NOT flattened here — it is
            // handed to the compiler thread as a `CompilerWork::Streaming`
            // spec so the thread resolves each branch lazily at the
            // frontier (see `compiler_thread_body`).
            //
            // Step E A4b open question #4: the SAME pass also derives the
            // `WaitSet` from the materialized order ("event only where
            // waited" — see `compiler_work_and_wait_set_for`).
            let (work, wait_set) =
                compiler_work_and_wait_set_for(&g, &effective_roots, &order_source);
            let mut layouts = HashMap::with_capacity(inputs.len());
            for &id in inputs.keys() {
                layouts.insert(id, g.layout(id));
            }
            (work, wait_set, layouts)
        };

        let (tx, rx) = channel::<Result<WorkItem>>();
        let graph_for_compiler = Arc::clone(&graph);

        // Compiler thread: read graph nodes, resolve kernels,
        // push WorkItems. On error, push the error and bail. PR-C1: for a
        // `Streaming` work spec the thread also resolves each branch's arm
        // at the frontier (the SAME VRAM-only selector the one-shot picker
        // uses) before compiling that arm's runs.
        let compiler = thread::spawn(move || {
            compiler_thread_body(graph_for_compiler, compiler_work, sym_env, tx);
        });

        // Executor on this thread: consume WorkItems, gather
        // inputs from the cache, allocate outputs, call kernels,
        // populate the cache. After each destructive op
        // (Op::Release / Op::Move) evict the destroyed input from
        // the cache, unless it's the realize target.
        //
        // Phase 4.3: at each dispatch-chunk boundary (target_backend
        // change), check the live SystemTopology generation against
        // the plan's stamped generation. Mismatch surfaces
        // `Error::TopologyChanged`; the realize layer
        // (`pipelined_bridge`) catches it, rebuilds the plan against
        // the fresh topology, and retries.
        let plan_generation: Option<u64> = generation_for(&order_source);
        let mut current_chunk_backend: Option<BackendId> = None;
        let mut cache: StorageCache = inputs;
        // Data-dependent dynamic-shapes: symbols BOUND BY PRODUCERS during
        // this pass (e.g. `Op::NonZeroIndices`'s runtime count). Distinct
        // from the compile-time `sym_env` (input-determined, moved into the
        // compiler thread): these become known only as ops execute. Returned
        // to the caller so a downstream pass can consume them.
        let mut produced_syms = SymEnv::new();
        // Step E A4b-1: per-node async-completion handles. Each producing node's
        // `execute_compiled` returns a `CompletionHandle` (CUDA → a recorded
        // `Event`; CPU/Vulkan → `Ready`) instead of being `wait`ed inline. We
        // store them here, wait the SOURCE producer's handle before a
        // cross-device copy (A4b-3, below), and drain whatever remains at
        // realize-end.
        let mut handles: HashMap<NodeId, CompletionHandle> = HashMap::new();
        // Step E A4b-4: eagerly-submitted-but-not-yet-waited Vulkan batches. The
        // OVERLAP enabler — when we LEAVE a Vulkan chunk (or are about to block on
        // a CUDA copy) we `submit_pending` the open Vulkan batch so the iGPU runs
        // it while the executor records/dispatches the next (CUDA) chunk, instead
        // of letting it sit merely-recorded until realize-end (the A4b-2 behavior,
        // which never overlapped). Each submitted batch is tracked here and waited
        // at the in-flight-lifetime guard (before a Vulkan read / Vulkan-referenced
        // eviction / realize-end). `multi_backend` gates ALL eager submits so a
        // single-device realize never touches this path (byte-identical to A4b-2).
        let mut inflight_vulkan: Vec<InflightVulkan> = Vec::new();
        // Step E A4b-4: set true the first time the dispatch order transitions
        // between two DIFFERENT non-CPU... actually any two distinct backends — a
        // genuine backend switch. On a pure-CUDA or pure-Vulkan graph there is no
        // switch, so this stays false and the eager-submit / in-flight machinery is
        // unreachable (single-device byte-identical + throughput-neutral, §5).
        let mut multi_backend = false;
        for item in rx {
            let item = item?;
            // Chunk-boundary hook (target_backend change). Existing duty: the
            // `TopologyChanged` generation check. Step E A4b-4 adds two duties:
            // (1) detect a genuine backend switch → arm `multi_backend`; (2) when
            // LEAVING a Vulkan chunk on a multi-backend graph, eager-submit the
            // open Vulkan batch so the iGPU starts it concurrently with the chunk
            // we are about to dispatch (the overlap win).
            if current_chunk_backend != Some(item.target_backend) {
                if let Some(plan_gen) = plan_generation {
                    let live = crate::dispatch::topology_generation();
                    if live != plan_gen {
                        return Err(Error::TopologyChanged {
                            plan_generation: plan_gen,
                            current_generation: live,
                        }
                        .bt());
                    }
                }
                let leaving = current_chunk_backend;
                if let Some(prev) = leaving
                    && prev != item.target_backend
                {
                    // A real backend switch occurred — this realize is
                    // genuinely multi-backend.
                    multi_backend = true;
                    // Leaving a Vulkan chunk → submit it NOW so it runs on the
                    // iGPU while we record/dispatch the next chunk. Whole-chunk
                    // granularity (the open batch IS the just-finished Vulkan
                    // chunk with all its intra-CB barriers) — UAF/race-safe per
                    // `eager_submit_all_vulkan`'s contract.
                    if prev == BackendId::Vulkan {
                        eager_submit_all_vulkan(&cache, &mut inflight_vulkan)?;
                    }
                }
                current_chunk_backend = Some(item.target_backend);
            }
            // Step E A4b-4 (Invariant E — same-device cross-chunk RAW safety):
            // before RECORDING a new Vulkan op while in-flight Vulkan batches
            // exist, wait them. Vulkan only guarantees batches BEGIN in submission
            // order (they may overlap / complete out of order), so a later Vulkan
            // op that reads a buffer an in-flight batch wrote would race without
            // this. The in-flight batch was submitted at the prior chunk boundary
            // and an intervening (CUDA) chunk has since run, so this fence is
            // typically already signalled — the iGPU work overlapped, and we only
            // re-sync as Vulkan work resumes. No-op when single-device
            // (`multi_backend` false) or no batches are in flight.
            if multi_backend
                && item.target_backend == BackendId::Vulkan
                && !inflight_vulkan.is_empty()
            {
                drain_inflight_vulkan(&mut inflight_vulkan)?;
            }
            // Step E A4b-3 + A4b-4: the cross-device residency boundary.
            if matches!(
                item.kind,
                WorkItemKind::Copy { .. } | WorkItemKind::Move { .. }
            ) {
                // A4b-4: before we (possibly) block the host on the CUDA producer,
                // eager-submit any open Vulkan chunk so the iGPU runs it WHILE we
                // wait on CUDA (the §5.1 ordering — submit Vulkan *before* draining
                // CUDA). Cheap + safe; gated to multi-backend (no-op single-device).
                if multi_backend {
                    eager_submit_all_vulkan(&cache, &mut inflight_vulkan)?;
                }
                if let Some(&producer) = item.inputs.first() {
                    // A4b-4: a VULKAN-source D2H must read COMPLETED data, and the
                    // Vulkan download path only flushes the OPEN batch (not our
                    // in-flight ones) — so wait the in-flight Vulkan batches here.
                    // For a CUDA/CPU source we do NOT wait Vulkan (the copy reads no
                    // Vulkan buffer); waiting it would needlessly serialize the
                    // independent Vulkan sub-DAG and kill the overlap. The Vulkan
                    // work that was eager-submitted at the chunk boundary keeps
                    // running concurrently until ITS result is the one being copied.
                    if multi_backend && copy_source_is_vulkan(&cache, producer)? {
                        drain_inflight_vulkan(&mut inflight_vulkan)?;
                    }
                    // Step E A4b-3: FINER CUDA source-drain — wait ONLY the source
                    // producer's recorded event (not the whole source device), so
                    // the OTHER sub-DAG's independent in-flight CUDA work keeps
                    // running. No-op for a Vulkan/CPU producer (no CUDA handle).
                    wait_producer_handle(&mut handles, producer)?;
                }
            }
            let handle = execute_work_item(
                &item,
                &mut cache,
                &mut layout_cache,
                &mut produced_syms,
                will_be_waited(&wait_set, item.node_id),
                // No persistent-output mode on the standard realize paths —
                // fresh alloc per node, byte-identical to pre-CapturedRun.
                None,
            )
            .map_err(|e| with_node_location(&graph, item.node_id, e))?;
            store_handle(&mut handles, item.node_id, handle);
            if let Some(d_idx) = item.destructive_input
                && let Some(&destroyed) = item.inputs.get(d_idx)
                && destroyed != target
            {
                // Step E A2.1 (in-flight-batch DATA-BUFFER lifetime — the
                // UAF guard, now DEFERRED-DELETION instead of a drain). A
                // `SubmittedBatch` owns its CB/descriptors/transients but
                // NOT the DATA buffers it reads (those live here in the
                // cache); recycling a data buffer an in-flight/open CB still
                // reads is a use-after-free. PRE-A2.1 we host-blocked here —
                // `drain_inflight_vulkan` (wait every in-flight batch) +
                // `force_flush` (submit+wait the open batch) — which stalled
                // the iGPU work we were overlapping with CUDA at EVERY such
                // eviction. A2.1 instead RETAINS the evicted buffer until the
                // reader fences signal: it eager-submits the open batch (no
                // wait → it becomes a fenced in-flight batch) and moves a
                // refcount clone of the buffer into every in-flight batch, so
                // the buffer frees POST-fence on each batch's `Drop`
                // (follow-on #2) with NO host block. Conservative over all
                // batches (no cheap buffer→batch map — the same reason the old
                // drain waited all of them). CUDA eviction is stream-ordered-
                // safe via A3. Single-device (multi_backend == false) keeps the
                // pre-A2.1 blocking force_flush of the OPEN batch (the else
                // branch) — byte-identical + UAF-safe there; the deferred path is
                // unsafe single-device (no eager-submit/in-flight tracking, and the
                // cross-batch RAW guard at :832 doesn't run — see
                // force_flush_vulkan's doc).
                if multi_backend {
                    defer_evicted_vulkan_buffer(&cache, destroyed, &mut inflight_vulkan)?;
                } else if let Some(d_arc) = cache.get(&destroyed) {
                    force_flush_vulkan(d_arc)?;
                }
                cache.remove(&destroyed);
                layout_cache.remove(&destroyed);
                // Drop the evicted node's CUDA handle (if still present)
                // so the realize-end drain's empty-map assert stays
                // meaningful. The free itself is stream-ordered-safe (A3).
                handles.remove(&destroyed);
            }
        }

        compiler
            .join()
            .map_err(|_| Error::Msg("compiler thread panicked".to_string()).bt())?;

        // Step E A4b-1: drain every outstanding async handle before the result is
        // read / the cache drops (freeing intermediates). For CUDA this waits the
        // recorded events (one stream/device ⇒ waiting the latest drains all prior
        // stream work too; see `drain_handles`).
        drain_handles(&mut handles)?;
        // Step E A4b open question #4 — "event only where waited": most CUDA
        // nodes in this realize never got an Event (elided per the `WaitSet`),
        // so `drain_handles` above only retired the (typically few) tracked
        // handles. This blanket per-device `cuStreamSynchronize` flushes the
        // elided work too — safe + sufficient because of single-stream-per-
        // device ordering (see `sync_active_cuda_devices`'s doc).
        sync_active_cuda_devices(&cache)?;
        // Step E A4b-4: wait every eagerly-submitted (in-flight) Vulkan batch
        // before the cache drops, freeing the data buffers their CBs read. Empty
        // on a single-device realize (nothing was eager-submitted).
        drain_inflight_vulkan(&mut inflight_vulkan)?;
        // Step E A4b-2: drain all deferred Vulkan work before reading/returning
        // the result (and before the cache drops, freeing buffers). The A2
        // submit+wait is now SPLIT — `drain_vulkan_pending` submits the open
        // batch then waits it via a `VulkanCompletion` handle. Byte-identical to
        // A2 for pure-Vulkan (one submission at realize-end, just split).
        drain_vulkan_pending(&cache)?;
        debug_assert!(
            inflight_vulkan.is_empty(),
            "A4b-4: in-flight Vulkan batch list must be empty after realize-end drain",
        );

        let storage = cache.remove(&target).ok_or_else(|| {
            Error::Msg(format!(
                "PipelinedExecutor::realize: target slot {:?} not populated after execution",
                target
            ))
            .bt()
        })?;
        let layout = layout_cache.remove(&target).ok_or_else(|| {
            Error::Msg(format!(
                "PipelinedExecutor::realize: target layout {:?} not populated after execution",
                target
            ))
            .bt()
        })?;
        Ok((storage, layout, produced_syms))
    }

    /// Realize multiple targets in one walk. Each target's transitive
    /// dependency set is collapsed into a single execution plan
    /// (via [`execution_plan`]); shared subgraphs are evaluated
    /// once. Returns a `Vec<(Storage, Layout)>` whose order matches
    /// `targets`.
    ///
    /// This is Phase A of Phase 7.6 step 9c — feature-parity with the
    /// legacy `GraphExecutor::realize_many_f32`. Multi-session
    /// migration target ([memory: project_phase_7_6_step_9c_parity_audit.md]).
    ///
    /// Pre-conditions: every reachable `Op::Const` must be in
    /// `inputs`; every reachable non-`Const` must have its
    /// `target_backend` set; the op + dtype must be registered in
    /// `global_bindings()`. Same as single-target [`Self::realize`].
    pub fn realize_many(
        graph: Arc<RwLock<Graph>>,
        targets: &[NodeId],
        inputs: StorageCache,
    ) -> Result<Vec<(Arc<RwLock<Storage>>, Layout)>> {
        Self::realize_many_inner(
            graph,
            targets,
            inputs,
            OrderSource::Default,
            SymEnv::default(),
        )
    }

    /// Multi-target PR-A3b-1 entry — the `realize_many` sibling of
    /// [`Self::realize_with_optimized`]. Drives the executor from the
    /// `OptimizedGraph`'s run lowering via the binding-table-lookup
    /// path, with the optimize-time `generation` keying the
    /// `TopologyChanged` chunk-boundary check.
    pub fn realize_many_with_optimized(
        graph: Arc<RwLock<Graph>>,
        targets: &[NodeId],
        inputs: StorageCache,
        optimized: &OptimizedGraph,
    ) -> Result<Vec<(Arc<RwLock<Storage>>, Layout)>> {
        Self::realize_many_inner(
            graph,
            targets,
            inputs,
            OrderSource::Optimized {
                optimized,
                route: None,
            },
            SymEnv::default(),
        )
    }

    /// Env-carrying sibling of [`Self::realize_many_with_optimized`] (Phase D
    /// symbolic extents). An empty env is byte-identical to it.
    pub fn realize_many_with_optimized_env(
        graph: Arc<RwLock<Graph>>,
        targets: &[NodeId],
        inputs: StorageCache,
        optimized: &OptimizedGraph,
        sym_env: SymEnv,
    ) -> Result<Vec<(Arc<RwLock<Storage>>, Layout)>> {
        Self::realize_many_inner(
            graph,
            targets,
            inputs,
            OrderSource::Optimized {
                optimized,
                route: None,
            },
            sym_env,
        )
    }

    /// Multi-target PR-C1 entry — the `realize_many` sibling of
    /// [`Self::realize_with_optimized_route`]. Lowers each target's runs
    /// following the picker's chosen arms via
    /// [`fuel_graph::lower_picked_route`].
    pub fn realize_many_with_optimized_route(
        graph: Arc<RwLock<Graph>>,
        targets: &[NodeId],
        inputs: StorageCache,
        optimized: &OptimizedGraph,
        route: &PickedRoute,
    ) -> Result<Vec<(Arc<RwLock<Storage>>, Layout)>> {
        Self::realize_many_inner(
            graph,
            targets,
            inputs,
            OrderSource::Optimized {
                optimized,
                route: Some(route),
            },
            SymEnv::default(),
        )
    }

    /// Env-carrying sibling of [`Self::realize_many_with_optimized_route`]
    /// (Phase D symbolic extents). An empty env is byte-identical to it.
    pub fn realize_many_with_optimized_route_env(
        graph: Arc<RwLock<Graph>>,
        targets: &[NodeId],
        inputs: StorageCache,
        optimized: &OptimizedGraph,
        route: &PickedRoute,
        sym_env: SymEnv,
    ) -> Result<Vec<(Arc<RwLock<Storage>>, Layout)>> {
        Self::realize_many_inner(
            graph,
            targets,
            inputs,
            OrderSource::Optimized {
                optimized,
                route: Some(route),
            },
            sym_env,
        )
    }

    /// Multi-target sibling of [`Self::realize_with_optimized_picking_env`] —
    /// the executor resolves one arm per `Op::Branch` (cleanup Step C/D) over
    /// the effective targets. PR C1: a branched graph routes to the
    /// `OrderSource::Streaming` walk (lazy per-branch resolution at the
    /// frontier); branchless / no-selector falls to the untouched arm-0 path.
    pub fn realize_many_with_optimized_picking_env(
        graph: Arc<RwLock<Graph>>,
        targets: &[NodeId],
        inputs: StorageCache,
        optimized: &OptimizedGraph,
        selector: Option<Arc<dyn RuntimeSelector>>,
        lookup: Option<BackendRuntimeLookup>,
        sym_env: SymEnv,
    ) -> Result<Vec<(Arc<RwLock<Storage>>, Layout)>> {
        match Self::streaming_pick_for(&graph, selector, lookup)? {
            Some(pick) => Self::realize_many_inner(
                graph,
                targets,
                inputs,
                OrderSource::Streaming {
                    optimized,
                    pick: &pick,
                },
                sym_env,
            ),
            None => {
                Self::realize_many_with_optimized_env(graph, targets, inputs, optimized, sym_env)
            }
        }
    }

    fn realize_many_inner(
        graph: Arc<RwLock<Graph>>,
        targets: &[NodeId],
        inputs: StorageCache,
        order_source: OrderSource<'_>,
        sym_env: SymEnv,
    ) -> Result<Vec<(Arc<RwLock<Storage>>, Layout)>> {
        if targets.is_empty() {
            return Ok(Vec::new());
        }

        // Auto-insert safety copies for in-place ops whose target
        // has additional readers in this realize set (residual-
        // connection cycle break). No-op when no destructive ops
        // are present.
        {
            let mut g = graph.write().map_err(|_| poisoned("graph lock"))?;
            let effective_roots = extend_with_side_effect_roots(&g, targets);
            insert_safety_copies(&mut g, &effective_roots);
        }

        // Execution plan covering every target's dependency set +
        // initial layouts for the input cache entries. Side-effect
        // roots merge in so their effects fire even when not
        // reachable from any target. `execution_plan` integrates
        // `derive_ordering`'s view-aware pinning so destructive ops
        // run AFTER non-destructive readers of their targets.
        let (compiler_work, wait_set, mut layout_cache): (
            CompilerWork,
            WaitSet,
            HashMap<NodeId, Layout>,
        ) = {
            let g = graph.read().map_err(|_| poisoned("graph lock"))?;
            let effective_roots = extend_with_side_effect_roots(&g, targets);
            // PR-A3b-1: derive from the OptimizedGraph's run lowering
            // when present (post-safety-copies); else the legacy
            // `execution_plan` walk. Identical on branchless graphs.
            //
            // PR-C1: a `Streaming` source is handed to the compiler thread
            // unflattened (resolves branches at the frontier).
            //
            // Step E A4b open question #4: also derives the `WaitSet`
            // ("event only where waited") from the same materialized order.
            let (work, wait_set) =
                compiler_work_and_wait_set_for(&g, &effective_roots, &order_source);
            let mut layouts = HashMap::with_capacity(inputs.len());
            for &id in inputs.keys() {
                layouts.insert(id, g.layout(id));
            }
            (work, wait_set, layouts)
        };

        // Caller's target set — used to gate destructive-input
        // eviction so we don't drop a tensor the caller asked for.
        let target_set: std::collections::HashSet<NodeId> = targets.iter().copied().collect();

        let (tx, rx) = channel::<Result<WorkItem>>();
        let graph_for_compiler = Arc::clone(&graph);

        let compiler = thread::spawn(move || {
            compiler_thread_body(graph_for_compiler, compiler_work, sym_env, tx);
        });

        // Phase 4.3: per-chunk SystemTopology generation check (see
        // realize_inner for the rationale). PR-A3b-1: the OptimizedGraph
        // path supplies its own optimize-time generation.
        let plan_generation: Option<u64> = generation_for(&order_source);
        let mut current_chunk_backend: Option<BackendId> = None;
        let mut cache: StorageCache = inputs;
        // Data-dependent dynamic-shapes producer-bound symbols (see
        // `realize_inner`). `realize_many` does not expose them today; the
        // env is dropped at pass end (producers still publish, keeping the
        // seam uniform across both realize loops).
        let mut produced_syms = SymEnv::new();
        // Step E A4b-1/A4b-3: per-node async-completion handles (see realize_inner
        // for the full rationale). Map drained at realize-end; the cross-device
        // copy waits the SOURCE producer's handle (A4b-3).
        let mut handles: HashMap<NodeId, CompletionHandle> = HashMap::new();
        // Step E A4b-4: eagerly-submitted in-flight Vulkan batches + the
        // multi-backend gate. See realize_inner for the full rationale — this is
        // the identical eager-submit / in-flight-lifetime machinery applied to the
        // multi-target walk. `multi_backend` arms on a genuine backend switch and
        // gates every eager submit, so a single-device realize_many is
        // byte-identical to A4b-2.
        let mut inflight_vulkan: Vec<InflightVulkan> = Vec::new();
        let mut multi_backend = false;
        for item in rx {
            let item = item?;
            // Chunk-boundary hook: TopologyChanged check + (A4b-4) backend-switch
            // detection + eager-submit-on-leaving-a-Vulkan-chunk.
            if current_chunk_backend != Some(item.target_backend) {
                if let Some(plan_gen) = plan_generation {
                    let live = crate::dispatch::topology_generation();
                    if live != plan_gen {
                        return Err(Error::TopologyChanged {
                            plan_generation: plan_gen,
                            current_generation: live,
                        }
                        .bt());
                    }
                }
                let leaving = current_chunk_backend;
                if let Some(prev) = leaving
                    && prev != item.target_backend
                {
                    multi_backend = true;
                    if prev == BackendId::Vulkan {
                        eager_submit_all_vulkan(&cache, &mut inflight_vulkan)?;
                    }
                }
                current_chunk_backend = Some(item.target_backend);
            }
            // Step E A4b-4 (Invariant E): wait in-flight Vulkan before recording a
            // new Vulkan op (same-device cross-chunk RAW safety). See realize_inner.
            if multi_backend
                && item.target_backend == BackendId::Vulkan
                && !inflight_vulkan.is_empty()
            {
                drain_inflight_vulkan(&mut inflight_vulkan)?;
            }
            // Step E A4b-3 + A4b-4: cross-device residency boundary. See
            // realize_inner for the full rationale.
            if matches!(
                item.kind,
                WorkItemKind::Copy { .. } | WorkItemKind::Move { .. }
            ) {
                // A4b-4: submit any open Vulkan chunk so the iGPU runs while the
                // host (possibly) waits on the CUDA producer.
                if multi_backend {
                    eager_submit_all_vulkan(&cache, &mut inflight_vulkan)?;
                }
                if let Some(&producer) = item.inputs.first() {
                    // A4b-4: wait in-flight Vulkan ONLY for a Vulkan-source D2H
                    // (reads completed data; download flushes only the OPEN batch).
                    // A CUDA/CPU source does NOT wait Vulkan (preserves overlap).
                    if multi_backend && copy_source_is_vulkan(&cache, producer)? {
                        drain_inflight_vulkan(&mut inflight_vulkan)?;
                    }
                    // Step E A4b-3: finer CUDA source-drain — wait ONLY the producer.
                    wait_producer_handle(&mut handles, producer)?;
                }
            }
            let handle = execute_work_item(
                &item,
                &mut cache,
                &mut layout_cache,
                &mut produced_syms,
                will_be_waited(&wait_set, item.node_id),
                // No persistent-output mode on the standard realize paths —
                // fresh alloc per node, byte-identical to pre-CapturedRun.
                None,
            )
            .map_err(|e| with_node_location(&graph, item.node_id, e))?;
            store_handle(&mut handles, item.node_id, handle);
            if let Some(d_idx) = item.destructive_input
                && let Some(&destroyed) = item.inputs.get(d_idx)
                && !target_set.contains(&destroyed)
            {
                // Step E A2.1 (in-flight DATA-BUFFER lifetime — UAF guard,
                // now DEFERRED-DELETION instead of a drain): retain the
                // evicted buffer until the reader fences signal — eager-
                // submit the open batch (no wait) + move a refcount clone of
                // the buffer into every in-flight batch, freeing it post-
                // fence on each batch's `Drop` with NO host block. Replaces
                // the pre-A2.1 `drain_inflight_vulkan` + open-batch
                // `force_flush` (two host-blocking fence waits). See
                // `defer_evicted_vulkan_buffer` / realize_inner for the
                // UAF/leak argument. CUDA eviction is A3 stream-safe.
                // Single-device (multi_backend == false) keeps the pre-A2.1
                // blocking force_flush of the OPEN batch (the else branch) —
                // byte-identical + UAF-safe; the deferred path is unsafe there
                // (see force_flush_vulkan's doc).
                if multi_backend {
                    defer_evicted_vulkan_buffer(&cache, destroyed, &mut inflight_vulkan)?;
                } else if let Some(d_arc) = cache.get(&destroyed) {
                    force_flush_vulkan(d_arc)?;
                }
                cache.remove(&destroyed);
                layout_cache.remove(&destroyed);
                // CUDA eviction is stream-ordered-safe (A3); drop the
                // handle so the realize-end empty-map assert stays valid.
                handles.remove(&destroyed);
            }
        }

        compiler
            .join()
            .map_err(|_| Error::Msg("compiler thread panicked".to_string()).bt())?;

        // Step E A4b-1: drain every outstanding async handle (CUDA events) before
        // results are read / the cache drops.
        drain_handles(&mut handles)?;
        // Step E A4b open question #4: blanket per-device stream sync for the
        // elided nodes `drain_handles` never tracked — see `realize_inner`'s
        // identical comment / `sync_active_cuda_devices`'s doc for the argument.
        sync_active_cuda_devices(&cache)?;
        // Step E A4b-4: wait every eagerly-submitted in-flight Vulkan batch before
        // the cache drops. Empty on a single-device realize.
        drain_inflight_vulkan(&mut inflight_vulkan)?;
        // Step E A4b-2: drain all deferred Vulkan work — submit the open batch
        // then wait it via a `VulkanCompletion` handle (the A2 submit+wait, split;
        // byte-identical for pure-Vulkan — one submission at realize-end).
        drain_vulkan_pending(&cache)?;
        debug_assert!(
            inflight_vulkan.is_empty(),
            "A4b-4: in-flight Vulkan batch list must be empty after realize-end drain",
        );

        // Verify every target was realized + collect outputs in
        // target order. We don't `remove` from the cache because the
        // same NodeId can appear twice in `targets` (caller wants the
        // same storage twice) — clone the Arc instead.
        let mut out = Vec::with_capacity(targets.len());
        for &target in targets {
            let storage = cache.get(&target).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor::realize_many: target {:?} not populated after execution",
                    target
                ))
                .bt()
            })?;
            let layout = layout_cache.get(&target).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor::realize_many: target layout {:?} not populated after execution",
                    target
                ))
                .bt()
            })?;
            out.push((storage, layout));
        }
        Ok(out)
    }
}

/// Resolve the executor's dispatch order from the configured
/// [`OrderSource`]. PR-A3b-1: `Optimized` lowers the in-place
/// "plan IS the graph" form via `extract_runs`/`lower_run`; `Default`
/// keeps the pre-A3b-1 `execution_plan` walk. Both produce the same
/// `NodeId` sequence on a branchless graph (A3a equivalence gate),
/// computed here AFTER safety-copy insertion so either path covers
/// any freshly inserted copies.
fn order_for(
    graph: &Graph,
    effective_roots: &[NodeId],
    order_source: &OrderSource<'_>,
) -> Vec<NodeId> {
    match order_source {
        OrderSource::Default => execution_plan(graph, effective_roots),
        // The streaming walk is driven in the compiler thread (it
        // interleaves arm resolution with compilation), so `order_for`
        // never flattens it on the production path. This arm exists only
        // for match totality / non-pipelined callers: it collects the SAME
        // lazily-resolved order eagerly (the streaming walk with the same
        // `resolve_branch` over the threaded selector), which equals the
        // one-shot `lower_picked_route(pick_route(..))` order.
        OrderSource::Streaming {
            optimized: _optimized,
            pick,
        } => {
            let bindings = global_bindings();
            fuel_graph::lower_picked_route_streaming(graph, effective_roots, |branch| {
                resolve_branch(graph, branch, &bindings, pick.selector.as_ref())
            })
        }
        OrderSource::Optimized { optimized, route } => {
            // Lower the runs over the *effective* roots (user targets +
            // any spliced/side-effect roots) so the order matches the
            // walk's reachable set rather than only the optimize-time
            // roots. On a branchless graph this is the same concatenated
            // `lower_run` sequence the OptimizedGraph reports.
            //
            // PR-C1: when the picker supplied a route, follow the chosen
            // arms (`lower_picked_route`); otherwise the arm-0 lowering
            // (`OptimizedGraph::dispatch_order`). An empty route is
            // byte-identical to arm-0, so the two paths agree whenever
            // the picker chose arm-0 everywhere (the no-pressure case).
            match route {
                Some(route) => fuel_graph::lower_picked_route(graph, effective_roots, route),
                None => {
                    // A transient view built solely to call `dispatch_order`;
                    // it does no planning, so it carries no placements. The
                    // real optimization's map lives on `optimized`.
                    let view = OptimizedGraph {
                        roots: effective_roots.to_vec(),
                        generation: optimized.generation,
                        placements: None,
                    };
                    view.dispatch_order(graph)
                }
            }
        }
        // Resolved by `compiler_work_and_wait_set_for` before delegating
        // here (the whole point of `Cached` is to skip this computation
        // on a hit) — never reached. Kept exhaustive rather than wildcarded
        // so a future `OrderSource` variant can't silently fall through.
        OrderSource::Cached { .. } => {
            unreachable!("OrderSource::Cached is resolved before order_for is called")
        }
    }
}

/// Step E A4b open question #4 — "event only where waited". The producer
/// [`NodeId`]s that some later `Op::Copy`/`Op::Move` node in `order` will
/// `wait_producer_handle` on (the executor's cross-device residency
/// boundary — see `wait_producer_handle`, whose `item.inputs.first()` read
/// this mirrors exactly). A CUDA node whose id is absent from the returned
/// set has NO mid-walk reader: same-device consumers are correct via stream
/// ordering alone (they never `wait`), and nothing touches its handle before
/// the realize-end drain, so [`crate::compiled::execute_compiled_with_wait_hint`]
/// can skip recording an `Event` for it entirely.
///
/// Requires a materialized `order` (works for [`OrderSource::Default`] /
/// [`OrderSource::Optimized`], whose [`CompilerWork::Order`] is a frozen
/// `Vec<NodeId>` computed up front) — see
/// [`compiler_work_and_wait_set_for`] for the [`OrderSource::Streaming`]
/// fallback.
fn build_wait_set(graph: &Graph, order: &[NodeId]) -> HashSet<NodeId> {
    let mut set = HashSet::new();
    for &id in order {
        let node = graph.node(id);
        if matches!(node.op, Op::Copy { .. } | Op::Move { .. })
            && let Some(&producer) = node.inputs.first()
        {
            set.insert(producer);
        }
    }
    set
}

/// Step E A4b open question #4: the executor's per-realize wait-set.
/// `None` is the CONSERVATIVE fallback — every CUDA node still gets an Event
/// (today's pre-elision behavior, byte-identical). `Some(set)` restricts
/// event production to `set`'s members ([`build_wait_set`]'s output);
/// everything else gets `CompletionHandle::Ready` with no Event recorded.
type WaitSet = Option<HashSet<NodeId>>;

/// Whether `node_id`'s completion handle must be individually observable
/// (i.e. `execute_compiled_with_wait_hint`'s `will_be_waited`). `None` (the
/// [`OrderSource::Streaming`] fallback — never elides) is always `true`.
#[inline]
fn will_be_waited(wait_set: &WaitSet, node_id: NodeId) -> bool {
    match wait_set {
        None => true,
        Some(set) => set.contains(&node_id),
    }
}

/// Build the [`CompilerWork`] the compiler thread walks for the configured
/// [`OrderSource`], plus the [`WaitSet`] "event only where waited" (open
/// question #4) consults. The non-streaming sources flatten up front via
/// [`order_for`] (unchanged) and the SAME materialized order feeds
/// [`build_wait_set`], so the pre-scan and the dispatch order never
/// disagree. [`OrderSource::Streaming`] is handed to the compiler thread as
/// a [`CompilerWork::Streaming`] spec — NOT flattened here — so the thread
/// resolves branches lazily at the frontier (Step E Phase C, PR C1); the lazy
/// order can't be pre-scanned without eagerly resolving it (which would
/// defeat the point of streaming), so it falls back to the conservative
/// `WaitSet::None` — event every node, unchanged behavior. Holds the graph
/// read-lock only for the flattening branches; the streaming branch defers
/// the lock to the compiler thread.
fn compiler_work_and_wait_set_for(
    graph: &Graph,
    effective_roots: &[NodeId],
    order_source: &OrderSource<'_>,
) -> (CompilerWork, WaitSet) {
    match order_source {
        OrderSource::Streaming { pick, .. } => (
            CompilerWork::Streaming {
                roots: effective_roots.to_vec(),
                pick: (*pick).clone(),
            },
            None,
        ),
        OrderSource::Cached {
            optimized,
            order_cache,
        } => {
            let order = order_cache.get_or_init(|| {
                order_for(
                    graph,
                    effective_roots,
                    &OrderSource::Optimized {
                        optimized,
                        route: None,
                    },
                )
            });
            let wait_set = build_wait_set(graph, order);
            (CompilerWork::Order(order.clone()), Some(wait_set))
        }
        other => {
            let order = order_for(graph, effective_roots, other);
            let wait_set = build_wait_set(graph, &order);
            (CompilerWork::Order(order), Some(wait_set))
        }
    }
}

/// The topology generation that keys the executor's `TopologyChanged`
/// chunk-boundary check. Uses the `OptimizedGraph`'s optimize-time
/// generation (PR-A3b-1 path). `None` ⇒ no check (the plain
/// `realize`/`realize_many` entries).
fn generation_for(order_source: &OrderSource<'_>) -> Option<u64> {
    match order_source {
        OrderSource::Optimized { optimized, .. } => Some(optimized.generation),
        OrderSource::Streaming { optimized, .. } => Some(optimized.generation),
        OrderSource::Cached { optimized, .. } => Some(optimized.generation),
        OrderSource::Default => None,
    }
}

/// What the compiler thread walks: either a pre-flattened dispatch
/// `Order` (the `Default` / `Optimized` paths — unchanged), or a
/// `Streaming` spec the thread resolves branch-by-branch at the frontier
/// (Step E Phase C, PR C1).
enum CompilerWork {
    /// A frozen topological order (computed up front by `order_for`). The
    /// compiler thread compiles each node and sends it in turn.
    Order(Vec<NodeId>),
    /// The streaming run-walk: the compiler thread drives
    /// [`fuel_graph::walk_picked_route_streaming`] over `roots`, resolving
    /// each `Op::Branch` lazily via [`resolve_branch`] as the frontier
    /// reaches its arm-entry, and compiling + sending only the chosen
    /// arm's runs. `roots` are the effective roots (same set `order_for`
    /// lowers over). Byte-identical to `Order(lower_picked_route(
    /// pick_route(..)))` in C1 (same VRAM-only selector).
    Streaming {
        roots: Vec<NodeId>,
        pick: StreamingPick,
    },
}

/// Compiler thread body. Holds the graph read-lock + a thread-local
/// layout cache and emits `WorkItem`s in topological order. For
/// [`CompilerWork::Order`] it walks the frozen order; for
/// [`CompilerWork::Streaming`] it resolves each branch at the frontier and
/// emits only the chosen arm's runs (Step E Phase C, PR C1). Sends
/// `Err(...)` and stops on the first failure.
fn compiler_thread_body(
    graph: Arc<RwLock<Graph>>,
    work: CompilerWork,
    sym_env: SymEnv,
    tx: Sender<Result<WorkItem>>,
) {
    let bindings = global_bindings();

    let g = match graph.read() {
        Ok(g) => g,
        Err(_) => {
            let _ = tx.send(Err(poisoned("graph lock in compiler")));
            return;
        }
    };

    // Compiler-thread-local layout cache. Populated as compile_one
    // walks topologically; downstream nodes look up their inputs'
    // layouts here (rather than from the graph side-table) to honor
    // the strided layouts emitted by metadata-only view ops earlier
    // in the same realize call.
    let mut layout_cache: HashMap<NodeId, Layout> = HashMap::new();
    // `compile_one` for one node + send; returns `false` to stop the walk
    // (channel closed or compile error). Shared by both work modes so the
    // streamed and frozen-order paths compile identically.
    let compile_and_send = |id: NodeId, layout_cache: &mut HashMap<NodeId, Layout>| -> bool {
        let item = compile_one(&g, id, layout_cache, &bindings, &sym_env);
        let stop_on_err = item.is_err();
        if tx.send(item).is_err() {
            return false;
        }
        !stop_on_err
    };

    match work {
        CompilerWork::Order(order) => {
            for id in order {
                if !compile_and_send(id, &mut layout_cache) {
                    return;
                }
            }
        }
        CompilerWork::Streaming { roots, pick } => {
            // Resolve each branch lazily at its arm-entry. C1 built the
            // streaming frontier; C2 made `pick.selector` load-aware (the
            // production `ChainedSelector` now reads each arm's live device
            // load — `pending_work_count / slot_capacity` via the
            // `BackendStreams` seam — as a key leg under the VRAM guard),
            // so this resolution reads live device load at THIS moment: the
            // executor has been submitting/draining (and updating the B1
            // counter) while the compiler ran ahead. Emit only the chosen
            // arm's runs, compiling + sending each node as the walk reaches
            // it — genuine streaming (resolution interleaved with
            // dispatch), not a pre-flattened order.
            let mut stopped = false;
            fuel_graph::walk_picked_route_streaming(
                &g,
                &roots,
                |branch| resolve_branch(&g, branch, &bindings, pick.selector.as_ref()),
                |members| {
                    if stopped {
                        return;
                    }
                    for &id in members {
                        if !compile_and_send(id, &mut layout_cache) {
                            stopped = true;
                            return;
                        }
                    }
                },
            );
        }
    }
}

/// Resolve the [`CompiledNode`] for a kernel-bearing graph node via
/// the first-registered [`compile_node`] binding-table lookup.
///
/// `op_params` always comes from the executor's
/// `op_to_op_params(graph, node, id, layout_cache, sym_env)` — the live
/// OpParams shape (reduce dims, conv geometry, the per-pass
/// `SymEnv`-resolved `WriteSlice` offset, etc.) derives from the graph
/// node + its resolved input layouts at execute time.
fn resolve_compiled(
    op_kind: OpKind,
    dtypes: &[DType],
    target_backend: BackendId,
    op_params: OpParams,
    bindings: &KernelBindingTable,
) -> Result<CompiledNode> {
    compile_node(op_kind, dtypes, target_backend, op_params, bindings)
}

/// Resolve one node into a `WorkItem` and update `layout_cache`
/// with the node's output layout. Three op shapes:
///
/// - `Op::Const` — adopts the entry from the input cache; layout is
///   read from the graph's side-table (or its contiguous fallback).
///
/// - Metadata-only view op — output layout is read from the graph's
///   side-table (populated by `Graph::push` at construction time);
///   the executor adopts the input's Storage Arc (no allocation, no
///   kernel).
///
/// - Computational op — output layout is `Layout::contiguous(node.shape)`
///   because today's kernels write contiguous output. The compiler
///   resolves the kernel ref and emits a Kernel work item.
fn compile_one(
    graph: &Graph,
    id: NodeId,
    layout_cache: &mut HashMap<NodeId, Layout>,
    bindings: &KernelBindingTable,
    sym_env: &SymEnv,
) -> Result<WorkItem> {
    let node = graph.node(id);
    let elem_count = node.shape.elem_count();
    let inputs = node.inputs.clone();
    // Snapshot destructive-input metadata from the graph at compile
    // time so the executor can evict the destroyed input from its
    // cache after the op runs (Op::Release / Op::Move semantics).
    let destructive_input = node.op.destructive_input();

    if matches!(node.op, Op::Const) {
        let output_layout = graph.layout(id);
        layout_cache.insert(id, output_layout.clone());
        return Ok(WorkItem {
            node_id: id,
            inputs,
            elem_count,
            dtype: node.dtype,
            target_backend: BackendId::Cpu,
            kind: WorkItemKind::ConstAdopt,
            compiled: None,
            output_layout,
            destructive_input,
            output_bundle: None,
        });
    }

    if node.op.is_view_op() {
        if inputs.len() != 1 {
            return Err(Error::Msg(format!(
                "view op {:?} expects 1 input, got {}",
                node.op,
                inputs.len(),
            ))
            .bt());
        }
        // Layout is read from the graph's side-table — populated by
        // `Graph::push` at construction time for view ops, and by
        // graph-rewriting opt passes that emit view nodes. The
        // compiler does not re-derive: graph.layout(id) is the single
        // source of truth.
        let output_layout = graph.layout(id);
        layout_cache.insert(id, output_layout.clone());
        // Inherit the upstream's target_backend (or default CPU) —
        // metadata-only adoption doesn't actually run on a backend,
        // but downstream consumers look at target_backend so it
        // needs to be set sensibly. Any device works.
        let target_backend = graph
            .target_backend(id)
            .or_else(|| graph.target_backend(inputs[0]))
            .unwrap_or(BackendId::Cpu);
        return Ok(WorkItem {
            node_id: id,
            inputs,
            elem_count,
            dtype: node.dtype,
            target_backend,
            kind: WorkItemKind::ViewOf {
                input: node.inputs[0],
            },
            compiled: None,
            output_layout,
            destructive_input,
            output_bundle: None,
        });
    }

    // Op::View — multi-output projection (Option C, Session 4).
    // Structurally identical to a metadata-only view op at realize
    // time: clone the producer's Storage Arc into the View's cache
    // slot. The graph's layout side-table, populated by
    // `Tensor::view`, already carries the slot's effective layout
    // (slot.byte_offset baked into start_offset in slot-dtype-element
    // units).
    if let Op::View { .. } = node.op {
        if inputs.len() != 1 {
            return Err(Error::Msg(format!(
                "Op::View expects 1 input (the producer), got {}",
                inputs.len(),
            ))
            .bt());
        }
        let output_layout = graph.layout(id);
        layout_cache.insert(id, output_layout.clone());
        let target_backend = graph
            .target_backend(id)
            .or_else(|| graph.target_backend(inputs[0]))
            .unwrap_or(BackendId::Cpu);
        return Ok(WorkItem {
            node_id: id,
            inputs: inputs.clone(),
            elem_count,
            dtype: node.dtype,
            target_backend,
            kind: WorkItemKind::SlotView {
                producer: inputs[0],
            },
            compiled: None,
            output_layout,
            destructive_input,
            output_bundle: None,
        });
    }

    // Op::ViewOwned — copies the slot's bytes into a fresh
    // contiguous Storage at realize time. Output layout is
    // contiguous starting at offset 0 (the memcpy produced an
    // independent buffer).
    if let Op::ViewOwned { slot } = node.op {
        if inputs.len() != 1 {
            return Err(Error::Msg(format!(
                "Op::ViewOwned expects 1 input (the producer), got {}",
                inputs.len(),
            ))
            .bt());
        }
        let output_layout = Layout::contiguous(node.shape.clone());
        layout_cache.insert(id, output_layout.clone());
        let target_backend = graph
            .target_backend(id)
            .or_else(|| graph.target_backend(inputs[0]))
            .unwrap_or(BackendId::Cpu);
        return Ok(WorkItem {
            node_id: id,
            inputs: inputs.clone(),
            elem_count,
            dtype: node.dtype,
            target_backend,
            kind: WorkItemKind::SlotOwn {
                producer: inputs[0],
                slot,
            },
            compiled: None,
            output_layout,
            destructive_input,
            output_bundle: None,
        });
    }

    if matches!(node.op, Op::Release) {
        // Op::Release produces a zero-element marker output. The
        // marker storage is CPU + 0 bytes — nothing reads it; it
        // exists so cache lookups on the Release NodeId resolve.
        // The actual deallocation of `inputs[0]` happens in the
        // realize loop via `destructive_input` cleanup.
        if inputs.len() != 1 {
            return Err(
                Error::Msg(format!("Op::Release expects 1 input, got {}", inputs.len(),)).bt(),
            );
        }
        let output_layout = Layout::contiguous(node.shape.clone());
        layout_cache.insert(id, output_layout.clone());
        // Inherit the source's target_backend so any downstream
        // backend-aware logic (telemetry, scheduling) sees a sensible
        // value. The marker doesn't actually run on any backend.
        let target_backend = graph
            .target_backend(id)
            .or_else(|| graph.target_backend(inputs[0]))
            .unwrap_or(BackendId::Cpu);
        return Ok(WorkItem {
            node_id: id,
            inputs,
            elem_count,
            dtype: node.dtype,
            target_backend,
            kind: WorkItemKind::ReleaseMarker,
            compiled: None,
            output_layout,
            destructive_input,
            output_bundle: None,
        });
    }

    // In-place ops (Phase 3 of in-place ops infrastructure): kernel
    // lookup goes through the standard binding-table path, but the
    // executor adopts the target's Arc as the output instead of
    // allocating fresh bytes. The compile-time predicate is exactly
    // `op.destructive_input().is_some()` AND NOT one of the
    // structural ops (WriteSlice / ZeroFill / Release / Move) which
    // have their own dedicated WorkItemKind arms in this function.
    if !matches!(
        node.op,
        Op::WriteSlice { .. }
            | Op::WriteSliceRotating { .. }
            | Op::WriteSliceDoff { .. }
            | Op::ZeroFill
            | Op::Release
            | Op::Move { .. },
    ) && let Some(target_idx) = node.op.destructive_input()
    {
        if inputs.len() <= target_idx {
            return Err(Error::Msg(format!(
                "in-place op {:?} declares destructive_input={target_idx} \
                     but has only {} input(s)",
                node.op,
                inputs.len(),
            ))
            .bt());
        }
        let target_backend = graph.target_backend(id).ok_or_else(|| {
            Error::Msg(format!(
                "PipelinedExecutor: in-place op node {id:?} has no target_backend set",
            ))
            .bt()
        })?;
        let op_kind = op_to_op_kind(&node.op).ok_or_else(|| {
            Error::Msg(format!(
                "PipelinedExecutor: in-place op {:?} has no op_to_op_kind mapping",
                node.op,
            ))
            .bt()
        })?;
        let op_params = op_to_op_params(graph, node, id, layout_cache, sym_env)?;
        let dtypes = build_lookup_dtypes(graph, node);
        let compiled = resolve_compiled(op_kind, &dtypes, target_backend, op_params, bindings)?;
        // Output adopts target's Layout (same Storage Arc, same shape).
        let output_layout = graph.layout(inputs[target_idx]);
        layout_cache.insert(id, output_layout.clone());
        return Ok(WorkItem {
            node_id: id,
            inputs: inputs.clone(),
            elem_count,
            dtype: node.dtype,
            target_backend,
            kind: WorkItemKind::InplaceKernel { target_idx },
            compiled: Some(compiled),
            output_layout,
            destructive_input,
            output_bundle: None,
        });
    }

    if matches!(node.op, Op::WriteSlice { .. }) {
        // WriteSlice: kernel lookup happens like a normal kernel, but
        // the work item carries a dedicated kind so the executor
        // knows to adopt the destination's Arc as the output rather
        // than allocating a fresh buffer. The destination's NodeId is
        // captured here so the executor doesn't have to re-derive it.
        if inputs.len() != 2 {
            return Err(Error::Msg(format!(
                "Op::WriteSlice expects 2 inputs (destination, source), got {}",
                inputs.len(),
            ))
            .bt());
        }
        let target_backend = graph.target_backend(id).ok_or_else(|| {
            Error::Msg(format!(
                "PipelinedExecutor: WriteSlice node {:?} has no target_backend set",
                id
            ))
            .bt()
        })?;
        let op_params = op_to_op_params(graph, node, id, layout_cache, sym_env)?;
        let dtypes = build_lookup_dtypes(graph, node);
        let compiled = resolve_compiled(
            OpKind::WriteSlice,
            &dtypes,
            target_backend,
            op_params,
            bindings,
        )?;
        // Output adopts the destination's Layout — same Storage Arc,
        // same shape. Downstream consumers that want a post-write
        // sub-extent compose an explicit Op::Slice (WriteSlice does
        // not encode partial extents).
        let output_layout = graph.layout(inputs[0]);
        layout_cache.insert(id, output_layout.clone());
        return Ok(WorkItem {
            node_id: id,
            inputs: inputs.clone(),
            elem_count,
            dtype: node.dtype,
            target_backend,
            kind: WorkItemKind::WriteSlice {
                dest: inputs[0],
                source: inputs[1],
            },
            compiled: Some(compiled),
            output_layout,
            destructive_input,
            output_bundle: None,
        });
    }

    if matches!(node.op, Op::WriteSliceRotating { .. }) {
        // WriteSliceRotating: same shape as WriteSlice but with a
        // third input (position scalar). Dispatch through the
        // binding table at `OpKind::WriteSliceRotating`; the
        // executor's arm reads the position from `position`'s
        // storage and the kernel splits across the ring boundary.
        if inputs.len() != 3 {
            return Err(Error::Msg(format!(
                "Op::WriteSliceRotating expects 3 inputs (destination, source, position), got {}",
                inputs.len(),
            ))
            .bt());
        }
        let target_backend = graph.target_backend(id).ok_or_else(|| {
            Error::Msg(format!(
                "PipelinedExecutor: WriteSliceRotating node {:?} has no target_backend set",
                id
            ))
            .bt()
        })?;
        let op_params = op_to_op_params(graph, node, id, layout_cache, sym_env)?;
        let dtypes = build_lookup_dtypes(graph, node);
        let compiled = resolve_compiled(
            OpKind::WriteSliceRotating,
            &dtypes,
            target_backend,
            op_params,
            bindings,
        )?;
        let output_layout = graph.layout(inputs[0]);
        layout_cache.insert(id, output_layout.clone());
        return Ok(WorkItem {
            node_id: id,
            inputs: inputs.clone(),
            elem_count,
            dtype: node.dtype,
            target_backend,
            kind: WorkItemKind::WriteSliceRotating {
                dest: inputs[0],
                source: inputs[1],
                position: inputs[2],
            },
            compiled: Some(compiled),
            output_layout,
            destructive_input,
            output_bundle: None,
        });
    }

    if matches!(node.op, Op::WriteSliceDoff { .. }) {
        // WriteSliceDoff: same shape as WriteSlice but with a third
        // input (device-resident I64 offset). Dispatch through the
        // binding table at `OpKind::WriteSliceDoff`; the executor's arm
        // threads the offset's device pointer straight to the kernel
        // (no D2H, capture-safe).
        if inputs.len() != 3 {
            return Err(Error::Msg(format!(
                "Op::WriteSliceDoff expects 3 inputs (destination, source, offset), got {}",
                inputs.len(),
            ))
            .bt());
        }
        let target_backend = graph.target_backend(id).ok_or_else(|| {
            Error::Msg(format!(
                "PipelinedExecutor: WriteSliceDoff node {:?} has no target_backend set",
                id
            ))
            .bt()
        })?;
        let op_params = op_to_op_params(graph, node, id, layout_cache, sym_env)?;
        let dtypes = build_lookup_dtypes(graph, node);
        let compiled = resolve_compiled(
            OpKind::WriteSliceDoff,
            &dtypes,
            target_backend,
            op_params,
            bindings,
        )?;
        let output_layout = graph.layout(inputs[0]);
        layout_cache.insert(id, output_layout.clone());
        return Ok(WorkItem {
            node_id: id,
            inputs: inputs.clone(),
            elem_count,
            dtype: node.dtype,
            target_backend,
            kind: WorkItemKind::WriteSliceDoff {
                dest: inputs[0],
                source: inputs[1],
                offset: inputs[2],
            },
            compiled: Some(compiled),
            output_layout,
            destructive_input,
            output_bundle: None,
        });
    }

    if let Op::Alloc { target } = node.op {
        // Op::Alloc { target }: produce a fresh zero-init Storage on
        // `target` with node.shape * node.dtype. Zero inputs.
        // Direct executor dispatch via WorkItemKind::Alloc — no
        // binding-table lookup (the device handle threading isn't
        // expressible through the binding table's current key shape).
        if !inputs.is_empty() {
            return Err(
                Error::Msg(format!("Op::Alloc expects 0 inputs, got {}", inputs.len(),)).bt(),
            );
        }
        let output_layout = Layout::contiguous(node.shape.clone());
        layout_cache.insert(id, output_layout.clone());
        // target_backend is informational here (the executor's Alloc
        // arm consults target_location for allocation routing); set
        // it to target's backend for consistency with how downstream
        // nodes inherit/read it.
        let target_backend = match target {
            DeviceLocation::Cpu => BackendId::Cpu,
            DeviceLocation::Cuda { .. } => BackendId::Cuda,
            DeviceLocation::Vulkan { .. } => BackendId::Vulkan,
            DeviceLocation::Metal { .. } => BackendId::Metal,
        };
        return Ok(WorkItem {
            node_id: id,
            inputs,
            elem_count,
            dtype: node.dtype,
            target_backend,
            kind: WorkItemKind::Alloc {
                target_location: target,
            },
            compiled: None,
            output_layout,
            destructive_input,
            output_bundle: None,
        });
    }

    if let Op::Iota { len } = node.op {
        // Op::Iota { len }: 0-input F32 position generator
        // [0.0, 1.0, …, (len-1)]. Direct executor dispatch via
        // WorkItemKind::Iota — no binding-table lookup (a leaf has no
        // inputs[0] to inherit a backend from). Modeled on Op::Alloc.
        if !inputs.is_empty() {
            return Err(
                Error::Msg(format!("Op::Iota expects 0 inputs, got {}", inputs.len(),)).bt(),
            );
        }
        let output_layout = Layout::contiguous(node.shape.clone());
        layout_cache.insert(id, output_layout.clone());
        // Leaf: the planner may not place a 0-input node, so fall back to
        // CPU. Iota is a tiny host-side generator; its only consumer today
        // is the CPU flash-attn decomposition's alibi bias.
        let target_backend = graph.target_backend(id).unwrap_or(BackendId::Cpu);
        return Ok(WorkItem {
            node_id: id,
            inputs,
            elem_count,
            dtype: node.dtype,
            target_backend,
            kind: WorkItemKind::Iota { len },
            compiled: None,
            output_layout,
            destructive_input,
            output_bundle: None,
        });
    }

    if matches!(node.op, Op::ZeroFill) {
        // Op::ZeroFill: 1 input (the buffer to zero); output aliases
        // input's Storage Arc (destructive in-place). No binding-
        // table lookup — per-backend dispatch in execute_work_item.
        if inputs.len() != 1 {
            return Err(Error::Msg(format!(
                "Op::ZeroFill expects 1 input, got {}",
                inputs.len(),
            ))
            .bt());
        }
        // Output layout: contiguous in the node's shape. We don't
        // require the input to be contiguous (auto_contiguize in the
        // executor handles strided inputs), but the destructive
        // semantic means callers should only ZeroFill an Op::Alloc
        // output today.
        let output_layout = Layout::contiguous(node.shape.clone());
        layout_cache.insert(id, output_layout.clone());
        let target_backend = graph
            .target_backend(id)
            .or_else(|| graph.target_backend(inputs[0]))
            .unwrap_or(BackendId::Cpu);
        return Ok(WorkItem {
            node_id: id,
            inputs,
            elem_count,
            dtype: node.dtype,
            target_backend,
            kind: WorkItemKind::ZeroFill,
            compiled: None,
            output_layout,
            destructive_input,
            output_bundle: None,
        });
    }

    if let Op::Copy { target } = node.op {
        // Op::Copy { target }: download bytes from the source residency
        // to a fresh Storage allocated on `target`. Kernel lookup uses
        // `target_backend` = source backend (set by `prepare()` on the
        // realize device the source is on); the wrapper does the actual
        // transfer (e.g. Vulkan::download_bytes → memcpy into a CPU
        // CpuStorageBytes). The output Storage's variant is determined
        // by `target_location` and allocated in `execute_work_item`'s
        // WorkItemKind::Copy arm.
        if inputs.len() != 1 {
            return Err(
                Error::Msg(format!("Op::Copy expects 1 input, got {}", inputs.len(),)).bt(),
            );
        }
        let target_backend = graph.target_backend(id).ok_or_else(|| {
            Error::Msg(format!(
                "PipelinedExecutor: Op::Copy node {:?} has no target_backend \
                 set (= source backend, the one whose kernel runs the \
                 download)",
                id
            ))
            .bt()
        })?;
        let op_params = OpParams::None;
        let dtypes = build_lookup_dtypes(graph, node);
        let compiled =
            resolve_compiled(OpKind::Copy, &dtypes, target_backend, op_params, bindings)?;
        // Output layout: contiguous in the node's shape (mirrors the
        // source's logical shape). Auto-contiguize on the input side
        // (in execute_work_item) handles strided source views by
        // materializing them into a contiguous source buffer before
        // the kernel runs.
        let output_layout = Layout::contiguous(node.shape.clone());
        layout_cache.insert(id, output_layout.clone());
        return Ok(WorkItem {
            node_id: id,
            inputs,
            elem_count,
            dtype: node.dtype,
            target_backend,
            kind: WorkItemKind::Copy {
                target_location: target,
            },
            compiled: Some(compiled),
            output_layout,
            destructive_input,
            output_bundle: None,
        });
    }

    if let Op::Move { target } = node.op {
        // Op::Move { target }: identical data-movement half to
        // Op::Copy — kernel lookup at (OpKind::Copy, [dt, dt],
        // source_backend); output allocated on `target` by the
        // executor's Copy/Move arm. The destructive half (release
        // the source) rides the realize loop's `destructive_input`
        // cleanup via the snapshot taken at the top of this function
        // (`Op::Move::destructive_input() == Some(0)`). Ordering
        // safety (a Move must not strand a sibling consumer of its
        // source) is `execution_plan`/`derive_ordering`'s job — the
        // Move is pinned after every non-destructive reader of the
        // source's alias set before any work item is emitted.
        if inputs.len() != 1 {
            return Err(
                Error::Msg(format!("Op::Move expects 1 input, got {}", inputs.len(),)).bt(),
            );
        }
        let target_backend = graph.target_backend(id).ok_or_else(|| {
            Error::Msg(format!(
                "PipelinedExecutor: Op::Move node {:?} has no target_backend \
                 set (= source backend, the one whose kernel runs the \
                 transfer)",
                id
            ))
            .bt()
        })?;
        let op_params = OpParams::None;
        let dtypes = build_lookup_dtypes(graph, node);
        let compiled =
            resolve_compiled(OpKind::Copy, &dtypes, target_backend, op_params, bindings)?;
        // Output layout: contiguous in the node's shape, same as
        // Op::Copy (the transfer materializes strided sources via
        // auto-contiguize on the input side).
        let output_layout = Layout::contiguous(node.shape.clone());
        layout_cache.insert(id, output_layout.clone());
        return Ok(WorkItem {
            node_id: id,
            inputs,
            elem_count,
            dtype: node.dtype,
            target_backend,
            kind: WorkItemKind::Move {
                target_location: target,
            },
            compiled: Some(compiled),
            output_layout,
            destructive_input,
            output_bundle: None,
        });
    }

    if matches!(node.op, Op::Reshape(_)) {
        if inputs.len() != 1 {
            return Err(
                Error::Msg(format!("Op::Reshape expects 1 input, got {}", inputs.len(),)).bt(),
            );
        }
        let input_layout = graph.layout(inputs[0]);
        // Output is contiguous in the new shape — bytes per element
        // are unchanged, so a contiguous input flows through with
        // zero copy. A non-contiguous input is auto-contiguized at
        // execute time and the result is naturally contiguous.
        let output_layout = Layout::contiguous(node.shape.clone());
        layout_cache.insert(id, output_layout.clone());
        let target_backend = graph
            .target_backend(id)
            .or_else(|| graph.target_backend(inputs[0]))
            .unwrap_or(BackendId::Cpu);
        // Sanity: same total element count.
        let in_elem_count = input_layout.shape().elem_count();
        if in_elem_count != elem_count {
            return Err(Error::Msg(format!(
                "Op::Reshape changes element count: input {} → output {}",
                in_elem_count, elem_count,
            ))
            .bt());
        }
        return Ok(WorkItem {
            node_id: id,
            inputs,
            elem_count,
            dtype: node.dtype,
            target_backend,
            kind: WorkItemKind::ContiguizeOf {
                input: node.inputs[0],
            },
            compiled: None,
            output_layout,
            destructive_input,
            output_bundle: None,
        });
    }

    if matches!(node.op, Op::Contiguize) {
        // Op::Contiguize: same execute-time path as Reshape, but
        // output shape == input shape so there's no element-count
        // check. Zero-copy when input is already contiguous +
        // zero-offset (the ContiguizeOf arm in execute_work_item
        // adopts the input Arc); otherwise auto_contiguize allocates
        // a fresh contiguous Storage on the input's backend.
        if inputs.len() != 1 {
            return Err(Error::Msg(format!(
                "Op::Contiguize expects 1 input, got {}",
                inputs.len(),
            ))
            .bt());
        }
        let output_layout = Layout::contiguous(node.shape.clone());
        layout_cache.insert(id, output_layout.clone());
        let target_backend = graph
            .target_backend(id)
            .or_else(|| graph.target_backend(inputs[0]))
            .unwrap_or(BackendId::Cpu);
        return Ok(WorkItem {
            node_id: id,
            inputs,
            elem_count,
            dtype: node.dtype,
            target_backend,
            kind: WorkItemKind::ContiguizeOf {
                input: node.inputs[0],
            },
            compiled: None,
            output_layout,
            destructive_input,
            output_bundle: None,
        });
    }

    // Runtime fused op — a JIT-adopted (Tier-2) kernel. `Op::Fused(fid, _)`
    // with `fid.is_runtime()` has no `OpKind` (the op was synthesized after
    // startup): its kernel resolves from the SAME binding table as every
    // static kernel, under `BindingKey::RuntimeFused(fid)` — the dtype tuple
    // is the key, so an absent kernel AND a dtype mismatch are one honest
    // `NoBackendForOp` miss. A deterministic *lookup*, not a decision: the
    // optimizer's gate emitted the arm only when this kernel was bound, and
    // pinned `target_backend` (`runtime-fused-op-registration.md` §6). The
    // guards are invariant checks, never live fallbacks.
    if let Op::Fused(fid, _) = &node.op
        && fid.is_runtime()
    {
        let target_backend = graph.target_backend(id).ok_or_else(|| {
            Error::Msg(format!(
                "PipelinedExecutor: runtime fused op {:?} (node {:?}) has no \
                     target_backend — the arm emitter pins it at emission",
                fid, id,
            ))
            .bt()
        })?;
        // v1: single-output only. Reject a multi-output bundle explicitly
        // rather than silently dropping it.
        if graph.output_views_arc(id).is_some() {
            return Err(Error::Msg(format!(
                "PipelinedExecutor: runtime fused op {:?} (node {:?}) carries a \
                     multi-output bundle — synthesized kernels are single-output in v1",
                fid, id,
            ))
            .bt());
        }
        let dtypes = build_lookup_dtypes(graph, node);
        let (kernel, caps) = bindings.lookup_with_caps(
            crate::kernel::BindingKey::RuntimeFused(*fid),
            &dtypes,
            target_backend,
        )?;
        let output_layout = Layout::contiguous(node.shape.clone());
        layout_cache.insert(id, output_layout.clone());
        // Deliberate, not a fallthrough to `op_to_op_params` (which doesn't
        // know runtime ops): the node's `Runtime { scalars }` — the live
        // `extract:` values, pattern pre-order — becomes `JitScalars` (the
        // launch appends them as the kernel's trailing `p{i}` args); a
        // param-free op carries `None` (its launch is layout-driven:
        // pointers + element count).
        let op_params = match &node.op {
            Op::Fused(_, FusedOpParams::Runtime { scalars }) if !scalars.is_empty() => {
                OpParams::JitScalars {
                    scalars: scalars.clone(),
                }
            }
            _ => OpParams::None,
        };
        let compiled = CompiledNode {
            op: OpKind::RuntimeFused,
            dtypes: KernelDTypes::from_slice(&dtypes),
            backend: target_backend,
            kernel,
            caps,
            op_params,
        };
        return Ok(WorkItem {
            node_id: id,
            inputs,
            elem_count,
            dtype: node.dtype,
            target_backend,
            kind: WorkItemKind::Kernel,
            compiled: Some(compiled),
            output_layout,
            destructive_input,
            output_bundle: None,
        });
    }

    let target_backend = graph.target_backend(id).ok_or_else(|| {
        Error::Msg(format!(
            "PipelinedExecutor: node {:?} ({:?}) has no target_backend set",
            id, node.op
        ))
        .bt()
    })?;

    let op_kind = op_to_op_kind(&node.op).ok_or_else(|| {
        Error::Msg(format!(
            "PipelinedExecutor: op {:?} not yet mapped to an OpKind \
             (Phase C migrates more ops as they're registered)",
            node.op,
        ))
        .bt()
    })?;

    let op_params = op_to_op_params(graph, node, id, layout_cache, sym_env)?;
    // Build the per-operand dtype list for the binding-table lookup —
    // inputs in order, then outputs. Variadic uniform-dtype ops
    // (Concat) collapse to the canonical `[T_in, T_out]` shorthand to
    // match how registrations index them.
    let dtypes = build_lookup_dtypes(graph, node);
    let compiled = resolve_compiled(op_kind, &dtypes, target_backend, op_params, bindings)?;
    let output_layout = Layout::contiguous(node.shape.clone());
    layout_cache.insert(id, output_layout.clone());
    // Multi-output: snapshot the producer's bundle metadata. When
    // `Some(_)`, the Kernel arm allocates a single Storage sized to
    // fit every slot and attaches the bundle via
    // `Storage::with_bundle` so downstream Op::View/Op::ViewOwned
    // resolve correctly.
    let output_bundle = graph.output_views_arc(id);
    Ok(WorkItem {
        node_id: id,
        inputs,
        elem_count,
        dtype: node.dtype,
        target_backend,
        kind: WorkItemKind::Kernel,
        compiled: Some(compiled),
        output_layout,
        destructive_input,
        output_bundle,
    })
}

/// Build the per-operand dtype list used as the binding-table lookup
/// key. Inputs in order, then the output. Variadic-uniform ops
/// (Concat) collapse to the canonical `[T_in, T_out]` shorthand to
/// match how those wrappers are registered (otherwise an N-way concat
/// would need N+1 distinct registrations per dtype).
///
/// `pub(crate)` — `crate::plan::compile_plan` (step 9b) reuses this
/// to key the route picker's binding-table lookup the same way the
/// pipelined path does.
pub(crate) fn build_lookup_dtypes(graph: &Graph, node: &Node) -> Vec<DType> {
    if matches!(node.op, Op::Concat { .. }) {
        // Concat: all inputs share node.dtype by construction.
        let in_dt = node
            .inputs
            .first()
            .map(|&id| graph.node(id).dtype)
            .unwrap_or(node.dtype);
        return vec![in_dt, node.dtype];
    }
    if matches!(node.op, Op::WriteSlice { .. }) {
        // WriteSlice: the destination (inputs[0]) is exposed to the
        // executor as the OUTPUT slot (in-place adoption), not as an
        // input the kernel reads. Canonicalize the binding-table key
        // to `[T_source, T_out]` so registrations match the kernel's
        // actual surface (one input slab + one output buffer).
        let src_dt = node
            .inputs
            .get(1)
            .map(|&id| graph.node(id).dtype)
            .unwrap_or(node.dtype);
        return vec![src_dt, node.dtype];
    }
    if matches!(node.op, Op::WriteSliceRotating { .. }) {
        // Same canonicalization as WriteSlice: dest = output slot;
        // position (inputs[2]) is read by the kernel via the
        // position-input parameter, not the binding-table key.
        let src_dt = node
            .inputs
            .get(1)
            .map(|&id| graph.node(id).dtype)
            .unwrap_or(node.dtype);
        return vec![src_dt, node.dtype];
    }
    if matches!(node.op, Op::WriteSliceDoff { .. }) {
        // Same canonicalization as WriteSlice: dest = output slot;
        // offset (inputs[2], I64) is a non-key runtime operand read by
        // the kernel via its device pointer, not the binding-table key.
        let src_dt = node
            .inputs
            .get(1)
            .map(|&id| graph.node(id).dtype)
            .unwrap_or(node.dtype);
        return vec![src_dt, node.dtype];
    }
    let mut dts: Vec<DType> = node.inputs.iter().map(|&id| graph.node(id).dtype).collect();
    dts.push(node.dtype);
    dts
}

/// SoftmaxLastDim is reachable only through `Op::Fused(FusedOps::
/// SOFTMAX_LAST_DIM, _)` post-step-5 (the legacy `Op::SoftmaxLastDim`
/// primitive variant was retired). This helper collapses the
/// op-to-OpKind/OpParams shapes for callers that want a single-arm
/// match.
fn op_is_softmax_last_dim(op: &Op) -> bool {
    matches!(op, Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::SOFTMAX_LAST_DIM)
}

/// FusedLinear is reachable only through `Op::Fused(FusedOps::
/// FUSED_LINEAR, _)` post-step-5.
fn op_is_fused_linear(op: &Op) -> bool {
    matches!(op, Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::FUSED_LINEAR)
}

/// Map a `fuel_graph::Op` to a `fuel_ir::dispatch::OpKind`.
/// Returns `None` for ops that haven't been wired into the new
/// dispatch path yet — Phase C extends this as op families migrate.
///
/// `pub(crate)` — `crate::plan::compile_plan` (step 9b) reuses this
/// to skip nodes the binding-table doesn't index (view ops,
/// `Op::Const`, not-yet-migrated ops) when building bindings.
pub(crate) fn op_to_op_kind(op: &Op) -> Option<OpKind> {
    match op {
        Op::Add => Some(OpKind::AddElementwise),
        Op::Sub => Some(OpKind::SubElementwise),
        Op::Mul => Some(OpKind::MulElementwise),
        Op::Div => Some(OpKind::DivElementwise),
        Op::Relu => Some(OpKind::ReluElementwise),
        Op::Neg => Some(OpKind::NegElementwise),
        Op::Sqr => Some(OpKind::SqrElementwise),
        Op::Sqrt => Some(OpKind::SqrtElementwise),
        Op::Tanh => Some(OpKind::TanhElementwise),
        Op::Exp => Some(OpKind::ExpElementwise),
        Op::Log => Some(OpKind::LogElementwise),
        Op::Sin => Some(OpKind::SinElementwise),
        Op::Cos => Some(OpKind::CosElementwise),
        Op::Sigmoid => Some(OpKind::SigmoidElementwise),
        Op::Silu => Some(OpKind::SiluElementwise),
        Op::GeluTanh => Some(OpKind::GeluTanhElementwise),
        Op::Step => Some(OpKind::StepElementwise),
        Op::Recip => Some(OpKind::RecipElementwise),
        Op::Abs => Some(OpKind::AbsElementwise),
        Op::Equal => Some(OpKind::EqualElementwise),
        Op::Ne => Some(OpKind::NotEqualElementwise),
        Op::Lt => Some(OpKind::LessElementwise),
        Op::Le => Some(OpKind::LessEqualElementwise),
        Op::Gt => Some(OpKind::GreaterElementwise),
        Op::Ge => Some(OpKind::GreaterEqualElementwise),
        Op::Where => Some(OpKind::Where),
        Op::Floor => Some(OpKind::FloorElementwise),
        Op::Ceil => Some(OpKind::CeilElementwise),
        Op::Round => Some(OpKind::RoundElementwise),
        Op::Sign => Some(OpKind::SignElementwise),
        Op::Erf => Some(OpKind::ErfElementwise),
        Op::Gelu => Some(OpKind::GeluElementwise),
        Op::Pow => Some(OpKind::PowElementwise),
        Op::Rsqrt => Some(OpKind::RsqrtElementwise),
        Op::Rem => Some(OpKind::RemElementwise),
        Op::Flip { .. } => Some(OpKind::Flip),
        Op::Roll { .. } => Some(OpKind::Roll),
        Op::CumSum { .. } => Some(OpKind::CumSum),
        Op::Triu { .. } => Some(OpKind::Triu),
        Op::Tril { .. } => Some(OpKind::Tril),
        Op::LogSoftmaxLastDim => Some(OpKind::LogSoftmaxLastDim),
        Op::LogSoftmaxLastDimBackward => Some(OpKind::LogSoftmaxLastDimBackward),
        Op::MaskedFill { .. } => Some(OpKind::MaskedFill),
        Op::Pad { .. } => Some(OpKind::Pad),
        Op::PadBackward { .. } => Some(OpKind::PadBackward),
        Op::SumDim(_) => Some(OpKind::SumReduce),
        Op::MaxDim(_) => Some(OpKind::MaxReduce),
        Op::MinDim(_) => Some(OpKind::MinReduce),
        Op::MeanDim(_) => Some(OpKind::MeanReduce),
        Op::SumAll => Some(OpKind::SumReduce),
        Op::MaxAll => Some(OpKind::MaxReduce),
        Op::MinAll => Some(OpKind::MinReduce),
        Op::MeanAll => Some(OpKind::MeanReduce),
        Op::MatMul => Some(OpKind::MatMul),
        Op::Cast(_) => Some(OpKind::Cast),
        // Phase 7.6 step 5: fused-op dispatch routes through
        // `Op::Fused(fid, _)`; the legacy `Op::Conv2D` /
        // `Op::FusedLinear` / `Op::SoftmaxLastDim` /
        // `Op::RmsNormLastDim` / `Op::LayerNormLastDim` / `Op::Rope`
        // arms were dropped together with the variants.
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::CONV2D => Some(OpKind::Conv2D),
        // Phase 7.6 step 5 (final): legacy `Op::ConvTranspose2D` /
        // `Op::FlashAttn` / `Op::PagedAttn` arms dropped with the
        // variants; dispatch flows only through `Op::Fused`.
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::CONV_TRANSPOSE2D => {
            Some(OpKind::ConvTranspose2D)
        }
        Op::ReduceSumTo(_) => Some(OpKind::ReduceSumTo),
        Op::ReduceMaxTo(_) => Some(OpKind::ReduceMaxTo),
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::FLASH_ATTN => {
            Some(OpKind::FlashAttn)
        }
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::PAGED_ATTN => {
            Some(OpKind::PagedAttn)
        }
        Op::AddScalar(_) => Some(OpKind::Affine),
        Op::MulScalar(_) => Some(OpKind::Affine),
        Op::Clamp { .. } => Some(OpKind::ClampElementwise),
        Op::PowI(_) => Some(OpKind::PowIElementwise),
        Op::Maximum => Some(OpKind::MaximumElementwise),
        Op::Minimum => Some(OpKind::MinimumElementwise),
        Op::Concat { .. } => Some(OpKind::Concat),
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::SOFTMAX_LAST_DIM => {
            Some(OpKind::SoftmaxLastDim)
        }
        // Phase 7.6 step 6 follow-up: backward helpers now route
        // through the byte-level binding table too.
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::SOFTMAX_LAST_DIM_BACKWARD => {
            Some(OpKind::SoftmaxLastDimBackward)
        }
        Op::Fused(fid, _)
            if *fid == fuel_graph::registry::FusedOps::LAYER_NORM_LAST_DIM_BACKWARD =>
        {
            Some(OpKind::LayerNormLastDimBackward)
        }
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::RMS_NORM_LAST_DIM_BACKWARD => {
            Some(OpKind::RmsNormLastDimBackward)
        }
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::REDUCE_MAX_TO_BACKWARD => {
            Some(OpKind::ReduceMaxToBackward)
        }
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::POWI_BACKWARD => {
            Some(OpKind::PowIElementwiseBackward)
        }
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::FUSED_LINEAR => {
            Some(OpKind::FusedLinear)
        }
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::RMS_NORM_LAST_DIM => {
            Some(OpKind::RmsNormLastDim)
        }
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::LAYER_NORM_LAST_DIM => {
            Some(OpKind::LayerNormLastDim)
        }
        Op::IndexSelect { .. } => Some(OpKind::IndexSelect),
        Op::Gather { .. } => Some(OpKind::Gather),
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::ROPE => Some(OpKind::Rope),
        Op::IndexAdd { .. } => Some(OpKind::IndexAdd),
        Op::ScatterAdd { .. } => Some(OpKind::ScatterAdd),
        Op::NonZeroIndices { .. } => Some(OpKind::NonZeroIndices),
        Op::ArgMaxDim(_) => Some(OpKind::ArgMaxDim),
        Op::ArgMinDim(_) => Some(OpKind::ArgMinDim),
        // Phase 7.6 step 5 (final): legacy `Op::QMatMul` arm dropped.
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::QMATMUL => {
            Some(OpKind::QMatMul)
        }
        Op::WriteSlice { .. } => Some(OpKind::WriteSlice),
        Op::WriteSliceRotating { .. } => Some(OpKind::WriteSliceRotating),
        Op::WriteSliceDoff { .. } => Some(OpKind::WriteSliceDoff),
        // Phase 2 of bridge-retirement (post-9c): Op::Copy dispatches
        // through the binding table at OpKind::Copy. The BackendId
        // axis encodes the source backend (the kernel runs there —
        // download from local memory). The executor handles output
        // allocation on the target via a dedicated WorkItemKind::Copy
        // arm, since target_location differs from target_backend for
        // this op (the only one with that property today).
        Op::Copy { .. } => Some(OpKind::Copy),
        // Op::Move dispatches the same data-movement kernel as
        // Op::Copy — the destructive release of the source is
        // realize-loop bookkeeping (`destructive_input` cleanup), not
        // a kernel. Same OpKind, same source-backend key convention.
        // `compile_plan` skips both (residency-determined, not a
        // picker decision).
        Op::Move { .. } => Some(OpKind::Copy),
        // Phase 3 of the in-place ops infrastructure: each in-place Op
        // variant maps to its OpKind so the binding-table dispatch can
        // resolve a kernel. The executor's dedicated WorkItemKind
        // arms (added alongside) handle the storage-Arc adoption.
        Op::ReluInplace => Some(OpKind::ReluInplace),
        Op::SiluInplace => Some(OpKind::SiluInplace),
        Op::GeluTanhInplace => Some(OpKind::GeluTanhInplace),
        Op::TanhInplace => Some(OpKind::TanhInplace),
        Op::SigmoidInplace => Some(OpKind::SigmoidInplace),
        Op::NegInplace => Some(OpKind::NegInplace),
        Op::AbsInplace => Some(OpKind::AbsInplace),
        Op::SqrInplace => Some(OpKind::SqrInplace),
        Op::SqrtInplace => Some(OpKind::SqrtInplace),
        Op::RsqrtInplace => Some(OpKind::RsqrtInplace),
        Op::RecipInplace => Some(OpKind::RecipInplace),
        Op::ExpInplace => Some(OpKind::ExpInplace),
        Op::LogInplace => Some(OpKind::LogInplace),
        Op::SinInplace => Some(OpKind::SinInplace),
        Op::CosInplace => Some(OpKind::CosInplace),
        Op::SignInplace => Some(OpKind::SignInplace),
        Op::FloorInplace => Some(OpKind::FloorInplace),
        Op::CeilInplace => Some(OpKind::CeilInplace),
        Op::RoundInplace => Some(OpKind::RoundInplace),
        Op::ErfInplace => Some(OpKind::ErfInplace),
        Op::GeluInplace => Some(OpKind::GeluInplace),
        Op::ClampInplace { .. } => Some(OpKind::ClampInplace),
        Op::PowIInplace(_) => Some(OpKind::PowIInplace),
        // In-place binary variants (Add/Sub/Mul/Div/MaskedFill) and
        // their OpKind siblings are tracked by a parallel session and
        // toggle in and out of the Op enum as that work progresses.
        // No multi-output arm references them; nothing to map here in
        // the current HEAD.
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::INPLACE_AFFINE => {
            Some(OpKind::InplaceAffine)
        }
        Op::Fused(fid, _)
            if *fid == fuel_graph::registry::FusedOps::FUSED_SOFTMAX_CROSS_ENTROPY =>
        {
            Some(OpKind::FusedSoftmaxCrossEntropy)
        }
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::CAUSAL_CONV1D => {
            Some(OpKind::CausalConv1d)
        }
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::SELECTIVE_SCAN => {
            Some(OpKind::SelectiveScan)
        }
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::SSD_CHUNK_SCAN => {
            Some(OpKind::SsdChunkScan)
        }
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::NF4_MATMUL => {
            Some(OpKind::Nf4Matmul)
        }
        // Phase 3a (post-9c): Op::Alloc + Op::ZeroFill are structural
        // ops dispatched directly via `WorkItemKind::Alloc` /
        // `WorkItemKind::ZeroFill` (no binding-table lookup).
        // `compile_plan` skips these.
        Op::Alloc { .. } | Op::ZeroFill => None,
        _ => None,
    }
}

/// Build the [`OpParams`] for `node`'s op. Most ops use
/// `OpParams::None`; reductions / matmul / conv / slice carry their
/// op-specific extras here. The graph is consulted to read input
/// shapes (e.g. reductions need the input shape to walk the
/// multi-index — Storage only carries bytes + dtype).
///
/// Phase C — extends as op families migrate. Returns Err if a
/// graph-shape lookup fails (currently can't, but the signature is
/// `Result` so future cases needing validation slot in cleanly).
/// Encode an `f64` value into the byte pattern of `dtype`. Used by
/// `Op::Pad` (Constant mode) to pre-convert the fill value once at
/// op_params time, so the kernel itself stays dtype-agnostic.
fn encode_value_to_bytes(dtype: DType, value: f64) -> Result<Vec<u8>> {
    match dtype {
        DType::F32 => Ok((value as f32).to_le_bytes().to_vec()),
        DType::F64 => Ok(value.to_le_bytes().to_vec()),
        DType::BF16 => Ok(half::bf16::from_f32(value as f32).to_le_bytes().to_vec()),
        DType::F16 => Ok(half::f16::from_f32(value as f32).to_le_bytes().to_vec()),
        DType::U8 => Ok(vec![value as u8]),
        DType::U32 => Ok((value as u32).to_le_bytes().to_vec()),
        other => Err(Error::Msg(format!(
            "encode_value_to_bytes: dtype {other:?} not yet supported for Pad fill",
        ))
        .bt()),
    }
}

/// Encode a typed `Scalar` to its little-endian byte representation.
/// Used by Op::MaskedFill so the kernel only sees bytes — never has
/// to know the value's dtype.
fn scalar_to_bytes(s: fuel_ir::Scalar) -> Vec<u8> {
    use fuel_ir::Scalar;
    match s {
        Scalar::U8(v) => vec![v],
        Scalar::I8(v) => vec![v as u8],
        Scalar::U32(v) => v.to_le_bytes().to_vec(),
        Scalar::I16(v) => v.to_le_bytes().to_vec(),
        Scalar::I32(v) => v.to_le_bytes().to_vec(),
        Scalar::I64(v) => v.to_le_bytes().to_vec(),
        Scalar::BF16(v) => v.to_le_bytes().to_vec(),
        Scalar::F16(v) => v.to_le_bytes().to_vec(),
        Scalar::F32(v) => v.to_le_bytes().to_vec(),
        Scalar::F64(v) => v.to_le_bytes().to_vec(),
        Scalar::Bool(b) => vec![b as u8],
        Scalar::F8E4M3(v) => vec![v.to_bits()],
        Scalar::F8E5M2(v) => vec![v.to_bits()],
        // OCP-MX E8M0 block scale is a raw byte on the wire (`2^(X-127)`,
        // decoded kernel-side); a single byte is its own LE encoding.
        Scalar::F8E8M0(v) => vec![v],
    }
}

fn op_to_op_params(
    graph: &Graph,
    node: &Node,
    node_id: NodeId,
    layout_cache: &HashMap<NodeId, Layout>,
    sym_env: &SymEnv,
) -> Result<OpParams> {
    // Helper: read an input's layout from the compiler-thread-local
    // cache (which is current within the realize call), falling back
    // to the graph's side-table if the input wasn't visited (which
    // shouldn't happen in topo order, but the fallback keeps the
    // path safe).
    let input_layout = |input_id: NodeId| -> Layout {
        layout_cache
            .get(&input_id)
            .cloned()
            .unwrap_or_else(|| graph.layout(input_id))
    };
    Ok(match &node.op {
        // Phase 7.6 step 5 (final): legacy `Op::QMatMul` arm dropped.
        Op::Fused(_, fuel_graph::registry::FusedOpParams::QMatMul { quant_type, k, n }) => {
            // Inputs: (activations f32 [..., m, k], weight_bytes
            // u32-typed). Output shape (this Node's shape) is
            // [..., m, n]. Flatten leading dims into batch_count.
            if node.inputs.len() != 2 {
                return Err(Error::Msg(format!(
                    "Op::QMatMul expects 2 inputs (activations, weight_bytes), got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let act_layout = input_layout(node.inputs[0]);
            let act_dims = act_layout.shape().dims();
            if act_dims.len() < 2 {
                return Err(Error::Msg(format!(
                    "Op::QMatMul: activations must be rank ≥ 2, got {act_dims:?}",
                ))
                .bt());
            }
            let m = act_dims[act_dims.len() - 2];
            let k_act = act_dims[act_dims.len() - 1];
            if k_act != *k {
                return Err(Error::Msg(format!(
                    "Op::QMatMul: activation last dim ({k_act}) must equal Op's k ({k})",
                ))
                .bt());
            }
            let batch_count: usize = act_dims[..act_dims.len() - 2].iter().product();
            OpParams::QMatMul {
                quant_type: *quant_type,
                batch_count,
                m,
                n: *n,
                k: *k,
            }
        }
        Op::ArgMaxDim(d) | Op::ArgMinDim(d) => {
            // Reuse OpParams::Reduce — same shape contract; the
            // single reduce dim is the argmax/argmin axis. Input
            // layout flows through KernelRef's `layouts[0]`.
            OpParams::Reduce {
                dims: vec![*d],
                keepdim: false,
            }
        }
        Op::SumDim(d) | Op::MaxDim(d) | Op::MinDim(d) | Op::MeanDim(d) => OpParams::Reduce {
            dims: vec![*d],
            keepdim: false,
        },
        Op::SumAll | Op::MaxAll | Op::MinAll | Op::MeanAll => {
            let il = input_layout(node.inputs[0]);
            let rank = il.shape().rank();
            OpParams::Reduce {
                dims: (0..rank).collect(),
                keepdim: false,
            }
        }
        Op::MatMul => {
            // Batched matmul: lhs `[..lhs_batch.., m, k]` @
            // rhs `[..rhs_batch.., k, n]` → out `[..lhs_batch.., m, n]`.
            // Per-axis the batch dims either match or follow GQA-style
            // divisibility (lhs_dim > rhs_dim && lhs_dim % rhs_dim == 0);
            // the kernel honors the latter via `rhs_idx = lhs_idx / n_rep`.
            if node.inputs.len() != 2 {
                return Err(Error::Msg(format!(
                    "Op::MatMul expects 2 inputs, got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let lhs = input_layout(node.inputs[0]);
            let rhs = input_layout(node.inputs[1]);
            let lhs_dims = lhs.shape().dims();
            let rhs_dims = rhs.shape().dims();
            if lhs_dims.len() < 2 || rhs_dims.len() < 2 {
                return Err(Error::Msg(format!(
                    "Op::MatMul requires both inputs rank ≥ 2; got lhs={:?} rhs={:?}",
                    lhs_dims, rhs_dims,
                ))
                .bt());
            }
            if lhs_dims.len() != rhs_dims.len() {
                return Err(Error::Msg(format!(
                    "Op::MatMul: ranks must match (auto-broadcast happens at \
                     graph construction time); got lhs rank {} vs rhs rank {}",
                    lhs_dims.len(),
                    rhs_dims.len(),
                ))
                .bt());
            }
            let rank = lhs_dims.len();
            let batch_rank = rank - 2;
            // Per-axis validation: equal or GQA-divisible.
            for i in 0..batch_rank {
                let la = lhs_dims[i];
                let ra = rhs_dims[i];
                let ok = la == ra || (ra > 0 && la > ra && la % ra == 0);
                if !ok {
                    return Err(Error::Msg(format!(
                        "Op::MatMul: batch dim {i} disallowed combination \
                         (lhs={la}, rhs={ra}); must be equal or \
                         GQA-divisible (lhs > rhs && lhs % rhs == 0)",
                    ))
                    .bt());
                }
            }
            let lhs_batch_dims: Vec<usize> = lhs_dims[..batch_rank].to_vec();
            let rhs_batch_dims: Vec<usize> = rhs_dims[..batch_rank].to_vec();
            let (m, k_lhs) = (lhs_dims[rank - 2], lhs_dims[rank - 1]);
            let (k_rhs, n) = (rhs_dims[rank - 2], rhs_dims[rank - 1]);
            if k_lhs != k_rhs {
                return Err(Error::Msg(format!(
                    "Op::MatMul: contracting dims disagree — lhs trailing is \
                     [{m}, {k_lhs}], rhs trailing is [{k_rhs}, {n}]",
                ))
                .bt());
            }
            // Data-determined-M (sparse-MoE): a side-table row count makes
            // `m` the capacity and this the compute count. Dual-path,
            // mirroring WriteSlice: resolve in the compile-time SymEnv
            // (static / input-determined) now; else DEFER a producer-bound
            // sym to execute; else it is a genuine caller error.
            let m_compute = match graph.matmul_row_count(node_id) {
                None => MatmulM::All,
                Some(dyn_m) => match dyn_m.resolve(sym_env) {
                    Some(c) => {
                        if c > m {
                            return Err(Error::Msg(format!(
                                "Op::MatMul: resolved row count {c} exceeds capacity {m}",
                            ))
                            .bt());
                        }
                        MatmulM::Rows(c)
                    }
                    None => match dyn_m {
                        fuel_ir::DynScalar::Sym(s) if graph.binds_data_determined_sym(s) => {
                            MatmulM::Deferred(dyn_m)
                        }
                        _ => {
                            return Err(Error::Msg(format!(
                                "Op::MatMul: dynamic row count {dyn_m:?} is unbound in the \
                                 SymEnv at compile and no producer binds it (not \
                                 data-determined)",
                            ))
                            .bt());
                        }
                    },
                },
            };
            OpParams::Matmul {
                lhs_batch_dims,
                rhs_batch_dims,
                m,
                n,
                k: k_lhs,
                m_compute,
            }
        }
        // Phase 7.6 step 4: legacy `Op::FusedLinear` and registry-extended
        // `Op::Fused(FUSED_LINEAR, _)` share the same shape contract and
        // params encoding (reuses OpParams::Matmul). One body covers both.
        op if op_is_fused_linear(op) => {
            // Inputs: [a, b, bias]. Same shape semantics as MatMul on
            // a/b; bias is rank-1 [N] and broadcasts along all leading
            // dims. We reuse OpParams::Matmul (kernel reads bias from
            // inputs[2] directly).
            if node.inputs.len() != 3 {
                return Err(Error::Msg(format!(
                    "Op::FusedLinear expects 3 inputs (a, b, bias), got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let lhs = input_layout(node.inputs[0]);
            let rhs = input_layout(node.inputs[1]);
            let bias = input_layout(node.inputs[2]);
            let lhs_dims = lhs.shape().dims();
            let rhs_dims = rhs.shape().dims();
            let bias_dims = bias.shape().dims();
            if lhs_dims.len() < 2 || rhs_dims.len() < 2 {
                return Err(Error::Msg(format!(
                    "Op::FusedLinear requires both a, b rank ≥ 2; got a={:?} b={:?}",
                    lhs_dims, rhs_dims,
                ))
                .bt());
            }
            if lhs_dims.len() != rhs_dims.len() {
                return Err(Error::Msg(format!(
                    "Op::FusedLinear: ranks must match (auto-broadcast happens at \
                     graph construction time); got a rank {} vs b rank {}",
                    lhs_dims.len(),
                    rhs_dims.len(),
                ))
                .bt());
            }
            let rank = lhs_dims.len();
            let batch_rank = rank - 2;
            for i in 0..batch_rank {
                let la = lhs_dims[i];
                let ra = rhs_dims[i];
                let ok = la == ra || (ra > 0 && la > ra && la % ra == 0);
                if !ok {
                    return Err(Error::Msg(format!(
                        "Op::FusedLinear: batch dim {i} disallowed (a={la}, b={ra})",
                    ))
                    .bt());
                }
            }
            let lhs_batch_dims: Vec<usize> = lhs_dims[..batch_rank].to_vec();
            let rhs_batch_dims: Vec<usize> = rhs_dims[..batch_rank].to_vec();
            let (m, k_lhs) = (lhs_dims[rank - 2], lhs_dims[rank - 1]);
            let (k_rhs, n) = (rhs_dims[rank - 2], rhs_dims[rank - 1]);
            if k_lhs != k_rhs {
                return Err(Error::Msg(format!(
                    "Op::FusedLinear: contracting dims disagree — a trailing is \
                     [{m}, {k_lhs}], b trailing is [{k_rhs}, {n}]",
                ))
                .bt());
            }
            if bias_dims.len() != 1 || bias_dims[0] != n {
                return Err(Error::Msg(format!(
                    "Op::FusedLinear: bias must be rank-1 [{n}], got {bias_dims:?}",
                ))
                .bt());
            }
            OpParams::Matmul {
                lhs_batch_dims,
                rhs_batch_dims,
                m,
                n,
                k: k_lhs,
                // FusedLinear is always static-M (dense linear layer).
                m_compute: MatmulM::All,
            }
        }
        // FusedSoftmaxCrossEntropy: flatten logits `[..., V]` to
        // `[n_rows, V]` for the kernel. n_rows is the product of all
        // leading dims (≥1 even for rank-1 logits, where targets are
        // a scalar). The kernel walks rows internally.
        Op::Fused(
            _,
            fuel_graph::registry::FusedOpParams::FusedSoftmaxCrossEntropy {
                reduction,
                ignore_index,
            },
        ) => {
            if node.inputs.len() != 2 {
                return Err(Error::Msg(format!(
                    "FusedSoftmaxCrossEntropy expects 2 inputs (logits, targets), got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let logits_layout = input_layout(node.inputs[0]);
            let logits_dims = logits_layout.shape().dims();
            if logits_dims.is_empty() {
                return Err(Error::Msg(
                    "FusedSoftmaxCrossEntropy: logits must have rank ≥ 1".to_string(),
                )
                .bt());
            }
            let vocab = *logits_dims.last().unwrap();
            let n_rows: usize = logits_dims[..logits_dims.len() - 1]
                .iter()
                .product::<usize>()
                .max(1);
            let targets_layout = input_layout(node.inputs[1]);
            let target_count: usize = targets_layout
                .shape()
                .dims()
                .iter()
                .product::<usize>()
                .max(1);
            if target_count != n_rows {
                return Err(Error::Msg(format!(
                    "FusedSoftmaxCrossEntropy: targets element count {target_count} must \
                     equal logits row count {n_rows} (logits {logits_dims:?}, targets {:?})",
                    targets_layout.shape().dims(),
                ))
                .bt());
            }
            OpParams::FusedSoftmaxCrossEntropy {
                n_rows,
                vocab,
                reduction: *reduction,
                ignore_index: *ignore_index,
            }
        }
        // CausalConv1d: derive (batch, channels, seq_in, seq_out, kernel)
        // from the input layouts. x is `[batch, channels, seq_in]`
        // (caller pre-pads with kernel-1 zeros), weight is
        // `[channels, 1, kernel]`. seq_out = seq_in - (kernel - 1).
        Op::Fused(_, fuel_graph::registry::FusedOpParams::CausalConv1d { use_silu }) => {
            if node.inputs.len() != 3 {
                return Err(Error::Msg(format!(
                    "CausalConv1d expects 3 inputs (x, weight, bias), got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let x_layout = input_layout(node.inputs[0]);
            let w_layout = input_layout(node.inputs[1]);
            let x_dims = x_layout.shape().dims();
            let w_dims = w_layout.shape().dims();
            if x_dims.len() != 3 {
                return Err(Error::Msg(format!(
                    "CausalConv1d: x must be rank 3 [batch, channels, seq+pad], got {x_dims:?}",
                ))
                .bt());
            }
            if w_dims.len() != 3 {
                return Err(Error::Msg(format!(
                    "CausalConv1d: weight must be rank 3 [channels, 1, kernel], got {w_dims:?}",
                ))
                .bt());
            }
            let batch = x_dims[0];
            let channels = x_dims[1];
            let seq_in = x_dims[2];
            let kernel = w_dims[2];
            if seq_in < kernel - 1 {
                return Err(Error::Msg(format!(
                    "CausalConv1d: x time dim {seq_in} must be ≥ kernel-1 = {} \
                     (caller must pre-pad with kernel-1 zeros)",
                    kernel - 1,
                ))
                .bt());
            }
            let seq_out = seq_in - (kernel - 1);
            OpParams::CausalConv1d {
                batch,
                channels,
                seq_in,
                seq_out,
                kernel,
                use_silu: *use_silu,
            }
        }
        // SelectiveScan: derive (batch, seqlen, dim, dstate) from the
        // input layouts. u: [batch, seqlen, dim]; a: [dim, dstate].
        Op::Fused(_, fuel_graph::registry::FusedOpParams::SelectiveScan { delta_softplus }) => {
            if node.inputs.len() != 5 {
                return Err(Error::Msg(format!(
                    "SelectiveScan expects 5 inputs (u, delta, a, b, c), got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let u_layout = input_layout(node.inputs[0]);
            let a_layout = input_layout(node.inputs[2]);
            let u_dims = u_layout.shape().dims();
            let a_dims = a_layout.shape().dims();
            if u_dims.len() != 3 {
                return Err(Error::Msg(format!(
                    "SelectiveScan: u must be rank 3 [batch, seqlen, dim], got {u_dims:?}",
                ))
                .bt());
            }
            if a_dims.len() != 2 {
                return Err(Error::Msg(format!(
                    "SelectiveScan: a must be rank 2 [dim, dstate], got {a_dims:?}",
                ))
                .bt());
            }
            let batch = u_dims[0];
            let seqlen = u_dims[1];
            let dim = u_dims[2];
            let dstate = a_dims[1];
            if a_dims[0] != dim {
                return Err(Error::Msg(format!(
                    "SelectiveScan: a's first dim {} must equal dim {dim}",
                    a_dims[0],
                ))
                .bt());
            }
            OpParams::SelectiveScan {
                batch,
                seqlen,
                dim,
                dstate,
                delta_softplus: *delta_softplus,
            }
        }
        // SsdChunkScan: derive (batch, seqlen, heads, head_dim,
        // state_dim) from x and b layouts. x: [batch, seqlen, heads,
        // head_dim]; b: [batch, seqlen, heads, state_dim].
        Op::Fused(_, fuel_graph::registry::FusedOpParams::SsdChunkScan { chunk_size }) => {
            if node.inputs.len() != 5 {
                return Err(Error::Msg(format!(
                    "SsdChunkScan expects 5 inputs (x, dt, a, b, c), got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let x_layout = input_layout(node.inputs[0]);
            let b_layout = input_layout(node.inputs[3]);
            let x_dims = x_layout.shape().dims();
            let b_dims = b_layout.shape().dims();
            if x_dims.len() != 4 {
                return Err(Error::Msg(format!(
                    "SsdChunkScan: x must be rank 4 [batch, seqlen, heads, head_dim], got {x_dims:?}",
                ))
                .bt());
            }
            if b_dims.len() != 4 {
                return Err(Error::Msg(format!(
                    "SsdChunkScan: b must be rank 4 [batch, seqlen, heads, state_dim], got {b_dims:?}",
                ))
                .bt());
            }
            let batch = x_dims[0];
            let seqlen = x_dims[1];
            let heads = x_dims[2];
            let head_dim = x_dims[3];
            let state_dim = b_dims[3];
            OpParams::SsdChunkScan {
                batch,
                seqlen,
                heads,
                head_dim,
                state_dim,
                chunk_size: *chunk_size,
            }
        }
        // Nf4Matmul: derive (batch, m, n, k) from activations + w_packed
        // layouts. activations: [..., m, k] (leading dims flatten into
        // batch); w_packed: [n, k/2] U8.
        Op::Fused(_, fuel_graph::registry::FusedOpParams::Nf4Matmul { block_size }) => {
            if node.inputs.len() != 3 {
                return Err(Error::Msg(format!(
                    "Nf4Matmul expects 3 inputs (activations, w_packed, absmax), got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let a_layout = input_layout(node.inputs[0]);
            let w_layout = input_layout(node.inputs[1]);
            let a_dims = a_layout.shape().dims();
            let w_dims = w_layout.shape().dims();
            if a_dims.len() < 2 {
                return Err(Error::Msg(format!(
                    "Nf4Matmul: activations must be rank ≥ 2, got {a_dims:?}",
                ))
                .bt());
            }
            if w_dims.len() != 2 {
                return Err(Error::Msg(format!(
                    "Nf4Matmul: w_packed must be rank 2 [n, k/2], got {w_dims:?}",
                ))
                .bt());
            }
            let m = a_dims[a_dims.len() - 2];
            let k = a_dims[a_dims.len() - 1];
            let n = w_dims[0];
            let batch: usize = a_dims[..a_dims.len() - 2].iter().product::<usize>().max(1);
            if k % 2 != 0 {
                return Err(Error::Msg(format!(
                    "Nf4Matmul: k={k} must be even (w_packed holds 2 nibbles per byte along k)",
                ))
                .bt());
            }
            if k % *block_size != 0 {
                return Err(Error::Msg(format!(
                    "Nf4Matmul: k={k} must be a multiple of block_size={block_size}",
                ))
                .bt());
            }
            if w_dims[1] != k / 2 {
                return Err(Error::Msg(format!(
                    "Nf4Matmul: w_packed second dim {} must equal k/2 = {}",
                    w_dims[1],
                    k / 2,
                ))
                .bt());
            }
            OpParams::Nf4Matmul {
                batch,
                m,
                n,
                k,
                block_size: *block_size,
            }
        }
        // Phase 7.6 step 3: SoftmaxLastDim flows through
        // `Op::Fused(FusedOps::SOFTMAX_LAST_DIM, _)`. The
        // SOFTMAX_LAST_DIM_BACKWARD variant added in step 6
        // follow-up shares the same `outer × last_dim` shape
        // contract (input 0 is `y` for forward / forward-output
        // for backward; both rank-≥1 with last-dim as the
        // reduction axis).
        op if op_is_softmax_last_dim(op)
            || matches!(op, Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::SOFTMAX_LAST_DIM_BACKWARD) =>
        {
            let il = input_layout(node.inputs[0]);
            let dims = il.shape().dims();
            if dims.is_empty() {
                return Err(Error::Msg("Op::SoftmaxLastDim requires rank ≥ 1".to_string()).bt());
            }
            let last_dim = *dims.last().unwrap();
            let outer_count: usize = dims[..dims.len() - 1].iter().product();
            OpParams::SoftmaxLastDim {
                outer_count,
                last_dim,
            }
        }
        Op::Flip { dim } => {
            // Single input. Precompute the flat-3-axis split
            // (outer × dim × inner) from the input shape.
            if node.inputs.len() != 1 {
                return Err(Error::Msg(format!(
                    "Op::Flip expects 1 input, got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let in_layout = input_layout(node.inputs[0]);
            let in_dims = in_layout.shape().dims();
            if *dim >= in_dims.len() {
                return Err(Error::Msg(format!(
                    "Op::Flip: dim {dim} out of range for rank {}",
                    in_dims.len(),
                ))
                .bt());
            }
            let outer_count: usize = in_dims[..*dim].iter().product();
            let dim_size = in_dims[*dim];
            let inner_count: usize = in_dims[*dim + 1..].iter().product();
            OpParams::Flip {
                outer_count,
                dim_size,
                inner_count,
                axis: *dim,
            }
        }
        Op::Roll { dim, shift } => {
            if node.inputs.len() != 1 {
                return Err(Error::Msg(format!(
                    "Op::Roll expects 1 input, got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let in_layout = input_layout(node.inputs[0]);
            let in_dims = in_layout.shape().dims();
            if *dim >= in_dims.len() {
                return Err(Error::Msg(format!(
                    "Op::Roll: dim {dim} out of range for rank {}",
                    in_dims.len(),
                ))
                .bt());
            }
            let outer_count: usize = in_dims[..*dim].iter().product();
            let dim_size = in_dims[*dim];
            let inner_count: usize = in_dims[*dim + 1..].iter().product();
            OpParams::Roll {
                outer_count,
                dim_size,
                inner_count,
                shift: *shift,
                axis: *dim,
            }
        }
        Op::CumSum { dim } => {
            if node.inputs.len() != 1 {
                return Err(Error::Msg(format!(
                    "Op::CumSum expects 1 input, got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let in_layout = input_layout(node.inputs[0]);
            let in_dims = in_layout.shape().dims();
            if *dim >= in_dims.len() {
                return Err(Error::Msg(format!(
                    "Op::CumSum: dim {dim} out of range for rank {}",
                    in_dims.len(),
                ))
                .bt());
            }
            let outer_count: usize = in_dims[..*dim].iter().product();
            let dim_size = in_dims[*dim];
            let inner_count: usize = in_dims[*dim + 1..].iter().product();
            OpParams::CumSum {
                outer_count,
                dim_size,
                inner_count,
                axis: *dim,
            }
        }
        Op::Triu { diagonal } | Op::Tril { diagonal } => {
            if node.inputs.len() != 1 {
                return Err(Error::Msg(format!(
                    "Op::Triu/Tril expects 1 input, got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let in_layout = input_layout(node.inputs[0]);
            let in_dims = in_layout.shape().dims();
            if in_dims.len() < 2 {
                return Err(Error::Msg(format!(
                    "Op::Triu/Tril requires rank >= 2, got {}",
                    in_dims.len(),
                ))
                .bt());
            }
            let rows = in_dims[in_dims.len() - 2];
            let cols = in_dims[in_dims.len() - 1];
            let batch_count: usize = in_dims[..in_dims.len() - 2]
                .iter()
                .product::<usize>()
                .max(1);
            OpParams::Triangular {
                batch_count,
                rows,
                cols,
                diagonal: *diagonal,
            }
        }
        Op::LogSoftmaxLastDim => {
            if node.inputs.len() != 1 {
                return Err(Error::Msg(format!(
                    "Op::LogSoftmaxLastDim expects 1 input, got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let in_layout = input_layout(node.inputs[0]);
            let in_dims = in_layout.shape().dims();
            if in_dims.is_empty() {
                return Err(Error::Msg("Op::LogSoftmaxLastDim requires rank ≥ 1".to_string()).bt());
            }
            let last_dim = *in_dims.last().unwrap();
            let outer_count: usize = in_dims[..in_dims.len() - 1].iter().product();
            OpParams::LogSoftmaxLastDim {
                outer_count,
                last_dim,
            }
        }
        Op::LogSoftmaxLastDimBackward => {
            // Inputs: (forward_output_y, upstream_grad). Same shape contract
            // as SoftmaxLastDimBackward — last-dim row-wise dispatch.
            if node.inputs.len() != 2 {
                return Err(Error::Msg(format!(
                    "Op::LogSoftmaxLastDimBackward expects 2 inputs, got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let in_layout = input_layout(node.inputs[0]);
            let in_dims = in_layout.shape().dims();
            if in_dims.is_empty() {
                return Err(Error::Msg(
                    "Op::LogSoftmaxLastDimBackward requires rank ≥ 1".to_string(),
                )
                .bt());
            }
            let last_dim = *in_dims.last().unwrap();
            let outer_count: usize = in_dims[..in_dims.len() - 1].iter().product();
            OpParams::LogSoftmaxLastDim {
                outer_count,
                last_dim,
            }
        }
        Op::MaskedFill { value } => {
            if node.inputs.len() != 2 {
                return Err(Error::Msg(format!(
                    "Op::MaskedFill expects 2 inputs (x, mask), got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let fill_bytes = scalar_to_bytes(*value);
            OpParams::MaskedFill { fill_bytes }
        }
        // Op::MaskedFillInplace tracked by the parallel in-place
        // binary family session — not in HEAD's Op enum right now.
        Op::PadBackward {
            in_shape,
            padding,
            mode,
        } => {
            // Single input is the upstream gradient; output is the
            // input gradient (shape == `in_shape`).
            if node.inputs.len() != 1 {
                return Err(Error::Msg(format!(
                    "Op::PadBackward expects 1 input, got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let in_dims: Vec<usize> = in_shape.dims().to_vec();
            // Output of the original forward Pad is the input shape
            // of the backward (== upstream's shape).
            let out_dims: Vec<usize> = in_dims
                .iter()
                .zip(padding.iter())
                .map(|(&d, &(b, a))| d + b + a)
                .collect();
            let mode_tag: u8 = match mode {
                fuel_graph::PadMode::Constant => 0,
                fuel_graph::PadMode::Reflect => 1,
                fuel_graph::PadMode::Replicate => 2,
            };
            OpParams::PadBackward {
                in_shape: in_dims,
                out_shape: out_dims,
                padding: padding.clone(),
                mode_tag,
            }
        }
        Op::Pad {
            padding,
            mode,
            value,
        } => {
            if node.inputs.len() != 1 {
                return Err(Error::Msg(format!(
                    "Op::Pad expects 1 input, got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let in_layout = input_layout(node.inputs[0]);
            let in_dims: Vec<usize> = in_layout.shape().dims().to_vec();
            if padding.len() != in_dims.len() {
                return Err(Error::Msg(format!(
                    "Op::Pad: padding.len() ({}) != input rank ({})",
                    padding.len(),
                    in_dims.len(),
                ))
                .bt());
            }
            let out_dims: Vec<usize> = in_dims
                .iter()
                .zip(padding.iter())
                .map(|(&d, &(b, a))| d + b + a)
                .collect();
            let mode_tag: u8 = match mode {
                fuel_graph::PadMode::Constant => 0,
                fuel_graph::PadMode::Reflect => 1,
                fuel_graph::PadMode::Replicate => 2,
            };
            // Encode fill value as bytes for the output dtype. The
            // kernel is dtype-agnostic — it just memcopies the
            // pattern. Conversion happens once here per node, not
            // per element in the kernel.
            let fill_bytes = encode_value_to_bytes(node.dtype, *value)?;
            OpParams::Pad {
                in_shape: in_dims,
                out_shape: out_dims,
                padding: padding.clone(),
                mode_tag,
                fill_bytes,
            }
        }
        Op::IndexAdd { dim } => {
            // Inputs: (base, indices, src). All same dtype except
            // indices is U32. Output shape == base shape.
            if node.inputs.len() != 3 {
                return Err(Error::Msg(format!(
                    "Op::IndexAdd expects 3 inputs (base, indices, src), got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let base_layout = input_layout(node.inputs[0]);
            let idx_layout = input_layout(node.inputs[1]);
            let src_layout = input_layout(node.inputs[2]);
            let base_dims = base_layout.shape().dims();
            let idx_dims = idx_layout.shape().dims();
            let src_dims = src_layout.shape().dims();
            if *dim >= base_dims.len() {
                return Err(Error::Msg(format!(
                    "Op::IndexAdd: dim {dim} out of range for base rank {}",
                    base_dims.len(),
                ))
                .bt());
            }
            if idx_dims.len() != 1 {
                return Err(Error::Msg(format!(
                    "Op::IndexAdd: indices must be rank 1, got {idx_dims:?}",
                ))
                .bt());
            }
            if base_dims.len() != src_dims.len() {
                return Err(Error::Msg(format!(
                    "Op::IndexAdd: base rank ({}) != src rank ({})",
                    base_dims.len(),
                    src_dims.len(),
                ))
                .bt());
            }
            let outer_count: usize = base_dims[..*dim].iter().product();
            let base_dim_size = base_dims[*dim];
            let inner_count: usize = base_dims[*dim + 1..].iter().product();
            let n_indices = idx_dims[0];
            OpParams::IndexAdd {
                outer_count,
                base_dim_size,
                n_indices,
                inner_count,
            }
        }
        Op::ScatterAdd { dim } => {
            if node.inputs.len() != 3 {
                return Err(Error::Msg(format!(
                    "Op::ScatterAdd expects 3 inputs (base, indices, src), got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let base_layout = input_layout(node.inputs[0]);
            let src_layout = input_layout(node.inputs[2]);
            let base_shape: Vec<usize> = base_layout.shape().dims().to_vec();
            let src_shape: Vec<usize> = src_layout.shape().dims().to_vec();
            if base_shape.len() != src_shape.len() {
                return Err(Error::Msg(format!(
                    "Op::ScatterAdd: base rank ({}) != src rank ({})",
                    base_shape.len(),
                    src_shape.len(),
                ))
                .bt());
            }
            if *dim >= base_shape.len() {
                return Err(Error::Msg(format!(
                    "Op::ScatterAdd: dim {dim} out of range for rank {}",
                    base_shape.len(),
                ))
                .bt());
            }
            OpParams::ScatterAdd {
                base_shape,
                src_shape,
                dim: *dim,
            }
        }
        // Phase 7.6 step 5: Rope routes through the registry; the
        // legacy `Op::Rope` arm was retired with the variant.
        Op::Fused(_, fuel_graph::registry::FusedOpParams::Rope) => {
            // Inputs: (x, cos, sin). x is [..., seq, head_dim];
            // cos/sin are [seq, head_dim] (validated at graph build).
            if node.inputs.len() != 3 {
                return Err(Error::Msg(format!(
                    "Op::Rope expects 3 inputs (x, cos, sin), got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let x_layout = input_layout(node.inputs[0]);
            let x_dims = x_layout.shape().dims();
            if x_dims.len() < 2 {
                return Err(
                    Error::Msg(format!("Op::Rope: x must have rank ≥ 2, got {x_dims:?}",)).bt(),
                );
            }
            let head_dim = *x_dims.last().unwrap();
            let seq = x_dims[x_dims.len() - 2];
            let outer_count: usize = x_dims[..x_dims.len() - 2].iter().product();
            OpParams::Rope {
                outer_count,
                seq,
                head_dim,
            }
        }
        Op::Gather { dim } => {
            // inputs[0] = source, inputs[1] = U32 indices (same rank).
            if node.inputs.len() != 2 {
                return Err(Error::Msg(format!(
                    "Op::Gather expects 2 inputs (source, indices), got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let src_layout = input_layout(node.inputs[0]);
            let idx_layout = input_layout(node.inputs[1]);
            let source_shape: Vec<usize> = src_layout.shape().dims().to_vec();
            let output_shape: Vec<usize> = idx_layout.shape().dims().to_vec();
            if source_shape.len() != output_shape.len() {
                return Err(Error::Msg(format!(
                    "Op::Gather: source rank ({}) != indices rank ({})",
                    source_shape.len(),
                    output_shape.len(),
                ))
                .bt());
            }
            if *dim >= source_shape.len() {
                return Err(Error::Msg(format!(
                    "Op::Gather: dim {dim} out of range for rank {}",
                    source_shape.len(),
                ))
                .bt());
            }
            OpParams::Gather {
                source_shape,
                output_shape,
                dim: *dim,
            }
        }
        Op::NonZeroIndices { count_sym } => {
            // inputs[0] = x (the value tensor). capacity == x.elem_count();
            // the bundled output is [indices [capacity] U32 ; count [1] U32].
            if node.inputs.len() != 1 {
                return Err(Error::Msg(format!(
                    "Op::NonZeroIndices expects 1 input, got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let x_layout = input_layout(node.inputs[0]);
            let capacity = x_layout.shape().elem_count();
            OpParams::NonZeroIndices {
                capacity,
                count_sym: *count_sym,
            }
        }
        Op::IndexSelect { dim } => {
            // inputs[0] = source, inputs[1] = U32 indices (rank 1).
            if node.inputs.len() != 2 {
                return Err(Error::Msg(format!(
                    "Op::IndexSelect expects 2 inputs (source, indices), got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let src_layout = input_layout(node.inputs[0]);
            let idx_layout = input_layout(node.inputs[1]);
            let src_dims = src_layout.shape().dims();
            let idx_dims = idx_layout.shape().dims();
            if *dim >= src_dims.len() {
                return Err(Error::Msg(format!(
                    "Op::IndexSelect: dim {dim} out of range for source rank {}",
                    src_dims.len(),
                ))
                .bt());
            }
            if idx_dims.len() != 1 {
                return Err(Error::Msg(format!(
                    "Op::IndexSelect: indices must be rank 1, got shape {idx_dims:?}",
                ))
                .bt());
            }
            let outer_count: usize = src_dims[..*dim].iter().product();
            let source_dim_size = src_dims[*dim];
            let inner_count: usize = src_dims[*dim + 1..].iter().product();
            let n_indices = idx_dims[0];
            OpParams::IndexSelect {
                outer_count,
                source_dim_size,
                n_indices,
                inner_count,
            }
        }
        // Phase 7.6 step 5: legacy `Op::RmsNormLastDim` /
        // `Op::LayerNormLastDim` arms retired with the variants;
        // both now route through `Op::Fused(NORM_LAST_DIM, _)`.
        // Step 6 follow-up extends this arm to also cover the
        // backward variants (same outer × last_dim + eps geometry).
        Op::Fused(fid, params)
            if *fid == fuel_graph::registry::FusedOps::RMS_NORM_LAST_DIM
                || *fid == fuel_graph::registry::FusedOps::LAYER_NORM_LAST_DIM
                || *fid == fuel_graph::registry::FusedOps::RMS_NORM_LAST_DIM_BACKWARD
                || *fid == fuel_graph::registry::FusedOps::LAYER_NORM_LAST_DIM_BACKWARD =>
        {
            let eps = match params {
                fuel_graph::registry::FusedOpParams::RmsNormLastDim { eps } => *eps,
                fuel_graph::registry::FusedOpParams::LayerNormLastDim { eps } => *eps,
                fuel_graph::registry::FusedOpParams::RmsNormLastDimBackward { eps } => *eps,
                fuel_graph::registry::FusedOpParams::LayerNormLastDimBackward { eps } => *eps,
                _ => {
                    return Err(Error::Msg(format!(
                        "Op::Fused(NORM_LAST_DIM[_BACKWARD], _) expected \
                     RmsNormLastDim[Backward] or LayerNormLastDim[Backward] params, got {params:?}",
                    ))
                    .bt());
                }
            };
            let il = input_layout(node.inputs[0]);
            let dims = il.shape().dims();
            if dims.is_empty() {
                return Err(Error::Msg(
                    "Op::Fused(NORM_LAST_DIM[_BACKWARD], _) requires rank ≥ 1".to_string(),
                )
                .bt());
            }
            let last_dim = *dims.last().unwrap();
            let outer_count: usize = dims[..dims.len() - 1].iter().product();
            OpParams::NormLastDim {
                outer_count,
                last_dim,
                eps,
            }
        }
        // ReduceMaxToBackward: shape pair via the new
        // OpParams::ReduceMaxToBackward variant. x's shape is
        // `input_shape`; upstream's shape is `output_shape`.
        Op::Fused(fid, _) if *fid == fuel_graph::registry::FusedOps::REDUCE_MAX_TO_BACKWARD => {
            if node.inputs.len() != 2 {
                return Err(Error::Msg(format!(
                    "Op::Fused(REDUCE_MAX_TO_BACKWARD) expects 2 inputs, got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let x_layout = input_layout(node.inputs[0]);
            let up_layout = input_layout(node.inputs[1]);
            let input_shape: Vec<usize> = x_layout.shape().dims().to_vec();
            let output_shape: Vec<usize> = up_layout.shape().dims().to_vec();
            OpParams::ReduceMaxToBackward {
                input_shape,
                output_shape,
            }
        }
        Op::Concat { dim } => {
            // Output's shape: [..., total_dim, ...]. Compute outer
            // and inner counts from output_shape[..dim] and
            // [dim+1..]; per-input dim sizes from each input's
            // layout shape at index `dim`.
            if node.inputs.is_empty() {
                return Err(Error::Msg("Op::Concat requires at least 1 input".to_string()).bt());
            }
            let out_dims = node.shape.dims();
            if *dim >= out_dims.len() {
                return Err(Error::Msg(format!(
                    "Op::Concat: dim {dim} out of range for output rank {}",
                    out_dims.len(),
                ))
                .bt());
            }
            let outer_count: usize = out_dims[..*dim].iter().product();
            let inner_count: usize = out_dims[*dim + 1..].iter().product();
            let mut input_dim_sizes: Vec<usize> = Vec::with_capacity(node.inputs.len());
            for in_id in &node.inputs {
                let il = input_layout(*in_id);
                let il_dims = il.shape().dims();
                if *dim >= il_dims.len() {
                    return Err(Error::Msg(format!(
                        "Op::Concat: input {in_id:?} has rank {} but concat dim is {dim}",
                        il_dims.len(),
                    ))
                    .bt());
                }
                input_dim_sizes.push(il_dims[*dim]);
            }
            OpParams::Concat {
                outer_count,
                input_dim_sizes,
                inner_count,
                axis: *dim,
            }
        }
        Op::AddScalar(c) => OpParams::Affine { mul: 1.0, add: *c },
        Op::MulScalar(c) => OpParams::Affine { mul: *c, add: 0.0 },
        Op::Clamp { min, max } => OpParams::Clamp {
            min: *min,
            max: *max,
        },
        Op::PowI(exp) => OpParams::PowI { exp: *exp },
        Op::ClampInplace { min, max } => OpParams::Clamp {
            min: *min,
            max: *max,
        },
        Op::PowIInplace(exp) => OpParams::PowI { exp: *exp },
        // PowI backward — `(x, upstream) → grad_x = exp · x^(exp-1) ·
        // upstream`. Pulls the same `exp` as the forward through
        // FusedOpParams::PowIBackward (autograd carries it across).
        Op::Fused(_, fuel_graph::registry::FusedOpParams::PowIBackward { exp }) => {
            OpParams::PowI { exp: *exp }
        }
        // In-place affine (Phase 3 of the in-place ops infrastructure):
        // reuse the existing `OpParams::Affine` payload — the kernel
        // side doesn't care whether the destination is fresh or aliases
        // the input; the structural decision lives in the executor's
        // dedicated `WorkItemKind::InplaceKernel` arm.
        Op::Fused(_, fuel_graph::registry::FusedOpParams::InplaceAffine { mul, add }) => {
            OpParams::Affine {
                mul: *mul,
                add: *add,
            }
        }
        // Phase 7.6 step 5: legacy `Op::Conv2D` arm retired with the
        // variant; Conv2D routes through `Op::Fused(CONV2D, _)`.
        Op::Fused(
            _,
            fuel_graph::registry::FusedOpParams::Conv2D {
                stride,
                padding,
                groups,
            },
        ) => {
            // Inputs[0] = x [N, Cin, Hin, Win]; inputs[1] = weight
            // [Cout, Cin/groups, Kh, Kw]; inputs[2] (optional) = bias [Cout].
            // Output (this Node's shape) = [N, Cout, Hout, Wout].
            if node.inputs.len() != 2 && node.inputs.len() != 3 {
                return Err(Error::Msg(format!(
                    "Op::Conv2D expects 2 or 3 inputs, got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let x_layout = input_layout(node.inputs[0]);
            let w_layout = input_layout(node.inputs[1]);
            let x_dims = x_layout.shape().dims();
            let w_dims = w_layout.shape().dims();
            if x_dims.len() != 4 || w_dims.len() != 4 {
                return Err(Error::Msg(format!(
                    "Op::Conv2D requires rank-4 x and weight; got x={x_dims:?} w={w_dims:?}",
                ))
                .bt());
            }
            let out_dims = node.shape.dims();
            if out_dims.len() != 4 {
                return Err(Error::Msg(format!(
                    "Op::Conv2D output must be rank 4, got {out_dims:?}",
                ))
                .bt());
            }
            let x_shape = [x_dims[0], x_dims[1], x_dims[2], x_dims[3]];
            let w_shape = [w_dims[0], w_dims[1], w_dims[2], w_dims[3]];
            let out_shape = [out_dims[0], out_dims[1], out_dims[2], out_dims[3]];
            OpParams::Conv2D {
                x_shape,
                w_shape,
                out_shape,
                stride: *stride,
                padding: *padding,
                dilation: (1, 1),
                groups: *groups,
            }
        }
        // Phase 7.6 step 5 (final): legacy `Op::PagedAttn` arm dropped.
        Op::Fused(
            _,
            fuel_graph::registry::FusedOpParams::PagedAttn {
                softmax_scale,
                block_size,
                softcap,
            },
        ) => {
            // Inputs[0]=q [B,Hq,Sq,D], inputs[1]=k_cache [num_blocks,
            // block_size, Hkv, D], inputs[2]=v_cache same shape,
            // inputs[3]=block_table [B, max_blocks_per_seq] U32,
            // inputs[4]=context_lens [B] U32, inputs[5]=alibi [Hq] (optional).
            if node.inputs.len() != 5 && node.inputs.len() != 6 {
                return Err(Error::Msg(format!(
                    "Op::PagedAttn expects 5 or 6 inputs, got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let q_layout = input_layout(node.inputs[0]);
            let kc_layout = input_layout(node.inputs[1]);
            let vc_layout = input_layout(node.inputs[2]);
            let bt_layout = input_layout(node.inputs[3]);
            let q_dims = q_layout.shape().dims();
            let kc_dims = kc_layout.shape().dims();
            let vc_dims = vc_layout.shape().dims();
            let bt_dims = bt_layout.shape().dims();
            if q_dims.len() != 4 || kc_dims.len() != 4 || vc_dims.len() != 4 {
                return Err(Error::Msg(format!(
                    "Op::PagedAttn requires rank-4 q/k_cache/v_cache; \
                     got q={q_dims:?} k_cache={kc_dims:?} v_cache={vc_dims:?}",
                ))
                .bt());
            }
            if kc_dims != vc_dims {
                return Err(Error::Msg(format!(
                    "Op::PagedAttn: k_cache {kc_dims:?} and v_cache {vc_dims:?} must match",
                ))
                .bt());
            }
            if bt_dims.len() != 2 {
                return Err(Error::Msg(format!(
                    "Op::PagedAttn: block_table must be rank 2 [B, max_blocks_per_seq], got {bt_dims:?}",
                ))
                .bt());
            }
            if kc_dims[1] != *block_size {
                return Err(Error::Msg(format!(
                    "Op::PagedAttn: k_cache block_size dim ({}) must equal Op's block_size ({})",
                    kc_dims[1], block_size,
                ))
                .bt());
            }
            OpParams::PagedAttn {
                b: q_dims[0],
                hq: q_dims[1],
                hkv: kc_dims[2],
                sq: q_dims[2],
                d: q_dims[3],
                block_size: *block_size,
                max_blocks_per_seq: bt_dims[1],
                num_blocks: kc_dims[0],
                softmax_scale: *softmax_scale,
                softcap: *softcap,
            }
        }
        // Phase 7.6 step 5 (final): legacy `Op::FlashAttn` arm dropped.
        Op::Fused(
            _,
            fuel_graph::registry::FusedOpParams::FlashAttn {
                softmax_scale,
                causal,
                window_size_left,
                window_size_right,
                softcap,
                k_len,
            },
        ) => {
            // Inputs[0]=q [B,Hq,Sq,D], inputs[1]=k [B,Hkv,Sk,D],
            // inputs[2]=v [B,Hkv,Sk,D], inputs[3]=alibi_slopes [Hq] (optional).
            // `sk` is the PHYSICAL K extent (capacity, for strides/bytes);
            // `k_len` (resolved below) is the LOGICAL attended length.
            if node.inputs.len() != 3 && node.inputs.len() != 4 {
                return Err(Error::Msg(format!(
                    "Op::FlashAttn expects 3 or 4 inputs, got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let q_layout = input_layout(node.inputs[0]);
            let k_layout = input_layout(node.inputs[1]);
            let v_layout = input_layout(node.inputs[2]);
            let q_dims = q_layout.shape().dims();
            let k_dims = k_layout.shape().dims();
            let v_dims = v_layout.shape().dims();
            if q_dims.len() != 4 || k_dims.len() != 4 || v_dims.len() != 4 {
                return Err(Error::Msg(format!(
                    "Op::FlashAttn requires rank-4 q/k/v; got q={q_dims:?} k={k_dims:?} v={v_dims:?}",
                ))
                .bt());
            }
            if k_dims != v_dims {
                return Err(Error::Msg(format!(
                    "Op::FlashAttn: k {k_dims:?} and v {v_dims:?} must share shape",
                ))
                .bt());
            }
            if q_dims[0] != k_dims[0] || q_dims[3] != k_dims[3] {
                return Err(Error::Msg(format!(
                    "Op::FlashAttn: q {q_dims:?} and k {k_dims:?} must share B and D",
                ))
                .bt());
            }
            let sk = k_dims[2];
            // Resolve the logical attended length. `None` ⇒ attend the
            // full K extent (today's behavior). `Some(dyn)` ⇒ resolve
            // against the per-pass SymEnv; the resolved length must fit
            // the capacity and cover Sq (a valid causal prefix).
            let attended_k_len = match k_len {
                None => sk,
                Some(dyn_k) => {
                    let kl = dyn_k.resolve(sym_env).ok_or_else(|| {
                        Error::Msg(format!(
                            "Op::FlashAttn: dynamic k_len {dyn_k:?} is unbound in the SymEnv at realize",
                        ))
                        .bt()
                    })?;
                    if kl > sk {
                        return Err(Error::Msg(format!(
                            "Op::FlashAttn: resolved k_len ({kl}) exceeds K capacity ({sk})",
                        ))
                        .bt());
                    }
                    if kl < q_dims[2] {
                        return Err(Error::Msg(format!(
                            "Op::FlashAttn: resolved k_len ({kl}) < Sq ({}) — not a valid causal prefix",
                            q_dims[2],
                        ))
                        .bt());
                    }
                    kl
                }
            };
            OpParams::FlashAttn {
                b: q_dims[0],
                hq: q_dims[1],
                hkv: k_dims[1],
                sq: q_dims[2],
                sk,
                d: q_dims[3],
                k_len: attended_k_len,
                softmax_scale: *softmax_scale,
                causal: *causal,
                window_size_left: *window_size_left,
                window_size_right: *window_size_right,
                softcap: *softcap,
            }
        }
        Op::ReduceSumTo(target_shape) => {
            if node.inputs.len() != 1 {
                return Err(Error::Msg(format!(
                    "Op::ReduceSumTo expects 1 input, got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let in_layout = input_layout(node.inputs[0]);
            let input_shape: Vec<usize> = in_layout.shape().dims().to_vec();
            let output_shape: Vec<usize> = target_shape.dims().to_vec();
            OpParams::ReduceSumTo {
                input_shape,
                output_shape,
            }
        }
        Op::ReduceMaxTo(target_shape) => {
            if node.inputs.len() != 1 {
                return Err(Error::Msg(format!(
                    "Op::ReduceMaxTo expects 1 input, got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let in_layout = input_layout(node.inputs[0]);
            let input_shape: Vec<usize> = in_layout.shape().dims().to_vec();
            let output_shape: Vec<usize> = target_shape.dims().to_vec();
            OpParams::ReduceMaxTo {
                input_shape,
                output_shape,
            }
        }
        // Phase 7.6 step 5 (final): legacy `Op::ConvTranspose2D` arm dropped.
        Op::Fused(
            _,
            fuel_graph::registry::FusedOpParams::ConvTranspose2D {
                stride,
                padding,
                output_padding,
                dilation,
                groups,
            },
        ) => {
            // Inputs[0] = x [N, Cin, Hin, Win]; inputs[1] = weight
            // [Cin, Cout/groups, Kh, Kw]; inputs[2] (optional) = bias [Cout].
            // Output (this Node's shape) = [N, Cout, Hout, Wout].
            if node.inputs.len() != 2 && node.inputs.len() != 3 {
                return Err(Error::Msg(format!(
                    "Op::ConvTranspose2D expects 2 or 3 inputs, got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let x_layout = input_layout(node.inputs[0]);
            let w_layout = input_layout(node.inputs[1]);
            let x_dims = x_layout.shape().dims();
            let w_dims = w_layout.shape().dims();
            if x_dims.len() != 4 || w_dims.len() != 4 {
                return Err(Error::Msg(format!(
                    "Op::ConvTranspose2D requires rank-4 x and weight; got x={x_dims:?} w={w_dims:?}",
                ))
                .bt());
            }
            let out_dims = node.shape.dims();
            if out_dims.len() != 4 {
                return Err(Error::Msg(format!(
                    "Op::ConvTranspose2D output must be rank 4, got {out_dims:?}",
                ))
                .bt());
            }
            let x_shape = [x_dims[0], x_dims[1], x_dims[2], x_dims[3]];
            let w_shape = [w_dims[0], w_dims[1], w_dims[2], w_dims[3]];
            let out_shape = [out_dims[0], out_dims[1], out_dims[2], out_dims[3]];
            OpParams::ConvTranspose2D {
                x_shape,
                w_shape,
                out_shape,
                stride: *stride,
                padding: *padding,
                output_padding: *output_padding,
                dilation: *dilation,
                groups: *groups,
            }
        }
        Op::WriteSlice { ranges, dyn_offset } => {
            // inputs[0] = destination (its shape == this node's shape
            // since WriteSlice's output adopts the destination's
            // Layout). inputs[1] = source slab; its shape is implied
            // by `ranges`. The kernel needs the destination shape to
            // compute strides for the slab walk.
            if node.inputs.len() != 2 {
                return Err(Error::Msg(format!(
                    "Op::WriteSlice expects 2 inputs (destination, source), got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let dest_dims = node.shape.dims().to_vec();
            if ranges.len() != dest_dims.len() {
                return Err(Error::Msg(format!(
                    "Op::WriteSlice: ranges.len() ({}) must equal destination rank ({})",
                    ranges.len(),
                    dest_dims.len(),
                ))
                .bt());
            }
            // Phase D symbolic extents: when present, resolve the runtime
            // start offset against the per-pass SymEnv, overriding
            // `ranges[axis].0`. The slab WIDTH on that axis stays
            // `ranges[axis].1 - ranges[axis].0`, so the effective range
            // becomes `(base, base + width)`. `None` ⇒ fully static, and
            // the SymEnv is never consulted (empty-env realize unchanged).
            //
            // DUAL-PATH (increment 2b — data-determined dynamic shapes):
            //  - The offset resolves in the compile-time `sym_env` (static,
            //    or an INPUT-determined `cached_len`): bake it into `ranges`
            //    now, exactly as before.
            //  - It is UNBOUND at compile AND a producer in this graph binds
            //    it (`Op::NonZeroIndices`'s count, etc.): it is
            //    DATA-DETERMINED — leave `ranges[axis]` as the static
            //    placeholder and DEFER, carrying the `DynScalar` in
            //    `deferred_dyn_offset`. The `WorkItemKind::WriteSlice`
            //    executor arm resolves it from `produced_syms` (after the
            //    producer runs) and bounds-checks the resolved range there.
            //  - It is unbound AND no producer binds it: a genuine caller
            //    error → surface it here (preserves the unbound-error path).
            let mut ranges = ranges.clone();
            let mut deferred_dyn_offset: Option<(usize, fuel_ir::DynScalar)> = None;
            if let Some((axis, off)) = dyn_offset {
                if *axis >= ranges.len() {
                    return Err(Error::Msg(format!(
                        "Op::WriteSlice: dyn_offset axis ({axis}) out of bounds for rank {}",
                        ranges.len(),
                    ))
                    .bt());
                }
                match off.resolve(sym_env) {
                    Some(base) => {
                        let (start, end) = ranges[*axis];
                        let width = end - start;
                        ranges[*axis] = (base, base + width);
                    }
                    None => {
                        let is_data_determined = matches!(off, fuel_ir::DynScalar::Sym(s)
                            if graph.binds_data_determined_sym(*s));
                        if is_data_determined {
                            deferred_dyn_offset = Some((*axis, *off));
                        } else {
                            return Err(Error::Msg(format!(
                                "Op::WriteSlice: dynamic offset {off:?} on axis {axis} is \
                                 unbound in the SymEnv at realize (and no producer in the \
                                 graph binds it, so it is not data-determined)",
                            ))
                            .bt());
                        }
                    }
                }
            }
            for (i, &(start, end)) in ranges.iter().enumerate() {
                // The deferred axis still holds its static placeholder; the
                // executor bounds-checks the resolved range after producing
                // the count. Every other axis is fully known now.
                if deferred_dyn_offset.is_some_and(|(a, _)| a == i) {
                    continue;
                }
                if end < start || end > dest_dims[i] {
                    return Err(Error::Msg(format!(
                        "Op::WriteSlice: ranges[{i}] = ({start}, {end}) invalid \
                         for destination dim {i} = {}",
                        dest_dims[i],
                    ))
                    .bt());
                }
            }
            OpParams::WriteSlice {
                dest_shape: dest_dims,
                ranges,
                deferred_dyn_offset,
            }
        }
        Op::WriteSliceRotating {
            axis,
            modulus,
            ranges,
        } => {
            // inputs[0] = destination, inputs[1] = source slab,
            // inputs[2] = dynamic position scalar (U32). The builder
            // already validated rank / dtype / shape parity; the
            // exec-time check below mirrors WriteSlice for the
            // structural invariants the executor depends on.
            if node.inputs.len() != 3 {
                return Err(Error::Msg(format!(
                    "Op::WriteSliceRotating expects 3 inputs (destination, source, position), got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let dest_dims = node.shape.dims().to_vec();
            if ranges.len() != dest_dims.len() {
                return Err(Error::Msg(format!(
                    "Op::WriteSliceRotating: ranges.len() ({}) must equal destination rank ({})",
                    ranges.len(),
                    dest_dims.len(),
                ))
                .bt());
            }
            if *axis >= dest_dims.len() {
                return Err(Error::Msg(format!(
                    "Op::WriteSliceRotating: axis {axis} out of bounds for rank {}",
                    dest_dims.len(),
                ))
                .bt());
            }
            if *modulus == 0 || *modulus > dest_dims[*axis] {
                return Err(Error::Msg(format!(
                    "Op::WriteSliceRotating: modulus {modulus} invalid for dest_shape[{axis}] = {}",
                    dest_dims[*axis],
                ))
                .bt());
            }
            OpParams::WriteSliceRotating {
                dest_shape: dest_dims,
                axis: *axis,
                modulus: *modulus,
                ranges: ranges.clone(),
            }
        }
        Op::WriteSliceDoff { axis, ranges } => {
            // inputs[0] = destination, inputs[1] = source slab,
            // inputs[2] = device-resident I64 offset scalar. The
            // builder validated rank / dtype / static-width parity; the
            // exec-time check below mirrors WriteSliceRotating minus the
            // modulus (no wrap). Bounds on the *dynamic* offset are the
            // caller's contract (device-only, un-checkable here).
            if node.inputs.len() != 3 {
                return Err(Error::Msg(format!(
                    "Op::WriteSliceDoff expects 3 inputs (destination, source, offset), got {}",
                    node.inputs.len(),
                ))
                .bt());
            }
            let dest_dims = node.shape.dims().to_vec();
            if ranges.len() != dest_dims.len() {
                return Err(Error::Msg(format!(
                    "Op::WriteSliceDoff: ranges.len() ({}) must equal destination rank ({})",
                    ranges.len(),
                    dest_dims.len(),
                ))
                .bt());
            }
            if *axis >= dest_dims.len() {
                return Err(Error::Msg(format!(
                    "Op::WriteSliceDoff: axis {axis} out of bounds for rank {}",
                    dest_dims.len(),
                ))
                .bt());
            }
            OpParams::WriteSliceDoff {
                dest_shape: dest_dims,
                axis: *axis,
                ranges: ranges.clone(),
            }
        }
        _ => OpParams::None,
    })
}

/// Execute one work item. Three branches by `WorkItemKind`:
///
/// - `ConstAdopt` — verify cache has an entry pre-seeded by the
///   caller; record the layout from the WorkItem.
/// - `ViewOf { input }` — clone the input's Storage Arc into the
///   output slot (bytes are shared); record the strided layout
///   from the WorkItem.
/// - `Kernel` — gather input Arcs, allocate the output, run the
///   compiled kernel, store the result; record the contiguous
///   layout from the WorkItem.
///
/// Best-effort: prefix a realize-time node error with the failing node's graph
/// location — `Graph::describe_node` renders `Node#N (op, out shape, dtype,
/// inputs=[...])`, enough to grep-locate the offending op in a large graph. The
/// legacy per-node executors (`fuel-graph-executor`, the retired
/// `fuel-reference-backend`) wrapped panics this way; the PipelinedExecutor
/// surfaces the raw kernel error, so a dtype/shape/OOB failure deep in a model
/// arrived with no graph coordinate. On a graph read-lock failure the original
/// error passes through unchanged — the location is a navigation aid, never
/// worth masking the real error with a lock issue. Zero happy-path cost: only
/// the error branch touches the graph.
fn with_node_location(graph: &Arc<RwLock<Graph>>, node_id: NodeId, err: Error) -> Error {
    match graph.read() {
        Ok(g) => err.context(g.describe_node(node_id)),
        Err(_) => err,
    }
}

/// Execute one [`WorkItem`] against the cache, returning the node's
/// [`CompletionHandle`].
///
/// Step E A4b-1: the producing arms (Kernel / WriteSlice / WriteSliceRotating /
/// in-place / cross-device Copy alloc) return the handle from `execute_compiled`
/// *without* waiting it inline — the realize loop stores it in its per-node
/// handle map and drains at realize-end. Non-producing arms (const adopt, view,
/// slot projection, release marker, contiguize) are pure Arc-clones / host
/// allocs whose work is already complete → [`CompletionHandle::Ready`].
/// Read a data-determined runtime count from bundle slot 1 of a
/// just-realized producer and bind it into the per-pass `SymEnv` under
/// `count_sym` — the data-dependent dynamic-shapes seam (increment 2a).
/// The producer's bundle is `[indices [capacity] U32 ; count [1] U32]`;
/// slot 1's single U32 starts at byte `capacity * 4` (after the
/// `capacity`-element U32 primary slot). CPU only for now — device
/// backends need a D2H after the completion handle is waited (a documented
/// follow-up; the whole data-dependent path is CPU-first). Called AFTER
/// `execute_compiled`, whose CPU path returns a `Ready` handle, so the
/// bytes are already valid.
fn bind_data_determined_count(
    arc: &Arc<RwLock<Storage>>,
    capacity: usize,
    count_sym: fuel_ir::SymId,
    produced_syms: &mut SymEnv,
) -> Result<()> {
    let guard = arc
        .read()
        .map_err(|_| poisoned("NonZeroIndices output storage"))?;
    let count_byte_off = capacity * std::mem::size_of::<u32>();
    let end = count_byte_off + std::mem::size_of::<u32>();
    // The non-CPU `BackendStorage` variants are cfg-gated; with GPU
    // features off, `Cpu` is the only variant and the fallback arm is
    // unreachable (a documented CPU-first limitation, not dead logic).
    #[allow(unreachable_patterns)]
    let count = match &guard.inner {
        fuel_memory::BackendStorage::Cpu(cpu_bytes) => {
            let bytes = cpu_bytes.bytes();
            if end > bytes.len() {
                return Err(Error::Msg(format!(
                    "NonZeroIndices: count slot byte range [{count_byte_off}..{end}) \
                     exceeds bundle byte length {}",
                    bytes.len(),
                ))
                .bt());
            }
            u32::from_le_bytes(
                bytes[count_byte_off..end]
                    .try_into()
                    .expect("4 bytes → u32"),
            )
        }
        other => {
            return Err(Error::Msg(format!(
                "NonZeroIndices: data-determined count read is CPU-only today; got \
                 BackendStorage::{:?} (device backends need a D2H after the completion \
                 handle is waited — a documented follow-up)",
                std::mem::discriminant(other),
            ))
            .bt());
        }
    };
    produced_syms.bind(count_sym, count as usize)?;
    Ok(())
}

/// Increment 2b — the consumer half of data-dependent dynamic shapes. When
/// a `WriteSlice`'s offset is DATA-DETERMINED (its `SymId` is produced
/// mid-pass, so `op_to_op_params` deferred it into
/// `OpParams::WriteSlice::deferred_dyn_offset` instead of baking it at
/// compile), resolve it HERE from `produced_syms` — populated by the
/// producer's execute step (single-threaded topological order guarantees
/// the producer already ran). Returns a patched clone of the `CompiledNode`
/// with the resolved `ranges` (and `deferred_dyn_offset` cleared) for the
/// kernel; `None` when there is nothing to defer (the common path — no
/// clone). Bounds-checks the resolved range (deferred past compile).
fn resolve_deferred_write_slice(
    compiled: &CompiledNode,
    produced_syms: &SymEnv,
    node_id: NodeId,
) -> Result<Option<CompiledNode>> {
    let OpParams::WriteSlice {
        dest_shape,
        ranges,
        deferred_dyn_offset,
    } = &compiled.op_params
    else {
        return Ok(None);
    };
    let Some((axis, off)) = deferred_dyn_offset else {
        return Ok(None);
    };
    let base = off.resolve(produced_syms).ok_or_else(|| {
        Error::Msg(format!(
            "Op::WriteSlice {node_id:?}: data-determined offset {off:?} on axis {axis} is \
             unbound in produced_syms at execute (its producer did not run / bind it)",
        ))
        .bt()
    })?;
    let mut ranges = ranges.clone();
    let (start, end) = ranges[*axis];
    let width = end - start;
    let resolved = (base, base + width);
    if resolved.1 > dest_shape[*axis] {
        return Err(Error::Msg(format!(
            "Op::WriteSlice {node_id:?}: resolved data-determined range {resolved:?} on axis \
             {axis} exceeds destination dim = {}",
            dest_shape[*axis],
        ))
        .bt());
    }
    ranges[*axis] = resolved;
    let mut patched = compiled.clone();
    patched.op_params = OpParams::WriteSlice {
        dest_shape: dest_shape.clone(),
        ranges,
        deferred_dyn_offset: None,
    };
    Ok(Some(patched))
}

/// Dynamic-M matmul: when a matmul's row count is DATA-DETERMINED
/// (`MatmulM::Deferred`, deferred by `op_to_op_params` because a producer
/// binds its `SymId` mid-pass), resolve it HERE from `produced_syms` and
/// run the kernel against a patched clone whose `m_compute` is
/// `MatmulM::Rows(count)`. Mirrors `resolve_deferred_write_slice`. `None`
/// when nothing is deferred (the common path — no clone). The kernel then
/// computes exactly `count` rows of the `m`-row capacity buffer.
fn resolve_deferred_matmul(
    compiled: &CompiledNode,
    produced_syms: &SymEnv,
    node_id: NodeId,
) -> Result<Option<CompiledNode>> {
    let OpParams::Matmul {
        lhs_batch_dims,
        rhs_batch_dims,
        m,
        n,
        k,
        m_compute,
    } = &compiled.op_params
    else {
        return Ok(None);
    };
    let MatmulM::Deferred(dyn_m) = m_compute else {
        return Ok(None);
    };
    let rows = dyn_m.resolve(produced_syms).ok_or_else(|| {
        Error::Msg(format!(
            "Op::MatMul {node_id:?}: data-determined row count {dyn_m:?} is unbound in \
             produced_syms at execute (its producer did not run / bind it)",
        ))
        .bt()
    })?;
    if rows > *m {
        return Err(Error::Msg(format!(
            "Op::MatMul {node_id:?}: resolved row count {rows} exceeds capacity {m}",
        ))
        .bt());
    }
    let mut patched = compiled.clone();
    patched.op_params = OpParams::Matmul {
        lhs_batch_dims: lhs_batch_dims.clone(),
        rhs_batch_dims: rhs_batch_dims.clone(),
        m: *m,
        n: *n,
        k: *k,
        m_compute: MatmulM::Rows(rows),
    };
    Ok(Some(patched))
}

fn execute_work_item(
    item: &WorkItem,
    cache: &mut StorageCache,
    layout_cache: &mut HashMap<NodeId, Layout>,
    produced_syms: &mut SymEnv,
    // Step E A4b open question #4 — "event only where waited": whether
    // THIS item's node id is in the realize's `WaitSet` (`will_be_waited`
    // in `pipelined.rs`, computed once per item by the caller). Forwarded
    // to every `execute_compiled_with_wait_hint` call below so a CUDA
    // kernel with no mid-walk reader skips its `Event` entirely.
    will_be_waited: bool,
    // CapturedRun executor build-out: persistent-output mode. `None` on
    // every existing realize path ⇒ byte-identical (fresh alloc, nothing
    // recorded). `Some(Record)` records each Kernel output's fixed Arc;
    // `Some(Reuse)` reuses the recorded buffer (no alloc — capture-safe)
    // and the write-into kernel path writes into it in place. Only the
    // `Kernel` arm consults it; every other arm ignores it.
    mut persistent: Option<&mut PersistentOutputs>,
) -> Result<CompletionHandle> {
    match &item.kind {
        WorkItemKind::ConstAdopt => {
            if !cache.contains_key(&item.node_id) {
                return Err(Error::Msg(format!(
                    "PipelinedExecutor: Const node {:?} not in input cache",
                    item.node_id
                ))
                .bt());
            }
            // Layout for input cache entries was seeded at realize
            // start; refresh from the WorkItem in case the side-table
            // was set after seeding.
            layout_cache.insert(item.node_id, item.output_layout.clone());
            Ok(CompletionHandle::Ready)
        }
        WorkItemKind::ViewOf { input } => {
            let input_arc = cache.get(input).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: view-op input {:?} of {:?} not realized",
                    input, item.node_id,
                ))
                .bt()
            })?;
            cache.insert(item.node_id, input_arc);
            layout_cache.insert(item.node_id, item.output_layout.clone());
            Ok(CompletionHandle::Ready)
        }
        WorkItemKind::SlotView { producer } => {
            // Op::View — multi-output projection. Same realization
            // shape as ViewOf: clone the producer's Storage Arc into
            // the View's cache slot. The WorkItem's output_layout was
            // computed at graph-build time by `Tensor::view`, with
            // the slot's byte_offset baked into start_offset (in
            // slot-dtype-element units). Downstream kernels reading
            // the producer's bytes as the slot's dtype starting at
            // that offset see the slot's typed window.
            let producer_arc = cache.get(producer).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: Op::View producer {:?} of {:?} not realized",
                    producer, item.node_id,
                ))
                .bt()
            })?;
            cache.insert(item.node_id, producer_arc);
            layout_cache.insert(item.node_id, item.output_layout.clone());
            Ok(CompletionHandle::Ready)
        }
        WorkItemKind::SlotOwn { producer, slot } => {
            // Op::ViewOwned — allocate a fresh Storage of the slot's
            // (shape, dtype) on the producer's device and copy the
            // slot's bytes in. Per-backend fast paths:
            //   - CPU:    direct &[u8]→&mut[u8] memcpy via CpuStorageBytes
            //   - CUDA:   D2D via CudaStorageBytes::slot_copy_to_new
            //             (cuMemcpyDtoDAsync with offset src pointer)
            //   - Vulkan: vkCmdCopyBuffer via
            //             VulkanBackend::slot_copy_to_new_handle
            //             (srcOffset/dstOffset/size)
            // For backends without a copy-with-offset hook we fall
            // back to a host round-trip (`to_host_buffer_dyn` →
            // slice → `storage_from_host_buffer_owned_dyn`), which
            // works but is wasteful.
            let producer_arc = cache.get(producer).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: Op::ViewOwned producer {:?} of {:?} not realized",
                    producer, item.node_id,
                ))
                .bt()
            })?;
            let new_storage = {
                let guard = producer_arc
                    .read()
                    .map_err(|_| poisoned("ViewOwned producer storage"))?;
                let bundle = guard.bundle().ok_or_else(|| {
                    Error::Msg(format!(
                        "Op::ViewOwned: producer {:?} has no bundle side-table \
                         — the multi-output authoring contract requires \
                         Storage::with_bundle / new_bundled at allocation \
                         time. Node {:?} slot {} cannot resolve its source.",
                        producer, item.node_id, slot,
                    ))
                    .bt()
                })?;
                let sv = bundle.get(*slot as usize).cloned().ok_or_else(|| {
                    Error::Msg(format!(
                        "Op::ViewOwned: slot {} out of range for producer {:?} \
                         (bundle has {} slots)",
                        slot,
                        producer,
                        bundle.len(),
                    ))
                    .bt()
                })?;
                use fuel_memory::BackendStorage;
                match &guard.inner {
                    BackendStorage::Cpu(cpu_bytes) => {
                        let src_bytes = cpu_bytes.bytes();
                        let end = sv.byte_offset + sv.len_bytes();
                        if end > src_bytes.len() {
                            return Err(Error::Msg(format!(
                                "Op::ViewOwned: slot {} byte range [{}..{}) \
                                 exceeds producer's CPU byte length {}",
                                slot,
                                sv.byte_offset,
                                end,
                                src_bytes.len(),
                            ))
                            .bt());
                        }
                        let owned = fuel_cpu_backend::CpuStorageBytes::from_bytes(
                            &src_bytes[sv.byte_offset..end],
                        );
                        fuel_memory::Storage::new(BackendStorage::Cpu(owned), sv.dtype)
                    }
                    #[cfg(feature = "cuda")]
                    BackendStorage::Cuda(cuda_bytes) => {
                        let dst = cuda_bytes.slot_copy_to_new(sv.byte_offset, sv.len_bytes())?;
                        fuel_memory::Storage::new(BackendStorage::Cuda(dst), sv.dtype)
                    }
                    #[cfg(feature = "vulkan")]
                    BackendStorage::Vulkan(vk_bytes) => {
                        let backend = vk_bytes.backend().ok_or_else(|| {
                            Error::Msg(format!(
                                "Op::ViewOwned: producer {:?} is on Vulkan but \
                                 its storage has no VulkanBackend handle. \
                                 Storages flowing through the pipelined executor \
                                 must be constructed via alloc_bytes_handle / \
                                 upload_bytes_handle so the bundle's slot extraction \
                                 path can reach the backend.",
                                producer,
                            ))
                            .bt()
                        })?;
                        let dst = backend.slot_copy_to_new_handle(
                            vk_bytes,
                            sv.byte_offset,
                            sv.len_bytes(),
                        )?;
                        fuel_memory::Storage::new(BackendStorage::Vulkan(dst), sv.dtype)
                    }
                    #[allow(unreachable_patterns)]
                    _ => {
                        return Err(Error::Msg(format!(
                            "Op::ViewOwned: producer {:?} is on a backend \
                             without a slot-copy-with-offset hook \
                             (CPU + CUDA + Vulkan wired). Add a per-backend \
                             `slot_copy_to_new`-equivalent and extend the \
                             match arm.",
                            producer,
                        ))
                        .bt());
                    }
                }
            };
            cache.insert(item.node_id, Arc::new(RwLock::new(new_storage)));
            layout_cache.insert(item.node_id, item.output_layout.clone());
            // A4b-1: Op::ViewOwned's slot copy (CUDA D2D) is same-stream; its
            // completion is carried by the realize-end full-stream sync in
            // `to_cpu_bytes` (unchanged) — `Ready` here is behavior-preserving.
            Ok(CompletionHandle::Ready)
        }
        WorkItemKind::WriteSlice { dest, source } => {
            let compiled = item.compiled.as_ref().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: WriteSlice work item {:?} has no compiled node",
                    item.node_id,
                ))
                .bt()
            })?;
            // Increment 2b: if the offset is DATA-DETERMINED, resolve it from
            // the producer-bound `produced_syms` and run the kernel against
            // the patched (resolved-range) node. `None` on the common path.
            let patched_ws = resolve_deferred_write_slice(compiled, produced_syms, item.node_id)?;
            let compiled = patched_ws.as_ref().unwrap_or(compiled);
            let dest_arc = cache.get(dest).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: WriteSlice destination {:?} of {:?} not realized",
                    dest, item.node_id,
                ))
                .bt()
            })?;
            let source_arc = cache.get(source).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: WriteSlice source {:?} of {:?} not realized",
                    source, item.node_id,
                ))
                .bt()
            })?;
            let dest_layout = layout_cache.get(dest).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: WriteSlice destination {:?} of {:?} has no cached layout",
                    dest, item.node_id,
                ))
                .bt()
            })?;
            let source_layout = layout_cache.get(source).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: WriteSlice source {:?} of {:?} has no cached layout",
                    source, item.node_id,
                ))
                .bt()
            })?;
            // v1 contract: destination must be contiguous + zero
            // offset. Strided dests would force the kernel to walk
            // both source and dest strides; defer until KV cache
            // exposes a non-contig dest. Source CAN be non-contig —
            // we auto-contiguize it here so the kernel sees a flat
            // slab.
            if !dest_layout.is_contiguous() || dest_layout.start_offset() != 0 {
                return Err(Error::Msg(format!(
                    "Op::WriteSlice (Phase E.3.2 v1): destination {:?} must be \
                     contiguous + zero-offset; got Layout {:?}",
                    dest, dest_layout,
                ))
                .bt());
            }
            let source_arc_contig = {
                let s_dtype = source_arc
                    .read()
                    .map_err(|_| poisoned("WriteSlice source storage"))?
                    .dtype;
                let s_len_bytes = source_arc
                    .read()
                    .map_err(|_| poisoned("WriteSlice source storage"))?
                    .inner
                    .len_bytes();
                let layout_bytes = source_layout.shape().elem_count() * s_dtype.size_in_bytes();
                let bytes_match_shape = s_len_bytes == layout_bytes;
                let already_contig = source_layout.is_contiguous()
                    && source_layout.start_offset() == 0
                    && bytes_match_shape;
                if already_contig {
                    source_arc
                } else {
                    auto_contiguize(&source_arc, &source_layout)?
                }
            };
            let source_layout_kernel = fuel_ir::Layout::contiguous(source_layout.shape().clone());
            // The kernel sees inputs=[source] and outputs=[dest_arc].
            // The dest input slot is intentionally absent — the kernel
            // doesn't read from dest; it only writes to its bytes
            // through the output slot's write lock.
            let input_arcs = vec![source_arc_contig];
            let mut output_arcs = vec![dest_arc.clone()];
            let kernel_layouts = vec![source_layout_kernel, dest_layout.clone()];
            // A4b-1: defer the wait — return the handle to the realize loop.
            let handle = execute_compiled_with_wait_hint(
                compiled,
                &input_arcs,
                &mut output_arcs,
                &kernel_layouts,
                will_be_waited,
            )?;
            // Adopt the dest Arc at this WriteSlice node's slot. The
            // realize loop's destructive_input cleanup evicts the
            // dest's own NodeId from the cache afterward — downstream
            // readers go through this node's NodeId.
            cache.insert(item.node_id, dest_arc);
            layout_cache.insert(item.node_id, item.output_layout.clone());
            Ok(handle)
        }
        WorkItemKind::WriteSliceRotating {
            dest,
            source,
            position,
        } => {
            // Same shape as WriteSlice but with a position input.
            // Kernel sees `[source, position]` as inputs; `dest` is the
            // pre-allocated output buffer (in-place adoption).
            let compiled = item.compiled.as_ref().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: WriteSliceRotating work item {:?} has no compiled node",
                    item.node_id,
                ))
                .bt()
            })?;
            let dest_arc = cache.get(dest).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: WriteSliceRotating destination {:?} of {:?} not realized",
                    dest, item.node_id,
                ))
                .bt()
            })?;
            let source_arc = cache.get(source).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: WriteSliceRotating source {:?} of {:?} not realized",
                    source, item.node_id,
                ))
                .bt()
            })?;
            let position_arc = cache.get(position).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: WriteSliceRotating position {:?} of {:?} not realized",
                    position, item.node_id,
                ))
                .bt()
            })?;
            let dest_layout = layout_cache.get(dest).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: WriteSliceRotating destination {:?} of {:?} has no cached layout",
                    dest, item.node_id,
                ))
                .bt()
            })?;
            let source_layout = layout_cache.get(source).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: WriteSliceRotating source {:?} of {:?} has no cached layout",
                    source, item.node_id,
                ))
                .bt()
            })?;
            // v1: destination contiguous + zero offset (same as WriteSlice).
            if !dest_layout.is_contiguous() || dest_layout.start_offset() != 0 {
                return Err(Error::Msg(format!(
                    "Op::WriteSliceRotating (v1): destination {:?} must be \
                     contiguous + zero-offset; got Layout {:?}",
                    dest, dest_layout,
                ))
                .bt());
            }
            let source_arc_contig = {
                let s_dtype = source_arc
                    .read()
                    .map_err(|_| poisoned("WriteSliceRotating source storage"))?
                    .dtype;
                let s_len_bytes = source_arc
                    .read()
                    .map_err(|_| poisoned("WriteSliceRotating source storage"))?
                    .inner
                    .len_bytes();
                let layout_bytes = source_layout.shape().elem_count() * s_dtype.size_in_bytes();
                let bytes_match_shape = s_len_bytes == layout_bytes;
                let already_contig = source_layout.is_contiguous()
                    && source_layout.start_offset() == 0
                    && bytes_match_shape;
                if already_contig {
                    source_arc
                } else {
                    auto_contiguize(&source_arc, &source_layout)?
                }
            };
            let source_layout_kernel = fuel_ir::Layout::contiguous(source_layout.shape().clone());
            let position_layout = fuel_ir::Layout::contiguous(fuel_ir::Shape::from_dims(&[]));
            let input_arcs = vec![source_arc_contig, position_arc];
            let mut output_arcs = vec![dest_arc.clone()];
            let kernel_layouts = vec![source_layout_kernel, position_layout, dest_layout.clone()];
            // A4b-1: defer the wait — return the handle to the realize loop.
            let handle = execute_compiled_with_wait_hint(
                compiled,
                &input_arcs,
                &mut output_arcs,
                &kernel_layouts,
                will_be_waited,
            )?;
            cache.insert(item.node_id, dest_arc);
            layout_cache.insert(item.node_id, item.output_layout.clone());
            Ok(handle)
        }
        WorkItemKind::WriteSliceDoff {
            dest,
            source,
            offset,
        } => {
            // Same shape as WriteSliceRotating: kernel sees
            // `[source, offset]` as inputs; `dest` is the pre-allocated
            // output buffer (in-place adoption). The offset (rank-0
            // I64) is passed through as a kernel input — the CUDA
            // wrapper threads its device pointer straight to the kernel
            // (no D2H), which is what makes a captured graph replay at
            // the host-updated position.
            let compiled = item.compiled.as_ref().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: WriteSliceDoff work item {:?} has no compiled node",
                    item.node_id,
                ))
                .bt()
            })?;
            let dest_arc = cache.get(dest).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: WriteSliceDoff destination {:?} of {:?} not realized",
                    dest, item.node_id,
                ))
                .bt()
            })?;
            let source_arc = cache.get(source).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: WriteSliceDoff source {:?} of {:?} not realized",
                    source, item.node_id,
                ))
                .bt()
            })?;
            let offset_arc = cache.get(offset).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: WriteSliceDoff offset {:?} of {:?} not realized",
                    offset, item.node_id,
                ))
                .bt()
            })?;
            let dest_layout = layout_cache.get(dest).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: WriteSliceDoff destination {:?} of {:?} has no cached layout",
                    dest, item.node_id,
                ))
                .bt()
            })?;
            let source_layout = layout_cache.get(source).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: WriteSliceDoff source {:?} of {:?} has no cached layout",
                    source, item.node_id,
                ))
                .bt()
            })?;
            // v1: destination contiguous + zero offset (same as WriteSlice).
            if !dest_layout.is_contiguous() || dest_layout.start_offset() != 0 {
                return Err(Error::Msg(format!(
                    "Op::WriteSliceDoff (v1): destination {:?} must be \
                     contiguous + zero-offset; got Layout {:?}",
                    dest, dest_layout,
                ))
                .bt());
            }
            let source_arc_contig = {
                let s_dtype = source_arc
                    .read()
                    .map_err(|_| poisoned("WriteSliceDoff source storage"))?
                    .dtype;
                let s_len_bytes = source_arc
                    .read()
                    .map_err(|_| poisoned("WriteSliceDoff source storage"))?
                    .inner
                    .len_bytes();
                let layout_bytes = source_layout.shape().elem_count() * s_dtype.size_in_bytes();
                let bytes_match_shape = s_len_bytes == layout_bytes;
                let already_contig = source_layout.is_contiguous()
                    && source_layout.start_offset() == 0
                    && bytes_match_shape;
                if already_contig {
                    source_arc
                } else {
                    auto_contiguize(&source_arc, &source_layout)?
                }
            };
            let source_layout_kernel = fuel_ir::Layout::contiguous(source_layout.shape().clone());
            let offset_layout = fuel_ir::Layout::contiguous(fuel_ir::Shape::from_dims(&[]));
            let input_arcs = vec![source_arc_contig, offset_arc];
            let mut output_arcs = vec![dest_arc.clone()];
            let kernel_layouts = vec![source_layout_kernel, offset_layout, dest_layout.clone()];
            // A4b-1: defer the wait — return the handle to the realize loop.
            let handle = execute_compiled_with_wait_hint(
                compiled,
                &input_arcs,
                &mut output_arcs,
                &kernel_layouts,
                will_be_waited,
            )?;
            cache.insert(item.node_id, dest_arc);
            layout_cache.insert(item.node_id, item.output_layout.clone());
            Ok(handle)
        }
        WorkItemKind::Alloc { target_location } => {
            // Op::Alloc { target }: allocate a fresh, zero-init storage
            // on target_location. No inputs, no kernel call — direct
            // executor dispatch. For non-CPU targets we derive the
            // per-backend device handle by searching the input cache
            // for any storage on the target backend (callers seed this
            // via `pipelined_bridge::device_seed_storage`).
            // Used only by the Vulkan arm below (alloc_bytes_handle); a
            // default-feature build sees it unused. GAP-229: 68161981's
            // unused_variables --fix renamed it `_n_bytes` at default
            // features and broke `--features vulkan` (E0425). Guard the
            // decline instead of the name so the binding stays usable.
            #[cfg_attr(not(feature = "vulkan"), allow(unused_variables))]
            let n_bytes = item.elem_count * item.dtype.size_in_bytes();
            let alloced: Storage = match target_location {
                // CPU has no separate uninit alloc primitive in safe
                // Rust (`vec![0; n]` is the canonical path). Op::Alloc
                // on CPU returns zero-init storage; the following
                // Op::ZeroFill is a redundant memset the optimizer
                // can elide and LLVM folds away regardless.
                DeviceLocation::Cpu => fuel_memory::alloc_cpu_zeroed(item.dtype, item.elem_count)?,
                #[cfg(feature = "cuda")]
                DeviceLocation::Cuda { gpu_id } => {
                    let cuda_dev = find_cuda_device_in_cache(cache, *gpu_id).ok_or_else(|| {
                        Error::Msg(format!(
                            "Op::Alloc on Cuda {{ gpu_id: {} }}: no CUDA \
                             storage in input cache to derive the device \
                             handle from. The caller must seed the cache \
                             (e.g. via `fuel-core::pipelined_bridge::\
                             device_seed_storage`) before realizing.",
                            gpu_id,
                        ))
                        .bt()
                    })?;
                    // Phase 3a follow-up: true uninit alloc via
                    // baracuda alpha.30's unsafe alloc. The bytes are
                    // uninit until a subsequent Op::ZeroFill or full-
                    // overwrite op runs. The byte-storage Arc has no
                    // typed-slice accessor that would dereference
                    // uninit bytes.
                    let cuda_bytes =
                        fuel_cuda_backend::CudaStorageBytes::alloc_uninit(&cuda_dev, n_bytes)?;
                    Storage::new(fuel_memory::BackendStorage::Cuda(cuda_bytes), item.dtype)
                }
                #[cfg(not(feature = "cuda"))]
                DeviceLocation::Cuda { .. } => {
                    return Err(Error::Msg(
                        "Op::Alloc on Cuda but fuel-storage wasn't built \
                         with --features cuda"
                            .to_string(),
                    )
                    .bt());
                }
                #[cfg(feature = "vulkan")]
                DeviceLocation::Vulkan { gpu_id } => {
                    let backend =
                        find_vulkan_backend_in_cache(cache, *gpu_id).ok_or_else(|| {
                            Error::Msg(format!(
                                "Op::Alloc on Vulkan {{ gpu_id: {} }}: no Vulkan \
                             storage in input cache to derive the backend \
                             handle from. The caller must seed the cache \
                             (e.g. via `fuel-core::pipelined_bridge::\
                             device_seed_storage`) before realizing.",
                                gpu_id,
                            ))
                            .bt()
                        })?;
                    // Phase 3a follow-up: true uninit alloc. The old
                    // `upload_bytes_handle(vec![0u8; n])` host-staged
                    // zeros (~2× the bandwidth of a device-side fill);
                    // we now allocate uninit and rely on a paired
                    // Op::ZeroFill (vkCmdFillBuffer) for the
                    // initialization. Faster KV-cache init on Vulkan.
                    let vk_bytes = backend.alloc_bytes_handle(n_bytes)?;
                    Storage::new(fuel_memory::BackendStorage::Vulkan(vk_bytes), item.dtype)
                }
                #[cfg(not(feature = "vulkan"))]
                DeviceLocation::Vulkan { .. } => {
                    return Err(Error::Msg(
                        "Op::Alloc on Vulkan but fuel-storage wasn't built \
                         with --features vulkan"
                            .to_string(),
                    )
                    .bt());
                }
                other => {
                    return Err(Error::Msg(format!(
                        "Op::Alloc on {:?}: target not yet wired (CPU + \
                         CUDA + Vulkan land in Phase 3a of bridge-retirement; \
                         Metal extends when its byte-storage substrate is \
                         ready).",
                        other,
                    ))
                    .bt());
                }
            };
            cache.insert(item.node_id, Arc::new(RwLock::new(alloced)));
            layout_cache.insert(item.node_id, item.output_layout.clone());
            Ok(CompletionHandle::Ready)
        }
        WorkItemKind::Iota { len } => {
            // Op::Iota { len }: 0-input F32 position generator
            // [0.0, 1.0, …, (len-1)]. Direct executor dispatch (no kernel /
            // binding-table lookup), like WorkItemKind::Alloc. CPU-only
            // today — GPU flash-attn keeps its fused kernel, so the alibi
            // decomposition (the sole Iota consumer) realizes on CPU.
            if item.target_backend != BackendId::Cpu {
                return Err(Error::Msg(format!(
                    "WorkItemKind::Iota: only CPU is wired today, got {:?}",
                    item.target_backend,
                ))
                .bt());
            }
            let v: Vec<f32> = (0..*len).map(|i| i as f32).collect();
            let iota = fuel_memory::from_slice_cpu(&v);
            cache.insert(item.node_id, Arc::new(RwLock::new(iota)));
            layout_cache.insert(item.node_id, item.output_layout.clone());
            Ok(CompletionHandle::Ready)
        }
        WorkItemKind::ZeroFill => {
            // Op::ZeroFill: in-place zero the input's bytes. Output
            // adopts the input's Storage Arc (same Storage; bytes
            // are mutated). Direct per-backend dispatch — no kernel
            // call, no binding-table lookup.
            if item.inputs.len() != 1 {
                return Err(Error::Msg(format!(
                    "PipelinedExecutor: ZeroFill work item {:?} expects \
                     1 input, got {}",
                    item.node_id,
                    item.inputs.len(),
                ))
                .bt());
            }
            let src_id = item.inputs[0];
            let src_arc = cache.get(&src_id).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: ZeroFill input {:?} of {:?} \
                     not realized",
                    src_id, item.node_id,
                ))
                .bt()
            })?;
            // Per-backend in-place fill. Acquired the write lock once
            // for the duration of the fill — kernels on async backends
            // (CUDA / Vulkan) issue the memset and return; the fence
            // happens when downstream readers acquire the read lock
            // through the executor's standard synchronization path.
            {
                let mut guard = src_arc
                    .write()
                    .map_err(|_| poisoned("ZeroFill destination storage"))?;
                match &mut guard.inner {
                    fuel_memory::BackendStorage::Cpu(c) => {
                        let bytes = c.bytes_mut();
                        bytes.fill(0);
                    }
                    #[cfg(feature = "cuda")]
                    fuel_memory::BackendStorage::Cuda(c) => {
                        c.zero_async()?;
                    }
                    #[cfg(feature = "vulkan")]
                    fuel_memory::BackendStorage::Vulkan(v) => {
                        let backend = v
                            .backend()
                            .ok_or_else(|| {
                                Error::Msg(
                                    "Op::ZeroFill on Vulkan: input has no \
                                 attached backend handle. Storage must \
                                 come from VulkanBackend::alloc_bytes_handle \
                                 / upload_bytes_handle."
                                        .to_string(),
                                )
                                .bt()
                            })?
                            .clone();
                        backend.fill_bytes_zero(v)?;
                    }
                    #[allow(unreachable_patterns)]
                    other => {
                        return Err(Error::Msg(format!(
                            "Op::ZeroFill: backend not wired ({other:?}); \
                             CPU + CUDA + Vulkan covered, Metal extends \
                             when its byte-storage substrate is ready.",
                        ))
                        .bt());
                    }
                }
            }
            // Adopt the (now-zero) Storage Arc at this node's slot.
            // Downstream readers go through `node_id`; the original
            // input's NodeId is evicted from the cache by the
            // realize loop's destructive_input cleanup.
            cache.insert(item.node_id, src_arc);
            layout_cache.insert(item.node_id, item.output_layout.clone());
            // A4b-1: ZeroFill's CUDA memset is same-stream (and Op::Alloc
            // typically zero-inits, so this is usually elided). Its completion
            // is carried by the realize-end full-stream sync in `to_cpu_bytes`
            // (unchanged) and same-stream ordering — `Ready` is behavior-preserving.
            Ok(CompletionHandle::Ready)
        }
        WorkItemKind::Copy { target_location } | WorkItemKind::Move { target_location } => {
            // Op::Copy / Op::Move { target }: kernel lookup at
            // (OpKind::Copy, [dt, dt], source_backend) — the wrapper
            // downloads from its own residency into a freshly-
            // allocated output on `target_location`. We auto-
            // contiguize the input first so the kernel always sees
            // the logical view's bytes (a transpose-view source
            // materializes into a contiguous buffer before download).
            //
            // Move shares this entire data-movement half; its
            // destructive half (release the source) is driven by the
            // realize loop's `destructive_input` cleanup after this
            // arm returns — see `WorkItemKind::Move`.
            let op_label = if matches!(item.kind, WorkItemKind::Move { .. }) {
                "Move"
            } else {
                "Copy"
            };
            let compiled = item.compiled.as_ref().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: {op_label} work item {:?} has no compiled node",
                    item.node_id,
                ))
                .bt()
            })?;
            if item.inputs.len() != 1 {
                return Err(Error::Msg(format!(
                    "PipelinedExecutor: {op_label} work item {:?} expects 1 input, got {}",
                    item.node_id,
                    item.inputs.len(),
                ))
                .bt());
            }
            let src_id = item.inputs[0];
            let src_arc = cache.get(&src_id).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: {op_label} input {:?} of {:?} not realized",
                    src_id, item.node_id,
                ))
                .bt()
            })?;
            // Multi-output Option C guard: Op::Copy on a bundled
            // producer would silently drop the bundle metadata on
            // the destination, and the auto-contiguize step below
            // would treat the bundle as a flat slot-0-shaped buffer
            // and corrupt non-primary slots. Until the per-backend
            // whole-bundle-Copy hook lands, point callers at the
            // supported per-slot path:
            // Op::View → Op::Copy → Op::ViewOwned.
            //
            // Exception: when the immediate input is an Op::View
            // projection (its layout's byte extent is strictly less
            // than the bundle's total bytes), the View has already
            // narrowed the slab. auto-contiguize walks the View's
            // layout and produces a flat slot-shaped buffer that's
            // correctly sized — no data loss, no slot corruption.
            // The guard only fires when the Copy targets the bundle
            // root directly.
            {
                let guard = src_arc
                    .read()
                    .map_err(|_| poisoned("Copy source storage"))?;
                if guard.is_bundled() {
                    let layout_bytes_for_guard = layout_cache
                        .get(&src_id)
                        .map(|l| l.shape().elem_count() * guard.dtype.size_in_bytes())
                        .unwrap_or(usize::MAX);
                    let bundle_bytes = guard.inner.len_bytes();
                    let is_whole_bundle = layout_bytes_for_guard >= bundle_bytes;
                    if is_whole_bundle {
                        return Err(Error::Msg(format!(
                            "PipelinedExecutor: Op::{op_label} on a bundled producer \
                             ({:?}) is not yet supported (Option C followup). \
                             Use Op::View → Op::Copy → Op::ViewOwned for \
                             per-slot cross-device moves, or wait for the \
                             per-backend whole-bundle-Copy hook.",
                            src_id,
                        ))
                        .bt());
                    }
                }
            }
            let src_layout = layout_cache.get(&src_id).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: {op_label} input {:?} of {:?} has no cached layout",
                    src_id, item.node_id,
                ))
                .bt()
            })?;
            let src_dtype = src_arc
                .read()
                .map_err(|_| poisoned("Copy source storage"))?
                .dtype;
            let src_len_bytes = src_arc
                .read()
                .map_err(|_| poisoned("Copy source storage"))?
                .inner
                .len_bytes();
            let layout_bytes = src_layout.shape().elem_count() * src_dtype.size_in_bytes();
            let bytes_match_shape = src_len_bytes == layout_bytes;
            let already_contig =
                src_layout.is_contiguous() && src_layout.start_offset() == 0 && bytes_match_shape;
            let src_input = if already_contig {
                src_arc
            } else {
                auto_contiguize(&src_arc, &src_layout)?
            };
            let kernel_input_layout = fuel_ir::Layout::contiguous(src_layout.shape().clone());
            // CapturedRun build-out (4b-ζ): in Reuse (capture/replay) mode,
            // reuse the fixed-address buffer recorded during the warm run
            // instead of allocating — device allocation is illegal inside a
            // capture scope. Only a same-device CUDA target_location ever
            // reaches Reuse mode for a Copy node in practice: capture_decode's
            // validation loop rejects every Cpu/Vulkan target_location before
            // a capture scope is ever entered (Move is rejected unconditionally
            // there too), so this arm trusts that invariant rather than
            // re-deriving it — the `DeviceLocation::Cuda` guard below is just
            // belt-and-suspenders for any other Reuse-mode caller.
            let reuse_arc = match &persistent {
                Some(p) if p.mode == PersistentMode::Reuse => match target_location {
                    DeviceLocation::Cuda { .. } => p.map.get(&item.node_id).cloned(),
                    _ => None,
                },
                _ => None,
            };
            // Allocate the output on `target_location`. Phase 2
            // covered D2H (target=Cpu); Phase 3b extends to H2D
            // (target=Cuda / Vulkan from a CPU source). For non-CPU
            // targets the executor's Op::Alloc-style device-handle
            // search applies (find_cuda_device_in_cache /
            // find_vulkan_backend_in_cache); callers seed the cache
            // (e.g. `pipelined_bridge::device_seed_storage`) before
            // realizing.
            // Used only by the Vulkan arm below (alloc_bytes_handle); a
            // default-feature build sees it unused. GAP-229: 68161981's
            // unused_variables --fix renamed it `_n_bytes` at default
            // features and broke `--features vulkan` (E0425). Guard the
            // decline instead of the name so the binding stays usable.
            #[cfg_attr(not(feature = "vulkan"), allow(unused_variables))]
            let n_bytes = item.elem_count * item.dtype.size_in_bytes();
            let input_arcs = vec![src_input];
            let mut output_arcs = if let Some(arc) = reuse_arc {
                vec![arc]
            } else {
                let output = match target_location {
                    DeviceLocation::Cpu => {
                        fuel_memory::alloc_cpu_zeroed(item.dtype, item.elem_count)?
                    }
                    #[cfg(feature = "cuda")]
                    DeviceLocation::Cuda { gpu_id } => {
                        let cuda_dev =
                            find_cuda_device_in_cache(cache, *gpu_id).ok_or_else(|| {
                                Error::Msg(format!(
                                    "Op::{op_label} on Cuda {{ gpu_id: {} }}: no CUDA \
                                 storage in input cache to derive the device \
                                 handle from. The caller must seed the cache \
                                 (e.g. via `fuel-core::pipelined_bridge::\
                                 device_seed_storage`) before realizing an \
                                 H2D Op::{op_label}.",
                                    gpu_id,
                                ))
                                .bt()
                            })?;
                        let cuda_bytes =
                            fuel_cuda_backend::CudaStorageBytes::alloc_uninit(&cuda_dev, n_bytes)?;
                        Storage::new(fuel_memory::BackendStorage::Cuda(cuda_bytes), item.dtype)
                    }
                    #[cfg(not(feature = "cuda"))]
                    DeviceLocation::Cuda { .. } => {
                        return Err(Error::Msg(format!(
                            "Op::{op_label} target Cuda but fuel-storage wasn't built \
                             with --features cuda",
                        ))
                        .bt());
                    }
                    #[cfg(feature = "vulkan")]
                    DeviceLocation::Vulkan { gpu_id } => {
                        let backend =
                            find_vulkan_backend_in_cache(cache, *gpu_id).ok_or_else(|| {
                                Error::Msg(format!(
                                    "Op::{op_label} on Vulkan {{ gpu_id: {} }}: no Vulkan \
                                 storage in input cache to derive the backend \
                                 handle from. The caller must seed the cache \
                                 (e.g. via `fuel-core::pipelined_bridge::\
                                 device_seed_storage`) before realizing an \
                                 H2D Op::{op_label}.",
                                    gpu_id,
                                ))
                                .bt()
                            })?;
                        let vk_bytes = backend.alloc_bytes_handle(n_bytes)?;
                        Storage::new(fuel_memory::BackendStorage::Vulkan(vk_bytes), item.dtype)
                    }
                    #[cfg(not(feature = "vulkan"))]
                    DeviceLocation::Vulkan { .. } => {
                        return Err(Error::Msg(format!(
                            "Op::{op_label} target Vulkan but fuel-storage wasn't built \
                             with --features vulkan",
                        ))
                        .bt());
                    }
                    other => {
                        return Err(Error::Msg(format!(
                            "PipelinedExecutor: Op::{op_label} target_location {:?} not yet \
                             wired (CPU + CUDA + Vulkan covered; Metal extends when \
                             its byte-storage substrate is ready).",
                            other
                        ))
                        .bt());
                    }
                };
                vec![Arc::new(RwLock::new(output))]
            };
            let kernel_layouts = vec![kernel_input_layout, item.output_layout.clone()];
            // A4b-1: defer the wait — return the handle to the realize loop.
            let handle = execute_compiled_with_wait_hint(
                compiled,
                &input_arcs,
                &mut output_arcs,
                &kernel_layouts,
                will_be_waited,
            )?;
            let arc = output_arcs.into_iter().next().expect("one output");
            // CapturedRun build-out (4b-ζ): Record (warm) mode remembers this
            // node's fixed output buffer so the capture run reuses the
            // identical device address. No-op in Reuse mode (the buffer came
            // FROM the map) and in None mode. Mirrors the Kernel arm's
            // Record-mode snapshot (above) exactly.
            if let Some(p) = persistent.as_deref_mut()
                && p.mode == PersistentMode::Record
            {
                p.map.insert(item.node_id, Arc::clone(&arc));
            }
            cache.insert(item.node_id, arc);
            layout_cache.insert(item.node_id, item.output_layout.clone());
            Ok(handle)
        }
        WorkItemKind::ReleaseMarker => {
            // Emit a zero-byte CPU storage at the Release node's
            // slot. Per Op::Release's contract the marker is never
            // read — this exists so any downstream cache lookup of
            // `release_id` resolves rather than failing. The actual
            // deallocation of `inputs[0]` is driven by the realize
            // loop's `destructive_input` cleanup (Phase B).
            let marker = fuel_memory::alloc_cpu_zeroed(item.dtype, 0)?;
            cache.insert(item.node_id, Arc::new(RwLock::new(marker)));
            layout_cache.insert(item.node_id, item.output_layout.clone());
            Ok(CompletionHandle::Ready)
        }
        WorkItemKind::ContiguizeOf { input } => {
            let input_arc = cache.get(input).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: reshape input {:?} of {:?} not realized",
                    input, item.node_id,
                ))
                .bt()
            })?;
            let input_layout = layout_cache.get(input).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: reshape input {:?} of {:?} has no cached layout",
                    input, item.node_id,
                ))
                .bt()
            })?;
            // CapturedRun build-out (4b-γ): in Reuse (capture/replay) mode,
            // reuse the fixed-address buffer recorded during the warm run —
            // device allocation is illegal inside a capture scope. Always
            // write-into when reusing (skip the zero-copy-vs-allocate
            // branch entirely): a given ContiguizeOf node's zero-copy-ness
            // is a static layout/stride property, identical across the warm
            // and capture passes (both replay the same compiled item list
            // with the same static layouts per capture_decode's own
            // design), so if this node ever needed the map entry it
            // allocated on warm too and the entry is guaranteed present.
            let reuse_arc = match &persistent {
                Some(p) if p.mode == PersistentMode::Reuse => p.map.get(&item.node_id).cloned(),
                _ => None,
            };
            let out_arc = if let Some(dest_arc) = reuse_arc {
                contiguize_into(&dest_arc, &input_arc, &input_layout)?;
                dest_arc
            } else if input_layout.is_contiguous() && input_layout.start_offset() == 0 {
                // Zero-copy when the input is already contiguous + zero
                // offset — never allocates, so Record mode has nothing to
                // record for this node (no fixed address to reuse).
                input_arc
            } else {
                // Allocate + copy via the contiguize kernel.
                let arc = auto_contiguize(&input_arc, &input_layout)?;
                if let Some(p) = persistent.as_deref_mut()
                    && p.mode == PersistentMode::Record
                {
                    p.map.insert(item.node_id, Arc::clone(&arc));
                }
                arc
            };
            cache.insert(item.node_id, out_arc);
            layout_cache.insert(item.node_id, item.output_layout.clone());
            // A4b-1: a materializing contiguize (CUDA) is same-stream; its
            // completion is carried by the realize-end full-stream sync in
            // `to_cpu_bytes` (unchanged) — `Ready` here is behavior-preserving.
            Ok(CompletionHandle::Ready)
        }
        WorkItemKind::Kernel => {
            let compiled = item.compiled.as_ref().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: Kernel work item {:?} has no compiled node",
                    item.node_id,
                ))
                .bt()
            })?;
            // Dynamic-M matmul (sparse MoE): resolve a DATA-DETERMINED row
            // count from the producer-bound `produced_syms` and run against
            // the patched (resolved-M) node. `None` on the common path.
            let patched_mm = resolve_deferred_matmul(compiled, produced_syms, item.node_id)?;
            let compiled = patched_mm.as_ref().unwrap_or(compiled);
            // Gather input Arcs from the cache, auto-contiguizing
            // any input whose layout is non-contiguous (typically
            // produced by an upstream metadata-only view op).
            // Today's kernels assume contiguous; this pass keeps
            // that invariant true at every kernel call site.
            //
            // We assemble a parallel `kernel_layouts` vec — the layout
            // of the bytes the kernel actually receives. After
            // auto-contiguize, that's `Layout::contiguous(shape)` for
            // the input's shape; for inputs already contiguous we use
            // the cached layout directly. Output layout comes last.
            let mut input_arcs: Vec<Arc<RwLock<Storage>>> = Vec::with_capacity(item.inputs.len());
            let mut kernel_layouts: Vec<fuel_ir::Layout> =
                Vec::with_capacity(item.inputs.len() + 1);
            // The kernel's `strided_input` cap lets non-contiguous
            // inputs (broadcast, transpose, etc.) flow through without
            // materialization — the kernel walks strides itself. Inputs
            // with non-zero `start_offset` still go through auto-
            // Contiguize today; offset honoring on the byte buffer is
            // a separate concern from stride support.
            let kernel_handles_strided = compiled.caps.strided_input;
            for in_id in &item.inputs {
                let in_arc = cache.get(in_id).cloned().ok_or_else(|| {
                    Error::Msg(format!(
                        "PipelinedExecutor: input {:?} of {:?} not realized",
                        in_id, item.node_id,
                    ))
                    .bt()
                })?;
                let in_layout = layout_cache.get(in_id).cloned().ok_or_else(|| {
                    Error::Msg(format!(
                        "PipelinedExecutor: input {:?} of {:?} has no cached layout",
                        in_id, item.node_id,
                    ))
                    .bt()
                })?;
                // "Already contig" means more than `is_contiguous()`:
                // the LAYOUT's element count must also match the
                // STORAGE's byte count for shape-validating kernels
                // (concat_cpu, etc.) that read `storage.len_bytes()`.
                // A first-N slice view has `is_contiguous() == true`
                // (strides match canonical [1] for shape [N]) but its
                // storage is the parent's full bytes — too big. In
                // that case we auto-contiguize to materialize only
                // the view's portion as a fresh storage.
                //
                // Same constraint applies to strided_ok: kernels that
                // declare strided support can walk strided inputs, but
                // the byte count check at the kernel surface still
                // assumes storage.len_bytes() == layout.elem_count() *
                // dtype.size_bytes(). Until kernels universally trust
                // layout over storage, we conservatively materialize.
                let in_dtype = in_arc.read().map_err(|_| poisoned("input storage"))?.dtype;
                let in_len_bytes = in_arc
                    .read()
                    .map_err(|_| poisoned("input storage"))?
                    .inner
                    .len_bytes();
                let layout_bytes = in_layout.shape().elem_count() * in_dtype.size_in_bytes();
                let bytes_match_shape = in_len_bytes == layout_bytes;
                let already_contig =
                    in_layout.is_contiguous() && in_layout.start_offset() == 0 && bytes_match_shape;
                let strided_ok =
                    kernel_handles_strided && in_layout.start_offset() == 0 && bytes_match_shape;
                if already_contig || strided_ok {
                    input_arcs.push(in_arc);
                    kernel_layouts.push(in_layout);
                } else {
                    let contig_arc = auto_contiguize(&in_arc, &in_layout)?;
                    input_arcs.push(contig_arc);
                    kernel_layouts.push(fuel_ir::Layout::contiguous(in_layout.shape().clone()));
                }
            }
            kernel_layouts.push(item.output_layout.clone());

            // Allocate output on the target backend. For GPU backends
            // we derive the device handle from the first input —
            // every kernel has ≥1 input (Op::Const is handled via
            // ConstAdopt, never via Kernel), and an input on the
            // target backend carries its own device. This avoids
            // threading a device handle through `realize`.
            //
            // Multi-output (Option C, item 3): when the WorkItem
            // carries an `output_bundle`, size the allocation to
            // span every slot's bytes (primary-dtype-element count
            // rounded up from the bundle's total byte span) and
            // attach the bundle metadata after allocation.
            let (alloc_elem_count, bundle_to_attach) = match &item.output_bundle {
                Some(bundle) => {
                    let dtype_bytes = item.dtype.size_in_bytes().max(1);
                    let total_bytes = bundle.iter().fold(0usize, |acc, s| {
                        let end = s.byte_offset.saturating_add(s.len_bytes());
                        acc.max(end)
                    });
                    let elems = total_bytes.div_ceil(dtype_bytes);
                    (elems, Some(Arc::clone(bundle)))
                }
                None => (item.elem_count, None),
            };
            // CapturedRun build-out: in Reuse (capture/replay) mode, reuse the
            // FIXED-ADDRESS buffer recorded during the warm run instead of
            // allocating — device allocation is illegal inside a capture scope,
            // and reusing the same address is what makes the captured graph's
            // baked kernel arguments valid on replay. In Record (warm) or None
            // mode, allocate as usual (below).
            let reuse_arc = match &persistent {
                Some(p) if p.mode == PersistentMode::Reuse => {
                    Some(p.map.get(&item.node_id).cloned().ok_or_else(|| {
                        Error::Msg(format!(
                            "PipelinedExecutor: capture (reuse) mode has no persistent \
                             output buffer for kernel node {:?}; the warm (record) run \
                             must realize every kernel node before capture",
                            item.node_id,
                        ))
                        .bt()
                    })?)
                }
                _ => None,
            };
            let mut output_arcs = if let Some(arc) = reuse_arc {
                vec![arc]
            } else {
                let output = match item.target_backend {
                    BackendId::Cpu => fuel_memory::alloc_cpu_zeroed(item.dtype, alloc_elem_count)?,
                    #[cfg(feature = "cuda")]
                    BackendId::Cuda => {
                        let first_in = input_arcs.first().ok_or_else(|| {
                            Error::Msg(format!(
                                "PipelinedExecutor: kernel {:?} on Cuda has no inputs; \
                             cannot derive device for output allocation",
                                item.node_id,
                            ))
                            .bt()
                        })?;
                        let guard = first_in.read().map_err(|_| poisoned("input storage"))?;
                        let cuda_in = match &guard.inner {
                            fuel_memory::BackendStorage::Cuda(c) => c,
                            other => {
                                return Err(Error::Msg(format!(
                                    "PipelinedExecutor: kernel {:?} target_backend=Cuda but \
                                 input has BackendStorage::{:?}; mixed-backend kernels \
                                 require an explicit Op::Copy first",
                                    item.node_id,
                                    std::mem::discriminant(other),
                                ))
                                .bt());
                            }
                        };
                        let n_bytes = alloc_elem_count * item.dtype.size_in_bytes();
                        let cuda_bytes =
                            fuel_cuda_backend::CudaStorageBytes::alloc(cuda_in.device(), n_bytes)?;
                        fuel_memory::Storage::new(
                            fuel_memory::BackendStorage::Cuda(cuda_bytes),
                            item.dtype,
                        )
                    }
                    #[cfg(feature = "vulkan")]
                    BackendId::Vulkan => {
                        // Mirror the CUDA path: derive the backend handle
                        // from the first input's `VulkanStorageBytes::backend()`.
                        // Vulkan catch-up V.1.B (2026-05-21) — requires
                        // inputs constructed via `alloc_bytes_handle` /
                        // `upload_bytes_handle` so they carry the
                        // `Arc<VulkanBackend>` (mirroring CUDA's
                        // `Arc<CudaDevice>`).
                        let first_in = input_arcs.first().ok_or_else(|| {
                            Error::Msg(format!(
                                "PipelinedExecutor: kernel {:?} on Vulkan has no inputs; \
                             cannot derive backend handle for output allocation",
                                item.node_id,
                            ))
                            .bt()
                        })?;
                        let guard = first_in.read().map_err(|_| poisoned("input storage"))?;
                        let vk_in = match &guard.inner {
                            fuel_memory::BackendStorage::Vulkan(v) => v,
                            other => {
                                return Err(Error::Msg(format!(
                                    "PipelinedExecutor: kernel {:?} target_backend=Vulkan but \
                                 input has BackendStorage::{:?}; mixed-backend kernels \
                                 require an explicit Op::Copy first",
                                    item.node_id,
                                    std::mem::discriminant(other),
                                ))
                                .bt());
                            }
                        };
                        let backend = vk_in.backend().ok_or_else(|| {
                            Error::Msg(format!(
                                "PipelinedExecutor: Vulkan kernel {:?} input has no backend \
                             handle. Storages flowing through the pipelined executor's \
                             Vulkan path must be constructed via \
                             VulkanBackend::alloc_bytes_handle / upload_bytes_handle \
                             (not the legacy alloc_bytes / upload_bytes which leave \
                             VulkanStorageBytes::backend = None).",
                                item.node_id,
                            ))
                            .bt()
                        })?;
                        let n_bytes = alloc_elem_count * item.dtype.size_in_bytes();
                        let vk_bytes = backend.alloc_bytes_handle(n_bytes)?;
                        fuel_memory::Storage::new(
                            fuel_memory::BackendStorage::Vulkan(vk_bytes),
                            item.dtype,
                        )
                    }
                    other => {
                        return Err(Error::Msg(format!(
                            "PipelinedExecutor: target_backend {:?} output allocation \
                         not yet implemented (CPU + CUDA + Vulkan wired; Metal extends later)",
                            other
                        ))
                        .bt());
                    }
                };
                // Attach bundle metadata when this is a multi-output op.
                // The kernel sees the allocated bytes; the bundle lives
                // at the Storage wrapper level so downstream Op::View /
                // Op::ViewOwned can resolve their slot windows.
                let output = match bundle_to_attach {
                    Some(bundle) => output.with_bundle(bundle)?,
                    None => output,
                };
                vec![Arc::new(RwLock::new(output))]
            };

            // A4b-1: defer the wait — return the handle to the realize loop.
            let handle = execute_compiled_with_wait_hint(
                compiled,
                &input_arcs,
                &mut output_arcs,
                &kernel_layouts,
                will_be_waited,
            )?;

            let arc = output_arcs.into_iter().next().expect("one output");

            // CapturedRun build-out: Record (warm) mode remembers this node's
            // fixed output buffer so the capture run reuses the identical
            // device address. No-op in Reuse mode (the buffer came FROM the
            // map) and in None mode.
            if let Some(p) = persistent
                && p.mode == PersistentMode::Record
            {
                p.map.insert(item.node_id, Arc::clone(&arc));
            }

            // Data-determined dynamic-shapes seam (increment 2a): a
            // producer that computes a runtime count publishes it into the
            // per-pass `SymEnv` so downstream ops can consume it as a
            // dynamic extent. `Op::NonZeroIndices` writes its count into
            // bundle slot 1; read it back and bind `count_sym`. This is the
            // FIRST op to bind a *data-determined* symbol mid-realize (all
            // prior binds are host-known before realize). Single-threaded:
            // the executor runs items topologically, so this bind precedes
            // any consumer's execute-time resolution.
            if let OpParams::NonZeroIndices {
                capacity,
                count_sym,
            } = &compiled.op_params
            {
                bind_data_determined_count(&arc, *capacity, *count_sym, produced_syms)?;
            }

            cache.insert(item.node_id, arc);
            layout_cache.insert(item.node_id, item.output_layout.clone());
            Ok(handle)
        }
        WorkItemKind::InplaceKernel { target_idx } => {
            let compiled = item.compiled.as_ref().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: InplaceKernel work item {:?} has no compiled node",
                    item.node_id,
                ))
                .bt()
            })?;
            let target_id = *item.inputs.get(*target_idx).ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: InplaceKernel work item {:?} target_idx={} \
                     out of bounds (inputs.len()={})",
                    item.node_id,
                    target_idx,
                    item.inputs.len(),
                ))
                .bt()
            })?;
            let target_arc = cache.get(&target_id).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: InplaceKernel target {:?} of {:?} not realized",
                    target_id, item.node_id,
                ))
                .bt()
            })?;
            let target_layout = layout_cache.get(&target_id).cloned().ok_or_else(|| {
                Error::Msg(format!(
                    "PipelinedExecutor: InplaceKernel target {:?} of {:?} has no cached layout",
                    target_id, item.node_id,
                ))
                .bt()
            })?;
            // v1 contract: target must be contiguous + zero offset.
            // Strided in-place targets would force the kernel to walk
            // strides for writes too; defer until a concrete consumer
            // needs it (mirrors `Op::WriteSlice`'s v1 dest contract).
            if !target_layout.is_contiguous() || target_layout.start_offset() != 0 {
                return Err(Error::Msg(format!(
                    "InplaceKernel (Phase 3 v1): target {:?} must be contiguous + \
                     zero-offset; got Layout {:?}",
                    target_id, target_layout,
                ))
                .bt());
            }
            // Kernel sees inputs=[non-target IR inputs in order] and
            // outputs=[target_arc]. For unary in-place (1 IR input at
            // target_idx=0) the input vec is empty, matching the
            // original Phase 3e contract. For binary in-place
            // (target_idx=0, 2 IR inputs) the wrapper sees the
            // non-destructive RHS as `inputs[0]`. The wrapper acquires
            // outputs[0]'s write lock and mutates through its
            // bytes_mut(). Layout vector contains the kernel's
            // inputs+outputs layouts in that order.
            let mut input_arcs: Vec<Arc<RwLock<Storage>>> = Vec::new();
            let mut kernel_layouts: Vec<Layout> = Vec::new();
            for (i, &input_id) in item.inputs.iter().enumerate() {
                if i == *target_idx {
                    continue;
                }
                let arc = cache.get(&input_id).cloned().ok_or_else(|| {
                    Error::Msg(format!(
                        "PipelinedExecutor: InplaceKernel non-target input {:?} of {:?} \
                         not realized",
                        input_id, item.node_id,
                    ))
                    .bt()
                })?;
                let lay = layout_cache.get(&input_id).cloned().ok_or_else(|| {
                    Error::Msg(format!(
                        "PipelinedExecutor: InplaceKernel non-target input {:?} of {:?} \
                         has no cached layout",
                        input_id, item.node_id,
                    ))
                    .bt()
                })?;
                input_arcs.push(arc);
                kernel_layouts.push(lay);
            }
            let mut output_arcs = vec![target_arc.clone()];
            kernel_layouts.push(target_layout.clone());
            // A4b-1: defer the wait — return the handle to the realize loop.
            let handle = execute_compiled_with_wait_hint(
                compiled,
                &input_arcs,
                &mut output_arcs,
                &kernel_layouts,
                will_be_waited,
            )?;
            // Adopt the target Arc at this node's slot. The realize
            // loop's destructive_input cleanup evicts the target's own
            // NodeId from the cache afterward.
            cache.insert(item.node_id, target_arc);
            layout_cache.insert(item.node_id, item.output_layout.clone());
            Ok(handle)
        }
    }
}

fn poisoned(what: &'static str) -> Error {
    Error::Msg(format!("PipelinedExecutor: {} poisoned", what)).bt()
}

/// Step E A4b-1: record a node's [`CompletionHandle`] in the executor's
/// per-node handle map.
///
/// `Ready` handles (CPU / Vulkan / view-and-alloc arms) carry no async work, so
/// we DON'T store them — keeping the map to genuinely-pending entries makes the
/// realize-end empty-after-drain assert (open question #6) meaningful and keeps
/// the drain O(pending) rather than O(nodes). A re-used NodeId (in-place adopt
/// re-keys an existing slot) waits the prior handle first so nothing leaks.
#[inline]
fn store_handle(
    handles: &mut HashMap<NodeId, CompletionHandle>,
    node_id: NodeId,
    handle: CompletionHandle,
) {
    if matches!(handle, CompletionHandle::Ready) {
        // A prior pending handle at this slot must still be waited (don't drop a
        // Pending silently). In practice NodeIds are unique per realize, so this
        // is just defensive.
        if let Some(prev) = handles.remove(&node_id) {
            let _ = prev.wait();
        }
        return;
    }
    if let Some(prev) = handles.insert(node_id, handle) {
        let _ = prev.wait();
    }
}

/// Step E A4b-3: the FINER cross-device producer wait.
///
/// Before dispatching a cross-device `Op::Copy` / `Op::Move` WorkItem, the
/// executor calls this on the copy's single source input. It removes the
/// source producer's [`CompletionHandle`] from the map and waits it — so the
/// specific producer P is complete before the D2H reads P's device buffer,
/// WITHOUT draining the whole source device (the conservative pre-A4b-3
/// behavior, where `to_cpu_bytes`'s `device.synchronize()` blocked on ALL
/// in-flight work on that device — including the OTHER sub-DAG's independent
/// work A4b-4 wants to overlap).
///
/// - **CUDA producer** → waits P's recorded `Event` (`cuEventSynchronize`):
///   fires after P + its already-ordered stream predecessors, nothing else.
/// - **Absent / `Ready` producer** (CPU producer, or a Vulkan producer — which
///   under A4b-2 is `Ready` mid-walk because nothing is submitted until
///   realize-end) → no-op. A Vulkan SOURCE's own ordering is handled inside
///   `download_bytes` (it `force_flush`es the open batch then reads), so there
///   is no mid-walk Vulkan handle to wait here; the cross-device Vulkan→CPU hop
///   stays correct without one.
///
/// Removing the handle here (rather than waiting it again at realize-end) keeps
/// `drain_handles`' post-drain empty-map assert meaningful and avoids a
/// double-wait. Waiting it twice would be harmless (`cuEventSynchronize` on an
/// already-signalled event is a fast no-op), but the consume-once contract is
/// cleaner. The producing node's *storage* stays in the cache for downstream
/// readers; only its completion handle is consumed.
fn wait_producer_handle(
    handles: &mut HashMap<NodeId, CompletionHandle>,
    producer: NodeId,
) -> Result<()> {
    if let Some(handle) = handles.remove(&producer) {
        handle.wait()?;
    }
    Ok(())
}

/// Step E A4b-1: realize-end drain — wait every outstanding async handle, then
/// clear the map.
///
/// **Event-recording / wait strategy (A4b-1 §1.2).** CUDA has one stream per
/// device, and an `Event` recorded after a node's launch signals when that
/// node *and all prior stream work* have completed. So once the
/// latest-recorded event on a device has signalled, every earlier event on that
/// device is already signalled too: waiting them is a non-blocking
/// `cuEventQuery` fast-path, not N host stalls. We therefore wait every stored
/// handle (correct + simple).
///
/// **Open question #4 — "event only where waited" — RESOLVED.** Recording an
/// `Event` for every node was measured negligible on a 32-op synthetic, but
/// real decode graphs run 150-200+ ops; `pipelined`'s `WaitSet` pre-scan (see
/// `compiler_work_and_wait_set_for` / `build_wait_set`) now restricts event
/// production to the nodes some later cross-device `Copy`/`Move` will
/// individually wait — same-device-only ("elided") nodes get
/// `CompletionHandle::Ready` with no `cuEventCreate`/`cuEventRecord` at all
/// (`execute_compiled_with_wait_hint`). This drain therefore now retires only
/// the (typically few) wait-set handles that were never consumed mid-walk by
/// `wait_producer_handle`; it is no longer sufficient on its own to guarantee
/// every enqueued CUDA kernel has completed — `sync_active_cuda_devices` below
/// covers the elided remainder. We keep a per-node handle (rather than a
/// single per-device event) for the wait-set members because A4b-3's finer
/// cross-device wait needs to wait a *specific producer's* completion — the
/// per-node handle is that seam.
fn drain_handles(handles: &mut HashMap<NodeId, CompletionHandle>) -> Result<()> {
    for (_node, handle) in handles.drain() {
        handle.wait()?;
    }
    // Open question #6: once handles are stored instead of `wait`ed inline, the
    // `#[must_use]` on `CompletionHandle` can't catch a leaked (never-waited)
    // handle at the call site. The drain consumes the whole map, so it is empty
    // here by construction; assert it to catch a future leak (e.g. a new
    // dispatch path that forgets to thread its handle through `store_handle`).
    debug_assert!(
        handles.is_empty(),
        "A4b-1: handle map must be empty after drain"
    );
    Ok(())
}

/// Step E A4b open question #4 step 3 — the realize-end complement to the
/// "event only where waited" elision. Because most CUDA nodes in a long
/// single-device chain no longer record an `Event` (see `drain_handles`'s
/// doc), this does ONE blanket `cuStreamSynchronize` per CUDA device that
/// still holds LIVE storage in `cache` — the realize target(s) plus any
/// surviving intermediates — instead of relying on per-node handles to cover
/// them.
///
/// **Why this is safe + sufficient.** CUDA has one stream per device, and
/// this executor dispatches strictly sequentially (one work item at a time,
/// in topological order) — so by the time a live node's own kernel was
/// launched, every same-device ancestor it depends on was ALREADY enqueued
/// earlier on the SAME stream. Stream ordering (not a host wait) guarantees
/// those ancestors run to completion no later than the live node's own
/// kernel. Syncing the live node's device therefore flushes its entire
/// same-device dependency chain, elided or not. Cross-device ancestors are
/// unaffected by elision — they always went through an `Op::Copy`/`Move`,
/// whose producer is in the `WaitSet` and was already waited explicitly at
/// the cross-device boundary (`wait_producer_handle`), mid-walk.
///
/// **Why evicted (non-live) intermediates don't need this.** A node whose
/// storage was evicted before realize-end (the `destructive_input` cleanup)
/// is never read by anything this realize returns; A3's stream-ordered
/// alloc/free already makes its eventual completion safe without a host
/// wait (the driver serializes the free after the freeing stream position,
/// same as any other stream-ordered op) — nothing to synchronize for.
///
/// No-op when `cache` holds no CUDA storage (CPU/Vulkan-only realize).
#[cfg(feature = "cuda")]
fn sync_active_cuda_devices(cache: &StorageCache) -> Result<()> {
    let mut devices: HashMap<fuel_ir::DeviceLocation, fuel_cuda_backend::CudaDevice> =
        HashMap::new();
    for arc in cache.values() {
        let guard = arc
            .read()
            .map_err(|_| poisoned("cache storage (cuda realize-end sync scan)"))?;
        if let fuel_memory::BackendStorage::Cuda(c) = &guard.inner {
            devices
                .entry(c.device().location())
                .or_insert_with(|| c.device().clone());
        }
    }
    for dev in devices.values() {
        dev.synchronize()?;
    }
    Ok(())
}

/// Non-CUDA builds never produce CUDA storage, so there is nothing to sync —
/// keeps the call site unconditional across both feature configurations.
#[cfg(not(feature = "cuda"))]
fn sync_active_cuda_devices(_cache: &StorageCache) -> Result<()> {
    Ok(())
}

/// Search the input cache for any `BackendStorage::Cuda(s)` whose
/// device matches `gpu_id`. Returns a clone of the `CudaDevice` Arc
/// the storage carries — the caller uses it to call
/// `CudaStorageBytes::alloc` for `Op::Alloc` on Cuda. Returns `None`
/// if no matching storage exists.
///
/// Phase 3a of bridge-retirement: callers seed the cache with a 0-byte
/// CUDA storage (`fuel-core::pipelined_bridge::device_seed_storage`)
/// so this lookup succeeds for the first Op::Alloc; subsequent
/// Op::Allocs see prior Op::Alloc outputs in the cache.
#[cfg(feature = "cuda")]
fn find_cuda_device_in_cache(
    cache: &StorageCache,
    gpu_id: usize,
) -> Option<fuel_cuda_backend::CudaDevice> {
    for arc in cache.values() {
        let guard = arc.read().ok()?;
        if let fuel_memory::BackendStorage::Cuda(c) = &guard.inner {
            // CudaDevice carries its DeviceLocation (gpu_id ordinal)
            // through `location()`; match against the target gpu_id.
            // Multi-GPU future work can refine the match; today's
            // single-GPU setups always match.
            if matches!(c.device().location(), DeviceLocation::Cuda { gpu_id: g } if g == gpu_id) {
                return Some(c.device().clone());
            }
        }
    }
    None
}

/// Step E A4b-2: a Vulkan [`Completion`](crate::compiled::Completion) over an
/// in-flight (submitted, not-yet-waited) batch. `wait` blocks on the batch's
/// fence (`vkWaitForFences`) — which fires when the whole submitted command
/// buffer has retired on the single compute queue — then releases the batch
/// (frees the CB / descriptor sets / transient buffers / retired pool, all now
/// idle on the GPU).
///
/// The fence is per-BATCH, not per-node: one `submit_pending` flushes every op
/// recorded since the last submit, so this single handle covers a contiguous run
/// of Vulkan nodes (the A2 batching win, preserved — fewer fences, larger
/// submissions). `backend` keeps the [`VulkanBackend`] alive so the
/// post-fence pool retire + the `DeviceInner` the batch's resources reference
/// outlive the wait.
///
/// `Send` (required by `Completion`): every owned field is `Send` — `Fence` /
/// `CommandBuffer` / `DescriptorSet` / `Buffer` / `Allocation` / `CommandPool`
/// are all `Send` in vulkane (Arc-of-`DeviceInner` + explicit `unsafe impl`s),
/// and `Arc<VulkanBackend>` is `Send + Sync`.
/// Step E B1: `batch` is an `Option` ONLY so [`Drop`] can coexist with the
/// by-value `into_wait` consume. A type with a `Drop` impl cannot be
/// destructured by move (`let VulkanCompletion { backend, batch } = self`,
/// E0509), so `into_wait` `take()`s the batch out of the `Option` instead of
/// moving it out of the struct, then lets the normal `Drop` run. In every
/// reachable state the batch is `Some` until exactly one `into_wait` consumes it.
#[cfg(feature = "vulkan")]
struct VulkanCompletion {
    backend: Arc<fuel_vulkan_backend::VulkanBackend>,
    batch: Option<fuel_vulkan_backend::SubmittedBatch>,
    /// The device this submitted batch counts against in the in-flight counter
    /// (Step E B1). `Drop` decrements this slot.
    loc: fuel_ir::DeviceLocation,
}

/// Step E A4b-4: the element type of the executor's in-flight-Vulkan-batch list.
/// A [`VulkanCompletion`] on a Vulkan build; a zero-sized placeholder otherwise
/// so the realize loops carry one `Vec<InflightVulkan>` local without `cfg` on
/// every use site (the helpers `eager_submit_all_vulkan` / `drain_inflight_vulkan`
/// are the only code that ever pushes/drains it, and they are `cfg`-gated).
#[cfg(feature = "vulkan")]
type InflightVulkan = VulkanCompletion;
#[cfg(not(feature = "vulkan"))]
type InflightVulkan = ();

/// Step E A4b-4: is the cross-device copy's SOURCE buffer Vulkan-resident?
///
/// Used at the cross-device `Op::Copy`/`Op::Move` boundary to decide whether the
/// executor must wait the in-flight Vulkan batches before the copy. A Vulkan→CPU
/// D2H reads the source on the host, and the Vulkan `download` path only
/// `force_flush`es the OPEN batch (never the executor's eagerly-submitted
/// in-flight ones), so for a VULKAN source we must `drain_inflight_vulkan` first
/// to read completed data. For a CUDA / CPU source we must NOT wait Vulkan — the
/// copy doesn't read any Vulkan buffer, and waiting Vulkan there would needlessly
/// serialize the independent Vulkan sub-DAG (killing the overlap A4b-4 exists to
/// win). Returns `false` when the producer's storage isn't in the cache (treated
/// as not-Vulkan → no Vulkan wait; the CUDA/CPU producer handle still covers it).
#[cfg(feature = "vulkan")]
fn copy_source_is_vulkan(cache: &StorageCache, producer: NodeId) -> Result<bool> {
    if let Some(arc) = cache.get(&producer) {
        let guard = arc
            .read()
            .map_err(|_| poisoned("storage lock probing copy-source backend"))?;
        return Ok(matches!(
            guard.inner,
            fuel_memory::BackendStorage::Vulkan(_)
        ));
    }
    Ok(false)
}

#[cfg(not(feature = "vulkan"))]
#[inline]
fn copy_source_is_vulkan(_cache: &StorageCache, _producer: NodeId) -> Result<bool> {
    Ok(false)
}

#[cfg(feature = "vulkan")]
impl crate::compiled::Completion for VulkanCompletion {
    fn wait(self: Box<Self>) -> Result<()> {
        // Move the whole `VulkanCompletion` out of the `Box` (allowed even with a
        // `Drop` impl — it's a full move, not a destructure) and consume by value.
        (*self).into_wait()
    }
}

#[cfg(feature = "vulkan")]
impl VulkanCompletion {
    /// Build the handle for an eagerly-submitted Vulkan batch and register it as
    /// one in-flight batch on `loc` (Step E B1). The matching decrement is in
    /// [`Drop`]. `loc` is the backend's `DeviceLocation::Vulkan { gpu_id }`.
    fn new(
        backend: Arc<fuel_vulkan_backend::VulkanBackend>,
        batch: fuel_vulkan_backend::SubmittedBatch,
        loc: fuel_ir::DeviceLocation,
    ) -> Self {
        crate::dispatch::inflight_inc(loc);
        Self {
            backend,
            batch: Some(batch),
            loc,
        }
    }

    /// Block on this batch's fence, retire pools, then free the batch. Consumes
    /// `self` by value (the by-value sibling of the boxed `Completion::wait`),
    /// used by the executor's `inflight_vulkan` drain (A4b-4) where we hold the
    /// `VulkanCompletion` unboxed in a `Vec`.
    ///
    /// Step E B1: `take()`s the batch out of the `Option` (NOT a destructure
    /// move — that's illegal with a `Drop` impl) and waits it; the in-flight
    /// decrement then rides `self`'s `Drop` as this returns. So a wait-consume
    /// and a bare drop (e.g. an error unwinding before the wait) BOTH decrement
    /// exactly once, and `into_wait` never double-counts.
    fn into_wait(mut self) -> Result<()> {
        // `wait_submitted` consumes the batch by value: waits the fence, retires
        // pools, then drops the batch (UAF-safe — nothing the in-flight CB
        // references frees before the fence signals). After a `new` the batch is
        // always `Some`; `take` leaves `None` so `Drop` doesn't touch it again.
        if let Some(batch) = self.batch.take() {
            self.backend.wait_submitted(batch)?;
        }
        Ok(())
        // `self` drops here → `inflight_dec(self.loc)`.
    }

    /// Step E A2.1 (deferred-deletion): retain an evicted DATA buffer until THIS
    /// in-flight batch's fence signals, then free it post-fence (on the batch's
    /// `Drop` — guaranteed post-fence by follow-on #2). Delegates to
    /// [`fuel_vulkan_backend::SubmittedBatch::retain_buffer`].
    ///
    /// The executor calls this for every in-flight batch when it evicts a Vulkan
    /// buffer those batches' CBs may read, INSTEAD of host-blocking on a drain.
    /// `batch` is `Some` in every reachable mid-walk state (it is `take`n only by
    /// `into_wait`/`wait`, which consume the whole `VulkanCompletion`), so the
    /// retain always lands on a live in-flight batch.
    fn retain_data(&mut self, buf: std::sync::Arc<fuel_vulkan_backend::VulkanBuffer>) {
        if let Some(batch) = self.batch.as_mut() {
            batch.retain_buffer(buf);
        }
    }
}

#[cfg(feature = "vulkan")]
impl Drop for VulkanCompletion {
    fn drop(&mut self) {
        // The single in-flight decrement for a Vulkan batch — fires whether the
        // batch was waited (`into_wait` / `wait`) or merely dropped (error
        // unwind), so the per-device count returns to baseline. Relaxed +
        // saturating (a scheduling hint, never a correctness gate).
        crate::dispatch::inflight_dec(self.loc);
    }
}

/// Step E A4b-4 — eager Vulkan submission during the walk (the OVERLAP enabler).
///
/// Submits each DISTINCT Vulkan backend's currently-open batch WITHOUT waiting,
/// pushing the resulting in-flight [`VulkanCompletion`]s onto `inflight` so the
/// executor waits them later (at the in-flight-lifetime guard / realize-end). An
/// empty open batch yields `None` and is skipped (idempotent).
///
/// This is what makes a Vulkan sub-DAG actually START on the iGPU while the
/// executor records/dispatches the next (CUDA) chunk — without it the Vulkan
/// batch sits merely *recorded* until realize-end and never overlaps the CUDA
/// stream (design §5). Called ONLY on a genuinely multi-backend realize (gated by
/// the executor's `multi_backend` flag); on a single-device graph it is never
/// reached, so pure-Vulkan stays byte-identical to A4b-2 (one realize-end submit).
///
/// **Whole-chunk granularity (UAF/race-safety, design §4 + the Vulkan
/// single-queue OOO-completion semantics).** We submit only at chunk boundaries
/// and before a cross-device wait — never mid-chunk — so the batch handed to the
/// GPU is a COMPLETE contiguous run of Vulkan nodes with all its intra-CB
/// dependency barriers (`recorder.rs` `vkCmdPipelineBarrier`) already in place.
/// Vulkan only guarantees that batches on one queue BEGIN in submission order;
/// they may overlap / complete out of order, so a split mid-chunk would lose the
/// write-before-read ordering for ops that read an earlier op's buffer in the
/// SAME chunk. By submitting whole chunks and re-syncing (`drain_inflight_vulkan`)
/// before any later Vulkan op records, before any cross-device copy, and before
/// any Vulkan-referenced eviction, no in-flight batch is ever read or freed while
/// still running.
#[cfg(feature = "vulkan")]
fn eager_submit_all_vulkan(
    cache: &StorageCache,
    inflight: &mut Vec<VulkanCompletion>,
) -> Result<()> {
    // Collect distinct Vulkan backends referenced by the cache (dedup by gpu_id).
    let mut seen: Vec<usize> = Vec::new();
    let mut backends: Vec<Arc<fuel_vulkan_backend::VulkanBackend>> = Vec::new();
    for arc in cache.values() {
        let guard = arc
            .read()
            .map_err(|_| poisoned("storage lock during eager vulkan submit"))?;
        if let fuel_memory::BackendStorage::Vulkan(v) = &guard.inner
            && let Some(backend) = v.backend()
            && !seen.contains(&backend.gpu_id)
        {
            seen.push(backend.gpu_id);
            backends.push(backend.clone());
        }
    }
    for backend in backends {
        // Step E B1: the per-device location for the in-flight counter. A batch
        // submitted here is one in-flight unit on this iGPU until its handle's
        // `Drop` (after `drain_inflight_vulkan` / realize-end).
        let loc = DeviceLocation::Vulkan {
            gpu_id: backend.gpu_id,
        };
        if let Some(batch) = backend.submit_pending()? {
            inflight.push(VulkanCompletion::new(backend, batch, loc));
        }
    }
    Ok(())
}

#[cfg(not(feature = "vulkan"))]
#[inline]
fn eager_submit_all_vulkan(
    _cache: &StorageCache,
    _inflight: &mut Vec<InflightVulkan>,
) -> Result<()> {
    Ok(())
}

/// Step E A4b-4 — wait every in-flight (eagerly-submitted) Vulkan batch, then
/// clear the list. The re-sync half of [`eager_submit_all_vulkan`].
///
/// Each `into_wait` blocks on the batch fence (`vkWaitForFences`), retires the
/// backend's descriptor pools, and frees the batch (CB / descriptor sets /
/// transient buffers / retired command pool — all now idle on the GPU). After
/// this returns, NO submitted Vulkan command buffer is still executing, so it is
/// safe to (a) record a new Vulkan op that may read a just-produced buffer,
/// (b) D2H-copy a Vulkan buffer on the host, or (c) free a buffer a batch read.
///
/// Because the eager submit happened at the PRIOR chunk boundary and an
/// intervening (e.g. CUDA) chunk has since been recorded/dispatched, the fence is
/// typically already signalled here — the wait is a fast no-op, and the iGPU work
/// genuinely overlapped the other device's stream. Waiting the Vulkan fence does
/// NOT touch the CUDA stream, so the other device keeps running (concurrency
/// preserved — design §5 "the per-device guards don't cross-stall").
#[cfg(feature = "vulkan")]
fn drain_inflight_vulkan(inflight: &mut Vec<VulkanCompletion>) -> Result<()> {
    for vc in inflight.drain(..) {
        vc.into_wait()?;
    }
    Ok(())
}

#[cfg(not(feature = "vulkan"))]
#[inline]
fn drain_inflight_vulkan(_inflight: &mut Vec<InflightVulkan>) -> Result<()> {
    Ok(())
}

/// Step E A4b-2: the realize-end Vulkan drain, SPLIT into submit-then-wait.
///
/// Replaces A2's `force_flush_all_vulkan` (which submitted + waited atomically
/// inside `force_flush`). For each DISTINCT Vulkan backend referenced by the
/// cache, `submit_pending` submits the still-open batch WITHOUT waiting (→ a
/// `SubmittedBatch`), wraps it in the executor's [`VulkanCompletion`] handle, and
/// `wait`s it. Net for a pure-Vulkan realize: the batch still accumulates across
/// the whole walk (the per-op path keeps returning `Ready`; no eager submit —
/// that is A4b-4) and is submitted ONCE here, then waited — **byte-identical to
/// A2**, same single submission at realize-end, just split through the handle.
///
/// Idempotent across repeated cache arcs: the first `submit_pending` on a backend
/// returns `Some(batch)`; subsequent calls on the same backend see an empty batch
/// and return `None` (skipped) — exactly the `force_flush` idempotence A2 relied
/// on. No-op for a non-Vulkan build.
#[cfg(feature = "vulkan")]
fn drain_vulkan_pending(cache: &StorageCache) -> Result<()> {
    use crate::compiled::CompletionHandle;
    for arc in cache.values() {
        let backend = {
            let guard = arc
                .read()
                .map_err(|_| poisoned("storage lock during vulkan submit_pending"))?;
            match &guard.inner {
                fuel_memory::BackendStorage::Vulkan(v) => v.backend().cloned(),
                _ => None,
            }
        };
        let Some(backend) = backend else { continue };
        // Submit the open batch (if any) without waiting, then wait via the
        // completion handle. Empty batch → None → nothing to wait.
        // Step E B1: `new` increments the in-flight counter for this device;
        // `handle.wait()` waits + drops the `VulkanCompletion`, decrementing it
        // — a balanced submit/retire even on this transient realize-end path.
        let loc = DeviceLocation::Vulkan {
            gpu_id: backend.gpu_id,
        };
        if let Some(batch) = backend.submit_pending()? {
            let handle: CompletionHandle =
                CompletionHandle::Pending(Box::new(VulkanCompletion::new(backend, batch, loc)));
            handle.wait()?;
        }
    }
    Ok(())
}

#[cfg(not(feature = "vulkan"))]
fn drain_vulkan_pending(_cache: &StorageCache) -> Result<()> {
    Ok(())
}

/// Step E A2: force the Vulkan backend behind `arc` (if any) to submit + WAIT
/// its open batch — the blocking drain before freeing a buffer the batch may
/// reference (a destructive eviction) so a recorded-but-unsubmitted command never
/// reads freed memory. No-op for non-Vulkan storage, an empty batch, or a
/// non-Vulkan build.
///
/// **Retained for the SINGLE-DEVICE path (byte-identical to pre-A2.1).** A2.1's
/// deferred-deletion ([`defer_evicted_vulkan_buffer`]) replaces this only on a
/// MULTI-backend realize, where eager-submit + in-flight tracking + the
/// realize-end `drain_inflight_vulkan` are all live. On a single-device Vulkan
/// realize the A4b machinery is gated off (`multi_backend == false`): the
/// cross-batch RAW guard (`pipelined.rs:832`) does not run, so a later Vulkan op
/// reading a buffer an eager-submitted batch wrote could race (separate
/// submissions only BEGIN in order, may overlap). The blocking `force_flush`
/// (submit + wait) is the coherent eviction drain there — exactly the original
/// behavior — so single-device stays byte-identical and UAF-safe.
#[cfg(feature = "vulkan")]
fn force_flush_vulkan(arc: &Arc<RwLock<Storage>>) -> Result<()> {
    let guard = arc
        .read()
        .map_err(|_| poisoned("storage lock during vulkan force_flush"))?;
    if let fuel_memory::BackendStorage::Vulkan(v) = &guard.inner
        && let Some(backend) = v.backend()
    {
        backend.force_flush()?;
    }
    Ok(())
}

#[cfg(not(feature = "vulkan"))]
fn force_flush_vulkan(_arc: &Arc<RwLock<Storage>>) -> Result<()> {
    Ok(())
}

/// Step E A2.1 — DEFERRED-DELETION of a Vulkan data buffer evicted while Vulkan
/// work is in flight (the throughput refinement that REPLACES the eviction-time
/// drain on a MULTI-backend realize). Called at the per-NodeId destructive
/// eviction (`Op::Release`/`Op::Move` cleanup) for the buffer about to leave the
/// cache, INSTEAD of `drain_inflight_vulkan` + `force_flush` (two host-blocking
/// fence waits) — so the iGPU work the executor was overlapping with CUDA is not
/// stalled at every eviction.
///
/// Two classes of Vulkan reader can hold a reference to `destroyed`'s buffer at
/// the eviction:
///   1. the **in-flight** `SubmittedBatch`es in `inflight` (submitted CBs, each
///      fenced), and
///   2. the **open**, not-yet-submitted recorder batch (recorded commands, NO
///      fence yet).
///
/// We make BOTH fence-gated without blocking the host:
///   - `eager_submit_all_vulkan` submits the open batch WITHOUT waiting, turning
///     it into one more in-flight `SubmittedBatch` (now fenced) pushed onto
///     `inflight`. (Whole-chunk granularity — UAF/race-safe per its contract; at a
///     destructive eviction the open batch is a contiguous Vulkan run.)
///   - We then move a refcount clone of the evicted buffer's `Arc<VulkanBuffer>`
///     into EVERY in-flight batch (conservative: we cannot cheaply map
///     buffer→batch, exactly the reason the old drain waited *all* of them). Each
///     batch frees its retained clone POST-fence on `Drop` (follow-on #2).
///
/// After this returns the caller `cache.remove`s `destroyed`; the cache's
/// `Arc<RwLock<Storage>>` drops, but the buffer's recycler
/// (`VulkanBuffer::drop`) reclaims it only when its LAST `Arc` drops — and the
/// retained clones keep it alive until their owning batches' fences signal. So the
/// GPU's last read of the buffer strictly precedes the free (NO use-after-free),
/// the host never blocks at the eviction (the iGPU keeps overlapping CUDA — the
/// throughput win), and every batch (hence every retained buffer) is drained by
/// realize-end at the latest (`drain_inflight_vulkan`, `pipelined.rs:916/1238`,
/// runs unconditionally → NO leak).
///
/// **Multi-backend only** (the call site gates `if multi_backend`). On a
/// single-device realize the open batch is drained by the blocking
/// [`force_flush_vulkan`] instead (byte-identical to pre-A2.1; see its doc for
/// why the deferred path is unsafe without the cross-batch RAW guard). No-op for a
/// non-Vulkan storage, an empty open + no in-flight batches, or a non-Vulkan build.
#[cfg(feature = "vulkan")]
fn defer_evicted_vulkan_buffer(
    cache: &StorageCache,
    destroyed: NodeId,
    inflight: &mut Vec<VulkanCompletion>,
) -> Result<()> {
    // Pull the evicted buffer's device Arc out of the cache BEFORE it is removed.
    // Non-Vulkan storage, a host-evicted (no device buffer) Vulkan storage, or an
    // absent entry → nothing to retain (no Vulkan reader can hold a device buffer).
    let buf = {
        let Some(arc) = cache.get(&destroyed) else {
            return Ok(());
        };
        let guard = arc
            .read()
            .map_err(|_| poisoned("storage lock during deferred-eviction retain"))?;
        match &guard.inner {
            fuel_memory::BackendStorage::Vulkan(v) => v.device_buffer_arc(),
            _ => None,
        }
    };
    let Some(buf) = buf else { return Ok(()) };

    // The OPEN-batch case: eager-submit (no wait) so the recorded-but-unsubmitted
    // commands that may read `destroyed` become a fenced in-flight batch on
    // `inflight`. This NEVER waits — it only flushes the open batch to the GPU.
    eager_submit_all_vulkan(cache, inflight)?;

    // Retain a clone of the evicted buffer in EVERY in-flight batch (now including
    // the just-submitted open batch). Each frees it POST-fence on `Drop` — the
    // buffer outlives the GPU's last read with no host block. Conservative over
    // all batches (no cheap buffer→batch map); over-retention only lengthens the
    // lifetime, never a UAF, and all batches drain by realize-end.
    for vc in inflight.iter_mut() {
        vc.retain_data(buf.clone());
    }
    // Step E A2.1 — throughput-observability. Count a DEFERRED retain that
    // actually replaced a host block: an evicted Vulkan buffer retained onto ≥1
    // in-flight batch (the open-batch eager-submit and/or pre-existing in-flight
    // batches). Pre-A2.1 every such site host-blocked (`force_flush` and/or
    // `drain_inflight_vulkan`); now it does not. A test reads
    // [`deferred_eviction_retains`] to assert the deferred path fired without a
    // drain (deterministic — no wall-clock, which is noisy on this box). When
    // `inflight` is empty here there was genuinely no in-flight/open Vulkan reader
    // to wait for, so no host block was avoided (and none was needed) — not counted.
    if !inflight.is_empty() {
        DEFERRED_EVICTION_RETAINS.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
    }
    Ok(())
}

/// Step E A2.1 — cumulative count of deferred-deletion retains that replaced a
/// host-blocking eviction drain (an evicted Vulkan buffer retained onto ≥1
/// in-flight batch instead of waiting it). Process-global, `Relaxed` — a pure
/// test/telemetry observation, never a correctness gate (mirrors the B1 in-flight
/// counter idiom). Read by the live deferred-eviction throughput test.
#[cfg(feature = "vulkan")]
pub static DEFERRED_EVICTION_RETAINS: std::sync::atomic::AtomicUsize =
    std::sync::atomic::AtomicUsize::new(0);

/// Read the A2.1 deferred-eviction-retain counter (see
/// [`DEFERRED_EVICTION_RETAINS`]). `pub` so the live throughput test can assert a
/// deferred retain fired (the eviction did NOT host-block on a Vulkan fence).
#[cfg(feature = "vulkan")]
pub fn deferred_eviction_retains() -> usize {
    DEFERRED_EVICTION_RETAINS.load(std::sync::atomic::Ordering::Relaxed)
}

#[cfg(not(feature = "vulkan"))]
#[inline]
fn defer_evicted_vulkan_buffer(
    _cache: &StorageCache,
    _destroyed: NodeId,
    _inflight: &mut Vec<InflightVulkan>,
) -> Result<()> {
    Ok(())
}

/// Vulkan counterpart of [`find_cuda_device_in_cache`]. Returns an
/// `Arc<VulkanBackend>` clone derived from any cached Vulkan storage
/// whose backend's `gpu_id` matches.
#[cfg(feature = "vulkan")]
fn find_vulkan_backend_in_cache(
    cache: &StorageCache,
    gpu_id: usize,
) -> Option<std::sync::Arc<fuel_vulkan_backend::VulkanBackend>> {
    for arc in cache.values() {
        let guard = arc.read().ok()?;
        if let fuel_memory::BackendStorage::Vulkan(v) = &guard.inner
            && let Some(backend) = v.backend()
            && backend.gpu_id == gpu_id
        {
            return Some(backend.clone());
        }
    }
    None
}

/// Materialize a contiguous Storage Arc from a non-contiguous one.
/// Allocates a fresh buffer on the input's backend and copies the
/// strided / offset / broadcast input into it via the backend's
/// contiguize kernel. The returned Arc is a brand-new buffer; the
/// caller is responsible for replacing the cache entry only for the
/// duration of one kernel call (the upstream view op's output stays
/// in the cache so other consumers still see the strided view).
///
/// Stage 4 of Layout-on-Node — auto-Contiguize.
fn auto_contiguize(arc: &Arc<RwLock<Storage>>, layout: &Layout) -> Result<Arc<RwLock<Storage>>> {
    let in_guard = arc
        .read()
        .map_err(|_| poisoned("input storage lock during auto_contiguize"))?;
    let dtype = in_guard.dtype;
    let dtype_size = dtype.size_in_bytes();
    let new_storage = match &in_guard.inner {
        fuel_memory::BackendStorage::Cpu(c) => {
            let new_bytes = fuel_cpu_backend::byte_kernels::contiguize_cpu(c, layout, dtype_size)?;
            Storage::new(fuel_memory::BackendStorage::Cpu(new_bytes), dtype)
        }
        #[cfg(feature = "cuda")]
        fuel_memory::BackendStorage::Cuda(c) => {
            // Native baracuda contiguize (alpha.29). Byte-width-
            // dispatched: 1/2/4/8/16 byte elements all route to the
            // appropriate kernel. Handles signed strides (Flip),
            // zero strides (BroadcastTo), and non-zero element offset.
            //
            // Three host-side fast paths bake into the baracuda
            // launchers: already-contiguous → single cuMemcpyDtoDAsync;
            // innermost-stride-1 → per-outer-coord run copy; generic
            // → one thread per output element. Retires the prior
            // D2H → CPU contiguize_cpu → H2D fallback (two device
            // round-trips per non-contig input).
            let contig = fuel_cuda_backend::baracuda::contiguize::contiguize_to_fresh(
                c, layout, dtype_size,
            )?;
            Storage::new(fuel_memory::BackendStorage::Cuda(contig), dtype)
        }
        #[cfg(feature = "vulkan")]
        fuel_memory::BackendStorage::Vulkan(v) => {
            // V.1.B stopgap: D2H → CPU contiguize_cpu → H2D. Mirrors
            // the pre-alpha.29 CUDA fallback. V.3.2 of the Vulkan
            // catch-up writes a native Slang contiguize kernel (with
            // signed-stride support that strided_copy.slang currently
            // lacks) and replaces this arm with a direct
            // VulkanStorageBytes → VulkanStorageBytes path.
            let backend = v.backend().ok_or_else(|| {
                Error::Msg(
                    "auto_contiguize: Vulkan input has no backend handle. \
                     Storages flowing through the pipelined executor's Vulkan \
                     path must be constructed via VulkanBackend::alloc_bytes_handle \
                     / upload_bytes_handle (not the legacy alloc_bytes / \
                     upload_bytes which leave VulkanStorageBytes::backend = None)."
                        .to_string(),
                )
                .bt()
            })?;
            let host_bytes = backend.download_bytes(v)?;
            let host_cpu = fuel_cpu_backend::byte_storage::CpuStorageBytes::from_bytes(&host_bytes);
            let contig_cpu =
                fuel_cpu_backend::byte_kernels::contiguize_cpu(&host_cpu, layout, dtype_size)?;
            let host_contig_bytes = contig_cpu.bytes().to_vec();
            let vk_bytes = backend.upload_bytes_handle(&host_contig_bytes)?;
            Storage::new(fuel_memory::BackendStorage::Vulkan(vk_bytes), dtype)
        }
        #[allow(unreachable_patterns)]
        _ => {
            return Err(Error::Msg(
                "auto_contiguize: backend not wired (CPU + CUDA + Vulkan are \
                 wired today; Metal extends this match when its byte-storage \
                 substrate lands)"
                    .to_string(),
            )
            .bt());
        }
    };
    Ok(Arc::new(RwLock::new(new_storage)))
}

/// Write-into sibling of [`auto_contiguize`] (CapturedRun executor
/// build-out, 4b-γ): materializes a contiguous copy of `arc` at `layout`
/// INTO `dest`'s EXISTING buffer instead of allocating a fresh one — the
/// path `WorkItemKind::ContiguizeOf`'s Reuse-mode branch needs (device
/// allocation is illegal inside a capture scope). `dest` must already be
/// sized to the contiguized output's byte count — true by construction,
/// since the persistent map only ever holds a buffer THIS SAME code path
/// (the warm-run `auto_contiguize` allocation) sized.
///
/// CUDA-only today: `PersistentOutputs::reuse` (Reuse mode) is only ever
/// constructed inside [`PipelinedExecutor::capture_decode`], which is
/// `#[cfg(feature = "cuda")]`-gated and validates every compute node
/// targets `Cuda` — a CPU or Vulkan storage can never actually reach this
/// function in practice. Per this project's "never panic on production
/// paths" rule, the other backends still surface a typed gap rather than
/// silently mishandling it (never a crash).
#[cfg_attr(not(feature = "cuda"), allow(unused_variables))]
fn contiguize_into(
    dest: &Arc<RwLock<Storage>>,
    arc: &Arc<RwLock<Storage>>,
    layout: &Layout,
) -> Result<()> {
    let in_guard = arc
        .read()
        .map_err(|_| poisoned("input storage lock during contiguize_into"))?;
    match &in_guard.inner {
        #[cfg(feature = "cuda")]
        fuel_memory::BackendStorage::Cuda(src_cuda) => {
            let dtype_size = in_guard.dtype.size_in_bytes();
            let mut dest_guard = dest
                .write()
                .map_err(|_| poisoned("dest storage lock during contiguize_into"))?;
            let dest_cuda = match &mut dest_guard.inner {
                fuel_memory::BackendStorage::Cuda(d) => d,
                other => {
                    return Err(Error::Msg(format!(
                        "contiguize_into: dest is BackendStorage::{:?}, not Cuda",
                        std::mem::discriminant(other),
                    ))
                    .bt());
                }
            };
            fuel_cuda_backend::baracuda::contiguize::contiguize_into(
                src_cuda, layout, dest_cuda, dtype_size,
            )
        }
        #[allow(unreachable_patterns)]
        _ => Err(Error::Msg(
            "contiguize_into: only CUDA is wired (PersistentOutputs::reuse / \
             capture_decode is CUDA-only today — no CPU/Vulkan consumer exists \
             yet; extend this match if one appears)"
                .to_string(),
        )
        .bt()),
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use fuel_graph::Node;
    use fuel_ir::Shape;

    /// Task 5 (C6): `Op::Scan` has no native kernel in Phase 1 — it is
    /// always lowered/re-fused before execution — so `op_to_op_kind` must
    /// fall through its `_ => None` wildcard for it, exactly like any other
    /// not-yet-wired op. Regression lock, not born-red: this is expected to
    /// pass immediately.
    #[test]
    fn scan_has_no_dispatch_op_kind() {
        use fuel_graph::{Op, ScanEmit};
        assert_eq!(
            crate::pipelined::op_to_op_kind(&Op::Scan {
                n_carries: 1,
                n_xs: 0,
                bound: 2,
                emit: ScanEmit::All,
                early_exit: None
            }),
            None,
        );
    }

    /// Step E A4b open question #4 — "event only where waited", step 1.
    /// `build_wait_set` collects exactly the producers of `Op::Copy`/`Move`
    /// nodes in the order: a same-device-only node (never copied/moved) must
    /// be ABSENT, and the Copy's own node id must not appear either (it is
    /// not its own producer).
    #[test]
    fn build_wait_set_collects_only_cross_device_copy_producers() {
        let mut g = Graph::new();
        let const_id = g.push(Node {
            op: Op::Const,
            inputs: vec![],
            shape: Shape::from_dims(&[4]),
            dtype: DType::F32,
        });
        // Same-device-only consumer of const_id — never fed into a Copy/Move.
        let same_device_id = g.push(Node {
            op: Op::Contiguize,
            inputs: vec![const_id],
            shape: Shape::from_dims(&[4]),
            dtype: DType::F32,
        });
        // A cross-device Copy whose producer is `same_device_id`.
        let copy_id = g.push(Node {
            op: Op::Copy {
                target: DeviceLocation::Cuda { gpu_id: 0 },
            },
            inputs: vec![same_device_id],
            shape: Shape::from_dims(&[4]),
            dtype: DType::F32,
        });
        let order = vec![const_id, same_device_id, copy_id];

        let set = build_wait_set(&g, &order);
        assert!(
            set.contains(&same_device_id),
            "the Copy's producer must be in the wait set"
        );
        assert!(
            !set.contains(&const_id),
            "a node with no Copy/Move consumer must be absent from the wait set"
        );
        assert!(
            !set.contains(&copy_id),
            "the Copy node itself is not its own producer"
        );
        assert_eq!(
            set.len(),
            1,
            "only the one true cross-device producer is in the set"
        );
    }

    /// A `Move` producer is collected the same way as a `Copy` producer.
    #[test]
    fn build_wait_set_collects_move_producers_too() {
        let mut g = Graph::new();
        let const_id = g.push(Node {
            op: Op::Const,
            inputs: vec![],
            shape: Shape::from_dims(&[4]),
            dtype: DType::F32,
        });
        let move_id = g.push(Node {
            op: Op::Move {
                target: DeviceLocation::Cpu,
            },
            inputs: vec![const_id],
            shape: Shape::from_dims(&[4]),
            dtype: DType::F32,
        });
        let order = vec![const_id, move_id];

        let set = build_wait_set(&g, &order);
        assert!(
            set.contains(&const_id),
            "the Move's producer must be in the wait set"
        );
        assert_eq!(set.len(), 1);
    }

    /// A CPU-only order (no `Op::Copy`/`Op::Move` anywhere) yields an empty
    /// wait set — every node in it is eligible for elision.
    #[test]
    fn build_wait_set_is_empty_for_cpu_only_order() {
        let mut g = Graph::new();
        let const_id = g.push(Node {
            op: Op::Const,
            inputs: vec![],
            shape: Shape::from_dims(&[4]),
            dtype: DType::F32,
        });
        let contig_id = g.push(Node {
            op: Op::Contiguize,
            inputs: vec![const_id],
            shape: Shape::from_dims(&[4]),
            dtype: DType::F32,
        });
        let order = vec![const_id, contig_id];

        let set = build_wait_set(&g, &order);
        assert!(
            set.is_empty(),
            "a CPU-only order (no Copy/Move) yields an empty wait set"
        );
    }

    /// `will_be_waited`'s two arms: the conservative `None` fallback (the
    /// `OrderSource::Streaming` case) is always `true`; a `Some(set)` checks
    /// membership.
    #[test]
    fn will_be_waited_none_is_conservative_true_for_any_node() {
        let ws: WaitSet = None;
        assert!(will_be_waited(&ws, NodeId(0)));
        assert!(will_be_waited(&ws, NodeId(12345)));
    }

    #[test]
    fn will_be_waited_some_checks_membership() {
        let mut set = HashSet::new();
        set.insert(NodeId(3));
        let ws: WaitSet = Some(set);
        assert!(will_be_waited(&ws, NodeId(3)), "member must be waited");
        assert!(!will_be_waited(&ws, NodeId(4)), "non-member must be elided");
    }

    /// A4b-1 test double: a `Completion` whose `wait` flips a shared flag, so a
    /// test can observe whether the drain actually waited it.
    struct FlagCompletion(std::sync::Arc<std::sync::atomic::AtomicUsize>);
    impl crate::compiled::Completion for FlagCompletion {
        fn wait(self: Box<Self>) -> Result<()> {
            self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
            Ok(())
        }
    }

    /// A4b-1: `store_handle` keeps only `Pending` handles (so the post-drain
    /// empty-map assert is meaningful and the drain is O(pending)), and
    /// `drain_handles` waits each pending handle exactly once then empties the map.
    #[test]
    fn handle_map_stores_pending_and_drains_once() {
        use std::sync::Arc as StdArc;
        use std::sync::atomic::{AtomicUsize, Ordering};

        let mut handles: HashMap<NodeId, CompletionHandle> = HashMap::new();

        // Ready handles are not stored.
        store_handle(&mut handles, NodeId(1), CompletionHandle::Ready);
        assert!(handles.is_empty(), "Ready handles must not be stored");

        // Two pending handles are stored and each waited once on drain.
        let waited = StdArc::new(AtomicUsize::new(0));
        store_handle(
            &mut handles,
            NodeId(2),
            CompletionHandle::Pending(Box::new(FlagCompletion(waited.clone()))),
        );
        store_handle(
            &mut handles,
            NodeId(3),
            CompletionHandle::Pending(Box::new(FlagCompletion(waited.clone()))),
        );
        assert_eq!(handles.len(), 2);

        drain_handles(&mut handles).expect("drain");
        assert_eq!(
            waited.load(Ordering::SeqCst),
            2,
            "each pending handle waited once"
        );
        assert!(handles.is_empty(), "map empty after drain");
    }

    /// A4b-1: re-keying a NodeId with a new Pending handle waits the prior one
    /// (no leaked async work), and storing `Ready` over a prior Pending also
    /// waits it. Defensive — NodeIds are unique per realize in practice.
    #[test]
    fn handle_map_rekey_waits_previous() {
        use std::sync::Arc as StdArc;
        use std::sync::atomic::{AtomicUsize, Ordering};

        let mut handles: HashMap<NodeId, CompletionHandle> = HashMap::new();
        let waited = StdArc::new(AtomicUsize::new(0));

        store_handle(
            &mut handles,
            NodeId(7),
            CompletionHandle::Pending(Box::new(FlagCompletion(waited.clone()))),
        );
        // Re-key with another Pending → prior is waited.
        store_handle(
            &mut handles,
            NodeId(7),
            CompletionHandle::Pending(Box::new(FlagCompletion(waited.clone()))),
        );
        assert_eq!(
            waited.load(Ordering::SeqCst),
            1,
            "rekey waited the prior pending"
        );

        // Store Ready over the live Pending → it is waited too.
        store_handle(&mut handles, NodeId(7), CompletionHandle::Ready);
        assert_eq!(
            waited.load(Ordering::SeqCst),
            2,
            "Ready-over-Pending waited it"
        );
        assert!(handles.is_empty());

        drain_handles(&mut handles).expect("drain empty");
    }

    /// A4b-3: `wait_producer_handle` waits the named producer's handle EXACTLY
    /// once and removes it from the map (the finer cross-device source-drain
    /// consumes the producer's completion before the Copy reads its buffer), and
    /// is a no-op for an absent producer (CPU / Vulkan-`Ready` source, or an
    /// already-consumed handle). The realize-end drain then waits only what
    /// remains — no double-wait.
    #[test]
    fn wait_producer_handle_consumes_named_handle_once() {
        use std::sync::Arc as StdArc;
        use std::sync::atomic::{AtomicUsize, Ordering};

        let mut handles: HashMap<NodeId, CompletionHandle> = HashMap::new();
        let waited = StdArc::new(AtomicUsize::new(0));

        // Two producers in flight; the Copy's source is NodeId(2).
        store_handle(
            &mut handles,
            NodeId(2),
            CompletionHandle::Pending(Box::new(FlagCompletion(waited.clone()))),
        );
        store_handle(
            &mut handles,
            NodeId(5),
            CompletionHandle::Pending(Box::new(FlagCompletion(waited.clone()))),
        );

        // Absent producer → no-op (a CPU/Vulkan-Ready source has no handle).
        wait_producer_handle(&mut handles, NodeId(99)).expect("absent is no-op");
        assert_eq!(
            waited.load(Ordering::SeqCst),
            0,
            "absent producer waited nothing"
        );
        assert_eq!(handles.len(), 2);

        // The cross-device copy's source: waited once, removed from the map.
        wait_producer_handle(&mut handles, NodeId(2)).expect("wait source");
        assert_eq!(
            waited.load(Ordering::SeqCst),
            1,
            "source producer waited once"
        );
        assert!(!handles.contains_key(&NodeId(2)), "source handle consumed");
        assert!(
            handles.contains_key(&NodeId(5)),
            "unrelated handle untouched"
        );

        // Re-waiting the (now absent) source is a no-op — no double-wait.
        wait_producer_handle(&mut handles, NodeId(2)).expect("re-wait is no-op");
        assert_eq!(waited.load(Ordering::SeqCst), 1, "no double-wait");

        // Realize-end drain waits only the remaining unrelated producer.
        drain_handles(&mut handles).expect("drain");
        assert_eq!(
            waited.load(Ordering::SeqCst),
            2,
            "drain waited the remaining one"
        );
        assert!(handles.is_empty());
    }

    /// Op::Contiguize on a contiguous-already input is zero-copy:
    /// the executor adopts the input Storage Arc unchanged.
    #[test]
    fn op_contiguize_zero_copy_on_contiguous_input() {
        let storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0]);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (const_id, contig_id) = {
            let mut g = graph.write().unwrap();
            let const_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 2]),
                dtype: DType::F32,
            });
            let contig_id = g.push(Node {
                op: Op::Contiguize,
                inputs: vec![const_id],
                shape: Shape::from_dims(&[2, 2]),
                dtype: DType::F32,
            });
            // No need to set target_backend on Contiguize — the
            // executor inherits or defaults to CPU.
            (const_id, contig_id)
        };

        let mut inputs = StorageCache::new();
        let input_arc = Arc::new(RwLock::new(storage));
        let input_arc_for_compare = Arc::clone(&input_arc);
        inputs.insert(const_id, input_arc);

        let (result_arc, _) =
            PipelinedExecutor::realize(graph, contig_id, inputs).expect("realize");

        // Same Arc → zero-copy adoption. Compare by raw pointer.
        assert!(
            Arc::ptr_eq(&result_arc, &input_arc_for_compare),
            "Op::Contiguize on contiguous input must adopt the input Arc (zero copy)",
        );
    }

    /// Op::Contiguize on a strided input (here: a transposed view)
    /// materializes a fresh contiguous Storage. The output bytes
    /// reflect the strided view's logical content, not the input
    /// storage's underlying memory order.
    #[test]
    fn op_contiguize_materializes_on_strided_input() {
        // 2x3 row-major: [[1,2,3],[4,5,6]] → transpose → 3x2:
        // [[1,4],[2,5],[3,6]]. The transposed view shares the
        // original buffer; Op::Contiguize on the transpose
        // materializes contiguous bytes matching the transposed
        // logical order.
        let storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0]);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (const_id, _transpose_id, contig_id) = {
            let mut g = graph.write().unwrap();
            let const_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let transpose_id = g.push(Node {
                op: Op::Transpose,
                inputs: vec![const_id],
                shape: Shape::from_dims(&[3, 2]),
                dtype: DType::F32,
            });
            let contig_id = g.push(Node {
                op: Op::Contiguize,
                inputs: vec![transpose_id],
                shape: Shape::from_dims(&[3, 2]),
                dtype: DType::F32,
            });
            (const_id, transpose_id, contig_id)
        };

        let mut inputs = StorageCache::new();
        inputs.insert(const_id, Arc::new(RwLock::new(storage)));

        let (result_arc, _) =
            PipelinedExecutor::realize(graph, contig_id, inputs).expect("realize");

        let guard = result_arc.read().unwrap();
        let cpu = if let fuel_memory::BackendStorage::Cpu(c) = &guard.inner {
            c
        } else {
            panic!("expected Cpu storage");
        };
        let typed: &[f32] = cpu.as_slice().expect("f32 cast");
        // Transposed logical order: [1,4,2,5,3,6].
        assert_eq!(
            typed,
            &[1.0, 4.0, 2.0, 5.0, 3.0, 6.0],
            "contiguized transpose must reflect logical (transposed) byte order",
        );
    }

    /// E2E: 3-node graph (Const + Const + Add), pre-seeded inputs,
    /// realized through the compiler+executor thread pair, returns
    /// expected sum bytes.
    #[test]
    fn pipelined_realize_const_const_add() {
        let lhs_storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0]);
        let rhs_storage = fuel_memory::from_slice_cpu(&[10.0_f32, 20.0, 30.0]);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (lhs_id, rhs_id, add_id) = {
            let mut g = graph.write().unwrap();
            let lhs_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let rhs_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let add_id = g.push(Node {
                op: Op::Add,
                inputs: vec![lhs_id, rhs_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            g.set_target_backend(add_id, BackendId::Cpu);
            (lhs_id, rhs_id, add_id)
        };

        let mut inputs = StorageCache::new();
        inputs.insert(lhs_id, Arc::new(RwLock::new(lhs_storage)));
        inputs.insert(rhs_id, Arc::new(RwLock::new(rhs_storage)));

        let (result_arc, _result_layout) =
            PipelinedExecutor::realize(graph, add_id, inputs).expect("realize");

        let guard = result_arc.read().unwrap();
        if let fuel_memory::BackendStorage::Cpu(c) = &guard.inner {
            let typed: &[f32] = c.as_slice().expect("f32 cast");
            assert_eq!(typed, &[11.0, 22.0, 33.0]);
        } else {
            panic!("expected CPU output");
        }
    }

    /// CapturedRun executor build-out (Increment 1): capture a 2-op binary
    /// chain `z = (a + b) * c` THROUGH the pipelined executor's
    /// persistent-output mode, then prove the three capture guarantees on a
    /// live GPU:
    ///   1. after capture, the fixed output buffer holds the warm-run result
    ///      (capture RECORDS launches, it does not execute them);
    ///   2. `replay()` recomputes bit-exactly into the SAME fixed buffer;
    ///   3. replay reads the FIXED input addresses — overwriting an input
    ///      buffer in place and replaying yields the new result, matching an
    ///      independent uncaptured `realize`. This is the whole mechanism:
    ///      the write-into binary wrapper + the executor's reuse mode +
    ///      `capture_run` compose into a replayable decode-step graph.
    #[test]
    #[ignore = "requires a live CUDA device"]
    #[cfg(feature = "cuda")]
    fn capture_decode_binary_chain_replays_bit_exact_cuda() {
        use fuel_cuda_backend::{CudaDevice, CudaStorageBytes};
        use fuel_ir::Shape;
        use fuel_memory::BackendStorage;

        let Ok(dev) = CudaDevice::new(0) else {
            return fuel_test_support::hardware::skip(
                fuel_test_support::hardware::Hardware::Cuda,
                fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
            );
        };

        // z = (a + b) * c, shape [4]. `y = a + b` is a persistent
        // INTERMEDIATE; `z` is the persistent target — both fixed-address.
        let build = || -> (Arc<RwLock<Graph>>, NodeId, NodeId, NodeId, NodeId) {
            let graph = Arc::new(RwLock::new(Graph::new()));
            let (a, b, c, z) = {
                let mut g = graph.write().unwrap();
                let a = g.push(Node {
                    op: Op::Const,
                    inputs: vec![],
                    shape: Shape::from_dims(&[4]),
                    dtype: DType::F32,
                });
                let b = g.push(Node {
                    op: Op::Const,
                    inputs: vec![],
                    shape: Shape::from_dims(&[4]),
                    dtype: DType::F32,
                });
                let c = g.push(Node {
                    op: Op::Const,
                    inputs: vec![],
                    shape: Shape::from_dims(&[4]),
                    dtype: DType::F32,
                });
                let y = g.push(Node {
                    op: Op::Add,
                    inputs: vec![a, b],
                    shape: Shape::from_dims(&[4]),
                    dtype: DType::F32,
                });
                let z = g.push(Node {
                    op: Op::Mul,
                    inputs: vec![y, c],
                    shape: Shape::from_dims(&[4]),
                    dtype: DType::F32,
                });
                g.set_target_backend(y, BackendId::Cuda);
                g.set_target_backend(z, BackendId::Cuda);
                (a, b, c, z)
            };
            (graph, a, b, c, z)
        };

        let up = |v: &[f32]| -> Arc<RwLock<Storage>> {
            let bytes: &[u8] = bytemuck::cast_slice(v);
            let cb = CudaStorageBytes::from_cpu_bytes(&dev, bytes).expect("h2d");
            Arc::new(RwLock::new(Storage::new(
                BackendStorage::Cuda(cb),
                DType::F32,
            )))
        };
        let read = |arc: &Arc<RwLock<Storage>>| -> Vec<f32> {
            let g = arc.read().unwrap();
            let BackendStorage::Cuda(c) = &g.inner else {
                panic!("not cuda")
            };
            let host = c.to_cpu_bytes().expect("d2h");
            bytemuck::cast_slice(&host).to_vec()
        };

        let a_data = [1.0_f32, 2.0, 3.0, 4.0];
        let b_data = [10.0_f32, 20.0, 30.0, 40.0];
        let c_data = [2.0_f32, 2.0, 2.0, 2.0];
        let expect0 = [22.0_f32, 44.0, 66.0, 88.0]; // (a+b)*c

        // Fixed input Arcs (shared into capture; overwritten in place later).
        let a_arc = up(&a_data);
        let b_arc = up(&b_data);
        let c_arc = up(&c_data);

        let (graph, a, b, c, z) = build();
        let mut inputs = StorageCache::new();
        inputs.insert(a, a_arc.clone());
        inputs.insert(b, b_arc.clone());
        inputs.insert(c, c_arc.clone());

        let captured =
            PipelinedExecutor::capture_decode(Arc::clone(&graph), z, inputs, SymEnv::default())
                .expect("capture_decode");

        // (1) warm-run result sits in the captured output buffer.
        assert_eq!(
            read(&captured.output),
            &expect0,
            "warm-run result after capture"
        );

        // (2) replay with the same inputs recomputes bit-exactly.
        captured.run.replay(&dev).expect("replay");
        dev.synchronize().expect("sync");
        assert_eq!(read(&captured.output), &expect0, "replay reproduces result");

        // (3) overwrite input `a` IN PLACE (same device address), replay ->
        //     the new result flows from the FIXED input address. Compare to
        //     an independent uncaptured realize with the same new input.
        let a2 = [100.0_f32, 200.0, 300.0, 400.0];
        {
            let g = a_arc.read().unwrap();
            let BackendStorage::Cuda(c) = &g.inner else {
                panic!("a not cuda")
            };
            c.write_from_host(bytemuck::cast_slice(&a2))
                .expect("in-place H2D");
        }
        captured.run.replay(&dev).expect("replay 2");
        dev.synchronize().expect("sync 2");
        let got = read(&captured.output);

        let (graph_ref, ar, br, cr, zr) = build();
        let mut inputs_ref = StorageCache::new();
        inputs_ref.insert(ar, up(&a2));
        inputs_ref.insert(br, up(&b_data));
        inputs_ref.insert(cr, up(&c_data));
        let (ref_arc, _) =
            PipelinedExecutor::realize(graph_ref, zr, inputs_ref).expect("ref realize");
        let expect1 = read(&ref_arc); // (a2+b)*c = [220,440,660,880]
        assert_eq!(
            got, expect1,
            "replay after in-place input update matches uncaptured realize"
        );
        assert_eq!(expect1, vec![220.0, 440.0, 660.0, 880.0]);
    }

    /// CapturedRun executor build-out (Increment 3): capture a DECODE-SHAPED
    /// graph — the full dense op-set composing in one capture — and prove it
    /// replays bit-exactly against an independent uncaptured realize.
    ///
    /// Graph (an FFN block + norms, batch=seq=1 like real decode):
    ///   n1 = rms_norm(x);  u = n1 · w_up;  s = silu(u);
    /// cuBLAS `gemm_dense` matmul IS graph-capturable (regression for the
    /// dangling-input bug that once masqueraded as "cuBLAS not replay-safe").
    /// Captures `y = a·w` where the caller does NOT retain `w`, FORCES the
    /// freed-`w` slot to be reused by garbage buffers, then replays 50× — all
    /// must equal the uncaptured realize. Passes iff `capture_decode` retains
    /// its input buffers (`CapturedDecode::retained_inputs`).
    #[test]
    #[ignore = "requires a live CUDA device"]
    #[cfg(feature = "cuda")]
    fn capture_decode_matmul_replays_bit_exact_cuda() {
        use fuel_cuda_backend::{CudaDevice, CudaStorageBytes};
        use fuel_ir::Shape;
        use fuel_memory::BackendStorage;
        let Ok(dev) = CudaDevice::new(0) else { return };
        const H: usize = 4;
        const F: usize = 8;
        let a_data: Vec<f32> = (0..H).map(|i| 0.5 + i as f32).collect();
        let w_data: Vec<f32> = (0..H * F).map(|i| 0.01 * (i as f32 + 1.0)).collect();
        let build = || -> (Arc<RwLock<Graph>>, NodeId, NodeId, NodeId) {
            let graph = Arc::new(RwLock::new(Graph::new()));
            let (a, w, y) = {
                let mut g = graph.write().unwrap();
                let a = g.push(Node {
                    op: Op::Const,
                    inputs: vec![],
                    shape: Shape::from_dims(&[1, H]),
                    dtype: DType::F32,
                });
                let w = g.push(Node {
                    op: Op::Const,
                    inputs: vec![],
                    shape: Shape::from_dims(&[H, F]),
                    dtype: DType::F32,
                });
                let y = g.push(Node {
                    op: Op::MatMul,
                    inputs: vec![a, w],
                    shape: Shape::from_dims(&[1, F]),
                    dtype: DType::F32,
                });
                g.set_target_backend(y, BackendId::Cuda);
                (a, w, y)
            };
            (graph, a, w, y)
        };
        let up = |v: &[f32]| {
            let b: Vec<u8> = v.iter().flat_map(|f| f.to_le_bytes()).collect();
            Arc::new(RwLock::new(Storage::new(
                BackendStorage::Cuda(CudaStorageBytes::from_cpu_bytes(&dev, &b).unwrap()),
                DType::F32,
            )))
        };
        let read = |arc: &Arc<RwLock<Storage>>| -> Vec<f32> {
            let g = arc.read().unwrap();
            let BackendStorage::Cuda(c) = &g.inner else {
                panic!("not cuda")
            };
            c.to_cpu_bytes()
                .unwrap()
                .chunks_exact(4)
                .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]]))
                .collect()
        };
        // Reference (uncaptured).
        let (gr, ar, wr, yr) = build();
        let mut insr = StorageCache::new();
        insr.insert(ar, up(&a_data));
        insr.insert(wr, up(&w_data));
        let (refarc, _) = PipelinedExecutor::realize(gr, yr, insr).unwrap();
        let expect = read(&refarc);
        // Capture (a + w NOT retained by caller).
        let (graph, a, w, y) = build();
        let mut inputs = StorageCache::new();
        inputs.insert(a, up(&a_data));
        inputs.insert(w, up(&w_data));
        let captured =
            PipelinedExecutor::capture_decode(Arc::clone(&graph), y, inputs, SymEnv::default())
                .expect("capture");
        assert_eq!(read(&captured.output), expect, "warm result after capture");
        // FORCE freed-memory reuse: same-size (H*F) garbage buffers. If `w`
        // were dangling, replay would read 999.0s.
        let mut squatters = Vec::new();
        for _ in 0..8 {
            let g: Vec<u8> = (0..H * F).flat_map(|_| 999.0f32.to_le_bytes()).collect();
            squatters.push(CudaStorageBytes::from_cpu_bytes(&dev, &g).unwrap());
        }
        dev.synchronize().unwrap();
        let mut bad = 0usize;
        for _ in 0..50 {
            captured.run.replay(&dev).unwrap();
            dev.synchronize().unwrap();
            if read(&captured.output) != expect {
                bad += 1;
            }
        }
        assert_eq!(
            bad, 0,
            "cuBLAS matmul capture replays bit-exact ({bad}/50 wrong)"
        );
    }

    /// `Op::MulScalar` / `OpKind::Affine` (the `1/√d` attention scale) is
    /// capture-safe (write-into variant + forced-reuse regression).
    #[test]
    #[ignore = "requires a live CUDA device"]
    #[cfg(feature = "cuda")]
    fn capture_decode_affine_replays_bit_exact_cuda() {
        use fuel_cuda_backend::{CudaDevice, CudaStorageBytes};
        use fuel_ir::Shape;
        use fuel_memory::BackendStorage;
        let Ok(dev) = CudaDevice::new(0) else { return };
        const N: usize = 6;
        let x_data: Vec<f32> = (0..N).map(|i| 1.0 + i as f32).collect();
        let expect: Vec<f32> = x_data.iter().map(|v| v * 0.5).collect();
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (x, y) = {
            let mut g = graph.write().unwrap();
            let x = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, N]),
                dtype: DType::F32,
            });
            let y = g.push(Node {
                op: Op::MulScalar(0.5),
                inputs: vec![x],
                shape: Shape::from_dims(&[1, N]),
                dtype: DType::F32,
            });
            g.set_target_backend(y, BackendId::Cuda);
            (x, y)
        };
        let up = |v: &[f32]| {
            let b: Vec<u8> = v.iter().flat_map(|f| f.to_le_bytes()).collect();
            Arc::new(RwLock::new(Storage::new(
                BackendStorage::Cuda(CudaStorageBytes::from_cpu_bytes(&dev, &b).unwrap()),
                DType::F32,
            )))
        };
        let read = |arc: &Arc<RwLock<Storage>>| -> Vec<f32> {
            let g = arc.read().unwrap();
            let BackendStorage::Cuda(c) = &g.inner else {
                panic!("not cuda")
            };
            c.to_cpu_bytes()
                .unwrap()
                .chunks_exact(4)
                .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]]))
                .collect()
        };
        let mut inputs = StorageCache::new();
        inputs.insert(x, up(&x_data));
        let captured =
            PipelinedExecutor::capture_decode(Arc::clone(&graph), y, inputs, SymEnv::default())
                .expect("capture");
        assert_eq!(read(&captured.output), expect, "warm result after capture");
        let mut sq = Vec::new();
        for _ in 0..8 {
            let g: Vec<u8> = (0..N).flat_map(|_| 999.0f32.to_le_bytes()).collect();
            sq.push(CudaStorageBytes::from_cpu_bytes(&dev, &g).unwrap());
        }
        dev.synchronize().unwrap();
        let mut bad = 0usize;
        for _ in 0..50 {
            captured.run.replay(&dev).unwrap();
            dev.synchronize().unwrap();
            if read(&captured.output) != expect {
                bad += 1;
            }
        }
        assert_eq!(bad, 0, "affine capture replays bit-exact ({bad}/50 wrong)");
    }

    /// `index_select` (embedding lookup) IS graph-capturable (regression for
    /// the dangling-input bug that once looked like "element 0 dropped on
    /// replay"). Captures `emb = index_select(wte, tok)` with `wte` NOT
    /// retained by the caller, FORCES the freed-`wte` slot to be reused by
    /// garbage buffers, then replays 50× — all must be `[1,2,3,4]`. Passes iff
    /// `capture_decode` retains its input buffers.
    #[test]
    #[ignore = "requires a live CUDA device"]
    #[cfg(feature = "cuda")]
    fn capture_decode_index_select_replays_bit_exact_cuda() {
        use fuel_cuda_backend::{CudaDevice, CudaStorageBytes};
        use fuel_ir::Shape;
        use fuel_memory::BackendStorage;
        let Ok(dev) = CudaDevice::new(0) else { return };
        const V: usize = 3;
        const H: usize = 4;
        let wte_data: Vec<f32> = (0..V)
            .flat_map(|v| (0..H).map(move |c| 1.0 + 10.0 * v as f32 + c as f32))
            .collect();
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (tok, wte, emb) = {
            let mut g = graph.write().unwrap();
            let tok = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1]),
                dtype: DType::U32,
            });
            let wte = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[V, H]),
                dtype: DType::F32,
            });
            let emb = g.push(Node {
                op: Op::IndexSelect { dim: 0 },
                inputs: vec![wte, tok],
                shape: Shape::from_dims(&[1, H]),
                dtype: DType::F32,
            });
            g.set_target_backend(emb, BackendId::Cuda);
            (tok, wte, emb)
        };
        let up_f32 = |v: &[f32]| {
            let b: Vec<u8> = v.iter().flat_map(|f| f.to_le_bytes()).collect();
            Arc::new(RwLock::new(Storage::new(
                BackendStorage::Cuda(CudaStorageBytes::from_cpu_bytes(&dev, &b).unwrap()),
                DType::F32,
            )))
        };
        let up_u32 = |v: &[u32]| {
            let b: Vec<u8> = v.iter().flat_map(|x| x.to_le_bytes()).collect();
            Arc::new(RwLock::new(Storage::new(
                BackendStorage::Cuda(CudaStorageBytes::from_cpu_bytes(&dev, &b).unwrap()),
                DType::U32,
            )))
        };
        let read = |arc: &Arc<RwLock<Storage>>| -> Vec<f32> {
            let g = arc.read().unwrap();
            let BackendStorage::Cuda(c) = &g.inner else {
                panic!("not cuda")
            };
            c.to_cpu_bytes()
                .unwrap()
                .chunks_exact(4)
                .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]]))
                .collect()
        };
        let tok_arc = up_u32(&[0]);
        let mut inputs = StorageCache::new();
        inputs.insert(tok, tok_arc.clone());
        inputs.insert(wte, up_f32(&wte_data));
        let captured =
            PipelinedExecutor::capture_decode(Arc::clone(&graph), emb, inputs, SymEnv::default())
                .expect("capture");
        assert_eq!(
            read(&captured.output),
            vec![1.0, 2.0, 3.0, 4.0],
            "warm result after capture"
        );
        // FORCE freed-memory reuse: same-size (V*H) garbage buffers reclaim
        // `wte`'s slot if it were dangling — replay would then read 999.0s.
        let mut squatters = Vec::new();
        for _ in 0..8 {
            let g: Vec<u8> = (0..V * H).flat_map(|_| 999.0f32.to_le_bytes()).collect();
            squatters.push(CudaStorageBytes::from_cpu_bytes(&dev, &g).unwrap());
        }
        dev.synchronize().unwrap();
        let mut bad = 0usize;
        for _ in 0..50 {
            captured.run.replay(&dev).unwrap();
            dev.synchronize().unwrap();
            if read(&captured.output) != vec![1.0, 2.0, 3.0, 4.0] {
                bad += 1;
            }
        }
        assert_eq!(
            bad, 0,
            "index_select capture replays bit-exact ({bad}/50 wrong)"
        );
    }

    ///   n1 = rms_norm(x);  s = silu(n1);  p = softmax(s);
    ///   r  = p + x;  n2 = rms_norm(r);  y = n2 + x
    /// Exercises rms_norm (×2 — proves the per-device RMS workspace is reused
    /// across sequential norms inside ONE captured graph), silu + softmax
    /// (write-into), and residual adds through the persistent-output capture
    /// path. Verified across two input value-sets (robust, not luck).
    ///
    /// Deliberately EXCLUDES matmul (cuBLAS not graph-replay-safe) and
    /// index_select (its host-side shape/stride arrays dangle under capture —
    /// proven to mis-replay). Both are surfaced `capture_decode` gaps, not
    /// silent mis-replays. See `op_kind_is_capture_writeinto`.
    #[test]
    #[ignore = "requires a live CUDA device"]
    #[cfg(feature = "cuda")]
    fn capture_decode_norm_silu_softmax_replays_bit_exact_cuda() {
        use fuel_cuda_backend::{CudaDevice, CudaStorageBytes};
        use fuel_graph::registry::{FusedOpParams, FusedOps};
        use fuel_ir::Shape;
        use fuel_memory::BackendStorage;

        let Ok(dev) = CudaDevice::new(0) else {
            return fuel_test_support::hardware::skip(
                fuel_test_support::hardware::Hardware::Cuda,
                fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
            );
        };
        const H: usize = 6;
        let eps = 1e-5f64;

        let build = || -> (Arc<RwLock<Graph>>, NodeId, NodeId) {
            let graph = Arc::new(RwLock::new(Graph::new()));
            let (x, y) = {
                let mut g = graph.write().unwrap();
                let mk = |g: &mut Graph, op: Op, ins: Vec<NodeId>, dims: &[usize]| {
                    g.push(Node {
                        op,
                        inputs: ins,
                        shape: Shape::from_dims(dims),
                        dtype: DType::F32,
                    })
                };
                let rms = || {
                    Op::Fused(
                        FusedOps::RMS_NORM_LAST_DIM,
                        FusedOpParams::RmsNormLastDim { eps },
                    )
                };
                let sm = || Op::Fused(FusedOps::SOFTMAX_LAST_DIM, FusedOpParams::SoftmaxLastDim);
                let x = mk(&mut g, Op::Const, vec![], &[1, H]);
                let n1 = mk(&mut g, rms(), vec![x], &[1, H]);
                let s = mk(&mut g, Op::Silu, vec![n1], &[1, H]);
                let p = mk(&mut g, sm(), vec![s], &[1, H]);
                let r = mk(&mut g, Op::Add, vec![p, x], &[1, H]);
                let n2 = mk(&mut g, rms(), vec![r], &[1, H]);
                let y = mk(&mut g, Op::Add, vec![n2, x], &[1, H]);
                for id in [n1, s, p, r, n2, y] {
                    g.set_target_backend(id, BackendId::Cuda);
                }
                (x, y)
            };
            (graph, x, y)
        };

        let up = |v: &[f32]| -> Arc<RwLock<Storage>> {
            let cb = CudaStorageBytes::from_cpu_bytes(&dev, bytemuck::cast_slice(v)).expect("h2d");
            Arc::new(RwLock::new(Storage::new(
                BackendStorage::Cuda(cb),
                DType::F32,
            )))
        };
        let read = |arc: &Arc<RwLock<Storage>>| -> Vec<f32> {
            let g = arc.read().unwrap();
            let BackendStorage::Cuda(c) = &g.inner else {
                panic!("not cuda")
            };
            bytemuck::cast_slice(&c.to_cpu_bytes().expect("d2h")).to_vec()
        };

        let x_data: Vec<f32> = (0..H).map(|c| 0.1 * (c as f32 + 1.0)).collect();
        let x_arc = up(&x_data);
        let (graph, x, y) = build();
        let mut inputs = StorageCache::new();
        inputs.insert(x, x_arc.clone());

        let captured =
            PipelinedExecutor::capture_decode(Arc::clone(&graph), y, inputs, SymEnv::default())
                .expect("capture_decode norm/silu/softmax");

        let realize_ref = |xv: &[f32]| -> Vec<f32> {
            let (g, xr, yr) = build();
            let mut ins = StorageCache::new();
            ins.insert(xr, up(xv));
            let (arc, _) = PipelinedExecutor::realize(g, yr, ins).expect("ref realize");
            read(&arc)
        };
        let expect0 = realize_ref(&x_data);

        // (1) after capture the output holds the warm result.
        assert_eq!(read(&captured.output), expect0, "warm result after capture");
        // (2) replay reproduces bit-exactly (incl. RMS workspace reuse on replay).
        captured.run.replay(&dev).expect("replay");
        dev.synchronize().expect("sync");
        assert_eq!(read(&captured.output), expect0, "replay reproduces result");

        // (3) overwrite x in place, replay -> matches uncaptured realize (a
        //     DIFFERENT value-set — so the chain is robust, not input-lucky).
        let x2: Vec<f32> = (0..H).map(|c| 0.3 * (c as f32 + 1.0)).collect();
        {
            let g = x_arc.read().unwrap();
            let BackendStorage::Cuda(c) = &g.inner else {
                panic!("x not cuda")
            };
            c.write_from_host(bytemuck::cast_slice(&x2))
                .expect("in-place H2D");
        }
        captured.run.replay(&dev).expect("replay 2");
        dev.synchronize().expect("sync 2");
        assert_eq!(
            read(&captured.output),
            realize_ref(&x2),
            "replay after x update matches realize"
        );
    }

    /// CapturedRun executor build-out (Increment 4b-γ): make `ContiguizeOf`
    /// (a non-contiguous reshape after `Permute` — the real decode graph's
    /// attention-merge reshape shape) AND a 2-input `Op::Concat` (the shape
    /// RoPE's CUDA decomposition emits for its rotate-half) BOTH
    /// capture-safe in ONE graph, so the crux 4b-γ exists for — allocation
    /// of an INTERMEDIATE, non-final buffer inside a capture scope — is
    /// actually exercised, not just each gap in isolation.
    ///
    /// Graph: `x [2,3,4] --Permute[1,0,2]--> [3,2,4]` (non-contiguous)
    /// `--Reshape[3,8]--> r` (`ContiguizeOf`, MUST allocate: its input is
    /// non-contiguous) `--Concat(dim=1, with b [3,8])--> y [3,16]` (target).
    #[test]
    #[ignore = "requires a live CUDA device"]
    #[cfg(feature = "cuda")]
    fn capture_decode_contiguize_then_concat_replays_bit_exact_cuda() {
        use fuel_cuda_backend::{CudaDevice, CudaStorageBytes};
        use fuel_ir::Shape;
        use fuel_memory::BackendStorage;

        let Ok(dev) = CudaDevice::new(0) else {
            return fuel_test_support::hardware::skip(
                fuel_test_support::hardware::Hardware::Cuda,
                fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
            );
        };

        let build = || -> (Arc<RwLock<Graph>>, NodeId, NodeId, NodeId) {
            let graph = Arc::new(RwLock::new(Graph::new()));
            let (x, b, y) = {
                let mut g = graph.write().unwrap();
                let x = g.push(Node {
                    op: Op::Const,
                    inputs: vec![],
                    shape: Shape::from_dims(&[2, 3, 4]),
                    dtype: DType::F32,
                });
                let p = g.push(Node {
                    op: Op::Permute(vec![1, 0, 2]),
                    inputs: vec![x],
                    shape: Shape::from_dims(&[3, 2, 4]),
                    dtype: DType::F32,
                });
                let r = g.push(Node {
                    op: Op::Reshape(Shape::from_dims(&[3, 8])),
                    inputs: vec![p],
                    shape: Shape::from_dims(&[3, 8]),
                    dtype: DType::F32,
                });
                let b = g.push(Node {
                    op: Op::Const,
                    inputs: vec![],
                    shape: Shape::from_dims(&[3, 8]),
                    dtype: DType::F32,
                });
                let y = g.push(Node {
                    op: Op::Concat { dim: 1 },
                    inputs: vec![r, b],
                    shape: Shape::from_dims(&[3, 16]),
                    dtype: DType::F32,
                });
                g.set_target_backend(p, BackendId::Cuda);
                g.set_target_backend(r, BackendId::Cuda);
                g.set_target_backend(y, BackendId::Cuda);
                (x, b, y)
            };
            (graph, x, b, y)
        };

        let up = |v: &[f32]| -> Arc<RwLock<Storage>> {
            let cb = CudaStorageBytes::from_cpu_bytes(&dev, bytemuck::cast_slice(v)).expect("h2d");
            Arc::new(RwLock::new(Storage::new(
                BackendStorage::Cuda(cb),
                DType::F32,
            )))
        };
        let read = |arc: &Arc<RwLock<Storage>>| -> Vec<f32> {
            let g = arc.read().unwrap();
            let BackendStorage::Cuda(c) = &g.inner else {
                panic!("not cuda")
            };
            bytemuck::cast_slice(&c.to_cpu_bytes().expect("d2h")).to_vec()
        };

        let x_data: Vec<f32> = (0..24).map(|i| i as f32).collect();
        let b_data: Vec<f32> = (0..24).map(|i| 100.0 + i as f32).collect();
        let x_arc = up(&x_data);
        let (graph, x, b, y) = build();
        let mut inputs = StorageCache::new();
        inputs.insert(x, x_arc.clone());
        // `b` deliberately NOT retained by a local variable — forces
        // reliance on `capture_decode`'s own `retained_inputs`, mirroring
        // the matmul/index_select regression precedent.
        inputs.insert(b, up(&b_data));

        let captured =
            PipelinedExecutor::capture_decode(Arc::clone(&graph), y, inputs, SymEnv::default())
                .expect("capture_decode contiguize+concat");

        let realize_ref = |xv: &[f32], bv: &[f32]| -> Vec<f32> {
            let (g, xr, br, yr) = build();
            let mut ins = StorageCache::new();
            ins.insert(xr, up(xv));
            ins.insert(br, up(bv));
            let (arc, _) = PipelinedExecutor::realize(g, yr, ins).expect("ref realize");
            read(&arc)
        };
        let expect0 = realize_ref(&x_data, &b_data);

        // (1) warm-run result sits in the captured output buffer (capture
        //     RECORDS launches, it does not execute them at capture time).
        assert_eq!(read(&captured.output), expect0, "warm result after capture");

        // (2) replay TWICE with unchanged inputs — proves the fixed-address
        //     reuse (both the ContiguizeOf intermediate and the Concat
        //     output) holds across multiple replay() calls, not just one.
        for i in 0..2 {
            captured
                .run
                .replay(&dev)
                .unwrap_or_else(|e| panic!("replay {i}: {e:?}"));
            dev.synchronize().expect("sync");
            assert_eq!(
                read(&captured.output),
                expect0,
                "replay {i} reproduces result"
            );
        }

        // (3) overwrite x in place (a DIFFERENT value-set), replay again ->
        //     the new result flows from the FIXED input address, through
        //     the fixed ContiguizeOf intermediate, into the fixed Concat
        //     output — matches an independent uncaptured realize.
        let x2: Vec<f32> = (0..24).map(|i| 1000.0 + i as f32).collect();
        {
            let g = x_arc.read().unwrap();
            let BackendStorage::Cuda(c) = &g.inner else {
                panic!("x not cuda")
            };
            c.write_from_host(bytemuck::cast_slice(&x2))
                .expect("in-place H2D");
        }
        captured.run.replay(&dev).expect("replay 3");
        dev.synchronize().expect("sync 3");
        assert_eq!(
            read(&captured.output),
            realize_ref(&x2, &b_data),
            "replay after x update matches independent uncaptured realize"
        );
    }

    /// CapturedRun executor build-out (Increment 4b-ζ): a same-device
    /// CUDA→CUDA `Op::Copy` — the shape `insert_safety_copies` emits for its
    /// aliasing-hazard snapshots in the real Llama decode graph — is
    /// capture-safe via `copy_from_cuda_wrapper`'s write-into
    /// `BackendStorage::Cuda` branch (`CudaStorageBytes::copy_from_device`).
    ///
    /// Graph: `x [2,4]`, `b [2,4]` --Copy{target: Cuda{0}}--> `c` (same-device
    /// snapshot of `x`) --Add(c, b)--> `y`. Capture, replay twice unchanged,
    /// then update `x` in place and replay again — every step must match an
    /// independent uncaptured `realize`.
    #[test]
    #[ignore = "requires a live CUDA device"]
    #[cfg(feature = "cuda")]
    fn capture_decode_same_device_copy_replays_bit_exact_cuda() {
        use fuel_cuda_backend::{CudaDevice, CudaStorageBytes};
        use fuel_ir::Shape;
        use fuel_memory::BackendStorage;

        let Ok(dev) = CudaDevice::new(0) else {
            return fuel_test_support::hardware::skip(
                fuel_test_support::hardware::Hardware::Cuda,
                fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
            );
        };

        let build = || -> (Arc<RwLock<Graph>>, NodeId, NodeId, NodeId) {
            let graph = Arc::new(RwLock::new(Graph::new()));
            let (x, b, y) = {
                let mut g = graph.write().unwrap();
                let x = g.push(Node {
                    op: Op::Const,
                    inputs: vec![],
                    shape: Shape::from_dims(&[2, 4]),
                    dtype: DType::F32,
                });
                let b = g.push(Node {
                    op: Op::Const,
                    inputs: vec![],
                    shape: Shape::from_dims(&[2, 4]),
                    dtype: DType::F32,
                });
                // Same-device CUDA->CUDA copy — the write-into-safe case.
                let c = g.push(Node {
                    op: Op::Copy {
                        target: DeviceLocation::Cuda { gpu_id: 0 },
                    },
                    inputs: vec![x],
                    shape: Shape::from_dims(&[2, 4]),
                    dtype: DType::F32,
                });
                let y = g.push(Node {
                    op: Op::Add,
                    inputs: vec![c, b],
                    shape: Shape::from_dims(&[2, 4]),
                    dtype: DType::F32,
                });
                // Copy's target_backend = SOURCE backend (x is Cuda-resident).
                g.set_target_backend(c, BackendId::Cuda);
                g.set_target_backend(y, BackendId::Cuda);
                (x, b, y)
            };
            (graph, x, b, y)
        };

        let up = |v: &[f32]| -> Arc<RwLock<Storage>> {
            let cb = CudaStorageBytes::from_cpu_bytes(&dev, bytemuck::cast_slice(v)).expect("h2d");
            Arc::new(RwLock::new(Storage::new(
                BackendStorage::Cuda(cb),
                DType::F32,
            )))
        };
        let read = |arc: &Arc<RwLock<Storage>>| -> Vec<f32> {
            let g = arc.read().unwrap();
            let BackendStorage::Cuda(c) = &g.inner else {
                panic!("not cuda")
            };
            bytemuck::cast_slice(&c.to_cpu_bytes().expect("d2h")).to_vec()
        };

        let x_data: Vec<f32> = (0..8).map(|i| i as f32).collect();
        let b_data: Vec<f32> = (0..8).map(|i| 100.0 + i as f32).collect();
        let x_arc = up(&x_data);
        let (graph, x, b, y) = build();
        let mut inputs = StorageCache::new();
        inputs.insert(x, x_arc.clone());
        // `b` deliberately NOT retained by a local variable — forces
        // reliance on `capture_decode`'s own `retained_inputs`.
        inputs.insert(b, up(&b_data));

        let captured =
            PipelinedExecutor::capture_decode(Arc::clone(&graph), y, inputs, SymEnv::default())
                .expect("capture_decode same-device copy");

        let realize_ref = |xv: &[f32], bv: &[f32]| -> Vec<f32> {
            let (g, xr, br, yr) = build();
            let mut ins = StorageCache::new();
            ins.insert(xr, up(xv));
            ins.insert(br, up(bv));
            let (arc, _) = PipelinedExecutor::realize(g, yr, ins).expect("ref realize");
            read(&arc)
        };
        let expect0 = realize_ref(&x_data, &b_data);

        // (1) warm-run result sits in the captured output buffer.
        assert_eq!(read(&captured.output), expect0, "warm result after capture");

        // (2) replay TWICE with unchanged inputs — the Copy's write-into
        //     destination buffer must stay bit-exact across replays.
        for i in 0..2 {
            captured
                .run
                .replay(&dev)
                .unwrap_or_else(|e| panic!("replay {i}: {e:?}"));
            dev.synchronize().expect("sync");
            assert_eq!(
                read(&captured.output),
                expect0,
                "replay {i} reproduces result"
            );
        }

        // (3) overwrite x in place (a DIFFERENT value-set), replay again ->
        //     the Copy re-runs on the fixed input address into its fixed
        //     destination buffer -> matches an independent uncaptured realize.
        let x2: Vec<f32> = (0..8).map(|i| 1000.0 + i as f32).collect();
        {
            let g = x_arc.read().unwrap();
            let BackendStorage::Cuda(c) = &g.inner else {
                panic!("x not cuda")
            };
            c.write_from_host(bytemuck::cast_slice(&x2))
                .expect("in-place H2D");
        }
        captured.run.replay(&dev).expect("replay 3");
        dev.synchronize().expect("sync 3");
        assert_eq!(
            read(&captured.output),
            realize_ref(&x2, &b_data),
            "replay after x update matches independent uncaptured realize"
        );
    }

    /// CapturedRun executor build-out (Increment 4b-ζ): a genuinely
    /// cross-device `Op::Copy` (D2H — `target: DeviceLocation::Cpu` from a
    /// CUDA-resident source) must still be REJECTED by `capture_decode`'s
    /// validation loop — a host round-trip can never be part of a CUDA
    /// graph capture. Fast check: no replay, just confirm the typed `Err`.
    #[test]
    #[ignore = "requires a live CUDA device"]
    #[cfg(feature = "cuda")]
    fn capture_decode_rejects_cross_device_copy_cuda() {
        use fuel_cuda_backend::{CudaDevice, CudaStorageBytes};
        use fuel_ir::Shape;
        use fuel_memory::BackendStorage;

        let Ok(dev) = CudaDevice::new(0) else {
            return fuel_test_support::hardware::skip(
                fuel_test_support::hardware::Hardware::Cuda,
                fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
            );
        };

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (x, y) = {
            let mut g = graph.write().unwrap();
            let x = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 4]),
                dtype: DType::F32,
            });
            // Genuinely cross-device: D2H.
            let y = g.push(Node {
                op: Op::Copy {
                    target: DeviceLocation::Cpu,
                },
                inputs: vec![x],
                shape: Shape::from_dims(&[2, 4]),
                dtype: DType::F32,
            });
            g.set_target_backend(y, BackendId::Cuda); // source backend = Cuda
            (x, y)
        };

        let x_data: Vec<f32> = (0..8).map(|i| i as f32).collect();
        let cb =
            CudaStorageBytes::from_cpu_bytes(&dev, bytemuck::cast_slice(&x_data)).expect("h2d");
        let x_arc = Arc::new(RwLock::new(Storage::new(
            BackendStorage::Cuda(cb),
            DType::F32,
        )));
        let mut inputs = StorageCache::new();
        inputs.insert(x, x_arc);

        let result = PipelinedExecutor::capture_decode(graph, y, inputs, SymEnv::default());
        let msg = match result {
            Ok(_) => panic!("D2H Op::Copy must be rejected by capture_decode"),
            Err(e) => e.to_string(),
        };
        assert!(
            msg.contains("cross-device"),
            "error should describe the cross-device rejection: {msg}"
        );
    }

    /// CapturedDecodeSession (Increment 4a): drive the warm→capture→replay
    /// orchestration over MANY tokens on a synthetic decode-shaped session —
    /// the real per-token decode pattern. Capture once, then per token
    /// H2D-update the fixed input buffer IN PLACE + replay, asserting each
    /// token's output is bit-exact against an independent uncaptured realize.
    /// A kernel-agnostic proof of the state machine + fixed-buffer mechanics
    /// (matmul + index_select excluded pending baracuda capture-safe kernels).
    #[test]
    #[ignore = "requires a live CUDA device"]
    #[cfg(feature = "cuda")]
    fn captured_decode_session_replays_many_tokens_cuda() {
        use fuel_cuda_backend::{CudaDevice, CudaStorageBytes};
        use fuel_graph::registry::{FusedOpParams, FusedOps};
        use fuel_ir::Shape;
        use fuel_memory::BackendStorage;

        let Ok(dev) = CudaDevice::new(0) else {
            return fuel_test_support::hardware::skip(
                fuel_test_support::hardware::Hardware::Cuda,
                fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
            );
        };
        const H: usize = 6;
        let eps = 1e-5f64;

        let build = || -> (Arc<RwLock<Graph>>, NodeId, NodeId) {
            let graph = Arc::new(RwLock::new(Graph::new()));
            let (x, y) = {
                let mut g = graph.write().unwrap();
                let mk = |g: &mut Graph, op: Op, ins: Vec<NodeId>, dims: &[usize]| {
                    g.push(Node {
                        op,
                        inputs: ins,
                        shape: Shape::from_dims(dims),
                        dtype: DType::F32,
                    })
                };
                let rms = || {
                    Op::Fused(
                        FusedOps::RMS_NORM_LAST_DIM,
                        FusedOpParams::RmsNormLastDim { eps },
                    )
                };
                let sm = || Op::Fused(FusedOps::SOFTMAX_LAST_DIM, FusedOpParams::SoftmaxLastDim);
                let x = mk(&mut g, Op::Const, vec![], &[1, H]);
                let n1 = mk(&mut g, rms(), vec![x], &[1, H]);
                let s = mk(&mut g, Op::Silu, vec![n1], &[1, H]);
                let p = mk(&mut g, sm(), vec![s], &[1, H]);
                let r = mk(&mut g, Op::Add, vec![p, x], &[1, H]);
                let n2 = mk(&mut g, rms(), vec![r], &[1, H]);
                let y = mk(&mut g, Op::Add, vec![n2, x], &[1, H]);
                for id in [n1, s, p, r, n2, y] {
                    g.set_target_backend(id, BackendId::Cuda);
                }
                (x, y)
            };
            (graph, x, y)
        };
        let up = |v: &[f32]| -> Arc<RwLock<Storage>> {
            let cb = CudaStorageBytes::from_cpu_bytes(&dev, bytemuck::cast_slice(v)).expect("h2d");
            Arc::new(RwLock::new(Storage::new(
                BackendStorage::Cuda(cb),
                DType::F32,
            )))
        };
        let read = |arc: &Arc<RwLock<Storage>>| -> Vec<f32> {
            let g = arc.read().unwrap();
            let BackendStorage::Cuda(c) = &g.inner else {
                panic!("not cuda")
            };
            bytemuck::cast_slice(&c.to_cpu_bytes().expect("d2h")).to_vec()
        };
        let realize_ref = |xv: &[f32]| -> Vec<f32> {
            let (g, xr, yr) = build();
            let mut ins = StorageCache::new();
            ins.insert(xr, up(xv));
            let (arc, _) = PipelinedExecutor::realize(g, yr, ins).expect("ref realize");
            read(&arc)
        };

        // 5 distinct "tokens" (distinct per-token input value-sets).
        let tokens: Vec<Vec<f32>> = (0..5)
            .map(|t| {
                (0..H)
                    .map(|c| 0.1 * (t as f32 + 1.0) * (c as f32 + 1.0))
                    .collect()
            })
            .collect();

        // Capture ONCE against token 0's fixed input buffer.
        let x_arc = up(&tokens[0]);
        let (graph, x, y) = build();
        let mut inputs = StorageCache::new();
        inputs.insert(x, x_arc.clone());
        let session =
            CapturedDecodeSession::capture(Arc::clone(&graph), y, inputs, &[x], SymEnv::default())
                .expect("capture session");

        // Warm result (token 0) already sits in the output buffer.
        assert_eq!(
            read(session.output()),
            realize_ref(&tokens[0]),
            "warm token 0"
        );

        // The decode loop: replay every token (incl. re-replaying token 0) via
        // in-place H2D update of the fixed input + one graph launch.
        for (t, tok) in tokens.iter().enumerate() {
            let out = session
                .replay_token(&[(x, bytemuck::cast_slice(tok))])
                .expect("replay_token");
            assert_eq!(
                read(&out),
                realize_ref(tok),
                "replayed token {t} matches uncaptured realize"
            );
        }
        // Distinct tokens must give distinct outputs (live recompute, not stale).
        assert_ne!(
            realize_ref(&tokens[0]),
            realize_ref(&tokens[1]),
            "tokens must differ"
        );
    }

    /// PR-C1 behavior contract: `realize_with_optimized_route` with an
    /// **empty** route (the no-pressure / no-telemetry case) is
    /// byte-identical to `realize_with_optimized` (arm-0). On a branchless
    /// graph the route has no branches to resolve, so both must produce
    /// the same bytes — realize is unchanged from Phase B.
    #[test]
    fn realize_with_optimized_route_empty_equals_arm0() {
        use crate::optimize::OptimizedGraph;

        let build = || {
            let graph = Arc::new(RwLock::new(Graph::new()));
            let (lhs_id, rhs_id, add_id) = {
                let mut g = graph.write().unwrap();
                let lhs_id = g.push(Node {
                    op: Op::Const,
                    inputs: vec![],
                    shape: Shape::from_dims(&[3]),
                    dtype: DType::F32,
                });
                let rhs_id = g.push(Node {
                    op: Op::Const,
                    inputs: vec![],
                    shape: Shape::from_dims(&[3]),
                    dtype: DType::F32,
                });
                let add_id = g.push(Node {
                    op: Op::Add,
                    inputs: vec![lhs_id, rhs_id],
                    shape: Shape::from_dims(&[3]),
                    dtype: DType::F32,
                });
                g.set_target_backend(add_id, BackendId::Cpu);
                (lhs_id, rhs_id, add_id)
            };
            let mut inputs = StorageCache::new();
            inputs.insert(
                lhs_id,
                Arc::new(RwLock::new(fuel_memory::from_slice_cpu(&[
                    1.0_f32, 2.0, 3.0,
                ]))),
            );
            inputs.insert(
                rhs_id,
                Arc::new(RwLock::new(fuel_memory::from_slice_cpu(&[
                    10.0_f32, 20.0, 30.0,
                ]))),
            );
            (graph, add_id, inputs)
        };

        let read_bytes = |arc: &Arc<RwLock<Storage>>| -> Vec<f32> {
            let guard = arc.read().unwrap();
            match &guard.inner {
                fuel_memory::BackendStorage::Cpu(c) => {
                    c.as_slice::<f32>().expect("f32 cast").to_vec()
                }
                _ => panic!("expected CPU output"),
            }
        };

        // Stamp the fixture with the LIVE topology generation rather than a
        // hard-coded 0. The executor's chunk-boundary check compares the plan's
        // stamped generation against `topology_generation()`; a real
        // `OptimizedGraph` carries the generation snapshotted at optimize time
        // (`plan.rs` `compile_plan`, which reads it *after* the bindings table
        // is warm), so it already reflects every registered backend. Force
        // `global_bindings()` init here so any feature-gated auto-registration
        // has run and settled the counter before we read it — otherwise a stale
        // read would be bumped out from under us the first time realize warms
        // the bindings. Under default features this is 0; under
        // `--features vulkan`, `register_derived_gpu_caps` bumps it to 1.
        drop(global_bindings());
        let live_gen = crate::dispatch::topology_generation();

        // Arm-0 path.
        let (g_a, add_a, in_a) = build();
        let opt_a = OptimizedGraph {
            roots: vec![add_a],
            generation: live_gen,
            placements: None,
        };
        let (arm0_arc, _) = PipelinedExecutor::realize_with_optimized(g_a, add_a, in_a, &opt_a)
            .expect("arm-0 realize");
        let arm0 = read_bytes(&arm0_arc);

        // Empty-route path — must match arm-0 exactly.
        let (g_r, add_r, in_r) = build();
        let opt_r = OptimizedGraph {
            roots: vec![add_r],
            generation: live_gen,
            placements: None,
        };
        let empty_route = PickedRoute::new();
        let (route_arc, _) =
            PipelinedExecutor::realize_with_optimized_route(g_r, add_r, in_r, &opt_r, &empty_route)
                .expect("empty-route realize");
        let routed = read_bytes(&route_arc);

        assert_eq!(arm0, vec![11.0, 22.0, 33.0]);
        assert_eq!(
            routed, arm0,
            "an empty route realizes byte-identically to arm-0 (Phase B contract)",
        );
    }

    /// `order_for` fix track (`docs/design/incremental-consumer-index.md`
    /// §5): [`PipelinedExecutor::realize_with_optimized_picking_cached_env`]
    /// over a shared `order_cache` must (a) realize byte-identically to the
    /// uncached arm-0 path, on both the cold (empty cache) and warm
    /// (populated cache) call, and (b) actually populate the cache on the
    /// first call rather than silently never using it.
    #[test]
    fn realize_with_optimized_picking_cached_env_matches_arm0_cold_and_warm() {
        use crate::optimize::OptimizedGraph;
        use std::sync::OnceLock;

        let build = || {
            let graph = Arc::new(RwLock::new(Graph::new()));
            let (lhs_id, rhs_id, add_id) = {
                let mut g = graph.write().unwrap();
                let lhs_id = g.push(Node {
                    op: Op::Const,
                    inputs: vec![],
                    shape: Shape::from_dims(&[3]),
                    dtype: DType::F32,
                });
                let rhs_id = g.push(Node {
                    op: Op::Const,
                    inputs: vec![],
                    shape: Shape::from_dims(&[3]),
                    dtype: DType::F32,
                });
                let add_id = g.push(Node {
                    op: Op::Add,
                    inputs: vec![lhs_id, rhs_id],
                    shape: Shape::from_dims(&[3]),
                    dtype: DType::F32,
                });
                g.set_target_backend(add_id, BackendId::Cpu);
                (lhs_id, rhs_id, add_id)
            };
            let mut inputs = StorageCache::new();
            inputs.insert(
                lhs_id,
                Arc::new(RwLock::new(fuel_memory::from_slice_cpu(&[
                    1.0_f32, 2.0, 3.0,
                ]))),
            );
            inputs.insert(
                rhs_id,
                Arc::new(RwLock::new(fuel_memory::from_slice_cpu(&[
                    10.0_f32, 20.0, 30.0,
                ]))),
            );
            (graph, add_id, inputs)
        };
        let read_bytes = |arc: &Arc<RwLock<Storage>>| -> Vec<f32> {
            let guard = arc.read().unwrap();
            match &guard.inner {
                fuel_memory::BackendStorage::Cpu(c) => {
                    c.as_slice::<f32>().expect("f32 cast").to_vec()
                }
                _ => panic!("expected CPU output"),
            }
        };

        drop(global_bindings());
        let live_gen = crate::dispatch::topology_generation();

        let (g, add, inputs) = build();
        let optimized = OptimizedGraph {
            roots: vec![add],
            generation: live_gen,
            placements: None,
        };
        let order_cache: OnceLock<Vec<NodeId>> = OnceLock::new();
        assert!(order_cache.get().is_none(), "sanity: cache starts empty");

        // Cold call: cache miss, must compute AND populate.
        let (cold_arc, _) = PipelinedExecutor::realize_with_optimized_picking_cached_env(
            g.clone(),
            add,
            inputs,
            &optimized,
            &order_cache,
            None,
            None,
            SymEnv::default(),
        )
        .expect("cold cached realize");
        assert_eq!(read_bytes(&cold_arc), vec![11.0, 22.0, 33.0]);
        assert!(
            order_cache.get().is_some(),
            "the cold call must populate order_cache, not leave it empty"
        );
        let cached_order_after_cold = order_cache.get().unwrap().clone();

        // Warm call: cache hit. Fresh inputs (same graph, new values) so the
        // realized OUTPUT differs from the cold call while the dispatch
        // ORDER (what's cached) must not change.
        let mut warm_inputs = StorageCache::new();
        warm_inputs.insert(
            g.read().unwrap().node(add).inputs[0],
            Arc::new(RwLock::new(fuel_memory::from_slice_cpu(&[
                100.0_f32, 200.0, 300.0,
            ]))),
        );
        warm_inputs.insert(
            g.read().unwrap().node(add).inputs[1],
            Arc::new(RwLock::new(fuel_memory::from_slice_cpu(&[
                1.0_f32, 2.0, 3.0,
            ]))),
        );
        let (warm_arc, _) = PipelinedExecutor::realize_with_optimized_picking_cached_env(
            g,
            add,
            warm_inputs,
            &optimized,
            &order_cache,
            None,
            None,
            SymEnv::default(),
        )
        .expect("warm cached realize");
        assert_eq!(
            read_bytes(&warm_arc),
            vec![101.0, 202.0, 303.0],
            "warm call must realize its OWN rebound inputs, not replay stale output",
        );
        assert_eq!(
            order_cache.get().unwrap(),
            &cached_order_after_cold,
            "the warm call must reuse the SAME cached order, not recompute a different one",
        );
    }

    /// `order_for` fix track (`docs/design/incremental-consumer-index.md`
    /// §5), PM review question: does `OrderSource::Cached` weaken the
    /// existing `TopologyChanged` chunk-boundary check? No — proven at the
    /// unit level (not a live topology-bump integration test, deliberately:
    /// `bump_topology_generation()` mutates a process-wide atomic shared by
    /// every concurrently-running test in this binary, including the cold/
    /// warm cache test above, which reads it into a local at the start of
    /// ITS OWN body and would flake if another test's bump landed mid-call;
    /// introducing that cross-test hazard isn't worth it when the two arms'
    /// `generation_for` expressions are provably identical). `generation_for`
    /// is private to this module, so it's tested directly here: given the
    /// SAME `&OptimizedGraph`, `Cached` and `Optimized` must report the
    /// IDENTICAL generation — the chunk-boundary check that compares this
    /// value against the LIVE `topology_generation()` fires exactly the
    /// same way for both, so caching cannot silently skip or weaken it.
    #[test]
    fn cached_generation_for_matches_optimized_generation_for() {
        let optimized = OptimizedGraph {
            roots: vec![NodeId(0)],
            generation: 42,
            placements: None,
        };
        let order_cache: std::sync::OnceLock<Vec<NodeId>> = std::sync::OnceLock::new();

        let optimized_gen = generation_for(&OrderSource::Optimized {
            optimized: &optimized,
            route: None,
        });
        let cached_gen = generation_for(&OrderSource::Cached {
            optimized: &optimized,
            order_cache: &order_cache,
        });
        assert_eq!(
            optimized_gen,
            Some(42),
            "sanity: Optimized reports the OptimizedGraph's own generation",
        );
        assert_eq!(
            cached_gen, optimized_gen,
            "Cached must report the SAME generation as Optimized for the same \
             OptimizedGraph -- the TopologyChanged check cannot tell the two apart, \
             by construction, so caching cannot silently weaken it",
        );
    }

    /// **PR-C3 reorder-invariance gate (CPU, runnable — ORDERING level).**
    /// The auto-overlap reorder is a HINT: it changes only *which order*
    /// independent device runs dispatch in, never *which* nodes run nor the
    /// result. The byte-identical-OUTPUT proof is necessarily LIVE (the
    /// reorder fires only on a ≥2-device graph, which needs ≥2 real backends
    /// to realize — see `cuda_vulkan_multidevice_realize_live`'s
    /// `[22,86,192,340]` oracle, which now flows through this reorder, plus
    /// the dual-GPU `cuda_vulkan_overlap_bench_live` arbitrary-graph
    /// auto-overlap gate). THIS test proves the ordering-level invariant on a
    /// CPU-only path: the C3 reorder of a two-device graph is a valid
    /// topological **permutation** of the un-reordered topo order — same node
    /// multiset, every node still after its inputs — that genuinely DIFFERS
    /// from the un-reordered order (so the reorder fired). Because the
    /// executor caches each node's result and every op is deterministic,
    /// realizing any valid topological order yields identical bytes; this
    /// test guards the precondition (a valid permutation) that makes that so.
    #[test]
    fn c3_reorder_is_a_valid_topo_permutation() {
        // Two INDEPENDENT device sub-chains over distinct consts reconverging
        // at a final CUDA add, the Vulkan root bridged to CUDA by an explicit
        // Op::Copy (a residency seam = its own run). Reconverge inputs in the
        // "wrong" (cuda-first) order, the order the un-reordered topo DFS pops
        // CUDA-last — exactly the arbitrary-graph shape the auto-overlap pass
        // must fix. The chains are UNARY (Relu) so each stays a single
        // contiguous run (no const-re-read fan-in fragmenting them), exactly
        // like the live benchmark's `relu(mul·add)` chains; the CUDA chain is
        // HEAVIER (4 vs 2) so the critical-path reorder emits it first.
        // (Stamps only; not realized here — the ordering proof, CPU-runnable.)
        let n = |g: &mut Graph, op: Op, inputs: Vec<NodeId>| {
            g.push(Node {
                op,
                inputs,
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            })
        };
        let mut g = Graph::new();
        let ca = n(&mut g, Op::Const, vec![]);
        let cb = n(&mut g, Op::Const, vec![]);
        // CUDA chain — HEAVIER (4 unary stages, one contiguous run).
        let mut cprev = ca;
        for _ in 0..4 {
            cprev = n(&mut g, Op::Relu, vec![cprev]);
            g.set_target_backend(cprev, BackendId::Cuda);
        }
        let c2 = cprev; // CUDA chunk root
        // Vulkan chain — lighter (2 unary stages).
        let mut vprev = cb;
        for _ in 0..2 {
            vprev = n(&mut g, Op::Relu, vec![vprev]);
            g.set_target_backend(vprev, BackendId::Vulkan);
        }
        let v2 = vprev; // Vulkan chunk root
        // Vulkan -> CUDA copy (residency seam), then reconverge on CUDA with
        // inputs [cuda_root, copy] (cuda-first).
        let copy = g.push(Node {
            op: Op::Copy {
                target: DeviceLocation::Cuda { gpu_id: 0 },
            },
            inputs: vec![v2],
            shape: Shape::from_dims(&[3]),
            dtype: DType::F32,
        });
        g.set_target_backend(copy, BackendId::Cuda);
        let out = n(&mut g, Op::Add, vec![c2, copy]);
        g.set_target_backend(out, BackendId::Cuda);

        let unreordered = execution_plan(&g, &[out]);
        let reordered = fuel_graph::lower_runs_arm0(&g, &[out]);

        // (1) The reorder FIRED — the order changed (else the test is
        //     vacuous + auto-overlap did nothing on an arbitrary graph).
        assert_ne!(
            reordered, unreordered,
            "the C3 reorder must change the order on a two-device graph; \
             reordered={reordered:?} unreordered={unreordered:?}",
        );

        // (2) Same node multiset — a permutation, no add/drop.
        let mut a = unreordered.clone();
        let mut b = reordered.clone();
        a.sort();
        b.sort();
        assert_eq!(a, b, "the reorder is a permutation of the same node set");

        // (3) The reordered order is a VALID topological order: every node
        //     appears after all of its inputs that are in the order.
        let pos: std::collections::HashMap<NodeId, usize> = reordered
            .iter()
            .enumerate()
            .map(|(i, &node)| (node, i))
            .collect();
        for (i, &node) in reordered.iter().enumerate() {
            for &inp in &g.node(node).inputs {
                if let Some(&j) = pos.get(&inp) {
                    assert!(
                        j < i,
                        "input {inp:?} of {node:?} must precede it in the reordered \
                         order; reordered={reordered:?}",
                    );
                }
            }
        }

        // (4) The overlap-relevant property: BOTH device producer chunks
        //     (the CUDA root c2, the Vulkan root v2) are emitted BEFORE the
        //     host-blocking cross-device copy — the A4b overlap topology the
        //     un-reordered cuda-last DFS violates — AND the HEAVIER CUDA chunk
        //     is emitted first (critical-path: the auto-submit device runs
        //     earliest, the live-overlap regression guard).
        let p = |node: NodeId| pos[&node];
        assert!(p(c2) < p(copy), "CUDA chunk before the cross-device copy");
        assert!(p(v2) < p(copy), "Vulkan chunk before the cross-device copy");
        assert!(
            p(c2) < p(v2),
            "the heavier CUDA chunk is emitted before the lighter Vulkan chunk \
             (critical-path heavy-first); reordered={reordered:?}",
        );
    }

    /// **C3 follow-on — the OPERAND-ORDER-INVARIANCE gate (CPU, runnable —
    /// the reliable operand-independence proof).** The residual the live
    /// `cuda_vulkan_overlap_bench_live` benchmark targets is whether the
    /// reconverge OPERAND order (`xc.add(&xv)` vs `xv.add(&xc)`) changes the
    /// wall-clock overlap. Wall-clock is thermally noisy on a heterogeneous
    /// dev box; the DETERMINISTIC root guarantee is this: the C3-reordered
    /// dispatch order the executor consumes is **byte-identical for both
    /// operand orders**. If the lowered order is the same, the executor
    /// processes the SAME WorkItem stream with the SAME cross-device
    /// submit/wait timing, so overlap cannot depend on the operand order.
    ///
    /// This builds the SAME two-device reconverge as
    /// `c3_reorder_is_a_valid_topo_permutation` but TWICE — once cuda-first
    /// (`add(cuda_root, copy)`), once vulkan-first (`add(copy, cuda_root)`)
    /// — over structurally identical graphs (same node ids, only the
    /// reconverge's two inputs swapped) and asserts `lower_runs_arm0` emits
    /// the identical op/device run sequence. The live executor trace confirms
    /// the same: both operand orders produce the identical WorkItem stream
    /// (the eager-submit/drain/wait sequence at every cross-device copy is the
    /// same, and the CUDA chunk is enqueued first in both).
    #[test]
    fn c3_reorder_is_operand_order_invariant() {
        // Build the two-device reconverge with the reconverge inputs in the
        // given order; return (graph, out) over a FRESH graph so node ids line
        // up 1:1 between the two builds (only the add's input order differs).
        let build = |cuda_first: bool| -> (Graph, NodeId) {
            let n = |g: &mut Graph, op: Op, inputs: Vec<NodeId>| {
                g.push(Node {
                    op,
                    inputs,
                    shape: Shape::from_dims(&[3]),
                    dtype: DType::F32,
                })
            };
            let mut g = Graph::new();
            let ca = n(&mut g, Op::Const, vec![]);
            let cb = n(&mut g, Op::Const, vec![]);
            let mut cprev = ca;
            for _ in 0..4 {
                cprev = n(&mut g, Op::Relu, vec![cprev]);
                g.set_target_backend(cprev, BackendId::Cuda);
            }
            let c2 = cprev;
            let mut vprev = cb;
            for _ in 0..2 {
                vprev = n(&mut g, Op::Relu, vec![vprev]);
                g.set_target_backend(vprev, BackendId::Vulkan);
            }
            let v2 = vprev;
            let copy = g.push(Node {
                op: Op::Copy {
                    target: DeviceLocation::Cuda { gpu_id: 0 },
                },
                inputs: vec![v2],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            g.set_target_backend(copy, BackendId::Cuda);
            // The ONE knob: the reconverge's two inputs, swapped.
            let inputs = if cuda_first {
                vec![c2, copy]
            } else {
                vec![copy, c2]
            };
            let out = n(&mut g, Op::Add, inputs);
            g.set_target_backend(out, BackendId::Cuda);
            (g, out)
        };

        let (g_cf, out_cf) = build(true);
        let (g_vf, out_vf) = build(false);

        // Lower both via the production path (dispatch_order → lower_runs_arm0
        // → device_alternating_order, the C3 reorder).
        let order_cf = fuel_graph::lower_runs_arm0(&g_cf, &[out_cf]);
        let order_vf = fuel_graph::lower_runs_arm0(&g_vf, &[out_vf]);

        // The decisive guarantee: the lowered dispatch order — as an
        // (op, device) sequence — is IDENTICAL for both operand orders. (Node
        // ids can differ if the build assigned them differently, but here the
        // builds are structurally identical so even the ids match; compare the
        // op/device sequence to be robust + descriptive on failure.)
        let tag = |g: &Graph, ids: &[NodeId]| -> Vec<(String, Option<BackendId>)> {
            ids.iter()
                .map(|&id| (format!("{:?}", g.node(id).op), g.target_backend(id)))
                .collect()
        };
        let seq_cf = tag(&g_cf, &order_cf);
        let seq_vf = tag(&g_vf, &order_vf);
        assert_eq!(
            seq_cf, seq_vf,
            "the C3-reordered dispatch order MUST be identical for both \
             reconverge operand orders (operand-order-invariance — the \
             executor then sees the same WorkItem stream regardless of \
             operand order, so overlap is operand-insensitive).\n\
             cuda-first = {seq_cf:?}\nvulkan-first = {seq_vf:?}",
        );
    }

    /// A selector that always takes the LAST arm (arm 1) so a non-arm-0
    /// pick is observable (a `WinnerSelector` picks arm-0 ⇒ empty route).
    #[cfg(test)]
    #[derive(Debug)]
    struct PickLast;
    #[cfg(test)]
    impl RuntimeSelector for PickLast {
        fn select<'a>(
            &self,
            set: &'a crate::ranker::AlternativeSet,
        ) -> Option<&'a crate::ranker::Candidate> {
            set.alternatives().last()
        }
    }

    /// Build the 2-arm diamond (mirrors `route_picker::tests::diamond`):
    /// arm-0 on CUDA, arm-1 on CPU; the arm exits carry their
    /// `target_backend` so the picker reads each arm's placement. Returns
    /// `(graph_arc, branch, post)`.
    #[cfg(test)]
    fn picking_diamond() -> (Arc<RwLock<Graph>>, NodeId, NodeId) {
        let node = |g: &mut Graph, op: Op, inputs: Vec<NodeId>| {
            g.push(Node {
                op,
                inputs,
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            })
        };
        let mut g = Graph::new();
        let pre = node(&mut g, Op::Const, vec![]);
        let diverge = node(&mut g, Op::Relu, vec![pre]);
        let arm0 = node(&mut g, Op::Silu, vec![diverge]);
        let arm1 = node(&mut g, Op::GeluTanh, vec![diverge]);
        g.set_target_backend(arm0, BackendId::Cuda);
        g.set_target_backend(arm1, BackendId::Cpu);
        let reconverge = node(&mut g, Op::Relu, vec![arm0]);
        let mut b = g.open_branch(diverge);
        b.add_arm(arm0);
        b.add_arm(arm1);
        let branch = b
            .finalize_branches(&mut g, reconverge)
            .expect("well-formed 2-arm branch")
            .expect("2 arms survive");
        let post = node(&mut g, Op::Tanh, vec![reconverge]);
        (Arc::new(RwLock::new(g)), branch, post)
    }

    /// Step E Phase C, PR C1: the streaming picking entry's gate.
    /// `streaming_pick_for` (the streaming successor of the old one-shot
    /// `pick_route_for`) must (1) return `Some(pick)` when a selector + a
    /// branch are present (⇒ the `OrderSource::Streaming` walk), (2) return
    /// `None` with no selector (the `runtime_selector_disabled` opt-out ⇒
    /// arm-0), and (3) short-circuit to `None` on a branchless graph (the
    /// fast-path ⇒ the untouched `dispatch_order` lowering).
    #[test]
    fn streaming_pick_for_gates() {
        let (graph, _branch, post) = picking_diamond();
        let _ = post;

        // (1) selector + branch ⇒ Some(pick).
        let sel: Arc<dyn RuntimeSelector> = Arc::new(PickLast);
        assert!(
            PipelinedExecutor::streaming_pick_for(&graph, Some(sel.clone()), None)
                .expect("ok")
                .is_some(),
            "selector + branch ⇒ a streaming pick",
        );

        // (2) no selector ⇒ None (arm-0 lowering).
        assert!(
            PipelinedExecutor::streaming_pick_for(&graph, None, None)
                .expect("ok")
                .is_none(),
            "no selector ⇒ no streaming pick (arm-0 lowering)",
        );

        // (3) branchless graph + selector ⇒ None (the fast-path).
        let node = |g: &mut Graph, op: Op, inputs: Vec<NodeId>| {
            g.push(Node {
                op,
                inputs,
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            })
        };
        let bl_graph = {
            let mut g = Graph::new();
            let c1 = node(&mut g, Op::Const, vec![]);
            let c2 = node(&mut g, Op::Const, vec![]);
            let add = node(&mut g, Op::Add, vec![c1, c2]);
            g.set_target_backend(add, BackendId::Cpu);
            Arc::new(RwLock::new(g))
        };
        assert!(
            PipelinedExecutor::streaming_pick_for(&bl_graph, Some(sel), None)
                .expect("ok")
                .is_none(),
            "branchless graph ⇒ no branches ⇒ no streaming pick (fast-path)",
        );
    }

    /// **C1 byte-identity gate (executor layer).** The STREAMED dispatch
    /// order — produced by the compiler thread's lazy per-branch resolution
    /// (`walk_picked_route_streaming` over `resolve_branch`, the same path
    /// `order_for(OrderSource::Streaming)` collects) — equals the ONE-SHOT
    /// order (`lower_picked_route` over the route the eager `pick_route`
    /// resolves) on the branched diamond, for both arm-0 (WinnerSelector)
    /// and arm-1 (PickLast). This is the proof that C1 changes *when* a
    /// branch resolves, not *which* arm — the same selector, the same
    /// `resolve_branch`, the same emitted order.
    #[test]
    fn streamed_order_equals_one_shot() {
        let (graph, _branch, post) = picking_diamond();
        let g = graph.read().unwrap();
        let bindings = global_bindings();

        for sel in [
            Arc::new(crate::ranker::WinnerSelector) as Arc<dyn RuntimeSelector>,
            Arc::new(PickLast) as Arc<dyn RuntimeSelector>,
        ] {
            // One-shot: resolve the whole route, then lower.
            let one_shot_route = pick_route(&g, &[post], &bindings, sel.as_ref(), None);
            let one_shot = fuel_graph::lower_picked_route(&g, &[post], &one_shot_route);

            // Streamed: resolve each branch lazily at the frontier via the
            // SAME `resolve_branch` the compiler thread uses.
            let streamed = fuel_graph::lower_picked_route_streaming(&g, &[post], |branch| {
                resolve_branch(&g, branch, &bindings, sel.as_ref())
            });

            assert_eq!(
                streamed, one_shot,
                "streamed order must equal one-shot for selector {sel:?}",
            );
        }
    }

    /// Realizing a node whose target_backend isn't set surfaces a
    /// typed error (no panic).
    #[test]
    fn pipelined_errors_on_unset_target_backend() {
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (lhs_id, rhs_id, add_id) = {
            let mut g = graph.write().unwrap();
            let lhs = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let rhs = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            // Deliberately do NOT call set_target_backend on the Add.
            let add = g.push(Node {
                op: Op::Add,
                inputs: vec![lhs, rhs],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            (lhs, rhs, add)
        };

        let mut inputs = StorageCache::new();
        inputs.insert(
            lhs_id,
            Arc::new(RwLock::new(fuel_memory::from_slice_cpu(&[1.0_f32, 2.0]))),
        );
        inputs.insert(
            rhs_id,
            Arc::new(RwLock::new(fuel_memory::from_slice_cpu(&[3.0_f32, 4.0]))),
        );

        let result = PipelinedExecutor::realize(graph, add_id, inputs);
        assert!(result.is_err(), "missing target_backend must error");
        let msg = result.unwrap_err().to_string();
        assert!(
            msg.contains("target_backend"),
            "error names the unmet precondition: {msg}"
        );
    }

    /// Realizing a Const-only graph adopts the pre-seeded input
    /// without calling any kernel.
    #[test]
    fn pipelined_realize_const_only() {
        let storage = fuel_memory::from_slice_cpu(&[5.0_f32, 6.0, 7.0]);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let const_id = {
            let mut g = graph.write().unwrap();
            g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            })
        };

        let mut inputs = StorageCache::new();
        inputs.insert(const_id, Arc::new(RwLock::new(storage)));

        let (result_arc, _) =
            PipelinedExecutor::realize(graph, const_id, inputs).expect("realize const");
        let guard = result_arc.read().unwrap();
        if let fuel_memory::BackendStorage::Cpu(c) = &guard.inner {
            let typed: &[f32] = c.as_slice().unwrap();
            assert_eq!(typed, &[5.0, 6.0, 7.0]);
        }
    }

    /// Missing input-cache entry for a Const surfaces a typed error.
    #[test]
    fn pipelined_errors_on_missing_const_input() {
        let graph = Arc::new(RwLock::new(Graph::new()));
        let const_id = {
            let mut g = graph.write().unwrap();
            g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            })
        };
        // Deliberately empty input cache.
        let inputs = StorageCache::new();
        let result = PipelinedExecutor::realize(graph, const_id, inputs);
        assert!(result.is_err());
    }

    /// E2E: 2-node graph (Const + Relu) — exercises the unary
    /// dispatch wrapper + kernel through the pipelined executor.
    #[test]
    fn pipelined_realize_const_relu() {
        let storage = fuel_memory::from_slice_cpu(&[-1.0_f32, 0.0, 0.5, -3.5, 7.25]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, relu_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            let relu_id = g.push(Node {
                op: Op::Relu,
                inputs: vec![in_id],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            g.set_target_backend(relu_id, BackendId::Cpu);
            (in_id, relu_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));

        let (result_arc, _) = PipelinedExecutor::realize(graph, relu_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        if let fuel_memory::BackendStorage::Cpu(c) = &guard.inner {
            let typed: &[f32] = c.as_slice().expect("f32 cast");
            assert_eq!(typed, &[0.0, 0.0, 0.5, 0.0, 7.25]);
        }
    }

    /// E2E: Const + Const + Sub + Mul + Div — exercises three more
    /// of the freshly-migrated binary kernels in one graph. Verifies
    /// that intermediates flow through the cache as expected.
    #[test]
    fn pipelined_realize_chained_binary_ops() {
        let a_storage = fuel_memory::from_slice_cpu(&[10.0_f32, 20.0]);
        let b_storage = fuel_memory::from_slice_cpu(&[3.0_f32, 5.0]);
        let c_storage = fuel_memory::from_slice_cpu(&[2.0_f32, 4.0]);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (a_id, b_id, c_id, sub_id, mul_id, div_id) = {
            let mut g = graph.write().unwrap();
            let a = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let b = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let c = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            // (a - b)         = [7, 15]
            let sub = g.push(Node {
                op: Op::Sub,
                inputs: vec![a, b],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            // (a - b) * c     = [14, 60]
            let mul = g.push(Node {
                op: Op::Mul,
                inputs: vec![sub, c],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            // ((a-b)*c) / b   = [14/3, 12]
            let div = g.push(Node {
                op: Op::Div,
                inputs: vec![mul, b],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            g.set_target_backend(sub, BackendId::Cpu);
            g.set_target_backend(mul, BackendId::Cpu);
            g.set_target_backend(div, BackendId::Cpu);
            (a, b, c, sub, mul, div)
        };

        let _ = (sub_id, mul_id);
        let mut inputs = StorageCache::new();
        inputs.insert(a_id, Arc::new(RwLock::new(a_storage)));
        inputs.insert(b_id, Arc::new(RwLock::new(b_storage)));
        inputs.insert(c_id, Arc::new(RwLock::new(c_storage)));

        let (result_arc, _) = PipelinedExecutor::realize(graph, div_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        if let fuel_memory::BackendStorage::Cpu(c) = &guard.inner {
            let typed: &[f32] = c.as_slice().unwrap();
            assert!((typed[0] - (14.0_f32 / 3.0)).abs() < 1e-6);
            assert!((typed[1] - 12.0).abs() < 1e-6);
        }
    }

    /// E2E: Const + Transpose — verifies metadata-only view ops
    /// share the input's Storage Arc and produce a strided Layout.
    /// Stage 3 of Layout-on-Node.
    #[test]
    fn pipelined_realize_transpose_is_metadata_only() {
        // shape [2, 3]; transpose → shape [3, 2], strided
        let storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, t_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let t_id = g.push(Node {
                op: Op::Transpose,
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3, 2]),
                dtype: DType::F32,
            });
            // Note: NO set_target_backend — transpose is metadata-only
            // and doesn't run on a backend.
            (in_id, t_id)
        };
        let mut inputs = StorageCache::new();
        let in_arc = Arc::new(RwLock::new(storage));
        inputs.insert(in_id, Arc::clone(&in_arc));

        let (result_arc, result_layout) =
            PipelinedExecutor::realize(graph, t_id, inputs).expect("realize");

        // The output Storage Arc is the SAME Arc as the input —
        // metadata-only adoption shares bytes.
        assert!(
            Arc::ptr_eq(&result_arc, &in_arc),
            "transpose must share input bytes"
        );

        // The output Layout is the transposed view.
        assert_eq!(result_layout.shape().dims(), &[3, 2]);
        assert_eq!(result_layout.stride(), &[1, 3]);
        assert!(!result_layout.is_contiguous());
    }

    /// E2E: Const + Flip(dim=0) — verifies Op::Flip is metadata-only.
    /// The output Storage Arc must be the SAME Arc (no copy);
    /// the output Layout has a negated stride at dim 0 and a
    /// shifted start_offset.
    #[test]
    fn pipelined_realize_flip_is_metadata_only() {
        // shape [3, 4]; flip dim 0 → shape [3, 4], stride [-4, 1], offset 8.
        let data: Vec<f32> = (1..=12).map(|x| x as f32).collect();
        let storage = fuel_memory::from_slice_cpu(&data);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, f_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3, 4]),
                dtype: DType::F32,
            });
            let f_id = g.push(Node {
                op: Op::Flip { dim: 0 },
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3, 4]),
                dtype: DType::F32,
            });
            (in_id, f_id)
        };
        let mut inputs = StorageCache::new();
        let in_arc = Arc::new(RwLock::new(storage));
        inputs.insert(in_id, Arc::clone(&in_arc));

        let (result_arc, result_layout) =
            PipelinedExecutor::realize(graph, f_id, inputs).expect("realize");

        assert!(
            Arc::ptr_eq(&result_arc, &in_arc),
            "flip must share input bytes"
        );
        assert_eq!(result_layout.shape().dims(), &[3, 4]);
        assert_eq!(result_layout.stride(), &[-4_isize, 1]);
        assert_eq!(result_layout.start_offset(), 8);
        assert!(!result_layout.is_contiguous());
    }

    /// E2E: Const + Permute(rank-3 axes [2, 0, 1]) — verifies the
    /// general permute path through metadata-only adoption.
    #[test]
    fn pipelined_realize_permute_is_metadata_only() {
        // shape [2, 3, 4]; permute axes [2, 0, 1] → shape [4, 2, 3]
        let data: Vec<f32> = (1..=24).map(|x| x as f32).collect();
        let storage = fuel_memory::from_slice_cpu(&data);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, p_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3, 4]),
                dtype: DType::F32,
            });
            let p_id = g.push(Node {
                op: Op::Permute(vec![2, 0, 1]),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[4, 2, 3]),
                dtype: DType::F32,
            });
            (in_id, p_id)
        };
        let mut inputs = StorageCache::new();
        let in_arc = Arc::new(RwLock::new(storage));
        inputs.insert(in_id, Arc::clone(&in_arc));

        let (result_arc, result_layout) =
            PipelinedExecutor::realize(graph, p_id, inputs).expect("realize");

        assert!(Arc::ptr_eq(&result_arc, &in_arc));
        assert_eq!(result_layout.shape().dims(), &[4, 2, 3]);
        // Original strides for shape [2, 3, 4] are [12, 4, 1].
        // After permute axes [2, 0, 1]: [strides[2], strides[0], strides[1]] = [1, 12, 4].
        assert_eq!(result_layout.stride(), &[1, 12, 4]);
    }

    /// E2E: Const + Cast(f32→f64) — verifies cast through the
    /// pipelined executor. Output Storage has the target dtype;
    /// bytes encode the widened values.
    #[test]
    fn pipelined_realize_cast_f32_to_f64() {
        let storage = fuel_memory::from_slice_cpu(&[1.5_f32, -2.25, 100.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, c_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let c_id = g.push(Node {
                op: Op::Cast(DType::F64),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F64,
            });
            g.set_target_backend(c_id, BackendId::Cpu);
            (in_id, c_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));

        let (result_arc, result_layout) =
            PipelinedExecutor::realize(graph, c_id, inputs).expect("realize");
        assert_eq!(result_layout.shape().dims(), &[3]);

        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::F64);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f64>().unwrap(), &[1.5_f64, -2.25, 100.0]);
    }

    /// E2E: Const + Cast(f32→bf16) + Cast(bf16→f32) — round trip
    /// through bf16; verifies the Cast wrapper's source-dtype
    /// dispatch (different sources hit different match arms).
    /// Inputs chosen to round-trip exactly through bf16.
    #[test]
    fn pipelined_realize_cast_round_trip_via_bf16() {
        let storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, -3.0, 0.5]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, c1_id, c2_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            let c1_id = g.push(Node {
                op: Op::Cast(DType::BF16),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[4]),
                dtype: DType::BF16,
            });
            let c2_id = g.push(Node {
                op: Op::Cast(DType::F32),
                inputs: vec![c1_id],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            g.set_target_backend(c1_id, BackendId::Cpu);
            g.set_target_backend(c2_id, BackendId::Cpu);
            (in_id, c1_id, c2_id)
        };
        let _ = c1_id;
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));

        let (result_arc, _) = PipelinedExecutor::realize(graph, c2_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::F32);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[1.0_f32, 2.0, -3.0, 0.5]);
    }

    /// E2E: Const + Slice — slice is metadata-only; the output Arc
    /// shares bytes with the input, and the Layout's start_offset
    /// + narrowed shape reflect the slice. Stage 3 of Layout-on-Node
    ///
    /// extended to cover Op::Slice via Layout::narrow.
    #[test]
    fn pipelined_realize_slice_is_metadata_only() {
        // shape [5]; slice dim 0 from index 1 with len 3 → shape [3]
        // Source: [10, 20, 30, 40, 50]; slice → [20, 30, 40]
        let storage = fuel_memory::from_slice_cpu(&[10.0_f32, 20.0, 30.0, 40.0, 50.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, s_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            let s_id = g.push(Node {
                op: Op::Slice {
                    dim: 0,
                    start: 1,
                    len: 3,
                },
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            (in_id, s_id)
        };
        let mut inputs = StorageCache::new();
        let in_arc = Arc::new(RwLock::new(storage));
        inputs.insert(in_id, Arc::clone(&in_arc));

        let (result_arc, result_layout) =
            PipelinedExecutor::realize(graph, s_id, inputs).expect("realize");

        // Bytes shared with the input.
        assert!(Arc::ptr_eq(&result_arc, &in_arc));
        assert_eq!(result_layout.shape().dims(), &[3]);
        // Slice into a contiguous source: the resulting layout has
        // start_offset = original_stride[0] * start = 1 * 1 = 1,
        // and stride [1] (still contiguous within the narrowed dim).
        assert_eq!(result_layout.start_offset(), 1);
        assert_eq!(result_layout.stride(), &[1]);
    }

    /// E2E: Const + Slice + SumAll — slice is metadata-only, but
    /// sum needs contiguous bytes, so auto-Contiguize materializes
    /// the slice before reduce. Tests the stage 3+4 integration
    /// through Op::Slice. Sum of `[20, 30, 40]` is 90.
    #[test]
    fn pipelined_realize_slice_then_sum_all() {
        let storage = fuel_memory::from_slice_cpu(&[10.0_f32, 20.0, 30.0, 40.0, 50.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, s_id, sum_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            let s_id = g.push(Node {
                op: Op::Slice {
                    dim: 0,
                    start: 1,
                    len: 3,
                },
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let sum_id = g.push(Node {
                op: Op::SumAll,
                inputs: vec![s_id],
                shape: Shape::from_dims(&[]),
                dtype: DType::F32,
            });
            g.set_target_backend(sum_id, BackendId::Cpu);
            (in_id, s_id, sum_id)
        };
        let _ = s_id;
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));

        let (result_arc, _) = PipelinedExecutor::realize(graph, sum_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[90.0]);
    }

    /// E2E: Const + Flip + ReluElementwise — Flip is metadata-only,
    /// then a kernel that doesn't yet handle negative strides forces
    /// auto-Contiguize to materialize through StridedIndex (which
    /// handles signed strides natively). Verifies the post-flip
    /// data ordering ([4, 3, 2, 1]) is correctly seen by the kernel.
    #[test]
    fn pipelined_realize_flip_then_relu_materializes_in_reverse() {
        let storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, _f_id, r_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            let f_id = g.push(Node {
                op: Op::Flip { dim: 0 },
                inputs: vec![in_id],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            let r_id = g.push(Node {
                op: Op::Relu,
                inputs: vec![f_id],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            g.set_target_backend(r_id, BackendId::Cpu);
            (in_id, f_id, r_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));

        let (result_arc, result_layout) =
            PipelinedExecutor::realize(graph, r_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[4.0, 3.0, 2.0, 1.0]);
        assert_eq!(result_layout.shape().dims(), &[4]);
        assert!(result_layout.is_contiguous(), "relu output is contiguous");
    }

    /// E2E: Const + BroadcastTo — verifies that broadcast layouts
    /// have stride 0 on the broadcast dim while sharing the input's
    /// bytes. Stage 3 of Layout-on-Node.
    #[test]
    fn pipelined_realize_broadcast_is_metadata_only() {
        // shape [3]; broadcast to [4, 3] — leading dim is 0-stride
        let storage = fuel_memory::from_slice_cpu(&[10.0_f32, 20.0, 30.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, b_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let b_id = g.push(Node {
                op: Op::BroadcastTo(Shape::from_dims(&[4, 3])),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[4, 3]),
                dtype: DType::F32,
            });
            (in_id, b_id)
        };
        let mut inputs = StorageCache::new();
        let in_arc = Arc::new(RwLock::new(storage));
        inputs.insert(in_id, Arc::clone(&in_arc));

        let (result_arc, result_layout) =
            PipelinedExecutor::realize(graph, b_id, inputs).expect("realize");

        assert!(Arc::ptr_eq(&result_arc, &in_arc));
        assert_eq!(result_layout.shape().dims(), &[4, 3]);
        // Broadcasted leading dim has stride 0.
        assert_eq!(result_layout.stride(), &[0, 1]);
    }

    /// E2E: Const + Const + MatMul — exercises rank-2 matmul
    /// through the pipelined executor. Inputs are contiguous (no
    /// auto-Contiguize needed); the kernel walks them via the
    /// (m, n, k) carried in OpParams::Matmul.
    #[test]
    fn pipelined_realize_matmul_2x3_times_3x2() {
        // [[1, 2, 3], [4, 5, 6]] @ [[7, 8], [9, 10], [11, 12]]
        //   = [[58, 64], [139, 154]]
        let lhs_storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0]);
        let rhs_storage = fuel_memory::from_slice_cpu(&[7.0_f32, 8.0, 9.0, 10.0, 11.0, 12.0]);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (lhs_id, rhs_id, mm_id) = {
            let mut g = graph.write().unwrap();
            let lhs = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let rhs = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3, 2]),
                dtype: DType::F32,
            });
            let mm = g.push(Node {
                op: Op::MatMul,
                inputs: vec![lhs, rhs],
                shape: Shape::from_dims(&[2, 2]),
                dtype: DType::F32,
            });
            g.set_target_backend(mm, BackendId::Cpu);
            (lhs, rhs, mm)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(lhs_id, Arc::new(RwLock::new(lhs_storage)));
        inputs.insert(rhs_id, Arc::new(RwLock::new(rhs_storage)));

        let (result_arc, result_layout) =
            PipelinedExecutor::realize(graph, mm_id, inputs).expect("realize");
        assert_eq!(result_layout.shape().dims(), &[2, 2]);
        assert!(result_layout.is_contiguous());

        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[58.0, 64.0, 139.0, 154.0]);
    }

    /// E2E: batched matmul through the pipelined executor. Two
    /// batches of [2, 2] @ [2, 2]; the kernel iterates over
    /// `batch_count` and produces concatenated outputs.
    #[test]
    fn pipelined_realize_matmul_batched_2x_2x2_times_2x2() {
        let lhs_storage = fuel_memory::from_slice_cpu(&[
            1.0_f32, 2.0, 3.0, 4.0, // batch 0
            1.0, 0.0, 0.0, 1.0, // batch 1 (identity)
        ]);
        let rhs_storage = fuel_memory::from_slice_cpu(&[
            5.0_f32, 6.0, 7.0, 8.0, // batch 0
            10.0, 20.0, 30.0, 40.0, // batch 1
        ]);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (lhs_id, rhs_id, mm_id) = {
            let mut g = graph.write().unwrap();
            let lhs = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 2, 2]),
                dtype: DType::F32,
            });
            let rhs = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 2, 2]),
                dtype: DType::F32,
            });
            let mm = g.push(Node {
                op: Op::MatMul,
                inputs: vec![lhs, rhs],
                shape: Shape::from_dims(&[2, 2, 2]),
                dtype: DType::F32,
            });
            g.set_target_backend(mm, BackendId::Cpu);
            (lhs, rhs, mm)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(lhs_id, Arc::new(RwLock::new(lhs_storage)));
        inputs.insert(rhs_id, Arc::new(RwLock::new(rhs_storage)));

        let (result_arc, result_layout) =
            PipelinedExecutor::realize(graph, mm_id, inputs).expect("realize");
        assert_eq!(result_layout.shape().dims(), &[2, 2, 2]);
        assert!(result_layout.is_contiguous());

        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        // batch 0: [[1,2],[3,4]] @ [[5,6],[7,8]] = [[19,22],[43,50]]
        // batch 1: identity @ [[10,20],[30,40]]   = [[10,20],[30,40]]
        assert_eq!(
            c.as_slice::<f32>().unwrap(),
            &[19.0, 22.0, 43.0, 50.0, 10.0, 20.0, 30.0, 40.0]
        );
    }

    /// E2E: F64 elementwise add through the pipelined executor.
    /// Verifies that capability-driven dispatch correctly routes
    /// (AddElementwise, F64) to the f64 wrapper/kernel.
    #[test]
    fn pipelined_realize_add_f64() {
        let lhs = fuel_memory::from_slice_cpu(&[1.0_f64, 2.0, 3.0]);
        let rhs = fuel_memory::from_slice_cpu(&[10.0_f64, 20.0, 30.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (l_id, r_id, op_id) = {
            let mut g = graph.write().unwrap();
            let l = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F64,
            });
            let r = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F64,
            });
            let op = g.push(Node {
                op: Op::Add,
                inputs: vec![l, r],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F64,
            });
            g.set_target_backend(op, BackendId::Cpu);
            (l, r, op)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(l_id, Arc::new(RwLock::new(lhs)));
        inputs.insert(r_id, Arc::new(RwLock::new(rhs)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::F64);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f64>().unwrap(), &[11.0_f64, 22.0, 33.0]);
    }

    /// E2E: Op::Equal F32 → Bool mask through the pipelined executor (GAP-168c).
    /// Verifies (a) the binding-table key `(EqualElementwise, [F32, F32, U8],
    /// Cpu)` resolves, (b) the executor allocates a U8-sized output
    /// buffer (1 byte per element, not 4), (c) the kernel writes the
    /// expected mask bits including IEEE-754 NaN handling
    /// (`NaN == NaN` is false).
    #[test]
    fn pipelined_realize_eq_f32_to_bool_mask() {
        let lhs = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, f32::NAN, 0.0]);
        let rhs = fuel_memory::from_slice_cpu(&[1.0_f32, 5.0, 3.0, f32::NAN, -0.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (l_id, r_id, eq_id) = {
            let mut g = graph.write().unwrap();
            let l = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            let r = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            let eq = g.push(Node {
                op: Op::Equal,
                inputs: vec![l, r],
                shape: Shape::from_dims(&[5]),
                dtype: DType::Bool,
            });
            g.set_target_backend(eq, BackendId::Cpu);
            (l, r, eq)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(l_id, Arc::new(RwLock::new(lhs)));
        inputs.insert(r_id, Arc::new(RwLock::new(rhs)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, eq_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::Bool);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let mask: &[u8] = c.as_slice().expect("u8 view");
        // Index 0: 1.0 == 1.0 → 1.
        // Index 1: 2.0 != 5.0 → 0.
        // Index 2: 3.0 == 3.0 → 1.
        // Index 3: NaN == NaN → 0 (IEEE-754).
        // Index 4: 0.0 == -0.0 → 1 (IEEE-754 zero equality).
        assert_eq!(mask, &[1, 0, 1, 0, 1]);
    }

    /// E2E: Op::Ne F32 → Bool mask. Mirrors the Eq F32 test with
    /// inverted predicate; NaN-vs-NaN slot now yields `1` (since
    /// `NaN != NaN` per IEEE-754).
    #[test]
    fn pipelined_realize_ne_f32_to_bool_mask() {
        let lhs = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, f32::NAN, 0.0]);
        let rhs = fuel_memory::from_slice_cpu(&[1.0_f32, 5.0, 3.0, f32::NAN, -0.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (l_id, r_id, ne_id) = {
            let mut g = graph.write().unwrap();
            let l = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            let r = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            let ne = g.push(Node {
                op: Op::Ne,
                inputs: vec![l, r],
                shape: Shape::from_dims(&[5]),
                dtype: DType::Bool,
            });
            g.set_target_backend(ne, BackendId::Cpu);
            (l, r, ne)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(l_id, Arc::new(RwLock::new(lhs)));
        inputs.insert(r_id, Arc::new(RwLock::new(rhs)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, ne_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::Bool);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let mask: &[u8] = c.as_slice().expect("u8 view");
        // Inverse of the Eq test:
        // 1.0 != 1.0 → 0;  2.0 != 5.0 → 1;  3.0 != 3.0 → 0;
        // NaN != NaN → 1 (IEEE-754);  0.0 != -0.0 → 0 (IEEE-754).
        assert_eq!(mask, &[0, 1, 0, 1, 0]);
    }

    /// E2E: Op::Lt F32 → Bool mask. Confirms strict-less-than semantics
    /// + IEEE-754 NaN handling (any comparison with NaN is unordered →
    ///
    /// `0`).
    #[test]
    fn pipelined_realize_lt_f32_to_bool_mask() {
        let lhs = fuel_memory::from_slice_cpu(&[1.0_f32, 5.0, 3.0, f32::NAN, -1.0]);
        let rhs = fuel_memory::from_slice_cpu(&[2.0_f32, 5.0, 3.0, 0.0, 0.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (l_id, r_id, lt_id) = {
            let mut g = graph.write().unwrap();
            let l = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            let r = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            let lt = g.push(Node {
                op: Op::Lt,
                inputs: vec![l, r],
                shape: Shape::from_dims(&[5]),
                dtype: DType::Bool,
            });
            g.set_target_backend(lt, BackendId::Cpu);
            (l, r, lt)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(l_id, Arc::new(RwLock::new(lhs)));
        inputs.insert(r_id, Arc::new(RwLock::new(rhs)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, lt_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::Bool);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let mask: &[u8] = c.as_slice().expect("u8 view");
        // 1.0 < 2.0 → 1;  5.0 < 5.0 → 0 (strict);  3.0 < 3.0 → 0;
        // NaN < 0.0 → 0 (unordered);  -1.0 < 0.0 → 1.
        assert_eq!(mask, &[1, 0, 0, 0, 1]);
    }

    /// E2E: Op::Le F32 → Bool mask. Distinct from Lt at the equal slot
    /// (`5.0 <= 5.0` = 1, vs Lt's `5.0 < 5.0` = 0). NaN unordered → 0.
    #[test]
    fn pipelined_realize_le_f32_to_bool_mask() {
        let lhs = fuel_memory::from_slice_cpu(&[1.0_f32, 5.0, 3.0, f32::NAN, -1.0]);
        let rhs = fuel_memory::from_slice_cpu(&[2.0_f32, 5.0, 2.0, 0.0, 0.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (l_id, r_id, le_id) = {
            let mut g = graph.write().unwrap();
            let l = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            let r = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            let le = g.push(Node {
                op: Op::Le,
                inputs: vec![l, r],
                shape: Shape::from_dims(&[5]),
                dtype: DType::Bool,
            });
            g.set_target_backend(le, BackendId::Cpu);
            (l, r, le)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(l_id, Arc::new(RwLock::new(lhs)));
        inputs.insert(r_id, Arc::new(RwLock::new(rhs)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, le_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::Bool);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let mask: &[u8] = c.as_slice().expect("u8 view");
        // 1.0 <= 2.0 → 1;  5.0 <= 5.0 → 1 (key Lt difference);
        // 3.0 <= 2.0 → 0;  NaN <= 0.0 → 0 (unordered);  -1.0 <= 0.0 → 1.
        assert_eq!(mask, &[1, 1, 0, 0, 1]);
    }

    /// E2E: Op::Gt F32 → Bool mask. Strict-greater: equality slot is
    /// `0`. NaN unordered → `0`.
    #[test]
    fn pipelined_realize_gt_f32_to_bool_mask() {
        let lhs = fuel_memory::from_slice_cpu(&[3.0_f32, 5.0, 2.0, f32::NAN, 1.0]);
        let rhs = fuel_memory::from_slice_cpu(&[2.0_f32, 5.0, 3.0, 0.0, 0.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (l_id, r_id, gt_id) = {
            let mut g = graph.write().unwrap();
            let l = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            let r = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            let gt = g.push(Node {
                op: Op::Gt,
                inputs: vec![l, r],
                shape: Shape::from_dims(&[5]),
                dtype: DType::Bool,
            });
            g.set_target_backend(gt, BackendId::Cpu);
            (l, r, gt)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(l_id, Arc::new(RwLock::new(lhs)));
        inputs.insert(r_id, Arc::new(RwLock::new(rhs)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, gt_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::Bool);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let mask: &[u8] = c.as_slice().expect("u8 view");
        // 3.0 > 2.0 → 1;  5.0 > 5.0 → 0 (strict);  2.0 > 3.0 → 0;
        // NaN > 0.0 → 0 (unordered);  1.0 > 0.0 → 1.
        assert_eq!(mask, &[1, 0, 0, 0, 1]);
    }

    /// E2E: Op::Ge F32 → Bool mask. Greater-or-equal: equality slot
    /// is `1` (distinguishes from Gt). NaN unordered → `0`. Closes
    /// the comparison family with full `[Eq, Ne, Lt, Le, Gt, Ge]`
    /// coverage.
    #[test]
    fn pipelined_realize_ge_f32_to_bool_mask() {
        let lhs = fuel_memory::from_slice_cpu(&[3.0_f32, 5.0, 2.0, f32::NAN, 0.0]);
        let rhs = fuel_memory::from_slice_cpu(&[2.0_f32, 5.0, 3.0, 0.0, 0.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (l_id, r_id, ge_id) = {
            let mut g = graph.write().unwrap();
            let l = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            let r = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            let ge = g.push(Node {
                op: Op::Ge,
                inputs: vec![l, r],
                shape: Shape::from_dims(&[5]),
                dtype: DType::Bool,
            });
            g.set_target_backend(ge, BackendId::Cpu);
            (l, r, ge)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(l_id, Arc::new(RwLock::new(lhs)));
        inputs.insert(r_id, Arc::new(RwLock::new(rhs)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, ge_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::Bool);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let mask: &[u8] = c.as_slice().expect("u8 view");
        // 3.0 >= 2.0 → 1;  5.0 >= 5.0 → 1 (key Gt difference);
        // 2.0 >= 3.0 → 0;  NaN >= 0.0 → 0 (unordered);  0.0 >= 0.0 → 1.
        assert_eq!(mask, &[1, 1, 0, 0, 1]);
    }

    /// E2E: Op::Equal F64 → Bool mask. Confirms the F64 wrapper is
    /// independently registered and routed (binding-table key
    /// `(EqualElementwise, [F64, F64, U8], Cpu)`).
    #[test]
    fn pipelined_realize_eq_f64_to_bool_mask() {
        let lhs = fuel_memory::from_slice_cpu(&[1.0_f64, 2.0, 3.0]);
        let rhs = fuel_memory::from_slice_cpu(&[1.0_f64, 2.0, 4.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (l_id, r_id, eq_id) = {
            let mut g = graph.write().unwrap();
            let l = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F64,
            });
            let r = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F64,
            });
            let eq = g.push(Node {
                op: Op::Equal,
                inputs: vec![l, r],
                shape: Shape::from_dims(&[3]),
                dtype: DType::Bool,
            });
            g.set_target_backend(eq, BackendId::Cpu);
            (l, r, eq)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(l_id, Arc::new(RwLock::new(lhs)));
        inputs.insert(r_id, Arc::new(RwLock::new(rhs)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, eq_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::Bool);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let mask: &[u8] = c.as_slice().expect("u8 view");
        assert_eq!(mask, &[1, 1, 0]);
    }

    /// E2E: Op::Where ternary select — `out[i] = if cond[i] != 0 { a[i] } else { b[i] }`.
    /// Validates (a) the binding-table key `(Where, [Bool, F32, F32, F32], Cpu)`
    /// resolves to the where_f32 wrapper, (b) the Bool cond input drives
    /// the per-slot pick, (c) outputs preserve the input dtype.
    ///
    /// GAP-168(c): the cond STORAGE is tagged Bool, not merely the graph node.
    /// Bool and U8 are byte-identical, so leaving the storage U8 under a Bool
    /// node is exactly the mismatch this cut exists to make visible.
    #[test]
    fn pipelined_realize_where_f32_picks_per_slot_from_bool_mask() {
        // cond = [1, 0, 1, 0, 1]; a = [1, 2, 3, 4, 5]; b = [10, 20, 30, 40, 50]
        // expected = [1, 20, 3, 40, 5]
        let mut cond_storage = fuel_memory::from_slice_cpu(&[1u8, 0, 1, 0, 1]);
        cond_storage.dtype = DType::Bool; // GAP-168(c): a Bool mask, not U8 bytes
        let a_storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0]);
        let b_storage = fuel_memory::from_slice_cpu(&[10.0_f32, 20.0, 30.0, 40.0, 50.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (cond_id, a_id, b_id, where_id) = {
            let mut g = graph.write().unwrap();
            let cond = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::Bool,
            });
            let a = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            let b = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            let w = g.push(Node {
                op: Op::Where,
                inputs: vec![cond, a, b],
                shape: Shape::from_dims(&[5]),
                dtype: DType::F32,
            });
            g.set_target_backend(w, BackendId::Cpu);
            (cond, a, b, w)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(cond_id, Arc::new(RwLock::new(cond_storage)));
        inputs.insert(a_id, Arc::new(RwLock::new(a_storage)));
        inputs.insert(b_id, Arc::new(RwLock::new(b_storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, where_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::F32);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let out: &[f32] = c.as_slice().expect("f32 view");
        assert_eq!(out, &[1.0, 20.0, 3.0, 40.0, 5.0]);
    }

    /// E2E: full chain `eq → where`. Compares two f32 vectors, then
    /// uses the resulting Bool mask to pick from a third tensor (or a
    /// fallback). Validates the comparison-family + Where ops compose
    /// end-to-end.
    #[test]
    fn pipelined_realize_eq_then_where_full_chain() {
        // a = [1, 2, 3]; b = [1, 5, 3] → eq = [1, 0, 1]
        // pick = [10, 20, 30]; fallback = [99, 99, 99]
        // result = [10, 99, 30]
        let a_storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0]);
        let b_storage = fuel_memory::from_slice_cpu(&[1.0_f32, 5.0, 3.0]);
        let pick_storage = fuel_memory::from_slice_cpu(&[10.0_f32, 20.0, 30.0]);
        let fb_storage = fuel_memory::from_slice_cpu(&[99.0_f32, 99.0, 99.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (a_id, b_id, pick_id, fb_id, where_id) = {
            let mut g = graph.write().unwrap();
            let a = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let b = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let pick = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let fb = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let eq = g.push(Node {
                op: Op::Equal,
                inputs: vec![a, b],
                shape: Shape::from_dims(&[3]),
                dtype: DType::Bool,
            });
            let w = g.push(Node {
                op: Op::Where,
                inputs: vec![eq, pick, fb],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            g.set_target_backend(eq, BackendId::Cpu);
            g.set_target_backend(w, BackendId::Cpu);
            (a, b, pick, fb, w)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(a_id, Arc::new(RwLock::new(a_storage)));
        inputs.insert(b_id, Arc::new(RwLock::new(b_storage)));
        inputs.insert(pick_id, Arc::new(RwLock::new(pick_storage)));
        inputs.insert(fb_id, Arc::new(RwLock::new(fb_storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, where_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let out: &[f32] = c.as_slice().expect("f32 view");
        assert_eq!(out, &[10.0, 99.0, 30.0]);
    }

    /// E2E: Q4_0 QMatMul through the pipelined executor — proves
    /// quantized weights can flow into the unified path. Activations
    /// are F32, weights are U32-typed (raw block bytes).
    /// Construct a Q4_0 weight tensor where every weight = 1.0
    /// (d=1.0, every nibble=9 → 1*(9-8)=1), so A @ W^T computes
    /// the per-row sum of activations.
    #[test]
    fn pipelined_realize_qmatmul_q4_0_unit_weight_sums_activations() {
        use fuel_graph::QuantType;
        use half::f16;
        let block_size = std::mem::size_of::<fuel_quantized::BlockQ4_0>();
        let mut w_bytes = vec![0u8; 2 * block_size];
        for block_idx in 0..2 {
            let off = block_idx * block_size;
            let d_bytes = f16::from_f32(1.0).to_le_bytes();
            w_bytes[off..off + 2].copy_from_slice(&d_bytes);
            for i in 0..16 {
                w_bytes[off + 2 + i] = 0x99;
            }
        }
        // Weight tensor is U32-typed (rank-1, length = bytes/4)
        let w_storage = fuel_memory::Storage::new(
            fuel_memory::BackendStorage::Cpu(fuel_cpu_backend::CpuStorageBytes::from_bytes(
                &w_bytes,
            )),
            DType::U32,
        );

        let act_vec: Vec<f32> = (1..=32).map(|x| x as f32).collect();
        let act_storage = fuel_memory::from_slice_cpu(&act_vec);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (act_id, w_id, mm_id) = {
            let mut g = graph.write().unwrap();
            let act = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 32]),
                dtype: DType::F32,
            });
            let w = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[w_bytes.len() / 4]),
                dtype: DType::U32,
            });
            let mm = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::QMATMUL,
                    fuel_graph::registry::FusedOpParams::QMatMul {
                        quant_type: QuantType::Q4_0,
                        k: 32,
                        n: 2,
                    },
                ),
                inputs: vec![act, w],
                shape: Shape::from_dims(&[1, 2]),
                dtype: DType::F32,
            });
            g.set_target_backend(mm, BackendId::Cpu);
            (act, w, mm)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(act_id, Arc::new(RwLock::new(act_storage)));
        inputs.insert(w_id, Arc::new(RwLock::new(w_storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, mm_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::F32);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let r: &[f32] = c.as_slice().unwrap();
        // Both rows = sum(1..=32) = 528, within Q8_1 round-trip
        // tolerance.
        assert!((r[0] - 528.0).abs() < 0.5, "got {}, want 528", r[0]);
        assert!((r[1] - 528.0).abs() < 0.5, "got {}, want 528", r[1]);
    }

    /// E2E: QMatMul with Q5_0 weights — verifies the new quant
    /// dispatch arm picks `qmatmul_q5_0_f32`. We build weights by
    /// quantizing all-ones via `BlockQ5_0::from_float`, then
    /// compare pipelined output against the direct fuel_quantized
    /// matmul on the same blocks.
    #[test]
    fn pipelined_realize_qmatmul_q5_0_against_reference() {
        use fuel_graph::QuantType;
        use fuel_quantized::{BlockQ5_0, GgmlType};
        let n = 2;
        let k = 64; // 2 blocks per row (Q5_0 vec_dot pairs blocks)
        // Quantize an all-ones [n, k] weight matrix.
        let w_f32 = vec![1.0_f32; n * k];
        let blocks_per_row = k / BlockQ5_0::BLCK_SIZE;
        let mut w_blocks = vec![BlockQ5_0::zeros(); n * blocks_per_row];
        BlockQ5_0::from_float(&w_f32, &mut w_blocks);
        // Reinterpret block slice as bytes (BlockQ5_0 is #[repr(C)]).
        let bytes_per_block = std::mem::size_of::<BlockQ5_0>();
        let w_bytes: Vec<u8> = unsafe {
            std::slice::from_raw_parts(
                w_blocks.as_ptr() as *const u8,
                w_blocks.len() * bytes_per_block,
            )
        }
        .to_vec();
        let w_storage = fuel_memory::Storage::new(
            fuel_memory::BackendStorage::Cpu(fuel_cpu_backend::CpuStorageBytes::from_bytes(
                &w_bytes,
            )),
            DType::U32,
        );
        let act_vec: Vec<f32> = (1..=k).map(|x| x as f32).collect();
        let act_storage = fuel_memory::from_slice_cpu(&act_vec);
        // Reference: direct matmul through fuel_quantized.
        let mut ref_out = vec![0.0_f32; n];
        fuel_quantized::matmul::<BlockQ5_0>((1, k, n), &act_vec, &w_blocks, &mut ref_out)
            .expect("ref matmul");

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (act_id, w_id, mm_id) = {
            let mut g = graph.write().unwrap();
            let act = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, k]),
                dtype: DType::F32,
            });
            let w = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[w_bytes.len() / 4]),
                dtype: DType::U32,
            });
            let mm = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::QMATMUL,
                    fuel_graph::registry::FusedOpParams::QMatMul {
                        quant_type: QuantType::Q5_0,
                        k,
                        n,
                    },
                ),
                inputs: vec![act, w],
                shape: Shape::from_dims(&[1, n]),
                dtype: DType::F32,
            });
            g.set_target_backend(mm, BackendId::Cpu);
            (act, w, mm)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(act_id, Arc::new(RwLock::new(act_storage)));
        inputs.insert(w_id, Arc::new(RwLock::new(w_storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, mm_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let r: &[f32] = c.as_slice().unwrap();
        // Bit-exact against same kernel via the trait, since both
        // paths run fuel_quantized::matmul<BlockQ5_0>.
        assert_eq!(
            r,
            ref_out.as_slice(),
            "pipelined Q5_0 differs from reference: got {r:?}, want {ref_out:?}"
        );
    }

    /// E2E: QMatMul with Q6K (256-element super-block k-quant).
    /// Same idea as the Q5_0 test — bit-exact against the
    /// reference fuel_quantized::matmul<BlockQ6K>. Confirms the
    /// dispatch arm wires `qmatmul_q6k_f32` correctly.
    #[test]
    fn pipelined_realize_qmatmul_q6k_against_reference() {
        use fuel_graph::QuantType;
        use fuel_quantized::{BlockQ6K, GgmlType};
        let n = 2;
        let k = 256; // 1 super-block per row
        let w_f32 = vec![1.0_f32; n * k];
        let blocks_per_row = k / BlockQ6K::BLCK_SIZE;
        let mut w_blocks = vec![BlockQ6K::zeros(); n * blocks_per_row];
        BlockQ6K::from_float(&w_f32, &mut w_blocks);
        let bytes_per_block = std::mem::size_of::<BlockQ6K>();
        let w_bytes: Vec<u8> = unsafe {
            std::slice::from_raw_parts(
                w_blocks.as_ptr() as *const u8,
                w_blocks.len() * bytes_per_block,
            )
        }
        .to_vec();
        let w_storage = fuel_memory::Storage::new(
            fuel_memory::BackendStorage::Cpu(fuel_cpu_backend::CpuStorageBytes::from_bytes(
                &w_bytes,
            )),
            DType::U32,
        );
        let act_vec: Vec<f32> = (1..=k).map(|x| x as f32 / 100.0).collect();
        let act_storage = fuel_memory::from_slice_cpu(&act_vec);
        let mut ref_out = vec![0.0_f32; n];
        fuel_quantized::matmul::<BlockQ6K>((1, k, n), &act_vec, &w_blocks, &mut ref_out)
            .expect("ref matmul");

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (act_id, w_id, mm_id) = {
            let mut g = graph.write().unwrap();
            let act = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, k]),
                dtype: DType::F32,
            });
            let w = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[w_bytes.len() / 4]),
                dtype: DType::U32,
            });
            let mm = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::QMATMUL,
                    fuel_graph::registry::FusedOpParams::QMatMul {
                        quant_type: QuantType::Q6K,
                        k,
                        n,
                    },
                ),
                inputs: vec![act, w],
                shape: Shape::from_dims(&[1, n]),
                dtype: DType::F32,
            });
            g.set_target_backend(mm, BackendId::Cpu);
            (act, w, mm)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(act_id, Arc::new(RwLock::new(act_storage)));
        inputs.insert(w_id, Arc::new(RwLock::new(w_storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, mm_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let r: &[f32] = c.as_slice().unwrap();
        assert_eq!(
            r,
            ref_out.as_slice(),
            "pipelined Q6K differs from reference: got {r:?}, want {ref_out:?}"
        );
    }

    /// E2E: BF16 RmsNormLastDim through the pipelined executor.
    /// Verifies that capability-driven dispatch routes the
    /// half-float norm op to the bf16-specific kernel (which
    /// accumulates in f32 internally).
    #[test]
    fn pipelined_realize_rms_norm_bf16() {
        let v: Vec<half::bf16> = [3.0_f32, 4.0]
            .iter()
            .map(|&x| half::bf16::from_f32(x))
            .collect();
        let storage = fuel_memory::from_slice_cpu(&v);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, op_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 2]),
                dtype: DType::BF16,
            });
            let op_id = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::RMS_NORM_LAST_DIM,
                    fuel_graph::registry::FusedOpParams::RmsNormLastDim { eps: 0.0 },
                ),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[1, 2]),
                dtype: DType::BF16,
            });
            g.set_target_backend(op_id, BackendId::Cpu);
            (in_id, op_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::BF16);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let r: &[half::bf16] = c.as_slice().unwrap();
        let rms = (12.5_f32).sqrt();
        // bf16's ~3-digit mantissa absorbs the divisor; allow ~5%.
        assert!((r[0].to_f32() - 3.0 / rms).abs() < 0.05);
        assert!((r[1].to_f32() - 4.0 / rms).abs() < 0.05);
    }

    /// E2E: BF16 matmul through the pipelined executor — proves
    /// the LLM forward-pass blocker (every transformer layer
    /// is dominated by matmul). Identity matmul on bf16 round-
    /// trips small integers exactly.
    #[test]
    fn pipelined_realize_matmul_bf16_identity() {
        let lhs_v: Vec<half::bf16> = [1.0_f32, 2.0, 3.0, 4.0]
            .iter()
            .map(|&x| half::bf16::from_f32(x))
            .collect();
        let rhs_v: Vec<half::bf16> = [1.0_f32, 0.0, 0.0, 1.0]
            .iter()
            .map(|&x| half::bf16::from_f32(x))
            .collect();
        let lhs = fuel_memory::from_slice_cpu(&lhs_v);
        let rhs = fuel_memory::from_slice_cpu(&rhs_v);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (l_id, r_id, mm_id) = {
            let mut g = graph.write().unwrap();
            let l = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 2]),
                dtype: DType::BF16,
            });
            let r = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 2]),
                dtype: DType::BF16,
            });
            let mm = g.push(Node {
                op: Op::MatMul,
                inputs: vec![l, r],
                shape: Shape::from_dims(&[2, 2]),
                dtype: DType::BF16,
            });
            g.set_target_backend(mm, BackendId::Cpu);
            (l, r, mm)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(l_id, Arc::new(RwLock::new(lhs)));
        inputs.insert(r_id, Arc::new(RwLock::new(rhs)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, mm_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::BF16);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let r: &[half::bf16] = c.as_slice().unwrap();
        let result_f32: Vec<f32> = r.iter().map(|x| x.to_f32()).collect();
        assert_eq!(result_f32, vec![1.0, 2.0, 3.0, 4.0]);
    }

    /// E2E: F16 sum-reduce — verifies bf16/f16 reduction dispatch
    /// works through the executor.
    #[test]
    fn pipelined_realize_sum_dim_f16() {
        let v: Vec<half::f16> = [1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0]
            .iter()
            .map(|&x| half::f16::from_f32(x))
            .collect();
        let storage = fuel_memory::from_slice_cpu(&v);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, sum_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F16,
            });
            let sum_id = g.push(Node {
                op: Op::SumDim(1),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F16,
            });
            g.set_target_backend(sum_id, BackendId::Cpu);
            (in_id, sum_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, sum_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::F16);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let r: &[half::f16] = c.as_slice().unwrap();
        let result_f32: Vec<f32> = r.iter().map(|x| x.to_f32()).collect();
        assert_eq!(result_f32, vec![6.0, 15.0]);
    }

    /// E2E: BF16 elementwise add through the pipelined executor.
    #[test]
    fn pipelined_realize_add_bf16() {
        let lhs_vec: Vec<half::bf16> = [1.0_f32, 2.0, 3.0]
            .iter()
            .map(|&x| half::bf16::from_f32(x))
            .collect();
        let rhs_vec: Vec<half::bf16> = [10.0_f32, 20.0, 30.0]
            .iter()
            .map(|&x| half::bf16::from_f32(x))
            .collect();
        let lhs = fuel_memory::from_slice_cpu(&lhs_vec);
        let rhs = fuel_memory::from_slice_cpu(&rhs_vec);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (l_id, r_id, op_id) = {
            let mut g = graph.write().unwrap();
            let l = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::BF16,
            });
            let r = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::BF16,
            });
            let op = g.push(Node {
                op: Op::Add,
                inputs: vec![l, r],
                shape: Shape::from_dims(&[3]),
                dtype: DType::BF16,
            });
            g.set_target_backend(op, BackendId::Cpu);
            (l, r, op)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(l_id, Arc::new(RwLock::new(lhs)));
        inputs.insert(r_id, Arc::new(RwLock::new(rhs)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::BF16);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let r: &[half::bf16] = c.as_slice().unwrap();
        let result_f32: Vec<f32> = r.iter().map(|x| x.to_f32()).collect();
        assert_eq!(result_f32, vec![11.0, 22.0, 33.0]);
    }

    /// E2E: F16 unary chain — Const + Sqr + Sqrt — verifies F16
    /// dispatch works and the via-f32 round-trip kernels behave
    /// correctly through the executor.
    #[test]
    fn pipelined_realize_sqr_then_sqrt_f16() {
        let v: Vec<half::f16> = [1.0_f32, 4.0, 9.0]
            .iter()
            .map(|&x| half::f16::from_f32(x))
            .collect();
        let storage = fuel_memory::from_slice_cpu(&v);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, sqrt_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F16,
            });
            let sqr_id = g.push(Node {
                op: Op::Sqr,
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F16,
            });
            let sqrt_id = g.push(Node {
                op: Op::Sqrt,
                inputs: vec![sqr_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F16,
            });
            g.set_target_backend(sqr_id, BackendId::Cpu);
            g.set_target_backend(sqrt_id, BackendId::Cpu);
            (in_id, sqrt_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, sqrt_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::F16);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let r: &[half::f16] = c.as_slice().unwrap();
        let result_f32: Vec<f32> = r.iter().map(|x| x.to_f32()).collect();
        // f16 has ~3 decimal digits; sqrt(sqr(x)) = |x| within rounding.
        for (got, want) in result_f32.iter().zip(&[1.0_f32, 4.0, 9.0]) {
            assert!((got - want).abs() < 0.05, "got {got}, want {want}");
        }
    }

    /// E2E: F64 sum-reduce along one dim through the pipelined
    /// executor.
    #[test]
    fn pipelined_realize_sum_dim_f64() {
        let storage = fuel_memory::from_slice_cpu(&[1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, sum_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F64,
            });
            let sum_id = g.push(Node {
                op: Op::SumDim(1),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F64,
            });
            g.set_target_backend(sum_id, BackendId::Cpu);
            (in_id, sum_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, sum_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::F64);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f64>().unwrap(), &[6.0_f64, 15.0]);
    }

    /// E2E: F64 matmul through the pipelined executor.
    #[test]
    fn pipelined_realize_matmul_2x3_times_3x2_f64() {
        let lhs = fuel_memory::from_slice_cpu(&[1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0]);
        let rhs = fuel_memory::from_slice_cpu(&[7.0_f64, 8.0, 9.0, 10.0, 11.0, 12.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (l_id, r_id, mm_id) = {
            let mut g = graph.write().unwrap();
            let l = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F64,
            });
            let r = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3, 2]),
                dtype: DType::F64,
            });
            let mm = g.push(Node {
                op: Op::MatMul,
                inputs: vec![l, r],
                shape: Shape::from_dims(&[2, 2]),
                dtype: DType::F64,
            });
            g.set_target_backend(mm, BackendId::Cpu);
            (l, r, mm)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(l_id, Arc::new(RwLock::new(lhs)));
        inputs.insert(r_id, Arc::new(RwLock::new(rhs)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, mm_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::F64);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(
            c.as_slice::<f64>().unwrap(),
            &[58.0_f64, 64.0, 139.0, 154.0]
        );
    }

    /// E2E: F64 unary chain — Const + Sqr + Sqrt — verifies that
    /// the kernel-binding lookup picks the f64 entries when the
    /// graph nodes carry DType::F64.
    #[test]
    fn pipelined_realize_sqr_then_sqrt_f64() {
        let storage = fuel_memory::from_slice_cpu(&[1.0_f64, 4.0, 9.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, sqrt_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F64,
            });
            let sqr_id = g.push(Node {
                op: Op::Sqr,
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F64,
            });
            let sqrt_id = g.push(Node {
                op: Op::Sqrt,
                inputs: vec![sqr_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F64,
            });
            g.set_target_backend(sqr_id, BackendId::Cpu);
            g.set_target_backend(sqrt_id, BackendId::Cpu);
            (in_id, sqrt_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, sqrt_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::F64);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f64>().unwrap(), &[1.0_f64, 4.0, 9.0]);
    }

    /// E2E: ArgMaxDim — produces U32 output indices.
    #[test]
    fn pipelined_realize_argmax_dim() {
        // input [2, 3] = [[1, 5, 2], [9, 0, 4]]
        // argmax dim=1 → [1, 0]
        let storage = fuel_memory::from_slice_cpu(&[1.0_f32, 5.0, 2.0, 9.0, 0.0, 4.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, op_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let op_id = g.push(Node {
                op: Op::ArgMaxDim(1),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[2]),
                dtype: DType::U32,
            });
            g.set_target_backend(op_id, BackendId::Cpu);
            (in_id, op_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        assert_eq!(guard.dtype, DType::U32);
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<u32>().unwrap(), &[1u32, 0]);
    }

    /// E2E: IndexAdd along outer dim — accumulate updates into a
    /// rank-1 base tensor at indexed positions.
    #[test]
    fn pipelined_realize_index_add_simple() {
        let base = fuel_memory::from_slice_cpu(&[10.0_f32, 20.0, 30.0]);
        let indices = fuel_memory::from_slice_cpu(&[0u32, 0]);
        let src = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (b_id, i_id, s_id, op_id) = {
            let mut g = graph.write().unwrap();
            let b = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let i = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::U32,
            });
            let s = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let op = g.push(Node {
                op: Op::IndexAdd { dim: 0 },
                inputs: vec![b, i, s],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            g.set_target_backend(op, BackendId::Cpu);
            (b, i, s, op)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(b_id, Arc::new(RwLock::new(base)));
        inputs.insert(i_id, Arc::new(RwLock::new(indices)));
        inputs.insert(s_id, Arc::new(RwLock::new(src)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        // 10 + 1 + 2 = 13; 20 untouched; 30 untouched.
        assert_eq!(c.as_slice::<f32>().unwrap(), &[13.0, 20.0, 30.0]);
    }

    /// E2E: ScatterAdd along outer dim — same-rank indices, base
    /// starts as zeros, src adds values at scatter positions.
    #[test]
    fn pipelined_realize_scatter_add_outer_dim() {
        // base [3, 2] = zeros; indices [2, 2] = [[0, 1], [2, 0]];
        // src [2, 2] = [[1, 2], [3, 4]]; dim=0
        // → out = [[1, 4], [0, 2], [3, 0]]
        let base = fuel_memory::from_slice_cpu(&[0.0_f32; 6]);
        let indices = fuel_memory::from_slice_cpu(&[0u32, 1, 2, 0]);
        let src = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (b_id, i_id, s_id, op_id) = {
            let mut g = graph.write().unwrap();
            let b = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3, 2]),
                dtype: DType::F32,
            });
            let i = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 2]),
                dtype: DType::U32,
            });
            let s = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 2]),
                dtype: DType::F32,
            });
            let op = g.push(Node {
                op: Op::ScatterAdd { dim: 0 },
                inputs: vec![b, i, s],
                shape: Shape::from_dims(&[3, 2]),
                dtype: DType::F32,
            });
            g.set_target_backend(op, BackendId::Cpu);
            (b, i, s, op)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(b_id, Arc::new(RwLock::new(base)));
        inputs.insert(i_id, Arc::new(RwLock::new(indices)));
        inputs.insert(s_id, Arc::new(RwLock::new(src)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(
            c.as_slice::<f32>().unwrap(),
            &[1.0, 4.0, 0.0, 2.0, 3.0, 0.0]
        );
    }

    /// E2E: Rope through the pipelined executor. cos=0, sin=1
    /// rotates the head_dim halves with sign per the rotate_half
    /// convention.
    #[test]
    fn pipelined_realize_rope_pi_over_two() {
        // x [1, 1, 4] = [1, 2, 3, 4]. cos=[0,0,0,0], sin=[1,1,1,1].
        // Expected: [-3, -4, 1, 2].
        let x = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0]);
        let cos = fuel_memory::from_slice_cpu(&[0.0_f32, 0.0, 0.0, 0.0]);
        let sin = fuel_memory::from_slice_cpu(&[1.0_f32, 1.0, 1.0, 1.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (x_id, cos_id, sin_id, r_id) = {
            let mut g = graph.write().unwrap();
            let x = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 4]),
                dtype: DType::F32,
            });
            let c = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 4]),
                dtype: DType::F32,
            });
            let s = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 4]),
                dtype: DType::F32,
            });
            let r = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::ROPE,
                    fuel_graph::registry::FusedOpParams::Rope,
                ),
                inputs: vec![x, c, s],
                shape: Shape::from_dims(&[1, 1, 4]),
                dtype: DType::F32,
            });
            g.set_target_backend(r, BackendId::Cpu);
            (x, c, s, r)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(x_id, Arc::new(RwLock::new(x)));
        inputs.insert(cos_id, Arc::new(RwLock::new(cos)));
        inputs.insert(sin_id, Arc::new(RwLock::new(sin)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, r_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[-3.0, -4.0, 1.0, 2.0]);
    }

    /// E2E: Gather along inner dim. Source [2, 4]; indices [2, 3];
    /// output [2, 3] = picks from each row by per-row indices.
    #[test]
    fn pipelined_realize_gather_inner_dim() {
        let source =
            fuel_memory::from_slice_cpu(&[10.0_f32, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0]);
        let indices = fuel_memory::from_slice_cpu(&[0u32, 2, 1, 3, 0, 0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (src_id, idx_id, g_id) = {
            let mut g = graph.write().unwrap();
            let s = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 4]),
                dtype: DType::F32,
            });
            let i = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::U32,
            });
            let g_id = g.push(Node {
                op: Op::Gather { dim: 1 },
                inputs: vec![s, i],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            g.set_target_backend(g_id, BackendId::Cpu);
            (s, i, g_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(src_id, Arc::new(RwLock::new(source)));
        inputs.insert(idx_id, Arc::new(RwLock::new(indices)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, g_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(
            c.as_slice::<f32>().unwrap(),
            &[10.0, 30.0, 20.0, 80.0, 50.0, 50.0]
        );
    }

    /// E2E: IndexSelect — embedding-table lookup. Source is a
    /// `[vocab=4, d_model=3]` table; indices are token IDs;
    /// output is `[seq=3, d_model=3]` with the picked rows.
    #[test]
    fn pipelined_realize_index_select_embedding_lookup() {
        let table = fuel_memory::from_slice_cpu(&[
            10.0_f32, 11.0, 12.0, // row 0
            20.0, 21.0, 22.0, // row 1
            30.0, 31.0, 32.0, // row 2
            40.0, 41.0, 42.0, // row 3
        ]);
        let indices = fuel_memory::from_slice_cpu(&[2u32, 0, 2]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (table_id, idx_id, sel_id) = {
            let mut g = graph.write().unwrap();
            let t = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[4, 3]),
                dtype: DType::F32,
            });
            let i = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::U32,
            });
            let s = g.push(Node {
                op: Op::IndexSelect { dim: 0 },
                inputs: vec![t, i],
                shape: Shape::from_dims(&[3, 3]),
                dtype: DType::F32,
            });
            g.set_target_backend(s, BackendId::Cpu);
            (t, i, s)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(table_id, Arc::new(RwLock::new(table)));
        inputs.insert(idx_id, Arc::new(RwLock::new(indices)));
        let (result_arc, result_layout) =
            PipelinedExecutor::realize(graph, sel_id, inputs).expect("realize");
        assert_eq!(result_layout.shape().dims(), &[3, 3]);
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(
            c.as_slice::<f32>().unwrap(),
            &[30.0, 31.0, 32.0, 10.0, 11.0, 12.0, 30.0, 31.0, 32.0]
        );
    }

    /// E2E: RmsNormLastDim on a 2-row input. Each row's output
    /// has unit RMS up to the eps-induced bias.
    #[test]
    fn pipelined_realize_rms_norm_last_dim() {
        let storage = fuel_memory::from_slice_cpu(&[
            3.0_f32, 4.0, // row 0: rms = sqrt(12.5)
            6.0, 8.0, // row 1: rms = sqrt(50.0) = 5*sqrt(2)
        ]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, op_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 2]),
                dtype: DType::F32,
            });
            let op_id = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::RMS_NORM_LAST_DIM,
                    fuel_graph::registry::FusedOpParams::RmsNormLastDim { eps: 0.0 },
                ),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[2, 2]),
                dtype: DType::F32,
            });
            g.set_target_backend(op_id, BackendId::Cpu);
            (in_id, op_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let result: &[f32] = c.as_slice().unwrap();
        // Row 0: rms = sqrt(12.5). Output = [3, 4] / sqrt(12.5).
        let rms0 = (12.5_f32).sqrt();
        assert!((result[0] - 3.0 / rms0).abs() < 1e-6);
        assert!((result[1] - 4.0 / rms0).abs() < 1e-6);
        // Row 1: rms = sqrt(50). Output = [6, 8] / sqrt(50).
        let rms1 = (50.0_f32).sqrt();
        assert!((result[2] - 6.0 / rms1).abs() < 1e-6);
        assert!((result[3] - 8.0 / rms1).abs() < 1e-6);
    }

    /// E2E: LayerNormLastDim — each row's output has zero mean and
    /// unit variance.
    #[test]
    fn pipelined_realize_layer_norm_last_dim() {
        let storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 10.0, 20.0, 30.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, op_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let op_id = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::LAYER_NORM_LAST_DIM,
                    fuel_graph::registry::FusedOpParams::LayerNormLastDim { eps: 0.0 },
                ),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            g.set_target_backend(op_id, BackendId::Cpu);
            (in_id, op_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let result: &[f32] = c.as_slice().unwrap();
        // Each row should have mean ~0 and var ~1.
        for row in 0..2 {
            let off = row * 3;
            let sum: f32 = result[off..off + 3].iter().sum();
            let mean = sum / 3.0;
            assert!(mean.abs() < 1e-6, "row {row} mean should be 0, got {mean}");
            let var: f32 = result[off..off + 3]
                .iter()
                .map(|v| (v - mean).powi(2))
                .sum::<f32>()
                / 3.0;
            assert!(
                (var - 1.0).abs() < 1e-6,
                "row {row} var should be 1, got {var}"
            );
        }
    }

    /// E2E: SoftmaxLastDim on a 2-row input. Each row should sum
    /// to 1; uniform row gives uniform output.
    #[test]
    fn pipelined_realize_softmax_last_dim() {
        // Row 0: [1, 1, 1, 1] → uniform 0.25 each
        // Row 1: [0, 0, 0, 100] → effectively a one-hot at position 3
        let storage = fuel_memory::from_slice_cpu(&[1.0_f32, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 100.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, op_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 4]),
                dtype: DType::F32,
            });
            let op_id = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::SOFTMAX_LAST_DIM,
                    fuel_graph::registry::FusedOpParams::SoftmaxLastDim,
                ),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[2, 4]),
                dtype: DType::F32,
            });
            g.set_target_backend(op_id, BackendId::Cpu);
            (in_id, op_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let result: &[f32] = c.as_slice().unwrap();

        // Row 0: uniform 0.25
        for v in &result[..4] {
            assert!((v - 0.25).abs() < 1e-7);
        }
        // Row 1: positions 0..3 ≈ 0, position 4 (= last column) ≈ 1
        // (e^100 dominates).
        for v in &result[4..7] {
            assert!(
                *v < 1e-30,
                "row-1 leading positions should be near 0, got {v}"
            );
        }
        assert!(
            result[7] > 0.999,
            "row-1 last position should dominate, got {}",
            result[7]
        );
        // Each row sums to 1.
        let row0_sum: f32 = result[..4].iter().sum();
        let row1_sum: f32 = result[4..].iter().sum();
        assert!((row0_sum - 1.0).abs() < 1e-6);
        assert!((row1_sum - 1.0).abs() < 1e-6);
    }

    /// E2E: Concat along inner dim — two [2, 3] tensors → [2, 6].
    #[test]
    fn pipelined_realize_concat_inner_dim() {
        let a = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0]);
        let b = fuel_memory::from_slice_cpu(&[7.0_f32, 8.0, 9.0, 10.0, 11.0, 12.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (a_id, b_id, c_id) = {
            let mut g = graph.write().unwrap();
            let a = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let b = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let c = g.push(Node {
                op: Op::Concat { dim: 1 },
                inputs: vec![a, b],
                shape: Shape::from_dims(&[2, 6]),
                dtype: DType::F32,
            });
            g.set_target_backend(c, BackendId::Cpu);
            (a, b, c)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(a_id, Arc::new(RwLock::new(a)));
        inputs.insert(b_id, Arc::new(RwLock::new(b)));
        let (result_arc, result_layout) =
            PipelinedExecutor::realize(graph, c_id, inputs).expect("realize");
        assert_eq!(result_layout.shape().dims(), &[2, 6]);
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(
            c.as_slice::<f32>().unwrap(),
            &[
                1.0, 2.0, 3.0, 7.0, 8.0, 9.0, 4.0, 5.0, 6.0, 10.0, 11.0, 12.0
            ]
        );
    }

    /// E2E: Concat with three inputs along outer dim — verifies
    /// variable-arity input handling through the executor.
    #[test]
    fn pipelined_realize_concat_three_inputs_outer() {
        let a = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0]);
        let b = fuel_memory::from_slice_cpu(&[3.0_f32, 4.0]);
        let c = fuel_memory::from_slice_cpu(&[5.0_f32, 6.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (a_id, b_id, c_id, cat_id) = {
            let mut g = graph.write().unwrap();
            let a = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let b = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let c = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let cat = g.push(Node {
                op: Op::Concat { dim: 0 },
                inputs: vec![a, b, c],
                shape: Shape::from_dims(&[6]),
                dtype: DType::F32,
            });
            g.set_target_backend(cat, BackendId::Cpu);
            (a, b, c, cat)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(a_id, Arc::new(RwLock::new(a)));
        inputs.insert(b_id, Arc::new(RwLock::new(b)));
        inputs.insert(c_id, Arc::new(RwLock::new(c)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, cat_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(
            c.as_slice::<f32>().unwrap(),
            &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]
        );
    }

    /// E2E: AddScalar — graph emits Op::AddScalar; the executor
    /// maps it to OpKind::Affine with mul=1, add=c.
    #[test]
    fn pipelined_realize_add_scalar() {
        let storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, op_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let op_id = g.push(Node {
                op: Op::AddScalar(10.0),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            g.set_target_backend(op_id, BackendId::Cpu);
            (in_id, op_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[11.0, 12.0, 13.0]);
    }

    /// E2E: Clamp — clamp values to [-2, 2].
    #[test]
    fn pipelined_realize_clamp() {
        let storage = fuel_memory::from_slice_cpu(&[-5.0_f32, 0.5, 100.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, op_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let op_id = g.push(Node {
                op: Op::Clamp {
                    min: -2.0,
                    max: 2.0,
                },
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            g.set_target_backend(op_id, BackendId::Cpu);
            (in_id, op_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[-2.0, 0.5, 2.0]);
    }

    /// E2E: Maximum — elementwise tensor max.
    #[test]
    fn pipelined_realize_maximum_elementwise() {
        let lhs_storage = fuel_memory::from_slice_cpu(&[1.0_f32, 5.0, -3.0]);
        let rhs_storage = fuel_memory::from_slice_cpu(&[2.0_f32, 1.0, -1.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (lhs_id, rhs_id, op_id) = {
            let mut g = graph.write().unwrap();
            let lhs = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let rhs = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let op = g.push(Node {
                op: Op::Maximum,
                inputs: vec![lhs, rhs],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            g.set_target_backend(op, BackendId::Cpu);
            (lhs, rhs, op)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(lhs_id, Arc::new(RwLock::new(lhs_storage)));
        inputs.insert(rhs_id, Arc::new(RwLock::new(rhs_storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[2.0, 5.0, -1.0]);
    }

    /// E2E: Const + Const + Conv2D — the 2×2 sum-kernel test from
    /// byte_kernels driven through the pipelined executor.
    #[test]
    fn pipelined_realize_conv2d_2x2_sum_kernel() {
        // x [1, 1, 3, 3]: [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
        // weight [1, 1, 2, 2]: all-ones
        // → out [1, 1, 2, 2]: [[12, 16], [24, 28]]
        let x_storage =
            fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]);
        let w_storage = fuel_memory::from_slice_cpu(&[1.0_f32, 1.0, 1.0, 1.0]);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (x_id, w_id, c_id) = {
            let mut g = graph.write().unwrap();
            let x = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 3, 3]),
                dtype: DType::F32,
            });
            let w = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F32,
            });
            let c = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::CONV2D,
                    fuel_graph::registry::FusedOpParams::Conv2D {
                        stride: (1, 1),
                        padding: (0, 0),
                        groups: 1,
                    },
                ),
                inputs: vec![x, w],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F32,
            });
            g.set_target_backend(c, BackendId::Cpu);
            (x, w, c)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(x_id, Arc::new(RwLock::new(x_storage)));
        inputs.insert(w_id, Arc::new(RwLock::new(w_storage)));

        let (result_arc, result_layout) =
            PipelinedExecutor::realize(graph, c_id, inputs).expect("realize");
        assert_eq!(result_layout.shape().dims(), &[1, 1, 2, 2]);

        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[12.0, 16.0, 24.0, 28.0]);
    }

    /// E2E: Conv2D with bias (3 inputs).
    #[test]
    fn pipelined_realize_conv2d_with_bias() {
        let x_storage =
            fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]);
        let w_storage = fuel_memory::from_slice_cpu(&[1.0_f32, 1.0, 1.0, 1.0]);
        let bias_storage = fuel_memory::from_slice_cpu(&[100.0_f32]);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (x_id, w_id, b_id, c_id) = {
            let mut g = graph.write().unwrap();
            let x = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 3, 3]),
                dtype: DType::F32,
            });
            let w = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F32,
            });
            let b = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1]),
                dtype: DType::F32,
            });
            let c = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::CONV2D,
                    fuel_graph::registry::FusedOpParams::Conv2D {
                        stride: (1, 1),
                        padding: (0, 0),
                        groups: 1,
                    },
                ),
                inputs: vec![x, w, b],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F32,
            });
            g.set_target_backend(c, BackendId::Cpu);
            (x, w, b, c)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(x_id, Arc::new(RwLock::new(x_storage)));
        inputs.insert(w_id, Arc::new(RwLock::new(w_storage)));
        inputs.insert(b_id, Arc::new(RwLock::new(bias_storage)));

        let (result_arc, _) = PipelinedExecutor::realize(graph, c_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[112.0, 116.0, 124.0, 128.0]);
    }

    /// E2E: Conv2D in F64 — same 2x2 sum-kernel test as F32, on doubles.
    #[test]
    fn pipelined_realize_conv2d_f64() {
        let x_storage =
            fuel_memory::from_slice_cpu(&[1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]);
        let w_storage = fuel_memory::from_slice_cpu(&[1.0_f64, 1.0, 1.0, 1.0]);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (x_id, w_id, c_id) = {
            let mut g = graph.write().unwrap();
            let x = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 3, 3]),
                dtype: DType::F64,
            });
            let w = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F64,
            });
            let c = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::CONV2D,
                    fuel_graph::registry::FusedOpParams::Conv2D {
                        stride: (1, 1),
                        padding: (0, 0),
                        groups: 1,
                    },
                ),
                inputs: vec![x, w],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F64,
            });
            g.set_target_backend(c, BackendId::Cpu);
            (x, w, c)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(x_id, Arc::new(RwLock::new(x_storage)));
        inputs.insert(w_id, Arc::new(RwLock::new(w_storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, c_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f64>().unwrap(), &[12.0, 16.0, 24.0, 28.0]);
    }

    /// E2E: Conv2D in BF16 — f32-accumulator path. Tolerant compare.
    #[test]
    fn pipelined_realize_conv2d_bf16() {
        let x_data: Vec<half::bf16> = [1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]
            .iter()
            .map(|v| half::bf16::from_f32(*v))
            .collect();
        let w_data: Vec<half::bf16> = [1.0_f32, 1.0, 1.0, 1.0]
            .iter()
            .map(|v| half::bf16::from_f32(*v))
            .collect();
        let x_storage = fuel_memory::from_slice_cpu(&x_data);
        let w_storage = fuel_memory::from_slice_cpu(&w_data);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (x_id, w_id, c_id) = {
            let mut g = graph.write().unwrap();
            let x = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 3, 3]),
                dtype: DType::BF16,
            });
            let w = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::BF16,
            });
            let c = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::CONV2D,
                    fuel_graph::registry::FusedOpParams::Conv2D {
                        stride: (1, 1),
                        padding: (0, 0),
                        groups: 1,
                    },
                ),
                inputs: vec![x, w],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::BF16,
            });
            g.set_target_backend(c, BackendId::Cpu);
            (x, w, c)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(x_id, Arc::new(RwLock::new(x_storage)));
        inputs.insert(w_id, Arc::new(RwLock::new(w_storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, c_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let got: Vec<f32> = c
            .as_slice::<half::bf16>()
            .unwrap()
            .iter()
            .map(|v| v.to_f32())
            .collect();
        let want = [12.0_f32, 16.0, 24.0, 28.0];
        for (g, w) in got.iter().zip(want.iter()) {
            assert!((g - w).abs() < 0.5, "got {got:?} want {want:?}");
        }
    }

    /// E2E: Conv2D in F16 — f32-accumulator path. Tolerant compare.
    #[test]
    fn pipelined_realize_conv2d_f16() {
        let x_data: Vec<half::f16> = [1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]
            .iter()
            .map(|v| half::f16::from_f32(*v))
            .collect();
        let w_data: Vec<half::f16> = [1.0_f32, 1.0, 1.0, 1.0]
            .iter()
            .map(|v| half::f16::from_f32(*v))
            .collect();
        let x_storage = fuel_memory::from_slice_cpu(&x_data);
        let w_storage = fuel_memory::from_slice_cpu(&w_data);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (x_id, w_id, c_id) = {
            let mut g = graph.write().unwrap();
            let x = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 3, 3]),
                dtype: DType::F16,
            });
            let w = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F16,
            });
            let c = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::CONV2D,
                    fuel_graph::registry::FusedOpParams::Conv2D {
                        stride: (1, 1),
                        padding: (0, 0),
                        groups: 1,
                    },
                ),
                inputs: vec![x, w],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F16,
            });
            g.set_target_backend(c, BackendId::Cpu);
            (x, w, c)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(x_id, Arc::new(RwLock::new(x_storage)));
        inputs.insert(w_id, Arc::new(RwLock::new(w_storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, c_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let got: Vec<f32> = c
            .as_slice::<half::f16>()
            .unwrap()
            .iter()
            .map(|v| v.to_f32())
            .collect();
        let want = [12.0_f32, 16.0, 24.0, 28.0];
        for (g, w) in got.iter().zip(want.iter()) {
            assert!((g - w).abs() < 0.05, "got {got:?} want {want:?}");
        }
    }

    /// E2E: Op::PagedAttn — F32, single-head, B=1, Sq=1.
    /// Same setup as the FlashAttn smoke test, but the K/V live in a
    /// paged cache that we look up via block_table.
    /// Layout:
    ///   block_size=2, num_blocks=1, max_blocks_per_seq=1.
    ///   k_cache shape [1, 2, 1, 2] = num_blocks × block_size × Hkv × D.
    ///   k_cache[block 0, slot 0, h 0] = [1, 0]
    ///   k_cache[block 0, slot 1, h 0] = [0, 1]
    ///   v_cache values: [10, 0] / [0, 10]
    ///   block_table[b=0, logical_block 0] = 0 (physical)
    ///   context_lens[0] = 2
    ///   q[0, 0, 0] = [2, 0]
    /// Causal is implicit (q_pos = ctx_len - Sq + sq = 2 - 1 + 0 = 1, both keys admissible).
    /// Same softmax math as FlashAttn → ~[8.808, 1.192].
    #[test]
    fn pipelined_realize_paged_attn_f32() {
        let q = fuel_memory::from_slice_cpu(&[2.0_f32, 0.0]);
        let k_cache = fuel_memory::from_slice_cpu(&[1.0_f32, 0.0, 0.0, 1.0]);
        let v_cache = fuel_memory::from_slice_cpu(&[10.0_f32, 0.0, 0.0, 10.0]);
        let block_table_u32 = fuel_memory::Storage::new(
            fuel_memory::BackendStorage::Cpu(fuel_cpu_backend::CpuStorageBytes::from_slice(&[
                0_u32,
            ])),
            DType::U32,
        );
        let context_lens_u32 = fuel_memory::Storage::new(
            fuel_memory::BackendStorage::Cpu(fuel_cpu_backend::CpuStorageBytes::from_slice(&[
                2_u32,
            ])),
            DType::U32,
        );
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (q_id, kc_id, vc_id, bt_id, cl_id, op_id) = {
            let mut g = graph.write().unwrap();
            let q = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 1, 2]),
                dtype: DType::F32,
            });
            let kc = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 2, 1, 2]),
                dtype: DType::F32,
            });
            let vc = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 2, 1, 2]),
                dtype: DType::F32,
            });
            let bt = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1]),
                dtype: DType::U32,
            });
            let cl = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1]),
                dtype: DType::U32,
            });
            let op = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::PAGED_ATTN,
                    fuel_graph::registry::FusedOpParams::PagedAttn {
                        softmax_scale: 1.0,
                        block_size: 2,
                        softcap: None,
                    },
                ),
                inputs: vec![q, kc, vc, bt, cl],
                shape: Shape::from_dims(&[1, 1, 1, 2]),
                dtype: DType::F32,
            });
            g.set_target_backend(op, BackendId::Cpu);
            (q, kc, vc, bt, cl, op)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(q_id, Arc::new(RwLock::new(q)));
        inputs.insert(kc_id, Arc::new(RwLock::new(k_cache)));
        inputs.insert(vc_id, Arc::new(RwLock::new(v_cache)));
        inputs.insert(bt_id, Arc::new(RwLock::new(block_table_u32)));
        inputs.insert(cl_id, Arc::new(RwLock::new(context_lens_u32)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let r = c.as_slice::<f32>().unwrap();
        let expected_p0 = 2.0_f32.exp() / (2.0_f32.exp() + 1.0);
        let expected_p1 = 1.0_f32 / (2.0_f32.exp() + 1.0);
        assert!(
            (r[0] - 10.0 * expected_p0).abs() < 1e-5,
            "row[0]: got {} expected {}",
            r[0],
            10.0 * expected_p0
        );
        assert!(
            (r[1] - 10.0 * expected_p1).abs() < 1e-5,
            "row[1]: got {} expected {}",
            r[1],
            10.0 * expected_p1
        );
    }

    /// E2E: PagedAttn BF16 — same single-row test, tolerant.
    #[test]
    fn pipelined_realize_paged_attn_bf16() {
        let q_v: Vec<half::bf16> = [2.0_f32, 0.0]
            .iter()
            .map(|x| half::bf16::from_f32(*x))
            .collect();
        let kc_v: Vec<half::bf16> = [1.0_f32, 0.0, 0.0, 1.0]
            .iter()
            .map(|x| half::bf16::from_f32(*x))
            .collect();
        let vc_v: Vec<half::bf16> = [10.0_f32, 0.0, 0.0, 10.0]
            .iter()
            .map(|x| half::bf16::from_f32(*x))
            .collect();
        let q = fuel_memory::from_slice_cpu(&q_v);
        let k_cache = fuel_memory::from_slice_cpu(&kc_v);
        let v_cache = fuel_memory::from_slice_cpu(&vc_v);
        let bt = fuel_memory::Storage::new(
            fuel_memory::BackendStorage::Cpu(fuel_cpu_backend::CpuStorageBytes::from_slice(&[
                0_u32,
            ])),
            DType::U32,
        );
        let cl = fuel_memory::Storage::new(
            fuel_memory::BackendStorage::Cpu(fuel_cpu_backend::CpuStorageBytes::from_slice(&[
                2_u32,
            ])),
            DType::U32,
        );
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (q_id, kc_id, vc_id, bt_id, cl_id, op_id) = {
            let mut g = graph.write().unwrap();
            let q = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 1, 2]),
                dtype: DType::BF16,
            });
            let kc = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 2, 1, 2]),
                dtype: DType::BF16,
            });
            let vc = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 2, 1, 2]),
                dtype: DType::BF16,
            });
            let bt = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1]),
                dtype: DType::U32,
            });
            let cl = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1]),
                dtype: DType::U32,
            });
            let op = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::PAGED_ATTN,
                    fuel_graph::registry::FusedOpParams::PagedAttn {
                        softmax_scale: 1.0,
                        block_size: 2,
                        softcap: None,
                    },
                ),
                inputs: vec![q, kc, vc, bt, cl],
                shape: Shape::from_dims(&[1, 1, 1, 2]),
                dtype: DType::BF16,
            });
            g.set_target_backend(op, BackendId::Cpu);
            (q, kc, vc, bt, cl, op)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(q_id, Arc::new(RwLock::new(q)));
        inputs.insert(kc_id, Arc::new(RwLock::new(k_cache)));
        inputs.insert(vc_id, Arc::new(RwLock::new(v_cache)));
        inputs.insert(bt_id, Arc::new(RwLock::new(bt)));
        inputs.insert(cl_id, Arc::new(RwLock::new(cl)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let got: Vec<f32> = c
            .as_slice::<half::bf16>()
            .unwrap()
            .iter()
            .map(|v| v.to_f32())
            .collect();
        let expected_p0 = 2.0_f32.exp() / (2.0_f32.exp() + 1.0);
        let expected_p1 = 1.0_f32 / (2.0_f32.exp() + 1.0);
        let want = [10.0 * expected_p0, 10.0 * expected_p1];
        for (g, w) in got.iter().zip(want.iter()) {
            assert!((g - w).abs() < 0.5, "got {got:?} want {want:?}");
        }
    }

    /// E2E: Op::FlashAttn — F32 single-head, single-batch, no mask.
    /// q = [[2.0, 0.0]], k = [[1.0, 0.0], [0.0, 1.0]], v = [[10, 0], [0, 10]]
    /// scale = 1.0
    /// scores = q · kᵀ = [2.0, 0.0]
    /// softmax = [e^2/(e^2+1), 1/(e^2+1)] ≈ [0.8808, 0.1192]
    /// out = softmax @ v = [10*0.8808, 10*0.1192] ≈ [8.808, 1.192]
    #[test]
    fn pipelined_realize_flash_attn_f32() {
        // [B=1, H=1, S=1or2, D=2]
        let q = fuel_memory::from_slice_cpu(&[2.0_f32, 0.0]);
        let k = fuel_memory::from_slice_cpu(&[1.0_f32, 0.0, 0.0, 1.0]);
        let v = fuel_memory::from_slice_cpu(&[10.0_f32, 0.0, 0.0, 10.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (q_id, k_id, v_id, op_id) = {
            let mut g = graph.write().unwrap();
            let q = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 1, 2]),
                dtype: DType::F32,
            });
            let k = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F32,
            });
            let v = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F32,
            });
            let op = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::FLASH_ATTN,
                    fuel_graph::registry::FusedOpParams::FlashAttn {
                        softmax_scale: 1.0,
                        causal: false,
                        window_size_left: None,
                        window_size_right: None,
                        softcap: None,
                        k_len: None,
                    },
                ),
                inputs: vec![q, k, v],
                shape: Shape::from_dims(&[1, 1, 1, 2]),
                dtype: DType::F32,
            });
            g.set_target_backend(op, BackendId::Cpu);
            (q, k, v, op)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(q_id, Arc::new(RwLock::new(q)));
        inputs.insert(k_id, Arc::new(RwLock::new(k)));
        inputs.insert(v_id, Arc::new(RwLock::new(v)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let r = c.as_slice::<f32>().unwrap();
        let expected_p0 = 2.0_f32.exp() / (2.0_f32.exp() + 1.0);
        let expected_p1 = 1.0_f32 / (2.0_f32.exp() + 1.0);
        assert!(
            (r[0] - 10.0 * expected_p0).abs() < 1e-5,
            "row[0]: got {} expected {}",
            r[0],
            10.0 * expected_p0
        );
        assert!(
            (r[1] - 10.0 * expected_p1).abs() < 1e-5,
            "row[1]: got {} expected {}",
            r[1],
            10.0 * expected_p1
        );
    }

    /// E2E: FlashAttn with causal mask — second query position
    /// attends to both keys (positions 0 and 1), first only attends
    /// to key 0 (everything beyond is masked).
    #[test]
    fn pipelined_realize_flash_attn_causal_f32() {
        // q [1,1,2,2]: query 0 = [1, 0], query 1 = [0, 1]
        // k [1,1,2,2]: keys = [[1, 0], [0, 1]]
        // v [1,1,2,2]: values = [[5, 6], [7, 8]]
        // softmax_scale=1, causal:
        //   query 0: only key 0 admissible → out = v[0] = [5, 6]
        //   query 1: both admissible. scores = q1·k = [0, 1]
        //            softmax = [1/(e+1), e/(e+1)]
        //            out = scores · v = [(5)/(e+1) + 7e/(e+1), 6/(e+1) + 8e/(e+1)]
        let q = fuel_memory::from_slice_cpu(&[1.0_f32, 0.0, 0.0, 1.0]);
        let k = fuel_memory::from_slice_cpu(&[1.0_f32, 0.0, 0.0, 1.0]);
        let v = fuel_memory::from_slice_cpu(&[5.0_f32, 6.0, 7.0, 8.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (q_id, k_id, v_id, op_id) = {
            let mut g = graph.write().unwrap();
            let q = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F32,
            });
            let k = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F32,
            });
            let v = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F32,
            });
            let op = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::FLASH_ATTN,
                    fuel_graph::registry::FusedOpParams::FlashAttn {
                        softmax_scale: 1.0,
                        causal: true,
                        window_size_left: None,
                        window_size_right: None,
                        softcap: None,
                        k_len: None,
                    },
                ),
                inputs: vec![q, k, v],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F32,
            });
            g.set_target_backend(op, BackendId::Cpu);
            (q, k, v, op)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(q_id, Arc::new(RwLock::new(q)));
        inputs.insert(k_id, Arc::new(RwLock::new(k)));
        inputs.insert(v_id, Arc::new(RwLock::new(v)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let r = c.as_slice::<f32>().unwrap();
        // Query 0 sees only key 0 → output = v[0]
        assert!((r[0] - 5.0).abs() < 1e-5, "got {}", r[0]);
        assert!((r[1] - 6.0).abs() < 1e-5, "got {}", r[1]);
        // Query 1 sees both. softmax([0, 1]) = [1/(e+1), e/(e+1)]
        let denom = (1.0_f32).exp() + 1.0;
        let expected_a = 5.0 / denom + 7.0 * (1.0_f32).exp() / denom;
        let expected_b = 6.0 / denom + 8.0 * (1.0_f32).exp() / denom;
        assert!(
            (r[2] - expected_a).abs() < 1e-5,
            "row1[0]: got {} expected {}",
            r[2],
            expected_a
        );
        assert!(
            (r[3] - expected_b).abs() < 1e-5,
            "row1[1]: got {} expected {}",
            r[3],
            expected_b
        );
    }

    /// E2E: FlashAttn over a fixed-**capacity** K/V with a runtime
    /// `k_len` resolved through a `SymEnv` (Phase D symbolic extents).
    /// Only the first `k_len` rows are attended; trailing "poison" rows
    /// in the capacity buffer are ignored, and the causal mask is
    /// bottom-right-aligned at offset `k_len - Sq`. This is flash decode
    /// over a persistent KV-cache: K/V are `[.., max_seq, ..]` capacity
    /// and the live prefix grows per token.
    #[test]
    fn pipelined_realize_flash_attn_dynamic_k_len() {
        use fuel_ir::{DynScalar, SymId};
        // q [1,1,2,2]; K/V capacity [1,1,4,2]; bind k_len = 3 so row 3
        // (the "poison" row) is never attended.
        let q = fuel_memory::from_slice_cpu(&[1.0_f32, 0.0, 0.0, 1.0]);
        let k = fuel_memory::from_slice_cpu(&[
            1.0_f32, 0.0, 0.0, 1.0, 1.0, 1.0, 100.0,
            100.0, // poison row 3 — must be ignored (k_len=3)
        ]);
        let v = fuel_memory::from_slice_cpu(&[
            5.0_f32, 6.0, 7.0, 8.0, 9.0, 10.0, 999.0,
            999.0, // poison row 3 — must be ignored
        ]);
        let sym = SymId(0);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (q_id, k_id, v_id, op_id) = {
            let mut g = graph.write().unwrap();
            let q = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F32,
            });
            let k = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 4, 2]),
                dtype: DType::F32,
            });
            let v = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 4, 2]),
                dtype: DType::F32,
            });
            let op = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::FLASH_ATTN,
                    fuel_graph::registry::FusedOpParams::FlashAttn {
                        softmax_scale: 1.0,
                        causal: true,
                        window_size_left: None,
                        window_size_right: None,
                        softcap: None,
                        k_len: Some(DynScalar::Sym(sym)),
                    },
                ),
                inputs: vec![q, k, v],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F32,
            });
            g.set_target_backend(op, BackendId::Cpu);
            (q, k, v, op)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(q_id, Arc::new(RwLock::new(q)));
        inputs.insert(k_id, Arc::new(RwLock::new(k)));
        inputs.insert(v_id, Arc::new(RwLock::new(v)));
        let mut env = SymEnv::new();
        env.bind(sym, 3).unwrap();
        let (result_arc, _) = PipelinedExecutor::realize_with_env(graph, op_id, inputs, env)
            .expect("realize_with_env");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let r = c.as_slice::<f32>().unwrap();
        let e = 1.0_f32.exp();
        // causal_offset = k_len - Sq = 3 - 2 = 1.
        // Query 0 (abs pos 1): keys {0,1}. scores [1,0] → softmax [e,1]/(e+1).
        let d0 = e + 1.0;
        let e0x = (5.0 * e + 7.0) / d0;
        let e0y = (6.0 * e + 8.0) / d0;
        // Query 1 (abs pos 2): keys {0,1,2}. scores [0,1,1] → [1,e,e]/(1+2e).
        let d1 = 1.0 + 2.0 * e;
        let e1x = (5.0 + 7.0 * e + 9.0 * e) / d1;
        let e1y = (6.0 + 8.0 * e + 10.0 * e) / d1;
        for (got, want) in r.iter().zip([e0x, e0y, e1x, e1y].iter()) {
            assert!(
                (got - want).abs() < 1e-4,
                "flash dyn k_len: got {got}, want {want} (full output {r:?}); \
                 a poison-contaminated value means the capacity rows past k_len leaked in",
            );
        }
    }

    /// A flash `k_len` symbol unbound in the `SymEnv` surfaces a typed
    /// error at realize (never a panic).
    #[test]
    fn pipelined_realize_flash_attn_dynamic_k_len_unbound_errors() {
        use fuel_ir::{DynScalar, SymId};
        let q = fuel_memory::from_slice_cpu(&[1.0_f32, 0.0, 0.0, 1.0]);
        let k = fuel_memory::from_slice_cpu(&[0.0_f32; 16]);
        let v = fuel_memory::from_slice_cpu(&[0.0_f32; 16]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (q_id, k_id, v_id, op_id) = {
            let mut g = graph.write().unwrap();
            let q = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F32,
            });
            let k = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 4, 2]),
                dtype: DType::F32,
            });
            let v = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 4, 2]),
                dtype: DType::F32,
            });
            let op = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::FLASH_ATTN,
                    fuel_graph::registry::FusedOpParams::FlashAttn {
                        softmax_scale: 1.0,
                        causal: true,
                        window_size_left: None,
                        window_size_right: None,
                        softcap: None,
                        k_len: Some(DynScalar::Sym(SymId(0))),
                    },
                ),
                inputs: vec![q, k, v],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F32,
            });
            g.set_target_backend(op, BackendId::Cpu);
            (q, k, v, op)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(q_id, Arc::new(RwLock::new(q)));
        inputs.insert(k_id, Arc::new(RwLock::new(k)));
        inputs.insert(v_id, Arc::new(RwLock::new(v)));
        // Empty env — the k_len symbol is unbound.
        let result = PipelinedExecutor::realize_with_env(graph, op_id, inputs, SymEnv::new());
        assert!(
            result.is_err(),
            "unbound flash k_len symbol must surface a typed error"
        );
    }

    /// E2E: FlashAttn BF16 — same single-row test as f32, tolerant.
    #[test]
    fn pipelined_realize_flash_attn_bf16() {
        let q_v: Vec<half::bf16> = [2.0_f32, 0.0]
            .iter()
            .map(|x| half::bf16::from_f32(*x))
            .collect();
        let k_v: Vec<half::bf16> = [1.0_f32, 0.0, 0.0, 1.0]
            .iter()
            .map(|x| half::bf16::from_f32(*x))
            .collect();
        let v_v: Vec<half::bf16> = [10.0_f32, 0.0, 0.0, 10.0]
            .iter()
            .map(|x| half::bf16::from_f32(*x))
            .collect();
        let q = fuel_memory::from_slice_cpu(&q_v);
        let k = fuel_memory::from_slice_cpu(&k_v);
        let v = fuel_memory::from_slice_cpu(&v_v);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (q_id, k_id, v_id, op_id) = {
            let mut g = graph.write().unwrap();
            let q = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 1, 2]),
                dtype: DType::BF16,
            });
            let k = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::BF16,
            });
            let v = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::BF16,
            });
            let op = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::FLASH_ATTN,
                    fuel_graph::registry::FusedOpParams::FlashAttn {
                        softmax_scale: 1.0,
                        causal: false,
                        window_size_left: None,
                        window_size_right: None,
                        softcap: None,
                        k_len: None,
                    },
                ),
                inputs: vec![q, k, v],
                shape: Shape::from_dims(&[1, 1, 1, 2]),
                dtype: DType::BF16,
            });
            g.set_target_backend(op, BackendId::Cpu);
            (q, k, v, op)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(q_id, Arc::new(RwLock::new(q)));
        inputs.insert(k_id, Arc::new(RwLock::new(k)));
        inputs.insert(v_id, Arc::new(RwLock::new(v)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let got: Vec<f32> = c
            .as_slice::<half::bf16>()
            .unwrap()
            .iter()
            .map(|v| v.to_f32())
            .collect();
        let expected_p0 = 2.0_f32.exp() / (2.0_f32.exp() + 1.0);
        let expected_p1 = 1.0_f32 / (2.0_f32.exp() + 1.0);
        let want = [10.0 * expected_p0, 10.0 * expected_p1];
        for (g, w) in got.iter().zip(want.iter()) {
            assert!((g - w).abs() < 0.5, "got {got:?} want {want:?}");
        }
    }

    /// E2E: Op::FusedLinear — F32. a[1,2,3] @ b[1,3,2] + bias[2].
    /// a = [[[1,2,3],[4,5,6]]], b = [[[1,0],[0,1],[1,1]]], bias=[10,20]
    /// matmul = [[1+0+3, 0+2+3], [4+0+6, 0+5+6]] = [[4, 5], [10, 11]]
    /// + bias = [[14, 25], [20, 31]]
    #[test]
    fn pipelined_realize_fused_linear_f32() {
        let a = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0]);
        let b = fuel_memory::from_slice_cpu(&[1.0_f32, 0.0, 0.0, 1.0, 1.0, 1.0]);
        let bias = fuel_memory::from_slice_cpu(&[10.0_f32, 20.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (a_id, b_id, bias_id, op_id) = {
            let mut g = graph.write().unwrap();
            let a = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 2, 3]),
                dtype: DType::F32,
            });
            let b = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 3, 2]),
                dtype: DType::F32,
            });
            let bias = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let op = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::FUSED_LINEAR,
                    fuel_graph::registry::FusedOpParams::FusedLinear,
                ),
                inputs: vec![a, b, bias],
                shape: Shape::from_dims(&[1, 2, 2]),
                dtype: DType::F32,
            });
            g.set_target_backend(op, BackendId::Cpu);
            (a, b, bias, op)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(a_id, Arc::new(RwLock::new(a)));
        inputs.insert(b_id, Arc::new(RwLock::new(b)));
        inputs.insert(bias_id, Arc::new(RwLock::new(bias)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[14.0, 25.0, 20.0, 31.0]);
    }

    /// E2E: FusedLinear F64 — same shape test on doubles.
    #[test]
    fn pipelined_realize_fused_linear_f64() {
        let a = fuel_memory::from_slice_cpu(&[1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0]);
        let b = fuel_memory::from_slice_cpu(&[1.0_f64, 0.0, 0.0, 1.0, 1.0, 1.0]);
        let bias = fuel_memory::from_slice_cpu(&[10.0_f64, 20.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (a_id, b_id, bias_id, op_id) = {
            let mut g = graph.write().unwrap();
            let a = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 2, 3]),
                dtype: DType::F64,
            });
            let b = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 3, 2]),
                dtype: DType::F64,
            });
            let bias = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F64,
            });
            let op = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::FUSED_LINEAR,
                    fuel_graph::registry::FusedOpParams::FusedLinear,
                ),
                inputs: vec![a, b, bias],
                shape: Shape::from_dims(&[1, 2, 2]),
                dtype: DType::F64,
            });
            g.set_target_backend(op, BackendId::Cpu);
            (a, b, bias, op)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(a_id, Arc::new(RwLock::new(a)));
        inputs.insert(b_id, Arc::new(RwLock::new(b)));
        inputs.insert(bias_id, Arc::new(RwLock::new(bias)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f64>().unwrap(), &[14.0, 25.0, 20.0, 31.0]);
    }

    /// E2E: FusedLinear BF16 — tolerant compare.
    #[test]
    fn pipelined_realize_fused_linear_bf16() {
        let a_v: Vec<half::bf16> = [1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0]
            .iter()
            .map(|x| half::bf16::from_f32(*x))
            .collect();
        let b_v: Vec<half::bf16> = [1.0_f32, 0.0, 0.0, 1.0, 1.0, 1.0]
            .iter()
            .map(|x| half::bf16::from_f32(*x))
            .collect();
        let bias_v: Vec<half::bf16> = [10.0_f32, 20.0]
            .iter()
            .map(|x| half::bf16::from_f32(*x))
            .collect();
        let a = fuel_memory::from_slice_cpu(&a_v);
        let b = fuel_memory::from_slice_cpu(&b_v);
        let bias = fuel_memory::from_slice_cpu(&bias_v);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (a_id, b_id, bias_id, op_id) = {
            let mut g = graph.write().unwrap();
            let a = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 2, 3]),
                dtype: DType::BF16,
            });
            let b = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 3, 2]),
                dtype: DType::BF16,
            });
            let bias = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::BF16,
            });
            let op = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::FUSED_LINEAR,
                    fuel_graph::registry::FusedOpParams::FusedLinear,
                ),
                inputs: vec![a, b, bias],
                shape: Shape::from_dims(&[1, 2, 2]),
                dtype: DType::BF16,
            });
            g.set_target_backend(op, BackendId::Cpu);
            (a, b, bias, op)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(a_id, Arc::new(RwLock::new(a)));
        inputs.insert(b_id, Arc::new(RwLock::new(b)));
        inputs.insert(bias_id, Arc::new(RwLock::new(bias)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let got: Vec<f32> = c
            .as_slice::<half::bf16>()
            .unwrap()
            .iter()
            .map(|v| v.to_f32())
            .collect();
        let want = [14.0_f32, 25.0, 20.0, 31.0];
        for (g, w) in got.iter().zip(want.iter()) {
            assert!((g - w).abs() < 0.5, "got {got:?} want {want:?}");
        }
    }

    /// E2E: Op::ReduceSumTo — sum the leading axis of a [2,3] tensor.
    /// Input [[1,2,3],[4,5,6]] → output [5,7,9].
    #[test]
    fn pipelined_realize_reduce_sum_to_f32_drops_leading_axis() {
        let v = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, op_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let op_id = g.push(Node {
                op: Op::ReduceSumTo(Shape::from_dims(&[3])),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            g.set_target_backend(op_id, BackendId::Cpu);
            (in_id, op_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(v)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[5.0, 7.0, 9.0]);
    }

    /// E2E: Op::ReduceSumTo — keep-dim with 1 in the middle.
    /// Input [2,3,4] → [2,1,4] sums along dim 1.
    #[test]
    fn pipelined_realize_reduce_sum_to_f32_keepdim_middle() {
        // [2,3,4]: layer 0 = [[1,2,3,4],[5,6,7,8],[9,10,11,12]]
        //          layer 1 = [[13..16],[17..20],[21..24]]
        let mut v: Vec<f32> = (1..=24).map(|x| x as f32).collect();
        let _ = &mut v;
        let s = fuel_memory::from_slice_cpu(&v);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, op_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3, 4]),
                dtype: DType::F32,
            });
            let op_id = g.push(Node {
                op: Op::ReduceSumTo(Shape::from_dims(&[2, 1, 4])),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[2, 1, 4]),
                dtype: DType::F32,
            });
            g.set_target_backend(op_id, BackendId::Cpu);
            (in_id, op_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(s)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        // layer0 dim1-sum: col j = 1+5+9, 2+6+10, 3+7+11, 4+8+12 = [15,18,21,24]
        // layer1 dim1-sum: col j = 13+17+21, 14+18+22, 15+19+23, 16+20+24 = [51,54,57,60]
        assert_eq!(
            c.as_slice::<f32>().unwrap(),
            &[15.0, 18.0, 21.0, 24.0, 51.0, 54.0, 57.0, 60.0],
        );
    }

    /// E2E: ReduceSumTo F64 — same drop-leading-axis test on doubles.
    #[test]
    fn pipelined_realize_reduce_sum_to_f64() {
        let v = fuel_memory::from_slice_cpu(&[1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, op_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F64,
            });
            let op_id = g.push(Node {
                op: Op::ReduceSumTo(Shape::from_dims(&[3])),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F64,
            });
            g.set_target_backend(op_id, BackendId::Cpu);
            (in_id, op_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(v)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f64>().unwrap(), &[5.0, 7.0, 9.0]);
    }

    /// E2E: Op::ReduceMaxTo F32 — drop the leading axis with max-reduce.
    #[test]
    fn pipelined_realize_reduce_max_to_f32_drops_leading_axis() {
        // Input [2,3]: row 0 = [1, 7, 3], row 1 = [4, 2, 6]. Max along
        // dim 0: [4, 7, 6].
        let v = fuel_memory::from_slice_cpu(&[1.0_f32, 7.0, 3.0, 4.0, 2.0, 6.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, op_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let op_id = g.push(Node {
                op: Op::ReduceMaxTo(Shape::from_dims(&[3])),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            g.set_target_backend(op_id, BackendId::Cpu);
            (in_id, op_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(v)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[4.0, 7.0, 6.0]);
    }

    /// E2E: Op::ReduceMaxTo F32 — keep-dim with 1 in the trailing axis.
    /// Mirrors the SoftmaxLastDim lowering's max-side shape: input
    /// [..., last] → [..., 1].
    #[test]
    fn pipelined_realize_reduce_max_to_f32_keepdim_trailing() {
        // Input [2, 3]: row maxes = [3, 6].
        let v = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, op_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let op_id = g.push(Node {
                op: Op::ReduceMaxTo(Shape::from_dims(&[2, 1])),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[2, 1]),
                dtype: DType::F32,
            });
            g.set_target_backend(op_id, BackendId::Cpu);
            (in_id, op_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(v)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[3.0, 6.0]);
    }

    /// E2E: ReduceSumTo BF16 — tolerant compare via f32-acc.
    #[test]
    fn pipelined_realize_reduce_sum_to_bf16() {
        let v: Vec<half::bf16> = [1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0]
            .iter()
            .map(|x| half::bf16::from_f32(*x))
            .collect();
        let s = fuel_memory::from_slice_cpu(&v);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, op_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::BF16,
            });
            let op_id = g.push(Node {
                op: Op::ReduceSumTo(Shape::from_dims(&[3])),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::BF16,
            });
            g.set_target_backend(op_id, BackendId::Cpu);
            (in_id, op_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(s)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, op_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let got: Vec<f32> = c
            .as_slice::<half::bf16>()
            .unwrap()
            .iter()
            .map(|v| v.to_f32())
            .collect();
        let want = [5.0_f32, 7.0, 9.0];
        for (g, w) in got.iter().zip(want.iter()) {
            assert!((g - w).abs() < 0.5, "got {got:?} want {want:?}");
        }
    }

    /// E2E: Op::ConvTranspose2D — F32 spread test.
    /// x = [[1, 2], [3, 4]] shape [1,1,2,2], all-ones kernel
    /// shape [1,1,2,2], stride=1, padding=0, dilation=1, no bias.
    /// Expected output (3x3):
    ///   [[1,  3, 2],
    ///    [4, 10, 6],
    ///    [3,  7, 4]]
    #[test]
    fn pipelined_realize_conv_transpose2d_f32() {
        let x_storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0]);
        let w_storage = fuel_memory::from_slice_cpu(&[1.0_f32, 1.0, 1.0, 1.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (x_id, w_id, c_id) = {
            let mut g = graph.write().unwrap();
            let x = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F32,
            });
            let w = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F32,
            });
            let c = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::CONV_TRANSPOSE2D,
                    fuel_graph::registry::FusedOpParams::ConvTranspose2D {
                        stride: (1, 1),
                        padding: (0, 0),
                        output_padding: (0, 0),
                        dilation: (1, 1),
                        groups: 1,
                    },
                ),
                inputs: vec![x, w],
                shape: Shape::from_dims(&[1, 1, 3, 3]),
                dtype: DType::F32,
            });
            g.set_target_backend(c, BackendId::Cpu);
            (x, w, c)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(x_id, Arc::new(RwLock::new(x_storage)));
        inputs.insert(w_id, Arc::new(RwLock::new(w_storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, c_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(
            c.as_slice::<f32>().unwrap(),
            &[1.0, 3.0, 2.0, 4.0, 10.0, 6.0, 3.0, 7.0, 4.0],
        );
    }

    /// E2E: ConvTranspose2D F64 — same shape test.
    #[test]
    fn pipelined_realize_conv_transpose2d_f64() {
        let x_storage = fuel_memory::from_slice_cpu(&[1.0_f64, 2.0, 3.0, 4.0]);
        let w_storage = fuel_memory::from_slice_cpu(&[1.0_f64, 1.0, 1.0, 1.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (x_id, w_id, c_id) = {
            let mut g = graph.write().unwrap();
            let x = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F64,
            });
            let w = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::F64,
            });
            let c = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::CONV_TRANSPOSE2D,
                    fuel_graph::registry::FusedOpParams::ConvTranspose2D {
                        stride: (1, 1),
                        padding: (0, 0),
                        output_padding: (0, 0),
                        dilation: (1, 1),
                        groups: 1,
                    },
                ),
                inputs: vec![x, w],
                shape: Shape::from_dims(&[1, 1, 3, 3]),
                dtype: DType::F64,
            });
            g.set_target_backend(c, BackendId::Cpu);
            (x, w, c)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(x_id, Arc::new(RwLock::new(x_storage)));
        inputs.insert(w_id, Arc::new(RwLock::new(w_storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, c_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(
            c.as_slice::<f64>().unwrap(),
            &[1.0, 3.0, 2.0, 4.0, 10.0, 6.0, 3.0, 7.0, 4.0],
        );
    }

    /// E2E: ConvTranspose2D BF16 — tolerant compare via f32-acc.
    #[test]
    fn pipelined_realize_conv_transpose2d_bf16() {
        let x: Vec<half::bf16> = [1.0_f32, 2.0, 3.0, 4.0]
            .iter()
            .map(|v| half::bf16::from_f32(*v))
            .collect();
        let w: Vec<half::bf16> = [1.0_f32, 1.0, 1.0, 1.0]
            .iter()
            .map(|v| half::bf16::from_f32(*v))
            .collect();
        let x_storage = fuel_memory::from_slice_cpu(&x);
        let w_storage = fuel_memory::from_slice_cpu(&w);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (x_id, w_id, c_id) = {
            let mut g = graph.write().unwrap();
            let x = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::BF16,
            });
            let w = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 1, 2, 2]),
                dtype: DType::BF16,
            });
            let c = g.push(Node {
                op: Op::Fused(
                    fuel_graph::registry::FusedOps::CONV_TRANSPOSE2D,
                    fuel_graph::registry::FusedOpParams::ConvTranspose2D {
                        stride: (1, 1),
                        padding: (0, 0),
                        output_padding: (0, 0),
                        dilation: (1, 1),
                        groups: 1,
                    },
                ),
                inputs: vec![x, w],
                shape: Shape::from_dims(&[1, 1, 3, 3]),
                dtype: DType::BF16,
            });
            g.set_target_backend(c, BackendId::Cpu);
            (x, w, c)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(x_id, Arc::new(RwLock::new(x_storage)));
        inputs.insert(w_id, Arc::new(RwLock::new(w_storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, c_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let got: Vec<f32> = c
            .as_slice::<half::bf16>()
            .unwrap()
            .iter()
            .map(|v| v.to_f32())
            .collect();
        let want = [1.0_f32, 3.0, 2.0, 4.0, 10.0, 6.0, 3.0, 7.0, 4.0];
        for (g, w) in got.iter().zip(want.iter()) {
            assert!((g - w).abs() < 0.5, "got {got:?} want {want:?}");
        }
    }

    /// E2E: GQA-style matmul through the pipelined executor.
    /// lhs has 4 batch heads, rhs has 2; each rhs head is shared
    /// by 2 lhs heads. Output's batch dim follows lhs (4 heads).
    #[test]
    fn pipelined_realize_matmul_gqa() {
        // lhs [4, 1, 2]: heads 0..3 are [[1,2]], [[3,4]], [[5,6]], [[7,8]]
        // rhs [2, 2, 1]: heads 0,1 are [[1],[0]], [[0],[1]]
        // Expected out [4, 1, 1]: [[1]], [[3]], [[6]], [[8]]
        let lhs_storage =
            fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]);
        let rhs_storage = fuel_memory::from_slice_cpu(&[1.0_f32, 0.0, 0.0, 1.0]);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (lhs_id, rhs_id, mm_id) = {
            let mut g = graph.write().unwrap();
            let lhs = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[4, 1, 2]),
                dtype: DType::F32,
            });
            let rhs = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 2, 1]),
                dtype: DType::F32,
            });
            let mm = g.push(Node {
                op: Op::MatMul,
                inputs: vec![lhs, rhs],
                shape: Shape::from_dims(&[4, 1, 1]),
                dtype: DType::F32,
            });
            g.set_target_backend(mm, BackendId::Cpu);
            (lhs, rhs, mm)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(lhs_id, Arc::new(RwLock::new(lhs_storage)));
        inputs.insert(rhs_id, Arc::new(RwLock::new(rhs_storage)));

        let (result_arc, result_layout) =
            PipelinedExecutor::realize(graph, mm_id, inputs).expect("realize");
        assert_eq!(result_layout.shape().dims(), &[4, 1, 1]);

        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[1.0, 3.0, 6.0, 8.0]);
    }

    /// E2E: matmul with a transposed rhs — proves stage 3+4
    /// integration carries through the matmul path. The transpose
    /// is metadata-only; auto-Contiguize materializes the strided
    /// rhs before the matmul kernel sees it.
    #[test]
    fn pipelined_realize_matmul_with_transposed_rhs() {
        // lhs [[1, 2], [3, 4]], rhs original [[5, 6], [7, 8]]
        // rhs.T = [[5, 7], [6, 8]]
        // lhs @ rhs.T = [[1*5+2*6, 1*7+2*8], [3*5+4*6, 3*7+4*8]]
        //             = [[17, 23], [39, 53]]
        let lhs_storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0]);
        let rhs_storage = fuel_memory::from_slice_cpu(&[5.0_f32, 6.0, 7.0, 8.0]);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (lhs_id, rhs_id, t_id, mm_id) = {
            let mut g = graph.write().unwrap();
            let lhs = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 2]),
                dtype: DType::F32,
            });
            let rhs = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 2]),
                dtype: DType::F32,
            });
            let t = g.push(Node {
                op: Op::Transpose,
                inputs: vec![rhs],
                shape: Shape::from_dims(&[2, 2]),
                dtype: DType::F32,
            });
            let mm = g.push(Node {
                op: Op::MatMul,
                inputs: vec![lhs, t],
                shape: Shape::from_dims(&[2, 2]),
                dtype: DType::F32,
            });
            g.set_target_backend(mm, BackendId::Cpu);
            (lhs, rhs, t, mm)
        };
        let _ = t_id;
        let mut inputs = StorageCache::new();
        inputs.insert(lhs_id, Arc::new(RwLock::new(lhs_storage)));
        inputs.insert(rhs_id, Arc::new(RwLock::new(rhs_storage)));

        let (result_arc, _) = PipelinedExecutor::realize(graph, mm_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[17.0, 23.0, 39.0, 53.0]);
    }

    /// E2E: Const + Reshape — contiguous-input reshape is zero
    /// copy. The output Storage Arc is the input Arc; the layout
    /// is contiguous in the new shape.
    #[test]
    fn pipelined_realize_reshape_zero_copy_when_contiguous() {
        let storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, r_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let r_id = g.push(Node {
                op: Op::Reshape(Shape::from_dims(&[3, 2])),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3, 2]),
                dtype: DType::F32,
            });
            (in_id, r_id)
        };
        let mut inputs = StorageCache::new();
        let in_arc = Arc::new(RwLock::new(storage));
        inputs.insert(in_id, Arc::clone(&in_arc));

        let (result_arc, result_layout) =
            PipelinedExecutor::realize(graph, r_id, inputs).expect("realize");

        // Zero copy — same Arc.
        assert!(
            Arc::ptr_eq(&result_arc, &in_arc),
            "contiguous reshape must zero-copy"
        );
        assert_eq!(result_layout.shape().dims(), &[3, 2]);
        assert!(result_layout.is_contiguous());

        // Bytes are unchanged; just reinterpreted as [3, 2].
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(
            c.as_slice::<f32>().unwrap(),
            &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]
        );
    }

    /// E2E: Const + Transpose + Reshape — reshape on a strided
    /// input auto-contiguizes the bytes. Output Arc is fresh
    /// (NOT the input Arc); the bytes are the materialized
    /// transposed layout.
    #[test]
    fn pipelined_realize_reshape_materializes_when_strided() {
        // shape [2, 3]: 1 2 3 / 4 5 6
        // Transpose → [3, 2] strided
        // Reshape → [6] (forces materialization)
        let storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, t_id, r_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let t_id = g.push(Node {
                op: Op::Transpose,
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3, 2]),
                dtype: DType::F32,
            });
            let r_id = g.push(Node {
                op: Op::Reshape(Shape::from_dims(&[6])),
                inputs: vec![t_id],
                shape: Shape::from_dims(&[6]),
                dtype: DType::F32,
            });
            (in_id, t_id, r_id)
        };
        let _ = t_id;
        let mut inputs = StorageCache::new();
        let in_arc = Arc::new(RwLock::new(storage));
        inputs.insert(in_id, Arc::clone(&in_arc));

        let (result_arc, result_layout) =
            PipelinedExecutor::realize(graph, r_id, inputs).expect("realize");

        // Fresh Arc — auto-contiguize allocated new bytes.
        assert!(!Arc::ptr_eq(&result_arc, &in_arc));
        assert_eq!(result_layout.shape().dims(), &[6]);
        assert!(result_layout.is_contiguous());

        // Materialized transposed bytes flattened: [1, 4, 2, 5, 3, 6].
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        assert_eq!(
            c.as_slice::<f32>().unwrap(),
            &[1.0, 4.0, 2.0, 5.0, 3.0, 6.0]
        );
    }

    /// E2E: Const + Transpose + SumDim — exercises stage 3
    /// (metadata-only Transpose, strided intermediate Layout) +
    /// stage 4 (auto-Contiguize before reduce kernel) end-to-end.
    /// The transpose makes the intermediate non-contiguous; the
    /// reduce wrapper would have failed in stage 2; with stage 4's
    /// auto-Contiguize, the kernel sees the materialized contiguous
    /// transposed bytes and produces the right answer.
    #[test]
    fn pipelined_realize_transpose_then_sum_dim_e2e() {
        // shape [2, 3]: rows are [1, 2, 3], [4, 5, 6]
        // After transpose: shape [3, 2], rows are [1, 4], [2, 5], [3, 6]
        // After SumDim(1): shape [3], values [5, 7, 9]
        let storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, t_id, sum_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let t_id = g.push(Node {
                op: Op::Transpose,
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3, 2]),
                dtype: DType::F32,
            });
            let sum_id = g.push(Node {
                op: Op::SumDim(1),
                inputs: vec![t_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            // Only the reduce kernel runs on the backend; the
            // transpose is metadata-only and doesn't need a target.
            g.set_target_backend(sum_id, BackendId::Cpu);
            (in_id, t_id, sum_id)
        };
        let _ = t_id;
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));

        let (result_arc, result_layout) =
            PipelinedExecutor::realize(graph, sum_id, inputs).expect("realize");

        // The reduce output is contiguous (kernel-produced).
        assert_eq!(result_layout.shape().dims(), &[3]);
        assert!(result_layout.is_contiguous());

        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let typed: &[f32] = c.as_slice().unwrap();
        assert_eq!(typed, &[5.0, 7.0, 9.0]);
    }

    /// E2E: Const + BroadcastTo + Add — broadcast intermediate
    /// auto-contiguizes for the Add kernel; the result is the
    /// expected sum.
    #[test]
    fn pipelined_realize_broadcast_then_add_e2e() {
        // shape [3]: [10, 20, 30]
        // BroadcastTo [2, 3]: [[10, 20, 30], [10, 20, 30]]
        // Plus shape [2, 3]: [[1, 2, 3], [4, 5, 6]]
        // Result: [[11, 22, 33], [14, 25, 36]]
        let bc_input = fuel_memory::from_slice_cpu(&[10.0_f32, 20.0, 30.0]);
        let plus_input = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (bc_in_id, plus_in_id, b_id, add_id) = {
            let mut g = graph.write().unwrap();
            let bc_in = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let plus_in = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let b = g.push(Node {
                op: Op::BroadcastTo(Shape::from_dims(&[2, 3])),
                inputs: vec![bc_in],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let add = g.push(Node {
                op: Op::Add,
                inputs: vec![b, plus_in],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            g.set_target_backend(add, BackendId::Cpu);
            (bc_in, plus_in, b, add)
        };
        let _ = b_id;
        let mut inputs = StorageCache::new();
        inputs.insert(bc_in_id, Arc::new(RwLock::new(bc_input)));
        inputs.insert(plus_in_id, Arc::new(RwLock::new(plus_input)));

        let (result_arc, _result_layout) =
            PipelinedExecutor::realize(graph, add_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let typed: &[f32] = c.as_slice().unwrap();
        assert_eq!(typed, &[11.0, 22.0, 33.0, 14.0, 25.0, 36.0]);
    }

    /// E2E: Const + SumDim — verifies that `OpParams::Reduce`
    /// flows from the graph (input shape via `op_to_op_params`)
    /// through compile_one and reaches the reduce kernel.
    #[test]
    fn pipelined_realize_sum_dim() {
        // shape [2, 3]; reduce dim 1 → output shape [2]
        let storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, sum_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let sum_id = g.push(Node {
                op: Op::SumDim(1),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            g.set_target_backend(sum_id, BackendId::Cpu);
            (in_id, sum_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));

        let (result_arc, _) = PipelinedExecutor::realize(graph, sum_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        if let fuel_memory::BackendStorage::Cpu(c) = &guard.inner {
            let typed: &[f32] = c.as_slice().unwrap();
            assert_eq!(typed, &[6.0, 15.0]);
        }
    }

    /// E2E: SumAll on a rank-3 input, exercising the all-dims branch
    /// of `op_to_op_params` (every dim reduced, rank-0 output).
    #[test]
    fn pipelined_realize_sum_all_rank3() {
        let data: Vec<f32> = (1..=24).map(|x| x as f32).collect();
        let storage = fuel_memory::from_slice_cpu(&data);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, sum_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3, 4]),
                dtype: DType::F32,
            });
            let sum_id = g.push(Node {
                op: Op::SumAll,
                inputs: vec![in_id],
                shape: Shape::from_dims(&[]),
                dtype: DType::F32,
            });
            g.set_target_backend(sum_id, BackendId::Cpu);
            (in_id, sum_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));

        let (result_arc, _) = PipelinedExecutor::realize(graph, sum_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        if let fuel_memory::BackendStorage::Cpu(c) = &guard.inner {
            let typed: &[f32] = c.as_slice().unwrap();
            // 1 + 2 + ... + 24 = 300
            assert_eq!(typed, &[300.0]);
        }
    }

    /// E2E: MaxDim + MeanDim chained — verifies all four reduce
    /// OpKinds reach their wrappers via the OpParams plumbing.
    #[test]
    fn pipelined_realize_max_then_mean() {
        // shape [2, 3], MaxDim(1) → [2], MeanDim(0) → []
        let storage = fuel_memory::from_slice_cpu(&[1.0_f32, 9.0, 3.0, 4.0, 2.0, 8.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, mean_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let max_id = g.push(Node {
                op: Op::MaxDim(1),
                inputs: vec![in_id],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let mean_id = g.push(Node {
                op: Op::MeanDim(0),
                inputs: vec![max_id],
                shape: Shape::from_dims(&[]),
                dtype: DType::F32,
            });
            g.set_target_backend(max_id, BackendId::Cpu);
            g.set_target_backend(mean_id, BackendId::Cpu);
            (in_id, mean_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));

        let (result_arc, _) = PipelinedExecutor::realize(graph, mean_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        if let fuel_memory::BackendStorage::Cpu(c) = &guard.inner {
            let typed: &[f32] = c.as_slice().unwrap();
            // MaxDim(1) on [[1,9,3],[4,2,8]] = [9, 8]; MeanDim(0) = 8.5
            assert_eq!(typed, &[8.5]);
        }
    }

    /// E2E: Sigmoid + Silu — exercises two of the more compositional
    /// new unary kernels through the pipelined executor. Verifies
    /// the additional `op_to_op_kind` mappings reach the right
    /// dispatch wrappers.
    #[test]
    fn pipelined_realize_sigmoid_then_silu() {
        let storage = fuel_memory::from_slice_cpu(&[0.0_f32, 1.0, -1.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, sig_id, silu_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let sig_id = g.push(Node {
                op: Op::Sigmoid,
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            // Silu of the sigmoid output — chains to confirm cache flow.
            let silu_id = g.push(Node {
                op: Op::Silu,
                inputs: vec![sig_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            g.set_target_backend(sig_id, BackendId::Cpu);
            g.set_target_backend(silu_id, BackendId::Cpu);
            (in_id, sig_id, silu_id)
        };
        let _ = sig_id;
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));

        let (result_arc, _) = PipelinedExecutor::realize(graph, silu_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        if let fuel_memory::BackendStorage::Cpu(c) = &guard.inner {
            let typed: &[f32] = c.as_slice().unwrap();
            // sigmoid(0) = 0.5; silu(0.5) = 0.5 * sigmoid(0.5) ≈ 0.3112
            assert!((typed[0] - 0.5 * (1.0 / (1.0 + (-0.5_f32).exp()))).abs() < 1e-6);
        }
    }

    /// E2E: chained unary ops — Const + Sqr + Sqrt should be a noop
    /// for non-negative inputs. Exercises the cache reuse path.
    #[test]
    fn pipelined_realize_chained_unary_sqr_then_sqrt() {
        let storage = fuel_memory::from_slice_cpu(&[1.0_f32, 4.0, 9.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, sqrt_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let sqr_id = g.push(Node {
                op: Op::Sqr,
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let sqrt_id = g.push(Node {
                op: Op::Sqrt,
                inputs: vec![sqr_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            g.set_target_backend(sqr_id, BackendId::Cpu);
            g.set_target_backend(sqrt_id, BackendId::Cpu);
            (in_id, sqrt_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));

        let (result_arc, _) = PipelinedExecutor::realize(graph, sqrt_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        if let fuel_memory::BackendStorage::Cpu(c) = &guard.inner {
            let typed: &[f32] = c.as_slice().unwrap();
            // sqrt(sqr(x)) == |x| == x for non-negative inputs.
            assert_eq!(typed, &[1.0, 4.0, 9.0]);
        }
    }

    /// Multi-stage pipelined: Const + Const + Add + Add (chain of
    /// two adds). Tests that work items are processed in topo order
    /// and intermediate results are cached and reused.
    #[test]
    fn pipelined_realize_chained_adds() {
        let a_storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0]);
        let b_storage = fuel_memory::from_slice_cpu(&[10.0_f32, 20.0]);
        let c_storage = fuel_memory::from_slice_cpu(&[100.0_f32, 200.0]);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (a_id, b_id, c_id, ab_id, abc_id) = {
            let mut g = graph.write().unwrap();
            let a = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let b = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let c = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let ab = g.push(Node {
                op: Op::Add,
                inputs: vec![a, b],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let abc = g.push(Node {
                op: Op::Add,
                inputs: vec![ab, c],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            g.set_target_backend(ab, BackendId::Cpu);
            g.set_target_backend(abc, BackendId::Cpu);
            (a, b, c, ab, abc)
        };

        let mut inputs = StorageCache::new();
        inputs.insert(a_id, Arc::new(RwLock::new(a_storage)));
        inputs.insert(b_id, Arc::new(RwLock::new(b_storage)));
        inputs.insert(c_id, Arc::new(RwLock::new(c_storage)));

        let (result_arc, _) = PipelinedExecutor::realize(graph, abc_id, inputs).expect("realize");
        // Suppress unused warning for the intermediate id.
        let _ = ab_id;

        let guard = result_arc.read().unwrap();
        if let fuel_memory::BackendStorage::Cpu(c) = &guard.inner {
            let typed: &[f32] = c.as_slice().unwrap();
            // (1+10) + 100 = 111;  (2+20) + 200 = 222.
            assert_eq!(typed, &[111.0, 222.0]);
        }
    }

    /// E2E: triu(diagonal=0) on a 3×3 matrix.
    /// Input:  [1 2 3 / 4 5 6 / 7 8 9]
    /// Output: [1 2 3 / 0 5 6 / 0 0 9]
    #[test]
    fn pipelined_realize_triu_3x3_diag0() {
        let storage =
            fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, out_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3, 3]),
                dtype: DType::F32,
            });
            let out_id = g.push(Node {
                op: Op::Triu { diagonal: 0 },
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3, 3]),
                dtype: DType::F32,
            });
            g.set_target_backend(out_id, BackendId::Cpu);
            (in_id, out_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, out_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let out: &[f32] = c.as_slice().unwrap();
        assert_eq!(out, &[1.0, 2.0, 3.0, 0.0, 5.0, 6.0, 0.0, 0.0, 9.0,]);
    }

    /// E2E: tril(diagonal=0) on a 3×3 matrix — the canonical causal mask.
    /// Output: [1 0 0 / 4 5 0 / 7 8 9]
    #[test]
    fn pipelined_realize_tril_3x3_diag0_causal_mask() {
        let storage =
            fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, out_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3, 3]),
                dtype: DType::F32,
            });
            let out_id = g.push(Node {
                op: Op::Tril { diagonal: 0 },
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3, 3]),
                dtype: DType::F32,
            });
            g.set_target_backend(out_id, BackendId::Cpu);
            (in_id, out_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, out_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let out: &[f32] = c.as_slice().unwrap();
        assert_eq!(out, &[1.0, 0.0, 0.0, 4.0, 5.0, 0.0, 7.0, 8.0, 9.0,]);
    }

    /// E2E: log_softmax over a 2×3 input. Rows are [1,2,3] and [3,2,1].
    /// log_softmax(row) = row - max - log(sum exp(row - max)).
    /// Row 0: max=3, exp(-2)+exp(-1)+exp(0) = 0.135+0.368+1.0 = 1.503;
    ///        log(1.503) ≈ 0.4076; out = [1-3-0.4076, 2-3-0.4076, 3-3-0.4076]
    ///                              = [-2.4076, -1.4076, -0.4076]
    /// Row 1 is row 0 reversed: [-0.4076, -1.4076, -2.4076].
    #[test]
    fn pipelined_realize_log_softmax_last_dim() {
        let storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 3.0, 2.0, 1.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, out_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let out_id = g.push(Node {
                op: Op::LogSoftmaxLastDim,
                inputs: vec![in_id],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            g.set_target_backend(out_id, BackendId::Cpu);
            (in_id, out_id)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, out_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let out: &[f32] = c.as_slice().unwrap();
        let expected = [
            -2.407_606,
            -1.4076059,
            -0.40760595_f32,
            -0.40760595,
            -1.4076059,
            -2.407_606,
        ];
        for (a, b) in out.iter().zip(expected.iter()) {
            assert!((a - b).abs() < 1e-5, "log_softmax mismatch: {a} vs {b}");
        }
        // The exp() of the output should sum to 1 per row.
        for row in out.chunks(3) {
            let sum: f32 = row.iter().map(|x| x.exp()).sum();
            assert!(
                (sum - 1.0).abs() < 1e-5,
                "softmax(log_softmax) row sum != 1: {sum}"
            );
        }
    }

    // --- side-effect roots + destructive cleanup (9c Phase B) -----

    /// Side-effect roots get merged into the realize walk even when
    /// not reachable from the user's targets. The production trigger
    /// is `Op::Release` (emitted by `ResidencyEvictionRule`); this
    /// test uses that exact shape.
    ///
    /// Graph: Const(a) + Const(b) → Add (user target); separate
    /// Const(c) → Release(c) (side-effect root, not reachable from
    /// Add). After realize_many on `[add]`, the Add should be in the
    /// output AND the Release should have fired (verified by the
    /// fact that realize succeeded — Op::Release in the walk used to
    /// fail compilation pre-Phase B+).
    #[test]
    fn pipelined_realize_merges_op_release_side_effect_root() {
        let a = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0]);
        let b = fuel_memory::from_slice_cpu(&[10.0_f32, 20.0]);
        let c = fuel_memory::from_slice_cpu(&[100.0_f32, 200.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (a_id, b_id, c_id, add_id, release_id) = {
            let mut g = graph.write().unwrap();
            let a = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let b = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            // c is NOT reachable from add; its Release IS marked as
            // a side-effect root.
            let c = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let add = g.push(Node {
                op: Op::Add,
                inputs: vec![a, b],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            // Production shape: Op::Release on c, marked as
            // side-effect root so it fires even though no graph
            // node reads its output.
            let release = g.push(Node {
                op: Op::Release,
                inputs: vec![c],
                shape: Shape::from_dims(&[0]),
                dtype: DType::F32,
            });
            g.set_target_backend(add, BackendId::Cpu);
            g.set_target_backend(release, BackendId::Cpu);
            g.add_side_effect_root(release);
            (a, b, c, add, release)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(a_id, Arc::new(RwLock::new(a)));
        inputs.insert(b_id, Arc::new(RwLock::new(b)));
        let c_arc = Arc::new(RwLock::new(c));
        let c_arc_external = Arc::clone(&c_arc);
        inputs.insert(c_id, c_arc);

        // Only `add` is requested. The Release on c is a side-effect
        // root that must still fire.
        let out = PipelinedExecutor::realize_many(graph, &[add_id], inputs).expect("realize_many");
        assert_eq!(out.len(), 1);
        let add_guard = out[0].0.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(arr) = &add_guard.inner else {
            panic!()
        };
        assert_eq!(arr.as_slice::<f32>().unwrap(), &[11.0, 22.0]);

        // The Release's destructive_input cleanup dropped the
        // cache's Arc to c. The only Arc to c that survives is the
        // external one this test holds — confirms the eviction
        // actually freed the cache's reference.
        assert_eq!(
            Arc::strong_count(&c_arc_external),
            1,
            "Op::Release should evict the cache's Arc to its source; \
             only the test's external clone should remain",
        );
        let _ = release_id;
    }

    /// `compile_one` emits a `WorkItemKind::ReleaseMarker` for
    /// `Op::Release` with `destructive_input = Some(0)`. Verify
    /// the WorkItem shape via a direct compile_one call.
    #[test]
    fn compile_one_emits_release_marker_with_destructive_input() {
        let graph_rc = Arc::new(RwLock::new(Graph::new()));
        let (src_id, release_id) = {
            let mut g = graph_rc.write().unwrap();
            let src = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let release = g.push(Node {
                op: Op::Release,
                inputs: vec![src],
                shape: Shape::from_dims(&[0]),
                dtype: DType::F32,
            });
            g.set_target_backend(release, BackendId::Cpu);
            (src, release)
        };
        let g = graph_rc.read().unwrap();
        let bindings = crate::dispatch::global_bindings();
        let mut layout_cache: HashMap<NodeId, Layout> = HashMap::new();
        let item = compile_one(
            &g,
            release_id,
            &mut layout_cache,
            &bindings,
            &SymEnv::default(),
        )
        .expect("compile_one Op::Release");
        assert!(matches!(item.kind, WorkItemKind::ReleaseMarker));
        assert_eq!(item.destructive_input, Some(0));
        assert_eq!(item.inputs, vec![src_id]);
        assert_eq!(
            item.elem_count, 0,
            "Release output is the zero-element marker"
        );
    }

    // --- Op::Move (executor-unification Session 1, gap 13) --------

    /// `compile_one` emits a `WorkItemKind::Move` for `Op::Move` with
    /// `destructive_input = Some(0)` and a resolved transfer kernel
    /// (binding-table lookup at `(OpKind::Copy, [dt, dt], source
    /// backend)` — Move shares Op::Copy's data-movement kernel).
    #[test]
    fn compile_one_emits_move_with_destructive_input() {
        let graph_rc = Arc::new(RwLock::new(Graph::new()));
        let (src_id, move_id) = {
            let mut g = graph_rc.write().unwrap();
            let src = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let mv = g.push(Node {
                op: Op::Move {
                    target: DeviceLocation::Cpu,
                },
                inputs: vec![src],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            g.set_target_backend(mv, BackendId::Cpu);
            (src, mv)
        };
        let g = graph_rc.read().unwrap();
        let bindings = crate::dispatch::global_bindings();
        let mut layout_cache: HashMap<NodeId, Layout> = HashMap::new();
        let item = compile_one(
            &g,
            move_id,
            &mut layout_cache,
            &bindings,
            &SymEnv::default(),
        )
        .expect("compile_one Op::Move");
        assert!(matches!(
            item.kind,
            WorkItemKind::Move {
                target_location: DeviceLocation::Cpu
            },
        ));
        assert_eq!(item.destructive_input, Some(0), "Move destroys its source");
        assert_eq!(item.inputs, vec![src_id]);
        assert!(
            item.compiled.is_some(),
            "Move resolves a transfer kernel at (OpKind::Copy, [dt, dt], source backend)",
        );
    }

    // --- runtime fused op (JIT-adopted Tier-2 kernel) --------------

    /// `compile_one` resolves an `Op::Fused(fid, Runtime)` node with
    /// `fid.is_runtime()` from the runtime-kernel sidecar (never the
    /// binding table): `WorkItemKind::Kernel`, `compiled.op ==
    /// OpKind::RuntimeFused`, and the kernel pointer is exactly the one
    /// `adopt_runtime_fused` bound.
    #[test]
    fn compile_one_resolves_runtime_fused_from_the_sidecar() {
        fn sentinel_kernel(
            _inputs: &[Arc<RwLock<fuel_memory::Storage>>],
            _outputs: &mut [Arc<RwLock<fuel_memory::Storage>>],
            _layouts: &[Layout],
            _params: &crate::kernel::OpParams,
        ) -> Result<()> {
            Ok(())
        }
        // A region shape unique to this test (process-global sidecar; see
        // runtime_fused_pathfinder's tests for the collision rationale).
        let region = fuel_graph::jit::PatternNode::Op {
            op: fuel_graph::jit::OpTag::Exp,
            attrs: fuel_graph::jit::OpAttrs::default(),
            operands: vec![fuel_graph::jit::PatternNode::Op {
                op: fuel_graph::jit::OpTag::Sub,
                attrs: fuel_graph::jit::OpAttrs::default(),
                operands: vec![
                    fuel_graph::jit::PatternNode::Bind { index: 0 },
                    fuel_graph::jit::PatternNode::Bind { index: 1 },
                ],
            }],
        };
        let rid = crate::runtime_fused_kernels::adopt_runtime_fused(
            "test::compile_one::exp_sub",
            region,
            sentinel_kernel as crate::kernel::KernelRef,
            vec![DType::F32, DType::F32, DType::F32],
            BackendId::Cpu,
        )
        .expect("registrable region");

        let graph_rc = Arc::new(RwLock::new(Graph::new()));
        let fused_id = {
            let mut g = graph_rc.write().unwrap();
            let s = Shape::from_dims(&[4]);
            let a = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: s.clone(),
                dtype: DType::F32,
            });
            let b = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: s.clone(),
                dtype: DType::F32,
            });
            let fused = g.push(Node {
                op: Op::Fused(
                    rid,
                    fuel_graph::registry::FusedOpParams::Runtime {
                        scalars: Vec::new(),
                    },
                ),
                inputs: vec![a, b],
                shape: s,
                dtype: DType::F32,
            });
            // The arm emitter pins the backend; mirror that here.
            g.set_target_backend(fused, BackendId::Cpu);
            fused
        };
        let g = graph_rc.read().unwrap();
        let bindings = crate::dispatch::global_bindings();
        let mut layout_cache: HashMap<NodeId, Layout> = HashMap::new();
        let item = compile_one(
            &g,
            fused_id,
            &mut layout_cache,
            &bindings,
            &SymEnv::default(),
        )
        .expect("compile_one runtime fused op");
        assert!(matches!(item.kind, WorkItemKind::Kernel));
        let compiled = item.compiled.expect("resolved a kernel");
        assert_eq!(compiled.op, fuel_ir::dispatch::OpKind::RuntimeFused);
        assert_eq!(compiled.backend, BackendId::Cpu);
        assert!(
            std::ptr::fn_addr_eq(compiled.kernel, sentinel_kernel as crate::kernel::KernelRef),
            "the kernel pointer is exactly the adopted one",
        );
        assert!(
            layout_cache.contains_key(&fused_id),
            "output layout cached for consumers"
        );
    }

    /// The dtype belt-and-suspenders guard: an `Op::Fused(runtime)` node
    /// whose per-operand dtypes disagree with the adopted kernel's tuple is
    /// a typed error, not a silent wrong-kernel dispatch.
    #[test]
    fn compile_one_rejects_runtime_fused_dtype_mismatch() {
        fn noop_kernel(
            _inputs: &[Arc<RwLock<fuel_memory::Storage>>],
            _outputs: &mut [Arc<RwLock<fuel_memory::Storage>>],
            _layouts: &[Layout],
            _params: &crate::kernel::OpParams,
        ) -> Result<()> {
            Ok(())
        }
        let region = fuel_graph::jit::PatternNode::Op {
            op: fuel_graph::jit::OpTag::Log,
            attrs: fuel_graph::jit::OpAttrs::default(),
            operands: vec![fuel_graph::jit::PatternNode::Op {
                op: fuel_graph::jit::OpTag::Maximum,
                attrs: fuel_graph::jit::OpAttrs::default(),
                operands: vec![
                    fuel_graph::jit::PatternNode::Bind { index: 0 },
                    fuel_graph::jit::PatternNode::Bind { index: 1 },
                ],
            }],
        };
        // Adopted for F32 operands…
        let rid = crate::runtime_fused_kernels::adopt_runtime_fused(
            "test::compile_one::log_maximum",
            region,
            noop_kernel as crate::kernel::KernelRef,
            vec![DType::F32, DType::F32, DType::F32],
            BackendId::Cpu,
        )
        .expect("registrable region");

        let graph_rc = Arc::new(RwLock::new(Graph::new()));
        let fused_id = {
            let mut g = graph_rc.write().unwrap();
            let s = Shape::from_dims(&[4]);
            // …but the node's operands are F64.
            let a = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: s.clone(),
                dtype: DType::F64,
            });
            let b = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: s.clone(),
                dtype: DType::F64,
            });
            let fused = g.push(Node {
                op: Op::Fused(
                    rid,
                    fuel_graph::registry::FusedOpParams::Runtime {
                        scalars: Vec::new(),
                    },
                ),
                inputs: vec![a, b],
                shape: s,
                dtype: DType::F64,
            });
            g.set_target_backend(fused, BackendId::Cpu);
            fused
        };
        let g = graph_rc.read().unwrap();
        let bindings = crate::dispatch::global_bindings();
        let mut layout_cache: HashMap<NodeId, Layout> = HashMap::new();
        let err = match compile_one(
            &g,
            fused_id,
            &mut layout_cache,
            &bindings,
            &SymEnv::default(),
        ) {
            Ok(_) => panic!("dtype mismatch must be a typed error"),
            Err(e) => e,
        };
        // Post-fold, the dtype tuple IS the binding key: a mismatch is the
        // same honest `NoBackendForOp` miss as an absent kernel — reported
        // with the `RuntimeFused` diagnostic op tag, never a wrong-kernel bind.
        assert!(
            matches!(
                err,
                fuel_ir::Error::NoBackendForOp {
                    op: fuel_ir::dispatch::OpKind::RuntimeFused,
                    ..
                }
            ),
            "a dtype mismatch is an honest binding miss: {err}",
        );
    }

    /// A runtime fused node carrying live `Runtime { scalars }` compiles with
    /// `OpParams::JitScalars` — the launch channel for the kernel's trailing
    /// `p{i}` args. (A param-free node keeps `OpParams::None`, covered by
    /// `compile_one_resolves_runtime_fused_from_the_sidecar`.)
    #[test]
    fn compile_one_maps_runtime_scalars_to_jit_scalars() {
        fn noop_kernel(
            _inputs: &[Arc<RwLock<fuel_memory::Storage>>],
            _outputs: &mut [Arc<RwLock<fuel_memory::Storage>>],
            _layouts: &[Layout],
            _params: &crate::kernel::OpParams,
        ) -> Result<()> {
            Ok(())
        }
        // Unique slot-template region (process-global sidecar).
        let region = fuel_graph::jit::PatternNode::Op {
            op: fuel_graph::jit::OpTag::Floor,
            attrs: fuel_graph::jit::OpAttrs::default(),
            operands: vec![fuel_graph::jit::PatternNode::Op {
                op: fuel_graph::jit::OpTag::MulScalar,
                attrs: fuel_graph::jit::OpAttrs::default(), // open slot
                operands: vec![fuel_graph::jit::PatternNode::Bind { index: 0 }],
            }],
        };
        let rid = crate::runtime_fused_kernels::adopt_runtime_fused(
            "test::compile_one::floor_mul_scalar",
            region,
            noop_kernel as crate::kernel::KernelRef,
            vec![DType::F32, DType::F32],
            BackendId::Cpu,
        )
        .expect("slot template registrable");

        let graph_rc = Arc::new(RwLock::new(Graph::new()));
        let fused_id = {
            let mut g = graph_rc.write().unwrap();
            let s = Shape::from_dims(&[4]);
            let x = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: s.clone(),
                dtype: DType::F32,
            });
            let fused = g.push(Node {
                op: Op::Fused(
                    rid,
                    fuel_graph::registry::FusedOpParams::Runtime {
                        scalars: vec![3.5, -1.25],
                    },
                ),
                inputs: vec![x],
                shape: s,
                dtype: DType::F32,
            });
            g.set_target_backend(fused, BackendId::Cpu);
            fused
        };
        let g = graph_rc.read().unwrap();
        let bindings = crate::dispatch::global_bindings();
        let mut layout_cache: HashMap<NodeId, Layout> = HashMap::new();
        let item = compile_one(
            &g,
            fused_id,
            &mut layout_cache,
            &bindings,
            &SymEnv::default(),
        )
        .expect("compile_one runtime fused op with scalars");
        let compiled = item.compiled.expect("resolved a kernel");
        assert!(
            matches!(
                &compiled.op_params,
                crate::kernel::OpParams::JitScalars { scalars } if scalars == &vec![3.5, -1.25]
            ),
            "Runtime scalars ride into OpParams::JitScalars, got {:?}",
            compiled.op_params,
        );
    }

    /// Move-to-same-device is the degenerate case: per the legacy
    /// `GraphExecutor` contract (`Op::Copy | Op::Move` shared arm →
    /// `backend.copy_to`) it is a plain copy — fresh storage on the
    /// same device, data intact — and the source is still evicted
    /// afterward (destructive semantics don't depend on the devices
    /// differing).
    #[test]
    fn pipelined_op_move_same_device_is_plain_copy() {
        let src = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (src_id, move_id) = {
            let mut g = graph.write().unwrap();
            let src = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            let mv = g.push(Node {
                op: Op::Move {
                    target: DeviceLocation::Cpu,
                },
                inputs: vec![src],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            g.set_target_backend(mv, BackendId::Cpu);
            (src, mv)
        };
        let src_arc = Arc::new(RwLock::new(src));
        let src_arc_external = Arc::clone(&src_arc);
        let mut inputs = StorageCache::new();
        inputs.insert(src_id, src_arc);

        let (out, layout) =
            PipelinedExecutor::realize(graph, move_id, inputs).expect("realize Op::Move");
        let guard = out.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!()
        };
        assert_eq!(
            c.as_slice::<f32>().unwrap(),
            &[1.0, 2.0, 3.0, 4.0],
            "Op::Move output should carry input's data to target",
        );
        assert!(layout.is_contiguous());
        drop(guard);
        assert!(
            !Arc::ptr_eq(&out, &src_arc_external),
            "same-device Move is a plain copy — fresh storage, not an alias",
        );
        // The realize loop's destructive_input cleanup dropped the
        // cache's Arc to the source; only the test's external clone
        // survives.
        assert_eq!(
            Arc::strong_count(&src_arc_external),
            1,
            "Op::Move should evict the cache's Arc to its source",
        );
    }

    /// The multiple-consumer-source case: a Move must NOT strand
    /// another consumer of its source. `execution_plan` (via
    /// `derive_ordering`, keyed off `destructive_input`) pins the
    /// Move AFTER the sibling reader; the executor's post-Move cache
    /// eviction then can't break the reader. Mirrors the legacy
    /// `op_move_pinned_after_sibling_reader_via_derive_ordering`
    /// test in fuel-graph-router/tests/cross_device.rs.
    ///
    ///   a   = const [1, 2, 3, 4]
    ///   b   = relu(a)        (non-destructive sibling reader)
    ///   m   = move(a, Cpu)   (destructive — evicts a once run)
    ///   out = add(b, m)      → [2, 4, 6, 8]
    #[test]
    fn pipelined_op_move_multi_consumer_source_not_stranded() {
        let a = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (a_id, out_id) = {
            let mut g = graph.write().unwrap();
            let a = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            let b = g.push(Node {
                op: Op::Relu,
                inputs: vec![a],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            let m = g.push(Node {
                op: Op::Move {
                    target: DeviceLocation::Cpu,
                },
                inputs: vec![a],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            let out = g.push(Node {
                op: Op::Add,
                inputs: vec![b, m],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            g.set_target_backend(b, BackendId::Cpu);
            g.set_target_backend(m, BackendId::Cpu);
            g.set_target_backend(out, BackendId::Cpu);
            (a, out)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(a_id, Arc::new(RwLock::new(a)));

        let (out, _) = PipelinedExecutor::realize(graph, out_id, inputs)
            .expect("realize add(relu(a), move(a))");
        let guard = out.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!()
        };
        assert_eq!(
            c.as_slice::<f32>().unwrap(),
            &[2.0, 4.0, 6.0, 8.0],
            "relu(a) must run before move(a) evicts a",
        );
    }

    /// A Move whose source is itself in the realize target set must
    /// not evict it — the caller asked for the source's storage. The
    /// destructive cleanup's `target_set` gate covers Move exactly
    /// like Release.
    #[test]
    fn pipelined_op_move_source_in_target_set_not_evicted() {
        let a = fuel_memory::from_slice_cpu(&[5.0_f32, 6.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (a_id, move_id) = {
            let mut g = graph.write().unwrap();
            let a = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let mv = g.push(Node {
                op: Op::Move {
                    target: DeviceLocation::Cpu,
                },
                inputs: vec![a],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            g.set_target_backend(mv, BackendId::Cpu);
            (a, mv)
        };
        let a_arc = Arc::new(RwLock::new(a));
        let a_arc_external = Arc::clone(&a_arc);
        let mut inputs = StorageCache::new();
        inputs.insert(a_id, a_arc);

        let out = PipelinedExecutor::realize_many(graph, &[move_id, a_id], inputs)
            .expect("realize_many [move, source]");
        assert_eq!(out.len(), 2);
        let mv_guard = out[0].0.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &mv_guard.inner else {
            panic!()
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[5.0, 6.0]);
        assert!(
            Arc::ptr_eq(&out[1].0, &a_arc_external),
            "requested source must survive the Move's destructive cleanup",
        );
    }

    // --- realize_many (Phase 7.6 step 9c Phase A) ----------------

    /// realize_many on empty targets returns an empty Vec — no graph
    /// walk, no panic.
    #[test]
    fn pipelined_realize_many_empty_targets() {
        let graph = Arc::new(RwLock::new(Graph::new()));
        let inputs = StorageCache::new();
        let out = PipelinedExecutor::realize_many(graph, &[], inputs).expect("realize_many");
        assert!(out.is_empty());
    }

    /// realize_many with two independent target chains. Each chain's
    /// output is in the cache; the shared topo walk realizes both.
    /// Verifies parallel chain handling + return order matches input.
    #[test]
    fn pipelined_realize_many_two_independent_targets() {
        let lhs = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0]);
        let rhs = fuel_memory::from_slice_cpu(&[10.0_f32, 20.0, 30.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (lhs_id, rhs_id, add_id, mul_id) = {
            let mut g = graph.write().unwrap();
            let lhs = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let rhs = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let add = g.push(Node {
                op: Op::Add,
                inputs: vec![lhs, rhs],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let mul = g.push(Node {
                op: Op::Mul,
                inputs: vec![lhs, rhs],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            g.set_target_backend(add, BackendId::Cpu);
            g.set_target_backend(mul, BackendId::Cpu);
            (lhs, rhs, add, mul)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(lhs_id, Arc::new(RwLock::new(lhs)));
        inputs.insert(rhs_id, Arc::new(RwLock::new(rhs)));

        // Pass mul first to confirm return order matches targets order
        // (not graph order — graph order would put add first).
        let out = PipelinedExecutor::realize_many(graph, &[mul_id, add_id], inputs)
            .expect("realize_many");
        assert_eq!(out.len(), 2);

        let mul_guard = out[0].0.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &mul_guard.inner else {
            panic!()
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[10.0, 40.0, 90.0]);

        let add_guard = out[1].0.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &add_guard.inner else {
            panic!()
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[11.0, 22.0, 33.0]);
    }

    /// realize_many with the same NodeId twice — caller asking for
    /// the same output twice. Both outputs are the same Arc (cheap).
    #[test]
    fn pipelined_realize_many_duplicate_target_returns_same_arc() {
        let storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (in_id, neg_id) = {
            let mut g = graph.write().unwrap();
            let in_id = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let neg = g.push(Node {
                op: Op::Neg,
                inputs: vec![in_id],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            g.set_target_backend(neg, BackendId::Cpu);
            (in_id, neg)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(in_id, Arc::new(RwLock::new(storage)));

        let out = PipelinedExecutor::realize_many(graph, &[neg_id, neg_id], inputs)
            .expect("realize_many");
        assert_eq!(out.len(), 2);
        assert!(
            Arc::ptr_eq(&out[0].0, &out[1].0),
            "duplicate target should share the same storage Arc",
        );
    }

    /// realize_many with shared upstream nodes evaluates the shared
    /// chunk exactly once. Two targets `f = a + b` and `g = (a + b) * a`
    /// — `a + b` is computed once.
    #[test]
    fn pipelined_realize_many_shared_subgraph_evaluates_once() {
        let a_storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0]);
        let b_storage = fuel_memory::from_slice_cpu(&[10.0_f32, 20.0, 30.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (a_id, b_id, add_id, mul_id) = {
            let mut g = graph.write().unwrap();
            let a = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let b = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let add = g.push(Node {
                op: Op::Add,
                inputs: vec![a, b],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            let mul = g.push(Node {
                op: Op::Mul,
                inputs: vec![add, a],
                shape: Shape::from_dims(&[3]),
                dtype: DType::F32,
            });
            g.set_target_backend(add, BackendId::Cpu);
            g.set_target_backend(mul, BackendId::Cpu);
            (a, b, add, mul)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(a_id, Arc::new(RwLock::new(a_storage)));
        inputs.insert(b_id, Arc::new(RwLock::new(b_storage)));

        let out = PipelinedExecutor::realize_many(graph, &[add_id, mul_id], inputs)
            .expect("realize_many");
        assert_eq!(out.len(), 2);

        let add_guard = out[0].0.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &add_guard.inner else {
            panic!()
        };
        assert_eq!(c.as_slice::<f32>().unwrap(), &[11.0, 22.0, 33.0]);

        let mul_guard = out[1].0.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &mul_guard.inner else {
            panic!()
        };
        // (a + b) * a = [11*1, 22*2, 33*3] = [11, 44, 99].
        assert_eq!(c.as_slice::<f32>().unwrap(), &[11.0, 44.0, 99.0]);
    }

    /// E2E: masked_fill with -inf — the attention masking pattern.
    /// x = [1, 2, 3, 4]; mask = [0, 1, 0, 1]; value = -1000.
    /// out = [1, -1000, 3, -1000].
    #[test]
    fn pipelined_realize_masked_fill_attention_pattern() {
        let x_storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0]);
        let mut mask_storage = fuel_memory::from_slice_cpu(&[0u8, 1, 0, 1]);
        mask_storage.dtype = DType::Bool; // GAP-168(c): a Bool mask, not U8 bytes
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (x_id, mask_id, out_id) = {
            let mut g = graph.write().unwrap();
            let x = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            let mask = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[4]),
                dtype: DType::Bool,
            });
            let out = g.push(Node {
                op: Op::MaskedFill {
                    value: fuel_ir::Scalar::F32(-1000.0),
                },
                inputs: vec![x, mask],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            g.set_target_backend(out, BackendId::Cpu);
            (x, mask, out)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(x_id, Arc::new(RwLock::new(x_storage)));
        inputs.insert(mask_id, Arc::new(RwLock::new(mask_storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, out_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let out: &[f32] = c.as_slice().unwrap();
        assert_eq!(out, &[1.0, -1000.0, 3.0, -1000.0]);
    }

    // ---- WriteSlice (Phase 7.6 step 9c E.3.2) -------------------------------

    /// E2E: pipelined realize of `Op::WriteSlice` writing a [1, 3, 2]
    /// source slab into a [4, 3, 2] destination at axis-0 row 2 — the
    /// canonical KV-cache append shape.
    #[test]
    fn pipelined_realize_write_slice_kv_cache_append() {
        let dest_storage = fuel_memory::from_slice_cpu(&[0.0_f32; 24]);
        let src_storage = fuel_memory::from_slice_cpu(&[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (dest_id, src_id, ws_id) = {
            let mut g = graph.write().unwrap();
            let dest = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[4, 3, 2]),
                dtype: DType::F32,
            });
            let src = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 3, 2]),
                dtype: DType::F32,
            });
            let ws = g.push(Node {
                op: Op::WriteSlice {
                    ranges: vec![(2, 3), (0, 3), (0, 2)],
                    dyn_offset: None,
                },
                inputs: vec![dest, src],
                shape: Shape::from_dims(&[4, 3, 2]), // adopts dest shape
                dtype: DType::F32,
            });
            g.set_target_backend(ws, BackendId::Cpu);
            (dest, src, ws)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(dest_id, Arc::new(RwLock::new(dest_storage)));
        inputs.insert(src_id, Arc::new(RwLock::new(src_storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, ws_id, inputs).expect("realize");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let out: &[f32] = c.as_slice().unwrap();
        let expected = [
            0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0,
            6.0, // row 2 = source
            0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
        ];
        assert_eq!(out, &expected);
    }

    /// E2E: pipelined realize of `Op::WriteSlice` adopts the
    /// destination's Storage Arc as the output slot (in-place
    /// alias) — the resulting Arc points to the SAME bytes the
    /// destination was constructed from.
    #[test]
    fn pipelined_realize_write_slice_aliases_dest_storage() {
        let dest_storage = fuel_memory::from_slice_cpu(&[0.0_f32; 6]);
        let src_storage = fuel_memory::from_slice_cpu(&[10.0_f32, 20.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (dest_id, src_id, ws_id, dest_arc_external) = {
            let mut g = graph.write().unwrap();
            let dest = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[3, 2]),
                dtype: DType::F32,
            });
            let src = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[1, 2]),
                dtype: DType::F32,
            });
            let ws = g.push(Node {
                op: Op::WriteSlice {
                    ranges: vec![(1, 2), (0, 2)],
                    dyn_offset: None,
                },
                inputs: vec![dest, src],
                shape: Shape::from_dims(&[3, 2]),
                dtype: DType::F32,
            });
            g.set_target_backend(ws, BackendId::Cpu);
            let arc = Arc::new(RwLock::new(dest_storage));
            (dest, src, ws, arc)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(dest_id, Arc::clone(&dest_arc_external));
        inputs.insert(src_id, Arc::new(RwLock::new(src_storage)));
        let (result_arc, _) = PipelinedExecutor::realize(graph, ws_id, inputs).expect("realize");
        // Bytes were written into the same Arc the caller held —
        // the WriteSlice op aliases dest's Storage Arc, not allocates
        // a fresh buffer.
        assert!(
            Arc::ptr_eq(&result_arc, &dest_arc_external),
            "WriteSlice output Arc must be the same Arc as the destination's"
        );
        let guard = dest_arc_external.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let out: &[f32] = c.as_slice().unwrap();
        assert_eq!(out, &[0.0, 0.0, 10.0, 20.0, 0.0, 0.0]);
    }

    // ---- WriteSlice dynamic offset (Phase D symbolic extents) ---------------

    /// E2E: pipelined realize of `Op::WriteSlice` with a runtime start
    /// offset (`dyn_offset`) resolved through a per-pass `SymEnv`. The
    /// width-2 source slab lands at the bound offset (3), not the static
    /// `ranges[0].0` placeholder (0) — the persistent-decode KV-cache
    /// write at a per-token `cached_len`.
    #[test]
    fn pipelined_realize_write_slice_dynamic_offset() {
        use fuel_ir::{DynScalar, SymId};
        let dest_storage = fuel_memory::from_slice_cpu(&[0.0_f32; 6]);
        let src_storage = fuel_memory::from_slice_cpu(&[7.0_f32, 8.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let sym = SymId(0);
        let (dest_id, src_id, ws_id) = {
            let mut g = graph.write().unwrap();
            let dest = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[6]),
                dtype: DType::F32,
            });
            let src = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let ws = g.push(Node {
                // ranges[0] = (0, 2): the start is a placeholder (ignored —
                // overridden by dyn_offset); only the width (2) is read.
                op: Op::WriteSlice {
                    ranges: vec![(0, 2)],
                    dyn_offset: Some((0, DynScalar::Sym(sym))),
                },
                inputs: vec![dest, src],
                shape: Shape::from_dims(&[6]),
                dtype: DType::F32,
            });
            g.set_target_backend(ws, BackendId::Cpu);
            (dest, src, ws)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(dest_id, Arc::new(RwLock::new(dest_storage)));
        inputs.insert(src_id, Arc::new(RwLock::new(src_storage)));

        // Bind cached_len = 3: the slab must land at indices [3, 4].
        let mut env = SymEnv::new();
        env.bind(sym, 3).unwrap();

        let (result_arc, _) = PipelinedExecutor::realize_with_env(graph, ws_id, inputs, env)
            .expect("realize_with_env");
        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let out: &[f32] = c.as_slice().unwrap();
        assert_eq!(
            out,
            &[0.0, 0.0, 0.0, 7.0, 8.0, 0.0],
            "slab must land at the SymEnv-bound offset 3, not the static placeholder 0",
        );
    }

    /// Increment 2a — the data-determined bind seam. After
    /// `Op::NonZeroIndices` realizes, the executor reads its runtime count
    /// from bundle slot 1 and PUBLISHES it into the produced `SymEnv` under
    /// `count_sym`. This is the first op to bind a *data-determined* symbol
    /// mid-realize (all prior binds are host-known before realize) — the
    /// keystone capability for SSM decode / MoE sparsity / MLA KV.
    #[test]
    fn pipelined_realize_nonzero_indices_publishes_count() {
        use fuel_ir::SymId;
        use fuel_ir::storage::{OutputViewSpec, compose_bundle};
        // flat [0,1,0,1,1,0] → 3 nonzeros at 1,3,4; capacity 6.
        let x_storage = fuel_memory::from_slice_cpu(&[0.0_f32, 1.0, 0.0, 1.0, 1.0, 0.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let count_sym = SymId(0);
        let (x_id, nzi_id) = {
            let mut g = graph.write().unwrap();
            let x = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2, 3]),
                dtype: DType::F32,
            });
            let capacity = 6usize;
            let idx_shape = Shape::from_dims(&[capacity]);
            let cnt_shape = Shape::from_dims(&[1]);
            let specs = vec![
                OutputViewSpec {
                    dtype: DType::U32,
                    shape: idx_shape.clone(),
                    layout: Layout::contiguous(idx_shape.clone()),
                    name: Some("indices"),
                },
                OutputViewSpec {
                    dtype: DType::U32,
                    shape: cnt_shape.clone(),
                    layout: Layout::contiguous(cnt_shape),
                    name: Some("count"),
                },
            ];
            let (_bytes, views) = compose_bundle(&specs).unwrap();
            let nzi = g.push(Node {
                op: Op::NonZeroIndices { count_sym },
                inputs: vec![x],
                shape: idx_shape,
                dtype: DType::U32,
            });
            g.set_output_views(nzi, Arc::from(views.into_boxed_slice()))
                .unwrap();
            g.set_target_backend(nzi, BackendId::Cpu);
            (x, nzi)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(x_id, Arc::new(RwLock::new(x_storage)));

        let (_arc, _layout, produced) =
            PipelinedExecutor::realize_with_env_producing(graph, nzi_id, inputs, SymEnv::new())
                .expect("realize_with_env_producing");

        assert_eq!(
            produced.get(count_sym),
            Some(3),
            "the executor must publish the data-determined nonzero count into the SymEnv",
        );
    }

    /// A `dyn_offset` whose symbol is unbound in the `SymEnv` surfaces a
    /// typed error at realize (never a panic).
    #[test]
    fn pipelined_realize_write_slice_dynamic_offset_unbound_errors() {
        use fuel_ir::{DynScalar, SymId};
        let dest_storage = fuel_memory::from_slice_cpu(&[0.0_f32; 6]);
        let src_storage = fuel_memory::from_slice_cpu(&[7.0_f32, 8.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let (dest_id, src_id, ws_id) = {
            let mut g = graph.write().unwrap();
            let dest = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[6]),
                dtype: DType::F32,
            });
            let src = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let ws = g.push(Node {
                op: Op::WriteSlice {
                    ranges: vec![(0, 2)],
                    dyn_offset: Some((0, DynScalar::Sym(SymId(0)))),
                },
                inputs: vec![dest, src],
                shape: Shape::from_dims(&[6]),
                dtype: DType::F32,
            });
            g.set_target_backend(ws, BackendId::Cpu);
            (dest, src, ws)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(dest_id, Arc::new(RwLock::new(dest_storage)));
        inputs.insert(src_id, Arc::new(RwLock::new(src_storage)));
        // Empty env — the symbol is unbound.
        let result = PipelinedExecutor::realize_with_env(graph, ws_id, inputs, SymEnv::new());
        assert!(
            result.is_err(),
            "unbound dyn_offset symbol must surface a typed error, not a panic",
        );
    }

    /// A runtime offset whose resolved slab runs past the destination
    /// capacity errors at realize (never an out-of-bounds write).
    #[test]
    fn pipelined_realize_write_slice_dynamic_offset_out_of_capacity_errors() {
        use fuel_ir::{DynScalar, SymId};
        let dest_storage = fuel_memory::from_slice_cpu(&[0.0_f32; 6]);
        let src_storage = fuel_memory::from_slice_cpu(&[7.0_f32, 8.0]);
        let graph = Arc::new(RwLock::new(Graph::new()));
        let sym = SymId(0);
        let (dest_id, src_id, ws_id) = {
            let mut g = graph.write().unwrap();
            let dest = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[6]),
                dtype: DType::F32,
            });
            let src = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let ws = g.push(Node {
                op: Op::WriteSlice {
                    ranges: vec![(0, 2)],
                    dyn_offset: Some((0, DynScalar::Sym(sym))),
                },
                inputs: vec![dest, src],
                shape: Shape::from_dims(&[6]),
                dtype: DType::F32,
            });
            g.set_target_backend(ws, BackendId::Cpu);
            (dest, src, ws)
        };
        let mut inputs = StorageCache::new();
        inputs.insert(dest_id, Arc::new(RwLock::new(dest_storage)));
        inputs.insert(src_id, Arc::new(RwLock::new(src_storage)));
        // offset 5 + width 2 = 7 > capacity 6.
        let mut env = SymEnv::new();
        env.bind(sym, 5).unwrap();
        let result = PipelinedExecutor::realize_with_env(graph, ws_id, inputs, env);
        assert!(
            result.is_err(),
            "resolved slab past destination capacity must surface a typed error",
        );
    }

    /// Regression for the PipelinedExecutor ordering-integration session.
    ///
    /// Graph: `x = Const`, `step_x = Step(x)`, `y = SigmoidInplace(x)`.
    /// `Step` and `SigmoidInplace` are both readers of `x`; `Step` is
    /// non-destructive, `SigmoidInplace` mutates `x`'s storage. The
    /// correct semantic is for `Step` to read PRE-mutation bytes.
    ///
    /// Pre-this-session (raw `topo_order_multi`): the two are siblings
    /// in topo order and the tie-break is unspecified — `SigmoidInplace`
    /// could run first, making `step_x` see post-sigmoid bytes
    /// (`[1, 1, 1, 1]` because every sigmoid output is positive).
    ///
    /// Post-this-session: `execution_plan` consults `derive_ordering`,
    /// which pins `SigmoidInplace` AFTER every other reader of `x`.
    /// `step_x` deterministically sees `x` and is `[1, 0, 1, 0]`.
    #[test]
    fn pipelined_inplace_with_multiple_readers_orders_correctly() {
        use fuel_ir::Shape;
        let data = [1.0_f32, -2.0, 3.0, -4.0];
        let src_storage = fuel_memory::from_slice_cpu(&data);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (x_id, step_id, sig_id) = {
            let mut g = graph.write().unwrap();
            let x = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            let step = g.push(Node {
                op: Op::Step,
                inputs: vec![x],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            let sig = g.push(Node {
                op: Op::SigmoidInplace,
                inputs: vec![x],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            g.set_target_backend(step, BackendId::Cpu);
            g.set_target_backend(sig, BackendId::Cpu);
            (x, step, sig)
        };

        let mut cache = StorageCache::new();
        cache.insert(x_id, Arc::new(RwLock::new(src_storage)));

        let results = PipelinedExecutor::realize_many(graph, &[step_id, sig_id], cache)
            .expect("realize_many");

        // step_x must reflect pre-mutation x: sign of [1, -2, 3, -4] →
        // [1, 0, 1, 0]. If the executor ran SigmoidInplace first, step
        // would see all-positive sigmoid outputs and produce [1, 1, 1, 1].
        let step_guard = results[0].0.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &step_guard.inner else {
            panic!("expected Cpu storage for step output");
        };
        let step_out: &[f32] = c.as_slice().expect("f32 cast");
        assert_eq!(
            step_out,
            &[1.0_f32, 0.0, 1.0, 0.0],
            "Step must run before SigmoidInplace; got post-mutation bytes"
        );
        drop(step_guard);

        // SigmoidInplace's output Arc IS the mutated cache entry — bytes
        // approximate sigmoid([1, -2, 3, -4]).
        let sig_guard = results[1].0.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &sig_guard.inner else {
            panic!("expected Cpu storage for sigmoid output");
        };
        let sig_out: &[f32] = c.as_slice().expect("f32 cast");
        let sig_ref = [
            1.0_f32 / (1.0 + (-1.0_f32).exp()),
            1.0_f32 / (1.0 + (2.0_f32).exp()),
            1.0_f32 / (1.0 + (-3.0_f32).exp()),
            1.0_f32 / (1.0 + (4.0_f32).exp()),
        ];
        for (got, want) in sig_out.iter().zip(sig_ref.iter()) {
            assert!((got - want).abs() < 1e-6, "sigmoid: got {got}, want {want}");
        }
    }

    /// Regression for the PipelinedExecutor ordering-integration session.
    ///
    /// Canonical residual-connection pattern:
    ///   `x = Const`, `y = ReluInplace(x)`, `z = Add(y, x)`.
    ///
    /// Pre-this-session: `derive_ordering` was never run, so the cycle
    /// (Add must precede ReluInplace per ordering edge; Add must follow
    /// ReluInplace per data-flow edge through y) was undetected. The
    /// executor produced `z = relu(x) + relu(x) = [2, 0, 6, 0]` —
    /// silently wrong.
    ///
    /// Post-this-session: `insert_safety_copies` runs before
    /// `execution_plan`, inserts `Op::Copy(x) → x_safe`, rewires Add's
    /// `x` input to `x_safe`. Result: `z = relu(x) + x = [2, -2, 6, -4]`.
    #[test]
    fn pipelined_residual_connection_inserts_safety_copy() {
        use fuel_ir::Shape;
        let data = [1.0_f32, -2.0, 3.0, -4.0];
        let src_storage = fuel_memory::from_slice_cpu(&data);

        let graph = Arc::new(RwLock::new(Graph::new()));
        let (x_id, _y_id, z_id) = {
            let mut g = graph.write().unwrap();
            let x = g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            let y = g.push(Node {
                op: Op::ReluInplace,
                inputs: vec![x],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            let z = g.push(Node {
                op: Op::Add,
                inputs: vec![y, x],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            g.set_target_backend(y, BackendId::Cpu);
            g.set_target_backend(z, BackendId::Cpu);
            (x, y, z)
        };

        let mut cache = StorageCache::new();
        cache.insert(x_id, Arc::new(RwLock::new(src_storage)));

        let (result_arc, _) = PipelinedExecutor::realize(graph, z_id, cache)
            .expect("realize z — insert_safety_copies must break the cycle");

        let guard = result_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let got: &[f32] = c.as_slice().expect("f32 cast");
        assert_eq!(
            got,
            &[2.0_f32, -2.0, 6.0, -4.0],
            "Add must read pre-mutation x via the inserted safety copy"
        );
    }

    // ------------------------------------------------------------------
    // Item 3 end-to-end: SelectiveScan through the bundled-output
    // PipelinedExecutor path. The producer allocates bundled
    // (y_bytes + last_state_bytes), the kernel writes both slots,
    // and the bundled-tuple builder returns each via Op::View.
    // ------------------------------------------------------------------

    fn cpu_const_f32(
        graph: &Arc<RwLock<Graph>>,
        cache: &mut StorageCache,
        data: &[f32],
        dims: &[usize],
    ) -> NodeId {
        let id = {
            let mut g = graph.write().unwrap();
            g.push(Node {
                op: Op::Const,
                inputs: vec![],
                shape: Shape::from_dims(dims),
                dtype: DType::F32,
            })
        };
        cache.insert(id, Arc::new(RwLock::new(fuel_memory::from_slice_cpu(data))));
        id
    }

    /// `selective_scan_bundled` realizes both `y` and `last_state` in
    /// a single `realize_many` pass; the Views project the same
    /// producer's bundled Storage Arc and the numerics match the
    /// hand-computed single-step kernel test (b=1,s=1,d=1,k=1).
    #[test]
    fn pipelined_realize_selective_scan_bundled_produces_y_and_last_state() {
        let graph: Arc<RwLock<Graph>> = Arc::new(RwLock::new(Graph::new()));
        let mut cache = StorageCache::new();

        let u_id = cpu_const_f32(&graph, &mut cache, &[3.0], &[1, 1, 1]);
        let delta_id = cpu_const_f32(&graph, &mut cache, &[1.0], &[1, 1, 1]);
        let a_id = cpu_const_f32(&graph, &mut cache, &[-1.0], &[1, 1]);
        let b_id = cpu_const_f32(&graph, &mut cache, &[2.0], &[1, 1, 1]);
        let c_id = cpu_const_f32(&graph, &mut cache, &[0.5], &[1, 1, 1]);

        let (y_id, last_state_id) = {
            let u_t = fuel_graph::NodeHandle::from_existing(Arc::clone(&graph), u_id);
            let delta_t = fuel_graph::NodeHandle::from_existing(Arc::clone(&graph), delta_id);
            let a_t = fuel_graph::NodeHandle::from_existing(Arc::clone(&graph), a_id);
            let b_t = fuel_graph::NodeHandle::from_existing(Arc::clone(&graph), b_id);
            let c_t = fuel_graph::NodeHandle::from_existing(Arc::clone(&graph), c_id);
            let (y, last_state) = u_t
                .selective_scan_bundled(&delta_t, &a_t, &b_t, &c_t, /* delta_softplus */ false)
                .expect("selective_scan_bundled");
            (y.id(), last_state.id())
        };
        // Set target_backend on the producer (View nodes inherit it).
        {
            let mut g = graph.write().unwrap();
            let producer = g.node(y_id).inputs[0];
            g.set_target_backend(producer, BackendId::Cpu);
        }

        let outputs =
            PipelinedExecutor::realize_many(Arc::clone(&graph), &[y_id, last_state_id], cache)
                .expect("realize_many");
        let (y_arc, y_layout) = outputs[0].clone();
        let (last_state_arc, last_state_layout) = outputs[1].clone();

        // Both Views project the same bundled producer Storage Arc.
        assert!(
            Arc::ptr_eq(&y_arc, &last_state_arc),
            "y and last_state Views project the SAME bundled producer Storage",
        );

        // Hand-computed numerics (single-step recurrence):
        // h[0,0,0] = exp(1.0 * -1.0) * 0 + 1.0 * 2.0 * 3.0 = 6.0
        // y[0,0,0] = h * c = 6.0 * 0.5 = 3.0
        // last_state[0,0,0] = h = 6.0
        let guard = y_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c_bytes) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let typed: &[f32] = c_bytes.as_slice().expect("f32 cast");
        assert_eq!(y_layout.start_offset(), 0);
        let y_value = typed[y_layout.start_offset()];
        assert!((y_value - 3.0).abs() < 1e-5, "y expected 3.0 got {y_value}");
        assert_eq!(
            last_state_layout.start_offset(),
            1,
            "slot 1 byte_offset 4 / sizeof(f32) = 1",
        );
        let ls_value = typed[last_state_layout.start_offset()];
        assert!(
            (ls_value - 6.0).abs() < 1e-5,
            "last_state expected 6.0 got {ls_value}"
        );
    }

    // ---- Ordering-defect repros (MLA decode multi-round WriteSlice) -----

    /// DIAGNOSTIC (not a repro by itself): a plain single-hop view chain
    /// `S = Slice(X); K = Relu(S)` competing with a SECOND destructive
    /// WriteSlice on `X` (`Y = WriteSlice(X, ...)`) within one
    /// `realize_many` call. `collect_alias_set` adds `S` to `X`'s alias
    /// set (Slice IS `is_view_op`), so `derive_ordering` pins `Y` after
    /// `S`'s reader `K` directly. This form is expected to STAY GREEN —
    /// recorded here as the control that isolates the Reshape-specific
    /// gap exercised by the next test.
    #[test]
    fn pipelined_write_slice_round2_plain_slice_reader_not_stranded() {
        let graph: Arc<RwLock<Graph>> = Arc::new(RwLock::new(Graph::new()));
        let mut cache = StorageCache::new();

        let b_id = cpu_const_f32(&graph, &mut cache, &[0.0, 0.0, 0.0, 0.0], &[4]);
        let src1_id = cpu_const_f32(&graph, &mut cache, &[1.0, 2.0], &[2]);
        let src2_id = cpu_const_f32(&graph, &mut cache, &[99.0, 98.0], &[2]);

        let (k_id, y_id) = {
            let mut g = graph.write().unwrap();
            let x_id = g.push(Node {
                op: Op::WriteSlice {
                    ranges: vec![(0, 2)],
                    dyn_offset: None,
                },
                inputs: vec![b_id, src1_id],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            let s_id = g.push(Node {
                op: Op::Slice {
                    dim: 0,
                    start: 0,
                    len: 2,
                },
                inputs: vec![x_id],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let k_id = g.push(Node {
                op: Op::Relu,
                inputs: vec![s_id],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            // Round 2 OVERLAPS round 1's [0,2) range so a wrong-order
            // execution would be numerically visible in K's output.
            let y_id = g.push(Node {
                op: Op::WriteSlice {
                    ranges: vec![(0, 2)],
                    dyn_offset: None,
                },
                inputs: vec![x_id, src2_id],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            g.set_target_backend(x_id, BackendId::Cpu);
            g.set_target_backend(k_id, BackendId::Cpu);
            g.set_target_backend(y_id, BackendId::Cpu);
            (k_id, y_id)
        };

        let plan = fuel_graph::opt::execution_plan(&graph.read().unwrap(), &[k_id, y_id]);
        eprintln!("[diagnostic plain-slice] execution_plan = {:?}", plan,);

        let outputs = PipelinedExecutor::realize_many(Arc::clone(&graph), &[k_id, y_id], cache)
            .expect("realize_many: plain slice reader must not error");
        let (k_arc, _) = &outputs[0];
        let guard = k_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let out: &[f32] = c.as_slice().unwrap();
        assert_eq!(
            out,
            &[1.0, 2.0],
            "K = relu(slice(X,0,2)) must read PRE-round-2 bytes; got {:?}",
            out,
        );
    }

    /// CRITICAL SUSPECT #2 repro: `Op::Reshape`/`Op::Contiguize` compile
    /// to `WorkItemKind::ContiguizeOf`, which zero-copy ADOPTS the
    /// input's Storage Arc when the input is contiguous at offset 0 —
    /// but `Op::Reshape` is NOT in `collect_alias_set`'s alias-extending
    /// predicate (only `Op::is_view_op` + `Op::View{..}`). So for
    /// `X = write_slice(B, src1); S = X.slice(0, 0, 2) [offset-0
    /// contiguous]; R = S.reshape([2]); K = relu(R); Y =
    /// write_slice(X, src2)`, `collect_alias_set(X)` stops at `{X, S}`
    /// — `R` never enters the alias set, so `derive_ordering` never
    /// scans `R`'s consumers (`K`) for pinning against `Y`. `K` and `Y`
    /// become simultaneously "ready" once `R` completes, and
    /// `execution_plan`'s stable tiebreak (smaller `topo_order_multi`
    /// position wins) can schedule `Y` (destructive) BEFORE `K` — `K`
    /// then dereferences `R`'s adopted Arc (== `X`'s Arc, mutated
    /// in-place by `Y`) and silently reads POST-`Y` bytes. No error;
    /// wrong numbers.
    #[test]
    fn pipelined_write_slice_reshape_adopted_arc_reader_reads_pre_write_bytes() {
        let graph: Arc<RwLock<Graph>> = Arc::new(RwLock::new(Graph::new()));
        let mut cache = StorageCache::new();

        let b_id = cpu_const_f32(&graph, &mut cache, &[0.0, 0.0, 0.0, 0.0], &[4]);
        let src1_id = cpu_const_f32(&graph, &mut cache, &[1.0, 2.0], &[2]);
        let src2_id = cpu_const_f32(&graph, &mut cache, &[99.0, 98.0], &[2]);

        let (x_id, k_id, y_id) = {
            let mut g = graph.write().unwrap();
            let x_id = g.push(Node {
                op: Op::WriteSlice {
                    ranges: vec![(0, 2)],
                    dyn_offset: None,
                },
                inputs: vec![b_id, src1_id],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            let s_id = g.push(Node {
                // OFFSET-0 contiguous prefix slice — ContiguizeOf's
                // zero-copy fast path requires this at execute time.
                op: Op::Slice {
                    dim: 0,
                    start: 0,
                    len: 2,
                },
                inputs: vec![x_id],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let r_id = g.push(Node {
                op: Op::Reshape(Shape::from_dims(&[2])),
                inputs: vec![s_id],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let k_id = g.push(Node {
                op: Op::Relu,
                inputs: vec![r_id],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            // Round 2 OVERLAPS round 1's [0,2) range (not the [2,4)
            // tail) so a wrong-order execution is numerically visible
            // in K's output — a disjoint round 2 would corrupt bytes
            // K never reads, masking the defect.
            let y_id = g.push(Node {
                op: Op::WriteSlice {
                    ranges: vec![(0, 2)],
                    dyn_offset: None,
                },
                inputs: vec![x_id, src2_id],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            g.set_target_backend(x_id, BackendId::Cpu);
            g.set_target_backend(k_id, BackendId::Cpu);
            g.set_target_backend(y_id, BackendId::Cpu);
            (x_id, k_id, y_id)
        };
        let _ = x_id;

        let plan = fuel_graph::opt::execution_plan(&graph.read().unwrap(), &[k_id, y_id]);
        eprintln!(
            "[diagnostic reshape-adopted-arc] execution_plan = {:?}",
            plan,
        );

        let outputs = PipelinedExecutor::realize_many(Arc::clone(&graph), &[k_id, y_id], cache)
            .expect("realize_many");
        let (k_arc, _) = &outputs[0];
        let guard = k_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let out: &[f32] = c.as_slice().unwrap();
        // K = relu(reshape(slice(X,0,2))) must read PRE-round-2 bytes:
        // relu([1.0, 2.0]) = [1.0, 2.0]. Round 2 overwrites the SAME
        // [0,2) range with [99.0, 98.0] — if Y (round 2) runs before K
        // (the ordering defect), K instead observes [99.0, 98.0].
        assert_eq!(
            out,
            &[1.0, 2.0],
            "K must read PRE-round-2 bytes through the adopted Arc; got {:?} \
             (post-round-2 corruption if this fails)",
            out,
        );
    }

    /// The MLA-decode shape from `rebind_latent_cache_fresh_graph`'s doc
    /// (`fuel-core/src/lazy_deepseek2.rs`): layer L's post-append buffer
    /// `X_L_N` (round N) is read TWICE within one realize call — once
    /// (non-destructively) through a Slice+Reshape "attention math" view
    /// chain whose result becomes an ancestor of layer L+1's round-N
    /// write, and again as the DESTINATION of layer L's round-(N+1)
    /// append. Reproduces the same Reshape-adoption gap as
    /// `pipelined_write_slice_reshape_adopted_arc_reader_reads_pre_write_bytes`
    /// at 2-layer scale, confirming the defect is not an artifact of the
    /// single-hop minimal form.
    #[test]
    fn pipelined_write_slice_mla_two_layer_round2_reshape_reader_not_stranded() {
        let graph: Arc<RwLock<Graph>> = Arc::new(RwLock::new(Graph::new()));
        let mut cache = StorageCache::new();

        let b_l_id = cpu_const_f32(&graph, &mut cache, &[0.0, 0.0, 0.0, 0.0], &[4]);
        let src_l_n_id = cpu_const_f32(&graph, &mut cache, &[1.0, 2.0], &[2]);
        let b_l1_id = cpu_const_f32(&graph, &mut cache, &[0.0, 0.0, 0.0, 0.0], &[4]);
        let src_l_n1_id = cpu_const_f32(&graph, &mut cache, &[99.0, 98.0], &[2]);

        let (x_l_n1_id, x_l1_n_id) = {
            let mut g = graph.write().unwrap();
            // Round N, layer L: append src_l_n into b_l.
            let x_l_n = g.push(Node {
                op: Op::WriteSlice {
                    ranges: vec![(0, 2)],
                    dyn_offset: None,
                },
                inputs: vec![b_l_id, src_l_n_id],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            // "Attention math" for round N at layer L: read the filled
            // prefix, reshape (head-split-style hop), then a kernel.
            let s_l_n = g.push(Node {
                op: Op::Slice {
                    dim: 0,
                    start: 0,
                    len: 2,
                },
                inputs: vec![x_l_n],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let rs_l_n = g.push(Node {
                op: Op::Reshape(Shape::from_dims(&[2])),
                inputs: vec![s_l_n],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            let r_l_n = g.push(Node {
                op: Op::Relu,
                inputs: vec![rs_l_n],
                shape: Shape::from_dims(&[2]),
                dtype: DType::F32,
            });
            // Round N, layer L+1: append the attention-math result.
            let x_l1_n = g.push(Node {
                op: Op::WriteSlice {
                    ranges: vec![(0, 2)],
                    dyn_offset: None,
                },
                inputs: vec![b_l1_id, r_l_n],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            // Round N+1, layer L: append into x_l_n directly (SAME node
            // s_l_n/rs_l_n/r_l_n read from) — OVERLAPPING range so a
            // wrong-order execution is numerically visible downstream.
            let x_l_n1 = g.push(Node {
                op: Op::WriteSlice {
                    ranges: vec![(0, 2)],
                    dyn_offset: None,
                },
                inputs: vec![x_l_n, src_l_n1_id],
                shape: Shape::from_dims(&[4]),
                dtype: DType::F32,
            });
            g.set_target_backend(x_l_n, BackendId::Cpu);
            g.set_target_backend(r_l_n, BackendId::Cpu);
            g.set_target_backend(x_l1_n, BackendId::Cpu);
            g.set_target_backend(x_l_n1, BackendId::Cpu);
            (x_l_n1, x_l1_n)
        };

        let plan = fuel_graph::opt::execution_plan(&graph.read().unwrap(), &[x_l1_n_id, x_l_n1_id]);
        eprintln!("[diagnostic mla-2-layer] execution_plan = {:?}", plan);

        let outputs =
            PipelinedExecutor::realize_many(Arc::clone(&graph), &[x_l1_n_id, x_l_n1_id], cache)
                .expect("realize_many");
        // outputs[0] = x_l1_n = write_slice(b_l1, relu(reshape(slice(x_l_n))))
        // Must reflect round-N's PRE-round-(N+1) bytes: relu([1,2]) = [1,2].
        let (x_l1_n_arc, _) = &outputs[0];
        let guard = x_l1_n_arc.read().unwrap();
        let fuel_memory::BackendStorage::Cpu(c) = &guard.inner else {
            panic!("expected Cpu storage");
        };
        let out: &[f32] = c.as_slice().unwrap();
        assert_eq!(
            out,
            &[1.0, 2.0, 0.0, 0.0],
            "layer L+1's round-N append must reflect PRE-round-(N+1) attention \
             math output; got {:?} (post-round-(N+1) corruption if this fails)",
            out,
        );
    }
}