hrx/fabric/
visibility.rs

1use super::*;
2
3/// Access site participating in an ordered transfer of ownership.
4#[derive(Clone, Copy)]
5pub enum MemorySite<'a> {
6    /// The buffer's actual CPU mapping.
7    Host,
8    /// An admitted device and an exact native queue family ordinal.
9    Device(&'a Device, u32),
10}
11/// Granularity of a qualified cache action.
12#[derive(Clone, Copy, Debug, PartialEq, Eq)]
13pub enum TransitionKind {
14    /// The provider cannot establish a transition.
15    Unknown,
16    /// No cache operation is necessary; ordering is still required.
17    None,
18    /// An explicit byte range must be transitioned.
19    Range,
20    /// The complete cache domain must be transitioned.
21    Global,
22}
23/// Participant responsible for the transition.
24#[derive(Clone, Copy, Debug, PartialEq, Eq)]
25pub enum CacheExecutor {
26    /// No operation or no qualified executor.
27    None,
28    /// Encode a queue command.
29    Queue,
30    /// Encode an operation in the running program.
31    Program,
32    /// Execute a host instruction and its prescribed fences.
33    HostDirect,
34    /// Call the native host cache API.
35    HostApi,
36}
37/// Semantic cache action; independent of ordering and atomic reach.
38#[derive(Clone, Copy, Debug, PartialEq, Eq)]
39pub enum CacheOperation {
40    /// No semantic queue operation.
41    None,
42    /// Publish writes to system memory.
43    ReleaseToSystem,
44    /// Acquire writes from system memory.
45    AcquireFromSystem,
46}
47/// Immutable facts for one half of a directional visibility relation.
48#[derive(Clone, Copy, Debug)]
49pub struct CacheTransition {
50    /// Scope of maintenance.
51    pub kind: TransitionKind,
52    /// Responsible participant.
53    pub executor: CacheExecutor,
54    /// Queue/program action.
55    pub operation: CacheOperation,
56    /// Minimum independently transitionable range in bytes.
57    pub range_granularity: u64,
58    native: amdf_cache_transition_t,
59}
60impl CacheTransition {
61    fn from_native(native: amdf_cache_transition_t) -> Result<Self> {
62        Ok(Self {
63            kind: match native.kind {
64                AMDF_CACHE_TRANSITION_KIND_UNKNOWN => TransitionKind::Unknown,
65                AMDF_CACHE_TRANSITION_KIND_NONE => TransitionKind::None,
66                AMDF_CACHE_TRANSITION_KIND_RANGE => TransitionKind::Range,
67                AMDF_CACHE_TRANSITION_KIND_GLOBAL => TransitionKind::Global,
68                _ => return Err(missing("cache transition kind")),
69            },
70            executor: match native.executor {
71                AMDF_CACHE_TRANSITION_EXECUTOR_NONE => CacheExecutor::None,
72                AMDF_CACHE_TRANSITION_EXECUTOR_QUEUE => CacheExecutor::Queue,
73                AMDF_CACHE_TRANSITION_EXECUTOR_PROGRAM => CacheExecutor::Program,
74                AMDF_CACHE_TRANSITION_EXECUTOR_HOST_DIRECT => CacheExecutor::HostDirect,
75                AMDF_CACHE_TRANSITION_EXECUTOR_HOST_API => CacheExecutor::HostApi,
76                _ => return Err(missing("cache transition executor")),
77            },
78            operation: match native.operation {
79                AMDF_CACHE_OPERATION_NONE => CacheOperation::None,
80                AMDF_CACHE_OPERATION_RELEASE_TO_SYSTEM => CacheOperation::ReleaseToSystem,
81                AMDF_CACHE_OPERATION_ACQUIRE_FROM_SYSTEM => CacheOperation::AcquireFromSystem,
82                _ => return Err(missing("cache operation")),
83            },
84            range_granularity: native.range_granularity,
85            native,
86        })
87    }
88}
89/// Largest scope of mutually atomic access reported for a width.
90#[derive(Clone, Copy, Debug, PartialEq, Eq)]
91pub enum AtomicScope {
92    /// Atomic reach is unavailable.
93    None,
94    /// Within one device.
95    Device,
96    /// Across the participating fabric.
97    Fabric,
98    /// Across the system.
99    System,
100}
101impl AtomicScope {
102    fn from_native(scope: amdf_atomic_scope_t) -> Result<Self> {
103        match scope {
104            AMDF_ATOMIC_SCOPE_NONE => Ok(Self::None),
105            AMDF_ATOMIC_SCOPE_DEVICE => Ok(Self::Device),
106            AMDF_ATOMIC_SCOPE_FABRIC => Ok(Self::Fabric),
107            AMDF_ATOMIC_SCOPE_SYSTEM => Ok(Self::System),
108            _ => Err(missing("atomic scope")),
109        }
110    }
111}
112/// Directional cache, reachability, atomic and provider cost facts.
113#[derive(Clone, Copy, Debug)]
114pub struct VisibilityFacts {
115    /// Whether the consumer can directly reach the producer's backing.
116    pub shared_backing_reachable: bool,
117    /// Producer action after writes.
118    pub release: CacheTransition,
119    /// Consumer action before reads.
120    pub acquire: CacheTransition,
121    /// Atomic reach for naturally aligned 32-bit accesses; operation support is separate.
122    pub atomic_scope_32: AtomicScope,
123    /// Atomic reach for naturally aligned 64-bit accesses; operation support is separate.
124    pub atomic_scope_64: AtomicScope,
125    /// Provider-qualified fixed cost, when known; not a measured workload latency.
126    pub estimated_fixed_cost_ns: Option<u64>,
127}
128impl VisibilityFacts {
129    pub(super) fn from_native(info: amdf_memory_pair_info_t) -> Result<Self> {
130        Ok(Self {
131            shared_backing_reachable: info.flags
132                & AMDF_MEMORY_PAIR_FLAG_SHARED_BACKING_REACHABLE as u64
133                != 0,
134            release: CacheTransition::from_native(info.release)?,
135            acquire: CacheTransition::from_native(info.acquire)?,
136            atomic_scope_32: AtomicScope::from_native(info.atomic_reach.scope_32)?,
137            atomic_scope_64: AtomicScope::from_native(info.atomic_reach.scope_64)?,
138            estimated_fixed_cost_ns: (info.flags & AMDF_MEMORY_PAIR_FLAG_FIXED_COST_KNOWN as u64
139                != 0)
140                .then_some(info.estimated_fixed_cost_nanoseconds),
141        })
142    }
143}
144/// Prepared directional visibility retaining its actual backing.
145/// Cache actions do not establish execution ordering.
146pub struct MemoryVisibility {
147    buffer: Buffer,
148    facts: VisibilityFacts,
149}
150impl std::ops::Deref for MemoryVisibility {
151    type Target = VisibilityFacts;
152    fn deref(&self) -> &Self::Target {
153        &self.facts
154    }
155}
156impl Buffer {
157    fn site(&self, site: MemorySite<'_>) -> Result<amdf_memory_site_t> {
158        let mut result = amdf_memory_site_t {
159            type_: AMDF_STRUCTURE_TYPE_MEMORY_SITE,
160            structure_size: size_of::<amdf_memory_site_t>() as u32,
161            ..Default::default()
162        };
163        match site {
164            MemorySite::Host => {
165                if let Some(source) = &self.0.source {
166                    return source.site(site);
167                }
168                result.kind = AMDF_MEMORY_SITE_KIND_HOST;
169                result.value.host_mapping = self.0.mapping;
170            }
171            MemorySite::Device(device, family) => {
172                let Some(index) = self
173                    .0
174                    .devices
175                    .iter()
176                    .position(|entry| Arc::ptr_eq(&entry.0, &device.0))
177                else {
178                    return self.0.source.as_ref().map_or_else(
179                        || {
180                            Err(Error::Message(
181                                "visibility device has no access to backing".into(),
182                            ))
183                        },
184                        |source| source.site(site),
185                    );
186                };
187                result.kind = AMDF_MEMORY_SITE_KIND_DEVICE;
188                result.value.device = amdf_memory_site_t__bindgen_ty_1__bindgen_ty_1 {
189                    memory: self.0.raw,
190                    access_ordinal: index as u32,
191                    queue_family_ordinal: family,
192                };
193            }
194        }
195        Ok(result)
196    }
197    /// Query exact producer-to-consumer visibility once during preparation.
198    /// Host aliases resolve to their original mapping; device sites resolve to
199    /// their actual native attachments rather than equating virtual addresses.
200    pub fn visibility(
201        &self,
202        producer: MemorySite<'_>,
203        consumer: MemorySite<'_>,
204    ) -> Result<MemoryVisibility> {
205        let mut info = amdf_memory_pair_info_t {
206            type_: AMDF_STRUCTURE_TYPE_MEMORY_PAIR_INFO,
207            structure_size: size_of::<amdf_memory_pair_info_t>() as u32,
208            ..Default::default()
209        };
210        check("memory_query_pair_info", unsafe {
211            entry!(self.0.instance.api.core, memory_query_pair_info)(
212                &self.site(producer)?,
213                &self.site(consumer)?,
214                &mut info,
215            )
216        })?;
217        Ok(MemoryVisibility {
218            buffer: self.clone(),
219            facts: VisibilityFacts::from_native(info)?,
220        })
221    }
222}
223impl MemoryVisibility {
224    /// Execute the prepared host producer action.
225    /// # Safety
226    /// Caller owns the range and its boundary cache lines, excludes conflicting
227    /// access, and supplies the ordering edge to the consumer.
228    pub unsafe fn release_host(&self, offset: usize, length: usize) -> Result<()> {
229        unsafe { self.host_transition(self.release, offset, length) }
230    }
231    /// Execute the prepared host consumer action after producer completion.
232    /// # Safety
233    /// Caller owns the range and its boundary cache lines and has established
234    /// producer completion before acquiring visibility.
235    pub unsafe fn acquire_host(&self, offset: usize, length: usize) -> Result<()> {
236        unsafe { self.host_transition(self.acquire, offset, length) }
237    }
238    unsafe fn host_transition(
239        &self,
240        transition: CacheTransition,
241        offset: usize,
242        length: usize,
243    ) -> Result<()> {
244        if offset
245            .checked_add(length)
246            .is_none_or(|end| end > self.buffer.len())
247        {
248            return Err(Error::Message("visibility range out of bounds".into()));
249        }
250        if !self.shared_backing_reachable {
251            return Err(Error::Unsupported("shared backing is unreachable".into()));
252        }
253        if transition.kind == TransitionKind::None {
254            std::sync::atomic::fence(std::sync::atomic::Ordering::SeqCst);
255            return Ok(());
256        }
257        if !matches!(
258            transition.executor,
259            CacheExecutor::HostApi | CacheExecutor::HostDirect
260        ) {
261            return Err(Error::Unsupported(
262                "transition is not a qualified host action".into(),
263            ));
264        }
265        let mut buffer = &self.buffer;
266        while let Some(source) = &buffer.0.source {
267            buffer = source;
268        }
269        check("prepared host visibility", unsafe {
270            entry!(buffer.0.instance.api.core, host_mapping_cache_control)(
271                buffer.0.mapping,
272                transition.native.host_operation,
273                offset as u64,
274                length as u64,
275            )
276        })
277    }
278}
279
280/// Coalesced host cache work prepared against one retained backing and direction.
281/// Preparation performs all range allocation and validation; execution does not
282/// query visibility or allocate storage.
283pub struct PreparedHostVisibility {
284    visibility: MemoryVisibility,
285    transition: CacheTransition,
286    ranges: Vec<std::ops::Range<usize>>,
287}
288impl MemoryVisibility {
289    /// Prepare producer cache work, merging overlapping and adjacent ranges.
290    pub fn prepare_release_host(
291        &self,
292        ranges: &[std::ops::Range<usize>],
293    ) -> Result<PreparedHostVisibility> {
294        self.prepare_host(self.release, ranges)
295    }
296    /// Prepare consumer cache work, merging overlapping and adjacent ranges.
297    pub fn prepare_acquire_host(
298        &self,
299        ranges: &[std::ops::Range<usize>],
300    ) -> Result<PreparedHostVisibility> {
301        self.prepare_host(self.acquire, ranges)
302    }
303    fn prepare_host(
304        &self,
305        transition: CacheTransition,
306        ranges: &[std::ops::Range<usize>],
307    ) -> Result<PreparedHostVisibility> {
308        if !self.shared_backing_reachable || transition.kind == TransitionKind::Unknown {
309            return Err(Error::Unsupported(
310                "unqualified host visibility relation".into(),
311            ));
312        }
313        if transition.kind != TransitionKind::None
314            && !matches!(
315                transition.executor,
316                CacheExecutor::HostDirect | CacheExecutor::HostApi
317            )
318        {
319            return Err(Error::Unsupported(
320                "transition is not a qualified host action".into(),
321            ));
322        }
323        let mut ranges = coalesce_ranges(ranges, self.buffer.len())?;
324        if !ranges.is_empty()
325            && matches!(
326                transition.kind,
327                TransitionKind::None | TransitionKind::Global
328            )
329        {
330            ranges.clear();
331            ranges.push(0..self.buffer.len());
332        }
333        Ok(PreparedHostVisibility {
334            visibility: MemoryVisibility {
335                buffer: self.buffer.clone(),
336                facts: self.facts,
337            },
338            transition,
339            ranges,
340        })
341    }
342}
343fn coalesce_ranges(
344    ranges: &[std::ops::Range<usize>],
345    length: usize,
346) -> Result<Vec<std::ops::Range<usize>>> {
347    if ranges.iter().any(|r| r.start > r.end || r.end > length) {
348        return Err(Error::Message("visibility range out of bounds".into()));
349    }
350    let mut sorted: Vec<_> = ranges.iter().filter(|r| !r.is_empty()).cloned().collect();
351    sorted.sort_unstable_by_key(|r| r.start);
352    let mut result: Vec<std::ops::Range<usize>> = Vec::with_capacity(sorted.len());
353    for range in sorted {
354        if let Some(last) = result.last_mut()
355            && last.end >= range.start
356        {
357            last.end = last.end.max(range.end);
358        } else {
359            result.push(range);
360        }
361    }
362    Ok(result)
363}
364impl PreparedHostVisibility {
365    /// Number of coalesced cache calls (or ordering fences for a no-op recipe).
366    pub fn operation_count(&self) -> usize {
367        self.ranges.len()
368    }
369    /// Execute the prepared host side of the ownership transfer.
370    /// # Safety
371    /// Caller owns all covered ranges and their boundary cache lines, excludes
372    /// conflicting accesses, and establishes the directional execution edge.
373    /// A global recipe additionally requires ownership of the entire cache domain.
374    pub unsafe fn execute(&self) -> Result<()> {
375        for range in &self.ranges {
376            unsafe {
377                self.visibility
378                    .host_transition(self.transition, range.start, range.len())
379            }?;
380        }
381        Ok(())
382    }
383}
384#[cfg(test)]
385mod tests {
386    use super::*;
387    #[test]
388    fn host_ranges_preserve_holes_and_reject_out_of_bounds() {
389        assert_eq!(
390            coalesce_ranges(&[8..16, 0..4, 3..10, 24..32, 16..16], 32).unwrap(),
391            vec![0..16, 24..32]
392        );
393        assert!(coalesce_ranges(std::slice::from_ref(&(0..33)), 32).is_err());
394        assert!(coalesce_ranges(&[std::ops::Range { start: 9, end: 8 }], 32).is_err());
395        assert!(
396            coalesce_ranges(std::slice::from_ref(&(0..0)), 0)
397                .unwrap()
398                .is_empty()
399        );
400    }
401}
hrx/fabric/visibility.rs

hrx/fabric/
visibility.rs
