1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
//! Per-operand local registrations, cached in [`crate::region_cache`].
//!
//! EFA requires `FI_MR_LOCAL`: every local RMA operand needs a registered descriptor.
//! Freed memory must not keep a registration. An EFA registration is frozen to the
//! physical pages present when it was made.
//!
//! Registered `FI_READ | FI_WRITE`, never remote-accessible, and the key is never exported, so
//! nothing here is reachable by a peer.
use std::os::raw::c_void;
use std::ptr;
use std::sync::Arc;
use crate::sys::{FI_READ, FI_WRITE, fi_mr_desc, fi_mr_reg, fid_domain, fid_mr};
use dma_libfabric_protocol::DmaError;
use crate::error::check;
use crate::operands::CacheableSpan;
use crate::region_cache::{self, Registration};
/// A local RMA operand's descriptor and the lease keeping its registration alive.
#[derive(Debug)]
pub(crate) struct LocalOperand {
pub(crate) descriptor: *mut c_void,
/// `None` only on providers that need no local registration at all, where the descriptor is
/// null and there is nothing to hold open.
pub(crate) lease: Option<Arc<Registration>>,
}
/// Per-endpoint handle for the operand-registration path, deferring registration lifetime to
/// [`crate::region_cache`].
#[derive(Debug)]
pub(crate) struct LocalRegions {
domain: *mut fid_domain,
}
impl LocalRegions {
pub(crate) fn new(domain: *mut fid_domain) -> Self {
Self { domain }
}
/// The local descriptor covering `[pointer, pointer + length)`, reusing a cached registration
/// that already covers it, else taking a fresh one. A cached registration is deregistered as its
/// memory is reclaimed, so it stays valid as long as the operand does — which spans the
/// transfer, since the transfer retains its operand.
///
/// `cacheable` yields the caller's [`crate::CacheableSpan`] for this operand. It is asked only
/// on a miss, so it pays once per registration rather than once per transfer.
/// `None` registers the operand's extent and keeps it out of the cache.
pub(crate) fn operand(
&mut self,
pointer: *mut u8,
length: usize,
cacheable: impl FnOnce() -> Option<CacheableSpan>,
) -> Result<LocalOperand, DmaError> {
let start = pointer as usize;
let end = start
.checked_add(length)
.ok_or_else(|| DmaError::Fabric("local operand range overflows".into()))?;
let domain = self.domain as usize;
if let Some(lease) = region_cache::covering(start, end, domain) {
// SAFETY: the lease holds the region open for as long as this operand lives, so the
// descriptor cannot be deregistered underneath the transfer that posts it.
let descriptor = unsafe { fi_mr_desc(lease.memory_region()) };
return Ok(LocalOperand {
descriptor,
lease: Some(lease),
});
}
// A span not containing the operand would pin the wrong memory and leave the operand itself
// unregistered, so it is refused here rather than trusted.
let cacheable = cacheable().filter(|span| span.contains(start, end));
// Register the caller's whole span when there is one, so the next operand landing inside it
// reuses this registration instead of taking its own.
let (base, limit) = cacheable.map_or((start, end), |span| (span.base, span.end));
// If a widened registration fails, take the operand's extent instead
let region = match self.register(base as *mut u8, limit - base) {
Ok(region) => Ok((base, limit, region)),
Err(_) if (base, limit) != (start, end) => self
.register(pointer, length)
.map(|region| (start, end, region)),
Err(error) => Err(error),
};
let (base, limit, region) = region?;
let lease = Registration::new(region);
// SAFETY: freshly registered above, and held by `lease`.
let descriptor = unsafe { fi_mr_desc(lease.memory_region()) };
// Only what the caller promised to `invalidate` may outlive this transfer in the registry —
// including the narrowed fallback above, which lies inside the span that promise covers.
if cacheable.is_some() {
// The registry takes its own lease; this operand keeps the one it already holds.
region_cache::track(base, limit, domain, &lease);
}
Ok(LocalOperand {
descriptor,
lease: Some(lease),
})
}
/// Register `[base, base + length)` up front and leave it in the registry.
///
/// every later operand inside should be a `covering()` hit, so no
/// [`crate::CacheableSpan`] is needed for these regions.
/// The registry is the sole holder. The span's lifetime belongs to the caller's
/// [`crate::MemoryRegion`], whose drop retires this entry.
pub(crate) fn pin(&mut self, base: usize, length: usize) -> Result<(), DmaError> {
let end = base
.checked_add(length)
.ok_or_else(|| DmaError::Fabric("pinned region range overflows".into()))?;
let lease = Registration::new(self.register(base as *mut u8, length)?);
region_cache::track(base, end, self.domain as usize, &lease);
Ok(())
}
fn register(&self, pointer: *mut u8, length: usize) -> Result<*mut fid_mr, DmaError> {
let mut memory_region: *mut fid_mr = ptr::null_mut();
// A root span, so first-touch registration cost is visible.
let _span = tracing::info_span!(parent: None, "region_register", bytes = length).entered();
check(
unsafe {
fi_mr_reg(
self.domain,
pointer as *const c_void,
length,
u64::from(FI_READ | FI_WRITE),
0,
0,
0,
&mut memory_region,
ptr::null_mut(),
)
},
"fi_mr_reg(operand)",
)?;
Ok(memory_region)
}
/// Close this domain's registrations before the endpoint closes it, since memory regions are
/// domain objects. Also stops a reclaim touching a region whose domain is gone.
pub(crate) fn clear(&mut self) {
region_cache::clear_domain(self.domain as usize);
}
}