use crate::{
error::{Error, Result},
TensorMemory, TensorTrait,
};
use num_traits::Num;
use std::{
fmt,
num::NonZero,
os::fd::{AsRawFd, OwnedFd},
};
#[allow(clippy::unnecessary_cast)]
fn identity_from_stat(stat: &nix::sys::stat::FileStat) -> crate::BufferIdentity {
let key = ((stat.st_dev as u64) << 32) ^ (stat.st_ino as u64);
crate::BufferIdentity::derived(crate::IdentityKind::DmaBuf, key)
}
#[derive(Debug)]
pub struct DmaTensor<T>
where
T: Num + Clone + fmt::Debug + Send + Sync,
{
pub name: String,
pub fd: OwnedFd,
pub shape: Vec<usize>,
pub _marker: std::marker::PhantomData<T>,
#[cfg(target_os = "linux")]
_drm_attachment: Option<crate::dmabuf::DrmAttachment>,
identity: crate::BufferIdentity,
pub(crate) buf_size: usize,
pub(crate) mmap_offset: usize,
#[cfg(target_os = "linux")]
pub(crate) is_imported: bool,
}
unsafe impl<T> Send for DmaTensor<T> where T: Num + Clone + fmt::Debug + Send + Sync {}
unsafe impl<T> Sync for DmaTensor<T> where T: Num + Clone + fmt::Debug + Send + Sync {}
impl<T> TensorTrait<T> for DmaTensor<T>
where
T: Num + Clone + fmt::Debug + Send + Sync,
{
#[cfg(all(target_os = "linux", not(miri)))]
fn new(shape: &[usize], name: Option<&str>) -> Result<Self> {
use log::debug;
use nix::sys::stat::fstat;
let logical_size = shape.iter().product::<usize>() * std::mem::size_of::<T>();
let name = match name {
Some(name) => name.to_owned(),
None => {
let uuid = uuid::Uuid::new_v4().as_simple().to_string();
format!("/{}", &uuid[..16])
}
};
let heap = match dma_heap::Heap::new(dma_heap::HeapKind::Cma) {
Ok(heap) => heap,
Err(_) => dma_heap::Heap::new(dma_heap::HeapKind::System)?,
};
let dma_fd = heap.allocate(logical_size)?;
let stat = fstat(&dma_fd)?;
debug!("DMA memory stat: {stat:?}");
let buf_size = if stat.st_size > 0 {
std::cmp::max(stat.st_size as usize, logical_size)
} else {
logical_size
};
let drm_attachment = crate::dmabuf::DrmAttachment::new(&dma_fd, false);
Ok(DmaTensor::<T> {
name: name.to_owned(),
fd: dma_fd,
shape: shape.to_vec(),
_marker: std::marker::PhantomData,
_drm_attachment: drm_attachment,
identity: identity_from_stat(&stat),
buf_size,
mmap_offset: 0,
is_imported: false,
})
}
#[cfg(not(all(target_os = "linux", not(miri))))]
fn new(_shape: &[usize], _name: Option<&str>) -> Result<Self> {
Err(Error::NotImplemented(
"DMA tensor allocation is unavailable (not Linux, or running \
under Miri, which cannot execute the real ioctls)"
.to_owned(),
))
}
fn from_fd(fd: OwnedFd, shape: &[usize], name: Option<&str>) -> Result<Self> {
if shape.is_empty() {
return Err(Error::InvalidSize(0));
}
let logical_size = shape.iter().product::<usize>() * std::mem::size_of::<T>();
if logical_size == 0 {
return Err(Error::InvalidSize(0));
}
let stat = nix::sys::stat::fstat(&fd)?;
let buf_size = if stat.st_size > 0 && stat.st_size as usize >= logical_size {
stat.st_size as usize
} else {
logical_size
};
#[cfg(target_os = "linux")]
let drm_attachment = None;
Ok(DmaTensor {
name: name.unwrap_or("").to_owned(),
fd,
shape: shape.to_vec(),
_marker: std::marker::PhantomData,
#[cfg(target_os = "linux")]
_drm_attachment: drm_attachment,
identity: identity_from_stat(&stat),
buf_size,
mmap_offset: 0,
#[cfg(target_os = "linux")]
is_imported: true,
})
}
fn clone_fd(&self) -> Result<OwnedFd> {
Ok(self.fd.try_clone()?)
}
fn memory(&self) -> TensorMemory {
TensorMemory::DmaBuf
}
fn name(&self) -> String {
self.name.clone()
}
fn shape(&self) -> &[usize] {
&self.shape
}
fn reshape(&mut self, shape: &[usize]) -> Result<()> {
if shape.is_empty() {
return Err(Error::InvalidSize(0));
}
let new_size = shape.iter().product::<usize>() * std::mem::size_of::<T>();
if new_size != self.size() {
return Err(Error::ShapeMismatch(format!(
"Cannot reshape incompatible shape: {:?} to {:?}",
self.shape, shape
)));
}
self.shape = shape.to_vec();
Ok(())
}
fn map_with<'a>(&self, access: crate::CpuAccess) -> Result<crate::view::HostView<'a, T>>
where
T: 'a,
{
self.map_inner(None, access)
}
fn buffer_identity(&self) -> &crate::BufferIdentity {
&self.identity
}
fn capacity_bytes(&self) -> usize {
self.buf_size
}
fn set_logical_shape(&mut self, shape: &[usize]) -> Result<()> {
if shape.is_empty() {
return Err(Error::InvalidSize(0));
}
let needed = shape.iter().product::<usize>() * std::mem::size_of::<T>();
if needed > self.buf_size {
return Err(Error::InsufficientCapacity {
needed,
capacity: self.buf_size,
});
}
self.shape = shape.to_vec();
Ok(())
}
#[cfg(target_os = "linux")]
fn view(&self, offset_bytes: usize, shape: &[usize]) -> Result<Self> {
let elem = std::mem::size_of::<T>();
if !offset_bytes.is_multiple_of(std::mem::align_of::<T>()) {
return Err(Error::InvalidOperation(format!(
"DmaTensor::view: offset {offset_bytes} not aligned to align_of::<T>()={}",
std::mem::align_of::<T>()
)));
}
let abs_offset = self
.mmap_offset
.checked_add(offset_bytes)
.ok_or(Error::InvalidSize(offset_bytes))?;
let logical = shape.iter().product::<usize>() * elem;
let needed = abs_offset
.checked_add(logical)
.ok_or(Error::InvalidSize(logical))?;
if needed > self.buf_size {
return Err(Error::InsufficientCapacity {
needed,
capacity: self.buf_size,
});
}
let mut v = self.try_clone()?;
v.shape = shape.to_vec();
v.mmap_offset = abs_offset;
Ok(v)
}
}
impl<T> AsRawFd for DmaTensor<T>
where
T: Num + Clone + fmt::Debug + Send + Sync,
{
fn as_raw_fd(&self) -> std::os::fd::RawFd {
self.fd.as_raw_fd()
}
}
pub(crate) struct DmaSyncBracket {
fd: std::os::fd::OwnedFd,
access: crate::CpuAccess,
_mapping: std::sync::Arc<crate::pin::MmapOwner>,
}
impl Drop for DmaSyncBracket {
fn drop(&mut self) {
if let Err(e) = crate::dmabuf::sync_access(&self.fd, false, self.access) {
log::error!("DMA_BUF_SYNC_END failed: {e}");
}
}
}
impl<T> DmaTensor<T>
where
T: Num + Clone + Send + Sync + std::fmt::Debug + Send + Sync,
{
#[cfg(all(target_os = "linux", feature = "static", not(miri)))]
pub(crate) fn new_with_byte_size(
shape: &[usize],
byte_size: usize,
name: Option<&str>,
) -> Result<Self> {
use log::debug;
use nix::sys::stat::fstat;
let logical_elems = shape
.iter()
.copied()
.try_fold(1usize, |acc, dim| acc.checked_mul(dim))
.ok_or_else(|| {
Error::InvalidArgument(format!(
"DmaTensor::new_with_byte_size: shape.product() overflows usize \
(shape={shape:?})"
))
})?;
let logical_size = logical_elems
.checked_mul(std::mem::size_of::<T>())
.ok_or_else(|| {
Error::InvalidArgument(format!(
"DmaTensor::new_with_byte_size: logical_elems {logical_elems} × \
sizeof::<T>={} overflows usize (shape={shape:?})",
std::mem::size_of::<T>()
))
})?;
if byte_size < logical_size {
return Err(Error::InvalidArgument(format!(
"DmaTensor::new_with_byte_size: byte_size {byte_size} < logical {logical_size} \
(shape={shape:?}, sizeof::<T>={})",
std::mem::size_of::<T>()
)));
}
let name = match name {
Some(name) => name.to_owned(),
None => {
let uuid = uuid::Uuid::new_v4().as_simple().to_string();
format!("/{}", &uuid[..16])
}
};
let heap = match dma_heap::Heap::new(dma_heap::HeapKind::Cma) {
Ok(heap) => heap,
Err(_) => dma_heap::Heap::new(dma_heap::HeapKind::System)?,
};
let dma_fd = heap.allocate(byte_size)?;
let stat = fstat(&dma_fd)?;
debug!("DMA padded memory stat: {stat:?}");
let buf_size = if stat.st_size > 0 {
std::cmp::max(stat.st_size as usize, byte_size)
} else {
byte_size
};
let drm_attachment = crate::dmabuf::DrmAttachment::new(&dma_fd, false);
Ok(DmaTensor::<T> {
name,
fd: dma_fd,
shape: shape.to_vec(),
_marker: std::marker::PhantomData,
_drm_attachment: drm_attachment,
identity: identity_from_stat(&stat),
buf_size,
mmap_offset: 0,
is_imported: false,
})
}
#[cfg(not(all(target_os = "linux", not(miri))))]
pub(crate) fn new_with_byte_size(
_shape: &[usize],
_byte_size: usize,
_name: Option<&str>,
) -> Result<Self> {
Err(Error::NotImplemented(
"DMA tensor allocation is unavailable (not Linux, or running \
under Miri, which cannot execute the real ioctls)"
.to_owned(),
))
}
#[cfg(feature = "static")]
pub(crate) fn map_with_byte_size<'a>(
&self,
byte_size: usize,
access: crate::CpuAccess,
) -> Result<crate::view::HostView<'a, T>>
where
T: 'a,
{
self.map_inner(Some(byte_size), access)
}
pub fn try_clone(&self) -> Result<Self> {
let fd = self.clone_fd()?;
#[cfg(target_os = "linux")]
let drm_attachment = if self.is_imported {
None
} else {
crate::dmabuf::DrmAttachment::new(&fd, false)
};
Ok(Self {
name: self.name.clone(),
fd,
shape: self.shape.clone(),
_marker: std::marker::PhantomData,
#[cfg(target_os = "linux")]
_drm_attachment: drm_attachment,
identity: self.identity.clone(),
buf_size: self.buf_size,
mmap_offset: self.mmap_offset,
#[cfg(target_os = "linux")]
is_imported: self.is_imported,
})
}
}
impl<T> DmaTensor<T>
where
T: Num + Clone + fmt::Debug + Send + Sync,
{
fn map_inner<'a>(
&self,
byte_size_override: Option<usize>,
access: crate::CpuAccess,
) -> Result<crate::view::HostView<'a, T>>
where
T: 'a,
{
let t_size = std::mem::size_of::<T>();
let logical_size = self.shape.iter().product::<usize>() * t_size;
if logical_size == 0 {
return Err(Error::InvalidSize(0));
}
let total_needed = self
.mmap_offset
.checked_add(logical_size)
.ok_or(Error::InvalidSize(0))?;
if total_needed > self.buf_size {
return Err(Error::InvalidSize(total_needed));
}
if t_size > 1 && !self.mmap_offset.is_multiple_of(std::mem::align_of::<T>()) {
return Err(Error::InvalidOperation(format!(
"DMA map: offset {} not aligned to align_of::<T>()={}",
self.mmap_offset,
std::mem::align_of::<T>()
)));
}
if let Some(byte_size) = byte_size_override {
if byte_size == 0 {
return Err(Error::InvalidSize(0));
}
if t_size > 1 && !byte_size.is_multiple_of(t_size) {
return Err(Error::InvalidOperation(format!(
"DMA map: byte_size {byte_size} is not a multiple of size_of::<T>()={t_size}"
)));
}
if self.mmap_offset.saturating_add(byte_size) > self.buf_size {
return Err(Error::InvalidSize(byte_size));
}
}
Ok(crate::view::HostView::new(
self.scoped_pin(access)?,
self.shape.clone(),
byte_size_override,
access,
))
}
pub(crate) fn scoped_pin<'a>(
&self,
access: crate::CpuAccess,
) -> crate::Result<crate::pin::HostPin<'a>>
where
T: 'a,
{
let mmap_size = self.buf_size;
let ptr = unsafe {
nix::sys::mman::mmap(
None,
NonZero::new(mmap_size).ok_or(Error::InvalidSize(mmap_size))?,
nix::sys::mman::ProtFlags::PROT_READ | nix::sys::mman::ProtFlags::PROT_WRITE,
nix::sys::mman::MapFlags::MAP_SHARED,
&self.fd,
0,
)?
};
let base = std::ptr::NonNull::new(ptr.as_ptr()).ok_or(Error::InvalidSize(mmap_size))?;
let mapping = std::sync::Arc::new(crate::pin::MmapOwner::new(base, mmap_size));
let fd = self.fd.try_clone()?;
crate::dmabuf::sync_access(&fd, true, access).map_err(Error::NixError)?;
let data = unsafe { mapping.base().add(self.mmap_offset) };
let len = mmap_size.saturating_sub(self.mmap_offset);
let keepalive = std::sync::Arc::new(DmaSyncBracket {
fd,
access,
_mapping: mapping,
});
Ok(crate::pin::HostPin::new(keepalive, data, len))
}
#[cfg(feature = "static")]
pub(crate) fn host_pin<'a>(&self) -> crate::Result<crate::pin::HostPin<'a>>
where
T: 'a,
{
let mmap_size = self.buf_size;
let ptr = unsafe {
nix::sys::mman::mmap(
None,
NonZero::new(mmap_size).ok_or(Error::InvalidSize(mmap_size))?,
nix::sys::mman::ProtFlags::PROT_READ | nix::sys::mman::ProtFlags::PROT_WRITE,
nix::sys::mman::MapFlags::MAP_SHARED,
&self.fd,
0,
)?
};
let base = std::ptr::NonNull::new(ptr.as_ptr()).ok_or(Error::InvalidSize(mmap_size))?;
let mapping = std::sync::Arc::new(crate::pin::MmapOwner::new(base, mmap_size));
let data = unsafe { mapping.base().add(self.mmap_offset) };
let len = mmap_size.saturating_sub(self.mmap_offset);
Ok(crate::pin::HostPin::new(mapping, data, len))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::TensorMapTrait;
#[cfg(target_os = "linux")]
fn dummy_fd() -> std::os::fd::OwnedFd {
use std::os::fd::FromRawFd;
use std::os::unix::io::IntoRawFd;
let f = std::fs::File::open("/dev/null").expect("open /dev/null");
unsafe { std::os::fd::OwnedFd::from_raw_fd(f.into_raw_fd()) }
}
#[test]
#[cfg(target_os = "linux")]
fn test_dma_map_offset_exceeds_buf_size() {
let fd = dummy_fd();
let mut t = DmaTensor::<u8>::from_fd(fd, &[4096], None).expect("import");
t.buf_size = 4096;
t.mmap_offset = 4096;
let result = t.map_inner(None, crate::CpuAccess::ReadWrite);
match result {
Err(Error::InvalidSize(n)) => assert_eq!(n, 8192),
other => panic!("expected InvalidSize(8192), got {other:?}"),
}
}
#[test]
#[cfg(target_os = "linux")]
fn test_dma_map_misaligned_offset() {
let fd = dummy_fd();
let mut t = DmaTensor::<u32>::from_fd(fd, &[1024], None).expect("import");
t.buf_size = 8192;
t.mmap_offset = 3;
let result = t.map_inner(None, crate::CpuAccess::ReadWrite);
assert!(
matches!(result, Err(Error::InvalidOperation(_))),
"expected InvalidOperation for misaligned offset, got {result:?}"
);
}
#[test]
#[cfg(target_os = "linux")]
fn test_dma_map_offset_overflow() {
let fd = dummy_fd();
let mut t = DmaTensor::<u8>::from_fd(fd, &[1], None).expect("import");
t.buf_size = 4096;
t.mmap_offset = usize::MAX;
let result = t.map_inner(None, crate::CpuAccess::ReadWrite);
match result {
Err(Error::InvalidSize(n)) => assert_eq!(n, 0),
other => panic!("expected InvalidSize(0), got {other:?}"),
}
}
#[test]
#[cfg(target_os = "linux")]
fn test_dma_map_with_offset() {
use crate::{Tensor, TensorMemory, TensorTrait};
let total_size: usize = 4096 * 4; let offset: usize = 4096; let data_size: usize = 4096;
let large_buf = match Tensor::<u8>::new(&[total_size], Some(TensorMemory::DmaBuf), None) {
Ok(buf) => buf,
Err(_) => {
eprintln!("SKIPPED: DMA not available");
return;
}
};
{
let mut map = large_buf.map().unwrap();
map.as_mut_slice().fill(0xAA);
}
let fd = large_buf.clone_fd().unwrap();
let mut offset_tensor = Tensor::<u8>::from_fd(fd, &[data_size], None).unwrap();
offset_tensor.set_plane_offset(offset);
let mut map = offset_tensor.map().unwrap();
let slice = map.as_mut_slice();
assert_eq!(slice.len(), data_size);
assert!(
slice.iter().all(|&b| b == 0xAA),
"Offset tensor map should see sentinel data at offset"
);
slice.fill(0xBB);
drop(map);
{
let map = large_buf.map().unwrap();
let buf = map.as_slice();
assert!(
buf[..offset].iter().all(|&b| b == 0xAA),
"Data before offset should be unchanged"
);
assert!(
buf[offset..offset + data_size].iter().all(|&b| b == 0xBB),
"Data at offset should be 0xBB"
);
}
}
}