#![cfg(target_os = "android")]
use crate::{
ahardwarebuffer_layout::{
checked_shape_bytes, desc_layout, image_format_and_bpe, lock_usage_for,
AHardwareBufferDesc, LockDecision, FORMAT_BLOB, USAGE_CPU_READ_MASK, USAGE_CPU_READ_OFTEN,
USAGE_CPU_WRITE_MASK, USAGE_CPU_WRITE_OFTEN,
},
error::{Error, Result},
packed_rgba16f_layout, BufferIdentity, DType, PixelFormat, TensorMap, TensorMapTrait,
TensorMemory, TensorTrait,
};
use log::trace;
use num_traits::Num;
use std::{
ffi::c_void,
fmt,
marker::PhantomData,
ops::{Deref, DerefMut},
ptr::NonNull,
sync::{Arc, Mutex},
};
pub(crate) type AHardwareBufferRef = *mut c_void;
#[repr(C)]
struct ARect {
left: i32,
top: i32,
right: i32,
bottom: i32,
}
const USAGE_GPU_SAMPLED_IMAGE: u64 = 0x100;
const USAGE_GPU_FRAMEBUFFER: u64 = 0x200;
const USAGE_GPU_DATA_BUFFER: u64 = 0x1000000;
fn cpu_usage_bits_for(access: crate::CpuAccess) -> u64 {
let mut bits = 0;
if access.reads() {
bits |= USAGE_CPU_READ_OFTEN;
}
if access.writes() {
bits |= USAGE_CPU_WRITE_OFTEN;
}
bits
}
#[link(name = "nativewindow")]
extern "C" {
fn AHardwareBuffer_allocate(
desc: *const AHardwareBufferDesc,
out_buffer: *mut AHardwareBufferRef,
) -> i32;
fn AHardwareBuffer_acquire(buffer: AHardwareBufferRef);
fn AHardwareBuffer_release(buffer: AHardwareBufferRef);
fn AHardwareBuffer_describe(buffer: AHardwareBufferRef, out_desc: *mut AHardwareBufferDesc);
fn AHardwareBuffer_lock(
buffer: AHardwareBufferRef,
usage: u64,
fence: i32,
rect: *const ARect,
out_virtual_address: *mut *mut c_void,
) -> i32;
fn AHardwareBuffer_unlock(buffer: AHardwareBufferRef, fence: *mut i32) -> i32;
fn __system_property_get(
name: *const std::os::raw::c_char,
value: *mut std::os::raw::c_char,
) -> i32;
}
fn ahardwarebuffer_get_id(buffer: AHardwareBufferRef) -> Option<u64> {
type GetIdFn = unsafe extern "C" fn(AHardwareBufferRef, *mut u64) -> i32;
static GET_ID: std::sync::OnceLock<Option<GetIdFn>> = std::sync::OnceLock::new();
let get_id = (*GET_ID.get_or_init(|| {
extern "C" {
fn dlsym(handle: *mut c_void, symbol: *const std::os::raw::c_char) -> *mut c_void;
}
#[cfg(target_pointer_width = "64")]
let rtld_default = std::ptr::null_mut();
#[cfg(target_pointer_width = "32")]
let rtld_default = 0xffff_ffffusize as *mut c_void;
let sym = unsafe { dlsym(rtld_default, c"AHardwareBuffer_getId".as_ptr()) };
if sym.is_null() {
log::debug!("AHardwareBuffer_getId unavailable (API < 31): identities stay fresh");
None
} else {
Some(unsafe { std::mem::transmute::<*mut c_void, GetIdFn>(sym) })
}
}))?;
let mut id = 0u64;
(unsafe { get_id(buffer, &mut id) } == 0).then_some(id)
}
fn interned_identity(buffer: &OwnedAHardwareBuffer) -> BufferIdentity {
let Some(key) = ahardwarebuffer_get_id(buffer.as_ptr()) else {
return BufferIdentity::new();
};
static INTERN: std::sync::LazyLock<Mutex<crate::ahardwarebuffer_layout::IdentityInternTable>> =
std::sync::LazyLock::new(|| {
Mutex::new(crate::ahardwarebuffer_layout::IdentityInternTable::new())
});
let mut table = match INTERN.lock() {
Ok(t) => t,
Err(poisoned) => poisoned.into_inner(),
};
let (id, guard, reused) = table.resolve(key, || {
let fresh = BufferIdentity::new();
(fresh.id(), fresh.guard_arc())
});
if reused {
trace!("AHardwareBuffer getId={key:#x} interned → identity {id} (EGL cache hit)");
}
BufferIdentity::from_parts(id, guard)
}
pub(crate) fn device_compression_scheme() -> Option<crate::CompressionScheme> {
static SCHEME: std::sync::OnceLock<Option<crate::CompressionScheme>> =
std::sync::OnceLock::new();
*SCHEME.get_or_init(|| {
const PROP_VALUE_MAX: usize = 92;
let mut buf = [0u8; PROP_VALUE_MAX];
let len = unsafe {
__system_property_get(
c"ro.hardware.egl".as_ptr(),
buf.as_mut_ptr() as *mut std::os::raw::c_char,
)
};
let vendor = usize::try_from(len)
.ok()
.filter(|&n| n <= PROP_VALUE_MAX)
.and_then(|n| std::str::from_utf8(&buf[..n]).ok())
.unwrap_or("");
let scheme = crate::ahardwarebuffer_layout::scheme_for_egl_vendor(vendor);
log::debug!("ro.hardware.egl={vendor:?} → compression scheme {scheme:?}");
scheme
})
}
#[derive(Debug, Clone)]
pub(crate) struct OwnedAHardwareBuffer {
inner: Arc<AhbHandle>,
}
#[derive(Debug)]
struct AhbHandle {
ptr: AHardwareBufferRef,
lock: Mutex<LockState>,
}
#[derive(Debug, Default)]
struct LockState {
count: usize,
base: *mut c_void,
usage: u64,
}
unsafe impl Send for AhbHandle {}
unsafe impl Sync for AhbHandle {}
impl AhbHandle {
fn new(ptr: AHardwareBufferRef) -> Self {
Self {
ptr,
lock: Mutex::new(LockState::default()),
}
}
}
impl Drop for AhbHandle {
fn drop(&mut self) {
if !self.ptr.is_null() {
unsafe { AHardwareBuffer_release(self.ptr) };
}
}
}
impl OwnedAHardwareBuffer {
fn allocate(mut desc: AHardwareBufferDesc) -> Result<(Self, AHardwareBufferDesc)> {
let mut ptr: AHardwareBufferRef = std::ptr::null_mut();
let rc = unsafe { AHardwareBuffer_allocate(&desc, &mut ptr) };
if rc != 0 || ptr.is_null() {
return Err(Error::IoError(std::io::Error::other(format!(
"AHardwareBuffer_allocate failed (rc={rc}, format=0x{:x}, {}x{})",
desc.format, desc.width, desc.height
))));
}
unsafe { AHardwareBuffer_describe(ptr, &mut desc) };
Ok((
Self {
inner: Arc::new(AhbHandle::new(ptr)),
},
desc,
))
}
unsafe fn from_external(ptr: AHardwareBufferRef) -> Result<(Self, AHardwareBufferDesc)> {
if ptr.is_null() {
return Err(Error::InvalidArgument(
"from_external: null AHardwareBuffer".into(),
));
}
AHardwareBuffer_acquire(ptr);
let mut desc = AHardwareBufferDesc {
width: 0,
height: 0,
layers: 0,
format: 0,
usage: 0,
stride: 0,
rfu0: 0,
rfu1: 0,
};
AHardwareBuffer_describe(ptr, &mut desc);
Ok((
Self {
inner: Arc::new(AhbHandle::new(ptr)),
},
desc,
))
}
pub(crate) fn as_ptr(&self) -> AHardwareBufferRef {
self.inner.ptr
}
fn lock_shared(&self, cpu_usage: u64) -> Result<*mut c_void> {
let mut st = self.inner.lock.lock().unwrap_or_else(|e| e.into_inner());
if st.count > 0 {
if cpu_usage & st.usage != cpu_usage {
return Err(Error::InvalidOperation(format!(
"concurrent AHardwareBuffer map with wider access \
(live lock usage {:#x}, requested {:#x}) — take the wider \
map first or drop the narrower one",
st.usage, cpu_usage
)));
}
st.count += 1;
return Ok(st.base);
}
let mut base: *mut c_void = std::ptr::null_mut();
let rc = unsafe {
AHardwareBuffer_lock(self.inner.ptr, cpu_usage, -1, std::ptr::null(), &mut base)
};
if rc != 0 {
return Err(Error::IoError(std::io::Error::other(format!(
"AHardwareBuffer_lock failed (rc={rc})"
))));
}
if base.is_null() {
unsafe { AHardwareBuffer_unlock(self.inner.ptr, std::ptr::null_mut()) };
return Err(Error::IoError(std::io::Error::other(
"AHardwareBuffer_lock returned a null address",
)));
}
st.base = base;
st.count = 1;
st.usage = cpu_usage;
Ok(base)
}
fn unlock_shared(&self) {
let mut st = self.inner.lock.lock().unwrap_or_else(|e| e.into_inner());
match st.count {
0 => log::error!("unbalanced AHardwareBuffer unlock (count already 0)"),
1 => {
let rc = unsafe { AHardwareBuffer_unlock(self.inner.ptr, std::ptr::null_mut()) };
if rc != 0 {
log::error!("AHardwareBuffer_unlock failed (rc={rc})");
}
st.count = 0;
st.base = std::ptr::null_mut();
}
_ => st.count -= 1,
}
}
}
#[derive(Debug)]
pub struct AHardwareBufferTensor<T>
where
T: Num + Clone + fmt::Debug + Send + Sync,
{
pub name: String,
pub(crate) buffer: OwnedAHardwareBuffer,
pub shape: Vec<usize>,
pub _marker: PhantomData<T>,
identity: BufferIdentity,
pub(crate) buf_size: usize,
pub(crate) bytes_per_row: usize,
physical_dims: (usize, usize),
cpu_usage: u64,
pub(crate) is_imported: bool,
pub(crate) view_offset: usize,
}
impl<T> TensorTrait<T> for AHardwareBufferTensor<T>
where
T: Num + Clone + fmt::Debug + Send + Sync,
{
fn new(shape: &[usize], name: Option<&str>) -> Result<Self> {
let byte_size = checked_shape_bytes::<T>(shape)?;
Self::new_with_byte_size(shape, byte_size, name)
}
fn from_fd(_fd: std::os::fd::OwnedFd, _shape: &[usize], _name: Option<&str>) -> Result<Self> {
Err(Error::NotImplemented(
"AHardwareBufferTensor::from_fd: AHardwareBuffer is not fd-backed; \
use from_hardware_buffer()"
.into(),
))
}
fn clone_fd(&self) -> Result<std::os::fd::OwnedFd> {
Err(Error::NotImplemented(
"AHardwareBufferTensor::clone_fd: share the AHardwareBuffer handle \
(hardware_buffer_ptr) via binder instead"
.into(),
))
}
fn memory(&self) -> TensorMemory {
TensorMemory::Dma
}
fn name(&self) -> String {
self.name.clone()
}
fn shape(&self) -> &[usize] {
&self.shape
}
fn reshape(&mut self, shape: &[usize]) -> Result<()> {
let new_bytes = checked_shape_bytes::<T>(shape)?;
let cur_bytes = checked_shape_bytes::<T>(&self.shape)?;
if new_bytes != cur_bytes {
return Err(Error::InvalidShape(format!(
"reshape: byte footprint mismatch ({cur_bytes} → {new_bytes})",
)));
}
self.shape = shape.to_vec();
self.view_offset = 0;
Ok(())
}
fn map_with(&self, access: crate::CpuAccess) -> Result<TensorMap<T>> {
let _span =
tracing::trace_span!("tensor.map", memory = "ahardwarebuffer", ?access).entered();
let m = AHardwareBufferMap::new(
self.buffer.clone(),
self.shape.clone(),
self.buf_size,
self.cpu_usage,
self.view_offset,
access,
self.identity.id(),
)?;
Ok(TensorMap::HardwareBuffer(m))
}
fn buffer_identity(&self) -> &BufferIdentity {
&self.identity
}
fn capacity_bytes(&self) -> usize {
self.buf_size
}
fn set_logical_shape(&mut self, shape: &[usize]) -> Result<()> {
if shape.is_empty() {
return Err(Error::InvalidSize(0));
}
let needed = checked_shape_bytes::<T>(shape)?;
let capacity = self.buf_size.saturating_sub(self.view_offset);
if needed > capacity {
return Err(Error::InsufficientCapacity { needed, capacity });
}
self.shape = shape.to_vec();
Ok(())
}
fn view(&self, offset_bytes: usize, shape: &[usize]) -> Result<Self> {
if !offset_bytes.is_multiple_of(std::mem::align_of::<T>()) {
return Err(Error::InvalidOperation(format!(
"AHardwareBufferTensor::view: offset {offset_bytes} not aligned to \
align_of::<T>()={}",
std::mem::align_of::<T>()
)));
}
let abs_offset = self
.view_offset
.checked_add(offset_bytes)
.ok_or(Error::InvalidSize(offset_bytes))?;
let logical = checked_shape_bytes::<T>(shape)?;
let needed = abs_offset
.checked_add(logical)
.ok_or(Error::InvalidSize(logical))?;
if needed > self.buf_size {
return Err(Error::InsufficientCapacity {
needed,
capacity: self.buf_size,
});
}
Ok(Self {
name: self.name.clone(),
buffer: self.buffer.clone(),
shape: shape.to_vec(),
_marker: PhantomData,
identity: self.identity.clone(),
buf_size: self.buf_size,
bytes_per_row: self.bytes_per_row,
physical_dims: self.physical_dims,
cpu_usage: self.cpu_usage,
is_imported: self.is_imported,
view_offset: abs_offset,
})
}
}
impl<T> AHardwareBufferTensor<T>
where
T: Num + Clone + fmt::Debug + Send + Sync,
{
pub(crate) fn new_with_byte_size(
shape: &[usize],
byte_size: usize,
name: Option<&str>,
) -> Result<Self> {
let _span =
tracing::trace_span!("tensor.alloc", memory = "ahardwarebuffer", byte_size,).entered();
let len: u32 = byte_size
.max(1)
.try_into()
.map_err(|_| Error::InvalidSize(byte_size))?;
let desc = AHardwareBufferDesc {
width: len,
height: 1,
layers: 1,
format: FORMAT_BLOB,
usage: USAGE_CPU_READ_OFTEN | USAGE_CPU_WRITE_OFTEN | USAGE_GPU_DATA_BUFFER,
stride: 0,
rfu0: 0,
rfu1: 0,
};
let (buffer, desc) = OwnedAHardwareBuffer::allocate(desc)?;
let (bytes_per_row, buf_size) = desc_layout(&desc).expect("BLOB layout is always known");
let name = match name {
Some(s) => s.to_owned(),
None => format!("ahardwarebuffer-{}", uuid::Uuid::new_v4()),
};
trace!("AHardwareBufferTensor::new: name={name} bytes={buf_size} shape={shape:?}",);
Ok(Self {
name,
identity: interned_identity(&buffer),
buffer,
shape: shape.to_vec(),
_marker: PhantomData,
buf_size,
bytes_per_row,
physical_dims: (desc.width as usize, desc.height as usize),
cpu_usage: desc.usage & (USAGE_CPU_READ_MASK | USAGE_CPU_WRITE_MASK),
is_imported: false,
view_offset: 0,
})
}
pub(crate) fn new_image(
width: usize,
height: usize,
format: PixelFormat,
dtype: DType,
shape: &[usize],
name: Option<&str>,
access: crate::CpuAccess,
) -> Result<Self> {
let _span = tracing::trace_span!(
"tensor.alloc",
memory = "ahardwarebuffer",
width,
height,
?format,
)
.entered();
let (ahb_format, bpe) = image_format_and_bpe(format, dtype).ok_or_else(|| {
Error::NotImplemented(format!(
"AHardwareBufferTensor::new_image: ({format:?}, {dtype:?}) has no \
AHardwareBuffer format mapping"
))
})?;
let (surface_width, surface_height) = match (format, dtype) {
(PixelFormat::PlanarRgb | PixelFormat::PlanarRgba, DType::F16) => {
let layout =
packed_rgba16f_layout(format, dtype, width, height).ok_or_else(|| {
Error::InvalidShape(format!(
"{format:?} F16 AHardwareBuffer requires width%4==0 for RGBA16F \
packing (got width={width})"
))
})?;
(layout.surface_w, layout.surface_h)
}
(PixelFormat::Rgb, DType::U8 | DType::I8) => {
let layout = crate::packed_rgb888_layout(width, height).ok_or_else(|| {
Error::InvalidShape(format!(
"Rgb u8/i8 AHardwareBuffer requires width%4==0 for RGBA8888 \
packing (got width={width})"
))
})?;
(layout.surface_w, layout.surface_h)
}
_ => (width, height),
};
let desc = AHardwareBufferDesc {
width: surface_width
.try_into()
.map_err(|_| Error::InvalidSize(surface_width))?,
height: surface_height
.try_into()
.map_err(|_| Error::InvalidSize(surface_height))?,
layers: 1,
format: ahb_format,
usage: USAGE_GPU_SAMPLED_IMAGE | USAGE_GPU_FRAMEBUFFER | cpu_usage_bits_for(access),
stride: 0,
rfu0: 0,
rfu1: 0,
};
let (buffer, desc) = match OwnedAHardwareBuffer::allocate(desc) {
Ok(ok) => ok,
Err(e) if access == crate::CpuAccess::None => {
log::warn!(
"AHardwareBuffer hardware-only allocation failed for \
{format:?}/{dtype:?} {width}x{height} ({e:?}); retrying \
with CPU access bits (linear layout)"
);
let retry = AHardwareBufferDesc {
usage: USAGE_GPU_SAMPLED_IMAGE
| USAGE_GPU_FRAMEBUFFER
| USAGE_CPU_READ_OFTEN
| USAGE_CPU_WRITE_OFTEN,
..desc
};
OwnedAHardwareBuffer::allocate(retry)?
}
Err(e) => return Err(e),
};
let (bytes_per_row, buf_size) = desc_layout(&desc).ok_or_else(|| {
Error::InvalidShape(format!(
"AHardwareBuffer layout overflow ({surface_width}x{surface_height}, bpe={bpe})"
))
})?;
let name = match name {
Some(s) => s.to_owned(),
None => format!("ahardwarebuffer-img-{}", uuid::Uuid::new_v4()),
};
trace!(
"AHardwareBufferTensor::new_image: name={name} surface={surface_width}x\
{surface_height} logical={width}x{height} {format:?}/{dtype:?} \
format=0x{ahb_format:x} stride_px={} bytes={buf_size} bytes_per_row={bytes_per_row}",
desc.stride,
);
Ok(Self {
name,
identity: interned_identity(&buffer),
buffer,
shape: shape.to_vec(),
_marker: PhantomData,
buf_size,
bytes_per_row,
physical_dims: (desc.width as usize, desc.height as usize),
cpu_usage: desc.usage & (USAGE_CPU_READ_MASK | USAGE_CPU_WRITE_MASK),
is_imported: false,
view_offset: 0,
})
}
pub unsafe fn from_hardware_buffer(
buffer_ptr: *mut c_void,
shape: &[usize],
name: Option<&str>,
) -> Result<Self> {
let (buffer, desc) = OwnedAHardwareBuffer::from_external(buffer_ptr)?;
let (bytes_per_row, buf_size) = desc_layout(&desc).ok_or_else(|| {
Error::NotImplemented(format!(
"from_hardware_buffer: AHardwareBuffer format 0x{:x} has no linear CPU \
layout the HAL supports (multi-planar YUV import is a planned follow-up)",
desc.format
))
})?;
let requested = checked_shape_bytes::<T>(shape)?;
if requested > buf_size {
return Err(Error::InvalidShape(format!(
"from_hardware_buffer: shape requires {requested} bytes but the buffer only \
has {buf_size} (shape={shape:?}, sizeof T={})",
std::mem::size_of::<T>(),
)));
}
let name = match name {
Some(s) => s.to_owned(),
None => format!("ahardwarebuffer-imported-{}", uuid::Uuid::new_v4()),
};
Ok(Self {
name,
identity: interned_identity(&buffer),
buffer,
shape: shape.to_vec(),
_marker: PhantomData,
buf_size,
bytes_per_row,
physical_dims: (desc.width as usize, desc.height as usize),
cpu_usage: desc.usage & (USAGE_CPU_READ_MASK | USAGE_CPU_WRITE_MASK),
is_imported: true,
view_offset: 0,
})
}
pub(crate) fn bytes_per_row(&self) -> usize {
self.bytes_per_row
}
pub(crate) fn physical_surface_dims(&self) -> (usize, usize) {
self.physical_dims
}
pub(crate) fn image_backing_row_stride(&self) -> Option<usize> {
let (_w, h) = self.physical_dims;
(h > 1).then_some(self.bytes_per_row)
}
pub(crate) fn map_with_byte_size(
&self,
byte_size: usize,
access: crate::CpuAccess,
) -> Result<TensorMap<T>> {
let m = AHardwareBufferMap::new_with_byte_size(
self.buffer.clone(),
self.shape.clone(),
self.buf_size,
byte_size,
self.cpu_usage,
self.view_offset,
access,
self.identity.id(),
)?;
Ok(TensorMap::HardwareBuffer(m))
}
pub fn hardware_buffer_ptr(&self) -> *mut c_void {
self.buffer.as_ptr()
}
}
impl<T> crate::Tensor<T>
where
T: Num + Clone + fmt::Debug + Send + Sync,
{
pub fn hardware_buffer_ptr(&self) -> Option<*mut c_void> {
match &self.storage {
crate::TensorStorage::Dma(ahb) => Some(ahb.hardware_buffer_ptr()),
_ => None,
}
}
pub fn hardware_buffer_physical_dims(&self) -> Option<(usize, usize)> {
match &self.storage {
crate::TensorStorage::Dma(ahb) => Some(ahb.physical_surface_dims()),
_ => None,
}
}
pub unsafe fn from_hardware_buffer(
buffer_ptr: *mut c_void,
shape: &[usize],
name: Option<&str>,
) -> Result<Self>
where
T: num_traits::Num,
{
let inner =
unsafe { AHardwareBufferTensor::<T>::from_hardware_buffer(buffer_ptr, shape, name)? };
let declared = match (
inner.cpu_usage & USAGE_CPU_READ_MASK != 0,
inner.cpu_usage & USAGE_CPU_WRITE_MASK != 0,
) {
(true, true) => crate::CpuAccess::ReadWrite,
(true, false) => crate::CpuAccess::Read,
(false, true) => crate::CpuAccess::Write,
(false, false) => crate::CpuAccess::None,
};
let mut t = crate::Tensor::wrap(crate::TensorStorage::Dma(inner));
t.set_cpu_access_unchecked(declared);
Ok(t)
}
}
#[derive(Debug)]
pub struct AHardwareBufferMap<T>
where
T: Num + Clone + fmt::Debug,
{
buffer: OwnedAHardwareBuffer,
shape: Vec<usize>,
base_ptr: NonNull<c_void>,
buf_size: usize,
byte_size_override: Option<usize>,
_marker: PhantomData<T>,
locked: bool,
writable: bool,
}
unsafe impl<T> Send for AHardwareBufferMap<T> where T: Num + Clone + fmt::Debug {}
unsafe impl<T> Sync for AHardwareBufferMap<T> where T: Num + Clone + fmt::Debug {}
impl<T> AHardwareBufferMap<T>
where
T: Num + Clone + fmt::Debug,
{
#[allow(clippy::too_many_arguments)]
fn new(
buffer: OwnedAHardwareBuffer,
shape: Vec<usize>,
buf_size: usize,
cpu_usage: u64,
view_offset: usize,
access: crate::CpuAccess,
identity_id: u64,
) -> Result<Self> {
Self::new_inner(
buffer,
shape,
buf_size,
None,
cpu_usage,
view_offset,
access,
identity_id,
)
}
#[allow(clippy::too_many_arguments)]
fn new_with_byte_size(
buffer: OwnedAHardwareBuffer,
shape: Vec<usize>,
buf_size: usize,
byte_size: usize,
cpu_usage: u64,
view_offset: usize,
access: crate::CpuAccess,
identity_id: u64,
) -> Result<Self> {
Self::new_inner(
buffer,
shape,
buf_size,
Some(byte_size),
cpu_usage,
view_offset,
access,
identity_id,
)
}
#[allow(clippy::too_many_arguments)]
fn new_inner(
buffer: OwnedAHardwareBuffer,
shape: Vec<usize>,
buf_size: usize,
byte_size_override: Option<usize>,
cpu_usage: u64,
view_offset: usize,
access: crate::CpuAccess,
identity_id: u64,
) -> Result<Self> {
let declared_writes = cpu_usage & USAGE_CPU_WRITE_MASK != 0;
let lock_usage = match lock_usage_for(cpu_usage, access.reads(), access.writes()) {
LockDecision::Refuse => {
let _ = identity_id;
return Err(Error::InvalidOperation(
"AHardwareBuffer was allocated without CPU access \
(CpuAccess::None) — allocate with CpuAccess::Read/Write/\
ReadWrite to map it on the CPU"
.into(),
));
}
LockDecision::Covered(bits) => bits,
LockDecision::Unplanned(bits) => {
log::debug!(
"AHardwareBuffer map exceeds declared access; replaying \
allocated usage {bits:#x}"
);
bits
}
};
if view_offset > buf_size {
return Err(Error::InsufficientCapacity {
needed: view_offset,
capacity: buf_size,
});
}
let window = buf_size - view_offset;
let exposed_bytes = match byte_size_override {
Some(bytes) => bytes,
None => checked_shape_bytes::<T>(&shape)?,
};
if exposed_bytes > window {
return Err(Error::InsufficientCapacity {
needed: exposed_bytes,
capacity: window,
});
}
let base = buffer.lock_shared(lock_usage)?;
let base_ptr = NonNull::new(base).expect("lock_shared validates the base address");
let base_ptr = NonNull::new(unsafe { base_ptr.as_ptr().byte_add(view_offset) })
.expect("offset base within locked buffer is non-null");
Ok(Self {
buffer,
shape,
base_ptr,
buf_size: buf_size - view_offset,
byte_size_override,
_marker: PhantomData,
locked: true,
writable: access.writes() && declared_writes,
})
}
fn elem_count(&self) -> usize {
match self.byte_size_override {
Some(bytes) => bytes / std::mem::size_of::<T>(),
None => self.shape.iter().product(),
}
}
}
impl<T> TensorMapTrait<T> for AHardwareBufferMap<T>
where
T: Num + Clone + fmt::Debug,
{
fn shape(&self) -> &[usize] {
&self.shape
}
fn len(&self) -> usize {
self.elem_count()
}
fn unmap(&mut self) {
if self.locked {
self.buffer.unlock_shared();
self.locked = false;
}
}
fn as_slice(&self) -> &[T] {
self.deref()
}
fn as_mut_slice(&mut self) -> &mut [T] {
self.deref_mut()
}
}
impl<T> Deref for AHardwareBufferMap<T>
where
T: Num + Clone + fmt::Debug,
{
type Target = [T];
fn deref(&self) -> &[T] {
let ptr = self.base_ptr.as_ptr() as *const T;
let len = self.elem_count();
debug_assert!(
len * std::mem::size_of::<T>() <= self.buf_size,
"AHardwareBufferMap deref: {} elems × {} bytes > buf_size {}",
len,
std::mem::size_of::<T>(),
self.buf_size,
);
unsafe { std::slice::from_raw_parts(ptr, len) }
}
}
impl<T> DerefMut for AHardwareBufferMap<T>
where
T: Num + Clone + fmt::Debug,
{
fn deref_mut(&mut self) -> &mut [T] {
crate::assert_map_writable(self.writable, "AHardwareBuffer");
let ptr = self.base_ptr.as_ptr() as *mut T;
let len = self.elem_count();
debug_assert!(
len * std::mem::size_of::<T>() <= self.buf_size,
"AHardwareBufferMap deref_mut: {} elems × {} bytes > buf_size {}",
len,
std::mem::size_of::<T>(),
self.buf_size,
);
unsafe { std::slice::from_raw_parts_mut(ptr, len) }
}
}
impl<T> Drop for AHardwareBufferMap<T>
where
T: Num + Clone + fmt::Debug,
{
fn drop(&mut self) {
self.unmap();
}
}
pub fn image_ahardwarebuffer_layout(format: PixelFormat, dtype: DType) -> Option<(u32, usize)> {
image_format_and_bpe(format, dtype)
}