use std::{
alloc::{Layout, alloc, alloc_zeroed, dealloc, handle_alloc_error},
borrow::{Borrow, BorrowMut},
fmt::{self, Debug, Formatter},
mem::{ManuallyDrop, MaybeUninit},
ops::{Deref, DerefMut},
ptr::{NonNull, copy_nonoverlapping},
slice::{from_raw_parts, from_raw_parts_mut},
sync::Arc,
};
use compio_buf::{IoBuf, IoBufMut, SetLen};
use super::{BufMeta, BufferPool, CachedBuf, validate_sector_size};
use crate::{
align::DEFAULT_SECTOR_SIZE,
error::{Error, Result},
};
pub struct AlignedBuf {
ptr: NonNull<u8>,
len: usize,
cap: usize,
align: usize,
pooled: Option<(Arc<BufferPool>, BufMeta)>,
}
unsafe impl Send for AlignedBuf {}
unsafe impl Sync for AlignedBuf {}
#[inline]
fn dangling(align: usize) -> NonNull<u8> {
unsafe { NonNull::new_unchecked(align as *mut u8) }
}
impl AlignedBuf {
pub fn new(cap: usize, align: usize) -> Result<Self> {
validate_sector_size(align)?;
if cap == 0 {
return Ok(Self {
ptr: dangling(align),
len: 0,
cap: 0,
align,
pooled: None,
});
}
let layout = Layout::from_size_align(cap, align)?;
let Some(ptr) = NonNull::new(unsafe { alloc_zeroed(layout) }) else {
return Err(Error::AllocFailed(layout));
};
Ok(Self {
ptr,
len: 0,
cap,
align,
pooled: None,
})
}
#[inline]
pub fn with_sector_size(cap: usize) -> Result<Self> {
Self::new(cap, DEFAULT_SECTOR_SIZE)
}
pub fn from_slice(data: &[u8], align: usize) -> Result<Self> {
validate_sector_size(align)?;
let len = data.len();
if len == 0 {
return Self::new(0, align);
}
let layout = Layout::from_size_align(len, align)?;
let Some(ptr) = NonNull::new(unsafe { alloc(layout) }) else {
return Err(Error::AllocFailed(layout));
};
unsafe {
copy_nonoverlapping(data.as_ptr(), ptr.as_ptr(), len);
}
Ok(Self {
ptr,
len,
cap: len,
align,
pooled: None,
})
}
pub fn zeroed(cap: usize, align: usize) -> Result<Self> {
let mut buf = Self::new(cap, align)?;
buf.len = cap;
Ok(buf)
}
#[inline]
pub fn len(&self) -> usize {
self.len
}
#[inline]
pub fn capacity(&self) -> usize {
self.cap
}
#[inline]
pub fn align(&self) -> usize {
self.align
}
#[inline]
pub fn is_empty(&self) -> bool {
self.len == 0
}
#[inline]
pub fn clear(&mut self) {
self.len = 0;
}
pub fn set_len(&mut self, new_len: usize) -> Result<()> {
if new_len > self.cap {
return Err(Error::SetLenExceeded {
len: new_len,
capacity: self.cap,
});
}
self.len = new_len;
Ok(())
}
#[inline]
pub fn as_slice(&self) -> &[u8] {
if self.len == 0 {
&[]
} else {
unsafe { from_raw_parts(self.ptr.as_ptr(), self.len) }
}
}
#[inline]
pub fn as_mut_slice(&mut self) -> &mut [u8] {
if self.len == 0 {
&mut []
} else {
unsafe { from_raw_parts_mut(self.ptr.as_ptr(), self.len) }
}
}
#[inline]
pub fn as_allocated_slice(&self) -> &[u8] {
if self.cap == 0 {
&[]
} else {
unsafe { from_raw_parts(self.ptr.as_ptr(), self.cap) }
}
}
#[inline]
pub fn as_allocated_slice_mut(&mut self) -> &mut [u8] {
if self.cap == 0 {
&mut []
} else {
unsafe { from_raw_parts_mut(self.ptr.as_ptr(), self.cap) }
}
}
#[inline]
#[must_use]
pub fn required_len(&self) -> usize {
self
.pooled
.as_ref()
.map_or(self.cap, |(_, meta)| meta.required)
}
#[inline]
pub(crate) fn set_required(&mut self, required: usize) {
if let Some((_, meta)) = self.pooled.as_mut() {
meta.required = required;
}
}
#[inline]
#[must_use]
pub fn clear_on_return(&self) -> bool {
self
.pooled
.as_ref()
.is_none_or(|(_, meta)| meta.clear_on_return)
}
pub(crate) fn from_cached(node: CachedBuf, pool: Arc<BufferPool>, meta: BufMeta) -> Self {
let node = ManuallyDrop::new(node);
Self {
ptr: node.ptr,
len: 0,
cap: node.cap,
align: node.align,
pooled: Some((pool, meta)),
}
}
pub(crate) fn attach(&mut self, pool: Arc<BufferPool>, meta: BufMeta) {
self.pooled = Some((pool, meta));
}
#[inline]
pub fn is_aligned_to(&self, align: usize) -> bool {
if align == 0 || !align.is_power_of_two() {
return false;
}
((self.ptr.as_ptr() as usize) & (align - 1)) == 0
}
#[inline]
pub fn is_ptr_aligned(&self) -> bool {
self.is_aligned_to(self.align)
}
#[inline]
pub fn as_mut_buf_ptr(&mut self) -> *mut u8 {
self.ptr.as_ptr()
}
#[inline]
pub unsafe fn set_len_unchecked(&mut self, len: usize) {
debug_assert!(len <= self.cap);
self.len = len;
}
}
impl Drop for AlignedBuf {
fn drop(&mut self) {
if self.cap == 0 {
return;
}
if let Some((pool, meta)) = self.pooled.take() {
pool.return_buf(self.ptr, self.cap, self.align, meta);
return;
}
unsafe {
let layout = Layout::from_size_align_unchecked(self.cap, self.align);
dealloc(self.ptr.as_ptr(), layout);
}
}
}
impl Deref for AlignedBuf {
type Target = [u8];
#[inline]
fn deref(&self) -> &Self::Target {
self.as_slice()
}
}
impl DerefMut for AlignedBuf {
#[inline]
fn deref_mut(&mut self) -> &mut Self::Target {
self.as_mut_slice()
}
}
impl AsRef<[u8]> for AlignedBuf {
#[inline]
fn as_ref(&self) -> &[u8] {
self.as_slice()
}
}
impl AsMut<[u8]> for AlignedBuf {
#[inline]
fn as_mut(&mut self) -> &mut [u8] {
self.as_mut_slice()
}
}
impl Borrow<[u8]> for AlignedBuf {
#[inline]
fn borrow(&self) -> &[u8] {
self.as_slice()
}
}
impl BorrowMut<[u8]> for AlignedBuf {
#[inline]
fn borrow_mut(&mut self) -> &mut [u8] {
self.as_mut_slice()
}
}
impl Clone for AlignedBuf {
fn clone(&self) -> Self {
if self.cap == 0 {
return Self {
ptr: dangling(self.align),
len: 0,
cap: 0,
align: self.align,
pooled: None,
};
}
let layout = unsafe { Layout::from_size_align_unchecked(self.cap, self.align) };
let raw = unsafe { alloc_zeroed(layout) };
let Some(ptr) = NonNull::new(raw) else {
handle_alloc_error(layout);
};
if self.len > 0 {
unsafe {
copy_nonoverlapping(self.ptr.as_ptr(), ptr.as_ptr(), self.len);
}
}
Self {
ptr,
len: self.len,
cap: self.cap,
align: self.align,
pooled: None,
}
}
}
impl PartialEq for AlignedBuf {
#[inline]
fn eq(&self, other: &Self) -> bool {
self.as_slice() == other.as_slice()
}
}
impl Eq for AlignedBuf {}
impl PartialEq<[u8]> for AlignedBuf {
#[inline]
fn eq(&self, other: &[u8]) -> bool {
self.as_slice() == other
}
}
impl PartialEq<&[u8]> for AlignedBuf {
#[inline]
fn eq(&self, other: &&[u8]) -> bool {
self.as_slice() == *other
}
}
impl Debug for AlignedBuf {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
f.debug_struct("AlignedBuf")
.field("len", &self.len)
.field("cap", &self.cap)
.field("required", &self.required_len())
.field("align", &self.align)
.field("pooled", &self.pooled.is_some())
.field("ptr", &self.ptr)
.finish()
}
}
impl IoBuf for AlignedBuf {
#[inline]
fn as_init(&self) -> &[u8] {
self.as_slice()
}
#[inline]
fn buf_len(&self) -> usize {
self.len
}
#[inline]
fn buf_ptr(&self) -> *const u8 {
self.ptr.as_ptr()
}
#[inline]
fn is_empty(&self) -> bool {
self.len == 0
}
}
impl SetLen for AlignedBuf {
#[inline]
unsafe fn set_len(&mut self, len: usize) {
unsafe { self.set_len_unchecked(len) };
}
}
impl IoBufMut for AlignedBuf {
#[inline]
fn as_uninit(&mut self) -> &mut [MaybeUninit<u8>] {
if self.cap == 0 {
&mut []
} else {
unsafe { from_raw_parts_mut(self.ptr.as_ptr() as *mut MaybeUninit<u8>, self.cap) }
}
}
#[inline]
fn buf_capacity(&mut self) -> usize {
self.cap
}
#[inline]
fn buf_mut_ptr(&mut self) -> *mut MaybeUninit<u8> {
self.ptr.as_ptr() as *mut MaybeUninit<u8>
}
}