#![allow(unsafe_code)]
use core::cell::Cell;
use core::ffi::c_void;
use core::ptr::NonNull;
use core::sync::atomic::Ordering;
use crate::core::line::{CompletionLine, Slot};
pub(crate) const PAGE: usize = 4096;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum LineError {
Status(i32),
Unavailable,
NoDomain,
ForeignStream,
}
impl LineError {
#[must_use]
pub fn status(self) -> i32 {
match self {
Self::Status(s) => s,
Self::Unavailable => -1,
Self::NoDomain => -3,
Self::ForeignStream => -4,
}
}
}
impl core::fmt::Display for LineError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
Self::Status(s) => write!(f, "driver status {s}"),
Self::Unavailable => f.write_str("the driver or one of its entry points is missing"),
Self::NoDomain => f.write_str("no current context on this thread"),
Self::ForeignStream => f.write_str("the stream belongs to another context"),
}
}
}
impl core::error::Error for LineError {}
impl From<LineError> for crate::Error {
fn from(e: LineError) -> Self {
crate::error!(Backend, "completion line", e.status())
}
}
pub(crate) trait LineDriver {
type Stream: Copy;
fn domain_current(&self) -> Result<u64, LineError>;
fn alloc_host(&self, bytes: usize) -> Result<NonNull<c_void>, LineError>;
fn device_address(&self, host: NonNull<c_void>) -> Result<u64, LineError>;
fn free_host(&self, host: NonNull<c_void>);
fn stream_domain(&self, stream: Self::Stream) -> Result<u64, LineError>;
fn write_value(&self, stream: Self::Stream, addr: u64, value: u64) -> Result<(), LineError>;
fn settle(&self, domain: u64) -> bool;
}
pub(crate) struct Line<D: LineDriver> {
host: NonNull<CompletionLine>,
device: u64,
driver: D,
domain: u64,
queued: Cell<bool>,
}
unsafe impl<D: LineDriver + Send> Send for Line<D> {}
impl<D: LineDriver> Line<D> {
pub(crate) fn with_driver(driver: D) -> Result<Self, LineError> {
let domain = driver.domain_current();
let page = driver.alloc_host(PAGE)?;
let device = driver.device_address(page);
let line = Self {
host: page.cast::<CompletionLine>(),
device: device.unwrap_or(0),
driver,
domain: domain.unwrap_or(0),
queued: Cell::new(false),
};
device?;
if domain? == 0 {
return Err(LineError::NoDomain);
}
for w in &line.line().words {
w.store(0, Ordering::Release);
}
Ok(line)
}
#[cfg(feature = "cuda")]
pub(crate) fn driver(&self) -> &D {
&self.driver
}
pub(crate) fn device_address(&self) -> u64 {
self.device
}
pub(crate) fn line(&self) -> &CompletionLine {
unsafe { self.host.as_ref() }
}
pub(crate) fn enqueue_store(
&self,
stream: D::Stream,
slot: Slot,
value: u64,
) -> Result<(), LineError> {
let offset = 8 * u64::try_from(slot.index()).map_err(|_| LineError::Unavailable)?;
if self.driver.stream_domain(stream)? != self.domain {
return Err(LineError::ForeignStream);
}
self.queued.set(true);
self.driver.write_value(stream, self.device + offset, value)
}
fn settled(&self) -> bool {
!self.queued.get() || self.driver.settle(self.domain)
}
}
impl<D: LineDriver> Drop for Line<D> {
fn drop(&mut self) {
if !self.settled() {
return;
}
self.driver.free_host(self.host.cast::<c_void>());
}
}
#[cfg(test)]
#[path = "mapped_tests.rs"]
mod tests;