use super::{
ControlOp, CpError, EffIndex, Location, MaybeUninit, PhantomData, PolicyMode, RendezvousId,
ResourceScope, UnsafeCell, fmt,
};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum DecisionArm {
Left,
Right,
}
impl DecisionArm {
#[inline]
pub const fn index(self) -> u8 {
match self {
Self::Left => 0,
Self::Right => 1,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum DecisionResolution {
Arm(DecisionArm),
Defer,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum DynamicPolicyResolution {
DecisionArm { arm: u8 },
Defer,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct ResolverErrorLocation {
location: &'static Location<'static>,
}
impl ResolverErrorLocation {
#[inline]
#[track_caller]
pub(crate) fn caller() -> Self {
Self {
location: Location::caller(),
}
}
#[inline]
const fn file(self) -> &'static str {
self.location.file()
}
#[inline]
const fn line(self) -> u32 {
self.location.line()
}
#[inline]
const fn column(self) -> u32 {
self.location.column()
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum ResolverOp {
Reject,
ResolveDecision,
SetResolver,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum ResolverErrorKind {
Reject,
Control(CpError),
}
#[derive(Clone, Copy, PartialEq, Eq)]
pub struct ResolverError {
pub(crate) op: ResolverOp,
location: ResolverErrorLocation,
kind: ResolverErrorKind,
}
impl fmt::Debug for ResolverError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ResolverError")
.field("operation", &self.operation())
.field("file", &self.file())
.field("line", &self.line())
.field("column", &self.column())
.field("kind", &self.kind)
.finish()
}
}
impl ResolverError {
#[inline]
#[track_caller]
pub fn reject() -> Self {
Self {
op: ResolverOp::Reject,
location: ResolverErrorLocation::caller(),
kind: ResolverErrorKind::Reject,
}
}
#[inline]
#[track_caller]
pub(crate) fn control(error: CpError) -> Self {
Self {
op: ResolverOp::SetResolver,
location: ResolverErrorLocation::caller(),
kind: ResolverErrorKind::Control(error),
}
}
#[inline]
pub(crate) const fn with_operation(mut self, op: ResolverOp) -> Self {
self.op = op;
self
}
#[inline]
pub(crate) const fn with_operation_at(
mut self,
op: ResolverOp,
location: ResolverErrorLocation,
) -> Self {
self.op = op;
self.location = location;
self
}
#[inline]
pub const fn operation(&self) -> &'static str {
match self.op {
ResolverOp::Reject => "reject",
ResolverOp::ResolveDecision => "resolve_decision",
ResolverOp::SetResolver => "set_resolver",
}
}
#[inline]
pub const fn file(&self) -> &'static str {
self.location.file()
}
#[inline]
pub const fn line(&self) -> u32 {
self.location.line()
}
#[inline]
pub const fn column(&self) -> u32 {
self.location.column()
}
}
impl From<CpError> for ResolverError {
#[inline]
#[track_caller]
fn from(error: CpError) -> Self {
Self::control(error)
}
}
#[repr(C)]
#[derive(Clone, Copy)]
struct DecisionResolverStatePayload<S> {
state: *const S,
pub(crate) resolver: fn(&S) -> Result<DecisionResolution, ResolverError>,
}
#[derive(Clone, Copy)]
union DecisionResolverStorage {
stateless: fn() -> Result<DecisionResolution, ResolverError>,
_stateful: DecisionResolverStatePayload<()>,
}
#[derive(Clone, Copy)]
pub struct ResolverRef<'cfg> {
storage: DecisionResolverStorage,
dispatch: unsafe fn(DecisionResolverStorage) -> Result<DecisionResolution, ResolverError>,
_marker: PhantomData<&'cfg ()>,
}
impl<'cfg> ResolverRef<'cfg> {
#[inline]
pub fn decision_state<S: 'cfg>(
state: &'cfg S,
resolver: fn(&S) -> Result<DecisionResolution, ResolverError>,
) -> Self {
const {
assert!(
core::mem::size_of::<DecisionResolverStatePayload<S>>()
== core::mem::size_of::<DecisionResolverStatePayload<()>>()
);
assert!(
core::mem::align_of::<DecisionResolverStatePayload<S>>()
== core::mem::align_of::<DecisionResolverStatePayload<()>>()
);
}
let payload = DecisionResolverStatePayload {
state: core::ptr::from_ref(state),
resolver,
};
let mut storage = MaybeUninit::<DecisionResolverStorage>::uninit();
unsafe {
storage
.as_mut_ptr()
.cast::<DecisionResolverStatePayload<S>>()
.write(payload);
}
Self {
storage: unsafe { storage.assume_init() },
dispatch: dispatch_decision_state::<S>,
_marker: PhantomData,
}
}
#[inline]
pub fn decision_fn(resolver: fn() -> Result<DecisionResolution, ResolverError>) -> Self {
Self {
storage: DecisionResolverStorage {
stateless: resolver,
},
dispatch: dispatch_decision_fn,
_marker: PhantomData,
}
}
#[inline]
pub(crate) const fn accepts_op(self, op: ControlOp) -> bool {
matches!(
op,
ControlOp::RouteDecision | ControlOp::LoopContinue | ControlOp::LoopBreak
)
}
#[inline]
pub(crate) fn resolve_decision(self) -> Result<DecisionResolution, ResolverError> {
unsafe {
(self.dispatch)(self.storage)
.map_err(|error| error.with_operation(ResolverOp::ResolveDecision))
}
}
}
unsafe fn dispatch_decision_state<S>(
storage: DecisionResolverStorage,
) -> Result<DecisionResolution, ResolverError> {
const {
assert!(
core::mem::size_of::<DecisionResolverStatePayload<S>>()
== core::mem::size_of::<DecisionResolverStatePayload<()>>()
);
assert!(
core::mem::align_of::<DecisionResolverStatePayload<S>>()
== core::mem::align_of::<DecisionResolverStatePayload<()>>()
);
}
let payload = unsafe {
(&storage as *const DecisionResolverStorage)
.cast::<DecisionResolverStatePayload<S>>()
.read()
};
let state = unsafe { &*payload.state };
(payload.resolver)(state)
}
unsafe fn dispatch_decision_fn(
storage: DecisionResolverStorage,
) -> Result<DecisionResolution, ResolverError> {
let resolver = unsafe { storage.stateless };
resolver()
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct DynamicResolverKey {
pub(crate) rv: RendezvousId,
pub(crate) eff_index: EffIndex,
pub(crate) op: ControlOp,
}
impl DynamicResolverKey {
pub(crate) const fn new(rv: RendezvousId, eff_index: EffIndex, op: ControlOp) -> Self {
Self { rv, eff_index, op }
}
}
#[derive(Clone, Copy)]
pub(crate) struct DynamicResolverEntry<'cfg> {
pub(crate) resolver: ResolverRef<'cfg>,
pub(crate) policy: PolicyMode,
}
#[inline]
pub(crate) const fn cluster_rendezvous_slot<const MAX_RV: usize>(
rv_id: RendezvousId,
) -> Option<usize> {
let raw = rv_id.raw() as usize;
if raw == 0 || raw > MAX_RV {
None
} else {
Some(raw - 1)
}
}
#[derive(Clone, Copy)]
pub(in crate::control::cluster::core) struct ResolverBucketEntry<'cfg> {
pub(crate) eff_index: EffIndex,
pub(crate) op: ControlOp,
entry: DynamicResolverEntry<'cfg>,
}
pub(crate) struct ResolverBucket<'cfg> {
entries: UnsafeCell<*mut Option<ResolverBucketEntry<'cfg>>>,
capacity: usize,
_no_send_sync: PhantomData<*mut ()>,
}
impl<'cfg> ResolverBucket<'cfg> {
pub(crate) const STORAGE_TAG_MASK: usize = Self::storage_align().saturating_sub(1);
pub(crate) unsafe fn init_empty(dst: *mut Self) {
unsafe {
core::ptr::addr_of_mut!((*dst).entries).write(UnsafeCell::new(core::ptr::null_mut()));
core::ptr::addr_of_mut!((*dst).capacity).write(0);
core::ptr::addr_of_mut!((*dst)._no_send_sync).write(PhantomData);
}
}
#[inline]
pub(crate) const fn storage_align() -> usize {
core::mem::align_of::<Option<ResolverBucketEntry<'cfg>>>()
}
#[inline]
pub(crate) const fn storage_bytes(capacity: usize) -> usize {
capacity.saturating_mul(core::mem::size_of::<Option<ResolverBucketEntry<'cfg>>>())
}
#[inline]
pub(in crate::control::cluster::core) fn raw_entries(
&self,
) -> *mut Option<ResolverBucketEntry<'cfg>> {
unsafe { *self.entries.get() }
}
#[inline]
pub(in crate::control::cluster::core) fn entries_ptr(
&self,
) -> *mut Option<ResolverBucketEntry<'cfg>> {
self.raw_entries()
.map_addr(|addr| addr & !Self::STORAGE_TAG_MASK)
}
#[inline]
fn encode_entries_ptr(
entries: *mut Option<ResolverBucketEntry<'cfg>>,
reclaim_delta: usize,
) -> *mut Option<ResolverBucketEntry<'cfg>> {
debug_assert_eq!(entries.addr() & Self::STORAGE_TAG_MASK, 0);
debug_assert!(reclaim_delta <= Self::STORAGE_TAG_MASK);
entries.map_addr(|addr| addr | reclaim_delta)
}
#[inline]
pub(crate) fn storage_ptr(&self) -> *mut u8 {
self.entries_ptr().cast::<u8>()
}
#[inline]
pub(crate) fn storage_reclaim_delta(&self) -> usize {
self.raw_entries().addr() & Self::STORAGE_TAG_MASK
}
#[inline]
pub(crate) fn storage_len(&self) -> usize {
Self::storage_bytes(self.capacity)
}
#[inline]
pub(crate) fn capacity(&self) -> usize {
self.capacity
}
pub(crate) fn occupied_len(&self) -> usize {
let entries = self.entries_ptr();
if entries.is_null() {
return 0;
}
let mut idx = 0usize;
let mut occupied = 0usize;
while idx < self.capacity {
unsafe {
if (*entries.add(idx)).is_some() {
occupied += 1;
}
}
idx += 1;
}
occupied
}
pub(crate) unsafe fn bind_from_storage(
&mut self,
storage: *mut u8,
capacity: usize,
reclaim_delta: usize,
) {
let entries = storage.cast::<Option<ResolverBucketEntry<'cfg>>>();
let mut idx = 0usize;
while idx < capacity {
unsafe {
entries.add(idx).write(None);
}
idx += 1;
}
*self.entries.get_mut() = Self::encode_entries_ptr(entries, reclaim_delta);
self.capacity = capacity;
}
pub(crate) unsafe fn rebind_from_storage(
&mut self,
storage: *mut u8,
new_capacity: usize,
reclaim_delta: usize,
) {
let old_entries = self.entries_ptr();
let old_capacity = self.capacity;
let new_entries = storage.cast::<Option<ResolverBucketEntry<'cfg>>>();
let mut idx = 0usize;
while idx < new_capacity {
unsafe {
new_entries.add(idx).write(None);
}
idx += 1;
}
if !old_entries.is_null() {
let mut next = 0usize;
let mut old_idx = 0usize;
while old_idx < old_capacity {
unsafe {
if let Some(entry) = (*old_entries.add(old_idx)).take() {
debug_assert!(next < new_capacity, "resolver bucket rebind overflow");
new_entries.add(next).write(Some(entry));
next += 1;
}
}
old_idx += 1;
}
}
*self.entries.get_mut() = Self::encode_entries_ptr(new_entries, reclaim_delta);
self.capacity = new_capacity;
}
pub(crate) fn insert(
&mut self,
eff_index: EffIndex,
op: ControlOp,
entry: DynamicResolverEntry<'cfg>,
) -> Result<(), CpError> {
let entries = self.entries_ptr();
if entries.is_null() {
return Err(CpError::resource_exhausted(ResourceScope::ResolverTable));
}
let mut first_empty = None;
let mut idx = 0usize;
while idx < self.capacity {
unsafe {
let slot = &mut *entries.add(idx);
match slot {
Some(stored) if stored.eff_index == eff_index && stored.op == op => {
stored.entry = entry;
return Ok(());
}
None if first_empty.is_none() => first_empty = Some(idx),
_ => {}
}
}
idx += 1;
}
let Some(idx) = first_empty else {
return Err(CpError::resource_exhausted(ResourceScope::ResolverTable));
};
unsafe {
*entries.add(idx) = Some(ResolverBucketEntry {
eff_index,
op,
entry,
});
}
Ok(())
}
pub(crate) fn get(
&self,
eff_index: EffIndex,
op: ControlOp,
) -> Option<&DynamicResolverEntry<'cfg>> {
let entries = self.entries_ptr();
if entries.is_null() {
return None;
}
let mut idx = 0usize;
while idx < self.capacity {
unsafe {
if let Some(stored) = (&*entries.add(idx)).as_ref()
&& stored.eff_index == eff_index
&& stored.op == op
{
return Some(&stored.entry);
}
}
idx += 1;
}
None
}
}
pub(crate) const fn is_dynamic_control_op(op: ControlOp) -> bool {
matches!(
op,
ControlOp::LoopContinue | ControlOp::LoopBreak | ControlOp::RouteDecision
)
}