use std::future::Future;
use std::ops::{Deref, DerefMut};
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use citadel_io::tokio::sync::mpsc::{Receiver, Sender};
use citadel_io::tokio::sync::{Mutex, OwnedMutexGuard};
use crate::reliable_conn::ReliableOrderedStreamToTargetExt;
use crate::sync::primitives::net_mutex::{sync_establish_init, InnerChannel};
use crate::sync::primitives::NetObject;
use crate::sync::subscription::Subscribable;
use crate::sync::subscription::SubscriptionBiStream;
use crate::sync::RelativeNodeType;
use crate::time_tracker::TimeTracker;
use crate::ScopedFutureResult;
use serde::{Deserialize, Serialize};
type InnerState<T> = (T, Sender<()>);
type OwnedLocalReadLock<T> = Arc<OwnedMutexGuard<InnerState<T>>>;
type OwnedLocalWriteLock<T> = OwnedMutexGuard<InnerState<T>>;
pub struct NetRwLock<T: NetObject, S: Subscribable + 'static> {
shared: Arc<Mutex<InnerState<T>>>,
channel: Arc<InnerChannel<S>>,
stop_tx: Option<citadel_io::tokio::sync::oneshot::Sender<()>>,
local_active_read_lock: Arc<citadel_io::RwLock<Option<OwnedLocalReadLock<T>>>>,
active_to_bg_signalled: Sender<()>,
}
impl<T: NetObject, S: Subscribable + 'static> NetRwLock<T, S> {
pub fn node_type(&self) -> RelativeNodeType {
self.channel.node_type()
}
pub async fn new_internal(
channel: InnerChannel<S>,
initial_value: T,
) -> Result<Self, anyhow::Error> {
let (stop_tx, stop_rx) = citadel_io::tokio::sync::oneshot::channel::<()>();
let (active_to_bg_tx, active_to_bg_rx) = citadel_io::tokio::sync::mpsc::channel(1);
let this = Self {
shared: Arc::new(Mutex::new((initial_value, active_to_bg_tx.clone()))),
channel: Arc::new(channel),
stop_tx: Some(stop_tx),
local_active_read_lock: Arc::new(citadel_io::RwLock::new(None)),
active_to_bg_signalled: active_to_bg_tx,
};
let shared_state = this.shared.clone();
let local_read_lock = this.local_active_read_lock.clone();
let channel = this.channel.clone();
citadel_io::spawn(async move {
if let Err(err) = passive_background_handler::<S, T>(
channel,
shared_state,
stop_rx,
active_to_bg_rx,
local_read_lock,
)
.await
{
log::error!(target: "citadel", "[NetRwLock] Err: {:?}", err.to_string());
}
log::trace!(target: "citadel", "[NetRwLock] Passive background handler ending")
});
Ok(this)
}
pub fn create(conn: &S, t: Option<T>) -> NetRwLockLoader<'_, T, S> {
NetRwLockLoader {
future: Box::pin(sync_establish_init(conn, t, Self::new_internal)),
}
}
pub fn read(&self) -> RwLockReadAcquirer<'_, T, S> {
RwLockReadAcquirer {
future: Box::pin(acquire_read(self)),
}
}
pub fn write(&self) -> RwLockWriteAcquirer<'_, T, S> {
RwLockWriteAcquirer {
future: Box::pin(acquire_write(self)),
}
}
pub fn active_local_reads(&self) -> usize {
self.local_active_read_lock
.read()
.as_ref()
.map(|r| Arc::strong_count(r) - 1)
.unwrap_or(0)
}
}
impl<T: NetObject, S: Subscribable + 'static> Drop for NetRwLock<T, S> {
fn drop(&mut self) {
log::trace!(target: "citadel", "DROPPING {:?} NetRwLock", self.channel.node_type());
let conn = self.channel.clone();
let stop_tx = self.stop_tx.take().unwrap();
let _ = stop_tx.send(());
if let Ok(rt) = citadel_io::tokio::runtime::Handle::try_current() {
rt.spawn(async move {
citadel_io::time::sleep(std::time::Duration::from_millis(200)).await;
let _ = conn.send_serialized(UpdatePacket::Halt).await;
});
}
}
}
pub struct NetRwLockLoader<'a, T: NetObject, S: Subscribable + 'static> {
future: ScopedFutureResult<'a, NetRwLock<T, S>>,
}
impl<T: NetObject, S: Subscribable + 'static> Future for NetRwLockLoader<'_, T, S> {
type Output = Result<NetRwLock<T, S>, anyhow::Error>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
self.future.as_mut().poll(cx)
}
}
pub struct RwLockReadAcquirer<'a, T: NetObject + 'static, S: Subscribable + 'static> {
pub(crate) future: ScopedFutureResult<'a, NetRwLockReadGuard<T, S>>,
}
pub struct NetRwLockReadGuard<T: NetObject + 'static, S: Subscribable + 'static> {
inner: Option<LocalLockHolder<T>>,
conn: Arc<InnerChannel<S>>,
shared_store: Arc<citadel_io::RwLock<Option<OwnedLocalReadLock<T>>>>,
}
impl<T: NetObject, S: Subscribable + 'static> Future for RwLockReadAcquirer<'_, T, S> {
type Output = Result<NetRwLockReadGuard<T, S>, anyhow::Error>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
self.future.as_mut().poll(cx)
}
}
pub(super) async fn acquire_read<T: NetObject + 'static, S: Subscribable + 'static>(
rwlock: &NetRwLock<T, S>,
) -> Result<NetRwLockReadGuard<T, S>, anyhow::Error> {
log::trace!(target: "citadel", "Running acquire_read for {:?}", rwlock.channel.node_type());
{
let pre_loaded = rwlock.local_active_read_lock.read();
log::trace!(target: "citadel", "Running acquire_read for {:?} | pre_loaded ? {}", rwlock.channel.node_type(), pre_loaded.is_some());
if pre_loaded
.as_ref()
.map(|r| Arc::strong_count(r) > 1)
.unwrap_or(false)
{
return Ok(NetRwLockReadGuard {
inner: Some(LocalLockHolder::Read(pre_loaded.clone(), false)),
conn: rwlock.channel.clone(),
shared_store: rwlock.local_active_read_lock.clone(),
});
}
}
acquire_lock(LockType::Read, rwlock, |inner| NetRwLockReadGuard {
inner: Some(inner),
conn: rwlock.channel.clone(),
shared_store: rwlock.local_active_read_lock.clone(),
})
.await
}
impl<T: NetObject, S: Subscribable> Deref for NetRwLockReadGuard<T, S> {
type Target = T;
fn deref(&self) -> &Self::Target {
self.inner.as_ref().unwrap().deref()
}
}
impl<T: NetObject, S: Subscribable + 'static> Drop for NetRwLockReadGuard<T, S> {
fn drop(&mut self) {
let this = self.inner.take().unwrap();
if this.arc_strong_count().unwrap() == 2 {
log::trace!(target: "citadel", "CALLING read drop code on {:?}", self.conn.node_type());
*self.shared_store.write() = None;
if let Ok(rt) = citadel_io::tokio::runtime::Handle::try_current() {
let future = NetRwLockEitherGuardDropCode::new::<T, S>(self.conn.clone(), this);
rt.spawn(future);
}
}
}
}
pub struct RwLockWriteAcquirer<'a, T: NetObject + 'static, S: Subscribable + 'static> {
pub(crate) future: ScopedFutureResult<'a, NetRwLockWriteGuard<T, S>>,
}
pub struct NetRwLockWriteGuard<T: NetObject + 'static, S: Subscribable + 'static> {
inner: Option<LocalLockHolder<T>>,
conn: Arc<InnerChannel<S>>,
}
impl<T: NetObject, S: Subscribable> Future for RwLockWriteAcquirer<'_, T, S> {
type Output = Result<NetRwLockWriteGuard<T, S>, anyhow::Error>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
self.future.as_mut().poll(cx)
}
}
pub(super) async fn acquire_write<T: NetObject + 'static, S: Subscribable + 'static>(
rwlock: &NetRwLock<T, S>,
) -> Result<NetRwLockWriteGuard<T, S>, anyhow::Error> {
acquire_lock(LockType::Write, rwlock, |inner| NetRwLockWriteGuard {
inner: Some(inner),
conn: rwlock.channel.clone(),
})
.await
}
impl<T: NetObject, S: Subscribable> Deref for NetRwLockWriteGuard<T, S> {
type Target = T;
fn deref(&self) -> &Self::Target {
self.inner.as_ref().unwrap()
}
}
impl<T: NetObject, S: Subscribable> DerefMut for NetRwLockWriteGuard<T, S> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.inner.as_mut().unwrap().assert_write_mut().0
}
}
impl<T: NetObject + 'static, S: Subscribable + 'static> Drop for NetRwLockWriteGuard<T, S> {
fn drop(&mut self) {
let this = self.inner.take().unwrap();
if let Ok(rt) = citadel_io::tokio::runtime::Handle::try_current() {
let future = NetRwLockEitherGuardDropCode::new::<T, S>(self.conn.clone(), this);
rt.spawn(future);
}
}
}
pub(super) struct NetRwLockEitherGuardDropCode {
future: Pin<Box<dyn Future<Output = Result<(), anyhow::Error>> + Send>>,
}
impl NetRwLockEitherGuardDropCode {
pub(super) fn new<T: NetObject + 'static, S: Subscribable + 'static>(
conn: Arc<InnerChannel<S>>,
guard: LocalLockHolder<T>,
) -> Self {
Self {
future: Box::pin(net_rwlock_guard_drop_code::<T, S>(conn, guard)),
}
}
}
impl Future for NetRwLockEitherGuardDropCode {
type Output = Result<(), anyhow::Error>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
self.future.as_mut().poll(cx)
}
}
async fn net_rwlock_guard_drop_code<T: NetObject, S: Subscribable + 'static>(
conn: Arc<InnerChannel<S>>,
lock: LocalLockHolder<T>,
) -> Result<(), anyhow::Error> {
log::trace!(target: "citadel", "[NetRwLock] Drop code initialized for {:?}...", conn.node_type());
match &lock {
LocalLockHolder::Read(..) => {
conn.send_serialized(UpdatePacket::ReleasedRead).await?;
}
LocalLockHolder::Write(_guard, ..) => {
conn.send_serialized(UpdatePacket::ReleasedWrite(bincode::serialize(
&lock.deref(),
)?))
.await?;
}
}
let mut adjacent_trying_to_acquire = None;
loop {
let packet = conn.recv_serialized::<UpdatePacket>().await?;
log::trace!(target: "citadel", "[NetRwLock] [Drop Code] RECV {:?} on {:?}", &packet, conn.node_type());
match packet {
UpdatePacket::ReleasedWrite(_new_value) => {
}
UpdatePacket::ReleasedRead => {
if let LocalLockHolder::Read(..) = &lock {
log::trace!(target: "citadel", "Yield:: Releasing Read lock");
conn.send_serialized(UpdatePacket::ReleasedVerified(LockType::Read))
.await?;
}
}
UpdatePacket::ReleasedVerified(_lock_type) => {
log::trace!(target: "citadel", "[NetRwLock] [Drop Code] Release has been verified for {:?}. Adjacent node updated; will drop local lock", conn.node_type());
if let Some(_lock_type) = adjacent_trying_to_acquire {
log::trace!(target: "citadel", "[NetRwLock] [Drop Code] Will not yet drop though, since remote requested lock access since dropping ...");
log::trace!(target: "citadel", "[KTX] {:?} yield_lock", conn.node_type());
return yield_lock::<S, T>(&conn, lock).await.map(|_| ());
}
return Ok(());
}
UpdatePacket::TryAcquire(_, lock_type) => {
adjacent_trying_to_acquire = Some(lock_type);
}
UpdatePacket::Halt => return Err(anyhow::Error::msg("Halted from background")),
UpdatePacket::LockAcquired(_) => {
}
}
}
}
#[derive(Serialize, Deserialize, Debug, Clone, Copy, Eq, PartialEq)]
enum LockType {
Read,
Write,
}
pub(super) enum LocalLockHolder<T> {
Write(Option<OwnedLocalWriteLock<T>>, bool),
Read(Option<OwnedLocalReadLock<T>>, bool),
}
impl<T> Deref for LocalLockHolder<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
match self {
Self::Write(val, _) => &val.as_ref().unwrap().0,
Self::Read(val, _) => &val.as_ref().unwrap().0,
}
}
}
impl<T> LocalLockHolder<T> {
fn is_from_background(&self) -> bool {
match self {
Self::Write(_, from_bg) | Self::Read(_, from_bg) => *from_bg,
}
}
fn lock_type(&self) -> LockType {
match self {
Self::Write(..) => LockType::Write,
Self::Read(..) => LockType::Read,
}
}
fn free_lock_and_get_sender(&mut self) -> Sender<()> {
match self {
Self::Read(r, _) => r.take().unwrap().1.clone(),
Self::Write(w, _) => w.take().unwrap().1.clone(),
}
}
fn assert_write_mut(&mut self) -> &mut OwnedLocalWriteLock<T> {
match self {
Self::Write(val, ..) => val.as_mut().unwrap(),
_ => {
panic!("Asserted write lock, but was a read lock");
}
}
}
fn assert_read(&self) -> &OwnedLocalReadLock<T> {
match self {
Self::Read(val, ..) => val.as_ref().unwrap(),
_ => {
panic!("Asserted write lock, but was a read lock");
}
}
}
fn assert_read_and_upgrade(&mut self) {
match self {
Self::Read(val, from_bg) => {
let val = val.take().unwrap();
log::trace!(target: "citadel", "Arc count: {}", Arc::strong_count(&val));
let upgraded = Arc::try_unwrap(val).map_err(|_| ()).unwrap();
let new = LocalLockHolder::Write(Some(upgraded), *from_bg);
*from_bg = true; *self = new
}
Self::Write(..) => {
panic!("Asserted read, but was write")
}
}
}
fn assert_write_and_downgrade(&mut self) {
match self {
Self::Write(val, from_bg) => {
let new = LocalLockHolder::Read(val.take().map(Arc::new), *from_bg);
*from_bg = true; *self = new;
}
Self::Read(..) => {
panic!("Asserted write, but was read")
}
}
}
fn arc_strong_count(&self) -> Option<usize> {
match self {
Self::Read(val, ..) => Some(Arc::strong_count(val.as_ref().unwrap())),
Self::Write(..) => None,
}
}
}
impl<T> Drop for LocalLockHolder<T> {
fn drop(&mut self) {
if !self.is_from_background() {
let sender = self.free_lock_and_get_sender(); if let Ok(rt) = citadel_io::tokio::runtime::Handle::try_current() {
rt.spawn(async move {
let _ = sender.send(()).await; });
}
}
}
}
async fn yield_lock<S: Subscribable + 'static, T: NetObject>(
channel: &Arc<InnerChannel<S>>,
mut lock: LocalLockHolder<T>,
) -> Result<LocalLockHolder<T>, anyhow::Error> {
match &lock {
LocalLockHolder::Read(..) => {
channel.send_serialized(UpdatePacket::ReleasedRead).await?;
}
LocalLockHolder::Write(val, _) => {
channel
.send_serialized(UpdatePacket::ReleasedWrite(
bincode::serialize(&val.as_ref().unwrap().0).unwrap(),
))
.await?;
}
}
loop {
let next_packet = channel.recv_serialized().await?;
log::trace!(target: "citadel", "Yield::RECV | {:?} received {:?}", channel.node_type(), &next_packet);
match next_packet {
UpdatePacket::ReleasedWrite(new_value) => match &mut lock {
LocalLockHolder::Write(val, _) => {
log::trace!(target: "citadel", "Yield:: Releasing Write lock");
val.as_mut().unwrap().0 = bincode::deserialize(&new_value)?;
channel
.send_serialized(UpdatePacket::ReleasedVerified(LockType::Write))
.await?;
return Ok(lock);
}
_ => {
log::warn!(target: "citadel", "{:?} | Invalid read/write packet. Expected a write, but local has a read", channel.node_type())
}
},
UpdatePacket::ReleasedRead => match &lock {
LocalLockHolder::Read(..) => {
log::trace!(target: "citadel", "Yield:: Releasing Read lock");
channel
.send_serialized(UpdatePacket::ReleasedVerified(LockType::Read))
.await?;
return Ok(lock);
}
_ => {
log::warn!(target: "citadel", "Invalid read/write packet. Expected a read, but local has a write")
}
},
UpdatePacket::Halt => return Err(anyhow::Error::msg("Halted from background")),
UpdatePacket::LockAcquired(_) => {
}
p => {
log::warn!(target: "citadel", "Received invalid packet type in inner_loop! {p:?}")
}
}
}
}
async fn passive_background_handler<S: Subscribable + 'static, T: NetObject>(
channel: Arc<InnerChannel<S>>,
shared_state: Arc<Mutex<InnerState<T>>>,
stop_rx: citadel_io::tokio::sync::oneshot::Receiver<()>,
mut active_to_background_rx: Receiver<()>,
read_lock_local: Arc<citadel_io::RwLock<Option<OwnedLocalReadLock<T>>>>,
) -> Result<(), anyhow::Error> {
let background_task = async move {
'outer_loop: loop {
match shared_state.clone().try_lock_owned() {
Ok(lock) => {
let packet = citadel_io::tokio::select! {
res0 = channel.recv_serialized::<UpdatePacket>() => res0?,
res1 = active_to_background_rx.recv() => {
res1.ok_or_else(|| anyhow::Error::msg("The active_to_background_tx died"))?;
continue 'outer_loop;
}
};
match packet {
UpdatePacket::TryAcquire(_, lock_type) => {
let lock_holder = match lock_type {
LockType::Read => {
let lock = Some(Arc::new(lock));
read_lock_local.write().clone_from(&lock);
LocalLockHolder::Read(lock, true)
}
LockType::Write => LocalLockHolder::Write(Some(lock), true),
};
log::trace!(target: "citadel", "[KTG] {:?} yield_lock", channel.node_type());
let lock_holder = yield_lock::<S, T>(&channel, lock_holder).await?;
if lock_type == LockType::Read
&& lock_holder.arc_strong_count().unwrap() == 2
{
*read_lock_local.write() = None;
std::mem::drop(lock_holder); }
}
UpdatePacket::ReleasedRead
| UpdatePacket::ReleasedWrite(..)
| UpdatePacket::ReleasedVerified(..)
| UpdatePacket::LockAcquired(..) => {
log::warn!(target: "citadel", "RELEASED/RELEASED_VERIFIED/LOCK_ACQUIRED should only be received in the yield_lock subroutine.");
}
UpdatePacket::Halt => {
return Err(anyhow::Error::msg("Halted from background"))
}
}
}
Err(_) => {
active_to_background_rx
.recv()
.await
.ok_or_else(|| anyhow::Error::msg("The active_to_background_tx died"))?;
}
}
}
};
citadel_io::tokio::select! {
res0 = background_task => res0,
_res1 = stop_rx => Ok(())
}
}
async fn acquire_lock<T: NetObject, S: Subscribable + 'static, R, F>(
lock_type: LockType,
rwlock: &NetRwLock<T, S>,
fx: F,
) -> Result<R, anyhow::Error>
where
F: Fn(LocalLockHolder<T>) -> R,
{
log::trace!(target: "citadel", "Attempting to acquire lock for {:?}", rwlock.channel.node_type());
rwlock.active_to_bg_signalled.send(()).await?;
let lock = rwlock.shared.clone().lock_owned().await;
let mut owned_local_lock = match lock_type {
LockType::Read => LocalLockHolder::Read(Some(Arc::new(lock)), false),
LockType::Write => LocalLockHolder::Write(Some(lock), false),
};
log::trace!(target: "citadel", "{:?} acquired local lock", rwlock.channel.node_type());
let conn = &rwlock.channel;
let local_request_time = TimeTracker::new().get_global_time_ns();
conn.send_serialized(UpdatePacket::TryAcquire(local_request_time, lock_type))
.await
.map_err(|err| anyhow::Error::msg(err.to_string()))?;
loop {
let packet = conn.recv_serialized().await?;
log::trace!(target: "citadel", "{:?}/active-channel || obtained packet {:?}", rwlock.channel.node_type(), &packet);
match packet {
UpdatePacket::ReleasedWrite(new_data) => {
let new_data = bincode::deserialize::<T>(&new_data)?;
match &mut owned_local_lock {
LocalLockHolder::Write(lock, ..) => {
lock.as_mut().unwrap().0 = new_data;
}
LocalLockHolder::Read(lock_orig, _) => {
let mut lock = Arc::try_unwrap(lock_orig.take().unwrap())
.map_err(|_err| anyhow::Error::msg("Unable to unwrap Arc"))?;
lock.0 = new_data;
*lock_orig = Some(Arc::new(lock));
}
}
conn.send_serialized(UpdatePacket::ReleasedVerified(lock_type))
.await?;
conn.send_serialized(UpdatePacket::LockAcquired(lock_type))
.await?;
if owned_local_lock.lock_type() == LockType::Read {
*rwlock.local_active_read_lock.write() =
Some(owned_local_lock.assert_read().clone())
}
return Ok((fx)(owned_local_lock));
}
UpdatePacket::ReleasedRead => {
conn.send_serialized(UpdatePacket::ReleasedVerified(lock_type))
.await?;
conn.send_serialized(UpdatePacket::LockAcquired(lock_type))
.await?;
if owned_local_lock.lock_type() == LockType::Read {
*rwlock.local_active_read_lock.write() =
Some(owned_local_lock.assert_read().clone())
}
return Ok((fx)(owned_local_lock));
}
UpdatePacket::LockAcquired(_lock_type) => {
}
UpdatePacket::TryAcquire(remote_request_time, lock_type) => {
log::trace!(target: "citadel", "[local: {:?}] BOTH trying to acquire! Local: {:?} | Remote: {:?}", conn.node_type(), owned_local_lock.lock_type(), lock_type);
if lock_type == LockType::Read && owned_local_lock.lock_type() == LockType::Read {
log::trace!(target: "citadel", "BOTH acquiring reads. Both win");
conn.send_serialized(UpdatePacket::ReleasedRead).await?;
if owned_local_lock.lock_type() == LockType::Read {
*rwlock.local_active_read_lock.write() =
Some(owned_local_lock.assert_read().clone())
}
return Ok(fx(owned_local_lock));
}
let local_wins = if remote_request_time == local_request_time {
rwlock.node_type() == RelativeNodeType::Initiator
} else {
remote_request_time > local_request_time
};
if local_wins {
conn.send_serialized(UpdatePacket::LockAcquired(lock_type))
.await?;
if owned_local_lock.lock_type() == LockType::Read {
*rwlock.local_active_read_lock.write() =
Some(owned_local_lock.assert_read().clone())
}
return Ok((fx)(owned_local_lock));
} else {
if owned_local_lock.lock_type() != lock_type {
log::trace!(target: "citadel", "Remote is trying to acquire lock type not equal to local type. Must transform. Local {:?}, Remote {:?}", owned_local_lock.lock_type(), lock_type);
if lock_type == LockType::Write {
*rwlock.local_active_read_lock.write() = None;
owned_local_lock.assert_read_and_upgrade();
}
if lock_type == LockType::Read {
owned_local_lock.assert_write_and_downgrade();
log::trace!(target: "citadel", "Asserted local is write and downgraded");
}
}
log::trace!(target: "citadel", "[KTZZ] {:?} yield_lock", conn.node_type());
owned_local_lock = yield_lock::<S, T>(conn, owned_local_lock).await?;
if owned_local_lock.lock_type() != lock_type {
log::trace!(target: "citadel", "[undo] Remote is trying to acquire lock type not equal to local type. Must transform");
if lock_type == LockType::Write {
owned_local_lock.assert_write_and_downgrade();
*rwlock.local_active_read_lock.write() =
Some(owned_local_lock.assert_read().clone()); assert_eq!(owned_local_lock.arc_strong_count().unwrap(), 2);
}
if lock_type == LockType::Read {
owned_local_lock.assert_read_and_upgrade();
*rwlock.local_active_read_lock.write() = None;
log::trace!(target: "citadel", "Asserted local is write and downgraded");
}
}
}
}
UpdatePacket::Halt => {
return Err(anyhow::Error::msg("The adjacent node dropped the Mutex"));
}
UpdatePacket::ReleasedVerified(..) => {
log::warn!(target: "citadel", "RELEASED_VERIFIED should only be received by the drop_lock subroutine")
}
}
}
}
#[derive(Serialize, Deserialize, Debug)]
enum UpdatePacket {
TryAcquire(i64, LockType),
ReleasedWrite(Vec<u8>),
ReleasedRead,
LockAcquired(LockType),
Halt,
ReleasedVerified(LockType),
}
#[cfg(test)]
mod tests {
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use crate::sync::test_utils::create_streams;
use citadel_io::tokio;
#[tokio::test]
async fn many_writes() {
citadel_logging::setup_log();
let (server_stream, client_stream) = create_streams().await;
const COUNT: u64 = 1000;
let init_value = 1000u64;
let final_value = 1001u64;
let (client_done_tx, client_done_rx) = citadel_io::tokio::sync::oneshot::channel::<()>();
let (server_done_tx, server_done_rx) = citadel_io::tokio::sync::oneshot::channel::<()>();
let server_ref = Arc::new(AtomicU64::new(init_value));
let client_ref = server_ref.clone();
let server = citadel_io::tokio::spawn(async move {
let rwlock = &server_stream.rwlock(Some(init_value)).await.unwrap();
log::trace!(target: "citadel", "Success establishing rwlock on server");
client_done_rx.await.unwrap();
let guard = rwlock.write().await.unwrap();
assert_eq!(*guard, final_value);
log::trace!(target: "citadel", "Server ASSERT_EQ valid");
std::mem::drop(guard);
for idx in 1..COUNT {
log::trace!(target: "citadel", "Server obtaining lock {idx}");
let mut lock = rwlock.write().await.unwrap();
log::trace!(target: "citadel", "****Server obtained lock {} w/val {:?}", idx, &*lock);
assert_eq!(idx + init_value, *lock);
*lock += 1;
server_ref.store(*lock, Ordering::SeqCst);
}
server_done_tx.send(()).unwrap();
});
let client = citadel_io::tokio::spawn(async move {
let rwlock = &client_stream.rwlock::<u64>(None).await.unwrap();
log::trace!(target: "citadel", "Success establishing rwlock on client");
let mut guard = rwlock.write().await.unwrap();
log::trace!(target: "citadel", "Client has successfully established a rwlock write lock");
*guard = 1001;
client_ref.store(*guard, Ordering::SeqCst);
std::mem::drop(guard);
client_done_tx.send(()).unwrap();
for _ in 1..COUNT {
let val = rwlock.write().await.unwrap();
let loaded = client_ref.load(Ordering::SeqCst);
if *val != loaded {
log::error!(target: "citadel", "Mutex value {} != loaded value {}", *val, loaded);
std::process::exit(-1);
}
}
server_done_rx.await.unwrap();
});
let (r0, r1) = citadel_io::tokio::join!(server, client);
r0.unwrap();
r1.unwrap();
}
#[tokio::test]
async fn many_reads() {
citadel_logging::setup_log();
let (server_stream, client_stream) = create_streams().await;
const COUNT: u64 = 1000;
let init_value = 1000u64;
let final_value = 1001u64;
let (client_done_tx, client_done_rx) = citadel_io::tokio::sync::oneshot::channel::<()>();
let (server_done_tx0, server_done_rx0) = citadel_io::tokio::sync::oneshot::channel::<()>();
let (server_done_tx, server_done_rx) = citadel_io::tokio::sync::oneshot::channel::<()>();
let (client_done_tx2, client_done_rx2) = citadel_io::tokio::sync::oneshot::channel::<()>();
let server_ref = Arc::new(AtomicU64::new(init_value));
let client_ref = server_ref.clone();
let server = citadel_io::tokio::spawn(async move {
let rwlock = &server_stream.rwlock(Some(init_value)).await.unwrap();
log::trace!(target: "citadel", "Success establishing rwlock on server");
client_done_rx.await.unwrap();
let guard = rwlock.write().await.unwrap();
assert_eq!(*guard, final_value);
log::trace!(target: "citadel", "Server ASSERT_EQ valid");
std::mem::drop(guard);
server_done_tx0.send(()).unwrap();
let mut reads = Vec::new();
for idx in 0..COUNT {
log::trace!(target: "citadel", "Server obtaining lock {idx}");
let lock = rwlock.read().await.unwrap();
log::trace!(target: "citadel", "****Server obtained read lock {} w/val {:?}", idx, &*lock);
reads.push(lock);
}
server_done_tx.send(()).unwrap();
assert_eq!(rwlock.active_local_reads(), COUNT as _);
log::trace!(target: "citadel", "**Server has acquired {COUNT} reads");
client_done_rx2.await.unwrap();
});
let client = citadel_io::tokio::spawn(async move {
let rwlock = &client_stream.rwlock::<u64>(None).await.unwrap();
log::trace!(target: "citadel", "Success establishing rwlock on client");
let mut guard = rwlock.write().await.unwrap();
log::trace!(target: "citadel", "Client has successfully established a rwlock write lock");
*guard = 1001;
client_ref.store(*guard, Ordering::SeqCst);
std::mem::drop(guard);
client_done_tx.send(()).unwrap();
server_done_rx0.await.unwrap();
let mut reads = Vec::new();
for idx in 0..COUNT {
log::trace!(target: "citadel", "Client obtaining lock {idx}");
let lock = rwlock.read().await.unwrap();
log::trace!(target: "citadel", "****Client obtained read lock {} w/val {:?}", idx, &*lock);
reads.push(lock);
}
log::trace!(target: "citadel", "**Client has acquired {COUNT} reads");
server_done_rx.await.unwrap();
assert_eq!(rwlock.active_local_reads(), COUNT as _);
std::mem::drop(reads);
client_done_tx2.send(()).unwrap();
});
let (r0, r1) = citadel_io::tokio::join!(server, client);
r0.unwrap();
r1.unwrap();
}
#[citadel_io::tokio::test]
async fn many_reads_no_initial_write() {
citadel_logging::setup_log();
let (server_stream, client_stream) = create_streams().await;
const COUNT: u64 = 1000;
let (client_done_tx, client_done_rx) = citadel_io::tokio::sync::oneshot::channel::<()>();
let server = citadel_io::tokio::spawn(async move {
let rwlock = &server_stream.rwlock::<u64>(Some(1000)).await.unwrap();
log::trace!(target: "citadel", "Success establishing rwlock on server");
let mut reads = Vec::new();
for idx in 0..COUNT {
log::trace!(target: "citadel", "Server obtaining lock {idx}");
let lock = rwlock.read().await.unwrap();
citadel_io::time::sleep(std::time::Duration::from_millis(1)).await;
log::trace!(target: "citadel", "****Server obtained read lock {} w/val {:?}", idx, &*lock);
reads.push(lock);
}
client_done_rx.await.unwrap();
});
let client = citadel_io::tokio::spawn(async move {
let rwlock = &client_stream.rwlock::<u64>(None).await.unwrap();
log::trace!(target: "citadel", "Success establishing rwlock on client");
let mut reads = Vec::new();
for idx in 0..COUNT {
log::trace!(target: "citadel", "Client obtaining lock {idx}");
let lock = rwlock.read().await.unwrap();
citadel_io::time::sleep(std::time::Duration::from_millis(1)).await;
log::trace!(target: "citadel", "****Client obtained read lock {} w/val {:?}", idx, &*lock);
reads.push(lock);
}
log::trace!(target: "citadel", "**Client has acquired {COUNT} reads");
std::mem::drop(reads);
client_done_tx.send(()).unwrap();
});
let (r0, r1) = citadel_io::tokio::join!(server, client);
r0.unwrap();
r1.unwrap();
}
#[citadel_io::tokio::test]
async fn many_reads_end_with_write() {
citadel_logging::setup_log();
let (server_stream, client_stream) = create_streams().await;
const COUNT: u64 = 1000;
let (client_done_tx, client_done_rx) = citadel_io::tokio::sync::oneshot::channel::<()>();
let (server_done_tx, server_done_rx) = citadel_io::tokio::sync::oneshot::channel::<()>();
let init_value = 1000u64;
let final_value = 1001u64;
let server_ref = Arc::new(AtomicU64::new(init_value));
let client_ref = server_ref.clone();
let server = citadel_io::tokio::spawn(async move {
let rwlock = &server_stream.rwlock::<u64>(Some(1000)).await.unwrap();
log::trace!(target: "citadel", "Success establishing rwlock on server");
let mut reads = Vec::new();
for idx in 0..COUNT {
log::trace!(target: "citadel", "Server obtaining lock {idx}");
let lock = rwlock.read().await.unwrap();
citadel_io::time::sleep(std::time::Duration::from_millis(1)).await;
log::trace!(target: "citadel", "****Server obtained read lock {} w/val {:?}", idx, &*lock);
reads.push(lock);
}
std::mem::drop(reads);
client_done_rx.await.unwrap();
let guard = rwlock.write().await.unwrap();
assert_eq!(*guard, final_value);
log::trace!(target: "citadel", "Server ASSERT_EQ valid");
std::mem::drop(guard);
server_done_tx.send(()).unwrap();
});
let client = citadel_io::tokio::spawn(async move {
let rwlock = &client_stream.rwlock::<u64>(None).await.unwrap();
log::trace!(target: "citadel", "Success establishing rwlock on client");
let mut reads = Vec::new();
for idx in 0..COUNT {
log::trace!(target: "citadel", "Client obtaining lock {idx}");
let lock = rwlock.read().await.unwrap();
citadel_io::time::sleep(std::time::Duration::from_millis(1)).await;
log::trace!(target: "citadel", "****Client obtained read lock {} w/val {:?}", idx, &*lock);
reads.push(lock);
}
log::trace!(target: "citadel", "**Client has acquired {COUNT} reads");
std::mem::drop(reads);
let mut guard = rwlock.write().await.unwrap();
log::trace!(target: "citadel", "Client has successfully established a rwlock write lock");
*guard = 1001;
client_ref.store(*guard, Ordering::SeqCst);
std::mem::drop(guard);
client_done_tx.send(()).unwrap();
server_done_rx.await.unwrap();
});
let (r0, r1) = citadel_io::tokio::join!(server, client);
r0.unwrap();
r1.unwrap();
}
#[citadel_io::tokio::test]
async fn many_interweaved() {
citadel_logging::setup_log();
let (server_stream, client_stream) = create_streams().await;
const COUNT: u64 = 1000;
let (init_tx, init_rx) = citadel_io::tokio::sync::oneshot::channel::<()>();
let (init2_tx, init2_rx) = citadel_io::tokio::sync::oneshot::channel::<()>();
let (client_done_tx, client_done_rx) = citadel_io::tokio::sync::oneshot::channel::<()>();
let (server_done_tx, server_done_rx) = citadel_io::tokio::sync::oneshot::channel::<()>();
let server_ref = Arc::new(AtomicU64::new(0));
let client_ref = server_ref.clone();
let server = citadel_io::tokio::spawn(async move {
let rwlock = &server_stream.rwlock::<u64>(Some(0)).await.unwrap();
log::trace!(target: "citadel", "Success establishing rwlock on server");
init_tx.send(()).unwrap();
init2_rx.await.unwrap();
let mut do_read = false;
for idx in 0..COUNT {
log::trace!(target: "citadel", "Server obtaining lock {idx}");
if do_read {
let lock = rwlock.read().await.unwrap();
log::trace!(target: "citadel", "****Server obtained read lock {} w/val {:?}", idx, &*lock);
do_read = false;
assert_eq!(server_ref.load(Ordering::Relaxed), *lock);
} else {
let mut lock = rwlock.write().await.unwrap();
log::trace!(target: "citadel", "****Server obtained write lock {} w/val {:?}", idx, &*lock);
do_read = true;
*lock = idx;
server_ref.store(*lock, Ordering::Relaxed);
}
}
client_done_rx.await.unwrap();
server_done_tx.send(()).unwrap();
});
let client = citadel_io::tokio::spawn(async move {
let rwlock = &client_stream.rwlock::<u64>(None).await.unwrap();
log::trace!(target: "citadel", "Success establishing rwlock on client");
init_rx.await.unwrap();
init2_tx.send(()).unwrap();
let mut do_read = false;
for idx in 0..COUNT {
if do_read {
let lock = rwlock.read().await.unwrap();
log::trace!(target: "citadel", "****Server obtained read lock {} w/val {:?}", idx, &*lock);
do_read = false;
assert_eq!(client_ref.load(Ordering::Relaxed), *lock);
} else {
let mut lock = rwlock.write().await.unwrap();
log::trace!(target: "citadel", "****Server obtained write lock {} w/val {:?}", idx, &*lock);
do_read = true;
*lock = idx;
client_ref.store(*lock, Ordering::Relaxed);
}
}
client_done_tx.send(()).unwrap();
server_done_rx.await.unwrap();
});
let (r0, r1) = citadel_io::tokio::join!(server, client);
r0.unwrap();
r1.unwrap();
}
}