use std::{
collections::VecDeque,
sync::{
Arc,
atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering},
},
};
use compio::runtime::spawn;
use crossfire::{
AsyncRx, MTx,
mpsc::{List, unbounded_async},
oneshot::{RxOneshot as OneshotAsyncRx, TxOneshot as OneshotTx, oneshot},
};
use parking_lot::Mutex;
use wbase::{
map::{ConcurrentMap, new_concurrent_map},
time::now_secs,
};
type ObserverQueue = Mutex<VecDeque<Arc<CollectionItemObserver>>>;
type KeysToObservers = ConcurrentMap<Vec<u8>, ObserverQueue>;
use wresp::RespCommand;
use crate::{
itembroker::{
collection_item_broker_event::{CollectionItemBrokerEvent, CollectionItemBrokerEventType},
collection_item_observer::{CollectionItemObserver, CollectionItemResult, ObserverStatus},
},
list::list_object::{ListObject, OperationDirection},
zset::sorted_set_object::SortedSetObject,
};
const MIN_SECS_BETWEEN_KEYS_TO_OBSERVERS_CLEANS: u64 = 5 * 60;
const MAIN_LOOP_NOT_STARTED: i32 = 0;
const MAIN_LOOP_STARTED: i32 = 1;
const MAIN_LOOP_DISPOSED: i32 = 2;
#[derive(Debug, Default)]
pub struct TryGetOutcome {
pub curr_count: usize,
pub result: Option<CollectionItemResult>,
pub notify_key: Option<Vec<u8>>,
}
impl TryGetOutcome {
pub fn none() -> Self {
Self::default()
}
pub fn with_count(curr_count: usize) -> Self {
Self {
curr_count,
..Self::none()
}
}
pub fn found(curr_count: usize, result: CollectionItemResult) -> Self {
Self {
curr_count,
result: Some(result),
notify_key: None,
}
}
pub fn moved(curr_count: usize, result: CollectionItemResult, notify_key: Vec<u8>) -> Self {
Self {
curr_count,
result: Some(result),
notify_key: Some(notify_key),
}
}
}
pub trait CollectionItemStore: Send + Sync {
fn try_get_result(
&self,
key: &[u8],
command: RespCommand,
cmd_args: &[Vec<u8>],
fail_on_src_type_mismatch: bool,
) -> TryGetOutcome;
}
impl<T: CollectionItemStore + ?Sized> CollectionItemStore for Arc<T> {
#[inline]
fn try_get_result(
&self,
key: &[u8],
command: RespCommand,
cmd_args: &[Vec<u8>],
fail_on_src_type_mismatch: bool,
) -> TryGetOutcome {
(**self).try_get_result(key, command, cmd_args, fail_on_src_type_mismatch)
}
}
pub trait TaskSpawner: Send + Sync {
fn spawn<F>(&self, fut: F)
where
F: Future<Output = ()> + Send + 'static;
}
impl<T: TaskSpawner> TaskSpawner for Arc<T> {
fn spawn<F>(&self, fut: F)
where
F: Future<Output = ()> + Send + 'static,
{
(**self).spawn(fut);
}
}
#[derive(Default, Clone, Copy)]
pub struct CompioTaskSpawner;
impl TaskSpawner for CompioTaskSpawner {
fn spawn<F>(&self, fut: F)
where
F: Future<Output = ()> + Send + 'static,
{
spawn(fut).detach();
}
}
pub struct CollectionItemBroker<S, Spawner = CompioTaskSpawner> {
events_tx: MTx<List<CollectionItemBrokerEvent>>,
events_rx: Mutex<Option<AsyncRx<List<CollectionItemBrokerEvent>>>>,
session_id_to_observer: ConcurrentMap<usize, Arc<CollectionItemObserver>>,
keys_to_observers: KeysToObservers,
keys_to_observers_time_last_clean: AtomicU64,
store: S,
main_loop_task_status: AtomicI32,
cts_cancelled: AtomicBool,
done_tx: Mutex<Option<OneshotTx<()>>>,
done_rx: Mutex<Option<OneshotAsyncRx<()>>>,
spawner: Mutex<Option<Arc<Spawner>>>,
}
impl<S: CollectionItemStore + 'static> CollectionItemBroker<S, CompioTaskSpawner> {
pub fn new(store: S) -> Self {
Self::new_with_spawner(store, CompioTaskSpawner)
}
}
impl<S: CollectionItemStore + 'static, Spawner: TaskSpawner + 'static>
CollectionItemBroker<S, Spawner>
{
pub fn new_with_spawner(store: S, spawner: Spawner) -> Self {
let (events_tx, events_rx) = unbounded_async();
let (done_tx, done_rx) = oneshot();
Self {
events_tx,
events_rx: Mutex::new(Some(events_rx)),
session_id_to_observer: new_concurrent_map(),
keys_to_observers: new_concurrent_map(),
keys_to_observers_time_last_clean: AtomicU64::new(now_secs()),
store,
main_loop_task_status: AtomicI32::new(MAIN_LOOP_NOT_STARTED),
cts_cancelled: AtomicBool::new(false),
done_tx: Mutex::new(Some(done_tx)),
done_rx: Mutex::new(Some(done_rx)),
spawner: Mutex::new(Some(Arc::new(spawner))),
}
}
pub fn set_spawner(&self, spawner: Arc<Spawner>) {
*self.spawner.lock() = Some(spawner);
}
pub fn try_get_observer(&self, session_id: usize) -> Option<Arc<CollectionItemObserver>> {
self.session_id_to_observer.pin().get(&session_id).cloned()
}
pub fn register_session_observer(&self, observer: Arc<CollectionItemObserver>) {
self
.session_id_to_observer
.pin()
.insert(observer.session_id, observer);
}
pub fn pop_broker_event(&self) -> Option<CollectionItemBrokerEvent> {
let mut lock = self.events_rx.lock();
lock.as_mut().and_then(|rx| rx.try_recv().ok())
}
pub async fn get_collection_item_async(
self: &Arc<Self>,
command: RespCommand,
keys: Vec<Vec<u8>>,
session_id: usize,
_timeout_seconds: f64,
cmd_args: Vec<Vec<u8>>,
) -> CollectionItemResult {
let observer = Arc::new(CollectionItemObserver::new(session_id, command, cmd_args));
self.get_collection_item_async_inner(observer, keys).await
}
pub fn start_wait(
self: &Arc<Self>,
command: RespCommand,
keys: Vec<Vec<u8>>,
session_id: usize,
cmd_args: Vec<Vec<u8>>,
) -> Arc<CollectionItemObserver> {
let observer = Arc::new(CollectionItemObserver::new(session_id, command, cmd_args));
self.register_observer(observer.clone(), keys);
self.start_main_loop();
observer
}
pub fn finish_wait(&self, observer: &Arc<CollectionItemObserver>) -> CollectionItemResult {
self
.session_id_to_observer
.pin()
.remove(&observer.session_id);
if observer.status() == ObserverStatus::WaitingForResult {
observer.handle_set_result(CollectionItemResult::empty());
}
observer.result()
}
async fn get_collection_item_async_inner(
self: &Arc<Self>,
observer: Arc<CollectionItemObserver>,
keys: Vec<Vec<u8>>,
) -> CollectionItemResult {
self.register_observer(observer.clone(), keys);
self.start_main_loop();
observer.wait_result().await;
self.finish_wait(&observer)
}
fn register_observer(&self, observer: Arc<CollectionItemObserver>, keys: Vec<Vec<u8>>) {
self
.session_id_to_observer
.pin()
.insert(observer.session_id, observer.clone());
self.enqueue_event(CollectionItemBrokerEvent::create_new_observer_event(
observer, keys,
));
}
pub fn start_main_loop(self: &Arc<Self>) {
if self.main_loop_task_status.load(Ordering::SeqCst) == MAIN_LOOP_NOT_STARTED
&& self
.main_loop_task_status
.compare_exchange(
MAIN_LOOP_NOT_STARTED,
MAIN_LOOP_STARTED,
Ordering::SeqCst,
Ordering::SeqCst,
)
.is_ok()
{
let spawner = self.spawner.lock().clone();
match spawner {
Some(spawner) => {
let broker = Arc::downgrade(self);
spawner.spawn(async move {
if let Some(broker) = broker.upgrade() {
broker.start_async().await;
}
});
}
None => {
self
.main_loop_task_status
.store(MAIN_LOOP_NOT_STARTED, Ordering::SeqCst);
}
}
}
}
pub fn handle_collection_update(&self, key: &[u8]) {
let pin = self.keys_to_observers.pin();
let Some(queue) = pin.get(key) else {
return;
};
if queue.lock().is_empty() {
return;
}
self.enqueue_event(CollectionItemBrokerEvent::create_collection_updated_event(
key.to_vec(),
));
}
pub fn handle_session_disposed(&self, session_id: usize) {
let removed = {
let pin = self.session_id_to_observer.pin();
pin.remove(&session_id).cloned()
};
let Some(observer) = removed else {
return;
};
observer.handle_session_disposed();
}
pub fn handle_broker_event(&self, broker_event: CollectionItemBrokerEvent) {
match broker_event.event_type {
CollectionItemBrokerEventType::NewObserver => {
if let (Some(observer), Some(keys)) = (broker_event.observer, broker_event.keys) {
self.initialize_observer(observer, &keys);
}
}
CollectionItemBrokerEventType::CollectionUpdated => {
if let Some(key) = broker_event.key {
self.try_assign_item_from_key(&key);
}
}
CollectionItemBrokerEventType::NotSet => {}
}
}
pub fn initialize_observer(&self, observer: Arc<CollectionItemObserver>, keys: &[Vec<u8>]) {
let pin = self.keys_to_observers.pin();
for key in keys {
let has_waiting = pin.get(key).is_some_and(|q| !q.lock().is_empty());
if has_waiting {
continue;
}
let outcome = self.try_get_result(key, observer.command, &observer.command_args, true);
let Some(result) = outcome.result else {
continue;
};
self
.session_id_to_observer
.pin()
.remove(&observer.session_id);
observer.handle_set_result(result);
if let Some(queue) = pin.get(key)
&& queue.lock().is_empty()
{
pin.remove(key);
}
if let Some(notify_key) = outcome.notify_key {
self.enqueue_event(CollectionItemBrokerEvent::create_collection_updated_event(
notify_key,
));
}
return;
}
for key in keys {
let queue = match pin.get(key) {
Some(q) => q,
None => pin.get_or_insert_with(key.clone(), || Mutex::new(VecDeque::new())),
};
queue.lock().push_back(observer.clone());
}
}
fn try_assign_item_from_key(&self, key: &[u8]) -> bool {
let mut assigned_any = false;
let pin = self.keys_to_observers.pin();
if let Some(queue) = pin.get(key) {
let mut queue = queue.lock();
while let Some(observer) = queue.front() {
if observer.status() != ObserverStatus::WaitingForResult {
queue.pop_front();
continue;
}
let outcome = self.try_get_result(key, observer.command, &observer.command_args, false);
let Some(result) = outcome.result else {
break;
};
let observer = queue.pop_front().unwrap();
self
.session_id_to_observer
.pin()
.remove(&observer.session_id);
observer.handle_set_result(result);
assigned_any = true;
if let Some(notify_key) = outcome.notify_key {
self.enqueue_event(CollectionItemBrokerEvent::create_collection_updated_event(
notify_key,
));
}
}
if queue.is_empty() {
drop(queue);
pin.remove(key);
}
}
assigned_any
}
fn try_get_result(
&self,
key: &[u8],
command: RespCommand,
cmd_args: &[Vec<u8>],
fail_on_src_type_mismatch: bool,
) -> TryGetOutcome {
self
.store
.try_get_result(key, command, cmd_args, fail_on_src_type_mismatch)
}
pub fn clean_keys_to_observers(&self) {
let pin = self.keys_to_observers.pin();
let mut empty_keys = Vec::new();
for (key, queue) in pin.iter() {
let mut queue = queue.lock();
while let Some(observer) = queue.front() {
if observer.status() != ObserverStatus::WaitingForResult {
queue.pop_front();
} else {
break;
}
}
if queue.is_empty() {
empty_keys.push(key.as_slice());
}
}
for key in empty_keys {
if let Some(queue) = pin.get(key)
&& queue.lock().is_empty()
{
pin.remove(key);
}
}
self
.keys_to_observers_time_last_clean
.store(now_secs(), Ordering::Relaxed);
}
pub async fn start_async(&self) {
let rx = {
let mut lock = self.events_rx.lock();
lock.take()
};
let Some(rx) = rx else {
return;
};
while !self.cts_cancelled.load(Ordering::SeqCst) {
let next_event = match rx.recv().await {
Ok(event) => event,
Err(_) => break,
};
self.handle_broker_event(next_event);
let now = now_secs();
let last = self
.keys_to_observers_time_last_clean
.load(Ordering::Relaxed);
if now.saturating_sub(last) >= MIN_SECS_BETWEEN_KEYS_TO_OBSERVERS_CLEANS {
self.clean_keys_to_observers();
}
}
if let Some(tx) = self.done_tx.lock().take() {
tx.send(());
}
}
pub async fn wait_done(&self) {
let rx = self.done_rx.lock().take();
if let Some(rx) = rx {
let _ = rx.await;
}
}
pub fn dispose(&self) {
self.cts_cancelled.store(true, Ordering::SeqCst);
self.session_id_to_observer.pin().iter().for_each(|(_, o)| {
if o.status() == ObserverStatus::WaitingForResult {
o.try_force_unblock(false);
}
});
let prev = self
.main_loop_task_status
.swap(MAIN_LOOP_DISPOSED, Ordering::SeqCst);
if prev == MAIN_LOOP_STARTED {
let _ = self.events_tx.try_send(CollectionItemBrokerEvent {
event_type: CollectionItemBrokerEventType::NotSet,
key: None,
keys: None,
observer: None,
});
}
}
#[inline]
pub fn is_disposed(&self) -> bool {
self.main_loop_task_status.load(Ordering::SeqCst) == MAIN_LOOP_DISPOSED
}
fn enqueue_event(&self, event: CollectionItemBrokerEvent) {
let _ = self.events_tx.try_send(event);
}
}
pub fn try_get_next_list_item(list_obj: &mut ListObject, command: RespCommand) -> Option<Vec<u8>> {
let list = &mut list_obj.list;
if list.is_empty() {
return None;
}
match command {
RespCommand::Brpop => list.pop_back(),
RespCommand::Blpop => list.pop_front(),
_ => None,
}
}
pub fn try_move_next_list_item(
src_list_obj: &mut ListObject,
dst_list_obj: &mut ListObject,
src_direction: OperationDirection,
dst_direction: OperationDirection,
) -> Option<Vec<u8>> {
if src_direction == OperationDirection::Unknown || dst_direction == OperationDirection::Unknown {
return None;
}
let next_item = {
let src = &mut src_list_obj.list;
if src.is_empty() {
return None;
}
match src_direction {
OperationDirection::Right => src.pop_back(),
OperationDirection::Left => src.pop_front(),
OperationDirection::Unknown => unreachable!(),
}
}?;
{
let dst = &mut dst_list_obj.list;
match dst_direction {
OperationDirection::Right => dst.push_back(next_item.clone()),
OperationDirection::Left => dst.push_front(next_item.clone()),
OperationDirection::Unknown => unreachable!(),
}
}
Some(next_item)
}
pub fn try_get_next_sorted_set_item(
key: &[u8],
sorted_set_obj: &mut SortedSetObject,
count: usize,
command: RespCommand,
cmd_args: &[Vec<u8>],
) -> Option<CollectionItemResult> {
if count == 0 {
return None;
}
match command {
RespCommand::Bzpopmin | RespCommand::Bzpopmax => {
let (score, element) = sorted_set_obj.pop_min_or_max(command == RespCommand::Bzpopmax)?;
Some(CollectionItemResult::single_with_score(
key.to_vec(),
score,
element,
))
}
RespCommand::Bzmpop => {
if cmd_args.len() < 2 || cmd_args[1].len() < 4 {
return None;
}
let low_scores_first = cmd_args[0].first() != Some(&0);
let Ok(pop_bytes) = cmd_args[1][..4].try_into() else {
return None;
};
let pop_count = usize::try_from(i32::from_le_bytes(pop_bytes))
.unwrap_or(0)
.min(count);
let mut scores = Vec::with_capacity(pop_count);
let mut items = Vec::with_capacity(pop_count);
for _ in 0..pop_count {
let Some((score, element)) = sorted_set_obj.pop_min_or_max(!low_scores_first) else {
break;
};
scores.push(score);
items.push(element);
}
Some(CollectionItemResult::multiple_with_scores(
key.to_vec(),
scores,
items,
))
}
_ => None,
}
}
#[cfg(test)]
mod tests {
use compio::runtime::{Runtime, spawn};
use super::*;
struct DummyStore;
impl CollectionItemStore for DummyStore {
fn try_get_result(
&self,
_key: &[u8],
_command: RespCommand,
_cmd_args: &[Vec<u8>],
_fail_on_src_type_mismatch: bool,
) -> TryGetOutcome {
TryGetOutcome::none()
}
}
#[test]
fn broker_done_flow() {
let broker = Arc::new(CollectionItemBroker::new(DummyStore));
let broker_clone = broker.clone();
Runtime::new().unwrap().block_on(async {
let handle = spawn(async move {
broker_clone.start_async().await;
});
broker.dispose();
assert!(broker.is_disposed());
broker.wait_done().await;
broker.wait_done().await;
handle.await.unwrap();
});
}
}