use std::{
collections::VecDeque,
pin::Pin,
sync::{
Arc,
atomic::{AtomicBool, AtomicI32, Ordering},
},
};
use parking_lot::{Mutex, RwLock};
use whasher::{GxPapayaMap as HashMap, new_papaya_map};
type ObserverQueue = Mutex<VecDeque<Arc<CollectionItemObserver>>>;
type KeysToObservers = HashMap<Vec<u8>, ObserverQueue>;
use wobject::list::list_object::{ListObject, OperationDirection};
use crate::{
objects::{
itembroker::{
collection_item_broker_event::{CollectionItemBrokerEvent, CollectionItemBrokerEventType},
collection_item_observer::{
CollectionItemObserver, CollectionItemResult, ObserverStatus, Wakeup,
},
},
sortedset::sorted_set_object::SortedSetObject,
},
types::RespCommand,
};
const MIN_SECS_BETWEEN_KEYS_TO_OBSERVERS_CLEANS: u64 = 5 * 60;
const MAIN_LOOP_NOT_STARTED: i32 = 0;
const MAIN_LOOP_STARTED: i32 = 1;
const MAIN_LOOP_DISPOSED: i32 = 2;
pub trait CollectionItemStore: Send + Sync {
fn try_get_result(
&self,
key: &[u8],
command: RespCommand,
cmd_args: &[Vec<u8>],
fail_on_src_type_mismatch: bool,
) -> (usize, Option<CollectionItemResult>);
}
pub trait TaskSpawner: Send + Sync {
fn spawn(&self, fut: Pin<Box<dyn Future<Output = ()> + Send>>);
}
pub struct CollectionItemBroker {
broker_events_queue: Mutex<VecDeque<CollectionItemBrokerEvent>>,
events_notify: Wakeup,
session_id_to_observer: HashMap<usize, Arc<CollectionItemObserver>>,
keys_to_observers: RwLock<Option<KeysToObservers>>,
keys_to_observers_time_last_clean: Mutex<coarsetime::Instant>,
store: Arc<dyn CollectionItemStore>,
main_loop_task_status: AtomicI32,
cts_cancelled: AtomicBool,
done: Wakeup,
spawner: Mutex<Option<Arc<dyn TaskSpawner>>>,
}
impl CollectionItemBroker {
pub fn new(store: Arc<dyn CollectionItemStore>) -> Self {
Self {
broker_events_queue: Mutex::new(VecDeque::new()),
events_notify: Wakeup::new(),
session_id_to_observer: new_papaya_map(),
keys_to_observers: RwLock::new(None),
keys_to_observers_time_last_clean: Mutex::new(coarsetime::Instant::now()),
store,
main_loop_task_status: AtomicI32::new(MAIN_LOOP_NOT_STARTED),
cts_cancelled: AtomicBool::new(false),
done: Wakeup::new(),
spawner: Mutex::new(None),
}
}
pub fn set_spawner(&self, spawner: Arc<dyn TaskSpawner>) {
*self.spawner.lock() = Some(spawner);
}
pub fn try_get_observer(&self, session_id: usize) -> Option<Arc<CollectionItemObserver>> {
self.session_id_to_observer.pin().get(&session_id).cloned()
}
pub async fn get_collection_item_async(
self: &Arc<Self>,
command: RespCommand,
keys: Vec<Vec<u8>>,
session_id: usize,
_timeout_seconds: f64,
cmd_args: Vec<Vec<u8>>,
) -> CollectionItemResult {
let observer = Arc::new(CollectionItemObserver::new(session_id, command, cmd_args));
self.get_collection_item_async_inner(observer, keys).await
}
pub async fn move_collection_item_async(
self: &Arc<Self>,
command: RespCommand,
src_key: Vec<u8>,
session_id: usize,
timeout_seconds: f64,
cmd_args: Vec<Vec<u8>>,
) -> CollectionItemResult {
self
.get_collection_item_async(
command,
vec![src_key],
session_id,
timeout_seconds,
cmd_args,
)
.await
}
async fn get_collection_item_async_inner(
self: &Arc<Self>,
observer: Arc<CollectionItemObserver>,
keys: Vec<Vec<u8>>,
) -> CollectionItemResult {
self
.session_id_to_observer
.pin()
.insert(observer.session_id, observer.clone());
self.start_main_loop();
self.enqueue_event(CollectionItemBrokerEvent::create_new_observer_event(
observer.clone(),
keys,
));
observer.wait_result().await;
self
.session_id_to_observer
.pin()
.remove(&observer.session_id);
if observer.status() == ObserverStatus::WaitingForResult {
observer.handle_set_result(CollectionItemResult::empty());
}
observer.result()
}
pub fn start_main_loop(self: &Arc<Self>) {
if self.main_loop_task_status.load(Ordering::SeqCst) == MAIN_LOOP_NOT_STARTED
&& self
.main_loop_task_status
.compare_exchange(
MAIN_LOOP_NOT_STARTED,
MAIN_LOOP_STARTED,
Ordering::SeqCst,
Ordering::SeqCst,
)
.is_ok()
{
let spawner = self.spawner.lock().clone();
match spawner {
Some(spawner) => {
let broker = Arc::downgrade(self);
spawner.spawn(Box::pin(async move {
if let Some(broker) = broker.upgrade() {
broker.start_async().await;
}
}));
}
None => {
self
.main_loop_task_status
.store(MAIN_LOOP_NOT_STARTED, Ordering::SeqCst);
}
}
}
}
pub fn handle_collection_update(&self, key: &[u8]) {
if self.keys_to_observers.read().is_none() {
return;
}
self.handle_collection_update_worker(key.to_vec());
}
fn handle_collection_update_worker(&self, key: Vec<u8>) {
let has_queue = match self.keys_to_observers.read().as_ref() {
Some(m) => {
let pin = m.pin();
pin.get(&key).map(|q| !q.lock().is_empty())
}
None => None,
};
let Some(has_waiting) = has_queue else {
return;
};
if !has_waiting {
let mut map = self.keys_to_observers.write();
if let Some(m) = map.as_mut() {
let empty = m.pin().get(&key).is_some_and(|q| q.lock().is_empty());
if empty {
m.pin().remove(&key);
}
}
}
self.enqueue_event(CollectionItemBrokerEvent::create_collection_updated_event(
key,
));
}
pub fn handle_session_disposed(&self, session_id: usize) {
let removed = {
let pin = self.session_id_to_observer.pin();
pin.remove(&session_id).cloned()
};
let Some(observer) = removed else {
return;
};
observer.handle_session_disposed();
}
fn handle_broker_event(&self, broker_event: CollectionItemBrokerEvent) {
match broker_event.event_type {
CollectionItemBrokerEventType::NewObserver => {
if let (Some(observer), Some(keys)) = (broker_event.observer, broker_event.keys) {
self.initialize_observer(observer, &keys);
}
}
CollectionItemBrokerEventType::CollectionUpdated => {
if let Some(key) = broker_event.key {
self.try_assign_item_from_key(&key);
}
}
CollectionItemBrokerEventType::NotSet => {}
}
}
fn initialize_observer(&self, observer: Arc<CollectionItemObserver>, keys: &[Vec<u8>]) {
let mut map = self.keys_to_observers.write();
for key in keys {
let has_waiting = match map.as_ref() {
Some(m) => {
let pin = m.pin();
pin.get(key).is_some_and(|q| !q.lock().is_empty())
}
None => false,
};
if has_waiting {
continue;
}
let (_, result) = self.try_get_result(key, observer.command, &observer.command_args, true);
let Some(result) = result else {
continue;
};
self
.session_id_to_observer
.pin()
.remove(&observer.session_id);
observer.handle_set_result(result);
if let Some(m) = map.as_mut()
&& let Some(queue) = m.pin().get(key)
&& queue.lock().is_empty()
{
m.pin().remove(key);
}
return;
}
let m = map.get_or_insert_with(new_papaya_map);
let pin = m.pin();
for key in keys {
match pin.get(key) {
Some(queue) => queue.lock().push_back(observer.clone()),
None => {
pin.insert(key.clone(), Mutex::new(VecDeque::from([observer.clone()])));
}
}
}
}
fn try_assign_item_from_key(&self, key: &[u8]) -> bool {
{
let map = self.keys_to_observers.read();
if let Some(m) = map.as_ref() {
let pin = m.pin();
if let Some(queue) = pin.get(key) {
let mut queue = queue.lock();
while let Some(observer) = queue.front().cloned() {
if observer.status() != ObserverStatus::WaitingForResult {
queue.pop_front();
continue;
}
let (curr_count, result) =
self.try_get_result(key, observer.command, &observer.command_args, false);
let Some(result) = result else {
if curr_count > 0 {
queue.pop_front();
continue;
}
return false;
};
queue.pop_front();
self
.session_id_to_observer
.pin()
.remove(&observer.session_id);
observer.handle_set_result(result);
return true;
}
}
}
}
let mut map = self.keys_to_observers.write();
if let Some(m) = map.as_mut() {
let pin = m.pin();
let empty = pin.get(key).is_some_and(|q| q.lock().is_empty());
if empty {
pin.remove(key);
}
}
false
}
fn try_get_result(
&self,
key: &[u8],
command: RespCommand,
cmd_args: &[Vec<u8>],
fail_on_src_type_mismatch: bool,
) -> (usize, Option<CollectionItemResult>) {
self
.store
.try_get_result(key, command, cmd_args, fail_on_src_type_mismatch)
}
pub fn clean_keys_to_observers(&self) {
let mut map = self.keys_to_observers.write();
if let Some(m) = map.as_mut() {
let pin = m.pin();
let keys: Vec<Vec<u8>> = pin.keys().cloned().collect();
for key in keys {
if let Some(queue) = pin.get(&key) {
let mut queue = queue.lock();
while let Some(observer) = queue.front() {
if observer.status() != ObserverStatus::WaitingForResult {
queue.pop_front();
} else {
break;
}
}
}
if pin.get(&key).is_some_and(|q| q.lock().is_empty()) {
pin.remove(&key);
}
}
}
*self.keys_to_observers_time_last_clean.lock() = coarsetime::Instant::now();
}
pub async fn start_async(&self) {
loop {
if self.cts_cancelled.load(Ordering::SeqCst) {
break;
}
let next_event = {
let mut queue = self.broker_events_queue.lock();
queue.pop_front()
};
let Some(next_event) = next_event else {
self.events_notify.wait().await;
continue;
};
self.handle_broker_event(next_event);
let elapsed = self.keys_to_observers_time_last_clean.lock().elapsed();
if elapsed > coarsetime::Duration::from_secs(MIN_SECS_BETWEEN_KEYS_TO_OBSERVERS_CLEANS) {
self.clean_keys_to_observers();
}
}
self.done.notify_one();
}
pub fn dispose(&self) {
self.cts_cancelled.store(true, Ordering::SeqCst);
self.session_id_to_observer.pin().iter().for_each(|(_, o)| {
if o.status() == ObserverStatus::WaitingForResult {
o.try_force_unblock(false);
}
});
let prev = self
.main_loop_task_status
.swap(MAIN_LOOP_DISPOSED, Ordering::SeqCst);
if prev == MAIN_LOOP_STARTED {
self.events_notify.notify_one();
}
}
fn enqueue_event(&self, event: CollectionItemBrokerEvent) {
self.broker_events_queue.lock().push_back(event);
self.events_notify.notify_one();
}
}
pub fn try_get_next_list_item(list_obj: &ListObject, command: RespCommand) -> Option<Vec<u8>> {
let mut list = list_obj.list.lock();
if list.is_empty() {
return None;
}
match command {
RespCommand::Brpop => list.pop_back(),
RespCommand::Blpop => list.pop_front(),
_ => None,
}
}
pub fn try_move_next_list_item(
src_list_obj: &ListObject,
dst_list_obj: &ListObject,
src_direction: OperationDirection,
dst_direction: OperationDirection,
) -> Option<Vec<u8>> {
let next_item = {
let mut src = src_list_obj.list.lock();
if src.is_empty() {
return None;
}
match src_direction {
OperationDirection::Right => src.pop_back(),
OperationDirection::Left => src.pop_front(),
OperationDirection::Unknown => return None,
}
}?;
{
let mut dst = dst_list_obj.list.lock();
match dst_direction {
OperationDirection::Right => dst.push_back(next_item.clone()),
OperationDirection::Left => dst.push_front(next_item.clone()),
OperationDirection::Unknown => return None,
}
}
Some(next_item)
}
pub fn try_get_next_sorted_set_item(
key: &[u8],
sorted_set_obj: &mut SortedSetObject,
count: usize,
command: RespCommand,
cmd_args: &[Vec<u8>],
) -> Option<CollectionItemResult> {
if count == 0 {
return None;
}
match command {
RespCommand::Bzpopmin | RespCommand::Bzpopmax => {
let (score, element) = sorted_set_obj.pop_min_or_max(command == RespCommand::Bzpopmax)?;
Some(CollectionItemResult::single_with_score(
key.to_vec(),
score,
element,
))
}
RespCommand::Bzmpop => {
if cmd_args.len() < 2 || cmd_args[1].len() < 4 {
return None;
}
let low_scores_first = cmd_args[0].first() != Some(&0);
let pop_count = usize::try_from(i32::from_le_bytes(cmd_args[1][..4].try_into().unwrap()))
.unwrap_or(0)
.min(count);
let mut scores = Vec::with_capacity(pop_count);
let mut items = Vec::with_capacity(pop_count);
for _ in 0..pop_count {
let Some((score, element)) = sorted_set_obj.pop_min_or_max(!low_scores_first) else {
break;
};
scores.push(score);
items.push(element);
}
Some(CollectionItemResult::multiple_with_scores(
key.to_vec(),
scores,
items,
))
}
_ => None,
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap as StdHashMap;
use wobject::list::list_object::ListOperation;
use super::*;
use crate::objects::itembroker::collection_item_observer::CollectionItemObserver as Obs;
struct MemStore(Mutex<StdHashMap<Vec<u8>, VecDeque<Vec<u8>>>>);
impl MemStore {
fn new() -> Self {
Self(Mutex::new(StdHashMap::new()))
}
fn push(&self, key: &[u8], item: &[u8]) {
self
.0
.lock()
.entry(key.to_vec())
.or_default()
.push_back(item.to_vec());
}
}
impl CollectionItemStore for MemStore {
fn try_get_result(
&self,
key: &[u8],
command: RespCommand,
_cmd_args: &[Vec<u8>],
_fail_on_src_type_mismatch: bool,
) -> (usize, Option<CollectionItemResult>) {
let mut map = self.0.lock();
let Some(queue) = map.get_mut(key) else {
return (0, None);
};
let count = queue.len();
match command {
RespCommand::Blpop | RespCommand::Brpop => {
let item = if command == RespCommand::Blpop {
queue.pop_front()
} else {
queue.pop_back()
};
match item {
Some(item) => (
count - 1,
Some(CollectionItemResult::single(key.to_vec(), item)),
),
None => (0, None),
}
}
_ => (count, None),
}
}
}
#[test]
fn broker_assigns_item_to_waiting_observer() {
let store = Arc::new(MemStore::new());
let broker = Arc::new(CollectionItemBroker::new(store.clone()));
let observer = Arc::new(Obs::new(1, RespCommand::Blpop, vec![]));
store.push(b"k", b"item-1");
broker.initialize_observer(observer.clone(), &[b"k".to_vec()]);
assert_eq!(observer.status(), ObserverStatus::ResultSet);
let result = observer.result();
assert_eq!(result.item.as_deref(), Some(b"item-1".as_slice()));
assert!(broker.try_get_observer(1).is_none());
}
#[test]
fn broker_queues_observer_until_update() {
let store = Arc::new(MemStore::new());
let broker = Arc::new(CollectionItemBroker::new(store.clone()));
let observer = Arc::new(Obs::new(2, RespCommand::Blpop, vec![]));
broker
.session_id_to_observer
.pin()
.insert(observer.session_id, observer.clone());
broker.initialize_observer(observer.clone(), &[b"k".to_vec()]);
assert_eq!(observer.status(), ObserverStatus::WaitingForResult);
assert!(broker.try_get_observer(2).is_some());
store.push(b"k", b"item-2");
broker.handle_collection_update(b"k");
while let Some(event) = broker.broker_events_queue.lock().pop_front() {
broker.handle_broker_event(event);
}
assert_eq!(observer.status(), ObserverStatus::ResultSet);
assert_eq!(
observer.result().item.as_deref(),
Some(b"item-2".as_slice())
);
}
#[test]
fn handle_session_disposed_removes_observer() {
let store = Arc::new(MemStore::new());
let broker = Arc::new(CollectionItemBroker::new(store));
let observer = Arc::new(Obs::new(9, RespCommand::Blpop, vec![]));
broker
.session_id_to_observer
.pin()
.insert(observer.session_id, observer.clone());
broker.initialize_observer(observer.clone(), &[b"k".to_vec()]);
assert!(broker.try_get_observer(9).is_some());
broker.handle_session_disposed(9);
assert!(broker.try_get_observer(9).is_none());
assert_eq!(observer.status(), ObserverStatus::SessionDisposed);
}
#[test]
fn clean_removes_finished_observers_and_empty_keys() {
let store = Arc::new(MemStore::new());
let broker = Arc::new(CollectionItemBroker::new(store));
let done = Arc::new(Obs::new(3, RespCommand::Blpop, vec![]));
let waiting = Arc::new(Obs::new(4, RespCommand::Blpop, vec![]));
broker
.session_id_to_observer
.pin()
.insert(done.session_id, done.clone());
broker.initialize_observer(done.clone(), &[b"k".to_vec()]);
broker.initialize_observer(waiting.clone(), &[b"k".to_vec()]);
broker.clean_keys_to_observers();
assert_eq!(waiting.status(), ObserverStatus::WaitingForResult);
}
#[test]
fn list_item_helpers() {
let src = ListObject::new();
let dst = ListObject::new();
src.operate(ListOperation::Rpush, b"l");
src.operate(ListOperation::Rpush, b"r");
assert_eq!(
try_get_next_list_item(&src, RespCommand::Blpop),
Some(b"l".to_vec())
);
assert_eq!(
try_get_next_list_item(&src, RespCommand::Brpop),
Some(b"r".to_vec())
);
assert_eq!(try_get_next_list_item(&src, RespCommand::Blpop), None);
src.operate(ListOperation::Rpush, b"a");
src.operate(ListOperation::Rpush, b"b");
let moved = try_move_next_list_item(
&src,
&dst,
OperationDirection::Right,
OperationDirection::Left,
);
assert_eq!(moved, Some(b"b".to_vec()));
assert_eq!(dst.index(0), Some(b"b".to_vec()));
}
#[test]
fn main_loop_wakes_waiting_observer() {
use std::task::{Context, Poll, Waker};
struct Collector(Mutex<Vec<Pin<Box<dyn Future<Output = ()> + Send>>>>);
impl TaskSpawner for Collector {
fn spawn(&self, fut: Pin<Box<dyn Future<Output = ()> + Send>>) {
self.0.lock().push(fut);
}
}
fn noop_waker() -> Waker {
Waker::noop().clone()
}
let store = Arc::new(MemStore::new());
let broker = Arc::new(CollectionItemBroker::new(store.clone()));
let collector = Arc::new(Collector(Mutex::new(Vec::new())));
broker.set_spawner(collector.clone());
let waiter_fut =
broker.get_collection_item_async(RespCommand::Blpop, vec![b"k".to_vec()], 6, 0.0, vec![]);
let mut waiter_fut = Box::pin(waiter_fut);
let waker = noop_waker();
let mut cx = Context::from_waker(&waker);
let mut fed = false;
for _ in 0..10_000 {
let _ = waiter_fut.as_mut().poll(&mut cx);
for fut in collector.0.lock().iter_mut() {
let _ = fut.as_mut().poll(&mut cx);
}
if !fed && broker.try_get_observer(6).is_some() {
store.push(b"k", b"late-item");
broker.handle_collection_update(b"k");
fed = true;
}
if let Poll::Ready(result) = waiter_fut.as_mut().poll(&mut cx) {
assert_eq!(result.item.as_deref(), Some(b"late-item".as_slice()));
assert!(broker.try_get_observer(6).is_none());
break;
}
}
assert!(fed, "observer never queued");
assert!(broker.try_get_observer(6).is_none());
}
}