use ahash::AHashSet;
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyList};
use std::collections::{BTreeMap, HashMap, VecDeque};
use std::sync::Mutex;
use std::time::Instant;
use super::pylock::lock;
use super::queue::INITIAL_CAPACITY_CAP;
fn check_window(window_size: usize) -> PyResult<()> {
if window_size == 0 {
return Err(pyo3::exceptions::PyValueError::new_err(
"window_size must be at least 1",
));
}
Ok(())
}
#[pyclass(frozen)]
pub struct RustSequencer {
state: Mutex<SequencerState>,
}
struct SequencerState {
sequence: u64,
seen_ids: AHashSet<String>,
seen_ids_order: VecDeque<(String, Instant)>,
window_size: usize,
}
impl SequencerState {
fn new(window_size: usize) -> Self {
Self {
sequence: 0,
seen_ids: AHashSet::with_capacity(window_size.min(INITIAL_CAPACITY_CAP)),
seen_ids_order: VecDeque::with_capacity(window_size.min(INITIAL_CAPACITY_CAP)),
window_size,
}
}
fn next_seq(&mut self) -> u64 {
self.sequence += 1;
self.sequence
}
fn is_duplicate(&mut self, event_id: &str) -> bool {
if self.seen_ids.contains(event_id) {
return true;
}
if self.seen_ids.len() >= self.window_size
&& let Some((oldest_id, _)) = self.seen_ids_order.pop_front()
{
self.seen_ids.remove(&oldest_id);
}
let owned = event_id.to_owned();
self.seen_ids.insert(owned.clone());
self.seen_ids_order.push_back((owned, Instant::now()));
false
}
fn cleanup(&mut self, max_age_secs: f64) {
let now = Instant::now();
while let Some((id, timestamp)) = self.seen_ids_order.front() {
let age = now.duration_since(*timestamp).as_secs_f64();
if age > max_age_secs {
let id = id.clone();
self.seen_ids_order.pop_front();
self.seen_ids.remove(&id);
} else {
break;
}
}
}
}
#[pymethods]
impl RustSequencer {
#[new]
fn new(window_size: usize) -> PyResult<Self> {
check_window(window_size)?;
Ok(Self {
state: Mutex::new(SequencerState::new(window_size)),
})
}
fn next_seq(&self, py: Python<'_>) -> u64 {
lock(py, &self.state).next_seq()
}
fn get_current_seq(&self, py: Python<'_>) -> u64 {
lock(py, &self.state).sequence
}
fn is_duplicate(&self, py: Python<'_>, event_id: &str) -> bool {
lock(py, &self.state).is_duplicate(event_id)
}
fn cleanup(&self, py: Python<'_>, max_age_secs: f64) {
lock(py, &self.state).cleanup(max_age_secs);
}
fn seen_count(&self, py: Python<'_>) -> usize {
lock(py, &self.state).seen_ids.len()
}
}
struct BufferedEvent {
timestamp: Instant,
payload: Py<PyAny>,
}
type Released = Vec<BufferedEvent>;
#[pyclass(frozen)]
pub struct RustEventSequencer {
state: Mutex<EventSequencerState>,
}
struct EventSequencerState {
sequence: u64,
seen_ids: AHashSet<String>,
seen_ids_queue: VecDeque<String>,
window_size: usize,
max_out_of_order: u64,
expected_sequences: HashMap<String, u64>,
buffered_events: HashMap<String, BTreeMap<u64, BufferedEvent>>,
last_activity: HashMap<String, Instant>,
duplicate_count: u64,
out_of_order_count: u64,
dropped_count: u64,
}
impl EventSequencerState {
fn new(window_size: usize, max_out_of_order: u64) -> Self {
Self {
sequence: 0,
seen_ids: AHashSet::with_capacity(window_size.min(INITIAL_CAPACITY_CAP)),
seen_ids_queue: VecDeque::with_capacity(window_size.min(INITIAL_CAPACITY_CAP)),
window_size,
max_out_of_order,
expected_sequences: HashMap::new(),
buffered_events: HashMap::new(),
last_activity: HashMap::new(),
duplicate_count: 0,
out_of_order_count: 0,
dropped_count: 0,
}
}
fn next_seq(&mut self) -> u64 {
self.sequence += 1;
self.sequence
}
fn is_duplicate(&mut self, event_id: &str) -> bool {
if self.seen_ids.contains(event_id) {
self.duplicate_count += 1;
return true;
}
if self.seen_ids_queue.len() >= self.window_size
&& let Some(old_id) = self.seen_ids_queue.pop_front()
{
self.seen_ids.remove(&old_id);
}
let owned = event_id.to_owned();
self.seen_ids.insert(owned.clone());
self.seen_ids_queue.push_back(owned);
false
}
fn process_sequenced_event<'py>(
&mut self,
py: Python<'py>,
topic: &str,
sequence: u64,
event: Bound<'py, PyAny>,
released: &mut Released,
) -> PyResult<Bound<'py, PyAny>> {
let topic_owned = topic.to_owned();
match self.last_activity.get_mut(topic) {
Some(ts) => *ts = Instant::now(),
None => {
self.last_activity
.insert(topic_owned.clone(), Instant::now());
}
}
if !self.expected_sequences.contains_key(&topic_owned) {
self.expected_sequences
.insert(topic_owned.clone(), sequence + 1);
if let Some(stale) = self.buffered_events.insert(topic_owned, BTreeMap::new()) {
released.extend(stale.into_values());
}
let result = PyList::new(py, [event.clone()])?;
return Ok(result.into_any());
}
let expected = *self.expected_sequences.get(&topic_owned).unwrap();
if sequence == expected {
let mut events_to_deliver: Vec<Bound<'py, PyAny>> = Vec::new();
events_to_deliver.push(event.clone());
let mut next_expected = sequence + 1;
if let Some(buffer) = self.buffered_events.get_mut(&topic_owned) {
while buffer.contains_key(&next_expected) {
let buffered = buffer.remove(&next_expected).unwrap();
events_to_deliver.push(buffered.payload.into_bound(py));
next_expected += 1;
}
}
self.expected_sequences.insert(topic_owned, next_expected);
let result = PyList::new(py, events_to_deliver)?;
Ok(result.into_any())
} else if sequence > expected {
self.out_of_order_count += 1;
if sequence - expected > self.max_out_of_order {
self.dropped_count += sequence - expected - 1;
self.expected_sequences
.insert(topic_owned.clone(), sequence + 1);
if let Some(buffer) = self.buffered_events.get_mut(&topic_owned) {
released.extend(std::mem::take(buffer).into_values());
}
let result = PyList::new(py, [event.clone()])?;
Ok(result.into_any())
} else {
let buffered = BufferedEvent {
timestamp: Instant::now(),
payload: event.clone().unbind(),
};
if let Some(replaced) = self
.buffered_events
.entry(topic_owned)
.or_default()
.insert(sequence, buffered)
{
released.push(replaced);
}
Ok(py.None().into_bound(py))
}
} else {
self.dropped_count += 1;
Ok(py.None().into_bound(py))
}
}
fn reset_sequence(&mut self, topic: Option<&str>, released: &mut Released) {
match topic {
Some(t) => {
self.expected_sequences.remove(t);
if let Some(buffer) = self.buffered_events.remove(t) {
released.extend(buffer.into_values());
}
self.last_activity.remove(t);
}
None => {
self.expected_sequences.clear();
released.extend(
self.buffered_events
.drain()
.flat_map(|(_, buffer)| buffer.into_values()),
);
self.last_activity.clear();
}
}
}
fn get_sequence_stats<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyDict>> {
let stats = PyDict::new(py);
let topic_stats = PyDict::new(py);
let mut total_buffered: usize = 0;
for (topic, &expected) in &self.expected_sequences {
let empty_tree = BTreeMap::new();
let buffered = self.buffered_events.get(topic).unwrap_or(&empty_tree);
total_buffered += buffered.len();
let topic_dict = PyDict::new(py);
topic_dict.set_item("expected", expected)?;
topic_dict.set_item("buffered", buffered.len())?;
let gaps = PyList::empty(py);
if !buffered.is_empty() {
let sequences: Vec<u64> = buffered.keys().copied().collect();
let mut last = expected;
for &seq in &sequences {
if seq - last > 1 {
let gap = PyDict::new(py);
gap.set_item("start", last + 1)?;
gap.set_item("end", seq - 1)?;
gap.set_item("size", seq - last - 1)?;
gaps.append(gap)?;
}
last = seq;
}
}
topic_dict.set_item("gaps", gaps)?;
let buffered_seqs: Vec<u64> = buffered.keys().copied().collect();
let buffered_seqs_list = PyList::new(py, &buffered_seqs)?;
topic_dict.set_item("buffered_sequences", buffered_seqs_list)?;
topic_stats.set_item(topic, topic_dict)?;
}
stats.set_item("current_sequence", self.sequence)?;
stats.set_item("duplicate_count", self.duplicate_count)?;
stats.set_item("out_of_order_count", self.out_of_order_count)?;
stats.set_item("dropped_count", self.dropped_count)?;
stats.set_item("topics", topic_stats)?;
stats.set_item("total_topics", self.expected_sequences.len())?;
stats.set_item("total_buffered", total_buffered)?;
stats.set_item("seen_ids_count", self.seen_ids.len())?;
Ok(stats)
}
fn get_buffer_stats<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyDict>> {
let stats = PyDict::new(py);
let topics_dict = PyDict::new(py);
let now = Instant::now();
let total_buffered: usize = self
.buffered_events
.values()
.map(|events| events.len())
.sum();
for (topic, &expected) in &self.expected_sequences {
if let Some(buffered) = self.buffered_events.get(topic)
&& !buffered.is_empty()
{
let sequences: Vec<u64> = buffered.keys().copied().collect();
let gap_size = sequences.first().map_or(0, |&first| first - expected);
let oldest_age = buffered
.values()
.map(|e| now.duration_since(e.timestamp).as_secs_f64())
.fold(0.0_f64, f64::max);
let topic_dict = PyDict::new(py);
topic_dict.set_item("expected_sequence", expected)?;
topic_dict.set_item("buffered_count", buffered.len())?;
let seqs_list = PyList::new(py, &sequences)?;
topic_dict.set_item("buffered_sequences", seqs_list)?;
topic_dict.set_item("gap_size", gap_size)?;
topic_dict.set_item("oldest_buffered_age", oldest_age)?;
topics_dict.set_item(topic, topic_dict)?;
}
}
stats.set_item("total_topics", self.expected_sequences.len())?;
stats.set_item("total_buffered", total_buffered)?;
stats.set_item("topics", topics_dict)?;
Ok(stats)
}
fn cleanup(&mut self, released: &mut Released) {
let now = Instant::now();
const TTL_SECS: f64 = 300.0;
for events in self.buffered_events.values_mut() {
let to_remove: Vec<u64> = events
.iter()
.filter(|(_, e)| now.duration_since(e.timestamp).as_secs_f64() > TTL_SECS)
.map(|(&seq, _)| seq)
.collect();
for seq in to_remove {
if let Some(expired) = events.remove(&seq) {
released.push(expired);
}
self.dropped_count += 1;
}
}
let idle_topics: Vec<String> = self
.expected_sequences
.keys()
.filter(|t| {
self.last_activity
.get(*t)
.is_none_or(|ts| now.duration_since(*ts).as_secs_f64() > TTL_SECS)
})
.cloned()
.collect();
for topic in idle_topics {
self.expected_sequences.remove(&topic);
if let Some(buffer) = self.buffered_events.remove(&topic) {
released.extend(buffer.into_values());
}
self.last_activity.remove(&topic);
}
let threshold = (self.window_size as f64 * 1.5) as usize;
if self.seen_ids.len() > threshold {
let keep_count = self.window_size;
let drain_count = self.seen_ids_queue.len().saturating_sub(keep_count);
for _ in 0..drain_count {
if let Some(old_id) = self.seen_ids_queue.pop_front() {
self.seen_ids.remove(&old_id);
}
}
}
}
}
#[pymethods]
impl RustEventSequencer {
#[new]
#[pyo3(signature = (window_size=10000, max_out_of_order=100))]
fn new(window_size: usize, max_out_of_order: u64) -> PyResult<Self> {
check_window(window_size)?;
Ok(Self {
state: Mutex::new(EventSequencerState::new(window_size, max_out_of_order)),
})
}
fn next_seq(&self, py: Python<'_>) -> u64 {
lock(py, &self.state).next_seq()
}
fn get_current_sequence(&self, py: Python<'_>) -> u64 {
lock(py, &self.state).sequence
}
fn is_duplicate(&self, py: Python<'_>, event_id: &str) -> bool {
lock(py, &self.state).is_duplicate(event_id)
}
#[pyo3(signature = (topic, sequence, event))]
fn process_sequenced_event<'py>(
&self,
py: Python<'py>,
topic: &str,
sequence: u64,
event: Bound<'py, PyAny>,
) -> PyResult<Bound<'py, PyAny>> {
let mut released = Released::new();
let result = {
let mut state = lock(py, &self.state);
state.process_sequenced_event(py, topic, sequence, event, &mut released)
};
drop(released); result
}
#[pyo3(signature = (topic=None))]
fn reset_sequence(&self, py: Python<'_>, topic: Option<&str>) {
let mut released = Released::new();
lock(py, &self.state).reset_sequence(topic, &mut released);
drop(released); }
fn get_sequence_stats<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyDict>> {
lock(py, &self.state).get_sequence_stats(py)
}
fn get_buffer_stats<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyDict>> {
lock(py, &self.state).get_buffer_stats(py)
}
fn cleanup(&self, py: Python<'_>) {
let mut released = Released::new();
lock(py, &self.state).cleanup(&mut released);
drop(released); }
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;
#[test]
fn cleanup_keeps_cursor_for_active_in_order_topic() {
let mut seq = EventSequencerState::new(100, 100);
seq.expected_sequences.insert("room".to_string(), 42);
seq.buffered_events
.insert("room".to_string(), BTreeMap::new());
seq.last_activity.insert("room".to_string(), Instant::now());
seq.cleanup(&mut Released::new());
assert_eq!(
seq.expected_sequences.get("room"),
Some(&42),
"an active in-order topic's ordering cursor must survive cleanup"
);
}
#[test]
fn cleanup_evicts_idle_topic_cursor() {
let mut seq = EventSequencerState::new(100, 100);
seq.expected_sequences.insert("stale".to_string(), 7);
seq.buffered_events
.insert("stale".to_string(), BTreeMap::new());
let old = Instant::now()
.checked_sub(Duration::from_secs(600))
.expect("instant 600s in the past is representable");
seq.last_activity.insert("stale".to_string(), old);
seq.cleanup(&mut Released::new());
assert!(
!seq.expected_sequences.contains_key("stale"),
"a topic idle past the TTL must have its cursor evicted"
);
assert!(!seq.last_activity.contains_key("stale"));
assert!(!seq.buffered_events.contains_key("stale"));
}
}