use std::{
collections::VecDeque,
fmt,
sync::{Arc, Mutex},
time::{SystemTime, UNIX_EPOCH},
};
use apalis_rtui_types::{ActivityBucket, EntryType, LogEntry, TaskStatusEvent};
use futures::{
StreamExt,
channel::mpsc::{Receiver, Sender, channel},
};
use crate::api::{
capability::{STATE_POISON, SharedState},
sse::{Client, TracingBroadcaster},
};
const REPLAY_CAPACITY: usize = 1000;
const ACTIVITY_MINUTES: usize = 60;
const CLIENT_CHANNEL_CAPACITY: usize = 100;
pub(crate) const RING_POISON: &str = "apalis-rtui: EventRing mutex poisoned";
#[derive(Debug, Clone, Default)]
pub(crate) struct StatusFilter {
pub(crate) queue: Option<String>,
pub(crate) task_id: Option<String>,
}
impl StatusFilter {
fn matches(&self, event: &TaskStatusEvent) -> bool {
self.queue.as_deref().is_none_or(|q| q == event.queue)
&& self.task_id.as_deref().is_none_or(|id| id == event.task_id)
}
}
#[derive(Debug, Clone, Copy, Default)]
struct ActivitySlot {
minute_epoch: Option<i64>,
start: u64,
done: u64,
failed: u64,
}
struct StatusClient {
filter: StatusFilter,
sender: Sender<TaskStatusEvent>,
}
impl fmt::Debug for StatusClient {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("StatusClient")
.field("filter", &self.filter)
.finish_non_exhaustive()
}
}
#[derive(Debug)]
pub(crate) struct EventRing {
activity: [ActivitySlot; ACTIVITY_MINUTES],
replay: VecDeque<TaskStatusEvent>,
clients: Vec<StatusClient>,
feeding: bool,
}
impl Default for EventRing {
fn default() -> Self {
Self {
activity: [ActivitySlot::default(); ACTIVITY_MINUTES],
replay: VecDeque::with_capacity(REPLAY_CAPACITY),
clients: Vec::new(),
feeding: false,
}
}
}
impl EventRing {
#[must_use]
pub(crate) fn create() -> Arc<Mutex<Self>> {
Arc::new(Mutex::new(Self::default()))
}
pub(crate) fn ensure_feeding(
ring: &Arc<Mutex<Self>>,
broadcaster: &Arc<Mutex<TracingBroadcaster>>,
state: Arc<Mutex<SharedState>>,
) {
let mut guard = ring.lock().expect(RING_POISON);
if guard.feeding {
return;
}
guard.feeding = true;
drop(guard);
let client = broadcaster
.lock()
.expect("invariant: broadcaster mutex is never held across a panic")
.new_client();
tokio::spawn(feed(Arc::clone(ring), client, state));
}
fn record(&mut self, event: &TaskStatusEvent) {
self.bump_activity(&event.status);
if self.replay.len() >= REPLAY_CAPACITY {
self.replay.pop_front();
}
self.replay.push_back(event.clone());
self.clients.retain_mut(|client| {
if !client.filter.matches(event) {
return !client.sender.is_closed();
}
match client.sender.try_send(event.clone()) {
Ok(()) => true,
Err(error) => !error.is_disconnected(),
}
});
}
fn bump_activity(&mut self, status: &str) {
let minute = unix_now_minute();
let idx = bucket_index(minute);
let slot = &mut self.activity[idx];
if slot.minute_epoch != Some(minute) {
*slot = ActivitySlot {
minute_epoch: Some(minute),
..ActivitySlot::default()
};
}
match status {
"Running" => slot.start += 1,
"Done" => slot.done += 1,
"Failed" | "Killed" => slot.failed += 1,
_ => {}
}
}
pub(crate) fn activity(&self) -> Vec<ActivityBucket> {
let now = unix_now_minute();
(0..ACTIVITY_MINUTES)
.map(|offset_from_oldest| {
let minute = now
- i64::try_from(ACTIVITY_MINUTES - 1 - offset_from_oldest).unwrap_or(i64::MAX);
let minute_epoch = minute.saturating_mul(60);
let slot = &self.activity[bucket_index(minute)];
if slot.minute_epoch == Some(minute) {
ActivityBucket {
minute_epoch,
start: slot.start,
done: slot.done,
failed: slot.failed,
}
} else {
ActivityBucket {
minute_epoch,
..ActivityBucket::default()
}
}
})
.collect()
}
pub(crate) fn subscribe(
&mut self,
filter: StatusFilter,
) -> (Vec<TaskStatusEvent>, Receiver<TaskStatusEvent>) {
let replay: Vec<_> = self
.replay
.iter()
.filter(|event| filter.matches(event))
.cloned()
.collect();
let (sender, receiver) = channel(CLIENT_CHANNEL_CAPACITY);
self.clients.push(StatusClient { filter, sender });
(replay, receiver)
}
}
fn bucket_index(minute: i64) -> usize {
usize::try_from(minute.rem_euclid(i64::try_from(ACTIVITY_MINUTES).unwrap_or(i64::MAX)))
.unwrap_or(0)
}
fn unix_now_minute() -> i64 {
let secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
i64::try_from(secs / 60).unwrap_or(i64::MAX)
}
async fn feed(ring: Arc<Mutex<EventRing>>, mut client: Client, state: Arc<Mutex<SharedState>>) {
while let Some(Ok(entry)) = client.next().await {
if let Some(event) = derive_event(&entry, &state) {
ring.lock().expect(RING_POISON).record(&event);
}
}
}
fn derive_event(entry: &LogEntry, state: &Mutex<SharedState>) -> Option<TaskStatusEvent> {
let span = entry.span.as_ref()?;
let message = entry.entry.message.as_deref()?;
let status = match message {
"task.start" => "Running",
"task.done" => "Done",
"task.failed" => "Failed",
_ => return None,
};
let (done_in_ms, result) = match &entry.entry.entry_type {
Some(EntryType::Result(result_entry)) => (
parse_done_in_ms(&result_entry.done_in),
Some(result_entry.result.clone()),
),
_ => (None, None),
};
let attempt = u32::try_from(span.attempt).unwrap_or(0);
let queue = state
.lock()
.expect(STATE_POISON)
.queue_for_task(&span.task_id)
.map(str::to_owned)
.unwrap_or_default();
Some(TaskStatusEvent {
task_id: span.task_id.clone(),
queue,
status: status.to_owned(),
attempt,
done_in_ms,
result,
})
}
fn parse_done_in_ms(raw: &str) -> Option<u64> {
let trimmed = raw.trim();
if let Some(value) = trimmed.strip_suffix("ms") {
value.trim().parse().ok()
} else if let Some(value) = trimmed.strip_suffix("μs") {
value.trim().parse::<u64>().ok().map(|us| us / 1_000)
} else if let Some(value) = trimmed.strip_suffix("ns") {
value.trim().parse::<u64>().ok().map(|ns| ns / 1_000_000)
} else if let Some(value) = trimmed.strip_suffix('s') {
value.trim().parse::<f64>().ok().and_then(|secs| {
let millis = secs * 1000.0;
(millis.is_finite() && millis >= 0.0).then_some(millis.round() as u64)
})
} else {
None
}
}
#[cfg(test)]
mod tests {
use apalis_rtui_types::{Entry, LineEntry, Span};
use super::*;
fn task_frame(message: &str, task_id: &str, attempt: i64) -> LogEntry {
LogEntry {
span: Some(Span {
attempt,
task_id: task_id.to_owned(),
name: "task".to_owned(),
}),
level: apalis_rtui_types::LogLevel::Debug,
timestamp: "2024-01-01T00:00:00Z".to_owned(),
target: "apalis::layers::tracing::on_request".to_owned(),
entry: Entry {
message: Some(message.to_owned()),
entry_type: None,
},
}
}
fn terminal_frame(
message: &str,
task_id: &str,
attempt: i64,
done_in: &str,
result: &str,
) -> LogEntry {
let mut entry = task_frame(message, task_id, attempt);
entry.entry.entry_type = Some(EntryType::Result(apalis_rtui_types::ResultEntry {
done_in: done_in.to_owned(),
result: result.to_owned(),
}));
entry
}
#[test]
fn parse_done_in_ms_handles_every_apalis_unit() {
assert_eq!(parse_done_in_ms("120ms"), Some(120));
assert_eq!(parse_done_in_ms("1500μs"), Some(1));
assert_eq!(parse_done_in_ms("2500000ns"), Some(2));
assert_eq!(parse_done_in_ms("1.5s"), Some(1500));
assert_eq!(parse_done_in_ms("garbage"), None);
}
#[test]
fn derive_event_skips_frames_with_no_span_or_unrecognized_message() {
let state = Arc::new(Mutex::new(SharedState::default()));
let mut no_span = task_frame("task.start", "t1", 0);
no_span.span = None;
assert!(derive_event(&no_span, &state).is_none());
let other_message = task_frame("something else", "t1", 0);
assert!(derive_event(&other_message, &state).is_none());
}
#[test]
fn derive_event_reports_running_for_task_start_with_no_terminal_fields() {
let state = Arc::new(Mutex::new(SharedState::default()));
let event = derive_event(&task_frame("task.start", "t1", 2), &state)
.expect("task.start derives an event");
assert_eq!(event.status, "Running");
assert_eq!(event.attempt, 2);
assert_eq!(event.done_in_ms, None);
assert_eq!(event.result, None);
assert_eq!(
event.queue, "",
"unrecorded task_id resolves to an empty queue"
);
}
#[test]
fn derive_event_resolves_queue_once_recorded() {
let state = Arc::new(Mutex::new(SharedState::default()));
state.lock().unwrap().record_task_queue("t1", "emails");
let event = derive_event(&task_frame("task.done", "t1", 1), &state)
.expect("task.done derives an event");
assert_eq!(event.queue, "emails");
}
#[test]
fn derive_event_carries_done_in_and_result_on_terminal_frames() {
let state = Arc::new(Mutex::new(SharedState::default()));
let frame = terminal_frame("task.failed", "t1", 1, "12ms", "boom");
let event = derive_event(&frame, &state).expect("task.failed derives an event");
assert_eq!(event.status, "Failed");
assert_eq!(event.done_in_ms, Some(12));
assert_eq!(event.result.as_deref(), Some("boom"));
}
#[test]
fn derive_event_ignores_a_line_entry_result_type() {
let state = Arc::new(Mutex::new(SharedState::default()));
let mut frame = task_frame("some log line", "t1", 0);
frame.entry.entry_type = Some(EntryType::Line(LineEntry::default()));
assert!(derive_event(&frame, &state).is_none());
}
#[test]
fn status_filter_matches_on_every_axis_independently() {
let event = TaskStatusEvent {
task_id: "t1".to_owned(),
queue: "emails".to_owned(),
status: "Done".to_owned(),
attempt: 0,
done_in_ms: None,
result: None,
};
assert!(StatusFilter::default().matches(&event));
assert!(
StatusFilter {
queue: Some("emails".to_owned()),
task_id: None,
}
.matches(&event)
);
assert!(
!StatusFilter {
queue: Some("notifications".to_owned()),
task_id: None,
}
.matches(&event)
);
assert!(
StatusFilter {
queue: None,
task_id: Some("t1".to_owned()),
}
.matches(&event)
);
assert!(
!StatusFilter {
queue: None,
task_id: Some("t2".to_owned()),
}
.matches(&event)
);
}
#[test]
fn record_bumps_the_matching_activity_bucket_and_replays_it() {
let mut ring = EventRing::default();
ring.record(&TaskStatusEvent {
task_id: "t1".to_owned(),
queue: "emails".to_owned(),
status: "Running".to_owned(),
attempt: 0,
done_in_ms: None,
result: None,
});
ring.record(&TaskStatusEvent {
task_id: "t1".to_owned(),
queue: "emails".to_owned(),
status: "Done".to_owned(),
attempt: 0,
done_in_ms: Some(5),
result: Some("ok".to_owned()),
});
let current = ring.activity().pop().expect("60 buckets, at least one");
assert_eq!(current.start, 1);
assert_eq!(current.done, 1);
assert_eq!(current.failed, 0);
assert_eq!(
current.minute_epoch,
unix_now_minute() * 60,
"minute_epoch is Unix epoch seconds truncated to the minute, not the minute counter"
);
let (replay, _live) = ring.subscribe(StatusFilter::default());
assert_eq!(replay.len(), 2);
}
#[test]
fn replay_buffer_is_bounded_at_capacity() {
let mut ring = EventRing::default();
for i in 0..REPLAY_CAPACITY + 10 {
ring.record(&TaskStatusEvent {
task_id: format!("t{i}"),
queue: "emails".to_owned(),
status: "Done".to_owned(),
attempt: 0,
done_in_ms: None,
result: None,
});
}
let (replay, _live) = ring.subscribe(StatusFilter::default());
assert_eq!(replay.len(), REPLAY_CAPACITY);
assert_eq!(replay[0].task_id, "t10", "the oldest 10 should be evicted");
}
#[test]
fn activity_is_always_exactly_60_zero_filled_buckets_when_nothing_recorded() {
let ring = EventRing::default();
let buckets = ring.activity();
assert_eq!(buckets.len(), ACTIVITY_MINUTES);
assert!(
buckets
.iter()
.all(|b| b.start == 0 && b.done == 0 && b.failed == 0)
);
}
#[test]
fn activity_buckets_are_consecutive_minute_truncated_epoch_seconds() {
let ring = EventRing::default();
let buckets = ring.activity();
let last = buckets.last().expect("60 buckets").minute_epoch;
assert_eq!(last, unix_now_minute() * 60);
for (index, bucket) in buckets.iter().enumerate() {
assert_eq!(
bucket.minute_epoch % 60,
0,
"bucket {index} is not truncated to the minute"
);
let steps_before_last = i64::try_from(ACTIVITY_MINUTES - 1 - index).expect("fits i64");
assert_eq!(bucket.minute_epoch, last - steps_before_last * 60);
}
}
#[tokio::test]
async fn subscribe_delivers_live_events_matching_the_filter_only() {
let mut ring = EventRing::default();
let (_replay, mut live) = ring.subscribe(StatusFilter {
queue: None,
task_id: Some("t1".to_owned()),
});
ring.record(&TaskStatusEvent {
task_id: "t2".to_owned(),
queue: "emails".to_owned(),
status: "Done".to_owned(),
attempt: 0,
done_in_ms: None,
result: None,
});
ring.record(&TaskStatusEvent {
task_id: "t1".to_owned(),
queue: "emails".to_owned(),
status: "Done".to_owned(),
attempt: 0,
done_in_ms: None,
result: None,
});
let event = live.next().await.expect("t1's event was delivered");
assert_eq!(event.task_id, "t1");
assert!(
futures::poll!(live.next()).is_pending(),
"t2's event must not have been delivered"
);
}
}