use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use a2a_protocol_types::error::{A2aError, A2aResult};
use a2a_protocol_types::events::StreamResponse;
use tokio::sync::{broadcast, mpsc};
use super::{EventQueueReader, EventQueueWriter, StreamEvent};
struct CountingWriter(usize);
impl std::io::Write for CountingWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0 += buf.len();
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
#[derive(Clone)]
pub struct InMemoryQueueWriter {
tx: broadcast::Sender<A2aResult<StreamEvent>>,
persistence_tx: Option<mpsc::Sender<A2aResult<StreamEvent>>>,
max_event_size: usize,
write_timeout: std::time::Duration,
metrics: Option<Arc<dyn crate::metrics::Metrics>>,
seq: Arc<AtomicU64>,
terminal_gate: Option<Arc<super::terminal_gate::TerminalGate>>,
}
impl std::fmt::Debug for InMemoryQueueWriter {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("InMemoryQueueWriter")
.field("persistence", &self.persistence_tx.is_some())
.field("max_event_size", &self.max_event_size)
.field("write_timeout", &self.write_timeout)
.field("metrics", &self.metrics.is_some())
.finish_non_exhaustive()
}
}
impl InMemoryQueueWriter {
pub(super) fn new(
tx: broadcast::Sender<A2aResult<StreamEvent>>,
max_event_size: usize,
write_timeout: std::time::Duration,
) -> Self {
Self {
tx,
persistence_tx: None,
max_event_size,
write_timeout,
metrics: None,
seq: Arc::new(AtomicU64::new(0)),
terminal_gate: None,
}
}
pub(super) fn new_with_persistence(
tx: broadcast::Sender<A2aResult<StreamEvent>>,
persistence_tx: mpsc::Sender<A2aResult<StreamEvent>>,
max_event_size: usize,
write_timeout: std::time::Duration,
) -> Self {
Self {
tx,
persistence_tx: Some(persistence_tx),
max_event_size,
write_timeout,
metrics: None,
seq: Arc::new(AtomicU64::new(0)),
terminal_gate: None,
}
}
pub(crate) fn seed_seq(&self, last: u64) {
self.seq.store(last, Ordering::Relaxed);
}
pub(crate) fn current_seq(&self) -> u64 {
self.seq.load(Ordering::Relaxed)
}
#[must_use]
pub(crate) fn with_metrics(mut self, metrics: Arc<dyn crate::metrics::Metrics>) -> Self {
self.metrics = Some(metrics);
self
}
#[must_use]
pub fn subscribe(&self) -> InMemoryQueueReader {
InMemoryQueueReader::new(self.tx.subscribe())
}
pub(crate) fn raw_subscribe(&self) -> broadcast::Receiver<A2aResult<StreamEvent>> {
self.tx.subscribe()
}
#[must_use]
pub(crate) fn with_terminal_gate(mut self) -> Self {
if self.persistence_tx.is_some() {
self.terminal_gate = Some(Arc::new(super::terminal_gate::TerminalGate::default()));
}
self
}
pub(crate) fn terminal_gate(&self) -> Option<Arc<super::terminal_gate::TerminalGate>> {
self.terminal_gate.clone()
}
fn ticket_for(&self, event: &StreamEvent) -> Option<Ticket<'_>> {
let gate = self.terminal_gate.as_deref()?;
let seq = event.seq?;
if !carries_terminal_state(&event.event) {
return None;
}
gate.arm(seq).map(|verdict| Ticket { gate, seq, verdict })
}
}
struct Ticket<'a> {
gate: &'a super::terminal_gate::TerminalGate,
seq: u64,
verdict: tokio::sync::oneshot::Receiver<StreamResponse>,
}
impl Ticket<'_> {
async fn verdict_within(mut self, timeout: std::time::Duration) -> Option<StreamResponse> {
tokio::time::timeout(timeout, &mut self.verdict)
.await
.ok()?
.ok()
}
}
impl Drop for Ticket<'_> {
fn drop(&mut self) {
self.gate.disarm(self.seq);
}
}
impl InMemoryQueueWriter {
fn check_event_size(&self, event: &StreamResponse) -> A2aResult<()> {
let mut counter = CountingWriter(0);
serde_json::to_writer(&mut counter, event)
.map_err(|e| A2aError::internal(format!("event serialization failed: {e}")))?;
let serialized_size = counter.0;
if serialized_size > self.max_event_size {
return Err(A2aError::internal(format!(
"event size {serialized_size} bytes exceeds maximum {} bytes",
self.max_event_size
)));
}
Ok(())
}
}
#[allow(clippy::manual_async_fn)]
impl EventQueueWriter for InMemoryQueueWriter {
fn write<'a>(
&'a self,
mut event: StreamResponse,
) -> Pin<Box<dyn Future<Output = A2aResult<()>> + Send + 'a>> {
Box::pin(async move {
super::status_stamp::stamp_status(&mut event);
self.check_event_size(&event)?;
let mut event = StreamEvent::at(self.seq.fetch_add(1, Ordering::Relaxed) + 1, event);
let mut ticket = self.ticket_for(&event);
let deadline = tokio::time::Instant::now() + self.write_timeout;
if let Some(ref persistence_tx) = self.persistence_tx {
match persistence_tx
.send_timeout(Ok(event.clone()), self.write_timeout)
.await
{
Ok(()) => {}
Err(mpsc::error::SendTimeoutError::Closed(_)) => {
ticket = None;
trace_warn!("persistence channel closed, event not persisted");
if let Some(metrics) = &self.metrics {
metrics.on_persistence_error(
crate::metrics::persistence_operation::QUEUE_HANDOFF,
crate::metrics::queue_handoff_error::CHANNEL_CLOSED,
);
}
}
Err(mpsc::error::SendTimeoutError::Timeout(_)) => {
trace_warn!(
timeout_ms =
u64::try_from(self.write_timeout.as_millis()).unwrap_or(u64::MAX),
"persistence channel full; background processor is not draining"
);
return Err(A2aError::internal(format!(
"event queue: the persistence channel was still full after {:?}; \
the background processor is not draining events",
self.write_timeout
)));
}
}
}
if let Some(ticket) = ticket
&& let Some(verdict) = ticket
.verdict_within(deadline.saturating_duration_since(tokio::time::Instant::now()))
.await
{
event.event = verdict;
}
match self.tx.send(Ok(event)) {
Ok(_) => Ok(()),
Err(_) if self.persistence_tx.is_some() => {
trace_warn!("no live event subscribers; event persisted only");
Ok(())
}
Err(_) => Err(A2aError::internal("event queue: no active receivers")),
}
})
}
fn close<'a>(&'a self) -> Pin<Box<dyn Future<Output = A2aResult<()>> + Send + 'a>> {
Box::pin(async move {
Ok(())
})
}
}
pub struct InMemoryQueueReader {
rx: broadcast::Receiver<A2aResult<StreamEvent>>,
pending: std::collections::VecDeque<A2aResult<StreamEvent>>,
reattach: Option<ReattachFn>,
reattaching: std::sync::Mutex<Option<Pin<Box<dyn Future<Output = Reattached> + Send>>>>,
saw_terminal: bool,
replayed_through: u64,
}
#[allow(clippy::missing_fields_in_debug)]
impl std::fmt::Debug for InMemoryQueueReader {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("InMemoryQueueReader")
.field("pending", &self.pending.len())
.field("reattach", &self.reattach.is_some())
.field("saw_terminal", &self.saw_terminal)
.finish()
}
}
#[allow(clippy::large_enum_variant)]
pub enum Reattached {
Channel(broadcast::Receiver<A2aResult<StreamEvent>>),
Final(StreamResponse),
End,
}
pub type ReattachFn =
Arc<dyn Fn() -> Pin<Box<dyn Future<Output = Reattached> + Send>> + Send + Sync>;
pub const fn carries_terminal_state(event: &StreamResponse) -> bool {
match event {
StreamResponse::Task(t) => t.status.state.is_terminal(),
StreamResponse::StatusUpdate(u) => u.status.state.is_terminal(),
_ => false,
}
}
impl InMemoryQueueReader {
pub(crate) fn with_reattach(mut self, reattach: ReattachFn) -> Self {
self.reattach = Some(reattach);
self
}
pub(crate) fn new(rx: broadcast::Receiver<A2aResult<StreamEvent>>) -> Self {
Self {
rx,
pending: std::collections::VecDeque::new(),
reattach: None,
reattaching: std::sync::Mutex::new(None),
saw_terminal: false,
replayed_through: 0,
}
}
pub fn set_first_event(&mut self, event: StreamResponse) {
self.pending
.push_front(Ok(StreamEvent::unpositioned(event)));
}
pub(crate) fn queue_replay(
&mut self,
events: impl IntoIterator<Item = crate::store::RecordedEvent>,
) {
for recorded in events {
self.replayed_through = self.replayed_through.max(recorded.seq);
self.pending
.push_back(Ok(StreamEvent::at(recorded.seq, recorded.event)));
}
}
pub(crate) fn with_first_event(
rx: broadcast::Receiver<A2aResult<StreamEvent>>,
first: StreamResponse,
) -> Self {
Self {
rx,
pending: std::iter::once(Ok(StreamEvent::unpositioned(first))).collect(),
reattach: None,
reattaching: std::sync::Mutex::new(None),
saw_terminal: false,
replayed_through: 0,
}
}
pub(crate) fn snapshot_then_end(first: StreamResponse) -> Self {
let (tx, rx) = broadcast::channel(1);
drop(tx);
Self {
rx,
pending: std::iter::once(Ok(StreamEvent::unpositioned(first))).collect(),
reattach: None,
reattaching: std::sync::Mutex::new(None),
saw_terminal: false,
replayed_through: 0,
}
}
}
fn lag_error(dropped: u64) -> a2a_protocol_types::error::A2aError {
a2a_protocol_types::error::A2aError::stream_lagged(dropped)
}
#[allow(clippy::redundant_pub_crate)] pub(crate) fn is_lag_error(err: &a2a_protocol_types::error::A2aError) -> bool {
err.is_stream_lagged()
}
impl EventQueueReader for InMemoryQueueReader {
fn read(
&mut self,
) -> Pin<Box<dyn Future<Output = Option<A2aResult<StreamEvent>>> + Send + '_>> {
Box::pin(async move {
if let Some(first) = self.pending.pop_front() {
if let Ok(ref ev) = first {
self.saw_terminal |= carries_terminal_state(&ev.event);
}
return Some(first);
}
loop {
match self.rx.recv().await {
Ok(event) => {
if let Ok(ref ev) = event {
if ev.seq.is_some_and(|s| s <= self.replayed_through) {
continue;
}
self.saw_terminal |= carries_terminal_state(&ev.event);
}
return Some(event);
}
Err(broadcast::error::RecvError::Lagged(n)) => {
trace_warn!(
dropped_events = n,
"event queue reader lagged, {n} events dropped"
);
return Some(Err(lag_error(n)));
}
Err(broadcast::error::RecvError::Closed) => {
if self.saw_terminal {
return None;
}
let reattach = self.reattach.as_ref()?;
let answer = {
let slot = self
.reattaching
.get_mut()
.unwrap_or_else(std::sync::PoisonError::into_inner);
slot.get_or_insert_with(|| reattach()).await
};
*self
.reattaching
.get_mut()
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
match answer {
Reattached::Channel(rx) => self.rx = rx,
Reattached::Final(event) => {
self.saw_terminal = true;
return Some(Ok(StreamEvent::unpositioned(event)));
}
Reattached::End => return None,
}
}
}
}
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::streaming::event_queue::{
DEFAULT_MAX_EVENT_SIZE, DEFAULT_WRITE_TIMEOUT, new_in_memory_queue,
new_in_memory_queue_with_options, new_in_memory_queue_with_persistence,
};
use a2a_protocol_types::events::{StreamResponse, TaskStatusUpdateEvent};
use a2a_protocol_types::task::{ContextId, TaskId, TaskState, TaskStatus};
fn make_status_event(task_id: &str, state: TaskState) -> StreamResponse {
StreamResponse::StatusUpdate(TaskStatusUpdateEvent {
task_id: TaskId::new(task_id),
context_id: ContextId::new("ctx-test"),
status: TaskStatus {
state,
message: None,
timestamp: None,
},
metadata: None,
})
}
fn counting_reattach(flag: &Arc<std::sync::atomic::AtomicBool>) -> ReattachFn {
let flag = Arc::clone(flag);
Arc::new(move || {
let flag = Arc::clone(&flag);
Box::pin(async move {
flag.store(true, std::sync::atomic::Ordering::SeqCst);
Reattached::End
})
})
}
#[tokio::test]
async fn terminal_status_update_ends_the_stream_without_reattaching() {
use std::sync::atomic::{AtomicBool, Ordering};
let (writer, reader) = new_in_memory_queue();
let called = Arc::new(AtomicBool::new(false));
let mut reader = reader.with_reattach(counting_reattach(&called));
writer
.write(make_status_event("t-term", TaskState::Completed))
.await
.unwrap();
drop(writer);
assert!(reader.read().await.is_some(), "the terminal frame arrives");
assert!(
reader.read().await.is_none(),
"a stream that delivered a terminal state ends at close"
);
assert!(
!called.load(Ordering::SeqCst),
"the reattach hook must not be consulted once a terminal state has been seen"
);
}
#[tokio::test]
async fn terminal_task_snapshot_ends_the_stream_without_reattaching() {
use a2a_protocol_types::task::Task;
use std::sync::atomic::{AtomicBool, Ordering};
let (writer, reader) = new_in_memory_queue();
let called = Arc::new(AtomicBool::new(false));
let mut reader = reader.with_reattach(counting_reattach(&called));
reader.set_first_event(StreamResponse::Task(Task {
id: TaskId::new("t-snap"),
context_id: ContextId::new("ctx-test"),
status: TaskStatus {
state: TaskState::Completed,
message: None,
timestamp: None,
},
history: None,
artifacts: None,
metadata: None,
}));
drop(writer);
assert!(reader.read().await.is_some(), "the snapshot arrives first");
assert!(
reader.read().await.is_none(),
"a terminal Task snapshot ends the stream at close"
);
assert!(
!called.load(Ordering::SeqCst),
"the reattach hook must not be consulted after a terminal Task snapshot"
);
}
#[tokio::test]
async fn non_terminal_event_still_consults_the_reattach_hook() {
use std::sync::atomic::{AtomicBool, Ordering};
let (writer, reader) = new_in_memory_queue();
let called = Arc::new(AtomicBool::new(false));
let mut reader = reader.with_reattach(counting_reattach(&called));
writer
.write(make_status_event("t-working", TaskState::Working))
.await
.unwrap();
drop(writer);
assert!(reader.read().await.is_some(), "the working frame arrives");
assert!(reader.read().await.is_none(), "the hook here returns End");
assert!(
called.load(Ordering::SeqCst),
"without a terminal state the reader must ask the hook whether the task is done"
);
}
#[tokio::test]
async fn event_of_exactly_max_size_is_accepted() {
let stamped = || {
let mut event = make_status_event("t-exact", TaskState::Working);
if let StreamResponse::StatusUpdate(u) = &mut event {
u.status.timestamp = Some("2026-01-02T03:04:05.000Z".into());
}
event
};
let event = stamped();
let exact = serde_json::to_vec(&event).expect("serializes").len();
let (writer, _reader) = new_in_memory_queue_with_options(16, exact, DEFAULT_WRITE_TIMEOUT);
assert!(
writer.write(event).await.is_ok(),
"an event of exactly max_event_size ({exact} bytes) is within the \
limit and must be accepted"
);
let event = stamped();
let (writer, _reader) =
new_in_memory_queue_with_options(16, exact - 1, DEFAULT_WRITE_TIMEOUT);
assert!(
writer.write(event).await.is_err(),
"one byte over the limit must still be rejected"
);
}
#[tokio::test]
async fn writer_debug_reports_the_decision_carrying_fields() {
let (writer, _reader) = new_in_memory_queue();
let rendered = format!("{writer:?}");
assert!(
rendered.contains("InMemoryQueueWriter"),
"the type name must appear: {rendered}"
);
assert!(
rendered.contains("persistence: false"),
"the persistence hand-off must be visible: {rendered}"
);
assert!(
rendered.contains("metrics: false"),
"the metrics hook must be visible: {rendered}"
);
}
#[tokio::test]
async fn reader_debug_reports_the_decision_carrying_fields() {
let (_writer, mut reader) = new_in_memory_queue();
reader.set_first_event(make_status_event("t-dbg", TaskState::Working));
let rendered = format!("{reader:?}");
assert!(
rendered.contains("InMemoryQueueReader"),
"the type name must appear: {rendered}"
);
assert!(
rendered.contains("pending: 1"),
"a pending snapshot must be visible: {rendered}"
);
assert!(
rendered.contains("saw_terminal: false"),
"terminal tracking must be visible: {rendered}"
);
}
#[tokio::test]
async fn write_with_no_subscribers_succeeds_when_persistence_attached() {
let (writer, reader, mut persistence_rx) =
crate::streaming::event_queue::new_in_memory_queue_with_persistence(
8,
1024 * 1024,
std::time::Duration::from_secs(1),
);
drop(reader);
writer
.write(make_status_event("t1", TaskState::Working))
.await
.expect("write must succeed with persistence attached");
let persisted = persistence_rx
.recv()
.await
.expect("persistence channel should have the event")
.expect("event should be Ok");
match persisted.event {
StreamResponse::StatusUpdate(evt) => {
assert_eq!(evt.status.state, TaskState::Working);
}
other => panic!("expected StatusUpdate, got: {other:?}"),
}
}
#[tokio::test]
async fn write_with_no_subscribers_fails_without_persistence() {
let (writer, reader) = new_in_memory_queue();
drop(reader);
let result = writer
.write(make_status_event("t1", TaskState::Working))
.await;
assert!(
result.is_err(),
"sync-mode write with no receivers must fail"
);
}
#[tokio::test]
async fn write_then_read_single_event() {
let (writer, mut reader) = new_in_memory_queue();
let event = make_status_event("t1", TaskState::Working);
writer.write(event).await.expect("write should succeed");
drop(writer);
let received = reader.read().await;
assert!(received.is_some(), "reader should return the written event");
let result = received.unwrap();
let event = result.expect("event should be Ok");
match &event.event {
StreamResponse::StatusUpdate(evt) => {
assert_eq!(
evt.status.state,
TaskState::Working,
"should be Working event"
);
}
other => panic!("expected StatusUpdate, got: {other:?}"),
}
let eof = reader.read().await;
assert!(
eof.is_none(),
"reader should return None after writer is dropped"
);
}
#[tokio::test]
async fn write_multiple_events_read_in_order() {
let (writer, mut reader) = new_in_memory_queue();
let e1 = make_status_event("t1", TaskState::Working);
let e2 = make_status_event("t1", TaskState::Completed);
writer.write(e1).await.expect("first write should succeed");
writer.write(e2).await.expect("second write should succeed");
drop(writer);
let r1 = reader.read().await.expect("should read first event");
let sr1 = r1.expect("first event should be Ok");
match &sr1.event {
StreamResponse::StatusUpdate(evt) => {
assert_eq!(
evt.status.state,
TaskState::Working,
"first event should be Working"
);
}
other => panic!("expected StatusUpdate, got: {other:?}"),
}
let r2 = reader.read().await.expect("should read second event");
let sr2 = r2.expect("second event should be Ok");
match &sr2.event {
StreamResponse::StatusUpdate(evt) => {
assert_eq!(
evt.status.state,
TaskState::Completed,
"second event should be Completed"
);
}
other => panic!("expected StatusUpdate, got: {other:?}"),
}
assert!(
reader.read().await.is_none(),
"should be EOF after all events"
);
}
#[tokio::test]
async fn read_returns_none_on_empty_closed_queue() {
let (writer, mut reader) = new_in_memory_queue();
drop(writer);
let result = reader.read().await;
assert!(
result.is_none(),
"reading from an empty closed queue should return None"
);
}
#[tokio::test]
async fn write_after_all_readers_dropped_returns_error() {
let (writer, reader) = new_in_memory_queue();
drop(reader);
let result = writer
.write(make_status_event("t1", TaskState::Working))
.await;
assert!(
result.is_err(),
"writing with no active receivers should return an error"
);
}
#[tokio::test]
async fn close_is_no_op_and_succeeds() {
let (writer, _reader) = new_in_memory_queue();
let result = writer.close().await;
assert!(result.is_ok(), "close() should succeed");
}
#[tokio::test]
async fn subscribe_creates_independent_reader() {
let (writer, mut reader1) = new_in_memory_queue();
let mut reader2 = writer.subscribe();
let event = make_status_event("t1", TaskState::Working);
writer.write(event).await.expect("write should succeed");
drop(writer);
let r1 = reader1.read().await;
assert!(r1.is_some(), "reader1 should receive the event");
let r2 = reader2.read().await;
assert!(r2.is_some(), "reader2 should receive the event");
assert!(reader1.read().await.is_none(), "reader1 should see EOF");
assert!(reader2.read().await.is_none(), "reader2 should see EOF");
}
#[tokio::test]
async fn subscriber_only_sees_events_after_subscribe() {
let (writer, mut reader1) = new_in_memory_queue();
writer
.write(make_status_event("t1", TaskState::Submitted))
.await
.expect("write should succeed");
let mut reader2 = writer.subscribe();
writer
.write(make_status_event("t1", TaskState::Working))
.await
.expect("write should succeed");
drop(writer);
let r1a = reader1
.read()
.await
.expect("reader1 should see first event");
let evt1a = r1a.expect("first event should be Ok");
assert!(
matches!(&evt1a.event, StreamResponse::StatusUpdate(e) if e.status.state == TaskState::Submitted),
"reader1 first event should be Submitted"
);
let r1b = reader1
.read()
.await
.expect("reader1 should see second event");
let evt_1b = r1b.expect("second event should be Ok");
assert!(
matches!(&evt_1b.event, StreamResponse::StatusUpdate(e) if e.status.state == TaskState::Working),
"reader1 second event should be Working"
);
assert!(reader1.read().await.is_none());
let r2a = reader2
.read()
.await
.expect("reader2 should see second event");
let evt2a = r2a.expect("event should be Ok");
assert!(
matches!(&evt2a.event, StreamResponse::StatusUpdate(e) if e.status.state == TaskState::Working),
"reader2 should see Working event"
);
assert!(
reader2.read().await.is_none(),
"reader2 should see EOF after the one event it received"
);
}
#[tokio::test]
async fn oversized_event_is_rejected() {
let (writer, _reader) = new_in_memory_queue_with_options(
16,
10, DEFAULT_WRITE_TIMEOUT,
);
let event = make_status_event("t1", TaskState::Working);
let result = writer.write(event).await;
assert!(
result.is_err(),
"event exceeding max_event_size should be rejected"
);
let err = result.unwrap_err();
let msg = format!("{err}");
assert!(
msg.contains("exceeds maximum"),
"error message should mention size limit, got: {msg}"
);
}
#[test]
fn counting_writer_flush_is_noop() {
use std::io::Write;
let mut cw = super::CountingWriter(0);
cw.write_all(b"hello").unwrap();
assert_eq!(cw.0, 5);
cw.flush().unwrap();
assert_eq!(cw.0, 5, "flush should not change the count");
}
#[tokio::test]
async fn event_within_size_limit_is_accepted() {
let (writer, mut reader) =
new_in_memory_queue_with_options(16, DEFAULT_MAX_EVENT_SIZE, DEFAULT_WRITE_TIMEOUT);
let event = make_status_event("t1", TaskState::Working);
writer
.write(event)
.await
.expect("event within size limit should be accepted");
drop(writer);
let r = reader.read().await;
assert!(r.is_some(), "reader should receive the event");
}
#[tokio::test]
async fn a_full_persistence_channel_fails_the_write_within_the_timeout() {
let timeout = std::time::Duration::from_millis(150);
let (writer, _sse, _held_receiver) =
new_in_memory_queue_with_persistence(1, DEFAULT_MAX_EVENT_SIZE, timeout);
let mut accepted = 0_usize;
let start = std::time::Instant::now();
loop {
let attempt = tokio::time::timeout(
timeout * 10,
writer.write(make_status_event("t1", TaskState::Working)),
)
.await
.expect("write parked past its own deadline: write_timeout is not being applied");
match attempt {
Ok(()) => accepted += 1,
Err(e) => {
assert!(
e.to_string().contains("persistence channel"),
"the error must name the channel that is full, got: {e}"
);
break;
}
}
assert!(
accepted <= 4096,
"write never reported a full channel after {accepted} events"
);
}
assert_eq!(
accepted, 1024,
"the persistence channel's capacity is max(capacity * 16, 1024)"
);
assert!(
start.elapsed() < timeout * 10,
"write should fail after roughly one timeout, took {:?}",
start.elapsed()
);
}
#[tokio::test]
async fn the_failing_write_waits_about_one_write_timeout() {
let timeout = std::time::Duration::from_millis(200);
let (writer, _sse, _held_receiver) =
new_in_memory_queue_with_persistence(1, DEFAULT_MAX_EVENT_SIZE, timeout);
for _ in 0..1024 {
writer
.write(make_status_event("t1", TaskState::Working))
.await
.expect("the channel has room until it is full");
}
let start = std::time::Instant::now();
let err = tokio::time::timeout(
timeout * 10,
writer.write(make_status_event("t1", TaskState::Working)),
)
.await
.expect("write parked past its own deadline: write_timeout is not being applied")
.expect_err("the channel is full");
let waited = start.elapsed();
assert!(waited >= timeout, "returned early after {waited:?}");
assert!(
waited < timeout * 4,
"waited far past the deadline: {waited:?}"
);
assert!(
err.to_string().contains("not draining"),
"the message must say what is wrong, got: {err}"
);
}
#[tokio::test]
async fn a_closed_persistence_channel_is_still_not_an_error() {
let (writer, _sse, persistence_rx) = new_in_memory_queue_with_persistence(
16,
DEFAULT_MAX_EVENT_SIZE,
std::time::Duration::from_millis(50),
);
drop(persistence_rx);
let start = std::time::Instant::now();
tokio::time::timeout(
std::time::Duration::from_secs(5),
writer.write(make_status_event("t1", TaskState::Working)),
)
.await
.expect("a closed channel must be observed at once, never waited out")
.expect("a closed persistence channel must not fail the write");
assert!(
start.elapsed() < std::time::Duration::from_millis(50),
"a closed channel must be detected immediately, not waited out"
);
}
#[tokio::test]
async fn a_dropped_event_moves_the_persistence_error_counter() {
#[derive(Default)]
struct Seen(std::sync::Mutex<Vec<(String, String)>>);
impl crate::metrics::Metrics for Seen {
fn on_persistence_error(&self, operation: &str, error_kind: &str) {
self.0
.lock()
.unwrap()
.push((operation.to_owned(), error_kind.to_owned()));
}
}
let seen = Arc::new(Seen::default());
let (writer, _sse, persistence_rx) = new_in_memory_queue_with_persistence(
16,
DEFAULT_MAX_EVENT_SIZE,
std::time::Duration::from_millis(50),
);
let writer = writer.with_metrics(Arc::clone(&seen) as Arc<dyn crate::metrics::Metrics>);
writer
.write(make_status_event("t1", TaskState::Working))
.await
.expect("an open channel takes the event");
assert!(
seen.0.lock().unwrap().is_empty(),
"nothing is dropped while the processor is alive"
);
drop(persistence_rx);
writer
.write(make_status_event("t1", TaskState::Working))
.await
.expect("a closed channel still does not fail the write");
assert_eq!(
*seen.0.lock().unwrap(),
vec![(
crate::metrics::persistence_operation::QUEUE_HANDOFF.to_owned(),
crate::metrics::queue_handoff_error::CHANNEL_CLOSED.to_owned()
)],
"the dropped event is counted, once, with the bounded labels"
);
}
#[tokio::test]
async fn a_live_event_the_replay_already_delivered_is_not_delivered_again() {
use crate::store::RecordedEvent;
let (writer, reader) = new_in_memory_queue();
let mut reader = reader;
for _ in 0..3 {
writer
.write(make_status_event("t-dup", TaskState::Working))
.await
.unwrap();
}
reader.queue_replay((1..=3).map(|seq| RecordedEvent {
seq,
event: make_status_event("t-dup", TaskState::Working),
}));
writer
.write(make_status_event("t-dup", TaskState::Working))
.await
.unwrap();
drop(writer);
let mut positions = Vec::new();
while let Some(item) = reader.read().await {
positions.push(item.expect("no errors").seq);
}
assert_eq!(
positions,
vec![Some(1), Some(2), Some(3), Some(4)],
"each position must be delivered exactly once, in order"
);
}
#[tokio::test]
async fn an_unpositioned_frame_is_never_suppressed_by_the_replay_mark() {
use crate::store::RecordedEvent;
let (writer, reader) = new_in_memory_queue();
let mut reader = reader;
reader.queue_replay(std::iter::once(RecordedEvent {
seq: 9,
event: make_status_event("t-unpos", TaskState::Working),
}));
reader.set_first_event(make_status_event("t-unpos", TaskState::Submitted));
drop(writer);
let mut positions = Vec::new();
while let Some(item) = reader.read().await {
positions.push(item.expect("no errors").seq);
}
assert_eq!(
positions,
vec![None, Some(9)],
"the snapshot leads, and carries no position"
);
}
}