apalis-rtui 0.1.0-rc.1

A drop-in replacement for apalis-board: HTTP surface, live updates, and the apalis-tui dashboard
Documentation
use std::sync::Arc;
use std::{io::LineWriter, sync::Mutex};

use tracing_subscriber::fmt::format::{Format, Json, JsonFields};
use tracing_subscriber::fmt::{Layer, MakeWriter};

use tracing_subscriber::registry::LookupSpan;

use crate::api::sse::broadcaster::TracingBroadcaster;

/// A tracing subscriber that sends log entries to a `TracingBroadcaster`.
#[derive(Debug, Clone)]
pub struct TracingSubscriber {
    broadcaster: Arc<Mutex<TracingBroadcaster>>,
}

impl TracingSubscriber {
    /// Create a new `TracingSubscriber` from a broadcaster reference.
    #[must_use]
    pub fn new(broadcaster: &Arc<Mutex<TracingBroadcaster>>) -> Self {
        Self {
            broadcaster: Arc::clone(broadcaster),
        }
    }

    /// Create a new `TracingSubscriber` from a raw broadcaster.
    #[must_use]
    pub fn new_inner(broadcaster: TracingBroadcaster) -> Self {
        Self {
            broadcaster: Arc::new(Mutex::new(broadcaster)),
        }
    }

    /// Get the underlying broadcaster.
    #[must_use]
    pub fn get_broadcaster(&self) -> &Arc<Mutex<TracingBroadcaster>> {
        &self.broadcaster
    }

    /// Create a tracing layer that uses this subscriber as a writer.
    #[must_use]
    pub fn layer<S>(self) -> Layer<S, JsonFields, Format<Json>, Self>
    where
        S: tracing_core::Subscriber + for<'a> LookupSpan<'a>,
    {
        tracing_subscriber::fmt::layer()
            .with_ansi(false)
            .fmt_fields(JsonFields::new())
            .event_format(tracing_subscriber::fmt::format().with_ansi(false).json())
            .with_writer(self)
    }
}

impl<'a> MakeWriter<'a> for TracingSubscriber {
    type Writer = LineWriter<Self>;

    fn make_writer(&self) -> Self::Writer {
        LineWriter::new(self.clone())
    }
}

impl std::io::Write for TracingSubscriber {
    /// Parse one formatted tracing line and hand it to the broadcaster.
    ///
    /// The lock here is a blocking [`Mutex::lock`], never `try_lock`. Two threads that emit a
    /// tracing event at the same moment both reach this line, and a `try_lock` gives the loser
    /// an error that the `tracing` fmt layer discards, so that thread's frame leaves no trace
    /// anywhere: not in `GET /events`, not in the `/events/status` replay buffer, not in the
    /// `/events/activity` counts. A task's one and only `task.done` or `task.failed` frame can
    /// go missing that way, and the loss grows with the number of workers a process runs. A
    /// measured 4-thread write of 100 frames lost 19 of them. Waiting for the lock costs a few
    /// microseconds, since the guarded work is a channel `try_send` per connected client and
    /// nothing else.
    ///
    /// Waiting cannot deadlock. Nothing under this mutex emits a tracing event of its own, and
    /// every other holder of it keeps the guard only for the length of
    /// [`TracingBroadcaster::new_client`].
    ///
    /// # Errors
    ///
    /// Returns [`std::io::ErrorKind::InvalidData`] when `buf` is not one of the JSON frames
    /// [`TracingSubscriber::layer`] formats, which is how a partial write is skipped.
    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
        let len = buf.len();
        let msg = std::str::from_utf8(buf).unwrap_or_default();
        let log_entry = serde_json::from_str(msg).map_err(|e| {
            std::io::Error::new(std::io::ErrorKind::InvalidData, format!("JSON Error: {e}"))
        })?;
        // A poisoned broadcaster still has every client's sender in it, so recover the value and
        // keep serving them. Dropping the frame here would be the same silent loss as above.
        self.broadcaster
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner)
            .send(&log_entry)
            .map_err(|e| std::io::Error::other(format!("Broadcast Error: {e}")))?;
        Ok(len)
    }

    fn flush(&mut self) -> std::io::Result<()> {
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use std::{
        io::Write as _,
        sync::{Arc, mpsc},
        thread,
        time::Duration,
    };

    use futures::StreamExt as _;

    use super::TracingSubscriber;
    use crate::api::sse::TracingBroadcaster;

    /// One line in the exact shape [`TracingSubscriber::layer`] formats, matching
    /// `tests/goldens/events-task-done.json`.
    fn task_done_line(seq: usize) -> String {
        format!(
            r#"{{"fields":{{"message":"task.done","done_in":"1ms","result":"ok {seq}"}},"level":"DEBUG","span":{{"attempt":1,"name":"task","task_id":"01ARZ3NDEKTSV4RRFFQ69G5FAV"}},"target":"apalis::layers::tracing::on_response","timestamp":"2024-01-01T00:00:00Z"}}
"#
        )
    }

    /// A frame written while another thread holds the broadcaster must still reach every client.
    ///
    /// The writer used to take the mutex with `try_lock` and discard the frame on failure, so a
    /// task's only `task.done` frame could vanish from `/events`, from the `/events/status`
    /// replay buffer, and from the `/events/activity` counts, with nothing on the wire to show
    /// it had happened. This test holds the mutex for a fixed window and writes inside it. The
    /// old behavior fails it on every run, never once in a while.
    #[test]
    fn a_frame_written_while_the_broadcaster_is_held_still_arrives() {
        let broadcaster = TracingBroadcaster::create();
        let client = broadcaster.lock().expect("fresh mutex").new_client();

        let (acquired_tx, acquired_rx) = mpsc::channel();
        let holder = thread::spawn({
            let broadcaster = Arc::clone(&broadcaster);
            move || {
                let guard = broadcaster.lock().expect("fresh mutex");
                acquired_tx.send(()).expect("the test thread is waiting");
                thread::sleep(Duration::from_millis(50));
                drop(guard);
            }
        });
        acquired_rx.recv().expect("the holder took the mutex");

        let mut writer = TracingSubscriber::new(&broadcaster);
        writer
            .write_all(task_done_line(0).as_bytes())
            .expect("a held broadcaster makes the write wait, never fail");
        holder.join().expect("holder thread");

        drop(writer);
        drop(broadcaster);
        let received: Vec<_> = futures::executor::block_on(client.0.collect::<Vec<_>>());
        assert_eq!(received.len(), 1, "the frame was dropped, not delivered");
    }

    /// Concurrent writers lose nothing. Four threads is what a process running several workers
    /// looks like: each worker emits its own `task.start`/`task.done` frames from whichever
    /// runtime thread picked it up, and those land on this one writer at the same moment.
    #[test]
    fn concurrent_writers_lose_no_frames() {
        const THREADS: usize = 4;
        // 100 frames total, the per-client channel capacity, so nothing is lost downstream
        // either and this test is only ever measuring the writer.
        const PER_THREAD: usize = 25;

        let broadcaster = TracingBroadcaster::create();
        let client = broadcaster.lock().expect("fresh mutex").new_client();
        let writers: Vec<_> = (0..THREADS)
            .map(|thread_index| {
                let mut writer = TracingSubscriber::new(&broadcaster);
                thread::spawn(move || {
                    for seq in 0..PER_THREAD {
                        writer
                            .write_all(task_done_line(thread_index * PER_THREAD + seq).as_bytes())
                            .expect("a contended broadcaster makes the write wait, never fail");
                    }
                })
            })
            .collect();
        for writer in writers {
            writer.join().expect("writer thread");
        }

        drop(broadcaster);
        let received: Vec<_> = futures::executor::block_on(client.0.collect::<Vec<_>>());
        assert_eq!(
            received.len(),
            THREADS * PER_THREAD,
            "frames went missing under concurrent writes"
        );
    }
}