1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
use std::sync::Arc;
use std::{io::LineWriter, sync::Mutex};
use tracing_subscriber::fmt::format::{Format, Json, JsonFields};
use tracing_subscriber::fmt::{Layer, MakeWriter};
use tracing_subscriber::registry::LookupSpan;
use crate::api::sse::broadcaster::TracingBroadcaster;
/// A tracing subscriber that sends log entries to a `TracingBroadcaster`.
#[derive(Debug, Clone)]
pub struct TracingSubscriber {
broadcaster: Arc<Mutex<TracingBroadcaster>>,
}
impl TracingSubscriber {
/// Create a new `TracingSubscriber` from a broadcaster reference.
#[must_use]
pub fn new(broadcaster: &Arc<Mutex<TracingBroadcaster>>) -> Self {
Self {
broadcaster: Arc::clone(broadcaster),
}
}
/// Create a new `TracingSubscriber` from a raw broadcaster.
#[must_use]
pub fn new_inner(broadcaster: TracingBroadcaster) -> Self {
Self {
broadcaster: Arc::new(Mutex::new(broadcaster)),
}
}
/// Get the underlying broadcaster.
#[must_use]
pub fn get_broadcaster(&self) -> &Arc<Mutex<TracingBroadcaster>> {
&self.broadcaster
}
/// Create a tracing layer that uses this subscriber as a writer.
#[must_use]
pub fn layer<S>(self) -> Layer<S, JsonFields, Format<Json>, Self>
where
S: tracing_core::Subscriber + for<'a> LookupSpan<'a>,
{
tracing_subscriber::fmt::layer()
.with_ansi(false)
.fmt_fields(JsonFields::new())
.event_format(tracing_subscriber::fmt::format().with_ansi(false).json())
.with_writer(self)
}
}
impl<'a> MakeWriter<'a> for TracingSubscriber {
type Writer = LineWriter<Self>;
fn make_writer(&self) -> Self::Writer {
LineWriter::new(self.clone())
}
}
impl std::io::Write for TracingSubscriber {
/// Parse one formatted tracing line and hand it to the broadcaster.
///
/// The lock here is a blocking [`Mutex::lock`], never `try_lock`. Two threads that emit a
/// tracing event at the same moment both reach this line, and a `try_lock` gives the loser
/// an error that the `tracing` fmt layer discards, so that thread's frame leaves no trace
/// anywhere: not in `GET /events`, not in the `/events/status` replay buffer, not in the
/// `/events/activity` counts. A task's one and only `task.done` or `task.failed` frame can
/// go missing that way, and the loss grows with the number of workers a process runs. A
/// measured 4-thread write of 100 frames lost 19 of them. Waiting for the lock costs a few
/// microseconds, since the guarded work is a channel `try_send` per connected client and
/// nothing else.
///
/// Waiting cannot deadlock. Nothing under this mutex emits a tracing event of its own, and
/// every other holder of it keeps the guard only for the length of
/// [`TracingBroadcaster::new_client`].
///
/// # Errors
///
/// Returns [`std::io::ErrorKind::InvalidData`] when `buf` is not one of the JSON frames
/// [`TracingSubscriber::layer`] formats, which is how a partial write is skipped.
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
let len = buf.len();
let msg = std::str::from_utf8(buf).unwrap_or_default();
let log_entry = serde_json::from_str(msg).map_err(|e| {
std::io::Error::new(std::io::ErrorKind::InvalidData, format!("JSON Error: {e}"))
})?;
// A poisoned broadcaster still has every client's sender in it, so recover the value and
// keep serving them. Dropping the frame here would be the same silent loss as above.
self.broadcaster
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.send(&log_entry)
.map_err(|e| std::io::Error::other(format!("Broadcast Error: {e}")))?;
Ok(len)
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
#[cfg(test)]
mod tests {
use std::{
io::Write as _,
sync::{Arc, mpsc},
thread,
time::Duration,
};
use futures::StreamExt as _;
use super::TracingSubscriber;
use crate::api::sse::TracingBroadcaster;
/// One line in the exact shape [`TracingSubscriber::layer`] formats, matching
/// `tests/goldens/events-task-done.json`.
fn task_done_line(seq: usize) -> String {
format!(
r#"{{"fields":{{"message":"task.done","done_in":"1ms","result":"ok {seq}"}},"level":"DEBUG","span":{{"attempt":1,"name":"task","task_id":"01ARZ3NDEKTSV4RRFFQ69G5FAV"}},"target":"apalis::layers::tracing::on_response","timestamp":"2024-01-01T00:00:00Z"}}
"#
)
}
/// A frame written while another thread holds the broadcaster must still reach every client.
///
/// The writer used to take the mutex with `try_lock` and discard the frame on failure, so a
/// task's only `task.done` frame could vanish from `/events`, from the `/events/status`
/// replay buffer, and from the `/events/activity` counts, with nothing on the wire to show
/// it had happened. This test holds the mutex for a fixed window and writes inside it. The
/// old behavior fails it on every run, never once in a while.
#[test]
fn a_frame_written_while_the_broadcaster_is_held_still_arrives() {
let broadcaster = TracingBroadcaster::create();
let client = broadcaster.lock().expect("fresh mutex").new_client();
let (acquired_tx, acquired_rx) = mpsc::channel();
let holder = thread::spawn({
let broadcaster = Arc::clone(&broadcaster);
move || {
let guard = broadcaster.lock().expect("fresh mutex");
acquired_tx.send(()).expect("the test thread is waiting");
thread::sleep(Duration::from_millis(50));
drop(guard);
}
});
acquired_rx.recv().expect("the holder took the mutex");
let mut writer = TracingSubscriber::new(&broadcaster);
writer
.write_all(task_done_line(0).as_bytes())
.expect("a held broadcaster makes the write wait, never fail");
holder.join().expect("holder thread");
drop(writer);
drop(broadcaster);
let received: Vec<_> = futures::executor::block_on(client.0.collect::<Vec<_>>());
assert_eq!(received.len(), 1, "the frame was dropped, not delivered");
}
/// Concurrent writers lose nothing. Four threads is what a process running several workers
/// looks like: each worker emits its own `task.start`/`task.done` frames from whichever
/// runtime thread picked it up, and those land on this one writer at the same moment.
#[test]
fn concurrent_writers_lose_no_frames() {
const THREADS: usize = 4;
// 100 frames total, the per-client channel capacity, so nothing is lost downstream
// either and this test is only ever measuring the writer.
const PER_THREAD: usize = 25;
let broadcaster = TracingBroadcaster::create();
let client = broadcaster.lock().expect("fresh mutex").new_client();
let writers: Vec<_> = (0..THREADS)
.map(|thread_index| {
let mut writer = TracingSubscriber::new(&broadcaster);
thread::spawn(move || {
for seq in 0..PER_THREAD {
writer
.write_all(task_done_line(thread_index * PER_THREAD + seq).as_bytes())
.expect("a contended broadcaster makes the write wait, never fail");
}
})
})
.collect();
for writer in writers {
writer.join().expect("writer thread");
}
drop(broadcaster);
let received: Vec<_> = futures::executor::block_on(client.0.collect::<Vec<_>>());
assert_eq!(
received.len(),
THREADS * PER_THREAD,
"frames went missing under concurrent writes"
);
}
}