Skip to main content

dbgflow_server/
lib.rs

1//! Read-only live transport for durable dbgflow recording history.
2//!
3//! The server intentionally has no dependency on the recorder or frontend.
4//! It follows immutable storage chunks and the synced active prefix, making
5//! slow clients independent from the writer's hot path.
6
7use std::{
8    collections::{BTreeMap, VecDeque},
9    convert::Infallible,
10    error::Error,
11    fmt, io,
12    net::{IpAddr, Ipv4Addr, SocketAddr},
13    path::{Path, PathBuf},
14    sync::{
15        Arc, Mutex, Weak,
16        atomic::{AtomicBool, AtomicU64, Ordering},
17    },
18    time::Duration,
19};
20
21use axum::{
22    Json, Router,
23    body::{Body, Bytes},
24    extract::{Query, Request, State},
25    http::{HeaderMap, HeaderValue, StatusCode, header::HOST},
26    middleware::{self, Next},
27    response::{
28        IntoResponse, Response,
29        sse::{Event as SseEvent, KeepAlive, Sse},
30    },
31    routing::get,
32};
33use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
34use dbgflow_protocol::{
35    CURRENT_PROJECTION_VERSION, CURRENT_SCHEMA_VERSION, EventEnvelope, EventKind, Payload,
36    PayloadEncoding, ProjectedSpan, ProjectionCheckpoint, ProjectionLimits, Sequence,
37};
38use dbgflow_storage::{
39    CURRENT_FORMAT_VERSION, DurableMetadata, DurableTailReader, Manifest, RecordingActivity,
40    RecordingReader, RecordingState, RetainedPrefix, StorageError, recording_activity,
41};
42use serde::{Deserialize, Serialize};
43use tokio::{
44    net::TcpListener,
45    sync::{mpsc, watch},
46    task::JoinHandle,
47};
48use tokio_stream::wrappers::ReceiverStream;
49
50pub const API_VERSION: u32 = 1;
51/// Absolute allocation guard even when an embedding application supplies its
52/// own page limits.
53pub const HARD_MAX_PAGE_LIMIT: usize = 100_000;
54/// Absolute response allocation guard for one historical page.
55pub const HARD_MAX_PAGE_BYTES: usize = 64 * 1024 * 1024;
56const MIN_PAGE_BYTES: usize = 1024;
57const EVENT_RESPONSE_OVERHEAD_BYTES: usize = 512;
58const LAST_EVENT_ID: &str = "last-event-id";
59const LAST_DURABLE_CURSOR: &str = "x-dbgflow-last-durable-cursor";
60const MAX_LIVE_CATCHUP_EVENTS: usize = 64;
61const MAX_LIVE_CATCHUP_BYTES: usize = 1024 * 1024;
62
63/// Loopback-only server, pagination limits, and live polling bounds.
64#[derive(Clone, Debug)]
65pub struct ServerConfig {
66    pub bind_addr: SocketAddr,
67    /// Optional directory containing an independently delivered browser UI.
68    ///
69    /// The four required assets are loaded and validated before the server
70    /// starts. When set, this directory takes precedence over the optional
71    /// embedded UI feature.
72    pub ui_dir: Option<PathBuf>,
73    pub default_page_limit: usize,
74    pub max_page_limit: usize,
75    /// Conservative maximum serialized size retained for one history page.
76    pub max_page_bytes: usize,
77    pub live_batch_limit: usize,
78    /// Maximum events retained by the shared live fanout.
79    pub live_buffer_events: usize,
80    /// Maximum accounted event bytes retained by the shared live fanout and
81    /// by one live history catch-up page.
82    pub live_buffer_bytes: usize,
83    pub live_poll_interval: Duration,
84    pub keep_alive_interval: Duration,
85}
86
87impl Default for ServerConfig {
88    fn default() -> Self {
89        Self {
90            bind_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0),
91            ui_dir: None,
92            default_page_limit: 500,
93            max_page_limit: 5_000,
94            max_page_bytes: 8 * 1024 * 1024,
95            live_batch_limit: 500,
96            live_buffer_events: 128,
97            live_buffer_bytes: 8 * 1024 * 1024,
98            live_poll_interval: Duration::from_millis(200),
99            keep_alive_interval: Duration::from_secs(15),
100        }
101    }
102}
103
104impl ServerConfig {
105    fn validate(&self) -> Result<(), ServerError> {
106        if !self.bind_addr.ip().is_loopback() {
107            return Err(ServerError::InvalidConfig(
108                "bind address must be loopback".into(),
109            ));
110        }
111        if self.default_page_limit == 0 || self.default_page_limit > self.max_page_limit {
112            return Err(ServerError::InvalidConfig(
113                "default_page_limit must be in 1..=max_page_limit".into(),
114            ));
115        }
116        if self.max_page_limit == 0
117            || self.max_page_limit > HARD_MAX_PAGE_LIMIT
118            || self.live_batch_limit == 0
119            || self.live_batch_limit > self.max_page_limit
120            || self.live_buffer_events == 0
121        {
122            return Err(ServerError::InvalidConfig(format!(
123                "page limits must be non-zero and no greater than {HARD_MAX_PAGE_LIMIT}; live limits must be non-zero and bounded"
124            )));
125        }
126        if !(MIN_PAGE_BYTES..=HARD_MAX_PAGE_BYTES).contains(&self.max_page_bytes) {
127            return Err(ServerError::InvalidConfig(format!(
128                "max_page_bytes must be in {MIN_PAGE_BYTES}..={HARD_MAX_PAGE_BYTES}"
129            )));
130        }
131        if !(MIN_PAGE_BYTES..=HARD_MAX_PAGE_BYTES).contains(&self.live_buffer_bytes) {
132            return Err(ServerError::InvalidConfig(format!(
133                "live_buffer_bytes must be in {MIN_PAGE_BYTES}..={HARD_MAX_PAGE_BYTES}"
134            )));
135        }
136        if self.live_poll_interval.is_zero() || self.keep_alive_interval.is_zero() {
137            return Err(ServerError::InvalidConfig(
138                "poll and keep-alive intervals must be non-zero".into(),
139            ));
140        }
141        Ok(())
142    }
143}
144
145#[derive(Debug)]
146pub enum ServerError {
147    InvalidConfig(String),
148    UiAsset { path: PathBuf, source: io::Error },
149    Io(io::Error),
150    Storage(StorageError),
151    Task(tokio::task::JoinError),
152}
153
154impl fmt::Display for ServerError {
155    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
156        match self {
157            Self::InvalidConfig(message) => {
158                write!(formatter, "invalid server configuration: {message}")
159            }
160            Self::UiAsset { path, source } => {
161                write!(
162                    formatter,
163                    "cannot load required UI asset {}: {source}",
164                    path.display()
165                )
166            }
167            Self::Io(error) => error.fmt(formatter),
168            Self::Storage(error) => error.fmt(formatter),
169            Self::Task(error) => write!(formatter, "server task failed: {error}"),
170        }
171    }
172}
173
174impl Error for ServerError {
175    fn source(&self) -> Option<&(dyn Error + 'static)> {
176        match self {
177            Self::Io(error) => Some(error),
178            Self::UiAsset { source, .. } => Some(source),
179            Self::Storage(error) => Some(error),
180            Self::Task(error) => Some(error),
181            Self::InvalidConfig(_) => None,
182        }
183    }
184}
185
186impl From<io::Error> for ServerError {
187    fn from(error: io::Error) -> Self {
188        Self::Io(error)
189    }
190}
191
192impl From<StorageError> for ServerError {
193    fn from(error: StorageError) -> Self {
194        Self::Storage(error)
195    }
196}
197
198/// A bound server which can be run in the current task or spawned.
199pub struct Server {
200    listener: TcpListener,
201    app: Router,
202    shutdown: watch::Sender<bool>,
203    local_addr: SocketAddr,
204    live_diagnostics: LiveDiagnostics,
205}
206
207impl Server {
208    pub async fn bind(
209        config: ServerConfig,
210        recording_path: impl AsRef<Path>,
211    ) -> Result<Self, ServerError> {
212        config.validate()?;
213        RecordingReader::open(recording_path.as_ref())?;
214        let ui = UiAssets::for_config(&config)?;
215        let listener = TcpListener::bind(config.bind_addr).await?;
216        let local_addr = listener.local_addr()?;
217        let (app, shutdown, live_diagnostics) = build_router(
218            recording_path.as_ref().to_path_buf(),
219            config,
220            local_addr.port(),
221            ui,
222        );
223        Ok(Self {
224            listener,
225            app,
226            shutdown,
227            local_addr,
228            live_diagnostics,
229        })
230    }
231
232    pub fn local_addr(&self) -> SocketAddr {
233        self.local_addr
234    }
235
236    /// Returns counters for verifying and observing the shared live tailer.
237    pub fn live_diagnostics(&self) -> LiveDiagnostics {
238        self.live_diagnostics.clone()
239    }
240
241    pub async fn serve(self) -> Result<(), ServerError> {
242        let mut shutdown = self.shutdown.subscribe();
243        axum::serve(self.listener, self.app)
244            .with_graceful_shutdown(async move {
245                while !*shutdown.borrow() {
246                    if shutdown.changed().await.is_err() {
247                        break;
248                    }
249                }
250            })
251            .await?;
252        Ok(())
253    }
254
255    pub fn spawn(self) -> ServerHandle {
256        let local_addr = self.local_addr;
257        let shutdown = self.shutdown.clone();
258        let task = tokio::spawn(self.serve());
259        ServerHandle {
260            local_addr,
261            shutdown: Some(shutdown),
262            task: Some(task),
263        }
264    }
265
266    /// Builds a router for embedding and in-process tests without binding a socket.
267    pub fn router(
268        config: ServerConfig,
269        recording_path: impl AsRef<Path>,
270    ) -> Result<Router, ServerError> {
271        config.validate()?;
272        RecordingReader::open(recording_path.as_ref())?;
273        let ui = UiAssets::for_config(&config)?;
274        let allowed_port = config.bind_addr.port();
275        Ok(build_router(
276            recording_path.as_ref().to_path_buf(),
277            config,
278            allowed_port,
279            ui,
280        )
281        .0)
282    }
283
284    /// Builds an embeddable router together with shared live-tailer diagnostics.
285    pub fn router_with_diagnostics(
286        config: ServerConfig,
287        recording_path: impl AsRef<Path>,
288    ) -> Result<(Router, LiveDiagnostics), ServerError> {
289        config.validate()?;
290        RecordingReader::open(recording_path.as_ref())?;
291        let ui = UiAssets::for_config(&config)?;
292        let allowed_port = config.bind_addr.port();
293        let (router, _shutdown, diagnostics) = build_router(
294            recording_path.as_ref().to_path_buf(),
295            config,
296            allowed_port,
297            ui,
298        );
299        Ok((router, diagnostics))
300    }
301}
302
303/// Handle for explicit graceful shutdown of a spawned server.
304///
305/// Dropping the handle broadcasts shutdown and aborts the listener task so a
306/// forgotten handle cannot detach a server indefinitely. Call [`Self::shutdown`]
307/// when graceful, awaitable completion is required.
308pub struct ServerHandle {
309    local_addr: SocketAddr,
310    shutdown: Option<watch::Sender<bool>>,
311    task: Option<JoinHandle<Result<(), ServerError>>>,
312}
313
314impl ServerHandle {
315    pub fn local_addr(&self) -> SocketAddr {
316        self.local_addr
317    }
318
319    pub async fn shutdown(mut self) -> Result<(), ServerError> {
320        if let Some(shutdown) = self.shutdown.take() {
321            shutdown.send_replace(true);
322        }
323        self.task
324            .take()
325            .expect("a live server handle owns its task")
326            .await
327            .map_err(ServerError::Task)?
328    }
329}
330
331impl Drop for ServerHandle {
332    fn drop(&mut self) {
333        if let Some(shutdown) = self.shutdown.take() {
334            shutdown.send_replace(true);
335        }
336        // Drop cannot await graceful completion. Aborting after broadcasting
337        // shutdown guarantees that the listener is not silently detached and
338        // leaked when its handle is forgotten.
339        if let Some(task) = self.task.take() {
340            task.abort();
341        }
342    }
343}
344
345/// Observable counters for the lazy shared durable tailer.
346///
347/// Cloning this handle does not keep the router or tailer alive.
348#[derive(Clone)]
349pub struct LiveDiagnostics {
350    metrics: Arc<LiveMetrics>,
351}
352
353impl LiveDiagnostics {
354    /// Number of shared tailer tasks started. It is at most one per router.
355    pub fn tailer_starts(&self) -> u64 {
356        self.metrics.starts.load(Ordering::Acquire)
357    }
358
359    /// Number of durable-storage polls performed by the shared tailer.
360    pub fn storage_polls(&self) -> u64 {
361        self.metrics.polls.load(Ordering::Acquire)
362    }
363
364    /// Whether the shared tailer task is currently alive.
365    pub fn tailer_active(&self) -> bool {
366        self.metrics.active.load(Ordering::Acquire)
367    }
368
369    /// Number of live SSE client tasks currently attached to this router.
370    pub fn subscribers(&self) -> u64 {
371        self.metrics.subscribers.load(Ordering::Acquire)
372    }
373
374    /// Number of clients that reached the shared stream after history catch-up.
375    pub fn completed_catchups(&self) -> u64 {
376        self.metrics.catchups.load(Ordering::Acquire)
377    }
378
379    /// Number of bounded history-page reads performed for SSE catch-up.
380    pub fn catchup_storage_reads(&self) -> u64 {
381        self.metrics.catchup_reads.load(Ordering::Acquire)
382    }
383
384    /// Number of durable events published once into the shared fanout.
385    pub fn published_events(&self) -> u64 {
386        self.metrics.published.load(Ordering::Acquire)
387    }
388
389    /// Directory scans performed by the stateful shared tail reader.
390    pub fn tail_directory_scans(&self) -> u64 {
391        self.metrics.tail_directory_scans.load(Ordering::Acquire)
392    }
393
394    /// Tail-reader reinitializations after a manifest lifecycle change.
395    pub fn tail_reinitializations(&self) -> u64 {
396        self.metrics.tail_reinitializations.load(Ordering::Acquire)
397    }
398
399    /// Durable frames decoded by the shared tail reader.
400    pub fn tail_frames_decoded(&self) -> u64 {
401        self.metrics.tail_frames_decoded.load(Ordering::Acquire)
402    }
403}
404
405#[derive(Default)]
406struct LiveMetrics {
407    starts: AtomicU64,
408    polls: AtomicU64,
409    active: AtomicBool,
410    subscribers: AtomicU64,
411    catchups: AtomicU64,
412    catchup_reads: AtomicU64,
413    published: AtomicU64,
414    tail_directory_scans: AtomicU64,
415    tail_reinitializations: AtomicU64,
416    tail_frames_decoded: AtomicU64,
417}
418
419struct LiveHub {
420    path: Arc<PathBuf>,
421    config: ServerConfig,
422    shutdown: watch::Sender<bool>,
423    notifications: watch::Sender<u64>,
424    state: Mutex<LiveFanoutState>,
425    started: AtomicBool,
426    metrics: Arc<LiveMetrics>,
427}
428
429#[derive(Default)]
430struct LiveFanoutState {
431    events: VecDeque<Arc<EventEnvelope>>,
432    event_bytes: usize,
433    evicted_through: Option<u64>,
434    initialized: bool,
435    tail_cursor: Option<u64>,
436    complete: bool,
437    terminal_error: Option<String>,
438}
439
440struct LiveSubscription {
441    notifications: watch::Receiver<u64>,
442    metrics: Arc<LiveMetrics>,
443    baseline: LiveBaseline,
444}
445
446impl Drop for LiveSubscription {
447    fn drop(&mut self) {
448        self.metrics.subscribers.fetch_sub(1, Ordering::Release);
449    }
450}
451
452enum LiveRead {
453    Event(Arc<EventEnvelope>),
454    Pending,
455    Complete,
456    TerminalError(String),
457    Lagged,
458}
459
460#[derive(Clone, Copy)]
461enum LiveBaseline {
462    Pending,
463    Ready(Option<u64>),
464}
465
466impl LiveHub {
467    fn new(path: Arc<PathBuf>, config: ServerConfig, shutdown: watch::Sender<bool>) -> Arc<Self> {
468        let (notifications, _) = watch::channel(0);
469        Arc::new(Self {
470            path,
471            config,
472            shutdown,
473            notifications,
474            state: Mutex::new(LiveFanoutState::default()),
475            started: AtomicBool::new(false),
476            metrics: Arc::new(LiveMetrics::default()),
477        })
478    }
479
480    fn subscribe(self: &Arc<Self>) -> LiveSubscription {
481        // Subscribe before starting or reading history so every durable event
482        // racing with catch-up either appears in history or wakes this receiver.
483        let notifications = self.notifications.subscribe();
484        let baseline = {
485            let state = self
486                .state
487                .lock()
488                .unwrap_or_else(std::sync::PoisonError::into_inner);
489            if state.initialized {
490                LiveBaseline::Ready(state.tail_cursor)
491            } else {
492                LiveBaseline::Pending
493            }
494        };
495        self.metrics.subscribers.fetch_add(1, Ordering::AcqRel);
496        self.ensure_started();
497        LiveSubscription {
498            notifications,
499            metrics: Arc::clone(&self.metrics),
500            baseline,
501        }
502    }
503
504    fn ensure_started(self: &Arc<Self>) {
505        if self
506            .started
507            .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
508            .is_err()
509        {
510            return;
511        }
512        self.metrics.starts.fetch_add(1, Ordering::Relaxed);
513        self.metrics.active.store(true, Ordering::Release);
514        let weak = Arc::downgrade(self);
515        let path = Arc::clone(&self.path);
516        let config = self.config.clone();
517        let shutdown = self.shutdown.subscribe();
518        let metrics = Arc::clone(&self.metrics);
519        tokio::spawn(shared_tailer(weak, path, config, shutdown, metrics));
520    }
521
522    fn publish(&self, events: Vec<EventEnvelope>) -> Result<(), String> {
523        if events.is_empty() {
524            return Ok(());
525        }
526        let event_count = events.len() as u64;
527        if let Some(event) = events
528            .iter()
529            .find(|event| live_event_weight(event) > self.config.live_buffer_bytes)
530        {
531            let weight = live_event_weight(event);
532            return Err(format!(
533                "live event {} requires {weight} bytes, exceeding live_buffer_bytes {}",
534                event.sequence.get(),
535                self.config.live_buffer_bytes
536            ));
537        }
538        let mut state = self
539            .state
540            .lock()
541            .unwrap_or_else(std::sync::PoisonError::into_inner);
542        for event in events {
543            let weight = live_event_weight(&event);
544            while state.events.len() >= self.config.live_buffer_events
545                || state.event_bytes.saturating_add(weight) > self.config.live_buffer_bytes
546            {
547                let Some(evicted) = state.events.pop_front() else {
548                    break;
549                };
550                state.event_bytes = state
551                    .event_bytes
552                    .saturating_sub(live_event_weight(&evicted));
553                state.evicted_through = Some(evicted.sequence.get());
554            }
555            state.event_bytes = state.event_bytes.saturating_add(weight);
556            state.tail_cursor = Some(event.sequence.get());
557            state.events.push_back(Arc::new(event));
558        }
559        drop(state);
560        self.metrics
561            .published
562            .fetch_add(event_count, Ordering::Release);
563        self.notify();
564        Ok(())
565    }
566
567    fn initialize(&self, baseline_cursor: Option<u64>, complete: bool) {
568        let mut state = self
569            .state
570            .lock()
571            .unwrap_or_else(std::sync::PoisonError::into_inner);
572        state.tail_cursor = baseline_cursor;
573        state.initialized = true;
574        state.complete = complete;
575        drop(state);
576        self.notify();
577    }
578
579    fn baseline(&self) -> LiveBaseline {
580        let state = self
581            .state
582            .lock()
583            .unwrap_or_else(std::sync::PoisonError::into_inner);
584        if state.initialized {
585            LiveBaseline::Ready(state.tail_cursor)
586        } else {
587            LiveBaseline::Pending
588        }
589    }
590
591    fn mark_complete(&self) {
592        self.state
593            .lock()
594            .unwrap_or_else(std::sync::PoisonError::into_inner)
595            .complete = true;
596        self.notify();
597    }
598
599    fn fail(&self, message: String) {
600        let mut state = self
601            .state
602            .lock()
603            .unwrap_or_else(std::sync::PoisonError::into_inner);
604        if state.terminal_error.is_none() {
605            state.terminal_error = Some(message);
606        }
607        if !state.initialized {
608            state.initialized = true;
609            state.tail_cursor = None;
610        }
611        drop(state);
612        self.notify();
613    }
614
615    fn read_after(&self, cursor: Option<u64>) -> LiveRead {
616        let state = self
617            .state
618            .lock()
619            .unwrap_or_else(std::sync::PoisonError::into_inner);
620        if state
621            .evicted_through
622            .is_some_and(|evicted| cursor.is_none_or(|cursor| cursor < evicted))
623        {
624            return LiveRead::Lagged;
625        }
626        let event = state
627            .events
628            .iter()
629            .find(|event| cursor.is_none_or(|cursor| event.sequence.get() > cursor))
630            .cloned();
631        if let Some(event) = event {
632            LiveRead::Event(event)
633        } else if let Some(message) = &state.terminal_error {
634            LiveRead::TerminalError(message.clone())
635        } else if state.complete {
636            LiveRead::Complete
637        } else {
638            LiveRead::Pending
639        }
640    }
641
642    fn notify(&self) {
643        self.notifications.send_modify(|version| {
644            *version = version.wrapping_add(1);
645        });
646    }
647}
648
649struct ActiveTailer(Arc<LiveMetrics>);
650
651impl Drop for ActiveTailer {
652    fn drop(&mut self) {
653        self.0.active.store(false, Ordering::Release);
654    }
655}
656
657async fn shared_tailer(
658    hub: Weak<LiveHub>,
659    path: Arc<PathBuf>,
660    config: ServerConfig,
661    mut shutdown: watch::Receiver<bool>,
662    metrics: Arc<LiveMetrics>,
663) {
664    let _active = ActiveTailer(Arc::clone(&metrics));
665    metrics.polls.fetch_add(1, Ordering::Relaxed);
666    let initialize_path = Arc::clone(&path);
667    let initialization = blocking(move || {
668        let reader = RecordingReader::open(initialize_path.as_ref())?;
669        Ok((
670            reader.last_durable_sequence()?.map(Sequence::get),
671            reader.manifest().state == RecordingState::Complete,
672        ))
673    })
674    .await;
675    let (cursor, initially_complete) = match initialization {
676        Ok(initialization) => initialization,
677        Err(error) => {
678            if let Some(hub) = hub.upgrade() {
679                hub.fail(error.message);
680            }
681            return;
682        }
683    };
684    let Some(initial_hub) = hub.upgrade() else {
685        return;
686    };
687    initial_hub.initialize(cursor, initially_complete);
688    drop(initial_hub);
689    if initially_complete {
690        return;
691    }
692    let tail_path = Arc::clone(&path);
693    let mut tail = match blocking(move || {
694        DurableTailReader::open(tail_path.as_ref(), cursor.map(Sequence::new))
695    })
696    .await
697    {
698        Ok(tail) => tail,
699        Err(error) => {
700            if let Some(hub) = hub.upgrade() {
701                hub.fail(error.message);
702            }
703            return;
704        }
705    };
706    loop {
707        if *shutdown.borrow() {
708            break;
709        }
710        let Some(current_hub) = hub.upgrade() else {
711            break;
712        };
713        drop(current_hub);
714        metrics.polls.fetch_add(1, Ordering::Relaxed);
715        let batch_limit = config.live_batch_limit;
716        let byte_limit = config.live_buffer_bytes;
717        let activity_path = Arc::clone(&path);
718        let result = blocking(move || {
719            let page = tail.poll_bounded(batch_limit, byte_limit, live_event_weight)?;
720            let activity = if tail.state() == RecordingState::Complete {
721                RecordingActivity::Complete
722            } else {
723                recording_activity(activity_path.as_ref())?
724            };
725            let stats = tail.stats();
726            Ok((tail, page, activity, stats))
727        })
728        .await;
729        let (returned_tail, page, activity, stats) = match result {
730            Ok(result) => result,
731            Err(error) => {
732                if let Some(hub) = hub.upgrade() {
733                    hub.fail(error.message);
734                }
735                break;
736            }
737        };
738        tail = returned_tail;
739        metrics
740            .tail_directory_scans
741            .store(stats.directory_scans, Ordering::Release);
742        metrics
743            .tail_reinitializations
744            .store(stats.reinitializations, Ordering::Release);
745        metrics
746            .tail_frames_decoded
747            .store(stats.frames_decoded, Ordering::Release);
748        if page.events.is_empty() {
749            if let Some(weight) = page.boundary_event_weight {
750                if let Some(hub) = hub.upgrade() {
751                    hub.fail(format!(
752                        "live event requires {weight} bytes, exceeding live_buffer_bytes {byte_limit}"
753                    ));
754                }
755                break;
756            }
757        }
758        let has_more = page.has_more;
759        if let Some(hub) = hub.upgrade() {
760            if let Err(message) = hub.publish(page.events) {
761                hub.fail(message);
762                break;
763            }
764            if activity != RecordingActivity::Active && !has_more {
765                hub.mark_complete();
766                break;
767            }
768        } else {
769            break;
770        }
771        if has_more {
772            continue;
773        }
774        tokio::select! {
775            _ = tokio::time::sleep(config.live_poll_interval) => {},
776            changed = shutdown.changed() => {
777                if changed.is_err() || *shutdown.borrow() { break; }
778            }
779        }
780    }
781}
782
783fn live_event_weight(event: &EventEnvelope) -> usize {
784    let retained_memory = std::mem::size_of::<EventEnvelope>()
785        .saturating_add(2 * std::mem::size_of::<usize>())
786        .saturating_add(
787            event
788                .payload
789                .as_ref()
790                .map_or(0, |payload| payload.data.capacity()),
791        );
792    retained_memory.max(event_response_size_bound(event))
793}
794
795#[derive(Clone)]
796struct AppState {
797    recording_path: Arc<PathBuf>,
798    config: ServerConfig,
799    shutdown: watch::Sender<bool>,
800    live: Arc<LiveHub>,
801    ui: Option<Arc<UiAssets>>,
802}
803
804#[derive(Debug)]
805struct UiAssets {
806    index_html: Bytes,
807    app_js: Bytes,
808    app_css: Bytes,
809    globals_css: Bytes,
810}
811
812impl UiAssets {
813    fn for_config(config: &ServerConfig) -> Result<Option<Arc<Self>>, ServerError> {
814        if let Some(directory) = &config.ui_dir {
815            return Self::from_directory(directory).map(Arc::new).map(Some);
816        }
817
818        #[cfg(feature = "ui")]
819        {
820            Ok(Some(Arc::new(Self {
821                index_html: Bytes::from_static(dbgflow_ui::index_html().as_bytes()),
822                app_js: Bytes::from_static(dbgflow_ui::app_js().as_bytes()),
823                app_css: Bytes::from_static(dbgflow_ui::app_css().as_bytes()),
824                globals_css: Bytes::from_static(dbgflow_ui::globals_css().as_bytes()),
825            })))
826        }
827
828        #[cfg(not(feature = "ui"))]
829        {
830            Ok(None)
831        }
832    }
833
834    fn from_directory(directory: &Path) -> Result<Self, ServerError> {
835        let read = |name: &str| {
836            let path = directory.join(name);
837            let metadata =
838                std::fs::symlink_metadata(&path).map_err(|source| ServerError::UiAsset {
839                    path: path.clone(),
840                    source,
841                })?;
842            if !metadata.file_type().is_file() {
843                return Err(ServerError::UiAsset {
844                    path,
845                    source: io::Error::new(
846                        io::ErrorKind::InvalidInput,
847                        "expected a regular file (symlinks and directories are rejected)",
848                    ),
849                });
850            }
851            std::fs::read(&path)
852                .map(Bytes::from)
853                .map_err(|source| ServerError::UiAsset { path, source })
854        };
855
856        Ok(Self {
857            index_html: read("index.html")?,
858            app_js: read("app.js")?,
859            app_css: read("app.css")?,
860            globals_css: read("globals.css")?,
861        })
862    }
863}
864
865#[derive(Clone, Copy)]
866struct HostPolicy {
867    port: u16,
868}
869
870fn build_router(
871    path: PathBuf,
872    config: ServerConfig,
873    allowed_port: u16,
874    ui: Option<Arc<UiAssets>>,
875) -> (Router, watch::Sender<bool>, LiveDiagnostics) {
876    let (shutdown, _) = watch::channel(false);
877    let live_hub = LiveHub::new(Arc::new(path.clone()), config.clone(), shutdown.clone());
878    let live_diagnostics = LiveDiagnostics {
879        metrics: Arc::clone(&live_hub.metrics),
880    };
881    let state = AppState {
882        recording_path: Arc::new(path),
883        config,
884        shutdown: shutdown.clone(),
885        live: live_hub,
886        ui: ui.clone(),
887    };
888    let router = Router::new()
889        .route("/api/v1/capabilities", get(capabilities))
890        .route("/api/v1/recording", get(recording))
891        .route("/api/v1/events", get(events))
892        .route("/api/v1/replay", get(replay))
893        .route("/api/v1/live", get(live));
894    let router = if ui.is_some() {
895        router
896            .route("/", get(ui_index))
897            .route("/app.js", get(ui_javascript))
898            .route("/app.css", get(ui_css))
899            .route("/globals.css", get(ui_globals_css))
900    } else {
901        router
902    };
903    let router = router
904        .with_state(state)
905        .layer(middleware::from_fn(security_headers))
906        .layer(middleware::from_fn_with_state(
907            HostPolicy { port: allowed_port },
908            enforce_loopback_host,
909        ));
910    (router, shutdown, live_diagnostics)
911}
912
913async fn enforce_loopback_host(
914    State(policy): State<HostPolicy>,
915    request: Request,
916    next: Next,
917) -> Response {
918    let mut values = request.headers().get_all(HOST).iter();
919    let host = values.next().and_then(|value| value.to_str().ok());
920    let single_host = host.is_some() && values.next().is_none();
921    if !single_host || !host.is_some_and(|value| allowed_host(value, policy.port)) {
922        return secure_response(
923            (
924                StatusCode::MISDIRECTED_REQUEST,
925                Json(ErrorResponse {
926                    error: "Host must name the loopback dbgflow listener".into(),
927                }),
928            )
929                .into_response(),
930        );
931    }
932    next.run(request).await
933}
934
935fn allowed_host(value: &str, port: u16) -> bool {
936    let value = value.to_ascii_lowercase();
937    value == format!("localhost:{port}")
938        || value == format!("127.0.0.1:{port}")
939        || value == format!("[::1]:{port}")
940}
941
942async fn security_headers(request: Request, next: Next) -> Response {
943    secure_response(next.run(request).await)
944}
945
946fn secure_response(mut response: Response) -> Response {
947    let headers = response.headers_mut();
948    headers.insert(
949        "content-security-policy",
950        HeaderValue::from_static(
951            "default-src 'none'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; font-src 'self'; worker-src 'self'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'",
952        ),
953    );
954    headers.insert(
955        "x-content-type-options",
956        HeaderValue::from_static("nosniff"),
957    );
958    headers.insert("x-frame-options", HeaderValue::from_static("DENY"));
959    headers.insert("referrer-policy", HeaderValue::from_static("no-referrer"));
960    headers.insert(
961        "permissions-policy",
962        HeaderValue::from_static("camera=(), microphone=(), geolocation=()"),
963    );
964    headers.insert(
965        "cross-origin-opener-policy",
966        HeaderValue::from_static("same-origin"),
967    );
968    headers.insert(
969        "cross-origin-resource-policy",
970        HeaderValue::from_static("same-origin"),
971    );
972    response
973}
974
975async fn ui_index(State(state): State<AppState>) -> Response {
976    ui_asset_response(
977        state
978            .ui
979            .expect("UI routes require loaded assets")
980            .index_html
981            .clone(),
982        "text/html; charset=utf-8",
983    )
984}
985
986async fn ui_javascript(State(state): State<AppState>) -> Response {
987    ui_asset_response(
988        state
989            .ui
990            .expect("UI routes require loaded assets")
991            .app_js
992            .clone(),
993        "application/javascript; charset=utf-8",
994    )
995}
996
997async fn ui_css(State(state): State<AppState>) -> Response {
998    ui_asset_response(
999        state
1000            .ui
1001            .expect("UI routes require loaded assets")
1002            .app_css
1003            .clone(),
1004        "text/css; charset=utf-8",
1005    )
1006}
1007
1008async fn ui_globals_css(State(state): State<AppState>) -> Response {
1009    ui_asset_response(
1010        state
1011            .ui
1012            .expect("UI routes require loaded assets")
1013            .globals_css
1014            .clone(),
1015        "text/css; charset=utf-8",
1016    )
1017}
1018
1019fn ui_asset_response(bytes: Bytes, content_type: &'static str) -> Response {
1020    Response::builder()
1021        .header("content-type", content_type)
1022        .body(Body::from(bytes))
1023        .expect("static UI response headers are valid")
1024}
1025
1026#[derive(Serialize)]
1027struct CapabilitiesResponse {
1028    api_version: u32,
1029    schema_version: u32,
1030    format_version: u32,
1031    transports: [&'static str; 1],
1032    cursor: CursorCapabilities,
1033    pagination: PaginationCapabilities,
1034    retention: RetentionCapabilities,
1035    replay: ReplayCapabilities,
1036}
1037
1038#[derive(Serialize)]
1039struct CursorCapabilities {
1040    kind: &'static str,
1041    semantics: &'static str,
1042    durability: &'static str,
1043}
1044
1045#[derive(Serialize)]
1046struct PaginationCapabilities {
1047    default_limit: usize,
1048    max_limit: usize,
1049    max_response_bytes: usize,
1050    directions: [&'static str; 2],
1051}
1052
1053#[derive(Serialize)]
1054struct RetentionCapabilities {
1055    rolling: bool,
1056    notice_event: &'static str,
1057}
1058
1059#[derive(Serialize)]
1060struct ReplayCapabilities {
1061    checkpoint_projection_version: u32,
1062    endpoint: &'static str,
1063    semantics: &'static str,
1064}
1065
1066async fn capabilities(State(state): State<AppState>) -> Json<CapabilitiesResponse> {
1067    Json(CapabilitiesResponse {
1068        api_version: API_VERSION,
1069        schema_version: CURRENT_SCHEMA_VERSION,
1070        format_version: CURRENT_FORMAT_VERSION,
1071        transports: ["sse"],
1072        cursor: CursorCapabilities {
1073            kind: "sequence",
1074            semantics: "exclusive",
1075            durability: "synced_frames",
1076        },
1077        pagination: PaginationCapabilities {
1078            default_limit: state.config.default_page_limit,
1079            max_limit: state.config.max_page_limit,
1080            max_response_bytes: state.config.max_page_bytes,
1081            directions: ["forward", "backward"],
1082        },
1083        retention: RetentionCapabilities {
1084            rolling: true,
1085            notice_event: "dbgflow.retention",
1086        },
1087        replay: ReplayCapabilities {
1088            checkpoint_projection_version: CURRENT_PROJECTION_VERSION,
1089            endpoint: "/api/v1/replay",
1090            semantics: "nearest_checkpoint_plus_deltas",
1091        },
1092    })
1093}
1094
1095#[derive(Serialize)]
1096struct RecordingResponse {
1097    format_version: u32,
1098    schema_version: u32,
1099    recording_id: String,
1100    state: RecordingState,
1101    activity: RecordingActivity,
1102    created_unix_ms: Option<i64>,
1103    metadata: BTreeMap<String, String>,
1104    durable: DurableResponse,
1105    /// Pre-v1 compatibility alias. New clients should use `durable`.
1106    committed: DurableResponse,
1107}
1108
1109#[derive(Clone, Serialize)]
1110struct DurableResponse {
1111    immutable_chunk_count: u64,
1112    has_active_prefix: bool,
1113    event_count: u64,
1114    first_cursor: Option<String>,
1115    last_cursor: Option<String>,
1116    retained_prefix: RetainedPrefixResponse,
1117}
1118
1119#[derive(Clone, Serialize)]
1120struct RetainedPrefixResponse {
1121    history_truncated: bool,
1122    generation: String,
1123    first_chunk_number: String,
1124    first_cursor: Option<String>,
1125    removed_through_cursor: Option<String>,
1126}
1127
1128async fn recording(State(state): State<AppState>) -> Result<Json<RecordingResponse>, ApiError> {
1129    let path = state.recording_path.clone();
1130    let (manifest, durable, activity) = blocking(move || {
1131        let reader = RecordingReader::open(path.as_ref())?;
1132        let durable = reader.durable_metadata()?;
1133        let activity = recording_activity(path.as_ref())?;
1134        Ok((reader.manifest().clone(), durable, activity))
1135    })
1136    .await?;
1137    Ok(Json(recording_response(manifest, durable, activity)))
1138}
1139
1140#[derive(Default, Deserialize)]
1141struct EventsQuery {
1142    after: Option<u64>,
1143    before: Option<u64>,
1144    limit: Option<usize>,
1145}
1146
1147#[derive(Serialize)]
1148struct EventsResponse {
1149    events: Vec<EventResponse>,
1150    next_cursor: Option<String>,
1151    previous_cursor: Option<String>,
1152    last_durable_cursor: Option<String>,
1153    has_more: bool,
1154    retained_prefix: RetainedPrefixResponse,
1155}
1156
1157async fn events(
1158    State(state): State<AppState>,
1159    Query(query): Query<EventsQuery>,
1160) -> Result<Json<EventsResponse>, ApiError> {
1161    let limit = query.limit.unwrap_or(state.config.default_page_limit);
1162    validate_limit(limit, state.config.max_page_limit)?;
1163    if query.after.is_some() && query.before.is_some() {
1164        return Err(ApiError::bad_request(
1165            "after and before cursors are mutually exclusive",
1166        ));
1167    }
1168    let direction = if let Some(before) = query.before {
1169        PageDirection::Backward(Sequence::new(before))
1170    } else {
1171        PageDirection::Forward(query.after.map(Sequence::new))
1172    };
1173    let path = state.recording_path.clone();
1174    let max_page_bytes = state.config.max_page_bytes;
1175    let snapshot = blocking(move || {
1176        let reader = RecordingReader::open(path.as_ref())?;
1177        for _ in 0..8 {
1178            let retained_before = reader.retained_prefix()?;
1179            let page = match direction {
1180                PageDirection::Forward(after) => reader.read_after_bounded(
1181                    after,
1182                    limit,
1183                    max_page_bytes,
1184                    event_response_size_bound,
1185                )?,
1186                PageDirection::Backward(before) => reader.read_before_bounded(
1187                    Some(before),
1188                    limit,
1189                    max_page_bytes,
1190                    event_response_size_bound,
1191                )?,
1192            };
1193            let last_durable_sequence = reader.last_durable_sequence()?;
1194            let retained_after = reader.retained_prefix()?;
1195            if retained_before == retained_after {
1196                return Ok(Some((page, last_durable_sequence, retained_after)));
1197            }
1198        }
1199        Ok(None)
1200    })
1201    .await?;
1202    let (page, last_durable_sequence, retained_prefix) = snapshot.ok_or_else(|| {
1203        ApiError::internal(
1204            "rolling retention changed repeatedly while capturing a coherent history page; retry",
1205        )
1206    })?;
1207    if page.events.is_empty() && page.boundary_event_weight.is_some() {
1208        return Err(ApiError::payload_too_large(format!(
1209            "boundary event exceeds the configured page response budget of {max_page_bytes} bytes"
1210        )));
1211    }
1212    let next_cursor = if query.before.is_none() {
1213        page.events
1214            .last()
1215            .map(|event| event.sequence.get().to_string())
1216            .or_else(|| query.after.map(|cursor| cursor.to_string()))
1217    } else {
1218        None
1219    };
1220    let previous_cursor = if query.before.is_some() {
1221        page.events
1222            .first()
1223            .map(|event| event.sequence.get().to_string())
1224    } else {
1225        None
1226    };
1227    Ok(Json(EventsResponse {
1228        events: page.events.iter().map(EventResponse::from).collect(),
1229        next_cursor,
1230        previous_cursor,
1231        last_durable_cursor: last_durable_sequence.map(|value| value.get().to_string()),
1232        has_more: page.has_more,
1233        retained_prefix: retained_prefix_response(retained_prefix),
1234    }))
1235}
1236
1237#[derive(Clone, Copy)]
1238enum PageDirection {
1239    Forward(Option<Sequence>),
1240    Backward(Sequence),
1241}
1242
1243#[derive(Deserialize)]
1244struct ReplayQuery {
1245    cursor: u64,
1246    boundary: Option<u64>,
1247    after: Option<u64>,
1248    limit: Option<usize>,
1249}
1250
1251#[derive(Serialize)]
1252struct ReplayResponse {
1253    requested_cursor: String,
1254    resolved_cursor: String,
1255    checkpoint: Option<CheckpointResponse>,
1256    deltas: Vec<EventResponse>,
1257    has_more: bool,
1258    next_cursor: Option<String>,
1259    exact: bool,
1260    retained_prefix: RetainedPrefixResponse,
1261    ignored_checkpoints: Vec<CheckpointIssueResponse>,
1262}
1263
1264#[derive(Serialize)]
1265struct CheckpointResponse {
1266    projection_version: u32,
1267    recording_id: String,
1268    durable_cursor: String,
1269    complete: bool,
1270    active_spans: Vec<ProjectedSpanResponse>,
1271    known_spans: Vec<ProjectedSpanResponse>,
1272}
1273
1274#[derive(Serialize)]
1275struct ProjectedSpanResponse {
1276    context: ContextResponse,
1277    opened_cursor: String,
1278    enter_depth: u32,
1279    is_open: bool,
1280    metadata: Option<PayloadResponse>,
1281}
1282
1283#[derive(Serialize)]
1284struct CheckpointIssueResponse {
1285    file: String,
1286    reason: String,
1287}
1288
1289async fn replay(
1290    State(state): State<AppState>,
1291    Query(query): Query<ReplayQuery>,
1292) -> Result<Json<ReplayResponse>, ApiError> {
1293    let limit = query.limit.unwrap_or(state.config.default_page_limit);
1294    validate_limit(limit, state.config.max_page_limit)?;
1295    if query.after.is_some_and(|after| after > query.cursor) {
1296        return Err(ApiError::bad_request(
1297            "replay after cursor must not exceed requested cursor",
1298        ));
1299    }
1300    if query
1301        .boundary
1302        .is_some_and(|boundary| boundary > query.cursor)
1303    {
1304        return Err(ApiError::bad_request(
1305            "replay boundary cursor must not exceed requested cursor",
1306        ));
1307    }
1308    let requested = Sequence::new(query.cursor);
1309    let path = state.recording_path.clone();
1310    let max_page_bytes = state.config.max_page_bytes;
1311    let captured = blocking(move || {
1312        let reader = RecordingReader::open(path.as_ref())?;
1313        for _ in 0..8 {
1314            let retained_before = reader.retained_prefix()?;
1315            let last = reader.last_durable_sequence()?.unwrap_or(Sequence::new(0));
1316            let resolved = Sequence::new(requested.get().min(last.get()));
1317            let lookup_cursor = Sequence::new(query.boundary.unwrap_or(resolved.get()));
1318            let lookup = if query.boundary.is_some() {
1319                reader.reconstruct_checkpoint_at(lookup_cursor, ProjectionLimits::default())?
1320            } else {
1321                reader.nearest_checkpoint_at_or_before(lookup_cursor)?
1322            };
1323            let checkpoint_after = lookup
1324                .checkpoint
1325                .as_ref()
1326                .map(|checkpoint| checkpoint.durable_cursor);
1327            let after = match (query.after, checkpoint_after) {
1328                (Some(after), Some(checkpoint)) => Some(Sequence::new(after.max(checkpoint.get()))),
1329                (Some(after), None) => Some(Sequence::new(after)),
1330                (None, checkpoint) => checkpoint,
1331            };
1332            let page = reader.read_range_bounded(
1333                after,
1334                resolved,
1335                limit,
1336                max_page_bytes,
1337                event_response_size_bound,
1338            )?;
1339            let retained_after = reader.retained_prefix()?;
1340            if retained_before == retained_after {
1341                return Ok(Some((resolved, lookup, page, retained_after)));
1342            }
1343        }
1344        Ok(None)
1345    })
1346    .await?;
1347    let (resolved, lookup, page, retained) = captured.ok_or_else(|| {
1348        ApiError::internal(
1349            "rolling retention changed repeatedly while capturing a coherent replay boundary; retry",
1350        )
1351    })?;
1352    if page.events.is_empty() && page.boundary_event_weight.is_some() {
1353        return Err(ApiError::payload_too_large(format!(
1354            "boundary replay event exceeds the configured response budget of {max_page_bytes} bytes"
1355        )));
1356    }
1357    let replayed_through = page
1358        .events
1359        .last()
1360        .map(|event| event.sequence.get())
1361        .or_else(|| {
1362            lookup
1363                .checkpoint
1364                .as_ref()
1365                .map(|checkpoint| checkpoint.durable_cursor.get())
1366        });
1367    let checkpoint_complete = lookup
1368        .checkpoint
1369        .as_ref()
1370        .is_none_or(|checkpoint| checkpoint.complete);
1371    let exact = checkpoint_complete
1372        && !page.has_more
1373        && replayed_through.is_some_and(|cursor| cursor == resolved.get())
1374        && !retention_notice_required(replayed_through, retained);
1375    let next_cursor = page
1376        .has_more
1377        .then(|| {
1378            page.events
1379                .last()
1380                .map(|event| event.sequence.get().to_string())
1381        })
1382        .flatten();
1383    Ok(Json(ReplayResponse {
1384        requested_cursor: requested.get().to_string(),
1385        resolved_cursor: resolved.get().to_string(),
1386        checkpoint: lookup.checkpoint.as_ref().map(CheckpointResponse::from),
1387        deltas: page.events.iter().map(EventResponse::from).collect(),
1388        has_more: page.has_more,
1389        next_cursor,
1390        exact,
1391        retained_prefix: retained_prefix_response(retained),
1392        ignored_checkpoints: lookup
1393            .ignored
1394            .into_iter()
1395            .map(|issue| CheckpointIssueResponse {
1396                file: issue
1397                    .path
1398                    .file_name()
1399                    .map_or_else(String::new, |name| name.to_string_lossy().into_owned()),
1400                reason: issue.reason,
1401            })
1402            .collect(),
1403    }))
1404}
1405
1406#[derive(Default, Deserialize)]
1407struct LiveQuery {
1408    after: Option<u64>,
1409}
1410
1411async fn live(
1412    State(state): State<AppState>,
1413    Query(query): Query<LiveQuery>,
1414    headers: HeaderMap,
1415) -> Result<Response, ApiError> {
1416    // Browsers reconnect EventSource with the original query string and a new
1417    // Last-Event-ID. Prefer the header when present so reconnects resume from
1418    // actual progress instead of replaying from the initial query cursor.
1419    let after = parse_last_event_id(&headers)?.or(query.after);
1420    let subscription = state.live.subscribe();
1421    let path = state.recording_path.clone();
1422    let (last_durable_sequence, retained_prefix) = blocking({
1423        let path = path.clone();
1424        move || {
1425            let reader = RecordingReader::open(path.as_ref())?;
1426            Ok((reader.last_durable_sequence()?, reader.retained_prefix()?))
1427        }
1428    })
1429    .await?;
1430
1431    // One queued serialized item plus the sender's current item keeps each
1432    // client bounded independently of the shared Arc<EventEnvelope> ring.
1433    let (sender, receiver) = mpsc::channel(1);
1434    let mut shutdown = state.shutdown.subscribe();
1435    let hub = Arc::clone(&state.live);
1436    tokio::spawn(async move {
1437        follow_shared(
1438            hub,
1439            after,
1440            retained_prefix,
1441            subscription,
1442            &mut shutdown,
1443            sender,
1444        )
1445        .await;
1446    });
1447
1448    let stream = ReceiverStream::new(receiver);
1449    let mut response = Sse::new(stream)
1450        .keep_alive(
1451            KeepAlive::new()
1452                .interval(state.config.keep_alive_interval)
1453                .text("keep-alive"),
1454        )
1455        .into_response();
1456    response.headers_mut().insert(
1457        LAST_DURABLE_CURSOR,
1458        HeaderValue::from_str(
1459            &last_durable_sequence.map_or_else(|| "".into(), |value| value.get().to_string()),
1460        )
1461        .expect("a decimal cursor is a valid header value"),
1462    );
1463    Ok(response)
1464}
1465
1466async fn follow_shared(
1467    hub: Arc<LiveHub>,
1468    after: Option<u64>,
1469    initial_retained_prefix: RetainedPrefix,
1470    mut subscription: LiveSubscription,
1471    shutdown: &mut watch::Receiver<bool>,
1472    sender: mpsc::Sender<Result<SseEvent, Infallible>>,
1473) {
1474    let mut cursor = after;
1475    let mut last_notice_generation = None;
1476    if retention_notice_required(cursor, initial_retained_prefix) {
1477        if !send_retention_notice(
1478            &sender,
1479            shutdown,
1480            &retained_prefix_response(initial_retained_prefix),
1481        )
1482        .await
1483        {
1484            return;
1485        }
1486        last_notice_generation = Some(initial_retained_prefix.generation);
1487        advance_cursor_to_retained_boundary(&mut cursor, initial_retained_prefix);
1488    }
1489    let baseline = loop {
1490        match subscription.baseline {
1491            LiveBaseline::Ready(cursor) => break cursor,
1492            LiveBaseline::Pending => {
1493                tokio::select! {
1494                    changed = subscription.notifications.changed() => {
1495                        if changed.is_err() { return; }
1496                    },
1497                    changed = shutdown.changed() => {
1498                        if changed.is_err() || *shutdown.borrow() { return; }
1499                    },
1500                    _ = sender.closed() => return,
1501                }
1502                subscription.baseline = hub.baseline();
1503            }
1504        }
1505    };
1506    // The subscription already exists, closing the classic history/live race:
1507    // anything that becomes durable during this bounded catch-up is either
1508    // read here or retained/notified by the shared fanout and deduplicated.
1509    while baseline.is_some_and(|baseline| cursor.is_none_or(|cursor| cursor < baseline)) {
1510        let read_path = Arc::clone(&hub.path);
1511        let read_after = cursor.map(Sequence::new);
1512        let batch_limit = hub.config.live_batch_limit.min(MAX_LIVE_CATCHUP_EVENTS);
1513        let byte_limit = hub.config.live_buffer_bytes.min(MAX_LIVE_CATCHUP_BYTES);
1514        hub.metrics.catchup_reads.fetch_add(1, Ordering::Relaxed);
1515        let result = blocking(move || {
1516            let reader = RecordingReader::open(read_path.as_ref())?;
1517            let page = reader.read_after_bounded(
1518                read_after,
1519                batch_limit,
1520                byte_limit,
1521                live_event_weight,
1522            )?;
1523            let retained_prefix = reader.retained_prefix()?;
1524            Ok((page, retained_prefix))
1525        })
1526        .await;
1527        let (mut page, mut page_retained_prefix) = match result {
1528            Ok(result) => result,
1529            Err(error) => {
1530                send_live_error(&sender, shutdown, error.message).await;
1531                return;
1532            }
1533        };
1534        // Keep normal catch-up pages small, but still make progress when a
1535        // single event is larger than that page budget and remains within the
1536        // configured per-event/live-ring byte bound.
1537        if page.events.is_empty()
1538            && page
1539                .boundary_event_weight
1540                .is_some_and(|weight| weight <= hub.config.live_buffer_bytes)
1541            && byte_limit < hub.config.live_buffer_bytes
1542        {
1543            let read_path = Arc::clone(&hub.path);
1544            let live_buffer_bytes = hub.config.live_buffer_bytes;
1545            hub.metrics.catchup_reads.fetch_add(1, Ordering::Relaxed);
1546            let result = blocking(move || {
1547                let reader = RecordingReader::open(read_path.as_ref())?;
1548                let page = reader.read_after_bounded(
1549                    read_after,
1550                    1,
1551                    live_buffer_bytes,
1552                    live_event_weight,
1553                )?;
1554                let retained_prefix = reader.retained_prefix()?;
1555                Ok((page, retained_prefix))
1556            })
1557            .await;
1558            (page, page_retained_prefix) = match result {
1559                Ok(result) => result,
1560                Err(error) => {
1561                    send_live_error(&sender, shutdown, error.message).await;
1562                    return;
1563                }
1564            };
1565        }
1566        if retention_notice_required(cursor, page_retained_prefix)
1567            && last_notice_generation != Some(page_retained_prefix.generation)
1568        {
1569            if !send_retention_notice(
1570                &sender,
1571                shutdown,
1572                &retained_prefix_response(page_retained_prefix),
1573            )
1574            .await
1575            {
1576                return;
1577            }
1578            last_notice_generation = Some(page_retained_prefix.generation);
1579            advance_cursor_to_retained_boundary(&mut cursor, page_retained_prefix);
1580        }
1581        for envelope in &page.events {
1582            if baseline.is_some_and(|baseline| envelope.sequence.get() > baseline) {
1583                break;
1584            }
1585            if cursor.is_some_and(|cursor| envelope.sequence.get() <= cursor) {
1586                continue;
1587            }
1588            if !send_live_event(&sender, shutdown, envelope).await {
1589                return;
1590            }
1591            cursor = Some(envelope.sequence.get());
1592        }
1593        if cursor.is_some_and(|cursor| baseline.is_some_and(|baseline| cursor >= baseline)) {
1594            break;
1595        }
1596        if page.events.is_empty() {
1597            if let Some(weight) = page.boundary_event_weight {
1598                send_live_error(
1599                    &sender,
1600                    shutdown,
1601                    format!(
1602                        "live event requires {weight} bytes, exceeding live_buffer_bytes {}",
1603                        hub.config.live_buffer_bytes
1604                    ),
1605                )
1606                .await;
1607                return;
1608            }
1609            send_live_error(
1610                &sender,
1611                shutdown,
1612                "durable history ended before the shared live baseline".into(),
1613            )
1614            .await;
1615            return;
1616        }
1617    }
1618
1619    hub.metrics.catchups.fetch_add(1, Ordering::Relaxed);
1620    loop {
1621        match hub.read_after(cursor) {
1622            LiveRead::Event(envelope) => {
1623                if cursor.is_some_and(|cursor| envelope.sequence.get() <= cursor) {
1624                    continue;
1625                }
1626                if !send_live_event(&sender, shutdown, &envelope).await {
1627                    return;
1628                }
1629                cursor = Some(envelope.sequence.get());
1630            }
1631            LiveRead::Pending => {
1632                tokio::select! {
1633                    changed = subscription.notifications.changed() => {
1634                        if changed.is_err() { return; }
1635                    },
1636                    changed = shutdown.changed() => {
1637                        if changed.is_err() || *shutdown.borrow() { return; }
1638                    },
1639                    _ = sender.closed() => return,
1640                }
1641            }
1642            LiveRead::Complete | LiveRead::Lagged => return,
1643            LiveRead::TerminalError(message) => {
1644                send_live_error(&sender, shutdown, message).await;
1645                return;
1646            }
1647        }
1648    }
1649}
1650
1651async fn send_live_event(
1652    sender: &mpsc::Sender<Result<SseEvent, Infallible>>,
1653    shutdown: &mut watch::Receiver<bool>,
1654    envelope: &EventEnvelope,
1655) -> bool {
1656    let permit = tokio::select! {
1657        permit = sender.reserve() => match permit {
1658            Ok(permit) => permit,
1659            Err(_) => return false,
1660        },
1661        _ = shutdown.changed() => return false,
1662    };
1663    let name = if envelope.kind == EventKind::Gap {
1664        "dbgflow.gap"
1665    } else {
1666        "dbgflow.event"
1667    };
1668    let event = SseEvent::default()
1669        .event(name)
1670        .id(envelope.sequence.get().to_string())
1671        .json_data(EventResponse::from(envelope))
1672        .expect("event response always serializes");
1673    permit.send(Ok(event));
1674    true
1675}
1676
1677async fn send_retention_notice(
1678    sender: &mpsc::Sender<Result<SseEvent, Infallible>>,
1679    shutdown: &mut watch::Receiver<bool>,
1680    notice: &RetainedPrefixResponse,
1681) -> bool {
1682    let permit = tokio::select! {
1683        permit = sender.reserve() => match permit {
1684            Ok(permit) => permit,
1685            Err(_) => return false,
1686        },
1687        _ = shutdown.changed() => return false,
1688    };
1689    let event = SseEvent::default()
1690        .event("dbgflow.retention")
1691        .json_data(notice)
1692        .expect("retention notice always serializes");
1693    permit.send(Ok(event));
1694    true
1695}
1696
1697async fn send_live_error(
1698    sender: &mpsc::Sender<Result<SseEvent, Infallible>>,
1699    shutdown: &mut watch::Receiver<bool>,
1700    message: String,
1701) {
1702    let permit = tokio::select! {
1703        permit = sender.reserve() => match permit {
1704            Ok(permit) => permit,
1705            Err(_) => return,
1706        },
1707        _ = shutdown.changed() => return,
1708    };
1709    let data = serde_json::json!({"error": message});
1710    let event = SseEvent::default()
1711        .event("dbgflow.error")
1712        .data(data.to_string());
1713    permit.send(Ok(event));
1714}
1715
1716fn parse_last_event_id(headers: &HeaderMap) -> Result<Option<u64>, ApiError> {
1717    let Some(value) = headers.get(LAST_EVENT_ID) else {
1718        return Ok(None);
1719    };
1720    let value = value
1721        .to_str()
1722        .map_err(|_| ApiError::bad_request("Last-Event-ID must be ASCII"))?;
1723    value
1724        .parse()
1725        .map(Some)
1726        .map_err(|_| ApiError::bad_request("Last-Event-ID must be an unsigned integer"))
1727}
1728
1729fn validate_limit(limit: usize, maximum: usize) -> Result<(), ApiError> {
1730    if limit == 0 || limit > maximum {
1731        Err(ApiError::bad_request(format!(
1732            "limit must be in 1..={maximum}"
1733        )))
1734    } else {
1735        Ok(())
1736    }
1737}
1738
1739async fn blocking<T: Send + 'static>(
1740    operation: impl FnOnce() -> Result<T, StorageError> + Send + 'static,
1741) -> Result<T, ApiError> {
1742    tokio::task::spawn_blocking(operation)
1743        .await
1744        .map_err(|error| ApiError::internal(format!("storage task failed: {error}")))?
1745        .map_err(ApiError::from)
1746}
1747
1748fn recording_response(
1749    manifest: Manifest,
1750    durable: DurableMetadata,
1751    activity: RecordingActivity,
1752) -> RecordingResponse {
1753    let retained = durable.retained_prefix;
1754    let durable = DurableResponse {
1755        immutable_chunk_count: durable.chunk_count,
1756        has_active_prefix: durable.has_active_prefix,
1757        event_count: durable.event_count,
1758        first_cursor: durable.first_sequence.map(|value| value.get().to_string()),
1759        last_cursor: durable.last_sequence.map(|value| value.get().to_string()),
1760        retained_prefix: retained_prefix_response(retained),
1761    };
1762    RecordingResponse {
1763        format_version: manifest.format_version,
1764        schema_version: manifest.schema_version,
1765        recording_id: manifest.recording_id,
1766        state: manifest.state,
1767        activity,
1768        created_unix_ms: manifest.created_unix_ms,
1769        metadata: manifest.metadata,
1770        durable: durable.clone(),
1771        committed: durable,
1772    }
1773}
1774
1775fn retained_prefix_response(retained: RetainedPrefix) -> RetainedPrefixResponse {
1776    RetainedPrefixResponse {
1777        history_truncated: retained.history_truncated,
1778        generation: retained.generation.to_string(),
1779        first_chunk_number: retained.first_chunk_number.to_string(),
1780        first_cursor: retained.first_sequence.map(|value| value.get().to_string()),
1781        removed_through_cursor: retained
1782            .removed_through_sequence
1783            .map(|value| value.get().to_string()),
1784    }
1785}
1786
1787fn retention_notice_required(after: Option<u64>, retained: RetainedPrefix) -> bool {
1788    if !retained.history_truncated {
1789        return false;
1790    }
1791    if let Some(removed) = retained.removed_through_sequence {
1792        return after.is_none_or(|after| after <= removed.get());
1793    }
1794    match retained.first_sequence {
1795        Some(first) => after.is_none_or(|after| after < first.get()),
1796        None => true,
1797    }
1798}
1799
1800fn advance_cursor_to_retained_boundary(cursor: &mut Option<u64>, retained: RetainedPrefix) {
1801    let boundary = retained
1802        .removed_through_sequence
1803        .map(Sequence::get)
1804        .or_else(|| {
1805            retained
1806                .first_sequence
1807                .and_then(|first| first.get().checked_sub(1))
1808        });
1809    if let Some(boundary) = boundary {
1810        *cursor = Some(cursor.map_or(boundary, |cursor| cursor.max(boundary)));
1811    }
1812}
1813
1814#[derive(Serialize)]
1815struct EventResponse {
1816    schema_version: u32,
1817    recording_id: String,
1818    sequence: String,
1819    timestamp: TimestampResponse,
1820    context: ContextResponse,
1821    kind: String,
1822    payload: Option<PayloadResponse>,
1823}
1824
1825#[derive(Serialize)]
1826struct TimestampResponse {
1827    monotonic_ns: String,
1828    wall_time_unix_ns: Option<String>,
1829}
1830
1831#[derive(Serialize)]
1832struct ContextResponse {
1833    process_id: u32,
1834    thread_id: String,
1835    flow_id: Option<String>,
1836    span_id: Option<String>,
1837    parent_span_id: Option<String>,
1838}
1839
1840#[derive(Serialize)]
1841struct PayloadResponse {
1842    encoding: String,
1843    flags: u32,
1844    data_base64: String,
1845}
1846
1847impl From<&EventEnvelope> for EventResponse {
1848    fn from(event: &EventEnvelope) -> Self {
1849        Self {
1850            schema_version: event.schema_version.get(),
1851            recording_id: format!("{:032x}", event.recording_id.as_u128()),
1852            sequence: event.sequence.get().to_string(),
1853            timestamp: TimestampResponse {
1854                monotonic_ns: event.timestamp.monotonic_ns.to_string(),
1855                wall_time_unix_ns: event
1856                    .timestamp
1857                    .wall_time_unix_ns
1858                    .map(|value| value.to_string()),
1859            },
1860            context: context_response(event.context),
1861            kind: event_kind(event.kind),
1862            payload: event.payload.as_ref().map(payload_response),
1863        }
1864    }
1865}
1866
1867impl From<&ProjectionCheckpoint> for CheckpointResponse {
1868    fn from(checkpoint: &ProjectionCheckpoint) -> Self {
1869        Self {
1870            projection_version: checkpoint.projection_version,
1871            recording_id: format!("{:032x}", checkpoint.recording_id.as_u128()),
1872            durable_cursor: checkpoint.durable_cursor.get().to_string(),
1873            complete: checkpoint.complete,
1874            active_spans: checkpoint
1875                .active_spans
1876                .iter()
1877                .map(ProjectedSpanResponse::from)
1878                .collect(),
1879            known_spans: checkpoint
1880                .known_spans
1881                .iter()
1882                .map(ProjectedSpanResponse::from)
1883                .collect(),
1884        }
1885    }
1886}
1887
1888impl From<&ProjectedSpan> for ProjectedSpanResponse {
1889    fn from(span: &ProjectedSpan) -> Self {
1890        Self {
1891            context: context_response(span.context),
1892            opened_cursor: span.opened_sequence.get().to_string(),
1893            enter_depth: span.enter_depth,
1894            is_open: span.is_open,
1895            metadata: span.metadata.as_ref().map(payload_response),
1896        }
1897    }
1898}
1899
1900fn context_response(context: dbgflow_protocol::EventContext) -> ContextResponse {
1901    ContextResponse {
1902        process_id: context.process_id.get(),
1903        thread_id: context.thread_id.get().to_string(),
1904        flow_id: context.flow_id.map(|value| value.get().to_string()),
1905        span_id: context.span_id.map(|value| value.get().to_string()),
1906        parent_span_id: context.parent_span_id.map(|value| value.get().to_string()),
1907    }
1908}
1909
1910fn payload_response(payload: &Payload) -> PayloadResponse {
1911    PayloadResponse {
1912        encoding: payload_encoding(payload.encoding),
1913        flags: payload.flags.bits(),
1914        data_base64: BASE64.encode(&payload.data),
1915    }
1916}
1917
1918fn event_response_size_bound(event: &EventEnvelope) -> usize {
1919    let encoded_payload = event.payload.as_ref().map_or(0, |payload| {
1920        payload.data.len().div_ceil(3).saturating_mul(4)
1921    });
1922    EVENT_RESPONSE_OVERHEAD_BYTES.saturating_add(encoded_payload)
1923}
1924
1925fn event_kind(kind: EventKind) -> String {
1926    match kind {
1927        EventKind::SpanOpen => "span_open".into(),
1928        EventKind::SpanEnter => "span_enter".into(),
1929        EventKind::SpanExit => "span_exit".into(),
1930        EventKind::SpanClose => "span_close".into(),
1931        EventKind::Snapshot => "snapshot".into(),
1932        EventKind::Checkpoint => "checkpoint".into(),
1933        EventKind::Log => "log".into(),
1934        EventKind::Metadata => "metadata".into(),
1935        EventKind::Gap => "gap".into(),
1936        EventKind::Custom(value) => format!("custom:{value}"),
1937    }
1938}
1939
1940fn payload_encoding(encoding: PayloadEncoding) -> String {
1941    match encoding {
1942        PayloadEncoding::Opaque => "opaque".into(),
1943        PayloadEncoding::Json => "json".into(),
1944        PayloadEncoding::Utf8 => "utf8".into(),
1945        PayloadEncoding::Protobuf => "protobuf".into(),
1946        PayloadEncoding::Custom(value) => format!("custom:{value}"),
1947    }
1948}
1949
1950#[derive(Serialize)]
1951struct ErrorResponse {
1952    error: String,
1953}
1954
1955struct ApiError {
1956    status: StatusCode,
1957    message: String,
1958}
1959
1960impl ApiError {
1961    fn bad_request(message: impl Into<String>) -> Self {
1962        Self {
1963            status: StatusCode::BAD_REQUEST,
1964            message: message.into(),
1965        }
1966    }
1967
1968    fn internal(message: impl Into<String>) -> Self {
1969        Self {
1970            status: StatusCode::INTERNAL_SERVER_ERROR,
1971            message: message.into(),
1972        }
1973    }
1974
1975    fn payload_too_large(message: impl Into<String>) -> Self {
1976        Self {
1977            status: StatusCode::PAYLOAD_TOO_LARGE,
1978            message: message.into(),
1979        }
1980    }
1981}
1982
1983impl From<StorageError> for ApiError {
1984    fn from(error: StorageError) -> Self {
1985        Self::internal(error.to_string())
1986    }
1987}
1988
1989impl IntoResponse for ApiError {
1990    fn into_response(self) -> Response {
1991        (
1992            self.status,
1993            Json(ErrorResponse {
1994                error: self.message,
1995            }),
1996        )
1997            .into_response()
1998    }
1999}
2000
2001#[cfg(test)]
2002mod tests;