1use std::{
8 collections::{BTreeMap, VecDeque},
9 convert::Infallible,
10 error::Error,
11 fmt, io,
12 net::{IpAddr, Ipv4Addr, SocketAddr},
13 path::{Path, PathBuf},
14 sync::{
15 Arc, Mutex, Weak,
16 atomic::{AtomicBool, AtomicU64, Ordering},
17 },
18 time::Duration,
19};
20
21use axum::{
22 Json, Router,
23 body::{Body, Bytes},
24 extract::{Query, Request, State},
25 http::{HeaderMap, HeaderValue, StatusCode, header::HOST},
26 middleware::{self, Next},
27 response::{
28 IntoResponse, Response,
29 sse::{Event as SseEvent, KeepAlive, Sse},
30 },
31 routing::get,
32};
33use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
34use dbgflow_protocol::{
35 CURRENT_PROJECTION_VERSION, CURRENT_SCHEMA_VERSION, EventEnvelope, EventKind, Payload,
36 PayloadEncoding, ProjectedSpan, ProjectionCheckpoint, ProjectionLimits, Sequence,
37};
38use dbgflow_storage::{
39 CURRENT_FORMAT_VERSION, DurableMetadata, DurableTailReader, Manifest, RecordingActivity,
40 RecordingReader, RecordingState, RetainedPrefix, StorageError, recording_activity,
41};
42use serde::{Deserialize, Serialize};
43use tokio::{
44 net::TcpListener,
45 sync::{mpsc, watch},
46 task::JoinHandle,
47};
48use tokio_stream::wrappers::ReceiverStream;
49
50pub const API_VERSION: u32 = 1;
51pub const HARD_MAX_PAGE_LIMIT: usize = 100_000;
54pub const HARD_MAX_PAGE_BYTES: usize = 64 * 1024 * 1024;
56const MIN_PAGE_BYTES: usize = 1024;
57const EVENT_RESPONSE_OVERHEAD_BYTES: usize = 512;
58const LAST_EVENT_ID: &str = "last-event-id";
59const LAST_DURABLE_CURSOR: &str = "x-dbgflow-last-durable-cursor";
60const MAX_LIVE_CATCHUP_EVENTS: usize = 64;
61const MAX_LIVE_CATCHUP_BYTES: usize = 1024 * 1024;
62
63#[derive(Clone, Debug)]
65pub struct ServerConfig {
66 pub bind_addr: SocketAddr,
67 pub ui_dir: Option<PathBuf>,
73 pub default_page_limit: usize,
74 pub max_page_limit: usize,
75 pub max_page_bytes: usize,
77 pub live_batch_limit: usize,
78 pub live_buffer_events: usize,
80 pub live_buffer_bytes: usize,
83 pub live_poll_interval: Duration,
84 pub keep_alive_interval: Duration,
85}
86
87impl Default for ServerConfig {
88 fn default() -> Self {
89 Self {
90 bind_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0),
91 ui_dir: None,
92 default_page_limit: 500,
93 max_page_limit: 5_000,
94 max_page_bytes: 8 * 1024 * 1024,
95 live_batch_limit: 500,
96 live_buffer_events: 128,
97 live_buffer_bytes: 8 * 1024 * 1024,
98 live_poll_interval: Duration::from_millis(200),
99 keep_alive_interval: Duration::from_secs(15),
100 }
101 }
102}
103
104impl ServerConfig {
105 fn validate(&self) -> Result<(), ServerError> {
106 if !self.bind_addr.ip().is_loopback() {
107 return Err(ServerError::InvalidConfig(
108 "bind address must be loopback".into(),
109 ));
110 }
111 if self.default_page_limit == 0 || self.default_page_limit > self.max_page_limit {
112 return Err(ServerError::InvalidConfig(
113 "default_page_limit must be in 1..=max_page_limit".into(),
114 ));
115 }
116 if self.max_page_limit == 0
117 || self.max_page_limit > HARD_MAX_PAGE_LIMIT
118 || self.live_batch_limit == 0
119 || self.live_batch_limit > self.max_page_limit
120 || self.live_buffer_events == 0
121 {
122 return Err(ServerError::InvalidConfig(format!(
123 "page limits must be non-zero and no greater than {HARD_MAX_PAGE_LIMIT}; live limits must be non-zero and bounded"
124 )));
125 }
126 if !(MIN_PAGE_BYTES..=HARD_MAX_PAGE_BYTES).contains(&self.max_page_bytes) {
127 return Err(ServerError::InvalidConfig(format!(
128 "max_page_bytes must be in {MIN_PAGE_BYTES}..={HARD_MAX_PAGE_BYTES}"
129 )));
130 }
131 if !(MIN_PAGE_BYTES..=HARD_MAX_PAGE_BYTES).contains(&self.live_buffer_bytes) {
132 return Err(ServerError::InvalidConfig(format!(
133 "live_buffer_bytes must be in {MIN_PAGE_BYTES}..={HARD_MAX_PAGE_BYTES}"
134 )));
135 }
136 if self.live_poll_interval.is_zero() || self.keep_alive_interval.is_zero() {
137 return Err(ServerError::InvalidConfig(
138 "poll and keep-alive intervals must be non-zero".into(),
139 ));
140 }
141 Ok(())
142 }
143}
144
145#[derive(Debug)]
146pub enum ServerError {
147 InvalidConfig(String),
148 UiAsset { path: PathBuf, source: io::Error },
149 Io(io::Error),
150 Storage(StorageError),
151 Task(tokio::task::JoinError),
152}
153
154impl fmt::Display for ServerError {
155 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
156 match self {
157 Self::InvalidConfig(message) => {
158 write!(formatter, "invalid server configuration: {message}")
159 }
160 Self::UiAsset { path, source } => {
161 write!(
162 formatter,
163 "cannot load required UI asset {}: {source}",
164 path.display()
165 )
166 }
167 Self::Io(error) => error.fmt(formatter),
168 Self::Storage(error) => error.fmt(formatter),
169 Self::Task(error) => write!(formatter, "server task failed: {error}"),
170 }
171 }
172}
173
174impl Error for ServerError {
175 fn source(&self) -> Option<&(dyn Error + 'static)> {
176 match self {
177 Self::Io(error) => Some(error),
178 Self::UiAsset { source, .. } => Some(source),
179 Self::Storage(error) => Some(error),
180 Self::Task(error) => Some(error),
181 Self::InvalidConfig(_) => None,
182 }
183 }
184}
185
186impl From<io::Error> for ServerError {
187 fn from(error: io::Error) -> Self {
188 Self::Io(error)
189 }
190}
191
192impl From<StorageError> for ServerError {
193 fn from(error: StorageError) -> Self {
194 Self::Storage(error)
195 }
196}
197
198pub struct Server {
200 listener: TcpListener,
201 app: Router,
202 shutdown: watch::Sender<bool>,
203 local_addr: SocketAddr,
204 live_diagnostics: LiveDiagnostics,
205}
206
207impl Server {
208 pub async fn bind(
209 config: ServerConfig,
210 recording_path: impl AsRef<Path>,
211 ) -> Result<Self, ServerError> {
212 config.validate()?;
213 RecordingReader::open(recording_path.as_ref())?;
214 let ui = UiAssets::for_config(&config)?;
215 let listener = TcpListener::bind(config.bind_addr).await?;
216 let local_addr = listener.local_addr()?;
217 let (app, shutdown, live_diagnostics) = build_router(
218 recording_path.as_ref().to_path_buf(),
219 config,
220 local_addr.port(),
221 ui,
222 );
223 Ok(Self {
224 listener,
225 app,
226 shutdown,
227 local_addr,
228 live_diagnostics,
229 })
230 }
231
232 pub fn local_addr(&self) -> SocketAddr {
233 self.local_addr
234 }
235
236 pub fn live_diagnostics(&self) -> LiveDiagnostics {
238 self.live_diagnostics.clone()
239 }
240
241 pub async fn serve(self) -> Result<(), ServerError> {
242 let mut shutdown = self.shutdown.subscribe();
243 axum::serve(self.listener, self.app)
244 .with_graceful_shutdown(async move {
245 while !*shutdown.borrow() {
246 if shutdown.changed().await.is_err() {
247 break;
248 }
249 }
250 })
251 .await?;
252 Ok(())
253 }
254
255 pub fn spawn(self) -> ServerHandle {
256 let local_addr = self.local_addr;
257 let shutdown = self.shutdown.clone();
258 let task = tokio::spawn(self.serve());
259 ServerHandle {
260 local_addr,
261 shutdown: Some(shutdown),
262 task: Some(task),
263 }
264 }
265
266 pub fn router(
268 config: ServerConfig,
269 recording_path: impl AsRef<Path>,
270 ) -> Result<Router, ServerError> {
271 config.validate()?;
272 RecordingReader::open(recording_path.as_ref())?;
273 let ui = UiAssets::for_config(&config)?;
274 let allowed_port = config.bind_addr.port();
275 Ok(build_router(
276 recording_path.as_ref().to_path_buf(),
277 config,
278 allowed_port,
279 ui,
280 )
281 .0)
282 }
283
284 pub fn router_with_diagnostics(
286 config: ServerConfig,
287 recording_path: impl AsRef<Path>,
288 ) -> Result<(Router, LiveDiagnostics), ServerError> {
289 config.validate()?;
290 RecordingReader::open(recording_path.as_ref())?;
291 let ui = UiAssets::for_config(&config)?;
292 let allowed_port = config.bind_addr.port();
293 let (router, _shutdown, diagnostics) = build_router(
294 recording_path.as_ref().to_path_buf(),
295 config,
296 allowed_port,
297 ui,
298 );
299 Ok((router, diagnostics))
300 }
301}
302
303pub struct ServerHandle {
309 local_addr: SocketAddr,
310 shutdown: Option<watch::Sender<bool>>,
311 task: Option<JoinHandle<Result<(), ServerError>>>,
312}
313
314impl ServerHandle {
315 pub fn local_addr(&self) -> SocketAddr {
316 self.local_addr
317 }
318
319 pub async fn shutdown(mut self) -> Result<(), ServerError> {
320 if let Some(shutdown) = self.shutdown.take() {
321 shutdown.send_replace(true);
322 }
323 self.task
324 .take()
325 .expect("a live server handle owns its task")
326 .await
327 .map_err(ServerError::Task)?
328 }
329}
330
331impl Drop for ServerHandle {
332 fn drop(&mut self) {
333 if let Some(shutdown) = self.shutdown.take() {
334 shutdown.send_replace(true);
335 }
336 if let Some(task) = self.task.take() {
340 task.abort();
341 }
342 }
343}
344
345#[derive(Clone)]
349pub struct LiveDiagnostics {
350 metrics: Arc<LiveMetrics>,
351}
352
353impl LiveDiagnostics {
354 pub fn tailer_starts(&self) -> u64 {
356 self.metrics.starts.load(Ordering::Acquire)
357 }
358
359 pub fn storage_polls(&self) -> u64 {
361 self.metrics.polls.load(Ordering::Acquire)
362 }
363
364 pub fn tailer_active(&self) -> bool {
366 self.metrics.active.load(Ordering::Acquire)
367 }
368
369 pub fn subscribers(&self) -> u64 {
371 self.metrics.subscribers.load(Ordering::Acquire)
372 }
373
374 pub fn completed_catchups(&self) -> u64 {
376 self.metrics.catchups.load(Ordering::Acquire)
377 }
378
379 pub fn catchup_storage_reads(&self) -> u64 {
381 self.metrics.catchup_reads.load(Ordering::Acquire)
382 }
383
384 pub fn published_events(&self) -> u64 {
386 self.metrics.published.load(Ordering::Acquire)
387 }
388
389 pub fn tail_directory_scans(&self) -> u64 {
391 self.metrics.tail_directory_scans.load(Ordering::Acquire)
392 }
393
394 pub fn tail_reinitializations(&self) -> u64 {
396 self.metrics.tail_reinitializations.load(Ordering::Acquire)
397 }
398
399 pub fn tail_frames_decoded(&self) -> u64 {
401 self.metrics.tail_frames_decoded.load(Ordering::Acquire)
402 }
403}
404
405#[derive(Default)]
406struct LiveMetrics {
407 starts: AtomicU64,
408 polls: AtomicU64,
409 active: AtomicBool,
410 subscribers: AtomicU64,
411 catchups: AtomicU64,
412 catchup_reads: AtomicU64,
413 published: AtomicU64,
414 tail_directory_scans: AtomicU64,
415 tail_reinitializations: AtomicU64,
416 tail_frames_decoded: AtomicU64,
417}
418
419struct LiveHub {
420 path: Arc<PathBuf>,
421 config: ServerConfig,
422 shutdown: watch::Sender<bool>,
423 notifications: watch::Sender<u64>,
424 state: Mutex<LiveFanoutState>,
425 started: AtomicBool,
426 metrics: Arc<LiveMetrics>,
427}
428
429#[derive(Default)]
430struct LiveFanoutState {
431 events: VecDeque<Arc<EventEnvelope>>,
432 event_bytes: usize,
433 evicted_through: Option<u64>,
434 initialized: bool,
435 tail_cursor: Option<u64>,
436 complete: bool,
437 terminal_error: Option<String>,
438}
439
440struct LiveSubscription {
441 notifications: watch::Receiver<u64>,
442 metrics: Arc<LiveMetrics>,
443 baseline: LiveBaseline,
444}
445
446impl Drop for LiveSubscription {
447 fn drop(&mut self) {
448 self.metrics.subscribers.fetch_sub(1, Ordering::Release);
449 }
450}
451
452enum LiveRead {
453 Event(Arc<EventEnvelope>),
454 Pending,
455 Complete,
456 TerminalError(String),
457 Lagged,
458}
459
460#[derive(Clone, Copy)]
461enum LiveBaseline {
462 Pending,
463 Ready(Option<u64>),
464}
465
466impl LiveHub {
467 fn new(path: Arc<PathBuf>, config: ServerConfig, shutdown: watch::Sender<bool>) -> Arc<Self> {
468 let (notifications, _) = watch::channel(0);
469 Arc::new(Self {
470 path,
471 config,
472 shutdown,
473 notifications,
474 state: Mutex::new(LiveFanoutState::default()),
475 started: AtomicBool::new(false),
476 metrics: Arc::new(LiveMetrics::default()),
477 })
478 }
479
480 fn subscribe(self: &Arc<Self>) -> LiveSubscription {
481 let notifications = self.notifications.subscribe();
484 let baseline = {
485 let state = self
486 .state
487 .lock()
488 .unwrap_or_else(std::sync::PoisonError::into_inner);
489 if state.initialized {
490 LiveBaseline::Ready(state.tail_cursor)
491 } else {
492 LiveBaseline::Pending
493 }
494 };
495 self.metrics.subscribers.fetch_add(1, Ordering::AcqRel);
496 self.ensure_started();
497 LiveSubscription {
498 notifications,
499 metrics: Arc::clone(&self.metrics),
500 baseline,
501 }
502 }
503
504 fn ensure_started(self: &Arc<Self>) {
505 if self
506 .started
507 .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
508 .is_err()
509 {
510 return;
511 }
512 self.metrics.starts.fetch_add(1, Ordering::Relaxed);
513 self.metrics.active.store(true, Ordering::Release);
514 let weak = Arc::downgrade(self);
515 let path = Arc::clone(&self.path);
516 let config = self.config.clone();
517 let shutdown = self.shutdown.subscribe();
518 let metrics = Arc::clone(&self.metrics);
519 tokio::spawn(shared_tailer(weak, path, config, shutdown, metrics));
520 }
521
522 fn publish(&self, events: Vec<EventEnvelope>) -> Result<(), String> {
523 if events.is_empty() {
524 return Ok(());
525 }
526 let event_count = events.len() as u64;
527 if let Some(event) = events
528 .iter()
529 .find(|event| live_event_weight(event) > self.config.live_buffer_bytes)
530 {
531 let weight = live_event_weight(event);
532 return Err(format!(
533 "live event {} requires {weight} bytes, exceeding live_buffer_bytes {}",
534 event.sequence.get(),
535 self.config.live_buffer_bytes
536 ));
537 }
538 let mut state = self
539 .state
540 .lock()
541 .unwrap_or_else(std::sync::PoisonError::into_inner);
542 for event in events {
543 let weight = live_event_weight(&event);
544 while state.events.len() >= self.config.live_buffer_events
545 || state.event_bytes.saturating_add(weight) > self.config.live_buffer_bytes
546 {
547 let Some(evicted) = state.events.pop_front() else {
548 break;
549 };
550 state.event_bytes = state
551 .event_bytes
552 .saturating_sub(live_event_weight(&evicted));
553 state.evicted_through = Some(evicted.sequence.get());
554 }
555 state.event_bytes = state.event_bytes.saturating_add(weight);
556 state.tail_cursor = Some(event.sequence.get());
557 state.events.push_back(Arc::new(event));
558 }
559 drop(state);
560 self.metrics
561 .published
562 .fetch_add(event_count, Ordering::Release);
563 self.notify();
564 Ok(())
565 }
566
567 fn initialize(&self, baseline_cursor: Option<u64>, complete: bool) {
568 let mut state = self
569 .state
570 .lock()
571 .unwrap_or_else(std::sync::PoisonError::into_inner);
572 state.tail_cursor = baseline_cursor;
573 state.initialized = true;
574 state.complete = complete;
575 drop(state);
576 self.notify();
577 }
578
579 fn baseline(&self) -> LiveBaseline {
580 let state = self
581 .state
582 .lock()
583 .unwrap_or_else(std::sync::PoisonError::into_inner);
584 if state.initialized {
585 LiveBaseline::Ready(state.tail_cursor)
586 } else {
587 LiveBaseline::Pending
588 }
589 }
590
591 fn mark_complete(&self) {
592 self.state
593 .lock()
594 .unwrap_or_else(std::sync::PoisonError::into_inner)
595 .complete = true;
596 self.notify();
597 }
598
599 fn fail(&self, message: String) {
600 let mut state = self
601 .state
602 .lock()
603 .unwrap_or_else(std::sync::PoisonError::into_inner);
604 if state.terminal_error.is_none() {
605 state.terminal_error = Some(message);
606 }
607 if !state.initialized {
608 state.initialized = true;
609 state.tail_cursor = None;
610 }
611 drop(state);
612 self.notify();
613 }
614
615 fn read_after(&self, cursor: Option<u64>) -> LiveRead {
616 let state = self
617 .state
618 .lock()
619 .unwrap_or_else(std::sync::PoisonError::into_inner);
620 if state
621 .evicted_through
622 .is_some_and(|evicted| cursor.is_none_or(|cursor| cursor < evicted))
623 {
624 return LiveRead::Lagged;
625 }
626 let event = state
627 .events
628 .iter()
629 .find(|event| cursor.is_none_or(|cursor| event.sequence.get() > cursor))
630 .cloned();
631 if let Some(event) = event {
632 LiveRead::Event(event)
633 } else if let Some(message) = &state.terminal_error {
634 LiveRead::TerminalError(message.clone())
635 } else if state.complete {
636 LiveRead::Complete
637 } else {
638 LiveRead::Pending
639 }
640 }
641
642 fn notify(&self) {
643 self.notifications.send_modify(|version| {
644 *version = version.wrapping_add(1);
645 });
646 }
647}
648
649struct ActiveTailer(Arc<LiveMetrics>);
650
651impl Drop for ActiveTailer {
652 fn drop(&mut self) {
653 self.0.active.store(false, Ordering::Release);
654 }
655}
656
657async fn shared_tailer(
658 hub: Weak<LiveHub>,
659 path: Arc<PathBuf>,
660 config: ServerConfig,
661 mut shutdown: watch::Receiver<bool>,
662 metrics: Arc<LiveMetrics>,
663) {
664 let _active = ActiveTailer(Arc::clone(&metrics));
665 metrics.polls.fetch_add(1, Ordering::Relaxed);
666 let initialize_path = Arc::clone(&path);
667 let initialization = blocking(move || {
668 let reader = RecordingReader::open(initialize_path.as_ref())?;
669 Ok((
670 reader.last_durable_sequence()?.map(Sequence::get),
671 reader.manifest().state == RecordingState::Complete,
672 ))
673 })
674 .await;
675 let (cursor, initially_complete) = match initialization {
676 Ok(initialization) => initialization,
677 Err(error) => {
678 if let Some(hub) = hub.upgrade() {
679 hub.fail(error.message);
680 }
681 return;
682 }
683 };
684 let Some(initial_hub) = hub.upgrade() else {
685 return;
686 };
687 initial_hub.initialize(cursor, initially_complete);
688 drop(initial_hub);
689 if initially_complete {
690 return;
691 }
692 let tail_path = Arc::clone(&path);
693 let mut tail = match blocking(move || {
694 DurableTailReader::open(tail_path.as_ref(), cursor.map(Sequence::new))
695 })
696 .await
697 {
698 Ok(tail) => tail,
699 Err(error) => {
700 if let Some(hub) = hub.upgrade() {
701 hub.fail(error.message);
702 }
703 return;
704 }
705 };
706 loop {
707 if *shutdown.borrow() {
708 break;
709 }
710 let Some(current_hub) = hub.upgrade() else {
711 break;
712 };
713 drop(current_hub);
714 metrics.polls.fetch_add(1, Ordering::Relaxed);
715 let batch_limit = config.live_batch_limit;
716 let byte_limit = config.live_buffer_bytes;
717 let activity_path = Arc::clone(&path);
718 let result = blocking(move || {
719 let page = tail.poll_bounded(batch_limit, byte_limit, live_event_weight)?;
720 let activity = if tail.state() == RecordingState::Complete {
721 RecordingActivity::Complete
722 } else {
723 recording_activity(activity_path.as_ref())?
724 };
725 let stats = tail.stats();
726 Ok((tail, page, activity, stats))
727 })
728 .await;
729 let (returned_tail, page, activity, stats) = match result {
730 Ok(result) => result,
731 Err(error) => {
732 if let Some(hub) = hub.upgrade() {
733 hub.fail(error.message);
734 }
735 break;
736 }
737 };
738 tail = returned_tail;
739 metrics
740 .tail_directory_scans
741 .store(stats.directory_scans, Ordering::Release);
742 metrics
743 .tail_reinitializations
744 .store(stats.reinitializations, Ordering::Release);
745 metrics
746 .tail_frames_decoded
747 .store(stats.frames_decoded, Ordering::Release);
748 if page.events.is_empty() {
749 if let Some(weight) = page.boundary_event_weight {
750 if let Some(hub) = hub.upgrade() {
751 hub.fail(format!(
752 "live event requires {weight} bytes, exceeding live_buffer_bytes {byte_limit}"
753 ));
754 }
755 break;
756 }
757 }
758 let has_more = page.has_more;
759 if let Some(hub) = hub.upgrade() {
760 if let Err(message) = hub.publish(page.events) {
761 hub.fail(message);
762 break;
763 }
764 if activity != RecordingActivity::Active && !has_more {
765 hub.mark_complete();
766 break;
767 }
768 } else {
769 break;
770 }
771 if has_more {
772 continue;
773 }
774 tokio::select! {
775 _ = tokio::time::sleep(config.live_poll_interval) => {},
776 changed = shutdown.changed() => {
777 if changed.is_err() || *shutdown.borrow() { break; }
778 }
779 }
780 }
781}
782
783fn live_event_weight(event: &EventEnvelope) -> usize {
784 let retained_memory = std::mem::size_of::<EventEnvelope>()
785 .saturating_add(2 * std::mem::size_of::<usize>())
786 .saturating_add(
787 event
788 .payload
789 .as_ref()
790 .map_or(0, |payload| payload.data.capacity()),
791 );
792 retained_memory.max(event_response_size_bound(event))
793}
794
795#[derive(Clone)]
796struct AppState {
797 recording_path: Arc<PathBuf>,
798 config: ServerConfig,
799 shutdown: watch::Sender<bool>,
800 live: Arc<LiveHub>,
801 ui: Option<Arc<UiAssets>>,
802}
803
804#[derive(Debug)]
805struct UiAssets {
806 index_html: Bytes,
807 app_js: Bytes,
808 app_css: Bytes,
809 globals_css: Bytes,
810}
811
812impl UiAssets {
813 fn for_config(config: &ServerConfig) -> Result<Option<Arc<Self>>, ServerError> {
814 if let Some(directory) = &config.ui_dir {
815 return Self::from_directory(directory).map(Arc::new).map(Some);
816 }
817
818 #[cfg(feature = "ui")]
819 {
820 Ok(Some(Arc::new(Self {
821 index_html: Bytes::from_static(dbgflow_ui::index_html().as_bytes()),
822 app_js: Bytes::from_static(dbgflow_ui::app_js().as_bytes()),
823 app_css: Bytes::from_static(dbgflow_ui::app_css().as_bytes()),
824 globals_css: Bytes::from_static(dbgflow_ui::globals_css().as_bytes()),
825 })))
826 }
827
828 #[cfg(not(feature = "ui"))]
829 {
830 Ok(None)
831 }
832 }
833
834 fn from_directory(directory: &Path) -> Result<Self, ServerError> {
835 let read = |name: &str| {
836 let path = directory.join(name);
837 let metadata =
838 std::fs::symlink_metadata(&path).map_err(|source| ServerError::UiAsset {
839 path: path.clone(),
840 source,
841 })?;
842 if !metadata.file_type().is_file() {
843 return Err(ServerError::UiAsset {
844 path,
845 source: io::Error::new(
846 io::ErrorKind::InvalidInput,
847 "expected a regular file (symlinks and directories are rejected)",
848 ),
849 });
850 }
851 std::fs::read(&path)
852 .map(Bytes::from)
853 .map_err(|source| ServerError::UiAsset { path, source })
854 };
855
856 Ok(Self {
857 index_html: read("index.html")?,
858 app_js: read("app.js")?,
859 app_css: read("app.css")?,
860 globals_css: read("globals.css")?,
861 })
862 }
863}
864
865#[derive(Clone, Copy)]
866struct HostPolicy {
867 port: u16,
868}
869
870fn build_router(
871 path: PathBuf,
872 config: ServerConfig,
873 allowed_port: u16,
874 ui: Option<Arc<UiAssets>>,
875) -> (Router, watch::Sender<bool>, LiveDiagnostics) {
876 let (shutdown, _) = watch::channel(false);
877 let live_hub = LiveHub::new(Arc::new(path.clone()), config.clone(), shutdown.clone());
878 let live_diagnostics = LiveDiagnostics {
879 metrics: Arc::clone(&live_hub.metrics),
880 };
881 let state = AppState {
882 recording_path: Arc::new(path),
883 config,
884 shutdown: shutdown.clone(),
885 live: live_hub,
886 ui: ui.clone(),
887 };
888 let router = Router::new()
889 .route("/api/v1/capabilities", get(capabilities))
890 .route("/api/v1/recording", get(recording))
891 .route("/api/v1/events", get(events))
892 .route("/api/v1/replay", get(replay))
893 .route("/api/v1/live", get(live));
894 let router = if ui.is_some() {
895 router
896 .route("/", get(ui_index))
897 .route("/app.js", get(ui_javascript))
898 .route("/app.css", get(ui_css))
899 .route("/globals.css", get(ui_globals_css))
900 } else {
901 router
902 };
903 let router = router
904 .with_state(state)
905 .layer(middleware::from_fn(security_headers))
906 .layer(middleware::from_fn_with_state(
907 HostPolicy { port: allowed_port },
908 enforce_loopback_host,
909 ));
910 (router, shutdown, live_diagnostics)
911}
912
913async fn enforce_loopback_host(
914 State(policy): State<HostPolicy>,
915 request: Request,
916 next: Next,
917) -> Response {
918 let mut values = request.headers().get_all(HOST).iter();
919 let host = values.next().and_then(|value| value.to_str().ok());
920 let single_host = host.is_some() && values.next().is_none();
921 if !single_host || !host.is_some_and(|value| allowed_host(value, policy.port)) {
922 return secure_response(
923 (
924 StatusCode::MISDIRECTED_REQUEST,
925 Json(ErrorResponse {
926 error: "Host must name the loopback dbgflow listener".into(),
927 }),
928 )
929 .into_response(),
930 );
931 }
932 next.run(request).await
933}
934
935fn allowed_host(value: &str, port: u16) -> bool {
936 let value = value.to_ascii_lowercase();
937 value == format!("localhost:{port}")
938 || value == format!("127.0.0.1:{port}")
939 || value == format!("[::1]:{port}")
940}
941
942async fn security_headers(request: Request, next: Next) -> Response {
943 secure_response(next.run(request).await)
944}
945
946fn secure_response(mut response: Response) -> Response {
947 let headers = response.headers_mut();
948 headers.insert(
949 "content-security-policy",
950 HeaderValue::from_static(
951 "default-src 'none'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; font-src 'self'; worker-src 'self'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'",
952 ),
953 );
954 headers.insert(
955 "x-content-type-options",
956 HeaderValue::from_static("nosniff"),
957 );
958 headers.insert("x-frame-options", HeaderValue::from_static("DENY"));
959 headers.insert("referrer-policy", HeaderValue::from_static("no-referrer"));
960 headers.insert(
961 "permissions-policy",
962 HeaderValue::from_static("camera=(), microphone=(), geolocation=()"),
963 );
964 headers.insert(
965 "cross-origin-opener-policy",
966 HeaderValue::from_static("same-origin"),
967 );
968 headers.insert(
969 "cross-origin-resource-policy",
970 HeaderValue::from_static("same-origin"),
971 );
972 response
973}
974
975async fn ui_index(State(state): State<AppState>) -> Response {
976 ui_asset_response(
977 state
978 .ui
979 .expect("UI routes require loaded assets")
980 .index_html
981 .clone(),
982 "text/html; charset=utf-8",
983 )
984}
985
986async fn ui_javascript(State(state): State<AppState>) -> Response {
987 ui_asset_response(
988 state
989 .ui
990 .expect("UI routes require loaded assets")
991 .app_js
992 .clone(),
993 "application/javascript; charset=utf-8",
994 )
995}
996
997async fn ui_css(State(state): State<AppState>) -> Response {
998 ui_asset_response(
999 state
1000 .ui
1001 .expect("UI routes require loaded assets")
1002 .app_css
1003 .clone(),
1004 "text/css; charset=utf-8",
1005 )
1006}
1007
1008async fn ui_globals_css(State(state): State<AppState>) -> Response {
1009 ui_asset_response(
1010 state
1011 .ui
1012 .expect("UI routes require loaded assets")
1013 .globals_css
1014 .clone(),
1015 "text/css; charset=utf-8",
1016 )
1017}
1018
1019fn ui_asset_response(bytes: Bytes, content_type: &'static str) -> Response {
1020 Response::builder()
1021 .header("content-type", content_type)
1022 .body(Body::from(bytes))
1023 .expect("static UI response headers are valid")
1024}
1025
1026#[derive(Serialize)]
1027struct CapabilitiesResponse {
1028 api_version: u32,
1029 schema_version: u32,
1030 format_version: u32,
1031 transports: [&'static str; 1],
1032 cursor: CursorCapabilities,
1033 pagination: PaginationCapabilities,
1034 retention: RetentionCapabilities,
1035 replay: ReplayCapabilities,
1036}
1037
1038#[derive(Serialize)]
1039struct CursorCapabilities {
1040 kind: &'static str,
1041 semantics: &'static str,
1042 durability: &'static str,
1043}
1044
1045#[derive(Serialize)]
1046struct PaginationCapabilities {
1047 default_limit: usize,
1048 max_limit: usize,
1049 max_response_bytes: usize,
1050 directions: [&'static str; 2],
1051}
1052
1053#[derive(Serialize)]
1054struct RetentionCapabilities {
1055 rolling: bool,
1056 notice_event: &'static str,
1057}
1058
1059#[derive(Serialize)]
1060struct ReplayCapabilities {
1061 checkpoint_projection_version: u32,
1062 endpoint: &'static str,
1063 semantics: &'static str,
1064}
1065
1066async fn capabilities(State(state): State<AppState>) -> Json<CapabilitiesResponse> {
1067 Json(CapabilitiesResponse {
1068 api_version: API_VERSION,
1069 schema_version: CURRENT_SCHEMA_VERSION,
1070 format_version: CURRENT_FORMAT_VERSION,
1071 transports: ["sse"],
1072 cursor: CursorCapabilities {
1073 kind: "sequence",
1074 semantics: "exclusive",
1075 durability: "synced_frames",
1076 },
1077 pagination: PaginationCapabilities {
1078 default_limit: state.config.default_page_limit,
1079 max_limit: state.config.max_page_limit,
1080 max_response_bytes: state.config.max_page_bytes,
1081 directions: ["forward", "backward"],
1082 },
1083 retention: RetentionCapabilities {
1084 rolling: true,
1085 notice_event: "dbgflow.retention",
1086 },
1087 replay: ReplayCapabilities {
1088 checkpoint_projection_version: CURRENT_PROJECTION_VERSION,
1089 endpoint: "/api/v1/replay",
1090 semantics: "nearest_checkpoint_plus_deltas",
1091 },
1092 })
1093}
1094
1095#[derive(Serialize)]
1096struct RecordingResponse {
1097 format_version: u32,
1098 schema_version: u32,
1099 recording_id: String,
1100 state: RecordingState,
1101 activity: RecordingActivity,
1102 created_unix_ms: Option<i64>,
1103 metadata: BTreeMap<String, String>,
1104 durable: DurableResponse,
1105 committed: DurableResponse,
1107}
1108
1109#[derive(Clone, Serialize)]
1110struct DurableResponse {
1111 immutable_chunk_count: u64,
1112 has_active_prefix: bool,
1113 event_count: u64,
1114 first_cursor: Option<String>,
1115 last_cursor: Option<String>,
1116 retained_prefix: RetainedPrefixResponse,
1117}
1118
1119#[derive(Clone, Serialize)]
1120struct RetainedPrefixResponse {
1121 history_truncated: bool,
1122 generation: String,
1123 first_chunk_number: String,
1124 first_cursor: Option<String>,
1125 removed_through_cursor: Option<String>,
1126}
1127
1128async fn recording(State(state): State<AppState>) -> Result<Json<RecordingResponse>, ApiError> {
1129 let path = state.recording_path.clone();
1130 let (manifest, durable, activity) = blocking(move || {
1131 let reader = RecordingReader::open(path.as_ref())?;
1132 let durable = reader.durable_metadata()?;
1133 let activity = recording_activity(path.as_ref())?;
1134 Ok((reader.manifest().clone(), durable, activity))
1135 })
1136 .await?;
1137 Ok(Json(recording_response(manifest, durable, activity)))
1138}
1139
1140#[derive(Default, Deserialize)]
1141struct EventsQuery {
1142 after: Option<u64>,
1143 before: Option<u64>,
1144 limit: Option<usize>,
1145}
1146
1147#[derive(Serialize)]
1148struct EventsResponse {
1149 events: Vec<EventResponse>,
1150 next_cursor: Option<String>,
1151 previous_cursor: Option<String>,
1152 last_durable_cursor: Option<String>,
1153 has_more: bool,
1154 retained_prefix: RetainedPrefixResponse,
1155}
1156
1157async fn events(
1158 State(state): State<AppState>,
1159 Query(query): Query<EventsQuery>,
1160) -> Result<Json<EventsResponse>, ApiError> {
1161 let limit = query.limit.unwrap_or(state.config.default_page_limit);
1162 validate_limit(limit, state.config.max_page_limit)?;
1163 if query.after.is_some() && query.before.is_some() {
1164 return Err(ApiError::bad_request(
1165 "after and before cursors are mutually exclusive",
1166 ));
1167 }
1168 let direction = if let Some(before) = query.before {
1169 PageDirection::Backward(Sequence::new(before))
1170 } else {
1171 PageDirection::Forward(query.after.map(Sequence::new))
1172 };
1173 let path = state.recording_path.clone();
1174 let max_page_bytes = state.config.max_page_bytes;
1175 let snapshot = blocking(move || {
1176 let reader = RecordingReader::open(path.as_ref())?;
1177 for _ in 0..8 {
1178 let retained_before = reader.retained_prefix()?;
1179 let page = match direction {
1180 PageDirection::Forward(after) => reader.read_after_bounded(
1181 after,
1182 limit,
1183 max_page_bytes,
1184 event_response_size_bound,
1185 )?,
1186 PageDirection::Backward(before) => reader.read_before_bounded(
1187 Some(before),
1188 limit,
1189 max_page_bytes,
1190 event_response_size_bound,
1191 )?,
1192 };
1193 let last_durable_sequence = reader.last_durable_sequence()?;
1194 let retained_after = reader.retained_prefix()?;
1195 if retained_before == retained_after {
1196 return Ok(Some((page, last_durable_sequence, retained_after)));
1197 }
1198 }
1199 Ok(None)
1200 })
1201 .await?;
1202 let (page, last_durable_sequence, retained_prefix) = snapshot.ok_or_else(|| {
1203 ApiError::internal(
1204 "rolling retention changed repeatedly while capturing a coherent history page; retry",
1205 )
1206 })?;
1207 if page.events.is_empty() && page.boundary_event_weight.is_some() {
1208 return Err(ApiError::payload_too_large(format!(
1209 "boundary event exceeds the configured page response budget of {max_page_bytes} bytes"
1210 )));
1211 }
1212 let next_cursor = if query.before.is_none() {
1213 page.events
1214 .last()
1215 .map(|event| event.sequence.get().to_string())
1216 .or_else(|| query.after.map(|cursor| cursor.to_string()))
1217 } else {
1218 None
1219 };
1220 let previous_cursor = if query.before.is_some() {
1221 page.events
1222 .first()
1223 .map(|event| event.sequence.get().to_string())
1224 } else {
1225 None
1226 };
1227 Ok(Json(EventsResponse {
1228 events: page.events.iter().map(EventResponse::from).collect(),
1229 next_cursor,
1230 previous_cursor,
1231 last_durable_cursor: last_durable_sequence.map(|value| value.get().to_string()),
1232 has_more: page.has_more,
1233 retained_prefix: retained_prefix_response(retained_prefix),
1234 }))
1235}
1236
1237#[derive(Clone, Copy)]
1238enum PageDirection {
1239 Forward(Option<Sequence>),
1240 Backward(Sequence),
1241}
1242
1243#[derive(Deserialize)]
1244struct ReplayQuery {
1245 cursor: u64,
1246 boundary: Option<u64>,
1247 after: Option<u64>,
1248 limit: Option<usize>,
1249}
1250
1251#[derive(Serialize)]
1252struct ReplayResponse {
1253 requested_cursor: String,
1254 resolved_cursor: String,
1255 checkpoint: Option<CheckpointResponse>,
1256 deltas: Vec<EventResponse>,
1257 has_more: bool,
1258 next_cursor: Option<String>,
1259 exact: bool,
1260 retained_prefix: RetainedPrefixResponse,
1261 ignored_checkpoints: Vec<CheckpointIssueResponse>,
1262}
1263
1264#[derive(Serialize)]
1265struct CheckpointResponse {
1266 projection_version: u32,
1267 recording_id: String,
1268 durable_cursor: String,
1269 complete: bool,
1270 active_spans: Vec<ProjectedSpanResponse>,
1271 known_spans: Vec<ProjectedSpanResponse>,
1272}
1273
1274#[derive(Serialize)]
1275struct ProjectedSpanResponse {
1276 context: ContextResponse,
1277 opened_cursor: String,
1278 enter_depth: u32,
1279 is_open: bool,
1280 metadata: Option<PayloadResponse>,
1281}
1282
1283#[derive(Serialize)]
1284struct CheckpointIssueResponse {
1285 file: String,
1286 reason: String,
1287}
1288
1289async fn replay(
1290 State(state): State<AppState>,
1291 Query(query): Query<ReplayQuery>,
1292) -> Result<Json<ReplayResponse>, ApiError> {
1293 let limit = query.limit.unwrap_or(state.config.default_page_limit);
1294 validate_limit(limit, state.config.max_page_limit)?;
1295 if query.after.is_some_and(|after| after > query.cursor) {
1296 return Err(ApiError::bad_request(
1297 "replay after cursor must not exceed requested cursor",
1298 ));
1299 }
1300 if query
1301 .boundary
1302 .is_some_and(|boundary| boundary > query.cursor)
1303 {
1304 return Err(ApiError::bad_request(
1305 "replay boundary cursor must not exceed requested cursor",
1306 ));
1307 }
1308 let requested = Sequence::new(query.cursor);
1309 let path = state.recording_path.clone();
1310 let max_page_bytes = state.config.max_page_bytes;
1311 let captured = blocking(move || {
1312 let reader = RecordingReader::open(path.as_ref())?;
1313 for _ in 0..8 {
1314 let retained_before = reader.retained_prefix()?;
1315 let last = reader.last_durable_sequence()?.unwrap_or(Sequence::new(0));
1316 let resolved = Sequence::new(requested.get().min(last.get()));
1317 let lookup_cursor = Sequence::new(query.boundary.unwrap_or(resolved.get()));
1318 let lookup = if query.boundary.is_some() {
1319 reader.reconstruct_checkpoint_at(lookup_cursor, ProjectionLimits::default())?
1320 } else {
1321 reader.nearest_checkpoint_at_or_before(lookup_cursor)?
1322 };
1323 let checkpoint_after = lookup
1324 .checkpoint
1325 .as_ref()
1326 .map(|checkpoint| checkpoint.durable_cursor);
1327 let after = match (query.after, checkpoint_after) {
1328 (Some(after), Some(checkpoint)) => Some(Sequence::new(after.max(checkpoint.get()))),
1329 (Some(after), None) => Some(Sequence::new(after)),
1330 (None, checkpoint) => checkpoint,
1331 };
1332 let page = reader.read_range_bounded(
1333 after,
1334 resolved,
1335 limit,
1336 max_page_bytes,
1337 event_response_size_bound,
1338 )?;
1339 let retained_after = reader.retained_prefix()?;
1340 if retained_before == retained_after {
1341 return Ok(Some((resolved, lookup, page, retained_after)));
1342 }
1343 }
1344 Ok(None)
1345 })
1346 .await?;
1347 let (resolved, lookup, page, retained) = captured.ok_or_else(|| {
1348 ApiError::internal(
1349 "rolling retention changed repeatedly while capturing a coherent replay boundary; retry",
1350 )
1351 })?;
1352 if page.events.is_empty() && page.boundary_event_weight.is_some() {
1353 return Err(ApiError::payload_too_large(format!(
1354 "boundary replay event exceeds the configured response budget of {max_page_bytes} bytes"
1355 )));
1356 }
1357 let replayed_through = page
1358 .events
1359 .last()
1360 .map(|event| event.sequence.get())
1361 .or_else(|| {
1362 lookup
1363 .checkpoint
1364 .as_ref()
1365 .map(|checkpoint| checkpoint.durable_cursor.get())
1366 });
1367 let checkpoint_complete = lookup
1368 .checkpoint
1369 .as_ref()
1370 .is_none_or(|checkpoint| checkpoint.complete);
1371 let exact = checkpoint_complete
1372 && !page.has_more
1373 && replayed_through.is_some_and(|cursor| cursor == resolved.get())
1374 && !retention_notice_required(replayed_through, retained);
1375 let next_cursor = page
1376 .has_more
1377 .then(|| {
1378 page.events
1379 .last()
1380 .map(|event| event.sequence.get().to_string())
1381 })
1382 .flatten();
1383 Ok(Json(ReplayResponse {
1384 requested_cursor: requested.get().to_string(),
1385 resolved_cursor: resolved.get().to_string(),
1386 checkpoint: lookup.checkpoint.as_ref().map(CheckpointResponse::from),
1387 deltas: page.events.iter().map(EventResponse::from).collect(),
1388 has_more: page.has_more,
1389 next_cursor,
1390 exact,
1391 retained_prefix: retained_prefix_response(retained),
1392 ignored_checkpoints: lookup
1393 .ignored
1394 .into_iter()
1395 .map(|issue| CheckpointIssueResponse {
1396 file: issue
1397 .path
1398 .file_name()
1399 .map_or_else(String::new, |name| name.to_string_lossy().into_owned()),
1400 reason: issue.reason,
1401 })
1402 .collect(),
1403 }))
1404}
1405
1406#[derive(Default, Deserialize)]
1407struct LiveQuery {
1408 after: Option<u64>,
1409}
1410
1411async fn live(
1412 State(state): State<AppState>,
1413 Query(query): Query<LiveQuery>,
1414 headers: HeaderMap,
1415) -> Result<Response, ApiError> {
1416 let after = parse_last_event_id(&headers)?.or(query.after);
1420 let subscription = state.live.subscribe();
1421 let path = state.recording_path.clone();
1422 let (last_durable_sequence, retained_prefix) = blocking({
1423 let path = path.clone();
1424 move || {
1425 let reader = RecordingReader::open(path.as_ref())?;
1426 Ok((reader.last_durable_sequence()?, reader.retained_prefix()?))
1427 }
1428 })
1429 .await?;
1430
1431 let (sender, receiver) = mpsc::channel(1);
1434 let mut shutdown = state.shutdown.subscribe();
1435 let hub = Arc::clone(&state.live);
1436 tokio::spawn(async move {
1437 follow_shared(
1438 hub,
1439 after,
1440 retained_prefix,
1441 subscription,
1442 &mut shutdown,
1443 sender,
1444 )
1445 .await;
1446 });
1447
1448 let stream = ReceiverStream::new(receiver);
1449 let mut response = Sse::new(stream)
1450 .keep_alive(
1451 KeepAlive::new()
1452 .interval(state.config.keep_alive_interval)
1453 .text("keep-alive"),
1454 )
1455 .into_response();
1456 response.headers_mut().insert(
1457 LAST_DURABLE_CURSOR,
1458 HeaderValue::from_str(
1459 &last_durable_sequence.map_or_else(|| "".into(), |value| value.get().to_string()),
1460 )
1461 .expect("a decimal cursor is a valid header value"),
1462 );
1463 Ok(response)
1464}
1465
1466async fn follow_shared(
1467 hub: Arc<LiveHub>,
1468 after: Option<u64>,
1469 initial_retained_prefix: RetainedPrefix,
1470 mut subscription: LiveSubscription,
1471 shutdown: &mut watch::Receiver<bool>,
1472 sender: mpsc::Sender<Result<SseEvent, Infallible>>,
1473) {
1474 let mut cursor = after;
1475 let mut last_notice_generation = None;
1476 if retention_notice_required(cursor, initial_retained_prefix) {
1477 if !send_retention_notice(
1478 &sender,
1479 shutdown,
1480 &retained_prefix_response(initial_retained_prefix),
1481 )
1482 .await
1483 {
1484 return;
1485 }
1486 last_notice_generation = Some(initial_retained_prefix.generation);
1487 advance_cursor_to_retained_boundary(&mut cursor, initial_retained_prefix);
1488 }
1489 let baseline = loop {
1490 match subscription.baseline {
1491 LiveBaseline::Ready(cursor) => break cursor,
1492 LiveBaseline::Pending => {
1493 tokio::select! {
1494 changed = subscription.notifications.changed() => {
1495 if changed.is_err() { return; }
1496 },
1497 changed = shutdown.changed() => {
1498 if changed.is_err() || *shutdown.borrow() { return; }
1499 },
1500 _ = sender.closed() => return,
1501 }
1502 subscription.baseline = hub.baseline();
1503 }
1504 }
1505 };
1506 while baseline.is_some_and(|baseline| cursor.is_none_or(|cursor| cursor < baseline)) {
1510 let read_path = Arc::clone(&hub.path);
1511 let read_after = cursor.map(Sequence::new);
1512 let batch_limit = hub.config.live_batch_limit.min(MAX_LIVE_CATCHUP_EVENTS);
1513 let byte_limit = hub.config.live_buffer_bytes.min(MAX_LIVE_CATCHUP_BYTES);
1514 hub.metrics.catchup_reads.fetch_add(1, Ordering::Relaxed);
1515 let result = blocking(move || {
1516 let reader = RecordingReader::open(read_path.as_ref())?;
1517 let page = reader.read_after_bounded(
1518 read_after,
1519 batch_limit,
1520 byte_limit,
1521 live_event_weight,
1522 )?;
1523 let retained_prefix = reader.retained_prefix()?;
1524 Ok((page, retained_prefix))
1525 })
1526 .await;
1527 let (mut page, mut page_retained_prefix) = match result {
1528 Ok(result) => result,
1529 Err(error) => {
1530 send_live_error(&sender, shutdown, error.message).await;
1531 return;
1532 }
1533 };
1534 if page.events.is_empty()
1538 && page
1539 .boundary_event_weight
1540 .is_some_and(|weight| weight <= hub.config.live_buffer_bytes)
1541 && byte_limit < hub.config.live_buffer_bytes
1542 {
1543 let read_path = Arc::clone(&hub.path);
1544 let live_buffer_bytes = hub.config.live_buffer_bytes;
1545 hub.metrics.catchup_reads.fetch_add(1, Ordering::Relaxed);
1546 let result = blocking(move || {
1547 let reader = RecordingReader::open(read_path.as_ref())?;
1548 let page = reader.read_after_bounded(
1549 read_after,
1550 1,
1551 live_buffer_bytes,
1552 live_event_weight,
1553 )?;
1554 let retained_prefix = reader.retained_prefix()?;
1555 Ok((page, retained_prefix))
1556 })
1557 .await;
1558 (page, page_retained_prefix) = match result {
1559 Ok(result) => result,
1560 Err(error) => {
1561 send_live_error(&sender, shutdown, error.message).await;
1562 return;
1563 }
1564 };
1565 }
1566 if retention_notice_required(cursor, page_retained_prefix)
1567 && last_notice_generation != Some(page_retained_prefix.generation)
1568 {
1569 if !send_retention_notice(
1570 &sender,
1571 shutdown,
1572 &retained_prefix_response(page_retained_prefix),
1573 )
1574 .await
1575 {
1576 return;
1577 }
1578 last_notice_generation = Some(page_retained_prefix.generation);
1579 advance_cursor_to_retained_boundary(&mut cursor, page_retained_prefix);
1580 }
1581 for envelope in &page.events {
1582 if baseline.is_some_and(|baseline| envelope.sequence.get() > baseline) {
1583 break;
1584 }
1585 if cursor.is_some_and(|cursor| envelope.sequence.get() <= cursor) {
1586 continue;
1587 }
1588 if !send_live_event(&sender, shutdown, envelope).await {
1589 return;
1590 }
1591 cursor = Some(envelope.sequence.get());
1592 }
1593 if cursor.is_some_and(|cursor| baseline.is_some_and(|baseline| cursor >= baseline)) {
1594 break;
1595 }
1596 if page.events.is_empty() {
1597 if let Some(weight) = page.boundary_event_weight {
1598 send_live_error(
1599 &sender,
1600 shutdown,
1601 format!(
1602 "live event requires {weight} bytes, exceeding live_buffer_bytes {}",
1603 hub.config.live_buffer_bytes
1604 ),
1605 )
1606 .await;
1607 return;
1608 }
1609 send_live_error(
1610 &sender,
1611 shutdown,
1612 "durable history ended before the shared live baseline".into(),
1613 )
1614 .await;
1615 return;
1616 }
1617 }
1618
1619 hub.metrics.catchups.fetch_add(1, Ordering::Relaxed);
1620 loop {
1621 match hub.read_after(cursor) {
1622 LiveRead::Event(envelope) => {
1623 if cursor.is_some_and(|cursor| envelope.sequence.get() <= cursor) {
1624 continue;
1625 }
1626 if !send_live_event(&sender, shutdown, &envelope).await {
1627 return;
1628 }
1629 cursor = Some(envelope.sequence.get());
1630 }
1631 LiveRead::Pending => {
1632 tokio::select! {
1633 changed = subscription.notifications.changed() => {
1634 if changed.is_err() { return; }
1635 },
1636 changed = shutdown.changed() => {
1637 if changed.is_err() || *shutdown.borrow() { return; }
1638 },
1639 _ = sender.closed() => return,
1640 }
1641 }
1642 LiveRead::Complete | LiveRead::Lagged => return,
1643 LiveRead::TerminalError(message) => {
1644 send_live_error(&sender, shutdown, message).await;
1645 return;
1646 }
1647 }
1648 }
1649}
1650
1651async fn send_live_event(
1652 sender: &mpsc::Sender<Result<SseEvent, Infallible>>,
1653 shutdown: &mut watch::Receiver<bool>,
1654 envelope: &EventEnvelope,
1655) -> bool {
1656 let permit = tokio::select! {
1657 permit = sender.reserve() => match permit {
1658 Ok(permit) => permit,
1659 Err(_) => return false,
1660 },
1661 _ = shutdown.changed() => return false,
1662 };
1663 let name = if envelope.kind == EventKind::Gap {
1664 "dbgflow.gap"
1665 } else {
1666 "dbgflow.event"
1667 };
1668 let event = SseEvent::default()
1669 .event(name)
1670 .id(envelope.sequence.get().to_string())
1671 .json_data(EventResponse::from(envelope))
1672 .expect("event response always serializes");
1673 permit.send(Ok(event));
1674 true
1675}
1676
1677async fn send_retention_notice(
1678 sender: &mpsc::Sender<Result<SseEvent, Infallible>>,
1679 shutdown: &mut watch::Receiver<bool>,
1680 notice: &RetainedPrefixResponse,
1681) -> bool {
1682 let permit = tokio::select! {
1683 permit = sender.reserve() => match permit {
1684 Ok(permit) => permit,
1685 Err(_) => return false,
1686 },
1687 _ = shutdown.changed() => return false,
1688 };
1689 let event = SseEvent::default()
1690 .event("dbgflow.retention")
1691 .json_data(notice)
1692 .expect("retention notice always serializes");
1693 permit.send(Ok(event));
1694 true
1695}
1696
1697async fn send_live_error(
1698 sender: &mpsc::Sender<Result<SseEvent, Infallible>>,
1699 shutdown: &mut watch::Receiver<bool>,
1700 message: String,
1701) {
1702 let permit = tokio::select! {
1703 permit = sender.reserve() => match permit {
1704 Ok(permit) => permit,
1705 Err(_) => return,
1706 },
1707 _ = shutdown.changed() => return,
1708 };
1709 let data = serde_json::json!({"error": message});
1710 let event = SseEvent::default()
1711 .event("dbgflow.error")
1712 .data(data.to_string());
1713 permit.send(Ok(event));
1714}
1715
1716fn parse_last_event_id(headers: &HeaderMap) -> Result<Option<u64>, ApiError> {
1717 let Some(value) = headers.get(LAST_EVENT_ID) else {
1718 return Ok(None);
1719 };
1720 let value = value
1721 .to_str()
1722 .map_err(|_| ApiError::bad_request("Last-Event-ID must be ASCII"))?;
1723 value
1724 .parse()
1725 .map(Some)
1726 .map_err(|_| ApiError::bad_request("Last-Event-ID must be an unsigned integer"))
1727}
1728
1729fn validate_limit(limit: usize, maximum: usize) -> Result<(), ApiError> {
1730 if limit == 0 || limit > maximum {
1731 Err(ApiError::bad_request(format!(
1732 "limit must be in 1..={maximum}"
1733 )))
1734 } else {
1735 Ok(())
1736 }
1737}
1738
1739async fn blocking<T: Send + 'static>(
1740 operation: impl FnOnce() -> Result<T, StorageError> + Send + 'static,
1741) -> Result<T, ApiError> {
1742 tokio::task::spawn_blocking(operation)
1743 .await
1744 .map_err(|error| ApiError::internal(format!("storage task failed: {error}")))?
1745 .map_err(ApiError::from)
1746}
1747
1748fn recording_response(
1749 manifest: Manifest,
1750 durable: DurableMetadata,
1751 activity: RecordingActivity,
1752) -> RecordingResponse {
1753 let retained = durable.retained_prefix;
1754 let durable = DurableResponse {
1755 immutable_chunk_count: durable.chunk_count,
1756 has_active_prefix: durable.has_active_prefix,
1757 event_count: durable.event_count,
1758 first_cursor: durable.first_sequence.map(|value| value.get().to_string()),
1759 last_cursor: durable.last_sequence.map(|value| value.get().to_string()),
1760 retained_prefix: retained_prefix_response(retained),
1761 };
1762 RecordingResponse {
1763 format_version: manifest.format_version,
1764 schema_version: manifest.schema_version,
1765 recording_id: manifest.recording_id,
1766 state: manifest.state,
1767 activity,
1768 created_unix_ms: manifest.created_unix_ms,
1769 metadata: manifest.metadata,
1770 durable: durable.clone(),
1771 committed: durable,
1772 }
1773}
1774
1775fn retained_prefix_response(retained: RetainedPrefix) -> RetainedPrefixResponse {
1776 RetainedPrefixResponse {
1777 history_truncated: retained.history_truncated,
1778 generation: retained.generation.to_string(),
1779 first_chunk_number: retained.first_chunk_number.to_string(),
1780 first_cursor: retained.first_sequence.map(|value| value.get().to_string()),
1781 removed_through_cursor: retained
1782 .removed_through_sequence
1783 .map(|value| value.get().to_string()),
1784 }
1785}
1786
1787fn retention_notice_required(after: Option<u64>, retained: RetainedPrefix) -> bool {
1788 if !retained.history_truncated {
1789 return false;
1790 }
1791 if let Some(removed) = retained.removed_through_sequence {
1792 return after.is_none_or(|after| after <= removed.get());
1793 }
1794 match retained.first_sequence {
1795 Some(first) => after.is_none_or(|after| after < first.get()),
1796 None => true,
1797 }
1798}
1799
1800fn advance_cursor_to_retained_boundary(cursor: &mut Option<u64>, retained: RetainedPrefix) {
1801 let boundary = retained
1802 .removed_through_sequence
1803 .map(Sequence::get)
1804 .or_else(|| {
1805 retained
1806 .first_sequence
1807 .and_then(|first| first.get().checked_sub(1))
1808 });
1809 if let Some(boundary) = boundary {
1810 *cursor = Some(cursor.map_or(boundary, |cursor| cursor.max(boundary)));
1811 }
1812}
1813
1814#[derive(Serialize)]
1815struct EventResponse {
1816 schema_version: u32,
1817 recording_id: String,
1818 sequence: String,
1819 timestamp: TimestampResponse,
1820 context: ContextResponse,
1821 kind: String,
1822 payload: Option<PayloadResponse>,
1823}
1824
1825#[derive(Serialize)]
1826struct TimestampResponse {
1827 monotonic_ns: String,
1828 wall_time_unix_ns: Option<String>,
1829}
1830
1831#[derive(Serialize)]
1832struct ContextResponse {
1833 process_id: u32,
1834 thread_id: String,
1835 flow_id: Option<String>,
1836 span_id: Option<String>,
1837 parent_span_id: Option<String>,
1838}
1839
1840#[derive(Serialize)]
1841struct PayloadResponse {
1842 encoding: String,
1843 flags: u32,
1844 data_base64: String,
1845}
1846
1847impl From<&EventEnvelope> for EventResponse {
1848 fn from(event: &EventEnvelope) -> Self {
1849 Self {
1850 schema_version: event.schema_version.get(),
1851 recording_id: format!("{:032x}", event.recording_id.as_u128()),
1852 sequence: event.sequence.get().to_string(),
1853 timestamp: TimestampResponse {
1854 monotonic_ns: event.timestamp.monotonic_ns.to_string(),
1855 wall_time_unix_ns: event
1856 .timestamp
1857 .wall_time_unix_ns
1858 .map(|value| value.to_string()),
1859 },
1860 context: context_response(event.context),
1861 kind: event_kind(event.kind),
1862 payload: event.payload.as_ref().map(payload_response),
1863 }
1864 }
1865}
1866
1867impl From<&ProjectionCheckpoint> for CheckpointResponse {
1868 fn from(checkpoint: &ProjectionCheckpoint) -> Self {
1869 Self {
1870 projection_version: checkpoint.projection_version,
1871 recording_id: format!("{:032x}", checkpoint.recording_id.as_u128()),
1872 durable_cursor: checkpoint.durable_cursor.get().to_string(),
1873 complete: checkpoint.complete,
1874 active_spans: checkpoint
1875 .active_spans
1876 .iter()
1877 .map(ProjectedSpanResponse::from)
1878 .collect(),
1879 known_spans: checkpoint
1880 .known_spans
1881 .iter()
1882 .map(ProjectedSpanResponse::from)
1883 .collect(),
1884 }
1885 }
1886}
1887
1888impl From<&ProjectedSpan> for ProjectedSpanResponse {
1889 fn from(span: &ProjectedSpan) -> Self {
1890 Self {
1891 context: context_response(span.context),
1892 opened_cursor: span.opened_sequence.get().to_string(),
1893 enter_depth: span.enter_depth,
1894 is_open: span.is_open,
1895 metadata: span.metadata.as_ref().map(payload_response),
1896 }
1897 }
1898}
1899
1900fn context_response(context: dbgflow_protocol::EventContext) -> ContextResponse {
1901 ContextResponse {
1902 process_id: context.process_id.get(),
1903 thread_id: context.thread_id.get().to_string(),
1904 flow_id: context.flow_id.map(|value| value.get().to_string()),
1905 span_id: context.span_id.map(|value| value.get().to_string()),
1906 parent_span_id: context.parent_span_id.map(|value| value.get().to_string()),
1907 }
1908}
1909
1910fn payload_response(payload: &Payload) -> PayloadResponse {
1911 PayloadResponse {
1912 encoding: payload_encoding(payload.encoding),
1913 flags: payload.flags.bits(),
1914 data_base64: BASE64.encode(&payload.data),
1915 }
1916}
1917
1918fn event_response_size_bound(event: &EventEnvelope) -> usize {
1919 let encoded_payload = event.payload.as_ref().map_or(0, |payload| {
1920 payload.data.len().div_ceil(3).saturating_mul(4)
1921 });
1922 EVENT_RESPONSE_OVERHEAD_BYTES.saturating_add(encoded_payload)
1923}
1924
1925fn event_kind(kind: EventKind) -> String {
1926 match kind {
1927 EventKind::SpanOpen => "span_open".into(),
1928 EventKind::SpanEnter => "span_enter".into(),
1929 EventKind::SpanExit => "span_exit".into(),
1930 EventKind::SpanClose => "span_close".into(),
1931 EventKind::Snapshot => "snapshot".into(),
1932 EventKind::Checkpoint => "checkpoint".into(),
1933 EventKind::Log => "log".into(),
1934 EventKind::Metadata => "metadata".into(),
1935 EventKind::Gap => "gap".into(),
1936 EventKind::Custom(value) => format!("custom:{value}"),
1937 }
1938}
1939
1940fn payload_encoding(encoding: PayloadEncoding) -> String {
1941 match encoding {
1942 PayloadEncoding::Opaque => "opaque".into(),
1943 PayloadEncoding::Json => "json".into(),
1944 PayloadEncoding::Utf8 => "utf8".into(),
1945 PayloadEncoding::Protobuf => "protobuf".into(),
1946 PayloadEncoding::Custom(value) => format!("custom:{value}"),
1947 }
1948}
1949
1950#[derive(Serialize)]
1951struct ErrorResponse {
1952 error: String,
1953}
1954
1955struct ApiError {
1956 status: StatusCode,
1957 message: String,
1958}
1959
1960impl ApiError {
1961 fn bad_request(message: impl Into<String>) -> Self {
1962 Self {
1963 status: StatusCode::BAD_REQUEST,
1964 message: message.into(),
1965 }
1966 }
1967
1968 fn internal(message: impl Into<String>) -> Self {
1969 Self {
1970 status: StatusCode::INTERNAL_SERVER_ERROR,
1971 message: message.into(),
1972 }
1973 }
1974
1975 fn payload_too_large(message: impl Into<String>) -> Self {
1976 Self {
1977 status: StatusCode::PAYLOAD_TOO_LARGE,
1978 message: message.into(),
1979 }
1980 }
1981}
1982
1983impl From<StorageError> for ApiError {
1984 fn from(error: StorageError) -> Self {
1985 Self::internal(error.to_string())
1986 }
1987}
1988
1989impl IntoResponse for ApiError {
1990 fn into_response(self) -> Response {
1991 (
1992 self.status,
1993 Json(ErrorResponse {
1994 error: self.message,
1995 }),
1996 )
1997 .into_response()
1998 }
1999}
2000
2001#[cfg(test)]
2002mod tests;