1use crate::{
2 adapters::{BackendAdapter, BackendError, ServeRequest},
3 identity::ProcessIdentity,
4 state::ServerState,
5};
6use std::{
7 collections::VecDeque,
8 future::Future,
9 sync::{Arc, Mutex},
10};
11use tokio::sync::mpsc::{self, Receiver, Sender};
12use tokio_util::sync::CancellationToken;
13
14pub const EVENT_CAPACITY: usize = 64;
15
16#[derive(Clone, Copy)]
17pub(crate) enum LifecycleWarning {
18 InstalledBypass,
19 EstimatedFit,
20 SizeOnly,
21 DrafterSkipped,
22}
23
24impl LifecycleWarning {
25 fn message(self) -> &'static str {
26 match self {
27 Self::InstalledBypass => {
28 "up: bypassing estimated fit; local content integrity is not catalog verification; throughput may be unknown"
29 }
30 Self::EstimatedFit => {
31 "up: requested quantization may not fit this hardware; continuing because it was explicitly requested"
32 }
33 Self::SizeOnly => {
34 "up: weights passed a size-floor check; no catalog SHA-256 was available"
35 }
36 Self::DrafterSkipped => {
37 "up: skipped the speculative drafter because it does not fit beside the model"
38 }
39 }
40 }
41}
42
43#[derive(Clone, Default)]
44pub struct DiagnosticObserver {
45 state: Arc<Mutex<DiagnosticSnapshot>>,
46}
47
48#[derive(Clone, Default)]
49pub struct DiagnosticSnapshot {
50 warnings: [u64; 3],
51 progress: VecDeque<(u64, u64)>,
52 omitted_progress: u64,
53}
54
55impl DiagnosticObserver {
56 pub fn snapshot(&self) -> DiagnosticSnapshot {
57 self.state
58 .lock()
59 .unwrap_or_else(|error| error.into_inner())
60 .clone()
61 }
62
63 fn warning(&self, warning: LifecycleWarning) {
64 let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
65 let count = &mut state.warnings[warning as usize];
66 *count = count.saturating_add(1);
67 }
68
69 fn progress(&self, completed: u64, total: u64) {
70 let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
71 if state.progress.len() == EVENT_CAPACITY {
72 state.progress.pop_front();
73 state.omitted_progress = state.omitted_progress.saturating_add(1);
74 }
75 state.progress.push_back((completed, total));
76 }
77}
78
79impl DiagnosticSnapshot {
80 pub fn text(&self) -> String {
81 let mut lines = Vec::new();
82 for warning in [
83 LifecycleWarning::InstalledBypass,
84 LifecycleWarning::EstimatedFit,
85 LifecycleWarning::SizeOnly,
86 ] {
87 let count = self.warnings[warning as usize];
88 if count > 0 {
89 lines.push(format!("{} ({count} occurrences)", warning.message()));
90 }
91 }
92 if self.omitted_progress > 0 {
93 lines.push(format!(
94 "{} earlier progress diagnostics omitted",
95 self.omitted_progress
96 ));
97 }
98 for (completed, total) in &self.progress {
99 lines.push(format!(" [redacted filename]: {completed}/{total} bytes"));
100 }
101 if lines.is_empty() {
102 String::new()
103 } else {
104 format!("{}\n", lines.join("\n"))
105 }
106 }
107}
108
109pub(crate) fn warning(observer: Option<&LifecycleObserver>, warning: LifecycleWarning) {
110 match observer.and_then(|observer| observer.diagnostics.as_ref()) {
111 Some(diagnostics) => diagnostics.warning(warning),
112 None => eprintln!("{}", warning.message()),
113 }
114}
115
116pub(crate) fn download_progress(
117 diagnostics: Option<&DiagnosticObserver>,
118 completed: u64,
119 total: u64,
120 file: &str,
121) {
122 match diagnostics {
123 Some(diagnostics) => diagnostics.progress(completed, total),
124 None => eprintln!(
125 " {}: {completed}/{total} bytes",
126 rigspark_core::reports::strip_control(file)
127 ),
128 }
129}
130
131#[derive(Debug, Clone, Copy, PartialEq, Eq)]
132pub enum LifecycleScope {
133 Runtime,
134 AcquisitionDaemon,
135}
136
137#[derive(Debug, Clone, Copy, PartialEq, Eq)]
138pub enum LifecycleStage {
139 Acquisition,
140 Verification,
141 AcquisitionVerification,
142 Start,
143 Attach,
144 Activation,
145 Readiness,
146 Stop,
147}
148
149#[derive(Debug, Clone, Copy, PartialEq, Eq)]
150pub enum LifecycleStatus {
151 Started,
152 Completed,
153 Failed,
154}
155
156#[derive(Debug, Clone, Copy, PartialEq, Eq)]
162pub struct LifecycleEvent {
163 pub scope: LifecycleScope,
164 pub stage: LifecycleStage,
165 pub status: LifecycleStatus,
166}
167
168pub struct LifecycleObserver {
171 sender: Sender<LifecycleEvent>,
172 pub(crate) diagnostics: Option<DiagnosticObserver>,
173}
174
175impl LifecycleObserver {
176 pub fn channel() -> (Self, Receiver<LifecycleEvent>) {
177 let (sender, receiver) = mpsc::channel(EVENT_CAPACITY);
178 (
179 Self {
180 sender,
181 diagnostics: None,
182 },
183 receiver,
184 )
185 }
186
187 pub fn with_diagnostics(mut self, diagnostics: DiagnosticObserver) -> Self {
188 self.diagnostics = Some(diagnostics);
189 self
190 }
191
192 fn emit(&self, scope: LifecycleScope, stage: LifecycleStage, status: LifecycleStatus) {
193 let _ = self.sender.try_send(LifecycleEvent {
194 scope,
195 stage,
196 status,
197 });
198 }
199}
200
201pub(crate) async fn observe<T>(
202 observer: Option<&LifecycleObserver>,
203 scope: LifecycleScope,
204 stage: LifecycleStage,
205 operation: impl Future<Output = Result<T, BackendError>>,
206) -> Result<T, BackendError> {
207 if let Some(observer) = observer {
208 observer.emit(scope, stage, LifecycleStatus::Started);
209 }
210 let result = operation.await;
211 if let Some(observer) = observer {
212 observer.emit(
213 scope,
214 stage,
215 if result.is_ok() {
216 LifecycleStatus::Completed
217 } else {
218 LifecycleStatus::Failed
219 },
220 );
221 }
222 result
223}
224
225pub(super) struct ObservedAdapter<'runtime> {
226 inner: &'runtime dyn BackendAdapter,
227 observer: Option<&'runtime LifecycleObserver>,
228 scope: LifecycleScope,
229}
230
231impl<'runtime> ObservedAdapter<'runtime> {
232 pub(super) fn new(
233 inner: &'runtime dyn BackendAdapter,
234 observer: Option<&'runtime LifecycleObserver>,
235 scope: LifecycleScope,
236 ) -> Self {
237 Self {
238 inner,
239 observer,
240 scope,
241 }
242 }
243}
244
245#[async_trait::async_trait]
246impl BackendAdapter for ObservedAdapter<'_> {
247 fn name(&self) -> &'static str {
248 self.inner.name()
249 }
250 fn trusts(&self, identity: &ProcessIdentity) -> bool {
251 self.inner.trusts(identity)
252 }
253 fn can_embed(&self) -> bool {
254 self.inner.can_embed()
255 }
256 fn can_stream(&self) -> bool {
257 self.inner.can_stream()
258 }
259
260 async fn serve(
261 &self,
262 request: &ServeRequest,
263 cancel: &CancellationToken,
264 ) -> Result<ServerState, BackendError> {
265 observe(
266 self.observer,
267 self.scope,
268 LifecycleStage::Start,
269 self.inner.serve(request, cancel),
270 )
271 .await
272 }
273 async fn attach_only(
274 &self,
275 request: &ServeRequest,
276 cancel: &CancellationToken,
277 ) -> Result<ServerState, BackendError> {
278 observe(
279 self.observer,
280 self.scope,
281 LifecycleStage::Attach,
282 self.inner.attach_only(request, cancel),
283 )
284 .await
285 }
286 async fn ready(
287 &self,
288 request: &ServeRequest,
289 cancel: &CancellationToken,
290 ) -> Result<(), BackendError> {
291 observe(
292 self.observer,
293 self.scope,
294 LifecycleStage::Readiness,
295 self.inner.ready(request, cancel),
296 )
297 .await
298 }
299 async fn ready_handle(
300 &self,
301 request: &ServeRequest,
302 handle: &ServerState,
303 cancel: &CancellationToken,
304 ) -> Result<(), BackendError> {
305 observe(
306 self.observer,
307 self.scope,
308 LifecycleStage::Readiness,
309 self.inner.ready_handle(request, handle, cancel),
310 )
311 .await
312 }
313 async fn stop(
314 &self,
315 handle: &ServerState,
316 cancel: &CancellationToken,
317 ) -> Result<(), BackendError> {
318 observe(
319 self.observer,
320 self.scope,
321 LifecycleStage::Stop,
322 self.inner.stop(handle, cancel),
323 )
324 .await
325 }
326}
327
328#[cfg(test)]
329mod tests {
330 use super::*;
331 use crate::{
332 adapters::{BackendAdapter, ServeRequest},
333 identity::ProcessIdentity,
334 state::{RuntimeState, ServerState},
335 };
336 use std::sync::Mutex;
337 use tokio_util::sync::CancellationToken;
338
339 #[test]
340 fn diagnostic_sink_bounds_progress_without_evicting_security_warnings() {
341 let diagnostics = DiagnosticObserver::default();
342 diagnostics.warning(LifecycleWarning::InstalledBypass);
343 diagnostics.warning(LifecycleWarning::EstimatedFit);
344 diagnostics.warning(LifecycleWarning::SizeOnly);
345 for completed in 0..100 {
346 download_progress(
347 Some(&diagnostics),
348 completed,
349 100,
350 "\u{1b}]52;c;secret\u{7}/token=secret.bin",
351 );
352 }
353 let snapshot = diagnostics.snapshot();
354 assert_eq!(snapshot.progress.len(), EVENT_CAPACITY);
355 assert_eq!(snapshot.omitted_progress, 36);
356 let text = snapshot.text();
357 assert!(text.contains("no catalog SHA-256"));
358 assert!(text.contains("bypassing estimated fit"));
359 assert!(text.contains("may not fit"));
360 assert!(text.contains("99/100 bytes"));
361 assert!(text.contains("36 earlier progress diagnostics omitted"));
362 assert!(text.contains("[redacted filename]"));
363 assert!(!text.contains("secret"));
364 assert!(!text.contains('\u{1b}'));
365 }
366
367 #[test]
368 fn diagnostic_sink_is_injected_and_keeps_repeated_warning_counts() {
369 let (plain, _) = LifecycleObserver::channel();
370 assert!(plain.diagnostics.is_none());
371 let diagnostics = DiagnosticObserver::default();
372 let observed = plain.with_diagnostics(diagnostics.clone());
373 for _ in 0..100 {
374 warning(Some(&observed), LifecycleWarning::SizeOnly);
375 }
376 assert!(diagnostics.snapshot().text().contains("100 occurrences"));
377 assert!(DiagnosticObserver::default().snapshot().text().is_empty());
378 }
379
380 struct Adapter {
381 fail: Option<LifecycleStage>,
382 calls: Mutex<Vec<LifecycleStage>>,
383 }
384
385 impl Adapter {
386 fn attempt(
387 &self,
388 stage: LifecycleStage,
389 cancel: &CancellationToken,
390 ) -> Result<(), BackendError> {
391 self.calls.lock().unwrap().push(stage);
392 if cancel.is_cancelled() {
393 Err(BackendError("cancelled".into()))
394 } else if self.fail == Some(stage) {
395 Err(BackendError("mock operation failed".into()))
396 } else {
397 Ok(())
398 }
399 }
400 }
401
402 fn handle() -> ServerState {
403 RuntimeState::parse(r#"{"schemaVersion":2,"active":{"backend":"ollama","modelId":"model:latest","endpoint":"http://127.0.0.1:11434","port":11434,"ownedByUs":true,"pid":123}}"#)
404 .unwrap().active.unwrap()
405 }
406
407 fn request() -> ServeRequest {
408 ServeRequest {
409 draft_model_path: None,
410 model_id: "model:latest".into(),
411 endpoint: "http://127.0.0.1:11434".into(),
412 model_path: None,
413 context: Some(4096),
414 cache: None,
415 }
416 }
417
418 #[async_trait::async_trait]
419 impl BackendAdapter for Adapter {
420 fn name(&self) -> &'static str {
421 "ollama"
422 }
423 fn trusts(&self, _identity: &ProcessIdentity) -> bool {
424 false
425 }
426 async fn serve(
427 &self,
428 request: &ServeRequest,
429 cancel: &CancellationToken,
430 ) -> Result<ServerState, BackendError> {
431 assert_eq!(
432 request.context,
433 if request.model_id == "llmup-acquisition" {
434 None
435 } else {
436 Some(4096)
437 }
438 );
439 self.attempt(LifecycleStage::Start, cancel)?;
440 Ok(handle())
441 }
442 async fn attach_only(
443 &self,
444 _request: &ServeRequest,
445 cancel: &CancellationToken,
446 ) -> Result<ServerState, BackendError> {
447 self.attempt(LifecycleStage::Attach, cancel)?;
448 let mut handle = handle();
449 handle.owned_by_us = false;
450 Ok(handle)
451 }
452 async fn ready(
453 &self,
454 _request: &ServeRequest,
455 _cancel: &CancellationToken,
456 ) -> Result<(), BackendError> {
457 panic!("ready_handle override must be forwarded")
458 }
459 async fn ready_handle(
460 &self,
461 _request: &ServeRequest,
462 active: &ServerState,
463 cancel: &CancellationToken,
464 ) -> Result<(), BackendError> {
465 assert_eq!(active.pid, Some(123));
466 self.attempt(LifecycleStage::Readiness, cancel)
467 }
468 async fn stop(
469 &self,
470 active: &ServerState,
471 cancel: &CancellationToken,
472 ) -> Result<(), BackendError> {
473 assert!(active.owned_by_us);
474 self.attempt(LifecycleStage::Stop, cancel)
475 }
476 }
477
478 #[tokio::test]
479 async fn runtime_events_adapter_orders_real_calls_and_preserves_failures() {
480 for failure in [
481 None,
482 Some(LifecycleStage::Start),
483 Some(LifecycleStage::Readiness),
484 Some(LifecycleStage::Stop),
485 ] {
486 let (observer, mut receiver) = LifecycleObserver::channel();
487 let adapter = Adapter {
488 fail: failure,
489 calls: Mutex::new(Vec::new()),
490 };
491 let observed = ObservedAdapter::new(&adapter, Some(&observer), LifecycleScope::Runtime);
492 let cancel = CancellationToken::new();
493 let outcome = async {
494 let active = observed.serve(&request(), &cancel).await?;
495 observed.ready_handle(&request(), &active, &cancel).await?;
496 observed.stop(&active, &cancel).await
497 }
498 .await;
499 assert_eq!(outcome.is_err(), failure.is_some());
500 if let Err(error) = outcome {
501 assert_eq!(error.0, "mock operation failed");
502 }
503 let mut expected = Vec::new();
504 for stage in [
505 LifecycleStage::Start,
506 LifecycleStage::Readiness,
507 LifecycleStage::Stop,
508 ] {
509 expected.push(stage);
510 assert_eq!(
511 receiver.try_recv().unwrap(),
512 LifecycleEvent {
513 scope: LifecycleScope::Runtime,
514 stage,
515 status: LifecycleStatus::Started
516 }
517 );
518 assert_eq!(
519 receiver.try_recv().unwrap(),
520 LifecycleEvent {
521 scope: LifecycleScope::Runtime,
522 stage,
523 status: if failure == Some(stage) {
524 LifecycleStatus::Failed
525 } else {
526 LifecycleStatus::Completed
527 }
528 }
529 );
530 if failure == Some(stage) {
531 break;
532 }
533 }
534 assert_eq!(*adapter.calls.lock().unwrap(), expected);
535 assert!(receiver.try_recv().is_err());
536 }
537 }
538
539 #[tokio::test]
540 async fn runtime_events_adapter_preserves_attach_and_cancellation() {
541 let (observer, mut receiver) = LifecycleObserver::channel();
542 let adapter = Adapter {
543 fail: None,
544 calls: Mutex::new(Vec::new()),
545 };
546 let observed =
547 ObservedAdapter::new(&adapter, Some(&observer), LifecycleScope::AcquisitionDaemon);
548 let cancel = CancellationToken::new();
549 assert!(
550 !observed
551 .attach_only(&request(), &cancel)
552 .await
553 .unwrap()
554 .owned_by_us
555 );
556 assert_eq!(receiver.try_recv().unwrap().stage, LifecycleStage::Attach);
557 assert_eq!(
558 receiver.try_recv().unwrap().status,
559 LifecycleStatus::Completed
560 );
561 cancel.cancel();
562 assert_eq!(
563 observed.serve(&request(), &cancel).await.unwrap_err().0,
564 "cancelled"
565 );
566 assert_eq!(
567 receiver.try_recv().unwrap().scope,
568 LifecycleScope::AcquisitionDaemon
569 );
570 assert_eq!(receiver.try_recv().unwrap().status, LifecycleStatus::Failed);
571 assert!(receiver.try_recv().is_err());
572 }
573
574 #[tokio::test]
575 async fn saturated_or_closed_observation_never_retries_or_replaces_runtime_result() {
576 for closed in [false, true] {
577 let (observer, mut receiver) = LifecycleObserver::channel();
578 for _ in 0..EVENT_CAPACITY {
579 observer.emit(
580 LifecycleScope::Runtime,
581 LifecycleStage::Start,
582 LifecycleStatus::Started,
583 );
584 }
585 if closed {
586 receiver.close();
587 }
588 let calls = std::sync::atomic::AtomicUsize::new(0);
589 let result = tokio::time::timeout(
590 std::time::Duration::from_secs(1),
591 observe(
592 Some(&observer),
593 LifecycleScope::Runtime,
594 LifecycleStage::Readiness,
595 async {
596 calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
597 Err::<(), _>(BackendError("authoritative runtime failure".into()))
598 },
599 ),
600 )
601 .await
602 .unwrap();
603 assert_eq!(result.unwrap_err().0, "authoritative runtime failure");
604 assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);
605 assert_eq!(receiver.len(), EVENT_CAPACITY);
606 let result = observe(
607 Some(&observer),
608 LifecycleScope::Runtime,
609 LifecycleStage::Stop,
610 async { Ok::<_, BackendError>("authoritative success") },
611 )
612 .await
613 .unwrap();
614 assert_eq!(result, "authoritative success");
615 assert_eq!(receiver.len(), EVENT_CAPACITY);
616 }
617 }
618
619 #[tokio::test]
620 async fn runtime_events_temporary_daemon_cleanup_does_not_claim_pull_success() {
621 let (observer, mut receiver) = LifecycleObserver::channel();
622 let adapter = Adapter {
623 fail: None,
624 calls: Mutex::new(Vec::new()),
625 };
626 let observed =
627 ObservedAdapter::new(&adapter, Some(&observer), LifecycleScope::AcquisitionDaemon);
628 let cancel = CancellationToken::new();
629 let operation = observe(
630 Some(&observer),
631 LifecycleScope::Runtime,
632 LifecycleStage::Acquisition,
633 async {
634 cancel.cancel();
635 Err::<(), _>(BackendError("pull cancelled".into()))
636 },
637 );
638 let result = crate::pull::with_ollama_daemon(
639 &observed,
640 "http://127.0.0.1:11434",
641 &cancel,
642 operation,
643 )
644 .await;
645 assert_eq!(result.unwrap_err().0, "pull cancelled");
646 assert_eq!(
647 *adapter.calls.lock().unwrap(),
648 [LifecycleStage::Start, LifecycleStage::Stop]
649 );
650 for (scope, stage, status) in [
651 (
652 LifecycleScope::AcquisitionDaemon,
653 LifecycleStage::Start,
654 LifecycleStatus::Started,
655 ),
656 (
657 LifecycleScope::AcquisitionDaemon,
658 LifecycleStage::Start,
659 LifecycleStatus::Completed,
660 ),
661 (
662 LifecycleScope::Runtime,
663 LifecycleStage::Acquisition,
664 LifecycleStatus::Started,
665 ),
666 (
667 LifecycleScope::Runtime,
668 LifecycleStage::Acquisition,
669 LifecycleStatus::Failed,
670 ),
671 (
672 LifecycleScope::AcquisitionDaemon,
673 LifecycleStage::Stop,
674 LifecycleStatus::Started,
675 ),
676 (
677 LifecycleScope::AcquisitionDaemon,
678 LifecycleStage::Stop,
679 LifecycleStatus::Completed,
680 ),
681 ] {
682 assert_eq!(
683 receiver.try_recv().unwrap(),
684 LifecycleEvent {
685 scope,
686 stage,
687 status
688 }
689 );
690 }
691 assert!(receiver.try_recv().is_err());
692 }
693
694 #[tokio::test]
695 async fn runtime_events_bracket_success_and_failure_without_false_completion() {
696 let (observer, mut receiver) = LifecycleObserver::channel();
697 let operation = async {
698 assert_eq!(
699 receiver.try_recv().unwrap(),
700 LifecycleEvent {
701 scope: LifecycleScope::Runtime,
702 stage: LifecycleStage::Verification,
703 status: LifecycleStatus::Started,
704 }
705 );
706 Ok::<_, BackendError>(42)
707 };
708 assert_eq!(
709 observe(
710 Some(&observer),
711 LifecycleScope::Runtime,
712 LifecycleStage::Verification,
713 operation
714 )
715 .await
716 .unwrap(),
717 42
718 );
719 assert_eq!(
720 receiver.try_recv().unwrap().status,
721 LifecycleStatus::Completed
722 );
723 let error = observe(
724 Some(&observer),
725 LifecycleScope::Runtime,
726 LifecycleStage::Start,
727 async { Err::<(), _>(BackendError("private diagnostic".into())) },
728 )
729 .await
730 .unwrap_err();
731 assert_eq!(error.0, "private diagnostic");
732 assert_eq!(
733 receiver.try_recv().unwrap().status,
734 LifecycleStatus::Started
735 );
736 let failed = receiver.try_recv().unwrap();
737 assert_eq!(failed.status, LifecycleStatus::Failed);
738 assert!(!format!("{failed:?}").contains("private diagnostic"));
739 assert!(receiver.try_recv().is_err());
740 }
741
742 #[tokio::test]
743 async fn runtime_events_full_closed_or_absent_observer_never_blocks_operations() {
744 let (observer, mut receiver) = LifecycleObserver::channel();
745 for _ in 0..EVENT_CAPACITY * 2 {
746 observe(
747 Some(&observer),
748 LifecycleScope::Runtime,
749 LifecycleStage::Readiness,
750 async { Ok::<_, BackendError>(()) },
751 )
752 .await
753 .unwrap();
754 }
755 assert_eq!(receiver.len(), EVENT_CAPACITY);
756 receiver.close();
757 for observer in [Some(&observer), None] {
758 assert_eq!(
759 observe(
760 observer,
761 LifecycleScope::Runtime,
762 LifecycleStage::Stop,
763 async { Ok::<_, BackendError>(7) }
764 )
765 .await
766 .unwrap(),
767 7
768 );
769 }
770 }
771
772 #[tokio::test]
773 async fn runtime_events_dropped_operation_has_no_completion() {
774 let (observer, mut receiver) = LifecycleObserver::channel();
775 {
776 let operation = observe(
777 Some(&observer),
778 LifecycleScope::Runtime,
779 LifecycleStage::Readiness,
780 std::future::pending::<Result<(), BackendError>>(),
781 );
782 tokio::pin!(operation);
783 tokio::select! {
784 biased;
785 _ = &mut operation => panic!("pending operation completed"),
786 event = receiver.recv() => assert_eq!(event.unwrap().status, LifecycleStatus::Started),
787 }
788 }
789 assert!(receiver.try_recv().is_err());
790 }
791}