1use std::sync::Arc;
36
37use astrs_wire::{Frame, FrameKind, NodeEvent, NodeRequest, WireMessage};
38use tokio::sync::mpsc;
39use tokio::task::JoinHandle;
40
41use crate::error::NodeError;
42use crate::events::{Event, QueuedEvent};
43use crate::payload::Payload;
44use crate::session::connect::{LinkReader, LinkWriter};
45use crate::session::routes::open_producer;
46use crate::session::{Outgoing, SessionShared};
47
48#[derive(Debug)]
50pub struct SessionTasks {
51 pub reader: JoinHandle<()>,
53 pub writer: JoinHandle<()>,
55}
56
57impl SessionTasks {
58 pub async fn join(self) {
63 let _ = self.reader.await;
64 let _ = self.writer.await;
65 }
66
67 pub fn abort(&self) {
69 self.reader.abort();
70 self.writer.abort();
71 }
72}
73
74#[must_use]
76pub fn spawn(
77 shared: &Arc<SessionShared>,
78 reader: LinkReader,
79 writer: LinkWriter,
80 outgoing: mpsc::Receiver<Outgoing>,
81) -> SessionTasks {
82 let reader_shared = Arc::clone(shared);
83 let writer_shared = Arc::clone(shared);
84 SessionTasks {
85 reader: shared
86 .runtime
87 .spawn(async move { reader_task(reader, reader_shared).await }),
88 writer: shared
89 .runtime
90 .spawn(async move { writer_task(writer, outgoing, writer_shared).await }),
91 }
92}
93
94async fn writer_task(
96 mut writer: LinkWriter,
97 mut outgoing: mpsc::Receiver<Outgoing>,
98 shared: Arc<SessionShared>,
99) {
100 while let Some(message) = outgoing.recv().await {
101 let result = match message {
102 Outgoing::Request(request) => writer.send_message(request.as_ref()).await,
103 Outgoing::Log(frame) => writer.send_message(frame.as_ref()).await,
104 Outgoing::Shutdown => {
105 let _ = writer.flush().await;
106 break;
107 }
108 };
109 if let Err(error) = result {
110 shared.source.push_control(Event::Error(format!(
111 "daemon connection failed while sending: {error}"
112 )));
113 break;
114 }
115 }
116 let _ = writer.shutdown().await;
117}
118
119async fn reader_task(mut reader: LinkReader, shared: Arc<SessionShared>) {
121 loop {
122 match reader.recv_frame().await {
123 Ok(Some(frame)) => {
124 if let Err(error) = handle_frame(&shared, &frame) {
125 shared
126 .source
127 .push_control(Event::Error(format!("undecodable frame: {error}")));
128 }
129 }
130 Ok(None) => break,
131 Err(error) => {
132 shared.source.push_control(Event::Error(format!(
133 "daemon connection failed while receiving: {error}"
134 )));
135 break;
136 }
137 }
138 }
139 shared.close();
140}
141
142pub fn handle_frame(shared: &Arc<SessionShared>, frame: &Frame) -> crate::Result<()> {
149 match frame.kind() {
150 FrameKind::NodeEvent => {
151 let event = NodeEvent::from_frame(&frame.as_view())?;
152 dispatch(shared, event);
153 Ok(())
154 }
155 other => Err(NodeError::Pattern(format!(
156 "a node received a `{other}` frame, which only flows the other way"
157 ))),
158 }
159}
160
161pub fn dispatch(shared: &Arc<SessionShared>, event: NodeEvent) {
163 shared.count_event();
164 match event {
165 NodeEvent::Input {
166 id,
167 source,
168 metadata,
169 payload,
170 } => {
171 let _accepted = shared.observe_remote(metadata.timestamp);
174 shared.source.push_input(
175 &id,
176 QueuedEvent::Input {
177 source,
178 metadata,
179 payload: Payload::inline(payload),
180 },
181 );
182 }
183 NodeEvent::InputClosed { id, source, reason } => {
184 let closed = QueuedEvent::Closed { source, reason };
191 if let Some(closed) = shared.inputs.close_input(&id, closed) {
192 shared.source.push_input(&id, closed);
193 }
194 }
195 NodeEvent::InputRecovered {
196 id,
197 source,
198 generation,
199 } => {
200 shared
201 .source
202 .push_input(&id, QueuedEvent::Recovered { source, generation });
203 }
204 NodeEvent::Stop { cause, .. } => {
205 shared.source.push_control(Event::Stop(cause));
206 }
207 NodeEvent::Reload { operator, path } => {
208 shared.source.push_control(Event::Reload { operator, path });
209 }
210 NodeEvent::AllInputsClosed => {
211 let _waited = shared.inputs.drain_all();
217 shared.source.push_control(Event::AllInputsClosed);
218 }
219 NodeEvent::NodeFailed { peer, cause } => {
220 shared
221 .source
222 .push_control(Event::NodeFailed { peer, cause });
223 }
224 NodeEvent::Restarted { peer, generation } => {
225 shared
226 .source
227 .push_control(Event::Restarted { peer, generation });
228 }
229 NodeEvent::ParamUpdate { scope, key, value } => {
230 shared
231 .source
232 .push_control(Event::ParamUpdate { scope, key, value });
233 }
234 NodeEvent::ParamDeleted { scope, key } => {
235 shared
236 .source
237 .push_control(Event::ParamDeleted { scope, key });
238 }
239 NodeEvent::ExtDropped { key, reason } => {
240 shared
241 .source
242 .push_control(Event::ExtDropped { key, reason });
243 }
244 NodeEvent::ExtValue { key, value } => {
245 let _delivered = shared.resolve_extension(&key, value);
246 }
247 NodeEvent::RouteUpgrade {
248 output,
249 segment,
250 consumers,
251 } => handle_route_upgrade(shared, output, segment, consumers),
252 NodeEvent::RouteDowngrade { output, reason } => {
253 shared.routes.slot(&output).offer_downgrade(reason);
254 }
255 NodeEvent::InputRouteUpgrade {
256 input,
257 source,
258 segment,
259 consumer,
260 } => handle_input_route_upgrade(shared, input, source, segment, consumer),
261 NodeEvent::InputRouteDowngrade { input, .. } => {
262 let _was_attached = shared.inputs.detach(&input);
266 }
267 NodeEvent::Registered { .. } => {
268 shared.source.push_control(Event::Error(
271 "the daemon sent a second `Registered` for a live session".to_owned(),
272 ));
273 }
274 other => {
277 shared
278 .source
279 .push_control(Event::Error(format!("unrecognised daemon event: {other}")));
280 }
281 }
282}
283
284fn handle_route_upgrade(
286 shared: &Arc<SessionShared>,
287 output: astrs_wire::DataId,
288 segment: astrs_wire::ShmSegmentSpec,
289 consumers: Vec<astrs_wire::PortRef>,
290) {
291 let slot = shared.routes.slot(&output);
292 match open_producer(
293 shared.dataflow,
294 &shared.node,
295 &output,
296 &segment,
297 shared.shm_broker.as_deref(),
298 ) {
299 Ok(producer) => {
300 slot.offer_upgrade(producer, segment, consumers);
301 let _ = shared.send_request(NodeRequest::RouteUpgradeAck {
302 output,
303 accepted: true,
304 reason: None,
305 });
306 }
307 Err(error) => {
308 let reason = error.to_string();
311 shared.count_shm_fallback();
312 shared.source.push_control(Event::Error(format!(
313 "route upgrade for `{output}` refused: {reason}"
314 )));
315 let _ = shared.send_request(NodeRequest::RouteUpgradeAck {
316 output,
317 accepted: false,
318 reason: Some(reason),
319 });
320 }
321 }
322}
323
324fn handle_input_route_upgrade(
337 shared: &Arc<SessionShared>,
338 input: astrs_wire::DataId,
339 source: astrs_wire::PortRef,
340 segment: astrs_wire::ShmSegmentSpec,
341 consumer: astrs_wire::PortRef,
342) {
343 if *consumer.node() != shared.node {
348 shared.inputs.count_refusal();
349 shared.source.push_control(Event::Error(format!(
350 "an input route upgrade for `{consumer}` reached node `{}`",
351 shared.node
352 )));
353 return;
354 }
355 if let Err(error) = shared.inputs.attach(shared, &input, &source, &segment) {
356 shared.inputs.count_refusal();
357 shared.count_shm_fallback();
358 shared.source.push_control(Event::Error(format!(
359 "input route upgrade for `{input}` refused: {error}"
360 )));
361 }
362}
363
364#[cfg(test)]
365mod tests {
366 #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
367
368 use super::*;
369 use crate::events::EventSource;
370 use crate::runtime::NodeRuntime;
371 use crate::session::{OUTGOING_CAPACITY, RoutePlane};
372 use astrs_time::HlcTimestamp;
373 use astrs_wire::{
374 DataId, DataflowId, ExtensionKey, FrameFlags, FrameLimits, InputSpec, Metadata, NodeId,
375 NodeSource, NodeSpawnSpec, PortRef, RouteCloseReason, RouteDowngradeReason, SessionId,
376 ShmSegmentSpec, StopCause,
377 };
378
379 fn spec() -> Arc<NodeSpawnSpec> {
380 Arc::new(
381 NodeSpawnSpec::new(
382 DataflowId::from_u128(1),
383 NodeId::new("detect").unwrap(),
384 0,
385 NodeSource::Dynamic,
386 )
387 .with_input(InputSpec::new(
388 DataId::new("frames").unwrap(),
389 PortRef::from_parts("camera", "image").unwrap(),
390 )),
391 )
392 }
393
394 fn shared() -> (Arc<SessionShared>, mpsc::Receiver<Outgoing>) {
395 let (sender, receiver) = mpsc::channel(OUTGOING_CAPACITY);
396 let source = Arc::new(EventSource::new());
397 let spec = spec();
398 for input in &spec.inputs {
399 source.register_input(input).unwrap();
400 }
401 let shared = Arc::new(SessionShared::new(
402 spec,
403 SessionId::from_u128(3),
404 sender,
405 source,
406 NodeRuntime::owned().unwrap(),
407 4096,
408 FrameLimits::uds(),
409 ));
410 (shared, receiver)
411 }
412
413 fn input_event(payload: Vec<u8>) -> NodeEvent {
414 NodeEvent::Input {
415 id: DataId::new("frames").unwrap(),
416 source: PortRef::from_parts("camera", "image").unwrap(),
417 metadata: Metadata::new(HlcTimestamp::new(1_000, 0)),
418 payload,
419 }
420 }
421
422 #[test]
423 fn an_input_reaches_its_queue_with_its_payload() {
424 let (shared, _rx) = shared();
425 dispatch(&shared, input_event(vec![1, 2, 3]));
426 let event = shared.source.try_next().unwrap();
427 let Event::Input { id, data, .. } = event else {
428 panic!("expected an input");
429 };
430 assert_eq!(id.as_str(), "frames");
431 assert_eq!(data.to_vec(), vec![1, 2, 3]);
432 assert_eq!(shared.stats().events_received, 1);
433 }
434
435 #[test]
436 fn every_control_event_reaches_the_control_lane() {
437 let (shared, _rx) = shared();
438 let events = vec![
439 NodeEvent::Stop {
440 cause: StopCause::Requested,
441 grace: None,
442 },
443 NodeEvent::Reload {
444 operator: None,
445 path: None,
446 },
447 NodeEvent::AllInputsClosed,
448 NodeEvent::NodeFailed {
449 peer: NodeId::new("camera").unwrap(),
450 cause: astrs_wire::NodeExitCause::Success,
451 },
452 NodeEvent::Restarted {
453 peer: NodeId::new("camera").unwrap(),
454 generation: 2,
455 },
456 NodeEvent::ExtDropped {
457 key: ExtensionKey::user("cal").unwrap(),
458 reason: "ttl".to_owned(),
459 },
460 ];
461 let expected = events.len();
462 for event in events {
463 dispatch(&shared, event);
464 }
465 let mut seen = 0;
466 while let Some(event) = shared.source.try_next() {
467 assert!(event.is_control(), "{event}");
468 seen += 1;
469 }
470 assert_eq!(seen, expected);
471 }
472
473 #[test]
474 fn input_closed_and_recovered_keep_their_input_queue() {
475 let (shared, _rx) = shared();
476 dispatch(&shared, input_event(vec![7]));
477 dispatch(
478 &shared,
479 NodeEvent::InputClosed {
480 id: DataId::new("frames").unwrap(),
481 source: PortRef::from_parts("camera", "image").unwrap(),
482 reason: RouteCloseReason::ProducerFinished,
483 },
484 );
485 dispatch(
486 &shared,
487 NodeEvent::InputRecovered {
488 id: DataId::new("frames").unwrap(),
489 source: PortRef::from_parts("camera", "image").unwrap(),
490 generation: 1,
491 },
492 );
493 assert!(shared.source.try_next().unwrap().is_input());
494 assert!(matches!(
495 shared.source.try_next().unwrap(),
496 Event::InputClosed { .. }
497 ));
498 assert!(matches!(
499 shared.source.try_next().unwrap(),
500 Event::InputRecovered { .. }
501 ));
502 }
503
504 #[test]
505 fn an_extension_value_goes_to_its_waiter_and_not_to_the_node() {
506 let (shared, _rx) = shared();
507 let key = ExtensionKey::user("cal").unwrap();
508 let receiver = shared.watch_extension(&key);
509 dispatch(
510 &shared,
511 NodeEvent::ExtValue {
512 key: key.clone(),
513 value: Some(vec![9]),
514 },
515 );
516 assert_eq!(receiver.blocking_recv().unwrap(), Some(vec![9]));
517 assert!(
518 shared.source.try_next().is_none(),
519 "the node never sees an ExtValue"
520 );
521 }
522
523 #[test]
524 fn a_route_downgrade_is_recorded_without_reaching_the_node() {
525 let (shared, _rx) = shared();
526 let output = DataId::new("detections").unwrap();
527 dispatch(
528 &shared,
529 NodeEvent::RouteDowngrade {
530 output: output.clone(),
531 reason: RouteDowngradeReason::ConsumerDetached {
532 consumer: PortRef::from_parts("detect", "frames").unwrap(),
533 },
534 },
535 );
536 assert!(shared.routes.slot(&output).has_update());
537 assert!(shared.source.try_next().is_none());
538 }
539
540 #[test]
541 fn an_input_route_downgrade_detaches_without_reaching_the_node() {
542 let (shared, _rx) = shared();
543 let input = DataId::new("frames").unwrap();
544 dispatch(
545 &shared,
546 NodeEvent::InputRouteDowngrade {
547 input: input.clone(),
548 reason: RouteDowngradeReason::SegmentClosed { generation: 2 },
549 },
550 );
551 assert_eq!(shared.inputs.plane(&input), RoutePlane::Daemon);
552 assert!(
553 shared.source.try_next().is_none(),
554 "a plane change is not an application event"
555 );
556 }
557
558 #[test]
559 fn an_unopenable_input_upgrade_is_refused_and_reported() {
560 let (shared, _rx) = shared();
561 let input = DataId::new("frames").unwrap();
562 dispatch(
563 &shared,
564 NodeEvent::InputRouteUpgrade {
565 input: input.clone(),
566 source: PortRef::from_parts("camera", "image").unwrap(),
567 segment: ShmSegmentSpec::new("astrs-bogus-name", 1, 4, 1024),
569 consumer: PortRef::from_parts("detect", "frames").unwrap(),
570 },
571 );
572 assert_eq!(shared.inputs.plane(&input), RoutePlane::Daemon);
574 assert_eq!(shared.inputs.stats().refusals, 1);
575 assert_eq!(shared.stats().shm_fallbacks, 1);
576 let Some(Event::Error(message)) = shared.source.try_next() else {
577 panic!("expected an error");
578 };
579 assert!(message.contains("frames"), "{message}");
580 }
581
582 #[test]
583 fn an_input_upgrade_addressed_to_another_node_is_refused() {
584 let (shared, _rx) = shared();
585 dispatch(
586 &shared,
587 NodeEvent::InputRouteUpgrade {
588 input: DataId::new("frames").unwrap(),
589 source: PortRef::from_parts("camera", "image").unwrap(),
590 segment: ShmSegmentSpec::new("astrs-anything", 1, 4, 1024),
591 consumer: PortRef::from_parts("planner", "frames").unwrap(),
593 },
594 );
595 assert_eq!(shared.inputs.stats().refusals, 1);
596 assert_eq!(
597 shared.inputs.stats().attaches,
598 0,
599 "nothing was mapped for another node's input"
600 );
601 let Some(Event::Error(message)) = shared.source.try_next() else {
602 panic!("expected an error");
603 };
604 assert!(message.contains("planner/frames"), "{message}");
605 }
606
607 #[test]
608 fn an_unopenable_upgrade_is_refused_and_reported() {
609 let (shared, mut rx) = shared();
610 let output = DataId::new("detections").unwrap();
611 dispatch(
612 &shared,
613 NodeEvent::RouteUpgrade {
614 output: output.clone(),
615 segment: ShmSegmentSpec::new("astrs-bogus-name", 1, 4, 1024),
616 consumers: Vec::new(),
617 },
618 );
619 let Some(Outgoing::Request(request)) = rx.try_recv().ok() else {
621 panic!("expected an ack");
622 };
623 let NodeRequest::RouteUpgradeAck {
624 accepted, reason, ..
625 } = *request
626 else {
627 panic!("expected a RouteUpgradeAck");
628 };
629 assert!(!accepted);
630 assert!(reason.is_some());
631 assert_eq!(shared.stats().shm_fallbacks, 1);
632
633 let event = shared.source.try_next().unwrap();
634 assert!(matches!(event, Event::Error(_)));
635 }
636
637 #[test]
638 fn a_second_registration_is_reported_as_an_error() {
639 let (shared, _rx) = shared();
640 dispatch(
641 &shared,
642 NodeEvent::Registered {
643 spec: Box::new((*shared.spec).clone()),
644 session: SessionId::from_u128(3),
645 },
646 );
647 let event = shared.source.try_next().unwrap();
648 let Event::Error(message) = event else {
649 panic!("expected an error");
650 };
651 assert!(message.contains("second"), "{message}");
652 }
653
654 #[test]
655 fn a_frame_of_the_wrong_kind_is_refused() {
656 let (shared, _rx) = shared();
657 let frame = Frame::new(FrameKind::NodeRequest, FrameFlags::EMPTY, vec![6]).unwrap();
658 let error = handle_frame(&shared, &frame).unwrap_err();
659 assert!(matches!(error, NodeError::Pattern(_)), "{error}");
660 }
661
662 #[test]
663 fn a_well_formed_frame_is_decoded_and_dispatched() {
664 let (shared, _rx) = shared();
665 let bytes = input_event(vec![4, 5])
666 .to_frame(FrameFlags::EMPTY, &FrameLimits::uds())
667 .unwrap();
668 let view = astrs_wire::decode_frame(&bytes, &FrameLimits::uds()).unwrap();
669 let frame = Frame::new(view.kind(), view.flags(), view.payload().to_vec()).unwrap();
670 handle_frame(&shared, &frame).unwrap();
671 assert!(shared.source.try_next().unwrap().is_input());
672 }
673
674 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
675 async fn the_pump_moves_frames_in_both_directions() {
676 use crate::session::connect::wrap_stream;
677
678 let (node_io, daemon_io) = tokio::io::duplex(64 * 1024);
679 let limits = FrameLimits::uds();
680 let node_link = wrap_stream(node_io, limits);
681 let mut daemon_link = wrap_stream(daemon_io, limits);
682
683 let (sender, receiver) = mpsc::channel(OUTGOING_CAPACITY);
684 let source = Arc::new(EventSource::new());
685 let spec = spec();
686 for input in &spec.inputs {
687 source.register_input(input).unwrap();
688 }
689 let shared = Arc::new(SessionShared::new(
690 spec,
691 SessionId::from_u128(1),
692 sender,
693 source,
694 crate::runtime::NodeRuntime::acquire().unwrap(),
695 4096,
696 limits,
697 ));
698 let (reader, writer) = node_link.into_halves();
699 let tasks = spawn(&shared, reader, writer, receiver);
700
701 shared
703 .send_request(NodeRequest::Subscribe { inputs: Vec::new() })
704 .unwrap();
705 let request = daemon_link
706 .expect_message::<NodeRequest>(FrameKind::NodeRequest)
707 .await
708 .unwrap();
709 assert!(matches!(request, NodeRequest::Subscribe { .. }));
710
711 daemon_link
713 .send_message(&input_event(vec![1]))
714 .await
715 .unwrap();
716 for _ in 0..100 {
717 if let Some(event) = shared.source.try_next() {
718 assert!(event.is_input());
719 break;
720 }
721 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
722 }
723
724 shared.close();
725 tasks.abort();
726 }
727}