1use std::sync::Arc;
27use std::time::Duration;
28
29use astrs_wire::{
30 DataId, DurationMs, NodeEvent, NodeHandshake, NodeRequest, NodeSpawnSpec, SessionId,
31 WireMessage,
32};
33use tokio::sync::mpsc;
34
35use crate::error::{NodeError, Result};
36use crate::events::{EventSource, EventStream};
37use crate::runtime::NodeRuntime;
38use crate::session::connect::{LinkDuplex, NodeLink};
39use crate::session::{OUTGOING_CAPACITY, SessionShared, pump};
40
41pub const REGISTRATION_TIMEOUT: Duration = Duration::from_secs(10);
43
44pub struct Registration {
46 pub shared: Arc<SessionShared>,
48 pub events: EventStream,
50 pub tasks: pump::SessionTasks,
52 pub spec: Arc<NodeSpawnSpec>,
54}
55
56impl core::fmt::Debug for Registration {
57 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
58 f.debug_struct("Registration")
59 .field("node", &self.spec.node)
60 .field("generation", &self.spec.generation)
61 .finish_non_exhaustive()
62 }
63}
64
65pub async fn register(
73 link: NodeLink,
74 handshake: NodeHandshake,
75 runtime: NodeRuntime,
76 zero_copy_threshold: u64,
77 subscribe_to: Vec<DataId>,
78) -> Result<Registration> {
79 register_with(
80 link,
81 handshake,
82 runtime,
83 zero_copy_threshold,
84 subscribe_to,
85 None,
86 )
87 .await
88}
89
90pub async fn register_with(
102 link: NodeLink,
103 handshake: NodeHandshake,
104 runtime: NodeRuntime,
105 zero_copy_threshold: u64,
106 subscribe_to: Vec<DataId>,
107 shm_broker: Option<std::path::PathBuf>,
108) -> Result<Registration> {
109 let limits = link.session.frame_limits();
110 let session_id = link.session.session_id;
111 let mut duplex = link.duplex;
112
113 duplex
114 .send_message(&NodeRequest::Register(handshake))
115 .await?;
116
117 let (spec, session, deferred) = await_registered(&mut duplex, session_id).await?;
118 let spec = Arc::new(spec);
119
120 let source = Arc::new(EventSource::new());
121 for input in &spec.inputs {
122 source.register_input(input)?;
123 }
124
125 let (sender, receiver) = mpsc::channel(OUTGOING_CAPACITY);
126 let shared = Arc::new(
127 SessionShared::new(
128 Arc::clone(&spec),
129 session,
130 sender,
131 Arc::clone(&source),
132 runtime.clone(),
133 zero_copy_threshold,
134 limits,
135 )
136 .with_shm_broker(shm_broker),
137 );
138
139 duplex
140 .send_message(&NodeRequest::Subscribe {
141 inputs: subscribe_to,
142 })
143 .await?;
144
145 let (reader, writer) = duplex.into_halves();
146 let tasks = pump::spawn(&shared, reader, writer, receiver);
147
148 {
151 let notify = Arc::clone(&shared);
152 source.set_abandon_hook(Box::new(move || {
153 let _ = notify.send_request(NodeRequest::EventStreamDropped);
154 }));
155 }
156
157 {
181 let notify = Arc::clone(&shared);
182 source.set_deadline_violation_hook(Box::new(move |input, budget, latency| {
183 let _ = notify.send_request(NodeRequest::ReportDeadlineViolation {
184 input,
185 budget: DurationMs::from_duration(budget),
186 latency: DurationMs::from_duration(latency),
187 });
188 }));
189 }
190
191 for event in deferred {
194 pump::dispatch(&shared, event);
195 }
196
197 let events = EventStream::new(source, runtime);
198 Ok(Registration {
199 shared,
200 events,
201 tasks,
202 spec,
203 })
204}
205
206async fn await_registered(
208 duplex: &mut LinkDuplex,
209 expected_session: SessionId,
210) -> Result<(NodeSpawnSpec, SessionId, Vec<NodeEvent>)> {
211 let mut deferred = Vec::new();
212 let deadline = tokio::time::Instant::now() + REGISTRATION_TIMEOUT;
213 loop {
214 let frame = tokio::time::timeout_at(deadline, duplex.recv_frame())
215 .await
216 .map_err(|_| NodeError::Timeout {
217 operation: "node registration",
218 millis: as_millis(REGISTRATION_TIMEOUT),
219 })??;
220 let Some(frame) = frame else {
221 return Err(NodeError::Registration(
222 "the daemon closed the connection before answering the registration".to_owned(),
223 ));
224 };
225 if frame.kind() != astrs_wire::FrameKind::NodeEvent {
226 return Err(NodeError::Registration(format!(
227 "the daemon answered a registration with a `{}` frame",
228 frame.kind()
229 )));
230 }
231 match NodeEvent::from_frame(&frame.as_view())? {
232 NodeEvent::Registered { spec, session } => {
233 if session != expected_session {
234 }
239 return Ok((*spec, session, deferred));
240 }
241 NodeEvent::Stop { cause, .. } => {
242 return Err(NodeError::Registration(format!(
243 "the daemon stopped this node during registration: {cause}"
244 )));
245 }
246 other => deferred.push(other),
247 }
248 }
249}
250
251fn as_millis(duration: Duration) -> u64 {
253 u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
254}
255
256#[cfg(test)]
257mod tests {
258 #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
259
260 use super::*;
261 use crate::session::connect::wrap_stream;
262 use astrs_wire::{
263 DataflowId, FrameKind, FrameLimits, InputSpec, Metadata, NodeId, NodeSource, OutputPayload,
264 PortRef, StopCause,
265 };
266
267 fn spec() -> NodeSpawnSpec {
268 NodeSpawnSpec::new(
269 DataflowId::from_u128(4),
270 NodeId::new("detect").unwrap(),
271 1,
272 NodeSource::Dynamic,
273 )
274 .with_input(InputSpec::new(
275 DataId::new("frames").unwrap(),
276 PortRef::from_parts("camera", "image").unwrap(),
277 ))
278 }
279
280 fn spec_with_deadline() -> NodeSpawnSpec {
286 NodeSpawnSpec::new(
287 DataflowId::from_u128(4),
288 NodeId::new("detect").unwrap(),
289 1,
290 NodeSource::Dynamic,
291 )
292 .with_input(InputSpec {
293 deadline: Some(DurationMs::new(1)),
294 ..InputSpec::new(
295 DataId::new("frames").unwrap(),
296 PortRef::from_parts("camera", "image").unwrap(),
297 )
298 })
299 }
300
301 fn link(io: tokio::io::DuplexStream) -> NodeLink {
302 NodeLink {
303 duplex: wrap_stream(io, FrameLimits::uds()),
304 session: astrs_wire::NegotiatedSession {
305 protocol: astrs_wire::PROTOCOL_VERSION,
306 session_id: SessionId::from_u128(11),
307 features: astrs_wire::FeatureFlags::EMPTY,
308 limits: astrs_wire::NegotiatedLimits::uds(),
309 role: astrs_wire::Role::Node,
310 peer_version: astrs_wire::AstrsVersion::current(),
311 resumed: false,
312 },
313 endpoint: "duplex".to_owned(),
314 }
315 }
316
317 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
318 async fn the_registration_exchange_completes_and_starts_the_pump() {
319 let (node_io, daemon_io) = tokio::io::duplex(64 * 1024);
320 let mut daemon = wrap_stream(daemon_io, FrameLimits::uds());
321
322 let daemon_task = tokio::spawn(async move {
323 let request = daemon
324 .expect_message::<NodeRequest>(FrameKind::NodeRequest)
325 .await
326 .unwrap();
327 assert!(matches!(request, NodeRequest::Register(_)));
328 daemon
329 .send_message(&NodeEvent::Registered {
330 spec: Box::new(spec()),
331 session: SessionId::from_u128(11),
332 })
333 .await
334 .unwrap();
335 let request = daemon
336 .expect_message::<NodeRequest>(FrameKind::NodeRequest)
337 .await
338 .unwrap();
339 assert!(matches!(request, NodeRequest::Subscribe { .. }));
340 daemon
341 .send_message(&NodeEvent::Input {
342 id: DataId::new("frames").unwrap(),
343 source: PortRef::from_parts("camera", "image").unwrap(),
344 metadata: Metadata::default(),
345 payload: vec![1, 2],
346 })
347 .await
348 .unwrap();
349 daemon
350 });
351
352 let handshake =
353 NodeHandshake::dynamic(DataflowId::from_u128(4), NodeId::new("detect").unwrap());
354 let registration = register(
355 link(node_io),
356 handshake,
357 NodeRuntime::acquire().unwrap(),
358 4096,
359 Vec::new(),
360 )
361 .await
362 .unwrap();
363
364 assert_eq!(registration.spec.generation, 1);
365 assert_eq!(registration.spec.inputs.len(), 1);
366 assert!(format!("{registration:?}").contains("detect"));
367
368 let mut events = registration.events;
369 let event = events
370 .recv_async_timeout(Duration::from_secs(5))
371 .await
372 .unwrap();
373 assert!(event.is_input());
374
375 registration.shared.close();
376 registration.tasks.abort();
377 let _daemon = daemon_task.await.unwrap();
378 }
379
380 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
381 async fn events_that_arrive_before_registered_are_replayed_not_lost() {
382 let (node_io, daemon_io) = tokio::io::duplex(64 * 1024);
383 let mut daemon = wrap_stream(daemon_io, FrameLimits::uds());
384
385 let daemon_task = tokio::spawn(async move {
386 let _ = daemon
387 .expect_message::<NodeRequest>(FrameKind::NodeRequest)
388 .await
389 .unwrap();
390 daemon
392 .send_message(&NodeEvent::Input {
393 id: DataId::new("frames").unwrap(),
394 source: PortRef::from_parts("camera", "image").unwrap(),
395 metadata: Metadata::default(),
396 payload: vec![9],
397 })
398 .await
399 .unwrap();
400 daemon
401 .send_message(&NodeEvent::Registered {
402 spec: Box::new(spec()),
403 session: SessionId::from_u128(11),
404 })
405 .await
406 .unwrap();
407 let _ = daemon
408 .expect_message::<NodeRequest>(FrameKind::NodeRequest)
409 .await
410 .unwrap();
411 daemon
412 });
413
414 let registration = register(
415 link(node_io),
416 NodeHandshake::dynamic(DataflowId::from_u128(4), NodeId::new("detect").unwrap()),
417 NodeRuntime::acquire().unwrap(),
418 4096,
419 Vec::new(),
420 )
421 .await
422 .unwrap();
423
424 let mut events = registration.events;
425 let event = events
426 .recv_async_timeout(Duration::from_secs(5))
427 .await
428 .unwrap();
429 let Some((id, _, data)) = event.into_input() else {
430 panic!("expected the replayed input");
431 };
432 assert_eq!(id.as_str(), "frames");
433 assert_eq!(data.to_vec(), vec![9]);
434
435 registration.shared.close();
436 registration.tasks.abort();
437 let _daemon = daemon_task.await.unwrap();
438 }
439
440 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
441 async fn a_stop_during_registration_is_reported() {
442 let (node_io, daemon_io) = tokio::io::duplex(64 * 1024);
443 let mut daemon = wrap_stream(daemon_io, FrameLimits::uds());
444 let daemon_task = tokio::spawn(async move {
445 let _ = daemon
446 .expect_message::<NodeRequest>(FrameKind::NodeRequest)
447 .await
448 .unwrap();
449 daemon
450 .send_message(&NodeEvent::Stop {
451 cause: StopCause::Destroyed,
452 grace: None,
453 })
454 .await
455 .unwrap();
456 daemon
457 });
458
459 let error = register(
460 link(node_io),
461 NodeHandshake::dynamic(DataflowId::from_u128(4), NodeId::new("detect").unwrap()),
462 NodeRuntime::acquire().unwrap(),
463 4096,
464 Vec::new(),
465 )
466 .await
467 .unwrap_err();
468 assert!(matches!(error, NodeError::Registration(_)), "{error}");
469 let _daemon = daemon_task.await.unwrap();
470 }
471
472 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
473 async fn a_daemon_that_hangs_up_is_reported() {
474 let (node_io, daemon_io) = tokio::io::duplex(64 * 1024);
475 drop(daemon_io);
476 let error = register(
477 link(node_io),
478 NodeHandshake::dynamic(DataflowId::from_u128(4), NodeId::new("detect").unwrap()),
479 NodeRuntime::acquire().unwrap(),
480 4096,
481 Vec::new(),
482 )
483 .await
484 .unwrap_err();
485 assert!(
486 matches!(error, NodeError::Registration(_) | NodeError::Transport(_)),
487 "{error}"
488 );
489 }
490
491 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
499 async fn a_deadline_violation_is_relayed_to_the_daemon() {
500 let (node_io, daemon_io) = tokio::io::duplex(64 * 1024);
501 let mut daemon = wrap_stream(daemon_io, FrameLimits::uds());
502
503 let daemon_task = tokio::spawn(async move {
504 let request = daemon
505 .expect_message::<NodeRequest>(FrameKind::NodeRequest)
506 .await
507 .unwrap();
508 assert!(matches!(request, NodeRequest::Register(_)));
509 daemon
510 .send_message(&NodeEvent::Registered {
511 spec: Box::new(spec_with_deadline()),
512 session: SessionId::from_u128(11),
513 })
514 .await
515 .unwrap();
516 let request = daemon
517 .expect_message::<NodeRequest>(FrameKind::NodeRequest)
518 .await
519 .unwrap();
520 assert!(matches!(request, NodeRequest::Subscribe { .. }));
521 daemon
522 .send_message(&NodeEvent::Input {
523 id: DataId::new("frames").unwrap(),
524 source: PortRef::from_parts("camera", "image").unwrap(),
525 metadata: Metadata::default(),
526 payload: vec![1, 2],
527 })
528 .await
529 .unwrap();
530
531 let request = daemon
536 .expect_message::<NodeRequest>(FrameKind::NodeRequest)
537 .await
538 .unwrap();
539 assert!(matches!(request, NodeRequest::SendMessage { .. }));
540
541 let request = daemon
545 .expect_message::<NodeRequest>(FrameKind::NodeRequest)
546 .await
547 .unwrap();
548 match request {
549 NodeRequest::ReportDeadlineViolation {
550 input,
551 budget,
552 latency,
553 } => {
554 assert_eq!(input.as_str(), "frames");
555 assert_eq!(budget, DurationMs::new(1));
556 assert!(latency >= budget, "{latency:?} >= {budget:?}");
557 }
558 other => panic!("expected ReportDeadlineViolation, got {other:?}"),
559 }
560 daemon
561 });
562
563 let registration = register(
564 link(node_io),
565 NodeHandshake::dynamic(DataflowId::from_u128(4), NodeId::new("detect").unwrap()),
566 NodeRuntime::acquire().unwrap(),
567 4096,
568 Vec::new(),
569 )
570 .await
571 .unwrap();
572
573 let mut events = registration.events;
574 let event = events
575 .recv_async_timeout(Duration::from_secs(5))
576 .await
577 .unwrap();
578 assert!(event.is_input(), "opens the deadline measurement");
579
580 tokio::time::sleep(Duration::from_millis(20)).await;
583
584 registration
585 .shared
586 .send_request(NodeRequest::SendMessage {
587 output: DataId::new("boxes").unwrap(),
588 metadata: Metadata::default(),
589 payload: OutputPayload::inline(vec![0]),
590 })
591 .unwrap();
592
593 let _daemon = daemon_task.await.unwrap();
600
601 registration.shared.close();
602 registration.tasks.abort();
603 }
604}