Skip to main content

astrs_node_api/node/
init.rs

1//! The registration handshake (blueprint §7.3).
2//!
3//! ```text
4//!   node                                       daemon
5//!    ├──────── Hello / Welcome ─────────────────►│   (§7.2, astrs-transport)
6//!    ├──────── Register(NodeHandshake) ─────────►│
7//!    │◄─────── Registered { spec, session } ─────┤
8//!    ├──────── Subscribe { inputs } ────────────►│
9//!    │                                            │
10//!    └──────── the pump starts here ─────────────┘
11//! ```
12//!
13//! The handshake runs on the link *before* the reader task exists, because
14//! everything after it depends on the specification `Registered` carries: a
15//! `path: dynamic` node (§8.3) has no `ASTRS_NODE_CONFIG` at all and learns
16//! its wiring here, and even a spawned one takes the daemon's word for its
17//! generation over its own.
18//!
19//! # Events that arrive during the handshake
20//!
21//! A daemon may legitimately send an input the instant it accepts the
22//! registration — it is under no obligation to wait for `Subscribe`. Those
23//! frames are **buffered** here and replayed through the ordinary dispatcher
24//! once the queues exist, so a message can be early but never lost.
25
26use std::sync::Arc;
27use std::time::Duration;
28
29use astrs_wire::{
30    DataId, DurationMs, NodeEvent, NodeHandshake, NodeRequest, NodeSpawnSpec, SessionId,
31    WireMessage,
32};
33use tokio::sync::mpsc;
34
35use crate::error::{NodeError, Result};
36use crate::events::{EventSource, EventStream};
37use crate::runtime::NodeRuntime;
38use crate::session::connect::{LinkDuplex, NodeLink};
39use crate::session::{OUTGOING_CAPACITY, SessionShared, pump};
40
41/// How long the registration exchange waits for its answer.
42pub const REGISTRATION_TIMEOUT: Duration = Duration::from_secs(10);
43
44/// What the registration produced.
45pub struct Registration {
46    /// The shared session state.
47    pub shared: Arc<SessionShared>,
48    /// The node's event stream.
49    pub events: EventStream,
50    /// The running reader and writer tasks.
51    pub tasks: pump::SessionTasks,
52    /// The effective specification the daemon assigned.
53    pub spec: Arc<NodeSpawnSpec>,
54}
55
56impl core::fmt::Debug for Registration {
57    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
58        f.debug_struct("Registration")
59            .field("node", &self.spec.node)
60            .field("generation", &self.spec.generation)
61            .finish_non_exhaustive()
62    }
63}
64
65/// Registers over an established link and starts the session.
66///
67/// # Errors
68///
69/// [`NodeError::Registration`] when the daemon refuses or answers with
70/// something else, [`NodeError::Timeout`] when it does not answer at all, and
71/// [`NodeError::Transport`] when the link fails mid-handshake.
72pub async fn register(
73    link: NodeLink,
74    handshake: NodeHandshake,
75    runtime: NodeRuntime,
76    zero_copy_threshold: u64,
77    subscribe_to: Vec<DataId>,
78) -> Result<Registration> {
79    register_with(
80        link,
81        handshake,
82        runtime,
83        zero_copy_threshold,
84        subscribe_to,
85        None,
86    )
87    .await
88}
89
90/// [`register`], told where the daemon's segment broker listens (§6.2).
91///
92/// The broker socket travels in [`astrs_wire::NodeConfig::shm_broker`], and it
93/// is what turns a `RouteUpgrade` into a mapped segment on every platform —
94/// see [`crate::session::routes::open_producer`]. Separate from [`register`]
95/// so a caller that has no configuration blob (the in-process test harness)
96/// keeps its shorter call.
97///
98/// # Errors
99///
100/// As [`register`].
101pub async fn register_with(
102    link: NodeLink,
103    handshake: NodeHandshake,
104    runtime: NodeRuntime,
105    zero_copy_threshold: u64,
106    subscribe_to: Vec<DataId>,
107    shm_broker: Option<std::path::PathBuf>,
108) -> Result<Registration> {
109    let limits = link.session.frame_limits();
110    let session_id = link.session.session_id;
111    let mut duplex = link.duplex;
112
113    duplex
114        .send_message(&NodeRequest::Register(handshake))
115        .await?;
116
117    let (spec, session, deferred) = await_registered(&mut duplex, session_id).await?;
118    let spec = Arc::new(spec);
119
120    let source = Arc::new(EventSource::new());
121    for input in &spec.inputs {
122        source.register_input(input)?;
123    }
124
125    let (sender, receiver) = mpsc::channel(OUTGOING_CAPACITY);
126    let shared = Arc::new(
127        SessionShared::new(
128            Arc::clone(&spec),
129            session,
130            sender,
131            Arc::clone(&source),
132            runtime.clone(),
133            zero_copy_threshold,
134            limits,
135        )
136        .with_shm_broker(shm_broker),
137    );
138
139    duplex
140        .send_message(&NodeRequest::Subscribe {
141            inputs: subscribe_to,
142        })
143        .await?;
144
145    let (reader, writer) = duplex.into_halves();
146    let tasks = pump::spawn(&shared, reader, writer, receiver);
147
148    // A node that stops reading must stop the daemon queueing for it (§7.3),
149    // and the only moment that is observable is when the stream is dropped.
150    {
151        let notify = Arc::clone(&shared);
152        source.set_abandon_hook(Box::new(move || {
153            let _ = notify.send_request(NodeRequest::EventStreamDropped);
154        }));
155    }
156
157    // The other half of §11.3: a deadline violation `EventSource` measures
158    // locally is always turned into a control-lane `Event::Error` (so the
159    // node itself sees it even offline), but only a *connected* node can
160    // relay it to the daemon for the `astrs/status` fan-out and the
161    // registered metric — this is the one place a live session exists to
162    // install that relay. `send_request` (not `_async`) matches
163    // `EventStreamDropped` above: the hook itself has no `.await` point to
164    // offer (its signature is a plain `Fn`, called synchronously from
165    // inside `EventSource::finish_deadlines`, itself invoked from *both*
166    // `SessionShared::send_request` and `send_request_async`'s publish
167    // path), so the sync call is the only one it *can* make — safe from
168    // either caller because `send_request` only ever blocks past
169    // `NodeRuntime::block_on` when the outgoing channel is momentarily
170    // full, and that call is what already makes blocking-from-async safe
171    // throughout this crate: `tokio::task::block_in_place` on a
172    // multi-threaded runtime, or a clean `NodeError::BlockingInAsync`
173    // (never a panic) on a single-threaded one. And, exactly like a
174    // dropped event stream, a relay that cannot be enqueued right now (the
175    // daemon is gone, the outgoing channel is closed, or that
176    // single-threaded-runtime error) is not a condition this node can act
177    // on differently, so the error is intentionally discarded rather than
178    // propagated out of a publish call that already succeeded on its own
179    // terms.
180    {
181        let notify = Arc::clone(&shared);
182        source.set_deadline_violation_hook(Box::new(move |input, budget, latency| {
183            let _ = notify.send_request(NodeRequest::ReportDeadlineViolation {
184                input,
185                budget: DurationMs::from_duration(budget),
186                latency: DurationMs::from_duration(latency),
187            });
188        }));
189    }
190
191    // Anything the daemon sent before the queues existed goes through the
192    // ordinary dispatcher now, in arrival order.
193    for event in deferred {
194        pump::dispatch(&shared, event);
195    }
196
197    let events = EventStream::new(source, runtime);
198    Ok(Registration {
199        shared,
200        events,
201        tasks,
202        spec,
203    })
204}
205
206/// Reads frames until `Registered` arrives, buffering everything else.
207async fn await_registered(
208    duplex: &mut LinkDuplex,
209    expected_session: SessionId,
210) -> Result<(NodeSpawnSpec, SessionId, Vec<NodeEvent>)> {
211    let mut deferred = Vec::new();
212    let deadline = tokio::time::Instant::now() + REGISTRATION_TIMEOUT;
213    loop {
214        let frame = tokio::time::timeout_at(deadline, duplex.recv_frame())
215            .await
216            .map_err(|_| NodeError::Timeout {
217                operation: "node registration",
218                millis: as_millis(REGISTRATION_TIMEOUT),
219            })??;
220        let Some(frame) = frame else {
221            return Err(NodeError::Registration(
222                "the daemon closed the connection before answering the registration".to_owned(),
223            ));
224        };
225        if frame.kind() != astrs_wire::FrameKind::NodeEvent {
226            return Err(NodeError::Registration(format!(
227                "the daemon answered a registration with a `{}` frame",
228                frame.kind()
229            )));
230        }
231        match NodeEvent::from_frame(&frame.as_view())? {
232            NodeEvent::Registered { spec, session } => {
233                if session != expected_session {
234                    // Not fatal — the greeting's session and the
235                    // registration's are allowed to differ if a daemon
236                    // resumes — but worth carrying the daemon's answer, which
237                    // is the authoritative one.
238                }
239                return Ok((*spec, session, deferred));
240            }
241            NodeEvent::Stop { cause, .. } => {
242                return Err(NodeError::Registration(format!(
243                    "the daemon stopped this node during registration: {cause}"
244                )));
245            }
246            other => deferred.push(other),
247        }
248    }
249}
250
251/// Milliseconds, saturating rather than wrapping.
252fn as_millis(duration: Duration) -> u64 {
253    u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
254}
255
256#[cfg(test)]
257mod tests {
258    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
259
260    use super::*;
261    use crate::session::connect::wrap_stream;
262    use astrs_wire::{
263        DataflowId, FrameKind, FrameLimits, InputSpec, Metadata, NodeId, NodeSource, OutputPayload,
264        PortRef, StopCause,
265    };
266
267    fn spec() -> NodeSpawnSpec {
268        NodeSpawnSpec::new(
269            DataflowId::from_u128(4),
270            NodeId::new("detect").unwrap(),
271            1,
272            NodeSource::Dynamic,
273        )
274        .with_input(InputSpec::new(
275            DataId::new("frames").unwrap(),
276            PortRef::from_parts("camera", "image").unwrap(),
277        ))
278    }
279
280    /// As [`spec`], but `frames` carries a 1ms deadline (§11.3) — short
281    /// enough that an ordinary test-thread scheduling delay between the
282    /// input arriving and the node's next publish reliably runs it over
283    /// budget, without a test-visible sleep longer than a couple of
284    /// milliseconds.
285    fn spec_with_deadline() -> NodeSpawnSpec {
286        NodeSpawnSpec::new(
287            DataflowId::from_u128(4),
288            NodeId::new("detect").unwrap(),
289            1,
290            NodeSource::Dynamic,
291        )
292        .with_input(InputSpec {
293            deadline: Some(DurationMs::new(1)),
294            ..InputSpec::new(
295                DataId::new("frames").unwrap(),
296                PortRef::from_parts("camera", "image").unwrap(),
297            )
298        })
299    }
300
301    fn link(io: tokio::io::DuplexStream) -> NodeLink {
302        NodeLink {
303            duplex: wrap_stream(io, FrameLimits::uds()),
304            session: astrs_wire::NegotiatedSession {
305                protocol: astrs_wire::PROTOCOL_VERSION,
306                session_id: SessionId::from_u128(11),
307                features: astrs_wire::FeatureFlags::EMPTY,
308                limits: astrs_wire::NegotiatedLimits::uds(),
309                role: astrs_wire::Role::Node,
310                peer_version: astrs_wire::AstrsVersion::current(),
311                resumed: false,
312            },
313            endpoint: "duplex".to_owned(),
314        }
315    }
316
317    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
318    async fn the_registration_exchange_completes_and_starts_the_pump() {
319        let (node_io, daemon_io) = tokio::io::duplex(64 * 1024);
320        let mut daemon = wrap_stream(daemon_io, FrameLimits::uds());
321
322        let daemon_task = tokio::spawn(async move {
323            let request = daemon
324                .expect_message::<NodeRequest>(FrameKind::NodeRequest)
325                .await
326                .unwrap();
327            assert!(matches!(request, NodeRequest::Register(_)));
328            daemon
329                .send_message(&NodeEvent::Registered {
330                    spec: Box::new(spec()),
331                    session: SessionId::from_u128(11),
332                })
333                .await
334                .unwrap();
335            let request = daemon
336                .expect_message::<NodeRequest>(FrameKind::NodeRequest)
337                .await
338                .unwrap();
339            assert!(matches!(request, NodeRequest::Subscribe { .. }));
340            daemon
341                .send_message(&NodeEvent::Input {
342                    id: DataId::new("frames").unwrap(),
343                    source: PortRef::from_parts("camera", "image").unwrap(),
344                    metadata: Metadata::default(),
345                    payload: vec![1, 2],
346                })
347                .await
348                .unwrap();
349            daemon
350        });
351
352        let handshake =
353            NodeHandshake::dynamic(DataflowId::from_u128(4), NodeId::new("detect").unwrap());
354        let registration = register(
355            link(node_io),
356            handshake,
357            NodeRuntime::acquire().unwrap(),
358            4096,
359            Vec::new(),
360        )
361        .await
362        .unwrap();
363
364        assert_eq!(registration.spec.generation, 1);
365        assert_eq!(registration.spec.inputs.len(), 1);
366        assert!(format!("{registration:?}").contains("detect"));
367
368        let mut events = registration.events;
369        let event = events
370            .recv_async_timeout(Duration::from_secs(5))
371            .await
372            .unwrap();
373        assert!(event.is_input());
374
375        registration.shared.close();
376        registration.tasks.abort();
377        let _daemon = daemon_task.await.unwrap();
378    }
379
380    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
381    async fn events_that_arrive_before_registered_are_replayed_not_lost() {
382        let (node_io, daemon_io) = tokio::io::duplex(64 * 1024);
383        let mut daemon = wrap_stream(daemon_io, FrameLimits::uds());
384
385        let daemon_task = tokio::spawn(async move {
386            let _ = daemon
387                .expect_message::<NodeRequest>(FrameKind::NodeRequest)
388                .await
389                .unwrap();
390            // An input *before* the registration answer.
391            daemon
392                .send_message(&NodeEvent::Input {
393                    id: DataId::new("frames").unwrap(),
394                    source: PortRef::from_parts("camera", "image").unwrap(),
395                    metadata: Metadata::default(),
396                    payload: vec![9],
397                })
398                .await
399                .unwrap();
400            daemon
401                .send_message(&NodeEvent::Registered {
402                    spec: Box::new(spec()),
403                    session: SessionId::from_u128(11),
404                })
405                .await
406                .unwrap();
407            let _ = daemon
408                .expect_message::<NodeRequest>(FrameKind::NodeRequest)
409                .await
410                .unwrap();
411            daemon
412        });
413
414        let registration = register(
415            link(node_io),
416            NodeHandshake::dynamic(DataflowId::from_u128(4), NodeId::new("detect").unwrap()),
417            NodeRuntime::acquire().unwrap(),
418            4096,
419            Vec::new(),
420        )
421        .await
422        .unwrap();
423
424        let mut events = registration.events;
425        let event = events
426            .recv_async_timeout(Duration::from_secs(5))
427            .await
428            .unwrap();
429        let Some((id, _, data)) = event.into_input() else {
430            panic!("expected the replayed input");
431        };
432        assert_eq!(id.as_str(), "frames");
433        assert_eq!(data.to_vec(), vec![9]);
434
435        registration.shared.close();
436        registration.tasks.abort();
437        let _daemon = daemon_task.await.unwrap();
438    }
439
440    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
441    async fn a_stop_during_registration_is_reported() {
442        let (node_io, daemon_io) = tokio::io::duplex(64 * 1024);
443        let mut daemon = wrap_stream(daemon_io, FrameLimits::uds());
444        let daemon_task = tokio::spawn(async move {
445            let _ = daemon
446                .expect_message::<NodeRequest>(FrameKind::NodeRequest)
447                .await
448                .unwrap();
449            daemon
450                .send_message(&NodeEvent::Stop {
451                    cause: StopCause::Destroyed,
452                    grace: None,
453                })
454                .await
455                .unwrap();
456            daemon
457        });
458
459        let error = register(
460            link(node_io),
461            NodeHandshake::dynamic(DataflowId::from_u128(4), NodeId::new("detect").unwrap()),
462            NodeRuntime::acquire().unwrap(),
463            4096,
464            Vec::new(),
465        )
466        .await
467        .unwrap_err();
468        assert!(matches!(error, NodeError::Registration(_)), "{error}");
469        let _daemon = daemon_task.await.unwrap();
470    }
471
472    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
473    async fn a_daemon_that_hangs_up_is_reported() {
474        let (node_io, daemon_io) = tokio::io::duplex(64 * 1024);
475        drop(daemon_io);
476        let error = register(
477            link(node_io),
478            NodeHandshake::dynamic(DataflowId::from_u128(4), NodeId::new("detect").unwrap()),
479            NodeRuntime::acquire().unwrap(),
480            4096,
481            Vec::new(),
482        )
483        .await
484        .unwrap_err();
485        assert!(
486            matches!(error, NodeError::Registration(_) | NodeError::Transport(_)),
487            "{error}"
488        );
489    }
490
491    /// The production wiring this module's own `register_with` adds beside
492    /// its abandon-hook install: a deadline violation `EventSource` measures
493    /// locally must reach the daemon as [`NodeRequest::ReportDeadlineViolation`],
494    /// not only the node's own control-lane `Event::Error` — see
495    /// `EventSource`'s `DeadlineViolationHook` docs for the half this test
496    /// does *not* cover (that local event is asserted directly in
497    /// `astrs_node_api::events::source`'s own tests).
498    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
499    async fn a_deadline_violation_is_relayed_to_the_daemon() {
500        let (node_io, daemon_io) = tokio::io::duplex(64 * 1024);
501        let mut daemon = wrap_stream(daemon_io, FrameLimits::uds());
502
503        let daemon_task = tokio::spawn(async move {
504            let request = daemon
505                .expect_message::<NodeRequest>(FrameKind::NodeRequest)
506                .await
507                .unwrap();
508            assert!(matches!(request, NodeRequest::Register(_)));
509            daemon
510                .send_message(&NodeEvent::Registered {
511                    spec: Box::new(spec_with_deadline()),
512                    session: SessionId::from_u128(11),
513                })
514                .await
515                .unwrap();
516            let request = daemon
517                .expect_message::<NodeRequest>(FrameKind::NodeRequest)
518                .await
519                .unwrap();
520            assert!(matches!(request, NodeRequest::Subscribe { .. }));
521            daemon
522                .send_message(&NodeEvent::Input {
523                    id: DataId::new("frames").unwrap(),
524                    source: PortRef::from_parts("camera", "image").unwrap(),
525                    metadata: Metadata::default(),
526                    payload: vec![1, 2],
527                })
528                .await
529                .unwrap();
530
531            // The node's simulated publish, which closes the open deadline
532            // measurement — always sent first (`finish_deadlines` runs
533            // synchronously *after* this request is already enqueued; see
534            // `SessionShared::send_request`'s own docs).
535            let request = daemon
536                .expect_message::<NodeRequest>(FrameKind::NodeRequest)
537                .await
538                .unwrap();
539            assert!(matches!(request, NodeRequest::SendMessage { .. }));
540
541            // The relay this test exists to prove: queued from inside the
542            // `DeadlineViolationHook` this module's `register_with` installs,
543            // strictly after the publish above on the same outgoing channel.
544            let request = daemon
545                .expect_message::<NodeRequest>(FrameKind::NodeRequest)
546                .await
547                .unwrap();
548            match request {
549                NodeRequest::ReportDeadlineViolation {
550                    input,
551                    budget,
552                    latency,
553                } => {
554                    assert_eq!(input.as_str(), "frames");
555                    assert_eq!(budget, DurationMs::new(1));
556                    assert!(latency >= budget, "{latency:?} >= {budget:?}");
557                }
558                other => panic!("expected ReportDeadlineViolation, got {other:?}"),
559            }
560            daemon
561        });
562
563        let registration = register(
564            link(node_io),
565            NodeHandshake::dynamic(DataflowId::from_u128(4), NodeId::new("detect").unwrap()),
566            NodeRuntime::acquire().unwrap(),
567            4096,
568            Vec::new(),
569        )
570        .await
571        .unwrap();
572
573        let mut events = registration.events;
574        let event = events
575            .recv_async_timeout(Duration::from_secs(5))
576            .await
577            .unwrap();
578        assert!(event.is_input(), "opens the deadline measurement");
579
580        // Comfortably past the 1ms budget, so the publish below is judged
581        // late regardless of test-runner scheduling jitter.
582        tokio::time::sleep(Duration::from_millis(20)).await;
583
584        registration
585            .shared
586            .send_request(NodeRequest::SendMessage {
587                output: DataId::new("boxes").unwrap(),
588                metadata: Metadata::default(),
589                payload: OutputPayload::inline(vec![0]),
590            })
591            .unwrap();
592
593        // `send_request` only enqueues onto the outgoing channel — the
594        // writer task still has to flush both the publish above and the
595        // relay it triggers onto the duplex. Awaiting the daemon side (whose
596        // own `expect_message` calls simply wait for those bytes to arrive)
597        // before tearing the session down avoids racing `tasks.abort()`
598        // against that still-in-flight write.
599        let _daemon = daemon_task.await.unwrap();
600
601        registration.shared.close();
602        registration.tasks.abort();
603    }
604}