Skip to main content

plexus_core/plexus/remote_child/
link.rs

1//! One connection to a child — its frames out, its answers routed back —
2//! and the supervisor that starts it, notices it gone and starts it again.
3//!
4//! A [`Conn`] is one process (stdio) or one socket (ws): a writer task that
5//! drains everything queued into one write per wakeup (spec/11), a reader
6//! that routes a response by `id` to the call that opened it (registering
7//! the subscription before the next message is read, so an item can never
8//! beat its id — spec/02) and an item by `subscription` to that call's
9//! bounded channel, and — when the far side goes — one `child_unavailable`
10//! to every call it held. A [`Shared`] outlives its connections: it keeps
11//! the schema's hash, the counters and the restart window.
12
13use std::borrow::Cow;
14use std::collections::{HashMap, HashSet, VecDeque};
15use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
16use std::sync::{Arc, Mutex, OnceLock, Weak};
17use std::time::{Duration, Instant};
18
19use serde::Deserialize;
20use serde_json::value::RawValue;
21use serde_json::{json, Value};
22use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWriteExt, BufReader};
23use tokio::sync::{mpsc, oneshot, Notify};
24
25use super::spec::{ChildOptions, ChildSpec};
26use super::upcall::{self, Served, Upcalls};
27use crate::plexus::types::{PlexusStreamItem, StreamMetadata};
28use plexus_client::RemoteError;
29
30/// A subscription id as the child spelled it (its JSON text: `4`, `"a"`).
31pub(super) type SubKey = String;
32
33pub(super) fn meta(name: &str) -> StreamMetadata {
34    StreamMetadata::new(vec![name.to_string()], crate::plexus::context::PlexusContext::hash())
35}
36
37/// `child_unavailable` (spec/12 rule 5): one error, and the end.
38pub(super) fn unavailable(name: &str, why: &str) -> PlexusStreamItem {
39    PlexusStreamItem::error(meta(name), RemoteError::ChildUnavailable { child: name.into(), reason: why.into() }, true)
40}
41
42/// `child_timeout`: the child did not open the call in the service's time.
43pub(super) fn timed_out(name: &str, timeout: Duration) -> PlexusStreamItem {
44    let error = RemoteError::ChildTimeout { child: name.into(), timeout_ms: timeout.as_millis() as u64 };
45    let mut item = PlexusStreamItem::error(meta(name), error, true);
46    if let PlexusStreamItem::Error { message, .. } = &mut item {
47        *message = format!("child {name} did not open the call in {timeout:?}");
48    }
49    item
50}
51
52/// `too_many_calls` (spec/05, 09): as a connection's limit answers.
53pub(super) fn too_many(name: &str, max: usize) -> PlexusStreamItem {
54    PlexusStreamItem::error(meta(name), RemoteError::TooManyCalls { max: max as u64, child: Some(name.into()) }, true)
55}
56
57/// A call the child refused at the JSON-RPC layer (`{code, message,
58/// data?}`): its `data` when that is a typed [`RemoteError`] (RETURNS-6),
59/// else what its code says; spec/05's `message` and `code` as it sent them.
60fn refused(name: &str, e: &Value) -> PlexusStreamItem {
61    let code = e.get("code").map(|c| match c {
62        Value::String(s) => s.clone(),
63        other => other.to_string(),
64    });
65    let message = e.get("message").and_then(Value::as_str).unwrap_or("the child refused the call").to_string();
66    let error = e
67        .get("data")
68        .and_then(|d| serde_json::from_value::<RemoteError>(d.clone()).ok())
69        .unwrap_or_else(|| super::peer::remote_error_of_code(e.get("code").and_then(Value::as_i64).unwrap_or(-32000), &message));
70    let fields = crate::plexus::types::spec_fields(&error);
71    PlexusStreamItem::Error { metadata: meta(name), message, code, recoverable: false, error, fields }
72}
73
74/// Counters that outlive a connection.
75#[derive(Default)]
76pub(super) struct Counters {
77    pub unasked: AtomicU64,
78    pub calls: AtomicU64,
79    pub restarts: AtomicU64,
80    pub open: AtomicUsize,
81}
82
83/// How a call's open failed.
84pub(super) enum OpenFail {
85    /// The connection went before the child answered.
86    Gone(String),
87    /// The child answered the open with a JSON-RPC error.
88    Refused(Value),
89}
90
91struct Opening {
92    reply: oneshot::Sender<Result<SubKey, OpenFail>>,
93    items: Sub,
94}
95
96/// An open call's items: where they go, and the method called — the
97/// `content_type` a progress item as main wrote it (none of its own) is
98/// named for (RETURNS-36, re-review #3).
99struct Sub {
100    tx: mpsc::Sender<PlexusStreamItem>,
101    method: Arc<str>,
102    /// KEEL-59 (N3): the call it was opened in, told when the peer is cut
103    /// off — until this call into the peer is let go.
104    mark: Option<Arc<PeerMark>>,
105}
106
107/// KEEL-59 (N3): one call into a peer, and the call it was opened in: set
108/// when the peer is cut off, taken when the call into it is let go.
109pub(super) struct PeerMark {
110    call: Option<crate::call::CallContext>,
111    set: std::sync::atomic::AtomicBool,
112}
113
114impl PeerMark {
115    /// For the call this runs in — only while it is routed (P2): a call
116    /// into the peer opened then is the dispatched call's own forward,
117    /// whose end is the call's next item. One a method opens from its
118    /// stream is its own business: its peer's end exempts nothing, read or
119    /// not.
120    pub(super) fn here() -> Arc<Self> {
121        let call = crate::call::context().ok().filter(crate::call::CallContext::routing);
122        Arc::new(Self { call, set: std::sync::atomic::AtomicBool::new(false) })
123    }
124
125    /// The peer was cut off: its end is on its way.
126    fn set(&self) {
127        if !self.set.swap(true, Ordering::AcqRel) {
128            if let Some(c) = &self.call {
129                c.peer_ended();
130            }
131        }
132    }
133
134    /// The call into the peer is let go.
135    pub(super) fn taken(&self) {
136        if self.set.swap(false, Ordering::AcqRel) {
137            if let Some(c) = &self.call {
138                c.peer_end_taken();
139            }
140        }
141    }
142}
143
144/// How long after its `true` an unsubscribed call's late item is still the
145/// cancel's tail: spec/04 lets an item already in flight land just after
146/// the response (the kit allows 50 ms).
147const TAIL: Duration = Duration::from_millis(250);
148
149#[derive(Default)]
150struct Routes {
151    pending: HashMap<u64, Opening>,
152    subs: HashMap<SubKey, Sub>,
153    /// Unsubscribes sent and not yet answered, and those answered within
154    /// [`TAIL`]: their late items are the cancel's tail, dropped quietly
155    /// (not "unasked").
156    closing: HashMap<u64, SubKey>,
157    closing_subs: HashSet<SubKey>,
158    answered: VecDeque<(Instant, SubKey)>,
159    /// Opens past their timeout: a late answer is unsubscribed at once.
160    abandoned: HashSet<u64>,
161}
162
163impl Routes {
164    /// Forget the cancels answered more than [`TAIL`] ago.
165    fn expire_tails(&mut self, now: Instant) {
166        while self.answered.front().is_some_and(|(t, _)| now.duration_since(*t) >= TAIL) {
167            if let Some((_, key)) = self.answered.pop_front() {
168                self.closing_subs.remove(&key);
169            }
170        }
171    }
172}
173
174/// One connection to the child.
175pub(super) struct Conn {
176    pub name: String,
177    pub pid: Option<u32>,
178    /// The child's own namespace (its `_info`'s `backend`).
179    pub ns: OnceLock<String>,
180    out: mpsc::UnboundedSender<Vec<u8>>,
181    routes: Mutex<Routes>,
182    next_id: AtomicU64,
183    alive: AtomicBool,
184    /// Asked to end (the service let go of the child, or refused its schema).
185    pub shutdown: Notify,
186    buffer: usize,
187    counters: Arc<Counters>,
188    /// NEST-11: the mount's calls back from its child, when it takes them
189    /// (a child it spawned or dialed); unset, a request from the far side
190    /// is refused.
191    pub up: OnceLock<Arc<Upcalls>>,
192    /// NEST-11: the calls back this connection is serving.
193    served: Arc<Served>,
194    /// DYNAUTH-8: on `wss://`, the sha256 (hex) of the key the far side's
195    /// certificate holds (its SubjectPublicKeyInfo).
196    pub tls_key: OnceLock<String>,
197    /// DYNAUTH-5: the Ed25519 key the far side proved at `_info`
198    /// (`key:<hex>`), if it proved one.
199    pub peer_key: OnceLock<String>,
200    /// KEEL-59: why the service cut this connection's calls off (a revoke,
201    /// a disconnect), once it has: nothing more is opened on it.
202    stopped: OnceLock<String>,
203}
204
205/// One message, read in one pass: a notification's item stays raw until
206/// its call is known (then it is read once, as the item).
207#[derive(Deserialize)]
208struct Msg<'a> {
209    #[serde(default)]
210    id: Option<Value>,
211    #[serde(default, borrow)]
212    result: Option<&'a RawValue>,
213    #[serde(default)]
214    error: Option<Value>,
215    #[serde(default, borrow)]
216    method: Option<Cow<'a, str>>,
217    #[serde(default, borrow)]
218    params: Option<Notif<'a>>,
219}
220
221#[derive(Deserialize)]
222struct Notif<'a> {
223    #[serde(default)]
224    subscription: Option<Value>,
225    #[serde(default, borrow)]
226    result: Option<&'a RawValue>,
227}
228
229/// A message whose `params` are not an object (a request with positional
230/// params): read loosely, only to answer it.
231#[derive(Deserialize)]
232struct Loose<'a> {
233    #[serde(default)]
234    id: Option<Value>,
235    #[serde(default, borrow)]
236    method: Option<Cow<'a, str>>,
237}
238
239#[derive(Deserialize)]
240struct Kind<'a> {
241    #[serde(rename = "type", borrow)]
242    kind: Cow<'a, str>,
243}
244
245impl Conn {
246    pub(super) fn new(name: &str, pid: Option<u32>, out: mpsc::UnboundedSender<Vec<u8>>, buffer: usize, counters: Arc<Counters>) -> Arc<Self> {
247        Arc::new(Self {
248            name: name.to_string(),
249            pid,
250            ns: OnceLock::new(),
251            out,
252            routes: Mutex::new(Routes::default()),
253            next_id: AtomicU64::new(1),
254            alive: AtomicBool::new(true),
255            shutdown: Notify::new(),
256            buffer,
257            counters,
258            up: OnceLock::new(),
259            served: Arc::default(),
260            tls_key: OnceLock::new(),
261            peer_key: OnceLock::new(),
262            stopped: OnceLock::new(),
263        })
264    }
265
266    pub(super) fn alive(&self) -> bool {
267        self.alive.load(Ordering::Acquire)
268    }
269
270    pub(super) fn ns(&self) -> &str {
271        self.ns.get().map(String::as_str).unwrap_or(&self.name)
272    }
273
274    fn send(&self, frame: &Value) {
275        let _ = self.out.send(serde_json::to_vec(frame).unwrap_or_default());
276    }
277
278    /// Open a call: `method` with `params` (a `{ns}.call`'s params, or
279    /// `_info`'s null); the subscription id back within `timeout`, its items
280    /// already routed to the receiver.
281    pub(super) async fn open(&self, method: &str, params: Value, timeout: Duration, mark: Option<Arc<PeerMark>>) -> Result<(SubKey, mpsc::Receiver<PlexusStreamItem>), PlexusStreamItem> {
282        let id = self.next_id.fetch_add(1, Ordering::Relaxed);
283        let (tx, rx) = mpsc::channel(self.buffer);
284        let (reply, answer) = oneshot::channel();
285        {
286            let mut r = self.routes.lock().unwrap();
287            if !self.alive() {
288                return Err(unavailable(&self.name, "the connection has closed"));
289            }
290            // KEEL-59: checked under the lock `stop` takes, so a call is
291            // either cut off with the rest or never opened.
292            if let Some(why) = self.stopped.get() {
293                return Err(unavailable(&self.name, why));
294            }
295            // a `{ns}.call`'s params name the method called (`echo.count`)
296            let called = params.get("method").and_then(Value::as_str).unwrap_or(method);
297            r.pending.insert(id, Opening { reply, items: Sub { tx, method: called.into(), mark } });
298        }
299        self.send(&json!({ "jsonrpc": "2.0", "id": id, "method": method, "params": params }));
300        match tokio::time::timeout(timeout, answer).await {
301            Ok(Ok(Ok(key))) => Ok((key, rx)),
302            Ok(Ok(Err(OpenFail::Gone(why)))) => Err(unavailable(&self.name, &why)),
303            Ok(Err(_)) => Err(unavailable(&self.name, "the connection closed before the call opened")),
304            Ok(Ok(Err(OpenFail::Refused(e)))) => Err(refused(&self.name, &e)),
305            Err(_) => {
306                let mut r = self.routes.lock().unwrap();
307                if r.pending.remove(&id).is_some() {
308                    r.abandoned.insert(id);
309                }
310                Err(timed_out(&self.name, timeout))
311            }
312        }
313    }
314
315    /// The caller let go of `key` before its end: forget it, and tell the
316    /// child (`{ns}.call_unsub [sid]`, spec/04).
317    pub(super) fn unsubscribe(&self, key: &str) {
318        let id = {
319            let mut r = self.routes.lock().unwrap();
320            if r.subs.remove(key).is_none() || !self.alive() {
321                return;
322            }
323            self.closing(&mut r, key)
324        };
325        self.send_unsub(id, key);
326    }
327
328    /// `key` is being unsubscribed: its late items are the cancel's tail.
329    fn closing(&self, r: &mut Routes, key: &str) -> u64 {
330        let id = self.next_id.fetch_add(1, Ordering::Relaxed);
331        r.closing.insert(id, key.to_string());
332        r.closing_subs.insert(key.to_string());
333        id
334    }
335
336    /// `{ns}.call_unsub [key]` to the child, as request `id`.
337    fn send_unsub(&self, id: u64, key: &str) {
338        let method = serde_json::to_string(&format!("{}.call_unsub", self.ns())).unwrap_or_default();
339        let frame = format!(r#"{{"jsonrpc":"2.0","id":{id},"method":{method},"params":[{key}]}}"#);
340        let _ = self.out.send(frame.into_bytes());
341    }
342
343    /// KEEL-59: the service cuts this connection's calls off (`dyn.revoke`,
344    /// `dyn.disconnect`): every call open into the child ends with one
345    /// `child_unavailable` item saying `why`, the child is told to stop each
346    /// (`call_unsub`, its late items the cancel's tail), a call still opening
347    /// is answered the same, and nothing more is opened on it. The socket
348    /// stays up: a reconnect is the service's to decide. The first `why`
349    /// stands.
350    pub(super) fn stop(&self, why: &str) {
351        let (subs, pending, unsubs) = {
352            let mut r = self.routes.lock().unwrap();
353            if self.stopped.set(why.to_string()).is_err() {
354                return;
355            }
356            let subs = std::mem::take(&mut r.subs);
357            let alive = self.alive();
358            let unsubs: Vec<(u64, SubKey)> = if alive { subs.keys().map(|k| (self.closing(&mut r, k), k.clone())).collect() } else { Vec::new() };
359            (subs, std::mem::take(&mut r.pending), unsubs)
360        };
361        for (id, key) in &unsubs {
362            self.send_unsub(*id, key);
363        }
364        for (_, Sub { tx, mark, .. }) in subs {
365            // Told before the item is sent: a grants move the cut-off makes
366            // leaves the call this end.
367            if let Some(m) = mark {
368                m.set();
369            }
370            Self::tell(tx, unavailable(&self.name, why));
371        }
372        for (_, open) in pending {
373            let _ = open.reply.send(Err(OpenFail::Gone(why.to_string())));
374        }
375    }
376
377    /// KEEL-59: why this connection's calls were cut off, if they were.
378    pub(super) fn stopped(&self) -> Option<&str> {
379        self.stopped.get().map(String::as_str)
380    }
381
382    /// A call's last item, without waiting on a caller that is not reading
383    /// (its buffer full: the item follows what it holds).
384    fn tell(tx: mpsc::Sender<PlexusStreamItem>, item: PlexusStreamItem) {
385        if let Err(mpsc::error::TrySendError::Full(item)) = tx.try_send(item) {
386            tokio::spawn(async move {
387                let _ = tx.send(item).await;
388            });
389        }
390    }
391
392    /// How many subscriptions are routed now.
393    pub(super) fn routed(&self) -> usize {
394        self.routes.lock().unwrap().subs.len()
395    }
396
397    /// One message from the child.
398    pub(super) async fn on_message(&self, raw: &[u8]) {
399        let msg: Msg<'_> = match serde_json::from_slice(raw) {
400            Ok(m) => m,
401            Err(e) => match serde_json::from_slice::<Loose<'_>>(raw) {
402                Ok(Loose { id: Some(id), method: Some(method) }) => return self.request(id, &method, raw),
403                _ => {
404                    tracing::warn!(target: "plexus::child", child = %self.name, "an unreadable message from the child: {e}");
405                    return;
406                }
407            },
408        };
409        if let Some(method) = msg.method {
410            if let Some(id) = msg.id {
411                return self.request(id, &method, raw);
412            }
413            match msg.params {
414                Some(Notif { subscription: Some(sub), result: Some(item) }) => self.route(sub.to_string(), item).await,
415                _ => tracing::warn!(target: "plexus::child", child = %self.name, %method, "a notification without subscription and result"),
416            }
417            return;
418        }
419        let Some(id) = msg.id.as_ref().and_then(Value::as_u64) else { return };
420        let mut r = self.routes.lock().unwrap();
421        if let Some(open) = r.pending.remove(&id) {
422            match (msg.result, msg.error) {
423                (Some(sid), _) => {
424                    let key = match serde_json::from_str::<Value>(sid.get()) {
425                        Ok(v) => v.to_string(),
426                        Err(_) => sid.get().to_string(),
427                    };
428                    r.subs.insert(key.clone(), open.items);
429                    let _ = open.reply.send(Ok(key));
430                }
431                (None, e) => {
432                    let _ = open.reply.send(Err(OpenFail::Refused(e.unwrap_or_else(|| json!({ "message": "an answer with neither result nor error" })))));
433                }
434            }
435        } else if let Some(key) = r.closing.remove(&id) {
436            // Answered: an item still in flight is the tail for TAIL more.
437            let now = Instant::now();
438            r.expire_tails(now);
439            r.answered.push_back((now, key));
440        } else if r.abandoned.remove(&id) {
441            // A call opened after its caller was told `child_timeout`: stop it.
442            if let Some(sid) = msg.result.and_then(|s| serde_json::from_str::<Value>(s.get()).ok()) {
443                let key = sid.to_string();
444                let uid = self.next_id.fetch_add(1, Ordering::Relaxed);
445                r.closing.insert(uid, key.clone());
446                r.closing_subs.insert(key.clone());
447                drop(r);
448                let method = serde_json::to_string(&format!("{}.call_unsub", self.ns())).unwrap_or_default();
449                let _ = self.out.send(format!(r#"{{"jsonrpc":"2.0","id":{uid},"method":{method},"params":[{key}]}}"#).into_bytes());
450            }
451        }
452    }
453
454    /// A request from the child: a call back under a call it serves
455    /// (NEST-11, `_host.call`), when this mount takes them; anything else is
456    /// refused.
457    fn request(&self, id: Value, method: &str, raw: &[u8]) {
458        if let (Some(up), true) = (self.up.get(), method == upcall::HOST_CALL || method == upcall::HOST_CALL_UNSUB) {
459            let params = serde_json::from_slice::<Value>(raw).ok().and_then(|mut v| v.get_mut("params").map(Value::take)).unwrap_or(Value::Null);
460            return upcall::on_request(up, &self.served, &self.out, &self.name, id, method, params);
461        }
462        self.refuse_request(id, method)
463    }
464
465    /// A child asks its service nothing else (spec/12): answered, not served.
466    fn refuse_request(&self, id: Value, method: &str) {
467        tracing::warn!(target: "plexus::child", child = %self.name, %method, "a request from the child refused");
468        self.send(&json!({ "jsonrpc": "2.0", "id": id, "error": { "code": -32601, "message": "a service answers no requests from its child" } }));
469    }
470
471    async fn route(&self, key: SubKey, raw: &RawValue) {
472        let (tx, method) = {
473            let mut r = self.routes.lock().unwrap();
474            match r.subs.get(&key) {
475                Some(sub) => (sub.tx.clone(), Arc::clone(&sub.method)),
476                None => {
477                    r.expire_tails(Instant::now());
478                    if !r.closing_subs.contains(&key) {
479                        drop(r);
480                        self.counters.unasked.fetch_add(1, Ordering::Relaxed);
481                        tracing::warn!(target: "plexus::child", child = %self.name, subscription = %key, "unasked item dropped");
482                    }
483                    return;
484                }
485            }
486        };
487        // RETURNS-36: a peer on plexus-wire's spec as main wrote it is read
488        // into 0.8's shape (`peer::item`) when — only when — it does not
489        // read as 0.8.
490        let parsed = serde_json::from_str::<PlexusStreamItem>(raw.get()).or_else(|e| {
491            super::peer::item(raw.get(), &method)
492                .and_then(|v| serde_json::from_value::<PlexusStreamItem>(v).ok())
493                .ok_or(e)
494        });
495        let item = match parsed {
496            Ok(item) => item,
497            Err(e) => {
498                let kind = serde_json::from_str::<Kind<'_>>(raw.get()).map(|k| k.kind.into_owned()).unwrap_or_default();
499                if kind != "error" {
500                    // An item kind this service does not know (spec/03: a
501                    // reader skips it) — or one that does not read at all.
502                    tracing::debug!(target: "plexus::child", child = %self.name, subscription = %key, %kind, "an item kind not relayed: {e}");
503                    return;
504                }
505                // RETURNS-36 (review #4): an error item is the call's end
506                // (spec/03); skipped, the call would wait for an end that
507                // never comes. It ends here, naming what could not be read,
508                // and the child is told to stop.
509                tracing::warn!(target: "plexus::child", child = %self.name, subscription = %key, "an error item not read: {e}");
510                self.unsubscribe(&key);
511                let error = plexus_client::RemoteError::Execution {
512                    message: format!("child {} sent an error item this service could not read: {e}", self.name),
513                };
514                PlexusStreamItem::error(meta(&self.name), error, false)
515            }
516        };
517        if matches!(item, PlexusStreamItem::Done { .. } | PlexusStreamItem::Error { .. }) {
518            self.routes.lock().unwrap().subs.remove(&key);
519        }
520        match tx.try_send(item) {
521            Ok(()) => {}
522            // The caller has not read the call's buffer: wait for it
523            // (lossless; the child's other calls wait behind it).
524            Err(mpsc::error::TrySendError::Full(item)) => {
525                let _ = tx.send(item).await;
526            }
527            Err(mpsc::error::TrySendError::Closed(_)) => {}
528        }
529    }
530
531    /// The far side is gone: every open call gets one `child_unavailable`.
532    pub(super) fn on_gone(&self, why: &str) {
533        let (subs, pending) = {
534            let mut r = self.routes.lock().unwrap();
535            self.alive.store(false, Ordering::Release);
536            self.served.abort_all();
537            r.closing.clear();
538            r.closing_subs.clear();
539            r.answered.clear();
540            r.abandoned.clear();
541            (std::mem::take(&mut r.subs), std::mem::take(&mut r.pending))
542        };
543        for (_, Sub { tx, mark, .. }) in subs {
544            // KEEL-59: as `stop` — the peer gone, its end is the call's.
545            if let Some(m) = mark {
546                m.set();
547            }
548            Self::tell(tx, unavailable(&self.name, why));
549        }
550        for (_, open) in pending {
551            let _ = open.reply.send(Err(OpenFail::Gone(why.to_string())));
552        }
553    }
554}
555
556/// Read newline-delimited messages from `r`, one wakeup per read: every
557/// whole line in the buffer is routed before the next read.
558pub(super) async fn read_lines<R: AsyncRead + Unpin>(conn: &Conn, r: R) {
559    let mut r = BufReader::with_capacity(1 << 17, r);
560    let mut carry: Vec<u8> = Vec::new();
561    loop {
562        let buf = match r.fill_buf().await {
563            Ok(b) if !b.is_empty() => b,
564            _ => break,
565        };
566        let n = buf.len();
567        let mut start = 0;
568        while let Some(i) = buf[start..].iter().position(|b| *b == b'\n') {
569            let end = start + i;
570            if carry.is_empty() {
571                let line = &buf[start..end];
572                if line.iter().any(|b| !b.is_ascii_whitespace()) {
573                    conn.on_message(line).await;
574                }
575            } else {
576                carry.extend_from_slice(&buf[start..end]);
577                let line = std::mem::take(&mut carry);
578                if line.iter().any(|b| !b.is_ascii_whitespace()) {
579                    conn.on_message(&line).await;
580                }
581            }
582            start = end + 1;
583        }
584        carry.extend_from_slice(&buf[start..]);
585        r.consume(n);
586    }
587}
588
589/// The writer: everything queued, one write per wakeup, each line whole.
590pub(super) async fn write_lines<W: tokio::io::AsyncWrite + Unpin>(mut w: W, mut rx: mpsc::UnboundedReceiver<Vec<u8>>) {
591    while let Some(first) = rx.recv().await {
592        let mut buf = first;
593        buf.push(b'\n');
594        while let Ok(more) = rx.try_recv() {
595            buf.extend_from_slice(&more);
596            buf.push(b'\n');
597        }
598        if w.write_all(&buf).await.is_err() || w.flush().await.is_err() {
599            break;
600        }
601    }
602}
603
604/// When the child was mounted, how long each step took.
605#[derive(Debug, Clone, Copy, Default, PartialEq)]
606pub struct MountTimings {
607    /// Spawn (or dial) → its `_info` answered.
608    pub spawn_ms: f64,
609    /// `_info` → its schema answered.
610    pub schema_ms: f64,
611    /// The schema requalified under the mount name and kept.
612    pub merge_ms: f64,
613}
614
615struct LinkState {
616    conn: Option<Arc<Conn>>,
617    down_since: Option<Instant>,
618    why: String,
619    window: VecDeque<Instant>,
620    /// Never again (a restart published another schema): the reason.
621    dead: Option<String>,
622}
623
624/// The child across its connections.
625/// KEEL-19: the word a mount's refusal starts with when a keyed peer meets
626/// a service with no dial key.
627pub const KEYED_MOUNT_NEEDS_A_KEY: &str = "keyed_mount_needs_a_key";
628
629pub(super) struct Shared {
630    pub spec: ChildSpec,
631    pub opts: ChildOptions,
632    pub name: String,
633    /// The hop's `principal`.
634    pub principal: String,
635    pub counters: Arc<Counters>,
636    state: Mutex<LinkState>,
637    dial: tokio::sync::Mutex<()>,
638    /// The mounted schema's hash, as the child published it.
639    pub mounted_hash: OnceLock<String>,
640    /// NEST-11: the child's calls back into the service.
641    pub upcalls: Arc<Upcalls>,
642    /// DYNAUTH-8 (TM-3): who the child proved it was at the mount, pinned:
643    /// a restart that proves another is refused, as one that publishes
644    /// another schema is.
645    pub identity: OnceLock<super::PeerIdentity>,
646    /// KEEL-59: why the service cut the child off (a revoke, a disconnect),
647    /// once it has: every call into it ended, and nothing more forwarded.
648    stopped: OnceLock<String>,
649}
650
651impl Drop for Shared {
652    fn drop(&mut self) {
653        if let Ok(st) = self.state.lock() {
654            if let Some(c) = &st.conn {
655                c.shutdown.notify_one();
656            }
657        }
658    }
659}
660
661/// The hash a child's schema is compared by across restarts.
662pub(super) fn schema_hash(raw: &Value) -> String {
663    match raw.get("hash").and_then(Value::as_str) {
664        Some(h) => h.to_string(),
665        None => raw.get("methods").map(Value::to_string).unwrap_or_default(),
666    }
667}
668
669/// NEST-4 · what a mount keeps of a child: its own schema and, walked at
670/// mount, every static child below it by dotted path (`a`, `a.b`), each as
671/// the child published it.
672#[derive(Debug, Clone)]
673pub(super) struct Kept {
674    pub top: Value,
675    pub nodes: Vec<(String, Value)>,
676}
677
678impl Kept {
679    /// The whole tree's hash, compared across restarts: a flat child's is its
680    /// schema's, as before; a child with children adds each one's, by path,
681    /// so a changed grandchild changes it whatever the child's own hash
682    /// covers.
683    pub fn hash(&self) -> String {
684        let mut h = schema_hash(&self.top);
685        for (path, raw) in &self.nodes {
686            h.push_str(&format!(" {path}={}", schema_hash(raw)));
687        }
688        h
689    }
690}
691
692/// The static children a schema names, by the name each is reached through:
693/// its methods whose role is `static_child` when any method states a child
694/// role; else its `children` side-table (IR-4) — a service's root, whose
695/// registered activations are listed there and nowhere else.
696pub(super) fn static_children(raw: &Value) -> Vec<String> {
697    let methods = raw.get("methods").and_then(Value::as_array).map(Vec::as_slice).unwrap_or_default();
698    let kind = |m: &Value| m.get("role").and_then(|r| r.get("kind").or(Some(r))).and_then(Value::as_str).map(str::to_string);
699    // A macro-built activation says its children by role (a static child's
700    // name is its method's, which may differ from the child's namespace);
701    // a service's root (a DynamicHub) lists its activations in `children`.
702    if methods.iter().any(|m| kind(m).is_some_and(|k| k.ends_with("_child"))) {
703        // NEST-6: a peer slot is a `static_child` marked `peer` on the
704        // wire (NEST-7) — a slot, reached by name, not a level to walk.
705        let peer = |m: &Value| m.get("role").and_then(|r| r.get("peer")).and_then(Value::as_bool).unwrap_or(false);
706        return methods.iter().filter(|m| kind(m).as_deref() == Some("static_child") && !peer(m)).filter_map(|m| m.get("name").and_then(Value::as_str)).map(str::to_string).collect();
707    }
708    raw.get("children").and_then(Value::as_array).map(|c| c.iter().filter_map(|c| c.get("namespace").and_then(Value::as_str)).map(str::to_string).collect()).unwrap_or_default()
709}
710
711impl Shared {
712    pub(super) fn new(spec: ChildSpec, opts: ChildOptions, principal: String) -> Arc<Self> {
713        let upcalls = Upcalls::new(opts.host.clone().or_else(crate::plexus::service_scope::current), opts.legacy_upcalls);
714        upcalls.set_schemas(&opts.schemas);
715        // KEEL-32 (R3-1): before the handshake — a child that asks at once
716        // finds nothing consented and, revoked, nothing at all.
717        if opts.consent_required {
718            upcalls.set_consent(Some(Vec::new()));
719        }
720        if let Some(why) = &opts.revoked {
721            upcalls.revoke(why);
722        }
723        Arc::new(Self {
724            upcalls,
725            name: spec.name().to_string(),
726            spec,
727            opts,
728            principal,
729            counters: Arc::new(Counters::default()),
730            state: Mutex::new(LinkState { conn: None, down_since: None, why: String::new(), window: VecDeque::new(), dead: None }),
731            dial: tokio::sync::Mutex::new(()),
732            mounted_hash: OnceLock::new(),
733            identity: OnceLock::new(),
734            stopped: OnceLock::new(),
735        })
736    }
737
738    pub(super) fn current(&self) -> Option<Arc<Conn>> {
739        self.state.lock().unwrap().conn.clone()
740    }
741
742    pub(super) fn install(&self, conn: Arc<Conn>) {
743        self.state.lock().unwrap().conn = Some(conn);
744    }
745
746    /// The service asks for itself (the mount): no caller, no verdict.
747    fn own_hop(&self) -> Value {
748        json!({ "principal": self.principal, "for": [], "limit": null, "call": uuid::Uuid::new_v4().to_string() })
749    }
750
751    /// Start (or dial) the child and ask its `_info` and its schema, then
752    /// walk its static children (NEST-4), all within one `timeout`.
753    pub(super) async fn connect_once(self: &Arc<Self>) -> Result<(Arc<Conn>, Kept, MountTimings), String> {
754        let t0 = Instant::now();
755        let conn = match &self.spec {
756            ChildSpec::Stdio { argv, .. } => self.spawn_stdio(argv)?,
757            ChildSpec::Ws { url, .. } => self.dial_ws(url).await?,
758        };
759        let asked = async {
760            // DYNAUTH-5: `_info` carries a nonce; a far side with a key of
761            // its own answers it signed (`key`, `proof`), and that key is
762            // registered and pinned with the mount. One that answers a key
763            // it cannot prove is refused; one that names none has none.
764            // KEEL-19: the proof is made for this service — the principal its
765            // connection proved (`asker` on a pipe, where nothing is proved):
766            // one made for another asker, relayed, is refused.
767            let nonce = crate::plexus::peer_token::random_nonce();
768            let info = ask(&conn, "_info", json!({ "nonce": nonce, "asker": self.principal }), self.opts.timeout).await.map_err(|e| format!("`_info`: {e}"))?;
769            let ns = info.get("backend").and_then(Value::as_str).ok_or("its `_info` names no backend")?.to_string();
770            if let Some(key) = info.get("key").and_then(Value::as_str) {
771                // KEEL-19: the asker a key proof is made for is this
772                // service's key, which its dial proved; a principal no
773                // transport proves (`service:dyn`) is every service's.
774                if crate::plexus::peer_token::key_of(&self.principal).is_none() {
775                    return Err(format!(
776                        "{KEYED_MOUNT_NEEDS_A_KEY}: its `_info` proves the key {key}, and this service dials as `{}`, which no transport proves: give the service a dial key (ChildOptions::key) to mount a keyed peer",
777                        self.principal
778                    ));
779                }
780                let proof = info.get("proof").and_then(Value::as_str).unwrap_or_default();
781                crate::plexus::peer_token::check_key_proof_for(key, proof, &nonce, &ns, &self.principal).map_err(|e| {
782                    let made_for = info.get("asker").and_then(Value::as_str).filter(|a| *a != self.principal).map(|a| format!(" (it was made for {a}, not for {})", self.principal)).unwrap_or_default();
783                    format!("its `_info` names the key {key} without proving it to this service: {e}{made_for}")
784                })?;
785                let _ = conn.peer_key.set(key.to_string());
786            }
787            let _ = conn.ns.set(ns.clone());
788            let t1 = Instant::now();
789            let deadline = t1 + self.opts.timeout;
790            let params = json!({ "method": format!("{ns}.schema"), "params": {}, "hop": self.own_hop() });
791            let schema = ask(&conn, &format!("{ns}.call"), params, self.opts.timeout).await.map_err(|e| format!("its schema: {e}"))?;
792            let nodes = self.walk(&conn, &ns, &schema, deadline).await?;
793            Ok::<_, String>((Kept { top: schema, nodes }, t1))
794        };
795        match asked.await {
796            Ok((schema, t1)) => {
797                let t = MountTimings { spawn_ms: (t1 - t0).as_secs_f64() * 1e3, schema_ms: t1.elapsed().as_secs_f64() * 1e3, merge_ms: 0.0 };
798                Ok((conn, schema, t))
799            }
800            Err(why) => {
801                conn.shutdown.notify_one();
802                Err(why)
803            }
804        }
805    }
806
807    /// NEST-4 · the static tree below `top`, depth-first: `{ns}.<path>.schema`
808    /// for each static child, at most `max_depth` levels down and before
809    /// `deadline` (the mount's one timeout), each asked as the mount asks the
810    /// child's own (no caller, no verdict). A dynamic child is not walked:
811    /// its role, on its parent's schema, is the slot. Past the depth, or the
812    /// time, the mount fails naming the path.
813    pub(super) async fn walk(&self, conn: &Conn, ns: &str, top: &Value, deadline: Instant) -> Result<Vec<(String, Value)>, String> {
814        let mut out = Vec::new();
815        let mut stack: Vec<String> = static_children(top).into_iter().rev().collect();
816        while let Some(path) = stack.pop() {
817            let depth = path.split('.').count();
818            if depth > self.opts.max_depth {
819                return Err(format!("its child `{path}` is {depth} levels down, past --child-max-depth ({})", self.opts.max_depth));
820            }
821            let left = deadline.saturating_duration_since(Instant::now());
822            if left.is_zero() {
823                return Err(format!("its tree was not walked within {:?}: stopped at `{path}`", self.opts.timeout));
824            }
825            let params = json!({ "method": format!("{ns}.{path}.schema"), "params": {}, "hop": self.own_hop() });
826            let raw = ask(conn, &format!("{ns}.call"), params, left).await.map_err(|e| format!("the schema of its child `{path}`: {e}"))?;
827            for child in static_children(&raw).into_iter().rev() {
828                stack.push(format!("{path}.{child}"));
829            }
830            out.push((path, raw));
831        }
832        Ok(out)
833    }
834
835    fn spawn_stdio(self: &Arc<Self>, argv: &[String]) -> Result<Arc<Conn>, String> {
836        let mut cmd = tokio::process::Command::new(&argv[0]);
837        cmd.args(&argv[1..])
838            .env_clear()
839            .envs(["PATH", "HOME", "LANG"].iter().map(|k| k.to_string()).chain(self.opts.env.iter().cloned()).filter_map(|k| std::env::var(&k).ok().map(|v| (k, v))))
840            .env("PLEXUS_CHILD_NAME", &self.name)
841            .stdin(std::process::Stdio::piped())
842            .stdout(std::process::Stdio::piped())
843            .stderr(std::process::Stdio::piped())
844            .kill_on_drop(true);
845        let mut child = cmd.spawn().map_err(|e| format!("spawning `{}`: {e}", argv[0]))?;
846        let (stdin, stdout, stderr) = (child.stdin.take(), child.stdout.take(), child.stderr.take());
847        let (Some(stdin), Some(stdout), Some(stderr)) = (stdin, stdout, stderr) else {
848            return Err("its pipes were not opened".into());
849        };
850        let (out, out_rx) = mpsc::unbounded_channel();
851        let conn = Conn::new(&self.name, child.id(), out, self.opts.buffer, Arc::clone(&self.counters));
852        let _ = conn.up.set(Arc::clone(&self.upcalls));
853        tokio::spawn(write_lines(stdin, out_rx));
854        let reader = Arc::clone(&conn);
855        let shared = Arc::downgrade(self);
856        tokio::spawn(async move {
857            tokio::select! {
858                _ = read_lines(&reader, stdout) => {}
859                _ = reader.shutdown.notified() => {
860                    let _ = child.start_kill();
861                }
862            }
863            let status = child.wait().await.map(|s| s.to_string()).unwrap_or_else(|e| e.to_string());
864            gone(&shared, &reader, &format!("the process exited ({status})"));
865        });
866        let name = self.name.clone();
867        tokio::spawn(async move {
868            let span = tracing::info_span!(target: "plexus::child", "plexus::child", name = %name);
869            let mut lines = BufReader::new(stderr).lines();
870            while let Ok(Some(l)) = lines.next_line().await {
871                let _e = span.enter();
872                // DYNAUTH-8: a child that logs its frames (plexus-transport's
873                // stdio at debug did) never puts a credential in this log.
874                tracing::info!(target: "plexus::child", child = %name, "{}", crate::plexus::redact::redact_str(&l));
875            }
876        });
877        Ok(conn)
878    }
879
880    #[cfg(feature = "child-ws")]
881    async fn dial_ws(self: &Arc<Self>, url: &str) -> Result<Arc<Conn>, String> {
882        super::ws::dial(self, url).await
883    }
884
885    #[cfg(not(feature = "child-ws"))]
886    async fn dial_ws(self: &Arc<Self>, _url: &str) -> Result<Arc<Conn>, String> {
887        Err("a WebSocket child needs plexus-core's `child-ws` feature".into())
888    }
889
890    /// The connection `conn` is gone; the next call starts another.
891    pub(super) fn lost(&self, conn: &Arc<Conn>, why: &str) {
892        let mut st = self.state.lock().unwrap();
893        if st.conn.as_ref().is_some_and(|c| Arc::ptr_eq(c, conn)) {
894            st.conn = None;
895            st.down_since = Some(Instant::now());
896            st.why = why.to_string();
897            drop(st);
898            tracing::warn!(target: "plexus::child", child = %self.name, "child gone: {why}");
899        }
900    }
901
902    /// A live connection, starting the child again if it died — within the
903    /// backoff, and never past `per_minute` restarts a minute.
904    pub(super) async fn ensure(self: &Arc<Self>) -> Result<Arc<Conn>, PlexusStreamItem> {
905        if let Some(c) = self.live()? {
906            return Ok(c);
907        }
908        let _g = self.dial.lock().await;
909        if let Some(c) = self.live()? {
910            return Ok(c);
911        }
912        let b = self.opts.backoff;
913        let wait = {
914            let mut st = self.state.lock().unwrap();
915            let now = Instant::now();
916            while st.window.front().is_some_and(|t| now.duration_since(*t) >= Duration::from_secs(60)) {
917                st.window.pop_front();
918            }
919            if st.window.len() >= b.per_minute {
920                let next = st.window.front().map(|t| Duration::from_secs(60).saturating_sub(now.duration_since(*t))).unwrap_or_default();
921                return Err(unavailable(&self.name, &format!("{} (restarted {} times in the last minute; the next restart in {:.1} s)", st.why, st.window.len(), next.as_secs_f64())));
922            }
923            let due = st.down_since.unwrap_or(now) + b.delay(st.window.len());
924            due.saturating_duration_since(now)
925        };
926        if wait > self.opts.timeout {
927            let why = self.state.lock().unwrap().why.clone();
928            return Err(unavailable(&self.name, &format!("{why} (restarting in {:.1} s)", wait.as_secs_f64())));
929        }
930        tokio::time::sleep(wait).await;
931        self.state.lock().unwrap().window.push_back(Instant::now());
932        self.counters.restarts.fetch_add(1, Ordering::Relaxed);
933        match self.connect_once().await {
934            Ok((conn, kept, t)) => {
935                let hash = kept.hash();
936                if self.mounted_hash.get().is_some_and(|h| *h != hash) {
937                    conn.shutdown.notify_one();
938                    let why = "schema changed; restart the service".to_string();
939                    tracing::error!(target: "plexus::child", child = %self.name, mounted = ?self.mounted_hash.get(), published = %hash, "the restarted child published another schema; not remounted: restart the service");
940                    self.state.lock().unwrap().dead = Some(why.clone());
941                    return Err(unavailable(&self.name, &why));
942                }
943                let proved = super::PeerIdentity::of(&conn);
944                if let Some(pinned) = self.identity.get().filter(|p| **p != proved) {
945                    conn.shutdown.notify_one();
946                    let why = format!("its identity changed: {pinned}, now {proved}; connect it again to trust the new one");
947                    tracing::error!(target: "plexus::child", child = %self.name, %pinned, %proved, "the restarted child proved another identity; not remounted");
948                    self.state.lock().unwrap().dead = Some(why.clone());
949                    return Err(unavailable(&self.name, &why));
950                }
951                tracing::info!(target: "plexus::child", child = %self.name, spawn_ms = t.spawn_ms, schema_ms = t.schema_ms, "child restarted");
952                self.state.lock().unwrap().conn = Some(Arc::clone(&conn));
953                Ok(conn)
954            }
955            Err(why) => {
956                let mut st = self.state.lock().unwrap();
957                st.down_since = Some(Instant::now());
958                st.why = why.clone();
959                Err(unavailable(&self.name, &why))
960            }
961        }
962    }
963
964    /// KEEL-59: cut the child off — every call open into it ends with one
965    /// item saying `why`, the child is told to stop each, and nothing more is
966    /// forwarded ([`Conn::stop`]). The first `why` stands.
967    pub(super) fn stop(&self, why: &str) {
968        let _ = self.stopped.set(why.to_string());
969        let conn = self.state.lock().unwrap().conn.clone();
970        if let Some(c) = conn {
971            c.stop(why);
972        }
973    }
974
975    /// KEEL-59: why the child was cut off, if it was.
976    pub(super) fn stopped(&self) -> Option<&str> {
977        self.stopped.get().map(String::as_str)
978    }
979
980    /// Why the child will never be mounted again, if it will not (a restart
981    /// published another schema).
982    pub(super) fn dead(&self) -> Option<String> {
983        self.state.lock().unwrap().dead.clone()
984    }
985
986    /// Why the child is down, when it is: the last connection's end, or the
987    /// last start that failed.
988    pub(super) fn why_down(&self) -> Option<String> {
989        let st = self.state.lock().unwrap();
990        if st.dead.is_some() {
991            return st.dead.clone();
992        }
993        match &st.conn {
994            Some(c) if c.alive() => None,
995            _ => Some(if st.why.is_empty() { "not connected".to_string() } else { st.why.clone() }),
996        }
997    }
998
999    /// Whether a live connection is up now.
1000    pub(super) fn is_up(&self) -> bool {
1001        let st = self.state.lock().unwrap();
1002        st.dead.is_none() && st.conn.as_ref().is_some_and(|c| c.alive())
1003    }
1004
1005    fn live(&self) -> Result<Option<Arc<Conn>>, PlexusStreamItem> {
1006        let st = self.state.lock().unwrap();
1007        if let Some(why) = &st.dead {
1008            return Err(unavailable(&self.name, why));
1009        }
1010        Ok(st.conn.clone().filter(|c| c.alive()))
1011    }
1012}
1013
1014/// The reader's end: every open call told, the supervisor told.
1015pub(super) fn gone(shared: &Weak<Shared>, conn: &Arc<Conn>, why: &str) {
1016    conn.on_gone(why);
1017    if let Some(s) = shared.upgrade() {
1018        s.lost(conn, why);
1019    }
1020}
1021
1022/// One unary call: the first `data`'s content (the rest read to the end).
1023async fn ask(conn: &Conn, method: &str, params: Value, timeout: Duration) -> Result<Value, String> {
1024    let (_, mut rx) = conn.open(method, params, timeout, None).await.map_err(|e| match e {
1025        PlexusStreamItem::Error { message, .. } => message,
1026        other => format!("{other:?}"),
1027    })?;
1028    let deadline = tokio::time::Instant::now() + timeout;
1029    let mut first = None;
1030    loop {
1031        match tokio::time::timeout_at(deadline, rx.recv()).await {
1032            Ok(Some(PlexusStreamItem::Data { content, .. })) => {
1033                if first.is_none() {
1034                    first = Some(content.into_value());
1035                }
1036            }
1037            Ok(Some(PlexusStreamItem::Error { message, .. })) => return Err(message),
1038            Ok(Some(PlexusStreamItem::Done { .. })) | Ok(None) => break,
1039            Ok(Some(_)) => {}
1040            Err(_) => return Err(format!("no answer in {timeout:?}")),
1041        }
1042    }
1043    first.ok_or_else(|| "it answered no data".to_string())
1044}