Skip to main content

plexus_core/plexus/remote_child/
link.rs

1//! One connection to a child — its frames out, its answers routed back —
2//! and the supervisor that starts it, notices it gone and starts it again.
3//!
4//! A [`Conn`] is one process (stdio) or one socket (ws): a writer task that
5//! drains everything queued into one write per wakeup (spec/11), a reader
6//! that routes a response by `id` to the call that opened it (registering
7//! the subscription before the next message is read, so an item can never
8//! beat its id — spec/02) and an item by `subscription` to that call's
9//! bounded channel, and — when the far side goes — one `child_unavailable`
10//! to every call it held. A [`Shared`] outlives its connections: it keeps
11//! the schema's hash, the counters and the restart window.
12
13use std::borrow::Cow;
14use std::collections::{HashMap, HashSet, VecDeque};
15use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
16use std::sync::{Arc, Mutex, OnceLock, Weak};
17use std::time::{Duration, Instant};
18
19use serde::Deserialize;
20use serde_json::value::RawValue;
21use serde_json::{json, Value};
22use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWriteExt, BufReader};
23use tokio::sync::{mpsc, oneshot, Notify};
24
25use super::spec::{ChildOptions, ChildSpec};
26use crate::plexus::types::{PlexusStreamItem, StreamMetadata};
27use plexus_client::RemoteError;
28
29/// A subscription id as the child spelled it (its JSON text: `4`, `"a"`).
30pub(super) type SubKey = String;
31
32pub(super) fn meta(name: &str) -> StreamMetadata {
33    StreamMetadata::new(vec![name.to_string()], crate::plexus::context::PlexusContext::hash())
34}
35
36/// `child_unavailable` (spec/12 rule 5): one error, and the end.
37pub(super) fn unavailable(name: &str, why: &str) -> PlexusStreamItem {
38    PlexusStreamItem::error(meta(name), RemoteError::ChildUnavailable { child: name.into(), reason: why.into() }, true)
39}
40
41/// `child_timeout`: the child did not open the call in the service's time.
42pub(super) fn timed_out(name: &str, timeout: Duration) -> PlexusStreamItem {
43    let error = RemoteError::ChildTimeout { child: name.into(), timeout_ms: timeout.as_millis() as u64 };
44    let mut item = PlexusStreamItem::error(meta(name), error, true);
45    if let PlexusStreamItem::Error { message, .. } = &mut item {
46        *message = format!("child {name} did not open the call in {timeout:?}");
47    }
48    item
49}
50
51/// `too_many_calls` (spec/05, 09): as a connection's limit answers.
52pub(super) fn too_many(name: &str, max: usize) -> PlexusStreamItem {
53    PlexusStreamItem::error(meta(name), RemoteError::TooManyCalls { max: max as u64, child: Some(name.into()) }, true)
54}
55
56/// A call the child refused at the JSON-RPC layer (`{code, message,
57/// data?}`): its `data` when that is a typed [`RemoteError`] (RETURNS-6),
58/// else what its code says; spec/05's `message` and `code` as it sent them.
59fn refused(name: &str, e: &Value) -> PlexusStreamItem {
60    let code = e.get("code").map(|c| match c {
61        Value::String(s) => s.clone(),
62        other => other.to_string(),
63    });
64    let message = e.get("message").and_then(Value::as_str).unwrap_or("the child refused the call").to_string();
65    let error = e
66        .get("data")
67        .and_then(|d| serde_json::from_value::<RemoteError>(d.clone()).ok())
68        .unwrap_or_else(|| super::peer::remote_error_of_code(e.get("code").and_then(Value::as_i64).unwrap_or(-32000), &message));
69    let fields = crate::plexus::types::spec_fields(&error);
70    PlexusStreamItem::Error { metadata: meta(name), message, code, recoverable: false, error, fields }
71}
72
73/// Counters that outlive a connection.
74#[derive(Default)]
75pub(super) struct Counters {
76    pub unasked: AtomicU64,
77    pub calls: AtomicU64,
78    pub restarts: AtomicU64,
79    pub open: AtomicUsize,
80}
81
82/// How a call's open failed.
83pub(super) enum OpenFail {
84    /// The connection went before the child answered.
85    Gone(String),
86    /// The child answered the open with a JSON-RPC error.
87    Refused(Value),
88}
89
90struct Opening {
91    reply: oneshot::Sender<Result<SubKey, OpenFail>>,
92    items: Sub,
93}
94
95/// An open call's items: where they go, and the method called — the
96/// `content_type` a progress item as main wrote it (none of its own) is
97/// named for (RETURNS-36, re-review #3).
98struct Sub {
99    tx: mpsc::Sender<PlexusStreamItem>,
100    method: Arc<str>,
101}
102
103/// How long after its `true` an unsubscribed call's late item is still the
104/// cancel's tail: spec/04 lets an item already in flight land just after
105/// the response (the kit allows 50 ms).
106const TAIL: Duration = Duration::from_millis(250);
107
108#[derive(Default)]
109struct Routes {
110    pending: HashMap<u64, Opening>,
111    subs: HashMap<SubKey, Sub>,
112    /// Unsubscribes sent and not yet answered, and those answered within
113    /// [`TAIL`]: their late items are the cancel's tail, dropped quietly
114    /// (not "unasked").
115    closing: HashMap<u64, SubKey>,
116    closing_subs: HashSet<SubKey>,
117    answered: VecDeque<(Instant, SubKey)>,
118    /// Opens past their timeout: a late answer is unsubscribed at once.
119    abandoned: HashSet<u64>,
120}
121
122impl Routes {
123    /// Forget the cancels answered more than [`TAIL`] ago.
124    fn expire_tails(&mut self, now: Instant) {
125        while self.answered.front().is_some_and(|(t, _)| now.duration_since(*t) >= TAIL) {
126            if let Some((_, key)) = self.answered.pop_front() {
127                self.closing_subs.remove(&key);
128            }
129        }
130    }
131}
132
133/// One connection to the child.
134pub(super) struct Conn {
135    pub name: String,
136    pub pid: Option<u32>,
137    /// The child's own namespace (its `_info`'s `backend`).
138    pub ns: OnceLock<String>,
139    out: mpsc::UnboundedSender<Vec<u8>>,
140    routes: Mutex<Routes>,
141    next_id: AtomicU64,
142    alive: AtomicBool,
143    /// Asked to end (the service let go of the child, or refused its schema).
144    pub shutdown: Notify,
145    buffer: usize,
146    counters: Arc<Counters>,
147}
148
149/// One message, read in one pass: a notification's item stays raw until
150/// its call is known (then it is read once, as the item).
151#[derive(Deserialize)]
152struct Msg<'a> {
153    #[serde(default)]
154    id: Option<Value>,
155    #[serde(default, borrow)]
156    result: Option<&'a RawValue>,
157    #[serde(default)]
158    error: Option<Value>,
159    #[serde(default, borrow)]
160    method: Option<Cow<'a, str>>,
161    #[serde(default, borrow)]
162    params: Option<Notif<'a>>,
163}
164
165#[derive(Deserialize)]
166struct Notif<'a> {
167    #[serde(default)]
168    subscription: Option<Value>,
169    #[serde(default, borrow)]
170    result: Option<&'a RawValue>,
171}
172
173/// A message whose `params` are not an object (a request with positional
174/// params): read loosely, only to answer it.
175#[derive(Deserialize)]
176struct Loose<'a> {
177    #[serde(default)]
178    id: Option<Value>,
179    #[serde(default, borrow)]
180    method: Option<Cow<'a, str>>,
181}
182
183#[derive(Deserialize)]
184struct Kind<'a> {
185    #[serde(rename = "type", borrow)]
186    kind: Cow<'a, str>,
187}
188
189impl Conn {
190    pub(super) fn new(name: &str, pid: Option<u32>, out: mpsc::UnboundedSender<Vec<u8>>, buffer: usize, counters: Arc<Counters>) -> Arc<Self> {
191        Arc::new(Self {
192            name: name.to_string(),
193            pid,
194            ns: OnceLock::new(),
195            out,
196            routes: Mutex::new(Routes::default()),
197            next_id: AtomicU64::new(1),
198            alive: AtomicBool::new(true),
199            shutdown: Notify::new(),
200            buffer,
201            counters,
202        })
203    }
204
205    pub(super) fn alive(&self) -> bool {
206        self.alive.load(Ordering::Acquire)
207    }
208
209    pub(super) fn ns(&self) -> &str {
210        self.ns.get().map(String::as_str).unwrap_or(&self.name)
211    }
212
213    fn send(&self, frame: &Value) {
214        let _ = self.out.send(serde_json::to_vec(frame).unwrap_or_default());
215    }
216
217    /// Open a call: `method` with `params` (a `{ns}.call`'s params, or
218    /// `_info`'s null); the subscription id back within `timeout`, its items
219    /// already routed to the receiver.
220    pub(super) async fn open(&self, method: &str, params: Value, timeout: Duration) -> Result<(SubKey, mpsc::Receiver<PlexusStreamItem>), PlexusStreamItem> {
221        let id = self.next_id.fetch_add(1, Ordering::Relaxed);
222        let (tx, rx) = mpsc::channel(self.buffer);
223        let (reply, answer) = oneshot::channel();
224        {
225            let mut r = self.routes.lock().unwrap();
226            if !self.alive() {
227                return Err(unavailable(&self.name, "the connection has closed"));
228            }
229            // a `{ns}.call`'s params name the method called (`echo.count`)
230            let called = params.get("method").and_then(Value::as_str).unwrap_or(method);
231            r.pending.insert(id, Opening { reply, items: Sub { tx, method: called.into() } });
232        }
233        self.send(&json!({ "jsonrpc": "2.0", "id": id, "method": method, "params": params }));
234        match tokio::time::timeout(timeout, answer).await {
235            Ok(Ok(Ok(key))) => Ok((key, rx)),
236            Ok(Ok(Err(OpenFail::Gone(why)))) => Err(unavailable(&self.name, &why)),
237            Ok(Err(_)) => Err(unavailable(&self.name, "the connection closed before the call opened")),
238            Ok(Ok(Err(OpenFail::Refused(e)))) => Err(refused(&self.name, &e)),
239            Err(_) => {
240                let mut r = self.routes.lock().unwrap();
241                if r.pending.remove(&id).is_some() {
242                    r.abandoned.insert(id);
243                }
244                Err(timed_out(&self.name, timeout))
245            }
246        }
247    }
248
249    /// The caller let go of `key` before its end: forget it, and tell the
250    /// child (`{ns}.call_unsub [sid]`, spec/04).
251    pub(super) fn unsubscribe(&self, key: &str) {
252        let id = {
253            let mut r = self.routes.lock().unwrap();
254            if r.subs.remove(key).is_none() || !self.alive() {
255                return;
256            }
257            let id = self.next_id.fetch_add(1, Ordering::Relaxed);
258            r.closing.insert(id, key.to_string());
259            r.closing_subs.insert(key.to_string());
260            id
261        };
262        let method = serde_json::to_string(&format!("{}.call_unsub", self.ns())).unwrap_or_default();
263        let frame = format!(r#"{{"jsonrpc":"2.0","id":{id},"method":{method},"params":[{key}]}}"#);
264        let _ = self.out.send(frame.into_bytes());
265    }
266
267    /// How many subscriptions are routed now.
268    pub(super) fn routed(&self) -> usize {
269        self.routes.lock().unwrap().subs.len()
270    }
271
272    /// One message from the child.
273    pub(super) async fn on_message(&self, raw: &[u8]) {
274        let msg: Msg<'_> = match serde_json::from_slice(raw) {
275            Ok(m) => m,
276            Err(e) => match serde_json::from_slice::<Loose<'_>>(raw) {
277                Ok(Loose { id: Some(id), method: Some(method) }) => return self.refuse_request(id, &method),
278                _ => {
279                    tracing::warn!(target: "plexus::child", child = %self.name, "an unreadable message from the child: {e}");
280                    return;
281                }
282            },
283        };
284        if let Some(method) = msg.method {
285            if let Some(id) = msg.id {
286                return self.refuse_request(id, &method);
287            }
288            match msg.params {
289                Some(Notif { subscription: Some(sub), result: Some(item) }) => self.route(sub.to_string(), item).await,
290                _ => tracing::warn!(target: "plexus::child", child = %self.name, %method, "a notification without subscription and result"),
291            }
292            return;
293        }
294        let Some(id) = msg.id.as_ref().and_then(Value::as_u64) else { return };
295        let mut r = self.routes.lock().unwrap();
296        if let Some(open) = r.pending.remove(&id) {
297            match (msg.result, msg.error) {
298                (Some(sid), _) => {
299                    let key = match serde_json::from_str::<Value>(sid.get()) {
300                        Ok(v) => v.to_string(),
301                        Err(_) => sid.get().to_string(),
302                    };
303                    r.subs.insert(key.clone(), open.items);
304                    let _ = open.reply.send(Ok(key));
305                }
306                (None, e) => {
307                    let _ = open.reply.send(Err(OpenFail::Refused(e.unwrap_or_else(|| json!({ "message": "an answer with neither result nor error" })))));
308                }
309            }
310        } else if let Some(key) = r.closing.remove(&id) {
311            // Answered: an item still in flight is the tail for TAIL more.
312            let now = Instant::now();
313            r.expire_tails(now);
314            r.answered.push_back((now, key));
315        } else if r.abandoned.remove(&id) {
316            // A call opened after its caller was told `child_timeout`: stop it.
317            if let Some(sid) = msg.result.and_then(|s| serde_json::from_str::<Value>(s.get()).ok()) {
318                let key = sid.to_string();
319                let uid = self.next_id.fetch_add(1, Ordering::Relaxed);
320                r.closing.insert(uid, key.clone());
321                r.closing_subs.insert(key.clone());
322                drop(r);
323                let method = serde_json::to_string(&format!("{}.call_unsub", self.ns())).unwrap_or_default();
324                let _ = self.out.send(format!(r#"{{"jsonrpc":"2.0","id":{uid},"method":{method},"params":[{key}]}}"#).into_bytes());
325            }
326        }
327    }
328
329    /// A child asks its service nothing (spec/12): answered, not served.
330    fn refuse_request(&self, id: Value, method: &str) {
331        tracing::warn!(target: "plexus::child", child = %self.name, %method, "a request from the child refused");
332        self.send(&json!({ "jsonrpc": "2.0", "id": id, "error": { "code": -32601, "message": "a service answers no requests from its child" } }));
333    }
334
335    async fn route(&self, key: SubKey, raw: &RawValue) {
336        let (tx, method) = {
337            let mut r = self.routes.lock().unwrap();
338            match r.subs.get(&key) {
339                Some(sub) => (sub.tx.clone(), Arc::clone(&sub.method)),
340                None => {
341                    r.expire_tails(Instant::now());
342                    if !r.closing_subs.contains(&key) {
343                        drop(r);
344                        self.counters.unasked.fetch_add(1, Ordering::Relaxed);
345                        tracing::warn!(target: "plexus::child", child = %self.name, subscription = %key, "unasked item dropped");
346                    }
347                    return;
348                }
349            }
350        };
351        // RETURNS-36: a peer on plexus-wire's spec as main wrote it is read
352        // into 0.8's shape (`peer::item`) when — only when — it does not
353        // read as 0.8.
354        let parsed = serde_json::from_str::<PlexusStreamItem>(raw.get()).or_else(|e| {
355            super::peer::item(raw.get(), &method)
356                .and_then(|v| serde_json::from_value::<PlexusStreamItem>(v).ok())
357                .ok_or(e)
358        });
359        let item = match parsed {
360            Ok(item) => item,
361            Err(e) => {
362                let kind = serde_json::from_str::<Kind<'_>>(raw.get()).map(|k| k.kind.into_owned()).unwrap_or_default();
363                if kind != "error" {
364                    // An item kind this service does not know (spec/03: a
365                    // reader skips it) — or one that does not read at all.
366                    tracing::debug!(target: "plexus::child", child = %self.name, subscription = %key, %kind, "an item kind not relayed: {e}");
367                    return;
368                }
369                // RETURNS-36 (review #4): an error item is the call's end
370                // (spec/03); skipped, the call would wait for an end that
371                // never comes. It ends here, naming what could not be read,
372                // and the child is told to stop.
373                tracing::warn!(target: "plexus::child", child = %self.name, subscription = %key, "an error item not read: {e}");
374                self.unsubscribe(&key);
375                let error = plexus_client::RemoteError::Execution {
376                    message: format!("child {} sent an error item this service could not read: {e}", self.name),
377                };
378                PlexusStreamItem::error(meta(&self.name), error, false)
379            }
380        };
381        if matches!(item, PlexusStreamItem::Done { .. } | PlexusStreamItem::Error { .. }) {
382            self.routes.lock().unwrap().subs.remove(&key);
383        }
384        match tx.try_send(item) {
385            Ok(()) => {}
386            // The caller has not read the call's buffer: wait for it
387            // (lossless; the child's other calls wait behind it).
388            Err(mpsc::error::TrySendError::Full(item)) => {
389                let _ = tx.send(item).await;
390            }
391            Err(mpsc::error::TrySendError::Closed(_)) => {}
392        }
393    }
394
395    /// The far side is gone: every open call gets one `child_unavailable`.
396    pub(super) fn on_gone(&self, why: &str) {
397        let (subs, pending) = {
398            let mut r = self.routes.lock().unwrap();
399            self.alive.store(false, Ordering::Release);
400            r.closing.clear();
401            r.closing_subs.clear();
402            r.answered.clear();
403            r.abandoned.clear();
404            (std::mem::take(&mut r.subs), std::mem::take(&mut r.pending))
405        };
406        for (_, Sub { tx, .. }) in subs {
407            match tx.try_send(unavailable(&self.name, why)) {
408                Err(mpsc::error::TrySendError::Full(item)) => {
409                    tokio::spawn(async move {
410                        let _ = tx.send(item).await;
411                    });
412                }
413                _ => {}
414            }
415        }
416        for (_, open) in pending {
417            let _ = open.reply.send(Err(OpenFail::Gone(why.to_string())));
418        }
419    }
420}
421
422/// Read newline-delimited messages from `r`, one wakeup per read: every
423/// whole line in the buffer is routed before the next read.
424pub(super) async fn read_lines<R: AsyncRead + Unpin>(conn: &Conn, r: R) {
425    let mut r = BufReader::with_capacity(1 << 17, r);
426    let mut carry: Vec<u8> = Vec::new();
427    loop {
428        let buf = match r.fill_buf().await {
429            Ok(b) if !b.is_empty() => b,
430            _ => break,
431        };
432        let n = buf.len();
433        let mut start = 0;
434        while let Some(i) = buf[start..].iter().position(|b| *b == b'\n') {
435            let end = start + i;
436            if carry.is_empty() {
437                let line = &buf[start..end];
438                if line.iter().any(|b| !b.is_ascii_whitespace()) {
439                    conn.on_message(line).await;
440                }
441            } else {
442                carry.extend_from_slice(&buf[start..end]);
443                let line = std::mem::take(&mut carry);
444                if line.iter().any(|b| !b.is_ascii_whitespace()) {
445                    conn.on_message(&line).await;
446                }
447            }
448            start = end + 1;
449        }
450        carry.extend_from_slice(&buf[start..]);
451        r.consume(n);
452    }
453}
454
455/// The writer: everything queued, one write per wakeup, each line whole.
456pub(super) async fn write_lines<W: tokio::io::AsyncWrite + Unpin>(mut w: W, mut rx: mpsc::UnboundedReceiver<Vec<u8>>) {
457    while let Some(first) = rx.recv().await {
458        let mut buf = first;
459        buf.push(b'\n');
460        while let Ok(more) = rx.try_recv() {
461            buf.extend_from_slice(&more);
462            buf.push(b'\n');
463        }
464        if w.write_all(&buf).await.is_err() || w.flush().await.is_err() {
465            break;
466        }
467    }
468}
469
470/// When the child was mounted, how long each step took.
471#[derive(Debug, Clone, Copy, Default, PartialEq)]
472pub struct MountTimings {
473    /// Spawn (or dial) → its `_info` answered.
474    pub spawn_ms: f64,
475    /// `_info` → its schema answered.
476    pub schema_ms: f64,
477    /// The schema requalified under the mount name and kept.
478    pub merge_ms: f64,
479}
480
481struct LinkState {
482    conn: Option<Arc<Conn>>,
483    down_since: Option<Instant>,
484    why: String,
485    window: VecDeque<Instant>,
486    /// Never again (a restart published another schema): the reason.
487    dead: Option<String>,
488}
489
490/// The child across its connections.
491pub(super) struct Shared {
492    pub spec: ChildSpec,
493    pub opts: ChildOptions,
494    pub name: String,
495    /// The hop's `principal`.
496    pub principal: String,
497    pub counters: Arc<Counters>,
498    state: Mutex<LinkState>,
499    dial: tokio::sync::Mutex<()>,
500    /// The mounted schema's hash, as the child published it.
501    pub mounted_hash: OnceLock<String>,
502}
503
504impl Drop for Shared {
505    fn drop(&mut self) {
506        if let Ok(st) = self.state.lock() {
507            if let Some(c) = &st.conn {
508                c.shutdown.notify_one();
509            }
510        }
511    }
512}
513
514/// The hash a child's schema is compared by across restarts.
515pub(super) fn schema_hash(raw: &Value) -> String {
516    match raw.get("hash").and_then(Value::as_str) {
517        Some(h) => h.to_string(),
518        None => raw.get("methods").map(Value::to_string).unwrap_or_default(),
519    }
520}
521
522impl Shared {
523    pub(super) fn new(spec: ChildSpec, opts: ChildOptions, principal: String) -> Arc<Self> {
524        Arc::new(Self {
525            name: spec.name().to_string(),
526            spec,
527            opts,
528            principal,
529            counters: Arc::new(Counters::default()),
530            state: Mutex::new(LinkState { conn: None, down_since: None, why: String::new(), window: VecDeque::new(), dead: None }),
531            dial: tokio::sync::Mutex::new(()),
532            mounted_hash: OnceLock::new(),
533        })
534    }
535
536    pub(super) fn current(&self) -> Option<Arc<Conn>> {
537        self.state.lock().unwrap().conn.clone()
538    }
539
540    pub(super) fn install(&self, conn: Arc<Conn>) {
541        self.state.lock().unwrap().conn = Some(conn);
542    }
543
544    /// The service asks for itself (the mount): no caller, no verdict.
545    fn own_hop(&self) -> Value {
546        json!({ "principal": self.principal, "for": [], "limit": null, "call": uuid::Uuid::new_v4().to_string() })
547    }
548
549    /// Start (or dial) the child and ask its `_info` and its schema.
550    pub(super) async fn connect_once(self: &Arc<Self>) -> Result<(Arc<Conn>, Value, MountTimings), String> {
551        let t0 = Instant::now();
552        let conn = match &self.spec {
553            ChildSpec::Stdio { argv, .. } => self.spawn_stdio(argv)?,
554            ChildSpec::Ws { url, .. } => self.dial_ws(url).await?,
555        };
556        let asked = async {
557            let info = ask(&conn, "_info", Value::Null, self.opts.timeout).await.map_err(|e| format!("`_info`: {e}"))?;
558            let ns = info.get("backend").and_then(Value::as_str).ok_or("its `_info` names no backend")?.to_string();
559            let _ = conn.ns.set(ns.clone());
560            let t1 = Instant::now();
561            let params = json!({ "method": format!("{ns}.schema"), "params": {}, "hop": self.own_hop() });
562            let schema = ask(&conn, &format!("{ns}.call"), params, self.opts.timeout).await.map_err(|e| format!("its schema: {e}"))?;
563            Ok::<_, String>((schema, t1))
564        };
565        match asked.await {
566            Ok((schema, t1)) => {
567                let t = MountTimings { spawn_ms: (t1 - t0).as_secs_f64() * 1e3, schema_ms: t1.elapsed().as_secs_f64() * 1e3, merge_ms: 0.0 };
568                Ok((conn, schema, t))
569            }
570            Err(why) => {
571                conn.shutdown.notify_one();
572                Err(why)
573            }
574        }
575    }
576
577    fn spawn_stdio(self: &Arc<Self>, argv: &[String]) -> Result<Arc<Conn>, String> {
578        let mut cmd = tokio::process::Command::new(&argv[0]);
579        cmd.args(&argv[1..])
580            .env_clear()
581            .envs(["PATH", "HOME", "LANG"].iter().map(|k| k.to_string()).chain(self.opts.env.iter().cloned()).filter_map(|k| std::env::var(&k).ok().map(|v| (k, v))))
582            .env("PLEXUS_CHILD_NAME", &self.name)
583            .stdin(std::process::Stdio::piped())
584            .stdout(std::process::Stdio::piped())
585            .stderr(std::process::Stdio::piped())
586            .kill_on_drop(true);
587        let mut child = cmd.spawn().map_err(|e| format!("spawning `{}`: {e}", argv[0]))?;
588        let (stdin, stdout, stderr) = (child.stdin.take(), child.stdout.take(), child.stderr.take());
589        let (Some(stdin), Some(stdout), Some(stderr)) = (stdin, stdout, stderr) else {
590            return Err("its pipes were not opened".into());
591        };
592        let (out, out_rx) = mpsc::unbounded_channel();
593        let conn = Conn::new(&self.name, child.id(), out, self.opts.buffer, Arc::clone(&self.counters));
594        tokio::spawn(write_lines(stdin, out_rx));
595        let reader = Arc::clone(&conn);
596        let shared = Arc::downgrade(self);
597        tokio::spawn(async move {
598            tokio::select! {
599                _ = read_lines(&reader, stdout) => {}
600                _ = reader.shutdown.notified() => {
601                    let _ = child.start_kill();
602                }
603            }
604            let status = child.wait().await.map(|s| s.to_string()).unwrap_or_else(|e| e.to_string());
605            gone(&shared, &reader, &format!("the process exited ({status})"));
606        });
607        let name = self.name.clone();
608        tokio::spawn(async move {
609            let span = tracing::info_span!(target: "plexus::child", "plexus::child", name = %name);
610            let mut lines = BufReader::new(stderr).lines();
611            while let Ok(Some(l)) = lines.next_line().await {
612                let _e = span.enter();
613                tracing::info!(target: "plexus::child", child = %name, "{l}");
614            }
615        });
616        Ok(conn)
617    }
618
619    #[cfg(feature = "child-ws")]
620    async fn dial_ws(self: &Arc<Self>, url: &str) -> Result<Arc<Conn>, String> {
621        super::ws::dial(self, url).await
622    }
623
624    #[cfg(not(feature = "child-ws"))]
625    async fn dial_ws(self: &Arc<Self>, _url: &str) -> Result<Arc<Conn>, String> {
626        Err("a WebSocket child needs plexus-core's `child-ws` feature".into())
627    }
628
629    /// The connection `conn` is gone; the next call starts another.
630    pub(super) fn lost(&self, conn: &Arc<Conn>, why: &str) {
631        let mut st = self.state.lock().unwrap();
632        if st.conn.as_ref().is_some_and(|c| Arc::ptr_eq(c, conn)) {
633            st.conn = None;
634            st.down_since = Some(Instant::now());
635            st.why = why.to_string();
636            drop(st);
637            tracing::warn!(target: "plexus::child", child = %self.name, "child gone: {why}");
638        }
639    }
640
641    /// A live connection, starting the child again if it died — within the
642    /// backoff, and never past `per_minute` restarts a minute.
643    pub(super) async fn ensure(self: &Arc<Self>) -> Result<Arc<Conn>, PlexusStreamItem> {
644        if let Some(c) = self.live()? {
645            return Ok(c);
646        }
647        let _g = self.dial.lock().await;
648        if let Some(c) = self.live()? {
649            return Ok(c);
650        }
651        let b = self.opts.backoff;
652        let wait = {
653            let mut st = self.state.lock().unwrap();
654            let now = Instant::now();
655            while st.window.front().is_some_and(|t| now.duration_since(*t) >= Duration::from_secs(60)) {
656                st.window.pop_front();
657            }
658            if st.window.len() >= b.per_minute {
659                let next = st.window.front().map(|t| Duration::from_secs(60).saturating_sub(now.duration_since(*t))).unwrap_or_default();
660                return Err(unavailable(&self.name, &format!("{} (restarted {} times in the last minute; the next restart in {:.1} s)", st.why, st.window.len(), next.as_secs_f64())));
661            }
662            let due = st.down_since.unwrap_or(now) + b.delay(st.window.len());
663            due.saturating_duration_since(now)
664        };
665        if wait > self.opts.timeout {
666            let why = self.state.lock().unwrap().why.clone();
667            return Err(unavailable(&self.name, &format!("{why} (restarting in {:.1} s)", wait.as_secs_f64())));
668        }
669        tokio::time::sleep(wait).await;
670        self.state.lock().unwrap().window.push_back(Instant::now());
671        self.counters.restarts.fetch_add(1, Ordering::Relaxed);
672        match self.connect_once().await {
673            Ok((conn, raw, t)) => {
674                let hash = schema_hash(&raw);
675                if self.mounted_hash.get().is_some_and(|h| *h != hash) {
676                    conn.shutdown.notify_one();
677                    let why = "schema changed; restart the service".to_string();
678                    tracing::error!(target: "plexus::child", child = %self.name, mounted = ?self.mounted_hash.get(), published = %hash, "the restarted child published another schema; not remounted: restart the service");
679                    self.state.lock().unwrap().dead = Some(why.clone());
680                    return Err(unavailable(&self.name, &why));
681                }
682                tracing::info!(target: "plexus::child", child = %self.name, spawn_ms = t.spawn_ms, schema_ms = t.schema_ms, "child restarted");
683                self.state.lock().unwrap().conn = Some(Arc::clone(&conn));
684                Ok(conn)
685            }
686            Err(why) => {
687                let mut st = self.state.lock().unwrap();
688                st.down_since = Some(Instant::now());
689                st.why = why.clone();
690                Err(unavailable(&self.name, &why))
691            }
692        }
693    }
694
695    fn live(&self) -> Result<Option<Arc<Conn>>, PlexusStreamItem> {
696        let st = self.state.lock().unwrap();
697        if let Some(why) = &st.dead {
698            return Err(unavailable(&self.name, why));
699        }
700        Ok(st.conn.clone().filter(|c| c.alive()))
701    }
702}
703
704/// The reader's end: every open call told, the supervisor told.
705pub(super) fn gone(shared: &Weak<Shared>, conn: &Arc<Conn>, why: &str) {
706    conn.on_gone(why);
707    if let Some(s) = shared.upgrade() {
708        s.lost(conn, why);
709    }
710}
711
712/// One unary call: the first `data`'s content (the rest read to the end).
713async fn ask(conn: &Conn, method: &str, params: Value, timeout: Duration) -> Result<Value, String> {
714    let (_, mut rx) = conn.open(method, params, timeout).await.map_err(|e| match e {
715        PlexusStreamItem::Error { message, .. } => message,
716        other => format!("{other:?}"),
717    })?;
718    let deadline = tokio::time::Instant::now() + timeout;
719    let mut first = None;
720    loop {
721        match tokio::time::timeout_at(deadline, rx.recv()).await {
722            Ok(Some(PlexusStreamItem::Data { content, .. })) => {
723                if first.is_none() {
724                    first = Some(content.into_value());
725                }
726            }
727            Ok(Some(PlexusStreamItem::Error { message, .. })) => return Err(message),
728            Ok(Some(PlexusStreamItem::Done { .. })) | Ok(None) => break,
729            Ok(Some(_)) => {}
730            Err(_) => return Err(format!("no answer in {timeout:?}")),
731        }
732    }
733    first.ok_or_else(|| "it answered no data".to_string())
734}