Skip to main content

plexus_core/plexus/remote_child/
link.rs

1//! One connection to a child — its frames out, its answers routed back —
2//! and the supervisor that starts it, notices it gone and starts it again.
3//!
4//! A [`Conn`] is one process (stdio) or one socket (ws): a writer task that
5//! drains everything queued into one write per wakeup (spec/11), a reader
6//! that routes a response by `id` to the call that opened it (registering
7//! the subscription before the next message is read, so an item can never
8//! beat its id — spec/02) and an item by `subscription` to that call's
9//! bounded channel, and — when the far side goes — one `child_unavailable`
10//! to every call it held. A [`Shared`] outlives its connections: it keeps
11//! the schema's hash, the counters and the restart window.
12
13use std::borrow::Cow;
14use std::collections::{HashMap, HashSet, VecDeque};
15use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
16use std::sync::{Arc, Mutex, OnceLock, Weak};
17use std::time::{Duration, Instant};
18
19use serde::Deserialize;
20use serde_json::value::RawValue;
21use serde_json::{json, Value};
22use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWriteExt, BufReader};
23use tokio::sync::{mpsc, oneshot, Notify};
24
25use super::spec::{ChildOptions, ChildSpec};
26use crate::plexus::types::{PlexusStreamItem, StreamMetadata};
27
28/// A subscription id as the child spelled it (its JSON text: `4`, `"a"`).
29pub(super) type SubKey = String;
30
31pub(super) fn meta(name: &str) -> StreamMetadata {
32    StreamMetadata::new(vec![name.to_string()], crate::plexus::context::PlexusContext::hash())
33}
34
35/// `child_unavailable` (spec/12 rule 5): one error, and the end.
36pub(super) fn unavailable(name: &str, why: &str) -> PlexusStreamItem {
37    let mut f = serde_json::Map::new();
38    f.insert("child".into(), name.into());
39    f.insert("reason".into(), why.into());
40    PlexusStreamItem::typed_error(meta(name), format!("child {name} is unavailable: {why}"), "child_unavailable", true, f)
41}
42
43/// `child_timeout`: the child did not open the call in the service's time.
44pub(super) fn timed_out(name: &str, timeout: Duration) -> PlexusStreamItem {
45    let mut f = serde_json::Map::new();
46    f.insert("child".into(), name.into());
47    f.insert("timeout_ms".into(), (timeout.as_millis() as u64).into());
48    PlexusStreamItem::typed_error(meta(name), format!("child {name} did not open the call in {timeout:?}"), "child_timeout", true, f)
49}
50
51/// `too_many_calls` (spec/05, 09): as a connection's limit answers.
52pub(super) fn too_many(name: &str, max: usize) -> PlexusStreamItem {
53    let mut f = serde_json::Map::new();
54    f.insert("max".into(), max.into());
55    f.insert("child".into(), name.into());
56    PlexusStreamItem::typed_error(meta(name), format!("too many calls open on child {name} (at most {max}): close one, or call again later"), "too_many_calls", true, f)
57}
58
59/// Counters that outlive a connection.
60#[derive(Default)]
61pub(super) struct Counters {
62    pub unasked: AtomicU64,
63    pub calls: AtomicU64,
64    pub restarts: AtomicU64,
65    pub open: AtomicUsize,
66}
67
68/// How a call's open failed.
69pub(super) enum OpenFail {
70    /// The connection went before the child answered.
71    Gone(String),
72    /// The child answered the open with a JSON-RPC error.
73    Refused(Value),
74}
75
76struct Opening {
77    reply: oneshot::Sender<Result<SubKey, OpenFail>>,
78    items: mpsc::Sender<PlexusStreamItem>,
79}
80
81/// How long after its `true` an unsubscribed call's late item is still the
82/// cancel's tail: spec/04 lets an item already in flight land just after
83/// the response (the kit allows 50 ms).
84const TAIL: Duration = Duration::from_millis(250);
85
86#[derive(Default)]
87struct Routes {
88    pending: HashMap<u64, Opening>,
89    subs: HashMap<SubKey, mpsc::Sender<PlexusStreamItem>>,
90    /// Unsubscribes sent and not yet answered, and those answered within
91    /// [`TAIL`]: their late items are the cancel's tail, dropped quietly
92    /// (not "unasked").
93    closing: HashMap<u64, SubKey>,
94    closing_subs: HashSet<SubKey>,
95    answered: VecDeque<(Instant, SubKey)>,
96    /// Opens past their timeout: a late answer is unsubscribed at once.
97    abandoned: HashSet<u64>,
98}
99
100impl Routes {
101    /// Forget the cancels answered more than [`TAIL`] ago.
102    fn expire_tails(&mut self, now: Instant) {
103        while self.answered.front().is_some_and(|(t, _)| now.duration_since(*t) >= TAIL) {
104            if let Some((_, key)) = self.answered.pop_front() {
105                self.closing_subs.remove(&key);
106            }
107        }
108    }
109}
110
111/// One connection to the child.
112pub(super) struct Conn {
113    pub name: String,
114    pub pid: Option<u32>,
115    /// The child's own namespace (its `_info`'s `backend`).
116    pub ns: OnceLock<String>,
117    out: mpsc::UnboundedSender<Vec<u8>>,
118    routes: Mutex<Routes>,
119    next_id: AtomicU64,
120    alive: AtomicBool,
121    /// Asked to end (the service let go of the child, or refused its schema).
122    pub shutdown: Notify,
123    buffer: usize,
124    counters: Arc<Counters>,
125}
126
127/// One message, read in one pass: a notification's item stays raw until
128/// its call is known (then it is read once, as the item).
129#[derive(Deserialize)]
130struct Msg<'a> {
131    #[serde(default)]
132    id: Option<Value>,
133    #[serde(default, borrow)]
134    result: Option<&'a RawValue>,
135    #[serde(default)]
136    error: Option<Value>,
137    #[serde(default, borrow)]
138    method: Option<Cow<'a, str>>,
139    #[serde(default, borrow)]
140    params: Option<Notif<'a>>,
141}
142
143#[derive(Deserialize)]
144struct Notif<'a> {
145    #[serde(default)]
146    subscription: Option<Value>,
147    #[serde(default, borrow)]
148    result: Option<&'a RawValue>,
149}
150
151/// A message whose `params` are not an object (a request with positional
152/// params): read loosely, only to answer it.
153#[derive(Deserialize)]
154struct Loose<'a> {
155    #[serde(default)]
156    id: Option<Value>,
157    #[serde(default, borrow)]
158    method: Option<Cow<'a, str>>,
159}
160
161#[derive(Deserialize)]
162struct Kind<'a> {
163    #[serde(rename = "type", borrow)]
164    kind: Cow<'a, str>,
165}
166
167impl Conn {
168    pub(super) fn new(name: &str, pid: Option<u32>, out: mpsc::UnboundedSender<Vec<u8>>, buffer: usize, counters: Arc<Counters>) -> Arc<Self> {
169        Arc::new(Self {
170            name: name.to_string(),
171            pid,
172            ns: OnceLock::new(),
173            out,
174            routes: Mutex::new(Routes::default()),
175            next_id: AtomicU64::new(1),
176            alive: AtomicBool::new(true),
177            shutdown: Notify::new(),
178            buffer,
179            counters,
180        })
181    }
182
183    pub(super) fn alive(&self) -> bool {
184        self.alive.load(Ordering::Acquire)
185    }
186
187    pub(super) fn ns(&self) -> &str {
188        self.ns.get().map(String::as_str).unwrap_or(&self.name)
189    }
190
191    fn send(&self, frame: &Value) {
192        let _ = self.out.send(serde_json::to_vec(frame).unwrap_or_default());
193    }
194
195    /// Open a call: `method` with `params` (a `{ns}.call`'s params, or
196    /// `_info`'s null); the subscription id back within `timeout`, its items
197    /// already routed to the receiver.
198    pub(super) async fn open(&self, method: &str, params: Value, timeout: Duration) -> Result<(SubKey, mpsc::Receiver<PlexusStreamItem>), PlexusStreamItem> {
199        let id = self.next_id.fetch_add(1, Ordering::Relaxed);
200        let (tx, rx) = mpsc::channel(self.buffer);
201        let (reply, answer) = oneshot::channel();
202        {
203            let mut r = self.routes.lock().unwrap();
204            if !self.alive() {
205                return Err(unavailable(&self.name, "the connection has closed"));
206            }
207            r.pending.insert(id, Opening { reply, items: tx });
208        }
209        self.send(&json!({ "jsonrpc": "2.0", "id": id, "method": method, "params": params }));
210        match tokio::time::timeout(timeout, answer).await {
211            Ok(Ok(Ok(key))) => Ok((key, rx)),
212            Ok(Ok(Err(OpenFail::Gone(why)))) => Err(unavailable(&self.name, &why)),
213            Ok(Err(_)) => Err(unavailable(&self.name, "the connection closed before the call opened")),
214            Ok(Ok(Err(OpenFail::Refused(e)))) => {
215                let code = e.get("code").map(|c| match c {
216                    Value::String(s) => s.clone(),
217                    other => other.to_string(),
218                });
219                let message = e.get("message").and_then(Value::as_str).unwrap_or("the child refused the call").to_string();
220                Err(PlexusStreamItem::Error { metadata: meta(&self.name), message, code, recoverable: false, fields: Default::default() })
221            }
222            Err(_) => {
223                let mut r = self.routes.lock().unwrap();
224                if r.pending.remove(&id).is_some() {
225                    r.abandoned.insert(id);
226                }
227                Err(timed_out(&self.name, timeout))
228            }
229        }
230    }
231
232    /// The caller let go of `key` before its end: forget it, and tell the
233    /// child (`{ns}.call_unsub [sid]`, spec/04).
234    pub(super) fn unsubscribe(&self, key: &str) {
235        let id = {
236            let mut r = self.routes.lock().unwrap();
237            if r.subs.remove(key).is_none() || !self.alive() {
238                return;
239            }
240            let id = self.next_id.fetch_add(1, Ordering::Relaxed);
241            r.closing.insert(id, key.to_string());
242            r.closing_subs.insert(key.to_string());
243            id
244        };
245        let method = serde_json::to_string(&format!("{}.call_unsub", self.ns())).unwrap_or_default();
246        let frame = format!(r#"{{"jsonrpc":"2.0","id":{id},"method":{method},"params":[{key}]}}"#);
247        let _ = self.out.send(frame.into_bytes());
248    }
249
250    /// How many subscriptions are routed now.
251    pub(super) fn routed(&self) -> usize {
252        self.routes.lock().unwrap().subs.len()
253    }
254
255    /// One message from the child.
256    pub(super) async fn on_message(&self, raw: &[u8]) {
257        let msg: Msg<'_> = match serde_json::from_slice(raw) {
258            Ok(m) => m,
259            Err(e) => match serde_json::from_slice::<Loose<'_>>(raw) {
260                Ok(Loose { id: Some(id), method: Some(method) }) => return self.refuse_request(id, &method),
261                _ => {
262                    tracing::warn!(target: "plexus::child", child = %self.name, "an unreadable message from the child: {e}");
263                    return;
264                }
265            },
266        };
267        if let Some(method) = msg.method {
268            if let Some(id) = msg.id {
269                return self.refuse_request(id, &method);
270            }
271            match msg.params {
272                Some(Notif { subscription: Some(sub), result: Some(item) }) => self.route(sub.to_string(), item).await,
273                _ => tracing::warn!(target: "plexus::child", child = %self.name, %method, "a notification without subscription and result"),
274            }
275            return;
276        }
277        let Some(id) = msg.id.as_ref().and_then(Value::as_u64) else { return };
278        let mut r = self.routes.lock().unwrap();
279        if let Some(open) = r.pending.remove(&id) {
280            match (msg.result, msg.error) {
281                (Some(sid), _) => {
282                    let key = match serde_json::from_str::<Value>(sid.get()) {
283                        Ok(v) => v.to_string(),
284                        Err(_) => sid.get().to_string(),
285                    };
286                    r.subs.insert(key.clone(), open.items);
287                    let _ = open.reply.send(Ok(key));
288                }
289                (None, e) => {
290                    let _ = open.reply.send(Err(OpenFail::Refused(e.unwrap_or_else(|| json!({ "message": "an answer with neither result nor error" })))));
291                }
292            }
293        } else if let Some(key) = r.closing.remove(&id) {
294            // Answered: an item still in flight is the tail for TAIL more.
295            let now = Instant::now();
296            r.expire_tails(now);
297            r.answered.push_back((now, key));
298        } else if r.abandoned.remove(&id) {
299            // A call opened after its caller was told `child_timeout`: stop it.
300            if let Some(sid) = msg.result.and_then(|s| serde_json::from_str::<Value>(s.get()).ok()) {
301                let key = sid.to_string();
302                let uid = self.next_id.fetch_add(1, Ordering::Relaxed);
303                r.closing.insert(uid, key.clone());
304                r.closing_subs.insert(key.clone());
305                drop(r);
306                let method = serde_json::to_string(&format!("{}.call_unsub", self.ns())).unwrap_or_default();
307                let _ = self.out.send(format!(r#"{{"jsonrpc":"2.0","id":{uid},"method":{method},"params":[{key}]}}"#).into_bytes());
308            }
309        }
310    }
311
312    /// A child asks its service nothing (spec/12): answered, not served.
313    fn refuse_request(&self, id: Value, method: &str) {
314        tracing::warn!(target: "plexus::child", child = %self.name, %method, "a request from the child refused");
315        self.send(&json!({ "jsonrpc": "2.0", "id": id, "error": { "code": -32601, "message": "a service answers no requests from its child" } }));
316    }
317
318    async fn route(&self, key: SubKey, raw: &RawValue) {
319        let tx = {
320            let mut r = self.routes.lock().unwrap();
321            match r.subs.get(&key) {
322                Some(tx) => tx.clone(),
323                None => {
324                    r.expire_tails(Instant::now());
325                    if !r.closing_subs.contains(&key) {
326                        drop(r);
327                        self.counters.unasked.fetch_add(1, Ordering::Relaxed);
328                        tracing::warn!(target: "plexus::child", child = %self.name, subscription = %key, "unasked item dropped");
329                    }
330                    return;
331                }
332            }
333        };
334        let item = match serde_json::from_str::<PlexusStreamItem>(raw.get()) {
335            Ok(item) => item,
336            Err(e) => {
337                // An item kind this service does not know (spec/03: a reader
338                // skips it) — or one that does not read at all.
339                let kind = serde_json::from_str::<Kind<'_>>(raw.get()).map(|k| k.kind.into_owned()).unwrap_or_default();
340                tracing::debug!(target: "plexus::child", child = %self.name, subscription = %key, %kind, "an item kind not relayed: {e}");
341                return;
342            }
343        };
344        if matches!(item, PlexusStreamItem::Done { .. } | PlexusStreamItem::Error { .. }) {
345            self.routes.lock().unwrap().subs.remove(&key);
346        }
347        match tx.try_send(item) {
348            Ok(()) => {}
349            // The caller has not read the call's buffer: wait for it
350            // (lossless; the child's other calls wait behind it).
351            Err(mpsc::error::TrySendError::Full(item)) => {
352                let _ = tx.send(item).await;
353            }
354            Err(mpsc::error::TrySendError::Closed(_)) => {}
355        }
356    }
357
358    /// The far side is gone: every open call gets one `child_unavailable`.
359    pub(super) fn on_gone(&self, why: &str) {
360        let (subs, pending) = {
361            let mut r = self.routes.lock().unwrap();
362            self.alive.store(false, Ordering::Release);
363            r.closing.clear();
364            r.closing_subs.clear();
365            r.answered.clear();
366            r.abandoned.clear();
367            (std::mem::take(&mut r.subs), std::mem::take(&mut r.pending))
368        };
369        for (_, tx) in subs {
370            match tx.try_send(unavailable(&self.name, why)) {
371                Err(mpsc::error::TrySendError::Full(item)) => {
372                    tokio::spawn(async move {
373                        let _ = tx.send(item).await;
374                    });
375                }
376                _ => {}
377            }
378        }
379        for (_, open) in pending {
380            let _ = open.reply.send(Err(OpenFail::Gone(why.to_string())));
381        }
382    }
383}
384
385/// Read newline-delimited messages from `r`, one wakeup per read: every
386/// whole line in the buffer is routed before the next read.
387pub(super) async fn read_lines<R: AsyncRead + Unpin>(conn: &Conn, r: R) {
388    let mut r = BufReader::with_capacity(1 << 17, r);
389    let mut carry: Vec<u8> = Vec::new();
390    loop {
391        let buf = match r.fill_buf().await {
392            Ok(b) if !b.is_empty() => b,
393            _ => break,
394        };
395        let n = buf.len();
396        let mut start = 0;
397        while let Some(i) = buf[start..].iter().position(|b| *b == b'\n') {
398            let end = start + i;
399            if carry.is_empty() {
400                let line = &buf[start..end];
401                if line.iter().any(|b| !b.is_ascii_whitespace()) {
402                    conn.on_message(line).await;
403                }
404            } else {
405                carry.extend_from_slice(&buf[start..end]);
406                let line = std::mem::take(&mut carry);
407                if line.iter().any(|b| !b.is_ascii_whitespace()) {
408                    conn.on_message(&line).await;
409                }
410            }
411            start = end + 1;
412        }
413        carry.extend_from_slice(&buf[start..]);
414        r.consume(n);
415    }
416}
417
418/// The writer: everything queued, one write per wakeup, each line whole.
419pub(super) async fn write_lines<W: tokio::io::AsyncWrite + Unpin>(mut w: W, mut rx: mpsc::UnboundedReceiver<Vec<u8>>) {
420    while let Some(first) = rx.recv().await {
421        let mut buf = first;
422        buf.push(b'\n');
423        while let Ok(more) = rx.try_recv() {
424            buf.extend_from_slice(&more);
425            buf.push(b'\n');
426        }
427        if w.write_all(&buf).await.is_err() || w.flush().await.is_err() {
428            break;
429        }
430    }
431}
432
433/// When the child was mounted, how long each step took.
434#[derive(Debug, Clone, Copy, Default, PartialEq)]
435pub struct MountTimings {
436    /// Spawn (or dial) → its `_info` answered.
437    pub spawn_ms: f64,
438    /// `_info` → its schema answered.
439    pub schema_ms: f64,
440    /// The schema requalified under the mount name and kept.
441    pub merge_ms: f64,
442}
443
444struct LinkState {
445    conn: Option<Arc<Conn>>,
446    down_since: Option<Instant>,
447    why: String,
448    window: VecDeque<Instant>,
449    /// Never again (a restart published another schema): the reason.
450    dead: Option<String>,
451}
452
453/// The child across its connections.
454pub(super) struct Shared {
455    pub spec: ChildSpec,
456    pub opts: ChildOptions,
457    pub name: String,
458    /// The hop's `principal`.
459    pub principal: String,
460    pub counters: Arc<Counters>,
461    state: Mutex<LinkState>,
462    dial: tokio::sync::Mutex<()>,
463    /// The mounted schema's hash, as the child published it.
464    pub mounted_hash: OnceLock<String>,
465}
466
467impl Drop for Shared {
468    fn drop(&mut self) {
469        if let Ok(st) = self.state.lock() {
470            if let Some(c) = &st.conn {
471                c.shutdown.notify_one();
472            }
473        }
474    }
475}
476
477/// The hash a child's schema is compared by across restarts.
478pub(super) fn schema_hash(raw: &Value) -> String {
479    match raw.get("hash").and_then(Value::as_str) {
480        Some(h) => h.to_string(),
481        None => raw.get("methods").map(Value::to_string).unwrap_or_default(),
482    }
483}
484
485impl Shared {
486    pub(super) fn new(spec: ChildSpec, opts: ChildOptions, principal: String) -> Arc<Self> {
487        Arc::new(Self {
488            name: spec.name().to_string(),
489            spec,
490            opts,
491            principal,
492            counters: Arc::new(Counters::default()),
493            state: Mutex::new(LinkState { conn: None, down_since: None, why: String::new(), window: VecDeque::new(), dead: None }),
494            dial: tokio::sync::Mutex::new(()),
495            mounted_hash: OnceLock::new(),
496        })
497    }
498
499    pub(super) fn current(&self) -> Option<Arc<Conn>> {
500        self.state.lock().unwrap().conn.clone()
501    }
502
503    pub(super) fn install(&self, conn: Arc<Conn>) {
504        self.state.lock().unwrap().conn = Some(conn);
505    }
506
507    /// The service asks for itself (the mount): no caller, no verdict.
508    fn own_hop(&self) -> Value {
509        json!({ "principal": self.principal, "for": [], "limit": null, "call": uuid::Uuid::new_v4().to_string() })
510    }
511
512    /// Start (or dial) the child and ask its `_info` and its schema.
513    pub(super) async fn connect_once(self: &Arc<Self>) -> Result<(Arc<Conn>, Value, MountTimings), String> {
514        let t0 = Instant::now();
515        let conn = match &self.spec {
516            ChildSpec::Stdio { argv, .. } => self.spawn_stdio(argv)?,
517            ChildSpec::Ws { url, .. } => self.dial_ws(url).await?,
518        };
519        let asked = async {
520            let info = ask(&conn, "_info", Value::Null, self.opts.timeout).await.map_err(|e| format!("`_info`: {e}"))?;
521            let ns = info.get("backend").and_then(Value::as_str).ok_or("its `_info` names no backend")?.to_string();
522            let _ = conn.ns.set(ns.clone());
523            let t1 = Instant::now();
524            let params = json!({ "method": format!("{ns}.schema"), "params": {}, "hop": self.own_hop() });
525            let schema = ask(&conn, &format!("{ns}.call"), params, self.opts.timeout).await.map_err(|e| format!("its schema: {e}"))?;
526            Ok::<_, String>((schema, t1))
527        };
528        match asked.await {
529            Ok((schema, t1)) => {
530                let t = MountTimings { spawn_ms: (t1 - t0).as_secs_f64() * 1e3, schema_ms: t1.elapsed().as_secs_f64() * 1e3, merge_ms: 0.0 };
531                Ok((conn, schema, t))
532            }
533            Err(why) => {
534                conn.shutdown.notify_one();
535                Err(why)
536            }
537        }
538    }
539
540    fn spawn_stdio(self: &Arc<Self>, argv: &[String]) -> Result<Arc<Conn>, String> {
541        let mut cmd = tokio::process::Command::new(&argv[0]);
542        cmd.args(&argv[1..])
543            .env_clear()
544            .envs(["PATH", "HOME", "LANG"].iter().map(|k| k.to_string()).chain(self.opts.env.iter().cloned()).filter_map(|k| std::env::var(&k).ok().map(|v| (k, v))))
545            .env("PLEXUS_CHILD_NAME", &self.name)
546            .stdin(std::process::Stdio::piped())
547            .stdout(std::process::Stdio::piped())
548            .stderr(std::process::Stdio::piped())
549            .kill_on_drop(true);
550        let mut child = cmd.spawn().map_err(|e| format!("spawning `{}`: {e}", argv[0]))?;
551        let (stdin, stdout, stderr) = (child.stdin.take(), child.stdout.take(), child.stderr.take());
552        let (Some(stdin), Some(stdout), Some(stderr)) = (stdin, stdout, stderr) else {
553            return Err("its pipes were not opened".into());
554        };
555        let (out, out_rx) = mpsc::unbounded_channel();
556        let conn = Conn::new(&self.name, child.id(), out, self.opts.buffer, Arc::clone(&self.counters));
557        tokio::spawn(write_lines(stdin, out_rx));
558        let reader = Arc::clone(&conn);
559        let shared = Arc::downgrade(self);
560        tokio::spawn(async move {
561            tokio::select! {
562                _ = read_lines(&reader, stdout) => {}
563                _ = reader.shutdown.notified() => {
564                    let _ = child.start_kill();
565                }
566            }
567            let status = child.wait().await.map(|s| s.to_string()).unwrap_or_else(|e| e.to_string());
568            gone(&shared, &reader, &format!("the process exited ({status})"));
569        });
570        let name = self.name.clone();
571        tokio::spawn(async move {
572            let span = tracing::info_span!(target: "plexus::child", "plexus::child", name = %name);
573            let mut lines = BufReader::new(stderr).lines();
574            while let Ok(Some(l)) = lines.next_line().await {
575                let _e = span.enter();
576                tracing::info!(target: "plexus::child", child = %name, "{l}");
577            }
578        });
579        Ok(conn)
580    }
581
582    #[cfg(feature = "child-ws")]
583    async fn dial_ws(self: &Arc<Self>, url: &str) -> Result<Arc<Conn>, String> {
584        super::ws::dial(self, url).await
585    }
586
587    #[cfg(not(feature = "child-ws"))]
588    async fn dial_ws(self: &Arc<Self>, _url: &str) -> Result<Arc<Conn>, String> {
589        Err("a WebSocket child needs plexus-core's `child-ws` feature".into())
590    }
591
592    /// The connection `conn` is gone; the next call starts another.
593    pub(super) fn lost(&self, conn: &Arc<Conn>, why: &str) {
594        let mut st = self.state.lock().unwrap();
595        if st.conn.as_ref().is_some_and(|c| Arc::ptr_eq(c, conn)) {
596            st.conn = None;
597            st.down_since = Some(Instant::now());
598            st.why = why.to_string();
599            drop(st);
600            tracing::warn!(target: "plexus::child", child = %self.name, "child gone: {why}");
601        }
602    }
603
604    /// A live connection, starting the child again if it died — within the
605    /// backoff, and never past `per_minute` restarts a minute.
606    pub(super) async fn ensure(self: &Arc<Self>) -> Result<Arc<Conn>, PlexusStreamItem> {
607        if let Some(c) = self.live()? {
608            return Ok(c);
609        }
610        let _g = self.dial.lock().await;
611        if let Some(c) = self.live()? {
612            return Ok(c);
613        }
614        let b = self.opts.backoff;
615        let wait = {
616            let mut st = self.state.lock().unwrap();
617            let now = Instant::now();
618            while st.window.front().is_some_and(|t| now.duration_since(*t) >= Duration::from_secs(60)) {
619                st.window.pop_front();
620            }
621            if st.window.len() >= b.per_minute {
622                let next = st.window.front().map(|t| Duration::from_secs(60).saturating_sub(now.duration_since(*t))).unwrap_or_default();
623                return Err(unavailable(&self.name, &format!("{} (restarted {} times in the last minute; the next restart in {:.1} s)", st.why, st.window.len(), next.as_secs_f64())));
624            }
625            let due = st.down_since.unwrap_or(now) + b.delay(st.window.len());
626            due.saturating_duration_since(now)
627        };
628        if wait > self.opts.timeout {
629            let why = self.state.lock().unwrap().why.clone();
630            return Err(unavailable(&self.name, &format!("{why} (restarting in {:.1} s)", wait.as_secs_f64())));
631        }
632        tokio::time::sleep(wait).await;
633        self.state.lock().unwrap().window.push_back(Instant::now());
634        self.counters.restarts.fetch_add(1, Ordering::Relaxed);
635        match self.connect_once().await {
636            Ok((conn, raw, t)) => {
637                let hash = schema_hash(&raw);
638                if self.mounted_hash.get().is_some_and(|h| *h != hash) {
639                    conn.shutdown.notify_one();
640                    let why = "schema changed; restart the service".to_string();
641                    tracing::error!(target: "plexus::child", child = %self.name, mounted = ?self.mounted_hash.get(), published = %hash, "the restarted child published another schema; not remounted: restart the service");
642                    self.state.lock().unwrap().dead = Some(why.clone());
643                    return Err(unavailable(&self.name, &why));
644                }
645                tracing::info!(target: "plexus::child", child = %self.name, spawn_ms = t.spawn_ms, schema_ms = t.schema_ms, "child restarted");
646                self.state.lock().unwrap().conn = Some(Arc::clone(&conn));
647                Ok(conn)
648            }
649            Err(why) => {
650                let mut st = self.state.lock().unwrap();
651                st.down_since = Some(Instant::now());
652                st.why = why.clone();
653                Err(unavailable(&self.name, &why))
654            }
655        }
656    }
657
658    fn live(&self) -> Result<Option<Arc<Conn>>, PlexusStreamItem> {
659        let st = self.state.lock().unwrap();
660        if let Some(why) = &st.dead {
661            return Err(unavailable(&self.name, why));
662        }
663        Ok(st.conn.clone().filter(|c| c.alive()))
664    }
665}
666
667/// The reader's end: every open call told, the supervisor told.
668pub(super) fn gone(shared: &Weak<Shared>, conn: &Arc<Conn>, why: &str) {
669    conn.on_gone(why);
670    if let Some(s) = shared.upgrade() {
671        s.lost(conn, why);
672    }
673}
674
675/// One unary call: the first `data`'s content (the rest read to the end).
676async fn ask(conn: &Conn, method: &str, params: Value, timeout: Duration) -> Result<Value, String> {
677    let (_, mut rx) = conn.open(method, params, timeout).await.map_err(|e| match e {
678        PlexusStreamItem::Error { message, .. } => message,
679        other => format!("{other:?}"),
680    })?;
681    let deadline = tokio::time::Instant::now() + timeout;
682    let mut first = None;
683    loop {
684        match tokio::time::timeout_at(deadline, rx.recv()).await {
685            Ok(Some(PlexusStreamItem::Data { content, .. })) => {
686                if first.is_none() {
687                    first = Some(content.into_value());
688                }
689            }
690            Ok(Some(PlexusStreamItem::Error { message, .. })) => return Err(message),
691            Ok(Some(PlexusStreamItem::Done { .. })) | Ok(None) => break,
692            Ok(Some(_)) => {}
693            Err(_) => return Err(format!("no answer in {timeout:?}")),
694        }
695    }
696    first.ok_or_else(|| "it answered no data".to_string())
697}