1use std::borrow::Cow;
14use std::collections::{HashMap, HashSet, VecDeque};
15use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
16use std::sync::{Arc, Mutex, OnceLock, Weak};
17use std::time::{Duration, Instant};
18
19use serde::Deserialize;
20use serde_json::value::RawValue;
21use serde_json::{json, Value};
22use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWriteExt, BufReader};
23use tokio::sync::{mpsc, oneshot, Notify};
24
25use super::spec::{ChildOptions, ChildSpec};
26use crate::plexus::types::{PlexusStreamItem, StreamMetadata};
27use plexus_client::RemoteError;
28
29pub(super) type SubKey = String;
31
32pub(super) fn meta(name: &str) -> StreamMetadata {
33 StreamMetadata::new(vec![name.to_string()], crate::plexus::context::PlexusContext::hash())
34}
35
36pub(super) fn unavailable(name: &str, why: &str) -> PlexusStreamItem {
38 PlexusStreamItem::error(meta(name), RemoteError::ChildUnavailable { child: name.into(), reason: why.into() }, true)
39}
40
41pub(super) fn timed_out(name: &str, timeout: Duration) -> PlexusStreamItem {
43 let error = RemoteError::ChildTimeout { child: name.into(), timeout_ms: timeout.as_millis() as u64 };
44 let mut item = PlexusStreamItem::error(meta(name), error, true);
45 if let PlexusStreamItem::Error { message, .. } = &mut item {
46 *message = format!("child {name} did not open the call in {timeout:?}");
47 }
48 item
49}
50
51pub(super) fn too_many(name: &str, max: usize) -> PlexusStreamItem {
53 PlexusStreamItem::error(meta(name), RemoteError::TooManyCalls { max: max as u64, child: Some(name.into()) }, true)
54}
55
56fn refused(name: &str, e: &Value) -> PlexusStreamItem {
60 let code = e.get("code").map(|c| match c {
61 Value::String(s) => s.clone(),
62 other => other.to_string(),
63 });
64 let message = e.get("message").and_then(Value::as_str).unwrap_or("the child refused the call").to_string();
65 let error = e
66 .get("data")
67 .and_then(|d| serde_json::from_value::<RemoteError>(d.clone()).ok())
68 .unwrap_or_else(|| super::peer::remote_error_of_code(e.get("code").and_then(Value::as_i64).unwrap_or(-32000), &message));
69 let fields = crate::plexus::types::spec_fields(&error);
70 PlexusStreamItem::Error { metadata: meta(name), message, code, recoverable: false, error, fields }
71}
72
73#[derive(Default)]
75pub(super) struct Counters {
76 pub unasked: AtomicU64,
77 pub calls: AtomicU64,
78 pub restarts: AtomicU64,
79 pub open: AtomicUsize,
80}
81
82pub(super) enum OpenFail {
84 Gone(String),
86 Refused(Value),
88}
89
90struct Opening {
91 reply: oneshot::Sender<Result<SubKey, OpenFail>>,
92 items: Sub,
93}
94
95struct Sub {
99 tx: mpsc::Sender<PlexusStreamItem>,
100 method: Arc<str>,
101}
102
103const TAIL: Duration = Duration::from_millis(250);
107
108#[derive(Default)]
109struct Routes {
110 pending: HashMap<u64, Opening>,
111 subs: HashMap<SubKey, Sub>,
112 closing: HashMap<u64, SubKey>,
116 closing_subs: HashSet<SubKey>,
117 answered: VecDeque<(Instant, SubKey)>,
118 abandoned: HashSet<u64>,
120}
121
122impl Routes {
123 fn expire_tails(&mut self, now: Instant) {
125 while self.answered.front().is_some_and(|(t, _)| now.duration_since(*t) >= TAIL) {
126 if let Some((_, key)) = self.answered.pop_front() {
127 self.closing_subs.remove(&key);
128 }
129 }
130 }
131}
132
133pub(super) struct Conn {
135 pub name: String,
136 pub pid: Option<u32>,
137 pub ns: OnceLock<String>,
139 out: mpsc::UnboundedSender<Vec<u8>>,
140 routes: Mutex<Routes>,
141 next_id: AtomicU64,
142 alive: AtomicBool,
143 pub shutdown: Notify,
145 buffer: usize,
146 counters: Arc<Counters>,
147}
148
149#[derive(Deserialize)]
152struct Msg<'a> {
153 #[serde(default)]
154 id: Option<Value>,
155 #[serde(default, borrow)]
156 result: Option<&'a RawValue>,
157 #[serde(default)]
158 error: Option<Value>,
159 #[serde(default, borrow)]
160 method: Option<Cow<'a, str>>,
161 #[serde(default, borrow)]
162 params: Option<Notif<'a>>,
163}
164
165#[derive(Deserialize)]
166struct Notif<'a> {
167 #[serde(default)]
168 subscription: Option<Value>,
169 #[serde(default, borrow)]
170 result: Option<&'a RawValue>,
171}
172
173#[derive(Deserialize)]
176struct Loose<'a> {
177 #[serde(default)]
178 id: Option<Value>,
179 #[serde(default, borrow)]
180 method: Option<Cow<'a, str>>,
181}
182
183#[derive(Deserialize)]
184struct Kind<'a> {
185 #[serde(rename = "type", borrow)]
186 kind: Cow<'a, str>,
187}
188
189impl Conn {
190 pub(super) fn new(name: &str, pid: Option<u32>, out: mpsc::UnboundedSender<Vec<u8>>, buffer: usize, counters: Arc<Counters>) -> Arc<Self> {
191 Arc::new(Self {
192 name: name.to_string(),
193 pid,
194 ns: OnceLock::new(),
195 out,
196 routes: Mutex::new(Routes::default()),
197 next_id: AtomicU64::new(1),
198 alive: AtomicBool::new(true),
199 shutdown: Notify::new(),
200 buffer,
201 counters,
202 })
203 }
204
205 pub(super) fn alive(&self) -> bool {
206 self.alive.load(Ordering::Acquire)
207 }
208
209 pub(super) fn ns(&self) -> &str {
210 self.ns.get().map(String::as_str).unwrap_or(&self.name)
211 }
212
213 fn send(&self, frame: &Value) {
214 let _ = self.out.send(serde_json::to_vec(frame).unwrap_or_default());
215 }
216
217 pub(super) async fn open(&self, method: &str, params: Value, timeout: Duration) -> Result<(SubKey, mpsc::Receiver<PlexusStreamItem>), PlexusStreamItem> {
221 let id = self.next_id.fetch_add(1, Ordering::Relaxed);
222 let (tx, rx) = mpsc::channel(self.buffer);
223 let (reply, answer) = oneshot::channel();
224 {
225 let mut r = self.routes.lock().unwrap();
226 if !self.alive() {
227 return Err(unavailable(&self.name, "the connection has closed"));
228 }
229 let called = params.get("method").and_then(Value::as_str).unwrap_or(method);
231 r.pending.insert(id, Opening { reply, items: Sub { tx, method: called.into() } });
232 }
233 self.send(&json!({ "jsonrpc": "2.0", "id": id, "method": method, "params": params }));
234 match tokio::time::timeout(timeout, answer).await {
235 Ok(Ok(Ok(key))) => Ok((key, rx)),
236 Ok(Ok(Err(OpenFail::Gone(why)))) => Err(unavailable(&self.name, &why)),
237 Ok(Err(_)) => Err(unavailable(&self.name, "the connection closed before the call opened")),
238 Ok(Ok(Err(OpenFail::Refused(e)))) => Err(refused(&self.name, &e)),
239 Err(_) => {
240 let mut r = self.routes.lock().unwrap();
241 if r.pending.remove(&id).is_some() {
242 r.abandoned.insert(id);
243 }
244 Err(timed_out(&self.name, timeout))
245 }
246 }
247 }
248
249 pub(super) fn unsubscribe(&self, key: &str) {
252 let id = {
253 let mut r = self.routes.lock().unwrap();
254 if r.subs.remove(key).is_none() || !self.alive() {
255 return;
256 }
257 let id = self.next_id.fetch_add(1, Ordering::Relaxed);
258 r.closing.insert(id, key.to_string());
259 r.closing_subs.insert(key.to_string());
260 id
261 };
262 let method = serde_json::to_string(&format!("{}.call_unsub", self.ns())).unwrap_or_default();
263 let frame = format!(r#"{{"jsonrpc":"2.0","id":{id},"method":{method},"params":[{key}]}}"#);
264 let _ = self.out.send(frame.into_bytes());
265 }
266
267 pub(super) fn routed(&self) -> usize {
269 self.routes.lock().unwrap().subs.len()
270 }
271
272 pub(super) async fn on_message(&self, raw: &[u8]) {
274 let msg: Msg<'_> = match serde_json::from_slice(raw) {
275 Ok(m) => m,
276 Err(e) => match serde_json::from_slice::<Loose<'_>>(raw) {
277 Ok(Loose { id: Some(id), method: Some(method) }) => return self.refuse_request(id, &method),
278 _ => {
279 tracing::warn!(target: "plexus::child", child = %self.name, "an unreadable message from the child: {e}");
280 return;
281 }
282 },
283 };
284 if let Some(method) = msg.method {
285 if let Some(id) = msg.id {
286 return self.refuse_request(id, &method);
287 }
288 match msg.params {
289 Some(Notif { subscription: Some(sub), result: Some(item) }) => self.route(sub.to_string(), item).await,
290 _ => tracing::warn!(target: "plexus::child", child = %self.name, %method, "a notification without subscription and result"),
291 }
292 return;
293 }
294 let Some(id) = msg.id.as_ref().and_then(Value::as_u64) else { return };
295 let mut r = self.routes.lock().unwrap();
296 if let Some(open) = r.pending.remove(&id) {
297 match (msg.result, msg.error) {
298 (Some(sid), _) => {
299 let key = match serde_json::from_str::<Value>(sid.get()) {
300 Ok(v) => v.to_string(),
301 Err(_) => sid.get().to_string(),
302 };
303 r.subs.insert(key.clone(), open.items);
304 let _ = open.reply.send(Ok(key));
305 }
306 (None, e) => {
307 let _ = open.reply.send(Err(OpenFail::Refused(e.unwrap_or_else(|| json!({ "message": "an answer with neither result nor error" })))));
308 }
309 }
310 } else if let Some(key) = r.closing.remove(&id) {
311 let now = Instant::now();
313 r.expire_tails(now);
314 r.answered.push_back((now, key));
315 } else if r.abandoned.remove(&id) {
316 if let Some(sid) = msg.result.and_then(|s| serde_json::from_str::<Value>(s.get()).ok()) {
318 let key = sid.to_string();
319 let uid = self.next_id.fetch_add(1, Ordering::Relaxed);
320 r.closing.insert(uid, key.clone());
321 r.closing_subs.insert(key.clone());
322 drop(r);
323 let method = serde_json::to_string(&format!("{}.call_unsub", self.ns())).unwrap_or_default();
324 let _ = self.out.send(format!(r#"{{"jsonrpc":"2.0","id":{uid},"method":{method},"params":[{key}]}}"#).into_bytes());
325 }
326 }
327 }
328
329 fn refuse_request(&self, id: Value, method: &str) {
331 tracing::warn!(target: "plexus::child", child = %self.name, %method, "a request from the child refused");
332 self.send(&json!({ "jsonrpc": "2.0", "id": id, "error": { "code": -32601, "message": "a service answers no requests from its child" } }));
333 }
334
335 async fn route(&self, key: SubKey, raw: &RawValue) {
336 let (tx, method) = {
337 let mut r = self.routes.lock().unwrap();
338 match r.subs.get(&key) {
339 Some(sub) => (sub.tx.clone(), Arc::clone(&sub.method)),
340 None => {
341 r.expire_tails(Instant::now());
342 if !r.closing_subs.contains(&key) {
343 drop(r);
344 self.counters.unasked.fetch_add(1, Ordering::Relaxed);
345 tracing::warn!(target: "plexus::child", child = %self.name, subscription = %key, "unasked item dropped");
346 }
347 return;
348 }
349 }
350 };
351 let parsed = serde_json::from_str::<PlexusStreamItem>(raw.get()).or_else(|e| {
355 super::peer::item(raw.get(), &method)
356 .and_then(|v| serde_json::from_value::<PlexusStreamItem>(v).ok())
357 .ok_or(e)
358 });
359 let item = match parsed {
360 Ok(item) => item,
361 Err(e) => {
362 let kind = serde_json::from_str::<Kind<'_>>(raw.get()).map(|k| k.kind.into_owned()).unwrap_or_default();
363 if kind != "error" {
364 tracing::debug!(target: "plexus::child", child = %self.name, subscription = %key, %kind, "an item kind not relayed: {e}");
367 return;
368 }
369 tracing::warn!(target: "plexus::child", child = %self.name, subscription = %key, "an error item not read: {e}");
374 self.unsubscribe(&key);
375 let error = plexus_client::RemoteError::Execution {
376 message: format!("child {} sent an error item this service could not read: {e}", self.name),
377 };
378 PlexusStreamItem::error(meta(&self.name), error, false)
379 }
380 };
381 if matches!(item, PlexusStreamItem::Done { .. } | PlexusStreamItem::Error { .. }) {
382 self.routes.lock().unwrap().subs.remove(&key);
383 }
384 match tx.try_send(item) {
385 Ok(()) => {}
386 Err(mpsc::error::TrySendError::Full(item)) => {
389 let _ = tx.send(item).await;
390 }
391 Err(mpsc::error::TrySendError::Closed(_)) => {}
392 }
393 }
394
395 pub(super) fn on_gone(&self, why: &str) {
397 let (subs, pending) = {
398 let mut r = self.routes.lock().unwrap();
399 self.alive.store(false, Ordering::Release);
400 r.closing.clear();
401 r.closing_subs.clear();
402 r.answered.clear();
403 r.abandoned.clear();
404 (std::mem::take(&mut r.subs), std::mem::take(&mut r.pending))
405 };
406 for (_, Sub { tx, .. }) in subs {
407 match tx.try_send(unavailable(&self.name, why)) {
408 Err(mpsc::error::TrySendError::Full(item)) => {
409 tokio::spawn(async move {
410 let _ = tx.send(item).await;
411 });
412 }
413 _ => {}
414 }
415 }
416 for (_, open) in pending {
417 let _ = open.reply.send(Err(OpenFail::Gone(why.to_string())));
418 }
419 }
420}
421
422pub(super) async fn read_lines<R: AsyncRead + Unpin>(conn: &Conn, r: R) {
425 let mut r = BufReader::with_capacity(1 << 17, r);
426 let mut carry: Vec<u8> = Vec::new();
427 loop {
428 let buf = match r.fill_buf().await {
429 Ok(b) if !b.is_empty() => b,
430 _ => break,
431 };
432 let n = buf.len();
433 let mut start = 0;
434 while let Some(i) = buf[start..].iter().position(|b| *b == b'\n') {
435 let end = start + i;
436 if carry.is_empty() {
437 let line = &buf[start..end];
438 if line.iter().any(|b| !b.is_ascii_whitespace()) {
439 conn.on_message(line).await;
440 }
441 } else {
442 carry.extend_from_slice(&buf[start..end]);
443 let line = std::mem::take(&mut carry);
444 if line.iter().any(|b| !b.is_ascii_whitespace()) {
445 conn.on_message(&line).await;
446 }
447 }
448 start = end + 1;
449 }
450 carry.extend_from_slice(&buf[start..]);
451 r.consume(n);
452 }
453}
454
455pub(super) async fn write_lines<W: tokio::io::AsyncWrite + Unpin>(mut w: W, mut rx: mpsc::UnboundedReceiver<Vec<u8>>) {
457 while let Some(first) = rx.recv().await {
458 let mut buf = first;
459 buf.push(b'\n');
460 while let Ok(more) = rx.try_recv() {
461 buf.extend_from_slice(&more);
462 buf.push(b'\n');
463 }
464 if w.write_all(&buf).await.is_err() || w.flush().await.is_err() {
465 break;
466 }
467 }
468}
469
470#[derive(Debug, Clone, Copy, Default, PartialEq)]
472pub struct MountTimings {
473 pub spawn_ms: f64,
475 pub schema_ms: f64,
477 pub merge_ms: f64,
479}
480
481struct LinkState {
482 conn: Option<Arc<Conn>>,
483 down_since: Option<Instant>,
484 why: String,
485 window: VecDeque<Instant>,
486 dead: Option<String>,
488}
489
490pub(super) struct Shared {
492 pub spec: ChildSpec,
493 pub opts: ChildOptions,
494 pub name: String,
495 pub principal: String,
497 pub counters: Arc<Counters>,
498 state: Mutex<LinkState>,
499 dial: tokio::sync::Mutex<()>,
500 pub mounted_hash: OnceLock<String>,
502}
503
504impl Drop for Shared {
505 fn drop(&mut self) {
506 if let Ok(st) = self.state.lock() {
507 if let Some(c) = &st.conn {
508 c.shutdown.notify_one();
509 }
510 }
511 }
512}
513
514pub(super) fn schema_hash(raw: &Value) -> String {
516 match raw.get("hash").and_then(Value::as_str) {
517 Some(h) => h.to_string(),
518 None => raw.get("methods").map(Value::to_string).unwrap_or_default(),
519 }
520}
521
522impl Shared {
523 pub(super) fn new(spec: ChildSpec, opts: ChildOptions, principal: String) -> Arc<Self> {
524 Arc::new(Self {
525 name: spec.name().to_string(),
526 spec,
527 opts,
528 principal,
529 counters: Arc::new(Counters::default()),
530 state: Mutex::new(LinkState { conn: None, down_since: None, why: String::new(), window: VecDeque::new(), dead: None }),
531 dial: tokio::sync::Mutex::new(()),
532 mounted_hash: OnceLock::new(),
533 })
534 }
535
536 pub(super) fn current(&self) -> Option<Arc<Conn>> {
537 self.state.lock().unwrap().conn.clone()
538 }
539
540 pub(super) fn install(&self, conn: Arc<Conn>) {
541 self.state.lock().unwrap().conn = Some(conn);
542 }
543
544 fn own_hop(&self) -> Value {
546 json!({ "principal": self.principal, "for": [], "limit": null, "call": uuid::Uuid::new_v4().to_string() })
547 }
548
549 pub(super) async fn connect_once(self: &Arc<Self>) -> Result<(Arc<Conn>, Value, MountTimings), String> {
551 let t0 = Instant::now();
552 let conn = match &self.spec {
553 ChildSpec::Stdio { argv, .. } => self.spawn_stdio(argv)?,
554 ChildSpec::Ws { url, .. } => self.dial_ws(url).await?,
555 };
556 let asked = async {
557 let info = ask(&conn, "_info", Value::Null, self.opts.timeout).await.map_err(|e| format!("`_info`: {e}"))?;
558 let ns = info.get("backend").and_then(Value::as_str).ok_or("its `_info` names no backend")?.to_string();
559 let _ = conn.ns.set(ns.clone());
560 let t1 = Instant::now();
561 let params = json!({ "method": format!("{ns}.schema"), "params": {}, "hop": self.own_hop() });
562 let schema = ask(&conn, &format!("{ns}.call"), params, self.opts.timeout).await.map_err(|e| format!("its schema: {e}"))?;
563 Ok::<_, String>((schema, t1))
564 };
565 match asked.await {
566 Ok((schema, t1)) => {
567 let t = MountTimings { spawn_ms: (t1 - t0).as_secs_f64() * 1e3, schema_ms: t1.elapsed().as_secs_f64() * 1e3, merge_ms: 0.0 };
568 Ok((conn, schema, t))
569 }
570 Err(why) => {
571 conn.shutdown.notify_one();
572 Err(why)
573 }
574 }
575 }
576
577 fn spawn_stdio(self: &Arc<Self>, argv: &[String]) -> Result<Arc<Conn>, String> {
578 let mut cmd = tokio::process::Command::new(&argv[0]);
579 cmd.args(&argv[1..])
580 .env_clear()
581 .envs(["PATH", "HOME", "LANG"].iter().map(|k| k.to_string()).chain(self.opts.env.iter().cloned()).filter_map(|k| std::env::var(&k).ok().map(|v| (k, v))))
582 .env("PLEXUS_CHILD_NAME", &self.name)
583 .stdin(std::process::Stdio::piped())
584 .stdout(std::process::Stdio::piped())
585 .stderr(std::process::Stdio::piped())
586 .kill_on_drop(true);
587 let mut child = cmd.spawn().map_err(|e| format!("spawning `{}`: {e}", argv[0]))?;
588 let (stdin, stdout, stderr) = (child.stdin.take(), child.stdout.take(), child.stderr.take());
589 let (Some(stdin), Some(stdout), Some(stderr)) = (stdin, stdout, stderr) else {
590 return Err("its pipes were not opened".into());
591 };
592 let (out, out_rx) = mpsc::unbounded_channel();
593 let conn = Conn::new(&self.name, child.id(), out, self.opts.buffer, Arc::clone(&self.counters));
594 tokio::spawn(write_lines(stdin, out_rx));
595 let reader = Arc::clone(&conn);
596 let shared = Arc::downgrade(self);
597 tokio::spawn(async move {
598 tokio::select! {
599 _ = read_lines(&reader, stdout) => {}
600 _ = reader.shutdown.notified() => {
601 let _ = child.start_kill();
602 }
603 }
604 let status = child.wait().await.map(|s| s.to_string()).unwrap_or_else(|e| e.to_string());
605 gone(&shared, &reader, &format!("the process exited ({status})"));
606 });
607 let name = self.name.clone();
608 tokio::spawn(async move {
609 let span = tracing::info_span!(target: "plexus::child", "plexus::child", name = %name);
610 let mut lines = BufReader::new(stderr).lines();
611 while let Ok(Some(l)) = lines.next_line().await {
612 let _e = span.enter();
613 tracing::info!(target: "plexus::child", child = %name, "{l}");
614 }
615 });
616 Ok(conn)
617 }
618
619 #[cfg(feature = "child-ws")]
620 async fn dial_ws(self: &Arc<Self>, url: &str) -> Result<Arc<Conn>, String> {
621 super::ws::dial(self, url).await
622 }
623
624 #[cfg(not(feature = "child-ws"))]
625 async fn dial_ws(self: &Arc<Self>, _url: &str) -> Result<Arc<Conn>, String> {
626 Err("a WebSocket child needs plexus-core's `child-ws` feature".into())
627 }
628
629 pub(super) fn lost(&self, conn: &Arc<Conn>, why: &str) {
631 let mut st = self.state.lock().unwrap();
632 if st.conn.as_ref().is_some_and(|c| Arc::ptr_eq(c, conn)) {
633 st.conn = None;
634 st.down_since = Some(Instant::now());
635 st.why = why.to_string();
636 drop(st);
637 tracing::warn!(target: "plexus::child", child = %self.name, "child gone: {why}");
638 }
639 }
640
641 pub(super) async fn ensure(self: &Arc<Self>) -> Result<Arc<Conn>, PlexusStreamItem> {
644 if let Some(c) = self.live()? {
645 return Ok(c);
646 }
647 let _g = self.dial.lock().await;
648 if let Some(c) = self.live()? {
649 return Ok(c);
650 }
651 let b = self.opts.backoff;
652 let wait = {
653 let mut st = self.state.lock().unwrap();
654 let now = Instant::now();
655 while st.window.front().is_some_and(|t| now.duration_since(*t) >= Duration::from_secs(60)) {
656 st.window.pop_front();
657 }
658 if st.window.len() >= b.per_minute {
659 let next = st.window.front().map(|t| Duration::from_secs(60).saturating_sub(now.duration_since(*t))).unwrap_or_default();
660 return Err(unavailable(&self.name, &format!("{} (restarted {} times in the last minute; the next restart in {:.1} s)", st.why, st.window.len(), next.as_secs_f64())));
661 }
662 let due = st.down_since.unwrap_or(now) + b.delay(st.window.len());
663 due.saturating_duration_since(now)
664 };
665 if wait > self.opts.timeout {
666 let why = self.state.lock().unwrap().why.clone();
667 return Err(unavailable(&self.name, &format!("{why} (restarting in {:.1} s)", wait.as_secs_f64())));
668 }
669 tokio::time::sleep(wait).await;
670 self.state.lock().unwrap().window.push_back(Instant::now());
671 self.counters.restarts.fetch_add(1, Ordering::Relaxed);
672 match self.connect_once().await {
673 Ok((conn, raw, t)) => {
674 let hash = schema_hash(&raw);
675 if self.mounted_hash.get().is_some_and(|h| *h != hash) {
676 conn.shutdown.notify_one();
677 let why = "schema changed; restart the service".to_string();
678 tracing::error!(target: "plexus::child", child = %self.name, mounted = ?self.mounted_hash.get(), published = %hash, "the restarted child published another schema; not remounted: restart the service");
679 self.state.lock().unwrap().dead = Some(why.clone());
680 return Err(unavailable(&self.name, &why));
681 }
682 tracing::info!(target: "plexus::child", child = %self.name, spawn_ms = t.spawn_ms, schema_ms = t.schema_ms, "child restarted");
683 self.state.lock().unwrap().conn = Some(Arc::clone(&conn));
684 Ok(conn)
685 }
686 Err(why) => {
687 let mut st = self.state.lock().unwrap();
688 st.down_since = Some(Instant::now());
689 st.why = why.clone();
690 Err(unavailable(&self.name, &why))
691 }
692 }
693 }
694
695 fn live(&self) -> Result<Option<Arc<Conn>>, PlexusStreamItem> {
696 let st = self.state.lock().unwrap();
697 if let Some(why) = &st.dead {
698 return Err(unavailable(&self.name, why));
699 }
700 Ok(st.conn.clone().filter(|c| c.alive()))
701 }
702}
703
704pub(super) fn gone(shared: &Weak<Shared>, conn: &Arc<Conn>, why: &str) {
706 conn.on_gone(why);
707 if let Some(s) = shared.upgrade() {
708 s.lost(conn, why);
709 }
710}
711
712async fn ask(conn: &Conn, method: &str, params: Value, timeout: Duration) -> Result<Value, String> {
714 let (_, mut rx) = conn.open(method, params, timeout).await.map_err(|e| match e {
715 PlexusStreamItem::Error { message, .. } => message,
716 other => format!("{other:?}"),
717 })?;
718 let deadline = tokio::time::Instant::now() + timeout;
719 let mut first = None;
720 loop {
721 match tokio::time::timeout_at(deadline, rx.recv()).await {
722 Ok(Some(PlexusStreamItem::Data { content, .. })) => {
723 if first.is_none() {
724 first = Some(content.into_value());
725 }
726 }
727 Ok(Some(PlexusStreamItem::Error { message, .. })) => return Err(message),
728 Ok(Some(PlexusStreamItem::Done { .. })) | Ok(None) => break,
729 Ok(Some(_)) => {}
730 Err(_) => return Err(format!("no answer in {timeout:?}")),
731 }
732 }
733 first.ok_or_else(|| "it answered no data".to_string())
734}