1use std::borrow::Cow;
14use std::collections::{HashMap, HashSet, VecDeque};
15use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
16use std::sync::{Arc, Mutex, OnceLock, Weak};
17use std::time::{Duration, Instant};
18
19use serde::Deserialize;
20use serde_json::value::RawValue;
21use serde_json::{json, Value};
22use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWriteExt, BufReader};
23use tokio::sync::{mpsc, oneshot, Notify};
24
25use super::spec::{ChildOptions, ChildSpec};
26use super::upcall::{self, Served, Upcalls};
27use crate::plexus::types::{PlexusStreamItem, StreamMetadata};
28use plexus_client::RemoteError;
29
30pub(super) type SubKey = String;
32
33pub(super) fn meta(name: &str) -> StreamMetadata {
34 StreamMetadata::new(vec![name.to_string()], crate::plexus::context::PlexusContext::hash())
35}
36
37pub(super) fn unavailable(name: &str, why: &str) -> PlexusStreamItem {
39 PlexusStreamItem::error(meta(name), RemoteError::ChildUnavailable { child: name.into(), reason: why.into() }, true)
40}
41
42pub(super) fn timed_out(name: &str, timeout: Duration) -> PlexusStreamItem {
44 let error = RemoteError::ChildTimeout { child: name.into(), timeout_ms: timeout.as_millis() as u64 };
45 let mut item = PlexusStreamItem::error(meta(name), error, true);
46 if let PlexusStreamItem::Error { message, .. } = &mut item {
47 *message = format!("child {name} did not open the call in {timeout:?}");
48 }
49 item
50}
51
52pub(super) fn too_many(name: &str, max: usize) -> PlexusStreamItem {
54 PlexusStreamItem::error(meta(name), RemoteError::TooManyCalls { max: max as u64, child: Some(name.into()) }, true)
55}
56
57fn refused(name: &str, e: &Value) -> PlexusStreamItem {
61 let code = e.get("code").map(|c| match c {
62 Value::String(s) => s.clone(),
63 other => other.to_string(),
64 });
65 let message = e.get("message").and_then(Value::as_str).unwrap_or("the child refused the call").to_string();
66 let error = e
67 .get("data")
68 .and_then(|d| serde_json::from_value::<RemoteError>(d.clone()).ok())
69 .unwrap_or_else(|| super::peer::remote_error_of_code(e.get("code").and_then(Value::as_i64).unwrap_or(-32000), &message));
70 let fields = crate::plexus::types::spec_fields(&error);
71 PlexusStreamItem::Error { metadata: meta(name), message, code, recoverable: false, error, fields }
72}
73
74#[derive(Default)]
76pub(super) struct Counters {
77 pub unasked: AtomicU64,
78 pub calls: AtomicU64,
79 pub restarts: AtomicU64,
80 pub open: AtomicUsize,
81}
82
83pub(super) enum OpenFail {
85 Gone(String),
87 Refused(Value),
89}
90
91struct Opening {
92 reply: oneshot::Sender<Result<SubKey, OpenFail>>,
93 items: Sub,
94}
95
96struct Sub {
100 tx: mpsc::Sender<PlexusStreamItem>,
101 method: Arc<str>,
102 mark: Option<Arc<PeerMark>>,
105}
106
107pub(super) struct PeerMark {
110 call: Option<crate::call::CallContext>,
111 set: std::sync::atomic::AtomicBool,
112}
113
114impl PeerMark {
115 pub(super) fn here() -> Arc<Self> {
121 let call = crate::call::context().ok().filter(crate::call::CallContext::routing);
122 Arc::new(Self { call, set: std::sync::atomic::AtomicBool::new(false) })
123 }
124
125 fn set(&self) {
127 if !self.set.swap(true, Ordering::AcqRel) {
128 if let Some(c) = &self.call {
129 c.peer_ended();
130 }
131 }
132 }
133
134 pub(super) fn taken(&self) {
136 if self.set.swap(false, Ordering::AcqRel) {
137 if let Some(c) = &self.call {
138 c.peer_end_taken();
139 }
140 }
141 }
142}
143
144const TAIL: Duration = Duration::from_millis(250);
148
149#[derive(Default)]
150struct Routes {
151 pending: HashMap<u64, Opening>,
152 subs: HashMap<SubKey, Sub>,
153 closing: HashMap<u64, SubKey>,
157 closing_subs: HashSet<SubKey>,
158 answered: VecDeque<(Instant, SubKey)>,
159 abandoned: HashSet<u64>,
161}
162
163impl Routes {
164 fn expire_tails(&mut self, now: Instant) {
166 while self.answered.front().is_some_and(|(t, _)| now.duration_since(*t) >= TAIL) {
167 if let Some((_, key)) = self.answered.pop_front() {
168 self.closing_subs.remove(&key);
169 }
170 }
171 }
172}
173
174pub(super) struct Conn {
176 pub name: String,
177 pub pid: Option<u32>,
178 pub ns: OnceLock<String>,
180 out: mpsc::UnboundedSender<Vec<u8>>,
181 routes: Mutex<Routes>,
182 next_id: AtomicU64,
183 alive: AtomicBool,
184 pub shutdown: Notify,
186 buffer: usize,
187 counters: Arc<Counters>,
188 pub up: OnceLock<Arc<Upcalls>>,
192 served: Arc<Served>,
194 pub tls_key: OnceLock<String>,
197 pub peer_key: OnceLock<String>,
200 stopped: OnceLock<String>,
203}
204
205#[derive(Deserialize)]
208struct Msg<'a> {
209 #[serde(default)]
210 id: Option<Value>,
211 #[serde(default, borrow)]
212 result: Option<&'a RawValue>,
213 #[serde(default)]
214 error: Option<Value>,
215 #[serde(default, borrow)]
216 method: Option<Cow<'a, str>>,
217 #[serde(default, borrow)]
218 params: Option<Notif<'a>>,
219}
220
221#[derive(Deserialize)]
222struct Notif<'a> {
223 #[serde(default)]
224 subscription: Option<Value>,
225 #[serde(default, borrow)]
226 result: Option<&'a RawValue>,
227}
228
229#[derive(Deserialize)]
232struct Loose<'a> {
233 #[serde(default)]
234 id: Option<Value>,
235 #[serde(default, borrow)]
236 method: Option<Cow<'a, str>>,
237}
238
239#[derive(Deserialize)]
240struct Kind<'a> {
241 #[serde(rename = "type", borrow)]
242 kind: Cow<'a, str>,
243}
244
245impl Conn {
246 pub(super) fn new(name: &str, pid: Option<u32>, out: mpsc::UnboundedSender<Vec<u8>>, buffer: usize, counters: Arc<Counters>) -> Arc<Self> {
247 Arc::new(Self {
248 name: name.to_string(),
249 pid,
250 ns: OnceLock::new(),
251 out,
252 routes: Mutex::new(Routes::default()),
253 next_id: AtomicU64::new(1),
254 alive: AtomicBool::new(true),
255 shutdown: Notify::new(),
256 buffer,
257 counters,
258 up: OnceLock::new(),
259 served: Arc::default(),
260 tls_key: OnceLock::new(),
261 peer_key: OnceLock::new(),
262 stopped: OnceLock::new(),
263 })
264 }
265
266 pub(super) fn alive(&self) -> bool {
267 self.alive.load(Ordering::Acquire)
268 }
269
270 pub(super) fn ns(&self) -> &str {
271 self.ns.get().map(String::as_str).unwrap_or(&self.name)
272 }
273
274 fn send(&self, frame: &Value) {
275 let _ = self.out.send(serde_json::to_vec(frame).unwrap_or_default());
276 }
277
278 pub(super) async fn open(&self, method: &str, params: Value, timeout: Duration, mark: Option<Arc<PeerMark>>) -> Result<(SubKey, mpsc::Receiver<PlexusStreamItem>), PlexusStreamItem> {
282 let id = self.next_id.fetch_add(1, Ordering::Relaxed);
283 let (tx, rx) = mpsc::channel(self.buffer);
284 let (reply, answer) = oneshot::channel();
285 {
286 let mut r = self.routes.lock().unwrap();
287 if !self.alive() {
288 return Err(unavailable(&self.name, "the connection has closed"));
289 }
290 if let Some(why) = self.stopped.get() {
293 return Err(unavailable(&self.name, why));
294 }
295 let called = params.get("method").and_then(Value::as_str).unwrap_or(method);
297 r.pending.insert(id, Opening { reply, items: Sub { tx, method: called.into(), mark } });
298 }
299 self.send(&json!({ "jsonrpc": "2.0", "id": id, "method": method, "params": params }));
300 match tokio::time::timeout(timeout, answer).await {
301 Ok(Ok(Ok(key))) => Ok((key, rx)),
302 Ok(Ok(Err(OpenFail::Gone(why)))) => Err(unavailable(&self.name, &why)),
303 Ok(Err(_)) => Err(unavailable(&self.name, "the connection closed before the call opened")),
304 Ok(Ok(Err(OpenFail::Refused(e)))) => Err(refused(&self.name, &e)),
305 Err(_) => {
306 let mut r = self.routes.lock().unwrap();
307 if r.pending.remove(&id).is_some() {
308 r.abandoned.insert(id);
309 }
310 Err(timed_out(&self.name, timeout))
311 }
312 }
313 }
314
315 pub(super) fn unsubscribe(&self, key: &str) {
318 let id = {
319 let mut r = self.routes.lock().unwrap();
320 if r.subs.remove(key).is_none() || !self.alive() {
321 return;
322 }
323 self.closing(&mut r, key)
324 };
325 self.send_unsub(id, key);
326 }
327
328 fn closing(&self, r: &mut Routes, key: &str) -> u64 {
330 let id = self.next_id.fetch_add(1, Ordering::Relaxed);
331 r.closing.insert(id, key.to_string());
332 r.closing_subs.insert(key.to_string());
333 id
334 }
335
336 fn send_unsub(&self, id: u64, key: &str) {
338 let method = serde_json::to_string(&format!("{}.call_unsub", self.ns())).unwrap_or_default();
339 let frame = format!(r#"{{"jsonrpc":"2.0","id":{id},"method":{method},"params":[{key}]}}"#);
340 let _ = self.out.send(frame.into_bytes());
341 }
342
343 pub(super) fn stop(&self, why: &str) {
351 let (subs, pending, unsubs) = {
352 let mut r = self.routes.lock().unwrap();
353 if self.stopped.set(why.to_string()).is_err() {
354 return;
355 }
356 let subs = std::mem::take(&mut r.subs);
357 let alive = self.alive();
358 let unsubs: Vec<(u64, SubKey)> = if alive { subs.keys().map(|k| (self.closing(&mut r, k), k.clone())).collect() } else { Vec::new() };
359 (subs, std::mem::take(&mut r.pending), unsubs)
360 };
361 for (id, key) in &unsubs {
362 self.send_unsub(*id, key);
363 }
364 for (_, Sub { tx, mark, .. }) in subs {
365 if let Some(m) = mark {
368 m.set();
369 }
370 Self::tell(tx, unavailable(&self.name, why));
371 }
372 for (_, open) in pending {
373 let _ = open.reply.send(Err(OpenFail::Gone(why.to_string())));
374 }
375 }
376
377 pub(super) fn stopped(&self) -> Option<&str> {
379 self.stopped.get().map(String::as_str)
380 }
381
382 fn tell(tx: mpsc::Sender<PlexusStreamItem>, item: PlexusStreamItem) {
385 if let Err(mpsc::error::TrySendError::Full(item)) = tx.try_send(item) {
386 tokio::spawn(async move {
387 let _ = tx.send(item).await;
388 });
389 }
390 }
391
392 pub(super) fn routed(&self) -> usize {
394 self.routes.lock().unwrap().subs.len()
395 }
396
397 pub(super) async fn on_message(&self, raw: &[u8]) {
399 let msg: Msg<'_> = match serde_json::from_slice(raw) {
400 Ok(m) => m,
401 Err(e) => match serde_json::from_slice::<Loose<'_>>(raw) {
402 Ok(Loose { id: Some(id), method: Some(method) }) => return self.request(id, &method, raw),
403 _ => {
404 tracing::warn!(target: "plexus::child", child = %self.name, "an unreadable message from the child: {e}");
405 return;
406 }
407 },
408 };
409 if let Some(method) = msg.method {
410 if let Some(id) = msg.id {
411 return self.request(id, &method, raw);
412 }
413 match msg.params {
414 Some(Notif { subscription: Some(sub), result: Some(item) }) => self.route(sub.to_string(), item).await,
415 _ => tracing::warn!(target: "plexus::child", child = %self.name, %method, "a notification without subscription and result"),
416 }
417 return;
418 }
419 let Some(id) = msg.id.as_ref().and_then(Value::as_u64) else { return };
420 let mut r = self.routes.lock().unwrap();
421 if let Some(open) = r.pending.remove(&id) {
422 match (msg.result, msg.error) {
423 (Some(sid), _) => {
424 let key = match serde_json::from_str::<Value>(sid.get()) {
425 Ok(v) => v.to_string(),
426 Err(_) => sid.get().to_string(),
427 };
428 r.subs.insert(key.clone(), open.items);
429 let _ = open.reply.send(Ok(key));
430 }
431 (None, e) => {
432 let _ = open.reply.send(Err(OpenFail::Refused(e.unwrap_or_else(|| json!({ "message": "an answer with neither result nor error" })))));
433 }
434 }
435 } else if let Some(key) = r.closing.remove(&id) {
436 let now = Instant::now();
438 r.expire_tails(now);
439 r.answered.push_back((now, key));
440 } else if r.abandoned.remove(&id) {
441 if let Some(sid) = msg.result.and_then(|s| serde_json::from_str::<Value>(s.get()).ok()) {
443 let key = sid.to_string();
444 let uid = self.next_id.fetch_add(1, Ordering::Relaxed);
445 r.closing.insert(uid, key.clone());
446 r.closing_subs.insert(key.clone());
447 drop(r);
448 let method = serde_json::to_string(&format!("{}.call_unsub", self.ns())).unwrap_or_default();
449 let _ = self.out.send(format!(r#"{{"jsonrpc":"2.0","id":{uid},"method":{method},"params":[{key}]}}"#).into_bytes());
450 }
451 }
452 }
453
454 fn request(&self, id: Value, method: &str, raw: &[u8]) {
458 if let (Some(up), true) = (self.up.get(), method == upcall::HOST_CALL || method == upcall::HOST_CALL_UNSUB) {
459 let params = serde_json::from_slice::<Value>(raw).ok().and_then(|mut v| v.get_mut("params").map(Value::take)).unwrap_or(Value::Null);
460 return upcall::on_request(up, &self.served, &self.out, &self.name, id, method, params);
461 }
462 self.refuse_request(id, method)
463 }
464
465 fn refuse_request(&self, id: Value, method: &str) {
467 tracing::warn!(target: "plexus::child", child = %self.name, %method, "a request from the child refused");
468 self.send(&json!({ "jsonrpc": "2.0", "id": id, "error": { "code": -32601, "message": "a service answers no requests from its child" } }));
469 }
470
471 async fn route(&self, key: SubKey, raw: &RawValue) {
472 let (tx, method) = {
473 let mut r = self.routes.lock().unwrap();
474 match r.subs.get(&key) {
475 Some(sub) => (sub.tx.clone(), Arc::clone(&sub.method)),
476 None => {
477 r.expire_tails(Instant::now());
478 if !r.closing_subs.contains(&key) {
479 drop(r);
480 self.counters.unasked.fetch_add(1, Ordering::Relaxed);
481 tracing::warn!(target: "plexus::child", child = %self.name, subscription = %key, "unasked item dropped");
482 }
483 return;
484 }
485 }
486 };
487 let parsed = serde_json::from_str::<PlexusStreamItem>(raw.get()).or_else(|e| {
491 super::peer::item(raw.get(), &method)
492 .and_then(|v| serde_json::from_value::<PlexusStreamItem>(v).ok())
493 .ok_or(e)
494 });
495 let item = match parsed {
496 Ok(item) => item,
497 Err(e) => {
498 let kind = serde_json::from_str::<Kind<'_>>(raw.get()).map(|k| k.kind.into_owned()).unwrap_or_default();
499 if kind != "error" {
500 tracing::debug!(target: "plexus::child", child = %self.name, subscription = %key, %kind, "an item kind not relayed: {e}");
503 return;
504 }
505 tracing::warn!(target: "plexus::child", child = %self.name, subscription = %key, "an error item not read: {e}");
510 self.unsubscribe(&key);
511 let error = plexus_client::RemoteError::Execution {
512 message: format!("child {} sent an error item this service could not read: {e}", self.name),
513 };
514 PlexusStreamItem::error(meta(&self.name), error, false)
515 }
516 };
517 if matches!(item, PlexusStreamItem::Done { .. } | PlexusStreamItem::Error { .. }) {
518 self.routes.lock().unwrap().subs.remove(&key);
519 }
520 match tx.try_send(item) {
521 Ok(()) => {}
522 Err(mpsc::error::TrySendError::Full(item)) => {
525 let _ = tx.send(item).await;
526 }
527 Err(mpsc::error::TrySendError::Closed(_)) => {}
528 }
529 }
530
531 pub(super) fn on_gone(&self, why: &str) {
533 let (subs, pending) = {
534 let mut r = self.routes.lock().unwrap();
535 self.alive.store(false, Ordering::Release);
536 self.served.abort_all();
537 r.closing.clear();
538 r.closing_subs.clear();
539 r.answered.clear();
540 r.abandoned.clear();
541 (std::mem::take(&mut r.subs), std::mem::take(&mut r.pending))
542 };
543 for (_, Sub { tx, mark, .. }) in subs {
544 if let Some(m) = mark {
546 m.set();
547 }
548 Self::tell(tx, unavailable(&self.name, why));
549 }
550 for (_, open) in pending {
551 let _ = open.reply.send(Err(OpenFail::Gone(why.to_string())));
552 }
553 }
554}
555
556pub(super) async fn read_lines<R: AsyncRead + Unpin>(conn: &Conn, r: R) {
559 let mut r = BufReader::with_capacity(1 << 17, r);
560 let mut carry: Vec<u8> = Vec::new();
561 loop {
562 let buf = match r.fill_buf().await {
563 Ok(b) if !b.is_empty() => b,
564 _ => break,
565 };
566 let n = buf.len();
567 let mut start = 0;
568 while let Some(i) = buf[start..].iter().position(|b| *b == b'\n') {
569 let end = start + i;
570 if carry.is_empty() {
571 let line = &buf[start..end];
572 if line.iter().any(|b| !b.is_ascii_whitespace()) {
573 conn.on_message(line).await;
574 }
575 } else {
576 carry.extend_from_slice(&buf[start..end]);
577 let line = std::mem::take(&mut carry);
578 if line.iter().any(|b| !b.is_ascii_whitespace()) {
579 conn.on_message(&line).await;
580 }
581 }
582 start = end + 1;
583 }
584 carry.extend_from_slice(&buf[start..]);
585 r.consume(n);
586 }
587}
588
589pub(super) async fn write_lines<W: tokio::io::AsyncWrite + Unpin>(mut w: W, mut rx: mpsc::UnboundedReceiver<Vec<u8>>) {
591 while let Some(first) = rx.recv().await {
592 let mut buf = first;
593 buf.push(b'\n');
594 while let Ok(more) = rx.try_recv() {
595 buf.extend_from_slice(&more);
596 buf.push(b'\n');
597 }
598 if w.write_all(&buf).await.is_err() || w.flush().await.is_err() {
599 break;
600 }
601 }
602}
603
604#[derive(Debug, Clone, Copy, Default, PartialEq)]
606pub struct MountTimings {
607 pub spawn_ms: f64,
609 pub schema_ms: f64,
611 pub merge_ms: f64,
613}
614
615struct LinkState {
616 conn: Option<Arc<Conn>>,
617 down_since: Option<Instant>,
618 why: String,
619 window: VecDeque<Instant>,
620 dead: Option<String>,
622}
623
624pub const KEYED_MOUNT_NEEDS_A_KEY: &str = "keyed_mount_needs_a_key";
628
629pub(super) struct Shared {
630 pub spec: ChildSpec,
631 pub opts: ChildOptions,
632 pub name: String,
633 pub principal: String,
635 pub counters: Arc<Counters>,
636 state: Mutex<LinkState>,
637 dial: tokio::sync::Mutex<()>,
638 pub mounted_hash: OnceLock<String>,
640 pub upcalls: Arc<Upcalls>,
642 pub identity: OnceLock<super::PeerIdentity>,
646 stopped: OnceLock<String>,
649}
650
651impl Drop for Shared {
652 fn drop(&mut self) {
653 if let Ok(st) = self.state.lock() {
654 if let Some(c) = &st.conn {
655 c.shutdown.notify_one();
656 }
657 }
658 }
659}
660
661pub(super) fn schema_hash(raw: &Value) -> String {
663 match raw.get("hash").and_then(Value::as_str) {
664 Some(h) => h.to_string(),
665 None => raw.get("methods").map(Value::to_string).unwrap_or_default(),
666 }
667}
668
669#[derive(Debug, Clone)]
673pub(super) struct Kept {
674 pub top: Value,
675 pub nodes: Vec<(String, Value)>,
676}
677
678impl Kept {
679 pub fn hash(&self) -> String {
684 let mut h = schema_hash(&self.top);
685 for (path, raw) in &self.nodes {
686 h.push_str(&format!(" {path}={}", schema_hash(raw)));
687 }
688 h
689 }
690}
691
692pub(super) fn static_children(raw: &Value) -> Vec<String> {
697 let methods = raw.get("methods").and_then(Value::as_array).map(Vec::as_slice).unwrap_or_default();
698 let kind = |m: &Value| m.get("role").and_then(|r| r.get("kind").or(Some(r))).and_then(Value::as_str).map(str::to_string);
699 if methods.iter().any(|m| kind(m).is_some_and(|k| k.ends_with("_child"))) {
703 let peer = |m: &Value| m.get("role").and_then(|r| r.get("peer")).and_then(Value::as_bool).unwrap_or(false);
706 return methods.iter().filter(|m| kind(m).as_deref() == Some("static_child") && !peer(m)).filter_map(|m| m.get("name").and_then(Value::as_str)).map(str::to_string).collect();
707 }
708 raw.get("children").and_then(Value::as_array).map(|c| c.iter().filter_map(|c| c.get("namespace").and_then(Value::as_str)).map(str::to_string).collect()).unwrap_or_default()
709}
710
711impl Shared {
712 pub(super) fn new(spec: ChildSpec, opts: ChildOptions, principal: String) -> Arc<Self> {
713 let upcalls = Upcalls::new(opts.host.clone().or_else(crate::plexus::service_scope::current), opts.legacy_upcalls);
714 upcalls.set_schemas(&opts.schemas);
715 if opts.consent_required {
718 upcalls.set_consent(Some(Vec::new()));
719 }
720 if let Some(why) = &opts.revoked {
721 upcalls.revoke(why);
722 }
723 Arc::new(Self {
724 upcalls,
725 name: spec.name().to_string(),
726 spec,
727 opts,
728 principal,
729 counters: Arc::new(Counters::default()),
730 state: Mutex::new(LinkState { conn: None, down_since: None, why: String::new(), window: VecDeque::new(), dead: None }),
731 dial: tokio::sync::Mutex::new(()),
732 mounted_hash: OnceLock::new(),
733 identity: OnceLock::new(),
734 stopped: OnceLock::new(),
735 })
736 }
737
738 pub(super) fn current(&self) -> Option<Arc<Conn>> {
739 self.state.lock().unwrap().conn.clone()
740 }
741
742 pub(super) fn install(&self, conn: Arc<Conn>) {
743 self.state.lock().unwrap().conn = Some(conn);
744 }
745
746 fn own_hop(&self) -> Value {
748 json!({ "principal": self.principal, "for": [], "limit": null, "call": uuid::Uuid::new_v4().to_string() })
749 }
750
751 pub(super) async fn connect_once(self: &Arc<Self>) -> Result<(Arc<Conn>, Kept, MountTimings), String> {
754 let t0 = Instant::now();
755 let conn = match &self.spec {
756 ChildSpec::Stdio { argv, .. } => self.spawn_stdio(argv)?,
757 ChildSpec::Ws { url, .. } => self.dial_ws(url).await?,
758 };
759 let asked = async {
760 let nonce = crate::plexus::peer_token::random_nonce();
768 let info = ask(&conn, "_info", json!({ "nonce": nonce, "asker": self.principal }), self.opts.timeout).await.map_err(|e| format!("`_info`: {e}"))?;
769 let ns = info.get("backend").and_then(Value::as_str).ok_or("its `_info` names no backend")?.to_string();
770 if let Some(key) = info.get("key").and_then(Value::as_str) {
771 if crate::plexus::peer_token::key_of(&self.principal).is_none() {
775 return Err(format!(
776 "{KEYED_MOUNT_NEEDS_A_KEY}: its `_info` proves the key {key}, and this service dials as `{}`, which no transport proves: give the service a dial key (ChildOptions::key) to mount a keyed peer",
777 self.principal
778 ));
779 }
780 let proof = info.get("proof").and_then(Value::as_str).unwrap_or_default();
781 crate::plexus::peer_token::check_key_proof_for(key, proof, &nonce, &ns, &self.principal).map_err(|e| {
782 let made_for = info.get("asker").and_then(Value::as_str).filter(|a| *a != self.principal).map(|a| format!(" (it was made for {a}, not for {})", self.principal)).unwrap_or_default();
783 format!("its `_info` names the key {key} without proving it to this service: {e}{made_for}")
784 })?;
785 let _ = conn.peer_key.set(key.to_string());
786 }
787 let _ = conn.ns.set(ns.clone());
788 let t1 = Instant::now();
789 let deadline = t1 + self.opts.timeout;
790 let params = json!({ "method": format!("{ns}.schema"), "params": {}, "hop": self.own_hop() });
791 let schema = ask(&conn, &format!("{ns}.call"), params, self.opts.timeout).await.map_err(|e| format!("its schema: {e}"))?;
792 let nodes = self.walk(&conn, &ns, &schema, deadline).await?;
793 Ok::<_, String>((Kept { top: schema, nodes }, t1))
794 };
795 match asked.await {
796 Ok((schema, t1)) => {
797 let t = MountTimings { spawn_ms: (t1 - t0).as_secs_f64() * 1e3, schema_ms: t1.elapsed().as_secs_f64() * 1e3, merge_ms: 0.0 };
798 Ok((conn, schema, t))
799 }
800 Err(why) => {
801 conn.shutdown.notify_one();
802 Err(why)
803 }
804 }
805 }
806
807 pub(super) async fn walk(&self, conn: &Conn, ns: &str, top: &Value, deadline: Instant) -> Result<Vec<(String, Value)>, String> {
814 let mut out = Vec::new();
815 let mut stack: Vec<String> = static_children(top).into_iter().rev().collect();
816 while let Some(path) = stack.pop() {
817 let depth = path.split('.').count();
818 if depth > self.opts.max_depth {
819 return Err(format!("its child `{path}` is {depth} levels down, past --child-max-depth ({})", self.opts.max_depth));
820 }
821 let left = deadline.saturating_duration_since(Instant::now());
822 if left.is_zero() {
823 return Err(format!("its tree was not walked within {:?}: stopped at `{path}`", self.opts.timeout));
824 }
825 let params = json!({ "method": format!("{ns}.{path}.schema"), "params": {}, "hop": self.own_hop() });
826 let raw = ask(conn, &format!("{ns}.call"), params, left).await.map_err(|e| format!("the schema of its child `{path}`: {e}"))?;
827 for child in static_children(&raw).into_iter().rev() {
828 stack.push(format!("{path}.{child}"));
829 }
830 out.push((path, raw));
831 }
832 Ok(out)
833 }
834
835 fn spawn_stdio(self: &Arc<Self>, argv: &[String]) -> Result<Arc<Conn>, String> {
836 let mut cmd = tokio::process::Command::new(&argv[0]);
837 cmd.args(&argv[1..])
838 .env_clear()
839 .envs(["PATH", "HOME", "LANG"].iter().map(|k| k.to_string()).chain(self.opts.env.iter().cloned()).filter_map(|k| std::env::var(&k).ok().map(|v| (k, v))))
840 .env("PLEXUS_CHILD_NAME", &self.name)
841 .stdin(std::process::Stdio::piped())
842 .stdout(std::process::Stdio::piped())
843 .stderr(std::process::Stdio::piped())
844 .kill_on_drop(true);
845 let mut child = cmd.spawn().map_err(|e| format!("spawning `{}`: {e}", argv[0]))?;
846 let (stdin, stdout, stderr) = (child.stdin.take(), child.stdout.take(), child.stderr.take());
847 let (Some(stdin), Some(stdout), Some(stderr)) = (stdin, stdout, stderr) else {
848 return Err("its pipes were not opened".into());
849 };
850 let (out, out_rx) = mpsc::unbounded_channel();
851 let conn = Conn::new(&self.name, child.id(), out, self.opts.buffer, Arc::clone(&self.counters));
852 let _ = conn.up.set(Arc::clone(&self.upcalls));
853 tokio::spawn(write_lines(stdin, out_rx));
854 let reader = Arc::clone(&conn);
855 let shared = Arc::downgrade(self);
856 tokio::spawn(async move {
857 tokio::select! {
858 _ = read_lines(&reader, stdout) => {}
859 _ = reader.shutdown.notified() => {
860 let _ = child.start_kill();
861 }
862 }
863 let status = child.wait().await.map(|s| s.to_string()).unwrap_or_else(|e| e.to_string());
864 gone(&shared, &reader, &format!("the process exited ({status})"));
865 });
866 let name = self.name.clone();
867 tokio::spawn(async move {
868 let span = tracing::info_span!(target: "plexus::child", "plexus::child", name = %name);
869 let mut lines = BufReader::new(stderr).lines();
870 while let Ok(Some(l)) = lines.next_line().await {
871 let _e = span.enter();
872 tracing::info!(target: "plexus::child", child = %name, "{}", crate::plexus::redact::redact_str(&l));
875 }
876 });
877 Ok(conn)
878 }
879
880 #[cfg(feature = "child-ws")]
881 async fn dial_ws(self: &Arc<Self>, url: &str) -> Result<Arc<Conn>, String> {
882 super::ws::dial(self, url).await
883 }
884
885 #[cfg(not(feature = "child-ws"))]
886 async fn dial_ws(self: &Arc<Self>, _url: &str) -> Result<Arc<Conn>, String> {
887 Err("a WebSocket child needs plexus-core's `child-ws` feature".into())
888 }
889
890 pub(super) fn lost(&self, conn: &Arc<Conn>, why: &str) {
892 let mut st = self.state.lock().unwrap();
893 if st.conn.as_ref().is_some_and(|c| Arc::ptr_eq(c, conn)) {
894 st.conn = None;
895 st.down_since = Some(Instant::now());
896 st.why = why.to_string();
897 drop(st);
898 tracing::warn!(target: "plexus::child", child = %self.name, "child gone: {why}");
899 }
900 }
901
902 pub(super) async fn ensure(self: &Arc<Self>) -> Result<Arc<Conn>, PlexusStreamItem> {
905 if let Some(c) = self.live()? {
906 return Ok(c);
907 }
908 let _g = self.dial.lock().await;
909 if let Some(c) = self.live()? {
910 return Ok(c);
911 }
912 let b = self.opts.backoff;
913 let wait = {
914 let mut st = self.state.lock().unwrap();
915 let now = Instant::now();
916 while st.window.front().is_some_and(|t| now.duration_since(*t) >= Duration::from_secs(60)) {
917 st.window.pop_front();
918 }
919 if st.window.len() >= b.per_minute {
920 let next = st.window.front().map(|t| Duration::from_secs(60).saturating_sub(now.duration_since(*t))).unwrap_or_default();
921 return Err(unavailable(&self.name, &format!("{} (restarted {} times in the last minute; the next restart in {:.1} s)", st.why, st.window.len(), next.as_secs_f64())));
922 }
923 let due = st.down_since.unwrap_or(now) + b.delay(st.window.len());
924 due.saturating_duration_since(now)
925 };
926 if wait > self.opts.timeout {
927 let why = self.state.lock().unwrap().why.clone();
928 return Err(unavailable(&self.name, &format!("{why} (restarting in {:.1} s)", wait.as_secs_f64())));
929 }
930 tokio::time::sleep(wait).await;
931 self.state.lock().unwrap().window.push_back(Instant::now());
932 self.counters.restarts.fetch_add(1, Ordering::Relaxed);
933 match self.connect_once().await {
934 Ok((conn, kept, t)) => {
935 let hash = kept.hash();
936 if self.mounted_hash.get().is_some_and(|h| *h != hash) {
937 conn.shutdown.notify_one();
938 let why = "schema changed; restart the service".to_string();
939 tracing::error!(target: "plexus::child", child = %self.name, mounted = ?self.mounted_hash.get(), published = %hash, "the restarted child published another schema; not remounted: restart the service");
940 self.state.lock().unwrap().dead = Some(why.clone());
941 return Err(unavailable(&self.name, &why));
942 }
943 let proved = super::PeerIdentity::of(&conn);
944 if let Some(pinned) = self.identity.get().filter(|p| **p != proved) {
945 conn.shutdown.notify_one();
946 let why = format!("its identity changed: {pinned}, now {proved}; connect it again to trust the new one");
947 tracing::error!(target: "plexus::child", child = %self.name, %pinned, %proved, "the restarted child proved another identity; not remounted");
948 self.state.lock().unwrap().dead = Some(why.clone());
949 return Err(unavailable(&self.name, &why));
950 }
951 tracing::info!(target: "plexus::child", child = %self.name, spawn_ms = t.spawn_ms, schema_ms = t.schema_ms, "child restarted");
952 self.state.lock().unwrap().conn = Some(Arc::clone(&conn));
953 Ok(conn)
954 }
955 Err(why) => {
956 let mut st = self.state.lock().unwrap();
957 st.down_since = Some(Instant::now());
958 st.why = why.clone();
959 Err(unavailable(&self.name, &why))
960 }
961 }
962 }
963
964 pub(super) fn stop(&self, why: &str) {
968 let _ = self.stopped.set(why.to_string());
969 let conn = self.state.lock().unwrap().conn.clone();
970 if let Some(c) = conn {
971 c.stop(why);
972 }
973 }
974
975 pub(super) fn stopped(&self) -> Option<&str> {
977 self.stopped.get().map(String::as_str)
978 }
979
980 pub(super) fn dead(&self) -> Option<String> {
983 self.state.lock().unwrap().dead.clone()
984 }
985
986 pub(super) fn why_down(&self) -> Option<String> {
989 let st = self.state.lock().unwrap();
990 if st.dead.is_some() {
991 return st.dead.clone();
992 }
993 match &st.conn {
994 Some(c) if c.alive() => None,
995 _ => Some(if st.why.is_empty() { "not connected".to_string() } else { st.why.clone() }),
996 }
997 }
998
999 pub(super) fn is_up(&self) -> bool {
1001 let st = self.state.lock().unwrap();
1002 st.dead.is_none() && st.conn.as_ref().is_some_and(|c| c.alive())
1003 }
1004
1005 fn live(&self) -> Result<Option<Arc<Conn>>, PlexusStreamItem> {
1006 let st = self.state.lock().unwrap();
1007 if let Some(why) = &st.dead {
1008 return Err(unavailable(&self.name, why));
1009 }
1010 Ok(st.conn.clone().filter(|c| c.alive()))
1011 }
1012}
1013
1014pub(super) fn gone(shared: &Weak<Shared>, conn: &Arc<Conn>, why: &str) {
1016 conn.on_gone(why);
1017 if let Some(s) = shared.upgrade() {
1018 s.lost(conn, why);
1019 }
1020}
1021
1022async fn ask(conn: &Conn, method: &str, params: Value, timeout: Duration) -> Result<Value, String> {
1024 let (_, mut rx) = conn.open(method, params, timeout, None).await.map_err(|e| match e {
1025 PlexusStreamItem::Error { message, .. } => message,
1026 other => format!("{other:?}"),
1027 })?;
1028 let deadline = tokio::time::Instant::now() + timeout;
1029 let mut first = None;
1030 loop {
1031 match tokio::time::timeout_at(deadline, rx.recv()).await {
1032 Ok(Some(PlexusStreamItem::Data { content, .. })) => {
1033 if first.is_none() {
1034 first = Some(content.into_value());
1035 }
1036 }
1037 Ok(Some(PlexusStreamItem::Error { message, .. })) => return Err(message),
1038 Ok(Some(PlexusStreamItem::Done { .. })) | Ok(None) => break,
1039 Ok(Some(_)) => {}
1040 Err(_) => return Err(format!("no answer in {timeout:?}")),
1041 }
1042 }
1043 first.ok_or_else(|| "it answered no data".to_string())
1044}