1mod host;
55mod link;
56mod names;
58mod peer;
60mod spec;
61#[cfg(feature = "child-ws")]
62mod ws;
63#[cfg(feature = "clap")]
64mod args;
65mod dial_in;
68#[cfg(feature = "child-ws")]
69mod dial_out;
70
71use std::pin::Pin;
72use std::sync::atomic::Ordering;
73use std::sync::Arc;
74use std::task::{Context, Poll};
75use std::time::Instant;
76
77use async_trait::async_trait;
78use futures::Stream;
79use plexus_auth_core::grants::{GrantsGeneration, Reach};
80use serde_json::{json, Value};
81use tokio::sync::mpsc;
82
83pub use host::{serve_child, serve_child_on, serve_child_stdio, ChildPeer};
84pub use link::MountTimings;
85pub use spec::{parse_duration, Backoff, ChildError, ChildOptions, ChildSpec};
86#[cfg(feature = "clap")]
87pub use args::ChildArgs;
88pub use dial_in::{PeerEvent, PeerInfo, PeerRoles, PeerSocket, Peers, PeersMethods};
89#[cfg(feature = "child-ws")]
90pub use dial_out::{key_principal, DialOut, DialState, Exposed};
91
92use link::{meta, too_many, unavailable, Conn, Counters, Shared, SubKey};
93
94use super::auth::AuthContext;
95use super::method_enum::MethodEnumSchema;
96use super::plexus::{Activation, ChildRouter, PlexusError};
97use super::schema::{MethodRole, PluginSchema};
98use super::streaming::PlexusStream;
99use super::types::PlexusStreamItem;
100
101#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
103pub struct ChildStats {
104 pub open: usize,
106 pub unasked: u64,
108 pub restarts: u64,
110 pub calls: u64,
112 pub name_schemas: u64,
115 pub name_schema_hits: u64,
117 pub names_kept: usize,
120}
121
122#[derive(Debug, Clone, Copy)]
125pub enum ChildMethods {}
126
127impl schemars::JsonSchema for ChildMethods {
128 fn schema_name() -> std::borrow::Cow<'static, str> {
129 "ChildMethods".into()
130 }
131
132 fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
133 schemars::json_schema!({
134 "description": "A child service's methods are data: its schema, asked when it was mounted (`<mount>.schema`)."
135 })
136 }
137}
138
139impl MethodEnumSchema for ChildMethods {
140 fn method_names() -> &'static [&'static str] {
141 &[]
142 }
143
144 fn schema_with_consts() -> Value {
145 serde_json::to_value(schemars::schema_for!(ChildMethods)).unwrap_or(Value::Null)
146 }
147}
148
149struct Inner {
150 shared: Arc<Shared>,
151 schema: PluginSchema,
152 nodes: Vec<(String, PluginSchema)>,
157 timings: MountTimings,
158 names: names::Names,
160}
161
162impl Inner {
163 pub(super) fn new(shared: Arc<Shared>, schema: PluginSchema, nodes: Vec<(String, PluginSchema)>, timings: MountTimings) -> Self {
164 let names = names::Names::new(shared.opts.schema_cache);
165 Self { shared, schema, nodes, timings, names }
166 }
167}
168
169#[derive(Clone)]
173pub struct RemoteChild {
174 inner: Arc<Inner>,
175}
176
177impl std::fmt::Debug for RemoteChild {
178 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
179 f.debug_struct("RemoteChild").field("spec", &self.inner.shared.spec.to_string()).field("stats", &self.stats()).finish()
180 }
181}
182
183fn read_schema(raw: Value) -> Result<PluginSchema, String> {
185 serde_json::from_value(peer::schema(raw)?).map_err(|e| format!("its schema does not read: {e}"))
186}
187
188pub(super) fn node_schema(path: &str, raw: Value) -> Result<PluginSchema, String> {
192 read_schema(raw).map_err(|e| format!("its child `{path}`: {e}"))
193}
194
195fn mount_schema(name: &str, raw: Value) -> Result<PluginSchema, String> {
201 let mut schema = read_schema(raw)?;
202 if schema.namespace != name {
203 tracing::info!(target: "plexus::child", child = %name, declared = %schema.namespace, "mounted under its mount name, not the namespace it declares");
204 }
205 let prefix = format!("{}.", schema.namespace);
206 let requalify = |g: &String| match g.strip_prefix(&prefix) {
207 Some(m) => format!("{name}.{m}"),
208 None => format!("{name}.{g}"),
209 };
210 schema.roles = schema.roles.iter().map(|(r, gs)| (r.clone(), gs.iter().map(requalify).collect())).collect();
211 for m in schema.methods.iter_mut() {
212 if let Some(g) = m.grant.as_ref() {
213 m.grant = Some(requalify(g));
214 }
215 }
216 schema.namespace = name.to_string();
217 Ok(schema)
218}
219
220impl RemoteChild {
221 pub async fn connect(spec: ChildSpec, opts: ChildOptions) -> Result<Self, ChildError> {
226 let name = spec.name().to_string();
227 let start = |why: String| ChildError::Start(name.clone(), why);
228 if opts.max_calls == 0 || opts.buffer == 0 {
229 return Err(start("max_calls and buffer are at least 1".into()));
230 }
231 let principal = Self::principal_for(&spec, &opts).map_err(start)?;
232 let shared = Shared::new(spec, opts, principal);
233 let (conn, kept, mut timings) = shared.connect_once().await.map_err(start)?;
234 let t = Instant::now();
235 let hash = kept.hash();
236 let read = mount_schema(&name, kept.top).and_then(|schema| {
237 let nodes = kept.nodes.into_iter().map(|(path, raw)| node_schema(&path, raw).map(|s| (path, s))).collect::<Result<Vec<_>, _>>()?;
238 Ok((schema, nodes))
239 });
240 let (schema, nodes) = match read {
241 Ok(s) => s,
242 Err(why) => {
243 conn.shutdown.notify_one();
244 return Err(start(why));
245 }
246 };
247 let _ = shared.mounted_hash.set(hash);
248 shared.install(conn);
249 timings.merge_ms = t.elapsed().as_secs_f64() * 1e3;
250 tracing::info!(target: "plexus::child", child = %name, spawn_ms = timings.spawn_ms, schema_ms = timings.schema_ms, merge_ms = timings.merge_ms, methods = schema.methods.len(), children = nodes.len(), "child mounted");
251 Ok(Self { inner: Arc::new(Inner::new(shared, schema, nodes, timings)) })
252 }
253
254 #[allow(unused_variables)]
255 fn principal_for(spec: &ChildSpec, opts: &ChildOptions) -> Result<String, String> {
256 #[cfg(feature = "child-ws")]
257 if let (ChildSpec::Ws { .. }, Some(path)) = (spec, &opts.key) {
258 return Ok(ws::principal_of(&ws::read_seed(path)?));
259 }
260 Ok(opts.principal.clone())
261 }
262
263 pub fn stats(&self) -> ChildStats {
265 let c: &Counters = &self.inner.shared.counters;
266 ChildStats {
267 open: c.open.load(Ordering::Relaxed),
268 unasked: c.unasked.load(Ordering::Relaxed),
269 restarts: c.restarts.load(Ordering::Relaxed),
270 calls: c.calls.load(Ordering::Relaxed),
271 name_schemas: self.inner.names.hops.load(Ordering::Relaxed),
272 name_schema_hits: self.inner.names.hits.load(Ordering::Relaxed),
273 names_kept: self.inner.names.len(c.restarts.load(Ordering::Relaxed)),
274 }
275 }
276
277 pub fn pid(&self) -> Option<u32> {
279 self.inner.shared.current().and_then(|c| c.pid)
280 }
281
282 pub fn name(&self) -> &str {
284 &self.inner.shared.name
285 }
286
287 pub fn spec(&self) -> &ChildSpec {
289 &self.inner.shared.spec
290 }
291
292 pub fn principal(&self) -> &str {
294 &self.inner.shared.principal
295 }
296
297 pub fn mount_timings(&self) -> MountTimings {
299 self.inner.timings
300 }
301
302 pub fn routed(&self) -> usize {
304 self.inner.shared.current().map_or(0, |c| c.routed())
305 }
306
307 fn hop(&self, conn: &Conn, auth: Option<&AuthContext>, reach: &Reach) -> Value {
311 let for_: Vec<String> = auth.and_then(|a| a.principal()).map(|p| p.to_string()).into_iter().collect();
312 let limit = reach.grant().map(|g| {
313 let grant = match g.as_str().strip_prefix(&format!("{}.", self.name())) {
314 Some(m) => format!("{}.{m}", conn.ns()),
315 None => g.as_str().to_string(),
316 };
317 let held_on: Vec<String> = reach.held_on().iter().map(|s| s.to_string()).collect();
318 json!({ "grant": grant, "held_on": held_on })
319 });
320 json!({ "principal": self.inner.shared.principal, "for": for_, "limit": limit, "call": uuid::Uuid::new_v4().to_string() })
321 }
322
323 fn node(&self, path: &str) -> Option<&PluginSchema> {
325 self.inner.nodes.iter().find(|(p, _)| p == path).map(|(_, s)| s)
326 }
327
328 fn slot_of(&self, path: &str) -> Option<(String, String)> {
334 let segs: Vec<&str> = path.split('.').collect();
335 let (n, schema) = (1..segs.len())
336 .rev()
337 .find_map(|n| self.node(&segs[..n].join(".")).map(|s| (n, s)))
338 .unwrap_or((0, &self.inner.schema));
339 let parent = std::iter::once(self.name()).chain(segs[..n].iter().copied()).collect::<Vec<_>>().join(".");
340 let next = segs[n];
341 if schema.methods.iter().any(|m| m.name == next && matches!(m.role, MethodRole::Peer { .. })) {
343 return Some((crate::plexus::slots::slot_path(&parent, next), segs[..(n + 2).min(segs.len())].join(".")));
344 }
345 let slot = schema.methods.iter().find(|m| matches!(m.role, MethodRole::DynamicChild { .. }))?;
346 Some((crate::plexus::slots::slot_path(&parent, &slot.name), segs[..=n].join(".")))
347 }
348
349 fn unreached(&self, method: &str) -> PlexusError {
351 PlexusError::MethodNotFound { activation: self.name().to_string(), method: method.to_string() }
352 }
353
354 async fn name_schema(&self, path: &str, params: &Value, auth: Option<&AuthContext>) -> PlexusStream {
361 let epoch = self.inner.shared.counters.restarts.load(Ordering::Relaxed);
362 if let Some(s) = self.inner.names.get(path, epoch) {
363 return self.schema_stream_of(&s, &s.namespace, params);
364 }
365 self.inner.names.crossed();
366 let items: Vec<PlexusStreamItem> = futures::StreamExt::collect(self.forward(&format!("{path}.schema"), json!({}), auth, None).await).await;
367 let kept = items.iter().find_map(|i| match i {
368 PlexusStreamItem::Data { metadata, content, .. } => Some((metadata.plexus_hash.clone(), content.clone().into_value())),
369 _ => None,
370 });
371 let Some((seen, raw)) = kept else {
372 return Box::pin(futures::stream::iter(items));
373 };
374 match read_schema(raw) {
375 Ok(schema) => {
376 let epoch = self.inner.shared.counters.restarts.load(Ordering::Relaxed);
378 self.inner.names.put(path, schema.clone(), seen, epoch);
379 self.schema_stream_of(&schema, &schema.namespace, params)
380 }
381 Err(_) => Box::pin(futures::stream::iter(items)),
382 }
383 }
384
385 fn schema_stream(&self, params: &Value) -> PlexusStream {
386 self.schema_stream_of(&self.inner.schema, self.name(), params)
387 }
388
389 fn schema_stream_of(&self, schema: &PluginSchema, label: &str, params: &Value) -> PlexusStream {
392 let ns = label.to_string();
393 let content = match params.get("method").and_then(Value::as_str) {
394 Some(m) => match schema.methods.iter().find(|x| x.name == m) {
395 Some(ms) => serde_json::to_value(ms).unwrap_or_default(),
396 None => {
397 let error = plexus_client::RemoteError::MethodNotFound { activation: ns.clone(), method: m.to_string() };
398 let mut e = PlexusStreamItem::error(meta(&ns), error, false);
399 if let PlexusStreamItem::Error { message, code, .. } = &mut e {
400 *message = format!("Method '{m}' not found");
401 *code = Some("-32602".into());
402 }
403 return Box::pin(futures::stream::iter([e]));
404 }
405 },
406 None => serde_json::to_value(schema).unwrap_or_default(),
407 };
408 Box::pin(futures::stream::iter([
409 PlexusStreamItem::Data { metadata: meta(&ns).into(), content_type: format!("{ns}.schema").into(), content: content.into() },
410 PlexusStreamItem::Done { metadata: meta(&ns) },
411 ]))
412 }
413
414 async fn forward(&self, method: &str, params: Value, auth: Option<&AuthContext>, instance: Option<String>) -> PlexusStream {
417 let shared = &self.inner.shared;
418 let one = |item: PlexusStreamItem| -> PlexusStream { Box::pin(futures::stream::iter([item])) };
419 let reach = match super::grant_gate::current_reach() {
422 Ok(reach) => reach,
423 Err(e) => return one(PlexusStreamItem::from_plexus_error(meta(self.name()), &e, false)),
424 };
425 let max = shared.opts.max_calls;
426 if shared.counters.open.fetch_add(1, Ordering::AcqRel) >= max {
427 shared.counters.open.fetch_sub(1, Ordering::AcqRel);
428 return one(too_many(self.name(), max));
429 }
430 let guard = OpenGuard(Arc::clone(&shared.counters));
431 let conn = match shared.ensure().await {
432 Ok(c) => c,
433 Err(item) => return one(item),
434 };
435 shared.counters.calls.fetch_add(1, Ordering::Relaxed);
436 let hop = self.hop(&conn, auth, &reach);
437 let path = format!("{}.{method}", conn.ns());
438 let call = json!({ "method": path, "params": params, "hop": hop });
439 match conn.open(&format!("{}.call", conn.ns()), call, shared.opts.timeout).await {
440 Ok((sid, rx)) => {
441 let generation = reach.generation().cloned();
442 let seen = generation.as_ref().map_or(0, GrantsGeneration::current);
443 let instance = instance.map(|path| (Arc::clone(&self.inner), path));
444 Box::pin(CallStream { rx, sid, conn, over: false, _guard: guard, reach, generation, seen, name: self.name().to_string(), instance })
445 }
446 Err(item) => one(item),
447 }
448 }
449}
450
451struct OpenGuard(Arc<Counters>);
453
454impl Drop for OpenGuard {
455 fn drop(&mut self) {
456 self.0.open.fetch_sub(1, Ordering::AcqRel);
457 }
458}
459
460struct CallStream {
463 rx: mpsc::Receiver<PlexusStreamItem>,
464 sid: SubKey,
465 conn: Arc<Conn>,
466 over: bool,
467 _guard: OpenGuard,
468 reach: Reach,
471 generation: Option<GrantsGeneration>,
472 seen: u64,
473 name: String,
474 instance: Option<(Arc<Inner>, String)>,
477}
478
479impl CallStream {
480 fn revoked(&self) -> PlexusStreamItem {
482 let grant = self.reach.grant().map(|g| g.as_str().to_string()).unwrap_or_default();
483 let error = plexus_client::RemoteError::Forbidden {
484 reason: plexus_client::DenyReason::MissingGrant {
485 grant: grant.clone(),
486 scope: "*".into(),
487 roles: Vec::new(),
488 held_on: Vec::new(),
489 },
490 };
491 let mut item = PlexusStreamItem::error(meta(&self.name), error, false);
492 if let PlexusStreamItem::Error { message, .. } = &mut item {
493 *message = format!("Forbidden: `{grant}` is no longer held — the call's grant moved while it ran");
494 }
495 item
496 }
497}
498
499impl Stream for CallStream {
500 type Item = PlexusStreamItem;
501
502 fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
503 if self.over {
504 return Poll::Ready(None);
505 }
506 match self.rx.poll_recv(cx) {
507 Poll::Ready(Some(item)) => {
508 if let (Some((inner, path)), Some(m)) = (&self.instance, item.metadata()) {
509 inner.names.observe(path, &m.plexus_hash);
510 }
511 let terminal = matches!(item, PlexusStreamItem::Done { .. } | PlexusStreamItem::Error { .. });
512 let moved = self.generation.as_ref().map(GrantsGeneration::current).filter(|g| *g != self.seen);
513 if let (false, Some(g)) = (terminal, moved) {
514 self.seen = g;
515 if self.reach.current().is_none() {
516 let sid = self.sid.clone();
517 self.conn.unsubscribe(&sid);
518 self.over = true;
519 return Poll::Ready(Some(self.revoked()));
520 }
521 }
522 if terminal {
523 self.over = true;
524 }
525 Poll::Ready(Some(item))
526 }
527 Poll::Ready(None) => {
528 self.over = true;
530 Poll::Ready(Some(unavailable(&self.name, "the connection closed")))
531 }
532 Poll::Pending => Poll::Pending,
533 }
534 }
535}
536
537impl Drop for CallStream {
538 fn drop(&mut self) {
539 if !self.over {
540 self.conn.unsubscribe(&self.sid);
541 }
542 }
543}
544
545#[async_trait]
546impl Activation for RemoteChild {
547 type Methods = ChildMethods;
548
549 fn namespace(&self) -> &str {
550 self.name()
551 }
552
553 fn version(&self) -> &str {
554 &self.inner.schema.version
555 }
556
557 fn description(&self) -> &str {
558 &self.inner.schema.description
559 }
560
561 fn long_description(&self) -> Option<&str> {
562 self.inner.schema.long_description.as_deref()
563 }
564
565 fn methods(&self) -> Vec<&str> {
566 self.inner.schema.methods.iter().map(|m| m.name.as_str()).collect()
567 }
568
569 fn method_help(&self, method: &str) -> Option<String> {
570 self.inner.schema.methods.iter().find(|m| m.name == method).map(|m| m.description.clone())
571 }
572
573 async fn call(&self, method: &str, params: Value, auth: Option<&AuthContext>, _raw: Option<&crate::request::RawRequestContext>) -> Result<PlexusStream, PlexusError> {
574 if method == "schema" {
575 return Ok(self.schema_stream(¶ms));
576 }
577 if let Some(path) = method.strip_suffix(".schema") {
579 if let Some(s) = self.node(path) {
580 return Ok(self.schema_stream_of(s, &s.namespace, ¶ms));
581 }
582 return match self.slot_of(path) {
585 Some(_) => Ok(self.name_schema(path, ¶ms, auth).await),
586 None => Err(self.unreached(method)),
587 };
588 }
589 let found = match method.rsplit_once('.') {
590 Some((path, name)) => match self.node(path) {
592 Some(s) => s.methods.iter().find(|m| m.name == name && m.role == MethodRole::Rpc),
593 None => match self.slot_of(path) {
598 Some((_, instance)) => return Ok(self.forward(method, params, auth, Some(instance)).await),
599 None => return Err(self.unreached(method)),
600 },
601 },
602 None => self.inner.schema.methods.iter().find(|m| m.name == method && m.role == MethodRole::Rpc),
603 };
604 let Some(m) = found else {
605 return Err(PlexusError::MethodNotFound { activation: self.name().to_string(), method: method.to_string() });
606 };
607 if m.input.is_some() {
610 return Err(PlexusError::ExecutionError(format!(
611 "`{}.{method}` takes an input stream, and a hop to a remote child does not forward one yet",
612 self.name()
613 )));
614 }
615 Ok(self.forward(method, params, auth, None).await)
616 }
617
618 fn into_rpc_methods(self) -> jsonrpsee::Methods {
621 jsonrpsee::Methods::new()
622 }
623
624 fn plugin_schema(&self) -> PluginSchema {
625 self.inner.schema.clone()
626 }
627
628 fn static_child_schemas(&self) -> Vec<(String, PluginSchema)> {
632 self.inner.nodes.clone()
633 }
634}
635
636#[async_trait]
637impl ChildRouter for RemoteChild {
638 fn router_namespace(&self) -> &str {
639 self.name()
640 }
641
642 async fn router_call(&self, method: &str, params: Value, auth: Option<&AuthContext>, raw: Option<&crate::request::RawRequestContext>) -> Result<PlexusStream, PlexusError> {
643 Activation::call(self, method, params, auth, raw).await
644 }
645
646 async fn get_child(&self, _name: &str) -> Option<Box<dyn ChildRouter>> {
647 None
648 }
649
650 fn router_schema(&self) -> Option<PluginSchema> {
651 Some(self.inner.schema.clone())
652 }
653
654 fn router_static_child_schemas(&self) -> Vec<(String, PluginSchema)> {
655 self.inner.nodes.clone()
656 }
657}
658
659#[cfg(test)]
660mod tests {
661 use super::*;
662
663 fn raw(methods: Value, roles: Value) -> Value {
664 json!({ "namespace": "stub", "version": "0.1.0", "description": "d", "self_hash": "h", "hash": "h", "methods": methods, "roles": roles })
665 }
666
667 fn m(name: &str) -> Value {
668 json!({ "name": name, "description": "", "hash": "x", "streaming": false, "bidirectional": false })
669 }
670
671 #[test]
675 fn a_method_stating_neither_cardinality_nor_streaming_is_refused() {
676 let silent = json!({ "name": "quiet", "description": "", "hash": "x" });
677 let e = mount_schema("echo", raw(json!([m("echo"), silent]), json!({}))).unwrap_err();
678 assert!(e.contains("`quiet`") && e.contains("neither cardinality"), "{e}");
679 }
680
681 #[test]
682 fn the_mount_name_wins_and_roles_are_requalified() {
683 let s = mount_schema("echo", raw(json!([m("echo"), m("shout")]), json!({ "viewer": ["stub.echo"], "editor": ["stub.echo", "stub.shout"] }))).unwrap();
684 assert_eq!(s.namespace, "echo");
685 assert_eq!(s.roles["viewer"], vec!["echo.echo".to_string()]);
686 assert_eq!(s.roles["editor"], vec!["echo.echo".to_string(), "echo.shout".to_string()]);
687 }
688
689 #[test]
691 fn a_child_with_children_mounts_and_its_method_grants_are_requalified() {
692 let mut inner = m("inner");
693 inner["role"] = json!({ "kind": "static_child" });
694 let mut own = m("echo");
695 own["grant"] = json!("stub.echo");
696 let s = mount_schema("echo", raw(json!([own, inner]), json!({}))).unwrap();
697 assert_eq!(s.methods[0].grant.as_deref(), Some("echo.echo"));
698 assert_eq!(s.methods[1].role, MethodRole::StaticChild);
699 let mut slot = m("item");
700 slot["role"] = json!({ "kind": "dynamic_child", "list_method": "names" });
701 let nested = raw(json!([m("echo"), s_inner(), slot]), json!({}));
702 assert_eq!(link::static_children(&nested), vec!["inner".to_string()], "a dynamic child is a slot, not walked");
703 let mut root = raw(json!([m("call")]), json!({}));
705 root["children"] = json!([{ "namespace": "greeter", "description": "", "hash": "g" }]);
706 assert_eq!(link::static_children(&root), vec!["greeter".to_string()]);
707 }
708
709 fn s_inner() -> Value {
710 let mut inner = m("inner");
711 inner["role"] = json!({ "kind": "static_child" });
712 inner
713 }
714
715 #[test]
717 fn the_tree_hash_covers_every_level() {
718 let top = raw(json!([m("echo")]), json!({}));
719 let flat = link::Kept { top: top.clone(), nodes: vec![] };
720 assert_eq!(flat.hash(), link::schema_hash(&top), "a flat child's hash is its schema's, as before");
721 let mut c = raw(json!([m("m")]), json!({}));
722 let one = link::Kept { top: top.clone(), nodes: vec![("a.b".into(), c.clone())] }.hash();
723 c["hash"] = json!("other");
724 let two = link::Kept { top, nodes: vec![("a.b".into(), c)] }.hash();
725 assert_ne!(one, two);
726 }
727
728 #[test]
729 fn typed_error_fields_cross_whole() {
730 let item = unavailable("echo", "gone");
731 let v = serde_json::to_value(&item).unwrap();
732 assert_eq!(v["error_kind"], "child_unavailable");
733 assert_eq!(v["code"], "child_unavailable");
734 assert_eq!(v["child"], "echo");
735 let back: PlexusStreamItem = serde_json::from_value(v.clone()).unwrap();
736 assert_eq!(back.error_kind(), Some("child_unavailable"));
737 assert_eq!(serde_json::to_value(&back).unwrap(), v);
738 let plain = PlexusStreamItem::error(meta("x"), plexus_client::RemoteError::Execution { message: "m".into() }, false);
741 let pv = serde_json::to_value(&plain).unwrap();
742 let keys: Vec<_> = pv.as_object().unwrap().keys().cloned().collect();
743 assert_eq!(keys, ["code", "error", "message", "metadata", "recoverable", "type"]);
744 }
745}