mod host;
mod link;
mod names;
mod peer;
mod spec;
mod upcall;
#[cfg(feature = "child-ws")]
mod ws;
#[cfg(feature = "clap")]
mod args;
mod dial_in;
#[cfg(feature = "child-ws")]
mod dial_out;
use std::pin::Pin;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::Instant;
use async_trait::async_trait;
use futures::Stream;
use plexus_auth_core::grants::{GrantsGeneration, Reach};
use serde_json::{json, Value};
use tokio::sync::mpsc;
pub use host::{host, serve_child, serve_child_on, serve_child_stdio, ChildPeer, HostTransport};
pub use upcall::{HOST_CALL, HOST_CALL_UNSUB};
pub use link::MountTimings;
pub use spec::{parse_duration, Backoff, ChildError, ChildOptions, ChildSpec};
#[cfg(feature = "clap")]
pub use args::ChildArgs;
pub use dial_in::{PeerEvent, PeerInfo, PeerRoles, PeerSocket, Peers, PeersMethods};
#[cfg(feature = "child-ws")]
pub use dial_out::{key_principal, DialOut, DialState, Exposed};
use link::{meta, too_many, unavailable, Conn, Counters, Shared, SubKey};
use super::auth::AuthContext;
use super::method_enum::MethodEnumSchema;
use super::plexus::{Activation, ChildRouter, PlexusError};
use super::schema::{MethodRole, PluginSchema};
use super::streaming::PlexusStream;
use super::types::PlexusStreamItem;
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct ChildStats {
pub open: usize,
pub unasked: u64,
pub restarts: u64,
pub calls: u64,
pub name_schemas: u64,
pub name_schema_hits: u64,
pub names_kept: usize,
pub calls_back: u64,
pub up: bool,
}
#[derive(Debug, Clone, Copy)]
pub enum ChildMethods {}
impl schemars::JsonSchema for ChildMethods {
fn schema_name() -> std::borrow::Cow<'static, str> {
"ChildMethods".into()
}
fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
schemars::json_schema!({
"description": "A child service's methods are data: its schema, asked when it was mounted (`<mount>.schema`)."
})
}
}
impl MethodEnumSchema for ChildMethods {
fn method_names() -> &'static [&'static str] {
&[]
}
fn schema_with_consts() -> Value {
serde_json::to_value(schemars::schema_for!(ChildMethods)).unwrap_or(Value::Null)
}
}
struct Inner {
shared: Arc<Shared>,
schema: PluginSchema,
nodes: Vec<(String, PluginSchema)>,
timings: MountTimings,
names: names::Names,
}
impl Inner {
pub(super) fn new(shared: Arc<Shared>, schema: PluginSchema, nodes: Vec<(String, PluginSchema)>, timings: MountTimings) -> Self {
let names = names::Names::new(shared.opts.schema_cache);
Self { shared, schema, nodes, timings, names }
}
}
#[derive(Clone)]
pub struct RemoteChild {
inner: Arc<Inner>,
}
impl std::fmt::Debug for RemoteChild {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RemoteChild").field("spec", &self.inner.shared.spec.to_string()).field("stats", &self.stats()).finish()
}
}
fn read_schema(raw: Value) -> Result<PluginSchema, String> {
serde_json::from_value(peer::schema(raw)?).map_err(|e| format!("its schema does not read: {e}"))
}
pub(super) fn node_schema(path: &str, raw: Value) -> Result<PluginSchema, String> {
read_schema(raw).map_err(|e| format!("its child `{path}`: {e}"))
}
fn mount_schema(name: &str, raw: Value) -> Result<PluginSchema, String> {
let mut schema = read_schema(raw)?;
if schema.namespace != name {
tracing::info!(target: "plexus::child", child = %name, declared = %schema.namespace, "mounted under its mount name, not the namespace it declares");
}
let prefix = format!("{}.", schema.namespace);
let requalify = |g: &String| match g.strip_prefix(&prefix) {
Some(m) => format!("{name}.{m}"),
None => format!("{name}.{g}"),
};
schema.roles = schema.roles.iter().map(|(r, gs)| (r.clone(), gs.iter().map(requalify).collect())).collect();
for m in schema.methods.iter_mut() {
if let Some(g) = m.grant.as_ref() {
m.grant = Some(requalify(g));
}
}
schema.namespace = name.to_string();
Ok(schema)
}
impl RemoteChild {
pub async fn connect(spec: ChildSpec, opts: ChildOptions) -> Result<Self, ChildError> {
let name = spec.name().to_string();
let start = |why: String| ChildError::Start(name.clone(), why);
if opts.max_calls == 0 || opts.buffer == 0 {
return Err(start("max_calls and buffer are at least 1".into()));
}
let principal = Self::principal_for(&spec, &opts).map_err(start)?;
let shared = Shared::new(spec, opts, principal);
let (conn, kept, mut timings) = shared.connect_once().await.map_err(start)?;
let t = Instant::now();
let hash = kept.hash();
let read = mount_schema(&name, kept.top).and_then(|schema| {
let nodes = kept.nodes.into_iter().map(|(path, raw)| node_schema(&path, raw).map(|s| (path, s))).collect::<Result<Vec<_>, _>>()?;
Ok((schema, nodes))
});
let (schema, nodes) = match read {
Ok(s) => s,
Err(why) => {
conn.shutdown.notify_one();
return Err(start(why));
}
};
let _ = shared.mounted_hash.set(hash);
shared.install(conn);
timings.merge_ms = t.elapsed().as_secs_f64() * 1e3;
tracing::info!(target: "plexus::child", child = %name, spawn_ms = timings.spawn_ms, schema_ms = timings.schema_ms, merge_ms = timings.merge_ms, methods = schema.methods.len(), children = nodes.len(), "child mounted");
Ok(Self { inner: Arc::new(Inner::new(shared, schema, nodes, timings)) })
}
#[allow(unused_variables)]
fn principal_for(spec: &ChildSpec, opts: &ChildOptions) -> Result<String, String> {
#[cfg(feature = "child-ws")]
if let (ChildSpec::Ws { .. }, Some(path)) = (spec, &opts.key) {
return Ok(ws::principal_of(&ws::read_seed(path)?));
}
Ok(opts.principal.clone())
}
pub fn stats(&self) -> ChildStats {
let c: &Counters = &self.inner.shared.counters;
ChildStats {
open: c.open.load(Ordering::Relaxed),
unasked: c.unasked.load(Ordering::Relaxed),
restarts: c.restarts.load(Ordering::Relaxed),
calls: c.calls.load(Ordering::Relaxed),
name_schemas: self.inner.names.hops.load(Ordering::Relaxed),
name_schema_hits: self.inner.names.hits.load(Ordering::Relaxed),
names_kept: self.inner.names.len(c.restarts.load(Ordering::Relaxed)),
calls_back: self.inner.shared.upcalls.made.load(Ordering::Relaxed),
up: self.inner.shared.is_up(),
}
}
pub fn is_up(&self) -> bool {
self.inner.shared.is_up()
}
pub fn dead(&self) -> Option<String> {
self.inner.shared.dead()
}
pub fn why_down(&self) -> Option<String> {
self.inner.shared.why_down()
}
pub async fn up(&self) -> Result<(), String> {
match self.inner.shared.ensure().await {
Ok(_) => Ok(()),
Err(PlexusStreamItem::Error { message, error, .. }) => Err(match error {
plexus_client::RemoteError::ChildUnavailable { reason, .. } => reason,
_ => message,
}),
Err(other) => Err(format!("{other:?}")),
}
}
pub fn calls_open_back(&self) -> usize {
self.inner.shared.upcalls.open_now()
}
pub fn mounted_hash(&self) -> &str {
self.inner.shared.mounted_hash.get().map(String::as_str).unwrap_or_default()
}
pub fn pid(&self) -> Option<u32> {
self.inner.shared.current().and_then(|c| c.pid)
}
pub fn name(&self) -> &str {
&self.inner.shared.name
}
pub fn spec(&self) -> &ChildSpec {
&self.inner.shared.spec
}
pub fn principal(&self) -> &str {
&self.inner.shared.principal
}
pub fn mount_timings(&self) -> MountTimings {
self.inner.timings
}
pub fn routed(&self) -> usize {
self.inner.shared.current().map_or(0, |c| c.routed())
}
fn hop(&self, conn: &Conn, auth: Option<&AuthContext>, reach: &Reach, method: &str) -> (Value, String) {
let for_: Vec<String> = auth.and_then(|a| a.principal()).map(|p| p.to_string()).into_iter().collect();
let limit = reach.grant().map(|g| {
let grant = match g.as_str().strip_prefix(&format!("{}.", self.name())) {
Some(m) => format!("{}.{m}", conn.ns()),
None => format!("{}.{method}", conn.ns()),
};
let held_on: Vec<String> = reach.held_on().iter().map(|s| s.to_string()).collect();
json!({ "grant": grant, "held_on": held_on })
});
let call = uuid::Uuid::new_v4().to_string();
(json!({ "principal": self.inner.shared.principal, "for": for_, "limit": limit, "call": call }), call)
}
fn node(&self, path: &str) -> Option<&PluginSchema> {
self.inner.nodes.iter().find(|(p, _)| p == path).map(|(_, s)| s)
}
fn slot_of(&self, path: &str) -> Option<(String, String)> {
let segs: Vec<&str> = path.split('.').collect();
let (n, schema) = (1..segs.len())
.rev()
.find_map(|n| self.node(&segs[..n].join(".")).map(|s| (n, s)))
.unwrap_or((0, &self.inner.schema));
let parent = std::iter::once(self.name()).chain(segs[..n].iter().copied()).collect::<Vec<_>>().join(".");
let next = segs[n];
if schema.methods.iter().any(|m| m.name == next && matches!(m.role, MethodRole::Peer { .. })) {
return Some((crate::plexus::slots::slot_path(&parent, next), segs[..(n + 2).min(segs.len())].join(".")));
}
let slot = schema.methods.iter().find(|m| matches!(m.role, MethodRole::DynamicChild { .. }))?;
Some((crate::plexus::slots::slot_path(&parent, &slot.name), segs[..=n].join(".")))
}
fn unreached(&self, method: &str) -> PlexusError {
PlexusError::MethodNotFound { activation: self.name().to_string(), method: method.to_string() }
}
async fn name_schema(&self, path: &str, params: &Value, auth: Option<&AuthContext>) -> PlexusStream {
let epoch = self.inner.shared.counters.restarts.load(Ordering::Relaxed);
if let Some(s) = self.inner.names.get(path, epoch) {
return self.schema_stream_of(&s, &s.namespace, params);
}
self.inner.names.crossed();
let items: Vec<PlexusStreamItem> = futures::StreamExt::collect(self.forward(&format!("{path}.schema"), json!({}), auth, None).await).await;
let kept = items.iter().find_map(|i| match i {
PlexusStreamItem::Data { metadata, content, .. } => Some((metadata.plexus_hash.clone(), content.clone().into_value())),
_ => None,
});
let Some((seen, raw)) = kept else {
return Box::pin(futures::stream::iter(items));
};
match read_schema(raw) {
Ok(schema) => {
let epoch = self.inner.shared.counters.restarts.load(Ordering::Relaxed);
self.inner.names.put(path, schema.clone(), seen, epoch);
self.schema_stream_of(&schema, &schema.namespace, params)
}
Err(_) => Box::pin(futures::stream::iter(items)),
}
}
fn schema_stream(&self, params: &Value) -> PlexusStream {
self.schema_stream_of(&self.inner.schema, self.name(), params)
}
fn schema_stream_of(&self, schema: &PluginSchema, label: &str, params: &Value) -> PlexusStream {
let ns = label.to_string();
let content = match params.get("method").and_then(Value::as_str) {
Some(m) => match schema.methods.iter().find(|x| x.name == m) {
Some(ms) => serde_json::to_value(ms).unwrap_or_default(),
None => {
let error = plexus_client::RemoteError::MethodNotFound { activation: ns.clone(), method: m.to_string() };
let mut e = PlexusStreamItem::error(meta(&ns), error, false);
if let PlexusStreamItem::Error { message, code, .. } = &mut e {
*message = format!("Method '{m}' not found");
*code = Some("-32602".into());
}
return Box::pin(futures::stream::iter([e]));
}
},
None => serde_json::to_value(schema).unwrap_or_default(),
};
Box::pin(futures::stream::iter([
PlexusStreamItem::Data { metadata: meta(&ns).into(), content_type: format!("{ns}.schema").into(), content: content.into() },
PlexusStreamItem::Done { metadata: meta(&ns) },
]))
}
async fn forward(&self, method: &str, params: Value, auth: Option<&AuthContext>, instance: Option<String>) -> PlexusStream {
let shared = &self.inner.shared;
let one = |item: PlexusStreamItem| -> PlexusStream { Box::pin(futures::stream::iter([item])) };
let reach = match super::grant_gate::current_reach() {
Ok(reach) => reach,
Err(e) => return one(PlexusStreamItem::from_plexus_error(meta(self.name()), &e, false)),
};
let max = shared.opts.max_calls;
if shared.counters.open.fetch_add(1, Ordering::AcqRel) >= max {
shared.counters.open.fetch_sub(1, Ordering::AcqRel);
return one(too_many(self.name(), max));
}
let guard = OpenGuard(Arc::clone(&shared.counters));
let conn = match shared.ensure().await {
Ok(c) => c,
Err(item) => return one(item),
};
shared.counters.calls.fetch_add(1, Ordering::Relaxed);
let (hop, call_id) = self.hop(&conn, auth, &reach, method);
let under = shared.upcalls.open(&call_id, auth, &reach);
let path = format!("{}.{method}", conn.ns());
let call = json!({ "method": path, "params": params, "hop": hop });
match conn.open(&format!("{}.call", conn.ns()), call, shared.opts.timeout).await {
Ok((sid, rx)) => {
let generation = reach.generation().cloned();
let seen = generation.as_ref().map_or(0, GrantsGeneration::current);
let instance = instance.map(|path| (Arc::clone(&self.inner), path));
Box::pin(CallStream { rx, sid, conn, over: false, _guard: guard, reach, generation, seen, name: self.name().to_string(), instance, under })
}
Err(item) => one(item),
}
}
}
struct OpenGuard(Arc<Counters>);
impl Drop for OpenGuard {
fn drop(&mut self) {
self.0.open.fetch_sub(1, Ordering::AcqRel);
}
}
struct CallStream {
rx: mpsc::Receiver<PlexusStreamItem>,
sid: SubKey,
conn: Arc<Conn>,
over: bool,
_guard: OpenGuard,
reach: Reach,
generation: Option<GrantsGeneration>,
seen: u64,
name: String,
instance: Option<(Arc<Inner>, String)>,
under: Option<upcall::UnderGuard>,
}
impl CallStream {
fn finish(&mut self, why: &str) {
self.over = true;
if let Some(u) = self.under.take() {
u.end(why);
}
}
fn revoked(&self) -> PlexusStreamItem {
let grant = self.reach.grant().map(|g| g.as_str().to_string()).unwrap_or_default();
let error = plexus_client::RemoteError::Forbidden {
reason: plexus_client::DenyReason::MissingGrant {
grant: grant.clone(),
scope: "*".into(),
roles: Vec::new(),
held_on: Vec::new(),
},
};
let mut item = PlexusStreamItem::error(meta(&self.name), error, false);
if let PlexusStreamItem::Error { message, .. } = &mut item {
*message = format!("Forbidden: `{grant}` is no longer held — the call's grant moved while it ran");
}
item
}
}
impl Stream for CallStream {
type Item = PlexusStreamItem;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
if self.over {
return Poll::Ready(None);
}
match self.rx.poll_recv(cx) {
Poll::Ready(Some(item)) => {
if let (Some((inner, path)), Some(m)) = (&self.instance, item.metadata()) {
inner.names.observe(path, &m.plexus_hash);
}
let terminal = matches!(item, PlexusStreamItem::Done { .. } | PlexusStreamItem::Error { .. });
let moved = self.generation.as_ref().map(GrantsGeneration::current).filter(|g| *g != self.seen);
if let (false, Some(g)) = (terminal, moved) {
self.seen = g;
if self.reach.current().is_none() {
let sid = self.sid.clone();
self.conn.unsubscribe(&sid);
self.finish("its grant was revoked");
return Poll::Ready(Some(self.revoked()));
}
}
if terminal {
self.finish("it ended");
}
Poll::Ready(Some(item))
}
Poll::Ready(None) => {
self.finish("the connection closed");
Poll::Ready(Some(unavailable(&self.name, "the connection closed")))
}
Poll::Pending => Poll::Pending,
}
}
}
impl Drop for CallStream {
fn drop(&mut self) {
if !self.over {
self.conn.unsubscribe(&self.sid);
}
}
}
#[async_trait]
impl Activation for RemoteChild {
type Methods = ChildMethods;
fn namespace(&self) -> &str {
self.name()
}
fn version(&self) -> &str {
&self.inner.schema.version
}
fn description(&self) -> &str {
&self.inner.schema.description
}
fn long_description(&self) -> Option<&str> {
self.inner.schema.long_description.as_deref()
}
fn methods(&self) -> Vec<&str> {
self.inner.schema.methods.iter().map(|m| m.name.as_str()).collect()
}
fn method_help(&self, method: &str) -> Option<String> {
self.inner.schema.methods.iter().find(|m| m.name == method).map(|m| m.description.clone())
}
async fn call(&self, method: &str, params: Value, auth: Option<&AuthContext>, _raw: Option<&crate::request::RawRequestContext>) -> Result<PlexusStream, PlexusError> {
if method == "schema" {
return Ok(self.schema_stream(¶ms));
}
if let Some(path) = method.strip_suffix(".schema") {
if let Some(s) = self.node(path) {
return Ok(self.schema_stream_of(s, &s.namespace, ¶ms));
}
return match self.slot_of(path) {
Some(_) => Ok(self.name_schema(path, ¶ms, auth).await),
None => Err(self.unreached(method)),
};
}
let found = match method.rsplit_once('.') {
Some((path, name)) => match self.node(path) {
Some(s) => s.methods.iter().find(|m| m.name == name && m.role == MethodRole::Rpc),
None => match self.slot_of(path) {
Some((_, instance)) => return Ok(self.forward(method, params, auth, Some(instance)).await),
None => return Err(self.unreached(method)),
},
},
None => self.inner.schema.methods.iter().find(|m| m.name == method && m.role == MethodRole::Rpc),
};
let Some(m) = found else {
return Err(PlexusError::MethodNotFound { activation: self.name().to_string(), method: method.to_string() });
};
if m.input.is_some() {
return Err(PlexusError::ExecutionError(format!(
"`{}.{method}` takes an input stream, and a hop to a remote child does not forward one yet",
self.name()
)));
}
Ok(self.forward(method, params, auth, None).await)
}
fn into_rpc_methods(self) -> jsonrpsee::Methods {
jsonrpsee::Methods::new()
}
fn plugin_schema(&self) -> PluginSchema {
self.inner.schema.clone()
}
fn static_child_schemas(&self) -> Vec<(String, PluginSchema)> {
self.inner.nodes.clone()
}
}
#[async_trait]
impl ChildRouter for RemoteChild {
fn router_namespace(&self) -> &str {
self.name()
}
async fn router_call(&self, method: &str, params: Value, auth: Option<&AuthContext>, raw: Option<&crate::request::RawRequestContext>) -> Result<PlexusStream, PlexusError> {
Activation::call(self, method, params, auth, raw).await
}
async fn get_child(&self, _name: &str) -> Option<Box<dyn ChildRouter>> {
None
}
fn router_schema(&self) -> Option<PluginSchema> {
Some(self.inner.schema.clone())
}
fn router_static_child_schemas(&self) -> Vec<(String, PluginSchema)> {
self.inner.nodes.clone()
}
}
#[cfg(test)]
mod tests {
use super::*;
fn raw(methods: Value, roles: Value) -> Value {
json!({ "namespace": "stub", "version": "0.1.0", "description": "d", "self_hash": "h", "hash": "h", "methods": methods, "roles": roles })
}
fn m(name: &str) -> Value {
json!({ "name": name, "description": "", "hash": "x", "streaming": false, "bidirectional": false })
}
#[test]
fn a_method_stating_neither_cardinality_nor_streaming_is_refused() {
let silent = json!({ "name": "quiet", "description": "", "hash": "x" });
let e = mount_schema("echo", raw(json!([m("echo"), silent]), json!({}))).unwrap_err();
assert!(e.contains("`quiet`") && e.contains("neither cardinality"), "{e}");
}
#[test]
fn the_mount_name_wins_and_roles_are_requalified() {
let s = mount_schema("echo", raw(json!([m("echo"), m("shout")]), json!({ "viewer": ["stub.echo"], "editor": ["stub.echo", "stub.shout"] }))).unwrap();
assert_eq!(s.namespace, "echo");
assert_eq!(s.roles["viewer"], vec!["echo.echo".to_string()]);
assert_eq!(s.roles["editor"], vec!["echo.echo".to_string(), "echo.shout".to_string()]);
}
#[test]
fn a_child_with_children_mounts_and_its_method_grants_are_requalified() {
let mut inner = m("inner");
inner["role"] = json!({ "kind": "static_child" });
let mut own = m("echo");
own["grant"] = json!("stub.echo");
let s = mount_schema("echo", raw(json!([own, inner]), json!({}))).unwrap();
assert_eq!(s.methods[0].grant.as_deref(), Some("echo.echo"));
assert_eq!(s.methods[1].role, MethodRole::StaticChild);
let mut slot = m("item");
slot["role"] = json!({ "kind": "dynamic_child", "list_method": "names" });
let nested = raw(json!([m("echo"), s_inner(), slot]), json!({}));
assert_eq!(link::static_children(&nested), vec!["inner".to_string()], "a dynamic child is a slot, not walked");
let mut root = raw(json!([m("call")]), json!({}));
root["children"] = json!([{ "namespace": "greeter", "description": "", "hash": "g" }]);
assert_eq!(link::static_children(&root), vec!["greeter".to_string()]);
}
fn s_inner() -> Value {
let mut inner = m("inner");
inner["role"] = json!({ "kind": "static_child" });
inner
}
#[test]
fn the_tree_hash_covers_every_level() {
let top = raw(json!([m("echo")]), json!({}));
let flat = link::Kept { top: top.clone(), nodes: vec![] };
assert_eq!(flat.hash(), link::schema_hash(&top), "a flat child's hash is its schema's, as before");
let mut c = raw(json!([m("m")]), json!({}));
let one = link::Kept { top: top.clone(), nodes: vec![("a.b".into(), c.clone())] }.hash();
c["hash"] = json!("other");
let two = link::Kept { top, nodes: vec![("a.b".into(), c)] }.hash();
assert_ne!(one, two);
}
#[test]
fn typed_error_fields_cross_whole() {
let item = unavailable("echo", "gone");
let v = serde_json::to_value(&item).unwrap();
assert_eq!(v["error_kind"], "child_unavailable");
assert_eq!(v["code"], "child_unavailable");
assert_eq!(v["child"], "echo");
let back: PlexusStreamItem = serde_json::from_value(v.clone()).unwrap();
assert_eq!(back.error_kind(), Some("child_unavailable"));
assert_eq!(serde_json::to_value(&back).unwrap(), v);
let plain = PlexusStreamItem::error(meta("x"), plexus_client::RemoteError::Execution { message: "m".into() }, false);
let pv = serde_json::to_value(&plain).unwrap();
let keys: Vec<_> = pv.as_object().unwrap().keys().cloned().collect();
assert_eq!(keys, ["code", "error", "message", "metadata", "recoverable", "type"]);
}
}