use std::any::{Any, TypeId};
use std::collections::HashMap;
use std::fmt;
use std::sync::{Arc, OnceLock};
use std::time::Duration;
use futures::future::BoxFuture;
use indexmap::IndexMap;
use crate::dsl::NodeDoc;
use crate::event::{Event, FrameId, PauseReason, Usage};
use crate::pool::{Selector, VariablePool};
use crate::snapshot::FrameSnapshot;
use crate::value::Value;
use crate::variables::{VarName, Variables};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum NodeRole {
Root,
ScopeStart,
Executable,
Branch,
Response,
Container,
}
#[derive(Clone, Debug, PartialEq, thiserror::Error)]
#[error("{kind}: {message}")]
pub struct NodeError {
pub kind: String,
pub message: String,
pub retryable: bool,
pub retry_after_ms: Option<u64>,
}
impl NodeError {
pub fn new(kind: impl Into<String>, message: impl Into<String>) -> Self {
Self { kind: kind.into(), message: message.into(), retryable: true, retry_after_ms: None }
}
pub fn fatal(kind: impl Into<String>, message: impl Into<String>) -> Self {
Self { kind: kind.into(), message: message.into(), retryable: false, retry_after_ms: None }
}
pub fn with_retry_after(mut self, retry_after_ms: Option<u64>) -> Self {
self.retry_after_ms = retry_after_ms;
self
}
pub fn with_message(mut self, message: impl Into<String>) -> Self {
self.message = message.into();
self
}
pub fn invalid_input(message: impl Into<String>) -> Self {
Self::fatal("invalid_input", message)
}
pub fn timeout(after: Duration) -> Self {
Self::new("timeout", format!("node timed out after {} ms", after.as_millis()))
}
}
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
#[error("node `{node_id}` ({kind}): {}{message}", field_prefix(.field))]
pub struct ConfigError {
pub node_id: String,
pub kind: String,
pub message: String,
pub field: Option<String>,
}
fn field_prefix(field: &Option<String>) -> String {
field.as_ref().map(|f| format!("{f}: ")).unwrap_or_default()
}
impl ConfigError {
pub fn new(doc: &NodeDoc, message: impl Into<String>) -> Self {
Self { node_id: doc.id.clone(), kind: doc.kind.clone(), message: message.into(), field: None }
}
#[must_use]
pub fn at_field(mut self, field: impl Into<String>) -> Self {
self.field = Some(field.into());
self
}
}
#[derive(Clone, Debug, Default, PartialEq)]
pub struct NodeOutput {
pub outputs: Variables,
pub branch: Option<String>,
pub writes: Vec<(Selector, Value)>,
pub usage: Option<Usage>,
pub metadata: IndexMap<String, Value>,
}
impl NodeOutput {
pub fn new() -> Self {
Self::default()
}
pub fn with(mut self, key: impl Into<VarName>, value: impl Into<Value>) -> Self {
self.outputs.insert(key.into(), value.into());
self
}
pub fn branch(mut self, handle: impl Into<String>) -> Self {
self.branch = Some(handle.into());
self
}
pub fn write(mut self, selector: Selector, value: Value) -> Self {
self.writes.push((selector, value));
self
}
pub fn usage(mut self, usage: Usage) -> Self {
self.usage = Some(usage);
self
}
pub fn meta(mut self, key: impl Into<String>, value: impl Into<Value>) -> Self {
self.metadata.insert(key.into(), value.into());
self
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct PauseRequest {
pub kind: String,
pub data: Value,
pub state: serde_json::Value,
}
#[derive(Clone, Debug, PartialEq)]
pub enum NodeOutcome {
Completed(NodeOutput),
Paused(PauseRequest),
}
impl From<NodeOutput> for NodeOutcome {
fn from(o: NodeOutput) -> Self {
NodeOutcome::Completed(o)
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct Resume {
pub state: serde_json::Value,
pub input: Option<Value>,
}
pub type NodeResult = Result<NodeOutcome, NodeError>;
pub trait Node: Send + Sync + fmt::Debug {
fn role(&self) -> NodeRole {
NodeRole::Executable
}
fn run(&self, ctx: NodeContext) -> BoxFuture<'_, NodeResult>;
fn inputs(&self) -> Vec<Selector> {
Vec::new()
}
fn branch_handles(&self) -> Option<Vec<String>> {
None
}
}
pub fn merge_writes(acc: &mut Vec<(Selector, Value)>, new: impl IntoIterator<Item = (Selector, Value)>) {
for (selector, value) in new {
if let Some(pos) = acc.iter().position(|(s, _)| *s == selector) {
acc.remove(pos);
}
acc.push((selector, value));
}
}
#[derive(Debug)]
pub struct SubgraphRequest {
pub index: usize,
pub pool: VariablePool,
pub resume: Option<FrameSnapshot>,
}
#[derive(Debug)]
#[allow(clippy::large_enum_variant)]
pub enum SubgraphResult {
Completed {
pool: VariablePool,
outputs: IndexMap<String, Value>,
writes: Vec<(Selector, Value)>,
usage: Usage,
broke: bool,
},
Failed {
error: NodeError,
usage: Usage,
},
Paused {
snapshot: FrameSnapshot,
reasons: Vec<PauseReason>,
},
}
pub trait Subgraph: Send + Sync + fmt::Debug {
fn run(&self, request: SubgraphRequest) -> BoxFuture<'_, SubgraphResult>;
}
pub trait Clock: Send + Sync + fmt::Debug {
fn now_ms(&self) -> u64;
fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()>;
fn supports_timers(&self) -> bool {
true
}
}
#[derive(Debug, Default, Clone, Copy)]
pub struct NoTimerClock;
impl Clock for NoTimerClock {
fn now_ms(&self) -> u64 {
system_now_ms()
}
fn sleep(&self, _duration: Duration) -> BoxFuture<'static, ()> {
Box::pin(futures::future::ready(()))
}
fn supports_timers(&self) -> bool {
false
}
}
#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]
fn system_now_ms() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX))
.unwrap_or(0)
}
#[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
fn system_now_ms() -> u64 {
use std::sync::atomic::{AtomicU64, Ordering};
static COUNTER: AtomicU64 = AtomicU64::new(0);
COUNTER.fetch_add(1, Ordering::Relaxed)
}
#[cfg(feature = "tokio")]
#[derive(Debug, Default, Clone, Copy)]
pub struct TokioClock;
#[cfg(feature = "tokio")]
impl Clock for TokioClock {
fn now_ms(&self) -> u64 {
NoTimerClock.now_ms()
}
fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()> {
if tokio::runtime::Handle::try_current().is_ok() {
Box::pin(tokio::time::sleep(duration))
} else {
Box::pin(futures::future::ready(()))
}
}
fn supports_timers(&self) -> bool {
tokio::runtime::Handle::try_current().is_ok()
}
}
pub fn default_clock() -> Arc<dyn Clock> {
#[cfg(feature = "tokio")]
{
Arc::new(TokioClock)
}
#[cfg(not(feature = "tokio"))]
{
Arc::new(NoTimerClock)
}
}
#[derive(Default, Clone)]
pub struct Services {
map: HashMap<TypeId, Arc<dyn Any + Send + Sync>>,
}
impl fmt::Debug for Services {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "Services({} entries)", self.map.len())
}
}
impl Services {
pub fn new() -> Self {
Self::default()
}
pub fn insert<T: Any + Send + Sync>(&mut self, service: T) -> &mut Self {
self.map.insert(TypeId::of::<T>(), Arc::new(service));
self
}
pub fn with<T: Any + Send + Sync>(mut self, service: T) -> Self {
self.insert(service);
self
}
pub fn get<T: Any + Send + Sync>(&self) -> Option<&T> {
self.map.get(&TypeId::of::<T>()).and_then(|s| s.downcast_ref::<T>())
}
}
pub type EventSink = Arc<dyn Fn(Event) + Send + Sync>;
#[derive(Clone)]
pub struct NodeContext {
pub node_id: Arc<str>,
pub exec_id: Arc<str>,
pub frame: FrameId,
pub attempt: u32,
pub pool: VariablePool,
pub services: Arc<Services>,
pub clock: Arc<dyn Clock>,
pub resume: Option<Resume>,
pub subgraph: Option<Arc<dyn Subgraph>>,
pub deadline_ms: Option<u64>,
sink: EventSink,
}
impl fmt::Debug for NodeContext {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("NodeContext")
.field("node_id", &self.node_id)
.field("exec_id", &self.exec_id)
.field("frame", &self.frame)
.field("attempt", &self.attempt)
.finish_non_exhaustive()
}
}
impl NodeContext {
#[allow(clippy::too_many_arguments)]
pub fn new(
node_id: impl Into<Arc<str>>,
exec_id: impl Into<Arc<str>>,
frame: FrameId,
attempt: u32,
pool: VariablePool,
services: Arc<Services>,
clock: Arc<dyn Clock>,
sink: EventSink,
) -> Self {
Self {
node_id: node_id.into(),
exec_id: exec_id.into(),
frame,
attempt,
pool,
services,
clock,
resume: None,
subgraph: None,
deadline_ms: None,
sink,
}
}
pub fn for_test(node_id: &str, pool: VariablePool) -> Self {
Self::new(
node_id,
format!("{node_id}-exec"),
Arc::from(""),
1,
pool,
Arc::new(Services::new()),
Arc::new(NoTimerClock),
Arc::new(|_| {}),
)
}
pub fn with_services(mut self, services: Arc<Services>) -> Self {
self.services = services;
self
}
pub fn with_sink(mut self, sink: EventSink) -> Self {
self.sink = sink;
self
}
pub fn with_resume(mut self, resume: Option<Resume>) -> Self {
self.resume = resume;
self
}
pub fn with_subgraph(mut self, subgraph: Option<Arc<dyn Subgraph>>) -> Self {
self.subgraph = subgraph;
self
}
pub fn with_deadline(mut self, deadline_ms: Option<u64>) -> Self {
self.deadline_ms = deadline_ms;
self
}
pub fn remaining(&self) -> Option<Duration> {
self.deadline_ms.map(|deadline| Duration::from_millis(deadline.saturating_sub(self.clock.now_ms())))
}
pub fn service<T: Any + Send + Sync>(&self) -> Option<&T> {
self.services.get::<T>()
}
pub fn require_service<T: Any + Send + Sync>(&self, what: &str) -> Result<&T, NodeError> {
self.services.get::<T>().ok_or_else(|| NodeError::fatal("missing_service", format!("no {what} configured")))
}
pub fn get(&self, selector: &Selector) -> Option<Value> {
self.pool.get(selector)
}
pub fn chunk(&self, field: &str, chunk: impl Into<String>) {
self.chunk_shared(&Arc::from(field), chunk);
}
pub fn chunk_shared(&self, field: &Arc<str>, chunk: impl Into<String>) {
(self.sink)(Event::NodeChunk {
frame: self.frame.clone(),
node_id: Arc::clone(&self.node_id),
exec_id: Arc::clone(&self.exec_id),
field: Arc::clone(field),
chunk: chunk.into(),
});
}
pub fn progress(&self, name: &str, data: Value) {
(self.sink)(Event::NodeProgress {
frame: self.frame.clone(),
node_id: Arc::clone(&self.node_id),
exec_id: Arc::clone(&self.exec_id),
name: name.to_owned(),
data,
});
}
pub fn sink(&self) -> &EventSink {
&self.sink
}
}
pub type NodeFactory = Arc<dyn Fn(&NodeDoc) -> Result<Arc<dyn Node>, ConfigError> + Send + Sync>;
pub type DescriptorFn = fn() -> crate::catalog::NodeDescriptor;
#[derive(Clone)]
enum DescriptorSlot {
Ready(Box<crate::catalog::NodeDescriptor>),
Lazy(Arc<LazyDescriptor>),
}
struct LazyDescriptor {
build: DescriptorFn,
built: OnceLock<crate::catalog::NodeDescriptor>,
}
#[derive(Clone, Default)]
pub struct Registry {
factories: HashMap<String, NodeFactory>,
descriptors: HashMap<String, DescriptorSlot>,
}
impl fmt::Debug for Registry {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let mut kinds: Vec<&String> = self.factories.keys().collect();
kinds.sort();
f.debug_struct("Registry").field("kinds", &kinds).finish()
}
}
impl Registry {
pub fn new() -> Self {
Self::default()
}
pub fn with_builtins() -> Self {
let mut r = Self::new();
crate::nodes::register_builtins(&mut r);
r
}
pub fn register<F>(&mut self, kind: &str, factory: F) -> &mut Self
where
F: Fn(&NodeDoc) -> Result<Arc<dyn Node>, ConfigError> + Send + Sync + 'static,
{
self.factories.insert(kind.to_owned(), Arc::new(factory));
self
}
pub fn register_described<F>(&mut self, descriptor: crate::catalog::NodeDescriptor, factory: F) -> &mut Self
where
F: Fn(&NodeDoc) -> Result<Arc<dyn Node>, ConfigError> + Send + Sync + 'static,
{
let kind = descriptor.kind.clone();
self.factories.insert(kind.clone(), Arc::new(factory));
self.descriptors.insert(kind, DescriptorSlot::Ready(Box::new(descriptor)));
self
}
pub fn register_lazily_described<F>(&mut self, kind: &str, descriptor: DescriptorFn, factory: F) -> &mut Self
where
F: Fn(&NodeDoc) -> Result<Arc<dyn Node>, ConfigError> + Send + Sync + 'static,
{
self.factories.insert(kind.to_owned(), Arc::new(factory));
let slot = LazyDescriptor { build: descriptor, built: OnceLock::new() };
self.descriptors.insert(kind.to_owned(), DescriptorSlot::Lazy(Arc::new(slot)));
self
}
pub fn describe(&mut self, descriptor: crate::catalog::NodeDescriptor) -> &mut Self {
if self.factories.contains_key(&descriptor.kind) {
self.descriptors.insert(descriptor.kind.clone(), DescriptorSlot::Ready(Box::new(descriptor)));
}
self
}
pub fn descriptor(&self, kind: &str) -> Option<crate::catalog::NodeDescriptor> {
if !self.factories.contains_key(kind) {
return None;
}
let descriptor = match self.descriptors.get(kind) {
None => crate::catalog::NodeDescriptor::bare(kind),
Some(DescriptorSlot::Ready(descriptor)) => (**descriptor).clone(),
Some(DescriptorSlot::Lazy(slot)) => slot.built.get_or_init(slot.build).clone(),
};
Some(descriptor)
}
pub fn descriptor_ref(&self, kind: &str) -> Option<&crate::catalog::NodeDescriptor> {
match self.descriptors.get(kind)? {
DescriptorSlot::Ready(descriptor) => Some(descriptor),
DescriptorSlot::Lazy(slot) => Some(slot.built.get_or_init(slot.build)),
}
}
pub fn descriptors(&self) -> Vec<crate::catalog::NodeDescriptor> {
let mut kinds: Vec<&String> = self.factories.keys().collect();
kinds.sort();
kinds.into_iter().filter_map(|k| self.descriptor(k)).collect()
}
pub fn alias(&mut self, alias: &str, kind: &str) -> &mut Self {
if let Some(f) = self.factories.get(kind).cloned() {
self.factories.insert(alias.to_owned(), f);
}
self
}
pub fn contains(&self, kind: &str) -> bool {
self.factories.contains_key(kind)
}
pub fn kinds(&self) -> impl Iterator<Item = &str> {
self.factories.keys().map(String::as_str)
}
pub fn build(&self, doc: &NodeDoc) -> Result<Arc<dyn Node>, ConfigError> {
let factory = self.factories.get(&doc.kind).ok_or_else(|| {
let suggestion = crate::validate::suggest(&doc.kind, self.kinds());
ConfigError::new(
doc,
format!("unknown node type `{}`{}", doc.kind, crate::validate::did_you_mean(&suggestion)),
)
})?;
factory(doc)
}
}
pub fn parse_config<T: serde::de::DeserializeOwned>(doc: &NodeDoc) -> Result<T, ConfigError> {
crate::dsl::parse_config_at(&doc.config).map_err(|(path, e)| {
let error = ConfigError::new(doc, e.to_string());
if path == "." { error } else { error.at_field(path) }
})
}