use std::future::{Future, poll_fn};
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use futures_util::FutureExt;
use serde_json::{Map, Value, json};
use crate::workflow::definition::JsonBoundary;
use crate::workflow::{
ActionContext, ActionMode, ActionOutcome, Checkpoint, GraphDescription, RunId, RunLineage,
RunSnapshot, WorkflowFailure, WorkflowState,
};
use crate::{AppError, WorkspaceId};
const MAX_LIFECYCLE_OBSERVATION: Duration = Duration::from_secs(30);
pub trait LifecycleHooks: Send + Sync + 'static {
fn observe(&self, _event: &LifecycleEvent<'_>) -> Result<(), AppError> {
Ok(())
}
fn observe_async<'a>(
&'a self,
_event: &'a LifecycleEvent<'a>,
) -> impl Future<Output = Result<(), AppError>> + Send + 'a {
async { Ok(()) }
}
}
pub enum LifecycleEvent<'a> {
Registered {
workspace: &'a WorkspaceId,
run: &'a RunId,
occurrence: &'a str,
graph: &'a GraphDescription,
parent: Option<&'a ActionContext>,
lineage: Option<&'a RunLineage>,
},
ChildFinished {
parent: &'a ActionContext,
snapshot: &'a RunSnapshot,
},
ExecuteStarted {
run: &'a RunId,
occurrence: &'a str,
},
ExecuteFinished {
snapshot: &'a RunSnapshot,
failure: Option<&'a WorkflowFailure>,
},
StepStarted {
context: &'a ActionContext,
state: &'a WorkflowState,
inputs: &'a Map<String, Value>,
},
StepFinished {
context: &'a ActionContext,
outcome: Option<Result<&'a ActionOutcome, &'a WorkflowFailure>>,
checkpoint: &'a Checkpoint,
},
StreamStarted {
context: &'a ActionContext,
span: Option<&'a str>,
},
StreamItem {
context: &'a ActionContext,
span: Option<&'a str>,
index: usize,
result: &'a Map<String, Value>,
},
StreamFinished {
context: &'a ActionContext,
span: Option<&'a str>,
items: usize,
failure: Option<&'a WorkflowFailure>,
},
SpanStarted {
context: &'a ActionContext,
id: &'a str,
name: &'a str,
parent: Option<&'a str>,
dependencies: &'a [String],
},
SpanAttribute {
context: &'a ActionContext,
span: Option<&'a str>,
key: &'a str,
value: &'a Value,
tags: &'a Map<String, Value>,
},
SpanFinished {
context: &'a ActionContext,
id: &'a str,
},
}
#[derive(Default)]
pub struct NoHooks;
impl LifecycleHooks for NoHooks {}
impl LifecycleEvent<'_> {
pub(crate) fn deadline(activation: Option<tokio::time::Instant>) -> tokio::time::Instant {
let ceiling = tokio::time::Instant::now() + MAX_LIFECYCLE_OBSERVATION;
activation.map_or(ceiling, |deadline| deadline.min(ceiling))
}
pub(crate) fn deliver_sync<H: LifecycleHooks>(&self, hooks: &H) -> Result<(), WorkflowFailure> {
catch_unwind(AssertUnwindSafe(|| hooks.observe(self)))
.map_err(|_| WorkflowFailure::Execution("lifecycle callback panicked".into()))?
.map_err(WorkflowFailure::hook)
}
pub(crate) async fn deliver<H: LifecycleHooks>(
&self,
hooks: &H,
mode: ActionMode,
deadline: tokio::time::Instant,
) -> Result<(), WorkflowFailure> {
if mode == ActionMode::Both {
return Err(WorkflowFailure::Execution(
"lifecycle requires one actual mode".into(),
));
}
self.deliver_sync(hooks)?;
if tokio::time::Instant::now() >= deadline {
return Err(WorkflowFailure::Timeout(
"lifecycle observation expired".into(),
));
}
if mode == ActionMode::Synchronous {
return Ok(());
}
let source = catch_unwind(AssertUnwindSafe(|| hooks.observe_async(self)))
.map_err(|_| WorkflowFailure::Execution("lifecycle callback panicked".into()))?;
let mut source = Box::pin(source);
let observed = tokio::select! {
biased;
_ = tokio::time::sleep_until(deadline) => {
Err(WorkflowFailure::Timeout("lifecycle observation expired".into()))
}
result = AssertUnwindSafe(poll_fn(|context| source.as_mut().poll(context))).catch_unwind() => {
match result {
Ok(result) => result.map_err(WorkflowFailure::hook),
Err(_) => Err(WorkflowFailure::Execution("lifecycle callback panicked".into())),
}
}
};
let destroyed = catch_unwind(AssertUnwindSafe(|| drop(source)))
.map_err(|_| WorkflowFailure::Execution("lifecycle callback panicked".into()));
observed.and(destroyed)
}
pub fn tracking_records(&self, observed_at: SystemTime) -> Result<Vec<Value>, AppError> {
let records = match self {
Self::Registered {
workspace,
run,
occurrence: _,
graph,
parent,
lineage,
} => {
let logical_parent = lineage.map(|lineage| {
json!({
"type":"pointer_data", "app_id":lineage.source,
"sequence_id":lineage.action_ordinal, "partition_key":workspace,
})
});
let spawning_parent = parent.map(|parent| {
json!({
"type":"pointer_data", "app_id":parent.run,
"sequence_id":parent.action_ordinal, "partition_key":parent.workspace,
})
});
let mut records = vec![
graph.tracking_record()?,
json!({
"type":"application_metadata", "partition_key":workspace,
"parent_pointer":logical_parent, "spawning_parent_pointer":spawning_parent,
}),
];
if lineage.is_some() || parent.is_some() {
let time = Self::tracking_time(observed_at)?;
for (kind, ordinal) in lineage
.map(|lineage| ("fork", lineage.action_ordinal))
.into_iter()
.chain(parent.map(|parent| ("spawn_start", Some(parent.action_ordinal))))
{
records.push(json!({
"type":"child_application_data", "event_type":kind,
"sequence_id":ordinal, "event_time":time,
"child":{"type":"pointer_data", "app_id":run,
"sequence_id":null, "partition_key":workspace},
}));
}
}
records
}
Self::StepStarted {
context, inputs, ..
} => vec![json!({
"type":"begin_entry", "start_time":Self::tracking_time(observed_at)?,
"action":context.node, "inputs":inputs, "sequence_id":context.action_ordinal,
})],
Self::StepFinished {
context,
outcome,
checkpoint,
} => {
let Some(outcome) = outcome else {
return Err(AppError::InvalidInput(
"step without an outcome has no tracking end-entry state".into(),
));
};
let mut state = checkpoint.state.clone();
state
.values
.insert("__SEQUENCE_ID".into(), context.action_ordinal.into());
match &checkpoint.position {
Some(position) => {
state
.values
.insert("__PRIOR_STEP".into(), position.clone().into());
}
None => {
state.values.remove("__PRIOR_STEP");
}
}
let (result, exception) = match outcome {
Ok(outcome) => (Some(&outcome.result), None),
Err(failure) => (None, Some(failure.message())),
};
vec![json!({
"type":"end_entry", "end_time":Self::tracking_time(observed_at)?,
"action":context.node, "result":result, "exception":exception,
"state":state.serialize()?, "sequence_id":context.action_ordinal,
})]
}
Self::SpanStarted {
context,
id,
name,
parent,
dependencies,
} => vec![json!({
"type":"begin_span", "start_time":Self::tracking_time(observed_at)?,
"action_sequence_id":context.action_ordinal, "span_id":id,
"span_name":name, "parent_span_id":parent, "span_dependencies":dependencies,
})],
Self::SpanFinished { context, id } => vec![json!({
"type":"end_span", "end_time":Self::tracking_time(observed_at)?,
"action_sequence_id":context.action_ordinal, "span_id":id,
})],
Self::SpanAttribute {
context,
span,
key,
value,
tags,
} => {
if tags.values().any(|value| !value.is_string()) {
return Err(AppError::InvalidInput(
"tracking attribute tags must be strings".into(),
));
}
vec![json!({
"type":"attribute", "key":key, "action_sequence_id":context.action_ordinal,
"span_id":span, "value":value, "tags":tags,
"time_logged":Self::tracking_time(observed_at)?,
})]
}
Self::StreamStarted { context, span } => vec![json!({
"type":"begin_stream", "action_sequence_id":context.action_ordinal,
"span_id":span, "stream_init_time":Self::tracking_time(observed_at)?,
})],
Self::StreamItem {
context,
span,
index: 0,
..
} => vec![json!({
"type":"first_item_stream", "action_sequence_id":context.action_ordinal,
"span_id":span, "first_item_time":Self::tracking_time(observed_at)?,
})],
Self::StreamFinished {
context,
span,
items,
..
} => vec![json!({
"type":"end_stream", "action_sequence_id":context.action_ordinal,
"span_id":span, "end_time":Self::tracking_time(observed_at)?,
"items_streamed":items,
})],
Self::ChildFinished { .. }
| Self::ExecuteStarted { .. }
| Self::ExecuteFinished { .. }
| Self::StreamItem { .. } => Vec::new(),
};
JsonBoundary::validate_size(&records)?;
Ok(records)
}
fn tracking_time(observed_at: SystemTime) -> Result<String, AppError> {
let elapsed = observed_at
.duration_since(UNIX_EPOCH)
.map_err(|_| AppError::InvalidInput("tracking timestamp precedes UNIX_EPOCH".into()))?;
if elapsed.as_secs() >= 253_402_300_800 {
return Err(AppError::InvalidInput(
"tracking timestamp exceeds year 9999".into(),
));
}
let mut days = elapsed.as_secs() / 86_400;
let mut year = 1970_u64;
let leap = loop {
let leap =
year.is_multiple_of(4) && (!year.is_multiple_of(100) || year.is_multiple_of(400));
let length = 365 + u64::from(leap);
if days < length {
break leap;
}
days -= length;
year += 1;
};
let months = [
31,
28 + u64::from(leap),
31,
30,
31,
30,
31,
31,
30,
31,
30,
31,
];
let mut month = 1;
for length in months {
if days < length {
break;
}
days -= length;
month += 1;
}
let time = elapsed.as_secs() % 86_400;
Ok(format!(
"{year:04}-{month:02}-{:02}T{:02}:{:02}:{:02}.{:09}Z",
days + 1,
time / 3_600,
(time / 60) % 60,
time % 60,
elapsed.subsec_nanos()
))
}
}
impl WorkflowFailure {
pub(crate) fn with_cause(self, cause: Option<Self>) -> Self {
match self {
Self::Hook {
message,
cause: prior,
} => Self::Hook {
message,
cause: prior.or_else(|| cause.map(Box::new)),
},
failure => failure,
}
}
fn hook(error: AppError) -> Self {
Self::Hook {
message: error.to_string(),
cause: None,
}
}
pub(crate) fn application_error(&self) -> AppError {
match self {
Self::Timeout(message) => AppError::Timeout(message.clone()),
_ => AppError::Integration(self.message().to_owned()),
}
}
pub(crate) fn message(&self) -> &str {
match self {
Self::Hook { message, .. } => message,
Self::Evaluation(message)
| Self::Action(message)
| Self::State(message)
| Self::Agent(message)
| Self::Child(message)
| Self::Execution(message)
| Self::Timeout(message)
| Self::Limit(message) => message,
}
}
}