#![allow(dead_code)]
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::RwLock;
use tokio_util::sync::CancellationToken;
#[derive(Clone)]
pub struct WorkflowCancellationRegistry {
inner: Arc<RegistryInner>,
}
struct RegistryInner {
runs: RwLock<HashMap<String, RunTokens>>,
}
#[derive(Default)]
struct RunTokens {
activities: HashMap<String, CancellationToken>,
nodes: HashMap<String, CancellationToken>,
}
impl WorkflowCancellationRegistry {
pub fn new() -> Self {
Self {
inner: Arc::new(RegistryInner {
runs: RwLock::new(HashMap::new()),
}),
}
}
pub fn register_activity(&self, run_id: &str, activity_id: &str) -> CancellationToken {
let mut runs = self.inner.runs.write().expect("registry lock poisoned");
let entry = runs.entry(run_id.to_string()).or_default();
let token = CancellationToken::new();
entry
.activities
.insert(activity_id.to_string(), token.clone());
token
}
pub fn unregister_activity(&self, run_id: &str, activity_id: &str) {
let mut runs = self.inner.runs.write().expect("registry lock poisoned");
if let Some(entry) = runs.get_mut(run_id) {
entry.activities.remove(activity_id);
if entry.activities.is_empty() && entry.nodes.is_empty() {
runs.remove(run_id);
}
}
}
pub fn register_node(&self, run_id: &str, node_id: &str) -> CancellationToken {
let mut runs = self.inner.runs.write().expect("registry lock poisoned");
let entry = runs.entry(run_id.to_string()).or_default();
let token = CancellationToken::new();
entry.nodes.insert(node_id.to_string(), token.clone());
token
}
pub fn unregister_node(&self, run_id: &str, node_id: &str) {
let mut runs = self.inner.runs.write().expect("registry lock poisoned");
if let Some(entry) = runs.get_mut(run_id) {
entry.nodes.remove(node_id);
if entry.activities.is_empty() && entry.nodes.is_empty() {
runs.remove(run_id);
}
}
}
pub fn cancel_activity(&self, run_id: &str, activity_id: &str) -> bool {
let mut runs = self.inner.runs.write().expect("registry lock poisoned");
let Some(entry) = runs.get_mut(run_id) else {
return false;
};
let Some(token) = entry.activities.remove(activity_id) else {
return false;
};
token.cancel();
if entry.activities.is_empty() && entry.nodes.is_empty() {
runs.remove(run_id);
}
true
}
pub fn cancel_node(&self, run_id: &str, node_id: &str) -> Vec<String> {
let mut runs = self.inner.runs.write().expect("registry lock poisoned");
let Some(entry) = runs.get_mut(run_id) else {
return Vec::new();
};
let mut cancelled = Vec::new();
if let Some(node_token) = entry.nodes.remove(node_id) {
node_token.cancel();
}
let run_prefix = format!("{}::", run_id);
let mut to_remove: Vec<String> = Vec::new();
for activity_id in entry.activities.keys() {
let matches = if let Some(rest) = activity_id.strip_prefix(&run_prefix) {
rest.split("::").any(|s| s == node_id)
} else {
activity_id == node_id
};
if matches {
to_remove.push(activity_id.clone());
}
}
for aid in &to_remove {
if let Some(t) = entry.activities.remove(aid) {
t.cancel();
cancelled.push(aid.clone());
}
}
if entry.activities.is_empty() && entry.nodes.is_empty() {
runs.remove(run_id);
}
cancelled
}
pub fn cancel_run(&self, run_id: &str) -> Vec<String> {
let mut runs = self.inner.runs.write().expect("registry lock poisoned");
let Some(entry) = runs.remove(run_id) else {
return Vec::new();
};
let cancelled: Vec<String> = entry.activities.keys().cloned().collect();
for (_, token) in entry.activities {
token.cancel();
}
for (_, token) in entry.nodes {
token.cancel();
}
cancelled
}
pub fn lookup_activity(&self, run_id: &str, activity_id: &str) -> Option<CancellationToken> {
let runs = self.inner.runs.read().expect("registry lock poisoned");
runs.get(run_id)?.activities.get(activity_id).cloned()
}
pub fn active_activity_ids(&self, run_id: &str) -> Vec<String> {
let runs = self.inner.runs.read().expect("registry lock poisoned");
runs.get(run_id)
.map(|e| e.activities.keys().cloned().collect())
.unwrap_or_default()
}
pub fn total_activities(&self) -> usize {
let runs = self.inner.runs.read().expect("registry lock poisoned");
runs.values().map(|e| e.activities.len()).sum()
}
pub fn total_nodes(&self) -> usize {
let runs = self.inner.runs.read().expect("registry lock poisoned");
runs.values().map(|e| e.nodes.len()).sum()
}
}
impl Default for WorkflowCancellationRegistry {
fn default() -> Self {
Self::new()
}
}
pub struct ActivityTokenGuard {
registry: WorkflowCancellationRegistry,
run_id: String,
activity_id: String,
pub token: CancellationToken,
}
impl ActivityTokenGuard {
pub fn register(
registry: &WorkflowCancellationRegistry,
run_id: &str,
activity_id: &str,
) -> Self {
let token = registry.register_activity(run_id, activity_id);
Self {
registry: registry.clone(),
run_id: run_id.to_string(),
activity_id: activity_id.to_string(),
token,
}
}
}
impl Drop for ActivityTokenGuard {
fn drop(&mut self) {
self.registry
.unregister_activity(&self.run_id, &self.activity_id);
}
}
pub fn global_cancellation_registry() -> &'static WorkflowCancellationRegistry {
static REGISTRY: std::sync::OnceLock<WorkflowCancellationRegistry> = std::sync::OnceLock::new();
REGISTRY.get_or_init(WorkflowCancellationRegistry::new)
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;
#[test]
fn register_and_lookup_activity() {
let reg = WorkflowCancellationRegistry::new();
let token = reg.register_activity("run-1", "run-1::a");
assert!(
reg.lookup_activity("run-1", "run-1::a").is_some(),
"token should be findable after register"
);
assert!(!token.is_cancelled());
assert_eq!(reg.total_activities(), 1);
}
#[test]
fn unregister_removes_token() {
let reg = WorkflowCancellationRegistry::new();
let _token = reg.register_activity("run-1", "run-1::a");
assert_eq!(reg.total_activities(), 1);
reg.unregister_activity("run-1", "run-1::a");
assert_eq!(reg.total_activities(), 0);
assert!(reg.lookup_activity("run-1", "run-1::a").is_none());
}
#[test]
fn unregister_is_idempotent() {
let reg = WorkflowCancellationRegistry::new();
let _token = reg.register_activity("run-1", "run-1::a");
reg.unregister_activity("run-1", "run-1::a");
reg.unregister_activity("run-1", "run-1::a"); assert_eq!(reg.total_activities(), 0);
}
#[test]
fn cancel_activity_cancels_token() {
let reg = WorkflowCancellationRegistry::new();
let token = reg.register_activity("run-1", "run-1::a");
assert!(!token.is_cancelled());
let found = reg.cancel_activity("run-1", "run-1::a");
assert!(found, "cancel_activity should return true");
assert!(token.is_cancelled(), "token should be cancelled");
assert_eq!(reg.total_activities(), 0);
}
#[test]
fn cancel_activity_nonexistent_returns_false() {
let reg = WorkflowCancellationRegistry::new();
let found = reg.cancel_activity("run-1", "run-1::ghost");
assert!(!found);
}
#[test]
fn cancel_activity_leaves_other_activities_untouched() {
let reg = WorkflowCancellationRegistry::new();
let token_a = reg.register_activity("run-1", "run-1::a");
let token_b = reg.register_activity("run-1", "run-1::b");
reg.cancel_activity("run-1", "run-1::a");
assert!(token_a.is_cancelled());
assert!(!token_b.is_cancelled());
assert_eq!(reg.total_activities(), 1);
assert!(reg.lookup_activity("run-1", "run-1::b").is_some());
}
#[test]
fn cancel_node_cancels_node_token_and_activity_with_same_node_id() {
let reg = WorkflowCancellationRegistry::new();
let node_token = reg.register_node("run-1", "node-a");
let act_token = reg.register_activity("run-1", "run-1::node-a");
let other_token = reg.register_activity("run-1", "run-1::node-b");
let cancelled = reg.cancel_node("run-1", "node-a");
assert_eq!(cancelled.len(), 1);
assert_eq!(cancelled[0], "run-1::node-a");
assert!(node_token.is_cancelled());
assert!(act_token.is_cancelled());
assert!(!other_token.is_cancelled(), "node-b should be untouched");
assert_eq!(reg.total_activities(), 1);
}
#[test]
fn cancel_node_matches_activity_id_with_node_segment() {
let reg = WorkflowCancellationRegistry::new();
let _node_tok = reg.register_node("run-1", "loop-1");
let child_a = reg.register_activity("run-1", "run-1::loop-1::work::step-a");
let child_b = reg.register_activity("run-1", "run-1::loop-1::work::step-b");
let unrelated = reg.register_activity("run-1", "run-1::other-node");
let cancelled = reg.cancel_node("run-1", "loop-1");
assert_eq!(cancelled.len(), 2);
assert!(cancelled.contains(&"run-1::loop-1::work::step-a".to_string()));
assert!(cancelled.contains(&"run-1::loop-1::work::step-b".to_string()));
assert!(child_a.is_cancelled());
assert!(child_b.is_cancelled());
assert!(!unrelated.is_cancelled());
assert_eq!(reg.total_activities(), 1);
}
#[test]
fn cancel_node_matches_actual_gate_and_work_ids() {
let reg = WorkflowCancellationRegistry::new();
let gate_tok = reg.register_activity("run-x", "run-x::gate::approve");
let work_tok = reg.register_activity("run-x", "run-x::work::approve");
let other = reg.register_activity("run-x", "run-x::work::unrelated");
let cancelled = reg.cancel_node("run-x", "approve");
assert_eq!(cancelled.len(), 2);
assert!(gate_tok.is_cancelled());
assert!(work_tok.is_cancelled());
assert!(!other.is_cancelled());
}
#[test]
fn cancel_node_nonexistent_returns_empty() {
let reg = WorkflowCancellationRegistry::new();
let cancelled = reg.cancel_node("run-1", "ghost");
assert!(cancelled.is_empty());
}
#[test]
fn cancel_run_cancels_all_tokens() {
let reg = WorkflowCancellationRegistry::new();
let t1 = reg.register_activity("run-1", "run-1::a");
let t2 = reg.register_activity("run-1", "run-1::b");
let n1 = reg.register_node("run-1", "node-a");
let cancelled = reg.cancel_run("run-1");
assert_eq!(cancelled.len(), 2);
assert!(t1.is_cancelled());
assert!(t2.is_cancelled());
assert!(n1.is_cancelled());
assert_eq!(reg.total_activities(), 0);
assert_eq!(reg.total_nodes(), 0);
}
#[test]
fn cancel_run_nonexistent_returns_empty() {
let reg = WorkflowCancellationRegistry::new();
let cancelled = reg.cancel_run("no-such-run");
assert!(cancelled.is_empty());
}
#[test]
fn cancel_run_only_affects_target_run() {
let reg = WorkflowCancellationRegistry::new();
let t1 = reg.register_activity("run-1", "run-1::a");
let t2 = reg.register_activity("run-2", "run-2::a");
reg.cancel_run("run-1");
assert!(t1.is_cancelled());
assert!(!t2.is_cancelled());
assert_eq!(reg.total_activities(), 1);
assert_eq!(reg.active_activity_ids("run-2"), vec!["run-2::a"]);
}
#[test]
fn active_activity_ids_returns_registered_ids() {
let reg = WorkflowCancellationRegistry::new();
reg.register_activity("run-1", "run-1::a");
reg.register_activity("run-1", "run-1::b");
reg.register_activity("run-2", "run-2::x");
let mut ids = reg.active_activity_ids("run-1");
ids.sort();
assert_eq!(ids, vec!["run-1::a", "run-1::b"]);
let ids2 = reg.active_activity_ids("run-2");
assert_eq!(ids2, vec!["run-2::x"]);
assert!(reg.active_activity_ids("run-3").is_empty());
}
#[tokio::test]
async fn concurrent_cancel_signals_active_dispatch() {
let reg = WorkflowCancellationRegistry::new();
let token = reg.register_activity("run-1", "run-1::slow");
let reg_clone = reg.clone();
let handle = tokio::spawn(async move {
tokio::select! {
_ = token.cancelled() => {
"cancelled"
}
_ = tokio::time::sleep(Duration::from_secs(5)) => {
"timeout"
}
}
});
tokio::time::sleep(Duration::from_millis(20)).await;
reg_clone.cancel_run("run-1");
let result = tokio::time::timeout(Duration::from_secs(2), handle)
.await
.expect("dispatch should finish quickly")
.expect("dispatch should not panic");
assert_eq!(result, "cancelled");
}
#[tokio::test]
async fn active_dispatch_observes_cancellation_after_cancel_requested() {
let reg = WorkflowCancellationRegistry::new();
let token = reg.register_activity("run-cancel-test", "run-cancel-test::work");
assert!(!token.is_cancelled());
assert_eq!(reg.total_activities(), 1);
let token_clone = token.clone();
let dispatch_handle = tokio::spawn(async move {
loop {
if token_clone.is_cancelled() {
return "dispatched_cancelled";
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
});
tokio::time::sleep(Duration::from_millis(10)).await;
let cancelled_ids = reg.cancel_run("run-cancel-test");
assert!(
!cancelled_ids.is_empty(),
"should have cancelled at least one activity"
);
assert!(
token.is_cancelled(),
"token must be cancelled after cancel_run"
);
let outcome = tokio::time::timeout(Duration::from_secs(2), dispatch_handle)
.await
.expect("dispatch should finish quickly")
.expect("dispatch should not panic");
assert_eq!(outcome, "dispatched_cancelled");
assert_eq!(reg.total_activities(), 0);
}
#[tokio::test]
async fn cancel_activity_targets_only_specified_activity() {
let reg = WorkflowCancellationRegistry::new();
let token_a = reg.register_activity("run-1", "run-1::a");
let token_b = reg.register_activity("run-1", "run-1::b");
let reg_clone = reg.clone();
let token_a_clone = token_a.clone();
let token_b_clone = token_b.clone();
let handle_a = tokio::spawn(async move {
tokio::select! {
_ = token_a_clone.cancelled() => "a_cancelled",
_ = tokio::time::sleep(Duration::from_secs(5)) => "a_timeout",
}
});
let handle_b = tokio::spawn(async move {
tokio::select! {
_ = token_b_clone.cancelled() => "b_cancelled",
_ = tokio::time::sleep(Duration::from_millis(200)) => "b_completed",
}
});
tokio::time::sleep(Duration::from_millis(10)).await;
reg_clone.cancel_activity("run-1", "run-1::a");
let result_a = tokio::time::timeout(Duration::from_secs(2), handle_a)
.await
.unwrap()
.unwrap();
let result_b = tokio::time::timeout(Duration::from_secs(2), handle_b)
.await
.unwrap()
.unwrap();
assert_eq!(result_a, "a_cancelled", "activity a should be cancelled");
assert_eq!(
result_b, "b_completed",
"activity b should complete normally"
);
assert!(token_a.is_cancelled());
assert!(!token_b.is_cancelled());
}
#[tokio::test]
async fn node_cancel_propagates_to_children() {
let reg = WorkflowCancellationRegistry::new();
let child_token = reg.register_activity("run-1", "run-1::parent-node::work::child");
let sibling_token = reg.register_activity("run-1", "run-1::other");
let child_clone = child_token.clone();
let sibling_clone = sibling_token.clone();
let handle_child = tokio::spawn(async move {
tokio::select! {
_ = child_clone.cancelled() => "child_cancelled",
_ = tokio::time::sleep(Duration::from_secs(5)) => "child_timeout",
}
});
let handle_sibling = tokio::spawn(async move {
tokio::select! {
_ = sibling_clone.cancelled() => "sibling_cancelled",
_ = tokio::time::sleep(Duration::from_millis(200)) => "sibling_ok",
}
});
tokio::time::sleep(Duration::from_millis(10)).await;
let cancelled = reg.cancel_node("run-1", "parent-node");
assert!(!cancelled.is_empty());
let child_result = tokio::time::timeout(Duration::from_secs(2), handle_child)
.await
.unwrap()
.unwrap();
let sibling_result = tokio::time::timeout(Duration::from_secs(2), handle_sibling)
.await
.unwrap()
.unwrap();
assert_eq!(child_result, "child_cancelled");
assert_eq!(sibling_result, "sibling_ok");
}
#[test]
fn guard_registers_and_auto_unregisters_on_drop() {
let reg = WorkflowCancellationRegistry::new();
assert_eq!(reg.total_activities(), 0);
{
let guard = ActivityTokenGuard::register(®, "run-1", "run-1::a");
assert_eq!(reg.total_activities(), 1);
assert!(!guard.token.is_cancelled());
}
assert_eq!(reg.total_activities(), 0);
}
#[test]
fn guard_token_is_independent_from_registry_clone() {
let reg = WorkflowCancellationRegistry::new();
let reg2 = reg.clone();
let guard = ActivityTokenGuard::register(®, "run-1", "run-1::a");
assert_eq!(reg2.total_activities(), 1);
reg2.cancel_run("run-1");
assert!(guard.token.is_cancelled());
drop(guard);
assert_eq!(reg.total_activities(), 0);
}
#[tokio::test]
async fn hooks_integration_guard_register_dispatch_observes_cancel() {
let reg = WorkflowCancellationRegistry::new();
let guard = ActivityTokenGuard::register(®, "run-hooks", "run-hooks::work");
let token = guard.token.clone();
let reg_clone = reg.clone();
let handle = tokio::spawn(async move {
loop {
if token.is_cancelled() {
return "cancelled_by_token";
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
});
tokio::time::sleep(Duration::from_millis(20)).await;
let count = reg_clone.cancel_run("run-hooks").len();
assert_eq!(count, 1, "should have cancelled 1 activity");
let result = tokio::time::timeout(Duration::from_secs(2), handle)
.await
.unwrap()
.unwrap();
assert_eq!(result, "cancelled_by_token");
drop(guard);
assert_eq!(reg.total_activities(), 0);
}
#[test]
fn guard_unregisters_on_early_exit() {
let reg = WorkflowCancellationRegistry::new();
fn simulate_dispatch(reg: &WorkflowCancellationRegistry, should_fail: bool) -> bool {
let guard = ActivityTokenGuard::register(reg, "run-1", "run-1::a");
if should_fail {
return false; }
let _cancelled = guard.token.is_cancelled();
true }
assert!(simulate_dispatch(®, false));
assert_eq!(reg.total_activities(), 0);
assert!(!simulate_dispatch(®, true));
assert_eq!(reg.total_activities(), 0);
}
#[tokio::test]
async fn cancellable_wait_yields_via_token_select() {
let reg = WorkflowCancellationRegistry::new();
let token = reg.register_activity("run-select", "run-select::poll");
let token_clone = token.clone();
let reg_clone = reg.clone();
let handle = tokio::spawn(async move {
tokio::select! {
_ = token_clone.cancelled() => "cancelled",
_ = tokio::time::sleep(Duration::from_secs(30)) => "timeout",
}
});
tokio::time::sleep(Duration::from_millis(10)).await;
reg_clone.cancel_run("run-select");
let result = tokio::time::timeout(Duration::from_secs(2), handle)
.await
.unwrap()
.unwrap();
assert_eq!(result, "cancelled");
}
#[tokio::test]
async fn full_hooks_integration_register_cancel_detect() {
let reg = WorkflowCancellationRegistry::new();
let guard = ActivityTokenGuard::register(®, "run-full", "run-full::work");
assert_eq!(reg.total_activities(), 1);
let reg_clone = reg.clone();
let dispatch_token = guard.token.clone();
let handle = tokio::spawn(async move {
if dispatch_token.is_cancelled() {
return "cancelled_early";
}
for _ in 0..100 {
if dispatch_token.is_cancelled() {
return "cancelled_during_work";
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
"completed"
});
tokio::time::sleep(Duration::from_millis(15)).await;
let cancelled_ids = reg_clone.cancel_run("run-full");
assert!(!cancelled_ids.is_empty());
let outcome = tokio::time::timeout(Duration::from_secs(5), handle)
.await
.unwrap()
.unwrap();
assert!(
outcome.starts_with("cancelled"),
"expected cancelled, got: {outcome}"
);
assert!(guard.token.is_cancelled());
drop(guard);
assert_eq!(reg.total_activities(), 0);
}
#[derive(Clone, Default)]
struct SignalTrace {
calls: std::sync::Arc<std::sync::Mutex<Vec<(u32, i32)>>>,
}
impl SignalTrace {
fn record(&self, pid: u32, signal: i32) {
self.calls.lock().unwrap().push((pid, signal));
}
fn snapshot(&self) -> Vec<(u32, i32)> {
self.calls.lock().unwrap().clone()
}
}
struct MockWorker {
pid: u32,
trace: SignalTrace,
}
impl MockWorker {
fn new(pid: u32, trace: SignalTrace) -> Self {
Self { pid, trace }
}
fn send_signal(&self, signal: i32) {
self.trace.record(self.pid, signal);
}
}
async fn escalate_signals<F>(
worker: &MockWorker,
grace: Duration,
poll_interval: Duration,
mut is_alive_after: F,
) where
F: FnMut() -> bool,
{
worker.send_signal(libc::SIGINT);
let deadline = tokio::time::Instant::now() + grace;
let mut exited = false;
while tokio::time::Instant::now() < deadline {
if !is_alive_after() {
exited = true;
break;
}
tokio::time::sleep(poll_interval).await;
}
if !exited {
worker.send_signal(libc::SIGKILL);
}
}
#[tokio::test]
async fn signal_escalation_sends_sigint_then_sigkill_when_process_ignores_sigint() {
let trace = SignalTrace::default();
let worker = MockWorker::new(42, trace.clone());
escalate_signals(
&worker,
Duration::from_millis(100), Duration::from_millis(10),
|| true, )
.await;
let calls = trace.snapshot();
assert_eq!(calls.len(), 2, "expected 2 signals: SIGINT then SIGKILL");
assert_eq!(
calls[0],
(42, libc::SIGINT),
"first signal should be SIGINT"
);
assert_eq!(
calls[1],
(42, libc::SIGKILL),
"second signal should be SIGKILL (process ignored SIGINT)"
);
}
#[tokio::test]
async fn signal_escalation_sends_only_sigint_when_process_exits_promptly() {
let trace = SignalTrace::default();
let worker = MockWorker::new(99, trace.clone());
let mut poll_count = 0;
escalate_signals(
&worker,
Duration::from_secs(5), Duration::from_millis(10),
|| {
poll_count += 1;
poll_count < 2
},
)
.await;
let calls = trace.snapshot();
assert_eq!(
calls.len(),
1,
"expected only SIGINT – process exited before grace expired"
);
assert_eq!(calls[0], (99, libc::SIGINT));
}
#[tokio::test]
async fn signal_escalation_grace_period_is_respected() {
let trace = SignalTrace::default();
let worker = MockWorker::new(7, trace.clone());
let start = tokio::time::Instant::now();
escalate_signals(
&worker,
Duration::from_millis(200),
Duration::from_millis(50),
|| true, )
.await;
let elapsed = start.elapsed();
assert!(
elapsed >= Duration::from_millis(180),
"escalation should respect grace period of 200ms, got {:?}",
elapsed
);
assert!(
elapsed < Duration::from_millis(500),
"escalation should not hang indefinitely, got {:?}",
elapsed
);
let calls = trace.snapshot();
assert_eq!(calls.len(), 2, "SIGINT + SIGKILL since process never died");
}
}