Skip to main content

adk_graph/
executor.rs

1//! Pregel-based execution engine for graphs
2//!
3//! Executes graphs using the Pregel model with super-steps.
4
5#[cfg(feature = "node-cache")]
6use crate::cache::compute_cache_key;
7use crate::deferred::{FAN_IN_METADATA_KEY, FanInSnapshot, FanInTracker};
8use crate::error::{GraphError, InterruptedExecution, Result};
9use crate::graph::CompiledGraph;
10use crate::interrupt::{GraphToolConfirmationPause, Interrupt};
11use crate::node::{ExecutionConfig, NodeContext};
12use crate::state::{Checkpoint, State};
13use crate::stream::{StreamEvent, StreamMode};
14use crate::timeout::{OnTimeout, ProgressHandle, execute_with_timeout, item_timeout_budget};
15use futures::stream::{self, StreamExt};
16use std::collections::HashMap;
17use std::sync::Arc;
18use std::time::Instant;
19
20/// Result of a super-step execution
21#[derive(Default)]
22pub struct SuperStepResult {
23    /// Nodes that were executed
24    pub executed_nodes: Vec<String>,
25    /// Interrupt if one occurred
26    pub interrupt: Option<Interrupt>,
27    /// Stream events generated
28    pub events: Vec<StreamEvent>,
29    /// Nodes that named their own successors, keyed by node name.
30    pub goto: HashMap<String, Vec<String>>,
31}
32
33/// What a completed run produced, and what it asks of its caller.
34#[derive(Debug, Clone)]
35pub struct GraphOutcome {
36    /// The final state.
37    pub state: State,
38    /// Nodes of the parent graph a node asked to run next, if any.
39    ///
40    /// Set by [`NodeOutput::with_goto_parent`](crate::node::NodeOutput::with_goto_parent).
41    /// A graph that is not a subgraph has no parent, so this is ignored.
42    pub goto_parent: Option<Vec<String>>,
43}
44
45/// Pregel-based executor for graphs
46pub struct PregelExecutor<'a> {
47    graph: &'a CompiledGraph,
48    config: ExecutionConfig,
49    /// ADK configuration supplied through an additive direct-graph API.
50    run_config: Option<adk_core::RunConfig>,
51    state: State,
52    step: usize,
53    pending_nodes: Vec<String>,
54    /// Parent nodes a node asked to run next; see `NodeOutput::with_goto_parent`.
55    goto_parent: Option<Vec<String>>,
56    /// Tracks deferred nodes waiting for all upstream paths to complete.
57    pending_deferred: HashMap<String, FanInTracker>,
58    /// Tracks when each deferred node first entered the pending state (for fan-in timeout).
59    deferred_start_times: HashMap<String, Instant>,
60    /// Attempts already spent per node, carried through a resume so a retry
61    /// budget is not restarted.
62    attempts: HashMap<String, u32>,
63    /// Outputs of children invoked imperatively, keyed by child path. Shared with
64    /// every node's invoker so a resumed parent serves finished children from it.
65    child_ledger: Arc<std::sync::Mutex<HashMap<String, serde_json::Value>>>,
66    /// The node whose static interrupt this run has already answered.
67    ///
68    /// Restored from the checkpoint on resume and cleared once that node has
69    /// executed, so the gate re-arms for a later arrival through a cycle.
70    cleared_interrupt: Option<String>,
71    /// Whether a thread whose last run finished starts again from its entry
72    /// nodes instead of returning the finished state.
73    restart_completed: bool,
74    /// The step this run started at; the recursion limit counts from here.
75    run_start_step: usize,
76}
77
78impl<'a> PregelExecutor<'a> {
79    /// Create a new executor
80    pub fn new(graph: &'a CompiledGraph, config: ExecutionConfig) -> Self {
81        Self::new_with_run_config(graph, config, None)
82    }
83
84    pub(crate) fn new_with_run_config(
85        graph: &'a CompiledGraph,
86        config: ExecutionConfig,
87        run_config: Option<adk_core::RunConfig>,
88    ) -> Self {
89        Self {
90            graph,
91            config,
92            run_config,
93            state: State::new(),
94            step: 0,
95            pending_nodes: vec![],
96            goto_parent: None,
97            pending_deferred: HashMap::new(),
98            deferred_start_times: HashMap::new(),
99            attempts: HashMap::new(),
100            child_ledger: Arc::new(std::sync::Mutex::new(HashMap::new())),
101            cleared_interrupt: None,
102            restart_completed: false,
103            run_start_step: 0,
104        }
105    }
106
107    /// Starts a thread whose last run finished again from its entry nodes.
108    ///
109    /// Used by [`GraphAgent`](crate::agent::GraphAgent), where every call is a
110    /// new turn on the session's thread. The finished state carries over and the
111    /// new input is merged on top; a paused thread still resumes where it paused.
112    pub(crate) fn restarting_completed_threads(mut self) -> Self {
113        self.restart_completed = true;
114        self
115    }
116
117    /// Attempt to resume from an existing checkpoint.
118    ///
119    /// If a checkpoint is found (either by explicit `resume_from` ID or by latest
120    /// checkpoint for the thread), restores state, pending_nodes, and step from it,
121    /// then merges the provided input on top. Returns `true` if resumed.
122    ///
123    /// If no checkpoint is found, returns `false` so the caller can proceed with
124    /// fresh-start logic.
125    async fn try_resume_from_checkpoint(&mut self, input: &State) -> Result<bool> {
126        let checkpoint = if let Some(checkpoint_id) = &self.config.resume_from {
127            // Resume from a specific checkpoint by ID
128            if let Some(cp) = self.graph.checkpointer.as_ref() {
129                cp.load_by_id(checkpoint_id).await?
130            } else {
131                None
132            }
133        } else if let Some(cp) = self.graph.checkpointer.as_ref() {
134            // Try to load the latest checkpoint for this thread
135            cp.load(&self.config.thread_id).await?
136        } else {
137            None
138        };
139
140        if let Some(checkpoint) = checkpoint {
141            let finished = checkpoint.pending_nodes.is_empty();
142            self.state = checkpoint.state;
143            self.step = checkpoint.step;
144
145            if finished && self.restart_completed && self.config.resume_from.is_none() {
146                // A new turn on a finished thread. The retry budget, child ledger,
147                // gate marker, and fan-in arrivals belonged to the finished run,
148                // so none of them carry over; only the state does.
149                self.pending_nodes = self.graph.get_entry_nodes();
150                self.run_start_step = checkpoint.step;
151            } else {
152                self.pending_nodes = checkpoint.pending_nodes;
153                self.cleared_interrupt = checkpoint.cleared_interrupt;
154                self.attempts = checkpoint.attempts;
155                *self.child_ledger.lock().expect("child ledger") = checkpoint.child_ledger;
156                if let Some(snapshot) = FanInSnapshot::from_metadata(&checkpoint.metadata)? {
157                    let (trackers, started) =
158                        snapshot.restore(|node| self.graph.get_upstream_nodes(node));
159                    self.pending_deferred = trackers;
160                    self.deferred_start_times = started;
161                }
162            }
163
164            // Merge input on top of restored state
165            for (key, value) in input {
166                self.graph.schema.apply_update(&mut self.state, key, value.clone());
167            }
168
169            Ok(true)
170        } else {
171            Ok(false)
172        }
173    }
174
175    /// Run the graph to completion
176    pub async fn run(&mut self, input: State) -> Result<State> {
177        // Check for existing checkpoint to resume from
178        let resumed = self.try_resume_from_checkpoint(&input).await?;
179
180        if !resumed {
181            // No checkpoint found — fresh start
182            self.state = self.initialize_state(input).await?;
183            self.pending_nodes = self.graph.get_entry_nodes();
184        }
185
186        // Main execution loop
187        while !self.pending_nodes.is_empty() {
188            // Check recursion limit
189            let steps_taken = self.step.saturating_sub(self.run_start_step);
190            if steps_taken >= self.config.recursion_limit {
191                return Err(GraphError::RecursionLimitExceeded(steps_taken));
192            }
193
194            // Execute super-step
195            let result = match self.execute_super_step().await {
196                Ok(result) => result,
197                Err(error) => {
198                    // Checkpoint before propagating, so a retry budget already
199                    // spent is not handed out again by the next invocation. The
200                    // frontier still holds the failed node, which is what makes
201                    // the run resumable at all.
202                    let any_retryable = self
203                        .pending_nodes
204                        .iter()
205                        .any(|node| self.graph.retry_policy_for(node).is_some());
206                    if any_retryable && let Err(save_error) = self.save_checkpoint().await {
207                        tracing::warn!(
208                            thread_id = %self.config.thread_id,
209                            step = self.step,
210                            error = %save_error,
211                            "failed to checkpoint a failing super-step; a resumed run restarts \
212                             its retry budget"
213                        );
214                    }
215                    return Err(error);
216                }
217            };
218
219            // Handle interrupts
220            if let Some(interrupt) = result.interrupt {
221                self.pause_frontier(&interrupt, &result.executed_nodes, &result.goto)?;
222                let checkpoint_id = self.save_checkpoint().await?;
223                return Err(GraphError::Interrupted(Box::new(InterruptedExecution::new(
224                    self.config.thread_id.clone(),
225                    checkpoint_id,
226                    interrupt,
227                    self.state.clone(),
228                    self.step,
229                ))));
230            }
231
232            // The gate re-arms once its node has run, so a cycle returning to
233            // the same node asks again.
234            if let Some(cleared) = &self.cleared_interrupt
235                && result.executed_nodes.iter().any(|n| n == cleared)
236            {
237                self.cleared_interrupt = None;
238            }
239
240            // Advance the frontier *before* checkpointing. A checkpoint records
241            // what still has to run, so saving while `pending_nodes` still holds
242            // the nodes that just finished would re-execute them on resume.
243            let next_candidates = self.next_frontier(&result.executed_nodes, &result.goto)?;
244            self.pending_nodes =
245                self.filter_deferred_nodes(next_candidates, &result.executed_nodes)?;
246            self.step += 1;
247
248            // An empty frontier is a terminal checkpoint: resuming it re-reads
249            // the final state instead of restarting the graph.
250            self.save_checkpoint().await?;
251
252            if self.pending_nodes.is_empty() {
253                break;
254            }
255        }
256
257        Ok(self.state.clone())
258    }
259
260    /// Run with streaming
261    pub fn run_stream(
262        mut self,
263        input: State,
264        mode: StreamMode,
265    ) -> impl futures::Stream<Item = Result<StreamEvent>> + 'a {
266        async_stream::stream! {
267            // Check for existing checkpoint to resume from
268            let resumed = match self.try_resume_from_checkpoint(&input).await {
269                Ok(r) => r,
270                Err(e) => {
271                    yield Err(e);
272                    return;
273                }
274            };
275
276            if resumed {
277                // Emit a resumed event indicating execution was restored from checkpoint
278                yield Ok(StreamEvent::resumed(self.step, self.pending_nodes.clone()));
279            } else {
280                // No checkpoint found — fresh start
281                match self.initialize_state(input).await {
282                    Ok(state) => self.state = state,
283                    Err(e) => {
284                        yield Err(e);
285                        return;
286                    }
287                }
288                self.pending_nodes = self.graph.get_entry_nodes();
289            }
290
291            // Stream initial state if requested
292            if matches!(mode, StreamMode::Values) {
293                yield Ok(StreamEvent::state(self.state.clone(), self.step));
294            }
295
296            // Main execution loop
297            while !self.pending_nodes.is_empty() {
298                // Check recursion limit
299                let steps_taken = self.step.saturating_sub(self.run_start_step);
300                if steps_taken >= self.config.recursion_limit {
301                    yield Err(GraphError::RecursionLimitExceeded(steps_taken));
302                    return;
303                }
304
305                // Emit node_start events BEFORE execution (in Debug mode)
306                if matches!(mode, StreamMode::Debug | StreamMode::Custom | StreamMode::Messages) {
307                    for node_name in &self.pending_nodes {
308                        yield Ok(StreamEvent::node_start(node_name, self.step));
309                    }
310                }
311
312                // For Messages mode, stream from nodes directly
313                if matches!(mode, StreamMode::Messages) {
314                    let mut result = SuperStepResult::default();
315
316                    // The same gate `execute_super_step` applies. This loop does
317                    // not call it, so without this the mode ignored every gate.
318                    if let Some(interrupt) = self.gate_before(&self.pending_nodes) {
319                        result.interrupt = Some(interrupt);
320                    }
321
322                    for node_name in &self.pending_nodes {
323                        if result.interrupt.is_some() {
324                            break;
325                        }
326                        if let Some(node) = self.graph.nodes.get(node_name) {
327                            let mut ctx = NodeContext::new(self.state.clone(), self.config.clone(), self.step);
328                            if let Some(run_config) = self.run_config.clone() {
329                                ctx.set_run_config(run_config);
330                            }
331                            ctx.set_parent_schema(Arc::new(self.graph.schema.clone()));
332                            ctx.set_child_invoker(Arc::new(crate::child::ChildInvoker::new(
333                                self.graph.nodes.clone(),
334                                Arc::clone(&self.child_ledger),
335                                node_name.clone(),
336                            )));
337
338                            // Attach progress handle if idle timeout is configured
339                            let policy = self.graph.timeout_policy_for(node_name).cloned();
340                            if let Some(ref p) = policy
341                                && p.idle_timeout.is_some() {
342                                    ctx.set_progress_handle(ProgressHandle::new());
343                                }
344
345                            let start = std::time::Instant::now();
346
347                            // The timeout policy now applies to the streamed
348                            // execution itself. For a stream, "idle" means no
349                            // event was produced within the idle timeout.
350                            let max_attempts = match policy.as_ref().map(|p| &p.on_timeout) {
351                                Some(OnTimeout::Retry { max_attempts }) => (*max_attempts).max(1),
352                                _ => 1,
353                            };
354                            let mut collected_events = Vec::new();
355                            let mut streamed_updates = Vec::new();
356                            let mut streamed_goto: Option<(String, Vec<String>)> = None;
357                            let mut streamed_interrupt: Option<Interrupt> = None;
358                            let mut timed_out_after;
359                            let mut attempt = 0;
360
361                            loop {
362                                attempt += 1;
363                                collected_events.clear();
364                                streamed_updates.clear();
365                                timed_out_after = None;
366                                let attempt_start = std::time::Instant::now();
367                                let mut node_stream = node.execute_stream(&ctx);
368                                let mut failure = None;
369
370                                loop {
371                                    let budget = policy
372                                        .as_ref()
373                                        .and_then(|p| item_timeout_budget(p, attempt_start.elapsed()));
374                                    let item = match budget {
375                                        Some(budget) => {
376                                            match tokio::time::timeout(budget, node_stream.next()).await {
377                                                Ok(item) => item,
378                                                Err(_) => {
379                                                    timed_out_after = Some(attempt_start.elapsed());
380                                                    break;
381                                                }
382                                            }
383                                        }
384                                        None => node_stream.next().await,
385                                    };
386
387                                    match item {
388                                        Some(Ok(event)) => {
389                                            // Yield Message events immediately
390                                            if matches!(event, StreamEvent::Message { .. }) {
391                                                yield Ok(event.clone());
392                                            }
393                                            // The node reports its state updates on the
394                                            // stream, so they are taken from the single
395                                            // execution that produced these events.
396                                            if let StreamEvent::Updates { ref updates, .. } = event {
397                                                streamed_updates.push(updates.clone());
398                                            }
399                                            // A node that routed itself reports it here.
400                                            if let StreamEvent::RouteDispatched {
401                                                ref source,
402                                                ref targets,
403                                            } = event
404                                            {
405                                                streamed_goto =
406                                                    Some((source.clone(), targets.clone()));
407                                            }
408                                            // As does a node asking to pause.
409                                            if let StreamEvent::NodeInterrupt {
410                                                ref message,
411                                                ref data,
412                                                ..
413                                            } = event
414                                            {
415                                                streamed_interrupt =
416                                                    Some(Interrupt::Dynamic {
417                                                        message: message.clone(),
418                                                        data: data.clone(),
419                                                    });
420                                            }
421                                            collected_events.push(event);
422                                        }
423                                        Some(Err(e)) => {
424                                            failure = Some(e);
425                                            break;
426                                        }
427                                        None => break,
428                                    }
429                                }
430                                drop(node_stream);
431
432                                if let Some(e) = failure {
433                                    yield Err(e);
434                                    return;
435                                }
436                                if timed_out_after.is_none() || attempt >= max_attempts {
437                                    break;
438                                }
439                            }
440
441                            if let Some(elapsed) = timed_out_after {
442                                let on_timeout =
443                                    policy.as_ref().map(|p| p.on_timeout.clone()).unwrap_or_default();
444                                match on_timeout {
445                                    OnTimeout::Skip => {
446                                        tracing::warn!(
447                                            node = %node_name,
448                                            elapsed = ?elapsed,
449                                            "node timed out while streaming, skipping"
450                                        );
451                                        streamed_updates.clear();
452                                    }
453                                    OnTimeout::Fail | OnTimeout::Retry { .. } => {
454                                        yield Err(GraphError::NodeTimedOut {
455                                            node: node_name.clone(),
456                                            elapsed,
457                                        });
458                                        return;
459                                    }
460                                }
461                            }
462
463                            let duration_ms = start.elapsed().as_millis() as u64;
464                            result.events.push(StreamEvent::node_end(node_name, self.step, duration_ms));
465                            result.events.extend(collected_events);
466
467                            let node_interrupt = streamed_interrupt;
468                            if let Some(interrupt) = node_interrupt {
469                                result.interrupt = Some(interrupt);
470                            } else {
471                                result.executed_nodes.push(node_name.clone());
472                                if let Some((source, targets)) = streamed_goto {
473                                    result.goto.insert(source, targets);
474                                }
475                                for updates in streamed_updates {
476                                    self.ensure_channels_declared(
477                                        node_name,
478                                        updates.keys().map(String::as_str),
479                                    )?;
480                                    for (key, value) in updates {
481                                        self.graph.schema.apply_update(&mut self.state, &key, value);
482                                    }
483                                }
484                            }
485                        }
486                    }
487
488                    // Yield node_end events
489                    for event in &result.events {
490                        if matches!(event, StreamEvent::NodeEnd { .. }) {
491                            yield Ok(event.clone());
492                        }
493                    }
494
495                    // A node that arms a gate on completion stops the run here, unless
496                    // it already asked to pause itself.
497                    if result.interrupt.is_none()
498                        && let Some(interrupt) = self.gate_after(&result.executed_nodes)
499                    {
500                        result.interrupt = Some(interrupt);
501                    }
502
503                    // This branch returns rather than falling through to the shared
504                    // handling below, so the pause is reported here.
505                    if let Some(interrupt) = result.interrupt {
506                        if let Err(error) =
507                            self.pause_frontier(&interrupt, &result.executed_nodes, &result.goto)
508                        {
509                            yield Err(error);
510                            return;
511                        }
512                        // Persist before reporting: without this the pause cannot be
513                        // resumed and the work already done is lost.
514                        let checkpoint_id = match self.save_checkpoint().await {
515                            Ok(checkpoint_id) => checkpoint_id,
516                            Err(error) => {
517                                yield Err(error);
518                                return;
519                            }
520                        };
521                        if let Some(pause) = GraphToolConfirmationPause::from_interrupted_execution(
522                            &InterruptedExecution::new(
523                                self.config.thread_id.clone(),
524                                checkpoint_id,
525                                interrupt.clone(),
526                                self.state.clone(),
527                                self.step,
528                            ),
529                        ) {
530                            yield Ok(StreamEvent::custom(
531                                &pause.node,
532                                GraphToolConfirmationPause::KIND,
533                                serde_json::to_value(&pause).unwrap_or(serde_json::Value::Null),
534                            ));
535                        }
536                        yield Ok(StreamEvent::interrupted(
537                            result.executed_nodes.first().map(|s| s.as_str()).unwrap_or("unknown"),
538                            &interrupt.to_string(),
539                        ));
540                        return;
541                    }
542
543                    // The gate re-arms once its node has run; see `run`.
544                    if let Some(cleared) = &self.cleared_interrupt
545                        && result.executed_nodes.iter().any(|n| n == cleared)
546                    {
547                        self.cleared_interrupt = None;
548                    }
549
550                    self.pending_nodes = {
551                        let next_candidates = self.next_frontier(&result.executed_nodes, &result.goto)?;
552                        match self.filter_deferred_nodes(next_candidates, &result.executed_nodes) {
553                            Ok(nodes) => nodes,
554                            Err(e) => {
555                                yield Err(e);
556                                return;
557                            }
558                        }
559                    };
560                    self.step += 1;
561
562                    // The other path checkpoints every super-step. This one did not,
563                    // so a run in this mode left no state to resume from and
564                    // `get_state` reported nothing.
565                    if let Err(e) = self.save_checkpoint().await {
566                        yield Err(e);
567                        return;
568                    }
569                    continue;
570                }
571
572                // Execute super-step (non-streaming)
573                let result = match self.execute_super_step().await {
574                    Ok(r) => r,
575                    Err(e) => {
576                        yield Err(e);
577                        return;
578                    }
579                };
580
581                // Yield events based on mode (node_end and custom events)
582                for event in &result.events {
583                    match (&mode, &event) {
584                        // Skip node_start since we already emitted it above
585                        (StreamMode::Custom | StreamMode::Debug, StreamEvent::NodeStart { .. }) => {}
586                        (StreamMode::Custom, _) => yield Ok(event.clone()),
587                        (StreamMode::Debug, _) => yield Ok(event.clone()),
588                        _ => {}
589                    }
590                }
591
592                // Yield state/updates
593                match mode {
594                    StreamMode::Values => {
595                        yield Ok(StreamEvent::state(self.state.clone(), self.step));
596                    }
597                    StreamMode::Updates => {
598                        yield Ok(StreamEvent::step_complete(
599                            self.step,
600                            result.executed_nodes.clone(),
601                        ));
602                    }
603                    _ => {}
604                }
605
606                // Handle interrupts
607                if let Some(interrupt) = result.interrupt {
608                    if let Err(error) =
609                        self.pause_frontier(&interrupt, &result.executed_nodes, &result.goto)
610                    {
611                        yield Err(error);
612                        return;
613                    }
614                    // Persist before reporting: without this the interrupt is
615                    // unresumable, because resuming loads the checkpoint for the
616                    // thread.
617                    let checkpoint_id = match self.save_checkpoint().await {
618                        Ok(checkpoint_id) => checkpoint_id,
619                        Err(error) => {
620                            yield Err(error);
621                            return;
622                        }
623                    };
624                    if let Some(pause) = GraphToolConfirmationPause::from_interrupted_execution(
625                        &InterruptedExecution::new(
626                            self.config.thread_id.clone(),
627                            checkpoint_id,
628                            interrupt.clone(),
629                            self.state.clone(),
630                            self.step,
631                        ),
632                    ) {
633                        yield Ok(StreamEvent::custom(
634                            &pause.node,
635                            GraphToolConfirmationPause::KIND,
636                            serde_json::to_value(&pause).unwrap_or(serde_json::Value::Null),
637                        ));
638                    }
639                    yield Ok(StreamEvent::interrupted(
640                        result.executed_nodes.first().map(|s| s.as_str()).unwrap_or("unknown"),
641                        &interrupt.to_string(),
642                    ));
643                    return;
644                }
645
646                // The gate re-arms once its node has run; see `run`.
647                if let Some(cleared) = &self.cleared_interrupt
648                    && result.executed_nodes.iter().any(|n| n == cleared)
649                {
650                    self.cleared_interrupt = None;
651                }
652
653                // Advance the frontier before checkpointing, so the checkpoint
654                // records what still has to run rather than what just finished.
655                //
656                // Reported only on the debug stream, because building it evaluates
657                // each router a second time.
658                if matches!(mode, StreamMode::Debug) {
659                    match self.graph.route_dispatches(&result.executed_nodes, &self.state) {
660                        Ok(dispatches) => {
661                            for (source, targets) in dispatches {
662                                yield Ok(StreamEvent::route_dispatched(&source, targets));
663                            }
664                        }
665                        Err(error) => {
666                            yield Err(error);
667                            return;
668                        }
669                    }
670                }
671
672                self.pending_nodes = {
673                    let next_candidates = self.next_frontier(&result.executed_nodes, &result.goto)?;
674                    match self.filter_deferred_nodes(next_candidates, &result.executed_nodes) {
675                        Ok(nodes) => nodes,
676                        Err(e) => {
677                            yield Err(e);
678                            return;
679                        }
680                    }
681                };
682                self.step += 1;
683
684                if let Err(e) = self.save_checkpoint().await {
685                    yield Err(e);
686                    return;
687                }
688            }
689
690            yield Ok(StreamEvent::done(self.state.clone(), self.step + 1));
691        }
692    }
693
694    /// Filter deferred nodes from the next candidates.
695    ///
696    /// For each candidate node that is configured as deferred, check whether all
697    /// upstream paths have completed. If not, hold the node in `pending_deferred`
698    /// and record the outputs from the just-executed nodes. If all upstream paths
699    /// have completed, inject the merged output into state and allow the node to
700    /// proceed.
701    ///
702    /// If a deferred node has a `fan_in_timeout` configured and the timeout has
703    /// elapsed:
704    /// - If at least one upstream path has completed, proceed with partial results.
705    /// - If zero upstream paths have completed, return `GraphError::FanInTimedOut`.
706    fn filter_deferred_nodes(
707        &mut self,
708        candidates: Vec<String>,
709        executed_nodes: &[String],
710    ) -> Result<Vec<String>> {
711        let mut ready_nodes = Vec::new();
712
713        for candidate in candidates {
714            if let Some(config) = self.graph.deferred_configs.get(&candidate) {
715                // This is a deferred node — check if all upstream paths are done
716                let upstream = self.graph.get_upstream_nodes(&candidate);
717
718                // Get or create the tracker for this deferred node
719                let tracker = self.pending_deferred.entry(candidate.clone()).or_insert_with(|| {
720                    let sources: Vec<&str> = upstream.iter().map(|s| s.as_str()).collect();
721                    FanInTracker::new(sources)
722                });
723
724                // Record the start time if this is the first time we see this deferred node
725                self.deferred_start_times.entry(candidate.clone()).or_insert_with(Instant::now);
726
727                // Record outputs from the just-executed nodes that are upstream of this deferred node
728                for executed in executed_nodes {
729                    if upstream.contains(executed) {
730                        // Use the current state as the output representation for this upstream node.
731                        // We capture a snapshot of the state that this upstream node contributed to.
732                        let output = self.state.get(executed).cloned().unwrap_or_else(|| {
733                            // If no state key matches the node name, capture the full state
734                            serde_json::Value::Object(
735                                self.state.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
736                            )
737                        });
738                        tracker.record(executed, output);
739                    }
740                }
741
742                if tracker.is_ready() {
743                    // All upstream paths have completed — merge and inject into state
744                    let merged = tracker.merge(&config.merge_strategy);
745                    let fan_in_key = format!("{candidate}_fan_in");
746                    self.graph.schema.apply_update(&mut self.state, &fan_in_key, merged);
747
748                    // Remove from pending_deferred and start times since it's now ready
749                    self.pending_deferred.remove(&candidate);
750                    self.deferred_start_times.remove(&candidate);
751                    ready_nodes.push(candidate);
752                } else if let Some(timeout_duration) = config.fan_in_timeout {
753                    // Check if the fan-in timeout has elapsed
754                    let start_time = self.deferred_start_times[&candidate];
755                    if start_time.elapsed() >= timeout_duration {
756                        let received = tracker.received_count();
757                        let expected = tracker.expected_count();
758
759                        // `min_predecessors` decides how many arrivals are enough
760                        // to release the node once the timeout expires.
761                        let required = config.min_predecessors.unwrap_or(1).max(1);
762                        if received >= required {
763                            // Proceed with partial results
764                            tracing::warn!(
765                                node = %candidate,
766                                received,
767                                expected,
768                                "fan-in timeout expired, proceeding with partial results"
769                            );
770                            let merged = tracker.merge(&config.merge_strategy);
771                            let fan_in_key = format!("{candidate}_fan_in");
772                            self.graph.schema.apply_update(&mut self.state, &fan_in_key, merged);
773
774                            // Clean up tracking state
775                            self.pending_deferred.remove(&candidate);
776                            self.deferred_start_times.remove(&candidate);
777                            ready_nodes.push(candidate);
778                        } else {
779                            // Too few arrived to release the node.
780                            self.pending_deferred.remove(&candidate);
781                            self.deferred_start_times.remove(&candidate);
782                            return Err(GraphError::FanInTimedOut {
783                                node: candidate,
784                                received,
785                                expected,
786                            });
787                        }
788                    }
789                }
790                // If not ready and no timeout (or timeout not yet elapsed), the node stays
791                // in pending_deferred and is NOT added to ready_nodes
792            } else {
793                // Not a deferred node — schedule normally
794                ready_nodes.push(candidate);
795            }
796        }
797
798        Ok(ready_nodes)
799    }
800
801    /// Initialize state from input and/or checkpoint
802    async fn initialize_state(&self, input: State) -> Result<State> {
803        // Start with schema defaults
804        let mut state = self.graph.schema.initialize_state();
805
806        // If resuming from checkpoint, load it
807        if let Some(checkpoint_id) = &self.config.resume_from {
808            if let Some(cp) = self.graph.checkpointer.as_ref()
809                && let Some(checkpoint) = cp.load_by_id(checkpoint_id).await?
810            {
811                state = checkpoint.state;
812            }
813        } else if let Some(cp) = self.graph.checkpointer.as_ref() {
814            // Try to load latest checkpoint for thread
815            if let Some(checkpoint) = cp.load(&self.config.thread_id).await? {
816                state = checkpoint.state;
817            }
818        }
819
820        // Merge input into state
821        for (key, value) in input {
822            self.graph.schema.apply_update(&mut state, &key, value);
823        }
824
825        Ok(state)
826    }
827
828    /// Execute one super-step (plan -> execute -> update)
829    async fn execute_super_step(&mut self) -> Result<SuperStepResult> {
830        let mut result = SuperStepResult::default();
831
832        if let Some(interrupt) = self.gate_before(&self.pending_nodes) {
833            return Ok(SuperStepResult { interrupt: Some(interrupt), ..Default::default() });
834        }
835
836        // Collect all updates and check for errors/interrupts
837        let mut all_updates = Vec::new();
838
839        // Node cache lookups. Every key is computed from the pre-step state, the
840        // state each node in this step reads, and a hit's updates join the same
841        // batch as computed ones, so a hit changes nothing a sibling sees and the
842        // key stored on a miss is the key the next identical lookup computes.
843        #[cfg(feature = "node-cache")]
844        let mut miss_keys: HashMap<String, String> = HashMap::new();
845        #[cfg(feature = "node-cache")]
846        let mut nodes_to_execute: Vec<String> = Vec::new();
847        #[cfg(feature = "node-cache")]
848        for node_name in &self.pending_nodes {
849            let Some(cache) = self.graph.node_caches.get(node_name) else {
850                nodes_to_execute.push(node_name.clone());
851                continue;
852            };
853            let cache_key = compute_cache_key(node_name, &self.state);
854            let cached_value = cache.get(&cache_key).await;
855            tracing::debug!(
856                node = %node_name,
857                cache_hit = cached_value.is_some(),
858                cache_key = %cache_key,
859                "node cache lookup"
860            );
861            match cached_value {
862                Some(serde_json::Value::Object(updates)) => {
863                    result.executed_nodes.push(node_name.clone());
864                    result.events.push(StreamEvent::node_end(node_name, self.step, 0));
865                    all_updates.push((node_name.clone(), updates.into_iter().collect()));
866                }
867                // A cached value is always the map of a node's updates; anything
868                // else is unusable, so the node runs.
869                Some(_) | None => {
870                    miss_keys.insert(node_name.clone(), cache_key);
871                    nodes_to_execute.push(node_name.clone());
872                }
873            }
874        }
875
876        // Determine which nodes to execute (all if cache feature is disabled).
877        //
878        // Sorted so that a bounded dispatch admits nodes in a fixed order rather
879        // than whatever order the frontier happened to be built in.
880        #[cfg(feature = "node-cache")]
881        let pending_for_execution = {
882            nodes_to_execute.sort();
883            &nodes_to_execute
884        };
885        #[cfg(not(feature = "node-cache"))]
886        let pending_for_execution = {
887            self.pending_nodes.sort();
888            &self.pending_nodes
889        };
890
891        // Execute all pending nodes in parallel
892        let nodes: Vec<_> = pending_for_execution
893            .iter()
894            .filter_map(|name| self.graph.nodes.get(name).map(|n| (name.clone(), n.clone())))
895            .collect();
896
897        // Look up timeout and retry policies for each node before spawning futures
898        let timeout_policies: Vec<_> =
899            nodes.iter().map(|(name, _)| self.graph.timeout_policy_for(name).cloned()).collect();
900        let retry_policies: Vec<_> =
901            nodes.iter().map(|(name, _)| self.graph.retry_policy_for(name).cloned()).collect();
902        // Attempts already spent, so a resumed run continues its budget rather
903        // than starting again. adk-python does not persist this.
904        let prior_attempts: Vec<u32> =
905            nodes.iter().map(|(name, _)| self.attempts.get(name).copied().unwrap_or(0)).collect();
906
907        let futures: Vec<_> = nodes
908            .into_iter()
909            .zip(timeout_policies)
910            .zip(retry_policies)
911            .zip(prior_attempts)
912            .map(|((((name, node), policy), retry), spent)| {
913                let mut ctx = NodeContext::new(self.state.clone(), self.config.clone(), self.step);
914                if let Some(run_config) = self.run_config.clone() {
915                    ctx.set_run_config(run_config);
916                }
917                // A node body may invoke other nodes. The invoker carries the
918                // graph's nodes and the shared ledger, so a resumed parent serves
919                // children that already finished. These invocations are awaited
920                // inline by the parent and are deliberately outside the
921                // concurrency budget: counting them could deadlock, because the
922                // parent holds its own slot while waiting.
923                ctx.set_parent_schema(Arc::new(self.graph.schema.clone()));
924                ctx.set_child_invoker(Arc::new(crate::child::ChildInvoker::new(
925                    self.graph.nodes.clone(),
926                    Arc::clone(&self.child_ledger),
927                    name.clone(),
928                )));
929
930                // Attach a ProgressHandle when idle timeout is configured
931                if let Some(ref p) = policy
932                    && p.idle_timeout.is_some()
933                {
934                    ctx.set_progress_handle(ProgressHandle::new());
935                }
936
937                let step = self.step;
938                async move {
939                    let start = Instant::now();
940                    let mut attempts = spent;
941                    let output = loop {
942                        let result = match policy {
943                            Some(ref timeout_policy) => {
944                                execute_with_timeout(node.as_ref(), &ctx, timeout_policy).await
945                            }
946                            None => node.execute(&ctx).await,
947                        };
948                        attempts += 1;
949
950                        let Err(ref error) = result else { break result };
951                        let Some(ref retry) = retry else { break result };
952                        if !retry.allows_another_attempt(attempts)
953                            || !retry.retry_on.should_retry(error)
954                        {
955                            break result;
956                        }
957
958                        let delay = retry.delay_for_attempt(attempts);
959                        tracing::warn!(
960                            node = %name,
961                            attempt = attempts,
962                            max_attempts = retry.max_attempts,
963                            delay_ms = delay.as_millis(),
964                            error = %error,
965                            "node failed, retrying after backoff"
966                        );
967                        tokio::time::sleep(delay).await;
968                    };
969                    let duration_ms = start.elapsed().as_millis() as u64;
970                    (name, output, duration_ms, step, attempts)
971                }
972            })
973            .collect();
974
975        // Bound the dispatch. `buffer_unordered` polls futures in the order they
976        // are produced, and the frontier is sorted above, so admission order does
977        // not depend on which node finished first.
978        let concurrency = self
979            .graph
980            .max_concurrency
981            .map_or(pending_for_execution.len(), |limit| limit.min(pending_for_execution.len()))
982            .max(1);
983        let mut outputs: Vec<_> =
984            stream::iter(futures).buffer_unordered(concurrency).collect().await;
985        // Completion order is intentionally nondeterministic under parallelism,
986        // but selecting which of several simultaneous pauses to report cannot
987        // be. Keep node event order stable and make the first node name win.
988        outputs.sort_by(|left, right| left.0.cmp(&right.0));
989
990        let mut interrupt = None;
991
992        for (node_name, output_result, duration_ms, step, attempts) in outputs {
993            // Record the budget spent, so a resumed run does not restart it. A
994            // node that finally succeeded keeps no entry: its budget is spent
995            // only while it is failing.
996            if output_result.is_err() {
997                self.attempts.insert(node_name.clone(), attempts);
998            } else {
999                self.attempts.remove(&node_name);
1000            }
1001            result.events.push(StreamEvent::node_end(&node_name, step, duration_ms));
1002
1003            match output_result {
1004                Ok(output) => {
1005                    // Check for dynamic interrupt
1006                    if let Some(node_interrupt) = output.interrupt {
1007                        if interrupt.is_none() {
1008                            interrupt = Some(node_interrupt);
1009                        }
1010                        continue;
1011                    }
1012                    result.executed_nodes.push(node_name.clone());
1013
1014                    // Collect custom events
1015                    result.events.extend(output.events);
1016
1017                    // Store result in cache on miss, under the key the lookup used.
1018                    #[cfg(feature = "node-cache")]
1019                    if let Some(cache_key) = miss_keys.get(&node_name)
1020                        && let Some(cache) = self.graph.node_caches.get(&node_name)
1021                    {
1022                        let updates_value = serde_json::to_value(&output.updates)
1023                            .unwrap_or(serde_json::Value::Object(serde_json::Map::new()));
1024                        let ttl = self.graph.cache_policies.get(&node_name).and_then(|p| p.ttl);
1025                        cache.set(cache_key, updates_value, ttl).await;
1026                    }
1027
1028                    // A node that named its successors overrides its declared edges.
1029                    if let Some(targets) = output.goto {
1030                        result.goto.insert(node_name.clone(), targets);
1031                    }
1032                    // A node handing control to the graph that holds this one. The
1033                    // run finishes normally; the caller reads this from the outcome.
1034                    if let Some(targets) = output.goto_parent {
1035                        self.goto_parent = Some(targets);
1036                    }
1037
1038                    // Collect updates with their node, so application order can be
1039                    // made independent of which future resolved first.
1040                    all_updates.push((node_name.clone(), output.updates));
1041                }
1042                Err(e) => {
1043                    // The retry budget is spent. A handler may record what
1044                    // happened and name a recovery node instead of ending the run.
1045                    // An interrupt never reaches here as a failure.
1046                    match self.graph.error_handler_for(&node_name) {
1047                        Some(handler) if !matches!(e, GraphError::Interrupted(_)) => {
1048                            let recovery = handler(&node_name, &e, &self.state)?;
1049                            if let Some(targets) = recovery.goto {
1050                                result.goto.insert(node_name.clone(), targets);
1051                            }
1052                            result.executed_nodes.push(node_name.clone());
1053                            all_updates.push((node_name, recovery.updates));
1054                        }
1055                        _ => {
1056                            return Err(GraphError::NodeExecutionFailed {
1057                                node: node_name,
1058                                message: e.to_string(),
1059                            });
1060                        }
1061                    }
1062                }
1063            }
1064        }
1065
1066        // Apply all updates atomically using reducers.
1067        //
1068        // `buffer_unordered` yields futures as they resolve, so the collected
1069        // order follows timing. A non-commutative reducer — `Append` builds an
1070        // array, so order is the result — would then give a different state for
1071        // the same input depending on which node finished first. Sorting by
1072        // (node, channel) makes the order total and timing-independent: node
1073        // names are unique within a graph, and a node's own updates are held in a
1074        // map whose iteration order is itself unspecified.
1075        all_updates.sort_by(|(left, _), (right, _)| left.cmp(right));
1076        for (node, updates) in all_updates {
1077            let mut keys: Vec<_> = updates.keys().cloned().collect();
1078            keys.sort();
1079            self.ensure_channels_declared(&node, keys.iter().map(String::as_str))?;
1080            for key in keys {
1081                if let Some(value) = updates.get(&key) {
1082                    self.graph.schema.apply_update(&mut self.state, &key, value.clone());
1083                }
1084            }
1085        }
1086
1087        if let Some(interrupt) = interrupt {
1088            return Ok(SuperStepResult { interrupt: Some(interrupt), ..result });
1089        }
1090
1091        if let Some(interrupt) = self.gate_after(&result.executed_nodes) {
1092            return Ok(SuperStepResult { interrupt: Some(interrupt), ..result });
1093        }
1094
1095        Ok(result)
1096    }
1097
1098    /// Save a checkpoint
1099    /// Returns the gate a pending node arms, if any.
1100    ///
1101    /// A node whose gate this run has already answered runs instead of
1102    /// interrupting again; without that a resume reaches the same conclusion and
1103    /// the node never executes.
1104    ///
1105    /// Shared by both execution paths. `StreamMode::Messages` runs nodes in its
1106    /// own loop, and when this check lived only in `execute_super_step` that mode
1107    /// ignored every gate.
1108    fn gate_before(&self, pending: &[String]) -> Option<Interrupt> {
1109        pending
1110            .iter()
1111            .find(|node| {
1112                self.graph.interrupt_before.contains(*node)
1113                    && self.cleared_interrupt.as_deref() != Some(node.as_str())
1114            })
1115            .map(|node| Interrupt::Before(node.clone()))
1116    }
1117
1118    /// Returns the gate an executed node arms, if any.
1119    fn gate_after(&self, executed: &[String]) -> Option<Interrupt> {
1120        executed
1121            .iter()
1122            .find(|node| self.graph.interrupt_after.contains(*node))
1123            .map(|node| Interrupt::After(node.clone()))
1124    }
1125
1126    /// Sets the frontier a paused super-step resumes from.
1127    ///
1128    /// | Interrupt | Saved frontier |
1129    /// |-----------|----------------|
1130    /// | `Before` | The frontier that was executing; the gated node produced no updates. |
1131    /// | `After` | The successors of the nodes that ran; the gated node already applied its updates. |
1132    /// | `Dynamic` (including tool confirmation) | The successors of every node that completed, plus each node that paused. |
1133    ///
1134    /// A completed node never re-runs on resume, and its successors and `goto`
1135    /// targets are not lost: they are computed now, while the step's results are
1136    /// still at hand, and arrivals at a fan-in node are recorded for the checkpoint.
1137    ///
1138    /// # Errors
1139    ///
1140    /// Returns the routing error from [`Self::next_frontier`] or the fan-in error
1141    /// from [`Self::filter_deferred_nodes`].
1142    fn pause_frontier(
1143        &mut self,
1144        interrupt: &Interrupt,
1145        executed: &[String],
1146        goto: &HashMap<String, Vec<String>>,
1147    ) -> Result<()> {
1148        match interrupt {
1149            Interrupt::Before(node) => {
1150                // Record the gate being answered so the resumed run executes this
1151                // node rather than stopping at it again.
1152                self.cleared_interrupt = Some(node.clone());
1153                return Ok(());
1154            }
1155            Interrupt::After(_) => {
1156                let next = self.next_frontier(executed, goto)?;
1157                self.pending_nodes = self.filter_deferred_nodes(next, executed)?;
1158            }
1159            Interrupt::Dynamic { .. } => {
1160                let paused: Vec<String> = self
1161                    .pending_nodes
1162                    .iter()
1163                    .filter(|node| !executed.contains(node))
1164                    .cloned()
1165                    .collect();
1166                let next = self.next_frontier(executed, goto)?;
1167                let mut frontier = self.filter_deferred_nodes(next, executed)?;
1168                for node in paused {
1169                    if !frontier.contains(&node) {
1170                        frontier.push(node);
1171                    }
1172                }
1173                self.pending_nodes = frontier;
1174            }
1175        }
1176        // A gate answered earlier re-arms once its node has run, as after an
1177        // uninterrupted step.
1178        if let Some(cleared) = &self.cleared_interrupt
1179            && executed.iter().any(|node| node == cleared)
1180        {
1181            self.cleared_interrupt = None;
1182        }
1183        Ok(())
1184    }
1185
1186    /// Computes the next frontier, letting a node's `goto` stand in for its edges.
1187    ///
1188    /// A node that named successors has its declared edges skipped, so a `goto`
1189    /// replaces an edge rather than adding to one. `END` is accepted and
1190    /// contributes no successor, which is how a branch stops.
1191    ///
1192    /// # Errors
1193    ///
1194    /// Returns [`GraphError::UnknownRouteTarget`] when a `goto` names a node the
1195    /// graph does not hold.
1196    fn next_frontier(
1197        &self,
1198        executed: &[String],
1199        goto: &HashMap<String, Vec<String>>,
1200    ) -> Result<Vec<String>> {
1201        // A node that routed itself does not also follow its declared edges.
1202        let followed_edges: Vec<String> =
1203            executed.iter().filter(|node| !goto.contains_key(*node)).cloned().collect();
1204        let mut next = self.graph.get_next_nodes(&followed_edges, &self.state)?;
1205
1206        // Sorted, so a multi-target goto admits its nodes in a fixed order.
1207        let mut routed: Vec<(&String, &Vec<String>)> = goto.iter().collect();
1208        routed.sort_by_key(|(node, _)| node.as_str());
1209
1210        for (node, targets) in routed {
1211            for target in targets {
1212                if target == crate::edge::END {
1213                    continue;
1214                }
1215                if self.graph.node(target).is_none() {
1216                    return Err(GraphError::UnknownRouteTarget(format!(
1217                        "node '{node}' routed to '{target}', which is not a node in this graph"
1218                    )));
1219                }
1220                if !next.contains(target) {
1221                    next.push(target.clone());
1222                }
1223            }
1224        }
1225        Ok(next)
1226    }
1227
1228    /// Rejects an update naming a channel the schema does not declare.
1229    ///
1230    /// Inert unless the graph asked for enforcement, and inert when the schema
1231    /// declares no channels, so an existing graph is unaffected either way.
1232    fn ensure_channels_declared<'k>(
1233        &self,
1234        node: &str,
1235        keys: impl IntoIterator<Item = &'k str>,
1236    ) -> Result<()> {
1237        if !self.graph.strict_channels {
1238            return Ok(());
1239        }
1240        match self.graph.schema.first_undeclared(keys) {
1241            Some(channel) => Err(GraphError::UndeclaredChannel {
1242                node: node.to_string(),
1243                channel: channel.to_string(),
1244            }),
1245            None => Ok(()),
1246        }
1247    }
1248
1249    async fn save_checkpoint(&self) -> Result<String> {
1250        if let Some(cp) = &self.graph.checkpointer {
1251            let mut checkpoint = Checkpoint::new(
1252                &self.config.thread_id,
1253                self.state.clone(),
1254                self.step,
1255                self.pending_nodes.clone(),
1256            );
1257            checkpoint.cleared_interrupt = self.cleared_interrupt.clone();
1258            checkpoint.attempts = self.attempts.clone();
1259            checkpoint.child_ledger = self.child_ledger.lock().expect("child ledger").clone();
1260            // Carried in metadata rather than a field, so every backend persists
1261            // it without a schema change and older checkpoints load as before.
1262            if let Some(snapshot) =
1263                FanInSnapshot::capture(&self.pending_deferred, &self.deferred_start_times)
1264            {
1265                checkpoint.metadata.insert(FAN_IN_METADATA_KEY.to_string(), snapshot.to_value());
1266            }
1267            let id = cp.save(&checkpoint).await?;
1268
1269            // Trimmed as the run proceeds, so the cost stays proportional to the run
1270            // and no external job is needed. After the save, so the newest counts.
1271            if let Some(policy) = &self.graph.retention {
1272                let removed = cp.prune(&self.config.thread_id, policy).await?;
1273                if removed > 0 {
1274                    tracing::debug!(
1275                        thread_id = %self.config.thread_id,
1276                        removed,
1277                        "pruned old checkpoints"
1278                    );
1279                }
1280            }
1281            return Ok(id);
1282        }
1283        Ok(String::new())
1284    }
1285}
1286
1287/// Convenience methods for CompiledGraph
1288impl CompiledGraph {
1289    /// Execute the graph synchronously
1290    pub async fn invoke(&self, input: State, config: ExecutionConfig) -> Result<State> {
1291        self.invoke_detailed(input, config).await.map(|outcome| outcome.state)
1292    }
1293
1294    /// Execute with ADK run configuration for agents inside a directly run graph.
1295    ///
1296    /// Use this to resume a tool-confirmation pause with decisions in
1297    /// [`adk_core::RunConfig`]. Existing [`Self::invoke`] callers remain
1298    /// standalone and use default ADK run configuration.
1299    pub async fn invoke_with_run_config(
1300        &self,
1301        input: State,
1302        config: ExecutionConfig,
1303        run_config: adk_core::RunConfig,
1304    ) -> Result<State> {
1305        let mut executor = PregelExecutor::new_with_run_config(self, config, Some(run_config));
1306        executor.run(input).await
1307    }
1308
1309    /// Executes and reports what the run asked of its caller.
1310    ///
1311    /// Only a graph run as a [`SubgraphNode`](crate::subgraph::SubgraphNode) has
1312    /// anything to report beyond its state, so [`Self::invoke`] is the usual
1313    /// entry point.
1314    pub async fn invoke_detailed(
1315        &self,
1316        input: State,
1317        config: ExecutionConfig,
1318    ) -> Result<GraphOutcome> {
1319        let mut executor = PregelExecutor::new(self, config);
1320        let state = executor.run(input).await?;
1321        Ok(GraphOutcome { state, goto_parent: executor.goto_parent })
1322    }
1323
1324    /// Execute with run configuration and retain a subgraph's parent routing outcome.
1325    pub async fn invoke_detailed_with_run_config(
1326        &self,
1327        input: State,
1328        config: ExecutionConfig,
1329        run_config: adk_core::RunConfig,
1330    ) -> Result<GraphOutcome> {
1331        let mut executor = PregelExecutor::new_with_run_config(self, config, Some(run_config));
1332        let state = executor.run(input).await?;
1333        Ok(GraphOutcome { state, goto_parent: executor.goto_parent })
1334    }
1335
1336    /// Execute with streaming
1337    pub fn stream(
1338        &self,
1339        input: State,
1340        config: ExecutionConfig,
1341        mode: StreamMode,
1342    ) -> impl futures::Stream<Item = Result<StreamEvent>> + '_ {
1343        tracing::debug!("CompiledGraph::stream called with mode {:?}", mode);
1344        let executor = PregelExecutor::new(self, config);
1345        executor.run_stream(input, mode)
1346    }
1347
1348    /// Stream with ADK run configuration for agents inside a directly run graph.
1349    ///
1350    /// A resumed tool-confirmation request can be approved or denied by passing
1351    /// [`adk_core::RunConfig::tool_confirmation_decisions`].
1352    pub fn stream_with_run_config(
1353        &self,
1354        input: State,
1355        config: ExecutionConfig,
1356        mode: StreamMode,
1357        run_config: adk_core::RunConfig,
1358    ) -> impl futures::Stream<Item = Result<StreamEvent>> + '_ {
1359        tracing::debug!("CompiledGraph::stream_with_run_config called with mode {:?}", mode);
1360        let executor = PregelExecutor::new_with_run_config(self, config, Some(run_config));
1361        executor.run_stream(input, mode)
1362    }
1363
1364    /// Get current state for a thread
1365    pub async fn get_state(&self, thread_id: &str) -> Result<Option<State>> {
1366        if let Some(cp) = &self.checkpointer {
1367            Ok(cp.load(thread_id).await?.map(|c| c.state))
1368        } else {
1369            Ok(None)
1370        }
1371    }
1372
1373    /// Update state for a thread (for human-in-the-loop)
1374    pub async fn update_state(
1375        &self,
1376        thread_id: &str,
1377        updates: impl IntoIterator<Item = (String, serde_json::Value)>,
1378    ) -> Result<()> {
1379        if let Some(cp) = &self.checkpointer
1380            && let Some(checkpoint) = cp.load(thread_id).await?
1381        {
1382            let mut state = checkpoint.state;
1383            for (key, value) in updates {
1384                self.schema.apply_update(&mut state, &key, value);
1385            }
1386            let mut new_checkpoint =
1387                Checkpoint::new(thread_id, state, checkpoint.step, checkpoint.pending_nodes);
1388            // Everything a resume restores carries over, so editing state between
1389            // a pause and its resume keeps the answered gate, the retry budget,
1390            // finished children, and the arrivals a waiting join has recorded.
1391            new_checkpoint.cleared_interrupt = checkpoint.cleared_interrupt;
1392            new_checkpoint.attempts = checkpoint.attempts;
1393            new_checkpoint.child_ledger = checkpoint.child_ledger;
1394            if let Some(fan_in) = checkpoint.metadata.get(FAN_IN_METADATA_KEY) {
1395                new_checkpoint.metadata.insert(FAN_IN_METADATA_KEY.to_string(), fan_in.clone());
1396            }
1397            cp.save(&new_checkpoint).await?;
1398        }
1399        Ok(())
1400    }
1401}
1402
1403#[cfg(test)]
1404mod tests {
1405    use super::*;
1406    use crate::edge::{END, START};
1407    use crate::graph::StateGraph;
1408    use crate::node::NodeOutput;
1409    use serde_json::json;
1410
1411    #[tokio::test]
1412    async fn test_simple_execution() {
1413        let graph = StateGraph::with_channels(&["value"])
1414            .add_node_fn("set_value", |_ctx| async {
1415                Ok(NodeOutput::new().with_update("value", json!(42)))
1416            })
1417            .add_edge(START, "set_value")
1418            .add_edge("set_value", END)
1419            .compile()
1420            .unwrap();
1421
1422        let result = graph.invoke(State::new(), ExecutionConfig::new("test")).await.unwrap();
1423
1424        assert_eq!(result.get("value"), Some(&json!(42)));
1425    }
1426
1427    #[tokio::test]
1428    async fn test_sequential_execution() {
1429        let graph = StateGraph::with_channels(&["value"])
1430            .add_node_fn("step1", |_ctx| async {
1431                Ok(NodeOutput::new().with_update("value", json!(1)))
1432            })
1433            .add_node_fn("step2", |ctx| async move {
1434                let current = ctx.get("value").and_then(|v| v.as_i64()).unwrap_or(0);
1435                Ok(NodeOutput::new().with_update("value", json!(current + 10)))
1436            })
1437            .add_edge(START, "step1")
1438            .add_edge("step1", "step2")
1439            .add_edge("step2", END)
1440            .compile()
1441            .unwrap();
1442
1443        let result = graph.invoke(State::new(), ExecutionConfig::new("test")).await.unwrap();
1444
1445        assert_eq!(result.get("value"), Some(&json!(11)));
1446    }
1447
1448    #[tokio::test]
1449    async fn test_conditional_routing() {
1450        let graph = StateGraph::with_channels(&["path", "result"])
1451            .add_node_fn("router", |ctx| async move {
1452                let path = ctx.get("path").and_then(|v| v.as_str()).unwrap_or("a");
1453                Ok(NodeOutput::new().with_update("route", json!(path)))
1454            })
1455            .add_node_fn("path_a", |_ctx| async {
1456                Ok(NodeOutput::new().with_update("result", json!("went to A")))
1457            })
1458            .add_node_fn("path_b", |_ctx| async {
1459                Ok(NodeOutput::new().with_update("result", json!("went to B")))
1460            })
1461            .add_edge(START, "router")
1462            .add_conditional_edges(
1463                "router",
1464                |state| state.get("route").and_then(|v| v.as_str()).unwrap_or(END).to_string(),
1465                [("a", "path_a"), ("b", "path_b"), (END, END)],
1466            )
1467            .add_edge("path_a", END)
1468            .add_edge("path_b", END)
1469            .compile()
1470            .unwrap();
1471
1472        // Test path A
1473        let mut input = State::new();
1474        input.insert("path".to_string(), json!("a"));
1475        let result = graph.invoke(input, ExecutionConfig::new("test")).await.unwrap();
1476        assert_eq!(result.get("result"), Some(&json!("went to A")));
1477
1478        // Test path B
1479        let mut input = State::new();
1480        input.insert("path".to_string(), json!("b"));
1481        let result = graph.invoke(input, ExecutionConfig::new("test")).await.unwrap();
1482        assert_eq!(result.get("result"), Some(&json!("went to B")));
1483    }
1484
1485    #[tokio::test]
1486    async fn test_cycle_with_limit() {
1487        let graph = StateGraph::with_channels(&["count"])
1488            .add_node_fn("increment", |ctx| async move {
1489                let count = ctx.get("count").and_then(|v| v.as_i64()).unwrap_or(0);
1490                Ok(NodeOutput::new().with_update("count", json!(count + 1)))
1491            })
1492            .add_edge(START, "increment")
1493            .add_conditional_edges(
1494                "increment",
1495                |state| {
1496                    let count = state.get("count").and_then(|v| v.as_i64()).unwrap_or(0);
1497                    if count < 5 { "increment".to_string() } else { END.to_string() }
1498                },
1499                [("increment", "increment"), (END, END)],
1500            )
1501            .compile()
1502            .unwrap();
1503
1504        let result = graph.invoke(State::new(), ExecutionConfig::new("test")).await.unwrap();
1505
1506        assert_eq!(result.get("count"), Some(&json!(5)));
1507    }
1508
1509    #[tokio::test]
1510    async fn test_recursion_limit() {
1511        let graph = StateGraph::with_channels(&["count"])
1512            .add_node_fn("loop", |ctx| async move {
1513                let count = ctx.get("count").and_then(|v| v.as_i64()).unwrap_or(0);
1514                Ok(NodeOutput::new().with_update("count", json!(count + 1)))
1515            })
1516            .add_edge(START, "loop")
1517            .add_edge("loop", "loop") // Infinite loop
1518            .compile()
1519            .unwrap()
1520            .with_recursion_limit(10);
1521
1522        let result = graph.invoke(State::new(), ExecutionConfig::new("test")).await;
1523
1524        // The recursion limit check happens when step >= limit, so it will exceed at step 10
1525        assert!(
1526            matches!(result, Err(GraphError::RecursionLimitExceeded(_))),
1527            "Expected RecursionLimitExceeded error, got: {:?}",
1528            result
1529        );
1530    }
1531}