1mod history;
12
13#[cfg(test)]
14use crate::move_bindings::tool::era::V1 as ToolWitnessV1;
15#[cfg(feature = "walrus")]
16use crate::walrus::StorageConf;
17use {
18 self::history::{fetch_shared_object_history, version_or_none, ObjectHistoryRequest},
19 crate::{
20 events::{NexusEvent, NexusEventDecoder, NexusEventKind},
21 move_bindings::{
22 interface::{
23 dag as dag_move,
24 era::V1 as InterfaceWitnessV1,
25 graph::{self as graph_move, RuntimeVertex},
26 meta_schema::MetaSchema,
27 onchain_tool_result::{OnchainToolResult, OnchainToolResultInnerV1},
28 payment::{ExecutionPaymentInnerV1, ExecutionPaymentVertexLock},
29 verifier::{FailureEvidenceKind, ToolVerifierMode},
30 },
31 move_std::type_name::TypeName,
32 sui_framework::{clock::Clock as SuiClock, linked_table, object::ID, vec_map::VecMap},
33 workflow::{
34 era::V1 as WorkflowWitnessV1,
35 execution::{self as execution_move, DAGExecution, DAGExecutionInnerV1, DAGWalk},
36 execution_events::{
37 EndStateReachedEvent,
38 ExecutionFinishedEvent,
39 TerminalErrEvalRecordedEvent,
40 },
41 execution_failure::WorkflowFailureClass,
42 invocation_adapter::InvocationLockedEvent,
43 },
44 FiniteCredits,
45 TimePass,
46 },
47 move_boundary,
48 nexus::{client::NexusClient, crawler::Crawler, error::NexusError, tap},
49 sui,
50 transactions::{
51 dag,
52 invocation::{self, InvocationPolicyCall},
53 },
54 types::{DagSpec, NexusContext, NexusData, PackageRole, SharedRoot, ToolRef},
55 },
56 anyhow::{anyhow, bail},
57 sha2::{Digest as _, Sha256},
58 std::{
59 collections::{BTreeMap, HashMap},
60 sync::Arc,
61 },
62 tokio::{
63 sync::mpsc::{unbounded_channel, UnboundedReceiver},
64 task::JoinHandle,
65 time::{Duration, Instant},
66 },
67};
68
69const DEFAULT_EXECUTION_INSPECTION_TIMEOUT: Duration = Duration::from_secs(60 * 60);
70const DEFAULT_EXECUTION_INSPECTION_POLL_INTERVAL: Duration = Duration::from_secs(1);
71pub const EXPIRED_WALK_NOT_DOUBLE_TIMEOUT_EXPIRED_REASON: &str =
72 "walk is not double timeout expired";
73pub const EXPIRED_WALK_ALREADY_TERMINAL_REASON: &str = "walk is already terminal";
74pub const EXPIRED_WALK_OTHER_PAYMENT_LOCKS_REASON: &str = "other walks still hold payment locks";
75
76#[derive(Clone, Debug)]
77pub struct PublishResult {
78 pub tx_digest: sui::types::Digest,
79 pub tx_checkpoint: u64,
80 pub dag_object_id: sui::types::Address,
81}
82
83#[derive(Clone, Debug, PartialEq, Eq)]
90pub enum InvocationPolicy {
91 FixedPrice,
92 FiniteCredits,
93 TimePass,
94 Custom(InvocationPolicyCall),
95}
96
97#[derive(Clone, Debug)]
99pub struct AuthorizeInvocationResult {
100 pub tx_digest: sui::types::Digest,
101 pub tx_checkpoint: u64,
102 pub lock: InvocationLockedEvent,
104}
105
106#[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
108pub struct DagSnapshot {
109 pub dag_id: sui::types::Address,
110 pub vertex_count: u64,
111 pub entry_groups: BTreeMap<String, BTreeMap<String, Vec<String>>>,
112 pub vertex_meta_schemas: BTreeMap<String, MetaSchema>,
113}
114
115#[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
120pub struct InvocationAbortCandidate {
121 pub walk_index: usize,
122 pub vertex: RuntimeVertex,
123 pub payment_vertex_key: Vec<u8>,
124 pub invocation_ref: sui::types::ObjectReference,
125}
126
127#[derive(Clone, Debug, PartialEq, Eq)]
128pub struct AbortExpiredExecutionResult {
129 pub tx_digest: sui::types::Digest,
130 pub tx_checkpoint: u64,
131 pub dag_id: sui::types::Address,
132 pub dag_execution_id: sui::types::Address,
133 pub selected_candidate: InvocationAbortCandidate,
134}
135
136#[derive(Clone, Debug, PartialEq, Eq)]
137pub struct AbortExecutionResult {
138 pub tx_digest: sui::types::Digest,
139 pub tx_checkpoint: u64,
140 pub dag_id: sui::types::Address,
141 pub dag_execution_id: sui::types::Address,
142 pub cleaned_broken_onchain_results: Vec<BrokenOnchainToolResultCleanup>,
143}
144
145#[derive(Clone, Debug, PartialEq, Eq)]
146pub struct SettleCommittedToolResultParams {
147 pub dag_execution_id: sui::types::Address,
148 pub walk_index: u64,
149}
150
151#[derive(Clone, Debug, PartialEq, Eq)]
152pub struct ResolveExpiredWalkParams {
153 pub dag_execution_id: sui::types::Address,
154 pub walk_index: u64,
155 pub invocation_id: Option<sui::types::Address>,
156}
157
158#[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
159#[serde(tag = "kind", rename_all = "snake_case")]
160pub enum ExpiredWalkResolutionKind {
161 Settled {
162 expected_vertex: RuntimeVertex,
163 invocation_ref: sui::types::ObjectReference,
164 },
165 SettledOnchainResult {
166 result_ref: sui::types::ObjectReference,
167 expected_vertex: RuntimeVertex,
168 invocation_ref: sui::types::ObjectReference,
169 tool_witness_id: sui::types::Address,
170 finalize_tx_digest: Vec<u8>,
171 },
172 Aborted,
173 AbortedWithInvocation {
174 selected_candidate: InvocationAbortCandidate,
175 },
176 Skipped {
177 reason: String,
178 },
179}
180
181impl ExpiredWalkResolutionKind {
182 pub fn as_str(&self) -> &'static str {
183 match self {
184 Self::Settled { .. } => "settled",
185 Self::SettledOnchainResult { .. } => "settled_onchain_result",
186 Self::Aborted => "aborted",
187 Self::AbortedWithInvocation { .. } => "aborted_with_invocation",
188 Self::Skipped { .. } => "skipped",
189 }
190 }
191
192 pub fn selected_candidate(&self) -> Option<&InvocationAbortCandidate> {
193 match self {
194 Self::AbortedWithInvocation { selected_candidate } => Some(selected_candidate),
195 _ => None,
196 }
197 }
198
199 pub fn skip_reason(&self) -> Option<&str> {
200 match self {
201 Self::Skipped { reason } => Some(reason),
202 _ => None,
203 }
204 }
205}
206
207#[derive(Clone, Debug, PartialEq, Eq)]
208pub struct ExpiredWalkResolutionPlan {
209 pub dag_id: sui::types::Address,
210 pub dag_execution_id: sui::types::Address,
211 pub walk_index: u64,
212 pub kind: ExpiredWalkResolutionKind,
213}
214
215#[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
216pub struct ExpiredWalkResolutionResult {
217 pub tx_digest: Option<sui::types::Digest>,
218 pub tx_checkpoint: Option<u64>,
219 pub dag_id: sui::types::Address,
220 pub dag_execution_id: sui::types::Address,
221 pub walk_index: u64,
222 pub resolution_kind: ExpiredWalkResolutionKind,
223}
224
225#[derive(Clone, Debug, PartialEq, Eq)]
226pub struct SettleCommittedToolResultByLeaderParams {
227 pub dag_execution_id: sui::types::Address,
228 pub leader_cap_id: sui::types::Address,
229 pub walk_index: u64,
230 pub expected_vertex: Option<RuntimeVertex>,
232 pub failed_onchain_tool_reason: Option<Vec<u8>>,
234 pub commit_tx_digest: Vec<u8>,
235 pub commit_gas_charge: u64,
236 pub settlement_gas_charge: u64,
237}
238
239#[derive(Clone, Debug, PartialEq, Eq)]
240pub struct RecordCommittedToolResultGasChargeParams {
241 pub dag_execution_id: sui::types::Address,
242 pub leader_cap_id: sui::types::Address,
243 pub walk_index: u64,
244 pub expected_vertex: Option<RuntimeVertex>,
246 pub failed_onchain_tool_reason: Option<Vec<u8>>,
248 pub commit_tx_digest: Vec<u8>,
249 pub commit_gas_charge: u64,
250 pub settlement_gas_charge: u64,
251}
252
253#[derive(Clone, Debug, PartialEq, Eq)]
254pub struct CommittedToolResultSettlementResult {
255 pub tx_digest: sui::types::Digest,
256 pub tx_checkpoint: u64,
257 pub dag_id: sui::types::Address,
258 pub dag_execution_id: sui::types::Address,
259 pub walk_index: u64,
260}
261
262#[derive(Clone, Debug, PartialEq, Eq)]
263pub struct BrokenOnchainToolResultCleanup {
264 pub walk_index: u64,
265 pub result_ref: sui::types::ObjectReference,
266 pub expected_vertex: RuntimeVertex,
267 pub tool_witness_id: sui::types::Address,
268}
269
270#[derive(Clone, Debug, PartialEq, Eq)]
271pub struct RecordCommittedToolResultGasChargeResult {
272 pub tx_digest: sui::types::Digest,
273 pub tx_checkpoint: u64,
274 pub dag_execution_id: sui::types::Address,
275 pub leader_cap_id: sui::types::Address,
276 pub walk_index: u64,
277}
278
279#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
284pub struct CommittedToolResultView {
285 pub expected_vertex: RuntimeVertex,
286 pub primary_failure_evidence_kind: Option<FailureEvidenceKind>,
287 pub secondary_failure_evidence_kind: Option<FailureEvidenceKind>,
288 pub current_leader_cap_id: sui::types::Address,
289 pub has_finalized_onchain_payload: bool,
290 pub leader_records: Vec<CommittedToolResultLeaderRecordView>,
291}
292
293impl CommittedToolResultView {
294 pub fn leader_record(
295 &self,
296 leader_cap_id: sui::types::Address,
297 ) -> Option<&CommittedToolResultLeaderRecordView> {
298 self.leader_records
299 .iter()
300 .find(|record| record.leader_cap_id == leader_cap_id)
301 }
302}
303
304#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
305pub struct CommittedToolResultLeaderRecordView {
306 pub leader_cap_id: sui::types::Address,
307 pub commit_tx_digest: Vec<u8>,
308 pub recipient: sui::types::Address,
309 pub commit_gas_charge: Option<u64>,
310 pub settlement_gas_charge: Option<u64>,
311}
312
313#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
314pub struct InsufficientSettlementMarkerView {
315 pub walks: Vec<u64>,
316}
317
318pub enum OnchainToolResultState {
319 NoResult,
320 Finalized {
321 result: OnchainToolResultInnerV1,
322 object_ref: sui::types::ObjectReference,
323 },
324 Committed {
325 committed_result: CommittedToolResultView,
326 },
327 InsufficientSettlement {
328 committed_result: Option<CommittedToolResultView>,
329 marker: InsufficientSettlementMarkerView,
330 },
331 InvalidEmpty {
332 result_id: sui::types::Address,
333 object_ref: sui::types::ObjectReference,
334 },
335}
336
337impl OnchainToolResultState {
338 pub fn consume_ready_result(
341 &self,
342 ) -> Option<(&OnchainToolResultInnerV1, &sui::types::ObjectReference)> {
343 match self {
344 Self::Finalized { result, object_ref } => Some((result, object_ref)),
345 Self::NoResult
346 | Self::Committed { .. }
347 | Self::InsufficientSettlement { .. }
348 | Self::InvalidEmpty { .. } => None,
349 }
350 }
351
352 pub fn consume_object_input(&self) -> Option<talus_sui_move_call::CallArg> {
355 self.consume_ready_result().map(|(_, object_ref)| {
356 talus_sui_move_call::CallArg::Shared(sui::types::SharedInput::new(
357 *object_ref.object_id(),
358 object_ref.version(),
359 true,
360 ))
361 })
362 }
363}
364
365fn onchain_tool_result_is_finalized(result: &OnchainToolResultInnerV1) -> bool {
366 result.finalized
367 && result.tag.as_option().is_some()
368 && result.named_payload.as_option().is_some()
369 && result.finalize_tx_digest.as_option().is_some()
370 && result.finalize_recipient.as_option().is_some()
371}
372
373impl From<execution_move::CommittedToolResult> for CommittedToolResultView {
374 fn from(value: execution_move::CommittedToolResult) -> Self {
375 Self {
376 expected_vertex: value.expected_vertex,
377 primary_failure_evidence_kind: value.primary_failure_evidence_kind.into_option(),
378 secondary_failure_evidence_kind: value.secondary_failure_evidence_kind.into_option(),
379 current_leader_cap_id: value.current_leader_cap_id.bytes,
380 has_finalized_onchain_payload: value.has_finalized_onchain_payload,
381 leader_records: value
382 .leader_records
383 .contents
384 .into_iter()
385 .map(|entry| CommittedToolResultLeaderRecordView {
386 leader_cap_id: entry.key.bytes,
387 commit_tx_digest: entry.value.commit_tx_digest,
388 recipient: entry.value.recipient,
389 commit_gas_charge: entry.value.commit_gas_charge.into_option(),
390 settlement_gas_charge: entry.value.settlement_gas_charge.into_option(),
391 })
392 .collect(),
393 }
394 }
395}
396
397pub struct InspectExecutionResult {
398 pub next_event: UnboundedReceiver<NexusEvent>,
399 pub poller: JoinHandle<Result<(), NexusError>>,
400}
401
402#[derive(Clone, Copy, Debug, PartialEq, Eq)]
405pub struct InspectExecutionOptions {
406 pub timeout: Duration,
407 pub poll_interval: Duration,
408}
409
410impl Default for InspectExecutionOptions {
411 fn default() -> Self {
412 Self {
413 timeout: DEFAULT_EXECUTION_INSPECTION_TIMEOUT,
414 poll_interval: DEFAULT_EXECUTION_INSPECTION_POLL_INTERVAL,
415 }
416 }
417}
418
419#[derive(Clone, Copy, Debug, PartialEq, Eq)]
420pub enum WorkflowExecutionTerminalState {
421 Succeeded,
422 Failed,
423 Aborted,
424 NoWalkOutcome,
425}
426
427#[derive(Clone, Debug)]
428pub struct ResolvedEndState {
429 pub event: EndStateReachedEvent,
430 pub resolved_ports_to_data: HashMap<String, NexusData>,
431}
432
433#[derive(Clone, Debug)]
434pub struct InspectExecutionCompletionResult {
435 pub terminal_state: WorkflowExecutionTerminalState,
436 pub execution_finished: ExecutionFinishedEvent,
437 pub end_states: Vec<ResolvedEndState>,
438 pub terminal_err_eval_recordings: Vec<TerminalErrEvalRecordedEvent>,
439 pub events: Vec<NexusEvent>,
440}
441
442pub struct ExecutionCostResult {
443 pub payment_id: sui::types::Address,
444 pub max_budget_mist: u64,
445 pub locked_budget_mist: u64,
446 pub consumed: u64,
447 pub outstanding_locks: u64,
448 pub accomplished: bool,
449 pub refunded: bool,
450}
451
452#[non_exhaustive]
454pub struct ExecutionCostDetails {
455 pub summary: ExecutionCostResult,
456 pub outstanding_invocation_ids: Vec<sui::types::Address>,
457}
458
459pub struct WorkflowActions {
460 pub(super) client: NexusClient,
461}
462
463struct ResolvedExecution {
464 context: Arc<NexusContext>,
465 object: crate::nexus::crawler::Response<DAGExecutionInnerV1>,
466}
467
468async fn fetch_execution(
469 client: &NexusClient,
470 execution_id: sui::types::Address,
471 required_roots: &[SharedRoot],
472) -> Result<ResolvedExecution, NexusError> {
473 let context = if required_roots.is_empty() {
474 client.context_for_object(execution_id).await?
475 } else {
476 client
477 .context_for_object_with_roots(execution_id, required_roots)
478 .await?
479 };
480 let object = client
481 .state_resolver()
482 .load_inner::<DAGExecution, WorkflowWitnessV1, DAGExecutionInnerV1>(execution_id, &context)
483 .await?;
484 Ok(ResolvedExecution { context, object })
485}
486
487async fn fetch_dag_in_context(
488 client: &NexusClient,
489 context: &NexusContext,
490 dag_id: sui::types::Address,
491) -> Result<crate::nexus::crawler::Response<dag_move::DAGInnerV1>, NexusError> {
492 client
493 .state_resolver()
494 .load_inner::<dag_move::DAG, InterfaceWitnessV1, dag_move::DAGInnerV1>(dag_id, context)
495 .await
496}
497
498fn event_execution_id(event: &NexusEventKind) -> Option<sui::types::Address> {
499 match event {
500 NexusEventKind::AgentSkillExecutionRequested(e) => Some(e.execution_id),
501 NexusEventKind::OccurrenceDispatched(e) => Some(e.execution_id.into()),
502 NexusEventKind::ExecutionPaymentFeesRecorded(e) => Some(e.execution_id),
503 NexusEventKind::ExecutionPaymentToolCostSnapshotted(e) => Some(e.execution_id),
504 NexusEventKind::InvocationAuthorizationRequired(e) => Some(e.execution.into()),
505 NexusEventKind::InvocationLocked(e) => Some(e.execution.bytes),
506 NexusEventKind::InvocationSettled(e) => Some(e.execution.bytes),
507 NexusEventKind::RequestWalkExecution(e) => Some(e.execution.into()),
508 NexusEventKind::CommittedToolResult(e) => Some(e.execution.into()),
509 NexusEventKind::WalkAdvanced(e) => Some(e.execution.into()),
510 NexusEventKind::WalkFailed(e) => Some(e.execution.into()),
511 NexusEventKind::SubmissionFailureEvidenceRecorded(e) => Some(e.execution.into()),
512 NexusEventKind::TerminalErrEvalRecorded(e) => Some(e.execution.into()),
513 NexusEventKind::ToolVerificationResolved(e) => Some(e.execution.into()),
514 NexusEventKind::WalkPendingAbort(e) => Some(e.execution.into()),
515 NexusEventKind::WalkAborted(e) => Some(e.execution.into()),
516 NexusEventKind::WalkCancelled(e) => Some(e.execution.into()),
517 NexusEventKind::EndStateReached(e) => Some(e.execution.into()),
518 NexusEventKind::ExecutionFinished(e) => Some(e.execution.into()),
519 NexusEventKind::ExecutionPaymentRefilled(e) => Some(e.execution_id),
520 _ => None,
521 }
522}
523
524fn is_transient_inspection_error(error: &NexusError) -> bool {
525 let NexusError::Rpc(error) = error else {
526 return false;
527 };
528
529 error.chain().any(|source| {
530 source
531 .downcast_ref::<tonic::Status>()
532 .is_some_and(|status| {
533 matches!(
534 status.code(),
535 tonic::Code::Unavailable
536 | tonic::Code::DeadlineExceeded
537 | tonic::Code::ResourceExhausted
538 | tonic::Code::Aborted
539 )
540 })
541 })
542}
543
544async fn fetch_execution_update_events(
545 crawler: &Crawler,
546 event_decoder: &NexusEventDecoder,
547 request: ObjectHistoryRequest<'_>,
548) -> Result<Vec<NexusEvent>, NexusError> {
549 let dag_execution_id = request.object_id;
550 let updates = fetch_shared_object_history(crawler, request).await?;
551
552 let mut events = Vec::new();
553 for update in updates {
554 for (index, event) in update.events.into_iter().enumerate() {
555 let Some(event) = event_decoder
556 .decode_sui_event(index as u64, update.digest, &event)
557 .await
558 .map_err(|error| {
559 NexusError::Parsing(anyhow::Error::new(error).context(format!(
560 "Could not decode event {index} from transaction '{}' while reconstructing execution '{dag_execution_id}'",
561 update.digest
562 )))
563 })?
564 else {
565 continue;
566 };
567 if event_execution_id(&event.data) == Some(dag_execution_id) {
568 events.push(event);
569 }
570 }
571 }
572 Ok(events)
573}
574
575#[cfg(feature = "walrus")]
576fn terminal_state_from_execution_finished(
577 execution_finished: &ExecutionFinishedEvent,
578) -> WorkflowExecutionTerminalState {
579 if execution_finished.was_aborted {
580 WorkflowExecutionTerminalState::Aborted
581 } else if execution_finished.has_any_walk_failed {
582 WorkflowExecutionTerminalState::Failed
583 } else if execution_finished.has_any_walk_succeeded {
584 WorkflowExecutionTerminalState::Succeeded
585 } else {
586 WorkflowExecutionTerminalState::NoWalkOutcome
587 }
588}
589
590#[cfg(feature = "walrus")]
591async fn build_execution_completion_result(
592 events: Vec<NexusEvent>,
593 dag_execution_id: sui::types::Address,
594 storage_conf: &StorageConf,
595) -> Result<InspectExecutionCompletionResult, NexusError> {
596 let mut end_states = Vec::new();
597 let mut terminal_err_eval_recordings = Vec::new();
598 let mut execution_finished = None;
599
600 for event in &events {
601 match &event.data {
602 NexusEventKind::EndStateReached(end_state) => {
603 let resolved_ports_to_data = end_state
604 .variant_ports_to_data
605 .clone()
606 .fetch_all(storage_conf)
607 .await
608 .map_err(|e| {
609 NexusError::Storage(anyhow!(
610 "Failed to fetch output data for execution '{dag_execution_id}': {e}"
611 ))
612 })?
613 .into_map();
614
615 end_states.push(ResolvedEndState {
616 event: end_state.clone(),
617 resolved_ports_to_data,
618 });
619 }
620 NexusEventKind::TerminalErrEvalRecorded(recorded) => {
621 terminal_err_eval_recordings.push(recorded.clone());
622 }
623 NexusEventKind::ExecutionFinished(finished) => {
624 execution_finished = Some(finished.clone());
625 }
626 _ => {}
627 }
628 }
629
630 let execution_finished = execution_finished.ok_or_else(|| {
631 NexusError::Channel(anyhow!(
632 "ExecutionFinished event not found while inspecting DAG execution '{dag_execution_id}'"
633 ))
634 })?;
635
636 Ok(InspectExecutionCompletionResult {
637 terminal_state: terminal_state_from_execution_finished(&execution_finished),
638 execution_finished,
639 end_states,
640 terminal_err_eval_recordings,
641 events,
642 })
643}
644
645pub fn dag_vertex_requires_tool_verification(vertex: &graph_move::VertexInfo) -> bool {
646 vertex.verifier_mode != ToolVerifierMode::None
647}
648
649pub async fn fetch_dag_vertices_bcs(
660 crawler: &Crawler,
661 dag: &dag_move::DAGInnerV1,
662) -> anyhow::Result<HashMap<graph_move::Vertex, graph_move::VertexInfo>> {
663 if dag.vertices.size() == 0 {
664 return Ok(HashMap::new());
665 }
666 Ok(crawler
667 .get_dynamic_fields::<
668 graph_move::Vertex,
669 linked_table::Node<graph_move::Vertex, graph_move::VertexInfo>,
670 >(dag.vertices.id(), dag.vertices.size())
671 .await?
672 .into_iter()
673 .map(|(vertex, node)| (vertex, node.value))
674 .collect())
675}
676
677pub async fn fetch_dag_vertices_from_links_bcs(
688 crawler: &Crawler,
689 context: &NexusContext,
690 dag: &dag_move::DAGInnerV1,
691) -> anyhow::Result<HashMap<graph_move::Vertex, graph_move::VertexInfo>> {
692 let expected_size = dag.vertices.size();
693 let head = dag.vertices.head.cloned_option();
694 let tail = dag.vertices.tail.cloned_option();
695 if expected_size == 0 {
696 if head.is_some() || tail.is_some() {
697 bail!("Empty DAG vertex table records a head or tail");
698 }
699 return Ok(HashMap::new());
700 }
701
702 let mut current = head.ok_or_else(|| anyhow!("DAG vertex table has no head"))?;
703 let key_type = crate::move_bindings::type_tag::<graph_move::Vertex>(context);
704 let mut previous = None;
705 let mut vertices = HashMap::with_capacity(expected_size);
706
707 for position in 0..expected_size {
708 if vertices.contains_key(¤t) {
709 bail!("DAG vertex table contains a cycle at position {position}");
710 }
711 let node = crawler
712 .get_dynamic_field_by_key::<
713 graph_move::Vertex,
714 linked_table::Node<graph_move::Vertex, graph_move::VertexInfo>,
715 >(dag.vertices.id(), current.clone(), &key_type)
716 .await?
717 .ok_or_else(|| anyhow!("DAG vertex table is missing position {position}"))?;
718 if node.prev.cloned_option() != previous {
719 bail!("DAG vertex table has an invalid previous link at position {position}");
720 }
721
722 let next = node.next.cloned_option();
723 previous = Some(current.clone());
724 vertices.insert(current.clone(), node.value);
725
726 if position + 1 == expected_size {
727 if next.is_some() {
728 bail!("DAG vertex table exceeds its declared size");
729 }
730 } else {
731 current = next.ok_or_else(|| {
732 anyhow!("DAG vertex table ends before its declared size at position {position}")
733 })?;
734 }
735 }
736
737 if previous != tail {
738 bail!("DAG vertex table tail does not match its final link");
739 }
740 Ok(vertices)
741}
742
743pub async fn fetch_dag_vertex_bcs(
754 crawler: &Crawler,
755 context: &NexusContext,
756 dag: &dag_move::DAGInnerV1,
757 vertex: &graph_move::Vertex,
758) -> anyhow::Result<Option<graph_move::VertexInfo>> {
759 crawler
760 .get_dynamic_field_by_key::<
761 graph_move::Vertex,
762 linked_table::Node<graph_move::Vertex, graph_move::VertexInfo>,
763 >(
764 dag.vertices.id(),
765 vertex.clone(),
766 &crate::move_bindings::type_tag::<graph_move::Vertex>(context),
767 )
768 .await
769 .map(|node| node.map(|node| node.value))
770}
771
772pub async fn fetch_dag_vertices_by_keys_bcs<I>(
783 crawler: &Crawler,
784 context: &NexusContext,
785 dag: &dag_move::DAGInnerV1,
786 vertices: I,
787) -> anyhow::Result<HashMap<graph_move::Vertex, graph_move::VertexInfo>>
788where
789 I: IntoIterator<Item = graph_move::Vertex>,
790{
791 Ok(crawler
792 .get_dynamic_fields_by_keys::<
793 graph_move::Vertex,
794 linked_table::Node<graph_move::Vertex, graph_move::VertexInfo>,
795 _,
796 >(
797 dag.vertices.id(),
798 vertices,
799 &crate::move_bindings::type_tag::<graph_move::Vertex>(context),
800 )
801 .await?
802 .into_iter()
803 .map(|(vertex, node)| (vertex, node.value))
804 .collect())
805}
806
807pub(crate) async fn fetch_dag_snapshot(
808 client: &NexusClient,
809 context: &NexusContext,
810 dag_id: sui::types::Address,
811) -> anyhow::Result<DagSnapshot> {
812 let dag = client
813 .state_resolver()
814 .load_inner::<dag_move::DAG, InterfaceWitnessV1, dag_move::DAGInnerV1>(dag_id, context)
815 .await?;
816 let entry_groups = dag
817 .data
818 .entry_groups
819 .contents
820 .iter()
821 .map(|entry_group| {
822 let vertices = entry_group
823 .value
824 .contents
825 .iter()
826 .map(|vertex| {
827 let mut ports = vertex
828 .value
829 .contents
830 .iter()
831 .map(|port| port.name.as_str().to_owned())
832 .collect::<Vec<_>>();
833 ports.sort();
834 (vertex.key.name.as_str().to_owned(), ports)
835 })
836 .collect::<BTreeMap<_, _>>();
837 (entry_group.key.name.as_str().to_owned(), vertices)
838 })
839 .collect();
840 let vertices = fetch_dag_vertices_from_links_bcs(client.crawler(), context, &dag.data).await?;
841 let vertex_meta_schemas = vertices
842 .into_iter()
843 .map(|(vertex, info)| {
844 let name = vertex.name.as_str().to_owned();
845 let schema = info.meta_schema.into_option().ok_or_else(|| {
846 anyhow!("DAG '{dag_id}' vertex '{name}' is missing its immutable MetaSchema")
847 })?;
848 Ok((name, schema))
849 })
850 .collect::<anyhow::Result<BTreeMap<_, _>>>()?;
851
852 Ok(DagSnapshot {
853 dag_id,
854 vertex_count: dag.data.vertices.size().try_into()?,
855 entry_groups,
856 vertex_meta_schemas,
857 })
858}
859
860pub async fn fetch_committed_tool_result_for_walk(
870 crawler: &Crawler,
871 context: &NexusContext,
872 execution_id: sui::types::Address,
873 walk_index: u64,
874) -> anyhow::Result<Option<CommittedToolResultView>> {
875 crawler
876 .get_dynamic_field_by_key::<
877 execution_move::CommittedToolResultKey,
878 execution_move::CommittedToolResult,
879 >(
880 execution_id,
881 execution_move::CommittedToolResultKey { walk_index },
882 &crate::move_bindings::type_tag::<execution_move::CommittedToolResultKey>(context),
883 )
884 .await
885 .map(|value| value.map(CommittedToolResultView::from))
886}
887
888pub fn committed_tool_result_from_finalized_output(
900 crawler: &Crawler,
901 context: &NexusContext,
902 executed: &sui::grpc::ExecutedTransaction,
903 transaction: sui::types::Digest,
904 execution_id: sui::types::Address,
905 walk_index: u64,
906) -> anyhow::Result<Option<CommittedToolResultView>> {
907 let key = execution_move::CommittedToolResultKey { walk_index };
908 let outputs = crawler.transaction_dynamic_field_outputs::<
909 execution_move::CommittedToolResultKey,
910 execution_move::CommittedToolResult,
911 >(
912 executed,
913 transaction,
914 &key,
915 &crate::move_bindings::type_tag::<execution_move::CommittedToolResultKey>(context),
916 &crate::move_bindings::type_tag::<execution_move::CommittedToolResult>(context),
917 )?;
918 let mut matching = outputs
919 .into_iter()
920 .filter(|output| output.object_id == execution_id);
921 let output = matching.next();
922 if matching.next().is_some() {
923 bail!(
924 "Transaction '{transaction}' wrote more than one committed result for execution \
925 '{execution_id}' walk {walk_index}"
926 );
927 }
928
929 Ok(output.map(|output| CommittedToolResultView::from(output.data)))
930}
931
932pub async fn inspect_expired_walk_resolution(
933 client: &NexusClient,
934 params: ResolveExpiredWalkParams,
935) -> anyhow::Result<ExpiredWalkResolutionPlan> {
936 let clock = client
937 .crawler()
938 .get_object::<SuiClock>(move_boundary::CLOCK_OBJECT_ID)
939 .await?
940 .data;
941 inspect_expired_walk_resolution_at(client, params, clock.timestamp_ms).await
942}
943
944pub async fn inspect_expired_walk_resolution_at(
945 client: &NexusClient,
946 params: ResolveExpiredWalkParams,
947 clock_ms: u64,
948) -> anyhow::Result<ExpiredWalkResolutionPlan> {
949 let resolved = fetch_execution(client, params.dag_execution_id, &[]).await?;
950 inspect_expired_walk_in_execution(client, params, clock_ms, &resolved).await
951}
952
953async fn inspect_expired_walk_in_execution(
954 client: &NexusClient,
955 params: ResolveExpiredWalkParams,
956 clock_ms: u64,
957 resolved: &ResolvedExecution,
958) -> anyhow::Result<ExpiredWalkResolutionPlan> {
959 let context = &resolved.context;
960 let execution = &resolved.object.data;
961 let crawler = client.crawler();
962 let Some(walk) = usize::try_from(params.walk_index)
963 .ok()
964 .and_then(|index| execution.walks.get(index))
965 else {
966 return Ok(ExpiredWalkResolutionPlan {
967 dag_id: execution.dag_id(),
968 dag_execution_id: params.dag_execution_id,
969 walk_index: params.walk_index,
970 kind: ExpiredWalkResolutionKind::Skipped {
971 reason: "walk index no longer exists in execution".to_string(),
972 },
973 });
974 };
975
976 let Some(timeout_vertex) = walk.timeout_expired_vertex(clock_ms) else {
977 return Ok(ExpiredWalkResolutionPlan {
978 dag_id: execution.dag_id(),
979 dag_execution_id: params.dag_execution_id,
980 walk_index: params.walk_index,
981 kind: ExpiredWalkResolutionKind::Skipped {
982 reason: unresolved_timeout_skip_reason(walk).to_string(),
983 },
984 });
985 };
986
987 match fetch_onchain_tool_result_state_for_walk(
988 client,
989 context,
990 params.dag_execution_id,
991 params.walk_index,
992 )
993 .await?
994 {
995 OnchainToolResultState::Committed { .. } => {
996 let invocation_ref = fetch_invocation_for_vertex(
997 client,
998 context,
999 params.dag_execution_id,
1000 execution.dag_id(),
1001 timeout_vertex,
1002 )
1003 .await?;
1004 return Ok(ExpiredWalkResolutionPlan {
1005 dag_id: execution.dag_id(),
1006 dag_execution_id: params.dag_execution_id,
1007 walk_index: params.walk_index,
1008 kind: ExpiredWalkResolutionKind::Settled {
1009 expected_vertex: timeout_vertex.clone(),
1010 invocation_ref,
1011 },
1012 });
1013 }
1014 OnchainToolResultState::Finalized { result, object_ref } => {
1015 let dag = fetch_dag_in_context(client, context, execution.dag_id()).await?;
1016 let vertex_info =
1017 fetch_dag_vertex_bcs(crawler, context, &dag.data, timeout_vertex.vertex())
1018 .await?
1019 .ok_or_else(|| {
1020 anyhow!(
1021 "DAG vertex '{}' missing from fetched DAG",
1022 timeout_vertex.vertex_name()
1023 )
1024 })?;
1025 let kind = finalized_onchain_result_resolution_kind(
1026 client,
1027 &vertex_info,
1028 timeout_vertex.clone(),
1029 fetch_invocation_for_vertex(
1030 client,
1031 context,
1032 params.dag_execution_id,
1033 execution.dag_id(),
1034 timeout_vertex,
1035 )
1036 .await?,
1037 &result,
1038 object_ref,
1039 )
1040 .await?;
1041 return Ok(ExpiredWalkResolutionPlan {
1042 dag_id: execution.dag_id(),
1043 dag_execution_id: params.dag_execution_id,
1044 walk_index: params.walk_index,
1045 kind,
1046 });
1047 }
1048 OnchainToolResultState::InsufficientSettlement { .. } => {
1049 return Ok(ExpiredWalkResolutionPlan {
1050 dag_id: execution.dag_id(),
1051 dag_execution_id: params.dag_execution_id,
1052 walk_index: params.walk_index,
1053 kind: ExpiredWalkResolutionKind::Skipped {
1054 reason: "walk has committed result with insufficient settlement".to_string(),
1055 },
1056 });
1057 }
1058 OnchainToolResultState::InvalidEmpty {
1059 result_id,
1060 object_ref,
1061 } => {
1062 return Ok(ExpiredWalkResolutionPlan {
1063 dag_id: execution.dag_id(),
1064 dag_execution_id: params.dag_execution_id,
1065 walk_index: params.walk_index,
1066 kind: ExpiredWalkResolutionKind::Skipped {
1067 reason: format!(
1068 "stored on-chain tool result {result_id}@{} is not finalized",
1069 object_ref.version()
1070 ),
1071 },
1072 });
1073 }
1074 OnchainToolResultState::NoResult => {}
1075 }
1076
1077 let Some(abort_vertex) = walk.abortable_timeout_expired_vertex(clock_ms) else {
1078 return Ok(ExpiredWalkResolutionPlan {
1079 dag_id: execution.dag_id(),
1080 dag_execution_id: params.dag_execution_id,
1081 walk_index: params.walk_index,
1082 kind: ExpiredWalkResolutionKind::Skipped {
1083 reason: format!(
1084 "timeout-expired walk at vertex '{}' is not abortable without a committed result",
1085 timeout_vertex.vertex_name()
1086 ),
1087 },
1088 });
1089 };
1090
1091 let payment =
1092 tap::fetch_execution_payment_for_execution(client, context, params.dag_execution_id)
1093 .await?
1094 .data;
1095 let dag = fetch_dag_in_context(client, context, execution.dag_id()).await?;
1096 let vertex_info = fetch_dag_vertex_bcs(crawler, context, &dag.data, abort_vertex.vertex())
1097 .await?
1098 .ok_or_else(|| {
1099 anyhow!(
1100 "DAG vertex '{}' missing from fetched DAG",
1101 abort_vertex.vertex_name()
1102 )
1103 })?;
1104 let tool_fqn = vertex_info.kind.tool_fqn()?;
1105 let vertex_key = payment_vertex_key(params.dag_execution_id, abort_vertex, &tool_fqn)?;
1106 let locked = payment
1107 .locked_vertices
1108 .iter()
1109 .find(|lock| lock.vertex_key == vertex_key);
1110
1111 let Some(locked) = locked else {
1112 return Ok(ExpiredWalkResolutionPlan {
1113 dag_id: execution.dag_id(),
1114 dag_execution_id: params.dag_execution_id,
1115 walk_index: params.walk_index,
1116 kind: if payment.locked_vertices.is_empty() {
1117 ExpiredWalkResolutionKind::Aborted
1118 } else {
1119 ExpiredWalkResolutionKind::Skipped {
1120 reason: EXPIRED_WALK_OTHER_PAYMENT_LOCKS_REASON.to_owned(),
1121 }
1122 },
1123 });
1124 };
1125
1126 let invocation_ref = crawler
1127 .get_object_metadata(locked.invocation_id.bytes)
1128 .await?
1129 .object_ref();
1130 let selected_candidate = InvocationAbortCandidate {
1131 walk_index: usize::try_from(params.walk_index)?,
1132 vertex: abort_vertex.clone(),
1133 payment_vertex_key: vertex_key,
1134 invocation_ref,
1135 };
1136 if params
1137 .invocation_id
1138 .is_some_and(|expected| expected != *selected_candidate.invocation_ref.object_id())
1139 {
1140 return Err(anyhow!(
1141 "Requested Invocation does not match the expired walk"
1142 ));
1143 }
1144
1145 Ok(ExpiredWalkResolutionPlan {
1146 dag_id: execution.dag_id(),
1147 dag_execution_id: params.dag_execution_id,
1148 walk_index: params.walk_index,
1149 kind: ExpiredWalkResolutionKind::AbortedWithInvocation { selected_candidate },
1150 })
1151}
1152
1153fn unresolved_timeout_skip_reason(walk: &DAGWalk) -> &'static str {
1154 match walk {
1155 DAGWalk::Active { .. } | DAGWalk::PendingSettlement { .. } => {
1156 EXPIRED_WALK_NOT_DOUBLE_TIMEOUT_EXPIRED_REASON
1157 }
1158 _ => EXPIRED_WALK_ALREADY_TERMINAL_REASON,
1159 }
1160}
1161
1162pub async fn inspect_expired_execution_resolution_at(
1166 client: &NexusClient,
1167 execution_id: sui::types::Address,
1168 clock_ms: u64,
1169) -> anyhow::Result<Option<ExpiredWalkResolutionPlan>> {
1170 let execution = fetch_execution(client, execution_id, &[]).await?;
1171 let mut selected: Option<ExpiredWalkResolutionPlan> = None;
1172 for (index, walk) in execution.object.data.walks.iter().enumerate() {
1173 if walk.timeout_expired_vertex(clock_ms).is_none() {
1174 continue;
1175 }
1176 let plan = inspect_expired_walk_in_execution(
1177 client,
1178 ResolveExpiredWalkParams {
1179 dag_execution_id: execution_id,
1180 walk_index: index as u64,
1181 invocation_id: None,
1182 },
1183 clock_ms,
1184 &execution,
1185 )
1186 .await?;
1187 if resolution_priority(&plan.kind) == 0 {
1188 return Ok(Some(plan));
1189 }
1190 if resolution_priority(&plan.kind)
1191 < selected
1192 .as_ref()
1193 .map_or(u8::MAX, |plan| resolution_priority(&plan.kind))
1194 {
1195 selected = Some(plan);
1196 }
1197 }
1198 Ok(selected)
1199}
1200
1201fn resolution_priority(kind: &ExpiredWalkResolutionKind) -> u8 {
1202 match kind {
1203 ExpiredWalkResolutionKind::Settled { .. }
1204 | ExpiredWalkResolutionKind::SettledOnchainResult { .. } => 0,
1205 ExpiredWalkResolutionKind::AbortedWithInvocation { .. } => 1,
1206 ExpiredWalkResolutionKind::Aborted => 2,
1207 ExpiredWalkResolutionKind::Skipped { .. } => u8::MAX,
1208 }
1209}
1210
1211async fn finalized_onchain_result_resolution_kind(
1212 client: &NexusClient,
1213 vertex_info: &graph_move::VertexInfo,
1214 timeout_vertex: RuntimeVertex,
1215 invocation_ref: sui::types::ObjectReference,
1216 result: &OnchainToolResultInnerV1,
1217 object_ref: sui::types::ObjectReference,
1218) -> anyhow::Result<ExpiredWalkResolutionKind> {
1219 let tool_fqn = vertex_info.kind.tool_fqn()?;
1220 let inspection = client.tool().inspect_tool(&tool_fqn).await?;
1221 let tool = inspection.tool.ok_or_else(|| {
1222 anyhow!(
1223 "Tool '{}' cannot be decoded: {}",
1224 tool_fqn,
1225 inspection
1226 .detail
1227 .as_deref()
1228 .unwrap_or("no supported Tool definition")
1229 )
1230 })?;
1231 let ToolRef::Sui {
1232 tool_witness_id, ..
1233 } = tool.reference()
1234 else {
1235 return Ok(ExpiredWalkResolutionKind::Skipped {
1236 reason: format!(
1237 "finalized on-chain result exists for non-Sui tool vertex '{}'",
1238 timeout_vertex.vertex_name()
1239 ),
1240 });
1241 };
1242 let finalize_tx_digest = result
1243 .finalize_tx_digest
1244 .as_option()
1245 .cloned()
1246 .ok_or_else(|| anyhow!("finalized on-chain result is missing finalize_tx_digest"))?;
1247 Ok(ExpiredWalkResolutionKind::SettledOnchainResult {
1248 result_ref: object_ref,
1249 expected_vertex: timeout_vertex,
1250 invocation_ref,
1251 tool_witness_id: tool_witness_id.bytes,
1252 finalize_tx_digest,
1253 })
1254}
1255
1256async fn broken_onchain_result_cleanups_for_abort(
1257 client: &NexusClient,
1258 context: &NexusContext,
1259 execution_id: sui::types::Address,
1260 execution: &DAGExecutionInnerV1,
1261 clock_ms: u64,
1262) -> anyhow::Result<Vec<BrokenOnchainToolResultCleanup>> {
1263 let crawler = client.crawler();
1264 let mut dag = None;
1265 let mut vertices = HashMap::new();
1266 let mut cleanups = Vec::new();
1267
1268 for (walk_index, walk) in execution.walks.iter().enumerate() {
1269 let Some(timeout_vertex) = walk.abortable_timeout_expired_vertex(clock_ms) else {
1270 continue;
1271 };
1272 let walk_index = u64::try_from(walk_index)?;
1273 match fetch_onchain_tool_result_state_for_walk(client, context, execution_id, walk_index)
1274 .await?
1275 {
1276 OnchainToolResultState::Finalized { result, object_ref } => {
1277 let vertex_key = timeout_vertex.vertex();
1278 if !vertices.contains_key(vertex_key) {
1279 if dag.is_none() {
1280 dag =
1281 Some(fetch_dag_in_context(client, context, execution.dag_id()).await?);
1282 }
1283 let dag = dag.as_ref().expect("DAG was fetched above");
1284 let vertex_info = fetch_dag_vertex_bcs(crawler, context, &dag.data, vertex_key)
1285 .await?
1286 .ok_or_else(|| {
1287 anyhow!(
1288 "DAG vertex '{}' missing from fetched DAG",
1289 timeout_vertex.vertex_name()
1290 )
1291 })?;
1292 vertices.insert(vertex_key.clone(), vertex_info);
1293 }
1294 let vertex_info = vertices.get(vertex_key).expect("vertex was fetched above");
1295 let kind = finalized_onchain_result_resolution_kind(
1296 client,
1297 vertex_info,
1298 timeout_vertex.clone(),
1299 fetch_invocation_for_vertex(
1300 client,
1301 context,
1302 execution_id,
1303 execution.dag_id(),
1304 timeout_vertex,
1305 )
1306 .await?,
1307 &result,
1308 object_ref.clone(),
1309 )
1310 .await?;
1311 let ExpiredWalkResolutionKind::SettledOnchainResult {
1312 expected_vertex,
1313 tool_witness_id,
1314 ..
1315 } = kind
1316 else {
1317 return Err(anyhow!(
1318 "expired abort is blocked by finalized on-chain result for walk {walk_index} that cannot be cleaned by the Sui-tool cleanup path"
1319 ));
1320 };
1321 if onchain_tool_result_has_required_stamps(
1322 &result,
1323 *object_ref.object_id(),
1324 execution_id,
1325 context.leader_registry.object_id(),
1326 tool_witness_id,
1327 ) {
1328 return Err(anyhow!(
1329 "expired abort is blocked by consumable on-chain result for walk {walk_index}; settle the on-chain result before aborting"
1330 ));
1331 }
1332 cleanups.push(BrokenOnchainToolResultCleanup {
1333 walk_index,
1334 result_ref: object_ref,
1335 expected_vertex,
1336 tool_witness_id,
1337 });
1338 }
1339 OnchainToolResultState::InvalidEmpty { result_id, .. } => {
1340 return Err(anyhow!(
1341 "expired abort is blocked by unfinalized on-chain result {result_id} for walk {walk_index}"
1342 ));
1343 }
1344 OnchainToolResultState::NoResult
1345 | OnchainToolResultState::Committed { .. }
1346 | OnchainToolResultState::InsufficientSettlement { .. } => {}
1347 }
1348 }
1349
1350 Ok(cleanups)
1351}
1352
1353fn onchain_tool_result_has_required_stamps(
1354 result: &OnchainToolResultInnerV1,
1355 result_id: sui::types::Address,
1356 execution_id: sui::types::Address,
1357 leader_registry_id: sui::types::Address,
1358 tool_witness_id: sui::types::Address,
1359) -> bool {
1360 let Some(stamps) = result.stamps.as_option() else {
1361 return false;
1362 };
1363 let execution_id = ID::new(execution_id);
1364 let leader_registry_id = ID::new(leader_registry_id);
1365 let tool_witness_id = ID::new(tool_witness_id);
1366 let result_id = ID::new(result_id);
1367
1368 stamps.contents.len() == 4
1369 && stamps
1370 .contents
1371 .iter()
1372 .any(|entry| entry.key == execution_id)
1373 && stamps
1374 .contents
1375 .iter()
1376 .any(|entry| entry.key == leader_registry_id)
1377 && stamps
1378 .contents
1379 .iter()
1380 .any(|entry| entry.key == tool_witness_id)
1381 && stamps.contents.iter().any(|entry| entry.key == result_id)
1382}
1383
1384pub fn onchain_tool_result_from_finalized_output(
1395 crawler: &Crawler,
1396 context: &NexusContext,
1397 executed: &sui::grpc::ExecutedTransaction,
1398 transaction: sui::types::Digest,
1399 execution_id: sui::types::Address,
1400 walk_index: u64,
1401) -> anyhow::Result<OnchainToolResultState> {
1402 use crate::move_bindings::primitives::object_state::Inner;
1403
1404 let pointers = crawler
1405 .transaction_dynamic_field_outputs::<execution_move::OnchainToolResultKey, ID>(
1406 executed,
1407 transaction,
1408 &execution_move::OnchainToolResultKey { walk_index },
1409 &crate::move_bindings::type_tag::<execution_move::OnchainToolResultKey>(context),
1410 &crate::move_bindings::type_tag::<ID>(context),
1411 )?;
1412 let mut pointers = pointers
1413 .into_iter()
1414 .filter(|output| output.object_id == execution_id);
1415 let pointer = pointers.next().ok_or_else(|| anyhow!(
1416 "Transaction '{transaction}' has no onchain result for execution '{execution_id}' walk {walk_index}"
1417 ))?;
1418 anyhow::ensure!(
1419 pointers.next().is_none(),
1420 "Transaction '{transaction}' has repeated onchain result pointers"
1421 );
1422 let result_id = pointer.data.bytes;
1423 let anchor =
1424 crawler.transaction_output_object::<OnchainToolResult>(executed, transaction, result_id)?;
1425 let initial_version = crawler.transaction_shared_initial_version(
1426 executed,
1427 transaction,
1428 result_id,
1429 &crate::move_bindings::struct_tag::<OnchainToolResult>(context),
1430 )?;
1431 anyhow::ensure!(
1432 anchor.data.id.address() == result_id,
1433 "Onchain result anchor identity does not match its walk pointer"
1434 );
1435
1436 let outputs = crawler.transaction_dynamic_field_outputs::<Inner, OnchainToolResultInnerV1>(
1437 executed,
1438 transaction,
1439 &Inner::new(false),
1440 &crate::move_bindings::type_tag::<Inner>(context),
1441 &crate::move_bindings::type_tag::<OnchainToolResultInnerV1>(context),
1442 )?;
1443 let mut outputs = outputs
1444 .into_iter()
1445 .filter(|output| output.object_id == result_id);
1446 let result = outputs
1447 .next()
1448 .ok_or_else(|| {
1449 anyhow!(
1450 "Transaction '{transaction}' has no stored output for onchain result '{result_id}'"
1451 )
1452 })?
1453 .data;
1454 anyhow::ensure!(
1455 outputs.next().is_none(),
1456 "Transaction '{transaction}' has repeated onchain result outputs"
1457 );
1458 anyhow::ensure!(
1459 result.execution_id.bytes == execution_id,
1460 "Onchain result belongs to another execution"
1461 );
1462 anyhow::ensure!(
1463 onchain_tool_result_is_finalized(&result),
1464 "Onchain result is not finalized"
1465 );
1466 anyhow::ensure!(
1467 result.finalize_tx_digest.as_option().map(Vec::as_slice) == Some(transaction.as_ref()),
1468 "Onchain result was finalized by another transaction"
1469 );
1470
1471 Ok(OnchainToolResultState::Finalized {
1472 result,
1473 object_ref: sui::types::ObjectReference::new(result_id, initial_version, anchor.digest),
1474 })
1475}
1476
1477pub async fn fetch_onchain_tool_result_state_for_walk(
1488 client: &NexusClient,
1489 context: &NexusContext,
1490 execution_id: sui::types::Address,
1491 walk_index: u64,
1492) -> anyhow::Result<OnchainToolResultState> {
1493 let crawler = client.crawler();
1494 let committed_result =
1495 fetch_committed_tool_result_for_walk(crawler, context, execution_id, walk_index).await?;
1496 let insufficient_settlement = crawler
1497 .get_dynamic_field_by_key::<
1498 execution_move::ExecutionPaymentInsufficientSettlementFieldKey,
1499 execution_move::ExecutionPaymentInsufficientSettlement,
1500 >(
1501 execution_id,
1502 insufficient_settlement_field_key(),
1503 &crate::move_bindings::type_tag::<
1504 execution_move::ExecutionPaymentInsufficientSettlementFieldKey,
1505 >(context),
1506 )
1507 .await?;
1508
1509 if let Some(marker) = insufficient_settlement {
1510 if marker.walks.contains(&walk_index) {
1511 return Ok(OnchainToolResultState::InsufficientSettlement {
1512 committed_result,
1513 marker: InsufficientSettlementMarkerView {
1514 walks: marker.walks,
1515 },
1516 });
1517 }
1518 }
1519
1520 if let Some(committed_result) = committed_result {
1521 return Ok(OnchainToolResultState::Committed { committed_result });
1522 }
1523
1524 let result_id = crawler
1525 .get_dynamic_field_by_key::<execution_move::OnchainToolResultKey, ID>(
1526 execution_id,
1527 execution_move::OnchainToolResultKey { walk_index },
1528 &crate::move_bindings::type_tag::<execution_move::OnchainToolResultKey>(context),
1529 )
1530 .await?;
1531 let Some(result_id) = result_id else {
1532 return Ok(OnchainToolResultState::NoResult);
1533 };
1534
1535 let result_id = result_id.bytes;
1536 let result_response = client
1537 .state_resolver()
1538 .load_inner::<OnchainToolResult, InterfaceWitnessV1, OnchainToolResultInnerV1>(
1539 result_id, context,
1540 )
1541 .await?;
1542 let object_ref = result_response.object_ref();
1543 let result = result_response.data;
1544
1545 if onchain_tool_result_is_finalized(&result) {
1546 Ok(OnchainToolResultState::Finalized { result, object_ref })
1547 } else {
1548 Ok(OnchainToolResultState::InvalidEmpty {
1549 result_id,
1550 object_ref,
1551 })
1552 }
1553}
1554
1555fn insufficient_settlement_field_key(
1556) -> execution_move::ExecutionPaymentInsufficientSettlementFieldKey {
1557 execution_move::ExecutionPaymentInsufficientSettlementFieldKey { dummy_field: false }
1558}
1559
1560pub async fn fetch_dag_default_values_bcs<T>(
1561 crawler: &Crawler,
1562 dag: &dag_move::DAGInnerV1,
1563) -> anyhow::Result<HashMap<graph_move::VertexInputPort, T>>
1564where
1565 T: serde::de::DeserializeOwned,
1566{
1567 crawler
1568 .get_dynamic_fields::<graph_move::VertexInputPort, T>(
1569 dag.defaults_to_input_ports.id(),
1570 dag.defaults_to_input_ports.size(),
1571 )
1572 .await
1573}
1574
1575pub async fn fetch_dag_default_values_by_keys_bcs<T, I>(
1587 crawler: &Crawler,
1588 context: &NexusContext,
1589 dag: &dag_move::DAGInnerV1,
1590 ports: I,
1591) -> anyhow::Result<HashMap<graph_move::VertexInputPort, T>>
1592where
1593 T: serde::de::DeserializeOwned,
1594 I: IntoIterator<Item = graph_move::VertexInputPort>,
1595{
1596 crawler
1597 .get_dynamic_fields_by_keys::<graph_move::VertexInputPort, T, _>(
1598 dag.defaults_to_input_ports.id(),
1599 ports,
1600 &crate::move_bindings::type_tag::<graph_move::VertexInputPort>(context),
1601 )
1602 .await
1603}
1604
1605pub async fn fetch_dag_edges_bcs(
1606 crawler: &Crawler,
1607 dag: &dag_move::DAGInnerV1,
1608) -> anyhow::Result<HashMap<graph_move::Vertex, Vec<graph_move::Edge>>> {
1609 crawler
1610 .get_dynamic_fields::<graph_move::Vertex, Vec<graph_move::Edge>>(
1611 dag.edges.id(),
1612 dag.edges.size(),
1613 )
1614 .await
1615}
1616
1617pub async fn fetch_dag_outputs_bcs(
1618 crawler: &Crawler,
1619 dag: &dag_move::DAGInnerV1,
1620) -> anyhow::Result<HashMap<graph_move::Vertex, Vec<graph_move::OutputVariantPort>>> {
1621 crawler
1622 .get_dynamic_fields::<graph_move::Vertex, Vec<graph_move::OutputVariantPort>>(
1623 dag.outputs.id(),
1624 dag.outputs.size(),
1625 )
1626 .await
1627}
1628
1629pub async fn offchain_success_requires_tool_verification(
1639 client: &NexusClient,
1640 context: &NexusContext,
1641 dag_object_id: sui::types::Address,
1642 next_vertex: &RuntimeVertex,
1643) -> anyhow::Result<bool> {
1644 let dag = fetch_dag_in_context(client, context, dag_object_id).await?;
1645 let vertex_name = next_vertex.vertex();
1646 let vertex = fetch_dag_vertex_bcs(client.crawler(), context, &dag.data, vertex_name)
1647 .await?
1648 .ok_or_else(|| {
1649 anyhow!(
1650 "Vertex '{}' not found in DAG verifier config",
1651 vertex_name.name
1652 )
1653 })?;
1654
1655 Ok(dag_vertex_requires_tool_verification(&vertex))
1656}
1657
1658pub async fn fetch_vertex_input_port_names(
1668 crawler: &Crawler,
1669 context: &NexusContext,
1670 dag: &dag_move::DAGInnerV1,
1671 vertex_name: &TypeName,
1672) -> anyhow::Result<Vec<String>> {
1673 let vertex_key = graph_move::Vertex::from(vertex_name);
1674 let vertex = fetch_dag_vertex_bcs(crawler, context, dag, &vertex_key)
1675 .await?
1676 .ok_or_else(|| {
1677 anyhow!("Vertex '{vertex_name}' not found in DAG vertices dynamic fields")
1678 })?;
1679
1680 Ok(vertex.declared_input_port_names())
1681}
1682
1683pub fn execution_terminal_record_matches_retryable_vertex(
1684 terminal_records: &VecMap<u64, execution_move::TerminalErrEvalRecord>,
1685 walk_index: u64,
1686 next_vertex: &RuntimeVertex,
1687) -> bool {
1688 terminal_records.get(&walk_index).is_some_and(|record| {
1689 &record.vertex == next_vertex && record.failure_class == WorkflowFailureClass::Retryable
1690 })
1691}
1692
1693pub async fn should_settle_tool_err_eval_gas(
1694 client: &NexusClient,
1695 execution: sui::types::Address,
1696 walk_index: u64,
1697 next_vertex: &RuntimeVertex,
1698) -> anyhow::Result<bool> {
1699 let execution = fetch_execution(client, execution, &[]).await?;
1700 Ok(execution_terminal_record_matches_retryable_vertex(
1701 &execution.object.data.terminal_records,
1702 walk_index,
1703 next_vertex,
1704 ))
1705}
1706
1707async fn resolve_invocation_policy_call(
1708 client: &NexusClient,
1709 context: &NexusContext,
1710 cashier_id: sui::types::Address,
1711 beneficiary: crate::move_bindings::interface::payment::PaymentSourceKind,
1712 policy: InvocationPolicy,
1713) -> Result<InvocationPolicyCall, NexusError> {
1714 match policy {
1715 InvocationPolicy::FixedPrice => {
1716 InvocationPolicyCall::fixed_price(context).map_err(NexusError::Parsing)
1717 }
1718 InvocationPolicy::FiniteCredits => {
1719 let credits_id = crate::move_bindings::derive_finite_credits_id(
1720 context,
1721 cashier_id,
1722 beneficiary.clone(),
1723 )
1724 .map_err(NexusError::Parsing)?;
1725 let credits = client
1726 .crawler()
1727 .get_object::<FiniteCredits>(credits_id)
1728 .await
1729 .map_err(|error| {
1730 NexusError::Configuration(format!(
1731 "Finite credits '{credits_id}' could not be resolved: {error}"
1732 ))
1733 })?;
1734 if credits.data.cashier.bytes != cashier_id || credits.data.beneficiary != beneficiary {
1735 return Err(NexusError::Configuration(format!(
1736 "Finite credits '{credits_id}' belong to another Tool"
1737 )));
1738 }
1739 let sui::types::Owner::Shared(initial_shared_version) = credits.owner else {
1740 return Err(NexusError::Configuration(format!(
1741 "Finite credits '{credits_id}' must be a shared object"
1742 )));
1743 };
1744 InvocationPolicyCall::finite_credits(context, credits_id, initial_shared_version)
1745 .map_err(NexusError::Parsing)
1746 }
1747 InvocationPolicy::TimePass => {
1748 let pass_id =
1749 crate::move_bindings::derive_time_pass_id(context, cashier_id, beneficiary.clone())
1750 .map_err(NexusError::Parsing)?;
1751 let pass = client
1752 .crawler()
1753 .get_object::<TimePass>(pass_id)
1754 .await
1755 .map_err(|error| {
1756 NexusError::Configuration(format!(
1757 "Time pass '{pass_id}' could not be resolved: {error}"
1758 ))
1759 })?;
1760 if pass.data.cashier.bytes != cashier_id || pass.data.beneficiary != beneficiary {
1761 return Err(NexusError::Configuration(format!(
1762 "Time pass '{pass_id}' belongs to another Tool"
1763 )));
1764 }
1765 let sui::types::Owner::Shared(initial_shared_version) = pass.owner else {
1766 return Err(NexusError::Configuration(format!(
1767 "Time pass '{pass_id}' must be a shared object"
1768 )));
1769 };
1770 InvocationPolicyCall::time_pass(context, pass_id, initial_shared_version)
1771 .map_err(NexusError::Parsing)
1772 }
1773 InvocationPolicy::Custom(call) => Ok(call),
1774 }
1775}
1776
1777impl WorkflowActions {
1778 pub async fn publish(
1780 &self,
1781 workflow_package: sui::types::Address,
1782 dag_spec: DagSpec,
1783 ) -> Result<PublishResult, NexusError> {
1784 let context = self
1785 .client
1786 .context_for_creator(workflow_package, PackageRole::Workflow, &[])
1787 .await?;
1788 let address = self.client.owner()?;
1789
1790 let tx = dag::publish_ptb(&context, dag_spec).map_err(NexusError::TransactionBuilding)?;
1793 let response = self.client.submit_transaction(tx, address).await?;
1794
1795 let dag_tag = crate::move_bindings::struct_tag::<dag_move::DAG>(&context);
1798 let dag_object_id = response
1799 .objects
1800 .into_iter()
1801 .find_map(|obj| {
1802 let sui::types::ObjectType::Struct(object_type) = obj.object_type() else {
1803 return None;
1804 };
1805
1806 if *object_type.address() == *dag_tag.address()
1807 && object_type.module() == dag_tag.module()
1808 && object_type.name() == dag_tag.name()
1809 {
1810 Some(obj.object_id())
1811 } else {
1812 None
1813 }
1814 })
1815 .ok_or_else(|| {
1816 NexusError::Parsing(anyhow!("DAG object ID not found in TX response"))
1817 })?;
1818
1819 Ok(PublishResult {
1820 tx_digest: response.digest,
1821 tx_checkpoint: response.checkpoint,
1822 dag_object_id,
1823 })
1824 }
1825
1826 pub async fn inspect_dag(
1829 &self,
1830 dag_id: sui::types::Address,
1831 ) -> Result<DagSnapshot, NexusError> {
1832 let context = self.client.context_for_object(dag_id).await?;
1833 fetch_dag_snapshot(&self.client, &context, dag_id)
1834 .await
1835 .map_err(NexusError::Rpc)
1836 }
1837
1838 pub async fn authorize_invocation(
1843 &self,
1844 dag_execution_id: sui::types::Address,
1845 leader_cap_id: sui::types::Address,
1846 walk_index: u64,
1847 vertex: RuntimeVertex,
1848 policy: InvocationPolicy,
1849 ) -> Result<AuthorizeInvocationResult, NexusError> {
1850 let client = &self.client;
1851 let objects = client.get_nexus_objects();
1852 let required_roots = [objects.tool_registry, objects.leader_registry];
1853 let crawler = client.crawler();
1854 let leader_cap = crawler
1855 .get_object_metadata(leader_cap_id)
1856 .await
1857 .map_err(NexusError::Rpc)?;
1858 let execution = fetch_execution(client, dag_execution_id, &[]).await?;
1859 let is_active = execution
1860 .object
1861 .data
1862 .walks
1863 .get(walk_index as usize)
1864 .is_some_and(|walk| {
1865 matches!(
1866 walk,
1867 DAGWalk::Active { next_vertex, .. } if next_vertex == &vertex
1868 )
1869 });
1870 if !is_active {
1871 return Err(NexusError::Configuration(format!(
1872 "Runtime vertex '{vertex}' is not active in execution '{dag_execution_id}'"
1873 )));
1874 }
1875
1876 let dag_id = execution.object.data.dag_id();
1877 let dag = fetch_dag_in_context(client, &execution.context, dag_id).await?;
1878 let vertex_info =
1879 fetch_dag_vertex_bcs(crawler, &execution.context, &dag.data, vertex.vertex())
1880 .await
1881 .map_err(NexusError::Rpc)?
1882 .ok_or_else(|| {
1883 NexusError::Configuration(format!(
1884 "Vertex '{}' is absent from DAG '{}'",
1885 vertex.vertex_name(),
1886 dag_id
1887 ))
1888 })?;
1889 let tool_fqn = vertex_info.kind.tool_fqn().map_err(NexusError::Parsing)?;
1890 let cashier = client.fetch_tool_cashier(&tool_fqn).await?;
1891 let cashier_id = *cashier.object_id();
1892 let payment = tap::fetch_execution_payment_for_execution(
1893 client,
1894 &execution.context,
1895 dag_execution_id,
1896 )
1897 .await
1898 .map_err(NexusError::Rpc)?;
1899 let runtime = client.runtime_context(&required_roots).await?;
1900 let policy = resolve_invocation_policy_call(
1901 client,
1902 &runtime,
1903 cashier_id,
1904 payment.data.source_kind,
1905 policy,
1906 )
1907 .await?;
1908 let transaction = invocation::authorize_ptb(
1909 &runtime,
1910 &cashier,
1911 &dag.object_ref(),
1912 &execution.object.object_ref(),
1913 &leader_cap.object_ref(),
1914 invocation::InvocationTarget {
1915 walk_index,
1916 vertex: &vertex,
1917 },
1918 &policy,
1919 0,
1920 )
1921 .map_err(NexusError::TransactionBuilding)?;
1922 let sender = client.owner()?;
1923 let response = client.submit_transaction(transaction, sender).await?;
1924 let lock = response
1925 .events
1926 .iter()
1927 .find_map(|event| match &event.data {
1928 NexusEventKind::InvocationLocked(lock)
1929 if lock.execution.bytes == dag_execution_id && lock.vertex == vertex =>
1930 {
1931 Some(lock.clone())
1932 }
1933 _ => None,
1934 })
1935 .ok_or_else(|| {
1936 NexusError::Parsing(anyhow!(
1937 "Authorization transaction '{}' emitted no exact Invocation lock",
1938 response.digest
1939 ))
1940 })?;
1941
1942 Ok(AuthorizeInvocationResult {
1943 tx_digest: response.digest,
1944 tx_checkpoint: response.checkpoint,
1945 lock,
1946 })
1947 }
1948
1949 pub async fn inspect_execution(
1955 &self,
1956 dag_execution_id: sui::types::Address,
1957 options: InspectExecutionOptions,
1958 ) -> Result<InspectExecutionResult, NexusError> {
1959 if options.poll_interval.is_zero() {
1960 return Err(NexusError::Configuration(
1961 "Execution inspection poll interval must be greater than zero".to_string(),
1962 ));
1963 }
1964
1965 let deadline = Instant::now() + options.timeout;
1966 let (tx, rx) = unbounded_channel::<NexusEvent>();
1967 let client = self.client.clone();
1968 let crawler = self.client.crawler().clone();
1969 let event_decoder = NexusEventDecoder::new(
1970 self.client.state_resolver().clone(),
1971 self.client.get_nexus_objects(),
1972 );
1973
1974 let poller = tokio::spawn(async move {
1975 let inspection = async move {
1976 let context = loop {
1977 match client.context_for_object(dag_execution_id).await {
1978 Ok(context) => break context,
1979 Err(error) if is_transient_inspection_error(&error) => {
1980 tokio::time::sleep(options.poll_interval).await;
1981 }
1982 Err(error) => return Err(error),
1983 }
1984 };
1985 let mut last_delivered_version = None;
1986
1987 loop {
1988 let latest = match crawler
1989 .get_object_update_reference(dag_execution_id, None)
1990 .await
1991 .map_err(|error| {
1992 NexusError::Rpc(error.context(format!(
1993 "Could not inspect execution '{dag_execution_id}' latest object; last successfully reconstructed version {}",
1994 version_or_none(last_delivered_version)
1995 )))
1996 }) {
1997 Ok(latest) => latest,
1998 Err(error) if is_transient_inspection_error(&error) => {
1999 tokio::time::sleep(options.poll_interval).await;
2000 continue;
2001 }
2002 Err(error) => return Err(error),
2003 };
2004
2005 if last_delivered_version == Some(latest.version) {
2006 tokio::time::sleep(options.poll_interval).await;
2007 continue;
2008 }
2009
2010 let latest_version = latest.version;
2011 let events = loop {
2012 match fetch_execution_update_events(
2013 &crawler,
2014 &event_decoder,
2015 ObjectHistoryRequest {
2016 object_name: "Execution",
2017 object_id: dag_execution_id,
2018 expected_type: crate::move_bindings::struct_tag::<DAGExecution>(
2019 &context,
2020 ),
2021 latest: latest.clone(),
2022 after_version: last_delivered_version,
2023 poll_interval: options.poll_interval,
2024 deadline,
2025 },
2026 )
2027 .await
2028 {
2029 Ok(events) => break events,
2030 Err(error) if is_transient_inspection_error(&error) => {
2031 tokio::time::sleep(options.poll_interval).await;
2032 }
2033 Err(error) => return Err(error),
2034 }
2035 };
2036 let execution_finished_seen = events
2037 .iter()
2038 .any(|event| matches!(event.data, NexusEventKind::ExecutionFinished(_)));
2039
2040 for event in events {
2041 tx.send(event)
2042 .map_err(|error| NexusError::Channel(error.into()))?;
2043 }
2044 last_delivered_version = Some(latest_version);
2045
2046 if execution_finished_seen {
2047 return Ok(());
2048 }
2049
2050 tokio::time::sleep(options.poll_interval).await;
2051 }
2052 };
2053
2054 tokio::time::timeout_at(deadline, inspection)
2055 .await
2056 .map_err(|_| {
2057 NexusError::Timeout(anyhow!(
2058 "Timeout {:?} reached while inspecting DAG execution '{dag_execution_id}'",
2059 options.timeout
2060 ))
2061 })?
2062 });
2063
2064 Ok(InspectExecutionResult {
2065 next_event: rx,
2066 poller,
2067 })
2068 }
2069
2070 #[cfg(feature = "walrus")]
2073 pub async fn inspect_execution_until_completion(
2074 &self,
2075 dag_execution_id: sui::types::Address,
2076 options: InspectExecutionOptions,
2077 storage_conf: &StorageConf,
2078 ) -> Result<InspectExecutionCompletionResult, NexusError> {
2079 let mut inspection = self.inspect_execution(dag_execution_id, options).await?;
2080
2081 let mut events = Vec::new();
2082
2083 while let Some(event) = inspection.next_event.recv().await {
2084 events.push(event);
2085 }
2086
2087 let poller_result = inspection.poller.await.map_err(|e| {
2088 NexusError::Channel(anyhow!(
2089 "Execution inspection task failed for DAG execution '{dag_execution_id}': {e}"
2090 ))
2091 })?;
2092 poller_result?;
2093
2094 build_execution_completion_result(events, dag_execution_id, storage_conf).await
2095 }
2096
2097 pub async fn execution_cost(
2100 &self,
2101 dag_execution_id: sui::types::Address,
2102 ) -> Result<ExecutionCostResult, NexusError> {
2103 Ok(self.execution_cost_details(dag_execution_id).await?.summary)
2104 }
2105
2106 pub async fn execution_cost_details(
2108 &self,
2109 dag_execution_id: sui::types::Address,
2110 ) -> Result<ExecutionCostDetails, NexusError> {
2111 let execution = fetch_execution(&self.client, dag_execution_id, &[]).await?;
2112 if execution
2113 .object
2114 .data
2115 .to_context()
2116 .map_err(NexusError::Parsing)?
2117 .is_none()
2118 {
2119 return Err(NexusError::Parsing(anyhow!(
2120 "DAG execution '{dag_execution_id}' has no TAP payment context"
2121 )));
2122 }
2123 let payment = tap::fetch_execution_payment_for_execution(
2124 &self.client,
2125 &execution.context,
2126 dag_execution_id,
2127 )
2128 .await
2129 .map_err(NexusError::Rpc)?;
2130
2131 Ok(ExecutionCostDetails::from_payment(
2132 payment.object_id,
2133 payment.data,
2134 ))
2135 }
2136
2137 pub async fn abort_expired_execution(
2147 &self,
2148 dag_execution_id: sui::types::Address,
2149 ) -> Result<AbortExecutionResult, NexusError> {
2150 self.abort_expired_execution_with(&self.client, dag_execution_id)
2151 .await
2152 }
2153
2154 async fn abort_expired_execution_with(
2155 &self,
2156 client: &NexusClient,
2157 dag_execution_id: sui::types::Address,
2158 ) -> Result<AbortExecutionResult, NexusError> {
2159 let crawler = client.crawler();
2160 let clock = crawler
2161 .get_object::<SuiClock>(move_boundary::CLOCK_OBJECT_ID)
2162 .await
2163 .map_err(NexusError::Rpc)?
2164 .data;
2165 let mut execution = fetch_execution(client, dag_execution_id, &[]).await?;
2166 let mut cleaned_broken_onchain_results = broken_onchain_result_cleanups_for_abort(
2167 client,
2168 &execution.context,
2169 dag_execution_id,
2170 &execution.object.data,
2171 clock.timestamp_ms,
2172 )
2173 .await
2174 .map_err(NexusError::TransactionBuilding)?;
2175
2176 if !cleaned_broken_onchain_results.is_empty() {
2177 execution = fetch_execution(client, dag_execution_id, &[]).await?;
2178 cleaned_broken_onchain_results = broken_onchain_result_cleanups_for_abort(
2179 client,
2180 &execution.context,
2181 dag_execution_id,
2182 &execution.object.data,
2183 clock.timestamp_ms,
2184 )
2185 .await
2186 .map_err(NexusError::TransactionBuilding)?;
2187 }
2188
2189 let dag_id = execution.object.data.dag_id();
2190 let dag = fetch_dag_in_context(client, &execution.context, dag_id).await?;
2191 let execution_ref = execution.object.object_ref();
2192 let objects = client.get_nexus_objects();
2193 let required_roots = if cleaned_broken_onchain_results.is_empty() {
2194 Vec::new()
2195 } else {
2196 vec![objects.tool_registry, objects.leader_registry]
2197 };
2198 let runtime = client.runtime_context(&required_roots).await?;
2199
2200 let address = client.owner()?;
2201 let tx = dag::abort_expired_execution_for_self_ptb(
2202 &runtime,
2203 &dag.object_ref(),
2204 &execution_ref,
2205 &cleaned_broken_onchain_results
2206 .iter()
2207 .map(|cleanup| dag::BrokenOnchainToolResultCleanupInput {
2208 walk_index: cleanup.walk_index,
2209 result_ref: cleanup.result_ref.clone(),
2210 tool_witness_id: cleanup.tool_witness_id,
2211 })
2212 .collect::<Vec<_>>(),
2213 None,
2214 )
2215 .map_err(NexusError::TransactionBuilding)?;
2216 let response = client.submit_transaction(tx, address).await?;
2217
2218 Ok(AbortExecutionResult {
2219 tx_digest: response.digest,
2220 tx_checkpoint: response.checkpoint,
2221 dag_id,
2222 dag_execution_id,
2223 cleaned_broken_onchain_results,
2224 })
2225 }
2226
2227 pub async fn settle_committed_tool_result_for_walk(
2229 &self,
2230 params: SettleCommittedToolResultParams,
2231 ) -> Result<CommittedToolResultSettlementResult, NexusError> {
2232 self.settle_committed_tool_result_for_walk_with(&self.client, params)
2233 .await
2234 }
2235
2236 async fn settle_committed_tool_result_for_walk_with(
2237 &self,
2238 client: &NexusClient,
2239 params: SettleCommittedToolResultParams,
2240 ) -> Result<CommittedToolResultSettlementResult, NexusError> {
2241 let objects = client.get_nexus_objects();
2242 let required_roots = [
2243 objects.tool_registry,
2244 objects.leader_registry,
2245 objects.priority_fee_vault,
2246 ];
2247 let execution = fetch_execution(client, params.dag_execution_id, &[]).await?;
2248 let runtime = client.runtime_context(&required_roots).await?;
2249 let dag_id = execution.object.data.dag_id();
2250 let dag = fetch_dag_in_context(client, &execution.context, dag_id).await?;
2251 let expected_vertex = pending_settlement_vertex(&execution.object.data, params.walk_index)?;
2252 let invocation_ref = fetch_invocation_for_vertex(
2253 client,
2254 &execution.context,
2255 params.dag_execution_id,
2256 dag_id,
2257 &expected_vertex,
2258 )
2259 .await?;
2260
2261 let address = client.owner()?;
2262 let tx = dag::settle_committed_tool_result_for_walk_for_self_ptb(
2263 &runtime,
2264 &dag.object_ref(),
2265 &execution.object.object_ref(),
2266 &invocation_ref,
2267 params.walk_index,
2268 &expected_vertex,
2269 None,
2270 )
2271 .map_err(NexusError::TransactionBuilding)?;
2272 let response = client.submit_transaction(tx, address).await?;
2273
2274 Ok(CommittedToolResultSettlementResult {
2275 tx_digest: response.digest,
2276 tx_checkpoint: response.checkpoint,
2277 dag_id,
2278 dag_execution_id: params.dag_execution_id,
2279 walk_index: params.walk_index,
2280 })
2281 }
2282
2283 pub async fn settle_committed_tool_result_for_walk_by_leader(
2285 &self,
2286 params: SettleCommittedToolResultByLeaderParams,
2287 ) -> Result<CommittedToolResultSettlementResult, NexusError> {
2288 let client = &self.client;
2289 let crawler = client.crawler();
2290 let objects = client.get_nexus_objects();
2291 let required_roots = [
2292 objects.tool_registry,
2293 objects.leader_registry,
2294 objects.priority_fee_vault,
2295 ];
2296 let execution = fetch_execution(client, params.dag_execution_id, &[]).await?;
2297 let runtime = client.runtime_context(&required_roots).await?;
2298 let dag_id = execution.object.data.dag_id();
2299 let dag = fetch_dag_in_context(client, &execution.context, dag_id).await?;
2300 let settlement_vertex =
2301 pending_settlement_vertex(&execution.object.data, params.walk_index)?;
2302 let invocation_ref = fetch_invocation_for_vertex(
2303 client,
2304 &execution.context,
2305 params.dag_execution_id,
2306 dag_id,
2307 &settlement_vertex,
2308 )
2309 .await?;
2310 let leader_cap_ref = crawler
2311 .get_object_metadata(params.leader_cap_id)
2312 .await
2313 .map_err(NexusError::Rpc)?;
2314 let (expected_vertex, failed_onchain_tool_reason) =
2315 match (params.expected_vertex, params.failed_onchain_tool_reason) {
2316 (Some(expected_vertex), Some(reason)) => {
2317 if expected_vertex != settlement_vertex {
2318 return Err(NexusError::TransactionBuilding(anyhow::anyhow!(
2319 "expected_vertex does not match the pending settlement vertex"
2320 )));
2321 }
2322 (expected_vertex, Some(reason))
2323 }
2324 (None, None) => (settlement_vertex, None),
2325 _ => {
2326 return Err(NexusError::TransactionBuilding(anyhow::anyhow!(
2327 "expected_vertex and failed_onchain_tool_reason must be supplied together"
2328 )));
2329 }
2330 };
2331 let address = client.owner()?;
2332 let tx = dag::settle_committed_tool_result_for_walk_by_leader_for_self_ptb(
2333 &runtime,
2334 &dag.object_ref(),
2335 &execution.object.object_ref(),
2336 &execution.object.owner,
2337 &leader_cap_ref.object_ref(),
2338 &leader_cap_ref.owner,
2339 &invocation_ref,
2340 params.walk_index,
2341 &expected_vertex,
2342 failed_onchain_tool_reason,
2343 params.commit_tx_digest,
2344 params.commit_gas_charge,
2345 params.settlement_gas_charge,
2346 )
2347 .map_err(NexusError::TransactionBuilding)?;
2348 let response = client.submit_transaction(tx, address).await?;
2349
2350 Ok(CommittedToolResultSettlementResult {
2351 tx_digest: response.digest,
2352 tx_checkpoint: response.checkpoint,
2353 dag_id,
2354 dag_execution_id: params.dag_execution_id,
2355 walk_index: params.walk_index,
2356 })
2357 }
2358
2359 pub async fn record_committed_tool_result_gas_charge_by_leader(
2361 &self,
2362 params: RecordCommittedToolResultGasChargeParams,
2363 ) -> Result<RecordCommittedToolResultGasChargeResult, NexusError> {
2364 let client = &self.client;
2365 let crawler = client.crawler();
2366 let objects = client.get_nexus_objects();
2367 let required_roots = [objects.leader_registry];
2368 let execution = fetch_execution(client, params.dag_execution_id, &[]).await?;
2369 let runtime = client.runtime_context(&required_roots).await?;
2370 let dag_id = execution.object.data.dag_id();
2371 let dag = fetch_dag_in_context(client, &execution.context, dag_id).await?;
2372 let leader_cap_ref = crawler
2373 .get_object_metadata(params.leader_cap_id)
2374 .await
2375 .map_err(NexusError::Rpc)?;
2376 let (expected_vertex, failed_onchain_tool_reason) =
2377 match (params.expected_vertex, params.failed_onchain_tool_reason) {
2378 (Some(expected_vertex), Some(reason)) => (expected_vertex, Some(reason)),
2379 (None, None) => (RuntimeVertex::plain(""), None),
2380 _ => {
2381 return Err(NexusError::TransactionBuilding(anyhow::anyhow!(
2382 "expected_vertex and failed_onchain_tool_reason must be supplied together"
2383 )));
2384 }
2385 };
2386
2387 let address = client.owner()?;
2388 let tx = dag::record_committed_tool_result_gas_charge_by_leader_for_self_ptb(
2389 &runtime,
2390 &dag.object_ref(),
2391 &execution.object.object_ref(),
2392 &execution.object.owner,
2393 &leader_cap_ref.object_ref(),
2394 &leader_cap_ref.owner,
2395 params.walk_index,
2396 &expected_vertex,
2397 failed_onchain_tool_reason,
2398 params.commit_tx_digest,
2399 params.commit_gas_charge,
2400 params.settlement_gas_charge,
2401 )
2402 .map_err(NexusError::TransactionBuilding)?;
2403 let response = client.submit_transaction(tx, address).await?;
2404
2405 Ok(RecordCommittedToolResultGasChargeResult {
2406 tx_digest: response.digest,
2407 tx_checkpoint: response.checkpoint,
2408 dag_execution_id: params.dag_execution_id,
2409 leader_cap_id: params.leader_cap_id,
2410 walk_index: params.walk_index,
2411 })
2412 }
2413
2414 pub async fn inspect_expired_walk_resolution(
2416 &self,
2417 params: ResolveExpiredWalkParams,
2418 ) -> Result<ExpiredWalkResolutionPlan, NexusError> {
2419 self.inspect_expired_walk_resolution_with(&self.client, params)
2420 .await
2421 }
2422
2423 async fn inspect_expired_walk_resolution_with(
2424 &self,
2425 client: &NexusClient,
2426 params: ResolveExpiredWalkParams,
2427 ) -> Result<ExpiredWalkResolutionPlan, NexusError> {
2428 inspect_expired_walk_resolution(client, params)
2429 .await
2430 .map_err(NexusError::Rpc)
2431 }
2432
2433 pub async fn resolve_expired_execution(
2440 &self,
2441 execution_id: sui::types::Address,
2442 ) -> Result<Vec<ExpiredWalkResolutionResult>, NexusError> {
2443 let mut results = Vec::new();
2444 let mut previous = None;
2445 loop {
2446 let clock = self
2447 .client
2448 .crawler()
2449 .get_object::<SuiClock>(move_boundary::CLOCK_OBJECT_ID)
2450 .await
2451 .map_err(NexusError::Rpc)?;
2452 let Some(plan) = inspect_expired_execution_resolution_at(
2453 &self.client,
2454 execution_id,
2455 clock.data.timestamp_ms,
2456 )
2457 .await
2458 .map_err(NexusError::Rpc)?
2459 else {
2460 break;
2461 };
2462 if previous.as_ref() == Some(&plan) {
2463 break;
2464 }
2465 let result = self
2466 .resolve_expired_walk(ResolveExpiredWalkParams {
2467 dag_execution_id: execution_id,
2468 walk_index: plan.walk_index,
2469 invocation_id: None,
2470 })
2471 .await?;
2472 previous = Some(plan);
2473 let progressed = result.tx_digest.is_some();
2474 results.push(result);
2475 if !progressed {
2476 break;
2477 }
2478 }
2479 Ok(results)
2480 }
2481
2482 pub async fn resolve_expired_walk(
2484 &self,
2485 params: ResolveExpiredWalkParams,
2486 ) -> Result<ExpiredWalkResolutionResult, NexusError> {
2487 let client = &self.client;
2488 let plan = self
2489 .inspect_expired_walk_resolution_with(client, params)
2490 .await?;
2491 let base = |resolution_kind| ExpiredWalkResolutionResult {
2492 tx_digest: None,
2493 tx_checkpoint: None,
2494 dag_id: plan.dag_id,
2495 dag_execution_id: plan.dag_execution_id,
2496 walk_index: plan.walk_index,
2497 resolution_kind,
2498 };
2499
2500 match plan.kind {
2501 ExpiredWalkResolutionKind::Settled {
2502 expected_vertex,
2503 invocation_ref,
2504 } => {
2505 let settled = self
2506 .settle_committed_tool_result_for_walk_with(
2507 client,
2508 SettleCommittedToolResultParams {
2509 dag_execution_id: plan.dag_execution_id,
2510 walk_index: plan.walk_index,
2511 },
2512 )
2513 .await?;
2514 Ok(ExpiredWalkResolutionResult {
2515 tx_digest: Some(settled.tx_digest),
2516 tx_checkpoint: Some(settled.tx_checkpoint),
2517 ..base(ExpiredWalkResolutionKind::Settled {
2518 expected_vertex,
2519 invocation_ref,
2520 })
2521 })
2522 }
2523 ExpiredWalkResolutionKind::SettledOnchainResult {
2524 result_ref,
2525 expected_vertex,
2526 invocation_ref,
2527 tool_witness_id,
2528 finalize_tx_digest,
2529 } => {
2530 let objects = client.get_nexus_objects();
2531 let required_roots = [
2532 objects.tool_registry,
2533 objects.leader_registry,
2534 objects.priority_fee_vault,
2535 ];
2536 let execution = fetch_execution(client, plan.dag_execution_id, &[]).await?;
2537 let runtime = client.runtime_context(&required_roots).await?;
2538 if execution.object.data.dag_id() != plan.dag_id {
2539 return Err(NexusError::Parsing(anyhow!(
2540 "DAG execution '{}' now references DAG '{}', expected '{}'",
2541 plan.dag_execution_id,
2542 execution.object.data.dag_id(),
2543 plan.dag_id
2544 )));
2545 }
2546 let dag = fetch_dag_in_context(client, &execution.context, plan.dag_id).await?;
2547 let result = client
2548 .state_resolver()
2549 .load_inner::<OnchainToolResult, InterfaceWitnessV1, OnchainToolResultInnerV1>(
2550 *result_ref.object_id(),
2551 &execution.context,
2552 )
2553 .await?;
2554 let address = client.owner()?;
2555 let context = &runtime;
2556 let tx = move_boundary::ptb(context, |tx| {
2557 let dag_arg = tx.immutable_object(&dag.object_ref())?;
2558 let execution_arg = tx.shared_object(&execution.object.object_ref(), true)?;
2559 let tool_registry = tx.shared_root(&context.tool_registry, false)?;
2560 let result_arg = tx.shared_object(&result.object_ref(), true)?;
2561 let leader_registry = tx.shared_root(&context.leader_registry, false)?;
2562 let priority_fee_vault = tx.shared_root(&context.priority_fee_vault, false)?;
2563 let clock = tx.clock()?;
2564 dag::settle_onchain_tool_result_for_walk(
2565 tx,
2566 dag_arg,
2567 execution_arg,
2568 tool_registry,
2569 result_arg,
2570 leader_registry,
2571 priority_fee_vault,
2572 plan.walk_index,
2573 &expected_vertex,
2574 tool_witness_id,
2575 clock,
2576 )?;
2577 invocation::settle(
2578 tx,
2579 dag_arg,
2580 execution_arg,
2581 &expected_vertex,
2582 &invocation_ref,
2583 )?;
2584 Ok(())
2585 })
2586 .map_err(NexusError::TransactionBuilding)?;
2587 let response = client.submit_transaction(tx, address).await?;
2588 Ok(ExpiredWalkResolutionResult {
2589 tx_digest: Some(response.digest),
2590 tx_checkpoint: Some(response.checkpoint),
2591 ..base(ExpiredWalkResolutionKind::SettledOnchainResult {
2592 result_ref: result.object_ref(),
2593 expected_vertex,
2594 invocation_ref,
2595 tool_witness_id,
2596 finalize_tx_digest,
2597 })
2598 })
2599 }
2600 ExpiredWalkResolutionKind::Aborted => {
2601 let aborted = self
2602 .abort_expired_execution_with(client, plan.dag_execution_id)
2603 .await?;
2604 Ok(ExpiredWalkResolutionResult {
2605 tx_digest: Some(aborted.tx_digest),
2606 tx_checkpoint: Some(aborted.tx_checkpoint),
2607 ..base(ExpiredWalkResolutionKind::Aborted)
2608 })
2609 }
2610 ExpiredWalkResolutionKind::AbortedWithInvocation { selected_candidate } => {
2611 let invocation_id = *selected_candidate.invocation_ref.object_id();
2612 let aborted = self
2613 .abort_expired_execution_with_invocation_with(
2614 client,
2615 plan.dag_execution_id,
2616 Some(invocation_id),
2617 )
2618 .await?;
2619 Ok(ExpiredWalkResolutionResult {
2620 tx_digest: Some(aborted.tx_digest),
2621 tx_checkpoint: Some(aborted.tx_checkpoint),
2622 ..base(ExpiredWalkResolutionKind::AbortedWithInvocation { selected_candidate })
2623 })
2624 }
2625 ExpiredWalkResolutionKind::Skipped { reason } => {
2626 Ok(base(ExpiredWalkResolutionKind::Skipped { reason }))
2627 }
2628 }
2629 }
2630
2631 pub async fn abort_expired_execution_invocation_candidates(
2633 &self,
2634 dag_execution_id: sui::types::Address,
2635 ) -> Result<Vec<InvocationAbortCandidate>, NexusError> {
2636 self.abort_expired_execution_invocation_candidates_with(&self.client, dag_execution_id)
2637 .await
2638 }
2639
2640 async fn abort_expired_execution_invocation_candidates_with(
2641 &self,
2642 client: &NexusClient,
2643 dag_execution_id: sui::types::Address,
2644 ) -> Result<Vec<InvocationAbortCandidate>, NexusError> {
2645 let execution = fetch_execution(client, dag_execution_id, &[]).await?;
2646 let context = &execution.context;
2647 let execution_state = &execution.object.data;
2648 let crawler = client.crawler();
2649 let dag = fetch_dag_in_context(client, context, execution_state.dag_id()).await?;
2650 let clock = crawler
2651 .get_object::<SuiClock>(move_boundary::CLOCK_OBJECT_ID)
2652 .await
2653 .map_err(NexusError::Rpc)?
2654 .data;
2655 let requested_vertices = execution_state
2656 .walks
2657 .iter()
2658 .filter_map(|walk| walk.abortable_timeout_expired_vertex(clock.timestamp_ms))
2659 .map(|vertex| vertex.vertex().clone());
2660 let vertices =
2661 fetch_dag_vertices_by_keys_bcs(crawler, context, &dag.data, requested_vertices)
2662 .await
2663 .map_err(NexusError::Rpc)?;
2664
2665 let payment = tap::fetch_execution_payment_for_execution(client, context, dag_execution_id)
2666 .await
2667 .map_err(NexusError::Rpc)?
2668 .data;
2669 let candidates = fetch_invocation_abort_candidates(
2670 crawler,
2671 filter_invocation_abort_candidate_walks(
2672 dag_execution_id,
2673 &vertices,
2674 &execution_state.walks,
2675 &payment.locked_vertices,
2676 clock.timestamp_ms,
2677 )
2678 .map_err(NexusError::Parsing)?,
2679 )
2680 .await?;
2681
2682 Ok(candidates)
2683 }
2684
2685 pub async fn abort_expired_execution_with_invocation(
2687 &self,
2688 dag_execution_id: sui::types::Address,
2689 invocation_id: Option<sui::types::Address>,
2690 ) -> Result<AbortExpiredExecutionResult, NexusError> {
2691 self.abort_expired_execution_with_invocation_with(
2692 &self.client,
2693 dag_execution_id,
2694 invocation_id,
2695 )
2696 .await
2697 }
2698
2699 async fn abort_expired_execution_with_invocation_with(
2700 &self,
2701 client: &NexusClient,
2702 dag_execution_id: sui::types::Address,
2703 invocation_id: Option<sui::types::Address>,
2704 ) -> Result<AbortExpiredExecutionResult, NexusError> {
2705 let candidates = self
2706 .abort_expired_execution_invocation_candidates_with(client, dag_execution_id)
2707 .await?;
2708 let selected_candidate = select_invocation_abort_candidate(candidates, invocation_id)?;
2709 let execution = fetch_execution(client, dag_execution_id, &[]).await?;
2710 let dag_id = execution.object.data.dag_id();
2711 let dag = fetch_dag_in_context(client, &execution.context, dag_id).await?;
2712 let runtime = client.runtime_context(&[]).await?;
2713
2714 let address = client.owner()?;
2715 let tx = invocation::abort_expired_ptb(
2716 &runtime,
2717 &dag.object_ref(),
2718 &execution.object.object_ref(),
2719 &selected_candidate.vertex,
2720 &selected_candidate.invocation_ref,
2721 None,
2722 )
2723 .map_err(NexusError::TransactionBuilding)?;
2724 let response = client.submit_transaction(tx, address).await?;
2725
2726 Ok(AbortExpiredExecutionResult {
2727 tx_digest: response.digest,
2728 tx_checkpoint: response.checkpoint,
2729 dag_id,
2730 dag_execution_id,
2731 selected_candidate,
2732 })
2733 }
2734}
2735
2736fn select_invocation_abort_candidate(
2737 candidates: Vec<InvocationAbortCandidate>,
2738 invocation_id: Option<sui::types::Address>,
2739) -> Result<InvocationAbortCandidate, NexusError> {
2740 if let Some(invocation_id) = invocation_id {
2741 candidates
2742 .into_iter()
2743 .find(|candidate| *candidate.invocation_ref.object_id() == invocation_id)
2744 .ok_or_else(|| {
2745 NexusError::Parsing(anyhow!(
2746 "Invocation '{invocation_id}' is not currently eligible to abort this execution"
2747 ))
2748 })
2749 } else {
2750 candidates.into_iter().next().ok_or_else(|| {
2751 NexusError::Parsing(anyhow!(
2752 "No Invocation abort candidates are currently eligible for this execution"
2753 ))
2754 })
2755 }
2756}
2757
2758impl ExecutionCostDetails {
2759 fn from_payment(payment_id: sui::types::Address, payment: ExecutionPaymentInnerV1) -> Self {
2760 Self {
2761 summary: ExecutionCostResult {
2762 payment_id,
2763 max_budget_mist: payment.max_budget_mist,
2764 locked_budget_mist: payment.locked_budget_mist,
2765 consumed: payment.consumed,
2766 outstanding_locks: payment.locks(),
2767 accomplished: payment.accomplished,
2768 refunded: payment.refunded,
2769 },
2770 outstanding_invocation_ids: payment
2771 .locked_vertices
2772 .iter()
2773 .map(|lock| lock.invocation_id.bytes)
2774 .collect(),
2775 }
2776 }
2777}
2778
2779pub fn payment_vertex_key(
2784 execution_id: sui::types::Address,
2785 vertex: &RuntimeVertex,
2786 tool_fqn: &crate::ToolFqn,
2787) -> anyhow::Result<Vec<u8>> {
2788 let mut bytes = Vec::new();
2789 bytes.extend_from_slice(b"nexus.payment.vertex.v1");
2790 bytes.extend(bcs::to_bytes(&execution_id)?);
2791 bytes.extend(bcs::to_bytes(vertex)?);
2792 bytes.extend(tool_fqn.to_string().as_bytes());
2793 Ok(Sha256::digest(bytes).to_vec())
2794}
2795
2796fn pending_settlement_vertex(
2797 execution: &DAGExecutionInnerV1,
2798 walk_index: u64,
2799) -> Result<RuntimeVertex, NexusError> {
2800 let walk_index = usize::try_from(walk_index).map_err(|error| {
2801 NexusError::Parsing(anyhow!("Walk index does not fit this platform: {error}"))
2802 })?;
2803 match execution.walks.get(walk_index) {
2804 Some(DAGWalk::PendingSettlement { next_vertex, .. }) => Ok(next_vertex.clone()),
2805 Some(_) => Err(NexusError::Parsing(anyhow!(
2806 "Walk {walk_index} is not pending settlement"
2807 ))),
2808 None => Err(NexusError::Parsing(anyhow!(
2809 "Walk index {walk_index} is outside the execution"
2810 ))),
2811 }
2812}
2813
2814async fn fetch_invocation_for_vertex(
2815 client: &NexusClient,
2816 context: &NexusContext,
2817 execution_id: sui::types::Address,
2818 dag_id: sui::types::Address,
2819 vertex: &RuntimeVertex,
2820) -> Result<sui::types::ObjectReference, NexusError> {
2821 let crawler = client.crawler();
2822 let dag = fetch_dag_in_context(client, context, dag_id).await?;
2823 let vertex_info = fetch_dag_vertex_bcs(crawler, context, &dag.data, vertex.vertex())
2824 .await
2825 .map_err(NexusError::Rpc)?
2826 .ok_or_else(|| {
2827 NexusError::Parsing(anyhow!("DAG vertex '{}' is missing", vertex.vertex_name()))
2828 })?;
2829 let tool_fqn = vertex_info.kind.tool_fqn().map_err(NexusError::Parsing)?;
2830 let vertex_key =
2831 payment_vertex_key(execution_id, vertex, &tool_fqn).map_err(NexusError::Parsing)?;
2832 let payment = tap::fetch_execution_payment_for_execution(client, context, execution_id)
2833 .await
2834 .map_err(NexusError::Rpc)?
2835 .data;
2836 let lock = payment
2837 .locked_vertices
2838 .iter()
2839 .find(|lock| lock.vertex_key == vertex_key)
2840 .ok_or_else(|| {
2841 NexusError::Parsing(anyhow!("Pending Tool result has no exact Invocation lock"))
2842 })?;
2843 crawler
2844 .get_object_metadata(lock.invocation_id.bytes)
2845 .await
2846 .map_err(NexusError::Rpc)
2847 .map(|metadata| metadata.object_ref())
2848}
2849
2850struct InvocationAbortCandidateSeed {
2851 walk_index: usize,
2852 vertex: RuntimeVertex,
2853 payment_vertex_key: Vec<u8>,
2854 invocation_id: sui::types::Address,
2855}
2856
2857fn filter_invocation_abort_candidate_walks(
2858 execution_id: sui::types::Address,
2859 vertices: &HashMap<graph_move::Vertex, graph_move::VertexInfo>,
2860 walks: &[DAGWalk],
2861 locks: &[ExecutionPaymentVertexLock],
2862 clock_ms: u64,
2863) -> anyhow::Result<Vec<InvocationAbortCandidateSeed>> {
2864 let mut candidates = Vec::new();
2865 for (walk_index, walk) in walks.iter().enumerate() {
2866 let Some(vertex) = walk.abortable_timeout_expired_vertex(clock_ms) else {
2867 continue;
2868 };
2869 let vertex_info = vertices.get(vertex.vertex()).ok_or_else(|| {
2870 anyhow!(
2871 "DAG vertex '{}' missing from fetched DAG",
2872 vertex.vertex_name()
2873 )
2874 })?;
2875 let tool_fqn = vertex_info.kind.tool_fqn()?;
2876 let vertex_key = payment_vertex_key(execution_id, vertex, &tool_fqn)?;
2877 if let Some(lock) = locks.iter().find(|lock| lock.vertex_key == vertex_key) {
2878 candidates.push(InvocationAbortCandidateSeed {
2879 walk_index,
2880 vertex: vertex.clone(),
2881 payment_vertex_key: vertex_key,
2882 invocation_id: lock.invocation_id.bytes,
2883 });
2884 }
2885 }
2886 Ok(candidates)
2887}
2888
2889async fn fetch_invocation_abort_candidates(
2890 crawler: &Crawler,
2891 candidates: Vec<InvocationAbortCandidateSeed>,
2892) -> Result<Vec<InvocationAbortCandidate>, NexusError> {
2893 let mut result = Vec::with_capacity(candidates.len());
2894 for candidate in candidates {
2895 let invocation_ref = crawler
2896 .get_object_metadata(candidate.invocation_id)
2897 .await
2898 .map_err(NexusError::Rpc)?
2899 .object_ref();
2900 result.push(InvocationAbortCandidate {
2901 walk_index: candidate.walk_index,
2902 vertex: candidate.vertex,
2903 payment_vertex_key: candidate.payment_vertex_key,
2904 invocation_ref,
2905 });
2906 }
2907 result.sort_by_key(|candidate| candidate.walk_index);
2908 Ok(result)
2909}
2910
2911#[cfg(test)]
2912mod tests {
2913 use {
2914 super::*,
2915 crate::{
2916 events::NexusEventKind,
2917 fqn,
2918 move_bindings::{
2919 interface::{
2920 dag as dag_move,
2921 graph::{self as graph_move, PostFailureAction, RuntimeVertex},
2922 payment::{
2923 ExecutionPayment,
2924 ExecutionPaymentFeesRecordedEvent,
2925 ExecutionPaymentFinalState,
2926 ExecutionPaymentToolCostSnapshottedEvent,
2927 PaymentSourceKind,
2928 SkillPaymentPolicy,
2929 },
2930 verifier::{ToolVerifierMode, VerifierDecision},
2931 version::InterfaceVersion,
2932 },
2933 move_std::{ascii::String as MoveString, option::Option as MoveOption},
2934 registry::{
2935 era::V1 as RegistryWitnessV1,
2936 leader::{LeaderRegistry, LeaderRegistryInnerV1},
2937 priority_fee_vault::{PriorityFeeVault, PriorityFeeVaultInnerV1},
2938 },
2939 scheduler::scheduler::OccurrenceDispatchedEvent,
2940 sui_framework::{table::Table as MoveTable, vec_set::VecSet},
2941 tool::tool_registry::{
2942 Tool as ToolAnchor,
2943 ToolInnerV1,
2944 ToolRef as ToolRegistryRef,
2945 ToolRegistry,
2946 ToolRegistryInnerV1,
2947 },
2948 workflow::{
2949 execution::DagExecutionPaymentFieldKey,
2950 execution_events::{
2951 AgentSkillExecutionRequestedEvent,
2952 CommittedToolResultEvent,
2953 EndStateReachedEvent,
2954 ExecutionFinishedEvent,
2955 ExecutionPaymentRefilledEvent,
2956 InvocationAuthorizationRequiredEvent,
2957 RequestWalkExecutionEvent,
2958 SubmissionFailureEvidenceRecordedEvent,
2959 TerminalErrEvalRecordedEvent,
2960 ToolVerificationResolvedEvent,
2961 WalkAdvancedEvent,
2962 WalkPendingAbortEvent,
2963 },
2964 execution_failure::WorkflowFailureClass,
2965 invocation_adapter::{InvocationLockedEvent, InvocationSettledEvent},
2966 },
2967 },
2968 sui::traits::*,
2969 test_utils::{nexus_mocks, sui_mocks},
2970 types::NexusValue,
2971 },
2972 serde::Serialize,
2973 std::sync::{
2974 atomic::{AtomicBool, Ordering},
2975 Arc,
2976 },
2977 };
2978
2979 fn clock_bcs(timestamp_ms: u64) -> Vec<u8> {
2980 bcs::to_bytes(&SuiClock::new(move_boundary::CLOCK_OBJECT_ID, timestamp_ms))
2981 .expect("clock BCS should serialize")
2982 }
2983
2984 #[test]
2985 fn generated_sui_clock_bcs_matches_live_object_shape() {
2986 let bytes = clock_bcs(61_000);
2987
2988 assert_eq!(bytes.len(), 40);
2989 let clock = bcs::from_bytes::<SuiClock>(&bytes).expect("clock BCS should decode");
2990 assert_eq!(clock.id.address(), move_boundary::CLOCK_OBJECT_ID);
2991 assert_eq!(clock.timestamp_ms, 61_000);
2992 }
2993
2994 #[test]
2995 fn unresolved_timeout_skip_reason_distinguishes_pending_from_terminal_walks() {
2996 let active = DAGWalk::Active {
2997 next_vertex: RuntimeVertex::plain("tool"),
2998 timeout_ms: 30_000,
2999 requires_vertex_authorization_grant: false,
3000 created_at: 1_000,
3001 };
3002 let pending_settlement = DAGWalk::PendingSettlement {
3003 next_vertex: RuntimeVertex::plain("tool"),
3004 timeout_ms: 30_000,
3005 requires_vertex_authorization_grant: false,
3006 created_at: 1_000,
3007 };
3008
3009 assert_eq!(
3010 unresolved_timeout_skip_reason(&active),
3011 EXPIRED_WALK_NOT_DOUBLE_TIMEOUT_EXPIRED_REASON
3012 );
3013 assert_eq!(
3014 unresolved_timeout_skip_reason(&pending_settlement),
3015 EXPIRED_WALK_NOT_DOUBLE_TIMEOUT_EXPIRED_REASON
3016 );
3017 assert_eq!(
3018 unresolved_timeout_skip_reason(&DAGWalk::Successful),
3019 EXPIRED_WALK_ALREADY_TERMINAL_REASON
3020 );
3021 }
3022
3023 fn committed_tool_result_bcs(
3024 expected_vertex: RuntimeVertex,
3025 primary_leader: sui::types::Address,
3026 _secondary_leader: sui::types::Address,
3027 primary_failure: Option<FailureEvidenceKind>,
3028 secondary_failure: Option<FailureEvidenceKind>,
3029 ) -> execution_move::CommittedToolResult {
3030 execution_move::CommittedToolResult {
3031 expected_vertex,
3032 variant: graph_move::OutputVariant::new("ok"),
3033 variant_ports_to_data: crate::move_bindings::sui_framework::vec_map::VecMap {
3034 contents: vec![],
3035 },
3036 failure_evidence_kind: MoveOption::from_option(primary_failure),
3037 primary_failure_evidence_kind: MoveOption::from_option(primary_failure),
3038 secondary_failure_evidence_kind: MoveOption::from_option(secondary_failure),
3039 current_leader_cap_id: object_id(primary_leader),
3040 has_finalized_onchain_payload: false,
3041 leader_records: crate::move_bindings::sui_framework::vec_map::VecMap {
3042 contents: vec![crate::move_bindings::sui_framework::vec_map::Entry {
3043 key: object_id(primary_leader),
3044 value: execution_move::CommittedToolResultLeaderRecord {
3045 commit_tx_digest: vec![1, 2, 3],
3046 recipient: sui::types::Address::from_static("0x44"),
3047 commit_gas_charge: MoveOption::from_option(Some(10)),
3048 settlement_gas_charge: MoveOption::from_option(None),
3049 },
3050 }],
3051 },
3052 }
3053 }
3054
3055 fn raw_inline_nexus_data_bcs(
3056 one: impl Into<Vec<u8>>,
3057 ) -> crate::move_bindings::primitives::data::NexusData {
3058 crate::move_bindings::primitives::data::NexusData::inline_data(one.into())
3059 .expect("fixture is bounded")
3060 }
3061
3062 fn object_id(bytes: sui::types::Address) -> crate::move_bindings::sui_framework::object::ID {
3063 crate::move_bindings::sui_framework::object::ID::new(bytes)
3064 }
3065
3066 fn invocation_lock(
3067 vertex_key: Vec<u8>,
3068 invocation_id: sui::types::Address,
3069 ) -> ExecutionPaymentVertexLock {
3070 ExecutionPaymentVertexLock {
3071 vertex_key,
3072 invocation_id: object_id(invocation_id),
3073 amount: 10,
3074 }
3075 }
3076
3077 fn output_variant(name: &str) -> crate::move_bindings::interface::graph::OutputVariant {
3078 crate::move_bindings::interface::graph::OutputVariant {
3079 name: MoveString::from(name),
3080 }
3081 }
3082
3083 fn dag_bcs(vertices_size: u64) -> dag_move::DAGInnerV1 {
3084 dag_move::DAGInnerV1 {
3085 finalized: true,
3086 vertices: linked_table::LinkedTable::new(sui_mocks::mock_sui_address(), vertices_size),
3087 entry_groups: crate::move_bindings::sui_framework::vec_map::VecMap { contents: vec![] },
3088 edges: MoveTable::new(sui_mocks::mock_sui_address(), 0),
3089 outputs: MoveTable::new(sui_mocks::mock_sui_address(), 0),
3090 defaults_to_input_ports: MoveTable::new(sui_mocks::mock_sui_address(), 0),
3091 post_failure_action: MoveOption::from_option(None::<graph_move::PostFailureAction>),
3092 }
3093 }
3094
3095 fn mock_get_dag_bcs(
3096 ledger_service_mock: &mut sui_mocks::grpc::MockLedgerService,
3097 state_service_mock: &mut sui_mocks::grpc::MockStateService,
3098 nexus_objects: &NexusContext,
3099 dag_ref: sui::types::ObjectReference,
3100 state: dag_move::DAGInnerV1,
3101 ) {
3102 let anchor = dag_move::DAG::new(crate::move_bindings::sui_framework::object::UID::new(
3103 *dag_ref.object_id(),
3104 ));
3105 sui_mocks::grpc::mock_object_state::<dag_move::DAG, InterfaceWitnessV1, dag_move::DAGInnerV1>(
3106 ledger_service_mock,
3107 state_service_mock,
3108 nexus_objects,
3109 dag_ref,
3110 sui::types::Owner::Shared(0),
3111 anchor,
3112 state,
3113 );
3114 }
3115
3116 #[tokio::test]
3117 async fn inspect_dag_succeeds_without_owned_coins() {
3118 let nexus_objects = sui_mocks::mock_nexus_context();
3119 let dag_ref = sui_mocks::mock_sui_object_ref();
3120 let mut state = dag_bcs(1);
3121 let vertices_id = state.vertices.id();
3122 let vertex = graph_move::Vertex::new("sum");
3123 state.vertices.head = Some(vertex.clone()).into();
3124 state.vertices.tail = Some(vertex.clone()).into();
3125 state
3126 .entry_groups
3127 .contents
3128 .push(crate::move_bindings::sui_framework::vec_map::Entry {
3129 key: graph_move::EntryGroup::new("main"),
3130 value: crate::move_bindings::sui_framework::vec_map::VecMap {
3131 contents: vec![crate::move_bindings::sui_framework::vec_map::Entry {
3132 key: graph_move::Vertex::new("sum"),
3133 value: VecSet {
3134 contents: vec![
3135 graph_move::InputPort::new("right"),
3136 graph_move::InputPort::new("left"),
3137 ],
3138 },
3139 }],
3140 },
3141 });
3142 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
3143 let mut state_service_mock = sui_mocks::grpc::MockStateService::new();
3144 sui_mocks::grpc::mock_get_dynamic_field_by_key(
3145 &mut ledger_service_mock,
3146 vertices_id,
3147 &crate::move_bindings::type_tag::<graph_move::Vertex>(&nexus_objects),
3148 vertex,
3149 offchain_vertex_node_bcs(&fqn!("xyz.taluslabs.fixture@1")),
3150 );
3151 mock_get_dag_bcs(
3152 &mut ledger_service_mock,
3153 &mut state_service_mock,
3154 &nexus_objects,
3155 dag_ref.clone(),
3156 state,
3157 );
3158 let package_service_mock = mock_package_graph(&mut ledger_service_mock, &nexus_objects);
3159 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
3160 ledger_service_mock: Some(ledger_service_mock),
3161 package_service_mock: Some(package_service_mock),
3162 state_service_mock: Some(state_service_mock),
3163 ..Default::default()
3164 });
3165 let client = nexus_mocks::mock_nexus_client_without_coins(&nexus_objects, &rpc_url).await;
3166
3167 let snapshot = client
3168 .workflow()
3169 .inspect_dag(*dag_ref.object_id())
3170 .await
3171 .expect("DAG inspection should not require a gas source");
3172
3173 assert!(client.gas_config().is_none());
3174 assert_eq!(snapshot.dag_id, *dag_ref.object_id());
3175 assert_eq!(snapshot.vertex_count, 1);
3176 assert_eq!(
3177 snapshot.entry_groups["main"]["sum"],
3178 ["left".to_owned(), "right".to_owned()]
3179 );
3180 }
3181
3182 #[tokio::test]
3183 async fn authorize_invocation_maps_leader_cap_metadata_failure() {
3184 let nexus_objects = sui_mocks::mock_nexus_context();
3185 let leader_cap_id = sui::types::Address::from_static("0x43");
3186 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
3187 sui_mocks::grpc::mock_get_object_not_found(&mut ledger_service_mock, leader_cap_id);
3188 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
3189 ledger_service_mock: Some(ledger_service_mock),
3190 ..Default::default()
3191 });
3192 let client = nexus_mocks::mock_nexus_client_without_coins(&nexus_objects, &rpc_url).await;
3193
3194 let error = client
3195 .workflow()
3196 .authorize_invocation(
3197 sui::types::Address::from_static("0x42"),
3198 leader_cap_id,
3199 0,
3200 RuntimeVertex::plain("worker"),
3201 InvocationPolicy::FixedPrice,
3202 )
3203 .await
3204 .expect_err("missing leader capability should stop authorization");
3205
3206 assert!(matches!(error, NexusError::Rpc(_)));
3207 }
3208
3209 #[tokio::test]
3210 async fn authorize_invocation_validates_the_active_walk() {
3211 let nexus_objects = sui_mocks::mock_nexus_context();
3212 let execution_ref = sui_mocks::mock_sui_object_ref();
3213 let dag_ref = sui_mocks::mock_sui_object_ref();
3214 let leader_cap_ref = sui_mocks::mock_sui_object_ref();
3215 let vertex = RuntimeVertex::plain("worker");
3216 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
3217 let mut state_service_mock = sui_mocks::grpc::MockStateService::new();
3218 sui_mocks::grpc::mock_get_object_metadata_exact(
3219 &mut ledger_service_mock,
3220 leader_cap_ref.clone(),
3221 sui::types::Owner::Address(sui::types::Address::from_static("0x99")),
3222 None,
3223 );
3224 mock_get_dag_execution_bcs(
3225 &mut ledger_service_mock,
3226 &mut state_service_mock,
3227 &nexus_objects,
3228 execution_ref.clone(),
3229 &dag_ref,
3230 vec![execution_move::DAGWalk::Successful],
3231 );
3232 let package_service_mock = mock_package_graph(&mut ledger_service_mock, &nexus_objects);
3233 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
3234 ledger_service_mock: Some(ledger_service_mock),
3235 package_service_mock: Some(package_service_mock),
3236 state_service_mock: Some(state_service_mock),
3237 ..Default::default()
3238 });
3239 let client = nexus_mocks::mock_nexus_client_without_coins(&nexus_objects, &rpc_url).await;
3240
3241 let error = client
3242 .workflow()
3243 .authorize_invocation(
3244 *execution_ref.object_id(),
3245 *leader_cap_ref.object_id(),
3246 0,
3247 vertex.clone(),
3248 InvocationPolicy::FixedPrice,
3249 )
3250 .await
3251 .expect_err("a terminal walk must not be authorized");
3252
3253 assert!(
3254 matches!(error, NexusError::Configuration(ref message) if message.contains("is not active")),
3255 "unexpected inactive walk error: {error:?}"
3256 );
3257 }
3258
3259 #[tokio::test]
3260 async fn authorize_invocation_forwards_the_leader_capability() {
3261 let nexus_objects = sui_mocks::mock_nexus_context();
3262 let execution_ref = sui_mocks::mock_sui_object_ref();
3263 let dag_ref = sui_mocks::mock_sui_object_ref();
3264 let leader_cap_ref = sui_mocks::mock_sui_object_ref();
3265 let vertex = RuntimeVertex::plain("worker");
3266 let tool_fqn = fqn!("xyz.taluslabs.authorize@1");
3267 let tool_id = crate::move_bindings::derive_tool_id(
3268 nexus_objects.tool_registry.object_id(),
3269 &tool_fqn,
3270 )
3271 .expect("tool ID derives");
3272 let cashier_origin = nexus_objects
3273 .type_origin(PackageRole::Tool, "tool_cashier", "ToolCashierKey")
3274 .expect("cashier origin resolves");
3275 let cashier_id = crate::move_bindings::derive_tool_cashier_id(cashier_origin, tool_id)
3276 .expect("cashier ID derives");
3277 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
3278 let mut state_service_mock = sui_mocks::grpc::MockStateService::new();
3279 sui_mocks::grpc::mock_get_object_metadata_exact(
3280 &mut ledger_service_mock,
3281 leader_cap_ref.clone(),
3282 sui::types::Owner::Address(sui::types::Address::from_static("0x99")),
3283 None,
3284 );
3285 mock_get_dag_execution_bcs(
3286 &mut ledger_service_mock,
3287 &mut state_service_mock,
3288 &nexus_objects,
3289 execution_ref.clone(),
3290 &dag_ref,
3291 vec![execution_move::DAGWalk::Active {
3292 next_vertex: vertex.clone(),
3293 timeout_ms: 30_000,
3294 requires_vertex_authorization_grant: false,
3295 created_at: 1_000,
3296 }],
3297 );
3298 let dag = dag_bcs(1);
3299 let vertices_id = dag.vertices.id();
3300 mock_get_dag_bcs(
3301 &mut ledger_service_mock,
3302 &mut state_service_mock,
3303 &nexus_objects,
3304 dag_ref.clone(),
3305 dag,
3306 );
3307 sui_mocks::grpc::mock_get_dynamic_field_by_key(
3308 &mut ledger_service_mock,
3309 vertices_id,
3310 &crate::move_bindings::type_tag::<graph_move::Vertex>(&nexus_objects),
3311 graph_move::Vertex::new("worker"),
3312 offchain_vertex_node_bcs(&tool_fqn),
3313 );
3314 mock_object_state_for_authorize(
3315 &mut ledger_service_mock,
3316 &mut state_service_mock,
3317 &nexus_objects,
3318 tool_id,
3319 &tool_fqn,
3320 );
3321 sui_mocks::grpc::mock_get_object_metadata_exact(
3322 &mut ledger_service_mock,
3323 sui_mocks::object_ref_for_id(cashier_id),
3324 sui::types::Owner::Shared(2),
3325 None,
3326 );
3327 let payment_ref = sui_mocks::mock_sui_object_ref();
3328 mock_execution_payment_field(
3329 &mut ledger_service_mock,
3330 &mut state_service_mock,
3331 &nexus_objects,
3332 *execution_ref.object_id(),
3333 payment_ref,
3334 authorize_payment_fixture(*execution_ref.object_id()),
3335 );
3336 mock_tool_registry_observation(
3337 &mut ledger_service_mock,
3338 &mut state_service_mock,
3339 &nexus_objects,
3340 );
3341 mock_leader_registry_observation(
3342 &mut ledger_service_mock,
3343 &mut state_service_mock,
3344 &nexus_objects,
3345 );
3346 sui_mocks::grpc::mock_runtime_authority(&mut ledger_service_mock, &nexus_objects, false);
3347 let package_service_mock = mock_package_graph(&mut ledger_service_mock, &nexus_objects);
3348 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
3349 ledger_service_mock: Some(ledger_service_mock),
3350 package_service_mock: Some(package_service_mock),
3351 state_service_mock: Some(state_service_mock),
3352 ..Default::default()
3353 });
3354 let client = nexus_mocks::mock_nexus_client_without_coins(&nexus_objects, &rpc_url).await;
3355
3356 let error = client
3357 .workflow()
3358 .authorize_invocation(
3359 *execution_ref.object_id(),
3360 *leader_cap_ref.object_id(),
3361 0,
3362 vertex,
3363 InvocationPolicy::Custom(InvocationPolicyCall::new(
3364 TypeName::new("malformed_policy"),
3365 vec![],
3366 )),
3367 )
3368 .await
3369 .expect_err("malformed policy should fail inside PTB construction");
3370
3371 assert!(matches!(error, NexusError::TransactionBuilding(_)));
3372 }
3373
3374 fn empty_object_table<T0, T1>(
3375 id: sui::types::Address,
3376 ) -> crate::move_bindings::sui_framework::object_table::ObjectTable<T0, T1> {
3377 crate::move_bindings::sui_framework::object_table::ObjectTable {
3378 id: crate::move_bindings::sui_framework::object::UID::new(id),
3379 size: 0,
3380 phantom_t0: std::marker::PhantomData,
3381 phantom_t1: std::marker::PhantomData,
3382 }
3383 }
3384
3385 fn dag_execution_bcs(
3386 _execution_ref: &sui::types::ObjectReference,
3387 dag_ref: &sui::types::ObjectReference,
3388 walks: Vec<execution_move::DAGWalk>,
3389 ) -> execution_move::DAGExecutionInnerV1 {
3390 execution_move::DAGExecutionInnerV1 {
3391 dag: object_id(*dag_ref.object_id()),
3392 entry_group: graph_move::EntryGroup::new("default"),
3393 invoker: sui::types::Address::from_static("0x1"),
3394 created_at: 0,
3395 priority_fee_percentage: 0,
3396 agent_id: object_id(sui::types::Address::from_static("0xa")),
3397 skill_id: 11,
3398 interface_version: InterfaceVersion::new(7),
3399 task_id: object_id(sui::types::Address::from_static("0xb")),
3400 occurrence_id: 0,
3401 last_request_for_execution_emitted_at_digest: vec![],
3402 last_request_for_execution_leaders: vec![],
3403 network: object_id(sui::types::Address::from_static("0xf")),
3404 evaluations: empty_object_table(sui_mocks::mock_sui_address()),
3405 terminal_records: crate::move_bindings::sui_framework::vec_map::VecMap {
3406 contents: vec![],
3407 },
3408 submission_failure_records: crate::move_bindings::sui_framework::vec_map::VecMap {
3409 contents: vec![],
3410 },
3411 pending_retry_handoff_cap_ids: crate::move_bindings::sui_framework::vec_map::VecMap {
3412 contents: vec![],
3413 },
3414 walk_request_authorities: crate::move_bindings::sui_framework::vec_map::VecMap {
3415 contents: vec![],
3416 },
3417 pending_payment_settlements: crate::move_bindings::sui_framework::vec_map::VecMap {
3418 contents: vec![],
3419 },
3420 active_walks: walks
3421 .iter()
3422 .filter(|walk| matches!(walk, execution_move::DAGWalk::Active { .. }))
3423 .count() as u64,
3424 pending_abort_walks: 0,
3425 pending_settlement_walks: 0,
3426 successful_walks: 0,
3427 failed_walks: 0,
3428 aborted_walks: 0,
3429 consumed_walks: 0,
3430 cancelled_walks: 0,
3431 walks,
3432 }
3433 }
3434
3435 fn mock_get_dag_execution_bcs(
3436 ledger_service_mock: &mut sui_mocks::grpc::MockLedgerService,
3437 state_service_mock: &mut sui_mocks::grpc::MockStateService,
3438 nexus_objects: &crate::types::NexusContext,
3439 execution_ref: sui::types::ObjectReference,
3440 dag_ref: &sui::types::ObjectReference,
3441 walks: Vec<execution_move::DAGWalk>,
3442 ) {
3443 let owner = sui::types::Owner::Shared(execution_ref.version());
3444 let execution = dag_execution_bcs(&execution_ref, dag_ref, walks);
3445 let anchor = execution_move::DAGExecution::new(
3446 crate::move_bindings::sui_framework::object::UID::new(*execution_ref.object_id()),
3447 );
3448 sui_mocks::grpc::mock_object_state::<
3449 execution_move::DAGExecution,
3450 WorkflowWitnessV1,
3451 execution_move::DAGExecutionInnerV1,
3452 >(
3453 ledger_service_mock,
3454 state_service_mock,
3455 nexus_objects,
3456 execution_ref,
3457 owner,
3458 anchor,
3459 execution,
3460 );
3461 }
3462
3463 fn mock_execution_observation_context(
3464 ledger_service_mock: &mut sui_mocks::grpc::MockLedgerService,
3465 state_service_mock: &mut sui_mocks::grpc::MockStateService,
3466 nexus_objects: &NexusContext,
3467 execution_ref: sui::types::ObjectReference,
3468 ) {
3469 mock_get_dag_execution_bcs(
3470 ledger_service_mock,
3471 state_service_mock,
3472 nexus_objects,
3473 execution_ref,
3474 &sui_mocks::mock_sui_object_ref(),
3475 vec![],
3476 );
3477 }
3478
3479 fn mock_package_graph(
3480 ledger_service_mock: &mut sui_mocks::grpc::MockLedgerService,
3481 nexus_objects: &NexusContext,
3482 ) -> sui_mocks::grpc::MockMovePackageService {
3483 let mut package_service_mock = sui_mocks::grpc::MockMovePackageService::new();
3484 sui_mocks::grpc::mock_nexus_package_graph(
3485 ledger_service_mock,
3486 &mut package_service_mock,
3487 nexus_objects.packages(),
3488 );
3489 package_service_mock
3490 }
3491
3492 fn mock_tool_registry_observation(
3493 ledger_service_mock: &mut sui_mocks::grpc::MockLedgerService,
3494 state_service_mock: &mut sui_mocks::grpc::MockStateService,
3495 nexus_objects: &NexusContext,
3496 ) {
3497 let root = nexus_objects.tool_registry;
3498 sui_mocks::grpc::mock_object_state_observation::<
3499 ToolRegistry,
3500 ToolWitnessV1,
3501 ToolRegistryInnerV1,
3502 >(
3503 ledger_service_mock,
3504 state_service_mock,
3505 nexus_objects,
3506 sui_mocks::object_ref_for_id(root.object_id()),
3507 sui::types::Owner::Shared(root.initial_shared_version),
3508 ToolRegistry::new(crate::move_bindings::sui_framework::object::UID::new(
3509 root.object_id(),
3510 )),
3511 );
3512 }
3513
3514 fn mock_leader_registry_observation(
3515 ledger_service_mock: &mut sui_mocks::grpc::MockLedgerService,
3516 state_service_mock: &mut sui_mocks::grpc::MockStateService,
3517 nexus_objects: &NexusContext,
3518 ) {
3519 let root = nexus_objects.leader_registry;
3520 sui_mocks::grpc::mock_object_state_observation::<
3521 LeaderRegistry,
3522 RegistryWitnessV1,
3523 LeaderRegistryInnerV1,
3524 >(
3525 ledger_service_mock,
3526 state_service_mock,
3527 nexus_objects,
3528 sui_mocks::object_ref_for_id(root.object_id()),
3529 sui::types::Owner::Shared(root.initial_shared_version),
3530 LeaderRegistry::new(crate::move_bindings::sui_framework::object::UID::new(
3531 root.object_id(),
3532 )),
3533 );
3534 }
3535
3536 fn mock_priority_fee_vault_observation(
3537 ledger_service_mock: &mut sui_mocks::grpc::MockLedgerService,
3538 state_service_mock: &mut sui_mocks::grpc::MockStateService,
3539 nexus_objects: &NexusContext,
3540 ) {
3541 let root = nexus_objects.priority_fee_vault;
3542 sui_mocks::grpc::mock_object_state_observation::<
3543 PriorityFeeVault,
3544 RegistryWitnessV1,
3545 PriorityFeeVaultInnerV1,
3546 >(
3547 ledger_service_mock,
3548 state_service_mock,
3549 nexus_objects,
3550 sui_mocks::object_ref_for_id(root.object_id()),
3551 sui::types::Owner::Shared(root.initial_shared_version),
3552 PriorityFeeVault::new(crate::move_bindings::sui_framework::object::UID::new(
3553 root.object_id(),
3554 )),
3555 );
3556 }
3557
3558 fn mock_execution_payment_field(
3559 ledger_service_mock: &mut sui_mocks::grpc::MockLedgerService,
3560 state_service_mock: &mut sui_mocks::grpc::MockStateService,
3561 nexus_objects: &NexusContext,
3562 execution_id: sui::types::Address,
3563 payment_ref: sui::types::ObjectReference,
3564 payment: ExecutionPaymentInnerV1,
3565 ) {
3566 let payment_id = *payment_ref.object_id();
3567 let anchor = ExecutionPayment::new(crate::move_bindings::sui_framework::object::UID::new(
3568 payment_id,
3569 ));
3570 sui_mocks::grpc::mock_get_dynamic_object_field_by_key(
3571 ledger_service_mock,
3572 execution_id,
3573 &crate::move_bindings::type_tag::<DagExecutionPaymentFieldKey>(nexus_objects),
3574 DagExecutionPaymentFieldKey::default(),
3575 payment_ref.clone(),
3576 sui::types::Owner::Shared(0),
3577 anchor.clone(),
3578 );
3579 sui_mocks::grpc::mock_object_state::<
3580 ExecutionPayment,
3581 InterfaceWitnessV1,
3582 ExecutionPaymentInnerV1,
3583 >(
3584 ledger_service_mock,
3585 state_service_mock,
3586 nexus_objects,
3587 payment_ref,
3588 sui::types::Owner::Shared(0),
3589 anchor,
3590 payment,
3591 );
3592 }
3593
3594 fn mock_object_state_for_authorize(
3595 ledger_service_mock: &mut sui_mocks::grpc::MockLedgerService,
3596 state_service_mock: &mut sui_mocks::grpc::MockStateService,
3597 nexus_objects: &NexusContext,
3598 tool_id: sui::types::Address,
3599 tool_fqn: &crate::ToolFqn,
3600 ) {
3601 sui_mocks::grpc::mock_object_state::<ToolAnchor, ToolWitnessV1, ToolInnerV1>(
3602 ledger_service_mock,
3603 state_service_mock,
3604 nexus_objects,
3605 sui_mocks::object_ref_for_id(tool_id),
3606 sui::types::Owner::Shared(1),
3607 ToolAnchor::new(crate::move_bindings::sui_framework::object::UID::new(
3608 tool_id,
3609 )),
3610 ToolInnerV1::new(
3611 crate::move_bindings::sui_framework::object::ID::new(
3612 nexus_objects.tool_registry.object_id(),
3613 ),
3614 MoveString::from(tool_fqn.to_string()),
3615 ToolRegistryRef::Http {
3616 url: b"https://example.com/tool".to_vec(),
3617 },
3618 b"coverage fixture".to_vec(),
3619 MetaSchema::new(vec![], vec![]),
3620 false,
3621 crate::move_bindings::sui_framework::balance::Balance {
3622 value: 0,
3623 phantom_t0: std::marker::PhantomData,
3624 },
3625 false,
3626 0,
3627 0,
3628 MoveOption::from(Some(1)),
3629 ),
3630 );
3631 }
3632
3633 fn authorize_payment_fixture(execution_id: sui::types::Address) -> ExecutionPaymentInnerV1 {
3634 ExecutionPaymentInnerV1 {
3635 execution_id,
3636 agent_id: object_id(sui::types::Address::from_static("0xa")),
3637 skill_id: 11,
3638 interface_revision: InterfaceVersion::new(7),
3639 payment_policy: SkillPaymentPolicy::UserFunded,
3640 source_kind: PaymentSourceKind::user_funded(sui::types::Address::from_static("0x1")),
3641 max_budget_mist: 100_000,
3642 gas_budget_mist: 83_334,
3643 priority_fee_reserve_mist: 16_666,
3644 locked_budget_mist: 0,
3645 funds: crate::move_bindings::sui_framework::balance::Balance {
3646 value: 100_000,
3647 phantom_t0: std::marker::PhantomData,
3648 },
3649 consumed: 0,
3650 tool_fee_charged: 0,
3651 priority_fee_charged: 0,
3652 priority_fee_percentage: 20,
3653 accomplished: false,
3654 refunded: false,
3655 final_state: ExecutionPaymentFinalState::Pending,
3656 tool_cost_snapshot: crate::move_bindings::sui_framework::vec_map::VecMap {
3657 contents: vec![],
3658 },
3659 locked_vertices: vec![],
3660 }
3661 }
3662
3663 fn pending_settlement_walks(
3664 walk_index: usize,
3665 vertex: RuntimeVertex,
3666 ) -> Vec<execution_move::DAGWalk> {
3667 let mut walks = vec![execution_move::DAGWalk::Successful; walk_index];
3668 walks.push(execution_move::DAGWalk::PendingSettlement {
3669 next_vertex: vertex,
3670 timeout_ms: 30_000,
3671 requires_vertex_authorization_grant: false,
3672 created_at: 1_000,
3673 });
3674 walks
3675 }
3676
3677 fn mock_exact_invocation_lock(
3678 ledger_service_mock: &mut sui_mocks::grpc::MockLedgerService,
3679 state_service_mock: &mut sui_mocks::grpc::MockStateService,
3680 nexus_objects: &NexusContext,
3681 execution_ref: &sui::types::ObjectReference,
3682 dag_ref: &sui::types::ObjectReference,
3683 vertex: &RuntimeVertex,
3684 tool_fqn: &crate::ToolFqn,
3685 ) -> sui::types::ObjectReference {
3686 let dag = dag_bcs(1);
3687 let vertices_id = dag.vertices.id();
3688 mock_get_dag_bcs(
3689 ledger_service_mock,
3690 state_service_mock,
3691 nexus_objects,
3692 dag_ref.clone(),
3693 dag,
3694 );
3695 sui_mocks::grpc::mock_get_dynamic_field_by_key(
3696 ledger_service_mock,
3697 vertices_id,
3698 &crate::move_bindings::type_tag::<graph_move::Vertex>(nexus_objects),
3699 vertex.vertex().clone(),
3700 offchain_vertex_node_bcs(tool_fqn),
3701 );
3702
3703 let payment_ref = sui_mocks::mock_sui_object_ref();
3704 let invocation_ref = sui_mocks::mock_sui_object_ref();
3705 let vertex_key = payment_vertex_key(*execution_ref.object_id(), vertex, tool_fqn)
3706 .expect("payment vertex key serializes");
3707 mock_execution_payment_field(
3708 ledger_service_mock,
3709 state_service_mock,
3710 nexus_objects,
3711 *execution_ref.object_id(),
3712 payment_ref.clone(),
3713 ExecutionPaymentInnerV1 {
3714 execution_id: *execution_ref.object_id(),
3715 agent_id: object_id(sui::types::Address::from_static("0xa")),
3716 skill_id: 11,
3717 interface_revision: InterfaceVersion::new(7),
3718 payment_policy: SkillPaymentPolicy::UserFunded,
3719 source_kind: PaymentSourceKind::user_funded(sui::types::Address::from_static(
3720 "0x1",
3721 )),
3722 max_budget_mist: 100_000,
3723 gas_budget_mist: 83_334,
3724 priority_fee_reserve_mist: 16_666,
3725 locked_budget_mist: 10,
3726 funds: crate::move_bindings::sui_framework::balance::Balance {
3727 value: 100_000,
3728 phantom_t0: std::marker::PhantomData,
3729 },
3730 consumed: 0,
3731 tool_fee_charged: 0,
3732 priority_fee_charged: 0,
3733 priority_fee_percentage: 20,
3734 accomplished: false,
3735 refunded: false,
3736 final_state: ExecutionPaymentFinalState::Pending,
3737 tool_cost_snapshot: crate::move_bindings::sui_framework::vec_map::VecMap {
3738 contents: vec![],
3739 },
3740 locked_vertices: vec![invocation_lock(vertex_key, *invocation_ref.object_id())],
3741 },
3742 );
3743 sui_mocks::grpc::mock_get_object_metadata_exact(
3744 ledger_service_mock,
3745 invocation_ref.clone(),
3746 sui::types::Owner::Object(*execution_ref.object_id()),
3747 None,
3748 );
3749 invocation_ref
3750 }
3751
3752 fn offchain_vertex_info(tool_fqn: &crate::ToolFqn) -> graph_move::VertexInfo {
3753 graph_move::VertexInfo {
3754 kind: graph_move::VertexKind::OffChain {
3755 tool_fqn: tool_fqn.to_string().into(),
3756 },
3757 input_ports: VecSet { contents: vec![] },
3758 post_failure_action: MoveOption::from_option(None::<graph_move::PostFailureAction>),
3759 tool_id: object_id(sui::types::Address::from_static("0x42")),
3760 meta_schema: MoveOption::from_option(Some(
3761 crate::move_bindings::interface::meta_schema::MetaSchema::new(vec![], vec![]),
3762 )),
3763 verifier_mode: ToolVerifierMode::None,
3764 }
3765 }
3766
3767 fn ports_data_map(
3768 entries: Vec<(&str, &'static [u8])>,
3769 ) -> crate::move_bindings::sui_framework::vec_map::VecMap<
3770 crate::move_bindings::interface::graph::OutputPort,
3771 crate::move_bindings::primitives::data::NexusData,
3772 > {
3773 crate::move_bindings::sui_framework::vec_map::VecMap {
3774 contents: entries
3775 .into_iter()
3776 .map(
3777 |(name, value)| crate::move_bindings::sui_framework::vec_map::Entry {
3778 key: crate::move_bindings::interface::graph::OutputPort {
3779 name: MoveString::from(name),
3780 },
3781 value: crate::move_bindings::primitives::data::NexusData::inline_data(
3782 value,
3783 )
3784 .expect("fixture is bounded"),
3785 },
3786 )
3787 .collect(),
3788 }
3789 }
3790
3791 async fn crawler_from_mocks(
3792 ledger_service_mock: sui_mocks::grpc::MockLedgerService,
3793 state_service_mock: sui_mocks::grpc::MockStateService,
3794 ) -> Crawler {
3795 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
3796 ledger_service_mock: Some(ledger_service_mock),
3797 state_service_mock: Some(state_service_mock),
3798 ..Default::default()
3799 });
3800 let client = sui::grpc::client(rpc_url).expect("mock client");
3801 Crawler::new(Arc::new(client))
3802 }
3803
3804 #[tokio::test]
3805 async fn fetch_committed_tool_result_for_walk_returns_none_when_absent() {
3806 let nexus_objects = sui_mocks::mock_nexus_context();
3807 let execution_id = sui::types::Address::from_static("0xe1");
3808 let key = execution_move::CommittedToolResultKey { walk_index: 7 };
3809 let key_type = crate::move_bindings::type_tag::<execution_move::CommittedToolResultKey>(
3810 &nexus_objects,
3811 );
3812 let expected_id = execution_id.derive_dynamic_child_id(
3813 &key_type,
3814 &bcs::to_bytes(&key).expect("committed result key serializes"),
3815 );
3816 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
3817 sui_mocks::grpc::mock_get_object_not_found(&mut ledger_service_mock, expected_id);
3818 let crawler = crawler_from_mocks(
3819 ledger_service_mock,
3820 sui_mocks::grpc::MockStateService::new(),
3821 )
3822 .await;
3823
3824 let result =
3825 fetch_committed_tool_result_for_walk(&crawler, &nexus_objects, execution_id, 7)
3826 .await
3827 .expect("fetch should succeed");
3828
3829 assert!(result.is_none());
3830 }
3831
3832 fn finalized_onchain_result(
3833 execution_id: sui::types::Address,
3834 transaction: sui::types::Digest,
3835 ) -> OnchainToolResultInnerV1 {
3836 OnchainToolResultInnerV1 {
3837 execution_id: ID::new(execution_id),
3838 finalized: true,
3839 stamps: Some(VecMap { contents: vec![] }).into(),
3840 tag: Some(b"ok".to_vec()).into(),
3841 named_payload: Some(VecMap { contents: vec![] }).into(),
3842 finalize_tx_digest: Some(transaction.into_inner().to_vec()).into(),
3843 finalize_recipient: Some(sui::types::Address::from_static("0xa1")).into(),
3844 }
3845 }
3846
3847 #[tokio::test]
3848 async fn onchain_result_receipt_resolves_consumption_without_current_state_reads() {
3849 let context = sui_mocks::mock_nexus_context();
3850 let transaction = sui::types::Digest::new([1; 32]);
3851 let execution_id = sui::types::Address::from_static("0xe1");
3852 let result_id = sui::types::Address::from_static("0xe2");
3853 let receipt = nexus_mocks::mock_finalized_onchain_result(
3854 &context,
3855 transaction,
3856 execution_id,
3857 7,
3858 result_id,
3859 finalized_onchain_result(execution_id, transaction),
3860 );
3861 let crawler = crawler_from_mocks(Default::default(), Default::default()).await;
3863 let state = onchain_tool_result_from_finalized_output(
3864 &crawler,
3865 &context,
3866 &receipt,
3867 transaction,
3868 execution_id,
3869 7,
3870 )
3871 .unwrap();
3872 let (_, result_ref) = state.consume_ready_result().unwrap();
3873 assert_eq!(*result_ref.object_id(), result_id);
3874 assert_eq!(
3875 result_ref.version(),
3876 11,
3877 "consumption uses the initial shared version"
3878 );
3879 assert!(onchain_tool_result_from_finalized_output(
3880 &crawler,
3881 &context,
3882 &receipt,
3883 transaction,
3884 execution_id,
3885 8
3886 )
3887 .is_err());
3888 assert!(onchain_tool_result_from_finalized_output(
3889 &crawler,
3890 &context,
3891 &receipt,
3892 transaction,
3893 result_id,
3894 7
3895 )
3896 .is_err());
3897 }
3898
3899 #[tokio::test]
3900 async fn onchain_result_receipt_rejects_unproven_results() {
3901 let context = sui_mocks::mock_nexus_context();
3902 let transaction = sui::types::Digest::new([1; 32]);
3903 let execution_id = sui::types::Address::from_static("0xe1");
3904 let result_id = sui::types::Address::from_static("0xe2");
3905 let crawler = crawler_from_mocks(Default::default(), Default::default()).await;
3906 let valid = finalized_onchain_result(execution_id, transaction);
3907 let mut wrong_execution = valid.clone();
3908 wrong_execution.execution_id = ID::new(result_id);
3909 let mut unfinished = valid.clone();
3910 unfinished.finalized = false;
3911 let mut wrong_transaction = valid.clone();
3912 wrong_transaction.finalize_tx_digest = Some(vec![2; 32]).into();
3913 let mut missing_payload = valid.clone();
3914 missing_payload.named_payload = None.into();
3915 for result in [
3916 wrong_execution,
3917 unfinished,
3918 wrong_transaction,
3919 missing_payload,
3920 ] {
3921 let receipt = nexus_mocks::mock_finalized_onchain_result(
3922 &context,
3923 transaction,
3924 execution_id,
3925 7,
3926 result_id,
3927 result,
3928 );
3929 assert!(onchain_tool_result_from_finalized_output(
3930 &crawler,
3931 &context,
3932 &receipt,
3933 transaction,
3934 execution_id,
3935 7
3936 )
3937 .is_err());
3938 }
3939 let receipt = nexus_mocks::mock_finalized_onchain_result(
3940 &context,
3941 transaction,
3942 execution_id,
3943 7,
3944 result_id,
3945 valid,
3946 );
3947 for omitted in 0..3 {
3948 let mut incomplete = receipt.clone();
3949 incomplete.objects.as_mut().unwrap().objects.remove(omitted);
3950 assert!(onchain_tool_result_from_finalized_output(
3951 &crawler,
3952 &context,
3953 &incomplete,
3954 transaction,
3955 execution_id,
3956 7
3957 )
3958 .is_err());
3959 }
3960 let mut repeated = receipt.clone();
3961 repeated
3962 .objects
3963 .as_mut()
3964 .unwrap()
3965 .objects
3966 .push(receipt.objects().objects[0].clone());
3967 assert!(onchain_tool_result_from_finalized_output(
3968 &crawler,
3969 &context,
3970 &repeated,
3971 transaction,
3972 execution_id,
3973 7
3974 )
3975 .is_err());
3976 let mut inherited = receipt.clone();
3977 inherited.objects.as_mut().unwrap().objects[2]
3978 .set_previous_transaction(sui::types::Digest::new([2; 32]));
3979 assert!(onchain_tool_result_from_finalized_output(
3980 &crawler,
3981 &context,
3982 &inherited,
3983 transaction,
3984 execution_id,
3985 7
3986 )
3987 .is_err());
3988 }
3989
3990 #[tokio::test]
3991 async fn onchain_result_state_reports_no_result_when_dynamic_fields_are_absent() {
3992 let nexus_objects = sui_mocks::mock_nexus_context();
3993 let execution_id = sui::types::Address::from_static("0xe1");
3994 let committed_key = execution_move::CommittedToolResultKey { walk_index: 7 };
3995 let insufficient_key = insufficient_settlement_field_key();
3996 let result_key = execution_move::OnchainToolResultKey { walk_index: 7 };
3997 let committed_id = execution_id.derive_dynamic_child_id(
3998 &crate::move_bindings::type_tag::<execution_move::CommittedToolResultKey>(
3999 &nexus_objects,
4000 ),
4001 &bcs::to_bytes(&committed_key).expect("committed key serializes"),
4002 );
4003 let insufficient_id = execution_id.derive_dynamic_child_id(
4004 &crate::move_bindings::type_tag::<
4005 execution_move::ExecutionPaymentInsufficientSettlementFieldKey,
4006 >(&nexus_objects),
4007 &bcs::to_bytes(&insufficient_key).expect("insufficient settlement key serializes"),
4008 );
4009 let result_id = execution_id.derive_dynamic_child_id(
4010 &crate::move_bindings::type_tag::<execution_move::OnchainToolResultKey>(&nexus_objects),
4011 &bcs::to_bytes(&result_key).expect("result key serializes"),
4012 );
4013 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
4014 for field_id in [committed_id, insufficient_id, result_id] {
4015 sui_mocks::grpc::mock_get_object_not_found(&mut ledger_service_mock, field_id);
4016 }
4017 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
4018 ledger_service_mock: Some(ledger_service_mock),
4019 ..Default::default()
4020 });
4021 let client = nexus_mocks::mock_nexus_client_without_coins(&nexus_objects, &rpc_url).await;
4022
4023 let state =
4024 fetch_onchain_tool_result_state_for_walk(&client, &nexus_objects, execution_id, 7)
4025 .await
4026 .expect("absent dynamic fields should be a valid empty state");
4027
4028 assert!(matches!(state, OnchainToolResultState::NoResult));
4029 }
4030
4031 #[tokio::test]
4032 async fn fetch_committed_tool_result_for_walk_reads_exact_key_and_decodes_match() {
4033 let nexus_objects = sui_mocks::mock_nexus_context();
4034 let execution_id = sui::types::Address::from_static("0xe1");
4035 let primary_leader = sui::types::Address::from_static("0xa1");
4036 let secondary_leader = sui::types::Address::from_static("0xa2");
4037 let committed = committed_tool_result_bcs(
4038 RuntimeVertex::plain("retryable"),
4039 primary_leader,
4040 secondary_leader,
4041 Some(FailureEvidenceKind::ToolEvidence),
4042 None,
4043 );
4044 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
4045 sui_mocks::grpc::mock_get_dynamic_field_by_key(
4046 &mut ledger_service_mock,
4047 execution_id,
4048 &crate::move_bindings::type_tag::<execution_move::CommittedToolResultKey>(
4049 &nexus_objects,
4050 ),
4051 execution_move::CommittedToolResultKey { walk_index: 7 },
4052 committed,
4053 );
4054 let crawler = crawler_from_mocks(
4055 ledger_service_mock,
4056 sui_mocks::grpc::MockStateService::new(),
4057 )
4058 .await;
4059
4060 let result =
4061 fetch_committed_tool_result_for_walk(&crawler, &nexus_objects, execution_id, 7)
4062 .await
4063 .expect("fetch should succeed")
4064 .expect("committed result should exist");
4065
4066 assert_eq!(result.expected_vertex, RuntimeVertex::plain("retryable"));
4067 assert_eq!(
4068 result.primary_failure_evidence_kind,
4069 Some(FailureEvidenceKind::ToolEvidence)
4070 );
4071 assert_eq!(result.secondary_failure_evidence_kind, None);
4072 assert!(result.leader_record(primary_leader).is_some());
4073 assert!(result.leader_record(secondary_leader).is_none());
4074 }
4075
4076 #[tokio::test]
4077 async fn fetch_committed_tool_result_for_walk_decodes_metadata_with_raw_output_payload() {
4078 let nexus_objects = sui_mocks::mock_nexus_context();
4079 let execution_id = sui::types::Address::from_static("0xe1");
4080 let primary_leader = sui::types::Address::from_static("0xa1");
4081 let secondary_leader = sui::types::Address::from_static("0xa2");
4082 let mut committed = committed_tool_result_bcs(
4083 RuntimeVertex::plain("retryable"),
4084 primary_leader,
4085 secondary_leader,
4086 Some(FailureEvidenceKind::ToolEvidence),
4087 None,
4088 );
4089 committed.variant_ports_to_data = crate::move_bindings::sui_framework::vec_map::VecMap {
4090 contents: vec![crate::move_bindings::sui_framework::vec_map::Entry {
4091 key: graph_move::OutputPort::new("reason"),
4092 value: raw_inline_nexus_data_bcs(b"not-json".to_vec()),
4093 }],
4094 };
4095 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
4096 sui_mocks::grpc::mock_get_dynamic_field_by_key(
4097 &mut ledger_service_mock,
4098 execution_id,
4099 &crate::move_bindings::type_tag::<execution_move::CommittedToolResultKey>(
4100 &nexus_objects,
4101 ),
4102 execution_move::CommittedToolResultKey { walk_index: 7 },
4103 committed,
4104 );
4105 let crawler = crawler_from_mocks(
4106 ledger_service_mock,
4107 sui_mocks::grpc::MockStateService::new(),
4108 )
4109 .await;
4110
4111 let result =
4112 fetch_committed_tool_result_for_walk(&crawler, &nexus_objects, execution_id, 7)
4113 .await
4114 .expect("fetch should ignore raw output payload bytes")
4115 .expect("committed result should exist");
4116
4117 assert_eq!(result.expected_vertex, RuntimeVertex::plain("retryable"));
4118 assert_eq!(
4119 result.primary_failure_evidence_kind,
4120 Some(FailureEvidenceKind::ToolEvidence)
4121 );
4122 assert_eq!(result.secondary_failure_evidence_kind, None);
4123 assert!(result.leader_record(primary_leader).is_some());
4124 assert!(result.leader_record(secondary_leader).is_none());
4125 }
4126
4127 fn offchain_vertex_node_bcs(
4128 tool_fqn: &crate::ToolFqn,
4129 ) -> linked_table::Node<graph_move::Vertex, graph_move::VertexInfo> {
4130 linked_table::Node {
4131 prev: crate::move_bindings::move_std::option::Option::from_option(
4132 None::<graph_move::Vertex>,
4133 ),
4134 next: crate::move_bindings::move_std::option::Option::from_option(
4135 None::<graph_move::Vertex>,
4136 ),
4137 value: offchain_vertex_info(tool_fqn),
4138 }
4139 }
4140
4141 fn supported_grpc_events(
4142 objects: &crate::types::NexusContext,
4143 nexus_events: Vec<NexusEventKind>,
4144 ) -> Vec<sui::grpc::Event> {
4145 #[derive(Serialize)]
4146 struct Wrapper<T> {
4147 event: T,
4148 }
4149
4150 let primitives = objects
4151 .require_package(PackageRole::Primitives)
4152 .unwrap()
4153 .initial_id;
4154 let interface = objects
4155 .require_package(PackageRole::Interface)
4156 .unwrap()
4157 .initial_id;
4158 let workflow = objects
4159 .require_package(PackageRole::Workflow)
4160 .unwrap()
4161 .initial_id;
4162
4163 nexus_events
4164 .into_iter()
4165 .map(|event| {
4166 let (event_pkg_id, event_type_origin, module) =
4167 if matches!(event, NexusEventKind::DAGCreated(_)) {
4168 (interface, interface, "dag")
4169 } else {
4170 (workflow, workflow, "execution")
4171 };
4172 let event_type = format!(
4173 "{}::event::EventWrapper<{}::{module}::{}>",
4174 primitives,
4175 event_type_origin,
4176 event.name()
4177 );
4178 let contents = match event {
4179 NexusEventKind::WalkAdvanced(event) => {
4180 bcs::to_bytes(&Wrapper { event }).unwrap()
4181 }
4182 NexusEventKind::EndStateReached(event) => {
4183 bcs::to_bytes(&Wrapper { event }).unwrap()
4184 }
4185 NexusEventKind::ExecutionFinished(event) => {
4186 bcs::to_bytes(&Wrapper { event }).unwrap()
4187 }
4188 NexusEventKind::TerminalErrEvalRecorded(event) => {
4189 bcs::to_bytes(&Wrapper { event }).unwrap()
4190 }
4191 NexusEventKind::DAGCreated(event) => bcs::to_bytes(&Wrapper { event }).unwrap(),
4192 _ => panic!("Unsupported event type for BCS serialization"),
4193 };
4194
4195 let mut grpc_event = sui::grpc::Event::default();
4196 grpc_event.set_package_id(event_pkg_id);
4197 grpc_event.set_module(module.to_string());
4198 grpc_event.set_sender(sui::types::Address::ZERO);
4199 grpc_event.set_event_type(event_type);
4200 grpc_event.set_contents(contents);
4201 grpc_event
4202 })
4203 .collect()
4204 }
4205
4206 fn expect_object_update_reference(
4207 ledger_service: &mut sui_mocks::grpc::MockLedgerService,
4208 objects: &crate::types::NexusContext,
4209 object_ref: sui::types::ObjectReference,
4210 initial_shared_version: sui::types::Version,
4211 expected_requested_version: Option<sui::types::Version>,
4212 previous_transaction: sui::types::Digest,
4213 ) {
4214 let object_type = crate::move_bindings::struct_tag::<DAGExecution>(objects).to_string();
4215 ledger_service
4216 .expect_get_object()
4217 .times(1)
4218 .returning(move |request| {
4219 assert_eq!(request.get_ref().version, expected_requested_version);
4220 let mut response = sui::grpc::GetObjectResponse::default();
4221 let mut object = sui::grpc::Object::default();
4222 object.set_object_id(*object_ref.object_id());
4223 object.set_owner(sui::grpc::Owner::from(sui::types::Owner::Shared(
4224 initial_shared_version,
4225 )));
4226 object.set_object_type(object_type.clone());
4227 object.set_version(object_ref.version());
4228 object.set_digest(*object_ref.digest());
4229 object.set_previous_transaction(previous_transaction.to_string());
4230 response.set_object(object);
4231 Ok(tonic::Response::new(response))
4232 });
4233 }
4234
4235 #[allow(clippy::too_many_arguments)]
4236 fn expect_transaction_update(
4237 ledger_service: &mut sui_mocks::grpc::MockLedgerService,
4238 objects: &crate::types::NexusContext,
4239 digest: sui::types::Digest,
4240 execution_id: sui::types::Address,
4241 version: sui::types::Version,
4242 output_digest: sui::types::Digest,
4243 input_state: sui::types::ObjectIn,
4244 nexus_events: Vec<NexusEventKind>,
4245 ) {
4246 let grpc_events = supported_grpc_events(objects, nexus_events);
4247 let served = Arc::new(AtomicBool::new(false));
4248 ledger_service
4249 .expect_get_transaction()
4250 .withf(move |request| {
4251 request.get_ref().digest_opt() == Some(digest.to_string().as_str())
4252 })
4253 .returning(move |request| {
4254 if served.swap(true, Ordering::SeqCst) {
4255 return Err(tonic::Status::failed_precondition(format!(
4256 "transaction '{digest}' was fetched again after its object version was already reconstructed"
4257 )));
4258 }
4259 assert_eq!(
4260 request.get_ref().digest_opt(),
4261 Some(digest.to_string().as_str())
4262 );
4263 let effects = sui::types::TransactionEffects::V2(Box::new(
4264 sui::types::TransactionEffectsV2 {
4265 status: sui::types::ExecutionStatus::Success,
4266 epoch: 1,
4267 gas_used: sui::types::GasCostSummary {
4268 computation_cost: 0,
4269 storage_cost: 0,
4270 storage_rebate: 0,
4271 non_refundable_storage_fee: 0,
4272 },
4273 transaction_digest: digest,
4274 gas_object_index: None,
4275 events_digest: None,
4276 dependencies: vec![],
4277 lamport_version: version,
4278 changed_objects: vec![sui::types::ChangedObject {
4279 object_id: execution_id,
4280 input_state: input_state.clone(),
4281 output_state: sui::types::ObjectOut::ObjectWrite {
4282 digest: output_digest,
4283 owner: sui::types::Owner::Shared(1),
4284 },
4285 id_operation: if matches!(input_state, sui::types::ObjectIn::NotExist) {
4286 sui::types::IdOperation::Created
4287 } else {
4288 sui::types::IdOperation::None
4289 },
4290 }],
4291 unchanged_consensus_objects: vec![],
4292 auxiliary_data_digest: None,
4293 },
4294 ));
4295 let mut grpc_effects = sui::grpc::TransactionEffects::default();
4296 grpc_effects.set_bcs(bcs::to_bytes(&effects).unwrap());
4297 let mut events = sui::grpc::TransactionEvents::default();
4298 events.set_events(grpc_events.clone());
4299 let mut transaction = sui::grpc::ExecutedTransaction::default();
4300 transaction.set_digest(digest);
4301 transaction.set_checkpoint(version);
4302 transaction.set_effects(grpc_effects);
4303 transaction.set_events(events);
4304 let mut response = sui::grpc::GetTransactionResponse::default();
4305 response.set_transaction(transaction);
4306 Ok(tonic::Response::new(response))
4307 });
4308 }
4309
4310 #[tokio::test]
4311 async fn test_workflow_actions_publish() {
4312 let mut rng = rand::thread_rng();
4313 let digest = sui::types::Digest::generate(&mut rng);
4314 let gas_coin_ref = sui_mocks::mock_sui_object_ref();
4315 let nexus_objects = sui_mocks::mock_nexus_context();
4316 let dag_object_id = sui::types::Address::generate(&mut rng);
4317
4318 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
4319 let mut state_service_mock = sui_mocks::grpc::MockStateService::new();
4320 let mut tx_service_mock = sui_mocks::grpc::MockTransactionExecutionService::new();
4321 let mut sub_service_mock = sui_mocks::grpc::MockSubscriptionService::new();
4322
4323 sui_mocks::grpc::mock_reference_gas_price(&mut ledger_service_mock, 1000);
4324 mock_tool_registry_observation(
4325 &mut ledger_service_mock,
4326 &mut state_service_mock,
4327 &nexus_objects,
4328 );
4329 let package_service_mock = mock_package_graph(&mut ledger_service_mock, &nexus_objects);
4330
4331 let dag_created = sui::types::Object::new(
4332 sui::types::ObjectData::Struct(
4333 sui::types::MoveStruct::new(
4334 crate::move_bindings::struct_tag::<dag_move::DAG>(&nexus_objects),
4335 true,
4336 0,
4337 dag_object_id.to_bcs().unwrap(),
4338 )
4339 .unwrap(),
4340 ),
4341 sui::types::Owner::Shared(0),
4342 digest,
4343 1000,
4344 );
4345
4346 let submitted = sui_mocks::grpc::mock_execute_transaction_and_wait_for_checkpoint(
4347 &mut tx_service_mock,
4348 &mut sub_service_mock,
4349 &mut ledger_service_mock,
4350 gas_coin_ref.clone(),
4351 vec![dag_created],
4352 vec![],
4353 vec![],
4354 );
4355
4356 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
4357 ledger_service_mock: Some(ledger_service_mock),
4358 package_service_mock: Some(package_service_mock),
4359 state_service_mock: Some(state_service_mock),
4360 execution_service_mock: Some(tx_service_mock),
4361 subscription_service_mock: Some(sub_service_mock),
4362 ..Default::default()
4363 });
4364
4365 let client = nexus_mocks::mock_nexus_client(&nexus_objects, &rpc_url).await;
4366
4367 let dag = DagSpec::default();
4368
4369 let result = client
4370 .workflow()
4371 .publish(
4372 nexus_objects
4373 .require_package(PackageRole::Workflow)
4374 .unwrap()
4375 .storage_id,
4376 dag,
4377 )
4378 .await
4379 .expect("Failed to publish DAG");
4380
4381 assert_eq!(result.dag_object_id, dag_object_id);
4382 assert_eq!(result.tx_digest, submitted.digest());
4383 assert_eq!(result.tx_checkpoint, 1);
4384 }
4385
4386 #[test]
4387 fn inspect_execution_options_defaults_are_stable() {
4388 assert_eq!(
4389 InspectExecutionOptions::default(),
4390 InspectExecutionOptions {
4391 timeout: Duration::from_secs(60 * 60),
4392 poll_interval: Duration::from_secs(1),
4393 }
4394 );
4395 }
4396
4397 #[tokio::test]
4398 async fn inspect_execution_rejects_zero_poll_interval_before_spawning() {
4399 let nexus_objects = sui_mocks::mock_nexus_context();
4400 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
4401 sui_mocks::grpc::mock_reference_gas_price(&mut ledger_service_mock, 1000);
4402 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
4403 ledger_service_mock: Some(ledger_service_mock),
4404 ..Default::default()
4405 });
4406 let client = nexus_mocks::mock_nexus_client(&nexus_objects, &rpc_url).await;
4407
4408 let result = client
4409 .workflow()
4410 .inspect_execution(
4411 sui::types::Address::ZERO,
4412 InspectExecutionOptions {
4413 timeout: Duration::from_secs(1),
4414 poll_interval: Duration::ZERO,
4415 },
4416 )
4417 .await;
4418
4419 assert!(matches!(result, Err(NexusError::Configuration(_))));
4420 }
4421 #[tokio::test]
4422 async fn inspect_execution_replays_update_chain_in_order() {
4423 let mut rng = rand::thread_rng();
4424 let nexus_objects = sui_mocks::mock_nexus_context();
4425 let dag_object_id = sui::types::Address::generate(&mut rng);
4426 let execution_object_id = sui::types::Address::generate(&mut rng);
4427 let version_1 = sui::types::ObjectReference::new(
4428 execution_object_id,
4429 1,
4430 sui::types::Digest::generate(&mut rng),
4431 );
4432 let version_9 = sui::types::ObjectReference::new(
4433 execution_object_id,
4434 9,
4435 sui::types::Digest::generate(&mut rng),
4436 );
4437 let version_20 = sui::types::ObjectReference::new(
4438 execution_object_id,
4439 20,
4440 sui::types::Digest::generate(&mut rng),
4441 );
4442 let tx_1 = sui::types::Digest::generate(&mut rng);
4443 let tx_9 = sui::types::Digest::generate(&mut rng);
4444 let tx_20 = sui::types::Digest::generate(&mut rng);
4445 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
4446 let mut state_service_mock = sui_mocks::grpc::MockStateService::new();
4447 mock_execution_observation_context(
4448 &mut ledger_service_mock,
4449 &mut state_service_mock,
4450 &nexus_objects,
4451 version_20.clone(),
4452 );
4453 let package_service_mock = mock_package_graph(&mut ledger_service_mock, &nexus_objects);
4454 expect_object_update_reference(
4455 &mut ledger_service_mock,
4456 &nexus_objects,
4457 version_20.clone(),
4458 1,
4459 None,
4460 tx_20,
4461 );
4462 expect_object_update_reference(
4463 &mut ledger_service_mock,
4464 &nexus_objects,
4465 version_9.clone(),
4466 1,
4467 Some(9),
4468 tx_9,
4469 );
4470 expect_object_update_reference(
4471 &mut ledger_service_mock,
4472 &nexus_objects,
4473 version_1.clone(),
4474 1,
4475 Some(1),
4476 tx_1,
4477 );
4478
4479 let walk_advanced = NexusEventKind::WalkAdvanced(WalkAdvancedEvent {
4480 dag: object_id(dag_object_id),
4481 execution: object_id(execution_object_id),
4482 walk_index: 0,
4483 vertex: RuntimeVertex::plain("initial"),
4484 variant: output_variant("ok"),
4485 variant_ports_to_data: ports_data_map(vec![]),
4486 });
4487 let end_state_reached = NexusEventKind::EndStateReached(EndStateReachedEvent {
4488 dag: object_id(dag_object_id),
4489 execution: object_id(execution_object_id),
4490 walk_index: 0,
4491 vertex: RuntimeVertex::plain("final"),
4492 variant: output_variant("ok"),
4493 variant_ports_to_data: ports_data_map(vec![("answer", b"42")]),
4494 });
4495 let execution_finished = NexusEventKind::ExecutionFinished(ExecutionFinishedEvent {
4496 dag: object_id(dag_object_id),
4497 execution: object_id(execution_object_id),
4498 has_any_walk_failed: false,
4499 has_any_walk_succeeded: true,
4500 was_aborted: false,
4501 });
4502 expect_transaction_update(
4503 &mut ledger_service_mock,
4504 &nexus_objects,
4505 tx_20,
4506 execution_object_id,
4507 20,
4508 *version_20.digest(),
4509 sui::types::ObjectIn::Exist {
4510 version: 9,
4511 digest: *version_9.digest(),
4512 owner: sui::types::Owner::Shared(1),
4513 },
4514 vec![end_state_reached, execution_finished],
4515 );
4516 expect_transaction_update(
4517 &mut ledger_service_mock,
4518 &nexus_objects,
4519 tx_9,
4520 execution_object_id,
4521 9,
4522 *version_9.digest(),
4523 sui::types::ObjectIn::Exist {
4524 version: 1,
4525 digest: *version_1.digest(),
4526 owner: sui::types::Owner::Shared(1),
4527 },
4528 vec![walk_advanced],
4529 );
4530 expect_transaction_update(
4531 &mut ledger_service_mock,
4532 &nexus_objects,
4533 tx_1,
4534 execution_object_id,
4535 1,
4536 *version_1.digest(),
4537 sui::types::ObjectIn::NotExist,
4538 vec![],
4539 );
4540
4541 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
4542 ledger_service_mock: Some(ledger_service_mock),
4543 package_service_mock: Some(package_service_mock),
4544 state_service_mock: Some(state_service_mock),
4545 ..Default::default()
4546 });
4547 let client = nexus_mocks::mock_nexus_client_without_coins(&nexus_objects, &rpc_url).await;
4548 let mut result = client
4549 .workflow()
4550 .inspect_execution(
4551 execution_object_id,
4552 InspectExecutionOptions {
4553 timeout: Duration::from_secs(2),
4554 poll_interval: Duration::from_millis(10),
4555 },
4556 )
4557 .await
4558 .expect("object inspection should start");
4559
4560 let mut events = Vec::new();
4561 while let Some(event) = result.next_event.recv().await {
4562 events.push(event);
4563 }
4564
4565 assert!(result.poller.await.unwrap().is_ok());
4566 assert_eq!(events.len(), 3);
4567 assert!(matches!(events[0].data, NexusEventKind::WalkAdvanced(_)));
4568 assert!(matches!(events[1].data, NexusEventKind::EndStateReached(_)));
4569 assert!(matches!(
4570 events[2].data,
4571 NexusEventKind::ExecutionFinished(_)
4572 ));
4573 }
4574
4575 #[tokio::test]
4576 async fn inspect_execution_retries_missing_transaction_until_deadline() {
4577 let mut rng = rand::thread_rng();
4578 let nexus_objects = sui_mocks::mock_nexus_context();
4579 let dag_object_id = sui::types::Address::generate(&mut rng);
4580 let execution_object_id = sui::types::Address::generate(&mut rng);
4581 let version_1 = sui::types::ObjectReference::new(
4582 execution_object_id,
4583 1,
4584 sui::types::Digest::generate(&mut rng),
4585 );
4586 let version_7 = sui::types::ObjectReference::new(
4587 execution_object_id,
4588 7,
4589 sui::types::Digest::generate(&mut rng),
4590 );
4591 let tx_1 = sui::types::Digest::generate(&mut rng);
4592 let tx_7 = sui::types::Digest::generate(&mut rng);
4593 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
4594 let mut state_service_mock = sui_mocks::grpc::MockStateService::new();
4595 mock_execution_observation_context(
4596 &mut ledger_service_mock,
4597 &mut state_service_mock,
4598 &nexus_objects,
4599 version_1.clone(),
4600 );
4601 let package_service_mock = mock_package_graph(&mut ledger_service_mock, &nexus_objects);
4602 sui_mocks::grpc::mock_reference_gas_price(&mut ledger_service_mock, 1000);
4603 expect_object_update_reference(
4604 &mut ledger_service_mock,
4605 &nexus_objects,
4606 version_1.clone(),
4607 1,
4608 None,
4609 tx_1,
4610 );
4611 ledger_service_mock
4612 .expect_get_object()
4613 .times(1)
4614 .returning(|_| Err(tonic::Status::aborted("retry object observation")));
4615 expect_object_update_reference(
4616 &mut ledger_service_mock,
4617 &nexus_objects,
4618 version_7.clone(),
4619 1,
4620 None,
4621 tx_7,
4622 );
4623 expect_transaction_update(
4624 &mut ledger_service_mock,
4625 &nexus_objects,
4626 tx_1,
4627 execution_object_id,
4628 1,
4629 *version_1.digest(),
4630 sui::types::ObjectIn::NotExist,
4631 vec![NexusEventKind::WalkAdvanced(WalkAdvancedEvent {
4632 dag: object_id(dag_object_id),
4633 execution: object_id(execution_object_id),
4634 walk_index: 0,
4635 vertex: RuntimeVertex::plain("initial"),
4636 variant: output_variant("ok"),
4637 variant_ports_to_data: ports_data_map(vec![]),
4638 })],
4639 );
4640 ledger_service_mock
4641 .expect_get_transaction()
4642 .withf(move |request| request.get_ref().digest_opt() == Some(tx_7.to_string().as_str()))
4643 .times(6)
4644 .returning(|_| Err(tonic::Status::not_found("transaction not visible yet")));
4645 expect_transaction_update(
4646 &mut ledger_service_mock,
4647 &nexus_objects,
4648 tx_7,
4649 execution_object_id,
4650 7,
4651 *version_7.digest(),
4652 sui::types::ObjectIn::Exist {
4653 version: 1,
4654 digest: *version_1.digest(),
4655 owner: sui::types::Owner::Shared(1),
4656 },
4657 vec![NexusEventKind::ExecutionFinished(ExecutionFinishedEvent {
4658 dag: object_id(dag_object_id),
4659 execution: object_id(execution_object_id),
4660 has_any_walk_failed: false,
4661 has_any_walk_succeeded: true,
4662 was_aborted: false,
4663 })],
4664 );
4665
4666 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
4667 ledger_service_mock: Some(ledger_service_mock),
4668 package_service_mock: Some(package_service_mock),
4669 state_service_mock: Some(state_service_mock),
4670 ..Default::default()
4671 });
4672 let client = nexus_mocks::mock_nexus_client(&nexus_objects, &rpc_url).await;
4673 let mut result = client
4674 .workflow()
4675 .inspect_execution(
4676 execution_object_id,
4677 InspectExecutionOptions {
4678 timeout: Duration::from_secs(2),
4679 poll_interval: Duration::from_millis(10),
4680 },
4681 )
4682 .await
4683 .expect("object inspection should start");
4684
4685 let mut events = Vec::new();
4686 while let Some(event) = result.next_event.recv().await {
4687 events.push(event);
4688 }
4689 assert_eq!(events.len(), 2);
4690 assert!(matches!(events[0].data, NexusEventKind::WalkAdvanced(_)));
4691 assert!(matches!(
4692 events[1].data,
4693 NexusEventKind::ExecutionFinished(_)
4694 ));
4695 assert!(result.poller.await.unwrap().is_ok());
4696 }
4697
4698 #[tokio::test]
4699 async fn inspect_execution_does_not_refetch_unchanged_transaction() {
4700 let mut rng = rand::thread_rng();
4701 let nexus_objects = sui_mocks::mock_nexus_context();
4702 let execution_object_id = sui::types::Address::generate(&mut rng);
4703 let object_ref = sui::types::ObjectReference::new(
4704 execution_object_id,
4705 1,
4706 sui::types::Digest::generate(&mut rng),
4707 );
4708 let update_tx = sui::types::Digest::generate(&mut rng);
4709 let execution_type =
4710 crate::move_bindings::struct_tag::<DAGExecution>(&nexus_objects).to_string();
4711 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
4712 let mut state_service_mock = sui_mocks::grpc::MockStateService::new();
4713 mock_execution_observation_context(
4714 &mut ledger_service_mock,
4715 &mut state_service_mock,
4716 &nexus_objects,
4717 object_ref.clone(),
4718 );
4719 let package_service_mock = mock_package_graph(&mut ledger_service_mock, &nexus_objects);
4720 sui_mocks::grpc::mock_reference_gas_price(&mut ledger_service_mock, 1000);
4721 let repeated_ref = object_ref.clone();
4722 ledger_service_mock
4723 .expect_get_object()
4724 .returning(move |request| {
4725 assert_eq!(request.get_ref().version, None);
4726 let mut response = sui::grpc::GetObjectResponse::default();
4727 let mut object = sui::grpc::Object::default();
4728 object.set_object_id(*repeated_ref.object_id());
4729 object.set_owner(sui::grpc::Owner::from(sui::types::Owner::Shared(1)));
4730 object.set_object_type(execution_type.clone());
4731 object.set_version(repeated_ref.version());
4732 object.set_digest(*repeated_ref.digest());
4733 object.set_previous_transaction(update_tx.to_string());
4734 response.set_object(object);
4735 Ok(tonic::Response::new(response))
4736 });
4737 expect_transaction_update(
4738 &mut ledger_service_mock,
4739 &nexus_objects,
4740 update_tx,
4741 execution_object_id,
4742 1,
4743 *object_ref.digest(),
4744 sui::types::ObjectIn::NotExist,
4745 vec![],
4746 );
4747
4748 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
4749 ledger_service_mock: Some(ledger_service_mock),
4750 package_service_mock: Some(package_service_mock),
4751 state_service_mock: Some(state_service_mock),
4752 ..Default::default()
4753 });
4754 let client = nexus_mocks::mock_nexus_client(&nexus_objects, &rpc_url).await;
4755 let mut result = client
4756 .workflow()
4757 .inspect_execution(
4758 execution_object_id,
4759 InspectExecutionOptions {
4760 timeout: Duration::from_millis(120),
4761 poll_interval: Duration::from_millis(20),
4762 },
4763 )
4764 .await
4765 .expect("object inspection should start");
4766
4767 assert!(result.next_event.recv().await.is_none());
4768 let poller_result = result.poller.await.unwrap();
4769 assert!(
4770 matches!(poller_result, Err(NexusError::Timeout(_))),
4771 "unexpected inspection result: {poller_result:?}"
4772 );
4773 }
4774
4775 #[tokio::test]
4776 async fn inspect_execution_surfaces_invalid_update_chain() {
4777 let mut rng = rand::thread_rng();
4778 let nexus_objects = sui_mocks::mock_nexus_context();
4779 let execution_object_id = sui::types::Address::generate(&mut rng);
4780 let object_ref = sui::types::ObjectReference::new(
4781 execution_object_id,
4782 1,
4783 sui::types::Digest::generate(&mut rng),
4784 );
4785 let update_tx = sui::types::Digest::generate(&mut rng);
4786 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
4787 let mut state_service_mock = sui_mocks::grpc::MockStateService::new();
4788 mock_execution_observation_context(
4789 &mut ledger_service_mock,
4790 &mut state_service_mock,
4791 &nexus_objects,
4792 object_ref.clone(),
4793 );
4794 let package_service_mock = mock_package_graph(&mut ledger_service_mock, &nexus_objects);
4795 sui_mocks::grpc::mock_reference_gas_price(&mut ledger_service_mock, 1000);
4796 expect_object_update_reference(
4797 &mut ledger_service_mock,
4798 &nexus_objects,
4799 object_ref,
4800 1,
4801 None,
4802 update_tx,
4803 );
4804 expect_transaction_update(
4805 &mut ledger_service_mock,
4806 &nexus_objects,
4807 update_tx,
4808 execution_object_id,
4809 1,
4810 sui::types::Digest::generate(&mut rng),
4811 sui::types::ObjectIn::NotExist,
4812 vec![],
4813 );
4814
4815 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
4816 ledger_service_mock: Some(ledger_service_mock),
4817 package_service_mock: Some(package_service_mock),
4818 state_service_mock: Some(state_service_mock),
4819 ..Default::default()
4820 });
4821 let client = nexus_mocks::mock_nexus_client(&nexus_objects, &rpc_url).await;
4822 let mut result = client
4823 .workflow()
4824 .inspect_execution(
4825 execution_object_id,
4826 InspectExecutionOptions {
4827 timeout: Duration::from_secs(1),
4828 poll_interval: Duration::from_millis(10),
4829 },
4830 )
4831 .await
4832 .expect("object inspection should start");
4833
4834 assert!(result.next_event.recv().await.is_none());
4835 let error = result
4836 .poller
4837 .await
4838 .unwrap()
4839 .expect_err("invalid update chain should fail");
4840 assert!(error.to_string().contains("output digest"));
4841 }
4842
4843 #[tokio::test]
4844 async fn inspect_execution_reports_missing_transaction_with_reconstruction_context() {
4845 let mut rng = rand::thread_rng();
4846 let nexus_objects = sui_mocks::mock_nexus_context();
4847 let execution_object_id = sui::types::Address::generate(&mut rng);
4848 let object_ref = sui::types::ObjectReference::new(
4849 execution_object_id,
4850 5,
4851 sui::types::Digest::generate(&mut rng),
4852 );
4853 let missing_tx = sui::types::Digest::generate(&mut rng);
4854 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
4855 let mut state_service_mock = sui_mocks::grpc::MockStateService::new();
4856 mock_execution_observation_context(
4857 &mut ledger_service_mock,
4858 &mut state_service_mock,
4859 &nexus_objects,
4860 object_ref.clone(),
4861 );
4862 let package_service_mock = mock_package_graph(&mut ledger_service_mock, &nexus_objects);
4863 sui_mocks::grpc::mock_reference_gas_price(&mut ledger_service_mock, 1000);
4864 expect_object_update_reference(
4865 &mut ledger_service_mock,
4866 &nexus_objects,
4867 object_ref,
4868 1,
4869 None,
4870 missing_tx,
4871 );
4872 ledger_service_mock
4873 .expect_get_transaction()
4874 .withf(move |request| {
4875 request.get_ref().digest_opt() == Some(missing_tx.to_string().as_str())
4876 })
4877 .times(1..)
4878 .returning(|_| Err(tonic::Status::not_found("transaction pruned")));
4879
4880 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
4881 ledger_service_mock: Some(ledger_service_mock),
4882 package_service_mock: Some(package_service_mock),
4883 state_service_mock: Some(state_service_mock),
4884 ..Default::default()
4885 });
4886 let client = nexus_mocks::mock_nexus_client(&nexus_objects, &rpc_url).await;
4887 let mut result = client
4888 .workflow()
4889 .inspect_execution(
4890 execution_object_id,
4891 InspectExecutionOptions {
4892 timeout: Duration::from_secs(1),
4893 poll_interval: Duration::from_millis(10),
4894 },
4895 )
4896 .await
4897 .expect("inspection task should start");
4898
4899 assert!(result.next_event.recv().await.is_none());
4900 let error = result.poller.await.unwrap().expect_err("history must fail");
4901 let message = error.to_string();
4902 assert!(
4903 message.contains(&execution_object_id.to_string()),
4904 "unexpected inspection error: {message}"
4905 );
4906 assert!(
4907 message.contains(&missing_tx.to_string()),
4908 "unexpected inspection error: {message}"
4909 );
4910 assert!(
4911 message.contains("last successfully reconstructed version none"),
4912 "unexpected inspection error: {message}"
4913 );
4914 }
4915
4916 #[tokio::test]
4917 async fn inspect_execution_reports_missing_historical_version() {
4918 let mut rng = rand::thread_rng();
4919 let nexus_objects = sui_mocks::mock_nexus_context();
4920 let execution_object_id = sui::types::Address::generate(&mut rng);
4921 let version_9 = sui::types::ObjectReference::new(
4922 execution_object_id,
4923 9,
4924 sui::types::Digest::generate(&mut rng),
4925 );
4926 let version_1_digest = sui::types::Digest::generate(&mut rng);
4927 let tx_9 = sui::types::Digest::generate(&mut rng);
4928 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
4929 let mut state_service_mock = sui_mocks::grpc::MockStateService::new();
4930 mock_execution_observation_context(
4931 &mut ledger_service_mock,
4932 &mut state_service_mock,
4933 &nexus_objects,
4934 version_9.clone(),
4935 );
4936 let package_service_mock = mock_package_graph(&mut ledger_service_mock, &nexus_objects);
4937 sui_mocks::grpc::mock_reference_gas_price(&mut ledger_service_mock, 1000);
4938 expect_object_update_reference(
4939 &mut ledger_service_mock,
4940 &nexus_objects,
4941 version_9.clone(),
4942 1,
4943 None,
4944 tx_9,
4945 );
4946 ledger_service_mock
4947 .expect_get_object()
4948 .times(1)
4949 .returning(|request| {
4950 assert_eq!(request.get_ref().version, Some(1));
4951 Err(tonic::Status::not_found("historical object pruned"))
4952 });
4953 expect_transaction_update(
4954 &mut ledger_service_mock,
4955 &nexus_objects,
4956 tx_9,
4957 execution_object_id,
4958 9,
4959 *version_9.digest(),
4960 sui::types::ObjectIn::Exist {
4961 version: 1,
4962 digest: version_1_digest,
4963 owner: sui::types::Owner::Shared(1),
4964 },
4965 vec![],
4966 );
4967
4968 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
4969 ledger_service_mock: Some(ledger_service_mock),
4970 package_service_mock: Some(package_service_mock),
4971 state_service_mock: Some(state_service_mock),
4972 ..Default::default()
4973 });
4974 let client = nexus_mocks::mock_nexus_client(&nexus_objects, &rpc_url).await;
4975 let mut result = client
4976 .workflow()
4977 .inspect_execution(
4978 execution_object_id,
4979 InspectExecutionOptions {
4980 timeout: Duration::from_secs(1),
4981 poll_interval: Duration::from_millis(10),
4982 },
4983 )
4984 .await
4985 .expect("inspection task should start");
4986
4987 assert!(result.next_event.recv().await.is_none());
4988 let error = result.poller.await.unwrap().expect_err("history must fail");
4989 let message = error.to_string();
4990 assert!(message.contains(&execution_object_id.to_string()));
4991 assert!(message.contains("missing object version 1"));
4992 assert!(message.contains("last successfully reconstructed version none"));
4993 }
4994
4995 #[cfg(feature = "walrus")]
4996 #[tokio::test]
4997 async fn test_workflow_actions_inspect_execution_until_completion() {
4998 let mut rng = rand::thread_rng();
4999 let nexus_objects = sui_mocks::mock_nexus_context();
5000 let dag_object_id = sui::types::Address::generate(&mut rng);
5001 let execution_object_id = sui::types::Address::generate(&mut rng);
5002 let object_ref = sui::types::ObjectReference::new(
5003 execution_object_id,
5004 42,
5005 sui::types::Digest::generate(&mut rng),
5006 );
5007 let update_tx = sui::types::Digest::generate(&mut rng);
5008
5009 let walk_advanced_event = NexusEventKind::WalkAdvanced(WalkAdvancedEvent {
5010 dag: object_id(dag_object_id),
5011 execution: object_id(execution_object_id),
5012 walk_index: 0,
5013 vertex: RuntimeVertex::plain("initial"),
5014 variant: output_variant("ok"),
5015 variant_ports_to_data: ports_data_map(vec![]),
5016 });
5017 let terminal_err_eval_event =
5018 NexusEventKind::TerminalErrEvalRecorded(TerminalErrEvalRecordedEvent {
5019 dag: object_id(dag_object_id),
5020 execution: object_id(execution_object_id),
5021 walk_index: 1,
5022 vertex: RuntimeVertex::plain("failable"),
5023 leader: sui::types::Address::THREE,
5024 failure_class: WorkflowFailureClass::TerminalToolFailure,
5025 outcome: MoveOption::from_option(Some(PostFailureAction::Terminate)),
5026 reason: MoveString::from("tool failed"),
5027 err_eval_hash: vec![9, 8, 7],
5028 duplicate: false,
5029 });
5030 let end_state_reached_event = NexusEventKind::EndStateReached(EndStateReachedEvent {
5031 dag: object_id(dag_object_id),
5032 execution: object_id(execution_object_id),
5033 walk_index: 0,
5034 vertex: RuntimeVertex::plain("final"),
5035 variant: output_variant("ok"),
5036 variant_ports_to_data: ports_data_map(vec![("answer", b"42")]),
5037 });
5038 let execution_finished_event = NexusEventKind::ExecutionFinished(ExecutionFinishedEvent {
5039 dag: object_id(dag_object_id),
5040 execution: object_id(execution_object_id),
5041 has_any_walk_failed: true,
5042 has_any_walk_succeeded: true,
5043 was_aborted: false,
5044 });
5045
5046 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
5047 let mut state_service_mock = sui_mocks::grpc::MockStateService::new();
5048 mock_execution_observation_context(
5049 &mut ledger_service_mock,
5050 &mut state_service_mock,
5051 &nexus_objects,
5052 object_ref.clone(),
5053 );
5054 let package_service_mock = mock_package_graph(&mut ledger_service_mock, &nexus_objects);
5055 sui_mocks::grpc::mock_reference_gas_price(&mut ledger_service_mock, 1000);
5056 expect_object_update_reference(
5057 &mut ledger_service_mock,
5058 &nexus_objects,
5059 object_ref.clone(),
5060 1,
5061 None,
5062 update_tx,
5063 );
5064 expect_transaction_update(
5065 &mut ledger_service_mock,
5066 &nexus_objects,
5067 update_tx,
5068 execution_object_id,
5069 object_ref.version(),
5070 *object_ref.digest(),
5071 sui::types::ObjectIn::NotExist,
5072 vec![
5073 walk_advanced_event,
5074 terminal_err_eval_event,
5075 end_state_reached_event,
5076 execution_finished_event,
5077 ],
5078 );
5079
5080 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
5081 ledger_service_mock: Some(ledger_service_mock),
5082 package_service_mock: Some(package_service_mock),
5083 state_service_mock: Some(state_service_mock),
5084 ..Default::default()
5085 });
5086 let client = nexus_mocks::mock_nexus_client(&nexus_objects, &rpc_url).await;
5087
5088 let result = client
5089 .workflow()
5090 .inspect_execution_until_completion(
5091 execution_object_id,
5092 InspectExecutionOptions {
5093 timeout: Duration::from_secs(2),
5094 poll_interval: Duration::from_millis(10),
5095 },
5096 &StorageConf::default(),
5097 )
5098 .await
5099 .expect("Failed to inspect execution until completion");
5100
5101 assert_eq!(
5102 result.terminal_state,
5103 WorkflowExecutionTerminalState::Failed
5104 );
5105 assert!(result.execution_finished.has_any_walk_failed);
5106 assert!(result.execution_finished.has_any_walk_succeeded);
5107 assert!(matches!(
5108 result.events.last().map(|event| &event.data),
5109 Some(NexusEventKind::ExecutionFinished(_))
5110 ));
5111 assert_eq!(result.terminal_err_eval_recordings.len(), 1);
5112 assert_eq!(
5113 result.terminal_err_eval_recordings[0].failure_class,
5114 WorkflowFailureClass::TerminalToolFailure
5115 );
5116 assert_eq!(result.events.len(), 4);
5117 }
5118
5119 #[tokio::test]
5120 async fn inspect_execution_total_timeout_covers_initial_request_retries() {
5121 let nexus_objects = sui_mocks::mock_nexus_context();
5122 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
5123 let mut state_service_mock = sui_mocks::grpc::MockStateService::new();
5124 sui_mocks::grpc::mock_reference_gas_price(&mut ledger_service_mock, 1000);
5125 ledger_service_mock
5126 .expect_get_object()
5127 .times(1..)
5128 .returning(|_| Err(tonic::Status::unavailable("temporary localnet outage")));
5129 state_service_mock
5130 .expect_list_dynamic_fields()
5131 .times(1..)
5132 .returning(|_| Err(tonic::Status::unavailable("temporary localnet outage")));
5133
5134 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
5135 ledger_service_mock: Some(ledger_service_mock),
5136 state_service_mock: Some(state_service_mock),
5137 ..Default::default()
5138 });
5139 let client = nexus_mocks::mock_nexus_client(&nexus_objects, &rpc_url).await;
5140 let mut result = client
5141 .workflow()
5142 .inspect_execution(
5143 sui::types::Address::TWO,
5144 InspectExecutionOptions {
5145 timeout: Duration::from_millis(80),
5146 poll_interval: Duration::from_millis(10),
5147 },
5148 )
5149 .await
5150 .expect("inspection task should start");
5151
5152 assert!(result.next_event.recv().await.is_none());
5153 let poller_result = result.poller.await.unwrap();
5154 assert!(
5155 matches!(poller_result, Err(NexusError::Timeout(_))),
5156 "unexpected inspection result: {poller_result:?}"
5157 );
5158 }
5159
5160 #[test]
5161 fn execution_inspection_retries_only_confirmed_transient_grpc_codes() {
5162 for code in [
5163 tonic::Code::Unavailable,
5164 tonic::Code::DeadlineExceeded,
5165 tonic::Code::ResourceExhausted,
5166 tonic::Code::Aborted,
5167 ] {
5168 let error = NexusError::Rpc(
5169 anyhow::Error::new(tonic::Status::new(code, "retry"))
5170 .context("inspection request failed"),
5171 );
5172 assert!(is_transient_inspection_error(&error), "{code:?}");
5173 }
5174
5175 let permanent = NexusError::Rpc(anyhow::Error::new(tonic::Status::not_found("missing")));
5176 assert!(!is_transient_inspection_error(&permanent));
5177 }
5178
5179 #[test]
5180 fn execution_history_includes_the_scheduler_dispatch() {
5181 let execution = sui::types::Address::TWO;
5182 let event = NexusEventKind::OccurrenceDispatched(OccurrenceDispatchedEvent {
5183 task_id: object_id(sui::types::Address::ZERO),
5184 occurrence_id: 7,
5185 execution_id: object_id(execution),
5186 dispatched_at_ms: 11,
5187 });
5188
5189 assert_eq!(event_execution_id(&event), Some(execution));
5190 }
5191
5192 #[test]
5193 fn execution_history_includes_the_agent_skill_request() {
5194 let execution = sui::types::Address::TWO;
5195 let event =
5196 NexusEventKind::AgentSkillExecutionRequested(AgentSkillExecutionRequestedEvent {
5197 execution_id: execution,
5198 agent_id: object_id(sui::types::Address::THREE),
5199 skill_id: 7,
5200 interface_revision: InterfaceVersion::new(1),
5201 payment_id: sui::types::Address::from_static("0x4"),
5202 });
5203
5204 assert_eq!(event_execution_id(&event), Some(execution));
5205 }
5206
5207 #[test]
5208 fn execution_history_includes_the_invocation_boundary() {
5209 let execution = sui::types::Address::TWO;
5210 let dag = object_id(sui::types::Address::ZERO);
5211 let events = [
5212 NexusEventKind::InvocationAuthorizationRequired(InvocationAuthorizationRequiredEvent {
5213 dag,
5214 execution: object_id(execution),
5215 walk_index: 0,
5216 vertex: RuntimeVertex::plain("tool"),
5217 }),
5218 NexusEventKind::RequestWalkExecution(RequestWalkExecutionEvent {
5219 dag,
5220 execution: object_id(execution),
5221 invocation: object_id(sui::types::Address::from_static("0x4")),
5222 invoker: sui::types::Address::THREE,
5223 walk_index: 0,
5224 next_vertex: RuntimeVertex::plain("tool"),
5225 evaluations: object_id(sui::types::Address::from_static("0x5")),
5226 agent_id: object_id(sui::types::Address::from_static("0x6")),
5227 skill_id: 7,
5228 interface_version: InterfaceVersion::new(1),
5229 task_id: object_id(sui::types::Address::from_static("0x7")),
5230 occurrence_id: 8,
5231 }),
5232 NexusEventKind::CommittedToolResult(CommittedToolResultEvent {
5233 dag,
5234 execution: object_id(execution),
5235 walk_index: 0,
5236 vertex: RuntimeVertex::plain("tool"),
5237 leader: object_id(sui::types::Address::from_static("0x8")),
5238 has_primary_failure_evidence: false,
5239 has_secondary_failure_evidence: false,
5240 }),
5241 ];
5242
5243 for event in events {
5244 assert_eq!(event_execution_id(&event), Some(execution));
5245 }
5246 }
5247
5248 #[test]
5249 fn test_event_execution_id_supports_terminal_err_eval_recorded() {
5250 let execution = sui::types::Address::TWO;
5251 let event = NexusEventKind::TerminalErrEvalRecorded(TerminalErrEvalRecordedEvent {
5252 dag: object_id(sui::types::Address::ZERO),
5253 execution: object_id(execution),
5254 walk_index: 2,
5255 vertex: RuntimeVertex::plain("failable"),
5256 leader: sui::types::Address::THREE,
5257 failure_class: WorkflowFailureClass::TerminalSubmissionFailure,
5258 outcome: MoveOption::from_option(Some(PostFailureAction::Terminate)),
5259 reason: MoveString::from("timeout"),
5260 err_eval_hash: vec![4, 5, 6],
5261 duplicate: true,
5262 });
5263
5264 assert_eq!(event_execution_id(&event), Some(execution));
5265 }
5266
5267 #[test]
5268 fn test_event_execution_id_supports_tool_verification_resolved() {
5269 let execution = sui::types::Address::TWO;
5270 let event = NexusEventKind::ToolVerificationResolved(ToolVerificationResolvedEvent {
5271 dag: object_id(sui::types::Address::ZERO),
5272 execution: object_id(execution),
5273 walk_index: 2,
5274 vertex: RuntimeVertex::plain("verified"),
5275 leader_cap_id: object_id(sui::types::Address::THREE),
5276 tool_id: object_id(sui::types::Address::from_static("0x4")),
5277 verifier_kind: ToolVerifierMode::External,
5278 verifier_witness_id: MoveOption::from_option(Some(object_id(
5279 sui::types::Address::from_static("0x5"),
5280 ))),
5281 decision: VerifierDecision::Accept,
5282 });
5283
5284 assert_eq!(event_execution_id(&event), Some(execution));
5285 }
5286 #[test]
5287 fn test_event_execution_id_supports_object_history_events() {
5288 let execution = sui::types::Address::TWO;
5289 let events = [
5290 NexusEventKind::ExecutionPaymentFeesRecorded(ExecutionPaymentFeesRecordedEvent {
5291 payment_id: sui::types::Address::ZERO,
5292 execution_id: execution,
5293 agent_id: object_id(sui::types::Address::THREE),
5294 skill_id: 1,
5295 gas_fee_mist: 2,
5296 tool_fee_mist: 3,
5297 priority_fee_mist: 4,
5298 priority_fee_percentage: 5,
5299 }),
5300 NexusEventKind::ExecutionPaymentToolCostSnapshotted(
5301 ExecutionPaymentToolCostSnapshottedEvent {
5302 payment_id: sui::types::Address::ZERO,
5303 execution_id: execution,
5304 agent_id: object_id(sui::types::Address::THREE),
5305 tool_fqn: b"demo::tool".to_vec(),
5306 cost: 6,
5307 },
5308 ),
5309 NexusEventKind::InvocationLocked(InvocationLockedEvent {
5310 execution: object_id(execution),
5311 vertex: RuntimeVertex::plain("tool"),
5312 tool: object_id(sui::types::Address::from_static("0x4")),
5313 tool_fqn: MoveString::from("demo::tool"),
5314 cashier: object_id(sui::types::Address::from_static("0x5")),
5315 invocation: object_id(sui::types::Address::from_static("0x6")),
5316 beneficiary: PaymentSourceKind::user_funded(sui::types::Address::THREE),
5317 policy: TypeName::new("0x7::fixed_price::Policy"),
5318 sources: vec![object_id(sui::types::Address::from_static("0x8"))],
5319 amount: 7,
5320 }),
5321 NexusEventKind::InvocationSettled(InvocationSettledEvent {
5322 execution: object_id(execution),
5323 vertex: RuntimeVertex::plain("tool"),
5324 tool: object_id(sui::types::Address::from_static("0x4")),
5325 tool_fqn: MoveString::from("demo::tool"),
5326 cashier: object_id(sui::types::Address::from_static("0x5")),
5327 invocation: object_id(sui::types::Address::from_static("0x6")),
5328 beneficiary: PaymentSourceKind::user_funded(sui::types::Address::THREE),
5329 policy: TypeName::new("0x7::fixed_price::Policy"),
5330 sources: vec![object_id(sui::types::Address::from_static("0x8"))],
5331 amount: 8,
5332 was_refunded: false,
5333 }),
5334 NexusEventKind::SubmissionFailureEvidenceRecorded(
5335 SubmissionFailureEvidenceRecordedEvent {
5336 dag: object_id(sui::types::Address::ZERO),
5337 execution: object_id(execution),
5338 walk_index: 9,
5339 vertex: RuntimeVertex::plain("submission_failure"),
5340 failed_leader: sui::types::Address::THREE,
5341 winning_leader: MoveOption::from_option(None),
5342 reason: MoveString::from("invalid evidence"),
5343 err_eval_hash: vec![10, 11, 12],
5344 },
5345 ),
5346 NexusEventKind::WalkPendingAbort(WalkPendingAbortEvent {
5347 dag: object_id(sui::types::Address::ZERO),
5348 execution: object_id(execution),
5349 walk_index: 13,
5350 vertex: RuntimeVertex::plain("pending_abort"),
5351 }),
5352 NexusEventKind::ExecutionPaymentRefilled(ExecutionPaymentRefilledEvent {
5353 execution_id: execution,
5354 payment_id: sui::types::Address::ZERO,
5355 source: sui::types::Address::THREE,
5356 refill_amount: 14,
5357 }),
5358 ];
5359
5360 for event in events {
5361 assert_eq!(event_execution_id(&event), Some(execution));
5362 }
5363 }
5364
5365 #[cfg(feature = "walrus")]
5366 #[test]
5367 fn test_terminal_state_from_execution_finished() {
5368 let success = ExecutionFinishedEvent {
5369 dag: object_id(sui::types::Address::ZERO),
5370 execution: object_id(sui::types::Address::TWO),
5371 has_any_walk_failed: false,
5372 has_any_walk_succeeded: true,
5373 was_aborted: false,
5374 };
5375 let failed = ExecutionFinishedEvent {
5376 has_any_walk_failed: true,
5377 has_any_walk_succeeded: false,
5378 ..success.clone()
5379 };
5380 let aborted = ExecutionFinishedEvent {
5381 has_any_walk_failed: true,
5382 has_any_walk_succeeded: true,
5383 was_aborted: true,
5384 ..success.clone()
5385 };
5386 let no_walk_outcome = ExecutionFinishedEvent {
5387 has_any_walk_failed: false,
5388 has_any_walk_succeeded: false,
5389 was_aborted: false,
5390 ..success.clone()
5391 };
5392
5393 assert_eq!(
5394 terminal_state_from_execution_finished(&success),
5395 WorkflowExecutionTerminalState::Succeeded
5396 );
5397 assert_eq!(
5398 terminal_state_from_execution_finished(&failed),
5399 WorkflowExecutionTerminalState::Failed
5400 );
5401 assert_eq!(
5402 terminal_state_from_execution_finished(&aborted),
5403 WorkflowExecutionTerminalState::Aborted
5404 );
5405 assert_eq!(
5406 terminal_state_from_execution_finished(&no_walk_outcome),
5407 WorkflowExecutionTerminalState::NoWalkOutcome
5408 );
5409 }
5410
5411 #[cfg(feature = "walrus")]
5412 #[tokio::test]
5413 async fn test_build_execution_completion_result_resolves_end_states() {
5414 let execution = sui::types::Address::TWO;
5415 let events = vec![
5416 NexusEvent {
5417 id: (sui::types::Digest::ZERO, 0),
5418 emitting_package: sui::types::Address::ZERO,
5419 generics: vec![],
5420 data: NexusEventKind::TerminalErrEvalRecorded(TerminalErrEvalRecordedEvent {
5421 dag: object_id(sui::types::Address::ZERO),
5422 execution: object_id(execution),
5423 walk_index: 1,
5424 vertex: RuntimeVertex::plain("failable"),
5425 leader: sui::types::Address::THREE,
5426 failure_class: WorkflowFailureClass::TerminalToolFailure,
5427 outcome: MoveOption::from_option(Some(PostFailureAction::Terminate)),
5428 reason: MoveString::from("tool failed"),
5429 err_eval_hash: vec![1, 2, 3],
5430 duplicate: false,
5431 }),
5432 distribution: None,
5433 },
5434 NexusEvent {
5435 id: (sui::types::Digest::ZERO, 1),
5436 emitting_package: sui::types::Address::ZERO,
5437 generics: vec![],
5438 data: NexusEventKind::EndStateReached(EndStateReachedEvent {
5439 dag: object_id(sui::types::Address::ZERO),
5440 execution: object_id(execution),
5441 walk_index: 0,
5442 vertex: RuntimeVertex::plain("final"),
5443 variant: output_variant("ok"),
5444 variant_ports_to_data: ports_data_map(vec![("answer", b"42")]),
5445 }),
5446 distribution: None,
5447 },
5448 NexusEvent {
5449 id: (sui::types::Digest::ZERO, 2),
5450 emitting_package: sui::types::Address::ZERO,
5451 generics: vec![],
5452 data: NexusEventKind::ExecutionFinished(ExecutionFinishedEvent {
5453 dag: object_id(sui::types::Address::ZERO),
5454 execution: object_id(execution),
5455 has_any_walk_failed: true,
5456 has_any_walk_succeeded: true,
5457 was_aborted: false,
5458 }),
5459 distribution: None,
5460 },
5461 ];
5462
5463 let result = build_execution_completion_result(events, execution, &StorageConf::default())
5464 .await
5465 .expect("summary should build");
5466
5467 assert_eq!(
5468 result.terminal_state,
5469 WorkflowExecutionTerminalState::Failed
5470 );
5471 assert_eq!(result.terminal_err_eval_recordings.len(), 1);
5472 assert_eq!(result.end_states.len(), 1);
5473 assert_eq!(
5474 result.end_states[0].event.vertex,
5475 RuntimeVertex::plain("final")
5476 );
5477 assert!(matches!(
5478 result.end_states[0]
5479 .resolved_ports_to_data
5480 .get("answer")
5481 .expect("answer port")
5482 .values()
5483 .expect("resolved answer is canonical")
5484 .as_slice(),
5485 [NexusValue::InlineData { bytes }] if bytes == b"42"
5486 ));
5487 }
5488
5489 #[tokio::test]
5490 async fn test_workflow_actions_execution_cost() {
5491 for detailed in [false, true] {
5492 let nexus_objects = sui_mocks::mock_nexus_context();
5493 let execution_ref = sui_mocks::mock_sui_object_ref();
5494 let payment_ref = sui_mocks::mock_sui_object_ref();
5495 let execution_id = *execution_ref.object_id();
5496 let invocation_id = sui::types::Address::from_static("0x123");
5497
5498 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
5499 let mut state_service_mock = sui_mocks::grpc::MockStateService::new();
5500
5501 mock_get_dag_execution_bcs(
5502 &mut ledger_service_mock,
5503 &mut state_service_mock,
5504 &nexus_objects,
5505 execution_ref,
5506 &sui::types::ObjectReference::new(
5507 sui::types::Address::from_static("0xd"),
5508 0,
5509 sui::types::Digest::ZERO,
5510 ),
5511 vec![],
5512 );
5513
5514 mock_execution_payment_field(
5515 &mut ledger_service_mock,
5516 &mut state_service_mock,
5517 &nexus_objects,
5518 execution_id,
5519 payment_ref.clone(),
5520 ExecutionPaymentInnerV1 {
5521 execution_id,
5522 agent_id: crate::move_bindings::sui_framework::object::ID::new(
5523 sui::types::Address::from_static("0xa"),
5524 ),
5525 skill_id: 11,
5526 interface_revision:
5527 crate::move_bindings::interface::version::InterfaceVersion::new(7),
5528 payment_policy:
5529 crate::move_bindings::interface::payment::SkillPaymentPolicy::UserFunded,
5530 source_kind:
5531 crate::move_bindings::interface::payment::PaymentSourceKind::user_funded(
5532 sui::types::Address::from_static("0x1"),
5533 ),
5534 max_budget_mist: 100_000,
5535 gas_budget_mist: 83_334,
5536 priority_fee_reserve_mist: 16_666,
5537 locked_budget_mist: 100_000,
5538 funds: crate::move_bindings::sui_framework::balance::Balance {
5539 value: 58_000,
5540 phantom_t0: std::marker::PhantomData,
5541 },
5542 consumed: 42_000,
5543 tool_fee_charged: 42_000,
5544 priority_fee_charged: 0,
5545 priority_fee_percentage: 20,
5546 accomplished: true,
5547 refunded: false,
5548 final_state: ExecutionPaymentFinalState::Accomplished,
5549 tool_cost_snapshot: crate::move_bindings::sui_framework::vec_map::VecMap {
5550 contents: vec![],
5551 },
5552 locked_vertices: vec![invocation_lock(vec![1, 2, 3], invocation_id)],
5553 },
5554 );
5555
5556 let package_service_mock = mock_package_graph(&mut ledger_service_mock, &nexus_objects);
5557 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
5558 ledger_service_mock: Some(ledger_service_mock),
5559 package_service_mock: Some(package_service_mock),
5560 state_service_mock: Some(state_service_mock),
5561 ..Default::default()
5562 });
5563
5564 let client =
5565 nexus_mocks::mock_nexus_client_without_coins(&nexus_objects, &rpc_url).await;
5566
5567 let result = if detailed {
5568 let details = client
5569 .workflow()
5570 .execution_cost_details(execution_id)
5571 .await
5572 .expect("Failed to fetch execution cost details");
5573 assert_eq!(details.outstanding_invocation_ids, vec![invocation_id]);
5574 details.summary
5575 } else {
5576 client
5577 .workflow()
5578 .execution_cost(execution_id)
5579 .await
5580 .expect("Failed to fetch execution cost")
5581 };
5582
5583 assert_eq!(result.payment_id, *payment_ref.object_id());
5584 assert_eq!(result.max_budget_mist, 100_000);
5585 assert_eq!(result.locked_budget_mist, 100_000);
5586 assert_eq!(result.consumed, 42_000);
5587 assert_eq!(result.outstanding_locks, 1);
5588 assert!(result.accomplished);
5589 assert!(!result.refunded);
5590 }
5591 }
5592
5593 #[tokio::test]
5594 async fn test_abort_expired_execution_invocation_candidates_returns_empty_snapshot() {
5595 let nexus_objects = sui_mocks::mock_nexus_context();
5596 let execution_ref = sui_mocks::mock_sui_object_ref();
5597 let dag_ref = sui_mocks::mock_sui_object_ref();
5598 let payment_ref = sui_mocks::mock_sui_object_ref();
5599 let dag = dag_bcs(0);
5600 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
5601 let mut state_service_mock = sui_mocks::grpc::MockStateService::new();
5602
5603 sui_mocks::grpc::mock_reference_gas_price(&mut ledger_service_mock, 1000);
5604 mock_get_dag_execution_bcs(
5605 &mut ledger_service_mock,
5606 &mut state_service_mock,
5607 &nexus_objects,
5608 execution_ref.clone(),
5609 &dag_ref,
5610 vec![],
5611 );
5612 mock_get_dag_bcs(
5613 &mut ledger_service_mock,
5614 &mut state_service_mock,
5615 &nexus_objects,
5616 dag_ref,
5617 dag,
5618 );
5619 sui_mocks::grpc::mock_get_object_bcs(
5620 &mut ledger_service_mock,
5621 sui::types::ObjectReference::new(
5622 move_boundary::CLOCK_OBJECT_ID,
5623 1,
5624 sui::types::Digest::from([1; 32]),
5625 ),
5626 sui::types::Owner::Shared(1),
5627 clock_bcs(61_000),
5628 );
5629 mock_execution_payment_field(
5630 &mut ledger_service_mock,
5631 &mut state_service_mock,
5632 &nexus_objects,
5633 *execution_ref.object_id(),
5634 payment_ref.clone(),
5635 ExecutionPaymentInnerV1 {
5636 execution_id: *execution_ref.object_id(),
5637 agent_id: crate::move_bindings::sui_framework::object::ID::new(
5638 sui::types::Address::from_static("0xa"),
5639 ),
5640 skill_id: 11,
5641 interface_revision: InterfaceVersion::new(7),
5642 payment_policy: SkillPaymentPolicy::UserFunded,
5643 source_kind:
5644 crate::move_bindings::interface::payment::PaymentSourceKind::user_funded(
5645 sui::types::Address::from_static("0x1"),
5646 ),
5647 max_budget_mist: 100_000,
5648 gas_budget_mist: 83_334,
5649 priority_fee_reserve_mist: 16_666,
5650 locked_budget_mist: 0,
5651 funds: crate::move_bindings::sui_framework::balance::Balance {
5652 value: 100_000,
5653 phantom_t0: std::marker::PhantomData,
5654 },
5655 consumed: 0,
5656 tool_fee_charged: 0,
5657 priority_fee_charged: 0,
5658 priority_fee_percentage: 20,
5659 accomplished: false,
5660 refunded: false,
5661 final_state: ExecutionPaymentFinalState::Pending,
5662 tool_cost_snapshot: crate::move_bindings::sui_framework::vec_map::VecMap {
5663 contents: vec![],
5664 },
5665 locked_vertices: vec![],
5666 },
5667 );
5668
5669 let package_service_mock = mock_package_graph(&mut ledger_service_mock, &nexus_objects);
5670 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
5671 ledger_service_mock: Some(ledger_service_mock),
5672 package_service_mock: Some(package_service_mock),
5673 state_service_mock: Some(state_service_mock),
5674 ..Default::default()
5675 });
5676 let client = nexus_mocks::mock_nexus_client(&nexus_objects, &rpc_url).await;
5677
5678 let candidates = client
5679 .workflow()
5680 .abort_expired_execution_invocation_candidates(*execution_ref.object_id())
5681 .await
5682 .expect("empty candidate snapshot should parse");
5683
5684 assert!(candidates.is_empty());
5685 }
5686
5687 #[tokio::test]
5688 async fn test_abort_expired_execution_with_invocation_submits_selected_candidate() {
5689 let mut rng = rand::thread_rng();
5690 let nexus_objects = sui_mocks::mock_nexus_context();
5691 let gas_coin_ref = sui_mocks::mock_sui_object_ref();
5692 let execution_ref = sui_mocks::mock_sui_object_ref();
5693 let dag_ref = sui_mocks::mock_sui_object_ref();
5694 let payment_ref = sui_mocks::mock_sui_object_ref();
5695 let tool_fqn = fqn!("xyz.taluslabs.payable@1");
5696 let invocation_id = sui::types::Address::from_static("0x404");
5697 let invocation_ref = sui_mocks::object_ref_for_id(invocation_id);
5698 let vertex = RuntimeVertex::plain("payable");
5699 let dag = dag_bcs(1);
5700 let vertex_key = graph_move::Vertex::new("payable");
5701 let vertex_field_id = dag.vertices.id().derive_dynamic_child_id(
5702 &crate::move_bindings::type_tag::<graph_move::Vertex>(&nexus_objects),
5703 &bcs::to_bytes(&vertex_key).expect("vertex key serializes"),
5704 );
5705 let vertex_field_ref = sui_mocks::object_ref_for_id(vertex_field_id);
5706 let execution_walks = vec![execution_move::DAGWalk::Active {
5707 next_vertex: vertex.clone(),
5708 timeout_ms: 30_000,
5709 requires_vertex_authorization_grant: false,
5710 created_at: 1_000,
5711 }];
5712 let payment_vertex_key =
5713 payment_vertex_key(*execution_ref.object_id(), &vertex, &tool_fqn).unwrap();
5714 let current_locked_vertices = vec![ExecutionPaymentVertexLock {
5715 vertex_key: payment_vertex_key.clone(),
5716 invocation_id: object_id(invocation_id),
5717 amount: 10,
5718 }];
5719 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
5720 let mut state_service_mock = sui_mocks::grpc::MockStateService::new();
5721 let mut tx_service_mock = sui_mocks::grpc::MockTransactionExecutionService::new();
5722 let mut sub_service_mock = sui_mocks::grpc::MockSubscriptionService::new();
5723
5724 sui_mocks::grpc::mock_reference_gas_price(&mut ledger_service_mock, 1000);
5725 mock_get_dag_execution_bcs(
5726 &mut ledger_service_mock,
5727 &mut state_service_mock,
5728 &nexus_objects,
5729 execution_ref.clone(),
5730 &dag_ref,
5731 execution_walks.clone(),
5732 );
5733 mock_get_dag_bcs(
5734 &mut ledger_service_mock,
5735 &mut state_service_mock,
5736 &nexus_objects,
5737 dag_ref.clone(),
5738 dag,
5739 );
5740 sui_mocks::grpc::mock_get_dynamic_table_values_bcs(
5741 &mut ledger_service_mock,
5742 vec![(
5743 vertex_field_ref,
5744 sui::types::Owner::Shared(1),
5745 vertex_key,
5746 offchain_vertex_node_bcs(&tool_fqn),
5747 )],
5748 );
5749 sui_mocks::grpc::mock_get_object_bcs(
5750 &mut ledger_service_mock,
5751 sui::types::ObjectReference::new(
5752 move_boundary::CLOCK_OBJECT_ID,
5753 1,
5754 sui::types::Digest::from([1; 32]),
5755 ),
5756 sui::types::Owner::Shared(1),
5757 clock_bcs(61_000),
5758 );
5759 mock_execution_payment_field(
5760 &mut ledger_service_mock,
5761 &mut state_service_mock,
5762 &nexus_objects,
5763 *execution_ref.object_id(),
5764 payment_ref.clone(),
5765 ExecutionPaymentInnerV1 {
5766 execution_id: *execution_ref.object_id(),
5767 agent_id: crate::move_bindings::sui_framework::object::ID::new(
5768 sui::types::Address::from_static("0xa"),
5769 ),
5770 skill_id: 11,
5771 interface_revision: InterfaceVersion::new(7),
5772 payment_policy: SkillPaymentPolicy::UserFunded,
5773 source_kind:
5774 crate::move_bindings::interface::payment::PaymentSourceKind::user_funded(
5775 sui::types::Address::from_static("0x1"),
5776 ),
5777 max_budget_mist: 100_000,
5778 gas_budget_mist: 83_334,
5779 priority_fee_reserve_mist: 16_666,
5780 locked_budget_mist: 10,
5781 funds: crate::move_bindings::sui_framework::balance::Balance {
5782 value: 100_000,
5783 phantom_t0: std::marker::PhantomData,
5784 },
5785 consumed: 0,
5786 tool_fee_charged: 0,
5787 priority_fee_charged: 0,
5788 priority_fee_percentage: 20,
5789 accomplished: false,
5790 refunded: false,
5791 final_state: ExecutionPaymentFinalState::Pending,
5792 tool_cost_snapshot: crate::move_bindings::sui_framework::vec_map::VecMap {
5793 contents: vec![],
5794 },
5795 locked_vertices: current_locked_vertices,
5796 },
5797 );
5798 sui_mocks::grpc::mock_get_object_metadata_exact(
5799 &mut ledger_service_mock,
5800 invocation_ref.clone(),
5801 sui::types::Owner::Object(*execution_ref.object_id()),
5802 None,
5803 );
5804 sui_mocks::grpc::mock_runtime_authority(&mut ledger_service_mock, &nexus_objects, false);
5805 let package_service_mock = mock_package_graph(&mut ledger_service_mock, &nexus_objects);
5806 let submitted = sui_mocks::grpc::mock_execute_transaction_and_wait_for_checkpoint(
5807 &mut tx_service_mock,
5808 &mut sub_service_mock,
5809 &mut ledger_service_mock,
5810 gas_coin_ref.clone(),
5811 vec![],
5812 vec![],
5813 vec![],
5814 );
5815
5816 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
5817 ledger_service_mock: Some(ledger_service_mock),
5818 package_service_mock: Some(package_service_mock),
5819 state_service_mock: Some(state_service_mock),
5820 execution_service_mock: Some(tx_service_mock),
5821 subscription_service_mock: Some(sub_service_mock),
5822 ..Default::default()
5823 });
5824 let pk = sui::crypto::Ed25519PrivateKey::generate(&mut rng);
5825 let client = NexusClient::builder()
5826 .with_private_key(pk)
5827 .with_rpc_url(&rpc_url)
5828 .with_nexus_objects(nexus_objects.objects().clone())
5829 .with_gas(vec![gas_coin_ref], 1000)
5830 .build()
5831 .await
5832 .expect("mock client should build");
5833
5834 let result = client
5835 .workflow()
5836 .abort_expired_execution_with_invocation(
5837 *execution_ref.object_id(),
5838 Some(invocation_id),
5839 )
5840 .await
5841 .expect("abort transaction should submit");
5842
5843 assert_eq!(result.tx_digest, submitted.digest());
5844 assert_eq!(result.tx_checkpoint, 1);
5845 assert_eq!(result.dag_id, *dag_ref.object_id());
5846 assert_eq!(result.dag_execution_id, *execution_ref.object_id());
5847 assert_eq!(result.selected_candidate.invocation_ref, invocation_ref);
5848 assert_eq!(result.selected_candidate.vertex, vertex);
5849 assert_eq!(
5850 result.selected_candidate.payment_vertex_key,
5851 payment_vertex_key
5852 );
5853 }
5854
5855 async fn mock_client_for_workflow_submit(
5856 nexus_objects: &crate::types::NexusObjects,
5857 gas_coin_ref: sui::types::ObjectReference,
5858 ledger_service_mock: sui_mocks::grpc::MockLedgerService,
5859 package_service_mock: sui_mocks::grpc::MockMovePackageService,
5860 state_service_mock: sui_mocks::grpc::MockStateService,
5861 tx_service_mock: sui_mocks::grpc::MockTransactionExecutionService,
5862 sub_service_mock: sui_mocks::grpc::MockSubscriptionService,
5863 ) -> NexusClient {
5864 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
5865 ledger_service_mock: Some(ledger_service_mock),
5866 package_service_mock: Some(package_service_mock),
5867 state_service_mock: Some(state_service_mock),
5868 execution_service_mock: Some(tx_service_mock),
5869 subscription_service_mock: Some(sub_service_mock),
5870 ..Default::default()
5871 });
5872 let pk = sui::crypto::Ed25519PrivateKey::generate(rand::thread_rng());
5873 NexusClient::builder()
5874 .with_private_key(pk)
5875 .with_rpc_url(&rpc_url)
5876 .with_nexus_objects(nexus_objects.clone())
5877 .with_gas(vec![gas_coin_ref], 1000)
5878 .build()
5879 .await
5880 .expect("mock client should build")
5881 }
5882
5883 #[tokio::test]
5884 async fn test_abort_expired_execution_submits_workflow_transaction() {
5885 let nexus_objects = sui_mocks::mock_nexus_context();
5886 let gas_coin_ref = sui_mocks::mock_sui_object_ref();
5887 let dag_ref = sui_mocks::mock_sui_object_ref();
5888 let execution_ref = sui_mocks::mock_sui_object_ref();
5889 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
5890 let mut state_service_mock = sui_mocks::grpc::MockStateService::new();
5891 let mut tx_service_mock = sui_mocks::grpc::MockTransactionExecutionService::new();
5892 let mut sub_service_mock = sui_mocks::grpc::MockSubscriptionService::new();
5893
5894 sui_mocks::grpc::mock_reference_gas_price(&mut ledger_service_mock, 1000);
5895 mock_get_dag_execution_bcs(
5896 &mut ledger_service_mock,
5897 &mut state_service_mock,
5898 &nexus_objects,
5899 execution_ref.clone(),
5900 &dag_ref,
5901 vec![],
5902 );
5903 mock_get_dag_bcs(
5904 &mut ledger_service_mock,
5905 &mut state_service_mock,
5906 &nexus_objects,
5907 dag_ref.clone(),
5908 dag_bcs(0),
5909 );
5910 sui_mocks::grpc::mock_get_object_bcs(
5911 &mut ledger_service_mock,
5912 sui::types::ObjectReference::new(
5913 move_boundary::CLOCK_OBJECT_ID,
5914 1,
5915 sui::types::Digest::from([1; 32]),
5916 ),
5917 sui::types::Owner::Shared(1),
5918 clock_bcs(61_000),
5919 );
5920 sui_mocks::grpc::mock_runtime_authority(&mut ledger_service_mock, &nexus_objects, false);
5921 let package_service_mock = mock_package_graph(&mut ledger_service_mock, &nexus_objects);
5922 let submitted = sui_mocks::grpc::mock_execute_transaction_and_wait_for_checkpoint(
5923 &mut tx_service_mock,
5924 &mut sub_service_mock,
5925 &mut ledger_service_mock,
5926 gas_coin_ref.clone(),
5927 vec![],
5928 vec![],
5929 vec![],
5930 );
5931 let client = mock_client_for_workflow_submit(
5932 &nexus_objects,
5933 gas_coin_ref,
5934 ledger_service_mock,
5935 package_service_mock,
5936 state_service_mock,
5937 tx_service_mock,
5938 sub_service_mock,
5939 )
5940 .await;
5941
5942 let result = client
5943 .workflow()
5944 .abort_expired_execution(*execution_ref.object_id())
5945 .await
5946 .expect("abort transaction should submit");
5947
5948 assert_eq!(result.tx_digest, submitted.digest());
5949 assert_eq!(result.tx_checkpoint, 1);
5950 assert_eq!(result.dag_id, *dag_ref.object_id());
5951 assert_eq!(result.dag_execution_id, *execution_ref.object_id());
5952 assert!(result.cleaned_broken_onchain_results.is_empty());
5953 }
5954
5955 #[tokio::test]
5956 async fn test_settle_committed_tool_result_for_walk_submits_workflow_transaction() {
5957 let nexus_objects = sui_mocks::mock_nexus_context();
5958 let gas_coin_ref = sui_mocks::mock_sui_object_ref();
5959 let dag_ref = sui_mocks::mock_sui_object_ref();
5960 let execution_ref = sui_mocks::mock_sui_object_ref();
5961 let vertex = RuntimeVertex::plain("tool");
5962 let tool_fqn = fqn!("xyz.taluslabs.fixture@1");
5963 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
5964 let mut state_service_mock = sui_mocks::grpc::MockStateService::new();
5965 let mut tx_service_mock = sui_mocks::grpc::MockTransactionExecutionService::new();
5966 let mut sub_service_mock = sui_mocks::grpc::MockSubscriptionService::new();
5967
5968 sui_mocks::grpc::mock_reference_gas_price(&mut ledger_service_mock, 1000);
5969 mock_get_dag_execution_bcs(
5970 &mut ledger_service_mock,
5971 &mut state_service_mock,
5972 &nexus_objects,
5973 execution_ref.clone(),
5974 &dag_ref,
5975 pending_settlement_walks(7, vertex.clone()),
5976 );
5977 mock_exact_invocation_lock(
5978 &mut ledger_service_mock,
5979 &mut state_service_mock,
5980 &nexus_objects,
5981 &execution_ref,
5982 &dag_ref,
5983 &vertex,
5984 &tool_fqn,
5985 );
5986 mock_tool_registry_observation(
5987 &mut ledger_service_mock,
5988 &mut state_service_mock,
5989 &nexus_objects,
5990 );
5991 mock_leader_registry_observation(
5992 &mut ledger_service_mock,
5993 &mut state_service_mock,
5994 &nexus_objects,
5995 );
5996 mock_priority_fee_vault_observation(
5997 &mut ledger_service_mock,
5998 &mut state_service_mock,
5999 &nexus_objects,
6000 );
6001 sui_mocks::grpc::mock_runtime_authority(&mut ledger_service_mock, &nexus_objects, false);
6002 let package_service_mock = mock_package_graph(&mut ledger_service_mock, &nexus_objects);
6003 let submitted = sui_mocks::grpc::mock_execute_transaction_and_wait_for_checkpoint(
6004 &mut tx_service_mock,
6005 &mut sub_service_mock,
6006 &mut ledger_service_mock,
6007 gas_coin_ref.clone(),
6008 vec![],
6009 vec![],
6010 vec![],
6011 );
6012 let client = mock_client_for_workflow_submit(
6013 &nexus_objects,
6014 gas_coin_ref,
6015 ledger_service_mock,
6016 package_service_mock,
6017 state_service_mock,
6018 tx_service_mock,
6019 sub_service_mock,
6020 )
6021 .await;
6022
6023 let result = client
6024 .workflow()
6025 .settle_committed_tool_result_for_walk(SettleCommittedToolResultParams {
6026 dag_execution_id: *execution_ref.object_id(),
6027 walk_index: 7,
6028 })
6029 .await
6030 .expect("settlement transaction should submit");
6031
6032 assert_eq!(result.tx_digest, submitted.digest());
6033 assert_eq!(result.tx_checkpoint, 1);
6034 assert_eq!(result.dag_id, *dag_ref.object_id());
6035 assert_eq!(result.dag_execution_id, *execution_ref.object_id());
6036 assert_eq!(result.walk_index, 7);
6037 }
6038
6039 #[tokio::test]
6040 async fn test_leader_settlement_and_record_gas_paths_submit_transactions() {
6041 let nexus_objects = sui_mocks::mock_nexus_context();
6042 let settle_gas_ref = sui_mocks::mock_sui_object_ref();
6043 let record_gas_ref = sui_mocks::mock_sui_object_ref();
6044 let dag_ref = sui_mocks::mock_sui_object_ref();
6045 let execution_ref = sui_mocks::mock_sui_object_ref();
6046 let leader_cap_ref = sui_mocks::mock_sui_object_ref();
6047 let vertex = RuntimeVertex::plain("tool");
6048 let tool_fqn = fqn!("xyz.taluslabs.fixture@1");
6049
6050 let mut settle_ledger = sui_mocks::grpc::MockLedgerService::new();
6051 let mut settle_state = sui_mocks::grpc::MockStateService::new();
6052 let mut settle_tx = sui_mocks::grpc::MockTransactionExecutionService::new();
6053 let mut settle_sub = sui_mocks::grpc::MockSubscriptionService::new();
6054 sui_mocks::grpc::mock_reference_gas_price(&mut settle_ledger, 1000);
6055 mock_get_dag_execution_bcs(
6056 &mut settle_ledger,
6057 &mut settle_state,
6058 &nexus_objects,
6059 execution_ref.clone(),
6060 &dag_ref,
6061 pending_settlement_walks(8, vertex.clone()),
6062 );
6063 mock_exact_invocation_lock(
6064 &mut settle_ledger,
6065 &mut settle_state,
6066 &nexus_objects,
6067 &execution_ref,
6068 &dag_ref,
6069 &vertex,
6070 &tool_fqn,
6071 );
6072 mock_tool_registry_observation(&mut settle_ledger, &mut settle_state, &nexus_objects);
6073 mock_leader_registry_observation(&mut settle_ledger, &mut settle_state, &nexus_objects);
6074 mock_priority_fee_vault_observation(&mut settle_ledger, &mut settle_state, &nexus_objects);
6075 sui_mocks::grpc::mock_runtime_authority(&mut settle_ledger, &nexus_objects, false);
6076 let settle_packages = mock_package_graph(&mut settle_ledger, &nexus_objects);
6077 sui_mocks::grpc::mock_get_object_metadata(
6078 &mut settle_ledger,
6079 leader_cap_ref.clone(),
6080 sui::types::Owner::Immutable,
6081 None,
6082 );
6083 let settle_submitted = sui_mocks::grpc::mock_execute_transaction_and_wait_for_checkpoint(
6084 &mut settle_tx,
6085 &mut settle_sub,
6086 &mut settle_ledger,
6087 settle_gas_ref.clone(),
6088 vec![],
6089 vec![],
6090 vec![],
6091 );
6092 let settle_client = mock_client_for_workflow_submit(
6093 &nexus_objects,
6094 settle_gas_ref,
6095 settle_ledger,
6096 settle_packages,
6097 settle_state,
6098 settle_tx,
6099 settle_sub,
6100 )
6101 .await;
6102 let settle_result = settle_client
6103 .workflow()
6104 .settle_committed_tool_result_for_walk_by_leader(
6105 SettleCommittedToolResultByLeaderParams {
6106 dag_execution_id: *execution_ref.object_id(),
6107 leader_cap_id: *leader_cap_ref.object_id(),
6108 walk_index: 8,
6109 expected_vertex: None,
6110 failed_onchain_tool_reason: None,
6111 commit_tx_digest: vec![1, 2, 3],
6112 commit_gas_charge: 11,
6113 settlement_gas_charge: 13,
6114 },
6115 )
6116 .await
6117 .expect("leader settlement transaction should submit");
6118
6119 assert_eq!(settle_result.tx_digest, settle_submitted.digest());
6120 assert_eq!(settle_result.tx_checkpoint, 1);
6121 assert_eq!(settle_result.dag_id, *dag_ref.object_id());
6122 assert_eq!(settle_result.walk_index, 8);
6123
6124 let mut record_ledger = sui_mocks::grpc::MockLedgerService::new();
6125 let mut record_state = sui_mocks::grpc::MockStateService::new();
6126 let mut record_tx = sui_mocks::grpc::MockTransactionExecutionService::new();
6127 let mut record_sub = sui_mocks::grpc::MockSubscriptionService::new();
6128 sui_mocks::grpc::mock_reference_gas_price(&mut record_ledger, 1000);
6129 mock_get_dag_execution_bcs(
6130 &mut record_ledger,
6131 &mut record_state,
6132 &nexus_objects,
6133 execution_ref.clone(),
6134 &dag_ref,
6135 vec![],
6136 );
6137 mock_get_dag_bcs(
6138 &mut record_ledger,
6139 &mut record_state,
6140 &nexus_objects,
6141 dag_ref.clone(),
6142 dag_bcs(0),
6143 );
6144 mock_leader_registry_observation(&mut record_ledger, &mut record_state, &nexus_objects);
6145 sui_mocks::grpc::mock_runtime_authority(&mut record_ledger, &nexus_objects, false);
6146 let record_packages = mock_package_graph(&mut record_ledger, &nexus_objects);
6147 sui_mocks::grpc::mock_get_object_metadata(
6148 &mut record_ledger,
6149 leader_cap_ref.clone(),
6150 sui::types::Owner::Address(sui::types::Address::from_static("0xabc")),
6151 None,
6152 );
6153 let record_submitted = sui_mocks::grpc::mock_execute_transaction_and_wait_for_checkpoint(
6154 &mut record_tx,
6155 &mut record_sub,
6156 &mut record_ledger,
6157 record_gas_ref.clone(),
6158 vec![],
6159 vec![],
6160 vec![],
6161 );
6162 let record_client = mock_client_for_workflow_submit(
6163 &nexus_objects,
6164 record_gas_ref,
6165 record_ledger,
6166 record_packages,
6167 record_state,
6168 record_tx,
6169 record_sub,
6170 )
6171 .await;
6172 let record_result = record_client
6173 .workflow()
6174 .record_committed_tool_result_gas_charge_by_leader(
6175 RecordCommittedToolResultGasChargeParams {
6176 dag_execution_id: *execution_ref.object_id(),
6177 leader_cap_id: *leader_cap_ref.object_id(),
6178 walk_index: 9,
6179 expected_vertex: None,
6180 failed_onchain_tool_reason: None,
6181 commit_tx_digest: vec![4, 5, 6],
6182 commit_gas_charge: 17,
6183 settlement_gas_charge: 19,
6184 },
6185 )
6186 .await
6187 .expect("leader gas record transaction should submit");
6188
6189 assert_eq!(record_result.tx_digest, record_submitted.digest());
6190 assert_eq!(record_result.tx_checkpoint, 1);
6191 assert_eq!(record_result.dag_execution_id, *execution_ref.object_id());
6192 assert_eq!(record_result.leader_cap_id, *leader_cap_ref.object_id());
6193 assert_eq!(record_result.walk_index, 9);
6194 }
6195
6196 #[test]
6197 fn dag_vertex_requires_tool_verification_reads_vertex_mode() {
6198 let mut vertex = offchain_vertex_info(&fqn!("xyz.example.tool@1"));
6199 assert!(!dag_vertex_requires_tool_verification(&vertex));
6200 vertex.verifier_mode = ToolVerifierMode::External;
6201 assert!(dag_vertex_requires_tool_verification(&vertex));
6202 }
6203
6204 #[test]
6205 fn execution_terminal_record_matches_retryable_vertex_uses_typed_records() {
6206 let vertex = RuntimeVertex::with_iterator("v1", 2, 5);
6207
6208 fn record(
6209 vertex: RuntimeVertex,
6210 failure_class: WorkflowFailureClass,
6211 ) -> execution_move::TerminalErrEvalRecord {
6212 execution_move::TerminalErrEvalRecord {
6213 walk_index: 9,
6214 vertex,
6215 leader: sui::types::Address::THREE,
6216 failure_class,
6217 outcome: MoveOption::from_option(None::<PostFailureAction>),
6218 reason: MoveString::from("retryable failure"),
6219 variant_ports_to_data: VecMap { contents: vec![] },
6220 err_eval_hash: vec![1, 2, 3],
6221 }
6222 }
6223
6224 let retryable_records = VecMap {
6225 contents: vec![crate::move_bindings::sui_framework::vec_map::Entry {
6226 key: 9,
6227 value: record(vertex.clone(), WorkflowFailureClass::Retryable),
6228 }],
6229 };
6230 let terminal_records = VecMap {
6231 contents: vec![crate::move_bindings::sui_framework::vec_map::Entry {
6232 key: 9,
6233 value: record(
6234 RuntimeVertex::plain("terminal_vertex"),
6235 WorkflowFailureClass::TerminalSubmissionFailure,
6236 ),
6237 }],
6238 };
6239
6240 assert!(execution_terminal_record_matches_retryable_vertex(
6241 &retryable_records,
6242 9,
6243 &vertex,
6244 ));
6245 assert!(!execution_terminal_record_matches_retryable_vertex(
6246 &retryable_records,
6247 10,
6248 &vertex,
6249 ));
6250 assert!(!execution_terminal_record_matches_retryable_vertex(
6251 &terminal_records,
6252 9,
6253 &RuntimeVertex::plain("terminal_vertex"),
6254 ));
6255 }
6256
6257 #[test]
6258 fn fetch_vertex_input_port_names_reads_declared_ports_from_typed_vertex() {
6259 let mut vertex = offchain_vertex_info(&fqn!("xyz.example.tool@1"));
6260 vertex.input_ports = VecSet {
6261 contents: vec![
6262 graph_move::InputPort::new("z_port"),
6263 graph_move::InputPort::new("a_port"),
6264 ],
6265 };
6266
6267 assert_eq!(
6268 vertex.declared_input_port_names(),
6269 vec!["a_port".to_string(), "z_port".to_string()]
6270 );
6271 }
6272
6273 #[test]
6274 fn invocation_abort_filter_returns_exact_expired_locked_vertex() {
6275 let execution_id = sui::types::Address::from_static("0xabc");
6276 let tool_fqn = fqn!("xyz.taluslabs.payable@1");
6277 let other_tool_fqn = fqn!("xyz.taluslabs.other@1");
6278 let invocation_id = sui::types::Address::from_static("0x704");
6279 let payable_vertex = RuntimeVertex::plain("payable");
6280 let idle_vertex = RuntimeVertex::plain("idle");
6281 let matching_key = payment_vertex_key(execution_id, &payable_vertex, &tool_fqn).unwrap();
6282 let mut vertices = HashMap::new();
6283 vertices.insert(
6284 graph_move::Vertex::new("payable"),
6285 offchain_vertex_info(&tool_fqn),
6286 );
6287 vertices.insert(
6288 graph_move::Vertex::new("idle"),
6289 offchain_vertex_info(&other_tool_fqn),
6290 );
6291 let walks = vec![
6292 DAGWalk::Active {
6293 next_vertex: payable_vertex.clone(),
6294 timeout_ms: 30_000,
6295 requires_vertex_authorization_grant: false,
6296 created_at: 1_000,
6297 },
6298 DAGWalk::Active {
6299 next_vertex: idle_vertex,
6300 timeout_ms: 30_000,
6301 requires_vertex_authorization_grant: false,
6302 created_at: 61_000,
6303 },
6304 DAGWalk::PendingAbort {
6305 at_vertex: RuntimeVertex::plain("already_pending"),
6306 },
6307 ];
6308 let locks = vec![invocation_lock(matching_key.clone(), invocation_id)];
6309
6310 let candidates = filter_invocation_abort_candidate_walks(
6311 execution_id,
6312 &vertices,
6313 &walks,
6314 &locks,
6315 61_000,
6316 )
6317 .unwrap();
6318
6319 assert_eq!(candidates.len(), 1);
6320 assert_eq!(candidates[0].walk_index, 0);
6321 assert_eq!(candidates[0].vertex, payable_vertex);
6322 assert_eq!(candidates[0].payment_vertex_key, matching_key);
6323 assert_eq!(candidates[0].invocation_id, invocation_id);
6324 }
6325
6326 #[test]
6327 fn invocation_abort_filter_ignores_nonmatching_payment_locks() {
6328 let execution_id = sui::types::Address::from_static("0xabc");
6329 let tool_fqn = fqn!("xyz.taluslabs.payable@1");
6330 let other_tool_fqn = fqn!("xyz.taluslabs.other@1");
6331 let payable_vertex = RuntimeVertex::plain("payable");
6332 let other_tool_key =
6333 payment_vertex_key(execution_id, &payable_vertex, &other_tool_fqn).unwrap();
6334 let mut vertices = HashMap::new();
6335 vertices.insert(
6336 graph_move::Vertex::new("payable"),
6337 offchain_vertex_info(&tool_fqn),
6338 );
6339 let walks = vec![DAGWalk::Active {
6340 next_vertex: payable_vertex,
6341 timeout_ms: 30_000,
6342 requires_vertex_authorization_grant: false,
6343 created_at: 1_000,
6344 }];
6345 let locks = vec![
6346 invocation_lock(other_tool_key, sui::types::Address::from_static("0x705")),
6347 invocation_lock(vec![1, 2, 3], sui::types::Address::from_static("0x706")),
6348 ];
6349
6350 let candidates = filter_invocation_abort_candidate_walks(
6351 execution_id,
6352 &vertices,
6353 &walks,
6354 &locks,
6355 61_000,
6356 )
6357 .unwrap();
6358
6359 assert!(candidates.is_empty());
6360 }
6361
6362 #[test]
6363 fn invocation_abort_filter_errors_when_expired_vertex_is_missing_from_dag() {
6364 let execution_id = sui::types::Address::from_static("0xabc");
6365 let walks = vec![DAGWalk::Active {
6366 next_vertex: RuntimeVertex::plain("missing"),
6367 timeout_ms: 30_000,
6368 requires_vertex_authorization_grant: false,
6369 created_at: 1_000,
6370 }];
6371
6372 let error = filter_invocation_abort_candidate_walks(
6373 execution_id,
6374 &HashMap::new(),
6375 &walks,
6376 &[],
6377 61_000,
6378 )
6379 .err()
6380 .expect("expired walk should require a fetched DAG vertex");
6381
6382 assert!(error
6383 .to_string()
6384 .contains("DAG vertex 'missing' missing from fetched DAG"));
6385 }
6386
6387 #[tokio::test]
6388 async fn fetch_invocation_abort_candidates_resolves_exact_invocation_refs() {
6389 let invocation_id = sui::types::Address::from_static("0x707");
6390 let invocation_ref = sui_mocks::object_ref_for_id(invocation_id);
6391 let candidate = InvocationAbortCandidateSeed {
6392 walk_index: 2,
6393 vertex: RuntimeVertex::plain("payable"),
6394 payment_vertex_key: vec![1, 2, 3],
6395 invocation_id,
6396 };
6397 let mut ledger_service_mock = sui_mocks::grpc::MockLedgerService::new();
6398
6399 sui_mocks::grpc::mock_get_object_metadata(
6400 &mut ledger_service_mock,
6401 invocation_ref.clone(),
6402 sui::types::Owner::Object(sui::types::Address::from_static("0x708")),
6403 None,
6404 );
6405
6406 let rpc_url = sui_mocks::grpc::mock_server(sui_mocks::grpc::ServerMocks {
6407 ledger_service_mock: Some(ledger_service_mock),
6408 ..Default::default()
6409 });
6410 let client = sui::grpc::client(rpc_url).expect("mock client");
6411 let crawler = Crawler::new(Arc::new(client));
6412 let candidates = fetch_invocation_abort_candidates(&crawler, vec![candidate])
6413 .await
6414 .expect("candidate refs should be fetched");
6415
6416 assert_eq!(candidates.len(), 1);
6417 assert_eq!(candidates[0].walk_index, 2);
6418 assert_eq!(candidates[0].vertex, RuntimeVertex::plain("payable"));
6419 assert_eq!(candidates[0].payment_vertex_key, vec![1, 2, 3]);
6420 assert_eq!(candidates[0].invocation_ref, invocation_ref);
6421 }
6422
6423 #[test]
6424 fn select_invocation_abort_candidate_uses_first_or_required_invocation() {
6425 let first_id = sui::types::Address::from_static("0x111");
6426 let second_id = sui::types::Address::from_static("0x222");
6427 let candidates = vec![
6428 InvocationAbortCandidate {
6429 walk_index: 0,
6430 vertex: RuntimeVertex::plain("first"),
6431 payment_vertex_key: vec![1],
6432 invocation_ref: sui::types::ObjectReference::new(
6433 first_id,
6434 1,
6435 sui::types::Digest::default(),
6436 ),
6437 },
6438 InvocationAbortCandidate {
6439 walk_index: 1,
6440 vertex: RuntimeVertex::plain("second"),
6441 payment_vertex_key: vec![2],
6442 invocation_ref: sui::types::ObjectReference::new(
6443 second_id,
6444 1,
6445 sui::types::Digest::default(),
6446 ),
6447 },
6448 ];
6449
6450 let selected = select_invocation_abort_candidate(candidates.clone(), None).unwrap();
6451 assert_eq!(*selected.invocation_ref.object_id(), first_id);
6452
6453 let selected =
6454 select_invocation_abort_candidate(candidates.clone(), Some(second_id)).unwrap();
6455 assert_eq!(*selected.invocation_ref.object_id(), second_id);
6456
6457 let missing = sui::types::Address::from_static("0x333");
6458 let error = select_invocation_abort_candidate(candidates, Some(missing)).unwrap_err();
6459 assert!(error
6460 .to_string()
6461 .contains("is not currently eligible to abort this execution"));
6462
6463 let error = select_invocation_abort_candidate(Vec::new(), None).unwrap_err();
6464 assert!(error
6465 .to_string()
6466 .contains("No Invocation abort candidates are currently eligible"));
6467 }
6468}