1use alloc::{sync::Arc, vec::Vec};
2use core::ops::ControlFlow;
3
4use miden_core::{
5 Word,
6 mast::{MastForest, MastNodeId},
7 program::{KernelDescriptor, MIN_STACK_DEPTH, Program, StackInputs, StackOutputs},
8};
9use miden_mast_package::debug_info::{
10 DebugSourceGraphLookupError, DebugSourceNodeId, PackageDebugInfo,
11};
12use tracing::instrument;
13
14use super::{
15 FastProcessor, NoopTracer,
16 external::maybe_use_caller_error_context,
17 step::{BreakReason, NeverStopper, ResumeContext, StepStopper},
18};
19#[cfg(feature = "std")]
20use crate::PrecompileWitness;
21use crate::{
22 ExecutionError, ExecutionOutput, ExecutionWitness, Host, LoadedMastForest, Stopper, SyncHost,
23 advice::AdviceError,
24 continuation_stack::{Continuation, ContinuationStack, SourceInlineCallContext},
25 errors::{
26 MapExecErr, MapExecErrNoCtx, PackageSourceDebugContext, malformed_mast_forest_with_context,
27 },
28 execution::{
29 InternalBreakReason, execute_impl, finish_emit_op_execution,
30 finish_load_mast_forest_from_dyn_start, finish_load_mast_forest_from_external,
31 },
32 trace::execution_tracer::ExecutionTracer,
33 tracer::Tracer,
34};
35
36impl FastProcessor {
37 pub fn execute_sync(
42 self,
43 program: &Program,
44 host: &mut impl SyncHost,
45 ) -> Result<ExecutionOutput, ExecutionError> {
46 self.execute_with_tracer_sync(program, host, &mut NoopTracer)
47 }
48
49 pub fn execute_with_package_debug_info_sync(
56 self,
57 program: &Program,
58 package_debug_info: &PackageDebugInfo,
59 host: &mut impl SyncHost,
60 ) -> Result<ExecutionOutput, ExecutionError> {
61 self.execute_with_package_debug_info_and_tracer_sync(
62 program,
63 package_debug_info,
64 None,
65 host,
66 &mut NoopTracer,
67 )
68 }
69
70 pub fn execute_with_package_debug_info_at_source_node_sync(
77 self,
78 program: &Program,
79 package_debug_info: &PackageDebugInfo,
80 entrypoint_source_node_id: DebugSourceNodeId,
81 host: &mut impl SyncHost,
82 ) -> Result<ExecutionOutput, ExecutionError> {
83 self.execute_with_package_debug_info_and_tracer_sync(
84 program,
85 package_debug_info,
86 Some(entrypoint_source_node_id),
87 host,
88 &mut NoopTracer,
89 )
90 }
91
92 #[inline(always)]
94 pub async fn execute(
95 self,
96 program: &Program,
97 host: &mut impl Host,
98 ) -> Result<ExecutionOutput, ExecutionError> {
99 self.execute_with_tracer(program, host, &mut NoopTracer).await
100 }
101
102 #[inline(always)]
107 pub async fn execute_with_package_debug_info(
108 self,
109 program: &Program,
110 package_debug_info: &PackageDebugInfo,
111 host: &mut impl Host,
112 ) -> Result<ExecutionOutput, ExecutionError> {
113 self.execute_with_package_debug_info_and_tracer(
114 program,
115 package_debug_info,
116 None,
117 host,
118 &mut NoopTracer,
119 )
120 .await
121 }
122
123 #[inline(always)]
125 pub async fn execute_with_package_debug_info_at_source_node(
126 self,
127 program: &Program,
128 package_debug_info: &PackageDebugInfo,
129 entrypoint_source_node_id: DebugSourceNodeId,
130 host: &mut impl Host,
131 ) -> Result<ExecutionOutput, ExecutionError> {
132 self.execute_with_package_debug_info_and_tracer(
133 program,
134 package_debug_info,
135 Some(entrypoint_source_node_id),
136 host,
137 &mut NoopTracer,
138 )
139 .await
140 }
141
142 #[cfg(feature = "std")]
157 #[instrument(name = "execute_and_build_trace_sync", skip_all)]
158 pub fn execute_and_build_trace_sync(
159 self,
160 program: &Program,
161 host: &mut impl SyncHost,
162 max_prover_memory_bytes: u64,
163 ) -> Result<(crate::trace::VmTrace, Option<PrecompileWitness>), ExecutionError> {
164 use miden_air::{config, memory};
165
166 use crate::trace::{
167 MAX_TRACE_LEN, build_hasher_chiplet, build_trace_with_budget,
168 build_trace_with_prebuilt_hasher,
169 };
170
171 if Self::rayon_has_no_parallel_worker() {
172 let (vm_witness, precompiles_witness) =
173 self.execute_for_proving_sync(program, host)?.into_parts();
174 let trace = build_trace_with_budget(vm_witness, max_prover_memory_bytes)?;
175 return Ok((trace, precompiles_witness));
176 }
177
178 let stack_inputs = self.initial_stack_inputs();
179 let max_trace_len = MAX_TRACE_LEN.min(memory::max_any_height_for_budget(
180 max_prover_memory_bytes,
181 &config::pcs_params(),
182 ));
183 let (sender, receiver) = std::sync::mpsc::channel();
184 let mut tracer = ExecutionTracer::new_with_streamed_hasher(
185 self.options.core_trace_fragment_size(),
186 self.options.max_stack_depth(),
187 sender,
188 );
189
190 let mut hasher = None;
191 let hasher_slot = &mut hasher;
192 let execution_output = rayon::in_place_scope(move |scope| {
195 let span = tracing::Span::current();
198 scope.spawn(move |_| {
199 let _span = span.entered();
200 let result = build_hasher_chiplet(receiver.into_iter().map(Ok), max_trace_len);
201 *hasher_slot = Some(result);
202 });
203
204 let execution_output = self.execute_with_tracer_sync(program, host, &mut tracer);
205
206 match execution_output {
207 Ok(output) => {
208 let (mut vm_witness, precompiles_witness) =
209 Self::execution_witness_from_parts(program, stack_inputs, output, tracer)
210 .into_parts();
211 drop(vm_witness.take_hasher_replay());
213 Ok((vm_witness, precompiles_witness))
214 },
215 Err(err) => {
216 drop(tracer);
220 Err(err)
221 },
222 }
223 });
224
225 let hasher = hasher.expect("hasher builder did not run");
226 let (vm_witness, precompiles_witness) = match execution_output {
227 Ok(output) => output,
228 Err(err) => {
229 if let Err(builder_err) = hasher {
230 tracing::debug!(%builder_err, "hasher builder also failed");
231 }
232 return Err(err);
233 },
234 };
235
236 let trace = build_trace_with_prebuilt_hasher(vm_witness, hasher?, max_prover_memory_bytes)?;
237 Ok((trace, precompiles_witness))
238 }
239
240 #[cfg(feature = "std")]
241 fn rayon_has_no_parallel_worker() -> bool {
242 rayon::current_num_threads() == 1 && rayon::current_thread_index().is_some()
244 }
245
246 #[instrument(name = "execute_for_proving_sync", skip_all)]
268 pub fn execute_for_proving_sync(
269 self,
270 program: &Program,
271 host: &mut impl SyncHost,
272 ) -> Result<ExecutionWitness, ExecutionError> {
273 let stack_inputs = self.initial_stack_inputs();
274 let mut tracer = ExecutionTracer::new(
275 self.options.core_trace_fragment_size(),
276 self.options.max_stack_depth(),
277 );
278 let execution_output = self.execute_with_tracer_sync(program, host, &mut tracer)?;
279 Ok(Self::execution_witness_from_parts(
280 program,
281 stack_inputs,
282 execution_output,
283 tracer,
284 ))
285 }
286
287 #[instrument(name = "execute_for_proving_with_package_debug_info_sync", skip_all)]
290 pub fn execute_for_proving_with_package_debug_info_sync(
291 self,
292 program: &Program,
293 package_debug_info: &PackageDebugInfo,
294 host: &mut impl SyncHost,
295 ) -> Result<ExecutionWitness, ExecutionError> {
296 let stack_inputs = self.initial_stack_inputs();
297 let mut tracer = ExecutionTracer::new(
298 self.options.core_trace_fragment_size(),
299 self.options.max_stack_depth(),
300 );
301 let execution_output = self.execute_with_package_debug_info_and_tracer_sync(
302 program,
303 package_debug_info,
304 None,
305 host,
306 &mut tracer,
307 )?;
308 Ok(Self::execution_witness_from_parts(
309 program,
310 stack_inputs,
311 execution_output,
312 tracer,
313 ))
314 }
315
316 #[instrument(
319 name = "execute_for_proving_with_package_debug_info_at_source_node_sync",
320 skip_all
321 )]
322 pub fn execute_for_proving_with_package_debug_info_at_source_node_sync(
323 self,
324 program: &Program,
325 package_debug_info: &PackageDebugInfo,
326 entrypoint_source_node_id: DebugSourceNodeId,
327 host: &mut impl SyncHost,
328 ) -> Result<ExecutionWitness, ExecutionError> {
329 let stack_inputs = self.initial_stack_inputs();
330 let mut tracer = ExecutionTracer::new(
331 self.options.core_trace_fragment_size(),
332 self.options.max_stack_depth(),
333 );
334 let execution_output = self.execute_with_package_debug_info_and_tracer_sync(
335 program,
336 package_debug_info,
337 Some(entrypoint_source_node_id),
338 host,
339 &mut tracer,
340 )?;
341 Ok(Self::execution_witness_from_parts(
342 program,
343 stack_inputs,
344 execution_output,
345 tracer,
346 ))
347 }
348
349 #[inline(always)]
351 #[instrument(name = "execute_for_proving", skip_all)]
352 pub async fn execute_for_proving(
353 self,
354 program: &Program,
355 host: &mut impl Host,
356 ) -> Result<ExecutionWitness, ExecutionError> {
357 let stack_inputs = self.initial_stack_inputs();
358 let mut tracer = ExecutionTracer::new(
359 self.options.core_trace_fragment_size(),
360 self.options.max_stack_depth(),
361 );
362 let execution_output = self.execute_with_tracer(program, host, &mut tracer).await?;
363 Ok(Self::execution_witness_from_parts(
364 program,
365 stack_inputs,
366 execution_output,
367 tracer,
368 ))
369 }
370
371 #[cfg(any(test, feature = "testing"))]
373 #[inline(always)]
374 #[instrument(name = "execute_for_proving_with_package_debug_info", skip_all)]
375 pub async fn execute_for_proving_with_package_debug_info(
376 self,
377 program: &Program,
378 package_debug_info: &PackageDebugInfo,
379 host: &mut impl Host,
380 ) -> Result<ExecutionWitness, ExecutionError> {
381 let stack_inputs = self.initial_stack_inputs();
382 let mut tracer = ExecutionTracer::new(
383 self.options.core_trace_fragment_size(),
384 self.options.max_stack_depth(),
385 );
386 let execution_output = self
387 .execute_with_package_debug_info_and_tracer(
388 program,
389 package_debug_info,
390 None,
391 host,
392 &mut tracer,
393 )
394 .await?;
395 Ok(Self::execution_witness_from_parts(
396 program,
397 stack_inputs,
398 execution_output,
399 tracer,
400 ))
401 }
402
403 #[cfg(any(test, feature = "testing"))]
406 #[inline(always)]
407 #[instrument(name = "execute_for_proving_with_package_debug_info_at_source_node", skip_all)]
408 pub async fn execute_for_proving_with_package_debug_info_at_source_node(
409 self,
410 program: &Program,
411 package_debug_info: &PackageDebugInfo,
412 entrypoint_source_node_id: DebugSourceNodeId,
413 host: &mut impl Host,
414 ) -> Result<ExecutionWitness, ExecutionError> {
415 let stack_inputs = self.initial_stack_inputs();
416 let mut tracer = ExecutionTracer::new(
417 self.options.core_trace_fragment_size(),
418 self.options.max_stack_depth(),
419 );
420 let execution_output = self
421 .execute_with_package_debug_info_and_tracer(
422 program,
423 package_debug_info,
424 Some(entrypoint_source_node_id),
425 host,
426 &mut tracer,
427 )
428 .await?;
429 Ok(Self::execution_witness_from_parts(
430 program,
431 stack_inputs,
432 execution_output,
433 tracer,
434 ))
435 }
436
437 pub async fn execute_with_tracer<T>(
439 mut self,
440 program: &Program,
441 host: &mut impl Host,
442 tracer: &mut T,
443 ) -> Result<ExecutionOutput, ExecutionError>
444 where
445 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
446 {
447 let mut continuation_stack = ContinuationStack::new(program);
448 let mut current_forest = program.mast_forest().clone();
449 let mut package_debug_info = None;
450 let mut inline_call_contexts = Vec::new();
451
452 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
453 let flow = self
454 .execute_impl_async(
455 &mut continuation_stack,
456 &mut current_forest,
457 program.kernel(),
458 host,
459 tracer,
460 &NeverStopper,
461 &mut package_debug_info,
462 &mut inline_call_contexts,
463 )
464 .await;
465 Self::execution_result_from_flow(flow, self)
466 }
467
468 async fn execute_with_package_debug_info_and_tracer<T>(
471 mut self,
472 program: &Program,
473 package_debug_info: &PackageDebugInfo,
474 entrypoint_source_node_id: Option<DebugSourceNodeId>,
475 host: &mut impl Host,
476 tracer: &mut T,
477 ) -> Result<ExecutionOutput, ExecutionError>
478 where
479 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
480 {
481 let mut continuation_stack = Self::source_aware_continuation_stack(
482 program,
483 package_debug_info,
484 entrypoint_source_node_id,
485 )?;
486 let mut current_forest = program.mast_forest().clone();
487 let mut package_debug_info = Some(Arc::new(package_debug_info.clone()));
488 let mut inline_call_contexts = Vec::new();
489
490 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
491 let flow = self
492 .execute_impl_async(
493 &mut continuation_stack,
494 &mut current_forest,
495 program.kernel(),
496 host,
497 tracer,
498 &NeverStopper,
499 &mut package_debug_info,
500 &mut inline_call_contexts,
501 )
502 .await;
503 Self::execution_result_from_flow(flow, self)
504 }
505
506 pub fn execute_with_tracer_sync<T>(
508 mut self,
509 program: &Program,
510 host: &mut impl SyncHost,
511 tracer: &mut T,
512 ) -> Result<ExecutionOutput, ExecutionError>
513 where
514 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
515 {
516 let mut continuation_stack = ContinuationStack::new(program);
517 let mut current_forest = program.mast_forest().clone();
518 let mut package_debug_info = None;
519 let mut inline_call_contexts = Vec::new();
520
521 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
522 let flow = self.execute_impl(
523 &mut continuation_stack,
524 &mut current_forest,
525 program.kernel(),
526 host,
527 tracer,
528 &NeverStopper,
529 &mut package_debug_info,
530 &mut inline_call_contexts,
531 );
532 Self::execution_result_from_flow(flow, self)
533 }
534
535 fn execute_with_package_debug_info_and_tracer_sync<T>(
538 mut self,
539 program: &Program,
540 package_debug_info: &PackageDebugInfo,
541 entrypoint_source_node_id: Option<DebugSourceNodeId>,
542 host: &mut impl SyncHost,
543 tracer: &mut T,
544 ) -> Result<ExecutionOutput, ExecutionError>
545 where
546 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
547 {
548 let mut continuation_stack = Self::source_aware_continuation_stack(
549 program,
550 package_debug_info,
551 entrypoint_source_node_id,
552 )?;
553 let mut current_forest = program.mast_forest().clone();
554 let mut package_debug_info = Some(Arc::new(package_debug_info.clone()));
555 let mut inline_call_contexts = Vec::new();
556
557 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
558 let flow = self.execute_impl(
559 &mut continuation_stack,
560 &mut current_forest,
561 program.kernel(),
562 host,
563 tracer,
564 &NeverStopper,
565 &mut package_debug_info,
566 &mut inline_call_contexts,
567 );
568 Self::execution_result_from_flow(flow, self)
569 }
570
571 pub fn step_sync(
573 &mut self,
574 host: &mut impl SyncHost,
575 resume_ctx: ResumeContext,
576 ) -> Result<Option<ResumeContext>, ExecutionError> {
577 let ResumeContext {
578 mut current_forest,
579 mut continuation_stack,
580 kernel,
581 mut package_debug_info,
582 mut inline_call_contexts,
583 } = resume_ctx;
584
585 let flow = self.execute_impl(
586 &mut continuation_stack,
587 &mut current_forest,
588 &kernel,
589 host,
590 &mut NoopTracer,
591 &StepStopper,
592 &mut package_debug_info,
593 &mut inline_call_contexts,
594 );
595 Self::resume_context_from_flow(
596 flow,
597 continuation_stack,
598 current_forest,
599 kernel,
600 package_debug_info,
601 inline_call_contexts,
602 )
603 }
604
605 pub fn step_with_package_debug_info_sync(
607 &mut self,
608 host: &mut impl SyncHost,
609 resume_ctx: ResumeContext,
610 package_debug_info: &PackageDebugInfo,
611 ) -> Result<Option<ResumeContext>, ExecutionError> {
612 let ResumeContext {
613 mut current_forest,
614 mut continuation_stack,
615 kernel,
616 package_debug_info: mut active_package_debug_info,
617 mut inline_call_contexts,
618 } = resume_ctx;
619 Self::ensure_source_aware_step_context(
620 &mut continuation_stack,
621 &mut active_package_debug_info,
622 package_debug_info,
623 )?;
624
625 let flow = self.execute_impl(
626 &mut continuation_stack,
627 &mut current_forest,
628 &kernel,
629 host,
630 &mut NoopTracer,
631 &StepStopper,
632 &mut active_package_debug_info,
633 &mut inline_call_contexts,
634 );
635 Self::resume_context_from_flow(
636 flow,
637 continuation_stack,
638 current_forest,
639 kernel,
640 active_package_debug_info,
641 inline_call_contexts,
642 )
643 }
644
645 #[inline(always)]
647 pub async fn step(
648 &mut self,
649 host: &mut impl Host,
650 resume_ctx: ResumeContext,
651 ) -> Result<Option<ResumeContext>, ExecutionError> {
652 let ResumeContext {
653 mut current_forest,
654 mut continuation_stack,
655 kernel,
656 mut package_debug_info,
657 mut inline_call_contexts,
658 } = resume_ctx;
659
660 let flow = self
661 .execute_impl_async(
662 &mut continuation_stack,
663 &mut current_forest,
664 &kernel,
665 host,
666 &mut NoopTracer,
667 &StepStopper,
668 &mut package_debug_info,
669 &mut inline_call_contexts,
670 )
671 .await;
672 Self::resume_context_from_flow(
673 flow,
674 continuation_stack,
675 current_forest,
676 kernel,
677 package_debug_info,
678 inline_call_contexts,
679 )
680 }
681
682 #[inline(always)]
684 pub async fn step_with_package_debug_info(
685 &mut self,
686 host: &mut impl Host,
687 resume_ctx: ResumeContext,
688 package_debug_info: &PackageDebugInfo,
689 ) -> Result<Option<ResumeContext>, ExecutionError> {
690 let ResumeContext {
691 mut current_forest,
692 mut continuation_stack,
693 kernel,
694 package_debug_info: mut active_package_debug_info,
695 mut inline_call_contexts,
696 } = resume_ctx;
697 Self::ensure_source_aware_step_context(
698 &mut continuation_stack,
699 &mut active_package_debug_info,
700 package_debug_info,
701 )?;
702
703 let flow = self
704 .execute_impl_async(
705 &mut continuation_stack,
706 &mut current_forest,
707 &kernel,
708 host,
709 &mut NoopTracer,
710 &StepStopper,
711 &mut active_package_debug_info,
712 &mut inline_call_contexts,
713 )
714 .await;
715 Self::resume_context_from_flow(
716 flow,
717 continuation_stack,
718 current_forest,
719 kernel,
720 active_package_debug_info,
721 inline_call_contexts,
722 )
723 }
724
725 #[inline(always)]
727 fn execution_witness_from_parts(
728 program: &Program,
729 stack_inputs: StackInputs,
730 execution_output: ExecutionOutput,
731 tracer: ExecutionTracer,
732 ) -> ExecutionWitness {
733 ExecutionWitness::from_execution(
734 program.to_info(),
735 stack_inputs,
736 execution_output,
737 tracer.into_trace_replay(),
738 )
739 }
740
741 #[inline(always)]
743 fn initial_stack_inputs(&self) -> StackInputs {
744 core::array::from_fn(|idx| self.stack_get(idx)).into()
745 }
746
747 pub(super) fn source_aware_continuation_stack(
748 program: &Program,
749 package_debug_info: &PackageDebugInfo,
750 entrypoint_source_node_id: Option<DebugSourceNodeId>,
751 ) -> Result<ContinuationStack<Arc<MastForest>>, ExecutionError> {
752 if let Some(source_node_id) = entrypoint_source_node_id {
753 let Some(source_node) = package_debug_info.source_node(source_node_id) else {
754 return Err(ExecutionError::Internal(
755 "package debug source graph is missing the entrypoint source node",
756 ));
757 };
758 if source_node.exec_node != program.entrypoint() {
759 return Err(ExecutionError::Internal(
760 "package debug entrypoint source node does not match the program entrypoint",
761 ));
762 }
763
764 return Ok(ContinuationStack::new_with_source_node_id(program, source_node_id));
765 }
766
767 let Some(source_node_id) = package_debug_info
768 .unique_source_root_for_exec_node(program.entrypoint())
769 .map_err(|_| {
770 ExecutionError::Internal(
771 "package debug source graph has ambiguous or malformed entrypoint roots",
772 )
773 })?
774 else {
775 return Ok(ContinuationStack::new_with_optional_source_node_id(program, None));
776 };
777
778 Ok(ContinuationStack::new_with_source_node_id(program, source_node_id))
779 }
780
781 #[cfg(any(test, feature = "testing"))]
782 fn source_aware_resume_context(
783 &mut self,
784 program: &Program,
785 package_debug_info: &PackageDebugInfo,
786 entrypoint_source_node_id: Option<DebugSourceNodeId>,
787 ) -> Result<ResumeContext, ExecutionError> {
788 self.advice
789 .extend_map(program.mast_forest().advice_map())
790 .map_exec_err_no_ctx()?;
791
792 Ok(ResumeContext {
793 current_forest: program.mast_forest().clone(),
794 continuation_stack: Self::source_aware_continuation_stack(
795 program,
796 package_debug_info,
797 entrypoint_source_node_id,
798 )?,
799 kernel: program.kernel().clone(),
800 package_debug_info: Some(Arc::new(package_debug_info.clone())),
801 inline_call_contexts: Vec::new(),
802 })
803 }
804
805 fn ensure_source_aware_step_context(
806 continuation_stack: &mut ContinuationStack<Arc<MastForest>>,
807 package_debug_info: &mut Option<Arc<PackageDebugInfo>>,
808 supplied_package_debug_info: &PackageDebugInfo,
809 ) -> Result<(), ExecutionError> {
810 if package_debug_info.is_none() {
811 *package_debug_info = Some(Arc::new(supplied_package_debug_info.clone()));
812 }
813
814 if !continuation_stack.tracks_source_nodes() {
815 let source_node_id = Self::source_root_for_next_continuation(
816 continuation_stack,
817 package_debug_info.as_deref().expect("package debug info was just initialized"),
818 )?;
819 continuation_stack.start_tracking_source_nodes(source_node_id);
820 }
821
822 Ok(())
823 }
824
825 fn source_root_for_next_continuation(
826 continuation_stack: &ContinuationStack<Arc<MastForest>>,
827 package_debug_info: &PackageDebugInfo,
828 ) -> Result<Option<DebugSourceNodeId>, ExecutionError> {
829 let Some((continuation, _)) = continuation_stack.peek_continuation_with_source_node_id()
830 else {
831 return Ok(None);
832 };
833
834 let Some(exec_node) = continuation.exec_node() else {
835 return Ok(None);
836 };
837
838 package_debug_info.unique_source_root_for_exec_node(exec_node).map_err(|_| {
839 ExecutionError::Internal(
840 "package debug source graph has ambiguous or malformed continuation roots",
841 )
842 })
843 }
844
845 #[inline(always)]
847 fn resume_context_from_flow(
848 flow: ControlFlow<BreakReason<Arc<MastForest>>, StackOutputs>,
849 mut continuation_stack: ContinuationStack<Arc<MastForest>>,
850 mut current_forest: Arc<MastForest>,
851 kernel: KernelDescriptor,
852 mut package_debug_info: Option<Arc<PackageDebugInfo>>,
853 mut inline_call_contexts: Vec<Option<SourceInlineCallContext>>,
854 ) -> Result<Option<ResumeContext>, ExecutionError> {
855 match flow {
856 ControlFlow::Continue(_) => Ok(None),
857 ControlFlow::Break(break_reason) => match break_reason {
858 BreakReason::Err(err) => Err(err),
859 BreakReason::Stopped(maybe_continuation) => {
860 if let Some((continuation, source_node_id)) = maybe_continuation {
861 continuation_stack
862 .push_resumed_with_source_node_id(continuation, source_node_id);
863 }
864
865 while matches!(
866 continuation_stack.peek_continuation(),
867 Some(Continuation::EnterForest { .. })
868 ) {
869 let Some((
870 Continuation::EnterForest {
871 forest,
872 package_debug_info: restored_debug_info,
873 inline_context_depth,
874 },
875 _,
876 )) = continuation_stack.pop_continuation_with_source_node_id()
877 else {
878 unreachable!("peeked continuation must still be EnterForest")
879 };
880 current_forest = forest;
881 package_debug_info = restored_debug_info;
882 inline_call_contexts.truncate(inline_context_depth);
883 }
884
885 Ok(Some(ResumeContext {
886 current_forest,
887 continuation_stack,
888 kernel,
889 package_debug_info,
890 inline_call_contexts,
891 }))
892 },
893 },
894 }
895 }
896
897 #[inline(always)]
899 fn current_stack_outputs(&self) -> StackOutputs {
900 StackOutputs::new(
901 &self.stack[self.stack_bot_idx..self.stack_top_idx]
902 .iter()
903 .rev()
904 .copied()
905 .collect::<Vec<_>>(),
906 )
907 .unwrap()
908 }
909
910 fn execute_impl<S, T>(
916 &mut self,
917 continuation_stack: &mut ContinuationStack<Arc<MastForest>>,
918 current_forest: &mut Arc<MastForest>,
919 kernel: &KernelDescriptor,
920 host: &mut impl SyncHost,
921 tracer: &mut T,
922 stopper: &S,
923 package_debug_info: &mut Option<Arc<PackageDebugInfo>>,
924 inline_call_contexts: &mut Vec<Option<SourceInlineCallContext>>,
925 ) -> ControlFlow<BreakReason<Arc<MastForest>>, StackOutputs>
926 where
927 S: Stopper<Processor = Self, Forest = Arc<MastForest>>,
928 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
929 {
930 while let ControlFlow::Break(internal_break_reason) = execute_impl(
931 self,
932 continuation_stack,
933 current_forest,
934 kernel,
935 host,
936 tracer,
937 stopper,
938 package_debug_info,
939 inline_call_contexts,
940 ) {
941 let current_package_debug_info = package_debug_info.as_deref();
942 let source_aware_execution =
943 current_package_debug_info.is_some() || continuation_stack.tracks_source_nodes();
944 match internal_break_reason {
945 InternalBreakReason::User(break_reason) => return ControlFlow::Break(break_reason),
946 InternalBreakReason::Emit { op_idx, continuation, source_node_id } => {
947 self.op_emit_sync(host, op_idx, current_package_debug_info, source_node_id)?;
948
949 finish_emit_op_execution(
950 continuation,
951 source_node_id,
952 self,
953 continuation_stack,
954 current_forest,
955 tracer,
956 stopper,
957 )?;
958 },
959 InternalBreakReason::LoadMastForestFromDyn { callee_hash, source_node_id } => {
960 let (root_id, new_forest, new_package_debug_info, new_source_node_id) =
961 match self.load_mast_forest_sync(
962 callee_hash,
963 host,
964 current_package_debug_info,
965 source_node_id,
966 source_aware_execution,
967 ) {
968 Ok(result) => result,
969 Err(err) => return ControlFlow::Break(BreakReason::Err(err)),
970 };
971
972 finish_load_mast_forest_from_dyn_start(
973 root_id,
974 new_forest,
975 new_package_debug_info,
976 new_source_node_id,
977 self,
978 current_forest,
979 package_debug_info,
980 inline_call_contexts.as_slice(),
981 continuation_stack,
982 tracer,
983 stopper,
984 )?;
985 },
986 InternalBreakReason::LoadMastForestFromExternal {
987 external_node_id,
988 procedure_hash,
989 source_node_id,
990 } => {
991 let inline_call_context = package_debug_info.clone().and_then(|debug_info| {
992 SourceInlineCallContext::for_source_boundary(debug_info, source_node_id)
993 });
994 let (root_id, new_forest, new_package_debug_info, new_source_node_id) =
995 match self.load_mast_forest_sync(
996 procedure_hash,
997 host,
998 current_package_debug_info,
999 source_node_id,
1000 source_aware_execution,
1001 ) {
1002 Ok(result) => result,
1003 Err(err) => {
1004 let maybe_enriched_err = maybe_use_caller_error_context(
1005 err,
1006 continuation_stack,
1007 current_package_debug_info,
1008 host,
1009 );
1010 return ControlFlow::Break(BreakReason::Err(maybe_enriched_err));
1011 },
1012 };
1013
1014 finish_load_mast_forest_from_external(
1015 root_id,
1016 new_forest,
1017 new_package_debug_info,
1018 new_source_node_id,
1019 inline_call_context,
1020 (external_node_id, source_node_id),
1021 current_forest,
1022 package_debug_info,
1023 inline_call_contexts,
1024 continuation_stack,
1025 tracer,
1026 )?;
1027 },
1028 }
1029 }
1030
1031 match StackOutputs::new(
1032 &self.stack[self.stack_bot_idx..self.stack_top_idx]
1033 .iter()
1034 .rev()
1035 .copied()
1036 .collect::<Vec<_>>(),
1037 ) {
1038 Ok(stack_outputs) => ControlFlow::Continue(stack_outputs),
1039 Err(_) => ControlFlow::Break(BreakReason::Err(ExecutionError::OutputStackOverflow(
1040 self.stack_top_idx - self.stack_bot_idx - MIN_STACK_DEPTH,
1041 ))),
1042 }
1043 }
1044
1045 async fn execute_impl_async<S, T>(
1046 &mut self,
1047 continuation_stack: &mut ContinuationStack<Arc<MastForest>>,
1048 current_forest: &mut Arc<MastForest>,
1049 kernel: &KernelDescriptor,
1050 host: &mut impl Host,
1051 tracer: &mut T,
1052 stopper: &S,
1053 package_debug_info: &mut Option<Arc<PackageDebugInfo>>,
1054 inline_call_contexts: &mut Vec<Option<SourceInlineCallContext>>,
1055 ) -> ControlFlow<BreakReason<Arc<MastForest>>, StackOutputs>
1056 where
1057 S: Stopper<Processor = Self, Forest = Arc<MastForest>>,
1058 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
1059 {
1060 while let ControlFlow::Break(internal_break_reason) = execute_impl(
1061 self,
1062 continuation_stack,
1063 current_forest,
1064 kernel,
1065 host,
1066 tracer,
1067 stopper,
1068 package_debug_info,
1069 inline_call_contexts,
1070 ) {
1071 let current_package_debug_info = package_debug_info.as_deref();
1072 let source_aware_execution =
1073 current_package_debug_info.is_some() || continuation_stack.tracks_source_nodes();
1074 match internal_break_reason {
1075 InternalBreakReason::User(break_reason) => return ControlFlow::Break(break_reason),
1076 InternalBreakReason::Emit { op_idx, continuation, source_node_id } => {
1077 self.op_emit(host, op_idx, current_package_debug_info, source_node_id).await?;
1078
1079 finish_emit_op_execution(
1080 continuation,
1081 source_node_id,
1082 self,
1083 continuation_stack,
1084 current_forest,
1085 tracer,
1086 stopper,
1087 )?;
1088 },
1089 InternalBreakReason::LoadMastForestFromDyn { callee_hash, source_node_id } => {
1090 let (root_id, new_forest, new_package_debug_info, new_source_node_id) =
1091 match self
1092 .load_mast_forest(
1093 callee_hash,
1094 host,
1095 current_package_debug_info,
1096 source_node_id,
1097 source_aware_execution,
1098 )
1099 .await
1100 {
1101 Ok(result) => result,
1102 Err(err) => return ControlFlow::Break(BreakReason::Err(err)),
1103 };
1104
1105 finish_load_mast_forest_from_dyn_start(
1106 root_id,
1107 new_forest,
1108 new_package_debug_info,
1109 new_source_node_id,
1110 self,
1111 current_forest,
1112 package_debug_info,
1113 inline_call_contexts.as_slice(),
1114 continuation_stack,
1115 tracer,
1116 stopper,
1117 )?;
1118 },
1119 InternalBreakReason::LoadMastForestFromExternal {
1120 external_node_id,
1121 procedure_hash,
1122 source_node_id,
1123 } => {
1124 let inline_call_context = package_debug_info.clone().and_then(|debug_info| {
1125 SourceInlineCallContext::for_source_boundary(debug_info, source_node_id)
1126 });
1127 let (root_id, new_forest, new_package_debug_info, new_source_node_id) =
1128 match self
1129 .load_mast_forest(
1130 procedure_hash,
1131 host,
1132 current_package_debug_info,
1133 source_node_id,
1134 source_aware_execution,
1135 )
1136 .await
1137 {
1138 Ok(result) => result,
1139 Err(err) => {
1140 let maybe_enriched_err = maybe_use_caller_error_context(
1141 err,
1142 continuation_stack,
1143 current_package_debug_info,
1144 host,
1145 );
1146 return ControlFlow::Break(BreakReason::Err(maybe_enriched_err));
1147 },
1148 };
1149
1150 finish_load_mast_forest_from_external(
1151 root_id,
1152 new_forest,
1153 new_package_debug_info,
1154 new_source_node_id,
1155 inline_call_context,
1156 (external_node_id, source_node_id),
1157 current_forest,
1158 package_debug_info,
1159 inline_call_contexts,
1160 continuation_stack,
1161 tracer,
1162 )?;
1163 },
1164 }
1165 }
1166
1167 match StackOutputs::new(
1168 &self.stack[self.stack_bot_idx..self.stack_top_idx]
1169 .iter()
1170 .rev()
1171 .copied()
1172 .collect::<Vec<_>>(),
1173 ) {
1174 Ok(stack_outputs) => ControlFlow::Continue(stack_outputs),
1175 Err(_) => ControlFlow::Break(BreakReason::Err(ExecutionError::OutputStackOverflow(
1176 self.stack_top_idx - self.stack_bot_idx - MIN_STACK_DEPTH,
1177 ))),
1178 }
1179 }
1180
1181 fn load_mast_forest_sync(
1185 &mut self,
1186 node_digest: Word,
1187 host: &mut impl SyncHost,
1188 package_debug_info: Option<&PackageDebugInfo>,
1189 source_node_id: Option<DebugSourceNodeId>,
1190 source_aware_execution: bool,
1191 ) -> Result<
1192 (
1193 MastNodeId,
1194 Arc<MastForest>,
1195 Option<Arc<PackageDebugInfo>>,
1196 Option<DebugSourceNodeId>,
1197 ),
1198 ExecutionError,
1199 > {
1200 let cached = self.loaded_mast_forests.get(&node_digest).cloned();
1201 let was_cached = cached.is_some();
1202 let loaded_mast_forest = match cached {
1203 Some(mast_forest) => mast_forest,
1204 None => host.get_mast_forest(&node_digest).ok_or_else(|| {
1205 match (package_debug_info, source_node_id) {
1206 (Some(debug_info), Some(source_node_id)) => {
1207 crate::errors::procedure_not_found_with_package_source_context(
1208 node_digest,
1209 PackageSourceDebugContext::new(debug_info, source_node_id),
1210 host,
1211 )
1212 },
1213 _ => crate::errors::procedure_not_found_with_context(node_digest),
1214 }
1215 })?,
1216 };
1217 let mast_forest = loaded_mast_forest.mast_forest().clone();
1218
1219 let root_id = mast_forest.find_procedure_root(node_digest).ok_or_else(|| {
1220 let context = match (package_debug_info, source_node_id) {
1221 (Some(debug_info), Some(source_node_id)) => {
1222 Some(PackageSourceDebugContext::new(debug_info, source_node_id))
1223 },
1224 _ => None,
1225 };
1226 malformed_mast_forest_with_context(node_digest, context, host)
1227 })?;
1228
1229 if !was_cached {
1230 self.cache_loaded_mast_forest(&loaded_mast_forest);
1231 }
1232 self.merge_mast_forest_advice(&mast_forest).map_exec_err()?;
1233 let (loaded_package_debug_info, loaded_source_node_id) =
1234 Self::loaded_package_source_context(
1235 &loaded_mast_forest,
1236 root_id,
1237 source_aware_execution,
1238 )?;
1239
1240 Ok((root_id, mast_forest, loaded_package_debug_info, loaded_source_node_id))
1241 }
1242
1243 async fn load_mast_forest(
1244 &mut self,
1245 node_digest: Word,
1246 host: &mut impl Host,
1247 package_debug_info: Option<&PackageDebugInfo>,
1248 source_node_id: Option<DebugSourceNodeId>,
1249 source_aware_execution: bool,
1250 ) -> Result<
1251 (
1252 MastNodeId,
1253 Arc<MastForest>,
1254 Option<Arc<PackageDebugInfo>>,
1255 Option<DebugSourceNodeId>,
1256 ),
1257 ExecutionError,
1258 > {
1259 let cached = self.loaded_mast_forests.get(&node_digest).cloned();
1260 let was_cached = cached.is_some();
1261 let loaded_mast_forest = match cached {
1262 Some(mast_forest) => mast_forest,
1263 None => {
1264 if let Some(mast_forest) = host.get_mast_forest(&node_digest).await {
1265 mast_forest
1266 } else {
1267 return Err(match (package_debug_info, source_node_id) {
1268 (Some(debug_info), Some(source_node_id)) => {
1269 crate::errors::procedure_not_found_with_package_source_context(
1270 node_digest,
1271 PackageSourceDebugContext::new(debug_info, source_node_id),
1272 host,
1273 )
1274 },
1275 _ => crate::errors::procedure_not_found_with_context(node_digest),
1276 });
1277 }
1278 },
1279 };
1280 let mast_forest = loaded_mast_forest.mast_forest().clone();
1281
1282 let root_id = mast_forest.find_procedure_root(node_digest).ok_or_else(|| {
1283 let context = match (package_debug_info, source_node_id) {
1284 (Some(debug_info), Some(source_node_id)) => {
1285 Some(PackageSourceDebugContext::new(debug_info, source_node_id))
1286 },
1287 _ => None,
1288 };
1289 malformed_mast_forest_with_context(node_digest, context, host)
1290 })?;
1291
1292 if !was_cached {
1293 self.cache_loaded_mast_forest(&loaded_mast_forest);
1294 }
1295 self.merge_mast_forest_advice(&mast_forest).map_exec_err()?;
1296 let (loaded_package_debug_info, loaded_source_node_id) =
1297 Self::loaded_package_source_context(
1298 &loaded_mast_forest,
1299 root_id,
1300 source_aware_execution,
1301 )?;
1302
1303 Ok((root_id, mast_forest, loaded_package_debug_info, loaded_source_node_id))
1304 }
1305
1306 fn cache_loaded_mast_forest(&mut self, loaded_mast_forest: &LoadedMastForest) {
1307 for procedure_digest in loaded_mast_forest.mast_forest().local_procedure_digests() {
1308 self.loaded_mast_forests
1309 .entry(procedure_digest)
1310 .or_insert_with(|| loaded_mast_forest.clone());
1311 }
1312 }
1313
1314 fn merge_mast_forest_advice(&mut self, mast_forest: &MastForest) -> Result<(), AdviceError> {
1315 let commitment = mast_forest.commitment();
1316 if self.merged_mast_forests.contains(&commitment) {
1317 return Ok(());
1318 }
1319
1320 self.advice.extend_map(mast_forest.advice_map())?;
1321 self.merged_mast_forests.insert(commitment);
1322 Ok(())
1323 }
1324
1325 fn loaded_package_source_context(
1326 loaded_mast_forest: &LoadedMastForest,
1327 root_id: MastNodeId,
1328 source_aware_execution: bool,
1329 ) -> Result<(Option<Arc<PackageDebugInfo>>, Option<DebugSourceNodeId>), ExecutionError> {
1330 if !source_aware_execution {
1331 return Ok((None, None));
1332 }
1333
1334 let Some(package_debug_info) = loaded_mast_forest
1335 .package_debug_info()
1336 .map_err(|_| ExecutionError::Internal("loaded package debug info is malformed"))?
1337 else {
1338 return Ok((None, None));
1339 };
1340
1341 let source_node_id = match package_debug_info.unique_source_root_for_exec_node(root_id) {
1342 Ok(source_node_id) => source_node_id,
1343 Err(DebugSourceGraphLookupError::AmbiguousRoot { .. }) => None,
1344 Err(_) => {
1345 return Err(ExecutionError::Internal(
1346 "loaded package debug source graph has malformed entrypoint roots",
1347 ));
1348 },
1349 };
1350
1351 Ok((Some(package_debug_info), source_node_id))
1352 }
1353
1354 pub fn execute_by_step_sync(
1356 mut self,
1357 program: &Program,
1358 host: &mut impl SyncHost,
1359 ) -> Result<StackOutputs, ExecutionError> {
1360 let mut current_resume_ctx = self.get_initial_resume_context(program)?;
1361
1362 loop {
1363 match self.step_sync(host, current_resume_ctx)? {
1364 Some(next_resume_ctx) => {
1365 current_resume_ctx = next_resume_ctx;
1366 },
1367 None => break Ok(self.current_stack_outputs()),
1368 }
1369 }
1370 }
1371
1372 #[cfg(any(test, feature = "testing"))]
1375 pub fn execute_by_step_with_package_debug_info_sync(
1376 mut self,
1377 program: &Program,
1378 package_debug_info: &PackageDebugInfo,
1379 host: &mut impl SyncHost,
1380 ) -> Result<StackOutputs, ExecutionError> {
1381 let mut current_resume_ctx =
1382 self.source_aware_resume_context(program, package_debug_info, None)?;
1383
1384 loop {
1385 match self.step_with_package_debug_info_sync(
1386 host,
1387 current_resume_ctx,
1388 package_debug_info,
1389 )? {
1390 Some(next_resume_ctx) => {
1391 current_resume_ctx = next_resume_ctx;
1392 },
1393 None => break Ok(self.current_stack_outputs()),
1394 }
1395 }
1396 }
1397
1398 #[cfg(any(test, feature = "testing"))]
1401 pub fn execute_by_step_with_package_debug_info_at_source_node_sync(
1402 mut self,
1403 program: &Program,
1404 package_debug_info: &PackageDebugInfo,
1405 entrypoint_source_node_id: DebugSourceNodeId,
1406 host: &mut impl SyncHost,
1407 ) -> Result<StackOutputs, ExecutionError> {
1408 let mut current_resume_ctx = self.source_aware_resume_context(
1409 program,
1410 package_debug_info,
1411 Some(entrypoint_source_node_id),
1412 )?;
1413
1414 loop {
1415 match self.step_with_package_debug_info_sync(
1416 host,
1417 current_resume_ctx,
1418 package_debug_info,
1419 )? {
1420 Some(next_resume_ctx) => {
1421 current_resume_ctx = next_resume_ctx;
1422 },
1423 None => break Ok(self.current_stack_outputs()),
1424 }
1425 }
1426 }
1427
1428 #[inline(always)]
1430 pub async fn execute_by_step(
1431 mut self,
1432 program: &Program,
1433 host: &mut impl Host,
1434 ) -> Result<StackOutputs, ExecutionError> {
1435 let mut current_resume_ctx = self.get_initial_resume_context(program)?;
1436 let mut processor = self;
1437
1438 loop {
1439 match processor.step(host, current_resume_ctx).await? {
1440 Some(next_resume_ctx) => {
1441 current_resume_ctx = next_resume_ctx;
1442 },
1443 None => break Ok(processor.current_stack_outputs()),
1444 }
1445 }
1446 }
1447
1448 #[cfg(any(test, feature = "testing"))]
1450 #[inline(always)]
1451 pub async fn execute_by_step_with_package_debug_info(
1452 mut self,
1453 program: &Program,
1454 package_debug_info: &PackageDebugInfo,
1455 host: &mut impl Host,
1456 ) -> Result<StackOutputs, ExecutionError> {
1457 let mut current_resume_ctx =
1458 self.source_aware_resume_context(program, package_debug_info, None)?;
1459 let mut processor = self;
1460
1461 loop {
1462 match processor
1463 .step_with_package_debug_info(host, current_resume_ctx, package_debug_info)
1464 .await?
1465 {
1466 Some(next_resume_ctx) => {
1467 current_resume_ctx = next_resume_ctx;
1468 },
1469 None => break Ok(processor.current_stack_outputs()),
1470 }
1471 }
1472 }
1473
1474 #[cfg(any(test, feature = "testing"))]
1477 #[inline(always)]
1478 pub async fn execute_by_step_with_package_debug_info_at_source_node(
1479 mut self,
1480 program: &Program,
1481 package_debug_info: &PackageDebugInfo,
1482 entrypoint_source_node_id: DebugSourceNodeId,
1483 host: &mut impl Host,
1484 ) -> Result<StackOutputs, ExecutionError> {
1485 let mut current_resume_ctx = self.source_aware_resume_context(
1486 program,
1487 package_debug_info,
1488 Some(entrypoint_source_node_id),
1489 )?;
1490 let mut processor = self;
1491
1492 loop {
1493 match processor
1494 .step_with_package_debug_info(host, current_resume_ctx, package_debug_info)
1495 .await?
1496 {
1497 Some(next_resume_ctx) => {
1498 current_resume_ctx = next_resume_ctx;
1499 },
1500 None => break Ok(processor.current_stack_outputs()),
1501 }
1502 }
1503 }
1504
1505 #[cfg(any(test, feature = "testing"))]
1509 pub fn execute_mut_sync(
1510 &mut self,
1511 program: &Program,
1512 host: &mut impl SyncHost,
1513 ) -> Result<StackOutputs, ExecutionError> {
1514 let mut continuation_stack = ContinuationStack::new(program);
1515 let mut current_forest = program.mast_forest().clone();
1516 let mut package_debug_info = None;
1517 let mut inline_call_contexts = Vec::new();
1518
1519 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
1520
1521 let flow = self.execute_impl(
1522 &mut continuation_stack,
1523 &mut current_forest,
1524 program.kernel(),
1525 host,
1526 &mut NoopTracer,
1527 &NeverStopper,
1528 &mut package_debug_info,
1529 &mut inline_call_contexts,
1530 );
1531 Self::stack_result_from_flow(flow)
1532 }
1533
1534 #[cfg(any(test, feature = "testing"))]
1536 #[inline(always)]
1537 pub async fn execute_mut(
1538 &mut self,
1539 program: &Program,
1540 host: &mut impl Host,
1541 ) -> Result<StackOutputs, ExecutionError> {
1542 let mut continuation_stack = ContinuationStack::new(program);
1543 let mut current_forest = program.mast_forest().clone();
1544 let mut package_debug_info = None;
1545 let mut inline_call_contexts = Vec::new();
1546
1547 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
1548
1549 let flow = self
1550 .execute_impl_async(
1551 &mut continuation_stack,
1552 &mut current_forest,
1553 program.kernel(),
1554 host,
1555 &mut NoopTracer,
1556 &NeverStopper,
1557 &mut package_debug_info,
1558 &mut inline_call_contexts,
1559 )
1560 .await;
1561 Self::stack_result_from_flow(flow)
1562 }
1563}
1564
1565#[cfg(all(test, feature = "std"))]
1566mod tests {
1567 use super::FastProcessor;
1568
1569 #[test]
1570 fn sole_rayon_worker_requires_buffered_trace_building() {
1571 rayon::ThreadPoolBuilder::new()
1572 .num_threads(1)
1573 .build()
1574 .unwrap()
1575 .install(|| assert!(FastProcessor::rayon_has_no_parallel_worker()));
1576 }
1577}