1use alloc::{sync::Arc, vec::Vec};
2use core::ops::ControlFlow;
3
4use miden_core::{
5 Word,
6 mast::{MastForest, MastNodeId},
7 program::{KernelDescriptor, MIN_STACK_DEPTH, Program, StackInputs, StackOutputs},
8};
9use miden_mast_package::debug_info::{
10 DebugSourceGraphLookupError, DebugSourceNodeId, PackageDebugInfo,
11};
12use tracing::instrument;
13
14use super::{
15 FastProcessor, NoopTracer,
16 external::maybe_use_caller_error_context,
17 step::{BreakReason, NeverStopper, ResumeContext, StepStopper},
18};
19#[cfg(feature = "std")]
20use crate::PrecompileWitness;
21use crate::{
22 ExecutionError, ExecutionOutput, ExecutionWitness, Host, LoadedMastForest, Stopper, SyncHost,
23 advice::AdviceError,
24 continuation_stack::{Continuation, ContinuationStack, SourceInlineCallContext},
25 errors::{
26 MapExecErr, MapExecErrNoCtx, PackageSourceDebugContext, malformed_mast_forest_with_context,
27 },
28 execution::{
29 InternalBreakReason, execute_impl, finish_emit_op_execution,
30 finish_load_mast_forest_from_dyn_start, finish_load_mast_forest_from_external,
31 },
32 trace::execution_tracer::ExecutionTracer,
33 tracer::Tracer,
34};
35
36impl FastProcessor {
37 pub fn execute_sync(
42 self,
43 program: &Program,
44 host: &mut impl SyncHost,
45 ) -> Result<ExecutionOutput, ExecutionError> {
46 self.execute_with_tracer_sync(program, host, &mut NoopTracer)
47 }
48
49 pub fn execute_with_package_debug_info_sync(
56 self,
57 program: &Program,
58 package_debug_info: &PackageDebugInfo,
59 host: &mut impl SyncHost,
60 ) -> Result<ExecutionOutput, ExecutionError> {
61 self.execute_with_package_debug_info_and_tracer_sync(
62 program,
63 package_debug_info,
64 None,
65 host,
66 &mut NoopTracer,
67 )
68 }
69
70 pub fn execute_with_package_debug_info_at_source_node_sync(
77 self,
78 program: &Program,
79 package_debug_info: &PackageDebugInfo,
80 entrypoint_source_node_id: DebugSourceNodeId,
81 host: &mut impl SyncHost,
82 ) -> Result<ExecutionOutput, ExecutionError> {
83 self.execute_with_package_debug_info_and_tracer_sync(
84 program,
85 package_debug_info,
86 Some(entrypoint_source_node_id),
87 host,
88 &mut NoopTracer,
89 )
90 }
91
92 #[inline(always)]
94 pub async fn execute(
95 self,
96 program: &Program,
97 host: &mut impl Host,
98 ) -> Result<ExecutionOutput, ExecutionError> {
99 self.execute_with_tracer(program, host, &mut NoopTracer).await
100 }
101
102 #[inline(always)]
107 pub async fn execute_with_package_debug_info(
108 self,
109 program: &Program,
110 package_debug_info: &PackageDebugInfo,
111 host: &mut impl Host,
112 ) -> Result<ExecutionOutput, ExecutionError> {
113 self.execute_with_package_debug_info_and_tracer(
114 program,
115 package_debug_info,
116 None,
117 host,
118 &mut NoopTracer,
119 )
120 .await
121 }
122
123 #[inline(always)]
125 pub async fn execute_with_package_debug_info_at_source_node(
126 self,
127 program: &Program,
128 package_debug_info: &PackageDebugInfo,
129 entrypoint_source_node_id: DebugSourceNodeId,
130 host: &mut impl Host,
131 ) -> Result<ExecutionOutput, ExecutionError> {
132 self.execute_with_package_debug_info_and_tracer(
133 program,
134 package_debug_info,
135 Some(entrypoint_source_node_id),
136 host,
137 &mut NoopTracer,
138 )
139 .await
140 }
141
142 #[cfg(feature = "std")]
157 #[instrument(name = "execute_and_build_trace_sync", skip_all)]
158 pub fn execute_and_build_trace_sync(
159 self,
160 program: &Program,
161 host: &mut impl SyncHost,
162 max_prover_memory_bytes: u64,
163 ) -> Result<(crate::trace::VmTrace, Option<PrecompileWitness>), ExecutionError> {
164 use miden_air::{config, memory};
165
166 use crate::trace::{
167 MAX_TRACE_LEN, build_hasher_chiplet, build_trace_with_budget,
168 build_trace_with_prebuilt_hasher,
169 };
170
171 if Self::rayon_has_no_parallel_worker() {
172 let (vm_witness, precompiles_witness) =
173 self.execute_for_proving_sync(program, host)?.into_parts();
174 let trace = build_trace_with_budget(vm_witness, max_prover_memory_bytes)?;
175 return Ok((trace, precompiles_witness));
176 }
177
178 let stack_inputs = self.initial_stack_inputs();
179 let max_trace_len = MAX_TRACE_LEN.min(memory::max_any_height_for_budget(
180 max_prover_memory_bytes,
181 &config::pcs_params(),
182 ));
183 let (sender, receiver) = std::sync::mpsc::channel();
184 let mut tracer = ExecutionTracer::new_with_streamed_hasher(
185 self.options.core_trace_fragment_size(),
186 self.options.max_stack_depth(),
187 sender,
188 );
189
190 let mut hasher = None;
191 let hasher_slot = &mut hasher;
192 let execution_output = rayon::in_place_scope(move |scope| {
195 let span = tracing::Span::current();
198 scope.spawn(move |_| {
199 let _span = span.entered();
200 let result = build_hasher_chiplet(receiver.into_iter().map(Ok), max_trace_len);
201 *hasher_slot = Some(result);
202 });
203
204 let execution_output = self.execute_with_tracer_sync(program, host, &mut tracer);
205
206 match execution_output {
207 Ok(output) => {
208 let (mut vm_witness, precompiles_witness) =
209 Self::execution_witness_from_parts(program, stack_inputs, output, tracer)
210 .into_parts();
211 drop(vm_witness.take_hasher_replay());
213 Ok((vm_witness, precompiles_witness))
214 },
215 Err(err) => {
216 drop(tracer);
220 Err(err)
221 },
222 }
223 });
224
225 let hasher = hasher.expect("hasher builder did not run");
226 let (vm_witness, precompiles_witness) = match execution_output {
227 Ok(output) => output,
228 Err(err) => {
229 if let Err(builder_err) = hasher {
230 tracing::debug!(%builder_err, "hasher builder also failed");
231 }
232 return Err(err);
233 },
234 };
235
236 let trace = build_trace_with_prebuilt_hasher(vm_witness, hasher?, max_prover_memory_bytes)?;
237 Ok((trace, precompiles_witness))
238 }
239
240 #[cfg(feature = "std")]
241 fn rayon_has_no_parallel_worker() -> bool {
242 rayon::current_num_threads() == 1 && rayon::current_thread_index().is_some()
244 }
245
246 #[instrument(name = "execute_for_proving_sync", skip_all)]
268 pub fn execute_for_proving_sync(
269 self,
270 program: &Program,
271 host: &mut impl SyncHost,
272 ) -> Result<ExecutionWitness, ExecutionError> {
273 let stack_inputs = self.initial_stack_inputs();
274 let mut tracer = ExecutionTracer::new(
275 self.options.core_trace_fragment_size(),
276 self.options.max_stack_depth(),
277 );
278 let execution_output = self.execute_with_tracer_sync(program, host, &mut tracer)?;
279 Ok(Self::execution_witness_from_parts(
280 program,
281 stack_inputs,
282 execution_output,
283 tracer,
284 ))
285 }
286
287 #[instrument(name = "execute_for_proving_with_package_debug_info_sync", skip_all)]
290 pub fn execute_for_proving_with_package_debug_info_sync(
291 self,
292 program: &Program,
293 package_debug_info: &PackageDebugInfo,
294 host: &mut impl SyncHost,
295 ) -> Result<ExecutionWitness, ExecutionError> {
296 let stack_inputs = self.initial_stack_inputs();
297 let mut tracer = ExecutionTracer::new(
298 self.options.core_trace_fragment_size(),
299 self.options.max_stack_depth(),
300 );
301 let execution_output = self.execute_with_package_debug_info_and_tracer_sync(
302 program,
303 package_debug_info,
304 None,
305 host,
306 &mut tracer,
307 )?;
308 Ok(Self::execution_witness_from_parts(
309 program,
310 stack_inputs,
311 execution_output,
312 tracer,
313 ))
314 }
315
316 #[instrument(
319 name = "execute_for_proving_with_package_debug_info_at_source_node_sync",
320 skip_all
321 )]
322 pub fn execute_for_proving_with_package_debug_info_at_source_node_sync(
323 self,
324 program: &Program,
325 package_debug_info: &PackageDebugInfo,
326 entrypoint_source_node_id: DebugSourceNodeId,
327 host: &mut impl SyncHost,
328 ) -> Result<ExecutionWitness, ExecutionError> {
329 let stack_inputs = self.initial_stack_inputs();
330 let mut tracer = ExecutionTracer::new(
331 self.options.core_trace_fragment_size(),
332 self.options.max_stack_depth(),
333 );
334 let execution_output = self.execute_with_package_debug_info_and_tracer_sync(
335 program,
336 package_debug_info,
337 Some(entrypoint_source_node_id),
338 host,
339 &mut tracer,
340 )?;
341 Ok(Self::execution_witness_from_parts(
342 program,
343 stack_inputs,
344 execution_output,
345 tracer,
346 ))
347 }
348
349 #[inline(always)]
351 #[instrument(name = "execute_for_proving", skip_all)]
352 pub async fn execute_for_proving(
353 self,
354 program: &Program,
355 host: &mut impl Host,
356 ) -> Result<ExecutionWitness, ExecutionError> {
357 let stack_inputs = self.initial_stack_inputs();
358 let mut tracer = ExecutionTracer::new(
359 self.options.core_trace_fragment_size(),
360 self.options.max_stack_depth(),
361 );
362 let execution_output = self.execute_with_tracer(program, host, &mut tracer).await?;
363 Ok(Self::execution_witness_from_parts(
364 program,
365 stack_inputs,
366 execution_output,
367 tracer,
368 ))
369 }
370
371 #[cfg(any(test, feature = "testing"))]
373 #[inline(always)]
374 #[instrument(name = "execute_for_proving_with_package_debug_info", skip_all)]
375 pub async fn execute_for_proving_with_package_debug_info(
376 self,
377 program: &Program,
378 package_debug_info: &PackageDebugInfo,
379 host: &mut impl Host,
380 ) -> Result<ExecutionWitness, ExecutionError> {
381 let stack_inputs = self.initial_stack_inputs();
382 let mut tracer = ExecutionTracer::new(
383 self.options.core_trace_fragment_size(),
384 self.options.max_stack_depth(),
385 );
386 let execution_output = self
387 .execute_with_package_debug_info_and_tracer(
388 program,
389 package_debug_info,
390 None,
391 host,
392 &mut tracer,
393 )
394 .await?;
395 Ok(Self::execution_witness_from_parts(
396 program,
397 stack_inputs,
398 execution_output,
399 tracer,
400 ))
401 }
402
403 #[cfg(any(test, feature = "testing"))]
406 #[inline(always)]
407 #[instrument(name = "execute_for_proving_with_package_debug_info_at_source_node", skip_all)]
408 pub async fn execute_for_proving_with_package_debug_info_at_source_node(
409 self,
410 program: &Program,
411 package_debug_info: &PackageDebugInfo,
412 entrypoint_source_node_id: DebugSourceNodeId,
413 host: &mut impl Host,
414 ) -> Result<ExecutionWitness, ExecutionError> {
415 let stack_inputs = self.initial_stack_inputs();
416 let mut tracer = ExecutionTracer::new(
417 self.options.core_trace_fragment_size(),
418 self.options.max_stack_depth(),
419 );
420 let execution_output = self
421 .execute_with_package_debug_info_and_tracer(
422 program,
423 package_debug_info,
424 Some(entrypoint_source_node_id),
425 host,
426 &mut tracer,
427 )
428 .await?;
429 Ok(Self::execution_witness_from_parts(
430 program,
431 stack_inputs,
432 execution_output,
433 tracer,
434 ))
435 }
436
437 pub async fn execute_with_tracer<T>(
439 mut self,
440 program: &Program,
441 host: &mut impl Host,
442 tracer: &mut T,
443 ) -> Result<ExecutionOutput, ExecutionError>
444 where
445 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
446 {
447 let mut continuation_stack = ContinuationStack::new(program);
448 let mut current_forest = program.mast_forest().clone();
449 let mut package_debug_info = None;
450 let mut inline_call_contexts = Vec::new();
451
452 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
453 let flow = self
454 .execute_impl_async(
455 &mut continuation_stack,
456 &mut current_forest,
457 program.kernel(),
458 host,
459 tracer,
460 &NeverStopper,
461 &mut package_debug_info,
462 &mut inline_call_contexts,
463 )
464 .await;
465 Self::execution_result_from_flow(flow, self)
466 }
467
468 async fn execute_with_package_debug_info_and_tracer<T>(
471 mut self,
472 program: &Program,
473 package_debug_info: &PackageDebugInfo,
474 entrypoint_source_node_id: Option<DebugSourceNodeId>,
475 host: &mut impl Host,
476 tracer: &mut T,
477 ) -> Result<ExecutionOutput, ExecutionError>
478 where
479 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
480 {
481 let mut continuation_stack = Self::source_aware_continuation_stack(
482 program,
483 package_debug_info,
484 entrypoint_source_node_id,
485 )?;
486 let mut current_forest = program.mast_forest().clone();
487 let mut package_debug_info = Some(Arc::new(package_debug_info.clone()));
488 let mut inline_call_contexts = Vec::new();
489
490 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
491 let flow = self
492 .execute_impl_async(
493 &mut continuation_stack,
494 &mut current_forest,
495 program.kernel(),
496 host,
497 tracer,
498 &NeverStopper,
499 &mut package_debug_info,
500 &mut inline_call_contexts,
501 )
502 .await;
503 Self::execution_result_from_flow(flow, self)
504 }
505
506 pub fn execute_with_tracer_sync<T>(
508 mut self,
509 program: &Program,
510 host: &mut impl SyncHost,
511 tracer: &mut T,
512 ) -> Result<ExecutionOutput, ExecutionError>
513 where
514 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
515 {
516 let mut continuation_stack = ContinuationStack::new(program);
517 let mut current_forest = program.mast_forest().clone();
518 let mut package_debug_info = None;
519 let mut inline_call_contexts = Vec::new();
520
521 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
522 let flow = self.execute_impl(
523 &mut continuation_stack,
524 &mut current_forest,
525 program.kernel(),
526 host,
527 tracer,
528 &NeverStopper,
529 &mut package_debug_info,
530 &mut inline_call_contexts,
531 );
532 Self::execution_result_from_flow(flow, self)
533 }
534
535 fn execute_with_package_debug_info_and_tracer_sync<T>(
538 mut self,
539 program: &Program,
540 package_debug_info: &PackageDebugInfo,
541 entrypoint_source_node_id: Option<DebugSourceNodeId>,
542 host: &mut impl SyncHost,
543 tracer: &mut T,
544 ) -> Result<ExecutionOutput, ExecutionError>
545 where
546 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
547 {
548 let mut continuation_stack = Self::source_aware_continuation_stack(
549 program,
550 package_debug_info,
551 entrypoint_source_node_id,
552 )?;
553 let mut current_forest = program.mast_forest().clone();
554 let mut package_debug_info = Some(Arc::new(package_debug_info.clone()));
555 let mut inline_call_contexts = Vec::new();
556
557 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
558 let flow = self.execute_impl(
559 &mut continuation_stack,
560 &mut current_forest,
561 program.kernel(),
562 host,
563 tracer,
564 &NeverStopper,
565 &mut package_debug_info,
566 &mut inline_call_contexts,
567 );
568 Self::execution_result_from_flow(flow, self)
569 }
570
571 pub fn step_sync(
573 &mut self,
574 host: &mut impl SyncHost,
575 resume_ctx: ResumeContext,
576 ) -> Result<Option<ResumeContext>, ExecutionError> {
577 let ResumeContext {
578 mut current_forest,
579 mut continuation_stack,
580 kernel,
581 mut package_debug_info,
582 mut inline_call_contexts,
583 } = resume_ctx;
584
585 let flow = self.execute_impl(
586 &mut continuation_stack,
587 &mut current_forest,
588 &kernel,
589 host,
590 &mut NoopTracer,
591 &StepStopper,
592 &mut package_debug_info,
593 &mut inline_call_contexts,
594 );
595 Self::resume_context_from_flow(
596 flow,
597 continuation_stack,
598 current_forest,
599 kernel,
600 package_debug_info,
601 inline_call_contexts,
602 )
603 }
604
605 pub fn step_with_package_debug_info_sync(
607 &mut self,
608 host: &mut impl SyncHost,
609 resume_ctx: ResumeContext,
610 package_debug_info: &PackageDebugInfo,
611 ) -> Result<Option<ResumeContext>, ExecutionError> {
612 let ResumeContext {
613 mut current_forest,
614 mut continuation_stack,
615 kernel,
616 package_debug_info: mut active_package_debug_info,
617 mut inline_call_contexts,
618 } = resume_ctx;
619 Self::ensure_source_aware_step_context(
620 &mut continuation_stack,
621 &mut active_package_debug_info,
622 package_debug_info,
623 )?;
624
625 let flow = self.execute_impl(
626 &mut continuation_stack,
627 &mut current_forest,
628 &kernel,
629 host,
630 &mut NoopTracer,
631 &StepStopper,
632 &mut active_package_debug_info,
633 &mut inline_call_contexts,
634 );
635 Self::resume_context_from_flow(
636 flow,
637 continuation_stack,
638 current_forest,
639 kernel,
640 active_package_debug_info,
641 inline_call_contexts,
642 )
643 }
644
645 #[inline(always)]
647 pub async fn step(
648 &mut self,
649 host: &mut impl Host,
650 resume_ctx: ResumeContext,
651 ) -> Result<Option<ResumeContext>, ExecutionError> {
652 let ResumeContext {
653 mut current_forest,
654 mut continuation_stack,
655 kernel,
656 mut package_debug_info,
657 mut inline_call_contexts,
658 } = resume_ctx;
659
660 let flow = self
661 .execute_impl_async(
662 &mut continuation_stack,
663 &mut current_forest,
664 &kernel,
665 host,
666 &mut NoopTracer,
667 &StepStopper,
668 &mut package_debug_info,
669 &mut inline_call_contexts,
670 )
671 .await;
672 Self::resume_context_from_flow(
673 flow,
674 continuation_stack,
675 current_forest,
676 kernel,
677 package_debug_info,
678 inline_call_contexts,
679 )
680 }
681
682 #[inline(always)]
684 pub async fn step_with_package_debug_info(
685 &mut self,
686 host: &mut impl Host,
687 resume_ctx: ResumeContext,
688 package_debug_info: &PackageDebugInfo,
689 ) -> Result<Option<ResumeContext>, ExecutionError> {
690 let ResumeContext {
691 mut current_forest,
692 mut continuation_stack,
693 kernel,
694 package_debug_info: mut active_package_debug_info,
695 mut inline_call_contexts,
696 } = resume_ctx;
697 Self::ensure_source_aware_step_context(
698 &mut continuation_stack,
699 &mut active_package_debug_info,
700 package_debug_info,
701 )?;
702
703 let flow = self
704 .execute_impl_async(
705 &mut continuation_stack,
706 &mut current_forest,
707 &kernel,
708 host,
709 &mut NoopTracer,
710 &StepStopper,
711 &mut active_package_debug_info,
712 &mut inline_call_contexts,
713 )
714 .await;
715 Self::resume_context_from_flow(
716 flow,
717 continuation_stack,
718 current_forest,
719 kernel,
720 active_package_debug_info,
721 inline_call_contexts,
722 )
723 }
724
725 #[inline(always)]
727 fn execution_witness_from_parts(
728 program: &Program,
729 stack_inputs: StackInputs,
730 execution_output: ExecutionOutput,
731 tracer: ExecutionTracer,
732 ) -> ExecutionWitness {
733 ExecutionWitness::from_execution(
734 program.to_info(),
735 stack_inputs,
736 execution_output,
737 tracer.into_trace_replay(),
738 )
739 }
740
741 #[inline(always)]
743 fn initial_stack_inputs(&self) -> StackInputs {
744 core::array::from_fn(|idx| self.stack_get(idx)).into()
745 }
746
747 pub(super) fn source_aware_continuation_stack(
748 program: &Program,
749 package_debug_info: &PackageDebugInfo,
750 entrypoint_source_node_id: Option<DebugSourceNodeId>,
751 ) -> Result<ContinuationStack<Arc<MastForest>>, ExecutionError> {
752 if let Some(source_node_id) = entrypoint_source_node_id {
753 let Some(source_node) = package_debug_info.source_node(source_node_id) else {
754 return Err(ExecutionError::Internal(
755 "package debug source graph is missing the entrypoint source node",
756 ));
757 };
758 if source_node.exec_node != program.entrypoint() {
759 return Err(ExecutionError::Internal(
760 "package debug entrypoint source node does not match the program entrypoint",
761 ));
762 }
763
764 return Ok(ContinuationStack::new_with_source_node_id(program, source_node_id));
765 }
766
767 let Some(source_node_id) = package_debug_info
768 .unique_source_root_for_exec_node(program.entrypoint())
769 .map_err(|_| {
770 ExecutionError::Internal(
771 "package debug source graph has ambiguous or malformed entrypoint roots",
772 )
773 })?
774 else {
775 return Ok(ContinuationStack::new_with_optional_source_node_id(program, None));
776 };
777
778 Ok(ContinuationStack::new_with_source_node_id(program, source_node_id))
779 }
780
781 #[cfg(any(test, feature = "testing"))]
782 fn source_aware_resume_context(
783 &mut self,
784 program: &Program,
785 package_debug_info: &PackageDebugInfo,
786 entrypoint_source_node_id: Option<DebugSourceNodeId>,
787 ) -> Result<ResumeContext, ExecutionError> {
788 self.advice
789 .extend_map(program.mast_forest().advice_map())
790 .map_exec_err_no_ctx()?;
791
792 Ok(ResumeContext {
793 current_forest: program.mast_forest().clone(),
794 continuation_stack: Self::source_aware_continuation_stack(
795 program,
796 package_debug_info,
797 entrypoint_source_node_id,
798 )?,
799 kernel: program.kernel().clone(),
800 package_debug_info: Some(Arc::new(package_debug_info.clone())),
801 inline_call_contexts: Vec::new(),
802 })
803 }
804
805 fn ensure_source_aware_step_context(
806 continuation_stack: &mut ContinuationStack<Arc<MastForest>>,
807 package_debug_info: &mut Option<Arc<PackageDebugInfo>>,
808 supplied_package_debug_info: &PackageDebugInfo,
809 ) -> Result<(), ExecutionError> {
810 if package_debug_info.is_none() {
811 *package_debug_info = Some(Arc::new(supplied_package_debug_info.clone()));
812 }
813
814 if !continuation_stack.tracks_source_nodes() {
815 let source_node_id = Self::source_root_for_next_continuation(
816 continuation_stack,
817 package_debug_info.as_deref().expect("package debug info was just initialized"),
818 )?;
819 continuation_stack.start_tracking_source_nodes(source_node_id);
820 }
821
822 Ok(())
823 }
824
825 fn source_root_for_next_continuation(
826 continuation_stack: &ContinuationStack<Arc<MastForest>>,
827 package_debug_info: &PackageDebugInfo,
828 ) -> Result<Option<DebugSourceNodeId>, ExecutionError> {
829 let Some((continuation, _)) = continuation_stack.peek_continuation_with_source_node_id()
830 else {
831 return Ok(None);
832 };
833
834 let Some(exec_node) = continuation.exec_node() else {
835 return Ok(None);
836 };
837
838 package_debug_info.unique_source_root_for_exec_node(exec_node).map_err(|_| {
839 ExecutionError::Internal(
840 "package debug source graph has ambiguous or malformed continuation roots",
841 )
842 })
843 }
844
845 #[inline(always)]
847 fn resume_context_from_flow(
848 flow: ControlFlow<BreakReason<Arc<MastForest>>, StackOutputs>,
849 mut continuation_stack: ContinuationStack<Arc<MastForest>>,
850 mut current_forest: Arc<MastForest>,
851 kernel: KernelDescriptor,
852 mut package_debug_info: Option<Arc<PackageDebugInfo>>,
853 mut inline_call_contexts: Vec<Option<SourceInlineCallContext>>,
854 ) -> Result<Option<ResumeContext>, ExecutionError> {
855 match flow {
856 ControlFlow::Continue(_) => Ok(None),
857 ControlFlow::Break(break_reason) => match break_reason {
858 BreakReason::Err(err) => Err(err),
859 BreakReason::Stopped(maybe_continuation) => {
860 if let Some((continuation, source_node_id)) = maybe_continuation {
861 continuation_stack.push_with_source_node_id(continuation, source_node_id);
862 }
863
864 while matches!(
865 continuation_stack.peek_continuation(),
866 Some(Continuation::EnterForest { .. })
867 ) {
868 let Some((
869 Continuation::EnterForest {
870 forest,
871 package_debug_info: restored_debug_info,
872 inline_context_depth,
873 },
874 _,
875 )) = continuation_stack.pop_continuation_with_source_node_id()
876 else {
877 unreachable!("peeked continuation must still be EnterForest")
878 };
879 current_forest = forest;
880 package_debug_info = restored_debug_info;
881 inline_call_contexts.truncate(inline_context_depth);
882 }
883
884 Ok(Some(ResumeContext {
885 current_forest,
886 continuation_stack,
887 kernel,
888 package_debug_info,
889 inline_call_contexts,
890 }))
891 },
892 },
893 }
894 }
895
896 #[inline(always)]
898 fn current_stack_outputs(&self) -> StackOutputs {
899 StackOutputs::new(
900 &self.stack[self.stack_bot_idx..self.stack_top_idx]
901 .iter()
902 .rev()
903 .copied()
904 .collect::<Vec<_>>(),
905 )
906 .unwrap()
907 }
908
909 fn execute_impl<S, T>(
915 &mut self,
916 continuation_stack: &mut ContinuationStack<Arc<MastForest>>,
917 current_forest: &mut Arc<MastForest>,
918 kernel: &KernelDescriptor,
919 host: &mut impl SyncHost,
920 tracer: &mut T,
921 stopper: &S,
922 package_debug_info: &mut Option<Arc<PackageDebugInfo>>,
923 inline_call_contexts: &mut Vec<Option<SourceInlineCallContext>>,
924 ) -> ControlFlow<BreakReason<Arc<MastForest>>, StackOutputs>
925 where
926 S: Stopper<Processor = Self, Forest = Arc<MastForest>>,
927 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
928 {
929 while let ControlFlow::Break(internal_break_reason) = execute_impl(
930 self,
931 continuation_stack,
932 current_forest,
933 kernel,
934 host,
935 tracer,
936 stopper,
937 package_debug_info,
938 inline_call_contexts,
939 ) {
940 let current_package_debug_info = package_debug_info.as_deref();
941 let source_aware_execution =
942 current_package_debug_info.is_some() || continuation_stack.tracks_source_nodes();
943 match internal_break_reason {
944 InternalBreakReason::User(break_reason) => return ControlFlow::Break(break_reason),
945 InternalBreakReason::Emit { op_idx, continuation, source_node_id } => {
946 self.op_emit_sync(host, op_idx, current_package_debug_info, source_node_id)?;
947
948 finish_emit_op_execution(
949 continuation,
950 source_node_id,
951 self,
952 continuation_stack,
953 current_forest,
954 tracer,
955 stopper,
956 )?;
957 },
958 InternalBreakReason::LoadMastForestFromDyn { callee_hash, source_node_id } => {
959 let (root_id, new_forest, new_package_debug_info, new_source_node_id) =
960 match self.load_mast_forest_sync(
961 callee_hash,
962 host,
963 current_package_debug_info,
964 source_node_id,
965 source_aware_execution,
966 ) {
967 Ok(result) => result,
968 Err(err) => return ControlFlow::Break(BreakReason::Err(err)),
969 };
970
971 finish_load_mast_forest_from_dyn_start(
972 root_id,
973 new_forest,
974 new_package_debug_info,
975 new_source_node_id,
976 self,
977 current_forest,
978 package_debug_info,
979 inline_call_contexts.as_slice(),
980 continuation_stack,
981 tracer,
982 stopper,
983 )?;
984 },
985 InternalBreakReason::LoadMastForestFromExternal {
986 external_node_id,
987 procedure_hash,
988 source_node_id,
989 } => {
990 let inline_call_context = package_debug_info.clone().and_then(|debug_info| {
991 SourceInlineCallContext::for_source_boundary(debug_info, source_node_id)
992 });
993 let (root_id, new_forest, new_package_debug_info, new_source_node_id) =
994 match self.load_mast_forest_sync(
995 procedure_hash,
996 host,
997 current_package_debug_info,
998 source_node_id,
999 source_aware_execution,
1000 ) {
1001 Ok(result) => result,
1002 Err(err) => {
1003 let maybe_enriched_err = maybe_use_caller_error_context(
1004 err,
1005 continuation_stack,
1006 current_package_debug_info,
1007 host,
1008 );
1009 return ControlFlow::Break(BreakReason::Err(maybe_enriched_err));
1010 },
1011 };
1012
1013 finish_load_mast_forest_from_external(
1014 root_id,
1015 new_forest,
1016 new_package_debug_info,
1017 new_source_node_id,
1018 inline_call_context,
1019 external_node_id,
1020 current_forest,
1021 package_debug_info,
1022 inline_call_contexts,
1023 continuation_stack,
1024 tracer,
1025 )?;
1026 },
1027 }
1028 }
1029
1030 match StackOutputs::new(
1031 &self.stack[self.stack_bot_idx..self.stack_top_idx]
1032 .iter()
1033 .rev()
1034 .copied()
1035 .collect::<Vec<_>>(),
1036 ) {
1037 Ok(stack_outputs) => ControlFlow::Continue(stack_outputs),
1038 Err(_) => ControlFlow::Break(BreakReason::Err(ExecutionError::OutputStackOverflow(
1039 self.stack_top_idx - self.stack_bot_idx - MIN_STACK_DEPTH,
1040 ))),
1041 }
1042 }
1043
1044 async fn execute_impl_async<S, T>(
1045 &mut self,
1046 continuation_stack: &mut ContinuationStack<Arc<MastForest>>,
1047 current_forest: &mut Arc<MastForest>,
1048 kernel: &KernelDescriptor,
1049 host: &mut impl Host,
1050 tracer: &mut T,
1051 stopper: &S,
1052 package_debug_info: &mut Option<Arc<PackageDebugInfo>>,
1053 inline_call_contexts: &mut Vec<Option<SourceInlineCallContext>>,
1054 ) -> ControlFlow<BreakReason<Arc<MastForest>>, StackOutputs>
1055 where
1056 S: Stopper<Processor = Self, Forest = Arc<MastForest>>,
1057 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
1058 {
1059 while let ControlFlow::Break(internal_break_reason) = execute_impl(
1060 self,
1061 continuation_stack,
1062 current_forest,
1063 kernel,
1064 host,
1065 tracer,
1066 stopper,
1067 package_debug_info,
1068 inline_call_contexts,
1069 ) {
1070 let current_package_debug_info = package_debug_info.as_deref();
1071 let source_aware_execution =
1072 current_package_debug_info.is_some() || continuation_stack.tracks_source_nodes();
1073 match internal_break_reason {
1074 InternalBreakReason::User(break_reason) => return ControlFlow::Break(break_reason),
1075 InternalBreakReason::Emit { op_idx, continuation, source_node_id } => {
1076 self.op_emit(host, op_idx, current_package_debug_info, source_node_id).await?;
1077
1078 finish_emit_op_execution(
1079 continuation,
1080 source_node_id,
1081 self,
1082 continuation_stack,
1083 current_forest,
1084 tracer,
1085 stopper,
1086 )?;
1087 },
1088 InternalBreakReason::LoadMastForestFromDyn { callee_hash, source_node_id } => {
1089 let (root_id, new_forest, new_package_debug_info, new_source_node_id) =
1090 match self
1091 .load_mast_forest(
1092 callee_hash,
1093 host,
1094 current_package_debug_info,
1095 source_node_id,
1096 source_aware_execution,
1097 )
1098 .await
1099 {
1100 Ok(result) => result,
1101 Err(err) => return ControlFlow::Break(BreakReason::Err(err)),
1102 };
1103
1104 finish_load_mast_forest_from_dyn_start(
1105 root_id,
1106 new_forest,
1107 new_package_debug_info,
1108 new_source_node_id,
1109 self,
1110 current_forest,
1111 package_debug_info,
1112 inline_call_contexts.as_slice(),
1113 continuation_stack,
1114 tracer,
1115 stopper,
1116 )?;
1117 },
1118 InternalBreakReason::LoadMastForestFromExternal {
1119 external_node_id,
1120 procedure_hash,
1121 source_node_id,
1122 } => {
1123 let inline_call_context = package_debug_info.clone().and_then(|debug_info| {
1124 SourceInlineCallContext::for_source_boundary(debug_info, source_node_id)
1125 });
1126 let (root_id, new_forest, new_package_debug_info, new_source_node_id) =
1127 match self
1128 .load_mast_forest(
1129 procedure_hash,
1130 host,
1131 current_package_debug_info,
1132 source_node_id,
1133 source_aware_execution,
1134 )
1135 .await
1136 {
1137 Ok(result) => result,
1138 Err(err) => {
1139 let maybe_enriched_err = maybe_use_caller_error_context(
1140 err,
1141 continuation_stack,
1142 current_package_debug_info,
1143 host,
1144 );
1145 return ControlFlow::Break(BreakReason::Err(maybe_enriched_err));
1146 },
1147 };
1148
1149 finish_load_mast_forest_from_external(
1150 root_id,
1151 new_forest,
1152 new_package_debug_info,
1153 new_source_node_id,
1154 inline_call_context,
1155 external_node_id,
1156 current_forest,
1157 package_debug_info,
1158 inline_call_contexts,
1159 continuation_stack,
1160 tracer,
1161 )?;
1162 },
1163 }
1164 }
1165
1166 match StackOutputs::new(
1167 &self.stack[self.stack_bot_idx..self.stack_top_idx]
1168 .iter()
1169 .rev()
1170 .copied()
1171 .collect::<Vec<_>>(),
1172 ) {
1173 Ok(stack_outputs) => ControlFlow::Continue(stack_outputs),
1174 Err(_) => ControlFlow::Break(BreakReason::Err(ExecutionError::OutputStackOverflow(
1175 self.stack_top_idx - self.stack_bot_idx - MIN_STACK_DEPTH,
1176 ))),
1177 }
1178 }
1179
1180 fn load_mast_forest_sync(
1184 &mut self,
1185 node_digest: Word,
1186 host: &mut impl SyncHost,
1187 package_debug_info: Option<&PackageDebugInfo>,
1188 source_node_id: Option<DebugSourceNodeId>,
1189 source_aware_execution: bool,
1190 ) -> Result<
1191 (
1192 MastNodeId,
1193 Arc<MastForest>,
1194 Option<Arc<PackageDebugInfo>>,
1195 Option<DebugSourceNodeId>,
1196 ),
1197 ExecutionError,
1198 > {
1199 let cached = self.loaded_mast_forests.get(&node_digest).cloned();
1200 let was_cached = cached.is_some();
1201 let loaded_mast_forest = match cached {
1202 Some(mast_forest) => mast_forest,
1203 None => host.get_mast_forest(&node_digest).ok_or_else(|| {
1204 match (package_debug_info, source_node_id) {
1205 (Some(debug_info), Some(source_node_id)) => {
1206 crate::errors::procedure_not_found_with_package_source_context(
1207 node_digest,
1208 PackageSourceDebugContext::new(debug_info, source_node_id),
1209 host,
1210 )
1211 },
1212 _ => crate::errors::procedure_not_found_with_context(node_digest),
1213 }
1214 })?,
1215 };
1216 let mast_forest = loaded_mast_forest.mast_forest().clone();
1217
1218 let root_id = mast_forest.find_procedure_root(node_digest).ok_or_else(|| {
1219 let context = match (package_debug_info, source_node_id) {
1220 (Some(debug_info), Some(source_node_id)) => {
1221 Some(PackageSourceDebugContext::new(debug_info, source_node_id))
1222 },
1223 _ => None,
1224 };
1225 malformed_mast_forest_with_context(node_digest, context, host)
1226 })?;
1227
1228 if !was_cached {
1229 self.cache_loaded_mast_forest(&loaded_mast_forest);
1230 }
1231 self.merge_mast_forest_advice(&mast_forest).map_exec_err()?;
1232 let (loaded_package_debug_info, loaded_source_node_id) =
1233 Self::loaded_package_source_context(
1234 &loaded_mast_forest,
1235 root_id,
1236 source_aware_execution,
1237 )?;
1238
1239 Ok((root_id, mast_forest, loaded_package_debug_info, loaded_source_node_id))
1240 }
1241
1242 async fn load_mast_forest(
1243 &mut self,
1244 node_digest: Word,
1245 host: &mut impl Host,
1246 package_debug_info: Option<&PackageDebugInfo>,
1247 source_node_id: Option<DebugSourceNodeId>,
1248 source_aware_execution: bool,
1249 ) -> Result<
1250 (
1251 MastNodeId,
1252 Arc<MastForest>,
1253 Option<Arc<PackageDebugInfo>>,
1254 Option<DebugSourceNodeId>,
1255 ),
1256 ExecutionError,
1257 > {
1258 let cached = self.loaded_mast_forests.get(&node_digest).cloned();
1259 let was_cached = cached.is_some();
1260 let loaded_mast_forest = match cached {
1261 Some(mast_forest) => mast_forest,
1262 None => {
1263 if let Some(mast_forest) = host.get_mast_forest(&node_digest).await {
1264 mast_forest
1265 } else {
1266 return Err(match (package_debug_info, source_node_id) {
1267 (Some(debug_info), Some(source_node_id)) => {
1268 crate::errors::procedure_not_found_with_package_source_context(
1269 node_digest,
1270 PackageSourceDebugContext::new(debug_info, source_node_id),
1271 host,
1272 )
1273 },
1274 _ => crate::errors::procedure_not_found_with_context(node_digest),
1275 });
1276 }
1277 },
1278 };
1279 let mast_forest = loaded_mast_forest.mast_forest().clone();
1280
1281 let root_id = mast_forest.find_procedure_root(node_digest).ok_or_else(|| {
1282 let context = match (package_debug_info, source_node_id) {
1283 (Some(debug_info), Some(source_node_id)) => {
1284 Some(PackageSourceDebugContext::new(debug_info, source_node_id))
1285 },
1286 _ => None,
1287 };
1288 malformed_mast_forest_with_context(node_digest, context, host)
1289 })?;
1290
1291 if !was_cached {
1292 self.cache_loaded_mast_forest(&loaded_mast_forest);
1293 }
1294 self.merge_mast_forest_advice(&mast_forest).map_exec_err()?;
1295 let (loaded_package_debug_info, loaded_source_node_id) =
1296 Self::loaded_package_source_context(
1297 &loaded_mast_forest,
1298 root_id,
1299 source_aware_execution,
1300 )?;
1301
1302 Ok((root_id, mast_forest, loaded_package_debug_info, loaded_source_node_id))
1303 }
1304
1305 fn cache_loaded_mast_forest(&mut self, loaded_mast_forest: &LoadedMastForest) {
1306 for procedure_digest in loaded_mast_forest.mast_forest().local_procedure_digests() {
1307 self.loaded_mast_forests
1308 .entry(procedure_digest)
1309 .or_insert_with(|| loaded_mast_forest.clone());
1310 }
1311 }
1312
1313 fn merge_mast_forest_advice(&mut self, mast_forest: &MastForest) -> Result<(), AdviceError> {
1314 let commitment = mast_forest.commitment();
1315 if self.merged_mast_forests.contains(&commitment) {
1316 return Ok(());
1317 }
1318
1319 self.advice.extend_map(mast_forest.advice_map())?;
1320 self.merged_mast_forests.insert(commitment);
1321 Ok(())
1322 }
1323
1324 fn loaded_package_source_context(
1325 loaded_mast_forest: &LoadedMastForest,
1326 root_id: MastNodeId,
1327 source_aware_execution: bool,
1328 ) -> Result<(Option<Arc<PackageDebugInfo>>, Option<DebugSourceNodeId>), ExecutionError> {
1329 if !source_aware_execution {
1330 return Ok((None, None));
1331 }
1332
1333 let Some(package_debug_info) = loaded_mast_forest
1334 .package_debug_info()
1335 .map_err(|_| ExecutionError::Internal("loaded package debug info is malformed"))?
1336 else {
1337 return Ok((None, None));
1338 };
1339
1340 let source_node_id = match package_debug_info.unique_source_root_for_exec_node(root_id) {
1341 Ok(source_node_id) => source_node_id,
1342 Err(DebugSourceGraphLookupError::AmbiguousRoot { .. }) => None,
1343 Err(_) => {
1344 return Err(ExecutionError::Internal(
1345 "loaded package debug source graph has malformed entrypoint roots",
1346 ));
1347 },
1348 };
1349
1350 Ok((Some(package_debug_info), source_node_id))
1351 }
1352
1353 pub fn execute_by_step_sync(
1355 mut self,
1356 program: &Program,
1357 host: &mut impl SyncHost,
1358 ) -> Result<StackOutputs, ExecutionError> {
1359 let mut current_resume_ctx = self.get_initial_resume_context(program)?;
1360
1361 loop {
1362 match self.step_sync(host, current_resume_ctx)? {
1363 Some(next_resume_ctx) => {
1364 current_resume_ctx = next_resume_ctx;
1365 },
1366 None => break Ok(self.current_stack_outputs()),
1367 }
1368 }
1369 }
1370
1371 #[cfg(any(test, feature = "testing"))]
1374 pub fn execute_by_step_with_package_debug_info_sync(
1375 mut self,
1376 program: &Program,
1377 package_debug_info: &PackageDebugInfo,
1378 host: &mut impl SyncHost,
1379 ) -> Result<StackOutputs, ExecutionError> {
1380 let mut current_resume_ctx =
1381 self.source_aware_resume_context(program, package_debug_info, None)?;
1382
1383 loop {
1384 match self.step_with_package_debug_info_sync(
1385 host,
1386 current_resume_ctx,
1387 package_debug_info,
1388 )? {
1389 Some(next_resume_ctx) => {
1390 current_resume_ctx = next_resume_ctx;
1391 },
1392 None => break Ok(self.current_stack_outputs()),
1393 }
1394 }
1395 }
1396
1397 #[cfg(any(test, feature = "testing"))]
1400 pub fn execute_by_step_with_package_debug_info_at_source_node_sync(
1401 mut self,
1402 program: &Program,
1403 package_debug_info: &PackageDebugInfo,
1404 entrypoint_source_node_id: DebugSourceNodeId,
1405 host: &mut impl SyncHost,
1406 ) -> Result<StackOutputs, ExecutionError> {
1407 let mut current_resume_ctx = self.source_aware_resume_context(
1408 program,
1409 package_debug_info,
1410 Some(entrypoint_source_node_id),
1411 )?;
1412
1413 loop {
1414 match self.step_with_package_debug_info_sync(
1415 host,
1416 current_resume_ctx,
1417 package_debug_info,
1418 )? {
1419 Some(next_resume_ctx) => {
1420 current_resume_ctx = next_resume_ctx;
1421 },
1422 None => break Ok(self.current_stack_outputs()),
1423 }
1424 }
1425 }
1426
1427 #[inline(always)]
1429 pub async fn execute_by_step(
1430 mut self,
1431 program: &Program,
1432 host: &mut impl Host,
1433 ) -> Result<StackOutputs, ExecutionError> {
1434 let mut current_resume_ctx = self.get_initial_resume_context(program)?;
1435 let mut processor = self;
1436
1437 loop {
1438 match processor.step(host, current_resume_ctx).await? {
1439 Some(next_resume_ctx) => {
1440 current_resume_ctx = next_resume_ctx;
1441 },
1442 None => break Ok(processor.current_stack_outputs()),
1443 }
1444 }
1445 }
1446
1447 #[cfg(any(test, feature = "testing"))]
1449 #[inline(always)]
1450 pub async fn execute_by_step_with_package_debug_info(
1451 mut self,
1452 program: &Program,
1453 package_debug_info: &PackageDebugInfo,
1454 host: &mut impl Host,
1455 ) -> Result<StackOutputs, ExecutionError> {
1456 let mut current_resume_ctx =
1457 self.source_aware_resume_context(program, package_debug_info, None)?;
1458 let mut processor = self;
1459
1460 loop {
1461 match processor
1462 .step_with_package_debug_info(host, current_resume_ctx, package_debug_info)
1463 .await?
1464 {
1465 Some(next_resume_ctx) => {
1466 current_resume_ctx = next_resume_ctx;
1467 },
1468 None => break Ok(processor.current_stack_outputs()),
1469 }
1470 }
1471 }
1472
1473 #[cfg(any(test, feature = "testing"))]
1476 #[inline(always)]
1477 pub async fn execute_by_step_with_package_debug_info_at_source_node(
1478 mut self,
1479 program: &Program,
1480 package_debug_info: &PackageDebugInfo,
1481 entrypoint_source_node_id: DebugSourceNodeId,
1482 host: &mut impl Host,
1483 ) -> Result<StackOutputs, ExecutionError> {
1484 let mut current_resume_ctx = self.source_aware_resume_context(
1485 program,
1486 package_debug_info,
1487 Some(entrypoint_source_node_id),
1488 )?;
1489 let mut processor = self;
1490
1491 loop {
1492 match processor
1493 .step_with_package_debug_info(host, current_resume_ctx, package_debug_info)
1494 .await?
1495 {
1496 Some(next_resume_ctx) => {
1497 current_resume_ctx = next_resume_ctx;
1498 },
1499 None => break Ok(processor.current_stack_outputs()),
1500 }
1501 }
1502 }
1503
1504 #[cfg(any(test, feature = "testing"))]
1508 pub fn execute_mut_sync(
1509 &mut self,
1510 program: &Program,
1511 host: &mut impl SyncHost,
1512 ) -> Result<StackOutputs, ExecutionError> {
1513 let mut continuation_stack = ContinuationStack::new(program);
1514 let mut current_forest = program.mast_forest().clone();
1515 let mut package_debug_info = None;
1516 let mut inline_call_contexts = Vec::new();
1517
1518 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
1519
1520 let flow = self.execute_impl(
1521 &mut continuation_stack,
1522 &mut current_forest,
1523 program.kernel(),
1524 host,
1525 &mut NoopTracer,
1526 &NeverStopper,
1527 &mut package_debug_info,
1528 &mut inline_call_contexts,
1529 );
1530 Self::stack_result_from_flow(flow)
1531 }
1532
1533 #[cfg(any(test, feature = "testing"))]
1535 #[inline(always)]
1536 pub async fn execute_mut(
1537 &mut self,
1538 program: &Program,
1539 host: &mut impl Host,
1540 ) -> Result<StackOutputs, ExecutionError> {
1541 let mut continuation_stack = ContinuationStack::new(program);
1542 let mut current_forest = program.mast_forest().clone();
1543 let mut package_debug_info = None;
1544 let mut inline_call_contexts = Vec::new();
1545
1546 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
1547
1548 let flow = self
1549 .execute_impl_async(
1550 &mut continuation_stack,
1551 &mut current_forest,
1552 program.kernel(),
1553 host,
1554 &mut NoopTracer,
1555 &NeverStopper,
1556 &mut package_debug_info,
1557 &mut inline_call_contexts,
1558 )
1559 .await;
1560 Self::stack_result_from_flow(flow)
1561 }
1562}
1563
1564#[cfg(all(test, feature = "std"))]
1565mod tests {
1566 use super::FastProcessor;
1567
1568 #[test]
1569 fn sole_rayon_worker_requires_buffered_trace_building() {
1570 rayon::ThreadPoolBuilder::new()
1571 .num_threads(1)
1572 .build()
1573 .unwrap()
1574 .install(|| assert!(FastProcessor::rayon_has_no_parallel_worker()));
1575 }
1576}