1use alloc::{sync::Arc, vec::Vec};
2use core::ops::ControlFlow;
3
4use miden_core::{
5 Word,
6 mast::{MastForest, MastNodeId},
7 program::{KernelDescriptor, MIN_STACK_DEPTH, Program, StackInputs, StackOutputs},
8};
9use miden_mast_package::debug_info::{
10 DebugSourceGraphLookupError, DebugSourceNodeId, PackageDebugInfo,
11};
12use tracing::instrument;
13
14use super::{
15 FastProcessor, NoopTracer,
16 external::maybe_use_caller_error_context,
17 step::{BreakReason, NeverStopper, ResumeContext, StepStopper},
18};
19#[cfg(feature = "std")]
20use crate::PrecompileWitness;
21use crate::{
22 ExecutionError, ExecutionOutput, ExecutionWitness, Host, LoadedMastForest, Stopper, SyncHost,
23 advice::AdviceError,
24 continuation_stack::{Continuation, ContinuationStack, SourceInlineCallContext},
25 errors::{
26 MapExecErr, MapExecErrNoCtx, PackageSourceDebugContext, malformed_mast_forest_with_context,
27 },
28 execution::{
29 InternalBreakReason, execute_impl, finish_emit_op_execution,
30 finish_load_mast_forest_from_dyn_start, finish_load_mast_forest_from_external,
31 },
32 trace::execution_tracer::ExecutionTracer,
33 tracer::Tracer,
34};
35
36impl FastProcessor {
37 pub fn execute_sync(
42 self,
43 program: &Program,
44 host: &mut impl SyncHost,
45 ) -> Result<ExecutionOutput, ExecutionError> {
46 self.execute_with_tracer_sync(program, host, &mut NoopTracer)
47 }
48
49 pub fn execute_with_package_debug_info_sync(
56 self,
57 program: &Program,
58 package_debug_info: &PackageDebugInfo,
59 host: &mut impl SyncHost,
60 ) -> Result<ExecutionOutput, ExecutionError> {
61 self.execute_with_package_debug_info_and_tracer_sync(
62 program,
63 package_debug_info,
64 None,
65 host,
66 &mut NoopTracer,
67 )
68 }
69
70 pub fn execute_with_package_debug_info_at_source_node_sync(
77 self,
78 program: &Program,
79 package_debug_info: &PackageDebugInfo,
80 entrypoint_source_node_id: DebugSourceNodeId,
81 host: &mut impl SyncHost,
82 ) -> Result<ExecutionOutput, ExecutionError> {
83 self.execute_with_package_debug_info_and_tracer_sync(
84 program,
85 package_debug_info,
86 Some(entrypoint_source_node_id),
87 host,
88 &mut NoopTracer,
89 )
90 }
91
92 #[inline(always)]
94 pub async fn execute(
95 self,
96 program: &Program,
97 host: &mut impl Host,
98 ) -> Result<ExecutionOutput, ExecutionError> {
99 self.execute_with_tracer(program, host, &mut NoopTracer).await
100 }
101
102 #[inline(always)]
107 pub async fn execute_with_package_debug_info(
108 self,
109 program: &Program,
110 package_debug_info: &PackageDebugInfo,
111 host: &mut impl Host,
112 ) -> Result<ExecutionOutput, ExecutionError> {
113 self.execute_with_package_debug_info_and_tracer(
114 program,
115 package_debug_info,
116 None,
117 host,
118 &mut NoopTracer,
119 )
120 .await
121 }
122
123 #[inline(always)]
125 pub async fn execute_with_package_debug_info_at_source_node(
126 self,
127 program: &Program,
128 package_debug_info: &PackageDebugInfo,
129 entrypoint_source_node_id: DebugSourceNodeId,
130 host: &mut impl Host,
131 ) -> Result<ExecutionOutput, ExecutionError> {
132 self.execute_with_package_debug_info_and_tracer(
133 program,
134 package_debug_info,
135 Some(entrypoint_source_node_id),
136 host,
137 &mut NoopTracer,
138 )
139 .await
140 }
141
142 #[cfg(feature = "std")]
157 #[instrument(name = "execute_and_build_trace_sync", skip_all)]
158 pub fn execute_and_build_trace_sync(
159 self,
160 program: &Program,
161 host: &mut impl SyncHost,
162 max_prover_memory_bytes: u64,
163 ) -> Result<(crate::trace::VmTrace, Option<PrecompileWitness>), ExecutionError> {
164 use miden_air::{config, memory};
165
166 use crate::trace::{
167 MAX_TRACE_LEN, build_hasher_chiplet, build_trace_with_budget,
168 build_trace_with_prebuilt_hasher,
169 };
170
171 if Self::rayon_has_no_parallel_worker() {
172 let (vm_witness, precompiles_witness) =
173 self.execute_for_proving_sync(program, host)?.into_parts();
174 let trace = build_trace_with_budget(vm_witness, max_prover_memory_bytes)?;
175 return Ok((trace, precompiles_witness));
176 }
177
178 let stack_inputs = self.initial_stack_inputs();
179 let max_trace_len = MAX_TRACE_LEN.min(memory::max_any_height_for_budget(
180 max_prover_memory_bytes,
181 &config::pcs_params(),
182 ));
183 let (sender, receiver) = std::sync::mpsc::channel();
184 let mut tracer = ExecutionTracer::new_with_streamed_hasher(
185 self.options.core_trace_fragment_size(),
186 self.options.max_stack_depth(),
187 sender,
188 );
189
190 let mut hasher = None;
191 let hasher_slot = &mut hasher;
192 let execution_output = rayon::in_place_scope(move |scope| {
195 let span = tracing::Span::current();
198 scope.spawn(move |_| {
199 let _span = span.entered();
200 let result = build_hasher_chiplet(
201 receiver.into_iter().map(Ok),
202 max_trace_len,
203 max_trace_len,
204 );
205 *hasher_slot = Some(result);
206 });
207
208 let execution_output = self.execute_with_tracer_sync(program, host, &mut tracer);
209
210 match execution_output {
211 Ok(output) => {
212 let (mut vm_witness, precompiles_witness) =
213 Self::execution_witness_from_parts(program, stack_inputs, output, tracer)
214 .into_parts();
215 drop(vm_witness.take_hasher_replay());
217 Ok((vm_witness, precompiles_witness))
218 },
219 Err(err) => {
220 drop(tracer);
224 Err(err)
225 },
226 }
227 });
228
229 let hasher = hasher.expect("hasher builder did not run");
230 let (vm_witness, precompiles_witness) = match execution_output {
231 Ok(output) => output,
232 Err(err) => {
233 if let Err(builder_err) = hasher {
234 tracing::debug!(%builder_err, "hasher builder also failed");
235 }
236 return Err(err);
237 },
238 };
239
240 let trace = build_trace_with_prebuilt_hasher(vm_witness, hasher?, max_prover_memory_bytes)?;
241 Ok((trace, precompiles_witness))
242 }
243
244 #[cfg(feature = "std")]
245 fn rayon_has_no_parallel_worker() -> bool {
246 rayon::current_num_threads() == 1 && rayon::current_thread_index().is_some()
248 }
249
250 #[instrument(name = "execute_for_proving_sync", skip_all)]
272 pub fn execute_for_proving_sync(
273 self,
274 program: &Program,
275 host: &mut impl SyncHost,
276 ) -> Result<ExecutionWitness, ExecutionError> {
277 let stack_inputs = self.initial_stack_inputs();
278 let mut tracer = ExecutionTracer::new(
279 self.options.core_trace_fragment_size(),
280 self.options.max_stack_depth(),
281 );
282 let execution_output = self.execute_with_tracer_sync(program, host, &mut tracer)?;
283 Ok(Self::execution_witness_from_parts(
284 program,
285 stack_inputs,
286 execution_output,
287 tracer,
288 ))
289 }
290
291 #[instrument(name = "execute_for_proving_with_package_debug_info_sync", skip_all)]
294 pub fn execute_for_proving_with_package_debug_info_sync(
295 self,
296 program: &Program,
297 package_debug_info: &PackageDebugInfo,
298 host: &mut impl SyncHost,
299 ) -> Result<ExecutionWitness, ExecutionError> {
300 let stack_inputs = self.initial_stack_inputs();
301 let mut tracer = ExecutionTracer::new(
302 self.options.core_trace_fragment_size(),
303 self.options.max_stack_depth(),
304 );
305 let execution_output = self.execute_with_package_debug_info_and_tracer_sync(
306 program,
307 package_debug_info,
308 None,
309 host,
310 &mut tracer,
311 )?;
312 Ok(Self::execution_witness_from_parts(
313 program,
314 stack_inputs,
315 execution_output,
316 tracer,
317 ))
318 }
319
320 #[instrument(
323 name = "execute_for_proving_with_package_debug_info_at_source_node_sync",
324 skip_all
325 )]
326 pub fn execute_for_proving_with_package_debug_info_at_source_node_sync(
327 self,
328 program: &Program,
329 package_debug_info: &PackageDebugInfo,
330 entrypoint_source_node_id: DebugSourceNodeId,
331 host: &mut impl SyncHost,
332 ) -> Result<ExecutionWitness, ExecutionError> {
333 let stack_inputs = self.initial_stack_inputs();
334 let mut tracer = ExecutionTracer::new(
335 self.options.core_trace_fragment_size(),
336 self.options.max_stack_depth(),
337 );
338 let execution_output = self.execute_with_package_debug_info_and_tracer_sync(
339 program,
340 package_debug_info,
341 Some(entrypoint_source_node_id),
342 host,
343 &mut tracer,
344 )?;
345 Ok(Self::execution_witness_from_parts(
346 program,
347 stack_inputs,
348 execution_output,
349 tracer,
350 ))
351 }
352
353 #[inline(always)]
355 #[instrument(name = "execute_for_proving", skip_all)]
356 pub async fn execute_for_proving(
357 self,
358 program: &Program,
359 host: &mut impl Host,
360 ) -> Result<ExecutionWitness, ExecutionError> {
361 let stack_inputs = self.initial_stack_inputs();
362 let mut tracer = ExecutionTracer::new(
363 self.options.core_trace_fragment_size(),
364 self.options.max_stack_depth(),
365 );
366 let execution_output = self.execute_with_tracer(program, host, &mut tracer).await?;
367 Ok(Self::execution_witness_from_parts(
368 program,
369 stack_inputs,
370 execution_output,
371 tracer,
372 ))
373 }
374
375 #[cfg(any(test, feature = "testing"))]
377 #[inline(always)]
378 #[instrument(name = "execute_for_proving_with_package_debug_info", skip_all)]
379 pub async fn execute_for_proving_with_package_debug_info(
380 self,
381 program: &Program,
382 package_debug_info: &PackageDebugInfo,
383 host: &mut impl Host,
384 ) -> Result<ExecutionWitness, ExecutionError> {
385 let stack_inputs = self.initial_stack_inputs();
386 let mut tracer = ExecutionTracer::new(
387 self.options.core_trace_fragment_size(),
388 self.options.max_stack_depth(),
389 );
390 let execution_output = self
391 .execute_with_package_debug_info_and_tracer(
392 program,
393 package_debug_info,
394 None,
395 host,
396 &mut tracer,
397 )
398 .await?;
399 Ok(Self::execution_witness_from_parts(
400 program,
401 stack_inputs,
402 execution_output,
403 tracer,
404 ))
405 }
406
407 #[cfg(any(test, feature = "testing"))]
410 #[inline(always)]
411 #[instrument(name = "execute_for_proving_with_package_debug_info_at_source_node", skip_all)]
412 pub async fn execute_for_proving_with_package_debug_info_at_source_node(
413 self,
414 program: &Program,
415 package_debug_info: &PackageDebugInfo,
416 entrypoint_source_node_id: DebugSourceNodeId,
417 host: &mut impl Host,
418 ) -> Result<ExecutionWitness, ExecutionError> {
419 let stack_inputs = self.initial_stack_inputs();
420 let mut tracer = ExecutionTracer::new(
421 self.options.core_trace_fragment_size(),
422 self.options.max_stack_depth(),
423 );
424 let execution_output = self
425 .execute_with_package_debug_info_and_tracer(
426 program,
427 package_debug_info,
428 Some(entrypoint_source_node_id),
429 host,
430 &mut tracer,
431 )
432 .await?;
433 Ok(Self::execution_witness_from_parts(
434 program,
435 stack_inputs,
436 execution_output,
437 tracer,
438 ))
439 }
440
441 pub async fn execute_with_tracer<T>(
443 mut self,
444 program: &Program,
445 host: &mut impl Host,
446 tracer: &mut T,
447 ) -> Result<ExecutionOutput, ExecutionError>
448 where
449 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
450 {
451 let mut continuation_stack = ContinuationStack::new(program);
452 let mut current_forest = program.mast_forest().clone();
453 let mut package_debug_info = None;
454 let mut inline_call_contexts = Vec::new();
455
456 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
457 let flow = self
458 .execute_impl_async(
459 &mut continuation_stack,
460 &mut current_forest,
461 program.kernel(),
462 host,
463 tracer,
464 &NeverStopper,
465 &mut package_debug_info,
466 &mut inline_call_contexts,
467 )
468 .await;
469 Self::execution_result_from_flow(flow, self)
470 }
471
472 async fn execute_with_package_debug_info_and_tracer<T>(
475 mut self,
476 program: &Program,
477 package_debug_info: &PackageDebugInfo,
478 entrypoint_source_node_id: Option<DebugSourceNodeId>,
479 host: &mut impl Host,
480 tracer: &mut T,
481 ) -> Result<ExecutionOutput, ExecutionError>
482 where
483 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
484 {
485 let mut continuation_stack = Self::source_aware_continuation_stack(
486 program,
487 package_debug_info,
488 entrypoint_source_node_id,
489 )?;
490 let mut current_forest = program.mast_forest().clone();
491 let mut package_debug_info = Some(Arc::new(package_debug_info.clone()));
492 let mut inline_call_contexts = Vec::new();
493
494 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
495 let flow = self
496 .execute_impl_async(
497 &mut continuation_stack,
498 &mut current_forest,
499 program.kernel(),
500 host,
501 tracer,
502 &NeverStopper,
503 &mut package_debug_info,
504 &mut inline_call_contexts,
505 )
506 .await;
507 Self::execution_result_from_flow(flow, self)
508 }
509
510 pub fn execute_with_tracer_sync<T>(
512 mut self,
513 program: &Program,
514 host: &mut impl SyncHost,
515 tracer: &mut T,
516 ) -> Result<ExecutionOutput, ExecutionError>
517 where
518 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
519 {
520 let mut continuation_stack = ContinuationStack::new(program);
521 let mut current_forest = program.mast_forest().clone();
522 let mut package_debug_info = None;
523 let mut inline_call_contexts = Vec::new();
524
525 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
526 let flow = self.execute_impl(
527 &mut continuation_stack,
528 &mut current_forest,
529 program.kernel(),
530 host,
531 tracer,
532 &NeverStopper,
533 &mut package_debug_info,
534 &mut inline_call_contexts,
535 );
536 Self::execution_result_from_flow(flow, self)
537 }
538
539 fn execute_with_package_debug_info_and_tracer_sync<T>(
542 mut self,
543 program: &Program,
544 package_debug_info: &PackageDebugInfo,
545 entrypoint_source_node_id: Option<DebugSourceNodeId>,
546 host: &mut impl SyncHost,
547 tracer: &mut T,
548 ) -> Result<ExecutionOutput, ExecutionError>
549 where
550 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
551 {
552 let mut continuation_stack = Self::source_aware_continuation_stack(
553 program,
554 package_debug_info,
555 entrypoint_source_node_id,
556 )?;
557 let mut current_forest = program.mast_forest().clone();
558 let mut package_debug_info = Some(Arc::new(package_debug_info.clone()));
559 let mut inline_call_contexts = Vec::new();
560
561 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
562 let flow = self.execute_impl(
563 &mut continuation_stack,
564 &mut current_forest,
565 program.kernel(),
566 host,
567 tracer,
568 &NeverStopper,
569 &mut package_debug_info,
570 &mut inline_call_contexts,
571 );
572 Self::execution_result_from_flow(flow, self)
573 }
574
575 pub fn step_sync(
577 &mut self,
578 host: &mut impl SyncHost,
579 resume_ctx: ResumeContext,
580 ) -> Result<Option<ResumeContext>, ExecutionError> {
581 let ResumeContext {
582 mut current_forest,
583 mut continuation_stack,
584 kernel,
585 mut package_debug_info,
586 mut inline_call_contexts,
587 } = resume_ctx;
588
589 let flow = self.execute_impl(
590 &mut continuation_stack,
591 &mut current_forest,
592 &kernel,
593 host,
594 &mut NoopTracer,
595 &StepStopper,
596 &mut package_debug_info,
597 &mut inline_call_contexts,
598 );
599 Self::resume_context_from_flow(
600 flow,
601 continuation_stack,
602 current_forest,
603 kernel,
604 package_debug_info,
605 inline_call_contexts,
606 )
607 }
608
609 pub fn step_with_package_debug_info_sync(
611 &mut self,
612 host: &mut impl SyncHost,
613 resume_ctx: ResumeContext,
614 package_debug_info: &PackageDebugInfo,
615 ) -> Result<Option<ResumeContext>, ExecutionError> {
616 let ResumeContext {
617 mut current_forest,
618 mut continuation_stack,
619 kernel,
620 package_debug_info: mut active_package_debug_info,
621 mut inline_call_contexts,
622 } = resume_ctx;
623 Self::ensure_source_aware_step_context(
624 &mut continuation_stack,
625 &mut active_package_debug_info,
626 package_debug_info,
627 )?;
628
629 let flow = self.execute_impl(
630 &mut continuation_stack,
631 &mut current_forest,
632 &kernel,
633 host,
634 &mut NoopTracer,
635 &StepStopper,
636 &mut active_package_debug_info,
637 &mut inline_call_contexts,
638 );
639 Self::resume_context_from_flow(
640 flow,
641 continuation_stack,
642 current_forest,
643 kernel,
644 active_package_debug_info,
645 inline_call_contexts,
646 )
647 }
648
649 #[inline(always)]
651 pub async fn step(
652 &mut self,
653 host: &mut impl Host,
654 resume_ctx: ResumeContext,
655 ) -> Result<Option<ResumeContext>, ExecutionError> {
656 let ResumeContext {
657 mut current_forest,
658 mut continuation_stack,
659 kernel,
660 mut package_debug_info,
661 mut inline_call_contexts,
662 } = resume_ctx;
663
664 let flow = self
665 .execute_impl_async(
666 &mut continuation_stack,
667 &mut current_forest,
668 &kernel,
669 host,
670 &mut NoopTracer,
671 &StepStopper,
672 &mut package_debug_info,
673 &mut inline_call_contexts,
674 )
675 .await;
676 Self::resume_context_from_flow(
677 flow,
678 continuation_stack,
679 current_forest,
680 kernel,
681 package_debug_info,
682 inline_call_contexts,
683 )
684 }
685
686 #[inline(always)]
688 pub async fn step_with_package_debug_info(
689 &mut self,
690 host: &mut impl Host,
691 resume_ctx: ResumeContext,
692 package_debug_info: &PackageDebugInfo,
693 ) -> Result<Option<ResumeContext>, ExecutionError> {
694 let ResumeContext {
695 mut current_forest,
696 mut continuation_stack,
697 kernel,
698 package_debug_info: mut active_package_debug_info,
699 mut inline_call_contexts,
700 } = resume_ctx;
701 Self::ensure_source_aware_step_context(
702 &mut continuation_stack,
703 &mut active_package_debug_info,
704 package_debug_info,
705 )?;
706
707 let flow = self
708 .execute_impl_async(
709 &mut continuation_stack,
710 &mut current_forest,
711 &kernel,
712 host,
713 &mut NoopTracer,
714 &StepStopper,
715 &mut active_package_debug_info,
716 &mut inline_call_contexts,
717 )
718 .await;
719 Self::resume_context_from_flow(
720 flow,
721 continuation_stack,
722 current_forest,
723 kernel,
724 active_package_debug_info,
725 inline_call_contexts,
726 )
727 }
728
729 #[inline(always)]
731 fn execution_witness_from_parts(
732 program: &Program,
733 stack_inputs: StackInputs,
734 execution_output: ExecutionOutput,
735 tracer: ExecutionTracer,
736 ) -> ExecutionWitness {
737 ExecutionWitness::from_execution(
738 program.to_info(),
739 stack_inputs,
740 execution_output,
741 tracer.into_trace_replay(),
742 )
743 }
744
745 #[inline(always)]
747 fn initial_stack_inputs(&self) -> StackInputs {
748 core::array::from_fn(|idx| self.stack_get(idx)).into()
749 }
750
751 pub(super) fn source_aware_continuation_stack(
752 program: &Program,
753 package_debug_info: &PackageDebugInfo,
754 entrypoint_source_node_id: Option<DebugSourceNodeId>,
755 ) -> Result<ContinuationStack<Arc<MastForest>>, ExecutionError> {
756 if let Some(source_node_id) = entrypoint_source_node_id {
757 let Some(source_node) = package_debug_info.source_node(source_node_id) else {
758 return Err(ExecutionError::Internal(
759 "package debug source graph is missing the entrypoint source node",
760 ));
761 };
762 if source_node.exec_node != program.entrypoint() {
763 return Err(ExecutionError::Internal(
764 "package debug entrypoint source node does not match the program entrypoint",
765 ));
766 }
767
768 return Ok(ContinuationStack::new_with_source_node_id(program, source_node_id));
769 }
770
771 let Some(source_node_id) = package_debug_info
772 .unique_source_root_for_exec_node(program.entrypoint())
773 .map_err(|_| {
774 ExecutionError::Internal(
775 "package debug source graph has ambiguous or malformed entrypoint roots",
776 )
777 })?
778 else {
779 return Ok(ContinuationStack::new_with_optional_source_node_id(program, None));
780 };
781
782 Ok(ContinuationStack::new_with_source_node_id(program, source_node_id))
783 }
784
785 #[cfg(any(test, feature = "testing"))]
786 fn source_aware_resume_context(
787 &mut self,
788 program: &Program,
789 package_debug_info: &PackageDebugInfo,
790 entrypoint_source_node_id: Option<DebugSourceNodeId>,
791 ) -> Result<ResumeContext, ExecutionError> {
792 self.advice
793 .extend_map(program.mast_forest().advice_map())
794 .map_exec_err_no_ctx()?;
795
796 Ok(ResumeContext {
797 current_forest: program.mast_forest().clone(),
798 continuation_stack: Self::source_aware_continuation_stack(
799 program,
800 package_debug_info,
801 entrypoint_source_node_id,
802 )?,
803 kernel: program.kernel().clone(),
804 package_debug_info: Some(Arc::new(package_debug_info.clone())),
805 inline_call_contexts: Vec::new(),
806 })
807 }
808
809 fn ensure_source_aware_step_context(
810 continuation_stack: &mut ContinuationStack<Arc<MastForest>>,
811 package_debug_info: &mut Option<Arc<PackageDebugInfo>>,
812 supplied_package_debug_info: &PackageDebugInfo,
813 ) -> Result<(), ExecutionError> {
814 if package_debug_info.is_none() {
815 *package_debug_info = Some(Arc::new(supplied_package_debug_info.clone()));
816 }
817
818 if !continuation_stack.tracks_source_nodes() {
819 let source_node_id = Self::source_root_for_next_continuation(
820 continuation_stack,
821 package_debug_info.as_deref().expect("package debug info was just initialized"),
822 )?;
823 continuation_stack.start_tracking_source_nodes(source_node_id);
824 }
825
826 Ok(())
827 }
828
829 fn source_root_for_next_continuation(
830 continuation_stack: &ContinuationStack<Arc<MastForest>>,
831 package_debug_info: &PackageDebugInfo,
832 ) -> Result<Option<DebugSourceNodeId>, ExecutionError> {
833 let Some((continuation, _)) = continuation_stack.peek_continuation_with_source_node_id()
834 else {
835 return Ok(None);
836 };
837
838 let Some(exec_node) = continuation.exec_node() else {
839 return Ok(None);
840 };
841
842 package_debug_info.unique_source_root_for_exec_node(exec_node).map_err(|_| {
843 ExecutionError::Internal(
844 "package debug source graph has ambiguous or malformed continuation roots",
845 )
846 })
847 }
848
849 #[inline(always)]
851 fn resume_context_from_flow(
852 flow: ControlFlow<BreakReason<Arc<MastForest>>, StackOutputs>,
853 mut continuation_stack: ContinuationStack<Arc<MastForest>>,
854 mut current_forest: Arc<MastForest>,
855 kernel: KernelDescriptor,
856 mut package_debug_info: Option<Arc<PackageDebugInfo>>,
857 mut inline_call_contexts: Vec<Option<SourceInlineCallContext>>,
858 ) -> Result<Option<ResumeContext>, ExecutionError> {
859 match flow {
860 ControlFlow::Continue(_) => Ok(None),
861 ControlFlow::Break(break_reason) => match break_reason {
862 BreakReason::Err(err) => Err(err),
863 BreakReason::Stopped(maybe_continuation) => {
864 if let Some((continuation, source_node_id)) = maybe_continuation {
865 continuation_stack
866 .push_resumed_with_source_node_id(continuation, source_node_id);
867 }
868
869 while matches!(
870 continuation_stack.peek_continuation(),
871 Some(Continuation::EnterForest { .. })
872 ) {
873 let Some((
874 Continuation::EnterForest {
875 forest,
876 package_debug_info: restored_debug_info,
877 inline_context_depth,
878 },
879 _,
880 )) = continuation_stack.pop_continuation_with_source_node_id()
881 else {
882 unreachable!("peeked continuation must still be EnterForest")
883 };
884 current_forest = forest;
885 package_debug_info = restored_debug_info;
886 inline_call_contexts.truncate(inline_context_depth);
887 }
888
889 Ok(Some(ResumeContext {
890 current_forest,
891 continuation_stack,
892 kernel,
893 package_debug_info,
894 inline_call_contexts,
895 }))
896 },
897 },
898 }
899 }
900
901 #[inline(always)]
903 fn current_stack_outputs(&self) -> StackOutputs {
904 StackOutputs::new(
905 &self.stack[self.stack_bot_idx..self.stack_top_idx]
906 .iter()
907 .rev()
908 .copied()
909 .collect::<Vec<_>>(),
910 )
911 .unwrap()
912 }
913
914 fn execute_impl<S, T>(
920 &mut self,
921 continuation_stack: &mut ContinuationStack<Arc<MastForest>>,
922 current_forest: &mut Arc<MastForest>,
923 kernel: &KernelDescriptor,
924 host: &mut impl SyncHost,
925 tracer: &mut T,
926 stopper: &S,
927 package_debug_info: &mut Option<Arc<PackageDebugInfo>>,
928 inline_call_contexts: &mut Vec<Option<SourceInlineCallContext>>,
929 ) -> ControlFlow<BreakReason<Arc<MastForest>>, StackOutputs>
930 where
931 S: Stopper<Processor = Self, Forest = Arc<MastForest>>,
932 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
933 {
934 while let ControlFlow::Break(internal_break_reason) = execute_impl(
935 self,
936 continuation_stack,
937 current_forest,
938 kernel,
939 host,
940 tracer,
941 stopper,
942 package_debug_info,
943 inline_call_contexts,
944 ) {
945 let current_package_debug_info = package_debug_info.as_deref();
946 let source_aware_execution =
947 current_package_debug_info.is_some() || continuation_stack.tracks_source_nodes();
948 match internal_break_reason {
949 InternalBreakReason::User(break_reason) => return ControlFlow::Break(break_reason),
950 InternalBreakReason::Emit { op_idx, continuation, source_node_id } => {
951 self.op_emit_sync(host, op_idx, current_package_debug_info, source_node_id)?;
952
953 finish_emit_op_execution(
954 continuation,
955 source_node_id,
956 self,
957 continuation_stack,
958 current_forest,
959 tracer,
960 stopper,
961 )?;
962 },
963 InternalBreakReason::LoadMastForestFromDyn { callee_hash, source_node_id } => {
964 let (root_id, new_forest, new_package_debug_info, new_source_node_id) =
965 match self.load_mast_forest_sync(
966 callee_hash,
967 host,
968 current_package_debug_info,
969 source_node_id,
970 source_aware_execution,
971 ) {
972 Ok(result) => result,
973 Err(err) => return ControlFlow::Break(BreakReason::Err(err)),
974 };
975
976 finish_load_mast_forest_from_dyn_start(
977 root_id,
978 new_forest,
979 new_package_debug_info,
980 new_source_node_id,
981 self,
982 current_forest,
983 package_debug_info,
984 inline_call_contexts.as_slice(),
985 continuation_stack,
986 tracer,
987 stopper,
988 )?;
989 },
990 InternalBreakReason::LoadMastForestFromExternal {
991 external_node_id,
992 procedure_hash,
993 source_node_id,
994 } => {
995 let inline_call_context = package_debug_info.clone().and_then(|debug_info| {
996 SourceInlineCallContext::for_source_boundary(debug_info, source_node_id)
997 });
998 let (root_id, new_forest, new_package_debug_info, new_source_node_id) =
999 match self.load_mast_forest_sync(
1000 procedure_hash,
1001 host,
1002 current_package_debug_info,
1003 source_node_id,
1004 source_aware_execution,
1005 ) {
1006 Ok(result) => result,
1007 Err(err) => {
1008 let maybe_enriched_err = maybe_use_caller_error_context(
1009 err,
1010 continuation_stack,
1011 current_package_debug_info,
1012 host,
1013 );
1014 return ControlFlow::Break(BreakReason::Err(maybe_enriched_err));
1015 },
1016 };
1017
1018 finish_load_mast_forest_from_external(
1019 root_id,
1020 new_forest,
1021 new_package_debug_info,
1022 new_source_node_id,
1023 inline_call_context,
1024 (external_node_id, source_node_id),
1025 current_forest,
1026 package_debug_info,
1027 inline_call_contexts,
1028 continuation_stack,
1029 tracer,
1030 )?;
1031 },
1032 }
1033 }
1034
1035 match StackOutputs::new(
1036 &self.stack[self.stack_bot_idx..self.stack_top_idx]
1037 .iter()
1038 .rev()
1039 .copied()
1040 .collect::<Vec<_>>(),
1041 ) {
1042 Ok(stack_outputs) => ControlFlow::Continue(stack_outputs),
1043 Err(_) => ControlFlow::Break(BreakReason::Err(ExecutionError::OutputStackOverflow(
1044 self.stack_top_idx - self.stack_bot_idx - MIN_STACK_DEPTH,
1045 ))),
1046 }
1047 }
1048
1049 async fn execute_impl_async<S, T>(
1050 &mut self,
1051 continuation_stack: &mut ContinuationStack<Arc<MastForest>>,
1052 current_forest: &mut Arc<MastForest>,
1053 kernel: &KernelDescriptor,
1054 host: &mut impl Host,
1055 tracer: &mut T,
1056 stopper: &S,
1057 package_debug_info: &mut Option<Arc<PackageDebugInfo>>,
1058 inline_call_contexts: &mut Vec<Option<SourceInlineCallContext>>,
1059 ) -> ControlFlow<BreakReason<Arc<MastForest>>, StackOutputs>
1060 where
1061 S: Stopper<Processor = Self, Forest = Arc<MastForest>>,
1062 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
1063 {
1064 while let ControlFlow::Break(internal_break_reason) = execute_impl(
1065 self,
1066 continuation_stack,
1067 current_forest,
1068 kernel,
1069 host,
1070 tracer,
1071 stopper,
1072 package_debug_info,
1073 inline_call_contexts,
1074 ) {
1075 let current_package_debug_info = package_debug_info.as_deref();
1076 let source_aware_execution =
1077 current_package_debug_info.is_some() || continuation_stack.tracks_source_nodes();
1078 match internal_break_reason {
1079 InternalBreakReason::User(break_reason) => return ControlFlow::Break(break_reason),
1080 InternalBreakReason::Emit { op_idx, continuation, source_node_id } => {
1081 self.op_emit(host, op_idx, current_package_debug_info, source_node_id).await?;
1082
1083 finish_emit_op_execution(
1084 continuation,
1085 source_node_id,
1086 self,
1087 continuation_stack,
1088 current_forest,
1089 tracer,
1090 stopper,
1091 )?;
1092 },
1093 InternalBreakReason::LoadMastForestFromDyn { callee_hash, source_node_id } => {
1094 let (root_id, new_forest, new_package_debug_info, new_source_node_id) =
1095 match self
1096 .load_mast_forest(
1097 callee_hash,
1098 host,
1099 current_package_debug_info,
1100 source_node_id,
1101 source_aware_execution,
1102 )
1103 .await
1104 {
1105 Ok(result) => result,
1106 Err(err) => return ControlFlow::Break(BreakReason::Err(err)),
1107 };
1108
1109 finish_load_mast_forest_from_dyn_start(
1110 root_id,
1111 new_forest,
1112 new_package_debug_info,
1113 new_source_node_id,
1114 self,
1115 current_forest,
1116 package_debug_info,
1117 inline_call_contexts.as_slice(),
1118 continuation_stack,
1119 tracer,
1120 stopper,
1121 )?;
1122 },
1123 InternalBreakReason::LoadMastForestFromExternal {
1124 external_node_id,
1125 procedure_hash,
1126 source_node_id,
1127 } => {
1128 let inline_call_context = package_debug_info.clone().and_then(|debug_info| {
1129 SourceInlineCallContext::for_source_boundary(debug_info, source_node_id)
1130 });
1131 let (root_id, new_forest, new_package_debug_info, new_source_node_id) =
1132 match self
1133 .load_mast_forest(
1134 procedure_hash,
1135 host,
1136 current_package_debug_info,
1137 source_node_id,
1138 source_aware_execution,
1139 )
1140 .await
1141 {
1142 Ok(result) => result,
1143 Err(err) => {
1144 let maybe_enriched_err = maybe_use_caller_error_context(
1145 err,
1146 continuation_stack,
1147 current_package_debug_info,
1148 host,
1149 );
1150 return ControlFlow::Break(BreakReason::Err(maybe_enriched_err));
1151 },
1152 };
1153
1154 finish_load_mast_forest_from_external(
1155 root_id,
1156 new_forest,
1157 new_package_debug_info,
1158 new_source_node_id,
1159 inline_call_context,
1160 (external_node_id, source_node_id),
1161 current_forest,
1162 package_debug_info,
1163 inline_call_contexts,
1164 continuation_stack,
1165 tracer,
1166 )?;
1167 },
1168 }
1169 }
1170
1171 match StackOutputs::new(
1172 &self.stack[self.stack_bot_idx..self.stack_top_idx]
1173 .iter()
1174 .rev()
1175 .copied()
1176 .collect::<Vec<_>>(),
1177 ) {
1178 Ok(stack_outputs) => ControlFlow::Continue(stack_outputs),
1179 Err(_) => ControlFlow::Break(BreakReason::Err(ExecutionError::OutputStackOverflow(
1180 self.stack_top_idx - self.stack_bot_idx - MIN_STACK_DEPTH,
1181 ))),
1182 }
1183 }
1184
1185 fn load_mast_forest_sync(
1189 &mut self,
1190 node_digest: Word,
1191 host: &mut impl SyncHost,
1192 package_debug_info: Option<&PackageDebugInfo>,
1193 source_node_id: Option<DebugSourceNodeId>,
1194 source_aware_execution: bool,
1195 ) -> Result<
1196 (
1197 MastNodeId,
1198 Arc<MastForest>,
1199 Option<Arc<PackageDebugInfo>>,
1200 Option<DebugSourceNodeId>,
1201 ),
1202 ExecutionError,
1203 > {
1204 let cached = self.loaded_mast_forests.get(&node_digest).cloned();
1205 let was_cached = cached.is_some();
1206 let loaded_mast_forest = match cached {
1207 Some(mast_forest) => mast_forest,
1208 None => host.get_mast_forest(&node_digest).ok_or_else(|| {
1209 match (package_debug_info, source_node_id) {
1210 (Some(debug_info), Some(source_node_id)) => {
1211 crate::errors::procedure_not_found_with_package_source_context(
1212 node_digest,
1213 PackageSourceDebugContext::new(debug_info, source_node_id),
1214 host,
1215 )
1216 },
1217 _ => crate::errors::procedure_not_found_with_context(node_digest),
1218 }
1219 })?,
1220 };
1221 let mast_forest = loaded_mast_forest.mast_forest().clone();
1222
1223 let root_id = mast_forest.find_procedure_root(node_digest).ok_or_else(|| {
1224 let context = match (package_debug_info, source_node_id) {
1225 (Some(debug_info), Some(source_node_id)) => {
1226 Some(PackageSourceDebugContext::new(debug_info, source_node_id))
1227 },
1228 _ => None,
1229 };
1230 malformed_mast_forest_with_context(node_digest, context, host)
1231 })?;
1232
1233 if !was_cached {
1234 self.cache_loaded_mast_forest(&loaded_mast_forest);
1235 }
1236 self.merge_mast_forest_advice(&mast_forest).map_exec_err()?;
1237 let (loaded_package_debug_info, loaded_source_node_id) =
1238 Self::loaded_package_source_context(
1239 &loaded_mast_forest,
1240 root_id,
1241 source_aware_execution,
1242 )?;
1243
1244 Ok((root_id, mast_forest, loaded_package_debug_info, loaded_source_node_id))
1245 }
1246
1247 async fn load_mast_forest(
1248 &mut self,
1249 node_digest: Word,
1250 host: &mut impl Host,
1251 package_debug_info: Option<&PackageDebugInfo>,
1252 source_node_id: Option<DebugSourceNodeId>,
1253 source_aware_execution: bool,
1254 ) -> Result<
1255 (
1256 MastNodeId,
1257 Arc<MastForest>,
1258 Option<Arc<PackageDebugInfo>>,
1259 Option<DebugSourceNodeId>,
1260 ),
1261 ExecutionError,
1262 > {
1263 let cached = self.loaded_mast_forests.get(&node_digest).cloned();
1264 let was_cached = cached.is_some();
1265 let loaded_mast_forest = match cached {
1266 Some(mast_forest) => mast_forest,
1267 None => {
1268 if let Some(mast_forest) = host.get_mast_forest(&node_digest).await {
1269 mast_forest
1270 } else {
1271 return Err(match (package_debug_info, source_node_id) {
1272 (Some(debug_info), Some(source_node_id)) => {
1273 crate::errors::procedure_not_found_with_package_source_context(
1274 node_digest,
1275 PackageSourceDebugContext::new(debug_info, source_node_id),
1276 host,
1277 )
1278 },
1279 _ => crate::errors::procedure_not_found_with_context(node_digest),
1280 });
1281 }
1282 },
1283 };
1284 let mast_forest = loaded_mast_forest.mast_forest().clone();
1285
1286 let root_id = mast_forest.find_procedure_root(node_digest).ok_or_else(|| {
1287 let context = match (package_debug_info, source_node_id) {
1288 (Some(debug_info), Some(source_node_id)) => {
1289 Some(PackageSourceDebugContext::new(debug_info, source_node_id))
1290 },
1291 _ => None,
1292 };
1293 malformed_mast_forest_with_context(node_digest, context, host)
1294 })?;
1295
1296 if !was_cached {
1297 self.cache_loaded_mast_forest(&loaded_mast_forest);
1298 }
1299 self.merge_mast_forest_advice(&mast_forest).map_exec_err()?;
1300 let (loaded_package_debug_info, loaded_source_node_id) =
1301 Self::loaded_package_source_context(
1302 &loaded_mast_forest,
1303 root_id,
1304 source_aware_execution,
1305 )?;
1306
1307 Ok((root_id, mast_forest, loaded_package_debug_info, loaded_source_node_id))
1308 }
1309
1310 fn cache_loaded_mast_forest(&mut self, loaded_mast_forest: &LoadedMastForest) {
1311 for procedure_digest in loaded_mast_forest.mast_forest().local_procedure_digests() {
1312 self.loaded_mast_forests
1313 .entry(procedure_digest)
1314 .or_insert_with(|| loaded_mast_forest.clone());
1315 }
1316 }
1317
1318 fn merge_mast_forest_advice(&mut self, mast_forest: &MastForest) -> Result<(), AdviceError> {
1319 let commitment = mast_forest.commitment();
1320 if self.merged_mast_forests.contains(&commitment) {
1321 return Ok(());
1322 }
1323
1324 self.advice.extend_map(mast_forest.advice_map())?;
1325 self.merged_mast_forests.insert(commitment);
1326 Ok(())
1327 }
1328
1329 fn loaded_package_source_context(
1330 loaded_mast_forest: &LoadedMastForest,
1331 root_id: MastNodeId,
1332 source_aware_execution: bool,
1333 ) -> Result<(Option<Arc<PackageDebugInfo>>, Option<DebugSourceNodeId>), ExecutionError> {
1334 if !source_aware_execution {
1335 return Ok((None, None));
1336 }
1337
1338 let Some(package_debug_info) = loaded_mast_forest
1339 .package_debug_info()
1340 .map_err(|_| ExecutionError::Internal("loaded package debug info is malformed"))?
1341 else {
1342 return Ok((None, None));
1343 };
1344
1345 let source_node_id = match package_debug_info.unique_source_root_for_exec_node(root_id) {
1346 Ok(source_node_id) => source_node_id,
1347 Err(DebugSourceGraphLookupError::AmbiguousRoot { .. }) => None,
1348 Err(_) => {
1349 return Err(ExecutionError::Internal(
1350 "loaded package debug source graph has malformed entrypoint roots",
1351 ));
1352 },
1353 };
1354
1355 Ok((Some(package_debug_info), source_node_id))
1356 }
1357
1358 pub fn execute_by_step_sync(
1360 mut self,
1361 program: &Program,
1362 host: &mut impl SyncHost,
1363 ) -> Result<StackOutputs, ExecutionError> {
1364 let mut current_resume_ctx = self.get_initial_resume_context(program)?;
1365
1366 loop {
1367 match self.step_sync(host, current_resume_ctx)? {
1368 Some(next_resume_ctx) => {
1369 current_resume_ctx = next_resume_ctx;
1370 },
1371 None => break Ok(self.current_stack_outputs()),
1372 }
1373 }
1374 }
1375
1376 #[cfg(any(test, feature = "testing"))]
1379 pub fn execute_by_step_with_package_debug_info_sync(
1380 mut self,
1381 program: &Program,
1382 package_debug_info: &PackageDebugInfo,
1383 host: &mut impl SyncHost,
1384 ) -> Result<StackOutputs, ExecutionError> {
1385 let mut current_resume_ctx =
1386 self.source_aware_resume_context(program, package_debug_info, None)?;
1387
1388 loop {
1389 match self.step_with_package_debug_info_sync(
1390 host,
1391 current_resume_ctx,
1392 package_debug_info,
1393 )? {
1394 Some(next_resume_ctx) => {
1395 current_resume_ctx = next_resume_ctx;
1396 },
1397 None => break Ok(self.current_stack_outputs()),
1398 }
1399 }
1400 }
1401
1402 #[cfg(any(test, feature = "testing"))]
1405 pub fn execute_by_step_with_package_debug_info_at_source_node_sync(
1406 mut self,
1407 program: &Program,
1408 package_debug_info: &PackageDebugInfo,
1409 entrypoint_source_node_id: DebugSourceNodeId,
1410 host: &mut impl SyncHost,
1411 ) -> Result<StackOutputs, ExecutionError> {
1412 let mut current_resume_ctx = self.source_aware_resume_context(
1413 program,
1414 package_debug_info,
1415 Some(entrypoint_source_node_id),
1416 )?;
1417
1418 loop {
1419 match self.step_with_package_debug_info_sync(
1420 host,
1421 current_resume_ctx,
1422 package_debug_info,
1423 )? {
1424 Some(next_resume_ctx) => {
1425 current_resume_ctx = next_resume_ctx;
1426 },
1427 None => break Ok(self.current_stack_outputs()),
1428 }
1429 }
1430 }
1431
1432 #[inline(always)]
1434 pub async fn execute_by_step(
1435 mut self,
1436 program: &Program,
1437 host: &mut impl Host,
1438 ) -> Result<StackOutputs, ExecutionError> {
1439 let mut current_resume_ctx = self.get_initial_resume_context(program)?;
1440 let mut processor = self;
1441
1442 loop {
1443 match processor.step(host, current_resume_ctx).await? {
1444 Some(next_resume_ctx) => {
1445 current_resume_ctx = next_resume_ctx;
1446 },
1447 None => break Ok(processor.current_stack_outputs()),
1448 }
1449 }
1450 }
1451
1452 #[cfg(any(test, feature = "testing"))]
1454 #[inline(always)]
1455 pub async fn execute_by_step_with_package_debug_info(
1456 mut self,
1457 program: &Program,
1458 package_debug_info: &PackageDebugInfo,
1459 host: &mut impl Host,
1460 ) -> Result<StackOutputs, ExecutionError> {
1461 let mut current_resume_ctx =
1462 self.source_aware_resume_context(program, package_debug_info, None)?;
1463 let mut processor = self;
1464
1465 loop {
1466 match processor
1467 .step_with_package_debug_info(host, current_resume_ctx, package_debug_info)
1468 .await?
1469 {
1470 Some(next_resume_ctx) => {
1471 current_resume_ctx = next_resume_ctx;
1472 },
1473 None => break Ok(processor.current_stack_outputs()),
1474 }
1475 }
1476 }
1477
1478 #[cfg(any(test, feature = "testing"))]
1481 #[inline(always)]
1482 pub async fn execute_by_step_with_package_debug_info_at_source_node(
1483 mut self,
1484 program: &Program,
1485 package_debug_info: &PackageDebugInfo,
1486 entrypoint_source_node_id: DebugSourceNodeId,
1487 host: &mut impl Host,
1488 ) -> Result<StackOutputs, ExecutionError> {
1489 let mut current_resume_ctx = self.source_aware_resume_context(
1490 program,
1491 package_debug_info,
1492 Some(entrypoint_source_node_id),
1493 )?;
1494 let mut processor = self;
1495
1496 loop {
1497 match processor
1498 .step_with_package_debug_info(host, current_resume_ctx, package_debug_info)
1499 .await?
1500 {
1501 Some(next_resume_ctx) => {
1502 current_resume_ctx = next_resume_ctx;
1503 },
1504 None => break Ok(processor.current_stack_outputs()),
1505 }
1506 }
1507 }
1508
1509 #[cfg(any(test, feature = "testing"))]
1513 pub fn execute_mut_sync(
1514 &mut self,
1515 program: &Program,
1516 host: &mut impl SyncHost,
1517 ) -> Result<StackOutputs, ExecutionError> {
1518 let mut continuation_stack = ContinuationStack::new(program);
1519 let mut current_forest = program.mast_forest().clone();
1520 let mut package_debug_info = None;
1521 let mut inline_call_contexts = Vec::new();
1522
1523 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
1524
1525 let flow = self.execute_impl(
1526 &mut continuation_stack,
1527 &mut current_forest,
1528 program.kernel(),
1529 host,
1530 &mut NoopTracer,
1531 &NeverStopper,
1532 &mut package_debug_info,
1533 &mut inline_call_contexts,
1534 );
1535 Self::stack_result_from_flow(flow)
1536 }
1537
1538 #[cfg(any(test, feature = "testing"))]
1540 #[inline(always)]
1541 pub async fn execute_mut(
1542 &mut self,
1543 program: &Program,
1544 host: &mut impl Host,
1545 ) -> Result<StackOutputs, ExecutionError> {
1546 let mut continuation_stack = ContinuationStack::new(program);
1547 let mut current_forest = program.mast_forest().clone();
1548 let mut package_debug_info = None;
1549 let mut inline_call_contexts = Vec::new();
1550
1551 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
1552
1553 let flow = self
1554 .execute_impl_async(
1555 &mut continuation_stack,
1556 &mut current_forest,
1557 program.kernel(),
1558 host,
1559 &mut NoopTracer,
1560 &NeverStopper,
1561 &mut package_debug_info,
1562 &mut inline_call_contexts,
1563 )
1564 .await;
1565 Self::stack_result_from_flow(flow)
1566 }
1567}
1568
1569#[cfg(all(test, feature = "std"))]
1570mod tests {
1571 use super::FastProcessor;
1572
1573 #[test]
1574 fn sole_rayon_worker_requires_buffered_trace_building() {
1575 rayon::ThreadPoolBuilder::new()
1576 .num_threads(1)
1577 .build()
1578 .unwrap()
1579 .install(|| assert!(FastProcessor::rayon_has_no_parallel_worker()));
1580 }
1581}