1use alloc::{sync::Arc, vec::Vec};
2use core::ops::ControlFlow;
3
4use miden_core::{
5 Word,
6 mast::{MastForest, MastNodeId},
7 program::{KernelDescriptor, MIN_STACK_DEPTH, Program, StackInputs, StackOutputs},
8};
9use miden_mast_package::debug_info::{
10 DebugSourceGraphLookupError, DebugSourceNodeId, PackageDebugInfo,
11};
12use tracing::instrument;
13
14use super::{
15 FastProcessor, NoopTracer,
16 external::maybe_use_caller_error_context,
17 step::{BreakReason, NeverStopper, ResumeContext, StepStopper},
18};
19#[cfg(feature = "std")]
20use crate::PrecompileWitness;
21use crate::{
22 ExecutionError, ExecutionOutput, ExecutionWitness, Host, LoadedMastForest, Stopper, SyncHost,
23 continuation_stack::{Continuation, ContinuationStack, SourceInlineCallContext},
24 errors::{
25 MapExecErr, MapExecErrNoCtx, PackageSourceDebugContext, malformed_mast_forest_with_context,
26 },
27 execution::{
28 InternalBreakReason, execute_impl, finish_emit_op_execution,
29 finish_load_mast_forest_from_dyn_start, finish_load_mast_forest_from_external,
30 },
31 trace::execution_tracer::ExecutionTracer,
32 tracer::Tracer,
33};
34
35impl FastProcessor {
36 pub fn execute_sync(
41 self,
42 program: &Program,
43 host: &mut impl SyncHost,
44 ) -> Result<ExecutionOutput, ExecutionError> {
45 self.execute_with_tracer_sync(program, host, &mut NoopTracer)
46 }
47
48 pub fn execute_with_package_debug_info_sync(
55 self,
56 program: &Program,
57 package_debug_info: &PackageDebugInfo,
58 host: &mut impl SyncHost,
59 ) -> Result<ExecutionOutput, ExecutionError> {
60 self.execute_with_package_debug_info_and_tracer_sync(
61 program,
62 package_debug_info,
63 None,
64 host,
65 &mut NoopTracer,
66 )
67 }
68
69 pub fn execute_with_package_debug_info_at_source_node_sync(
76 self,
77 program: &Program,
78 package_debug_info: &PackageDebugInfo,
79 entrypoint_source_node_id: DebugSourceNodeId,
80 host: &mut impl SyncHost,
81 ) -> Result<ExecutionOutput, ExecutionError> {
82 self.execute_with_package_debug_info_and_tracer_sync(
83 program,
84 package_debug_info,
85 Some(entrypoint_source_node_id),
86 host,
87 &mut NoopTracer,
88 )
89 }
90
91 #[inline(always)]
93 pub async fn execute(
94 self,
95 program: &Program,
96 host: &mut impl Host,
97 ) -> Result<ExecutionOutput, ExecutionError> {
98 self.execute_with_tracer(program, host, &mut NoopTracer).await
99 }
100
101 #[inline(always)]
106 pub async fn execute_with_package_debug_info(
107 self,
108 program: &Program,
109 package_debug_info: &PackageDebugInfo,
110 host: &mut impl Host,
111 ) -> Result<ExecutionOutput, ExecutionError> {
112 self.execute_with_package_debug_info_and_tracer(
113 program,
114 package_debug_info,
115 None,
116 host,
117 &mut NoopTracer,
118 )
119 .await
120 }
121
122 #[inline(always)]
124 pub async fn execute_with_package_debug_info_at_source_node(
125 self,
126 program: &Program,
127 package_debug_info: &PackageDebugInfo,
128 entrypoint_source_node_id: DebugSourceNodeId,
129 host: &mut impl Host,
130 ) -> Result<ExecutionOutput, ExecutionError> {
131 self.execute_with_package_debug_info_and_tracer(
132 program,
133 package_debug_info,
134 Some(entrypoint_source_node_id),
135 host,
136 &mut NoopTracer,
137 )
138 .await
139 }
140
141 #[cfg(feature = "std")]
156 #[instrument(name = "execute_and_build_trace_sync", skip_all)]
157 pub fn execute_and_build_trace_sync(
158 self,
159 program: &Program,
160 host: &mut impl SyncHost,
161 max_prover_memory_bytes: u64,
162 ) -> Result<(crate::trace::VmTrace, Option<PrecompileWitness>), ExecutionError> {
163 use miden_air::{config, memory};
164
165 use crate::trace::{
166 MAX_TRACE_LEN, build_hasher_chiplet, build_trace_with_budget,
167 build_trace_with_prebuilt_hasher,
168 };
169
170 if Self::rayon_has_no_parallel_worker() {
171 let (vm_witness, precompiles_witness) =
172 self.execute_for_proving_sync(program, host)?.into_parts();
173 let trace = build_trace_with_budget(vm_witness, max_prover_memory_bytes)?;
174 return Ok((trace, precompiles_witness));
175 }
176
177 let stack_inputs = self.initial_stack_inputs();
178 let max_trace_len = MAX_TRACE_LEN.min(memory::max_any_height_for_budget(
179 max_prover_memory_bytes,
180 &config::pcs_params(),
181 ));
182 let (sender, receiver) = std::sync::mpsc::channel();
183 let mut tracer = ExecutionTracer::new_with_streamed_hasher(
184 self.options.core_trace_fragment_size(),
185 self.options.max_stack_depth(),
186 sender,
187 );
188
189 let mut hasher = None;
190 let hasher_slot = &mut hasher;
191 let execution_output = rayon::in_place_scope(move |scope| {
194 let span = tracing::Span::current();
197 scope.spawn(move |_| {
198 let _span = span.entered();
199 let result = build_hasher_chiplet(receiver.into_iter().map(Ok), max_trace_len);
200 *hasher_slot = Some(result);
201 });
202
203 let execution_output = self.execute_with_tracer_sync(program, host, &mut tracer);
204
205 match execution_output {
206 Ok(output) => {
207 let (mut vm_witness, precompiles_witness) =
208 Self::execution_witness_from_parts(program, stack_inputs, output, tracer)
209 .into_parts();
210 drop(vm_witness.take_hasher_replay());
212 Ok((vm_witness, precompiles_witness))
213 },
214 Err(err) => {
215 drop(tracer);
219 Err(err)
220 },
221 }
222 });
223
224 let hasher = hasher.expect("hasher builder did not run");
225 let (vm_witness, precompiles_witness) = match execution_output {
226 Ok(output) => output,
227 Err(err) => {
228 if let Err(builder_err) = hasher {
229 tracing::debug!(%builder_err, "hasher builder also failed");
230 }
231 return Err(err);
232 },
233 };
234
235 let trace = build_trace_with_prebuilt_hasher(vm_witness, hasher?, max_prover_memory_bytes)?;
236 Ok((trace, precompiles_witness))
237 }
238
239 #[cfg(feature = "std")]
240 fn rayon_has_no_parallel_worker() -> bool {
241 rayon::current_num_threads() == 1 && rayon::current_thread_index().is_some()
243 }
244
245 #[instrument(name = "execute_for_proving_sync", skip_all)]
267 pub fn execute_for_proving_sync(
268 self,
269 program: &Program,
270 host: &mut impl SyncHost,
271 ) -> Result<ExecutionWitness, ExecutionError> {
272 let stack_inputs = self.initial_stack_inputs();
273 let mut tracer = ExecutionTracer::new(
274 self.options.core_trace_fragment_size(),
275 self.options.max_stack_depth(),
276 );
277 let execution_output = self.execute_with_tracer_sync(program, host, &mut tracer)?;
278 Ok(Self::execution_witness_from_parts(
279 program,
280 stack_inputs,
281 execution_output,
282 tracer,
283 ))
284 }
285
286 #[instrument(name = "execute_for_proving_with_package_debug_info_sync", skip_all)]
289 pub fn execute_for_proving_with_package_debug_info_sync(
290 self,
291 program: &Program,
292 package_debug_info: &PackageDebugInfo,
293 host: &mut impl SyncHost,
294 ) -> Result<ExecutionWitness, ExecutionError> {
295 let stack_inputs = self.initial_stack_inputs();
296 let mut tracer = ExecutionTracer::new(
297 self.options.core_trace_fragment_size(),
298 self.options.max_stack_depth(),
299 );
300 let execution_output = self.execute_with_package_debug_info_and_tracer_sync(
301 program,
302 package_debug_info,
303 None,
304 host,
305 &mut tracer,
306 )?;
307 Ok(Self::execution_witness_from_parts(
308 program,
309 stack_inputs,
310 execution_output,
311 tracer,
312 ))
313 }
314
315 #[instrument(
318 name = "execute_for_proving_with_package_debug_info_at_source_node_sync",
319 skip_all
320 )]
321 pub fn execute_for_proving_with_package_debug_info_at_source_node_sync(
322 self,
323 program: &Program,
324 package_debug_info: &PackageDebugInfo,
325 entrypoint_source_node_id: DebugSourceNodeId,
326 host: &mut impl SyncHost,
327 ) -> Result<ExecutionWitness, ExecutionError> {
328 let stack_inputs = self.initial_stack_inputs();
329 let mut tracer = ExecutionTracer::new(
330 self.options.core_trace_fragment_size(),
331 self.options.max_stack_depth(),
332 );
333 let execution_output = self.execute_with_package_debug_info_and_tracer_sync(
334 program,
335 package_debug_info,
336 Some(entrypoint_source_node_id),
337 host,
338 &mut tracer,
339 )?;
340 Ok(Self::execution_witness_from_parts(
341 program,
342 stack_inputs,
343 execution_output,
344 tracer,
345 ))
346 }
347
348 #[inline(always)]
350 #[instrument(name = "execute_for_proving", skip_all)]
351 pub async fn execute_for_proving(
352 self,
353 program: &Program,
354 host: &mut impl Host,
355 ) -> Result<ExecutionWitness, ExecutionError> {
356 let stack_inputs = self.initial_stack_inputs();
357 let mut tracer = ExecutionTracer::new(
358 self.options.core_trace_fragment_size(),
359 self.options.max_stack_depth(),
360 );
361 let execution_output = self.execute_with_tracer(program, host, &mut tracer).await?;
362 Ok(Self::execution_witness_from_parts(
363 program,
364 stack_inputs,
365 execution_output,
366 tracer,
367 ))
368 }
369
370 #[cfg(any(test, feature = "testing"))]
372 #[inline(always)]
373 #[instrument(name = "execute_for_proving_with_package_debug_info", skip_all)]
374 pub async fn execute_for_proving_with_package_debug_info(
375 self,
376 program: &Program,
377 package_debug_info: &PackageDebugInfo,
378 host: &mut impl Host,
379 ) -> Result<ExecutionWitness, ExecutionError> {
380 let stack_inputs = self.initial_stack_inputs();
381 let mut tracer = ExecutionTracer::new(
382 self.options.core_trace_fragment_size(),
383 self.options.max_stack_depth(),
384 );
385 let execution_output = self
386 .execute_with_package_debug_info_and_tracer(
387 program,
388 package_debug_info,
389 None,
390 host,
391 &mut tracer,
392 )
393 .await?;
394 Ok(Self::execution_witness_from_parts(
395 program,
396 stack_inputs,
397 execution_output,
398 tracer,
399 ))
400 }
401
402 #[cfg(any(test, feature = "testing"))]
405 #[inline(always)]
406 #[instrument(name = "execute_for_proving_with_package_debug_info_at_source_node", skip_all)]
407 pub async fn execute_for_proving_with_package_debug_info_at_source_node(
408 self,
409 program: &Program,
410 package_debug_info: &PackageDebugInfo,
411 entrypoint_source_node_id: DebugSourceNodeId,
412 host: &mut impl Host,
413 ) -> Result<ExecutionWitness, ExecutionError> {
414 let stack_inputs = self.initial_stack_inputs();
415 let mut tracer = ExecutionTracer::new(
416 self.options.core_trace_fragment_size(),
417 self.options.max_stack_depth(),
418 );
419 let execution_output = self
420 .execute_with_package_debug_info_and_tracer(
421 program,
422 package_debug_info,
423 Some(entrypoint_source_node_id),
424 host,
425 &mut tracer,
426 )
427 .await?;
428 Ok(Self::execution_witness_from_parts(
429 program,
430 stack_inputs,
431 execution_output,
432 tracer,
433 ))
434 }
435
436 pub async fn execute_with_tracer<T>(
438 mut self,
439 program: &Program,
440 host: &mut impl Host,
441 tracer: &mut T,
442 ) -> Result<ExecutionOutput, ExecutionError>
443 where
444 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
445 {
446 let mut continuation_stack = ContinuationStack::new(program);
447 let mut current_forest = program.mast_forest().clone();
448 let mut package_debug_info = None;
449 let mut inline_call_contexts = Vec::new();
450
451 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
452 let flow = self
453 .execute_impl_async(
454 &mut continuation_stack,
455 &mut current_forest,
456 program.kernel(),
457 host,
458 tracer,
459 &NeverStopper,
460 &mut package_debug_info,
461 &mut inline_call_contexts,
462 )
463 .await;
464 Self::execution_result_from_flow(flow, self)
465 }
466
467 async fn execute_with_package_debug_info_and_tracer<T>(
470 mut self,
471 program: &Program,
472 package_debug_info: &PackageDebugInfo,
473 entrypoint_source_node_id: Option<DebugSourceNodeId>,
474 host: &mut impl Host,
475 tracer: &mut T,
476 ) -> Result<ExecutionOutput, ExecutionError>
477 where
478 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
479 {
480 let mut continuation_stack = Self::source_aware_continuation_stack(
481 program,
482 package_debug_info,
483 entrypoint_source_node_id,
484 )?;
485 let mut current_forest = program.mast_forest().clone();
486 let mut package_debug_info = Some(Arc::new(package_debug_info.clone()));
487 let mut inline_call_contexts = Vec::new();
488
489 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
490 let flow = self
491 .execute_impl_async(
492 &mut continuation_stack,
493 &mut current_forest,
494 program.kernel(),
495 host,
496 tracer,
497 &NeverStopper,
498 &mut package_debug_info,
499 &mut inline_call_contexts,
500 )
501 .await;
502 Self::execution_result_from_flow(flow, self)
503 }
504
505 pub fn execute_with_tracer_sync<T>(
507 mut self,
508 program: &Program,
509 host: &mut impl SyncHost,
510 tracer: &mut T,
511 ) -> Result<ExecutionOutput, ExecutionError>
512 where
513 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
514 {
515 let mut continuation_stack = ContinuationStack::new(program);
516 let mut current_forest = program.mast_forest().clone();
517 let mut package_debug_info = None;
518 let mut inline_call_contexts = Vec::new();
519
520 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
521 let flow = self.execute_impl(
522 &mut continuation_stack,
523 &mut current_forest,
524 program.kernel(),
525 host,
526 tracer,
527 &NeverStopper,
528 &mut package_debug_info,
529 &mut inline_call_contexts,
530 );
531 Self::execution_result_from_flow(flow, self)
532 }
533
534 fn execute_with_package_debug_info_and_tracer_sync<T>(
537 mut self,
538 program: &Program,
539 package_debug_info: &PackageDebugInfo,
540 entrypoint_source_node_id: Option<DebugSourceNodeId>,
541 host: &mut impl SyncHost,
542 tracer: &mut T,
543 ) -> Result<ExecutionOutput, ExecutionError>
544 where
545 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
546 {
547 let mut continuation_stack = Self::source_aware_continuation_stack(
548 program,
549 package_debug_info,
550 entrypoint_source_node_id,
551 )?;
552 let mut current_forest = program.mast_forest().clone();
553 let mut package_debug_info = Some(Arc::new(package_debug_info.clone()));
554 let mut inline_call_contexts = Vec::new();
555
556 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
557 let flow = self.execute_impl(
558 &mut continuation_stack,
559 &mut current_forest,
560 program.kernel(),
561 host,
562 tracer,
563 &NeverStopper,
564 &mut package_debug_info,
565 &mut inline_call_contexts,
566 );
567 Self::execution_result_from_flow(flow, self)
568 }
569
570 pub fn step_sync(
572 &mut self,
573 host: &mut impl SyncHost,
574 resume_ctx: ResumeContext,
575 ) -> Result<Option<ResumeContext>, ExecutionError> {
576 let ResumeContext {
577 mut current_forest,
578 mut continuation_stack,
579 kernel,
580 mut package_debug_info,
581 mut inline_call_contexts,
582 } = resume_ctx;
583
584 let flow = self.execute_impl(
585 &mut continuation_stack,
586 &mut current_forest,
587 &kernel,
588 host,
589 &mut NoopTracer,
590 &StepStopper,
591 &mut package_debug_info,
592 &mut inline_call_contexts,
593 );
594 Self::resume_context_from_flow(
595 flow,
596 continuation_stack,
597 current_forest,
598 kernel,
599 package_debug_info,
600 inline_call_contexts,
601 )
602 }
603
604 pub fn step_with_package_debug_info_sync(
606 &mut self,
607 host: &mut impl SyncHost,
608 resume_ctx: ResumeContext,
609 package_debug_info: &PackageDebugInfo,
610 ) -> Result<Option<ResumeContext>, ExecutionError> {
611 let ResumeContext {
612 mut current_forest,
613 mut continuation_stack,
614 kernel,
615 package_debug_info: mut active_package_debug_info,
616 mut inline_call_contexts,
617 } = resume_ctx;
618 Self::ensure_source_aware_step_context(
619 &mut continuation_stack,
620 &mut active_package_debug_info,
621 package_debug_info,
622 )?;
623
624 let flow = self.execute_impl(
625 &mut continuation_stack,
626 &mut current_forest,
627 &kernel,
628 host,
629 &mut NoopTracer,
630 &StepStopper,
631 &mut active_package_debug_info,
632 &mut inline_call_contexts,
633 );
634 Self::resume_context_from_flow(
635 flow,
636 continuation_stack,
637 current_forest,
638 kernel,
639 active_package_debug_info,
640 inline_call_contexts,
641 )
642 }
643
644 #[inline(always)]
646 pub async fn step(
647 &mut self,
648 host: &mut impl Host,
649 resume_ctx: ResumeContext,
650 ) -> Result<Option<ResumeContext>, ExecutionError> {
651 let ResumeContext {
652 mut current_forest,
653 mut continuation_stack,
654 kernel,
655 mut package_debug_info,
656 mut inline_call_contexts,
657 } = resume_ctx;
658
659 let flow = self
660 .execute_impl_async(
661 &mut continuation_stack,
662 &mut current_forest,
663 &kernel,
664 host,
665 &mut NoopTracer,
666 &StepStopper,
667 &mut package_debug_info,
668 &mut inline_call_contexts,
669 )
670 .await;
671 Self::resume_context_from_flow(
672 flow,
673 continuation_stack,
674 current_forest,
675 kernel,
676 package_debug_info,
677 inline_call_contexts,
678 )
679 }
680
681 #[inline(always)]
683 pub async fn step_with_package_debug_info(
684 &mut self,
685 host: &mut impl Host,
686 resume_ctx: ResumeContext,
687 package_debug_info: &PackageDebugInfo,
688 ) -> Result<Option<ResumeContext>, ExecutionError> {
689 let ResumeContext {
690 mut current_forest,
691 mut continuation_stack,
692 kernel,
693 package_debug_info: mut active_package_debug_info,
694 mut inline_call_contexts,
695 } = resume_ctx;
696 Self::ensure_source_aware_step_context(
697 &mut continuation_stack,
698 &mut active_package_debug_info,
699 package_debug_info,
700 )?;
701
702 let flow = self
703 .execute_impl_async(
704 &mut continuation_stack,
705 &mut current_forest,
706 &kernel,
707 host,
708 &mut NoopTracer,
709 &StepStopper,
710 &mut active_package_debug_info,
711 &mut inline_call_contexts,
712 )
713 .await;
714 Self::resume_context_from_flow(
715 flow,
716 continuation_stack,
717 current_forest,
718 kernel,
719 active_package_debug_info,
720 inline_call_contexts,
721 )
722 }
723
724 #[inline(always)]
726 fn execution_witness_from_parts(
727 program: &Program,
728 stack_inputs: StackInputs,
729 execution_output: ExecutionOutput,
730 tracer: ExecutionTracer,
731 ) -> ExecutionWitness {
732 ExecutionWitness::from_execution(
733 program.to_info(),
734 stack_inputs,
735 execution_output,
736 tracer.into_trace_replay(),
737 )
738 }
739
740 #[inline(always)]
742 fn initial_stack_inputs(&self) -> StackInputs {
743 core::array::from_fn(|idx| self.stack_get(idx)).into()
744 }
745
746 pub(super) fn source_aware_continuation_stack(
747 program: &Program,
748 package_debug_info: &PackageDebugInfo,
749 entrypoint_source_node_id: Option<DebugSourceNodeId>,
750 ) -> Result<ContinuationStack<Arc<MastForest>>, ExecutionError> {
751 if let Some(source_node_id) = entrypoint_source_node_id {
752 let Some(source_node) = package_debug_info.source_node(source_node_id) else {
753 return Err(ExecutionError::Internal(
754 "package debug source graph is missing the entrypoint source node",
755 ));
756 };
757 if source_node.exec_node != program.entrypoint() {
758 return Err(ExecutionError::Internal(
759 "package debug entrypoint source node does not match the program entrypoint",
760 ));
761 }
762
763 return Ok(ContinuationStack::new_with_source_node_id(program, source_node_id));
764 }
765
766 let Some(source_node_id) = package_debug_info
767 .unique_source_root_for_exec_node(program.entrypoint())
768 .map_err(|_| {
769 ExecutionError::Internal(
770 "package debug source graph has ambiguous or malformed entrypoint roots",
771 )
772 })?
773 else {
774 return Ok(ContinuationStack::new_with_optional_source_node_id(program, None));
775 };
776
777 Ok(ContinuationStack::new_with_source_node_id(program, source_node_id))
778 }
779
780 #[cfg(any(test, feature = "testing"))]
781 fn source_aware_resume_context(
782 &mut self,
783 program: &Program,
784 package_debug_info: &PackageDebugInfo,
785 entrypoint_source_node_id: Option<DebugSourceNodeId>,
786 ) -> Result<ResumeContext, ExecutionError> {
787 self.advice
788 .extend_map(program.mast_forest().advice_map())
789 .map_exec_err_no_ctx()?;
790
791 Ok(ResumeContext {
792 current_forest: program.mast_forest().clone(),
793 continuation_stack: Self::source_aware_continuation_stack(
794 program,
795 package_debug_info,
796 entrypoint_source_node_id,
797 )?,
798 kernel: program.kernel().clone(),
799 package_debug_info: Some(Arc::new(package_debug_info.clone())),
800 inline_call_contexts: Vec::new(),
801 })
802 }
803
804 fn ensure_source_aware_step_context(
805 continuation_stack: &mut ContinuationStack<Arc<MastForest>>,
806 package_debug_info: &mut Option<Arc<PackageDebugInfo>>,
807 supplied_package_debug_info: &PackageDebugInfo,
808 ) -> Result<(), ExecutionError> {
809 if package_debug_info.is_none() {
810 *package_debug_info = Some(Arc::new(supplied_package_debug_info.clone()));
811 }
812
813 if !continuation_stack.tracks_source_nodes() {
814 let source_node_id = Self::source_root_for_next_continuation(
815 continuation_stack,
816 package_debug_info.as_deref().expect("package debug info was just initialized"),
817 )?;
818 continuation_stack.start_tracking_source_nodes(source_node_id);
819 }
820
821 Ok(())
822 }
823
824 fn source_root_for_next_continuation(
825 continuation_stack: &ContinuationStack<Arc<MastForest>>,
826 package_debug_info: &PackageDebugInfo,
827 ) -> Result<Option<DebugSourceNodeId>, ExecutionError> {
828 let Some((continuation, _)) = continuation_stack.peek_continuation_with_source_node_id()
829 else {
830 return Ok(None);
831 };
832
833 let Some(exec_node) = continuation.exec_node() else {
834 return Ok(None);
835 };
836
837 package_debug_info.unique_source_root_for_exec_node(exec_node).map_err(|_| {
838 ExecutionError::Internal(
839 "package debug source graph has ambiguous or malformed continuation roots",
840 )
841 })
842 }
843
844 #[inline(always)]
846 fn resume_context_from_flow(
847 flow: ControlFlow<BreakReason<Arc<MastForest>>, StackOutputs>,
848 mut continuation_stack: ContinuationStack<Arc<MastForest>>,
849 mut current_forest: Arc<MastForest>,
850 kernel: KernelDescriptor,
851 mut package_debug_info: Option<Arc<PackageDebugInfo>>,
852 mut inline_call_contexts: Vec<Option<SourceInlineCallContext>>,
853 ) -> Result<Option<ResumeContext>, ExecutionError> {
854 match flow {
855 ControlFlow::Continue(_) => Ok(None),
856 ControlFlow::Break(break_reason) => match break_reason {
857 BreakReason::Err(err) => Err(err),
858 BreakReason::Stopped(maybe_continuation) => {
859 if let Some((continuation, source_node_id)) = maybe_continuation {
860 continuation_stack.push_with_source_node_id(continuation, source_node_id);
861 }
862
863 while matches!(
864 continuation_stack.peek_continuation(),
865 Some(Continuation::EnterForest { .. })
866 ) {
867 let Some((
868 Continuation::EnterForest {
869 forest,
870 package_debug_info: restored_debug_info,
871 inline_context_depth,
872 },
873 _,
874 )) = continuation_stack.pop_continuation_with_source_node_id()
875 else {
876 unreachable!("peeked continuation must still be EnterForest")
877 };
878 current_forest = forest;
879 package_debug_info = restored_debug_info;
880 inline_call_contexts.truncate(inline_context_depth);
881 }
882
883 Ok(Some(ResumeContext {
884 current_forest,
885 continuation_stack,
886 kernel,
887 package_debug_info,
888 inline_call_contexts,
889 }))
890 },
891 },
892 }
893 }
894
895 #[inline(always)]
897 fn current_stack_outputs(&self) -> StackOutputs {
898 StackOutputs::new(
899 &self.stack[self.stack_bot_idx..self.stack_top_idx]
900 .iter()
901 .rev()
902 .copied()
903 .collect::<Vec<_>>(),
904 )
905 .unwrap()
906 }
907
908 fn execute_impl<S, T>(
914 &mut self,
915 continuation_stack: &mut ContinuationStack<Arc<MastForest>>,
916 current_forest: &mut Arc<MastForest>,
917 kernel: &KernelDescriptor,
918 host: &mut impl SyncHost,
919 tracer: &mut T,
920 stopper: &S,
921 package_debug_info: &mut Option<Arc<PackageDebugInfo>>,
922 inline_call_contexts: &mut Vec<Option<SourceInlineCallContext>>,
923 ) -> ControlFlow<BreakReason<Arc<MastForest>>, StackOutputs>
924 where
925 S: Stopper<Processor = Self, Forest = Arc<MastForest>>,
926 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
927 {
928 while let ControlFlow::Break(internal_break_reason) = execute_impl(
929 self,
930 continuation_stack,
931 current_forest,
932 kernel,
933 host,
934 tracer,
935 stopper,
936 package_debug_info,
937 inline_call_contexts,
938 ) {
939 let current_package_debug_info = package_debug_info.as_deref();
940 let source_aware_execution =
941 current_package_debug_info.is_some() || continuation_stack.tracks_source_nodes();
942 match internal_break_reason {
943 InternalBreakReason::User(break_reason) => return ControlFlow::Break(break_reason),
944 InternalBreakReason::Emit { op_idx, continuation, source_node_id } => {
945 self.op_emit_sync(host, op_idx, current_package_debug_info, source_node_id)?;
946
947 finish_emit_op_execution(
948 continuation,
949 source_node_id,
950 self,
951 continuation_stack,
952 current_forest,
953 tracer,
954 stopper,
955 )?;
956 },
957 InternalBreakReason::LoadMastForestFromDyn { callee_hash, source_node_id } => {
958 let (root_id, new_forest, new_package_debug_info, new_source_node_id) =
959 match self.load_mast_forest_sync(
960 callee_hash,
961 host,
962 current_package_debug_info,
963 source_node_id,
964 source_aware_execution,
965 ) {
966 Ok(result) => result,
967 Err(err) => return ControlFlow::Break(BreakReason::Err(err)),
968 };
969
970 finish_load_mast_forest_from_dyn_start(
971 root_id,
972 new_forest,
973 new_package_debug_info,
974 new_source_node_id,
975 self,
976 current_forest,
977 package_debug_info,
978 inline_call_contexts.as_slice(),
979 continuation_stack,
980 tracer,
981 stopper,
982 )?;
983 },
984 InternalBreakReason::LoadMastForestFromExternal {
985 external_node_id,
986 procedure_hash,
987 source_node_id,
988 } => {
989 let inline_call_context = package_debug_info.clone().and_then(|debug_info| {
990 SourceInlineCallContext::for_source_boundary(debug_info, source_node_id)
991 });
992 let (root_id, new_forest, new_package_debug_info, new_source_node_id) =
993 match self.load_mast_forest_sync(
994 procedure_hash,
995 host,
996 current_package_debug_info,
997 source_node_id,
998 source_aware_execution,
999 ) {
1000 Ok(result) => result,
1001 Err(err) => {
1002 let maybe_enriched_err = maybe_use_caller_error_context(
1003 err,
1004 continuation_stack,
1005 current_package_debug_info,
1006 host,
1007 );
1008 return ControlFlow::Break(BreakReason::Err(maybe_enriched_err));
1009 },
1010 };
1011
1012 finish_load_mast_forest_from_external(
1013 root_id,
1014 new_forest,
1015 new_package_debug_info,
1016 new_source_node_id,
1017 inline_call_context,
1018 external_node_id,
1019 current_forest,
1020 package_debug_info,
1021 inline_call_contexts,
1022 continuation_stack,
1023 tracer,
1024 )?;
1025 },
1026 }
1027 }
1028
1029 match StackOutputs::new(
1030 &self.stack[self.stack_bot_idx..self.stack_top_idx]
1031 .iter()
1032 .rev()
1033 .copied()
1034 .collect::<Vec<_>>(),
1035 ) {
1036 Ok(stack_outputs) => ControlFlow::Continue(stack_outputs),
1037 Err(_) => ControlFlow::Break(BreakReason::Err(ExecutionError::OutputStackOverflow(
1038 self.stack_top_idx - self.stack_bot_idx - MIN_STACK_DEPTH,
1039 ))),
1040 }
1041 }
1042
1043 async fn execute_impl_async<S, T>(
1044 &mut self,
1045 continuation_stack: &mut ContinuationStack<Arc<MastForest>>,
1046 current_forest: &mut Arc<MastForest>,
1047 kernel: &KernelDescriptor,
1048 host: &mut impl Host,
1049 tracer: &mut T,
1050 stopper: &S,
1051 package_debug_info: &mut Option<Arc<PackageDebugInfo>>,
1052 inline_call_contexts: &mut Vec<Option<SourceInlineCallContext>>,
1053 ) -> ControlFlow<BreakReason<Arc<MastForest>>, StackOutputs>
1054 where
1055 S: Stopper<Processor = Self, Forest = Arc<MastForest>>,
1056 T: Tracer<Processor = Self, Forest = Arc<MastForest>>,
1057 {
1058 while let ControlFlow::Break(internal_break_reason) = execute_impl(
1059 self,
1060 continuation_stack,
1061 current_forest,
1062 kernel,
1063 host,
1064 tracer,
1065 stopper,
1066 package_debug_info,
1067 inline_call_contexts,
1068 ) {
1069 let current_package_debug_info = package_debug_info.as_deref();
1070 let source_aware_execution =
1071 current_package_debug_info.is_some() || continuation_stack.tracks_source_nodes();
1072 match internal_break_reason {
1073 InternalBreakReason::User(break_reason) => return ControlFlow::Break(break_reason),
1074 InternalBreakReason::Emit { op_idx, continuation, source_node_id } => {
1075 self.op_emit(host, op_idx, current_package_debug_info, source_node_id).await?;
1076
1077 finish_emit_op_execution(
1078 continuation,
1079 source_node_id,
1080 self,
1081 continuation_stack,
1082 current_forest,
1083 tracer,
1084 stopper,
1085 )?;
1086 },
1087 InternalBreakReason::LoadMastForestFromDyn { callee_hash, source_node_id } => {
1088 let (root_id, new_forest, new_package_debug_info, new_source_node_id) =
1089 match self
1090 .load_mast_forest(
1091 callee_hash,
1092 host,
1093 current_package_debug_info,
1094 source_node_id,
1095 source_aware_execution,
1096 )
1097 .await
1098 {
1099 Ok(result) => result,
1100 Err(err) => return ControlFlow::Break(BreakReason::Err(err)),
1101 };
1102
1103 finish_load_mast_forest_from_dyn_start(
1104 root_id,
1105 new_forest,
1106 new_package_debug_info,
1107 new_source_node_id,
1108 self,
1109 current_forest,
1110 package_debug_info,
1111 inline_call_contexts.as_slice(),
1112 continuation_stack,
1113 tracer,
1114 stopper,
1115 )?;
1116 },
1117 InternalBreakReason::LoadMastForestFromExternal {
1118 external_node_id,
1119 procedure_hash,
1120 source_node_id,
1121 } => {
1122 let inline_call_context = package_debug_info.clone().and_then(|debug_info| {
1123 SourceInlineCallContext::for_source_boundary(debug_info, source_node_id)
1124 });
1125 let (root_id, new_forest, new_package_debug_info, new_source_node_id) =
1126 match self
1127 .load_mast_forest(
1128 procedure_hash,
1129 host,
1130 current_package_debug_info,
1131 source_node_id,
1132 source_aware_execution,
1133 )
1134 .await
1135 {
1136 Ok(result) => result,
1137 Err(err) => {
1138 let maybe_enriched_err = maybe_use_caller_error_context(
1139 err,
1140 continuation_stack,
1141 current_package_debug_info,
1142 host,
1143 );
1144 return ControlFlow::Break(BreakReason::Err(maybe_enriched_err));
1145 },
1146 };
1147
1148 finish_load_mast_forest_from_external(
1149 root_id,
1150 new_forest,
1151 new_package_debug_info,
1152 new_source_node_id,
1153 inline_call_context,
1154 external_node_id,
1155 current_forest,
1156 package_debug_info,
1157 inline_call_contexts,
1158 continuation_stack,
1159 tracer,
1160 )?;
1161 },
1162 }
1163 }
1164
1165 match StackOutputs::new(
1166 &self.stack[self.stack_bot_idx..self.stack_top_idx]
1167 .iter()
1168 .rev()
1169 .copied()
1170 .collect::<Vec<_>>(),
1171 ) {
1172 Ok(stack_outputs) => ControlFlow::Continue(stack_outputs),
1173 Err(_) => ControlFlow::Break(BreakReason::Err(ExecutionError::OutputStackOverflow(
1174 self.stack_top_idx - self.stack_bot_idx - MIN_STACK_DEPTH,
1175 ))),
1176 }
1177 }
1178
1179 fn load_mast_forest_sync(
1183 &mut self,
1184 node_digest: Word,
1185 host: &mut impl SyncHost,
1186 package_debug_info: Option<&PackageDebugInfo>,
1187 source_node_id: Option<DebugSourceNodeId>,
1188 source_aware_execution: bool,
1189 ) -> Result<
1190 (
1191 MastNodeId,
1192 Arc<MastForest>,
1193 Option<Arc<PackageDebugInfo>>,
1194 Option<DebugSourceNodeId>,
1195 ),
1196 ExecutionError,
1197 > {
1198 let loaded_mast_forest = host.get_mast_forest(&node_digest).ok_or_else(|| {
1199 match (package_debug_info, source_node_id) {
1200 (Some(debug_info), Some(source_node_id)) => {
1201 crate::errors::procedure_not_found_with_package_source_context(
1202 node_digest,
1203 PackageSourceDebugContext::new(debug_info, source_node_id),
1204 host,
1205 )
1206 },
1207 _ => crate::errors::procedure_not_found_with_context(node_digest),
1208 }
1209 })?;
1210 let mast_forest = loaded_mast_forest.mast_forest().clone();
1211
1212 let root_id = mast_forest.find_procedure_root(node_digest).ok_or_else(|| {
1213 let context = match (package_debug_info, source_node_id) {
1214 (Some(debug_info), Some(source_node_id)) => {
1215 Some(PackageSourceDebugContext::new(debug_info, source_node_id))
1216 },
1217 _ => None,
1218 };
1219 malformed_mast_forest_with_context(node_digest, context, host)
1220 })?;
1221
1222 self.advice.extend_map(mast_forest.advice_map()).map_exec_err()?;
1223 let (loaded_package_debug_info, loaded_source_node_id) =
1224 Self::loaded_package_source_context(
1225 &loaded_mast_forest,
1226 root_id,
1227 source_aware_execution,
1228 )?;
1229
1230 Ok((root_id, mast_forest, loaded_package_debug_info, loaded_source_node_id))
1231 }
1232
1233 async fn load_mast_forest(
1234 &mut self,
1235 node_digest: Word,
1236 host: &mut impl Host,
1237 package_debug_info: Option<&PackageDebugInfo>,
1238 source_node_id: Option<DebugSourceNodeId>,
1239 source_aware_execution: bool,
1240 ) -> Result<
1241 (
1242 MastNodeId,
1243 Arc<MastForest>,
1244 Option<Arc<PackageDebugInfo>>,
1245 Option<DebugSourceNodeId>,
1246 ),
1247 ExecutionError,
1248 > {
1249 let loaded_mast_forest = if let Some(mast_forest) = host.get_mast_forest(&node_digest).await
1250 {
1251 mast_forest
1252 } else {
1253 return Err(match (package_debug_info, source_node_id) {
1254 (Some(debug_info), Some(source_node_id)) => {
1255 crate::errors::procedure_not_found_with_package_source_context(
1256 node_digest,
1257 PackageSourceDebugContext::new(debug_info, source_node_id),
1258 host,
1259 )
1260 },
1261 _ => crate::errors::procedure_not_found_with_context(node_digest),
1262 });
1263 };
1264 let mast_forest = loaded_mast_forest.mast_forest().clone();
1265
1266 let root_id = mast_forest.find_procedure_root(node_digest).ok_or_else(|| {
1267 let context = match (package_debug_info, source_node_id) {
1268 (Some(debug_info), Some(source_node_id)) => {
1269 Some(PackageSourceDebugContext::new(debug_info, source_node_id))
1270 },
1271 _ => None,
1272 };
1273 malformed_mast_forest_with_context(node_digest, context, host)
1274 })?;
1275
1276 self.advice.extend_map(mast_forest.advice_map()).map_exec_err()?;
1277 let (loaded_package_debug_info, loaded_source_node_id) =
1278 Self::loaded_package_source_context(
1279 &loaded_mast_forest,
1280 root_id,
1281 source_aware_execution,
1282 )?;
1283
1284 Ok((root_id, mast_forest, loaded_package_debug_info, loaded_source_node_id))
1285 }
1286
1287 fn loaded_package_source_context(
1288 loaded_mast_forest: &LoadedMastForest,
1289 root_id: MastNodeId,
1290 source_aware_execution: bool,
1291 ) -> Result<(Option<Arc<PackageDebugInfo>>, Option<DebugSourceNodeId>), ExecutionError> {
1292 if !source_aware_execution {
1293 return Ok((None, None));
1294 }
1295
1296 let Some(package_debug_info) = loaded_mast_forest
1297 .package_debug_info()
1298 .map_err(|_| ExecutionError::Internal("loaded package debug info is malformed"))?
1299 else {
1300 return Ok((None, None));
1301 };
1302
1303 let source_node_id = match package_debug_info.unique_source_root_for_exec_node(root_id) {
1304 Ok(source_node_id) => source_node_id,
1305 Err(DebugSourceGraphLookupError::AmbiguousRoot { .. }) => None,
1306 Err(_) => {
1307 return Err(ExecutionError::Internal(
1308 "loaded package debug source graph has malformed entrypoint roots",
1309 ));
1310 },
1311 };
1312
1313 Ok((Some(package_debug_info), source_node_id))
1314 }
1315
1316 pub fn execute_by_step_sync(
1318 mut self,
1319 program: &Program,
1320 host: &mut impl SyncHost,
1321 ) -> Result<StackOutputs, ExecutionError> {
1322 let mut current_resume_ctx = self.get_initial_resume_context(program)?;
1323
1324 loop {
1325 match self.step_sync(host, current_resume_ctx)? {
1326 Some(next_resume_ctx) => {
1327 current_resume_ctx = next_resume_ctx;
1328 },
1329 None => break Ok(self.current_stack_outputs()),
1330 }
1331 }
1332 }
1333
1334 #[cfg(any(test, feature = "testing"))]
1337 pub fn execute_by_step_with_package_debug_info_sync(
1338 mut self,
1339 program: &Program,
1340 package_debug_info: &PackageDebugInfo,
1341 host: &mut impl SyncHost,
1342 ) -> Result<StackOutputs, ExecutionError> {
1343 let mut current_resume_ctx =
1344 self.source_aware_resume_context(program, package_debug_info, None)?;
1345
1346 loop {
1347 match self.step_with_package_debug_info_sync(
1348 host,
1349 current_resume_ctx,
1350 package_debug_info,
1351 )? {
1352 Some(next_resume_ctx) => {
1353 current_resume_ctx = next_resume_ctx;
1354 },
1355 None => break Ok(self.current_stack_outputs()),
1356 }
1357 }
1358 }
1359
1360 #[cfg(any(test, feature = "testing"))]
1363 pub fn execute_by_step_with_package_debug_info_at_source_node_sync(
1364 mut self,
1365 program: &Program,
1366 package_debug_info: &PackageDebugInfo,
1367 entrypoint_source_node_id: DebugSourceNodeId,
1368 host: &mut impl SyncHost,
1369 ) -> Result<StackOutputs, ExecutionError> {
1370 let mut current_resume_ctx = self.source_aware_resume_context(
1371 program,
1372 package_debug_info,
1373 Some(entrypoint_source_node_id),
1374 )?;
1375
1376 loop {
1377 match self.step_with_package_debug_info_sync(
1378 host,
1379 current_resume_ctx,
1380 package_debug_info,
1381 )? {
1382 Some(next_resume_ctx) => {
1383 current_resume_ctx = next_resume_ctx;
1384 },
1385 None => break Ok(self.current_stack_outputs()),
1386 }
1387 }
1388 }
1389
1390 #[inline(always)]
1392 pub async fn execute_by_step(
1393 mut self,
1394 program: &Program,
1395 host: &mut impl Host,
1396 ) -> Result<StackOutputs, ExecutionError> {
1397 let mut current_resume_ctx = self.get_initial_resume_context(program)?;
1398 let mut processor = self;
1399
1400 loop {
1401 match processor.step(host, current_resume_ctx).await? {
1402 Some(next_resume_ctx) => {
1403 current_resume_ctx = next_resume_ctx;
1404 },
1405 None => break Ok(processor.current_stack_outputs()),
1406 }
1407 }
1408 }
1409
1410 #[cfg(any(test, feature = "testing"))]
1412 #[inline(always)]
1413 pub async fn execute_by_step_with_package_debug_info(
1414 mut self,
1415 program: &Program,
1416 package_debug_info: &PackageDebugInfo,
1417 host: &mut impl Host,
1418 ) -> Result<StackOutputs, ExecutionError> {
1419 let mut current_resume_ctx =
1420 self.source_aware_resume_context(program, package_debug_info, None)?;
1421 let mut processor = self;
1422
1423 loop {
1424 match processor
1425 .step_with_package_debug_info(host, current_resume_ctx, package_debug_info)
1426 .await?
1427 {
1428 Some(next_resume_ctx) => {
1429 current_resume_ctx = next_resume_ctx;
1430 },
1431 None => break Ok(processor.current_stack_outputs()),
1432 }
1433 }
1434 }
1435
1436 #[cfg(any(test, feature = "testing"))]
1439 #[inline(always)]
1440 pub async fn execute_by_step_with_package_debug_info_at_source_node(
1441 mut self,
1442 program: &Program,
1443 package_debug_info: &PackageDebugInfo,
1444 entrypoint_source_node_id: DebugSourceNodeId,
1445 host: &mut impl Host,
1446 ) -> Result<StackOutputs, ExecutionError> {
1447 let mut current_resume_ctx = self.source_aware_resume_context(
1448 program,
1449 package_debug_info,
1450 Some(entrypoint_source_node_id),
1451 )?;
1452 let mut processor = self;
1453
1454 loop {
1455 match processor
1456 .step_with_package_debug_info(host, current_resume_ctx, package_debug_info)
1457 .await?
1458 {
1459 Some(next_resume_ctx) => {
1460 current_resume_ctx = next_resume_ctx;
1461 },
1462 None => break Ok(processor.current_stack_outputs()),
1463 }
1464 }
1465 }
1466
1467 #[cfg(any(test, feature = "testing"))]
1471 pub fn execute_mut_sync(
1472 &mut self,
1473 program: &Program,
1474 host: &mut impl SyncHost,
1475 ) -> Result<StackOutputs, ExecutionError> {
1476 let mut continuation_stack = ContinuationStack::new(program);
1477 let mut current_forest = program.mast_forest().clone();
1478 let mut package_debug_info = None;
1479 let mut inline_call_contexts = Vec::new();
1480
1481 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
1482
1483 let flow = self.execute_impl(
1484 &mut continuation_stack,
1485 &mut current_forest,
1486 program.kernel(),
1487 host,
1488 &mut NoopTracer,
1489 &NeverStopper,
1490 &mut package_debug_info,
1491 &mut inline_call_contexts,
1492 );
1493 Self::stack_result_from_flow(flow)
1494 }
1495
1496 #[cfg(any(test, feature = "testing"))]
1498 #[inline(always)]
1499 pub async fn execute_mut(
1500 &mut self,
1501 program: &Program,
1502 host: &mut impl Host,
1503 ) -> Result<StackOutputs, ExecutionError> {
1504 let mut continuation_stack = ContinuationStack::new(program);
1505 let mut current_forest = program.mast_forest().clone();
1506 let mut package_debug_info = None;
1507 let mut inline_call_contexts = Vec::new();
1508
1509 self.advice.extend_map(current_forest.advice_map()).map_exec_err_no_ctx()?;
1510
1511 let flow = self
1512 .execute_impl_async(
1513 &mut continuation_stack,
1514 &mut current_forest,
1515 program.kernel(),
1516 host,
1517 &mut NoopTracer,
1518 &NeverStopper,
1519 &mut package_debug_info,
1520 &mut inline_call_contexts,
1521 )
1522 .await;
1523 Self::stack_result_from_flow(flow)
1524 }
1525}
1526
1527#[cfg(all(test, feature = "std"))]
1528mod tests {
1529 use super::FastProcessor;
1530
1531 #[test]
1532 fn sole_rayon_worker_requires_buffered_trace_building() {
1533 rayon::ThreadPoolBuilder::new()
1534 .num_threads(1)
1535 .build()
1536 .unwrap()
1537 .install(|| assert!(FastProcessor::rayon_has_no_parallel_worker()));
1538 }
1539}