1use super::{
2 begin_participant_flights_dispatch, begin_submission_wave_participant_flights_dispatch,
3 finalize_session_frames_with_boundary, fmt, invalid_resource, issue_batch_invocation_id,
4 poison_session_frame, prepare_participant_flights, prepare_submission_wave_participant_flights,
5 reset_participant_flights_after_definitely_not_submitted,
6 reset_submission_wave_participant_flights_after_definitely_not_submitted,
7 session_participant_key, ActiveInvocationWaveGuard, ActiveSequenceFrame, AdmissionDeferred,
8 AdmissionFitPolicy, AdmissionPressureAction, AdmissionRejected, AdmittedRequestResources,
9 AdmittedSequenceResources, AdmittedStepParticipant, AllocationLifetime, Arc, AtomicU64,
10 BackingInitializationEncodeError, BackingPrepareDecision, BatchCapacityClaimDecision,
11 BatchInvocationId, BatchParticipantAuthority, BatchParticipantTokenSpan, BatchStepId,
12 BatchWorkShape, ClaimedBackingTransaction, ClaimedSubmissionWaveBacking,
13 DeferredDeviceCleanupDomainId, DeviceCommandBatch, DeviceRuntime, Digest,
14 DynamicBackingDeferred, DynamicDeferredMaintenanceOutcome, DynamicResourceDescriptor,
15 ExecutionLane, ExecutionLaneId, InvocationRegistry, InvocationResourceAdmissionRequest,
16 LaneBackingPrepareDecision, LogicalAdmissionCoordinatorId, LogicalBackingBufferView,
17 LogicalBackingSliceAuthority, LogicalBatchCapacityLease, NodeId, Ordering,
18 ParticipantFlightCandidate, ParticipantFlightPhase, ParticipantNodeKey, PlanBackingDeferral,
19 PlanCapacityWaitRegistration, PreparedBackingInitializations, PreparedParticipantFlightHold,
20 PreparedSubmissionWaveParticipantFlightHold, RequestStateHazardAcquireDecision,
21 RequestStateHazardDeferral, RequestStateHazardParticipant, RequestStateHazardPermit,
22 RequestStateHazardPoison, RequestStateHazardSplitRequired,
23 RequestStateHazardTerminalDisposition, ResourceId, SequenceAuthorityId,
24 SequenceBackingSnapshot, SequenceRecoveryRegistry, SequenceSessionEpoch,
25 SequenceSessionFingerprint, Serialize, Sha256, StaticProvisioningLease,
26 StepFinalizationFailure, StepFrameFinalization, StepParticipantFrameAssignment,
27 StepParticipantRetirement, StepParticipantRetirementDisposition, StepResourceLease,
28 StepRetirementReceipt, TokenSpanWork, TrustedPlanRuntimeEvidence, VNextError,
29 SEQUENCE_DISPATCH_POISONED_BIT,
30};
31use crate::vnext::ReusableExecutionBucketSpec;
32use std::time::{Duration, Instant};
33
34#[derive(Debug, Clone, Copy, PartialEq, Eq)]
35pub enum StepResourceAdmissionProfilePhase {
36 AuthorityAndPolicyValidate,
37 DemandEvaluate,
38 BackingClaim,
39 LogicalCapacityClaim,
40 TransactionValidateAndFingerprint,
41 FrameCaptureAndLease,
42}
43
44#[inline(always)]
45pub(super) fn step_admission_profile_start<const PROFILE: bool>() -> Option<Instant> {
46 PROFILE.then(Instant::now)
47}
48
49#[inline(always)]
50pub(super) fn record_step_admission_profile<const PROFILE: bool, F>(
51 observer: &mut F,
52 phase: StepResourceAdmissionProfilePhase,
53 started: Option<Instant>,
54) where
55 F: FnMut(StepResourceAdmissionProfilePhase, Duration),
56{
57 if PROFILE {
58 observer(
59 phase,
60 started
61 .expect("profiled step admission phase owns a start instant")
62 .elapsed(),
63 );
64 }
65}
66
67#[derive(Debug)]
68pub enum ExecutionStreamCreationError<E> {
69 Contract(VNextError),
70 Runtime(E),
71}
72
73#[derive(Debug, Clone, Copy, PartialEq, Eq)]
77pub(super) enum BoundExecutionStreamState {
78 Ready,
79 InUse,
80 Poisoned,
81}
82
83#[must_use = "an execution stream must be activated through its resource lease"]
84pub struct BoundExecutionStream<R>
85where
86 R: DeviceRuntime,
87{
88 pub(super) runtime: Arc<R>,
91 pub(super) coordinator_id: LogicalAdmissionCoordinatorId,
92 pub(super) sequence_authority: SequenceAuthorityId,
93 pub(super) stream: Option<R::Stream>,
94 pub(super) state: BoundExecutionStreamState,
95 pub(super) sequence_recovery: Arc<SequenceRecoveryRegistry<R>>,
96 pub(super) sequence_dispatch_gate: Arc<AtomicU64>,
97 pub(super) abandoned_sequence: Option<(u32, u64)>,
98 pub(super) resources: Arc<AdmittedSequenceResources<R>>,
99}
100
101impl<R> BoundExecutionStream<R>
102where
103 R: DeviceRuntime,
104{
105 pub(super) fn stream(&self) -> &R::Stream {
106 self.stream
107 .as_ref()
108 .expect("bound execution stream retains its raw stream")
109 }
110
111 pub(super) fn stream_mut(&mut self) -> &mut R::Stream {
112 self.stream
113 .as_mut()
114 .expect("bound execution stream retains its raw stream")
115 }
116}
117
118impl<R> Drop for BoundExecutionStream<R>
119where
120 R: DeviceRuntime,
121{
122 fn drop(&mut self) {
123 let Some(key) = self.abandoned_sequence.take() else {
124 return;
125 };
126 self.sequence_dispatch_gate
127 .fetch_or(SEQUENCE_DISPATCH_POISONED_BIT, Ordering::AcqRel);
128 if let Some(stream) = self.stream.take() {
129 self.sequence_recovery.attach_stream(key, stream);
130 }
131 }
132}
133
134impl<R> StepResourceLease<R>
135where
136 R: DeviceRuntime,
137{
138 pub(super) fn new(
139 participants: Vec<AdmittedStepParticipant<R>>,
140 execution_lane: Arc<ExecutionLane<R>>,
141 reusable_execution_bucket: Option<ReusableExecutionBucketSpec>,
142 batch_step_id: BatchStepId,
143 claimed_backing: ClaimedBackingTransaction,
144 ) -> Result<Self, VNextError> {
145 if participants.is_empty() {
146 return Err(invalid_resource(
147 "step resources require a non-empty participant set",
148 ));
149 }
150 let coordinator = participants[0]
151 .session
152 .resources()
153 .request
154 .plan
155 .logical_admission();
156 if claimed_backing.work_shape().participants().len() != participants.len()
157 || claimed_backing
158 .work_shape()
159 .participants()
160 .iter()
161 .zip(&participants)
162 .any(|(authority, participant)| {
163 authority.sequence_authority() != participant.session.sequence_authority()
164 || authority.request_authority() != participant.session.request_authority()
165 })
166 {
167 return Err(invalid_resource(
168 "step work shape differs from its exact batch participants",
169 ));
170 }
171 if let Some(capacity) = claimed_backing.logical_capacity() {
172 let parents_match = capacity
173 .parents()
174 .iter()
175 .map(|parent| (parent.sequence(), parent.request()))
176 .eq(participants.iter().map(|participant| {
177 (
178 participant.session.sequence_authority(),
179 participant.session.request_authority(),
180 )
181 }));
182 if !coordinator.owns_batch_capacity_claim(capacity) || !parents_match {
183 return Err(invalid_resource(
184 "step capacity authority differs from its exact batch participants",
185 ));
186 }
187 }
188 Ok(Self {
189 claimed_backing,
190 participants,
191 invocation_registry: Arc::new(InvocationRegistry::default()),
192 execution_lane,
193 reusable_execution_bucket,
194 batch_step_id,
195 completed_boundary: super::Mutex::new(super::StepCompletedBoundarySlot::default()),
196 finalized: false,
197 })
198 }
199
200 pub const fn batch_step_id(&self) -> BatchStepId {
201 self.batch_step_id
202 }
203
204 pub fn execution_lane(&self) -> &Arc<ExecutionLane<R>> {
205 &self.execution_lane
206 }
207
208 pub fn reusable_execution_bucket(&self) -> Option<&ReusableExecutionBucketSpec> {
209 self.reusable_execution_bucket.as_ref()
210 }
211
212 pub fn try_retire_normal(
213 self: Arc<Self>,
214 ) -> Result<StepRetirementReceipt, StepFinalizationFailure<R>> {
215 Self::try_finalize(self, StepFrameFinalization::Commit)
216 }
217
218 pub fn try_abort(self: Arc<Self>) -> Result<StepRetirementReceipt, StepFinalizationFailure<R>> {
219 Self::try_finalize(self, StepFrameFinalization::Abort)
220 }
221
222 pub fn try_rollback_unsubmitted(
226 self: Arc<Self>,
227 ) -> Result<StepRetirementReceipt, StepFinalizationFailure<R>> {
228 Self::try_finalize(self, StepFrameFinalization::RollbackUnsubmitted)
229 }
230
231 fn try_finalize(
232 step: Arc<Self>,
233 finalization: StepFrameFinalization,
234 ) -> Result<StepRetirementReceipt, StepFinalizationFailure<R>> {
235 let mut step = match Arc::try_unwrap(step) {
236 Ok(step) => step,
237 Err(step) => {
238 return Err(StepFinalizationFailure {
239 step,
240 error: invalid_resource(
241 "step cannot finalize while an invocation or scheduler clone retains it",
242 ),
243 });
244 }
245 };
246 if finalization == StepFrameFinalization::RollbackUnsubmitted {
247 if let Err(error) = step.invocation_registry.ensure_pristine_for_step_rollback() {
248 return Err(StepFinalizationFailure {
249 step: Arc::new(step),
250 error,
251 });
252 }
253 }
254 let dispositions = match step.finalize_participants(finalization) {
255 Ok(dispositions) => dispositions,
256 Err(error) => {
257 return Err(StepFinalizationFailure {
258 step: Arc::new(step),
259 error,
260 });
261 }
262 };
263 let participants = step
264 .participants
265 .iter()
266 .zip(dispositions)
267 .map(|(participant, disposition)| StepParticipantRetirement {
268 assignment: StepParticipantFrameAssignment::new(
269 participant.session.sequence_authority(),
270 participant.session.request_authority(),
271 participant.frame.frame_id,
272 ),
273 disposition,
274 })
275 .collect();
276 Ok(StepRetirementReceipt {
277 batch_step_id: step.batch_step_id,
278 participants,
279 })
280 }
281
282 fn finalize_participants(
283 &mut self,
284 finalization: StepFrameFinalization,
285 ) -> Result<Vec<StepParticipantRetirementDisposition>, VNextError> {
286 if self.finalized {
287 return Err(invalid_resource("step resources are already finalized"));
288 }
289 let mut holds = self
290 .participants
291 .iter_mut()
292 .map(|participant| &mut participant.frame)
293 .collect::<Vec<_>>();
294 let completed = self
295 .completed_boundary
296 .get_mut()
297 .map_err(|_| invalid_resource("step completed-boundary mutex is poisoned"))?;
298 let dispositions =
299 finalize_session_frames_with_boundary(&mut holds, finalization, completed.proof())?;
300 completed.consume();
301 self.finalized = true;
302 Ok(dispositions)
303 }
304
305 pub fn participant_count(&self) -> u32 {
306 u32::try_from(self.participants.len())
307 .expect("step participant count was validated before admission")
308 }
309
310 pub fn coordinator_id(&self) -> LogicalAdmissionCoordinatorId {
311 self.participants[0].session.resources().coordinator_id()
312 }
313
314 pub fn participants(
315 &self,
316 ) -> impl ExactSizeIterator<Item = &Arc<AdmittedSequenceResources<R>>> {
317 self.participants
318 .iter()
319 .map(|participant| participant.session.resources())
320 }
321
322 pub(crate) fn participant_backing_snapshot(
323 &self,
324 participant: BatchParticipantAuthority,
325 ) -> Result<&Arc<SequenceBackingSnapshot<R>>, VNextError> {
326 let index = self
327 .participants
328 .binary_search_by_key(&participant.canonical_key(), |candidate| {
329 session_participant_key(&candidate.session)
330 })
331 .map_err(|_| invalid_resource("step does not own that backing participant"))?;
332 self.participants
333 .get(index)
334 .map(|participant| &participant.backing_snapshot)
335 .ok_or_else(|| invalid_resource("step backing participant mapping is inconsistent"))
336 }
337
338 pub(crate) fn participant_backing_view(
339 &self,
340 authority: BatchParticipantAuthority,
341 resource_id: &ResourceId,
342 ) -> Result<LogicalBackingBufferView<'_, R::Buffer>, VNextError> {
343 let index = self
344 .participants
345 .binary_search_by_key(&authority.canonical_key(), |candidate| {
346 session_participant_key(&candidate.session)
347 })
348 .map_err(|_| invalid_resource("step does not own that backing participant"))?;
349 let participant = self
350 .participants
351 .get(index)
352 .ok_or_else(|| invalid_resource("step backing participant mapping is inconsistent"))?;
353 let authorities = participant.backing_snapshot.backing_slices_for(resource_id);
354 if !authorities.is_empty() {
355 return participant
356 .session
357 .resources()
358 .request
359 .plan
360 .dynamic_pools()
361 .view_many(authorities);
362 }
363 participant
364 .session
365 .resources()
366 .request
367 .backing_view(resource_id)
368 }
369
370 pub fn participant_frames(
371 &self,
372 ) -> impl ExactSizeIterator<Item = StepParticipantFrameAssignment> + '_ {
373 self.participants.iter().map(|participant| {
374 StepParticipantFrameAssignment::new(
375 participant.session.sequence_authority(),
376 participant.session.request_authority(),
377 participant.frame.frame_id,
378 )
379 })
380 }
381
382 pub fn backing_slices(&self) -> &[LogicalBackingSliceAuthority] {
383 self.claimed_backing.backing_slices()
384 }
385
386 pub fn logical_capacity(&self) -> Option<&LogicalBatchCapacityLease> {
387 self.claimed_backing.logical_capacity()
388 }
389
390 pub fn work_shape(&self) -> &BatchWorkShape {
391 self.claimed_backing.work_shape()
392 }
393
394 pub fn bind_invocation_work_shape(
395 &self,
396 mut participant_tokens: Vec<(BatchParticipantAuthority, TokenSpanWork)>,
397 ) -> Result<BatchWorkShape, VNextError> {
398 participant_tokens.sort_by_key(|(participant, _)| participant.canonical_key());
399 if participant_tokens.is_empty()
400 || participant_tokens
401 .windows(2)
402 .any(|pair| pair[0].0.canonical_key() == pair[1].0.canonical_key())
403 || participant_tokens.iter().any(|(authority, _)| {
404 self.participants
405 .binary_search_by_key(&authority.canonical_key(), |participant| {
406 session_participant_key(&participant.session)
407 })
408 .is_err()
409 })
410 {
411 return Err(invalid_resource(
412 "invocation token work must bind a unique non-empty step participant subset",
413 ));
414 }
415 BatchWorkShape::new(
416 participant_tokens
417 .into_iter()
418 .map(|(participant, token_span)| {
419 BatchParticipantTokenSpan::new(participant, token_span)
420 })
421 .collect(),
422 )
423 }
424
425 pub fn bind_all_invocation_work_shape(
426 &self,
427 token_spans: Vec<TokenSpanWork>,
428 ) -> Result<BatchWorkShape, VNextError> {
429 if token_spans.len() != self.participants.len() {
430 return Err(invalid_resource(
431 "invocation token work count differs from all step participants",
432 ));
433 }
434 self.bind_invocation_work_shape(
435 self.participants
436 .iter()
437 .zip(token_spans)
438 .map(|(participant, token_span)| {
439 (
440 BatchParticipantAuthority::new(
441 participant.session.sequence_authority(),
442 participant.session.request_authority(),
443 ),
444 token_span,
445 )
446 })
447 .collect(),
448 )
449 }
450
451 pub fn shared_all_invocation_work_shape(
454 &self,
455 token_spans: &[TokenSpanWork],
456 ) -> Result<Arc<BatchWorkShape>, VNextError> {
457 let work_shape = self.claimed_backing.work_shape();
458 if token_spans.len() != self.participants.len()
459 || work_shape
460 .participant_work()
461 .iter()
462 .zip(token_spans)
463 .any(|(bound, requested)| bound.token_span() != requested)
464 {
465 return Err(invalid_resource(
466 "submission wave token work differs from its admitted step work authority",
467 ));
468 }
469 Ok(Arc::clone(self.claimed_backing.work_shape_arc()))
470 }
471
472 pub fn claimed_backing(&self) -> &ClaimedBackingTransaction {
473 &self.claimed_backing
474 }
475
476 pub fn static_provisioning(&self) -> Option<&StaticProvisioningLease<R>> {
477 self.participants[0]
478 .session
479 .resources()
480 .static_provisioning()
481 }
482
483 pub fn plan_evidence(&self) -> TrustedPlanRuntimeEvidence {
484 self.participants[0].session.resources().plan_evidence()
485 }
486
487 pub(crate) fn backing_view(
488 &self,
489 resource_id: &ResourceId,
490 ) -> Result<LogicalBackingBufferView<'_, R::Buffer>, VNextError> {
491 if let Some(authority) = self
492 .claimed_backing
493 .backing_slices()
494 .iter()
495 .find(|authority| authority.resource_id() == resource_id)
496 {
497 return self.participants[0]
498 .session
499 .resources()
500 .request
501 .plan
502 .dynamic_pools()
503 .view(authority);
504 }
505 Err(invalid_resource(format!(
506 "resource `{resource_id}` is not step-shared backing"
507 )))
508 }
509
510 pub(crate) fn dynamic_descriptor(
511 &self,
512 resource_id: &ResourceId,
513 ) -> Result<&DynamicResourceDescriptor, VNextError> {
514 let pools = self.participants[0]
515 .session
516 .resources()
517 .request
518 .plan
519 .dynamic_pools();
520 let mut matches = pools.domains.iter().filter_map(|domain| {
521 domain
522 .descriptors
523 .binary_search_by(|descriptor| descriptor.base_resource_id().cmp(resource_id))
524 .ok()
525 .map(|index| &domain.descriptors[index])
526 });
527 let descriptor = matches.next().ok_or_else(|| {
528 invalid_resource(format!(
529 "resource `{resource_id}` has no dynamic plan descriptor"
530 ))
531 })?;
532 if matches.next().is_some() {
533 return Err(invalid_resource(format!(
534 "resource `{resource_id}` has duplicate dynamic plan descriptors"
535 )));
536 }
537 Ok(descriptor)
538 }
539
540 pub(crate) fn participant_backing_views(
541 &self,
542 resource_id: &ResourceId,
543 ) -> Result<Vec<(SequenceAuthorityId, LogicalBackingBufferView<'_, R::Buffer>)>, VNextError>
544 {
545 self.participants
546 .iter()
547 .map(|participant| {
548 let authority = BatchParticipantAuthority::new(
549 participant.session.sequence_authority(),
550 participant.session.request_authority(),
551 );
552 Ok((
553 participant.session.sequence_authority(),
554 self.participant_backing_view(authority, resource_id)?,
555 ))
556 })
557 .collect()
558 }
559
560 pub fn try_admit_invocation(
561 self: &Arc<Self>,
562 request: InvocationResourceAdmissionRequest,
563 ) -> Result<InvocationResourceAdmissionDecision<R>, VNextError> {
564 let _lifecycle = self.participants[0]
565 .session
566 .resources()
567 .request
568 .plan
569 .resources
570 .read_lifecycle("admit a step invocation")?;
571 if self.claimed_backing.has_shared_physical_claims() {
572 return Err(invalid_resource(
573 "shared Step activation backing requires an ordered single-fence submission wave",
574 ));
575 }
576 let deferred_node_id = request.node_id().clone();
577 let deferred_work_fingerprint = request.work_shape().fingerprint().to_owned();
578 let prepared = match self.prepare_invocation_scope(request)? {
579 PreparedInvocationScopeDecision::Prepared(prepared) => prepared,
580 PreparedInvocationScopeDecision::Deferred(deferred) => {
581 return Ok(InvocationResourceAdmissionDecision::Deferred(deferred));
582 }
583 PreparedInvocationScopeDecision::BackingDeferred(deferred) => {
584 return Ok(InvocationResourceAdmissionDecision::BackingDeferred(
585 InvocationAdmissionBackingDeferral::new(
586 Arc::clone(self),
587 deferred,
588 deferred_node_id,
589 deferred_work_fingerprint,
590 )?,
591 ));
592 }
593 PreparedInvocationScopeDecision::PermanentRejected(rejected) => {
594 return Ok(InvocationResourceAdmissionDecision::PermanentRejected(
595 rejected,
596 ));
597 }
598 PreparedInvocationScopeDecision::RequestStateDeferred(deferred) => {
599 return Ok(InvocationResourceAdmissionDecision::RequestStateDeferred(
600 deferred,
601 ));
602 }
603 PreparedInvocationScopeDecision::RequestStateSplitRequired(split) => {
604 return Ok(InvocationResourceAdmissionDecision::RequestStateSplitRequired(split));
605 }
606 PreparedInvocationScopeDecision::RequestStatePoisoned(poison) => {
607 return Ok(InvocationResourceAdmissionDecision::RequestStatePoisoned(
608 poison,
609 ));
610 }
611 };
612 let PreparedInvocationScope {
613 participants,
614 participant_frames,
615 node_id,
616 claimed_backing,
617 flight_candidates,
618 request_state_hazards,
619 } = prepared;
620 let batch_invocation_id = issue_batch_invocation_id()?;
621 let prepared_participant_flights =
622 prepare_participant_flights(&flight_candidates, &node_id)?;
623 let topology_keys = participant_frames
624 .iter()
625 .map(|assignment| {
626 ParticipantNodeKey::new(
627 assignment.participant(),
628 assignment.frame_id(),
629 node_id.clone(),
630 )
631 })
632 .collect();
633 let active_wave = self.invocation_registry.enter(
634 topology_keys,
635 batch_invocation_id,
636 claimed_backing.work_shape().fingerprint(),
637 )?;
638 Ok(InvocationResourceAdmissionDecision::Admitted(
639 InvocationResourceLease::new(
640 Arc::clone(self),
641 participants,
642 participant_frames,
643 node_id,
644 batch_invocation_id,
645 claimed_backing,
646 prepared_participant_flights,
647 active_wave,
648 request_state_hazards,
649 )?,
650 ))
651 }
652
653 pub fn try_prepare_submission_wave(
654 self: &Arc<Self>,
655 requests: Vec<InvocationResourceAdmissionRequest>,
656 ) -> Result<StepSubmissionWaveAdmissionDecision<R>, VNextError> {
657 let plan = &self.participants[0].session.resources().request.plan;
658 let plan_nodes = plan.nodes();
659 if requests.is_empty()
660 || requests.len() != plan_nodes.len()
661 || requests
662 .iter()
663 .zip(plan_nodes)
664 .any(|(request, node)| request.node_id() != node.id())
665 {
666 return Err(invalid_resource(
667 "submission wave must cover every plan node exactly once in immutable plan order",
668 ));
669 }
670 let fit_policy = requests[0].fit_policy();
671 let pressure_action = requests[0].pressure_action();
672 if requests.iter().any(|request| {
673 request.fit_policy() != fit_policy || request.pressure_action() != pressure_action
674 }) {
675 return Err(invalid_resource(
676 "submission wave nodes require one admission fit and pressure policy",
677 ));
678 }
679 let work_shape = Arc::clone(&requests[0].work_shape);
680 if requests
681 .iter()
682 .any(|request| request.work_shape.as_ref() != work_shape.as_ref())
683 {
684 return Err(invalid_resource(
685 "submission wave nodes must share one canonical work authority",
686 ));
687 }
688 self.prepare_full_plan_submission_wave(work_shape, fit_policy, pressure_action)
689 }
690
691 pub fn try_prepare_determinism_submission_wave(
694 self: &Arc<Self>,
695 requests: Vec<InvocationResourceAdmissionRequest>,
696 ) -> Result<StepSubmissionWaveAdmissionDecision<R>, VNextError> {
697 let plan = &self.participants[0].session.resources().request.plan;
698 let plan_nodes = plan.nodes();
699 if requests.is_empty() {
700 return Err(invalid_resource(
701 "determinism submission wave requires a non-empty plan node scope",
702 ));
703 }
704 let node_indices = requests
705 .iter()
706 .map(|request| {
707 plan_nodes
708 .iter()
709 .position(|node| node.id() == request.node_id())
710 .ok_or_else(|| {
711 invalid_resource(
712 "determinism submission wave references an unknown plan node",
713 )
714 })
715 })
716 .collect::<Result<Vec<_>, _>>()?;
717 if node_indices.windows(2).any(|pair| pair[0] >= pair[1]) {
718 return Err(invalid_resource(
719 "determinism submission wave nodes must be unique and in immutable plan order",
720 ));
721 }
722 let fit_policy = requests[0].fit_policy();
723 let pressure_action = requests[0].pressure_action();
724 if requests.iter().any(|request| {
725 request.fit_policy() != fit_policy || request.pressure_action() != pressure_action
726 }) {
727 return Err(invalid_resource(
728 "determinism submission wave nodes require one admission fit and pressure policy",
729 ));
730 }
731 let work_shape = Arc::clone(&requests[0].work_shape);
732 if requests
733 .iter()
734 .any(|request| request.work_shape.as_ref() != work_shape.as_ref())
735 {
736 return Err(invalid_resource(
737 "determinism submission wave nodes must share one canonical work authority",
738 ));
739 }
740 self.prepare_submission_wave(
741 work_shape,
742 fit_policy,
743 pressure_action,
744 &node_indices,
745 SubmissionWavePurpose::DeterminismProbe,
746 )
747 }
748
749 pub fn try_prepare_full_plan_submission_wave(
754 self: &Arc<Self>,
755 work_shape: Arc<BatchWorkShape>,
756 fit_policy: AdmissionFitPolicy,
757 pressure_action: AdmissionPressureAction,
758 ) -> Result<StepSubmissionWaveAdmissionDecision<R>, VNextError> {
759 self.prepare_full_plan_submission_wave(work_shape, fit_policy, pressure_action)
760 }
761
762 fn prepare_full_plan_submission_wave(
763 self: &Arc<Self>,
764 work_shape: Arc<BatchWorkShape>,
765 fit_policy: AdmissionFitPolicy,
766 pressure_action: AdmissionPressureAction,
767 ) -> Result<StepSubmissionWaveAdmissionDecision<R>, VNextError> {
768 let node_count = self.participants[0]
769 .session
770 .resources()
771 .request
772 .plan
773 .nodes()
774 .len();
775 let node_indices = (0..node_count).collect::<Vec<_>>();
776 self.prepare_submission_wave(
777 work_shape,
778 fit_policy,
779 pressure_action,
780 &node_indices,
781 SubmissionWavePurpose::FullPlan,
782 )
783 }
784
785 fn prepare_submission_wave(
786 self: &Arc<Self>,
787 work_shape: Arc<BatchWorkShape>,
788 fit_policy: AdmissionFitPolicy,
789 pressure_action: AdmissionPressureAction,
790 node_indices: &[usize],
791 purpose: SubmissionWavePurpose,
792 ) -> Result<StepSubmissionWaveAdmissionDecision<R>, VNextError> {
793 let _lifecycle = self.participants[0]
794 .session
795 .resources()
796 .request
797 .plan
798 .resources
799 .read_lifecycle("prepare a step submission wave")?;
800 let plan = &self.participants[0].session.resources().request.plan;
801 let plan_nodes = plan.nodes();
802 if plan_nodes.is_empty()
803 || node_indices.is_empty()
804 || node_indices.windows(2).any(|pair| pair[0] >= pair[1])
805 || node_indices
806 .last()
807 .is_some_and(|node_index| *node_index >= plan_nodes.len())
808 {
809 return Err(invalid_resource(
810 "submission wave requires a non-empty canonical immutable-plan node scope",
811 ));
812 }
813 if work_shape.participants().len() != self.participants.len()
814 || work_shape
815 .participants()
816 .iter()
817 .zip(&self.participants)
818 .any(|(authority, participant)| {
819 authority.canonical_key() != session_participant_key(&participant.session)
820 })
821 {
822 return Err(invalid_resource(
823 "submission wave must bind every step participant exactly once",
824 ));
825 }
826 let participant_authority =
827 Arc::new(self.prepare_participant_authority(Arc::clone(&work_shape), fit_policy)?);
828 let prepared_nodes = node_indices
829 .iter()
830 .copied()
831 .map(|plan_node_index| {
832 PreparedStepSubmissionNode::new(plan_node_index, Arc::clone(&participant_authority))
833 })
834 .collect::<Vec<_>>();
835 let request_participants = participant_authority
836 .participants
837 .iter()
838 .map(|participant| {
839 let request = Arc::clone(participant.request_resources());
840 RequestStateHazardParticipant::new(
841 Arc::clone(&request.plan.dynamic_pools().request_state_hazards),
842 request.request_authority(),
843 request,
844 )
845 })
846 .collect::<Vec<_>>();
847 let request_state_hazards = match plan
848 .dynamic_pools()
849 .request_state_hazards
850 .try_acquire(&request_participants, node_indices)?
851 {
852 RequestStateHazardAcquireDecision::Acquired(permit) => permit,
853 RequestStateHazardAcquireDecision::Deferred(deferred) => {
854 return Ok(StepSubmissionWaveAdmissionDecision::RequestStateDeferred(
855 deferred,
856 ));
857 }
858 RequestStateHazardAcquireDecision::SplitRequired(split) => {
859 return Ok(StepSubmissionWaveAdmissionDecision::RequestStateSplitRequired(split));
860 }
861 RequestStateHazardAcquireDecision::Poisoned(poison) => {
862 return Ok(StepSubmissionWaveAdmissionDecision::RequestStatePoisoned(
863 poison,
864 ));
865 }
866 };
867 let immediate_shape = work_shape.immediate_shape();
868 let fit_shape = match fit_policy {
869 AdmissionFitPolicy::ImmediateOnly => immediate_shape,
870 AdmissionFitPolicy::FullInputMustFit => work_shape.fit_shape(),
871 };
872 let (demand, requested_slices) = plan.submission_wave_demand(
873 immediate_shape,
874 fit_shape,
875 self.reusable_execution_bucket.as_ref(),
876 fit_policy,
877 pressure_action,
878 )?;
879 let prepared_backing = match plan
880 .prepare_lane_stable_backing_slices(&self.execution_lane, requested_slices)?
881 {
882 LaneBackingPrepareDecision::Prepared(prepared) => prepared,
883 LaneBackingPrepareDecision::Deferred(deferred) => {
884 let deferred_node_work_fingerprints = prepared_nodes
885 .iter()
886 .map(|node| {
887 (
888 node.node_id().clone(),
889 node.work_shape().fingerprint().to_owned(),
890 )
891 })
892 .collect();
893 return Ok(StepSubmissionWaveAdmissionDecision::BackingDeferred(
894 StepSubmissionWaveBackingDeferral::new(
895 Arc::clone(self),
896 deferred,
897 deferred_node_work_fingerprints,
898 )?,
899 ));
900 }
901 };
902 let logical_capacity = if demand.immediate_claim().is_empty() {
903 None
904 } else {
905 let parents = self
906 .participants
907 .iter()
908 .map(|participant| participant.session.resources().logical_lease())
909 .collect::<Vec<_>>();
910 match plan
911 .logical_admission()
912 .try_claim_for_sequences(&parents, &demand)?
913 {
914 BatchCapacityClaimDecision::Claimed(capacity) => {
915 if !plan
916 .logical_admission()
917 .owns_batch_capacity_claim(&capacity)
918 {
919 return Err(invalid_resource(
920 "submission wave admission returned foreign capacity authority",
921 ));
922 }
923 Some(capacity)
924 }
925 BatchCapacityClaimDecision::Deferred(deferred) => {
926 return Ok(StepSubmissionWaveAdmissionDecision::Deferred(deferred));
927 }
928 BatchCapacityClaimDecision::PermanentRejected(rejected) => {
929 return Ok(StepSubmissionWaveAdmissionDecision::PermanentRejected(
930 rejected,
931 ));
932 }
933 }
934 };
935 let committed_backing = prepared_backing.commit();
936 let has_program_binding_nodes = prepared_nodes.iter().any(|node| {
937 plan_nodes[node.plan_node_index()]
938 .binding_resource()
939 .is_some()
940 });
941 let program_binding_layout = match (
942 self.reusable_execution_bucket.as_ref(),
943 has_program_binding_nodes,
944 ) {
945 (Some(bucket), true) => Some(
946 plan.dynamic_pools()
947 .program_binding_layout(bucket.bucket_id())
948 .cloned()
949 .ok_or_else(|| {
950 invalid_resource(
951 "reusable submission wave has no compiled program binding layout",
952 )
953 })?,
954 ),
955 _ => None,
956 };
957 let claimed_backing = ClaimedSubmissionWaveBacking::new(
958 plan.plan_hash().clone(),
959 prepared_nodes.len(),
960 plan_nodes.len(),
961 Arc::clone(&work_shape),
962 demand,
963 logical_capacity,
964 committed_backing,
965 program_binding_layout,
966 )?;
967 let wave_fingerprint =
968 submission_wave_fingerprint(self, &prepared_nodes, &claimed_backing, purpose)?;
969 let batch_invocation_id = issue_batch_invocation_id()?;
970 let prepared_participant_flights =
971 prepare_submission_wave_participant_flights(&participant_authority.flight_candidates)?;
972 let covered_participant_nodes = prepared_nodes
973 .len()
974 .checked_mul(participant_authority.participants.len())
975 .ok_or_else(|| invalid_resource("submission wave topology size exceeds usize"))?;
976 let active_wave = self.invocation_registry.enter_submission_wave(
977 covered_participant_nodes,
978 batch_invocation_id,
979 &wave_fingerprint,
980 )?;
981 Ok(StepSubmissionWaveAdmissionDecision::Prepared(
982 PreparedStepSubmissionWave {
983 claimed_backing,
984 initializations: None,
985 request_state_hazards,
986 nodes: prepared_nodes,
987 prepared_participant_flights,
988 active_wave,
989 step: Arc::clone(self),
990 execution_lane_id: self.execution_lane.id(),
991 batch_invocation_id,
992 fingerprint: wave_fingerprint,
993 purpose,
994 },
995 ))
996 }
997
998 fn prepare_invocation_scope(
999 &self,
1000 request: InvocationResourceAdmissionRequest,
1001 ) -> Result<PreparedInvocationScopeDecision<R>, VNextError> {
1002 let PreparedInvocationMetadata {
1003 participants,
1004 participant_frames,
1005 participant_session_identities: _,
1006 flight_candidates,
1007 node_id,
1008 work_shape,
1009 fit_policy,
1010 pressure_action,
1011 } = self.prepare_invocation_metadata(request)?;
1012 let plan = &participants[0].request.plan;
1013 let node_index = plan
1014 .nodes()
1015 .iter()
1016 .position(|node| node.id() == &node_id)
1017 .ok_or_else(|| invalid_resource("invocation references an unknown plan node"))?;
1018 let request_participants = participants
1019 .iter()
1020 .map(|participant| {
1021 let request = Arc::clone(participant.request_resources());
1022 RequestStateHazardParticipant::new(
1023 Arc::clone(&request.plan.dynamic_pools().request_state_hazards),
1024 request.request_authority(),
1025 request,
1026 )
1027 })
1028 .collect::<Vec<_>>();
1029 let request_state_hazards = match plan
1030 .dynamic_pools()
1031 .request_state_hazards
1032 .try_acquire(&request_participants, &[node_index])?
1033 {
1034 RequestStateHazardAcquireDecision::Acquired(permit) => permit,
1035 RequestStateHazardAcquireDecision::Deferred(deferred) => {
1036 return Ok(PreparedInvocationScopeDecision::RequestStateDeferred(
1037 deferred,
1038 ));
1039 }
1040 RequestStateHazardAcquireDecision::SplitRequired(split) => {
1041 return Ok(PreparedInvocationScopeDecision::RequestStateSplitRequired(
1042 split,
1043 ));
1044 }
1045 RequestStateHazardAcquireDecision::Poisoned(poison) => {
1046 return Ok(PreparedInvocationScopeDecision::RequestStatePoisoned(
1047 poison,
1048 ));
1049 }
1050 };
1051 let immediate_shape = work_shape.immediate_shape();
1052 let fit_shape = match fit_policy {
1053 AdmissionFitPolicy::ImmediateOnly => immediate_shape,
1054 AdmissionFitPolicy::FullInputMustFit => work_shape.fit_shape(),
1055 };
1056 let (demand, requested_slices) = plan.scoped_demand(
1057 AllocationLifetime::Invocation,
1058 Some(&node_id),
1059 immediate_shape,
1060 fit_shape,
1061 self.reusable_execution_bucket.as_ref(),
1062 fit_policy,
1063 pressure_action,
1064 )?;
1065 let prepared = match plan.prepare_backing_slices(requested_slices)? {
1066 BackingPrepareDecision::Prepared(prepared) => prepared,
1067 BackingPrepareDecision::Deferred(deferred) => {
1068 return Ok(PreparedInvocationScopeDecision::BackingDeferred(deferred));
1069 }
1070 };
1071 let logical_capacity = if demand.immediate_claim().is_empty() {
1072 None
1073 } else {
1074 let parents = participants
1075 .iter()
1076 .map(|participant| participant.logical_lease())
1077 .collect::<Vec<_>>();
1078 match plan
1079 .logical_admission()
1080 .try_claim_for_sequences(&parents, &demand)?
1081 {
1082 BatchCapacityClaimDecision::Claimed(capacity) => {
1083 let parents_match = capacity
1084 .parents()
1085 .iter()
1086 .map(|parent| (parent.sequence(), parent.request()))
1087 .eq(participants.iter().map(|participant| {
1088 (
1089 participant.sequence_authority(),
1090 participant.request_authority(),
1091 )
1092 }));
1093 if !plan
1094 .logical_admission()
1095 .owns_batch_capacity_claim(&capacity)
1096 || !parents_match
1097 {
1098 return Err(invalid_resource(
1099 "invocation admission returned capacity for another participant set",
1100 ));
1101 }
1102 Some(capacity)
1103 }
1104 BatchCapacityClaimDecision::Deferred(deferred) => {
1105 return Ok(PreparedInvocationScopeDecision::Deferred(deferred));
1106 }
1107 BatchCapacityClaimDecision::PermanentRejected(rejected) => {
1108 return Ok(PreparedInvocationScopeDecision::PermanentRejected(rejected));
1109 }
1110 }
1111 };
1112 let backing_slices = prepared.commit();
1113 let claimed_backing = ClaimedBackingTransaction::new(
1114 work_shape,
1115 demand,
1116 logical_capacity,
1117 backing_slices,
1118 None,
1119 )?;
1120 Ok(PreparedInvocationScopeDecision::Prepared(
1121 PreparedInvocationScope {
1122 participants,
1123 participant_frames,
1124 node_id,
1125 claimed_backing,
1126 flight_candidates,
1127 request_state_hazards,
1128 },
1129 ))
1130 }
1131
1132 fn prepare_invocation_metadata(
1133 &self,
1134 request: InvocationResourceAdmissionRequest,
1135 ) -> Result<PreparedInvocationMetadata<R>, VNextError> {
1136 let InvocationResourceAdmissionRequest {
1137 node_id,
1138 work_shape,
1139 fit_policy,
1140 pressure_action,
1141 } = request;
1142 let PreparedParticipantAuthority {
1143 plan_evidence: _,
1144 participants,
1145 participant_frames,
1146 participant_session_identities,
1147 flight_candidates,
1148 work_shape,
1149 } = self.prepare_participant_authority(work_shape, fit_policy)?;
1150 Ok(PreparedInvocationMetadata {
1151 participants,
1152 participant_frames,
1153 participant_session_identities,
1154 flight_candidates,
1155 node_id,
1156 work_shape,
1157 fit_policy,
1158 pressure_action,
1159 })
1160 }
1161
1162 fn prepare_participant_authority(
1163 &self,
1164 work_shape: Arc<BatchWorkShape>,
1165 fit_policy: AdmissionFitPolicy,
1166 ) -> Result<PreparedParticipantAuthority<R>, VNextError> {
1167 let immediate_shape = work_shape.immediate_shape();
1168 let fit_shape = match fit_policy {
1169 AdmissionFitPolicy::ImmediateOnly => immediate_shape,
1170 AdmissionFitPolicy::FullInputMustFit => work_shape.fit_shape(),
1171 };
1172 let participant_sessions = work_shape
1173 .participants()
1174 .iter()
1175 .map(|authority| {
1176 self.participants
1177 .binary_search_by_key(&authority.canonical_key(), |participant| {
1178 session_participant_key(&participant.session)
1179 })
1180 .map(|index| Arc::clone(&self.participants[index].session))
1181 .map_err(|_| {
1182 invalid_resource(
1183 "invocation participant is not a member of its execution frame",
1184 )
1185 })
1186 })
1187 .collect::<Result<Vec<_>, _>>()?;
1188 let mut participant_frames = Vec::with_capacity(participant_sessions.len());
1189 let mut flight_candidates = Vec::with_capacity(participant_sessions.len());
1190 let mut canonical_work = None;
1191 for (request_index, participant) in participant_sessions.iter().enumerate() {
1192 let key = session_participant_key(participant);
1193 let index = self
1194 .participants
1195 .binary_search_by_key(&key, |step_participant| {
1196 session_participant_key(&step_participant.session)
1197 })
1198 .map_err(|_| {
1199 invalid_resource("invocation participant lost its execution-frame assignment")
1200 })?;
1201 let step_participant = &self.participants[index];
1202 let frame = ActiveSequenceFrame {
1203 frame_id: step_participant.frame.frame_id,
1204 batch_step_id: self.batch_step_id,
1205 };
1206 let admitted = self.work_shape().participant_work()[index].token_span();
1207 if admitted.checkpoint_tokens().is_some() {
1211 let state =
1213 participant.slot.state.lock().map_err(|_| {
1214 invalid_resource("sequence session state mutex is poisoned")
1215 })?;
1216 let super::SequenceSessionSlotState::Active(active) = &*state else {
1217 return Err(invalid_resource("invocation session is no longer active"));
1218 };
1219 if active.epoch != participant.epoch
1220 || active.fingerprint != participant.fingerprint
1221 || active.active_frame != Some(frame)
1222 {
1223 return Err(invalid_resource("invocation lost its exact Step frame"));
1224 }
1225 if let Some(span) = active.completed_boundary.bind_imported_invocation_work(
1226 admitted,
1227 work_shape.participant_work()[request_index].token_span(),
1228 participant.resources().request.plan.plan_hash(),
1229 )? {
1230 let work = canonical_work
1231 .get_or_insert_with(|| work_shape.participant_work().to_vec());
1232 work[request_index] = BatchParticipantTokenSpan::new(
1233 work_shape.participants()[request_index],
1234 span,
1235 );
1236 }
1237 }
1238 participant_frames.push(StepParticipantFrameAssignment::new(
1239 participant.sequence_authority(),
1240 participant.request_authority(),
1241 step_participant.frame.frame_id,
1242 ));
1243 flight_candidates.push(ParticipantFlightCandidate {
1244 slot: Arc::clone(&participant.slot),
1245 epoch: participant.epoch,
1246 fingerprint: participant.fingerprint.clone(),
1247 frame,
1248 participant: BatchParticipantAuthority::new(
1249 participant.sequence_authority(),
1250 participant.request_authority(),
1251 ),
1252 });
1253 }
1254 let participants = participant_sessions
1255 .iter()
1256 .map(|participant| Arc::clone(participant.resources()))
1257 .collect::<Vec<_>>();
1258 let participant_count = u32::try_from(participants.len())
1259 .map_err(|_| invalid_resource("invocation participant count exceeds u32"))?;
1260 if participant_count == 0
1261 || immediate_shape.sequences() != participant_count
1262 || fit_shape.sequences() != participant_count
1263 {
1264 return Err(invalid_resource(
1265 "invocation shape sequence count differs from its exact participant set",
1266 ));
1267 }
1268 let participant_authorities = participants
1269 .iter()
1270 .map(|participant| {
1271 BatchParticipantAuthority::new(
1272 participant.sequence_authority(),
1273 participant.request_authority(),
1274 )
1275 })
1276 .collect::<Vec<_>>();
1277 if work_shape.participants() != participant_authorities {
1278 return Err(invalid_resource(
1279 "invocation work authority differs from selected participants",
1280 ));
1281 }
1282 let participant_session_identities = flight_candidates
1283 .iter()
1284 .map(|candidate| (candidate.epoch, candidate.fingerprint.clone()))
1285 .collect();
1286 let work_shape = match canonical_work {
1287 Some(work) => Arc::new(BatchWorkShape::new(work)?),
1288 None => work_shape,
1289 };
1290 Ok(PreparedParticipantAuthority {
1291 plan_evidence: self.plan_evidence(),
1292 participants,
1293 participant_frames,
1294 participant_session_identities,
1295 flight_candidates,
1296 work_shape,
1297 })
1298 }
1299}
1300
1301impl<R> Drop for StepResourceLease<R>
1302where
1303 R: DeviceRuntime,
1304{
1305 fn drop(&mut self) {
1306 if !self.finalized {
1307 for participant in &self.participants {
1308 poison_session_frame(&participant.frame);
1309 }
1310 }
1311 }
1312}
1313
1314pub enum InvocationResourceAdmissionDecision<R>
1315where
1316 R: DeviceRuntime,
1317{
1318 Admitted(InvocationResourceLease<R>),
1319 Deferred(AdmissionDeferred),
1320 BackingDeferred(InvocationAdmissionBackingDeferral<R>),
1321 PermanentRejected(AdmissionRejected),
1322 RequestStateDeferred(RequestStateHazardDeferral),
1323 RequestStateSplitRequired(RequestStateHazardSplitRequired),
1324 RequestStatePoisoned(RequestStateHazardPoison),
1325}
1326
1327#[must_use = "invocation backing deferral retains its exact step parent"]
1330pub struct InvocationAdmissionBackingDeferral<R>
1331where
1332 R: DeviceRuntime,
1333{
1334 backing: PlanBackingDeferral<R>,
1335 step: Arc<StepResourceLease<R>>,
1336 node_id: NodeId,
1337 work_fingerprint: String,
1338}
1339
1340impl<R> InvocationAdmissionBackingDeferral<R>
1341where
1342 R: DeviceRuntime,
1343{
1344 fn new(
1345 step: Arc<StepResourceLease<R>>,
1346 evidence: DynamicBackingDeferred,
1347 node_id: NodeId,
1348 work_fingerprint: String,
1349 ) -> Result<Self, VNextError> {
1350 let resources = Arc::clone(
1351 &step.participants[0]
1352 .session
1353 .resources()
1354 .request
1355 .plan
1356 .resources,
1357 );
1358 Ok(Self {
1359 backing: PlanBackingDeferral::new(resources, evidence)?,
1360 step,
1361 node_id,
1362 work_fingerprint,
1363 })
1364 }
1365
1366 pub fn evidence(&self) -> &DynamicBackingDeferred {
1367 self.backing.evidence()
1368 }
1369
1370 pub fn node_id(&self) -> &NodeId {
1371 &self.node_id
1372 }
1373
1374 pub fn work_fingerprint(&self) -> &str {
1375 &self.work_fingerprint
1376 }
1377
1378 pub fn maintain(&self) -> Result<DynamicDeferredMaintenanceOutcome, VNextError> {
1379 if self.step.finalized {
1380 return Err(invalid_resource(
1381 "finalized step cannot maintain invocation backing",
1382 ));
1383 }
1384 self.backing.maintain()
1385 }
1386
1387 pub fn register_waiter(&self) -> Result<PlanCapacityWaitRegistration<R>, VNextError> {
1388 self.backing.register_waiter()
1389 }
1390}
1391
1392enum PreparedInvocationScopeDecision<R>
1393where
1394 R: DeviceRuntime,
1395{
1396 Prepared(PreparedInvocationScope<R>),
1397 Deferred(AdmissionDeferred),
1398 BackingDeferred(DynamicBackingDeferred),
1399 PermanentRejected(AdmissionRejected),
1400 RequestStateDeferred(RequestStateHazardDeferral),
1401 RequestStateSplitRequired(RequestStateHazardSplitRequired),
1402 RequestStatePoisoned(RequestStateHazardPoison),
1403}
1404
1405struct PreparedInvocationScope<R>
1406where
1407 R: DeviceRuntime,
1408{
1409 claimed_backing: ClaimedBackingTransaction,
1410 participants: Vec<Arc<AdmittedSequenceResources<R>>>,
1411 participant_frames: Vec<StepParticipantFrameAssignment>,
1412 flight_candidates: Vec<ParticipantFlightCandidate>,
1413 node_id: NodeId,
1414 request_state_hazards: Option<RequestStateHazardPermit<Arc<AdmittedRequestResources<R>>>>,
1415}
1416
1417struct PreparedInvocationMetadata<R>
1418where
1419 R: DeviceRuntime,
1420{
1421 participants: Vec<Arc<AdmittedSequenceResources<R>>>,
1422 participant_frames: Vec<StepParticipantFrameAssignment>,
1423 participant_session_identities: Vec<(SequenceSessionEpoch, SequenceSessionFingerprint)>,
1424 flight_candidates: Vec<ParticipantFlightCandidate>,
1425 node_id: NodeId,
1426 work_shape: Arc<BatchWorkShape>,
1427 fit_policy: AdmissionFitPolicy,
1428 pressure_action: AdmissionPressureAction,
1429}
1430
1431struct PreparedParticipantAuthority<R>
1432where
1433 R: DeviceRuntime,
1434{
1435 plan_evidence: TrustedPlanRuntimeEvidence,
1436 participants: Vec<Arc<AdmittedSequenceResources<R>>>,
1437 participant_frames: Vec<StepParticipantFrameAssignment>,
1438 participant_session_identities: Vec<(SequenceSessionEpoch, SequenceSessionFingerprint)>,
1439 flight_candidates: Vec<ParticipantFlightCandidate>,
1440 work_shape: Arc<BatchWorkShape>,
1441}
1442
1443fn submission_wave_fingerprint<R>(
1444 step: &StepResourceLease<R>,
1445 nodes: &[PreparedStepSubmissionNode<R>],
1446 claimed_backing: &ClaimedSubmissionWaveBacking,
1447 purpose: SubmissionWavePurpose,
1448) -> Result<String, VNextError>
1449where
1450 R: DeviceRuntime,
1451{
1452 #[derive(Serialize)]
1453 struct WaveNodeInput<'a> {
1454 plan_node_index: usize,
1455 node_id: &'a NodeId,
1456 }
1457
1458 #[derive(Serialize)]
1459 struct WaveInput<'a> {
1460 domain: &'static str,
1461 purpose: SubmissionWavePurpose,
1462 batch_step_id: BatchStepId,
1463 plan_hash: &'a super::PlanHash,
1464 node_count: usize,
1465 nodes: &'a [WaveNodeInput<'a>],
1466 step_backing_fingerprint: &'a str,
1467 invocation_backing_fingerprint: &'a str,
1468 participant_frames: &'a [StepParticipantFrameAssignment],
1469 work_fingerprint: &'a str,
1470 }
1471
1472 let first = nodes
1473 .first()
1474 .ok_or_else(|| invalid_resource("submission wave fingerprint requires plan nodes"))?;
1475 if nodes.len() != claimed_backing.node_count()
1476 || nodes
1477 .iter()
1478 .any(|node| !Arc::ptr_eq(&node.participant_authority, &first.participant_authority))
1479 || nodes
1480 .windows(2)
1481 .any(|pair| pair[0].plan_node_index() >= pair[1].plan_node_index())
1482 || first.work_shape() != claimed_backing.work_shape()
1483 {
1484 return Err(invalid_resource(
1485 "submission wave topology differs from its shared plan/work authority",
1486 ));
1487 }
1488 let node_scope = nodes
1489 .iter()
1490 .map(|node| WaveNodeInput {
1491 plan_node_index: node.plan_node_index(),
1492 node_id: node.node_id(),
1493 })
1494 .collect::<Vec<_>>();
1495 let bytes = serde_json::to_vec(&WaveInput {
1496 domain: "ferrum.runtime-vnext.step-submission-wave.v4",
1497 purpose,
1498 batch_step_id: step.batch_step_id,
1499 plan_hash: claimed_backing.plan_hash(),
1500 node_count: claimed_backing.node_count(),
1501 nodes: &node_scope,
1502 step_backing_fingerprint: step.claimed_backing.fingerprint(),
1503 invocation_backing_fingerprint: claimed_backing.fingerprint(),
1504 participant_frames: first.participant_frames(),
1505 work_fingerprint: claimed_backing.work_shape().fingerprint(),
1506 })
1507 .map_err(|error| {
1508 invalid_resource(format!(
1509 "submission wave fingerprint encode failed: {error}"
1510 ))
1511 })?;
1512 Ok(format!("{:x}", Sha256::digest(bytes)))
1513}
1514
1515#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
1516#[serde(rename_all = "snake_case")]
1517pub(crate) enum SubmissionWavePurpose {
1518 FullPlan,
1519 DeterminismProbe,
1520}
1521
1522pub enum StepSubmissionWaveAdmissionDecision<R>
1523where
1524 R: DeviceRuntime,
1525{
1526 Prepared(PreparedStepSubmissionWave<R>),
1527 Deferred(AdmissionDeferred),
1528 BackingDeferred(StepSubmissionWaveBackingDeferral<R>),
1529 PermanentRejected(AdmissionRejected),
1530 RequestStateDeferred(RequestStateHazardDeferral),
1531 RequestStateSplitRequired(RequestStateHazardSplitRequired),
1532 RequestStatePoisoned(RequestStateHazardPoison),
1533}
1534
1535#[must_use = "submission-wave backing deferral retains its exact step parent"]
1537pub struct StepSubmissionWaveBackingDeferral<R>
1538where
1539 R: DeviceRuntime,
1540{
1541 backing: PlanBackingDeferral<R>,
1542 step: Arc<StepResourceLease<R>>,
1543 node_work_fingerprints: Vec<(NodeId, String)>,
1544}
1545
1546impl<R> StepSubmissionWaveBackingDeferral<R>
1547where
1548 R: DeviceRuntime,
1549{
1550 fn new(
1551 step: Arc<StepResourceLease<R>>,
1552 evidence: DynamicBackingDeferred,
1553 node_work_fingerprints: Vec<(NodeId, String)>,
1554 ) -> Result<Self, VNextError> {
1555 let resources = Arc::clone(
1556 &step.participants[0]
1557 .session
1558 .resources()
1559 .request
1560 .plan
1561 .resources,
1562 );
1563 Ok(Self {
1564 backing: PlanBackingDeferral::new(resources, evidence)?,
1565 step,
1566 node_work_fingerprints,
1567 })
1568 }
1569
1570 pub fn evidence(&self) -> &DynamicBackingDeferred {
1571 self.backing.evidence()
1572 }
1573
1574 pub fn node_work_fingerprints(&self) -> &[(NodeId, String)] {
1575 &self.node_work_fingerprints
1576 }
1577
1578 pub fn maintain(&self) -> Result<DynamicDeferredMaintenanceOutcome, VNextError> {
1579 if self.step.finalized {
1580 return Err(invalid_resource(
1581 "finalized step cannot maintain submission-wave backing",
1582 ));
1583 }
1584 self.backing.maintain()
1585 }
1586
1587 pub fn register_waiter(&self) -> Result<PlanCapacityWaitRegistration<R>, VNextError> {
1588 self.backing.register_waiter()
1589 }
1590}
1591
1592pub struct PreparedStepSubmissionNode<R>
1596where
1597 R: DeviceRuntime,
1598{
1599 participant_authority: Arc<PreparedParticipantAuthority<R>>,
1600 node_index: usize,
1601}
1602
1603impl<R> PreparedStepSubmissionNode<R>
1604where
1605 R: DeviceRuntime,
1606{
1607 fn new(node_index: usize, participant_authority: Arc<PreparedParticipantAuthority<R>>) -> Self {
1608 Self {
1609 participant_authority,
1610 node_index,
1611 }
1612 }
1613
1614 pub fn node_id(&self) -> &NodeId {
1615 self.participant_authority.participants[0]
1616 .request
1617 .plan
1618 .nodes()
1619 .get(self.node_index)
1620 .expect("prepared submission node index was derived from the immutable plan")
1621 .id()
1622 }
1623
1624 pub(crate) const fn plan_node_index(&self) -> usize {
1625 self.node_index
1626 }
1627
1628 pub fn participant_count(&self) -> u32 {
1629 u32::try_from(self.participant_authority.participants.len())
1630 .expect("wave participant count was validated before admission")
1631 }
1632
1633 pub fn participants(
1634 &self,
1635 ) -> impl ExactSizeIterator<Item = &Arc<AdmittedSequenceResources<R>>> {
1636 self.participant_authority.participants.iter()
1637 }
1638
1639 pub fn participant_frames(&self) -> &[StepParticipantFrameAssignment] {
1640 &self.participant_authority.participant_frames
1641 }
1642
1643 pub fn work_shape(&self) -> &BatchWorkShape {
1644 self.participant_authority.work_shape.as_ref()
1645 }
1646
1647 pub fn plan_evidence(&self) -> TrustedPlanRuntimeEvidence {
1648 self.participant_authority.plan_evidence.clone()
1649 }
1650
1651 pub(crate) fn plan_evidence_ref(&self) -> &TrustedPlanRuntimeEvidence {
1652 &self.participant_authority.plan_evidence
1653 }
1654
1655 pub(crate) fn runtime(&self) -> &Arc<R> {
1656 &self.participant_authority.participants[0]
1657 .request
1658 .plan
1659 .runtime()
1660 }
1661
1662 pub(crate) fn participant_session_identities(
1663 &self,
1664 ) -> impl ExactSizeIterator<Item = (SequenceSessionEpoch, &SequenceSessionFingerprint)> {
1665 self.participant_authority
1666 .participant_session_identities
1667 .iter()
1668 .map(|(epoch, fingerprint)| (*epoch, fingerprint))
1669 }
1670}
1671
1672#[must_use = "a prepared submission wave must be dispatched or explicitly dropped"]
1677pub struct PreparedStepSubmissionWave<R>
1678where
1679 R: DeviceRuntime,
1680{
1681 claimed_backing: ClaimedSubmissionWaveBacking,
1683 initializations: Option<PreparedBackingInitializations>,
1684 request_state_hazards: Option<RequestStateHazardPermit<Arc<AdmittedRequestResources<R>>>>,
1685 nodes: Vec<PreparedStepSubmissionNode<R>>,
1686 prepared_participant_flights: Vec<PreparedSubmissionWaveParticipantFlightHold>,
1687 active_wave: ActiveInvocationWaveGuard,
1688 step: Arc<StepResourceLease<R>>,
1689 execution_lane_id: ExecutionLaneId,
1690 batch_invocation_id: BatchInvocationId,
1691 fingerprint: String,
1692 purpose: SubmissionWavePurpose,
1693}
1694
1695impl<R> PreparedStepSubmissionWave<R>
1696where
1697 R: DeviceRuntime,
1698{
1699 pub fn batch_step_id(&self) -> BatchStepId {
1700 self.step.batch_step_id()
1701 }
1702
1703 pub const fn batch_invocation_id(&self) -> BatchInvocationId {
1704 self.batch_invocation_id
1705 }
1706
1707 pub const fn execution_lane_id(&self) -> ExecutionLaneId {
1708 self.execution_lane_id
1709 }
1710
1711 pub fn fingerprint(&self) -> &str {
1712 &self.fingerprint
1713 }
1714
1715 pub fn nodes(&self) -> &[PreparedStepSubmissionNode<R>] {
1716 &self.nodes
1717 }
1718
1719 pub(crate) const fn purpose(&self) -> SubmissionWavePurpose {
1720 self.purpose
1721 }
1722
1723 pub(crate) fn record_full_plan_success(
1724 &self,
1725 seal: &crate::vnext::SuccessfulWaveCompletionSeal,
1726 ) -> Result<(), VNextError> {
1727 super::record_completed_wave(self, seal)
1728 }
1729
1730 pub fn claimed_backing(&self) -> &ClaimedSubmissionWaveBacking {
1731 &self.claimed_backing
1732 }
1733
1734 pub fn node_count(&self) -> usize {
1735 self.nodes.len()
1736 }
1737
1738 pub fn prepared_participant_flight_count(&self) -> usize {
1739 self.prepared_participant_flights.len()
1740 }
1741
1742 pub fn node_participant_projection_count(&self) -> usize {
1743 self.nodes
1744 .iter()
1745 .map(|node| node.participant_count() as usize)
1746 .sum()
1747 }
1748
1749 pub fn physical_invocation_ledger_entry_count(&self) -> usize {
1750 self.active_wave.physical_entry_count()
1751 }
1752
1753 pub fn step_resources(&self) -> &Arc<StepResourceLease<R>> {
1754 &self.step
1755 }
1756
1757 pub(crate) fn runtime(&self) -> &Arc<R> {
1758 self.nodes[0].runtime()
1759 }
1760
1761 pub(crate) fn deferred_cleanup_domain(&self) -> DeferredDeviceCleanupDomainId {
1762 self.nodes[0].participant_authority.participants[0]
1763 .request
1764 .plan
1765 .resources
1766 .deferred_cleanup_domain
1767 }
1768
1769 pub fn has_shared_step_backing(&self) -> bool {
1770 self.step.claimed_backing().has_shared_physical_claims()
1771 }
1772
1773 pub(crate) fn backing_view(
1774 &self,
1775 node_index: usize,
1776 resource_id: &ResourceId,
1777 ) -> Result<LogicalBackingBufferView<'_, R::Buffer>, VNextError> {
1778 let node = self
1779 .nodes
1780 .get(node_index)
1781 .ok_or_else(|| invalid_resource("submission wave node index is out of bounds"))?;
1782 if let Some(authority) = self
1783 .claimed_backing
1784 .backing_slices()
1785 .iter()
1786 .find(|authority| authority.resource_id() == resource_id)
1787 {
1788 return node.participant_authority.participants[0]
1789 .request
1790 .plan
1791 .dynamic_pools()
1792 .view(authority);
1793 }
1794 self.step.backing_view(resource_id)
1795 }
1796
1797 pub(crate) fn begin_dispatch(&mut self) -> Result<(), VNextError> {
1798 match &self.initializations {
1799 Some(initializations) => initializations.ensure_wave(&self.fingerprint)?,
1800 None => {
1801 self.initializations = Some(PreparedBackingInitializations::prepare(
1802 &self.step,
1803 &self.fingerprint,
1804 )?);
1805 }
1806 }
1807 begin_submission_wave_participant_flights_dispatch(&mut self.prepared_participant_flights)
1808 }
1809
1810 pub(crate) fn encode_backing_initializations(
1811 &self,
1812 runtime: &R,
1813 commands: &mut DeviceCommandBatch<R::Command>,
1814 ) -> Result<usize, BackingInitializationEncodeError<R::Error>> {
1815 self.initializations
1816 .as_ref()
1817 .ok_or_else(|| {
1818 BackingInitializationEncodeError::Contract(invalid_resource(
1819 "submission wave initialization was not prepared",
1820 ))
1821 })?
1822 .encode(&self.step, runtime, commands)
1823 }
1824
1825 pub(crate) fn mark_submission_fence_installed(&mut self) -> Result<(), VNextError> {
1826 self.initializations
1827 .as_mut()
1828 .ok_or_else(|| invalid_resource("submission wave initialization was not prepared"))?
1829 .mark_in_flight()?;
1830 self.active_wave.mark_in_flight()?;
1831 if let Some(hazards) = &mut self.request_state_hazards {
1832 hazards.mark_submission_fence_installed()?;
1833 }
1834 Ok(())
1835 }
1836
1837 pub(crate) fn mark_submission_indeterminate(&mut self) {
1838 if let Some(initializations) = &mut self.initializations {
1839 initializations.mark_indeterminate();
1840 }
1841 if let Some(hazards) = &mut self.request_state_hazards {
1842 hazards.mark_submission_indeterminate();
1843 }
1844 }
1845
1846 pub(crate) fn finish_backing_initializations(
1847 &mut self,
1848 succeeded: bool,
1849 ) -> Result<(), VNextError> {
1850 self.initializations
1851 .as_mut()
1852 .ok_or_else(|| invalid_resource("submission wave initialization was not prepared"))?
1853 .finish(succeeded)
1854 }
1855
1856 pub(crate) fn finish_request_state_hazards(
1857 &mut self,
1858 disposition: RequestStateHazardTerminalDisposition,
1859 ) -> Result<(), VNextError> {
1860 match &mut self.request_state_hazards {
1861 Some(hazards) => hazards.finish(disposition),
1862 None => Ok(()),
1863 }
1864 }
1865
1866 pub(crate) fn definitely_not_submitted(
1867 mut self,
1868 ) -> Result<DefinitelyNotSubmittedWaveRetryAuthority<R>, VNextError> {
1869 self.active_wave.mark_not_submitted()?;
1870 reset_submission_wave_participant_flights_after_definitely_not_submitted(
1871 &mut self.prepared_participant_flights,
1872 )?;
1873 let topology_fingerprint = self.fingerprint.clone();
1874 let prior_attempt = self.batch_invocation_id;
1875 Ok(DefinitelyNotSubmittedWaveRetryAuthority {
1876 wave: Some(self),
1877 topology_fingerprint,
1878 prior_attempt,
1879 })
1880 }
1881
1882 fn prepare_definitely_not_submitted_retry(
1883 &mut self,
1884 fresh_attempt: BatchInvocationId,
1885 topology_fingerprint: &str,
1886 ) -> Result<(), VNextError> {
1887 if self.fingerprint != topology_fingerprint
1888 || self
1889 .prepared_participant_flights
1890 .iter()
1891 .any(|hold| hold.phase != ParticipantFlightPhase::Prepared)
1892 {
1893 return Err(invalid_resource(
1894 "definitely-not-submitted wave retry topology changed",
1895 ));
1896 }
1897 self.initializations
1898 .as_ref()
1899 .ok_or_else(|| invalid_resource("wave retry lost backing initialization authority"))?
1900 .ensure_wave(topology_fingerprint)?;
1901 self.active_wave.prepare_retry(fresh_attempt)?;
1902 self.batch_invocation_id = fresh_attempt;
1903 Ok(())
1904 }
1905}
1906
1907#[must_use = "a definitely-not-submitted wave retry must be retried or retired"]
1908pub struct DefinitelyNotSubmittedWaveRetryAuthority<R>
1909where
1910 R: DeviceRuntime,
1911{
1912 wave: Option<PreparedStepSubmissionWave<R>>,
1913 topology_fingerprint: String,
1914 prior_attempt: BatchInvocationId,
1915}
1916
1917impl<R> fmt::Debug for DefinitelyNotSubmittedWaveRetryAuthority<R>
1918where
1919 R: DeviceRuntime,
1920{
1921 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1922 formatter
1923 .debug_struct("DefinitelyNotSubmittedWaveRetryAuthority")
1924 .field("prior_attempt", &self.prior_attempt)
1925 .field("topology_fingerprint", &self.topology_fingerprint)
1926 .finish_non_exhaustive()
1927 }
1928}
1929
1930impl<R> DefinitelyNotSubmittedWaveRetryAuthority<R>
1931where
1932 R: DeviceRuntime,
1933{
1934 pub const fn prior_attempt(&self) -> BatchInvocationId {
1935 self.prior_attempt
1936 }
1937
1938 pub fn topology_fingerprint(&self) -> &str {
1939 &self.topology_fingerprint
1940 }
1941
1942 pub fn retry(mut self) -> Result<PreparedStepSubmissionWave<R>, VNextError> {
1943 let fresh_attempt = issue_batch_invocation_id()?;
1944 let wave = self
1945 .wave
1946 .as_mut()
1947 .ok_or_else(|| invalid_resource("wave retry authority no longer owns its wave"))?;
1948 wave.prepare_definitely_not_submitted_retry(fresh_attempt, &self.topology_fingerprint)?;
1949 self.wave
1950 .take()
1951 .ok_or_else(|| invalid_resource("validated wave retry authority lost its wave"))
1952 }
1953}
1954
1955#[must_use = "prepared invocation resources must be dispatched or explicitly dropped"]
1959pub struct InvocationResourceLease<R>
1960where
1961 R: DeviceRuntime,
1962{
1963 claimed_backing: ClaimedBackingTransaction,
1966 initializations: Option<PreparedBackingInitializations>,
1967 request_state_hazards: Option<RequestStateHazardPermit<Arc<AdmittedRequestResources<R>>>>,
1968 prepared_participant_flights: Vec<PreparedParticipantFlightHold>,
1969 active_wave: ActiveInvocationWaveGuard,
1970 participants: Vec<Arc<AdmittedSequenceResources<R>>>,
1971 participant_frames: Vec<StepParticipantFrameAssignment>,
1972 step: Arc<StepResourceLease<R>>,
1973 node_id: NodeId,
1974 batch_invocation_id: BatchInvocationId,
1975}
1976
1977impl<R> InvocationResourceLease<R>
1978where
1979 R: DeviceRuntime,
1980{
1981 fn new(
1982 step: Arc<StepResourceLease<R>>,
1983 participants: Vec<Arc<AdmittedSequenceResources<R>>>,
1984 participant_frames: Vec<StepParticipantFrameAssignment>,
1985 node_id: NodeId,
1986 batch_invocation_id: BatchInvocationId,
1987 claimed_backing: ClaimedBackingTransaction,
1988 prepared_participant_flights: Vec<PreparedParticipantFlightHold>,
1989 active_wave: ActiveInvocationWaveGuard,
1990 request_state_hazards: Option<RequestStateHazardPermit<Arc<AdmittedRequestResources<R>>>>,
1991 ) -> Result<Self, VNextError> {
1992 if participants.is_empty() {
1993 return Err(invalid_resource(
1994 "invocation resources require a non-empty participant set",
1995 ));
1996 }
1997 if participant_frames.len() != participants.len()
1998 || prepared_participant_flights.len() != participants.len()
1999 || participant_frames
2000 .iter()
2001 .zip(&participants)
2002 .any(|(assignment, participant)| {
2003 assignment.sequence_authority() != participant.sequence_authority()
2004 || assignment.request_authority() != participant.request_authority()
2005 })
2006 {
2007 return Err(invalid_resource(
2008 "invocation frame mapping differs from its exact participant set",
2009 ));
2010 }
2011 if claimed_backing.work_shape().participants().len() != participants.len()
2012 || claimed_backing
2013 .work_shape()
2014 .participants()
2015 .iter()
2016 .zip(&participants)
2017 .any(|(authority, participant)| {
2018 authority.sequence_authority() != participant.sequence_authority()
2019 || authority.request_authority() != participant.request_authority()
2020 })
2021 || claimed_backing.work_shape().immediate_tokens()
2022 > step.work_shape().immediate_tokens()
2023 || claimed_backing.work_shape().immediate_pages() > step.work_shape().immediate_pages()
2024 || claimed_backing.work_shape().fit_tokens() > step.work_shape().fit_tokens()
2025 || claimed_backing.work_shape().fit_pages() > step.work_shape().fit_pages()
2026 {
2027 return Err(invalid_resource(
2028 "invocation work shape differs from participants or exceeds its step",
2029 ));
2030 }
2031 if let Some(capacity) = claimed_backing.logical_capacity() {
2032 let coordinator = participants[0].request.plan.logical_admission();
2033 let parents_match = capacity
2034 .parents()
2035 .iter()
2036 .map(|parent| (parent.sequence(), parent.request()))
2037 .eq(participants.iter().map(|participant| {
2038 (
2039 participant.sequence_authority(),
2040 participant.request_authority(),
2041 )
2042 }));
2043 if !coordinator.owns_batch_capacity_claim(capacity) || !parents_match {
2044 return Err(invalid_resource(
2045 "invocation capacity authority differs from its exact participants",
2046 ));
2047 }
2048 }
2049 Ok(Self {
2050 claimed_backing,
2051 initializations: None,
2052 request_state_hazards,
2053 prepared_participant_flights,
2054 active_wave,
2055 participants,
2056 participant_frames,
2057 step,
2058 node_id,
2059 batch_invocation_id,
2060 })
2061 }
2062
2063 pub fn node_id(&self) -> &NodeId {
2064 &self.node_id
2065 }
2066
2067 pub const fn batch_invocation_id(&self) -> BatchInvocationId {
2068 self.batch_invocation_id
2069 }
2070
2071 pub fn batch_step_id(&self) -> BatchStepId {
2072 self.step.batch_step_id()
2073 }
2074
2075 pub fn participant_count(&self) -> u32 {
2076 u32::try_from(self.participants.len())
2077 .expect("invocation participant count was validated before admission")
2078 }
2079
2080 pub fn prepared_participant_count(&self) -> u32 {
2081 u32::try_from(self.prepared_participant_flights.len())
2082 .expect("prepared participant count was validated at construction")
2083 }
2084
2085 pub(crate) fn participant_session_identities(
2086 &self,
2087 ) -> impl ExactSizeIterator<Item = (SequenceSessionEpoch, &SequenceSessionFingerprint)> {
2088 self.prepared_participant_flights
2089 .iter()
2090 .map(|hold| (hold.epoch, &hold.fingerprint))
2091 }
2092
2093 pub(crate) fn begin_dispatch(&mut self) -> Result<(), VNextError> {
2094 let topology_fingerprint = self.retry_topology_fingerprint()?;
2095 match &self.initializations {
2096 Some(initializations) => {
2097 initializations.ensure_wave(&topology_fingerprint)?;
2098 }
2099 None => {
2100 self.initializations = Some(PreparedBackingInitializations::prepare(
2101 &self.step,
2102 &topology_fingerprint,
2103 )?);
2104 }
2105 }
2106 begin_participant_flights_dispatch(&mut self.prepared_participant_flights)
2107 }
2108
2109 pub(crate) fn encode_backing_initializations(
2110 &self,
2111 runtime: &R,
2112 commands: &mut DeviceCommandBatch<R::Command>,
2113 ) -> Result<usize, BackingInitializationEncodeError<R::Error>> {
2114 self.initializations
2115 .as_ref()
2116 .ok_or_else(|| {
2117 BackingInitializationEncodeError::Contract(invalid_resource(
2118 "invocation backing initialization was not prepared",
2119 ))
2120 })?
2121 .encode(&self.step, runtime, commands)
2122 }
2123
2124 pub(crate) fn mark_submission_fence_installed(&mut self) -> Result<(), VNextError> {
2125 self.initializations
2126 .as_mut()
2127 .ok_or_else(|| invalid_resource("invocation initialization was not prepared"))?
2128 .mark_in_flight()?;
2129 self.active_wave.mark_in_flight()?;
2130 if let Some(hazards) = &mut self.request_state_hazards {
2131 hazards.mark_submission_fence_installed()?;
2132 }
2133 Ok(())
2134 }
2135
2136 pub(crate) fn mark_submission_indeterminate(&mut self) {
2137 if let Some(initializations) = &mut self.initializations {
2138 initializations.mark_indeterminate();
2139 }
2140 if let Some(hazards) = &mut self.request_state_hazards {
2141 hazards.mark_submission_indeterminate();
2142 }
2143 }
2144
2145 pub(crate) fn finish_backing_initializations(
2146 &mut self,
2147 succeeded: bool,
2148 ) -> Result<(), VNextError> {
2149 self.initializations
2150 .as_mut()
2151 .ok_or_else(|| invalid_resource("invocation initialization was not prepared"))?
2152 .finish(succeeded)
2153 }
2154
2155 pub(crate) fn finish_request_state_hazards(
2156 &mut self,
2157 disposition: RequestStateHazardTerminalDisposition,
2158 ) -> Result<(), VNextError> {
2159 match &mut self.request_state_hazards {
2160 Some(hazards) => hazards.finish(disposition),
2161 None => Ok(()),
2162 }
2163 }
2164
2165 pub(crate) fn definitely_not_submitted(
2166 mut self,
2167 ) -> Result<DefinitelyNotSubmittedRetryAuthority<R>, VNextError> {
2168 self.active_wave.mark_not_submitted()?;
2169 reset_participant_flights_after_definitely_not_submitted(
2170 &mut self.prepared_participant_flights,
2171 )?;
2172 let topology_fingerprint = self.retry_topology_fingerprint()?;
2173 let work_fingerprint = self.work_shape().fingerprint().to_owned();
2174 let prior_attempt = self.batch_invocation_id;
2175 Ok(DefinitelyNotSubmittedRetryAuthority {
2176 invocation: Some(self),
2177 topology_fingerprint,
2178 work_fingerprint,
2179 prior_attempt,
2180 })
2181 }
2182
2183 fn prepare_definitely_not_submitted_retry(
2184 &mut self,
2185 fresh_attempt: BatchInvocationId,
2186 topology_fingerprint: &str,
2187 work_fingerprint: &str,
2188 ) -> Result<(), VNextError> {
2189 if self.retry_topology_fingerprint()? != topology_fingerprint
2190 || self.work_shape().fingerprint() != work_fingerprint
2191 || self
2192 .prepared_participant_flights
2193 .iter()
2194 .any(|hold| hold.phase != ParticipantFlightPhase::Prepared)
2195 {
2196 return Err(invalid_resource(
2197 "definitely-not-submitted retry topology or work fingerprint changed",
2198 ));
2199 }
2200 self.initializations
2201 .as_ref()
2202 .ok_or_else(|| {
2203 invalid_resource("invocation retry lost backing initialization authority")
2204 })?
2205 .ensure_wave(topology_fingerprint)?;
2206 self.active_wave.prepare_retry(fresh_attempt)?;
2207 self.batch_invocation_id = fresh_attempt;
2208 Ok(())
2209 }
2210
2211 fn retry_topology_fingerprint(&self) -> Result<String, VNextError> {
2212 #[derive(Serialize)]
2213 struct FingerprintInput<'a> {
2214 domain: &'static str,
2215 node_id: &'a NodeId,
2216 participant_frames: &'a [StepParticipantFrameAssignment],
2217 work_fingerprint: &'a str,
2218 }
2219 let bytes = serde_json::to_vec(&FingerprintInput {
2220 domain: "ferrum.runtime-vnext.invocation-retry-topology.v1",
2221 node_id: &self.node_id,
2222 participant_frames: &self.participant_frames,
2223 work_fingerprint: self.work_shape().fingerprint(),
2224 })
2225 .map_err(|error| {
2226 invalid_resource(format!(
2227 "invocation retry topology fingerprint encode failed: {error}"
2228 ))
2229 })?;
2230 Ok(format!("{:x}", Sha256::digest(bytes)))
2231 }
2232
2233 pub fn coordinator_id(&self) -> LogicalAdmissionCoordinatorId {
2234 self.participants[0].coordinator_id()
2235 }
2236
2237 pub fn participants(
2238 &self,
2239 ) -> impl ExactSizeIterator<Item = &Arc<AdmittedSequenceResources<R>>> {
2240 self.participants.iter()
2241 }
2242
2243 pub fn participant_frames(&self) -> &[StepParticipantFrameAssignment] {
2244 &self.participant_frames
2245 }
2246
2247 pub fn step_resources(&self) -> &Arc<StepResourceLease<R>> {
2248 &self.step
2249 }
2250
2251 pub(crate) fn participant_backing_snapshot(
2252 &self,
2253 index: usize,
2254 ) -> Result<&Arc<SequenceBackingSnapshot<R>>, VNextError> {
2255 let participant = self.participants.get(index).ok_or_else(|| {
2256 invalid_resource("invocation backing participant index is out of range")
2257 })?;
2258 self.step
2259 .participant_backing_snapshot(BatchParticipantAuthority::new(
2260 participant.sequence_authority(),
2261 participant.request_authority(),
2262 ))
2263 }
2264
2265 pub fn backing_slices(&self) -> &[LogicalBackingSliceAuthority] {
2266 self.claimed_backing.backing_slices()
2267 }
2268
2269 pub fn logical_capacity(&self) -> Option<&LogicalBatchCapacityLease> {
2270 self.claimed_backing.logical_capacity()
2271 }
2272
2273 pub fn work_shape(&self) -> &BatchWorkShape {
2274 self.claimed_backing.work_shape()
2275 }
2276
2277 pub fn claimed_backing(&self) -> &ClaimedBackingTransaction {
2278 &self.claimed_backing
2279 }
2280
2281 pub fn plan_evidence(&self) -> TrustedPlanRuntimeEvidence {
2282 self.step.plan_evidence()
2283 }
2284
2285 pub(crate) fn runtime(&self) -> &Arc<R> {
2286 self.participants[0].request.plan.runtime()
2287 }
2288
2289 pub(crate) fn deferred_cleanup_domain(&self) -> DeferredDeviceCleanupDomainId {
2290 self.participants[0]
2291 .request
2292 .plan
2293 .resources
2294 .deferred_cleanup_domain
2295 }
2296
2297 pub(crate) fn backing_view(
2298 &self,
2299 resource_id: &ResourceId,
2300 ) -> Result<LogicalBackingBufferView<'_, R::Buffer>, VNextError> {
2301 if let Some(authority) = self
2302 .claimed_backing
2303 .backing_slices()
2304 .iter()
2305 .find(|authority| authority.resource_id() == resource_id)
2306 {
2307 return self.step.participants[0]
2308 .session
2309 .resources()
2310 .request
2311 .plan
2312 .dynamic_pools()
2313 .view(authority);
2314 }
2315 self.step.backing_view(resource_id)
2316 }
2317
2318 pub(crate) fn participant_backing_views(
2319 &self,
2320 resource_id: &ResourceId,
2321 ) -> Result<Vec<(SequenceAuthorityId, LogicalBackingBufferView<'_, R::Buffer>)>, VNextError>
2322 {
2323 self.participants
2324 .iter()
2325 .map(|participant| {
2326 let authority = BatchParticipantAuthority::new(
2327 participant.sequence_authority(),
2328 participant.request_authority(),
2329 );
2330 Ok((
2331 participant.sequence_authority(),
2332 self.step.participant_backing_view(authority, resource_id)?,
2333 ))
2334 })
2335 .collect()
2336 }
2337}
2338
2339#[must_use = "a definitely-not-submitted retry authority must be retried or retired"]
2343pub struct DefinitelyNotSubmittedRetryAuthority<R>
2344where
2345 R: DeviceRuntime,
2346{
2347 invocation: Option<InvocationResourceLease<R>>,
2348 topology_fingerprint: String,
2349 work_fingerprint: String,
2350 prior_attempt: BatchInvocationId,
2351}
2352
2353impl<R> fmt::Debug for DefinitelyNotSubmittedRetryAuthority<R>
2354where
2355 R: DeviceRuntime,
2356{
2357 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
2358 formatter
2359 .debug_struct("DefinitelyNotSubmittedRetryAuthority")
2360 .field("prior_attempt", &self.prior_attempt)
2361 .field("topology_fingerprint", &self.topology_fingerprint)
2362 .field("work_fingerprint", &self.work_fingerprint)
2363 .finish_non_exhaustive()
2364 }
2365}
2366
2367impl<R> DefinitelyNotSubmittedRetryAuthority<R>
2368where
2369 R: DeviceRuntime,
2370{
2371 pub const fn prior_attempt(&self) -> BatchInvocationId {
2372 self.prior_attempt
2373 }
2374
2375 pub fn topology_fingerprint(&self) -> &str {
2376 &self.topology_fingerprint
2377 }
2378
2379 pub fn work_fingerprint(&self) -> &str {
2380 &self.work_fingerprint
2381 }
2382
2383 pub fn retry(mut self) -> Result<InvocationResourceLease<R>, VNextError> {
2384 let fresh_attempt = issue_batch_invocation_id()?;
2385 let invocation = self
2386 .invocation
2387 .as_mut()
2388 .ok_or_else(|| invalid_resource("retry authority no longer owns its invocation"))?;
2389 invocation.prepare_definitely_not_submitted_retry(
2390 fresh_attempt,
2391 &self.topology_fingerprint,
2392 &self.work_fingerprint,
2393 )?;
2394 Ok(self
2395 .invocation
2396 .take()
2397 .expect("validated retry authority still owns its invocation"))
2398 }
2399}