1use std::collections::{BTreeMap, BTreeSet};
2use std::sync::Arc;
3
4use super::super::{
5 AdmittedSequenceResources, AllocationKind, AllocationLifetime, BatchInvocationId,
6 BatchParticipantAuthority, BatchParticipantTokenRange, BatchStepId, BatchWorkShape,
7 BufferDescriptor, BufferUsage, DeviceId, DeviceRuntime, DynamicResourceDemand,
8 EncodedDeviceOperation, ExecutablePlanView, ExecutionIdentityEnvelope, InvocationResourceLease,
9 LogicalAdmissionCoordinatorId, LogicalBackingBufferView, NodeId, NodeWorkContract, PlanHash,
10 PlanId, PlanNode, PreparedStepSubmissionNode, PreparedStepSubmissionWave,
11 ProgramBindingNodeBinding, ProviderId, ProviderWorkspaceRequirement, ResourceId, SemanticValue,
12 SequenceBackingSnapshot, SequenceSessionEpoch, SequenceSessionFingerprint,
13 StepParticipantFrameAssignment, StepResourceLease, TrustedActiveSequenceBinding,
14 TrustedPlanRuntimeEvidence, VNextError,
15};
16use super::buffer_view::{
17 sequence_execution_shape, validate_value_binding_physical_coverage,
18 ValueBindingPhysicalCoverage,
19};
20use super::foundation::invalid_operation;
21use super::resolved_value::resource_uses_packed_batch_coordinates;
22use super::{
23 AttributeId, BatchOperationIdentity, BatchOperationNodeIdentity, ElementType,
24 OperationBufferView, OperationDescriptor, OperationProviderDescriptor, ResolvedValueBinding,
25 ResolvedValueRole,
26};
27
28pub(super) enum OperationInvocationResources<'a, R: DeviceRuntime> {
29 Invocation(&'a InvocationResourceLease<R>),
30 Wave {
31 wave: &'a PreparedStepSubmissionWave<R>,
32 node_index: usize,
33 },
34}
35
36impl<R: DeviceRuntime> Copy for OperationInvocationResources<'_, R> {}
37
38impl<R: DeviceRuntime> Clone for OperationInvocationResources<'_, R> {
39 fn clone(&self) -> Self {
40 *self
41 }
42}
43
44impl<'a, R: DeviceRuntime> OperationInvocationResources<'a, R> {
45 fn wave_node(self) -> Result<&'a PreparedStepSubmissionNode<R>, VNextError> {
46 match self {
47 Self::Wave { wave, node_index } => wave
48 .nodes()
49 .get(node_index)
50 .ok_or_else(|| invalid_operation("submission wave node index is out of bounds")),
51 Self::Invocation(_) => Err(invalid_operation(
52 "single-operation resources do not contain a wave node",
53 )),
54 }
55 }
56
57 pub(super) fn node_id(self) -> Result<&'a NodeId, VNextError> {
58 match self {
59 Self::Invocation(invocation) => Ok(invocation.node_id()),
60 Self::Wave { .. } => Ok(self.wave_node()?.node_id()),
61 }
62 }
63
64 fn program_binding_node(self) -> Option<ProgramBindingNodeBinding> {
65 match self {
66 Self::Invocation(_) => None,
67 Self::Wave { wave, node_index } => wave.nodes().get(node_index).and_then(|node| {
68 wave.claimed_backing()
69 .program_binding_node(node.plan_node_index())
70 }),
71 }
72 }
73
74 pub(super) fn participant_count(self) -> Result<usize, VNextError> {
75 match self {
76 Self::Invocation(invocation) => usize::try_from(invocation.participant_count())
77 .map_err(|_| invalid_operation("operation participant count exceeds usize")),
78 Self::Wave { .. } => usize::try_from(self.wave_node()?.participant_count())
79 .map_err(|_| invalid_operation("wave participant count exceeds usize")),
80 }
81 }
82
83 pub(super) fn prepared_participant_count(self) -> Result<usize, VNextError> {
84 match self {
85 Self::Invocation(invocation) => {
86 usize::try_from(invocation.prepared_participant_count()).map_err(|_| {
87 invalid_operation("prepared operation participant count exceeds usize")
88 })
89 }
90 Self::Wave { .. } => Ok(self.wave_node()?.participant_session_identities().len()),
91 }
92 }
93
94 pub(super) fn participant(
95 self,
96 index: usize,
97 ) -> Result<&'a Arc<AdmittedSequenceResources<R>>, VNextError> {
98 match self {
99 Self::Invocation(invocation) => invocation
100 .participants()
101 .nth(index)
102 .ok_or_else(|| invalid_operation("operation participant index is out of range")),
103 Self::Wave { .. } => self
104 .wave_node()?
105 .participants()
106 .nth(index)
107 .ok_or_else(|| invalid_operation("wave participant index is out of range")),
108 }
109 }
110
111 fn participant_backing_snapshot(
112 self,
113 index: usize,
114 ) -> Result<&'a Arc<SequenceBackingSnapshot<R>>, VNextError> {
115 let participant = self.participant(index)?;
116 self.step_resources()
117 .participant_backing_snapshot(BatchParticipantAuthority::new(
118 participant.sequence_authority(),
119 participant.request_authority(),
120 ))
121 }
122
123 fn participant_backing_view(
124 self,
125 index: usize,
126 resource_id: &ResourceId,
127 ) -> Result<LogicalBackingBufferView<'a, R::Buffer>, VNextError> {
128 let participant = self.participant(index)?;
129 self.step_resources().participant_backing_view(
130 BatchParticipantAuthority::new(
131 participant.sequence_authority(),
132 participant.request_authority(),
133 ),
134 resource_id,
135 )
136 }
137
138 pub(super) fn participant_frames(
139 self,
140 ) -> Result<&'a [StepParticipantFrameAssignment], VNextError> {
141 match self {
142 Self::Invocation(invocation) => Ok(invocation.participant_frames()),
143 Self::Wave { .. } => Ok(self.wave_node()?.participant_frames()),
144 }
145 }
146
147 pub(super) fn participant_session_identity(
148 self,
149 index: usize,
150 ) -> Result<(SequenceSessionEpoch, &'a SequenceSessionFingerprint), VNextError> {
151 match self {
152 Self::Invocation(invocation) => invocation
153 .participant_session_identities()
154 .nth(index)
155 .ok_or_else(|| invalid_operation("operation participant session is missing")),
156 Self::Wave { .. } => self
157 .wave_node()?
158 .participant_session_identities()
159 .nth(index)
160 .ok_or_else(|| invalid_operation("wave participant session is missing")),
161 }
162 }
163
164 pub(super) fn batch_step_id(self) -> BatchStepId {
165 match self {
166 Self::Invocation(invocation) => invocation.batch_step_id(),
167 Self::Wave { wave, .. } => wave.batch_step_id(),
168 }
169 }
170
171 pub(super) fn batch_invocation_id(self) -> BatchInvocationId {
172 match self {
173 Self::Invocation(invocation) => invocation.batch_invocation_id(),
174 Self::Wave { wave, .. } => wave.batch_invocation_id(),
175 }
176 }
177
178 pub(super) fn coordinator_id(self) -> Result<LogicalAdmissionCoordinatorId, VNextError> {
179 Ok(self.participant(0)?.coordinator_id())
180 }
181
182 pub(super) fn work_shape(self) -> Result<&'a BatchWorkShape, VNextError> {
183 match self {
184 Self::Invocation(invocation) => Ok(invocation.work_shape()),
185 Self::Wave { .. } => Ok(self.wave_node()?.work_shape()),
186 }
187 }
188
189 pub(super) fn step_resources(self) -> &'a Arc<StepResourceLease<R>> {
190 match self {
191 Self::Invocation(invocation) => invocation.step_resources(),
192 Self::Wave { wave, .. } => wave.step_resources(),
193 }
194 }
195
196 pub(super) fn runtime(self) -> &'a Arc<R> {
197 match self {
198 Self::Invocation(invocation) => invocation.runtime(),
199 Self::Wave { wave, .. } => wave.runtime(),
200 }
201 }
202
203 pub(super) fn plan_identity_matches(
204 self,
205 plan_id: &PlanId,
206 plan_hash: &PlanHash,
207 device_id: &DeviceId,
208 ) -> Result<bool, VNextError> {
209 match self {
210 Self::Invocation(invocation) => {
211 let evidence = invocation.plan_evidence();
212 Ok(evidence.plan_id() == plan_id
213 && evidence.plan_hash() == plan_hash
214 && evidence.device_id() == device_id)
215 }
216 Self::Wave { .. } => {
217 let evidence = self.wave_node()?.plan_evidence_ref();
218 Ok(evidence.plan_id() == plan_id
219 && evidence.plan_hash() == plan_hash
220 && evidence.device_id() == device_id)
221 }
222 }
223 }
224
225 fn plan_evidence_matches(
226 self,
227 expected: &TrustedPlanRuntimeEvidence,
228 ) -> Result<bool, VNextError> {
229 match self {
230 Self::Invocation(invocation) => Ok(invocation.plan_evidence() == *expected),
231 Self::Wave { .. } => Ok(self.wave_node()?.plan_evidence_ref() == expected),
232 }
233 }
234
235 pub(super) fn backing_fingerprint(self) -> &'a str {
236 match self {
237 Self::Invocation(invocation) => invocation.claimed_backing().fingerprint(),
238 Self::Wave { wave, .. } => wave.fingerprint(),
239 }
240 }
241
242 fn backing_view(
243 self,
244 resource_id: &ResourceId,
245 ) -> Result<LogicalBackingBufferView<'a, R::Buffer>, VNextError> {
246 match self {
247 Self::Invocation(invocation) => invocation.backing_view(resource_id),
248 Self::Wave { wave, node_index } => wave.backing_view(node_index, resource_id),
249 }
250 }
251}
252
253#[derive(Debug, Clone, Copy, PartialEq, Eq)]
254enum PreparedOperationResourceSource {
255 PlanStatic { slot_index: usize },
256 Dynamic { descriptor_index: usize },
257}
258
259#[derive(Debug, Clone, PartialEq, Eq)]
260struct PreparedOperationResource {
261 resource_id: ResourceId,
262 source: PreparedOperationResourceSource,
263}
264
265#[derive(Debug, Clone, PartialEq, Eq)]
270pub(super) struct PreparedOperationDispatchBinding {
271 node_index: usize,
272 resources: Vec<PreparedOperationResource>,
273 binding_component_views: Vec<Vec<usize>>,
274 scratch_view: Option<usize>,
275 binding_view: Option<usize>,
276 persistent_view: Option<usize>,
277}
278
279impl PreparedOperationDispatchBinding {
280 pub(super) fn prepare(
281 resolved: &dyn ExecutablePlanView,
282 provider: &OperationProviderDescriptor,
283 node_id: &NodeId,
284 ) -> Result<Self, VNextError> {
285 let plan = resolved.execution_plan();
286 let (node_index, node) = plan
287 .payload()
288 .nodes()
289 .iter()
290 .enumerate()
291 .find(|(_, node)| node.id() == node_id)
292 .ok_or_else(|| invalid_operation(format!("plan has no node `{node_id}`")))?;
293 let operation = resolved.capabilities().operation(node.operation_id())?;
294 let registered = resolved
295 .capabilities()
296 .providers_for(node.operation_id())?
297 .iter()
298 .find(|candidate| candidate.provider_id() == provider.provider_id())
299 .ok_or_else(|| invalid_operation("operation provider is absent from the catalog"))?;
300 if registered != provider
301 || provider.provider_id() != node.selection().selected_provider()
302 || provider.operation_id() != node.operation_id()
303 || provider.operation_fingerprint() != node.operation_fingerprint()
304 || provider.provider_implementation_fingerprint()
305 != node.provider_implementation_fingerprint()
306 || provider.execution_semantics() != node.provider_execution_semantics()
307 || provider.device_id() != plan.payload().device_id()
308 || !provider.version().satisfies(node.operation_version())
309 {
310 return Err(invalid_operation(
311 "operation provider is not the exact catalog entry selected by the plan",
312 ));
313 }
314 operation.validate_attributes(node.attributes())?;
315 operation.validate_resolved_bindings(node.values())?;
316
317 let provider_resources = node.provider_resources();
318 if provider_resources.provider_id() != provider.provider_id()
319 || provider_resources.estimator_id() != provider.resource_estimator_id()
320 || provider_resources.estimator_version() != provider.resource_estimator_version()
321 || provider_resources.estimator_implementation_fingerprint()
322 != provider.resource_estimator_implementation_fingerprint()
323 || provider_resources.value_alignment_bytes()
324 < operation.resources.minimum_value_alignment_bytes
325 || provider_resources.value_alignment_bytes()
326 % operation.resources.minimum_value_alignment_bytes
327 != 0
328 || !operation
329 .resources
330 .scratch
331 .accepts(provider_resources.scratch().is_some())
332 || !operation
333 .resources
334 .binding
335 .accepts(provider_resources.binding().is_some())
336 || !operation
337 .resources
338 .persistent
339 .accepts(provider_resources.persistent().is_some())
340 {
341 return Err(invalid_operation(
342 "plan provider resource estimate is not bound to the selected provider and operation contract",
343 ));
344 }
345 let scratch_resource = select_workspace_resource(
346 provider_resources.scratch(),
347 node.scratch_resource(),
348 "scratch",
349 )?;
350 let binding_resource = select_workspace_resource(
351 provider_resources.binding(),
352 node.binding_resource(),
353 "binding",
354 )?;
355 let persistent_resource = select_workspace_resource(
356 provider_resources.persistent(),
357 node.persistent_resource(),
358 "persistent",
359 )?;
360
361 let memory = plan.payload().memory();
362 let mut required_resources = node
363 .values()
364 .iter()
365 .flat_map(|binding| binding.storage().components())
366 .map(|component| component.resource_id().clone())
367 .collect::<BTreeSet<_>>();
368 required_resources.extend(scratch_resource.iter().map(|resource| (*resource).clone()));
369 required_resources.extend(binding_resource.iter().map(|resource| (*resource).clone()));
370 required_resources.extend(
371 persistent_resource
372 .iter()
373 .map(|resource| (*resource).clone()),
374 );
375 let resources = required_resources
376 .into_iter()
377 .map(|resource_id| {
378 let static_index = memory
379 .static_allocations()
380 .binary_search_by(|allocation| allocation.resource_id().cmp(&resource_id));
381 let dynamic_index = memory
382 .dynamic_descriptors()
383 .binary_search_by(|descriptor| descriptor.base_resource_id().cmp(&resource_id));
384 let source = match (static_index, dynamic_index) {
385 (Ok(slot_index), Err(_)) => {
386 PreparedOperationResourceSource::PlanStatic { slot_index }
387 }
388 (Err(_), Ok(descriptor_index)) => {
389 PreparedOperationResourceSource::Dynamic { descriptor_index }
390 }
391 (Ok(_), Ok(_)) => {
392 return Err(invalid_operation(format!(
393 "plan resource `{resource_id}` is both static and dynamic"
394 )))
395 }
396 (Err(_), Err(_)) => {
397 return Err(invalid_operation(format!(
398 "plan has no static allocation or dynamic descriptor for `{resource_id}`"
399 )));
400 }
401 };
402 Ok(PreparedOperationResource {
403 resource_id,
404 source,
405 })
406 })
407 .collect::<Result<Vec<_>, VNextError>>()?;
408 let view_index_for = |resource_id: &ResourceId, kind: &str| {
409 resources
410 .binary_search_by(|resource| resource.resource_id.cmp(resource_id))
411 .map_err(|_| invalid_operation(format!("{kind} resource view is missing")))
412 };
413 let binding_component_views = node
414 .values()
415 .iter()
416 .map(|binding| {
417 binding
418 .storage()
419 .components()
420 .iter()
421 .map(|component| view_index_for(component.resource_id(), "value binding"))
422 .collect::<Result<Vec<_>, _>>()
423 })
424 .collect::<Result<Vec<_>, _>>()?;
425 let scratch_view = scratch_resource
426 .map(|resource| view_index_for(resource, "scratch"))
427 .transpose()?;
428 let binding_view = binding_resource
429 .map(|resource| view_index_for(resource, "binding"))
430 .transpose()?;
431 let persistent_view = persistent_resource
432 .map(|resource| view_index_for(resource, "persistent"))
433 .transpose()?;
434 Ok(Self {
435 node_index,
436 resources,
437 binding_component_views,
438 scratch_view,
439 binding_view,
440 persistent_view,
441 })
442 }
443
444 pub(super) fn node<'plan>(
445 &self,
446 resolved: &'plan dyn ExecutablePlanView,
447 node_id: &NodeId,
448 ) -> Result<&'plan PlanNode, VNextError> {
449 resolved
450 .execution_plan()
451 .payload()
452 .nodes()
453 .get(self.node_index)
454 .filter(|node| node.id() == node_id)
455 .ok_or_else(|| {
456 invalid_operation("prepared operation binding differs from its plan node")
457 })
458 }
459}
460
461pub struct OperationInvocation<'a, B> {
464 identity: &'a ExecutionIdentityEnvelope,
465 operation: &'a OperationDescriptor,
466 node_id: &'a NodeId,
467 provider_id: &'a ProviderId,
468 views: Vec<OperationBufferView<'a, B>>,
469 bindings: &'a [ResolvedValueBinding],
470 attributes: &'a BTreeMap<AttributeId, SemanticValue>,
471 work: &'a NodeWorkContract,
472 scratch_view: Option<usize>,
473 binding_view: Option<usize>,
474 persistent_view: Option<usize>,
475 work_shape: &'a BatchWorkShape,
476 claimed_backing_fingerprint: &'a str,
477}
478
479impl<'a, B> OperationInvocation<'a, B> {
480 #[allow(clippy::too_many_arguments)]
481 fn from_prepared<R>(
482 runtime: &R,
483 resolved: &'a dyn ExecutablePlanView,
484 prepared: &PreparedOperationDispatchBinding,
485 node: &'a PlanNode,
486 operation: &'a OperationDescriptor,
487 identity: &'a ExecutionIdentityEnvelope,
488 node_id: &'a NodeId,
489 resources: OperationInvocationResources<'a, R>,
490 active_binding: &TrustedActiveSequenceBinding,
491 participant_index: usize,
492 ) -> Result<Self, VNextError>
493 where
494 R: DeviceRuntime<Buffer = B>,
495 {
496 let plan = resolved.execution_plan();
497 let parts = identity.parts();
498 let participant = resources.participant(participant_index)?;
499 let participant_backing = resources.participant_backing_snapshot(participant_index)?;
500 let participant_frame = resources
501 .participant_frames()?
502 .get(participant_index)
503 .ok_or_else(|| invalid_operation("operation participant frame is missing"))?;
504 let participant_session = resources.participant_session_identity(participant_index)?;
505 let static_lease = participant.static_provisioning();
506 let lease_identity = static_lease.map(|lease| lease.identity());
507 let admission = active_binding.plan().static_provisioning_binding();
508 let pool_fingerprint = active_binding.static_pool_identity_fingerprint_ref();
509 let memory = plan.payload().memory();
510 if resources.participant_count()? != resources.prepared_participant_count()?
511 || resources.node_id()? != node_id
512 || participant_frame.sequence_authority() != participant.sequence_authority()
513 || participant_frame.request_authority() != participant.request_authority()
514 || !resources.plan_evidence_matches(active_binding.plan())?
515 || resources.coordinator_id()? != active_binding.coordinator_id()
516 || participant.sequence_authority() != active_binding.sequence_authority()
517 || participant.run_id() != active_binding.run_id()
518 || participant.request_id() != active_binding.request_id()
519 || !active_binding
520 .matches_sequence_session(participant_session.0, participant_session.1)
521 || runtime.descriptor() != resolved.device()
522 || runtime.descriptor() != resolved.capabilities().device()
523 || runtime.descriptor().runtime_implementation_fingerprint
524 != plan.payload().device_runtime_implementation_fingerprint()
525 || parts.plan_id.as_ref() != Some(plan.payload().plan_id())
526 || parts.plan_hash.as_ref() != Some(plan.plan_hash())
527 || parts.frame_id != Some(participant_frame.frame_id())
528 || parts.node_invocation_id.is_none()
529 || parts.node_id.as_ref() != Some(node.id())
530 || parts.operation_id.as_ref() != Some(node.operation_id())
531 || parts.provider_id.as_ref() != Some(node.selection().selected_provider())
532 || parts.device_id.as_ref() != Some(plan.payload().device_id())
533 || parts.run_id != *active_binding.run_id()
534 || parts.request_id != *active_binding.request_id()
535 || parts.transaction_id.as_ref()
536 != lease_identity.map(|identity| identity.transaction_id())
537 || parts.resource_pool_id != active_binding.static_pool_id()
538 || parts.resource_pool_identity_fingerprint.as_deref() != pool_fingerprint
539 || parts.provisioning_run_id.as_ref()
540 != lease_identity.map(|identity| identity.run_id())
541 || parts.provisioning_request_id.as_ref()
542 != lease_identity.map(|identity| identity.request_id())
543 || parts.active_sequence_slot != Some(active_binding.sequence_authority().sparse_id())
544 || parts.admission_generation != Some(active_binding.sequence_authority().generation())
545 || parts.activation_epoch != Some(active_binding.activation_epoch())
546 || parts.runtime_implementation_fingerprint.as_deref()
547 != Some(active_binding.runtime_implementation_fingerprint())
548 || parts.active_sequence_fingerprint.as_deref() != Some(active_binding.fingerprint())
549 || parts.completed_sequence_fingerprint.is_some()
550 || parts.aborted_sequence_fingerprint.is_some()
551 || active_binding.plan().plan_id() != plan.payload().plan_id()
552 || active_binding.plan().plan_hash() != plan.plan_hash()
553 || active_binding.plan().device_id() != plan.payload().device_id()
554 || active_binding.plan().runtime_implementation_fingerprint()
555 != plan.payload().device_runtime_implementation_fingerprint()
556 || active_binding.runtime_implementation_fingerprint()
557 != runtime.descriptor().runtime_implementation_fingerprint
558 || active_binding.static_provisioning_identity() != lease_identity
559 || admission != static_lease.map(|lease| lease.admission())
560 || admission.is_some_and(|admission| {
561 admission.device_capacity_bytes() != memory.device_capacity_bytes()
562 || admission.usable_capacity_bytes() != memory.usable_capacity_bytes()
563 || admission.plan_static_bytes() != memory.static_bytes()
564 || admission.maximum_active_sequences() != memory.maximum_active_sequences()
565 })
566 || parts.resource_id.is_some()
567 || parts.resource_generation.is_some()
568 || parts.resource_batch_fingerprint.is_some()
569 {
570 return Err(invalid_operation(
571 "operation invocation does not close over the runtime device, selected plan, node, provider, request, and lease transaction",
572 ));
573 }
574 let provider_resources = node.provider_resources();
575 let mut views = Vec::with_capacity(prepared.resources.len());
576 for resource in &prepared.resources {
577 let resource_id = &resource.resource_id;
578 match resource.source {
579 PreparedOperationResourceSource::PlanStatic { slot_index } => {
580 let allocation = memory
581 .static_allocations()
582 .get(slot_index)
583 .filter(|allocation| allocation.resource_id() == resource_id)
584 .ok_or_else(|| {
585 invalid_operation(
586 "prepared static resource index differs from the memory plan",
587 )
588 })?;
589 let lease = static_lease.ok_or_else(|| {
590 invalid_operation(format!(
591 "plan-static resource `{resource_id}` lacks static provisioning"
592 ))
593 })?;
594 let leased = lease.plan_static_view(slot_index, allocation)?;
595 views.push(OperationBufferView::from_static(
596 leased,
597 participant.device_buffer_retention(),
598 ));
599 }
600 PreparedOperationResourceSource::Dynamic { descriptor_index } => {
601 let descriptor = memory
602 .dynamic_descriptors()
603 .get(descriptor_index)
604 .filter(|descriptor| descriptor.base_resource_id() == resource_id)
605 .ok_or_else(|| {
606 invalid_operation(
607 "prepared dynamic resource index differs from the memory plan",
608 )
609 })?;
610 let descriptor_lifetime = descriptor.lifetime();
611 let packed_batch_coordinates = resource_uses_packed_batch_coordinates(
612 memory,
613 descriptor.base_resource_id(),
614 )?;
615 let backing = resources.backing_view(resource_id).or_else(|_| {
616 resources.participant_backing_view(participant_index, resource_id)
617 })?;
618 let expected_backing_bytes = match descriptor.lifetime() {
619 AllocationLifetime::Invocation => descriptor
620 .evaluate_request_bytes_for_shape(
621 resources.work_shape()?.immediate_shape(),
622 )?,
623 AllocationLifetime::Step => descriptor.evaluate_request_bytes_for_shape(
624 resources.step_resources().work_shape().immediate_shape(),
625 )?,
626 AllocationLifetime::Sequence => {
627 let participant_token_range = resources
628 .work_shape()?
629 .participant_token_ranges()
630 .get(participant_index)
631 .ok_or_else(|| {
632 invalid_operation(
633 "operation participant token range is missing",
634 )
635 })?;
636 let execution_shape = sequence_execution_shape(
637 participant_backing.committed_shape(),
638 participant_token_range.source_token_range().end,
639 )?;
640 descriptor.evaluate_request_bytes_for_shape(execution_shape)?
641 }
642 AllocationLifetime::Request => descriptor.evaluate_fit_request_bytes(
643 participant.request_resources().work_shape(),
644 )?,
645 AllocationLifetime::Plan => {
646 return Err(invalid_operation(format!(
647 "plan-lifetime resource `{resource_id}` cannot use dynamic backing"
648 )))
649 }
650 };
651 let size_matches = match descriptor.lifetime() {
652 AllocationLifetime::Sequence => {
653 backing.size_bytes() >= expected_backing_bytes
654 }
655 _ => backing.size_bytes() == expected_backing_bytes,
656 };
657 if !size_matches
658 || backing.capacity_size_bytes() < backing.size_bytes()
659 || backing.alignment_bytes() != descriptor.alignment_bytes()
660 || backing.usage() != descriptor.usage()
661 || backing.element_type() != descriptor.element_type()
662 || backing.storage_profile() != descriptor.storage().profile()
663 {
664 return Err(invalid_operation(format!(
665 "logical backing extent differs from plan descriptor `{resource_id}`"
666 )));
667 }
668 let participant_window = match (
669 descriptor.lifetime(),
670 descriptor.kind(),
671 descriptor.demand(),
672 ) {
673 (
674 AllocationLifetime::Step,
675 AllocationKind::Value,
676 DynamicResourceDemand::ActualSequences {
677 bytes_per_sequence,
678 maximum_sequences,
679 },
680 ) => {
681 let work_shape = resources.step_resources().work_shape();
682 if work_shape.immediate_sequences() > *maximum_sequences
683 || participant_index >= work_shape.participants().len()
684 {
685 return Err(invalid_operation(
686 "participant fixed resource exceeds its Step work shape",
687 ));
688 }
689 let offset = bytes_per_sequence
690 .checked_mul(u64::try_from(participant_index).map_err(|_| {
691 invalid_operation(
692 "participant fixed resource index exceeds u64",
693 )
694 })?)
695 .ok_or_else(|| {
696 invalid_operation(
697 "participant fixed resource offset overflows u64",
698 )
699 })?;
700 Some((offset, *bytes_per_sequence))
701 }
702 _ => None,
703 };
704 let view_bytes = participant_window
705 .map(|(_, bytes_per_sequence)| bytes_per_sequence)
706 .unwrap_or(expected_backing_bytes);
707 let descriptor = BufferDescriptor {
708 resource_id: resource_id.clone(),
709 size_bytes: view_bytes,
710 alignment_bytes: backing.alignment_bytes(),
711 usage: backing.usage(),
712 element_type: backing.element_type(),
713 };
714 let view = if let Some((offset, _)) = participant_window {
715 OperationBufferView::from_backing_window(
716 descriptor,
717 backing,
718 offset,
719 descriptor_lifetime,
720 )
721 } else if backing.capacity_size_bytes() > expected_backing_bytes {
722 OperationBufferView::from_backing_prefix(
723 descriptor,
724 backing,
725 descriptor_lifetime,
726 )
727 } else {
728 OperationBufferView::from_backing_exact(
729 descriptor,
730 backing,
731 descriptor_lifetime,
732 )
733 };
734 views.push(view.with_packed_batch_coordinates(packed_batch_coordinates));
735 }
736 }
737 }
738
739 for view in &views {
740 view.validate_runtime(runtime, lease_identity)?;
741 let translated = view.translate(0, view.descriptor().size_bytes)?;
742 let translated_bytes = translated.iter().try_fold(0_u64, |total, region| {
743 total
744 .checked_add(region.length_bytes())
745 .ok_or_else(|| invalid_operation("translated operation regions overflow u64"))
746 })?;
747 if translated_bytes != view.descriptor().size_bytes {
748 return Err(invalid_operation(format!(
749 "operation resource `{}` is not fully backed by physical regions",
750 view.resource_id()
751 )));
752 }
753 }
754 if node.values().len() != prepared.binding_component_views.len() {
755 return Err(invalid_operation(
756 "prepared value-binding recipe differs from its plan node",
757 ));
758 }
759 for (binding, component_views) in
760 node.values().iter().zip(&prepared.binding_component_views)
761 {
762 if binding.storage().components().len() != component_views.len() {
763 return Err(invalid_operation(
764 "prepared component recipe differs from its value binding",
765 ));
766 }
767 for (component, view_index) in
768 binding.storage().components().iter().zip(component_views)
769 {
770 let view = views.get(*view_index).ok_or_else(|| {
771 invalid_operation("value binding lacks a committed resource view")
772 })?;
773 if view.resource_id() != component.resource_id() {
774 return Err(invalid_operation(
775 "prepared value-binding view differs from its resource",
776 ));
777 }
778 let dynamic_demand = match prepared
779 .resources
780 .get(*view_index)
781 .map(|resource| resource.source)
782 {
783 Some(PreparedOperationResourceSource::PlanStatic { .. }) => None,
784 Some(PreparedOperationResourceSource::Dynamic { descriptor_index }) => Some(
785 memory
786 .dynamic_descriptors()
787 .get(descriptor_index)
788 .filter(|descriptor| {
789 descriptor.base_resource_id() == component.resource_id()
790 })
791 .ok_or_else(|| {
792 invalid_operation(
793 "prepared component descriptor differs from the memory plan",
794 )
795 })?
796 .demand(),
797 ),
798 None => {
799 return Err(invalid_operation(
800 "prepared component view index is out of range",
801 ))
802 }
803 };
804 let coverage = validate_value_binding_physical_coverage(
805 node.work(),
806 binding,
807 component,
808 view.descriptor(),
809 dynamic_demand,
810 provider_resources.value_alignment_bytes(),
811 )?;
812 if coverage == ValueBindingPhysicalCoverage::CanonicalComponent {
813 let translated =
814 view.translate(component.offset_bytes(), component.length_bytes())?;
815 let translated_bytes = translated.iter().try_fold(0_u64, |total, region| {
816 total.checked_add(region.length_bytes()).ok_or_else(|| {
817 invalid_operation("translated value-binding regions overflow u64")
818 })
819 })?;
820 if translated_bytes != component.length_bytes() {
821 return Err(invalid_operation(format!(
822 "resource `{}` does not physically cover its value binding",
823 component.resource_id()
824 )));
825 }
826 }
827 }
828 }
829 validate_workspace(
830 &views,
831 prepared.scratch_view,
832 BufferUsage::Scratch,
833 provider_resources.scratch(),
834 "scratch",
835 )?;
836 validate_workspace(
837 &views,
838 prepared.binding_view,
839 BufferUsage::Binding,
840 provider_resources.binding(),
841 "binding",
842 )?;
843 validate_workspace(
844 &views,
845 prepared.persistent_view,
846 BufferUsage::Persistent,
847 provider_resources.persistent(),
848 "persistent",
849 )?;
850 Ok(Self {
851 identity,
852 operation,
853 node_id,
854 provider_id: node.selection().selected_provider(),
855 views,
856 bindings: node.values(),
857 attributes: node.attributes(),
858 work: node.work(),
859 scratch_view: prepared.scratch_view,
860 binding_view: prepared.binding_view,
861 persistent_view: prepared.persistent_view,
862 work_shape: resources.work_shape()?,
863 claimed_backing_fingerprint: resources.backing_fingerprint(),
864 })
865 }
866
867 pub fn identity(&self) -> &ExecutionIdentityEnvelope {
868 self.identity
869 }
870
871 pub fn operation(&self) -> &OperationDescriptor {
872 self.operation
873 }
874
875 pub fn node_id(&self) -> &NodeId {
876 self.node_id
877 }
878
879 pub fn provider_id(&self) -> &ProviderId {
880 self.provider_id
881 }
882
883 pub fn views(&self) -> &[OperationBufferView<'a, B>] {
884 &self.views
885 }
886
887 pub fn bindings(&self) -> &[ResolvedValueBinding] {
888 self.bindings
889 }
890
891 pub fn attributes(&self) -> &BTreeMap<AttributeId, SemanticValue> {
892 self.attributes
893 }
894
895 pub fn work(&self) -> &NodeWorkContract {
896 self.work
897 }
898
899 pub fn scratch_view(&self) -> Option<&OperationBufferView<'a, B>> {
900 self.scratch_view.map(|index| &self.views[index])
901 }
902
903 pub fn binding_view(&self) -> Option<&OperationBufferView<'a, B>> {
904 self.binding_view.map(|index| &self.views[index])
905 }
906
907 pub fn persistent_view(&self) -> Option<&OperationBufferView<'a, B>> {
908 self.persistent_view.map(|index| &self.views[index])
909 }
910
911 pub fn work_shape(&self) -> &BatchWorkShape {
912 self.work_shape
913 }
914
915 pub fn claimed_backing_fingerprint(&self) -> &str {
916 self.claimed_backing_fingerprint
917 }
918}
919
920pub struct BatchedOperationInvocation<'a, B> {
924 batch_identity: &'a BatchOperationIdentity,
925 node_identity: &'a BatchOperationNodeIdentity,
926 participants: Vec<OperationInvocation<'a, B>>,
927 program_binding: Option<ProgramBindingNodeBinding>,
928}
929
930impl<'a, B> BatchedOperationInvocation<'a, B> {
931 pub(super) fn from_resolved<R>(
932 runtime: &R,
933 resolved: &'a dyn ExecutablePlanView,
934 prepared: &PreparedOperationDispatchBinding,
935 batch_identity: &'a BatchOperationIdentity,
936 resources: &'a InvocationResourceLease<R>,
937 active_bindings: &'a [TrustedActiveSequenceBinding],
938 ) -> Result<Self, VNextError>
939 where
940 R: DeviceRuntime<Buffer = B>,
941 {
942 let node_identity = batch_identity.single_node().ok_or_else(|| {
943 invalid_operation("single-operation invocation received a multi-node batch identity")
944 })?;
945 Self::from_resources(
946 runtime,
947 resolved,
948 prepared,
949 batch_identity,
950 node_identity,
951 OperationInvocationResources::Invocation(resources),
952 active_bindings.iter(),
953 )
954 }
955
956 #[allow(clippy::too_many_arguments)]
957 pub(super) fn from_wave_node<'binding, R, I>(
958 runtime: &R,
959 resolved: &'a dyn ExecutablePlanView,
960 prepared: &PreparedOperationDispatchBinding,
961 batch_identity: &'a BatchOperationIdentity,
962 node_identity: &'a BatchOperationNodeIdentity,
963 wave: &'a PreparedStepSubmissionWave<R>,
964 node_index: usize,
965 active_bindings: I,
966 ) -> Result<Self, VNextError>
967 where
968 R: DeviceRuntime<Buffer = B>,
969 I: ExactSizeIterator<Item = &'binding TrustedActiveSequenceBinding>,
970 {
971 Self::from_resources(
972 runtime,
973 resolved,
974 prepared,
975 batch_identity,
976 node_identity,
977 OperationInvocationResources::Wave { wave, node_index },
978 active_bindings,
979 )
980 }
981
982 #[allow(clippy::too_many_arguments)]
983 fn from_resources<'binding, R, I>(
984 runtime: &R,
985 resolved: &'a dyn ExecutablePlanView,
986 prepared: &PreparedOperationDispatchBinding,
987 batch_identity: &'a BatchOperationIdentity,
988 node_identity: &'a BatchOperationNodeIdentity,
989 resources: OperationInvocationResources<'a, R>,
990 active_bindings: I,
991 ) -> Result<Self, VNextError>
992 where
993 R: DeviceRuntime<Buffer = B>,
994 I: ExactSizeIterator<Item = &'binding TrustedActiveSequenceBinding>,
995 {
996 let participant_count = resources.participant_count()?;
997 let participant_frames = resources.participant_frames()?;
998 if participant_count == 0
999 || participant_count != active_bindings.len()
1000 || participant_count != node_identity.participants().len()
1001 || participant_count != participant_frames.len()
1002 || batch_identity.batch_step_id() != resources.batch_step_id()
1003 || batch_identity.batch_invocation_id() != resources.batch_invocation_id()
1004 || node_identity.node_id() != resources.node_id()?
1005 || node_identity.work_shape_fingerprint() != resources.work_shape()?.fingerprint()
1006 || batch_identity.claimed_backing_fingerprint() != resources.backing_fingerprint()
1007 || node_identity
1008 .participants()
1009 .iter()
1010 .zip(participant_frames)
1011 .any(|(participant, frame)| {
1012 let key = participant.node_key();
1013 key.sequence_authority() != frame.sequence_authority()
1014 || key.request_authority() != frame.request_authority()
1015 || key.frame_id() != frame.frame_id()
1016 || key.node_id() != node_identity.node_id()
1017 })
1018 {
1019 return Err(invalid_operation(
1020 "batched operation identity differs from its exact invocation resources",
1021 ));
1022 }
1023 let node = prepared.node(resolved, node_identity.node_id())?;
1024 let operation = resolved.capabilities().operation(node.operation_id())?;
1025 let participants = node_identity
1026 .participants()
1027 .iter()
1028 .zip(active_bindings)
1029 .enumerate()
1030 .map(|(index, (participant, active_binding))| {
1031 OperationInvocation::from_prepared(
1032 runtime,
1033 resolved,
1034 prepared,
1035 node,
1036 operation,
1037 participant.identity(),
1038 node_identity.node_id(),
1039 resources,
1040 active_binding,
1041 index,
1042 )
1043 })
1044 .collect::<Result<Vec<_>, _>>()?;
1045 let program_binding = resources.program_binding_node();
1046 Ok(Self {
1047 batch_identity,
1048 node_identity,
1049 participants,
1050 program_binding,
1051 })
1052 }
1053
1054 pub fn batch_identity(&self) -> &BatchOperationIdentity {
1055 self.batch_identity
1056 }
1057
1058 pub fn participants(&self) -> &[OperationInvocation<'a, B>] {
1059 &self.participants
1060 }
1061
1062 pub fn operation(&self) -> &OperationDescriptor {
1063 self.participants[0].operation()
1064 }
1065
1066 pub fn node_id(&self) -> &NodeId {
1067 self.node_identity.node_id()
1068 }
1069
1070 pub fn provider_id(&self) -> &ProviderId {
1071 self.node_identity.provider_id()
1072 }
1073
1074 pub fn work_shape(&self) -> &BatchWorkShape {
1075 self.participants[0].work_shape()
1076 }
1077
1078 pub fn work_contract(&self) -> &NodeWorkContract {
1079 self.participants[0].work()
1080 }
1081
1082 pub fn program_binding(&self) -> Option<&ProgramBindingNodeBinding> {
1083 self.program_binding.as_ref()
1084 }
1085
1086 pub fn attach_binding_command<C>(
1091 &self,
1092 operation: EncodedDeviceOperation<C>,
1093 command: C,
1094 ) -> EncodedDeviceOperation<C> {
1095 if self.program_binding.is_some() {
1096 operation.with_program_binding(command)
1097 } else {
1098 operation.with_dynamic_binding(command)
1099 }
1100 }
1101
1102 pub fn participant_token_ranges(&self) -> &[BatchParticipantTokenRange] {
1103 self.work_shape().participant_token_ranges()
1104 }
1105
1106 pub fn binding_uses_packed_batch_coordinates(
1111 &self,
1112 role: ResolvedValueRole,
1113 ordinal: u32,
1114 ) -> Result<bool, VNextError> {
1115 let mut packed_batch_coordinates = None;
1116 for participant in &self.participants {
1117 let binding = participant
1118 .bindings()
1119 .iter()
1120 .find(|binding| binding.role() == role && binding.ordinal() == ordinal)
1121 .ok_or_else(|| {
1122 invalid_operation(format!(
1123 "operation participant lacks {role:?} binding {ordinal}"
1124 ))
1125 })?;
1126 let [component] = binding.storage().components() else {
1127 return Err(invalid_operation(
1128 "batch sharing requires a single-resource value binding",
1129 ));
1130 };
1131 let view = participant
1132 .views()
1133 .iter()
1134 .find(|view| view.resource_id() == component.resource_id())
1135 .ok_or_else(|| {
1136 invalid_operation("operation value binding has no physical resource view")
1137 })?;
1138 match packed_batch_coordinates {
1139 Some(expected) if expected != view.uses_packed_batch_coordinates() => {
1140 return Err(invalid_operation(
1141 "operation participants disagree on value coordinate space",
1142 ));
1143 }
1144 None => packed_batch_coordinates = Some(view.uses_packed_batch_coordinates()),
1145 Some(_) => {}
1146 }
1147 }
1148 Ok(packed_batch_coordinates.expect("batched operation invocations are non-empty"))
1149 }
1150}
1151
1152fn select_workspace_resource<'a>(
1153 requirement: Option<&ProviderWorkspaceRequirement>,
1154 resource: Option<&'a ResourceId>,
1155 kind: &str,
1156) -> Result<Option<&'a ResourceId>, VNextError> {
1157 let Some(requirement) = requirement else {
1158 if resource.is_none() {
1159 return Ok(None);
1160 }
1161 return Err(invalid_operation(format!(
1162 "plan has unrequested {kind} resources"
1163 )));
1164 };
1165 resource.map(Some).ok_or_else(|| {
1166 invalid_operation(format!(
1167 "{kind} workspace base identity is missing for {:?} scope",
1168 requirement.scope()
1169 ))
1170 })
1171}
1172
1173fn validate_workspace<B>(
1174 views: &[OperationBufferView<'_, B>],
1175 index: Option<usize>,
1176 usage: BufferUsage,
1177 requirement: Option<&ProviderWorkspaceRequirement>,
1178 kind: &str,
1179) -> Result<(), VNextError> {
1180 match (requirement, index) {
1181 (None, None) => Ok(()),
1182 (None, Some(_)) | (Some(_), None) => Err(invalid_operation(format!(
1183 "{kind} workspace presence differs from the operation contract"
1184 ))),
1185 (Some(requirement), Some(index)) => {
1186 let descriptor = views[index].descriptor();
1187 let required_bytes = requirement.minimum_bytes()?;
1188 if descriptor.usage != usage
1189 || descriptor.element_type != ElementType::U8
1190 || descriptor.size_bytes < required_bytes
1191 || descriptor.alignment_bytes < requirement.alignment_bytes()
1192 || descriptor.alignment_bytes % requirement.alignment_bytes() != 0
1193 {
1194 return Err(invalid_operation(format!(
1195 "{kind} workspace descriptor is invalid"
1196 )));
1197 }
1198 let translated = views[index].translate(0, required_bytes)?;
1199 let translated_bytes = translated.iter().try_fold(0_u64, |total, region| {
1200 total.checked_add(region.length_bytes()).ok_or_else(|| {
1201 invalid_operation(format!("{kind} workspace region coverage overflows u64"))
1202 })
1203 })?;
1204 if translated_bytes != required_bytes {
1205 return Err(invalid_operation(format!(
1206 "{kind} workspace is not fully backed by physical regions"
1207 )));
1208 }
1209 Ok(())
1210 }
1211 }
1212}